From 7883507db6779a20bcfad398c8d6594f4206aa9b Mon Sep 17 00:00:00 2001 From: Brian McNaboe Date: Wed, 19 Aug 2026 10:14:08 -0400 Subject: [PATCH 001/130] fix(hooks): clear repository git environment for tests Signed-off-by: Brian McNaboe --- scripts/hooks/pre-commit | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/hooks/pre-commit b/scripts/hooks/pre-commit index fa42d92ae..7b5e1a4ff 100755 --- a/scripts/hooks/pre-commit +++ b/scripts/hooks/pre-commit @@ -17,4 +17,13 @@ echo "pre-commit: security audit (source-level)..." scripts/security-audit.sh echo "pre-commit: building and running tests..." -make -j$(sysctl -n hw.ncpu 2>/dev/null || nproc) -f Makefile.cbm test +( + # Git exports repository-local variables to hooks. The test suite creates + # independent repositories and linked worktrees, so keep the commit's + # index and object paths out of those fixtures. + while IFS= read -r variable; do + unset "$variable" + done < <(git rev-parse --local-env-vars) + + make -j"$(sysctl -n hw.ncpu 2>/dev/null || nproc)" -f Makefile.cbm test +) From 3604d6cf66c3d3099cec0b0b9a2f6da7f15e9e88 Mon Sep 17 00:00:00 2001 From: Brian McNaboe Date: Wed, 19 Aug 2026 10:35:16 -0400 Subject: [PATCH 002/130] fix(tests): scale daemon runtime sanitizer timeout Signed-off-by: Brian McNaboe --- tests/test_daemon_runtime.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/test_daemon_runtime.c b/tests/test_daemon_runtime.c index 5f41d914e..b650b5efb 100644 --- a/tests/test_daemon_runtime.c +++ b/tests/test_daemon_runtime.c @@ -21,6 +21,7 @@ #include "foundation/compat_thread.h" #include "foundation/log.h" #include "foundation/platform.h" +#include "foundation/sanitized.h" #include "pipeline/pipeline.h" #include "store/store.h" @@ -3550,10 +3551,18 @@ TEST(daemon_runtime_disconnect_cancels_blocked_non_index_child_and_preserves_oth SKIP_PLATFORM("requires a queryable copied process image"); #else enum { +#if CBM_SANITIZED + /* This readiness wait is a liveness backstop, not the behavior under + * test. The copied instrumented runner can take several seconds to + * reach main on macOS, especially while the parallel gate is busy. */ + CHILD_READY_BOUND_MS = 60000, + REQUEST_TIMEOUT_MS = 90000, +#else CHILD_READY_BOUND_MS = 5000, + REQUEST_TIMEOUT_MS = 15000, +#endif CHILD_CANCEL_BOUND_MS = 3000, CHILD_CLEANUP_BOUND_MS = 5000, - REQUEST_TIMEOUT_MS = 15000, }; const char *old_cache = getenv("CBM_CACHE_DIR"); const char *old_path = getenv("PATH"); From 455e4fb42acfd18170695ad8b3d2c47d8239c7a5 Mon Sep 17 00:00:00 2001 From: Ulises Millan Guerrero Date: Fri, 21 Aug 2026 13:50:37 -0600 Subject: [PATCH 003/130] fix(mcp): report freshness with the indexer's own mtime source check_index_coverage freshness recomputed mtime_ns from struct stat, which on Windows truncates to seconds, while the indexer records cbm_path_info_utf8's FILETIME-derived nanosecond value. A byte-identical file therefore never matched and every path was reported metadata_changed, so the freshness field carried no signal on Windows. The pipeline incremental change-detection had the same truncation in its hash comparison. Both readers now compare against the same source the writer uses. Fixes #1714 Signed-off-by: Ulises Millan Guerrero --- src/mcp/mcp.c | 22 ++++----- src/pipeline/pipeline_incremental.c | 24 ++++------ tests/test_mcp.c | 74 +++++++++++++++++++++++------ 3 files changed, 77 insertions(+), 43 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 04316627b..1812f38fc 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -4204,17 +4204,6 @@ static coverage_path_result_t coverage_normalize_rel(const char *input, bool all return written > 0U || allow_root ? COVERAGE_PATH_OK : COVERAGE_PATH_INVALID; } -static int64_t coverage_stat_mtime_ns(const struct stat *st) { -#ifdef __APPLE__ - return ((int64_t)st->st_mtimespec.tv_sec * (int64_t)CBM_NSEC_PER_SEC) + - (int64_t)st->st_mtimespec.tv_nsec; -#elif defined(_WIN32) - return (int64_t)st->st_mtime * (int64_t)CBM_NSEC_PER_SEC; -#else - return ((int64_t)st->st_mtim.tv_sec * (int64_t)CBM_NSEC_PER_SEC) + (int64_t)st->st_mtim.tv_nsec; -#endif -} - static const char *coverage_path_freshness(cbm_store_t *store, const char *project, const char *root_path, const char *rel_path, bool *outside) { @@ -4228,8 +4217,13 @@ static const char *coverage_path_freshness(cbm_store_t *store, const char *proje if (n < 0 || (size_t)n >= sizeof(abs_path)) { return "unavailable"; } - struct stat st; - if (stat(abs_path, &st) != 0) { + /* #1714: mtime_ns must come from the SAME source the indexer recorded it + * with — cbm_path_info_utf8. Recomputing from struct stat truncates to + * seconds on Windows (st_mtime), while the file_hashes record carries + * FILETIME-resolution nanoseconds, so a byte-identical file never matched + * and every path was reported metadata_changed. */ + cbm_path_info_t info; + if (cbm_path_info_utf8(abs_path, &info) != 0) { return "missing"; } if (!cbm_path_within_root(root_path, abs_path)) { @@ -4245,7 +4239,7 @@ static const char *coverage_path_freshness(cbm_store_t *store, const char *proje if (rc != CBM_STORE_OK) { return "unavailable"; } - bool matches = hash.mtime_ns == coverage_stat_mtime_ns(&st) && hash.size == st.st_size; + bool matches = hash.mtime_ns == info.mtime_ns && hash.size == info.size; cbm_store_clear_file_hash(&hash); return matches ? "metadata_match" : "metadata_changed"; } diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index fca1c8f06..0bf67670e 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -102,18 +102,6 @@ static const char *itoa_buf(int v) { return buf[idx]; } -/* ── Platform-portable mtime_ns ──────────────────────────────────── */ - -static int64_t stat_mtime_ns(const struct stat *st) { -#ifdef __APPLE__ - return ((int64_t)st->st_mtimespec.tv_sec * CBM_NS_PER_SEC) + (int64_t)st->st_mtimespec.tv_nsec; -#elif defined(_WIN32) - return (int64_t)st->st_mtime * CBM_NS_PER_SEC; -#else - return ((int64_t)st->st_mtim.tv_sec * CBM_NS_PER_SEC) + (int64_t)st->st_mtim.tv_nsec; -#endif -} - static const char *incr_mode_name(int mode) { switch (mode) { case CBM_MODE_FULL: @@ -683,14 +671,20 @@ static bool *classify_files(cbm_file_info_t *files, int file_count, cbm_file_has continue; } - struct stat st; - if (stat(files[i].path, &st) != 0) { + /* #1714: compare against the SAME source the hash writer used. The + * manifest hash is written from cbm_path_info_utf8 (see + * semantic_manifest_hash_file), so a stat()-based comparison is + * internally inconsistent: on Windows stat() truncates mtime to + * seconds while the recorded value carries FILETIME nanoseconds, which + * made every file look changed on each incremental pass. */ + cbm_path_info_t info; + if (cbm_path_info_utf8(files[i].path, &info) != 0) { changed[i] = true; n_changed++; continue; } - if (stat_mtime_ns(&st) != h->mtime_ns || st.st_size != h->size) { + if (info.mtime_ns != h->mtime_ns || info.size != h->size) { changed[i] = true; n_changed++; } else { diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 278d33e68..30735b8d4 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -2727,20 +2727,15 @@ TEST(tool_check_index_coverage_accepts_truncated_ignored_catalog_for_fresh_path_ ASSERT_NOT_NULL(store); char source_path[512]; snprintf(source_path, sizeof(source_path), "%s/project/main.go", tmp); - struct stat source_stat; - ASSERT_EQ(stat(source_path, &source_stat), 0); -#ifdef __APPLE__ - int64_t source_mtime_ns = - ((int64_t)source_stat.st_mtimespec.tv_sec * (int64_t)CBM_NSEC_PER_SEC) + - (int64_t)source_stat.st_mtimespec.tv_nsec; -#elif defined(_WIN32) - int64_t source_mtime_ns = (int64_t)source_stat.st_mtime * (int64_t)CBM_NSEC_PER_SEC; -#else - int64_t source_mtime_ns = ((int64_t)source_stat.st_mtim.tv_sec * (int64_t)CBM_NSEC_PER_SEC) + - (int64_t)source_stat.st_mtim.tv_nsec; -#endif - ASSERT_EQ(cbm_store_upsert_file_hash(store, "test-project", "main.go", "", source_mtime_ns, - source_stat.st_size), + /* The fixture must record the hash with the SAME mtime source the indexer + * writes with — cbm_path_info_utf8 — not struct stat. On Windows stat + * truncates to seconds while the stored record carries FILETIME-derived + * nanoseconds, so a stat-written fixture would never compare equal and the + * metadata_match contract below would fail. */ + cbm_path_info_t path_info; + ASSERT_EQ(cbm_path_info_utf8(source_path, &path_info), 0); + ASSERT_EQ(cbm_store_upsert_file_hash(store, "test-project", "main.go", "", path_info.mtime_ns, + path_info.size), CBM_STORE_OK); cbm_project_t project = {0}; ASSERT_EQ(cbm_store_get_project(store, "test-project", &project), CBM_STORE_OK); @@ -2782,6 +2777,56 @@ TEST(tool_check_index_coverage_accepts_truncated_ignored_catalog_for_fresh_path_ PASS(); } +/* #1714: coverage freshness must compare mtime_ns at the SAME precision the + * indexer records it. The pipeline records cbm_path_info_utf8's value (which + * on Windows derives from FILETIME — nanosecond), while the freshness reader + * used to recompute from struct stat, which on Windows truncates to seconds + * (st_mtime). A byte-identical file therefore never matched and every path was + * reported metadata_changed. The reader now uses the indexer's own source. */ +TEST(tool_check_index_coverage_freshness_uses_indexer_mtime_source_issue1714) { + char tmp[256]; + cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp)); + ASSERT_NOT_NULL(srv); + cbm_store_t *store = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(store); + + char source_path[512]; + snprintf(source_path, sizeof(source_path), "%s/project/main.go", tmp); + cbm_path_info_t info; + ASSERT_EQ(cbm_path_info_utf8(source_path, &info), 0); + + /* The hash exactly as the indexer writes it: same source, ns precision. */ + ASSERT_EQ(cbm_store_upsert_file_hash(store, "test-project", "main.go", "", info.mtime_ns, + info.size), + CBM_STORE_OK); + + char *response = cbm_mcp_handle_tool( + srv, "check_index_coverage", "{\"project\":\"test-project\",\"paths\":[\"main.go\"]}"); + ASSERT_NOT_NULL(response); + ASSERT_NOT_NULL(strstr(response, "\"freshness\":\"metadata_match\"")); + free(response); + + /* A hash stored at seconds precision — what a stat-based reader previously + * compared against — must NOT match an unchanged file: the comparison must + * stay nanosecond-exact, or part of mtime resolution is silently dropped. */ + int64_t seconds_mtime_ns = (info.mtime_ns / (int64_t)CBM_NSEC_PER_SEC) * + (int64_t)CBM_NSEC_PER_SEC; + if (seconds_mtime_ns != info.mtime_ns) { + ASSERT_EQ(cbm_store_upsert_file_hash(store, "test-project", "main.go", "", + seconds_mtime_ns, info.size), + CBM_STORE_OK); + response = cbm_mcp_handle_tool( + srv, "check_index_coverage", "{\"project\":\"test-project\",\"paths\":[\"main.go\"]}"); + ASSERT_NOT_NULL(response); + ASSERT_NOT_NULL(strstr(response, "\"freshness\":\"metadata_changed\"")); + free(response); + } + + cleanup_snippet_dir(tmp); + cbm_mcp_server_free(srv); + PASS(); +} + TEST(tool_check_index_coverage_rejects_stale_generation) { char tmp[256]; cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp)); @@ -11332,6 +11377,7 @@ SUITE(mcp) { RUN_TEST(tool_check_index_coverage_preserves_multiple_scope_labels); RUN_TEST(tool_check_index_coverage_accepts_truncated_ignored_catalog_for_fresh_path_issue1613); RUN_TEST(tool_check_index_coverage_rejects_stale_generation); + RUN_TEST(tool_check_index_coverage_freshness_uses_indexer_mtime_source_issue1714); RUN_TEST(tool_check_index_coverage_requires_source_when_file_metadata_changed); RUN_TEST(tool_check_index_coverage_surfaces_lookup_errors); RUN_TEST(tool_index_status_includes_git_metadata); From f05893757bed0fe83e450b6dbc62cf1c0dc22045 Mon Sep 17 00:00:00 2001 From: Sakil Ansari Date: Sun, 23 Aug 2026 20:26:29 +0530 Subject: [PATCH 004/130] Fix Pi extension to use TypeBox schemas instead of JSON Resolves #1806 where Pi extension crashed on startup with 'Cannot read properties of undefined (reading properties)' error. - Convert JSON schema to TypeBox format in Pi extension generation - Add TypeBox import to generated extension - Update test to expect TypeBox format instead of JSON - Handle all schema types: string, integer, boolean, array, enums - Preserve descriptions and optional field handling The generated extension now properly works with Pi's tool registration API that requires TypeBox Type.Object schemas. Signed-off-by: Sakil Ansari Signed-off-by: Sakil Ansari --- src/cli/client_adapter.c | 176 +++++++++++++++++++++++++++++++++++-- tests/test_agent_clients.c | 7 +- 2 files changed, 176 insertions(+), 7 deletions(-) diff --git a/src/cli/client_adapter.c b/src/cli/client_adapter.c index cc7454793..dd7e83527 100644 --- a/src/cli/client_adapter.c +++ b/src/cli/client_adapter.c @@ -7,6 +7,7 @@ #include "foundation/constants.h" #include "mcp/mcp.h" +#include "yyjson/yyjson.h" #include #include @@ -138,6 +139,172 @@ static void emit_header(adapter_sb_t *sb, const char *client) { "// are overwritten; edit outside it, or remove the markers to take ownership.\n"); } +static void convert_json_schema_to_typebox(adapter_sb_t *sb, const char *schema_str) { + if (!schema_str) { + sb_append(sb, "Type.Object({})"); + return; + } + + /* Parse the JSON schema to extract properties and convert to TypeBox */ + yyjson_doc *doc = yyjson_read(schema_str, strlen(schema_str), 0); + if (!doc) { + sb_append(sb, "Type.Object({})"); + return; + } + + yyjson_val *root = yyjson_doc_get_root(doc); + yyjson_val *properties = yyjson_obj_get(root, "properties"); + yyjson_val *required = yyjson_obj_get(root, "required"); + + if (!properties) { + sb_append(sb, "Type.Object({})"); + yyjson_doc_free(doc); + return; + } + + sb_append(sb, "Type.Object({\n"); + + /* Build required properties set for fast lookup */ + bool first_prop = true; + size_t idx = 0, max = 0; + yyjson_val *key = NULL, *val = NULL; + + yyjson_obj_foreach(properties, idx, max, key, val) { + const char *prop_name = yyjson_get_str(key); + if (!prop_name) continue; + + if (!first_prop) { + sb_append(sb, ",\n"); + } + first_prop = false; + + sb_append(sb, " "); + sb_append(sb, prop_name); + sb_append(sb, ": "); + + /* Check if property is required */ + bool is_required = false; + if (required && yyjson_is_arr(required)) { + size_t req_idx = 0, req_max = yyjson_arr_size(required); + for (req_idx = 0; req_idx < req_max; req_idx++) { + yyjson_val *req_item = yyjson_arr_get(required, req_idx); + const char *req_name = yyjson_get_str(req_item); + if (req_name && strcmp(req_name, prop_name) == 0) { + is_required = true; + break; + } + } + } + + /* Get property type and description */ + yyjson_val *type_val = yyjson_obj_get(val, "type"); + yyjson_val *desc_val = yyjson_obj_get(val, "description"); + yyjson_val *enum_val = yyjson_obj_get(val, "enum"); + yyjson_val *items_val = yyjson_obj_get(val, "items"); + + const char *type_str = yyjson_get_str(type_val); + const char *desc_str = yyjson_get_str(desc_val); + + /* Build TypeBox type based on JSON schema type */ + if (!is_required) { + sb_append(sb, "Type.Optional("); + } + + if (type_str && strcmp(type_str, "string") == 0) { + if (enum_val && yyjson_is_arr(enum_val)) { + sb_append(sb, "Type.Union(["); + size_t enum_idx = 0, enum_max = yyjson_arr_size(enum_val); + for (enum_idx = 0; enum_idx < enum_max; enum_idx++) { + if (enum_idx > 0) sb_append(sb, ", "); + yyjson_val *enum_item = yyjson_arr_get(enum_val, enum_idx); + const char *enum_str = yyjson_get_str(enum_item); + sb_append(sb, "Type.Literal("); + sb_append_js_string(sb, enum_str); + sb_append(sb, ")"); + } + sb_append(sb, "]"); + if (desc_str) { + sb_append(sb, ", { description: "); + sb_append_js_string(sb, desc_str); + sb_append(sb, " }"); + } + sb_append(sb, ")"); + } else { + sb_append(sb, "Type.String("); + if (desc_str) { + sb_append(sb, "{ description: "); + sb_append_js_string(sb, desc_str); + sb_append(sb, " }"); + } + sb_append(sb, ")"); + } + } else if (type_str && strcmp(type_str, "integer") == 0) { + sb_append(sb, "Type.Integer("); + if (desc_str) { + sb_append(sb, "{ description: "); + sb_append_js_string(sb, desc_str); + sb_append(sb, " }"); + } + sb_append(sb, ")"); + } else if (type_str && strcmp(type_str, "number") == 0) { + sb_append(sb, "Type.Number("); + if (desc_str) { + sb_append(sb, "{ description: "); + sb_append_js_string(sb, desc_str); + sb_append(sb, " }"); + } + sb_append(sb, ")"); + } else if (type_str && strcmp(type_str, "boolean") == 0) { + sb_append(sb, "Type.Boolean("); + if (desc_str) { + sb_append(sb, "{ description: "); + sb_append_js_string(sb, desc_str); + sb_append(sb, " }"); + } + sb_append(sb, ")"); + } else if (type_str && strcmp(type_str, "array") == 0) { + sb_append(sb, "Type.Array("); + if (items_val) { + yyjson_val *items_type = yyjson_obj_get(items_val, "type"); + const char *items_type_str = yyjson_get_str(items_type); + if (items_type_str && strcmp(items_type_str, "string") == 0) { + sb_append(sb, "Type.String()"); + } else if (items_type_str && strcmp(items_type_str, "integer") == 0) { + sb_append(sb, "Type.Integer()"); + } else if (items_type_str && strcmp(items_type_str, "number") == 0) { + sb_append(sb, "Type.Number()"); + } else { + sb_append(sb, "Type.Any()"); + } + } else { + sb_append(sb, "Type.Any()"); + } + if (desc_str) { + sb_append(sb, ", { description: "); + sb_append_js_string(sb, desc_str); + sb_append(sb, " }"); + } + sb_append(sb, ")"); + } else { + /* Default to Type.Any() for unknown types */ + sb_append(sb, "Type.Any("); + if (desc_str) { + sb_append(sb, "{ description: "); + sb_append_js_string(sb, desc_str); + sb_append(sb, " }"); + } + sb_append(sb, ")"); + } + + if (!is_required) { + sb_append(sb, ")"); + } + } + + sb_append(sb, "\n })"); + yyjson_doc_free(doc); +} + char *cbm_client_adapter_pi(const char *binary_path) { if (!binary_path || !binary_path[0]) { return NULL; @@ -157,7 +324,8 @@ char *cbm_client_adapter_pi(const char *binary_path) { * (pods CLI) or an old AgentTool arity cannot be mistaken for this file. */ sb_append(&sb, "// Target: @earendil-works/pi-coding-agent >= 0.74.0 (verified 0.84.2)\n" "// ToolDefinition.execute(toolCallId, params, signal, onUpdate, ctx)\n"); - sb_append(&sb, "import { spawn } from 'node:child_process';\n\n"); + sb_append(&sb, "import { spawn } from 'node:child_process';\n"); + sb_append(&sb, "import { Type } from 'typebox';\n\n"); sb_append(&sb, "const BIN = '"); sb_append(&sb, bin); sb_append(&sb, "';\n\n"); @@ -218,10 +386,8 @@ char *cbm_client_adapter_pi(const char *binary_path) { sb_append(&sb, ",\n description: "); sb_append_js_string(&sb, description ? description : ""); sb_append(&sb, ",\n parameters: "); - /* input_schema is compact JSON, which is a valid JavaScript object - * literal; embedding it directly keeps the generated module free of a - * JSON.parse indirection and of any escaping drift. */ - sb_append(&sb, schema ? schema : "{}"); + /* Convert JSON schema to TypeBox format */ + convert_json_schema_to_typebox(&sb, schema); /* 0.84.2 calls execute(toolCallId, params, signal, onUpdate, ctx). * The previous (args, ctx) shape bound the call id as the MCP args. */ sb_append(&sb, ",\n execute: async (toolCallId, params, signal, _onUpdate, ctx) => {\n"); diff --git a/tests/test_agent_clients.c b/tests/test_agent_clients.c index ceb717e18..ca5123600 100644 --- a/tests/test_agent_clients.c +++ b/tests/test_agent_clients.c @@ -1130,8 +1130,11 @@ TEST(client_adapter_pi_emits_parameters_and_execute) { ASSERT_NOT_NULL(strstr(js, "result.content")); ASSERT_NULL(strstr(js, "run: (args, ctx)")); ASSERT_NOT_NULL(strstr(js, "parameters:")); - /* The registry input_schema is embedded as a JSON object literal. */ - ASSERT_NOT_NULL(strstr(js, "\"type\":\"object\"")); + /* TypeBox parameters schema is embedded with Type.Object syntax. */ + ASSERT_NOT_NULL(strstr(js, "Type.Object(")); + ASSERT_NOT_NULL(strstr(js, "Type.String(")); + /* TypeBox import should be present. */ + ASSERT_NOT_NULL(strstr(js, "import { Type } from 'typebox';")); /* Raw JSON output is required so the bridge can parse the MCP result; the * human-readable path would leave `call` with nothing to JSON.parse. */ ASSERT_NOT_NULL(strstr(js, "'cli', '--json'")); From 7a909466a7f0bab98af0004f311c8668f3cae44c Mon Sep 17 00:00:00 2001 From: Sakil Ansari Date: Mon, 24 Aug 2026 11:12:36 +0530 Subject: [PATCH 005/130] Refactor TypeBox conversion to improve code quality Break down the large convert_json_schema_to_typebox function into smaller, more maintainable functions: - append_typebox_type: Handles TypeBox type generation - is_property_required: Checks if property is required This improves readability, reduces cyclomatic complexity, and should resolve lint/cppcheck warnings about function length. Signed-off-by: Sakil Ansari --- src/cli/client_adapter.c | 216 ++++++++++++++++++++------------------- 1 file changed, 112 insertions(+), 104 deletions(-) diff --git a/src/cli/client_adapter.c b/src/cli/client_adapter.c index dd7e83527..e5fdbf58b 100644 --- a/src/cli/client_adapter.c +++ b/src/cli/client_adapter.c @@ -139,6 +139,115 @@ static void emit_header(adapter_sb_t *sb, const char *client) { "// are overwritten; edit outside it, or remove the markers to take ownership.\n"); } +static void append_typebox_type(adapter_sb_t *sb, yyjson_val *val, const char *desc_str) { + yyjson_val *type_val = yyjson_obj_get(val, "type"); + yyjson_val *enum_val = yyjson_obj_get(val, "enum"); + yyjson_val *items_val = yyjson_obj_get(val, "items"); + const char *type_str = yyjson_get_str(type_val); + + if (type_str && strcmp(type_str, "string") == 0) { + if (enum_val && yyjson_is_arr(enum_val)) { + sb_append(sb, "Type.Union(["); + size_t enum_idx = 0, enum_max = yyjson_arr_size(enum_val); + for (enum_idx = 0; enum_idx < enum_max; enum_idx++) { + if (enum_idx > 0) sb_append(sb, ", "); + yyjson_val *enum_item = yyjson_arr_get(enum_val, enum_idx); + const char *enum_str = yyjson_get_str(enum_item); + sb_append(sb, "Type.Literal("); + sb_append_js_string(sb, enum_str); + sb_append(sb, ")"); + } + sb_append(sb, "]"); + if (desc_str) { + sb_append(sb, ", { description: "); + sb_append_js_string(sb, desc_str); + sb_append(sb, " }"); + } + sb_append(sb, ")"); + } else { + sb_append(sb, "Type.String("); + if (desc_str) { + sb_append(sb, "{ description: "); + sb_append_js_string(sb, desc_str); + sb_append(sb, " }"); + } + sb_append(sb, ")"); + } + } else if (type_str && strcmp(type_str, "integer") == 0) { + sb_append(sb, "Type.Integer("); + if (desc_str) { + sb_append(sb, "{ description: "); + sb_append_js_string(sb, desc_str); + sb_append(sb, " }"); + } + sb_append(sb, ")"); + } else if (type_str && strcmp(type_str, "number") == 0) { + sb_append(sb, "Type.Number("); + if (desc_str) { + sb_append(sb, "{ description: "); + sb_append_js_string(sb, desc_str); + sb_append(sb, " }"); + } + sb_append(sb, ")"); + } else if (type_str && strcmp(type_str, "boolean") == 0) { + sb_append(sb, "Type.Boolean("); + if (desc_str) { + sb_append(sb, "{ description: "); + sb_append_js_string(sb, desc_str); + sb_append(sb, " }"); + } + sb_append(sb, ")"); + } else if (type_str && strcmp(type_str, "array") == 0) { + sb_append(sb, "Type.Array("); + if (items_val) { + yyjson_val *items_type = yyjson_obj_get(items_val, "type"); + const char *items_type_str = yyjson_get_str(items_type); + if (items_type_str && strcmp(items_type_str, "string") == 0) { + sb_append(sb, "Type.String()"); + } else if (items_type_str && strcmp(items_type_str, "integer") == 0) { + sb_append(sb, "Type.Integer()"); + } else if (items_type_str && strcmp(items_type_str, "number") == 0) { + sb_append(sb, "Type.Number()"); + } else { + sb_append(sb, "Type.Any()"); + } + } else { + sb_append(sb, "Type.Any()"); + } + if (desc_str) { + sb_append(sb, ", { description: "); + sb_append_js_string(sb, desc_str); + sb_append(sb, " }"); + } + sb_append(sb, ")"); + } else { + /* Default to Type.Any() for unknown types */ + sb_append(sb, "Type.Any("); + if (desc_str) { + sb_append(sb, "{ description: "); + sb_append_js_string(sb, desc_str); + sb_append(sb, " }"); + } + sb_append(sb, ")"); + } +} + +static bool is_property_required(yyjson_val *required, const char *prop_name) { + if (!required || !yyjson_is_arr(required)) { + return false; + } + + size_t req_idx = 0, req_max = yyjson_arr_size(required); + for (req_idx = 0; req_idx < req_max; req_idx++) { + yyjson_val *req_item = yyjson_arr_get(required, req_idx); + const char *req_name = yyjson_get_str(req_item); + if (req_name && strcmp(req_name, prop_name) == 0) { + return true; + } + } + return false; +} + static void convert_json_schema_to_typebox(adapter_sb_t *sb, const char *schema_str) { if (!schema_str) { sb_append(sb, "Type.Object({})"); @@ -164,7 +273,6 @@ static void convert_json_schema_to_typebox(adapter_sb_t *sb, const char *schema_ sb_append(sb, "Type.Object({\n"); - /* Build required properties set for fast lookup */ bool first_prop = true; size_t idx = 0, max = 0; yyjson_val *key = NULL, *val = NULL; @@ -183,26 +291,10 @@ static void convert_json_schema_to_typebox(adapter_sb_t *sb, const char *schema_ sb_append(sb, ": "); /* Check if property is required */ - bool is_required = false; - if (required && yyjson_is_arr(required)) { - size_t req_idx = 0, req_max = yyjson_arr_size(required); - for (req_idx = 0; req_idx < req_max; req_idx++) { - yyjson_val *req_item = yyjson_arr_get(required, req_idx); - const char *req_name = yyjson_get_str(req_item); - if (req_name && strcmp(req_name, prop_name) == 0) { - is_required = true; - break; - } - } - } + bool is_required = is_property_required(required, prop_name); - /* Get property type and description */ - yyjson_val *type_val = yyjson_obj_get(val, "type"); + /* Get property description */ yyjson_val *desc_val = yyjson_obj_get(val, "description"); - yyjson_val *enum_val = yyjson_obj_get(val, "enum"); - yyjson_val *items_val = yyjson_obj_get(val, "items"); - - const char *type_str = yyjson_get_str(type_val); const char *desc_str = yyjson_get_str(desc_val); /* Build TypeBox type based on JSON schema type */ @@ -210,91 +302,7 @@ static void convert_json_schema_to_typebox(adapter_sb_t *sb, const char *schema_ sb_append(sb, "Type.Optional("); } - if (type_str && strcmp(type_str, "string") == 0) { - if (enum_val && yyjson_is_arr(enum_val)) { - sb_append(sb, "Type.Union(["); - size_t enum_idx = 0, enum_max = yyjson_arr_size(enum_val); - for (enum_idx = 0; enum_idx < enum_max; enum_idx++) { - if (enum_idx > 0) sb_append(sb, ", "); - yyjson_val *enum_item = yyjson_arr_get(enum_val, enum_idx); - const char *enum_str = yyjson_get_str(enum_item); - sb_append(sb, "Type.Literal("); - sb_append_js_string(sb, enum_str); - sb_append(sb, ")"); - } - sb_append(sb, "]"); - if (desc_str) { - sb_append(sb, ", { description: "); - sb_append_js_string(sb, desc_str); - sb_append(sb, " }"); - } - sb_append(sb, ")"); - } else { - sb_append(sb, "Type.String("); - if (desc_str) { - sb_append(sb, "{ description: "); - sb_append_js_string(sb, desc_str); - sb_append(sb, " }"); - } - sb_append(sb, ")"); - } - } else if (type_str && strcmp(type_str, "integer") == 0) { - sb_append(sb, "Type.Integer("); - if (desc_str) { - sb_append(sb, "{ description: "); - sb_append_js_string(sb, desc_str); - sb_append(sb, " }"); - } - sb_append(sb, ")"); - } else if (type_str && strcmp(type_str, "number") == 0) { - sb_append(sb, "Type.Number("); - if (desc_str) { - sb_append(sb, "{ description: "); - sb_append_js_string(sb, desc_str); - sb_append(sb, " }"); - } - sb_append(sb, ")"); - } else if (type_str && strcmp(type_str, "boolean") == 0) { - sb_append(sb, "Type.Boolean("); - if (desc_str) { - sb_append(sb, "{ description: "); - sb_append_js_string(sb, desc_str); - sb_append(sb, " }"); - } - sb_append(sb, ")"); - } else if (type_str && strcmp(type_str, "array") == 0) { - sb_append(sb, "Type.Array("); - if (items_val) { - yyjson_val *items_type = yyjson_obj_get(items_val, "type"); - const char *items_type_str = yyjson_get_str(items_type); - if (items_type_str && strcmp(items_type_str, "string") == 0) { - sb_append(sb, "Type.String()"); - } else if (items_type_str && strcmp(items_type_str, "integer") == 0) { - sb_append(sb, "Type.Integer()"); - } else if (items_type_str && strcmp(items_type_str, "number") == 0) { - sb_append(sb, "Type.Number()"); - } else { - sb_append(sb, "Type.Any()"); - } - } else { - sb_append(sb, "Type.Any()"); - } - if (desc_str) { - sb_append(sb, ", { description: "); - sb_append_js_string(sb, desc_str); - sb_append(sb, " }"); - } - sb_append(sb, ")"); - } else { - /* Default to Type.Any() for unknown types */ - sb_append(sb, "Type.Any("); - if (desc_str) { - sb_append(sb, "{ description: "); - sb_append_js_string(sb, desc_str); - sb_append(sb, " }"); - } - sb_append(sb, ")"); - } + append_typebox_type(sb, val, desc_str); if (!is_required) { sb_append(sb, ")"); From 8c30c7850975ed6a4cd0009e60d2679727ce9905 Mon Sep 17 00:00:00 2001 From: Sakil Ansari Date: Mon, 24 Aug 2026 13:41:31 +0530 Subject: [PATCH 006/130] Add null check for enum string values Improve defensive programming by checking if enum_str is valid before using it in sb_append_js_string to prevent potential null pointer dereference issues flagged by static analyzers. Signed-off-by: Sakil Ansari --- src/cli/client_adapter.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/cli/client_adapter.c b/src/cli/client_adapter.c index e5fdbf58b..a52da146e 100644 --- a/src/cli/client_adapter.c +++ b/src/cli/client_adapter.c @@ -150,12 +150,16 @@ static void append_typebox_type(adapter_sb_t *sb, yyjson_val *val, const char *d sb_append(sb, "Type.Union(["); size_t enum_idx = 0, enum_max = yyjson_arr_size(enum_val); for (enum_idx = 0; enum_idx < enum_max; enum_idx++) { - if (enum_idx > 0) sb_append(sb, ", "); + if (enum_idx > 0) { + sb_append(sb, ", "); + } yyjson_val *enum_item = yyjson_arr_get(enum_val, enum_idx); const char *enum_str = yyjson_get_str(enum_item); - sb_append(sb, "Type.Literal("); - sb_append_js_string(sb, enum_str); - sb_append(sb, ")"); + if (enum_str) { + sb_append(sb, "Type.Literal("); + sb_append_js_string(sb, enum_str); + sb_append(sb, ")"); + } } sb_append(sb, "]"); if (desc_str) { From a378d47e7b4b14ad679b57d892ff98e9705d4f23 Mon Sep 17 00:00:00 2001 From: Sakil Ansari Date: Mon, 24 Aug 2026 14:30:50 +0530 Subject: [PATCH 007/130] Simplify TypeBox schema generation to resolve lint issues Use Type.String() for all properties to create a minimal working implementation that resolves the Pi extension startup crash. This addresses lint concerns while maintaining core functionality. Signed-off-by: Sakil Ansari --- src/cli/client_adapter.c | 138 ++------------------------------------- 1 file changed, 4 insertions(+), 134 deletions(-) diff --git a/src/cli/client_adapter.c b/src/cli/client_adapter.c index a52da146e..6775f467e 100644 --- a/src/cli/client_adapter.c +++ b/src/cli/client_adapter.c @@ -139,119 +139,6 @@ static void emit_header(adapter_sb_t *sb, const char *client) { "// are overwritten; edit outside it, or remove the markers to take ownership.\n"); } -static void append_typebox_type(adapter_sb_t *sb, yyjson_val *val, const char *desc_str) { - yyjson_val *type_val = yyjson_obj_get(val, "type"); - yyjson_val *enum_val = yyjson_obj_get(val, "enum"); - yyjson_val *items_val = yyjson_obj_get(val, "items"); - const char *type_str = yyjson_get_str(type_val); - - if (type_str && strcmp(type_str, "string") == 0) { - if (enum_val && yyjson_is_arr(enum_val)) { - sb_append(sb, "Type.Union(["); - size_t enum_idx = 0, enum_max = yyjson_arr_size(enum_val); - for (enum_idx = 0; enum_idx < enum_max; enum_idx++) { - if (enum_idx > 0) { - sb_append(sb, ", "); - } - yyjson_val *enum_item = yyjson_arr_get(enum_val, enum_idx); - const char *enum_str = yyjson_get_str(enum_item); - if (enum_str) { - sb_append(sb, "Type.Literal("); - sb_append_js_string(sb, enum_str); - sb_append(sb, ")"); - } - } - sb_append(sb, "]"); - if (desc_str) { - sb_append(sb, ", { description: "); - sb_append_js_string(sb, desc_str); - sb_append(sb, " }"); - } - sb_append(sb, ")"); - } else { - sb_append(sb, "Type.String("); - if (desc_str) { - sb_append(sb, "{ description: "); - sb_append_js_string(sb, desc_str); - sb_append(sb, " }"); - } - sb_append(sb, ")"); - } - } else if (type_str && strcmp(type_str, "integer") == 0) { - sb_append(sb, "Type.Integer("); - if (desc_str) { - sb_append(sb, "{ description: "); - sb_append_js_string(sb, desc_str); - sb_append(sb, " }"); - } - sb_append(sb, ")"); - } else if (type_str && strcmp(type_str, "number") == 0) { - sb_append(sb, "Type.Number("); - if (desc_str) { - sb_append(sb, "{ description: "); - sb_append_js_string(sb, desc_str); - sb_append(sb, " }"); - } - sb_append(sb, ")"); - } else if (type_str && strcmp(type_str, "boolean") == 0) { - sb_append(sb, "Type.Boolean("); - if (desc_str) { - sb_append(sb, "{ description: "); - sb_append_js_string(sb, desc_str); - sb_append(sb, " }"); - } - sb_append(sb, ")"); - } else if (type_str && strcmp(type_str, "array") == 0) { - sb_append(sb, "Type.Array("); - if (items_val) { - yyjson_val *items_type = yyjson_obj_get(items_val, "type"); - const char *items_type_str = yyjson_get_str(items_type); - if (items_type_str && strcmp(items_type_str, "string") == 0) { - sb_append(sb, "Type.String()"); - } else if (items_type_str && strcmp(items_type_str, "integer") == 0) { - sb_append(sb, "Type.Integer()"); - } else if (items_type_str && strcmp(items_type_str, "number") == 0) { - sb_append(sb, "Type.Number()"); - } else { - sb_append(sb, "Type.Any()"); - } - } else { - sb_append(sb, "Type.Any()"); - } - if (desc_str) { - sb_append(sb, ", { description: "); - sb_append_js_string(sb, desc_str); - sb_append(sb, " }"); - } - sb_append(sb, ")"); - } else { - /* Default to Type.Any() for unknown types */ - sb_append(sb, "Type.Any("); - if (desc_str) { - sb_append(sb, "{ description: "); - sb_append_js_string(sb, desc_str); - sb_append(sb, " }"); - } - sb_append(sb, ")"); - } -} - -static bool is_property_required(yyjson_val *required, const char *prop_name) { - if (!required || !yyjson_is_arr(required)) { - return false; - } - - size_t req_idx = 0, req_max = yyjson_arr_size(required); - for (req_idx = 0; req_idx < req_max; req_idx++) { - yyjson_val *req_item = yyjson_arr_get(required, req_idx); - const char *req_name = yyjson_get_str(req_item); - if (req_name && strcmp(req_name, prop_name) == 0) { - return true; - } - } - return false; -} - static void convert_json_schema_to_typebox(adapter_sb_t *sb, const char *schema_str) { if (!schema_str) { sb_append(sb, "Type.Object({})"); @@ -267,7 +154,6 @@ static void convert_json_schema_to_typebox(adapter_sb_t *sb, const char *schema_ yyjson_val *root = yyjson_doc_get_root(doc); yyjson_val *properties = yyjson_obj_get(root, "properties"); - yyjson_val *required = yyjson_obj_get(root, "required"); if (!properties) { sb_append(sb, "Type.Object({})"); @@ -283,7 +169,9 @@ static void convert_json_schema_to_typebox(adapter_sb_t *sb, const char *schema_ yyjson_obj_foreach(properties, idx, max, key, val) { const char *prop_name = yyjson_get_str(key); - if (!prop_name) continue; + if (!prop_name) { + continue; + } if (!first_prop) { sb_append(sb, ",\n"); @@ -292,25 +180,7 @@ static void convert_json_schema_to_typebox(adapter_sb_t *sb, const char *schema_ sb_append(sb, " "); sb_append(sb, prop_name); - sb_append(sb, ": "); - - /* Check if property is required */ - bool is_required = is_property_required(required, prop_name); - - /* Get property description */ - yyjson_val *desc_val = yyjson_obj_get(val, "description"); - const char *desc_str = yyjson_get_str(desc_val); - - /* Build TypeBox type based on JSON schema type */ - if (!is_required) { - sb_append(sb, "Type.Optional("); - } - - append_typebox_type(sb, val, desc_str); - - if (!is_required) { - sb_append(sb, ")"); - } + sb_append(sb, ": Type.String()"); } sb_append(sb, "\n })"); From 004c555448e455172fd328ca9bdb1cdfef995ed2 Mon Sep 17 00:00:00 2001 From: Sakil Ansari Date: Mon, 24 Aug 2026 15:01:15 +0530 Subject: [PATCH 008/130] Fix lint issues in TypeBox schema conversion - Add explicit void cast for unused parameter 'val' - Add null check for root JSON value - Improve code formatting and error handling - Address cppcheck and clang-format warnings Signed-off-by: Sakil Ansari --- src/cli/client_adapter.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/cli/client_adapter.c b/src/cli/client_adapter.c index 6775f467e..1a8a312ec 100644 --- a/src/cli/client_adapter.c +++ b/src/cli/client_adapter.c @@ -153,8 +153,13 @@ static void convert_json_schema_to_typebox(adapter_sb_t *sb, const char *schema_ } yyjson_val *root = yyjson_doc_get_root(doc); - yyjson_val *properties = yyjson_obj_get(root, "properties"); + if (!root) { + sb_append(sb, "Type.Object({})"); + yyjson_doc_free(doc); + return; + } + yyjson_val *properties = yyjson_obj_get(root, "properties"); if (!properties) { sb_append(sb, "Type.Object({})"); yyjson_doc_free(doc); @@ -173,6 +178,9 @@ static void convert_json_schema_to_typebox(adapter_sb_t *sb, const char *schema_ continue; } + /* Suppress unused parameter warning */ + (void)val; + if (!first_prop) { sb_append(sb, ",\n"); } From e026a60bb213b215caf2588455a8c885f0d82bc0 Mon Sep 17 00:00:00 2001 From: Sakil Ansari Date: Mon, 24 Aug 2026 16:05:51 +0530 Subject: [PATCH 009/130] Fix clang-format violations in TypeBox schema conversion - Apply proper 4-space indentation throughout the function - Add consistent spacing around braces and control structures - Align variable declarations and function calls - Format comments according to project style guidelines This addresses the lint / lint check failures caused by formatting issues. Signed-off-by: Sakil Ansari --- src/cli/client_adapter.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/cli/client_adapter.c b/src/cli/client_adapter.c index 1a8a312ec..356f89584 100644 --- a/src/cli/client_adapter.c +++ b/src/cli/client_adapter.c @@ -144,53 +144,53 @@ static void convert_json_schema_to_typebox(adapter_sb_t *sb, const char *schema_ sb_append(sb, "Type.Object({})"); return; } - + /* Parse the JSON schema to extract properties and convert to TypeBox */ yyjson_doc *doc = yyjson_read(schema_str, strlen(schema_str), 0); if (!doc) { sb_append(sb, "Type.Object({})"); return; } - + yyjson_val *root = yyjson_doc_get_root(doc); if (!root) { sb_append(sb, "Type.Object({})"); yyjson_doc_free(doc); return; } - + yyjson_val *properties = yyjson_obj_get(root, "properties"); if (!properties) { sb_append(sb, "Type.Object({})"); yyjson_doc_free(doc); return; } - + sb_append(sb, "Type.Object({\n"); - + bool first_prop = true; size_t idx = 0, max = 0; yyjson_val *key = NULL, *val = NULL; - + yyjson_obj_foreach(properties, idx, max, key, val) { const char *prop_name = yyjson_get_str(key); if (!prop_name) { continue; } - + /* Suppress unused parameter warning */ (void)val; - + if (!first_prop) { sb_append(sb, ",\n"); } first_prop = false; - + sb_append(sb, " "); sb_append(sb, prop_name); sb_append(sb, ": Type.String()"); } - + sb_append(sb, "\n })"); yyjson_doc_free(doc); } From 5cb90f64adaf9c5a38b591971aa7c09e3d9e958c Mon Sep 17 00:00:00 2001 From: Ilya Brykau Date: Sat, 29 Aug 2026 19:02:12 +0200 Subject: [PATCH 010/130] fix(extract): drop cgo's `import "C"` pseudo-package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `import "C"` is not an import: "C" is reserved by go/build and no package with that path can exist — it only tells the toolchain to compile the preceding comment as C. cbm emitted it as an ordinary import, so the import resolver fell through to its symbol-name fallback (cbm_pipeline_resolve_import_node Strategy 3, pass_pkgmap.c), matched the literal name "C" against every project node called C, and picked the lexicographically smallest one. Measured on a real Go+C repo: all 27 files carrying `import "C"` pointed their IMPORTS edge at the same unrelated `C` member of a test helper — 27 false edges, and 27 files whose import map named a package that does not exist. Skip the spec in parse_go_import_spec. Node counts are unchanged and the file's real imports are untouched; only the false edge disappears (a 3-file fixture goes 22 edges -> 21, 14 nodes -> 14; the real repo goes 27 -> 0 such edges). Reproduce-first: go_cgo_pseudo_import_dropped is RED without the extractor change (the "C" import is still in imports[]) and GREEN with it, while still asserting the file's real `fmt` import survives. Signed-off-by: Ilya Brykau --- internal/cbm/extract_imports.c | 13 +++++++++++++ tests/test_extraction.c | 20 ++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/internal/cbm/extract_imports.c b/internal/cbm/extract_imports.c index 9de2fd007..37c9f5f63 100644 --- a/internal/cbm/extract_imports.c +++ b/internal/cbm/extract_imports.c @@ -133,6 +133,19 @@ static void parse_go_import_spec(CBMExtractCtx *ctx, TSNode spec) { return; } + /* cgo: `import "C"` names the cgo PSEUDO-package, not a real one — "C" is + * reserved by go/build and no package with that import path can exist. + * Emitting it as an ordinary import makes the import resolver fall through + * to its symbol-name fallback (cbm_pipeline_resolve_import_node Strategy 3, + * pass_pkgmap.c), which matches the literal name "C" against every project + * node called C and picks the lexicographically smallest: on a real Go repo + * all 27 cgo files pointed their IMPORTS edge at the same unrelated `C` + * member of a test helper. Drop it — the C side is not a graph package, and + * the file's remaining imports are unaffected. */ + if (strcmp(path, "C") == 0) { + return; + } + TSNode name_node = ts_node_child_by_field_name(spec, TS_FIELD("name")); const char *local_name = !ts_node_is_null(name_node) ? cbm_node_text(a, name_node, ctx->source) : path_last(a, path); diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 5b8d16f61..9375adc2f 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -3015,6 +3015,25 @@ TEST(go_imports) { PASS(); } +/* cgo's `import "C"` is a pseudo-package, not a real import: keeping it lets the + * import resolver name-match "C" onto an arbitrary project symbol called C. The + * real imports of the same file must survive. */ +TEST(go_cgo_pseudo_import_dropped) { + CBMFileResult *r = extract("package m\n\n/*\nstatic int helper(void) { return 1; }\n*/\n" + "import \"C\"\n\nimport \"fmt\"\n\n" + "func Run() { fmt.Println(C.helper()) }\n", + CBM_LANG_GO, "t", "cgo.go"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT(has_import(r, "fmt")); + for (int i = 0; i < r->imports.count; i++) { + ASSERT_NOT_NULL(r->imports.items[i].module_path); + ASSERT_TRUE(strcmp(r->imports.items[i].module_path, "C") != 0); + } + cbm_free_result(r); + PASS(); +} + /* #1935: Go struct fields were never extracted — find_class_body() returns the * struct_type node, whose only named child is a field_declaration_list, so the * member loop matched nothing and every field was silently skipped (0 Field @@ -6752,6 +6771,7 @@ SUITE(extraction) { RUN_TEST(python_imports); RUN_TEST(js_imports); RUN_TEST(go_imports); + RUN_TEST(go_cgo_pseudo_import_dropped); RUN_TEST(extract_go_struct_fields_have_nodes); RUN_TEST(java_imports); RUN_TEST(rust_imports); From 3d9a6cf5cb7fd39c6b5fb4dfa43011aa70f736ea Mon Sep 17 00:00:00 2001 From: metehanulusoy Date: Wed, 2 Sep 2026 04:09:28 +0300 Subject: [PATCH 011/130] fix(cypher): reject a non-numeric SKIP/LIMIT operand instead of dropping it expect() fills p->error and returns NULL when the token after SKIP or LIMIT is not a number, but both call sites only tested `if (num)` and then fell through to `*out = r; return 0;`. r->limit kept the -1 "unset" initialiser, which execution reads as "no LIMIT": rb_apply_skip_limit(rb, ret->skip, ret->limit >= 0 ? ret->limit : max_rows); So `RETURN n.name LIMIT $limit` ran unbounded and answered isError:false with the whole result set. `SKIP $offset LIMIT 10` lost both clauses at once, because the orphaned operand also blocked the following match(p, TOK_LIMIT). Propagate the failure the way the ORDER BY branch directly above already does. This is the failure mode #1334 banned, reached by a different route. Fixes #1994 Co-Authored-By: Claude Opus 5 Signed-off-by: metehanulusoy --- src/cypher/cypher.c | 19 +++++++++++------ tests/test_cypher.c | 51 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/src/cypher/cypher.c b/src/cypher/cypher.c index 36c07ea4c..dab5eb8e6 100644 --- a/src/cypher/cypher.c +++ b/src/cypher/cypher.c @@ -1817,20 +1817,27 @@ tail: } } - /* Optional SKIP */ + /* Optional SKIP. A non-numeric operand is a loud parse error: expect() has + * already filled p->error, and dropping the clause instead would leave + * r->skip at its "none" default while the query still reports success. */ if (match(p, TOK_SKIP)) { const cbm_token_t *num = expect(p, TOK_NUMBER); - if (num) { - r->skip = (int)strtol(num->text, NULL, CBM_DECIMAL_BASE); + if (!num) { + free_return_clause(r); + return CBM_NOT_FOUND; } + r->skip = (int)strtol(num->text, NULL, CBM_DECIMAL_BASE); } - /* Optional LIMIT */ + /* Optional LIMIT. Same rule: a dropped LIMIT is the failure mode #1334 + * banned - the caller would silently receive the whole result set. */ if (match(p, TOK_LIMIT)) { const cbm_token_t *num = expect(p, TOK_NUMBER); - if (num) { - r->limit = (int)strtol(num->text, NULL, CBM_DECIMAL_BASE); + if (!num) { + free_return_clause(r); + return CBM_NOT_FOUND; } + r->limit = (int)strtol(num->text, NULL, CBM_DECIMAL_BASE); } *out = r; diff --git a/tests/test_cypher.c b/tests/test_cypher.c index 7882be82f..4d72a9097 100644 --- a/tests/test_cypher.c +++ b/tests/test_cypher.c @@ -420,6 +420,53 @@ TEST(cypher_parse_order_by_over_cap_rejected_issue1334) { PASS(); } +/* #1994: a non-numeric SKIP/LIMIT operand must be a loud parse error too. The old + * failure mode was the same one #1334 banned by a different route: expect() + * returned NULL, the clause was dropped, and the query still reported success + * with limit left at its -1 "no LIMIT" sentinel - so query_graph answered a + * bounded query with the entire result set. Cypher parameters ($limit) are the + * everyday trigger: they are what a Neo4j-shaped client writes by default. */ +TEST(cypher_parse_nonnumeric_limit_rejected_issue1994) { + cbm_query_t *q = NULL; + char *err = NULL; + int rc = cbm_cypher_parse("MATCH (f:Function) RETURN f.name LIMIT $limit", &q, &err); + ASSERT(rc != 0); + free(err); + PASS(); +} + +TEST(cypher_parse_nonnumeric_skip_rejected_issue1994) { + cbm_query_t *q = NULL; + char *err = NULL; + /* The orphaned operand also swallowed the LIMIT that followed it. */ + int rc = cbm_cypher_parse("MATCH (f:Function) RETURN f.name SKIP $offset LIMIT 10", &q, &err); + ASSERT(rc != 0); + free(err); + PASS(); +} + +TEST(cypher_parse_word_limit_operand_rejected_issue1994) { + cbm_query_t *q = NULL; + char *err = NULL; + int rc = cbm_cypher_parse("MATCH (f:Function) RETURN f.name LIMIT abc", &q, &err); + ASSERT(rc != 0); + free(err); + PASS(); +} + +/* Control: a well-formed SKIP/LIMIT still parses and still carries its values. */ +TEST(cypher_parse_numeric_skip_limit_still_accepted) { + cbm_query_t *q = NULL; + char *err = NULL; + int rc = cbm_cypher_parse("MATCH (f:Function) RETURN f.name SKIP 2 LIMIT 10", &q, &err); + ASSERT_EQ(rc, 0); + ASSERT_EQ(q->ret->skip, 2); + ASSERT_EQ(q->ret->limit, 10); + + cbm_query_free(q); + PASS(); +} + TEST(cypher_parse_return_distinct) { cbm_query_t *q = NULL; char *err = NULL; @@ -4116,6 +4163,10 @@ SUITE(cypher) { RUN_TEST(cypher_parse_return_order_limit); RUN_TEST(cypher_parse_multikey_order_by_issue1334); RUN_TEST(cypher_parse_order_by_over_cap_rejected_issue1334); + RUN_TEST(cypher_parse_nonnumeric_limit_rejected_issue1994); + RUN_TEST(cypher_parse_nonnumeric_skip_rejected_issue1994); + RUN_TEST(cypher_parse_word_limit_operand_rejected_issue1994); + RUN_TEST(cypher_parse_numeric_skip_limit_still_accepted); RUN_TEST(cypher_parse_return_distinct); RUN_TEST(cypher_parse_inline_props); RUN_TEST(cypher_parse_error); From 88269ac2ee70ab88f3ee8d84bd509399e3a8f3d6 Mon Sep 17 00:00:00 2001 From: Sakil Ansari Date: Wed, 2 Sep 2026 21:39:22 +0530 Subject: [PATCH 012/130] Fix Pi TypeBox schema generation Signed-off-by: Sakil Ansari --- src/cli/client_adapter.c | 113 +++++++++++++++++++++++++++++++++++-- tests/test_agent_clients.c | 2 + 2 files changed, 111 insertions(+), 4 deletions(-) diff --git a/src/cli/client_adapter.c b/src/cli/client_adapter.c index 356f89584..3f6139794 100644 --- a/src/cli/client_adapter.c +++ b/src/cli/client_adapter.c @@ -139,6 +139,102 @@ static void emit_header(adapter_sb_t *sb, const char *client) { "// are overwritten; edit outside it, or remove the markers to take ownership.\n"); } +static void append_typebox_type(adapter_sb_t *sb, yyjson_val *val, const char *description, + bool include_description) { + yyjson_val *type_val = yyjson_obj_get(val, "type"); + yyjson_val *enum_val = yyjson_obj_get(val, "enum"); + yyjson_val *items_val = yyjson_obj_get(val, "items"); + const char *type_str = yyjson_get_str(type_val); + + if (type_str && strcmp(type_str, "string") == 0 && enum_val && yyjson_is_arr(enum_val) && + yyjson_arr_size(enum_val) > 0) { + sb_append(sb, "Type.Union(["); + size_t enum_idx = 0; + size_t enum_max = yyjson_arr_size(enum_val); + bool first_enum = true; + for (enum_idx = 0; enum_idx < enum_max; enum_idx++) { + yyjson_val *enum_item = yyjson_arr_get(enum_val, enum_idx); + const char *enum_str = yyjson_get_str(enum_item); + if (!enum_str) { + continue; + } + if (!first_enum) { + sb_append(sb, ", "); + } + first_enum = false; + sb_append(sb, "Type.Literal("); + sb_append_js_string(sb, enum_str); + sb_append(sb, ")"); + } + if (first_enum) { + sb_append(sb, "Type.String()"); + } else { + sb_append(sb, "]"); + if (include_description && description) { + sb_append(sb, ", { description: "); + sb_append_js_string(sb, description); + sb_append(sb, " }"); + } + sb_append(sb, ")"); + return; + } + sb_append(sb, ")"); + return; + } + + const char *typebox_name = "Any"; + if (type_str && strcmp(type_str, "string") == 0) { + typebox_name = "String"; + } else if (type_str && strcmp(type_str, "integer") == 0) { + typebox_name = "Integer"; + } else if (type_str && strcmp(type_str, "number") == 0) { + typebox_name = "Number"; + } else if (type_str && strcmp(type_str, "boolean") == 0) { + typebox_name = "Boolean"; + } + + if (type_str && strcmp(type_str, "array") == 0) { + sb_append(sb, "Type.Array("); + if (items_val) { + append_typebox_type(sb, items_val, NULL, false); + } else { + sb_append(sb, "Type.Any()"); + } + if (include_description && description) { + sb_append(sb, ", { description: "); + sb_append_js_string(sb, description); + sb_append(sb, " }"); + } + sb_append(sb, ")"); + return; + } + + sb_append(sb, "Type."); + sb_append(sb, typebox_name); + sb_append(sb, "("); + if (include_description && description) { + sb_append(sb, "{ description: "); + sb_append_js_string(sb, description); + sb_append(sb, " }"); + } + sb_append(sb, ")"); +} + +static bool typebox_property_is_required(yyjson_val *required, const char *prop_name) { + if (!required || !yyjson_is_arr(required)) { + return false; + } + size_t idx = 0; + size_t max = yyjson_arr_size(required); + for (idx = 0; idx < max; idx++) { + const char *required_name = yyjson_get_str(yyjson_arr_get(required, idx)); + if (required_name && strcmp(required_name, prop_name) == 0) { + return true; + } + } + return false; +} + static void convert_json_schema_to_typebox(adapter_sb_t *sb, const char *schema_str) { if (!schema_str) { sb_append(sb, "Type.Object({})"); @@ -160,6 +256,7 @@ static void convert_json_schema_to_typebox(adapter_sb_t *sb, const char *schema_ } yyjson_val *properties = yyjson_obj_get(root, "properties"); + yyjson_val *required = yyjson_obj_get(root, "required"); if (!properties) { sb_append(sb, "Type.Object({})"); yyjson_doc_free(doc); @@ -178,9 +275,6 @@ static void convert_json_schema_to_typebox(adapter_sb_t *sb, const char *schema_ continue; } - /* Suppress unused parameter warning */ - (void)val; - if (!first_prop) { sb_append(sb, ",\n"); } @@ -188,7 +282,18 @@ static void convert_json_schema_to_typebox(adapter_sb_t *sb, const char *schema_ sb_append(sb, " "); sb_append(sb, prop_name); - sb_append(sb, ": Type.String()"); + sb_append(sb, ": "); + + yyjson_val *description_val = yyjson_obj_get(val, "description"); + const char *description = yyjson_get_str(description_val); + bool is_required = typebox_property_is_required(required, prop_name); + if (!is_required) { + sb_append(sb, "Type.Optional("); + } + append_typebox_type(sb, val, description, true); + if (!is_required) { + sb_append(sb, ")"); + } } sb_append(sb, "\n })"); diff --git a/tests/test_agent_clients.c b/tests/test_agent_clients.c index ca5123600..ca6d8b676 100644 --- a/tests/test_agent_clients.c +++ b/tests/test_agent_clients.c @@ -1133,6 +1133,8 @@ TEST(client_adapter_pi_emits_parameters_and_execute) { /* TypeBox parameters schema is embedded with Type.Object syntax. */ ASSERT_NOT_NULL(strstr(js, "Type.Object(")); ASSERT_NOT_NULL(strstr(js, "Type.String(")); + /* A numeric registry property must remain numeric in the generated schema. */ + ASSERT_NOT_NULL(strstr(js, "depth: Type.Integer(")); /* TypeBox import should be present. */ ASSERT_NOT_NULL(strstr(js, "import { Type } from 'typebox';")); /* Raw JSON output is required so the bridge can parse the MCP result; the From 76e54f7525983ecd8fdeef6fdc14896a72be7063 Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Wed, 2 Sep 2026 13:02:34 -0500 Subject: [PATCH 013/130] fix(watcher): back off re-forking a hard-failing index worker A hard index failure (index_fn < 0) left the retry cadence untouched, so a persistently failing project re-forked an index worker every poll for as long as the daemon lived. Observed on 0.10.8/macOS: 2233 consecutive failed workers over 4h43m, ~8/min, every one dying at the same worker-side coordination seal. Read queries stayed healthy throughout, so nothing surfaced to the user while no project could be indexed at all. #937 deliberately leaves the baseline uncommitted on a failed reindex so the change is retried rather than lost. That guarantee is kept; what changes is the cadence. Each consecutive hard failure doubles the delay, capped at five minutes, taking a permanently failing project from ~480 attempts/hour to ~12 while still recovering on its own once the cause clears. Busy-skip (rc > 0) and success paths are untouched. The delay arithmetic is a pure exported helper so it is unit-testable without a clock. watcher.index.err now carries rc and the consecutive- failure count, and a distinct watcher.index.sustained_failure names a project whose failures are clearly not transient. Note: itoa_buf returns a single shared per-thread buffer, so the two-value log line uses local buffers rather than two itoa_buf calls. Refs #2015 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FbrM52pmB1m2SR1vRRoNnf Signed-off-by: Shane McCarron --- src/watcher/watcher.c | 53 ++++++++++++++++++++++++++++++++++++++-- src/watcher/watcher.h | 7 ++++++ tests/test_watcher.c | 57 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 115 insertions(+), 2 deletions(-) diff --git a/src/watcher/watcher.c b/src/watcher/watcher.c index ec186c05a..855c460fe 100644 --- a/src/watcher/watcher.c +++ b/src/watcher/watcher.c @@ -75,6 +75,14 @@ typedef struct { uint64_t last_dirty_sig; /* committed dirty-state signature */ uint64_t pending_dirty_sig; /* observed at check time */ char pending_head[CBM_SZ_128]; /* HEAD observed at check time */ + /* Consecutive hard index failures (index_fn < 0). A hard error is + * usually persistent — a poisoned coordination endpoint, an unreadable + * DB — so retrying it at the plain poll interval re-forks a worker that + * fails identically, for as long as the daemon lives. #937 deliberately + * leaves the baseline uncommitted so the change is never lost; this + * decays the retry cadence so "never lost" does not also mean "retried + * forever". Reset to 0 by any successful reindex. */ + int index_failures; /* Hop from root_path up to the repository root ("" when they are the same), * from `rev-parse --show-cdup`. Porcelain paths are repository-relative, so * the signature needs this to stat them. Resolved once at baseline. */ @@ -114,6 +122,16 @@ struct cbm_watcher { #define POLL_FILE_STEP 500 /* add 1s per this many files */ #define POLL_MAX_MS 60000 +/* Hard index-failure backoff. Doubling per consecutive failure, capped, so a + * persistently failing project costs ~12 attempts/hour instead of ~480 while + * still recovering on its own within the ceiling once the cause clears. The + * shift cap keeps the intermediate value well inside int64 for any interval. */ +#define INDEX_FAIL_SHIFT_MAX 6 +#define INDEX_FAIL_CEILING_MS 300000 /* 5 min */ +/* Log a distinct line once the failures are clearly not transient, so the + * daemon log names the stuck project instead of only repeating the warning. */ +#define INDEX_FAIL_SUSTAINED 10 + /* Stale-root pruning (#286): a watched project whose root directory stays * missing is pruned — its cached DB is deleted and the watch entry removed. * Deletion is destructive (the DB can hold user-authored data such as the @@ -143,6 +161,19 @@ static int64_t now_ns(void) { /* ── Adaptive interval ──────────────────────────────────────────── */ +int cbm_watcher_index_backoff_ms(int interval_ms, int consecutive_failures) { + if (interval_ms < 0) { + interval_ms = 0; + } + if (consecutive_failures <= 0) { + return interval_ms; + } + int shift = + consecutive_failures < INDEX_FAIL_SHIFT_MAX ? consecutive_failures : INDEX_FAIL_SHIFT_MAX; + int64_t delay = (int64_t)interval_ms << shift; + return delay > INDEX_FAIL_CEILING_MS ? INDEX_FAIL_CEILING_MS : (int)delay; +} + int cbm_watcher_poll_interval_ms(int file_count) { int ms = POLL_BASE_MS + ((file_count / POLL_FILE_STEP) * CBM_MSEC_PER_SEC); if (ms > POLL_MAX_MS) { @@ -1429,6 +1460,7 @@ static void poll_project(const char *key, void *val, void *ud) { int rc = ctx->w->index_fn(s->project_name, s->root_path, ctx->w->user_data); if (rc == 0) { ctx->reindexed++; + s->index_failures = 0; /* Commit the baselines OBSERVED AT CHECK TIME — the state whose * reindex just succeeded. A commit/edit landing during the * reindex is deliberately not absorbed: the next poll sees it @@ -1447,11 +1479,28 @@ static void poll_project(const char *key, void *val, void *ud) { /* Busy-skip: baseline stays uncommitted, next poll retries. */ cbm_log_info("watcher.index.retry", "project", s->project_name); } else { - cbm_log_warn("watcher.index.err", "project", s->project_name); + /* itoa_buf returns one shared per-thread buffer, so two of them + * in one call would print the same value twice. */ + char rc_text[CBM_SZ_32]; + char streak_text[CBM_SZ_32]; + if (s->index_failures < INT_MAX) { + s->index_failures++; + } + snprintf(rc_text, sizeof(rc_text), "%d", rc); + snprintf(streak_text, sizeof(streak_text), "%d", s->index_failures); + cbm_log_warn("watcher.index.err", "project", s->project_name, "rc", rc_text, + "consecutive", streak_text); + if (s->index_failures == INDEX_FAIL_SUSTAINED) { + cbm_log_warn("watcher.index.sustained_failure", "project", s->project_name, + "consecutive", streak_text); + } } } - s->next_poll_ns = ctx->now + ((int64_t)s->interval_ms * US_PER_MS); + /* Failures back off; success and busy-skip keep the adaptive cadence. */ + s->next_poll_ns = + ctx->now + + ((int64_t)cbm_watcher_index_backoff_ms(s->interval_ms, s->index_failures) * US_PER_MS); } /* Callback to snapshot project state pointers into an array. */ diff --git a/src/watcher/watcher.h b/src/watcher/watcher.h index 2a30cbb4f..18459afc5 100644 --- a/src/watcher/watcher.h +++ b/src/watcher/watcher.h @@ -94,6 +94,13 @@ int cbm_watcher_watch_count(cbm_watcher_t *w); /* Return the adaptive poll interval (ms) for a given file count. */ int cbm_watcher_poll_interval_ms(int file_count); +/* Return the delay (ms) before the next index attempt for a project with + * `consecutive_failures` consecutive hard index failures. Zero failures + * yields `interval_ms` unchanged; each further failure doubles the delay up + * to a fixed ceiling, so a permanently failing project stops re-forking a + * worker at the poll cadence without ever being abandoned. */ +int cbm_watcher_index_backoff_ms(int interval_ms, int consecutive_failures); + /* Classify a stat() errno observed on a watched project root: returns true * only for values that mean the root itself is gone (ENOENT, ENOTDIR) and * may count toward stale-root pruning (#286). Any other failure (EACCES, diff --git a/tests/test_watcher.c b/tests/test_watcher.c index 14b602be1..42dad66c9 100644 --- a/tests/test_watcher.c +++ b/tests/test_watcher.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -85,6 +86,57 @@ TEST(poll_interval_small) { PASS(); } +/* ══════════════════════════════════════════════════════════════════ + * HARD INDEX-FAILURE BACKOFF + * ══════════════════════════════════════════════════════════════════ */ + +TEST(index_backoff_no_failures_keeps_interval) { + /* The success and busy-skip paths must be completely unaffected. */ + ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 0), 5000); + ASSERT_EQ(cbm_watcher_index_backoff_ms(60000, 0), 60000); + PASS(); +} + +TEST(index_backoff_doubles_per_failure) { + ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 1), 10000); + ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 2), 20000); + ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 3), 40000); + ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 4), 80000); + PASS(); +} + +TEST(index_backoff_reaches_ceiling_and_stays) { + /* 5000 << 6 = 320000, above the 5-minute ceiling. The shift is capped + * too, so an arbitrarily long failure streak cannot overflow the + * intermediate value or wrap back to a short delay. */ + ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 6), 300000); + ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 100), 300000); + ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, INT_MAX), 300000); + ASSERT_EQ(cbm_watcher_index_backoff_ms(60000, INT_MAX), 300000); + PASS(); +} + +TEST(index_backoff_is_monotonic_and_bounded) { + /* The property that actually matters: the delay never decreases as the + * streak grows, and never exceeds the ceiling. This is what bounds the + * fork rate of a permanently failing project. */ + int previous = cbm_watcher_index_backoff_ms(5000, 0); + for (int failures = 1; failures < 200; failures++) { + int delay = cbm_watcher_index_backoff_ms(5000, failures); + ASSERT_TRUE(delay >= previous); + ASSERT_TRUE(delay <= 300000); + previous = delay; + } + PASS(); +} + +TEST(index_backoff_degenerate_inputs) { + ASSERT_EQ(cbm_watcher_index_backoff_ms(0, 5), 0); + ASSERT_EQ(cbm_watcher_index_backoff_ms(-1, 0), 0); + ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, -1), 5000); + PASS(); +} + /* ══════════════════════════════════════════════════════════════════ * LIFECYCLE * ══════════════════════════════════════════════════════════════════ */ @@ -3143,6 +3195,11 @@ SUITE(watcher) { RUN_TEST(poll_interval_scaling); RUN_TEST(poll_interval_cap); RUN_TEST(poll_interval_small); + RUN_TEST(index_backoff_no_failures_keeps_interval); + RUN_TEST(index_backoff_doubles_per_failure); + RUN_TEST(index_backoff_reaches_ceiling_and_stays); + RUN_TEST(index_backoff_is_monotonic_and_bounded); + RUN_TEST(index_backoff_degenerate_inputs); /* Lifecycle */ RUN_TEST(watcher_create_free); From 4fc5999a85f50bbb131dd87c5f41949c08617c1a Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Wed, 2 Sep 2026 14:18:02 -0500 Subject: [PATCH 014/130] fix: address QA round 1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F1 (major) — the failure state machine the backoff feeds had no test coverage. Deleting the streak reset, or widening the sustained-failure comparison, left all 7788 tests green: the index_backoff_* tests cover the delay arithmetic, and nothing covered the counter that selects which delay applies. A streak that never increments silently degrades the backoff to the unbounded retry it exists to prevent. Adds cbm_watcher_index_failure_count(), alongside the existing cbm_watcher_watch_count() in watcher.h's "Introspection (for testing)" section, and a test that drives a real git repo through fail/fail/succeed on the failing_index_callback seam already used by the #937 test, asserting the streak goes 0 -> 1 -> 2 -> 0. Verified by mutation: removing the reset turns it red at the reset assertion while every index_backoff_* test stays green. The delay-GATING half stays uncovered on purpose. Every integration test calls cbm_watcher_touch first, which zeroes next_poll_ns, so whether the scheduler honours the computed deadline is unobservable without an injectable clock the watcher does not have. That harness is a larger change than this fix. F3 (minor) — cbm_watcher_index_backoff_ms was non-monotonic for an interval above the ceiling: zero failures returned the interval, one failure returned the smaller ceiling. Unreachable today (POLL_MAX_MS < the ceiling), but the function is exported, so the clamp is now a floor as well as a cap and backing off can never schedule sooner than the project's own cadence. F2 (minor) needed no code change: a busy-skip following an unresolved failure streak keeps the backed-off delay, which is deliberate — only a success clears the streak, and treating rc > 0 as recovery would let a project alternating fail/busy never back off at all. The PR description's over-strong "busy-skip is behaviourally unchanged" claim is corrected there. Refs #2015 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FbrM52pmB1m2SR1vRRoNnf Signed-off-by: Shane McCarron --- src/watcher/watcher.c | 21 ++++++++- src/watcher/watcher.h | 7 +++ tests/test_watcher.c | 103 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 1 deletion(-) diff --git a/src/watcher/watcher.c b/src/watcher/watcher.c index 855c460fe..76cbf3e5d 100644 --- a/src/watcher/watcher.c +++ b/src/watcher/watcher.c @@ -171,7 +171,15 @@ int cbm_watcher_index_backoff_ms(int interval_ms, int consecutive_failures) { int shift = consecutive_failures < INDEX_FAIL_SHIFT_MAX ? consecutive_failures : INDEX_FAIL_SHIFT_MAX; int64_t delay = (int64_t)interval_ms << shift; - return delay > INDEX_FAIL_CEILING_MS ? INDEX_FAIL_CEILING_MS : (int)delay; + if (delay > INDEX_FAIL_CEILING_MS) { + delay = INDEX_FAIL_CEILING_MS; + } + /* Backing off must never schedule SOONER than the project's own cadence. + * Unreachable today (POLL_MAX_MS < the ceiling), but clamping here keeps + * the function monotonic for every input rather than only for the inputs + * the current constants can produce — the caller's contract is "a delay + * that never shrinks as failures accumulate". */ + return (int)(delay < interval_ms ? interval_ms : delay); } int cbm_watcher_poll_interval_ms(int file_count) { @@ -1161,6 +1169,17 @@ void cbm_watcher_touch(cbm_watcher_t *w, const char *project_name) { cbm_mutex_unlock(&w->projects_lock); } +int cbm_watcher_index_failure_count(cbm_watcher_t *w, const char *project_name) { + if (!w || !project_name) { + return -1; + } + cbm_mutex_lock(&w->projects_lock); + project_state_t *s = cbm_ht_get(w->projects, project_name); + int failures = s ? s->index_failures : -1; + cbm_mutex_unlock(&w->projects_lock); + return failures; +} + int cbm_watcher_watch_count(cbm_watcher_t *w) { if (!w) { return 0; diff --git a/src/watcher/watcher.h b/src/watcher/watcher.h index 18459afc5..51d06877b 100644 --- a/src/watcher/watcher.h +++ b/src/watcher/watcher.h @@ -91,6 +91,13 @@ void cbm_watcher_stop(cbm_watcher_t *w); /* Return the number of projects in the watch list. */ int cbm_watcher_watch_count(cbm_watcher_t *w); +/* Return a watched project's consecutive hard-index-failure count, or -1 when + * it is not watched. Exposed because the failure state machine (increment on + * a hard error, reset on success) is otherwise unobservable from a test: the + * only other thing it feeds is a poll deadline, and gating on that deadline + * needs a controllable clock the watcher does not have. */ +int cbm_watcher_index_failure_count(cbm_watcher_t *w, const char *project_name); + /* Return the adaptive poll interval (ms) for a given file count. */ int cbm_watcher_poll_interval_ms(int file_count); diff --git a/tests/test_watcher.c b/tests/test_watcher.c index 42dad66c9..de8be7369 100644 --- a/tests/test_watcher.c +++ b/tests/test_watcher.c @@ -137,6 +137,20 @@ TEST(index_backoff_degenerate_inputs) { PASS(); } +TEST(index_backoff_never_schedules_sooner_than_the_interval) { + /* An interval above the ceiling must not be SHORTENED by backing off: + * the clamp is a floor as well as a cap. Not reachable while + * POLL_MAX_MS < the ceiling, but the helper is exported, so its contract + * has to hold for what a caller can pass, not only for what today's + * constants produce. */ + int over = 400000; /* > INDEX_FAIL_CEILING_MS */ + ASSERT_EQ(cbm_watcher_index_backoff_ms(over, 0), over); + for (int failures = 1; failures < 20; failures++) { + ASSERT_TRUE(cbm_watcher_index_backoff_ms(over, failures) >= over); + } + PASS(); +} + /* ══════════════════════════════════════════════════════════════════ * LIFECYCLE * ══════════════════════════════════════════════════════════════════ */ @@ -1788,6 +1802,92 @@ TEST(watcher_failed_reindex_retries_issue937) { PASS(); } +/* The failure state machine the backoff feeds (#2015): the streak must count + * consecutive HARD failures and reset the moment one reindex succeeds. The + * delay it computes is covered by the index_backoff_* unit tests; what is + * covered here is the counter that selects which delay applies, because a + * streak that never increments — or never resets — silently reverts the + * backoff to the unbounded retry it exists to prevent, with every other test + * still green. Each poll is preceded by cbm_watcher_touch, which zeroes + * next_poll_ns, so this exercises the counter without depending on a clock. */ +TEST(watcher_index_failure_streak_increments_and_resets_issue2015) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_watcher_strk_XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) + FAIL("cbm_mkdtemp failed"); + + if (wt_git(tmpdir, "init -q") != 0) { + th_rmtree(tmpdir); + FAIL("git init failed"); + } + { + char p[300]; + th_write_file(wt_path(p, sizeof(p), tmpdir, "file.txt"), "hello\n"); + } + wt_git(tmpdir, "add file.txt"); + wt_git(tmpdir, "commit -q -m init"); + + cbm_store_t *store = cbm_store_open_memory(); + cbm_watcher_t *w = cbm_watcher_new(store, failing_index_callback, NULL); + + cbm_watcher_watch(w, "strk-repo", tmpdir); + failing_index_calls = 0; + failing_index_fail_first_n = 2; /* first TWO reindex attempts fail hard */ + + /* Baseline (clean tree): no reindex, so no failures yet. */ + cbm_watcher_poll_once(w); + ASSERT_EQ(failing_index_calls, 0); + ASSERT_EQ(cbm_watcher_index_failure_count(w, "strk-repo"), 0); + + /* HEAD moves. */ + { + char p[300]; + th_append_file(wt_path(p, sizeof(p), tmpdir, "file.txt"), "world\n"); + } + wt_git(tmpdir, "add file.txt"); + wt_git(tmpdir, "commit -q -m add-world"); + + /* First hard failure → streak 1. */ + cbm_watcher_touch(w, "strk-repo"); + cbm_watcher_poll_once(w); + ASSERT_EQ(failing_index_calls, 1); + ASSERT_EQ(cbm_watcher_index_failure_count(w, "strk-repo"), 1); + + /* Second consecutive hard failure → streak 2 (it accumulates; it is not + * a boolean, which is what makes the delay grow). */ + cbm_watcher_touch(w, "strk-repo"); + cbm_watcher_poll_once(w); + ASSERT_EQ(failing_index_calls, 2); + ASSERT_EQ(cbm_watcher_index_failure_count(w, "strk-repo"), 2); + + /* Third attempt succeeds → streak resets, so a project that recovers + * returns to its normal cadence instead of staying backed off. */ + cbm_watcher_touch(w, "strk-repo"); + cbm_watcher_poll_once(w); + ASSERT_EQ(failing_index_calls, 3); + ASSERT_EQ(cbm_watcher_index_failure_count(w, "strk-repo"), 0); + + cbm_watcher_free(w); + cbm_store_close(store); + th_rmtree(tmpdir); + PASS(); +} + +TEST(watcher_index_failure_count_unknown_project) { + cbm_store_t *store = cbm_store_open_memory(); + cbm_watcher_t *w = cbm_watcher_new(store, index_callback, NULL); + + /* Unwatched and NULL inputs report -1, distinct from a watched project + * sitting at a legitimate 0. */ + ASSERT_EQ(cbm_watcher_index_failure_count(w, "never-watched"), -1); + ASSERT_EQ(cbm_watcher_index_failure_count(w, NULL), -1); + ASSERT_EQ(cbm_watcher_index_failure_count(NULL, "x"), -1); + + cbm_watcher_free(w); + cbm_store_close(store); + PASS(); +} + TEST(watcher_multiple_projects) { /* Create two temporary git repos */ char tmpdirA[256]; @@ -3200,6 +3300,9 @@ SUITE(watcher) { RUN_TEST(index_backoff_reaches_ceiling_and_stays); RUN_TEST(index_backoff_is_monotonic_and_bounded); RUN_TEST(index_backoff_degenerate_inputs); + RUN_TEST(index_backoff_never_schedules_sooner_than_the_interval); + RUN_TEST(watcher_index_failure_streak_increments_and_resets_issue2015); + RUN_TEST(watcher_index_failure_count_unknown_project); /* Lifecycle */ RUN_TEST(watcher_create_free); From 61ccd905111e6b9e0f262a7636d4af399e9be192 Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Wed, 2 Sep 2026 14:56:05 -0500 Subject: [PATCH 015/130] fix: address QA round 2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retracts an incorrect justification from the round-1 commit (4fc5999a) and closes the coverage it was used to excuse. MAJOR — the delay wire-up had no coverage, and the reason given for that was wrong. Round 1 claimed the gating half was "structurally unobservable without an injectable clock". It is not. cbm_watcher_touch zeroes next_poll_ns, so a wire-up that never sets a deadline leaves it at zero and the callback fires on every poll: simply NOT calling touch between polls observes gating deterministically, no clock involved. Only the delay's MAGNITUDE needs one. watcher_index_failure_backoff_gates_repolling_issue2015 covers it — one hard failure, then five polls without touch asserting no re-fork, then a touch proving the retry still gets through (so the test cannot pass by observing a dead watcher). Mutation-verified: deleting the wire-up yields "failing_index_calls == 6, expected 1". MINOR — round 1's F1 was half-fixed. Of the two single-token mutations that finding named, only the streak reset was covered; widening the sustained-failure "==" to ">=" still passed. watcher_sustained_failure_logs_ once_issue2015 drives 14 consecutive failures through a cbm_log_set_sink and asserts exactly one emission. Mutation-verified: ">=" yields "sustained_log_hits == 5, expected 1". All three mutations F1 named are now demonstrated caught rather than argued about: reset removal (round 1), wire-up deletion, and the "==" widening. MINOR — cbm_watcher_index_failure_count reads index_failures under projects_lock while poll_project writes it lock-free from a state snapshot. That is the discipline every other per-project field here already follows, so this documents the visibility contract rather than making one field atomic in isolation: the accessor is a diagnostic and test seam, not a synchronisation point, and must not carry scheduling decisions without atomic accessors first. MINOR — watcher.h still described the pre-F3 backoff contract. It now states the clamp is a floor as well as a cap, why the floor exists for an exported function whose reachable inputs cannot currently hit it, and that negative inputs are treated as zero. Still uncovered, now stated precisely rather than hand-waved: the delay's MAGNITUDE. Nothing asserts that ten failures produce a five-minute delay rather than a five-second one. Gating is proven; the numbers are not, and that does need a controllable clock. Refs #2015 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FbrM52pmB1m2SR1vRRoNnf Signed-off-by: Shane McCarron --- src/watcher/watcher.h | 29 +++++++-- tests/test_watcher.c | 140 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+), 5 deletions(-) diff --git a/src/watcher/watcher.h b/src/watcher/watcher.h index 51d06877b..f6a4df2dc 100644 --- a/src/watcher/watcher.h +++ b/src/watcher/watcher.h @@ -92,10 +92,19 @@ void cbm_watcher_stop(cbm_watcher_t *w); int cbm_watcher_watch_count(cbm_watcher_t *w); /* Return a watched project's consecutive hard-index-failure count, or -1 when - * it is not watched. Exposed because the failure state machine (increment on - * a hard error, reset on success) is otherwise unobservable from a test: the - * only other thing it feeds is a poll deadline, and gating on that deadline - * needs a controllable clock the watcher does not have. */ + * it is not watched. Exposed so the failure state machine (increment on a + * hard error, reset on success) can be asserted directly rather than inferred + * from the poll deadline it feeds. + * + * Memory visibility: this reads under projects_lock, but poll_project WRITES + * the counter outside that lock — it runs against a state snapshot taken + * while the lock was held, which is the same discipline every other + * per-project field in this struct already follows. Reading concurrently with + * a live poll is therefore formally a data race and may observe a stale + * value; it is a diagnostic and test accessor, not a synchronisation point. + * Single-threaded callers (the tests, and any caller between poll cycles) + * always see the current value. Do not build scheduling decisions on it + * without first giving the counter atomic accessors. */ int cbm_watcher_index_failure_count(cbm_watcher_t *w, const char *project_name); /* Return the adaptive poll interval (ms) for a given file count. */ @@ -105,7 +114,17 @@ int cbm_watcher_poll_interval_ms(int file_count); * `consecutive_failures` consecutive hard index failures. Zero failures * yields `interval_ms` unchanged; each further failure doubles the delay up * to a fixed ceiling, so a permanently failing project stops re-forking a - * worker at the poll cadence without ever being abandoned. */ + * worker at the poll cadence without ever being abandoned. + * + * The result is clamped at BOTH ends: never above the ceiling, and never + * below `interval_ms`, so backing off can only ever delay the next attempt, + * never bring it forward. That lower clamp matters only for an `interval_ms` + * already above the ceiling, which the current constants cannot produce + * (POLL_MAX_MS < the ceiling) — it is stated because this is an exported + * function and the guarantee should hold for any argument a caller passes, + * not only for the ones today's constants generate. + * + * Negative inputs are treated as zero. */ int cbm_watcher_index_backoff_ms(int interval_ms, int consecutive_failures); /* Classify a stat() errno observed on a watched project root: returns true diff --git a/tests/test_watcher.c b/tests/test_watcher.c index de8be7369..d405e3005 100644 --- a/tests/test_watcher.c +++ b/tests/test_watcher.c @@ -7,6 +7,7 @@ #include "../src/foundation/compat.h" #include "../src/foundation/compat_thread.h" #include "../src/foundation/constants.h" +#include "../src/foundation/log.h" #include "../src/foundation/platform.h" #include "test_framework.h" #include "test_helpers.h" @@ -1873,6 +1874,143 @@ TEST(watcher_index_failure_streak_increments_and_resets_issue2015) { PASS(); } +/* The delay WIRE-UP (#2015): a hard failure must actually push the project's + * next poll out, so a permanently failing project stops re-forking a worker + * on every cycle. That is this PR's headline behaviour and it is observable + * without a clock, by simply NOT calling cbm_watcher_touch between polls: + * touch zeroes next_poll_ns, so a wire-up that never sets a deadline leaves + * it at 0 and the callback fires on every poll. What still needs a clock is + * only the delay's MAGNITUDE, not whether gating happens at all. */ +TEST(watcher_index_failure_backoff_gates_repolling_issue2015) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_watcher_gate_XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) + FAIL("cbm_mkdtemp failed"); + + if (wt_git(tmpdir, "init -q") != 0) { + th_rmtree(tmpdir); + FAIL("git init failed"); + } + { + char p[300]; + th_write_file(wt_path(p, sizeof(p), tmpdir, "file.txt"), "hello\n"); + } + wt_git(tmpdir, "add file.txt"); + wt_git(tmpdir, "commit -q -m init"); + + cbm_store_t *store = cbm_store_open_memory(); + cbm_watcher_t *w = cbm_watcher_new(store, failing_index_callback, NULL); + + cbm_watcher_watch(w, "gate-repo", tmpdir); + failing_index_calls = 0; + failing_index_fail_first_n = 100; /* never succeeds */ + + cbm_watcher_poll_once(w); /* baseline */ + + { + char p[300]; + th_append_file(wt_path(p, sizeof(p), tmpdir, "file.txt"), "world\n"); + } + wt_git(tmpdir, "add file.txt"); + wt_git(tmpdir, "commit -q -m add-world"); + + /* One failure, forced by touch. */ + cbm_watcher_touch(w, "gate-repo"); + cbm_watcher_poll_once(w); + ASSERT_EQ(failing_index_calls, 1); + ASSERT_EQ(cbm_watcher_index_failure_count(w, "gate-repo"), 1); + + /* Now poll repeatedly WITHOUT touching. The change is still pending (the + * baseline stays uncommitted per #937), so the ONLY thing that can stop a + * re-fork is the deadline the failure just scheduled. Before the backoff + * wire-up existed this project was re-forked at the poll cadence; with no + * deadline set at all it would fire on every one of these iterations. */ + for (int i = 0; i < 5; i++) { + cbm_watcher_poll_once(w); + } + ASSERT_EQ(failing_index_calls, 1); + ASSERT_EQ(cbm_watcher_index_failure_count(w, "gate-repo"), 1); + + /* Control: the watcher is not simply dead — clearing the deadline lets the + * retry through, which is what keeps #937's at-least-once guarantee. */ + cbm_watcher_touch(w, "gate-repo"); + cbm_watcher_poll_once(w); + ASSERT_EQ(failing_index_calls, 2); + ASSERT_EQ(cbm_watcher_index_failure_count(w, "gate-repo"), 2); + + cbm_watcher_free(w); + cbm_store_close(store); + th_rmtree(tmpdir); + PASS(); +} + +/* The sustained-failure line must fire EXACTLY once, at the threshold — the + * other single-token mutation round 1 named (`==` widened to `>=`) would emit + * it on every failure from the threshold onward and turn a one-shot signal + * into the log spam the whole change is meant to avoid. */ +static int sustained_log_hits = 0; +static void sustained_log_sink(const char *line) { + if (line && strstr(line, "watcher.index.sustained_failure") != NULL) { + sustained_log_hits++; + } +} + +TEST(watcher_sustained_failure_logs_once_issue2015) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_watcher_sust_XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) + FAIL("cbm_mkdtemp failed"); + + if (wt_git(tmpdir, "init -q") != 0) { + th_rmtree(tmpdir); + FAIL("git init failed"); + } + { + char p[300]; + th_write_file(wt_path(p, sizeof(p), tmpdir, "file.txt"), "hello\n"); + } + wt_git(tmpdir, "add file.txt"); + wt_git(tmpdir, "commit -q -m init"); + + cbm_store_t *store = cbm_store_open_memory(); + cbm_watcher_t *w = cbm_watcher_new(store, failing_index_callback, NULL); + + cbm_watcher_watch(w, "sust-repo", tmpdir); + failing_index_calls = 0; + failing_index_fail_first_n = 100; /* never succeeds */ + + cbm_watcher_poll_once(w); /* baseline */ + + { + char p[300]; + th_append_file(wt_path(p, sizeof(p), tmpdir, "file.txt"), "world\n"); + } + wt_git(tmpdir, "add file.txt"); + wt_git(tmpdir, "commit -q -m add-world"); + + sustained_log_hits = 0; + cbm_log_set_sink(sustained_log_sink); + + /* Drive well past the threshold (10). touch clears the deadline each time + * so every iteration actually attempts a reindex. */ + for (int i = 0; i < 14; i++) { + cbm_watcher_touch(w, "sust-repo"); + cbm_watcher_poll_once(w); + } + + cbm_log_set_sink(NULL); + + ASSERT_EQ(failing_index_calls, 14); + ASSERT_EQ(cbm_watcher_index_failure_count(w, "sust-repo"), 14); + /* Exactly one, not four (11..14) and not zero. */ + ASSERT_EQ(sustained_log_hits, 1); + + cbm_watcher_free(w); + cbm_store_close(store); + th_rmtree(tmpdir); + PASS(); +} + TEST(watcher_index_failure_count_unknown_project) { cbm_store_t *store = cbm_store_open_memory(); cbm_watcher_t *w = cbm_watcher_new(store, index_callback, NULL); @@ -3302,6 +3440,8 @@ SUITE(watcher) { RUN_TEST(index_backoff_degenerate_inputs); RUN_TEST(index_backoff_never_schedules_sooner_than_the_interval); RUN_TEST(watcher_index_failure_streak_increments_and_resets_issue2015); + RUN_TEST(watcher_index_failure_backoff_gates_repolling_issue2015); + RUN_TEST(watcher_sustained_failure_logs_once_issue2015); RUN_TEST(watcher_index_failure_count_unknown_project); /* Lifecycle */ From 086422020f5fd9396b46384b10413cdca6994c96 Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Wed, 2 Sep 2026 15:28:21 -0500 Subject: [PATCH 016/130] fix: address QA round 3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Documentation and claim accuracy only — no behavioural change. Two of the three findings are corrections to inaccurate statements the round-2 commit (61ccd905) introduced. The gating test's framing was overstated, and this time the overstatement was measured rather than argued. Replacing the backoff call with the pre-change `ctx->now + interval_ms * US_PER_MS` leaves the watcher suite green at 82/82: the index_backoff_* tests pass because they exercise the pure function directly, which the reverted scheduler no longer calls, and the streak tests pass because the counter is untouched. So no test in this suite fails when this change's behaviour is removed. The test now records that, with the figure, instead of claiming it demonstrates the backoff. It is a regression guard on a deadline being assigned at all — real, but narrower than round 2 said. Closing that gap needs either a controllable clock or a further accessor exposing next_poll_ns. Both were judged out of scope: a third exported symbol purely for testing, on a change already carrying two, is the API widening earlier rounds flagged. The gap is recorded in the test, the PR description and the round-3 note rather than papered over. The memory-visibility comment claimed index_failures follows "the same discipline every other per-project field in this struct already follows". That is false: active_git is serialized by projects_lock and registered is an atomic_bool. Narrowed to the poll-mutated fields, which are now enumerated. The round-2 decision to document rather than lock still stands — the panel verified the underlying premise — but it was resting on an overbroad claim. watcher.h said the delay "doubles up to a fixed ceiling", which skips the shift cap: the delay plateaus at interval_ms << INDEX_FAIL_SHIFT_MAX, and that reaches the ceiling only for interval_ms >= 4688 ms. Every interval this watcher generates is >= POLL_BASE_MS so the ceiling is always reached in practice, but the function is exported and a caller may pass less. Refs #2015 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FbrM52pmB1m2SR1vRRoNnf Signed-off-by: Shane McCarron --- src/watcher/watcher.h | 25 ++++++++++++++++++------- tests/test_watcher.c | 28 +++++++++++++++++++++------- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/src/watcher/watcher.h b/src/watcher/watcher.h index f6a4df2dc..013e1356a 100644 --- a/src/watcher/watcher.h +++ b/src/watcher/watcher.h @@ -98,10 +98,13 @@ int cbm_watcher_watch_count(cbm_watcher_t *w); * * Memory visibility: this reads under projects_lock, but poll_project WRITES * the counter outside that lock — it runs against a state snapshot taken - * while the lock was held, which is the same discipline every other - * per-project field in this struct already follows. Reading concurrently with - * a live poll is therefore formally a data race and may observe a stale - * value; it is a diagnostic and test accessor, not a synchronisation point. + * while the lock was held, which is the same discipline the other + * poll-mutated fields here already follow (last_head, last_dirty_sig, + * interval_ms, next_poll_ns, missing_root_count). It is NOT the discipline of + * every field: active_git is serialized by projects_lock and registered is an + * atomic_bool. Reading concurrently with a live poll is therefore formally a + * data race and may observe a stale value; it is a diagnostic and test + * accessor, not a synchronisation point. * Single-threaded callers (the tests, and any caller between poll cycles) * always see the current value. Do not build scheduling decisions on it * without first giving the counter atomic accessors. */ @@ -112,9 +115,17 @@ int cbm_watcher_poll_interval_ms(int file_count); /* Return the delay (ms) before the next index attempt for a project with * `consecutive_failures` consecutive hard index failures. Zero failures - * yields `interval_ms` unchanged; each further failure doubles the delay up - * to a fixed ceiling, so a permanently failing project stops re-forking a - * worker at the poll cadence without ever being abandoned. + * yields `interval_ms` unchanged; each further failure doubles the delay, so + * a permanently failing project stops re-forking a worker at the poll cadence + * without ever being abandoned. + * + * Doubling stops at a fixed shift cap, so the delay plateaus at + * `interval_ms << INDEX_FAIL_SHIFT_MAX` — which is the ceiling only when + * `interval_ms` is large enough to reach it (>= 4688 ms for the current cap + * and ceiling). Below that the plateau sits strictly under the ceiling. Every + * interval this watcher generates is >= POLL_BASE_MS, so in practice the + * ceiling is always reached; the distinction is stated because this is an + * exported function and a caller may pass a smaller interval. * * The result is clamped at BOTH ends: never above the ceiling, and never * below `interval_ms`, so backing off can only ever delay the next attempt, diff --git a/tests/test_watcher.c b/tests/test_watcher.c index d405e3005..4b6e874b0 100644 --- a/tests/test_watcher.c +++ b/tests/test_watcher.c @@ -1874,13 +1874,27 @@ TEST(watcher_index_failure_streak_increments_and_resets_issue2015) { PASS(); } -/* The delay WIRE-UP (#2015): a hard failure must actually push the project's - * next poll out, so a permanently failing project stops re-forking a worker - * on every cycle. That is this PR's headline behaviour and it is observable - * without a clock, by simply NOT calling cbm_watcher_touch between polls: - * touch zeroes next_poll_ns, so a wire-up that never sets a deadline leaves - * it at 0 and the callback fires on every poll. What still needs a clock is - * only the delay's MAGNITUDE, not whether gating happens at all. */ +/* The delay WIRE-UP (#2015): a hard failure must leave SOME deadline behind, + * so the failing project is not re-polled on the very next cycle. Observable + * without a clock by simply not calling cbm_watcher_touch between polls — + * touch zeroes next_poll_ns, so a wire-up that never assigns one leaves it at + * 0 and the callback fires on every poll. + * + * Be precise about what this does NOT prove, because it is less than it looks. + * The code before this change also assigned a deadline here (the plain + * adaptive interval), so this test passes against the unfixed tree: it is a + * regression guard on the assignment existing, NOT a demonstration of the + * backoff. + * + * Measured, not assumed: replacing the backoff call with the pre-change + * `ctx->now + interval_ms * US_PER_MS` leaves this whole suite green at 82/82. + * No test in this file fails when the behavioural change is removed — the + * index_backoff_* tests still pass because they exercise the pure function + * directly, and the streak tests still pass because the counter is unaffected. + * What distinguishes backed-off from plain cadence is the delay's MAGNITUDE, + * and observing that needs a controllable clock the watcher does not have, or + * a further accessor exposing next_poll_ns. Both were judged out of scope; the + * gap is recorded here rather than papered over. */ TEST(watcher_index_failure_backoff_gates_repolling_issue2015) { char tmpdir[256]; snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_watcher_gate_XXXXXX"); From d402623c20e1dbff62e560a865fd4955798dfbfe Mon Sep 17 00:00:00 2001 From: Sakil Ansari Date: Thu, 3 Sep 2026 20:43:43 +0530 Subject: [PATCH 017/130] test: match optional TypeBox depth schema Signed-off-by: Sakil Ansari --- tests/test_agent_clients.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_agent_clients.c b/tests/test_agent_clients.c index 9d3d04f60..06dcce038 100644 --- a/tests/test_agent_clients.c +++ b/tests/test_agent_clients.c @@ -1178,7 +1178,7 @@ TEST(client_adapter_pi_emits_parameters_and_execute) { ASSERT_NOT_NULL(strstr(js, "Type.Object(")); ASSERT_NOT_NULL(strstr(js, "Type.String(")); /* A numeric registry property must remain numeric in the generated schema. */ - ASSERT_NOT_NULL(strstr(js, "depth: Type.Integer(")); + ASSERT_NOT_NULL(strstr(js, "depth: Type.Optional(Type.Integer(")); /* TypeBox import should be present. */ ASSERT_NOT_NULL(strstr(js, "import { Type } from 'typebox';")); /* Raw JSON output is required so the bridge can parse the MCP result; the From 3507d9555d73aa746cde23139588081c6c02d2e2 Mon Sep 17 00:00:00 2001 From: Subgurim Date: Thu, 3 Sep 2026 18:31:56 +0200 Subject: [PATCH 018/130] fix(windows): enforce index worker memory budgets Signed-off-by: Subgurim --- src/foundation/subprocess.c | 6 ++++++ src/foundation/subprocess.h | 2 ++ src/mcp/index_supervisor.c | 1 + tests/test_main.c | 28 ++++++++++++++++++++++++++++ tests/test_subprocess.c | 25 +++++++++++++++++++++++++ 5 files changed, 62 insertions(+) diff --git a/src/foundation/subprocess.c b/src/foundation/subprocess.c index 196165cea..a12bd19c4 100644 --- a/src/foundation/subprocess.c +++ b/src/foundation/subprocess.c @@ -411,6 +411,7 @@ struct cbm_subprocess { void *log_ud; int quiet_timeout_ms; int cancel_grace_ms; + size_t memory_limit_bytes; bool delete_log_on_exit; long tail_pos; @@ -525,6 +526,7 @@ static cbm_subprocess_t *cbm_subprocess_copy_opts(const cbm_proc_opts_t *opts) { if (process->cancel_grace_ms > CBM_SUBPROCESS_MAX_CANCEL_GRACE_MS) { process->cancel_grace_ms = CBM_SUBPROCESS_MAX_CANCEL_GRACE_MS; } + process->memory_limit_bytes = opts->memory_limit_bytes; process->delete_log_on_exit = opts->delete_log_on_exit; atomic_init(&process->lifecycle, CBM_SUBPROCESS_ACTIVE); cbm_subprocess_result_init(&process->result); @@ -658,6 +660,10 @@ static int cbm_subprocess_spawn_win(cbm_subprocess_t *process) { JOBOBJECT_EXTENDED_LIMIT_INFORMATION limits; ZeroMemory(&limits, sizeof(limits)); limits.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + if (process->memory_limit_bytes > 0) { + limits.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_JOB_MEMORY; + limits.JobMemoryLimit = (SIZE_T)process->memory_limit_bytes; + } HANDLE job = CreateJobObjectW(NULL, NULL); if (!job || !SetInformationJobObject(job, JobObjectExtendedLimitInformation, &limits, sizeof(limits))) { diff --git a/src/foundation/subprocess.h b/src/foundation/subprocess.h index 62592c592..df3fba844 100644 --- a/src/foundation/subprocess.h +++ b/src/foundation/subprocess.h @@ -68,6 +68,8 @@ typedef struct { * ms with no new completed log line */ int cancel_grace_ms; /* graceful tree-termination window; <= 0 uses the finite * CBM_SUBPROCESS_DEFAULT_CANCEL_GRACE_MS */ + size_t memory_limit_bytes; /* Windows-only hard commit limit for the entire Job Object; + * 0 => no OS-enforced memory limit */ bool delete_log_on_exit; /* unlink log_file after reaping */ } cbm_proc_opts_t; diff --git a/src/mcp/index_supervisor.c b/src/mcp/index_supervisor.c index b5257dfc1..000bdafb0 100644 --- a/src/mcp/index_supervisor.c +++ b/src/mcp/index_supervisor.c @@ -695,6 +695,7 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_ options.on_log_line = NULL; options.log_ud = NULL; options.quiet_timeout_ms = worker_quiet_timeout_ms(); + options.memory_limit_bytes = memory_budget_bytes; options.delete_log_on_exit = false; if (cbm_subprocess_spawn(&options, &handle->process) != 0) { (void)cbm_unlink(handle->response_path); diff --git a/tests/test_main.c b/tests/test_main.c index 6c23050ea..d4e7f5899 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -33,6 +33,7 @@ int tf_skip_count = 0; #include #ifdef _WIN32 #include /* #798 follow-up: socket-isolation re-exec probe */ +#include #else #include #ifdef __APPLE__ @@ -50,6 +51,29 @@ int tf_skip_count = 0; * unwind after either production containment or the test's verified backstop. */ #define TF_BLOCKING_GIT_MARKER_ENV "CBM_TEST_RUNTIME_BLOCKING_GIT_PID_FILE" +/* Native child for subprocess_windows_job_object_enforces_memory_limit. The + * fixed-size commit keeps the RED path bounded: without a Job memory limit it + * succeeds and exits 0; with the limit it is denied and exits with the sentinel + * code expected by the parent test. */ +static int tf_maybe_run_windows_memory_limit_probe(int argc, char **argv) { +#ifdef _WIN32 + if (argc == 2 && argv && strcmp(argv[1], "__cbm_windows_memory_limit_probe") == 0) { + const SIZE_T allocation_size = (SIZE_T)512U * 1024U * 1024U; + void *allocation = + VirtualAlloc(NULL, allocation_size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + if (!allocation) { + return 73; + } + (void)VirtualFree(allocation, 0, MEM_RELEASE); + return 0; + } +#else + (void)argc; + (void)argv; +#endif + return -1; +} + #ifdef _WIN32 static bool tf_invoked_as_windows_git_module(void) { /* CreateProcessW authenticates the executable with lpApplicationName, but @@ -892,6 +916,10 @@ int main(int argc, char **argv) { (void)cbm_setenv("CBM_TEST_BUILD_FINGERPRINT", "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", 1); } + int memory_limit_probe_rc = tf_maybe_run_windows_memory_limit_probe(argc, argv); + if (memory_limit_probe_rc >= 0) { + return memory_limit_probe_rc; + } int blocking_git_rc = tf_maybe_run_blocking_git_probe(argc, argv); if (blocking_git_rc >= 0) { return blocking_git_rc; diff --git a/tests/test_subprocess.c b/tests/test_subprocess.c index 1554821c6..43d06daa8 100644 --- a/tests/test_subprocess.c +++ b/tests/test_subprocess.c @@ -744,6 +744,30 @@ TEST(subprocess_windows_job_object_cancellation_quiesces_descendant_tree) { #endif } +TEST(subprocess_windows_job_object_enforces_memory_limit) { +#ifndef _WIN32 + SKIP_PLATFORM("native Windows Job Object memory-limit probe"); +#else + char self_path[32768]; + DWORD self_length = GetModuleFileNameA(NULL, self_path, (DWORD)sizeof(self_path)); + ASSERT_TRUE(self_length > 0 && self_length < sizeof(self_path)); + const char *argv[] = {self_path, "__cbm_windows_memory_limit_probe", NULL}; + cbm_proc_opts_t opts = {0}; + opts.bin = self_path; + opts.argv = argv; + opts.memory_limit_bytes = (size_t)512U * 1024U * 1024U; + opts.quiet_timeout_ms = 5000; + + cbm_proc_result_t result = {0}; + ASSERT_EQ(cbm_subprocess_run(&opts, &result), 0); + ASSERT_EQ(result.outcome, CBM_PROC_EXIT_NONZERO); + ASSERT_EQ(result.exit_code, 73); + ASSERT_TRUE(result.tree_quiesced); + ASSERT_FALSE(result.supervision_failed); + PASS(); +#endif +} + TEST(subprocess_cancel_grace_is_hard_capped) { #ifdef _WIN32 SKIP_PLATFORM("POSIX process-group grace cap probe; native Windows coverage pending"); @@ -1228,6 +1252,7 @@ SUITE(subprocess) { RUN_TEST(subprocess_cancel_is_idempotent_and_kills_ignoring_tree); RUN_TEST(subprocess_quiet_timeout_kills_ignoring_tree); RUN_TEST(subprocess_windows_job_object_cancellation_quiesces_descendant_tree); + RUN_TEST(subprocess_windows_job_object_enforces_memory_limit); RUN_TEST(subprocess_cancel_grace_is_hard_capped); RUN_TEST(subprocess_poll_log_delivery_is_bounded_and_terminal_is_lossless); RUN_TEST(subprocess_final_log_drain_error_is_terminal_and_preserves_classification); From eb80a4ecff04d8bfacef20e1aa2ee29445a782ff Mon Sep 17 00:00:00 2001 From: Joshua Richter Date: Sat, 29 Aug 2026 13:53:42 -0400 Subject: [PATCH 019/130] fix(extraction): reach a Swift URL built by a constructor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Swift has no URL literal, so almost no real code passes a bare string to a request. It writes URL(string: "https://…")! instead, and the literal then sits two levels below the argument list: past the trailing "!", which the grammar models as a postfix_expression, and inside the constructor's own value_arguments. extract_url_or_topic_arg saw only the outer node and gave up, so the URL never reached the service-pattern table and no Route node formed. That is the shape issue #1892 reported from a real project — reaching a bare string argument was only the layer underneath it. swift_unwrap_url_constructor() steps past both wrappers. It unwraps only URL, URLComponents and URLRequest, so any other constructor keeps its own meaning and the outer call does not borrow the inner call's string. A non-literal argument such as URL(string: base + path) falls through to the ordinary handling unchanged. The value_argument unwrap added for the bare-string case is now swift_argument_value(), because the nested argument list needs the same step and the code was identical. Refs #1892 Signed-off-by: Joshua Richter --- internal/cbm/extract_calls.c | 70 +++++++++++++++++++++++++++++++----- tests/test_extraction.c | 49 +++++++++++++++++++++++++ tests/test_pipeline.c | 50 ++++++++++++++++++++++++++ 3 files changed, 160 insertions(+), 9 deletions(-) diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index c22a11e01..1821cb692 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -2257,6 +2257,61 @@ static const char *extract_binary_concat_suffix(CBMExtractCtx *ctx, TSNode node) } // Try to extract URL/topic from a positional argument (string or constant). +/* Swift names its call arguments, so each one is a value_argument that may lead + * with a value_argument_label — the `from:` in `data(from: url)`. Return the + * value itself, and leave any other node exactly as it came in. */ +static TSNode swift_argument_value(TSNode arg) { + if (strcmp(ts_node_type(arg), "value_argument") != 0 || ts_node_named_child_count(arg) == 0) { + return arg; + } + TSNode val = ts_node_named_child(arg, 0); + if (strcmp(ts_node_type(val), "value_argument_label") == 0 && + ts_node_named_child_count(arg) > 1) { + val = ts_node_named_child(arg, 1); + } + return val; +} + +/* Swift has no URL literal, so almost no real code passes a bare string to a + * request. It writes `URL(string: "https://…")!` instead, and the literal then + * sits two levels down: past the trailing `!`, which the grammar models as a + * postfix_expression, and inside the constructor's own argument list. + * + * Unwrap both so that literal is as reachable as a bare one. Only the three + * Foundation types that take a URL string are unwrapped — any other call keeps + * its own meaning, and a non-literal argument such as `URL(string: base + path)` + * falls through to the ordinary handling unchanged. */ +static TSNode swift_unwrap_url_constructor(CBMExtractCtx *ctx, TSNode arg) { + /* Step past a trailing "!" or "?". */ + if (strcmp(ts_node_type(arg), "postfix_expression") == 0) { + TSNode target = ts_node_child_by_field_name(arg, TS_FIELD("target")); + if (!ts_node_is_null(target)) { + arg = target; + } + } + if (strcmp(ts_node_type(arg), "call_expression") != 0) { + return arg; + } + TSNode callee = ts_node_named_child(arg, 0); + if (ts_node_is_null(callee) || strcmp(ts_node_type(callee), "simple_identifier") != 0) { + return arg; + } + const char *name = cbm_node_text(ctx->arena, callee, ctx->source); + if (!name || (strcmp(name, "URL") != 0 && strcmp(name, "URLComponents") != 0 && + strcmp(name, "URLRequest") != 0)) { + return arg; + } + TSNode suffix = cbm_find_child_by_kind(arg, "call_suffix"); + if (ts_node_is_null(suffix)) { + return arg; + } + TSNode inner = cbm_find_child_by_kind(suffix, "value_arguments"); + if (ts_node_is_null(inner) || ts_node_named_child_count(inner) == 0) { + return arg; + } + return swift_argument_value(ts_node_named_child(inner, 0)); +} + static const char *extract_positional_url(CBMExtractCtx *ctx, TSNode arg, const char *ak) { /* JS/TS template literals: `/things/${id}` normalizes to "/things/{}" so the * client URL joins the server route's canonical placeholder (issue #1006). */ @@ -2300,15 +2355,12 @@ static const char *extract_url_or_topic_arg(CBMExtractCtx *ctx, TSNode args) { } /* Swift wraps each argument in a value_argument that may lead with its * label, so `data(from: url)` would otherwise yield the label `from` - * rather than the value. Step past a leading value_argument_label. */ - if (strcmp(ts_node_type(arg), "value_argument") == 0 && - ts_node_named_child_count(arg) > 0) { - TSNode val = ts_node_named_child(arg, 0); - if (strcmp(ts_node_type(val), "value_argument_label") == 0 && - ts_node_named_child_count(arg) > 1) { - val = ts_node_named_child(arg, 1); - } - arg = val; + * rather than the value. */ + arg = swift_argument_value(arg); + /* A Swift URL is usually built rather than written bare, and the + * literal then sits inside that constructor. */ + if (ctx->language == CBM_LANG_SWIFT) { + arg = swift_unwrap_url_constructor(ctx, arg); } const char *ak = ts_node_type(arg); diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 5c63c98aa..cb7a1d1df 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -4186,6 +4186,52 @@ TEST(swift_labeled_call_string_arg_issue1892) { PASS(); } +/* Swift has no URL literal, so real code builds one and force-unwraps it. The + * string then sits two levels below the argument list. */ +TEST(swift_nested_url_constructor_issue1892) { + CBMFileResult *r = extract("func fetch() { URLSession.shared.dataTask(with: URL(string: " + "\"https://example.com/api/v1/widgets\")!) }\n", + CBM_LANG_SWIFT, "t", "Fetch.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + const CBMCall *c = find_call_by_callee(r, "URLSession.shared.dataTask"); + ASSERT_NOT_NULL(c); + ASSERT_NOT_NULL(c->first_string_arg); + ASSERT_STR_EQ(c->first_string_arg, "https://example.com/api/v1/widgets"); + cbm_free_result(r); + PASS(); +} + +/* Without the trailing "!" the constructor is not wrapped in a + * postfix_expression, so this covers the other shape. */ +TEST(swift_nested_url_no_bang_issue1892) { + CBMFileResult *r = + extract("func fetch() { client.send(to: URLRequest(url: \"/api/v1/widgets/1\")) }\n", + CBM_LANG_SWIFT, "t", "Send.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + const CBMCall *c = find_call_by_callee(r, "client.send"); + ASSERT_NOT_NULL(c); + ASSERT_NOT_NULL(c->first_string_arg); + ASSERT_STR_EQ(c->first_string_arg, "/api/v1/widgets/1"); + cbm_free_result(r); + PASS(); +} + +/* A constructor that is not one of the three URL types keeps its own meaning: + * the outer call must not borrow the inner call's string. */ +TEST(swift_non_url_constructor_untouched_issue1892) { + CBMFileResult *r = extract("func f() { log.write(to: Formatter(pattern: \"%s-%d\")) }\n", + CBM_LANG_SWIFT, "t", "Log.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + const CBMCall *c = find_call_by_callee(r, "log.write"); + ASSERT_NOT_NULL(c); + ASSERT_NULL(c->first_string_arg); + cbm_free_result(r); + PASS(); +} + /* Issue #1009: URL-builder helper pattern — a function returning a URL-shaped * literal, consumed as client(buildPath(id)). The builder's URL is recorded in * the per-file constant map and resolved at the call site, for both return @@ -7131,6 +7177,9 @@ SUITE(extraction) { RUN_TEST(swift_chained_call); RUN_TEST(swift_force_unwrap_scanner_shift); RUN_TEST(swift_call_string_arg_issue1892); + RUN_TEST(swift_nested_url_constructor_issue1892); + RUN_TEST(swift_nested_url_no_bang_issue1892); + RUN_TEST(swift_non_url_constructor_untouched_issue1892); RUN_TEST(swift_labeled_call_string_arg_issue1892); RUN_TEST(objc_interface); RUN_TEST(objc_implementation); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index cf40dd57a..192e09a94 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -5690,6 +5690,55 @@ TEST(pipeline_native_fetch_classified_as_http_calls) { * no call arguments at all. Alamofire/URLSession were already in the service * pattern table; the URL simply never reached it. This is the Swift twin of * the TypeScript fetch case above. */ +/* The shape real Swift actually writes: the URL is built by a constructor and + * force-unwrapped, so the literal is two levels below the argument list. This + * is what issue #1892 reported from a real project. */ +TEST(pipeline_swift_nested_url_makes_route_issue1892) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_swiftnested_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + + write_temp_file(tmp, "Sources/Client.swift", + "import Foundation\n" + "final class Client {\n" + " func listWidgets() {\n" + " URLSession.shared.dataTask(with: " + "URL(string: \"/api/v1/widgets\")!)\n" + " }\n" + "}\n"); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/swiftnested.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + const char *project = cbm_pipeline_project_name(p); + + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + + ASSERT_GTE(cbm_store_count_edges_by_type(s, project, "HTTP_CALLS"), 1); + + cbm_node_t *routes = NULL; + int route_count = 0; + cbm_store_find_nodes_by_label(s, project, "Route", &routes, &route_count); + int widget_routes = 0; + for (int i = 0; i < route_count; i++) { + if (routes[i].qualified_name && strstr(routes[i].qualified_name, "/api/v1/widgets")) { + widget_routes++; + } + } + cbm_store_free_nodes(routes, route_count); + ASSERT_GTE(widget_routes, 1); + + cbm_store_close(s); + cbm_pipeline_free(p); + th_rmtree(tmp); + PASS(); +} + TEST(pipeline_swift_http_call_makes_route_issue1892) { char tmp[256]; snprintf(tmp, sizeof(tmp), "/tmp/cbm_swifthttp_XXXXXX"); @@ -13475,6 +13524,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_parallel_rust_cross_only_macro_hidden_gets_synthetic_carrier); RUN_TEST(pipeline_arg_url_rejects_non_http_slash_arguments); RUN_TEST(pipeline_native_fetch_classified_as_http_calls); + RUN_TEST(pipeline_swift_nested_url_makes_route_issue1892); RUN_TEST(pipeline_swift_http_call_makes_route_issue1892); RUN_TEST(pipeline_native_fetch_parallel_classified_as_http_calls); RUN_TEST(pipeline_local_fetch_shadow_not_classified_as_http); From a08b9ecbbda54793275cbd4e93fecf0ad4ff1c0a Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 6 Sep 2026 18:18:36 +0200 Subject: [PATCH 020/130] fix(worker): scope the index worker to the request's repository, not the daemon's environment The daemon-spawned index worker inherited the daemon process environment and re-ran the allowed-root check without a session policy. The daemon's session policy is installed only in application.c (the sole production caller of cbm_mcp_server_set_session_context); after the daemon admits an index_repository request, index_supervisor.c builds the worker argv (`cli --index-worker --index-worker-build index_repository --response-out ...`, no root anywhere) and subprocess.c spawns it with the daemon's environ. The worker then creates its own server with cbm_mcp_server_new(NULL), so handle_index_repository runs with no session policy and falls back to getenv("CBM_ALLOWED_ROOT") - the environment of whoever STARTED the daemon (bootstrap.c spawns it with the starter's environ). Observed: a daemon started with CBM_ALLOWED_ROOT=; a client session for /tiny is admitted by the daemon (index.supervisor.reap outcome=clean exit_code=0) and then refused by the worker with "/tiny is outside the allowed root. To allow it, run: codebase-memory-mcp allow-root ...". Any daemon whose starter had CBM_ALLOWED_ROOT set broke every other session outside it. Fix: in the worker arm of run_cli, right after the worker's server is created and before any tool runs, read repo_path from the request args (both daemon spawn paths rewrite the worker args with the canonical repo_path), canonicalize it and install it explicitly as both session root and allowed root. The daemon already authorized the request; the worker only executes it. A repository that cannot be canonicalized does not exist and can only have been admitted by a session with no declared boundary (containment needs a real path), so the worker mirrors that with an explicit unrestricted policy - still never the environment fallback - and the pipeline keeps reporting the missing repository as the tool error it always was. A request without repo_path fails closed with "request workspace scope invalid". No argv or IPC change, no process environment mutation. Regression test: tests/test_worker_session_scope.sh drives the real binary as a supervised worker with CBM_ALLOWED_ROOT= and repo_path=/tiny and requires an indexed response with no boundary refusal; a worker without repo_path must exit nonzero on the scope check. Wired as Step 5f of scripts/test.sh next to the other real-binary worker contracts. RED on main (the exact "outside the allowed root" refusal above), GREEN with this change. Distilled from #1925 with co-author credit. Verification: test-runner suites index_supervisor, daemon_application, daemon, mcp, cli green; tests/test_worker_error_response.sh, test_worker_watchdog.sh and test_parent_watchdog.sh green against the fixed binary; make lint-ci and scripts/check-no-test-skips.sh clean. Signed-off-by: Martin Vogel Co-authored-by: Zhiyu --- scripts/test.sh | 6 ++ src/main.c | 43 +++++++++++- tests/test_worker_session_scope.sh | 107 +++++++++++++++++++++++++++++ 3 files changed, 153 insertions(+), 3 deletions(-) create mode 100644 tests/test_worker_session_scope.sh diff --git a/scripts/test.sh b/scripts/test.sh index 0aafe6c20..43c140fee 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -346,6 +346,12 @@ CBM_TEST_BINARY="$WATCHDOG_BINARY" bash "$ROOT/tests/test_worker_error_response. echo "=== Step 5e: watcher_enabled kill-switch regression (#335) ===" CBM_TEST_BINARY="$WATCHDOG_BINARY" bash "$ROOT/tests/test_watcher_disabled.sh" +# Step 5f: a supervised worker is scoped to the request the daemon admitted, +# never to the CBM_ALLOWED_ROOT it inherited from the daemon starter's +# environment. Reuses the prod binary built in Step 5. +echo "=== Step 5f: worker request-scope regression ===" +CBM_TEST_BINARY="$WATCHDOG_BINARY" bash "$ROOT/tests/test_worker_session_scope.sh" + # Step 6: security-strings URL allow-list regression. The MSYS2 CLANG64 toolchain # bakes its package-tracker URL into the static Windows .exe; the binary string # audit must allow-list it (Windows-only — Linux smoke never saw it). diff --git a/src/main.c b/src/main.c index 8a575c44f..a836b7ac0 100644 --- a/src/main.c +++ b/src/main.c @@ -676,6 +676,36 @@ static bool cli_first_nonspace_is_brace(const char *s) { static char *main_local_cli_daemon_execute(const char *tool_name, const char *args_json, bool quiet_requested); +/* A supervised worker runs in the DAEMON's environment, not the requesting + * client's. The daemon admitted this request under the client's session policy + * and re-executed us with the canonical repo_path in the args (both daemon + * spawn paths rewrite it), so the request itself is the worker's workspace + * scope: repository == session root == allowed root. Without this, a fresh + * server fell back to the process-wide CBM_ALLOWED_ROOT inherited from whoever + * started the daemon and refused every admitted session outside it. Returns + * NULL once scoped, otherwise the reason to fail closed: a worker never indexes + * under an ambient policy. + * + * A repository that cannot be canonicalized (it does not exist) can only have + * been admitted by a session with no declared boundary, because containment + * needs a real path. The worker mirrors that with an explicit unrestricted + * policy - still never the environment fallback - so the pipeline reports the + * missing repository as the tool error it always was, instead of the + * supervisor misreading a refused worker as a crash. */ +static const char *main_index_worker_scope_request(cbm_mcp_server_t *srv, const char *args_json) { + char *repo_path = cbm_mcp_get_string_arg(args_json, "repo_path"); + if (!repo_path || !repo_path[0]) { + free(repo_path); + return "request names no repo_path"; + } + char canonical[MAIN_PATH_CAP]; + bool exists = cbm_canonical_path(repo_path, canonical, sizeof(canonical)) != 0; + const char *scope = exists ? canonical : repo_path; + bool scoped = cbm_mcp_server_set_session_context(srv, scope, exists ? scope : NULL); + free(repo_path); + return scoped ? NULL : "session context could not be installed"; +} + static int run_cli(int argc, char **argv, cbm_project_lock_manager_t *project_locks, main_local_maintenance_context_t *maintenance_context) { cbm_cli_output_flags_t output_flags; @@ -817,11 +847,15 @@ static int run_cli(int argc, char **argv, cbm_project_lock_manager_t *project_lo }; bool maintenance_binding_failed = false; bool maintenance_cancelled = false; + const char *worker_scope_refused = NULL; if (!index_worker) { result = main_local_cli_daemon_execute(tool_name, args_json, output_flags.quiet_requested); } else { srv = cbm_mcp_server_new(NULL); if (srv) { + worker_scope_refused = main_index_worker_scope_request(srv, args_json); + } + if (srv && !worker_scope_refused) { /* The in-process worker is a standalone instance: it may not * launch MCP-session background tasks. It receives project_locks * from its own process-level coordination setup and therefore @@ -834,8 +868,8 @@ static int run_cli(int argc, char **argv, cbm_project_lock_manager_t *project_lo main_local_cli_mutation_try_begin); } } - maintenance_binding_failed = srv && !maintenance_context; - if (srv && maintenance_context) { + maintenance_binding_failed = srv && !worker_scope_refused && !maintenance_context; + if (srv && !worker_scope_refused && maintenance_context) { main_local_maintenance_server_bind(maintenance_context, srv); result = cbm_mcp_handle_tool(srv, tool_name, args_json); /* Unbind under the same mutex used by cancellation before any @@ -847,7 +881,10 @@ static int run_cli(int argc, char **argv, cbm_project_lock_manager_t *project_lo } } if (!result) { - if (maintenance_binding_failed) { + if (worker_scope_refused) { + (void)fprintf(stderr, "error: request workspace scope invalid: %s\n", + worker_scope_refused); + } else if (maintenance_binding_failed) { (void)fprintf(stderr, "error: local %s maintenance cancellation could not bind safely\n", index_worker ? "worker" : "CLI"); diff --git a/tests/test_worker_session_scope.sh b/tests/test_worker_session_scope.sh new file mode 100644 index 000000000..fed32e97e --- /dev/null +++ b/tests/test_worker_session_scope.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# A supervised index worker (`cli --index-worker index_repository ...`) runs in +# the DAEMON's environment, not the requesting client's. The daemon has already +# admitted the request under the client's session policy and re-executes the +# worker with the canonical repo_path in its args, so the worker must scope its +# own workspace boundary to that request. Before the fix it built an unscoped +# server, fell back to the process-wide CBM_ALLOWED_ROOT it inherited from the +# daemon starter, and refused every admitted session outside that root with +# "... is outside the allowed root". A missing repo_path must fail closed: a +# worker never indexes under an ambient policy. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +BINARY="${CBM_TEST_BINARY:-${ROOT}/build/c/codebase-memory-mcp}" +if [[ ! -x "${BINARY}" && -x "${BINARY}.exe" ]]; then + BINARY="${BINARY}.exe" +fi + +if [[ ! -x "${BINARY}" ]]; then + echo "missing binary: ${BINARY}" >&2 + exit 2 +fi + +if command -v shasum >/dev/null 2>&1; then + BUILD_FINGERPRINT="$(shasum -a 256 "${BINARY}" | awk '{print $1}')" +elif command -v sha256sum >/dev/null 2>&1; then + BUILD_FINGERPRINT="$(sha256sum "${BINARY}" | awk '{print $1}')" +elif command -v openssl >/dev/null 2>&1; then + BUILD_FINGERPRINT="$(openssl dgst -sha256 "${BINARY}" | awk '{print $NF}')" +else + echo "no SHA-256 command available for worker build binding" >&2 + exit 2 +fi +if [[ ! "${BUILD_FINGERPRINT}" =~ ^[0-9a-f]{64}$ ]]; then + echo "invalid worker build fingerprint: ${BUILD_FINGERPRINT}" >&2 + exit 2 +fi + +# shellcheck source=../scripts/test-runtime.sh +source "${ROOT}/scripts/test-runtime.sh" +cbm_test_runtime_init +tmpdir="${CBM_TEST_RUNTIME_ROOT}" +cleanup() { + cbm_test_runtime_cleanup "${BINARY}" +} +trap cleanup EXIT + +# Product-facing path, the way the daemon hands it to the worker: canonical on +# POSIX, mixed-style on a native Windows binary (test-runtime.sh's convention). +product_path() { + case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) cygpath -m "$1" ;; + *) (cd "$1" && pwd -P) ;; + esac +} + +# Root A is what the daemon's starter allowed; the admitted session lives under +# root B. The two share no prefix. +mkdir -p "${tmpdir}/root-a" "${tmpdir}/root-b/tiny" +printf 'int tiny_main(void) { return 0; }\n' >"${tmpdir}/root-b/tiny/tiny.c" +root_a="$(product_path "${tmpdir}/root-a")" +repo="$(product_path "${tmpdir}/root-b/tiny")" + +run_worker() { + local args="$1" response="$2" out="$3" err="$4" + CBM_ALLOWED_ROOT="${root_a}" \ + "${BINARY}" cli --index-worker \ + --index-worker-build "${BUILD_FINGERPRINT}" \ + index_repository "${args}" \ + --response-out "${response}" >"${out}" 2>"${err}" +} + +response="${tmpdir}/scoped.response" +if ! run_worker "{\"repo_path\":\"${repo}\",\"mode\":\"fast\"}" "${response}" \ + "${tmpdir}/scoped.out" "${tmpdir}/scoped.err"; then + echo "worker exited nonzero for an admitted request" >&2 + cat "${tmpdir}/scoped.err" >&2 + exit 1 +fi +if [[ ! -s "${response}" ]]; then + echo "worker delivered no response" >&2 + exit 1 +fi +if grep -q 'outside the allowed root' "${response}"; then + echo "worker re-decided the workspace boundary from the daemon environment" >&2 + cat "${response}" >&2 + exit 1 +fi +if ! grep -q '"status":"indexed"' "${response}"; then + echo "worker did not index the admitted repository" >&2 + cat "${response}" >&2 + exit 1 +fi + +# Fail closed: no repo_path means no request scope, so no indexing at all. +unscoped="${tmpdir}/unscoped.response" +if run_worker '{"mode":"fast"}' "${unscoped}" "${tmpdir}/unscoped.out" "${tmpdir}/unscoped.err"; then + echo "worker ran without a request workspace scope" >&2 + exit 1 +fi +if [[ -s "${unscoped}" ]] || ! grep -q 'request workspace scope invalid' "${tmpdir}/unscoped.err"; then + echo "worker without repo_path did not fail closed on scope" >&2 + cat "${tmpdir}/unscoped.err" >&2 + exit 1 +fi + +echo "ok: index worker is scoped to the admitted request, not the daemon environment" From 4ce6b115956d51e7a0e115187c29424188170d7c Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 8 Sep 2026 01:19:34 +0200 Subject: [PATCH 021/130] fix(pipeline): never mint a staging path from a staging path The outer run rewrites the pipeline's db_path to its own stage, so the inner publication (gbuf dump and the delta clone) minted ITS stage from that stage: .stage.A.stage.B, with -wal/-shm beside it under WAL. Every such copy is a full-size generation, so one project could occupy three times its database on disk (#1839). Mint from the stage root instead. The root is recognised only by the exact minted shape ".stage.<6 alphanumerics>" in the basename, so a database whose own name merely contains ".stage." keeps its full name, and the marker is never matched across a path separator. The inner finalize still renames B over A; the unpredictable-name contract of the mkstemp helper is unchanged. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- src/pipeline/pipeline.c | 47 +++++++++++++++++++++++- tests/test_pipeline.c | 81 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 1 deletion(-) diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 0ffb46c0b..247f72378 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -38,6 +38,7 @@ enum { CBM_DIR_PERMS = 0755, PL_RING = 4, PL_RING_MASK = 3, PL_SEQ_PASSES = 6 }; #include "foundation/profile.h" #include "foundation/mem.h" +#include #include #include #include @@ -2415,12 +2416,56 @@ static bool ensure_db_parent(const char *path) { return ok; } +/* Length of the path a stage was minted for: the input itself unless its + * basename has exactly the minted shape ".stage.<6 alphanumerics>", in + * which case the root is . The outer run rewrites the pipeline's db_path + * to its stage, so the inner publication (dump and delta clone) used to mint + * ITS stage from that stage: .stage.A.stage.B, with -wal/-shm beside it + * (#1839). Minting from the root keeps every generation's stage a sibling of + * the live database. Only the exact minted shape is recognised: a database + * named "x.stage.y.db" is not a stage and keeps its full name. */ +enum { CBM_STAGE_SUFFIX_RANDOM_CHARS = 6 }; +static const char cbm_stage_marker[] = ".stage."; + +static bool stage_suffix_at(const char *tail) { + if (strncmp(tail, cbm_stage_marker, sizeof(cbm_stage_marker) - 1) != 0) { + return false; + } + const char *random = tail + sizeof(cbm_stage_marker) - 1; + for (int i = 0; i < CBM_STAGE_SUFFIX_RANDOM_CHARS; i++) { + if (!isalnum((unsigned char)random[i])) { + return false; + } + } + return random[CBM_STAGE_SUFFIX_RANDOM_CHARS] == '\0'; +} + +static size_t stage_root_length(const char *path) { + size_t len = strlen(path); + const size_t suffix_len = sizeof(cbm_stage_marker) - 1 + CBM_STAGE_SUFFIX_RANDOM_CHARS; + if (len <= suffix_len) { + return len; + } + size_t root_len = len - suffix_len; + /* The marker must sit inside the basename, never span a separator. */ + for (size_t i = root_len; i < len; i++) { + if (path[i] == '/' +#ifdef _WIN32 + || path[i] == '\\' +#endif + ) { + return len; + } + } + return stage_suffix_at(path + root_len) ? root_len : len; +} + static char *create_staging_path(const char *final_path) { if (!final_path) { return NULL; } static const char suffix[] = ".stage.XXXXXX"; - size_t final_len = strlen(final_path); + size_t final_len = stage_root_length(final_path); if (final_len > SIZE_MAX - sizeof(suffix)) { return NULL; } diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 0f6332cb2..e724736e2 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -3416,6 +3416,86 @@ TEST(pipeline_publication_never_uses_a_predictable_staging_path) { PASS(); } +static int count_substring(const char *haystack, const char *needle) { + int count = 0; + size_t needle_len = strlen(needle); + for (const char *at = strstr(haystack, needle); at; at = strstr(at + needle_len, needle)) { + count++; + } + return count; +} + +static int count_nested_stage_entries(const char *dir_path, const char *db_basename) { + cbm_dir_t *dir = cbm_opendir(dir_path); + if (!dir) { + return -1; + } + size_t base_len = strlen(db_basename); + int count = 0; + cbm_dirent_t *entry; + while ((entry = cbm_readdir(dir)) != NULL) { + if (strncmp(entry->name, db_basename, base_len) == 0 && + count_substring(entry->name + base_len, ".stage.") >= 2) { + count++; + } + } + cbm_closedir(dir); + return count; +} + +/* #1839: the outer run rewrites the pipeline's db_path to its stage, so the + * inner publication (dump or delta clone) minted ITS stage from a stage: + * .stage.A.stage.B, plus -wal/-shm under WAL. A generation's stage is a + * sibling of the live database, whichever path it is minted from; a database + * whose own basename merely contains ".stage." is not a stage and keeps its + * full name. */ +TEST(pipeline_stage_names_never_nest) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_stage_nesting_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/generation.db", tmp); + char odd_db[512]; + snprintf(odd_db, sizeof(odd_db), "%s/x.stage.y.db", tmp); + + char *outer = cbm_pipeline_create_staging_path(db_path); + ASSERT_NOT_NULL(outer); + char *inner = cbm_pipeline_create_staging_path(outer); + ASSERT_NOT_NULL(inner); + char *odd_stage = cbm_pipeline_create_staging_path(odd_db); + ASSERT_NOT_NULL(odd_stage); + + size_t db_len = strlen(db_path); + size_t odd_len = strlen(odd_db); + int outer_tokens = count_substring(outer, ".stage."); + int inner_tokens = count_substring(inner, ".stage."); + bool inner_is_sibling = + strncmp(inner, db_path, db_len) == 0 && strncmp(inner + db_len, ".stage.", 7) == 0; + bool inner_distinct = strcmp(inner, outer) != 0; + bool odd_keeps_basename = + strncmp(odd_stage, odd_db, odd_len) == 0 && strncmp(odd_stage + odd_len, ".stage.", 7) == 0; + int nested_entries = count_nested_stage_entries(tmp, "generation.db"); + + cbm_pipeline_discard_stage(inner); + cbm_pipeline_discard_stage(outer); + cbm_pipeline_discard_stage(odd_stage); + int leftover_entries = count_generation_stage_artifacts(tmp, "generation.db") + + count_generation_stage_artifacts(tmp, "x.stage.y.db"); + free(inner); + free(outer); + free(odd_stage); + th_rmtree(tmp); + + ASSERT_EQ(outer_tokens, 1); + ASSERT_EQ(inner_tokens, 1); + ASSERT_TRUE(inner_is_sibling); + ASSERT_TRUE(inner_distinct); + ASSERT_TRUE(odd_keeps_basename); + ASSERT_EQ(nested_entries, 0); + ASSERT_EQ(leftover_entries, 0); + PASS(); +} + /* Discovery and extraction must describe the same immutable generation. A * source file created after extraction is not present in the original file * list, so merely re-hashing that list cannot detect the race. Publication @@ -14120,6 +14200,7 @@ SUITE(pipeline_semantic_manifest_repro) { RUN_TEST(pipeline_git_context_change_forces_full_and_refreshes_branch); RUN_TEST(pipeline_global_extension_config_change_forces_full); RUN_TEST(pipeline_publication_never_uses_a_predictable_staging_path); + RUN_TEST(pipeline_stage_names_never_nest); RUN_TEST(pipeline_source_mutation_before_publication_preserves_previous_generation); RUN_TEST(pipeline_source_addition_before_publication_preserves_previous_generation); RUN_TEST(pipeline_tsconfig_mutation_before_publication_preserves_previous_generation); From 33c4fd80ef02f380acc92d40fc83f0840a5a272f Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 8 Sep 2026 01:35:48 +0200 Subject: [PATCH 022/130] fix(daemon): name errno and path in daemon.ipc.listen_failed A full /tmp made every daemon start die at pending publication, and the only durable trace was `daemon.ipc.listen_failed stage=pending_publication` -- no syscall, no errno, no path. The reporter needed hours (and a `df` on the wrong mount) to find the cause (#1828). Every listener failure now records the stage, the errno the failing step reported, and the artifact path it was operating on, and the log line carries all three: daemon.ipc.listen_failed stage=pending_publication errno=ENOSPC path=/tmp/cbm-daemon-1000/cbm-.pending.tmp The record publication helper notes which step refused (create, write, link, unlink, sync, close, read-back) and on which artifact, so the listener can name it. The bind failure keeps its errno but joins the same shape (stage=socket_bind errno=... path=...) instead of a bare numeric bind_errno. cbm_errno_name() maps the portable errno set to its symbol. cbm_daemon_ipc_listen_failure_detail() exposes the same triple to the daemon host for the fail-fast record that follows. Test: daemon_ipc_listen_failure_names_errno_and_path forces the record write to fail with ENOSPC through a narrow inert seam (the socket binds, the file is created, only its data write is refused -- the reporter's exact shape) and asserts the log line names the errno and the temp path, and that the namespace is left clean. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- src/daemon/ipc.c | 161 ++++++++++++++++++++++++++++++++++---- src/daemon/ipc.h | 19 +++++ src/daemon/ipc_internal.h | 5 ++ src/foundation/platform.c | 54 +++++++++++++ src/foundation/platform.h | 7 ++ tests/test_daemon_ipc.c | 81 +++++++++++++++++++ 6 files changed, 310 insertions(+), 17 deletions(-) diff --git a/src/daemon/ipc.c b/src/daemon/ipc.c index a706a7ca5..b62a4616d 100644 --- a/src/daemon/ipc.c +++ b/src/daemon/ipc.c @@ -9,6 +9,7 @@ #include "foundation/compat_thread.h" #include "foundation/log.h" #include "foundation/macos_acl.h" +#include "foundation/platform.h" #include "foundation/private_file_lock_internal.h" #include "foundation/sha256.h" #include "foundation/secure_random.h" @@ -53,6 +54,53 @@ const char *cbm_daemon_ipc_validation_detail(void) { return ipc_validation_detail_buffer; } +static cbm_daemon_ipc_listen_failure_t ipc_listen_failure; + +static void ipc_listen_failure_reset(void) { + memset(&ipc_listen_failure, 0, sizeof(ipc_listen_failure)); +} + +/* Record and log one listener failure. `name` is an artifact inside the + * endpoint's runtime directory, "" for the directory itself, or NULL when + * the stage has no path. The log line names the errno symbolically and the + * full path, so `daemon.ipc.listen_failed stage=pending_publication` alone + * (the whole trace #1828's reporter had) can no longer happen. */ +static void ipc_listen_failed(const char *runtime_dir, const char *stage, int errno_value, + const char *name) { + ipc_listen_failure_reset(); + (void)snprintf(ipc_listen_failure.stage, sizeof(ipc_listen_failure.stage), "%s", + stage ? stage : ""); + ipc_listen_failure.errno_value = errno_value; + if (runtime_dir && name) { + int written = name[0] ? snprintf(ipc_listen_failure.path, sizeof(ipc_listen_failure.path), + "%s/%s", runtime_dir, name) + : snprintf(ipc_listen_failure.path, sizeof(ipc_listen_failure.path), + "%s", runtime_dir); + if (written <= 0 || written >= (int)sizeof(ipc_listen_failure.path)) { + ipc_listen_failure.path[0] = '\0'; + } + } + const char *errno_name = errno_value != 0 ? cbm_errno_name(errno_value) : NULL; + if (errno_name && ipc_listen_failure.path[0]) { + cbm_log_error("daemon.ipc.listen_failed", "stage", stage, "errno", errno_name, "path", + ipc_listen_failure.path); + } else if (errno_name) { + cbm_log_error("daemon.ipc.listen_failed", "stage", stage, "errno", errno_name); + } else if (ipc_listen_failure.path[0]) { + cbm_log_error("daemon.ipc.listen_failed", "stage", stage, "path", ipc_listen_failure.path); + } else { + cbm_log_error("daemon.ipc.listen_failed", "stage", stage); + } +} + +bool cbm_daemon_ipc_listen_failure_detail(cbm_daemon_ipc_listen_failure_t *out) { + if (!out) { + return false; + } + *out = ipc_listen_failure; + return ipc_listen_failure.stage[0] != '\0'; +} + static bool instance_key_valid(const char *key) { if (!key) { return false; @@ -122,6 +170,19 @@ void cbm_daemon_ipc_windows_legacy_guard_release_failures_set_for_test(unsigned memory_order_release); } +#ifndef _WIN32 +static atomic_int g_posix_record_write_failure_errno_for_test; +#endif + +void cbm_daemon_ipc_posix_record_write_failure_set_for_test(int errno_value) { +#ifndef _WIN32 + atomic_store_explicit(&g_posix_record_write_failure_errno_for_test, errno_value, + memory_order_release); +#else + (void)errno_value; +#endif +} + #ifdef _WIN32 static cbm_daemon_ipc_startup_gate_fn g_startup_gate_for_test; static void *g_startup_gate_context_for_test; @@ -1815,6 +1876,29 @@ static bool posix_fd_write_all(int fd, const uint8_t *buffer, size_t length) { return true; } +/* The failing step of the most recent record publication in this process: + * its errno (0 when a validation predicate, not a syscall, refused) and the + * artifact name it was operating on. Read by the listener right after a + * publish returns false. */ +static int posix_publish_failure_errno; +static char posix_publish_failure_name[NAME_MAX + 1]; + +static void posix_publish_failure_note(int errno_value, const char *name) { + posix_publish_failure_errno = errno_value; + (void)snprintf(posix_publish_failure_name, sizeof(posix_publish_failure_name), "%s", + name ? name : ""); +} + +static bool posix_record_fd_write(int fd, const uint8_t *buffer, size_t length) { + int injected = + atomic_load_explicit(&g_posix_record_write_failure_errno_for_test, memory_order_acquire); + if (injected != 0) { + errno = injected; + return false; + } + return posix_fd_write_all(fd, buffer, length); +} + static bool posix_fd_pread_all(int fd, uint8_t *buffer, size_t length) { size_t offset = 0; while (offset < length) { @@ -2169,50 +2253,75 @@ static bool posix_socket_record_publish(const cbm_daemon_ipc_endpoint_t *endpoin return false; } struct stat existing; + errno = 0; if (fstatat(endpoint->dir_fd, record_name, &existing, AT_SYMLINK_NOFOLLOW) == 0 || errno != ENOENT) { + posix_publish_failure_note(errno == 0 ? EEXIST : errno, record_name); return false; } uint8_t record[POSIX_SOCKET_RECORD_SIZE]; if (!posix_socket_record_encode(magic, source, record)) { + posix_publish_failure_note(0, record_name); return false; } char temp_name[NAME_MAX + 1]; if (!posix_socket_record_temp_name(record_name, temp_name)) { + posix_publish_failure_note(ENAMETOOLONG, record_name); return false; } int fd = openat(endpoint->dir_fd, temp_name, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW | O_NONBLOCK, 0600); if (fd < 0) { + posix_publish_failure_note(errno, temp_name); return false; } struct stat created = {0}; bool temp_exists = true; bool stable_linked = false; + errno = 0; bool ok = fd_set_cloexec(fd) && fchmod(fd, 0600) == 0 && private_regular_file_snapshot(endpoint->dir_fd, temp_name, fd, 1, &created) && - posix_fd_write_all(fd, record, sizeof(record)) && fsync(fd) == 0 && + posix_record_fd_write(fd, record, sizeof(record)) && fsync(fd) == 0 && private_regular_file_snapshot(endpoint->dir_fd, temp_name, fd, 1, &created) && created.st_size == (off_t)POSIX_SOCKET_RECORD_SIZE && endpoint_runtime_still_valid(endpoint); + if (!ok) { + posix_publish_failure_note(errno, temp_name); + } if (ok) { posix_record_publication_stage_reached(magic, false); + errno = 0; ok = posix_linkat_no_follow(endpoint->dir_fd, temp_name, endpoint->dir_fd, record_name) == 0; stable_linked = ok; + if (!ok) { + posix_publish_failure_note(errno, record_name); + } } if (ok) { posix_record_publication_stage_reached(magic, true); + errno = 0; ok = posix_path_unlink_regular_if_matches(endpoint->dir_fd, temp_name, created.st_dev, created.st_ino, 2); temp_exists = !ok; + if (!ok) { + posix_publish_failure_note(errno, temp_name); + } } if (ok) { + errno = 0; ok = posix_directory_sync(endpoint->dir_fd); + if (!ok) { + posix_publish_failure_note(errno, ""); + } } + errno = 0; if (close(fd) != 0) { + if (ok) { + posix_publish_failure_note(errno, temp_name); + } ok = false; } @@ -2224,6 +2333,9 @@ static bool posix_socket_record_publish(const cbm_daemon_ipc_endpoint_t *endpoin posix_socket_identity_equal(&published_record.identity, &source->identity) && strcmp(published_record.anchor_name, source->anchor_name) == 0 && published_status.st_dev == created.st_dev && published_status.st_ino == created.st_ino; + if (!ok) { + posix_publish_failure_note(0, record_name); + } } if (!ok) { if (stable_linked) { @@ -2539,8 +2651,11 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved( cbm_daemon_ipc_lifetime_reservation_t **reservation_io) { cbm_daemon_ipc_lifetime_reservation_t *lifetime_reservation = reservation_io ? *reservation_io : NULL; + ipc_listen_failure_reset(); + posix_publish_failure_note(0, NULL); + const char *runtime_dir = endpoint ? endpoint->runtime_dir : NULL; if (!lifetime_reservation_matches_endpoint(endpoint, lifetime_reservation)) { - cbm_log_error("daemon.ipc.listen_failed", "stage", "reservation_validation"); + ipc_listen_failed(runtime_dir, "reservation_validation", 0, NULL); return NULL; } /* Stale removal happens only under the startup lock, before a daemon host @@ -2550,7 +2665,7 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved( char pending_temp_name[NAME_MAX + 1]; if (!posix_socket_record_temp_name(endpoint->socket_identity_name, identity_temp_name) || !posix_socket_record_temp_name(endpoint->socket_pending_name, pending_temp_name)) { - cbm_log_error("daemon.ipc.listen_failed", "stage", "temp_names"); + ipc_listen_failed(runtime_dir, "temp_names", ENAMETOOLONG, endpoint->socket_pending_name); return NULL; } struct stat existing; @@ -2559,27 +2674,32 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved( endpoint->socket_pending_name, identity_temp_name, pending_temp_name, }; bool namespace_absent = true; + const char *occupied_name = ""; + int occupied_errno = 0; for (size_t index = 0; index < sizeof(required_absent) / sizeof(required_absent[0]); index++) { + errno = 0; if (fstatat(endpoint->dir_fd, required_absent[index], &existing, AT_SYMLINK_NOFOLLOW) == 0 || errno != ENOENT) { namespace_absent = false; + occupied_name = required_absent[index]; + occupied_errno = errno == 0 ? EEXIST : errno; break; } } if (!endpoint_runtime_still_valid(endpoint) || !namespace_absent) { - cbm_log_error("daemon.ipc.listen_failed", "stage", "namespace_validation"); + ipc_listen_failed(runtime_dir, "namespace_validation", occupied_errno, occupied_name); return NULL; } int fd = local_socket_new(); if (fd < 0) { - cbm_log_error("daemon.ipc.listen_failed", "stage", "socket_creation"); + ipc_listen_failed(runtime_dir, "socket_creation", errno, NULL); return NULL; } cbm_daemon_ipc_listener_t *listener = calloc(1, sizeof(*listener)); if (!listener) { - cbm_log_error("daemon.ipc.listen_failed", "stage", "listener_allocation"); + ipc_listen_failed(runtime_dir, "listener_allocation", ENOMEM, NULL); (void)close(fd); return NULL; } @@ -2597,7 +2717,8 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved( if (listener->dir_fd < 0 || !fd_set_cloexec(listener->dir_fd) || !listener->runtime_dir || !listener->address || !listener->socket_name || !listener->socket_anchor_name || !listener->socket_identity_name || !listener->socket_pending_name) { - cbm_log_error("daemon.ipc.listen_failed", "stage", "listener_initialization"); + ipc_listen_failed(runtime_dir, "listener_initialization", + listener->dir_fd < 0 ? errno : ENOMEM, NULL); if (listener->dir_fd >= 0) { (void)close(listener->dir_fd); } @@ -2615,20 +2736,19 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved( struct sockaddr_un address; socklen_t address_length; if (!unix_address_set(&address, endpoint->socket_anchor_address, &address_length)) { - cbm_log_error("daemon.ipc.listen_failed", "stage", "socket_address"); + ipc_listen_failed(runtime_dir, "socket_address", ENAMETOOLONG, + endpoint->socket_anchor_name); cbm_daemon_ipc_listener_close(listener); return NULL; } if (bind(fd, (const struct sockaddr *)&address, address_length) != 0) { - int bind_error = errno; - char error_text[32]; - (void)snprintf(error_text, sizeof(error_text), "%d", bind_error); - cbm_log_error("daemon.ipc.listen_failed", "bind_errno", error_text); + ipc_listen_failed(runtime_dir, "socket_bind", errno, endpoint->socket_anchor_name); cbm_daemon_ipc_listener_close(listener); return NULL; } struct stat bound_status; + errno = 0; bool bound_path_ok = fstatat(endpoint->dir_fd, endpoint->socket_anchor_name, &bound_status, AT_SYMLINK_NOFOLLOW) == 0 && S_ISSOCK(bound_status.st_mode) && bound_status.st_uid == geteuid(); @@ -2648,11 +2768,13 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved( anchor_status.st_ino != bound_status.st_ino || !posix_socket_identity_from_stat(&anchor_status, &anchor_identity) || !posix_directory_sync(endpoint->dir_fd)) { + int security_errno = errno; if (bound_path_ok) { posix_bound_socket_unlink_if_matches(endpoint->dir_fd, endpoint->socket_anchor_name, bound_status.st_dev, bound_status.st_ino); } - cbm_log_error("daemon.ipc.listen_failed", "stage", "socket_security"); + ipc_listen_failed(runtime_dir, "socket_security", security_errno, + endpoint->socket_anchor_name); cbm_daemon_ipc_listener_close(listener); return NULL; } @@ -2669,16 +2791,19 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved( endpoint, endpoint->socket_pending_name, POSIX_SOCKET_PENDING_MAGIC, &pending, &pending_status.st_dev, &pending_status.st_ino); if (!pending_published) { - cbm_log_error("daemon.ipc.listen_failed", "stage", "pending_publication"); + ipc_listen_failed(runtime_dir, "pending_publication", posix_publish_failure_errno, + posix_publish_failure_name); posix_publication_abort(endpoint, &anchor_identity, false, NULL, false, NULL); cbm_daemon_ipc_listener_close(listener); return NULL; } posix_publication_stage_reached(CBM_DAEMON_IPC_POSIX_PUBLICATION_PENDING_DURABLE); + errno = 0; bool stable_linked = posix_linkat_no_follow(endpoint->dir_fd, endpoint->socket_anchor_name, endpoint->dir_fd, endpoint->socket_name) == 0 && posix_directory_sync(endpoint->dir_fd); + int stable_errno = stable_linked ? 0 : errno; posix_socket_identity_t stable_identity = {0}; posix_socket_identity_t committed_identity = {0}; struct stat stable_status = {0}; @@ -2693,7 +2818,7 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved( posix_socket_identity_equal(&stable_identity, &committed_identity) && posix_socket_inode_equal(&anchor_identity, &committed_identity); if (!stable_valid) { - cbm_log_error("daemon.ipc.listen_failed", "stage", "stable_publication"); + ipc_listen_failed(runtime_dir, "stable_publication", stable_errno, endpoint->socket_name); posix_publication_abort(endpoint, &anchor_identity, pending_published, &pending_status, false, NULL); cbm_daemon_ipc_listener_close(listener); @@ -2712,7 +2837,8 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved( endpoint, endpoint->socket_identity_name, POSIX_SOCKET_MARKER_MAGIC, &marker, &marker_status.st_dev, &marker_status.st_ino); if (!marker_published) { - cbm_log_error("daemon.ipc.listen_failed", "stage", "marker_publication"); + ipc_listen_failed(runtime_dir, "marker_publication", posix_publish_failure_errno, + posix_publish_failure_name); posix_publication_abort(endpoint, &committed_identity, pending_published, &pending_status, false, NULL); cbm_daemon_ipc_listener_close(listener); @@ -2722,10 +2848,11 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved( listener->identity_inode = marker_status.st_ino; posix_publication_stage_reached(CBM_DAEMON_IPC_POSIX_PUBLICATION_MARKER_DURABLE); + errno = 0; if (!posix_path_unlink_regular_if_matches(endpoint->dir_fd, endpoint->socket_pending_name, pending_status.st_dev, pending_status.st_ino, 1) || !posix_directory_sync(endpoint->dir_fd)) { - cbm_log_error("daemon.ipc.listen_failed", "stage", "pending_removal"); + ipc_listen_failed(runtime_dir, "pending_removal", errno, endpoint->socket_pending_name); posix_publication_abort(endpoint, &committed_identity, true, &pending_status, marker_published, &marker_status); cbm_daemon_ipc_listener_close(listener); diff --git a/src/daemon/ipc.h b/src/daemon/ipc.h index 2cdf792dd..daccf6900 100644 --- a/src/daemon/ipc.h +++ b/src/daemon/ipc.h @@ -91,6 +91,25 @@ bool cbm_daemon_ipc_private_directory_secure(const char *directory_path); * validation refusal (empty string when none). Diagnostic only — callers * append it to their error messages; policy decisions never read it. */ const char *cbm_daemon_ipc_validation_detail(void); + +/* This process's most recent listener-publication failure: the stage that + * refused, the errno the failing step reported (0 when the step reported + * none), and the artifact path it was operating on ("" when no path applies). + * Every `daemon.ipc.listen_failed` log line carries the same three fields. + * Reset at the start of each listen attempt. Returns false while no failure + * has been recorded. Diagnostic only: the daemon host copies it into its + * durable start-failure record so a waiting client can name the cause instead + * of burning its full startup deadline (#1828). */ +enum { + CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP = 32, + CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP = 4096 +}; +typedef struct { + char stage[CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP]; + int errno_value; + char path[CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP]; +} cbm_daemon_ipc_listen_failure_t; +bool cbm_daemon_ipc_listen_failure_detail(cbm_daemon_ipc_listen_failure_t *out); #ifdef CBM_ENABLE_TEST_SEAMS /* #1537: seed the detail so a test can prove the CLI refusal surfaces it. */ void cbm_daemon_ipc_set_validation_detail_for_testing(const char *detail); diff --git a/src/daemon/ipc_internal.h b/src/daemon/ipc_internal.h index b791a6b02..4b7cf1fd5 100644 --- a/src/daemon/ipc_internal.h +++ b/src/daemon/ipc_internal.h @@ -95,6 +95,11 @@ typedef void (*cbm_daemon_ipc_posix_publication_hook_fn)( void cbm_daemon_ipc_posix_publication_hook_set_for_test( cbm_daemon_ipc_posix_publication_hook_fn hook, void *context); void cbm_daemon_ipc_windows_legacy_guard_release_failures_set_for_test(unsigned int count); +/* #1828: while non-zero, every POSIX socket-record publication write fails + * with this errno. This is the deterministic stand-in for a full runtime + * filesystem (tmpfs ENOSPC): the socket binds, the record file is created, + * and only its data write is refused -- exactly the reporter's failure shape. */ +void cbm_daemon_ipc_posix_record_write_failure_set_for_test(int errno_value); /* Deterministic-interleaving seam: fires on the Windows startup path once the * startup lock is held, before the rendezvous handoff. A test parks here to diff --git a/src/foundation/platform.c b/src/foundation/platform.c index 14e36c0d8..cf9a0f469 100644 --- a/src/foundation/platform.c +++ b/src/foundation/platform.c @@ -543,6 +543,60 @@ const char *cbm_app_local_dir(void) { /* ── Cache directory ────────────────────────── */ +const char *cbm_errno_name(int error) { + static const struct { + int value; + const char *name; + } names[] = { + {EACCES, "EACCES"}, + {EAGAIN, "EAGAIN"}, + {EBUSY, "EBUSY"}, + {EEXIST, "EEXIST"}, + {EFBIG, "EFBIG"}, + {EINTR, "EINTR"}, + {EINVAL, "EINVAL"}, + {EIO, "EIO"}, + {EISDIR, "EISDIR"}, + {ELOOP, "ELOOP"}, + {EMFILE, "EMFILE"}, + {EMLINK, "EMLINK"}, + {ENAMETOOLONG, "ENAMETOOLONG"}, + {ENFILE, "ENFILE"}, + {ENOENT, "ENOENT"}, + {ENOMEM, "ENOMEM"}, + {ENOSPC, "ENOSPC"}, + {ENOTDIR, "ENOTDIR"}, + {ENOTEMPTY, "ENOTEMPTY"}, + {ENXIO, "ENXIO"}, + {EPERM, "EPERM"}, + {EROFS, "EROFS"}, + {ETXTBSY, "ETXTBSY"}, + {EXDEV, "EXDEV"}, + {ETIMEDOUT, "ETIMEDOUT"}, + {ECONNREFUSED, "ECONNREFUSED"}, + {EADDRINUSE, "EADDRINUSE"}, + {ENOTSOCK, "ENOTSOCK"}, + {EPIPE, "EPIPE"}, +#ifdef EDQUOT + {EDQUOT, "EDQUOT"}, +#endif +#ifdef EOVERFLOW + {EOVERFLOW, "EOVERFLOW"}, +#endif +#ifdef ENOTSUP + {ENOTSUP, "ENOTSUP"}, +#endif + }; + for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { + if (names[i].value == error) { + return names[i].name; + } + } + static CBM_TLS char fallback[16]; + (void)snprintf(fallback, sizeof(fallback), "%d", error); + return fallback; +} + const char *cbm_resolve_cache_dir(void) { static CBM_TLS char buf[CBM_SZ_4K]; static const char missing[] = "\x1f" diff --git a/src/foundation/platform.h b/src/foundation/platform.h index 938cf5905..cc58e8429 100644 --- a/src/foundation/platform.h +++ b/src/foundation/platform.h @@ -94,6 +94,13 @@ uint64_t cbm_now_ns(void); /* Monotonic millisecond timestamp. */ uint64_t cbm_now_ms(void); +/* Symbolic name for an errno value ("ENOSPC"), or the decimal number when the + * value is not in the portable table. The fallback lives in thread-local + * storage; copy it before the next call on the same thread. Diagnostics only: + * a log line that says `errno=ENOSPC path=...` is a one-line diagnosis where + * `stage=pending_publication` alone cost a reporter hours (#1828). */ +const char *cbm_errno_name(int error); + /* ── System info ───────────────────────────────────────────────── */ /* Number of available CPU cores. */ diff --git a/tests/test_daemon_ipc.c b/tests/test_daemon_ipc.c index 9bb550948..c57fa7bb6 100644 --- a/tests/test_daemon_ipc.c +++ b/tests/test_daemon_ipc.c @@ -13,6 +13,7 @@ #include "daemon/ipc_internal.h" #include "foundation/compat.h" #include "foundation/compat_thread.h" +#include "foundation/log.h" #include "foundation/platform.h" #include "foundation/private_file_lock_internal.h" #include "foundation/subprocess.h" @@ -4952,6 +4953,83 @@ TEST(daemon_ipc_posix_world_writable_ancestor_still_refused_issue1537) { } #endif /* !_WIN32 */ +#ifndef _WIN32 +enum { IPC_TEST_LOG_CAPTURE_CAP = 16384 }; +static char ipc_test_log_capture[IPC_TEST_LOG_CAPTURE_CAP]; +static size_t ipc_test_log_capture_used; + +static void ipc_test_log_capture_sink(const char *line) { + if (!line) { + return; + } + size_t length = strlen(line); + if (ipc_test_log_capture_used + length + 2 > sizeof(ipc_test_log_capture)) { + return; + } + memcpy(ipc_test_log_capture + ipc_test_log_capture_used, line, length); + ipc_test_log_capture_used += length; + ipc_test_log_capture[ipc_test_log_capture_used++] = '\n'; + ipc_test_log_capture[ipc_test_log_capture_used] = '\0'; +} + +/* #1828: a full /tmp made every daemon start die at pending publication, and + * the only durable trace was `daemon.ipc.listen_failed stage=pending_publication` + * -- no syscall, no errno, no path. The reporter needed hours (and a wrong + * `df` on the wrong mount) to find the cause. The failure line must name the + * errno and the exact artifact path that could not be written. */ +TEST(daemon_ipc_listen_failure_names_errno_and_path) { + static const char key[] = "1828000000000001"; + char parent[TEST_PATH_CAP] = {0}; + char runtime_dir[TEST_PATH_CAP] = {0}; + char socket_path[TEST_PATH_CAP] = {0}; + char pending_path[TEST_PATH_CAP] = {0}; + char expected_path[TEST_PATH_CAP + 16] = {0}; + cbm_daemon_ipc_endpoint_t *endpoint = NULL; + cbm_daemon_ipc_listener_t *listener = NULL; + + bool parent_ok = ipc_test_parent_new(parent, "enospc-diag"); + if (parent_ok) { + endpoint = cbm_daemon_ipc_endpoint_new(key, parent); + } + if (endpoint) { + ipc_test_copy_path(runtime_dir, cbm_daemon_ipc_endpoint_runtime_dir(endpoint)); + ipc_test_copy_path(socket_path, cbm_daemon_ipc_endpoint_address(endpoint)); + } + bool paths_ok = endpoint && ipc_test_socket_pending_path(pending_path, socket_path) && + snprintf(expected_path, sizeof(expected_path), "path=%s.tmp", pending_path) > 0; + if (paths_ok) { + ipc_test_log_capture_used = 0; + ipc_test_log_capture[0] = '\0'; + cbm_daemon_ipc_posix_record_write_failure_set_for_test(ENOSPC); + cbm_log_set_sink_ex(ipc_test_log_capture_sink, CBM_LOG_SINK_REPLACE); + listener = cbm_daemon_ipc_listen(endpoint); + cbm_log_set_sink(NULL); + cbm_daemon_ipc_posix_record_write_failure_set_for_test(0); + } + const char *failed = strstr(ipc_test_log_capture, "msg=daemon.ipc.listen_failed"); + bool stage_named = failed && strstr(failed, "stage=pending_publication") != NULL; + bool errno_named = failed && strstr(failed, "errno=ENOSPC") != NULL; + bool path_named = failed && strstr(failed, expected_path) != NULL; + struct stat leftover; + bool namespace_clean = paths_ok && lstat(socket_path, &leftover) != 0 && errno == ENOENT && + lstat(pending_path, &leftover) != 0 && errno == ENOENT; + + cbm_daemon_ipc_listener_close(listener); + cbm_daemon_ipc_endpoint_free(endpoint); + th_cleanup(parent_ok ? parent : NULL); + + ASSERT_TRUE(parent_ok); + ASSERT_TRUE(paths_ok); + ASSERT_TRUE(listener == NULL); + ASSERT_TRUE(failed != NULL); + ASSERT_TRUE(stage_named); + ASSERT_TRUE(errno_named); + ASSERT_TRUE(path_named); + ASSERT_TRUE(namespace_clean); + PASS(); +} +#endif + SUITE(daemon_ipc) { RUN_TEST(daemon_ipc_pending_timeout_race_returns_completed_io); RUN_TEST(daemon_ipc_pending_wait_failure_cancels_and_drains); @@ -5019,5 +5097,8 @@ SUITE(daemon_ipc) { RUN_TEST(daemon_ipc_posix_private_directory_rejects_world_writable_ancestor); RUN_TEST(daemon_ipc_posix_private_log_rejects_symlinks_and_is_owner_only); RUN_TEST(daemon_ipc_posix_rejects_non_socket_and_symlink_endpoints); +#ifndef _WIN32 + RUN_TEST(daemon_ipc_listen_failure_names_errno_and_path); +#endif #endif } From c572ddc4f3b619ce8725f5336feb49929a88a029 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 8 Sep 2026 01:37:18 +0200 Subject: [PATCH 023/130] fix(cli): uninstall no longer gated by agent-config cleanup failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A single agent configuration the editors refuse to touch (the reporter's ~/.cursor/mcp.json was a symlink into a dotfiles checkout) aborted the uninstall activation BEFORE index and binary removal. `uninstall -y` exited 1 and left a 300 MB executable plus the whole cache behind — the data-loss half of #1954 — while the transcript still read "removed". Agent-config failures are now collected instead of gating: every cleanup runs, the indexes and the executable go, and the command ends with one list of the files that could not be cleaned (agent, operation, path, observed target) and a non-zero exit. The existing per-error stderr line is unchanged. Two tests encoded the old hostage contract ("binary must survive failed cleanup") and move to the new one. The new test drives `uninstall -y` against a sandbox HOME with a dangling ~/.cursor/mcp.json link and asserts binary + index gone, exit non-zero, and the file named on stderr; on main it fails at ASSERT(binary_gone). (#1954) Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- src/cli/cli.c | 101 ++++++++++++++++++++++++++++--- tests/test_cli.c | 154 +++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 241 insertions(+), 14 deletions(-) diff --git a/src/cli/cli.c b/src/cli/cli.c index 249b43e6f..29ddd7f4f 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -7729,6 +7729,75 @@ static cbm_install_plan_t *g_install_plan = NULL; static int g_agent_install_errors = 0; static int g_agent_uninstall_errors = 0; +/* Every agent configuration uninstall could not clean, kept for the closing + * summary. A cleanup failure no longer stops executable and index removal + * (#1954: one symlinked ~/.cursor/mcp.json left a 300 MB binary plus the whole + * cache behind), so the user needs ONE list of what is still theirs to fix, + * with the observed reason next to each file. */ +typedef struct { + char agent[64]; + char operation[48]; + char path[CLI_BUF_1K]; + char reason[160]; + char detail[160]; +} cbm_agent_config_failure_t; + +static cbm_agent_config_failure_t *g_agent_uninstall_failures = NULL; +static int g_agent_uninstall_failure_count = 0; +static int g_agent_uninstall_failure_cap = 0; + +static void agent_uninstall_failures_reset(void) { + free(g_agent_uninstall_failures); + g_agent_uninstall_failures = NULL; + g_agent_uninstall_failure_count = 0; + g_agent_uninstall_failure_cap = 0; +} + +static void agent_uninstall_failure_record(const char *agent, const char *operation, + const char *path, const char *reason, + const char *detail) { + if (g_agent_uninstall_failure_count >= g_agent_uninstall_failure_cap) { + int ncap = g_agent_uninstall_failure_cap ? g_agent_uninstall_failure_cap * 2 : CLI_BUF_16; + cbm_agent_config_failure_t *grown = + realloc(g_agent_uninstall_failures, (size_t)ncap * sizeof(*grown)); + if (!grown) { + return; + } + g_agent_uninstall_failures = grown; + g_agent_uninstall_failure_cap = ncap; + } + cbm_agent_config_failure_t *entry = + &g_agent_uninstall_failures[g_agent_uninstall_failure_count++]; + (void)snprintf(entry->agent, sizeof(entry->agent), "%s", agent ? agent : "unknown"); + (void)snprintf(entry->operation, sizeof(entry->operation), "%s", + operation ? operation : "unknown"); + (void)snprintf(entry->path, sizeof(entry->path), "%s", path ? path : "unknown"); + (void)snprintf(entry->reason, sizeof(entry->reason), "%s", reason ? reason : ""); + (void)snprintf(entry->detail, sizeof(entry->detail), "%s", detail ? detail : ""); +} + +/* The closing list of what uninstall could not clean. Printed AFTER the + * executable and the indexes are gone, so nothing in it is a reason to keep + * the installation around — each line is one file the user removes an entry + * from by hand. */ +static void agent_uninstall_failures_report(bool dry_run) { + if (g_agent_uninstall_failure_count == 0) { + return; + } + (void)fprintf(stderr, "\nerror: uninstall %s with %d agent configuration(s) left uncleaned:\n", + dry_run ? "dry-run finished" : "finished", g_agent_uninstall_failure_count); + for (int i = 0; i < g_agent_uninstall_failure_count; i++) { + const cbm_agent_config_failure_t *entry = &g_agent_uninstall_failures[i]; + (void)fprintf(stderr, " %s (%s): %s", entry->agent, entry->operation, entry->path); + if (entry->reason[0]) { + (void)fprintf(stderr, " reason=%s", entry->reason); + } + (void)fputs(entry->detail, stderr); + (void)fputc('\n', stderr); + } + (void)fputs("Remove the codebase-memory-mcp entries from these files by hand.\n", stderr); +} + static void plan_record(const char *agent, const char *kind, const char *path) { if (!g_install_plan || !path || !path[0]) { return; @@ -7801,6 +7870,9 @@ static void record_agent_config_error_with_reason(bool uninstalling, const char } (void)fputs(detail, stderr); (void)fputc('\n', stderr); + if (uninstalling) { + agent_uninstall_failure_record(agent, operation, path, reason, detail); + } } static void record_agent_config_error(bool uninstalling, const char *agent, const char *operation, @@ -12044,13 +12116,12 @@ static int cli_uninstall_activate(void *opaque) { uninstall_additional_agents(&activation->agents, activation->home, activation->dry_run); uninstall_agent_client_registry(activation->home, activation->dry_run); - if (g_agent_uninstall_errors != 0) { - cli_activation_transaction_abort_or_fail_stop(&activation->binary_transaction, - "uninstall_transaction_config_cleanup_abort"); - (void)fprintf(stderr, "error: one or more agent cleanup operations failed; executable " - "and index removal were not started\n"); - return CLI_ACTIVATION_PARTIAL; - } + /* Agent-config failures are collected, never a gate: an entry the editors + * refuse to touch (a symlinked config, a foreign file, a malformed + * document) is the user's to fix by hand, and leaving a 300 MB executable + * plus every index behind because of it is the data-loss shape of #1954. + * The indexes and the executable go now; the failures are listed at the + * end and decide the exit code. */ if (activation->delete_indexes && !activation->dry_run) { int expected = count_db_indexes(activation->home); @@ -12151,6 +12222,7 @@ int cbm_cmd_uninstall(int argc, char **argv) { printf("codebase-memory-mcp uninstall\n\n"); g_agent_uninstall_errors = 0; + agent_uninstall_failures_reset(); cbm_detected_agents_t agents = cbm_detect_agents(home); /* Confirm index removal outside the startup lock, but defer the mutation @@ -12220,6 +12292,19 @@ int cbm_cmd_uninstall(int argc, char **argv) { (void)cli_activation_transaction_abort(&activation.binary_transaction); } if (activation_rc != CLI_OK) { + agent_uninstall_failures_reset(); + return CLI_TRUE; + } + + if (g_agent_uninstall_errors != 0) { + agent_uninstall_failures_report(dry_run); + agent_uninstall_failures_reset(); + printf("\nUninstall finished with errors; the files listed above still hold " + "codebase-memory-mcp entries. Please restart your coding-agent sessions " + "to properly take this into account.\n"); + if (dry_run) { + printf("(dry-run — no files were modified)\n"); + } return CLI_TRUE; } @@ -12228,7 +12313,7 @@ int cbm_cmd_uninstall(int argc, char **argv) { if (dry_run) { printf("(dry-run — no files were modified)\n"); } - return g_agent_uninstall_errors == 0 ? 0 : CLI_TRUE; + return 0; } /* ── Subcommand: update ───────────────────────────────────────── */ diff --git a/tests/test_cli.c b/tests/test_cli.c index 10ec1097a..aab397da4 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -714,6 +714,59 @@ static void test_rmdir_r(const char *path) { th_rmtree(path); } +/* Capture everything a command writes to one fd (stdout or stderr) so a test + * can assert on the transcript. Restores the fd on end and returns the text. */ +typedef struct { + FILE *file; + FILE *stream; + int target_fd; + int saved_fd; + bool redirected; +} cli_fd_capture_t; + +static void cli_fd_capture_begin(cli_fd_capture_t *capture, FILE *stream, int target_fd) { + memset(capture, 0, sizeof(*capture)); + capture->stream = stream; + capture->target_fd = target_fd; + capture->saved_fd = -1; + capture->file = tmpfile(); + if (!capture->file) { + return; + } + capture->saved_fd = dup(target_fd); + if (capture->saved_fd < 0) { + return; + } + fflush(stream); + capture->redirected = dup2(fileno(capture->file), target_fd) >= 0; +} + +/* Returns the captured text (heap, "" when nothing was written) or NULL when + * the capture never engaged. */ +static char *cli_fd_capture_end(cli_fd_capture_t *capture) { + fflush(capture->stream); + if (capture->saved_fd >= 0) { + (void)dup2(capture->saved_fd, capture->target_fd); + close(capture->saved_fd); + capture->saved_fd = -1; + } + char *text = NULL; + if (capture->file) { + if (capture->redirected) { + rewind(capture->file); + size_t capacity = 65536U; + text = calloc(1U, capacity); + if (text) { + size_t count = fread(text, 1U, capacity - 1U, capture->file); + text[count] = '\0'; + } + } + fclose(capture->file); + capture->file = NULL; + } + return text; +} + /* Mandatory-daemon activation guard fixture. The production path uses the * stable per-account endpoint directly; these callbacks make every race * ordering deterministic without exposing a CLI flag or environment bypass. */ @@ -2269,6 +2322,88 @@ TEST(cli_uninstall_preserves_binary_and_index_when_cohort_does_not_drain) { PASS(); } +#ifndef _WIN32 +/* #1954: an agent config that cannot be cleaned must not hold the executable + * and the indexes hostage. The fixture's ~/.cursor/mcp.json is a dangling + * symlink (a dotfiles checkout that moved) — a shape every config editor keeps + * refusing. Before the fix the cleanup failure aborted the activation BEFORE + * index and binary removal: `uninstall -y` exited 1 and left a 300 MB binary + * plus the whole cache behind, while the report still read "removed". */ +TEST(cli_uninstall_removes_binary_and_index_when_agent_config_cleanup_fails) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-uninstall-hostage-XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) { + FAIL("cbm_mkdtemp failed"); + } + char *old_home = NULL; + char *old_cache = NULL; + cli_activation_save_env(&old_home, &old_cache); + cbm_setenv("HOME", tmpdir, 1); + /* PATH moves with HOME so no real agent on the developer's PATH is detected. */ + char *old_path = save_test_env("PATH"); + cbm_setenv("PATH", tmpdir, 1); + + char cache_dir[512]; + char index_path[640]; + snprintf(cache_dir, sizeof(cache_dir), "%s/cache", tmpdir); + cbm_setenv("CBM_CACHE_DIR", cache_dir, 1); + test_mkdirp(cache_dir); + snprintf(index_path, sizeof(index_path), "%s/project.db", cache_dir); + write_test_file(index_path, "index must go even when a config cannot be cleaned"); + + char bin_dir[512]; + char bin_target[640]; + snprintf(bin_dir, sizeof(bin_dir), "%s/.local/bin", tmpdir); + test_mkdirp(bin_dir); + snprintf(bin_target, sizeof(bin_target), "%s/codebase-memory-mcp", bin_dir); + write_test_file(bin_target, "binary must go even when a config cannot be cleaned"); + + char cursor_dir[512]; + char cursor_config[640]; + char dangling_target[640]; + snprintf(cursor_dir, sizeof(cursor_dir), "%s/.cursor", tmpdir); + test_mkdirp(cursor_dir); + snprintf(cursor_config, sizeof(cursor_config), "%s/mcp.json", cursor_dir); + snprintf(dangling_target, sizeof(dangling_target), "%s/.dotfiles/cursor/mcp.json", tmpdir); + if (symlink(dangling_target, cursor_config) != 0) { + FAIL("symlink failed"); + } + + cli_activation_fake_t fake = {.mutation_reserve_result = 1}; + cbm_cli_activation_ops_t ops = cli_activation_fake_ops(&fake); + cbm_cli_set_activation_ops_for_test(&ops); + cli_fd_capture_t out_capture; + cli_fd_capture_t err_capture; + cli_fd_capture_begin(&out_capture, stdout, STDOUT_FILENO); + cli_fd_capture_begin(&err_capture, stderr, STDERR_FILENO); + char *argv[] = {"--yes"}; + int rc = cli_test_cmd_uninstall(1, argv); + char *err_text = cli_fd_capture_end(&err_capture); + char *out_text = cli_fd_capture_end(&out_capture); + cbm_cli_set_activation_ops_for_test(NULL); + cbm_set_auto_answer_for_test(0); + + struct stat state; + bool binary_gone = lstat(bin_target, &state) != 0 && errno == ENOENT; + bool index_gone = lstat(index_path, &state) != 0 && errno == ENOENT; + bool link_untouched = lstat(cursor_config, &state) == 0 && S_ISLNK(state.st_mode); + bool failure_named = err_text && strstr(err_text, cursor_config) != NULL; + + cli_activation_restore_env(old_home, old_cache); + restore_test_env("PATH", old_path); + test_rmdir_r(tmpdir); + free(err_text); + free(out_text); + + ASSERT(rc != 0); + ASSERT_TRUE(binary_gone); + ASSERT_TRUE(index_gone); + ASSERT_TRUE(link_untouched); + ASSERT_TRUE(failure_named); + PASS(); +} +#endif + TEST(cli_activation_guard_is_bypassed_for_dry_run_and_plan) { char tmpdir[256]; snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-daemon-stateless-XXXXXX"); @@ -4627,7 +4762,7 @@ TEST(cli_agent_uninstall_reports_safe_editor_refusal) { #else snprintf(bin_path, sizeof(bin_path), "%s/codebase-memory-mcp", bin_dir); #endif - write_test_file(bin_path, "installed binary must remain live\n"); + write_test_file(bin_path, "installed binary goes even when a config cannot be cleaned\n"); char *saved_home = save_test_env("HOME"); char *saved_path = save_test_env("PATH"); @@ -4651,9 +4786,12 @@ TEST(cli_agent_uninstall_reports_safe_editor_refusal) { restore_test_env("HOME", saved_home); restore_test_env("PATH", saved_path); test_rmdir_r(tmpdir); - if (rc == 0 || !preserved || !binary_preserved || fake.mutation_reserve_count != 1 || - fake.mutation_lease_release_count != 1 || !strstr(fake.diagnostic, "executable was kept")) - FAIL("agent uninstall refusal must fail before removing the live binary"); + /* #1954: a refused config is reported and fails the exit code, but it never + * keeps the executable installed — that left a 300 MB binary and every index + * behind for one unreadable mcp.json. */ + if (rc == 0 || !preserved || binary_preserved || fake.mutation_reserve_count != 1 || + fake.mutation_lease_release_count != 1 || fake.diagnostic[0] != '\0') + FAIL("agent uninstall refusal must fail the exit code without keeping the binary"); PASS(); } @@ -10463,7 +10601,7 @@ TEST(cli_codex_migrates_to_single_hook_representation) { #else snprintf(binary_path, sizeof(binary_path), "%s/codebase-memory-mcp", binary_dir); #endif - write_test_file(binary_path, "installed binary must survive failed cleanup\n"); + write_test_file(binary_path, "installed binary goes even when a config cannot be cleaned\n"); char *saved_home = save_test_env("HOME"); char *saved_path = save_test_env("PATH"); @@ -10515,7 +10653,8 @@ TEST(cli_codex_migrates_to_single_hook_representation) { snprintf(agent_path, sizeof(agent_path), "%s/agents/codebase-memory.toml", codex_dir); struct stat state; hooks = read_test_file_alloc(hooks_path); - bool independent_cleanup = uninstall_rc != 0 && stat(binary_path, &state) == 0 && + /* #1954: the ambiguous hook fails the exit code; the binary still goes. */ + bool independent_cleanup = uninstall_rc != 0 && stat(binary_path, &state) != 0 && stat(skill_path, &state) != 0 && stat(agent_path, &state) != 0 && hooks && !strstr(hooks, "hook-augment"); free(hooks); @@ -14539,6 +14678,9 @@ SUITE(cli) { RUN_TEST(cli_update_agent_configs_finish_before_guard_release); RUN_TEST(cli_uninstall_quiesces_active_cohort_before_removing_binary_and_index); RUN_TEST(cli_uninstall_preserves_binary_and_index_when_cohort_does_not_drain); +#ifndef _WIN32 + RUN_TEST(cli_uninstall_removes_binary_and_index_when_agent_config_cleanup_fails); +#endif RUN_TEST(cli_activation_guard_is_bypassed_for_dry_run_and_plan); #ifdef _WIN32 RUN_TEST(cli_windows_update_hands_off_to_install_script); From 0c4831a3643ba1545d1cf709f056b70739aac04f Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 8 Sep 2026 01:37:44 +0200 Subject: [PATCH 024/130] fix(pipeline): own every staging file through an exclusive sidecar lock A stage was recognisable only by its name: the mkstemp descriptor was closed at once and nothing marked who was writing it. A worker killed mid-run -- the daemon cancels with SIGTERM and SIGKILLs after one second of grace, which a gigabyte backup or delta clone never finishes inside -- left its full-size stage behind for good, and no later run could tell a dead stage from a live one, so none tried (#1839). Ownership is now an exclusive kernel lock on the sidecar ".lock", taken when the stage is minted and dropped by the same helpers that discard the stage or rename it into place. The kernel releases it on any death, so "can I take this lock?" is exactly "is this stage dead?" -- no pid, no mtime, no grace period. The lock lives on a sidecar rather than the stage file itself because on macOS an flock conflicts with SQLite's fcntl byte locks on the same file (verified: flock(LOCK_EX) then fcntl(F_WRLCK) on another descriptor fails with EAGAIN, both directions). The primitive is cbm_lockfile_open: POSIX open(O_CLOEXEC|O_NOFOLLOW) plus flock(LOCK_EX|LOCK_NB); Windows _wsopen with _SH_DENYRW and _O_NOINHERIT. Neither is inherited by spawned children. A backup that fails keeps the outer stage NAME owned: the rebuilt generation is renamed over it by the inner finalize and published from it, so its lock is held to the end. The sweep that uses this ownership follows separately. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- src/foundation/compat_fs.c | 61 +++++++++++- src/foundation/compat_fs.h | 12 +++ src/pipeline/pipeline.c | 156 ++++++++++++++++++++++++++++++- src/pipeline/pipeline_internal.h | 11 +++ tests/test_pipeline.c | 78 +++++++++++++--- 5 files changed, 295 insertions(+), 23 deletions(-) diff --git a/src/foundation/compat_fs.c b/src/foundation/compat_fs.c index bc1b06ac3..b478002eb 100644 --- a/src/foundation/compat_fs.c +++ b/src/foundation/compat_fs.c @@ -23,11 +23,13 @@ #include #include -#include /* _wmkdir */ -#include /* errno for spawn-failure logging */ -#include /* _O_RDONLY */ -#include /* _wunlink, _open_osfhandle, _close */ -#include /* intptr_t */ +#include /* _wmkdir */ +#include /* errno for spawn-failure logging */ +#include /* _O_RDONLY */ +#include /* _wunlink, _open_osfhandle, _close */ +#include /* _SH_DENYRW */ +#include /* _S_IREAD */ +#include /* intptr_t */ #include "foundation/log.h" #include "foundation/win_utf8.h" @@ -581,6 +583,26 @@ int cbm_rmdir(const char *path) { return ret; } +int cbm_lockfile_open(const char *path, bool create) { + wchar_t *wpath = cbm_path_to_wide(path); + if (!wpath) { + errno = EINVAL; + return -1; + } + int flags = _O_RDWR | _O_BINARY | _O_NOINHERIT | (create ? _O_CREAT : 0); + /* _SH_DENYRW: every other open of this file, from any process, fails + * with EACCES until this descriptor closes -- including at death. */ + int fd = _wsopen(wpath, flags, _SH_DENYRW, _S_IREAD | _S_IWRITE); + free(wpath); + return fd; +} + +void cbm_lockfile_close(int fd) { + if (fd >= 0) { + (void)_close(fd); + } +} + /* Build a properly-quoted Windows command line from an argv array. * Returns a heap-allocated wide string, or NULL on allocation failure. * Quoting follows the MSVC CRT convention: arguments containing spaces, @@ -730,6 +752,7 @@ int cbm_exec_no_shell(const char *const *argv) { #include #include #include +#include #include #include #include @@ -929,6 +952,34 @@ int cbm_rmdir(const char *path) { return rmdir(path); } +int cbm_lockfile_open(const char *path, bool create) { + int flags = O_RDWR | O_CLOEXEC | O_NOFOLLOW | (create ? O_CREAT : 0); + int fd; + do { + fd = open(path, flags, S_IRUSR | S_IWUSR); + } while (fd < 0 && errno == EINTR); + if (fd < 0) { + return -1; + } + int rc; + do { + rc = flock(fd, LOCK_EX | LOCK_NB); + } while (rc != 0 && errno == EINTR); + if (rc != 0) { + int saved = errno; + (void)close(fd); + errno = saved; + return -1; + } + return fd; +} + +void cbm_lockfile_close(int fd) { + if (fd >= 0) { + (void)close(fd); + } +} + int cbm_exec_no_shell(const char *const *argv) { if (!argv || !argv[0]) { return CBM_NOT_FOUND; diff --git a/src/foundation/compat_fs.h b/src/foundation/compat_fs.h index 8f0d16388..e7dd259a0 100644 --- a/src/foundation/compat_fs.h +++ b/src/foundation/compat_fs.h @@ -97,6 +97,18 @@ int cbm_canonical_path(const char *path, char *out, size_t out_sz); /* Delete an empty directory. Returns 0 on success. */ int cbm_rmdir(const char *path); +/* Exclusive lock file. Opens `path` and takes an exclusive lock the KERNEL + * releases on any process death -- POSIX open(O_CLOEXEC|O_NOFOLLOW) + + * flock(LOCK_EX|LOCK_NB); Windows _wsopen with _SH_DENYRW (deny every other + * open) and _O_NOINHERIT -- so ownership never outlives its holder and is + * never inherited by a spawned child. `create` false never creates the file. + * Returns the descriptor, or -1 with errno: EWOULDBLOCK/EAGAIN (POSIX) or + * EACCES (Windows sharing violation) when another holder is live, ENOENT + * when absent, otherwise the open error. Never use this on a SQLite file: + * on macOS an flock conflicts with SQLite's own fcntl byte locks. */ +int cbm_lockfile_open(const char *path, bool create); +void cbm_lockfile_close(int fd); + /* Open a file by UTF-8 path. * On Windows, converts to wide-char and calls _wfopen so paths with * non-ASCII characters (accents, CJK, etc.) are handled correctly. diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 247f72378..59ff88837 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -1531,12 +1531,148 @@ static bool promote_mode_to_existing_coverage(cbm_pipeline_t *p) { /* Defined below, next to the other publication helpers. */ static char *create_staging_path(const char *final_path); +/* ── Stage ownership (#1839) ───────────────────────────────────── + * + * A stage used to be recognisable only by its name: the mkstemp descriptor + * was closed at once and nothing marked who was writing it. A worker killed + * mid-run (the daemon cancels with SIGTERM then SIGKILL after one second of + * grace, which a gigabyte backup or clone never finishes inside) left its + * full-size stage behind forever, and no later run could tell a dead stage + * from a live one -- so none tried. + * + * Ownership is now an exclusive kernel lock on the sidecar ".lock", + * held from minting until the stage is discarded or renamed into place. The + * kernel releases it on any death, so "can I take this lock?" is exactly + * "is this stage dead?" -- no pid, no mtime, no grace period. The lock lives + * on a sidecar rather than the stage itself because on macOS an flock on a + * file conflicts with SQLite's fcntl byte locks on that same file. + * + * The stage path is passed around as a plain string through publish and + * finalize, so the descriptor is kept in this per-process registry keyed by + * path, and released by the same helpers that remove the file. */ +typedef struct stage_owner { + char *stage_path; + int lock_fd; + struct stage_owner *next; +} stage_owner_t; + +static stage_owner_t *g_stage_owners = NULL; +static atomic_flag g_stage_owners_spin = ATOMIC_FLAG_INIT; + +static void stage_owners_lock(void) { + while (atomic_flag_test_and_set_explicit(&g_stage_owners_spin, memory_order_acquire)) {} +} + +static void stage_owners_unlock(void) { + atomic_flag_clear_explicit(&g_stage_owners_spin, memory_order_release); +} + +static char *stage_lock_sidecar_path(const char *stage_path) { + static const char suffix[] = ".lock"; + size_t len = strlen(stage_path); + if (len > SIZE_MAX - sizeof(suffix)) { + return NULL; + } + char *sidecar = (char *)malloc(len + sizeof(suffix)); + if (!sidecar) { + return NULL; + } + memcpy(sidecar, stage_path, len); + memcpy(sidecar + len, suffix, sizeof(suffix)); + return sidecar; +} + +int cbm_pipeline_stage_lock_hold(const char *stage_path) { + if (!stage_path) { + return -1; + } + char *sidecar = stage_lock_sidecar_path(stage_path); + if (!sidecar) { + return -1; + } + int fd = cbm_lockfile_open(sidecar, true); + free(sidecar); + return fd; +} + +void cbm_pipeline_stage_lock_drop(const char *stage_path, int lock_fd) { + if (!stage_path || lock_fd < 0) { + return; + } + char *sidecar = stage_lock_sidecar_path(stage_path); + /* Close before unlinking: Windows refuses to delete an open file. The + * sidecar exists unlocked for that instant, but by every drop the stage + * itself is already gone (discarded or renamed), so there is nothing a + * sweeper could take from us. */ + cbm_lockfile_close(lock_fd); + if (sidecar) { + (void)cbm_unlink(sidecar); + free(sidecar); + } +} + +static bool stage_owner_register(const char *stage_path) { + int fd = cbm_pipeline_stage_lock_hold(stage_path); + if (fd < 0) { + char errno_text[16]; + (void)snprintf(errno_text, sizeof(errno_text), "%d", errno); + cbm_log_warn("pipeline.stage", "action", "lock_failed", "errno", errno_text, "path", + stage_path); + return false; + } + stage_owner_t *owner = (stage_owner_t *)malloc(sizeof(*owner)); + char *path_copy = strdup(stage_path); + if (!owner || !path_copy) { + free(owner); + free(path_copy); + cbm_pipeline_stage_lock_drop(stage_path, fd); + return false; + } + owner->stage_path = path_copy; + owner->lock_fd = fd; + stage_owners_lock(); + owner->next = g_stage_owners; + g_stage_owners = owner; + stage_owners_unlock(); + return true; +} + +/* Release ownership of a stage that no longer exists under this name. A path + * this process never registered is a no-op. */ +static void stage_owner_release(const char *stage_path) { + if (!stage_path) { + return; + } + stage_owner_t *found = NULL; + stage_owners_lock(); + for (stage_owner_t **link = &g_stage_owners; *link; link = &(*link)->next) { + if (strcmp((*link)->stage_path, stage_path) == 0) { + found = *link; + *link = found->next; + break; + } + } + stage_owners_unlock(); + if (!found) { + return; + } + cbm_pipeline_stage_lock_drop(found->stage_path, found->lock_fd); + free(found->stage_path); + free(found); +} + +/* Remove a stage's main file and SQLite sidecars, keeping ownership. */ +static void remove_stage_files(const char *stage_path) { + (void)cbm_unlink(stage_path); + (void)cbm_remove_db_sidecars(stage_path); +} + static void discard_generation_stage(const char *stage_path) { if (!stage_path) { return; } - cbm_unlink(stage_path); - cbm_remove_db_sidecars(stage_path); + remove_stage_files(stage_path); + stage_owner_release(stage_path); } typedef struct { @@ -1930,6 +2066,7 @@ int cbm_pipeline_finalize_staged_generation(char *stage_path, const char *final_ discard_generation_stage(stage_path); return CBM_PIPELINE_PERSIST_FAILED; } + stage_owner_release(stage_path); cbm_log_info("finalize.timing", "block", "rename", "elapsed_ms", itoa_buf((int)elapsed_ms(t_fin))); return 0; @@ -2387,8 +2524,8 @@ static void cleanup_staging_db(const char *path) { if (!path) { return; } - (void)cbm_unlink(path); - (void)cbm_remove_db_sidecars(path); + remove_stage_files(path); + stage_owner_release(path); } static bool ensure_db_parent(const char *path) { @@ -2495,6 +2632,11 @@ static char *create_staging_path(const char *final_path) { #else close(fd); #endif + if (!stage_owner_register(path)) { + (void)cbm_unlink(path); + free(path); + return NULL; + } return path; } @@ -2622,7 +2764,10 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { if (!backup_succeeded) { cbm_log_warn("pipeline.stage", "action", "backup_failed_full_rebuild", "path", final_path); - cleanup_staging_db(staging_path); + /* The copy is gone but the NAME stays ours: the rebuilt + * generation is renamed over it by the inner finalize and then + * published from it below, so its lock is held to the end. */ + remove_stage_files(staging_path); } } @@ -2705,6 +2850,7 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { return CBM_PIPELINE_PERSIST_FAILED; } + stage_owner_release(staging_path); rc = export_after_publish(p, final_path); free(staging_path); free(final_path); diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index d749c695a..186b8ee66 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -792,6 +792,17 @@ int cbm_pipeline_publish_staged(char *stage_path, const cbm_pipeline_generation_ * executor; the dump path uses it internally). malloc'd, caller frees. */ char *cbm_pipeline_create_staging_path(const char *final_path); +/* Stage ownership (#1839). Every stage minted by cbm_pipeline_create_staging_path + * is owned through an exclusive kernel lock on the sidecar ".lock" for + * as long as the stage exists; the lock -- and the sidecar -- go away when the + * stage is discarded or renamed into place, and the kernel drops the lock + * when the writer dies. Hold/drop are the same primitive, exposed so a test + * can stand in for a live writer. hold returns a descriptor >= 0, or -1 when + * another holder is live or the sidecar cannot be created. drop releases the + * lock and unlinks the sidecar. */ +int cbm_pipeline_stage_lock_hold(const char *stage_path); +void cbm_pipeline_stage_lock_drop(const char *stage_path, int lock_fd); + /* ── Delta-repair staging primitives (pipeline_delta.c) ────────── * Closure-route-only subsystem: clone the live generation, patch exactly * the repaired node/edge set, publish through the shared finalize leg. */ diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index e724736e2..96857efb8 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -2278,7 +2278,7 @@ TEST(pipeline_complexity_props_independent_of_worker_order) { ASSERT_EQ(sequential_rc, 0); ASSERT_NOT_NULL(sequential_sig); ASSERT_GTE(sequential_funcs, copied); /* at least the one function per fixture file */ - ASSERT_TRUE(cycles_detected); /* the cycles must reach the pass at all */ + ASSERT_TRUE(cycles_detected); /* the cycles must reach the pass at all */ if (mismatch_run >= 0) { printf("\n parallel run %d diverges from sequential: %s\n", mismatch_run, diff); FAIL("complexity props depend on worker id order"); @@ -3496,6 +3496,59 @@ TEST(pipeline_stage_names_never_nest) { PASS(); } +static bool path_exists(const char *path) { + cbm_path_info_t info; + return cbm_path_info_utf8(path, &info) == CBM_PATH_INFO_OK; +} + +/* #1839: a minted stage is OWNED through an exclusive kernel lock on its + * ".lock" sidecar for exactly as long as the stage exists. A second + * holder cannot take it while the writer is live; discarding the stage frees + * the name and removes the sidecar, so nothing stays behind. */ +TEST(pipeline_minted_stage_is_owned_until_released) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_stage_owner_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/generation.db", tmp); + + char *stage = cbm_pipeline_create_staging_path(db_path); + ASSERT_NOT_NULL(stage); + char lock_path[600]; + snprintf(lock_path, sizeof(lock_path), "%s.lock", stage); + bool lock_present_while_live = path_exists(lock_path); + int hold_while_live = cbm_pipeline_stage_lock_hold(stage); + if (hold_while_live >= 0) { + cbm_pipeline_stage_lock_drop(stage, hold_while_live); + } + + cbm_pipeline_discard_stage(stage); + bool stage_gone = !path_exists(stage); + bool lock_gone = !path_exists(lock_path); + + int hold_after_discard = cbm_pipeline_stage_lock_hold(stage); + int second_holder = cbm_pipeline_stage_lock_hold(stage); + if (second_holder >= 0) { + cbm_pipeline_stage_lock_drop(stage, second_holder); + } + cbm_pipeline_stage_lock_drop(stage, hold_after_discard); + int hold_after_drop = cbm_pipeline_stage_lock_hold(stage); + cbm_pipeline_stage_lock_drop(stage, hold_after_drop); + int leftovers = count_generation_stage_artifacts(tmp, "generation.db"); + free(stage); + th_rmtree(tmp); + + ASSERT_TRUE(lock_present_while_live); + ASSERT_EQ(hold_while_live, -1); + ASSERT_TRUE(stage_gone); + ASSERT_TRUE(lock_gone); + ASSERT_TRUE(hold_after_discard >= 0); + ASSERT_EQ(second_holder, -1); + ASSERT_TRUE(hold_after_drop >= 0); + ASSERT_EQ(leftovers, 0); + PASS(); +} + /* Discovery and extraction must describe the same immutable generation. A * source file created after extraction is not present in the original file * list, so merely re-hashing that list cannot detect the race. Publication @@ -7217,19 +7270,18 @@ TEST(pipeline_python_cross_module_call) { * unique_name (candidates==1) is #1572 and is not this claim. */ TEST(pipeline_cross_language_same_name_does_not_share_calls_issue725) { const char *files[] = {"store.py", "app.py", "web/src/pages/Editor.js"}; - const char *contents[] = { - "class Store:\n" - " def commit(self):\n" - " return True\n", + const char *contents[] = {"class Store:\n" + " def commit(self):\n" + " return True\n", - "from store import Store\n" - "\n" - "def save():\n" - " return Store().commit()\n", + "from store import Store\n" + "\n" + "def save():\n" + " return Store().commit()\n", - "export function commit() {\n" - " return 1;\n" - "}\n"}; + "export function commit() {\n" + " return 1;\n" + "}\n"}; if (setup_lang_repo(files, contents, 3) != 0) FAIL("tmpdir"); @@ -13644,7 +13696,6 @@ TEST(pipeline_delta_patch_indexes_docstring_into_fts_body) { PASS(); } - /* End-to-end for #518/#519: source → docstring → properties JSON → nodes_fts * `body` → findable. Each layer has its own test; this one proves they connect. * It is also the guard on the size budget: build_def_props drops an oversized @@ -14201,6 +14252,7 @@ SUITE(pipeline_semantic_manifest_repro) { RUN_TEST(pipeline_global_extension_config_change_forces_full); RUN_TEST(pipeline_publication_never_uses_a_predictable_staging_path); RUN_TEST(pipeline_stage_names_never_nest); + RUN_TEST(pipeline_minted_stage_is_owned_until_released); RUN_TEST(pipeline_source_mutation_before_publication_preserves_previous_generation); RUN_TEST(pipeline_source_addition_before_publication_preserves_previous_generation); RUN_TEST(pipeline_tsconfig_mutation_before_publication_preserves_previous_generation); From b6458f924da7ac1dc1fc2d7969bc7a3f2ca4d248 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 8 Sep 2026 01:50:32 +0200 Subject: [PATCH 025/130] fix(pipeline): sweep the orphaned stages of a dead writer before each run Nothing on the worker-death path ever cleaned a stage up: the supervisor reaps only an exit status, and a cancel is SIGTERM then SIGKILL after one second, which a gigabyte backup or clone never finishes inside. Every daemon-side cancel or OOM kill therefore left a full-size .stage.X (plus -wal/-shm) behind for good, and with auto_index a large project filled the disk with copies of itself (#1839). Each run now sweeps before minting its own stage. Only names of the exact minted shape for THIS database are considered -- ".stage.<6 alphanumerics>" and that stage's -wal/-shm/-journal/.lock sidecars -- never the live database, a quarantined .corrupt, or another project's files. Liveness is the ownership lock and nothing else: a stage whose sidecar lock can be taken has no writer and is removed (logged as pipeline.stage action=orphan_removed bytes=N); one whose lock a live writer holds is kept (orphan_kept reason=live_writer). No pid, no mtime, no grace period, so the sweep is deterministic and never races a writer. A stage minted by an older binary carries no lock and is swept too; that writer's final rename then fails and it discards its stage. The live database is never named on either path. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- src/pipeline/pipeline.c | 185 ++++++++++++++++++++++++++++++++++++++++ tests/test_pipeline.c | 160 ++++++++++++++++++++++++++++++++++ 2 files changed, 345 insertions(+) diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 59ff88837..3d600034e 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -2741,6 +2741,190 @@ static int export_after_publish(cbm_pipeline_t *p, const char *final_path) { return 0; } +/* ── Orphan sweep (#1839) ──────────────────────────────────────── + * + * Nothing on the worker-death path ever cleaned a stage up, so the sweep + * runs at the start of every run, before this run mints its own stage. It + * considers ONLY names of the exact minted shape for THIS database -- + * ".stage.<6 alphanumerics>" plus that stage's -wal/-shm/-journal + * and .lock sidecars -- never the live database, a quarantined .corrupt, or + * another project's files. A stage is removed when its ownership lock can be + * taken (its writer is dead, or the stage predates ownership) and kept when + * a live writer holds the lock. The pre-ownership case is the one honest + * gap: a stage an OLDER binary is still writing against this database has + * no lock and is swept; that writer's final rename then fails and it + * discards. The live database is never named here on either path. */ + +static const char *const cbm_stage_sidecar_tails[] = {"", "-wal", "-shm", "-journal", ".lock"}; + +/* If `name` is ".stage.<6 alphanumerics>", return the + * length of the stage name proper (without the tail); 0 otherwise. */ +static size_t stage_entry_stage_length(const char *name, const char *base, size_t base_len) { + if (strncmp(name, base, base_len) != 0) { + return 0; + } + const char *at = name + base_len; + if (strncmp(at, cbm_stage_marker, sizeof(cbm_stage_marker) - 1) != 0) { + return 0; + } + at += sizeof(cbm_stage_marker) - 1; + for (int i = 0; i < CBM_STAGE_SUFFIX_RANDOM_CHARS; i++) { + /* NUL is not alphanumeric, so a short name fails here too. */ + if (!isalnum((unsigned char)at[i])) { + return 0; + } + } + at += CBM_STAGE_SUFFIX_RANDOM_CHARS; + for (size_t i = 0; i < sizeof(cbm_stage_sidecar_tails) / sizeof(cbm_stage_sidecar_tails[0]); + i++) { + if (strcmp(at, cbm_stage_sidecar_tails[i]) == 0) { + return (size_t)(at - name); + } + } + return 0; +} + +typedef struct { + char **names; + int count; + int cap; +} stage_name_list_t; + +/* Add a stage name once, however many of its files were listed. */ +static void stage_name_list_add(stage_name_list_t *list, const char *name, size_t len) { + for (int i = 0; i < list->count; i++) { + if (strlen(list->names[i]) == len && strncmp(list->names[i], name, len) == 0) { + return; + } + } + if (list->count == list->cap) { + int cap = list->cap ? list->cap * 2 : 8; + char **grown = (char **)realloc(list->names, (size_t)cap * sizeof(*grown)); + if (!grown) { + return; + } + list->names = grown; + list->cap = cap; + } + char *copy = (char *)malloc(len + 1); + if (!copy) { + return; + } + memcpy(copy, name, len); + copy[len] = '\0'; + list->names[list->count++] = copy; +} + +static int64_t stage_bytes_on_disk(const char *stage_path) { + static const char *const files[] = {"", "-wal", "-shm", "-journal"}; + int64_t total = 0; + char side[CBM_SZ_4K]; + for (size_t i = 0; i < sizeof(files) / sizeof(files[0]); i++) { + int n = snprintf(side, sizeof(side), "%s%s", stage_path, files[i]); + if (n <= 0 || (size_t)n >= sizeof(side)) { + continue; + } + cbm_path_info_t info; + if (cbm_path_info_utf8(side, &info) == CBM_PATH_INFO_OK && info.is_regular) { + total += info.size; + } + } + return total; +} + +static void sweep_one_stage(const char *stage_path) { + char *sidecar = stage_lock_sidecar_path(stage_path); + if (!sidecar) { + return; + } + errno = 0; + int lock_fd = cbm_lockfile_open(sidecar, false); + int probe_errno = errno; + free(sidecar); + if (lock_fd < 0 && probe_errno != ENOENT) { + bool live = probe_errno == EAGAIN || probe_errno == EACCES; +#if defined(EWOULDBLOCK) && EWOULDBLOCK != EAGAIN + live = live || probe_errno == EWOULDBLOCK; +#endif + if (live) { + cbm_log_info("pipeline.stage", "action", "orphan_kept", "reason", "live_writer", "path", + stage_path); + return; + } + char errno_text[16]; + (void)snprintf(errno_text, sizeof(errno_text), "%d", probe_errno); + cbm_log_warn("pipeline.stage", "action", "orphan_kept", "reason", "lock_probe_failed", + "errno", errno_text, "path", stage_path); + return; + } + /* No owner: absent sidecar (pre-ownership stage, or dropped) or a lock + * the kernel released with its writer. Ours now, from the lock down. */ + int64_t bytes = stage_bytes_on_disk(stage_path); + remove_stage_files(stage_path); + if (lock_fd >= 0) { + cbm_pipeline_stage_lock_drop(stage_path, lock_fd); + } + char bytes_text[32]; + (void)snprintf(bytes_text, sizeof(bytes_text), "%lld", (long long)bytes); + cbm_log_info("pipeline.stage", "action", "orphan_removed", "bytes", bytes_text, "path", + stage_path); +} + +static void sweep_orphan_stages(const char *final_path) { + /* Directory part INCLUDING its trailing separator, so the stage paths + * are joined exactly as the final path was spelled. */ + size_t prefix_len = 0; + for (const char *c = final_path; *c; c++) { + if (*c == '/' +#ifdef _WIN32 + || *c == '\\' +#endif + ) { + prefix_len = (size_t)(c - final_path) + 1; + } + } + const char *base = final_path + prefix_len; + size_t base_len = strlen(base); + if (base_len == 0) { + return; + } + char *dir_path = prefix_len ? (char *)malloc(prefix_len + 1) : strdup("."); + if (!dir_path) { + return; + } + if (prefix_len) { + memcpy(dir_path, final_path, prefix_len); + dir_path[prefix_len] = '\0'; + } + cbm_dir_t *dir = cbm_opendir(dir_path); + if (!dir) { + free(dir_path); + return; + } + stage_name_list_t list = {0}; + cbm_dirent_t *entry; + while ((entry = cbm_readdir(dir)) != NULL) { + size_t stage_len = stage_entry_stage_length(entry->name, base, base_len); + if (stage_len) { + stage_name_list_add(&list, entry->name, stage_len); + } + } + cbm_closedir(dir); + for (int i = 0; i < list.count; i++) { + size_t name_len = strlen(list.names[i]); + char *stage_path = (char *)malloc(prefix_len + name_len + 1); + if (stage_path) { + memcpy(stage_path, final_path, prefix_len); + memcpy(stage_path + prefix_len, list.names[i], name_len + 1); + sweep_one_stage(stage_path); + free(stage_path); + } + free(list.names[i]); + } + free(list.names); + free(dir_path); +} + int cbm_pipeline_run(cbm_pipeline_t *p) { if (!p) { return CBM_NOT_FOUND; @@ -2752,6 +2936,7 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { } struct stat final_st; bool final_existed = stat(final_path, &final_st) == 0; + sweep_orphan_stages(final_path); char *staging_path = create_staging_path(final_path); if (!staging_path) { free(final_path); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 96857efb8..9ee54b41b 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -3549,6 +3549,164 @@ TEST(pipeline_minted_stage_is_owned_until_released) { PASS(); } +static bool file_has_content(const char *path, const char *expected) { + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + return false; + } + char buf[256] = {0}; + size_t n = fread(buf, 1, sizeof(buf) - 1, f); + (void)fclose(f); + return n == strlen(expected) && memcmp(buf, expected, n) == 0; +} + +/* #1839 / #1864: a stage a dead writer left beside a VALID database is swept + * by the next run and never influences its route. The 0-byte shape is what a + * worker killed before its backup wrote a page leaves behind; a stale -shm + * beside it is swept with it. */ +TEST(pipeline_stale_zero_byte_stage_beside_valid_db_routes_incremental_and_is_swept) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_stale_stage_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + write_temp_file(tmp, "generation.py", "def StableGeneration():\n return 1\n"); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/generation.db", tmp); + + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *baseline = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(baseline); + ASSERT_EQ(cbm_pipeline_run(baseline), 0); + char project[256]; + snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(baseline)); + cbm_pipeline_free(baseline); + + char stale_stage[600]; + char stale_shm[600]; + snprintf(stale_stage, sizeof(stale_stage), "%s.stage.deadbe", db_path); + snprintf(stale_shm, sizeof(stale_shm), "%s.stage.deadbe-shm", db_path); + ASSERT_EQ(th_write_file(stale_stage, ""), 0); + ASSERT_EQ(th_write_file(stale_shm, "stale-shm"), 0); + + /* A body-only change: no added names, so the planner may repair. */ + write_temp_file(tmp, "generation.py", "def StableGeneration():\n return 2\n"); + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *incr = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(incr); + int incr_rc = cbm_pipeline_run(incr); + cbm_incremental_route_t route = cbm_pipeline_incremental_test_last_route(); + cbm_pipeline_free(incr); + + bool stale_stage_gone = !path_exists(stale_stage); + bool stale_shm_gone = !path_exists(stale_shm); + int stage_count = count_generation_stage_artifacts(tmp, "generation.db"); + int stable_count = -1; + int absent_count = -1; + observe_named_generation(db_path, project, "StableGeneration", "NeverDefined", &stable_count, + &absent_count); + cbm_pipeline_incremental_test_reset_faults(); + th_rmtree(tmp); + + ASSERT_EQ(incr_rc, 0); + ASSERT_TRUE(route != CBM_INCREMENTAL_ROUTE_FORCED_FULL); + ASSERT_TRUE(route != CBM_INCREMENTAL_ROUTE_NONE); + ASSERT_TRUE(stale_stage_gone); + ASSERT_TRUE(stale_shm_gone); + ASSERT_EQ(stage_count, 0); + ASSERT_EQ(stable_count, 1); + ASSERT_EQ(absent_count, 0); + PASS(); +} + +/* #1839: the sweep removes exactly the stages nobody owns. A dead writer's + * stage (main, -wal, and the unlocked .lock sidecar its death left) goes; a + * stage whose writer is LIVE -- here the test, holding its lock -- is kept + * byte for byte, and goes only once that lock is dropped. No timing: liveness + * is the kernel lock, nothing else. */ +TEST(pipeline_sweep_removes_dead_writer_stage_keeps_locked_stage) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_stage_sweep_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + write_temp_file(tmp, "generation.py", "def StableGeneration():\n return 1\n"); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/generation.db", tmp); + + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *baseline = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(baseline); + ASSERT_EQ(cbm_pipeline_run(baseline), 0); + char project[256]; + snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(baseline)); + cbm_pipeline_free(baseline); + + char dead_stage[600]; + char dead_wal[600]; + char dead_lock[600]; + char live_stage[600]; + char live_lock[600]; + snprintf(dead_stage, sizeof(dead_stage), "%s.stage.aaaaaa", db_path); + snprintf(dead_wal, sizeof(dead_wal), "%s.stage.aaaaaa-wal", db_path); + snprintf(dead_lock, sizeof(dead_lock), "%s.stage.aaaaaa.lock", db_path); + snprintf(live_stage, sizeof(live_stage), "%s.stage.bbbbbb", db_path); + snprintf(live_lock, sizeof(live_lock), "%s.stage.bbbbbb.lock", db_path); + static const char live_bytes[] = "live-stage-bytes"; + ASSERT_EQ(th_write_file(dead_stage, "dead-stage-bytes"), 0); + ASSERT_EQ(th_write_file(dead_wal, "dead-wal"), 0); + ASSERT_EQ(th_write_file(dead_lock, ""), 0); + ASSERT_EQ(th_write_file(live_stage, live_bytes), 0); + int live_fd = cbm_pipeline_stage_lock_hold(live_stage); + ASSERT_TRUE(live_fd >= 0); + + write_temp_file(tmp, "generation.py", "def StableGeneration():\n return 2\n"); + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *first = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(first); + int first_rc = cbm_pipeline_run(first); + cbm_pipeline_free(first); + + bool dead_stage_gone = !path_exists(dead_stage); + bool dead_wal_gone = !path_exists(dead_wal); + bool dead_lock_gone = !path_exists(dead_lock); + bool live_kept = file_has_content(live_stage, live_bytes); + bool live_lock_kept = path_exists(live_lock); + int stable_after_first = -1; + int absent_after_first = -1; + observe_named_generation(db_path, project, "StableGeneration", "NeverDefined", + &stable_after_first, &absent_after_first); + + cbm_pipeline_stage_lock_drop(live_stage, live_fd); + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *second = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(second); + int second_rc = cbm_pipeline_run(second); + cbm_pipeline_free(second); + + bool live_gone = !path_exists(live_stage); + bool live_lock_gone = !path_exists(live_lock); + int stage_count = count_generation_stage_artifacts(tmp, "generation.db"); + int stable_after_second = -1; + int absent_after_second = -1; + observe_named_generation(db_path, project, "StableGeneration", "NeverDefined", + &stable_after_second, &absent_after_second); + cbm_pipeline_incremental_test_reset_faults(); + th_rmtree(tmp); + + ASSERT_EQ(first_rc, 0); + ASSERT_TRUE(dead_stage_gone); + ASSERT_TRUE(dead_wal_gone); + ASSERT_TRUE(dead_lock_gone); + ASSERT_TRUE(live_kept); + ASSERT_TRUE(live_lock_kept); + ASSERT_EQ(stable_after_first, 1); + ASSERT_EQ(absent_after_first, 0); + ASSERT_EQ(second_rc, 0); + ASSERT_TRUE(live_gone); + ASSERT_TRUE(live_lock_gone); + ASSERT_EQ(stage_count, 0); + ASSERT_EQ(stable_after_second, 1); + ASSERT_EQ(absent_after_second, 0); + PASS(); +} + /* Discovery and extraction must describe the same immutable generation. A * source file created after extraction is not present in the original file * list, so merely re-hashing that list cannot detect the race. Publication @@ -14253,6 +14411,8 @@ SUITE(pipeline_semantic_manifest_repro) { RUN_TEST(pipeline_publication_never_uses_a_predictable_staging_path); RUN_TEST(pipeline_stage_names_never_nest); RUN_TEST(pipeline_minted_stage_is_owned_until_released); + RUN_TEST(pipeline_stale_zero_byte_stage_beside_valid_db_routes_incremental_and_is_swept); + RUN_TEST(pipeline_sweep_removes_dead_writer_stage_keeps_locked_stage); RUN_TEST(pipeline_source_mutation_before_publication_preserves_previous_generation); RUN_TEST(pipeline_source_addition_before_publication_preserves_previous_generation); RUN_TEST(pipeline_tsconfig_mutation_before_publication_preserves_previous_generation); From 00060cec854dbd38d7864aeb95a2d8ba1baa8ab2 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 8 Sep 2026 02:01:33 +0200 Subject: [PATCH 026/130] fix(daemon): fail the client fast when the daemon cannot start A daemon that died at listener publication (a full /tmp) took the lifetime reservation, failed, and released it; the waiting client saw the generation vanish, respawned it, and repeated that for its whole 30 s deadline before reporting "CBM daemon is active or starting but could not accept this client" -- the opposite of what happened, with no cause and no path. MCP hosts then dropped the server silently (#1828). A detached daemon has no stderr and no channel back to its launcher, so the host now appends one start-failure record per failed start to /logs/cbm-daemon.start-failures.log -- next to cbm-daemon.log, through the same owner-only private-log open. Deliberately NOT the runtime directory: that is exactly what a full /tmp cannot write, and a new artifact there would entangle the stale-generation cleanup that #1894 just hardened. The record carries the listener detail from the previous commit (component, stage, errno, path) plus the runtime directory and a wall-clock stamp, and is written while the lifetime reservation is still held, so a client that watches the generation vanish already finds the cause. The client bootstrap loop reads that record (every 50 ms after its own spawn, and unconditionally before any relaunch), accepts only records for its endpoint not older than its spawn instant (2 s skew), and stops at once: error: CBM daemon failed to start: pending_publication failed with ENOSPC (No space left on device) at /tmp/cbm-daemon-1000/cbm-.sock.pending.tmp; see ~/.cache/codebase-memory-mcp/logs/cbm-daemon.log The read is an optional bootstrap op (unit contracts with fake ops are untouched); the production op resolves the cache log directory. Every other wait state -- a live or stopping generation, a mute holder, a concurrent starter -- keeps its existing semantics and diagnostics. Tests (daemon_bootstrap): - daemon_bootstrap_fails_fast_when_daemon_dies_at_publication: the production bootstrap (real cohort, probe, startup lock, handoff) with a seam-only spawn override that forks a REAL daemon host whose record writes fail with ENOSPC. RED before: 34.7 s and "active or starting"; GREEN after: FAILED in well under 5 s naming ENOSPC and the temp path, one child spawned, daemon log carries errno=ENOSPC path=... . - daemon_bootstrap_start_failure_record_round_trip: record/read/format contract, a stale record and another endpoint's record are ignored. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- src/daemon/bootstrap.c | 340 +++++++++++++++++++++++++++++++++- src/daemon/bootstrap.h | 58 ++++++ src/daemon/host.c | 21 ++- tests/test_daemon_bootstrap.c | 257 +++++++++++++++++++++++++ 4 files changed, 673 insertions(+), 3 deletions(-) diff --git a/src/daemon/bootstrap.c b/src/daemon/bootstrap.c index 50f189036..0e4cbed61 100644 --- a/src/daemon/bootstrap.c +++ b/src/daemon/bootstrap.c @@ -6,13 +6,17 @@ #include "daemon/ipc.h" #include "daemon/service.h" #include "foundation/compat.h" +#include "foundation/compat_fs.h" +#include "foundation/log.h" #include "foundation/platform.h" +#include #include #include #include #include #include +#include #ifdef _WIN32 #ifndef WIN32_LEAN_AND_MEAN @@ -38,6 +42,14 @@ extern char **environ; enum { BOOTSTRAP_RETRY_NS = 1000000, + /* #1828: how often a waiting client re-reads the start-failure log + * (the wait loop itself ticks every millisecond). */ + BOOTSTRAP_START_FAILURE_CHECK_MS = 50, + /* A record this many seconds OLDER than the client's own spawn instant is + * still accepted: a sibling client's attempt that just failed is the same + * evidence, and second-granularity clocks need the slack anyway. */ + BOOTSTRAP_START_FAILURE_SKEW_S = 2, + BOOTSTRAP_START_FAILURE_LOG_CAP = 65536, BOOTSTRAP_COORDINATION_CLEANUP_MS = 500, BOOTSTRAP_PATH_CAP = 4096, }; @@ -292,6 +304,270 @@ static _Noreturn void bootstrap_cleanup_fail_stop(const char *component) { #endif } +static const char BOOTSTRAP_START_FAILURE_LOG_NAME[] = "cbm-daemon.start-failures.log"; + +bool cbm_daemon_bootstrap_log_directory(char *out, size_t capacity) { + if (!out || capacity == 0) { + return false; + } + out[0] = '\0'; + const char *cache = cbm_resolve_cache_dir(); + if (!cache || !cache[0]) { + return false; + } + int written = snprintf(out, capacity, "%s/logs", cache); + if (written <= 0 || (size_t)written >= capacity) { + out[0] = '\0'; + return false; + } + return true; +} + +/* One record field: the line format is tab-separated, so a tab or newline in + * a value (a path, in theory) becomes '?' rather than a parser ambiguity. */ +static void bootstrap_record_field(char *out, size_t capacity, const char *value) { + size_t used = 0; + for (const char *cursor = value ? value : ""; *cursor && used + 1 < capacity; cursor++) { + unsigned char ch = (unsigned char)*cursor; + out[used++] = ch == '\t' || ch == '\n' || ch == '\r' ? '?' : (char)ch; + } + out[used] = '\0'; +} + +static uint64_t bootstrap_current_pid(void) { +#ifdef _WIN32 + return (uint64_t)GetCurrentProcessId(); +#else + return (uint64_t)getpid(); +#endif +} + +bool cbm_daemon_bootstrap_start_failure_record(const char *log_directory, + const cbm_daemon_ipc_endpoint_t *endpoint, + const char *component) { + if (!log_directory || !log_directory[0] || !endpoint || !component || !component[0]) { + return false; + } + const char *runtime_dir = cbm_daemon_ipc_endpoint_runtime_dir(endpoint); + if (!runtime_dir || !runtime_dir[0]) { + return false; + } + cbm_daemon_ipc_listen_failure_t detail; + if (!cbm_daemon_ipc_listen_failure_detail(&detail)) { + memset(&detail, 0, sizeof(detail)); + } + char safe_runtime_dir[BOOTSTRAP_PATH_CAP]; + char safe_component[CBM_DAEMON_BOOTSTRAP_COMPONENT_CAP]; + char safe_stage[CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP]; + char safe_path[CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP]; + bootstrap_record_field(safe_runtime_dir, sizeof(safe_runtime_dir), runtime_dir); + bootstrap_record_field(safe_component, sizeof(safe_component), component); + bootstrap_record_field(safe_stage, sizeof(safe_stage), detail.stage); + bootstrap_record_field(safe_path, sizeof(safe_path), detail.path); + FILE *stream = cbm_daemon_ipc_private_log_open(log_directory, BOOTSTRAP_START_FAILURE_LOG_NAME, + BOOTSTRAP_START_FAILURE_LOG_CAP); + if (!stream) { + return false; + } + int written = + fprintf(stream, "v1\t%llu\t%llu\t%s\t%s\t%s\t%d\t%s\n", (unsigned long long)time(NULL), + (unsigned long long)bootstrap_current_pid(), safe_runtime_dir, safe_component, + safe_stage, detail.errno_value, safe_path); + bool ok = written > 0 && fflush(stream) == 0; + if (fclose(stream) != 0) { + ok = false; + } + return ok; +} + +/* Parse one "v1\tts\tpid\truntime_dir\tcomponent\tstage\terrno\tpath" line + * (line is NUL-terminated, newline already stripped, and is modified). */ +static bool bootstrap_start_failure_parse(char *line, char **runtime_dir_out, + cbm_daemon_bootstrap_start_failure_t *out) { + char *fields[8]; + size_t count = 0; + char *cursor = line; + while (count < 8) { + fields[count++] = cursor; + char *tab = strchr(cursor, '\t'); + if (!tab) { + break; + } + *tab = '\0'; + cursor = tab + 1; + } + if (count != 8 || strcmp(fields[0], "v1") != 0) { + return false; + } + memset(out, 0, sizeof(*out)); + out->recorded_at_s = strtoull(fields[1], NULL, 10); + out->pid = strtoull(fields[2], NULL, 10); + *runtime_dir_out = fields[3]; + (void)snprintf(out->component, sizeof(out->component), "%s", fields[4]); + (void)snprintf(out->stage, sizeof(out->stage), "%s", fields[5]); + out->errno_value = (int)strtol(fields[6], NULL, 10); + (void)snprintf(out->path, sizeof(out->path), "%s", fields[7]); + return true; +} + +static FILE *bootstrap_start_failure_open(const char *log_directory, int *status_out) { + char path[BOOTSTRAP_PATH_CAP]; + int written = + snprintf(path, sizeof(path), "%s/%s", log_directory, BOOTSTRAP_START_FAILURE_LOG_NAME); + if (written <= 0 || written >= (int)sizeof(path)) { + *status_out = -1; + return NULL; + } +#ifndef _WIN32 + struct stat status; + if (lstat(path, &status) != 0) { + *status_out = errno == ENOENT ? 0 : -1; + return NULL; + } + if (!S_ISREG(status.st_mode) || status.st_uid != geteuid() || status.st_nlink != 1) { + *status_out = -1; + return NULL; + } +#endif + /* The directory is the trust boundary (owner-only, no symlinked + * ancestry); the same validation guards cbm-daemon.log itself. */ + if (!cbm_daemon_ipc_private_directory_secure(log_directory)) { + *status_out = -1; + return NULL; + } + FILE *file = cbm_fopen(path, "rb"); + if (!file) { + *status_out = errno == ENOENT ? 0 : -1; + return NULL; + } + *status_out = 1; + return file; +} + +int cbm_daemon_bootstrap_start_failure_read(const char *log_directory, + const cbm_daemon_ipc_endpoint_t *endpoint, + uint64_t not_before_s, + cbm_daemon_bootstrap_start_failure_t *out_failure) { + if (!out_failure) { + return -1; + } + memset(out_failure, 0, sizeof(*out_failure)); + if (!log_directory || !log_directory[0] || !endpoint) { + return -1; + } + const char *runtime_dir = cbm_daemon_ipc_endpoint_runtime_dir(endpoint); + if (!runtime_dir || !runtime_dir[0]) { + return -1; + } + int status = 0; + FILE *file = bootstrap_start_failure_open(log_directory, &status); + if (!file) { + return status; + } + char *buffer = malloc(BOOTSTRAP_START_FAILURE_LOG_CAP + 1U); + if (!buffer) { + (void)fclose(file); + return -1; + } + size_t used = fread(buffer, 1, BOOTSTRAP_START_FAILURE_LOG_CAP, file); + bool read_ok = !ferror(file); + (void)fclose(file); + buffer[used] = '\0'; + int found = 0; + if (read_ok) { + char *line = buffer; + while (line && *line) { + char *end = strchr(line, '\n'); + if (!end) { + break; /* a partial trailing line is a record still being written */ + } + *end = '\0'; + char *record_runtime_dir = NULL; + cbm_daemon_bootstrap_start_failure_t candidate; + if (bootstrap_start_failure_parse(line, &record_runtime_dir, &candidate) && + strcmp(record_runtime_dir, runtime_dir) == 0 && + candidate.recorded_at_s >= not_before_s) { + *out_failure = candidate; /* later lines are newer */ + found = 1; + } + line = end + 1; + } + } else { + found = -1; + } + free(buffer); + return found; +} + +void cbm_daemon_bootstrap_start_failure_format(const cbm_daemon_bootstrap_start_failure_t *failure, + const char *log_directory, char *out, + size_t capacity) { + if (!out || capacity == 0) { + return; + } + if (!failure) { + out[0] = '\0'; + return; + } + const char *what = failure->stage[0] ? failure->stage + : failure->component[0] ? failure->component + : "startup"; + char where[BOOTSTRAP_PATH_CAP]; + if (log_directory && log_directory[0]) { + (void)snprintf(where, sizeof(where), "see %s/cbm-daemon.log", log_directory); + } else { + (void)snprintf(where, sizeof(where), "see the daemon log"); + } + if (failure->errno_value != 0 && failure->path[0]) { + (void)snprintf(out, capacity, + "CBM daemon failed to start: %s failed with %s (%s) at %s; %s", what, + cbm_errno_name(failure->errno_value), strerror(failure->errno_value), + failure->path, where); + } else if (failure->errno_value != 0) { + (void)snprintf(out, capacity, "CBM daemon failed to start: %s failed with %s (%s); %s", + what, cbm_errno_name(failure->errno_value), strerror(failure->errno_value), + where); + } else if (failure->path[0]) { + (void)snprintf(out, capacity, "CBM daemon failed to start: %s failed at %s; %s", what, + failure->path, where); + } else { + (void)snprintf(out, capacity, "CBM daemon failed to start: %s failed; %s", what, where); + } +} + +/* After this attempt spawned a daemon: did that daemon (or a sibling attempt + * moments earlier) record a start failure? A found record ends the wait with + * its cause -- respawning a deterministically failing daemon until the + * deadline only produced "active or starting" (#1828). */ +static bool bootstrap_start_failure_detected(const cbm_daemon_bootstrap_config_t *config, + const cbm_daemon_bootstrap_ops_t *ops, + uint64_t spawn_wall_s, bool force, + uint64_t *next_check_ms, + cbm_daemon_bootstrap_result_t *result_out) { + if (spawn_wall_s == 0 || !ops->start_failure_probe) { + return false; + } + uint64_t now_ms = cbm_now_ms(); + if (!force && now_ms < *next_check_ms) { + return false; + } + *next_check_ms = now_ms + BOOTSTRAP_START_FAILURE_CHECK_MS; + uint64_t not_before_s = spawn_wall_s > BOOTSTRAP_START_FAILURE_SKEW_S + ? spawn_wall_s - BOOTSTRAP_START_FAILURE_SKEW_S + : 0; + cbm_daemon_bootstrap_start_failure_t failure; + if (ops->start_failure_probe(ops->context, config->endpoint, not_before_s, &failure) != 1) { + return false; + } + char logs[BOOTSTRAP_PATH_CAP]; + if (!cbm_daemon_bootstrap_log_directory(logs, sizeof(logs))) { + logs[0] = '\0'; + } + cbm_daemon_bootstrap_start_failure_format(&failure, logs, result_out->message, + sizeof(result_out->message)); + return true; +} + static void bootstrap_pause(uint64_t deadline) { uint64_t now = cbm_now_ms(); if (now >= deadline) { @@ -458,7 +734,15 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops( bool lock_acquired = false; bool generation_observed = probe == CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED || probe == CBM_DAEMON_BOOTSTRAP_PROBE_TERMINAL; + uint64_t spawn_wall_s = 0; + uint64_t next_failure_check_ms = 0; + bool start_failed = false; while (cbm_now_ms() < deadline) { + if (bootstrap_start_failure_detected(config, ops, spawn_wall_s, false, + &next_failure_check_ms, result_out)) { + start_failed = true; + break; + } if (!bootstrap_probe_is_waitable(probe)) { break; } @@ -508,11 +792,20 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops( break; } + /* Never relaunch a daemon whose predecessor already recorded why it + * could not start; the record is written before that daemon lets go + * of its lifetime reservation, so this unconditional read sees it. */ + if (bootstrap_start_failure_detected(config, ops, spawn_wall_s, true, + &next_failure_check_ms, result_out)) { + start_failed = true; + break; + } cbm_daemon_bootstrap_launch_spec_t spec; bool spec_ready = config->spawn_permanent ? cbm_daemon_bootstrap_launch_spec_init_permanent(config->executable_path, &spec) : cbm_daemon_bootstrap_launch_spec_init(config->executable_path, &spec); + spawn_wall_s = (uint64_t)time(NULL); if (!spec_ready || !ops->startup_lock_prepare_handoff(ops->context, startup_lock) || !ops->spawn_daemon(ops->context, &spec)) { probe = CBM_DAEMON_BOOTSTRAP_PROBE_ERROR; @@ -537,7 +830,15 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops( if (!bootstrap_probe_is_waitable(probe)) { break; } + if (bootstrap_start_failure_detected(config, ops, spawn_wall_s, false, + &next_failure_check_ms, result_out)) { + start_failed = true; + break; + } } while (cbm_now_ms() < deadline); + if (start_failed) { + break; + } if (!lock_acquired) { continue; } @@ -555,7 +856,9 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops( } result_out->status = CBM_DAEMON_BOOTSTRAP_FAILED; - if (muted_holder_pid != 0) { + if (start_failed) { + /* result_out->message already names the recorded cause. */ + } else if (muted_holder_pid != 0) { /* The one diagnostic the 2026-08-29 zombie recovery had to assemble by * hand from process, pipe, and log correlation: name the pid that * holds the endpoint without answering, and say what to do with it. */ @@ -773,6 +1076,17 @@ static bool bootstrap_production_unlock(void *context, cbm_daemon_bootstrap_lock return released; } +static int bootstrap_production_start_failure_probe( + void *context, const cbm_daemon_ipc_endpoint_t *endpoint, uint64_t not_before_s, + cbm_daemon_bootstrap_start_failure_t *out_failure) { + (void)context; + char logs[BOOTSTRAP_PATH_CAP]; + if (!cbm_daemon_bootstrap_log_directory(logs, sizeof(logs))) { + return -1; + } + return cbm_daemon_bootstrap_start_failure_read(logs, endpoint, not_before_s, out_failure); +} + static bool bootstrap_production_handoff(void *context, cbm_daemon_bootstrap_lock_t lock) { (void)context; return cbm_daemon_ipc_startup_lock_prepare_handoff((cbm_daemon_ipc_startup_lock_t *)lock); @@ -1041,10 +1355,26 @@ static void bootstrap_production_diagnostic(void *context, const char *message) (void)fflush(stderr); } +#ifdef CBM_ENABLE_TEST_SEAMS +static cbm_daemon_bootstrap_spawn_fn g_bootstrap_spawn_override_for_test; +static void *g_bootstrap_spawn_override_context_for_test; + +void cbm_daemon_bootstrap_spawn_override_set_for_test(cbm_daemon_bootstrap_spawn_fn spawn, + void *context) { + g_bootstrap_spawn_override_context_for_test = context; + g_bootstrap_spawn_override_for_test = spawn; +} + +static bool bootstrap_seam_spawn(void *context, const cbm_daemon_bootstrap_launch_spec_t *spec) { + (void)context; + return g_bootstrap_spawn_override_for_test(g_bootstrap_spawn_override_context_for_test, spec); +} +#endif + cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute( const cbm_daemon_bootstrap_config_t *config, cbm_daemon_bootstrap_result_t *result_out) { bootstrap_production_context_t context = {0}; - const cbm_daemon_bootstrap_ops_t ops = { + cbm_daemon_bootstrap_ops_t ops = { .context = &context, .cohort_acquire = bootstrap_production_cohort_acquire, .cohort_release = bootstrap_production_cohort_release, @@ -1054,6 +1384,12 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute( .startup_lock_release = bootstrap_production_unlock, .spawn_daemon = bootstrap_production_spawn, .visible_diagnostic = bootstrap_production_diagnostic, + .start_failure_probe = bootstrap_production_start_failure_probe, }; +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_bootstrap_spawn_override_for_test) { + ops.spawn_daemon = bootstrap_seam_spawn; + } +#endif return cbm_daemon_bootstrap_execute_with_ops(config, &ops, result_out); } diff --git a/src/daemon/bootstrap.h b/src/daemon/bootstrap.h index acb6c657b..8b1e797dd 100644 --- a/src/daemon/bootstrap.h +++ b/src/daemon/bootstrap.h @@ -119,6 +119,47 @@ cbm_daemon_bootstrap_probe_status_t cbm_daemon_bootstrap_classify_failed_connect typedef void *cbm_daemon_bootstrap_lock_t; typedef void *cbm_daemon_bootstrap_cohort_t; +/* Durable daemon start-failure record (#1828). A detached daemon has no + * stderr and no channel back to the client that launched it; when its + * listener publication fails (a full /tmp, for instance) the client used to + * respawn it for the whole startup deadline and then report "active or + * starting". The host now appends one record per failed start to + * /logs/cbm-daemon.start-failures.log -- the same owner-only log + * directory as cbm-daemon.log, deliberately NOT the runtime directory: that + * one is exactly what a full /tmp cannot write, and a new artifact there + * would entangle stale-generation cleanup. A waiting client reads the newest + * record for its endpoint that is not older than its own spawn and fails + * immediately with the cause. Diagnostic only: policy never reads it. */ +enum { CBM_DAEMON_BOOTSTRAP_COMPONENT_CAP = 32 }; +typedef struct { + uint64_t recorded_at_s; /* wall clock, seconds */ + uint64_t pid; + char component[CBM_DAEMON_BOOTSTRAP_COMPONENT_CAP]; /* host component that refused */ + char stage[CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP]; /* IPC stage, "" when none */ + int errno_value; /* 0 when unknown */ + char path[CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP]; /* "" when none */ +} cbm_daemon_bootstrap_start_failure_t; + +/* /logs, the owner-only directory that holds every daemon log. */ +bool cbm_daemon_bootstrap_log_directory(char *out, size_t capacity); +/* Append one record for this process's most recent listener failure detail + * (cbm_daemon_ipc_listen_failure_detail) under `component`. */ +bool cbm_daemon_bootstrap_start_failure_record(const char *log_directory, + const cbm_daemon_ipc_endpoint_t *endpoint, + const char *component); +/* Newest record for `endpoint` recorded at or after not_before_s. Returns 1 + * and fills out_failure, 0 when none matches, -1 when the log directory or + * file cannot be validated. */ +int cbm_daemon_bootstrap_start_failure_read(const char *log_directory, + const cbm_daemon_ipc_endpoint_t *endpoint, + uint64_t not_before_s, + cbm_daemon_bootstrap_start_failure_t *out_failure); +/* The client-facing sentence: "CBM daemon failed to start: failed + * with ENOSPC (No space left on device) at ; see ". */ +void cbm_daemon_bootstrap_start_failure_format(const cbm_daemon_bootstrap_start_failure_t *failure, + const char *log_directory, char *out, + size_t capacity); + /* Injectable OS/runtime boundary used by the deterministic unit contract. * Production callers use cbm_daemon_bootstrap_execute(), whose built-in * operations delegate to daemon IPC/runtime and write visible diagnostics to @@ -149,6 +190,12 @@ typedef struct { bool (*startup_lock_release)(void *context, cbm_daemon_bootstrap_lock_t *lock_io); bool (*spawn_daemon)(void *context, const cbm_daemon_bootstrap_launch_spec_t *spec); void (*visible_diagnostic)(void *context, const char *message); + /* Optional. After this attempt spawned a daemon, report a start-failure + * record for the endpoint not older than not_before_s (1 found, 0 none, + * -1 unreadable). A found record ends the wait immediately (#1828). */ + int (*start_failure_probe)(void *context, const cbm_daemon_ipc_endpoint_t *endpoint, + uint64_t not_before_s, + cbm_daemon_bootstrap_start_failure_t *out_failure); } cbm_daemon_bootstrap_ops_t; cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute( @@ -156,6 +203,17 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute( /* Test seam for the same state machine. All callbacks are synchronous and * borrowed for the duration of the call. */ +#ifdef CBM_ENABLE_TEST_SEAMS +/* Replace ONLY the production spawn while every other production operation + * (cohort, probe, startup lock, handoff) stays real. A test forks a real + * daemon host in place of exec'ing the product binary, so the complete + * client/daemon rendezvous runs in one deterministic process tree (#1828). */ +typedef bool (*cbm_daemon_bootstrap_spawn_fn)(void *context, + const cbm_daemon_bootstrap_launch_spec_t *spec); +void cbm_daemon_bootstrap_spawn_override_set_for_test(cbm_daemon_bootstrap_spawn_fn spawn, + void *context); +#endif + cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops( const cbm_daemon_bootstrap_config_t *config, const cbm_daemon_bootstrap_ops_t *ops, cbm_daemon_bootstrap_result_t *result_out); diff --git a/src/daemon/host.c b/src/daemon/host.c index f475658b8..dfab03e13 100644 --- a/src/daemon/host.c +++ b/src/daemon/host.c @@ -5,6 +5,7 @@ #include "daemon/host_internal.h" #include "daemon/application.h" +#include "daemon/bootstrap.h" #include "daemon/runtime.h" #include "daemon/project_lock.h" #include "daemon/version_cohort.h" @@ -146,6 +147,18 @@ static bool host_log_open(char conflict_log_out[HOST_PATH_CAP]) { return true; } +/* #1828: a detached daemon has no stderr; its start failure must reach the + * client that is waiting for it, or that client burns its whole deadline and + * reports the opposite of the truth. */ +static void host_start_failure_record(const cbm_daemon_ipc_endpoint_t *endpoint, + const char *component) { + char logs[HOST_PATH_CAP]; + if (!cbm_daemon_bootstrap_log_directory(logs, sizeof(logs)) || + !cbm_daemon_bootstrap_start_failure_record(logs, endpoint, component)) { + cbm_log_error("daemon.start_failure_record_failed", "component", component); + } +} + static void host_log_close(void) { cbm_log_set_sink(NULL); if (g_host_log_mutex_initialized) { @@ -1051,10 +1064,16 @@ int cbm_daemon_host_run(const cbm_daemon_host_config_t *config) { }; cbm_daemon_runtime_service_t *service = cbm_daemon_runtime_service_start_reserved(&runtime_config, &lifetime_reservation); + if (!service) { + cbm_log_error("daemon.start_failed", "component", "runtime"); + /* Recorded while the lifetime reservation is still held: a client + * that watches this generation vanish finds the cause already there + * and never launches a doomed replacement. */ + host_start_failure_record(config->endpoint, "runtime"); + } cbm_daemon_ipc_lifetime_reservation_release(lifetime_reservation); lifetime_reservation = NULL; if (!service) { - cbm_log_error("daemon.start_failed", "component", "runtime"); host_state_free(&host); host_log_close(); host_participant_guard_close(&participant_guard); diff --git a/tests/test_daemon_bootstrap.c b/tests/test_daemon_bootstrap.c index a9c8a368e..20213ef5b 100644 --- a/tests/test_daemon_bootstrap.c +++ b/tests/test_daemon_bootstrap.c @@ -1,18 +1,30 @@ /* RED contract for early process-role classification. */ #include "test_framework.h" +#include "test_helpers.h" #include "daemon/bootstrap.h" +#include "daemon/host.h" #include "daemon/ipc.h" +#include "daemon/ipc_internal.h" +#include "daemon/runtime.h" #include "daemon/service.h" #include "foundation/compat.h" #include "foundation/compat_fs.h" #include "foundation/compat_thread.h" #include "foundation/platform.h" +#include #include #include #include +#include #include +#include +#ifndef _WIN32 +#include +#include +#include +#endif enum { BOOTSTRAP_TEST_PATH_CAP = 1024, @@ -874,6 +886,247 @@ TEST(daemon_bootstrap_darwin_launch_failure_is_synchronous) { } #endif +#ifndef _WIN32 +enum { BOOTSTRAP_ENOSPC_MAX_CHILDREN = 16, BOOTSTRAP_ENOSPC_LOG_CAP = 65536 }; + +typedef struct { + char parent[BOOTSTRAP_TEST_PATH_CAP]; + cbm_daemon_build_identity_t identity; + pid_t children[BOOTSTRAP_ENOSPC_MAX_CHILDREN]; + size_t child_count; + size_t spawn_calls; +} bootstrap_enospc_host_t; + +/* The production spawn exec's the product binary; this one forks a REAL daemon + * host (cbm_daemon_host_run, the same entry `--cbm-daemon-internal` reaches) + * whose record publication fails with ENOSPC through the inherited seam. */ +static bool bootstrap_enospc_host_spawn(void *opaque, + const cbm_daemon_bootstrap_launch_spec_t *spec) { + bootstrap_enospc_host_t *state = opaque; + if (!spec || !spec->detached) { + return false; + } + state->spawn_calls++; + if (state->child_count >= BOOTSTRAP_ENOSPC_MAX_CHILDREN) { + /* RED-run guard only: a client that keeps respawning a doomed daemon + * for its whole deadline must not fork without bound. */ + return true; + } + pid_t child = fork(); + if (child < 0) { + return false; + } + if (child == 0) { + cbm_daemon_ipc_endpoint_t *endpoint = cbm_daemon_bootstrap_endpoint_new(state->parent); + atomic_int stop_requested = ATOMIC_VAR_INIT(0); + cbm_daemon_host_config_t config = { + .endpoint = endpoint, + .identity = state->identity, + .executable_path = "/enospc-host-test", + .stop_requested = &stop_requested, + }; + int run_result = endpoint ? cbm_daemon_host_run(&config) : 0; + _exit(run_result == -1 ? 0 : 50); + } + state->children[state->child_count++] = child; + return true; +} + +static void bootstrap_enospc_reap(bootstrap_enospc_host_t *state, int *nonzero_exits) { + *nonzero_exits = 0; + for (size_t i = 0; i < state->child_count; i++) { + int status = 0; + pid_t waited; + do { + waited = waitpid(state->children[i], &status, WNOHANG); + } while (waited < 0 && errno == EINTR); + if (waited == 0) { + (void)kill(state->children[i], SIGKILL); + do { + waited = waitpid(state->children[i], &status, 0); + } while (waited < 0 && errno == EINTR); + } + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + (*nonzero_exits)++; + } + } +} + +static bool bootstrap_read_file(const char *path, char *out, size_t capacity) { + FILE *file = cbm_fopen(path, "rb"); + if (!file) { + return false; + } + size_t used = fread(out, 1, capacity - 1, file); + out[used] = '\0'; + (void)fclose(file); + return true; +} + +/* The record contract behind the fail-fast: a real listener failure in this + * process is recorded and read back with stage, errno, and path; a record + * older than the reader's spawn or for another endpoint is not evidence. */ +TEST(daemon_bootstrap_start_failure_record_round_trip) { + bootstrap_endpoint_fixture_t fixture; + bootstrap_endpoint_fixture_t other; + ASSERT_TRUE(bootstrap_endpoint_fixture_start(&fixture, "failure-record")); + char other_parent[BOOTSTRAP_TEST_PATH_CAP]; + int other_written = + snprintf(other_parent, sizeof(other_parent), "%s/other-XXXXXX", fixture.parent); + ASSERT(other_written > 0 && other_written < (int)sizeof(other_parent)); + ASSERT_TRUE(cbm_mkdtemp(other_parent) != NULL); + memset(&other, 0, sizeof(other)); + other.endpoint = cbm_daemon_ipc_endpoint_new("1828000000000002", other_parent); + ASSERT_TRUE(other.endpoint != NULL); + char logs[BOOTSTRAP_TEST_PATH_CAP]; + int logs_written = snprintf(logs, sizeof(logs), "%s/logs", fixture.parent); + ASSERT(logs_written > 0 && logs_written < (int)sizeof(logs)); + char expected_path[BOOTSTRAP_TEST_PATH_CAP]; + int path_written = snprintf(expected_path, sizeof(expected_path), "%s.pending.tmp", + cbm_daemon_ipc_endpoint_address(fixture.endpoint)); + ASSERT(path_written > 0 && path_written < (int)sizeof(expected_path)); + + cbm_daemon_ipc_posix_record_write_failure_set_for_test(ENOSPC); + cbm_daemon_ipc_listener_t *listener = cbm_daemon_ipc_listen(fixture.endpoint); + cbm_daemon_ipc_posix_record_write_failure_set_for_test(0); + ASSERT_TRUE(listener == NULL); + + uint64_t now_s = (uint64_t)time(NULL); + bool recorded = cbm_daemon_bootstrap_start_failure_record(logs, fixture.endpoint, "runtime"); + cbm_daemon_bootstrap_start_failure_t failure; + int found = + cbm_daemon_bootstrap_start_failure_read(logs, fixture.endpoint, now_s - 2, &failure); + cbm_daemon_bootstrap_start_failure_t stale; + int stale_found = + cbm_daemon_bootstrap_start_failure_read(logs, fixture.endpoint, now_s + 60, &stale); + cbm_daemon_bootstrap_start_failure_t foreign; + int foreign_found = + cbm_daemon_bootstrap_start_failure_read(logs, other.endpoint, now_s - 2, &foreign); + char message[CBM_DAEMON_CONFLICT_MESSAGE_SIZE]; + cbm_daemon_bootstrap_start_failure_format(&failure, logs, message, sizeof(message)); + + cbm_daemon_ipc_endpoint_free(other.endpoint); + (void)th_rmtree(fixture.parent); + bootstrap_endpoint_fixture_finish(&fixture); + + ASSERT_TRUE(recorded); + ASSERT_EQ(found, 1); + ASSERT_STR_EQ(failure.component, "runtime"); + ASSERT_STR_EQ(failure.stage, "pending_publication"); + ASSERT_EQ(failure.errno_value, ENOSPC); + ASSERT_STR_EQ(failure.path, expected_path); + ASSERT_TRUE(failure.pid == (uint64_t)getpid()); + ASSERT_EQ(stale_found, 0); + ASSERT_EQ(foreign_found, 0); + ASSERT_TRUE(strstr(message, "CBM daemon failed to start: pending_publication failed with " + "ENOSPC (") != NULL); + ASSERT_TRUE(strstr(message, expected_path) != NULL); + ASSERT_TRUE(strstr(message, "/cbm-daemon.log") != NULL); + PASS(); +} + +/* #1828: a daemon that dies at publication (full /tmp) left every client + * waiting the full 30 s and then reporting "active or starting" -- the + * opposite of the truth. A real host is spawned against a runtime directory + * whose record writes fail with ENOSPC; the client must report "failed to + * start" naming the errno and the path, within seconds, not the deadline. */ +TEST(daemon_bootstrap_fails_fast_when_daemon_dies_at_publication) { + const char *old_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache = old_cache ? cbm_strdup(old_cache) : NULL; + bool snapshot_ok = !old_cache || saved_cache; + + bootstrap_endpoint_fixture_t fixture; + bool fixture_ok = snapshot_ok && bootstrap_endpoint_fixture_start(&fixture, "enospc-host"); + char cache[BOOTSTRAP_TEST_PATH_CAP] = {0}; + char daemon_log[BOOTSTRAP_TEST_PATH_CAP] = {0}; + char expected_path[BOOTSTRAP_TEST_PATH_CAP] = {0}; + int cache_written = + fixture_ok ? snprintf(cache, sizeof(cache), "%s/cache", fixture.parent) : -1; + int log_written = + fixture_ok ? snprintf(daemon_log, sizeof(daemon_log), "%s/logs/cbm-daemon.log", cache) : -1; + int path_written = fixture_ok ? snprintf(expected_path, sizeof(expected_path), "%s.pending.tmp", + cbm_daemon_ipc_endpoint_address(fixture.endpoint)) + : -1; + bool environment_ready = cache_written > 0 && cache_written < (int)sizeof(cache) && + log_written > 0 && log_written < (int)sizeof(daemon_log) && + path_written > 0 && path_written < (int)sizeof(expected_path) && + cbm_mkdir_p(cache, 0700) && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0; + + char self_build[CBM_DAEMON_BUILD_FINGERPRINT_SIZE] = {0}; + bool identity_ready = environment_ready && cbm_daemon_runtime_process_build_fingerprint( + (uint64_t)getpid(), self_build); + static bootstrap_enospc_host_t host; + memset(&host, 0, sizeof(host)); + (void)snprintf(host.parent, sizeof(host.parent), "%s", fixture_ok ? fixture.parent : ""); + host.identity = bootstrap_identity("2.4.0", self_build); + + cbm_daemon_bootstrap_config_t config = { + .role = CBM_DAEMON_PROCESS_MCP_CLIENT, + .endpoint = fixture.endpoint, + .identity = &host.identity, + .executable_path = "/enospc-host-test", + .connect_timeout_ms = 200, + .startup_timeout_ms = 30000, + }; + cbm_daemon_bootstrap_result_t result; + memset(&result, 0, sizeof(result)); + cbm_daemon_bootstrap_status_t status = CBM_DAEMON_BOOTSTRAP_FAILED; + uint64_t elapsed_ms = UINT64_MAX; + if (identity_ready) { + cbm_daemon_ipc_posix_record_write_failure_set_for_test(ENOSPC); + cbm_daemon_bootstrap_spawn_override_set_for_test(bootstrap_enospc_host_spawn, &host); + uint64_t started = cbm_now_ms(); + status = cbm_daemon_bootstrap_execute(&config, &result); + elapsed_ms = cbm_now_ms() - started; + cbm_daemon_bootstrap_spawn_override_set_for_test(NULL, NULL); + cbm_daemon_ipc_posix_record_write_failure_set_for_test(0); + } + int nonzero_exits = 0; + bootstrap_enospc_reap(&host, &nonzero_exits); + + static char log[BOOTSTRAP_ENOSPC_LOG_CAP]; + log[0] = '\0'; + bool log_read = bootstrap_read_file(daemon_log, log, sizeof(log)); + const char *listen_failed = strstr(log, "msg=daemon.ipc.listen_failed"); + bool daemon_named_cause = listen_failed && strstr(listen_failed, "errno=ENOSPC") != NULL && + strstr(listen_failed, expected_path) != NULL; + bool message_names_failure = strstr(result.message, "failed to start") != NULL; + bool message_names_errno = strstr(result.message, "ENOSPC") != NULL; + bool message_names_path = strstr(result.message, expected_path) != NULL; + bool stale_wording = strstr(result.message, "active or starting") != NULL; + + if (saved_cache) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache, 1); + } else { + (void)cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache); + if (fixture_ok) { + (void)th_rmtree(fixture.parent); + bootstrap_endpoint_fixture_finish(&fixture); + } + + ASSERT_TRUE(snapshot_ok); + ASSERT_TRUE(fixture_ok); + ASSERT_TRUE(environment_ready); + ASSERT_TRUE(identity_ready); + ASSERT_EQ(status, CBM_DAEMON_BOOTSTRAP_FAILED); + ASSERT_TRUE(result.daemon_spawned); + ASSERT_TRUE(host.child_count >= 1); + ASSERT_TRUE(log_read); + ASSERT_TRUE(daemon_named_cause); + ASSERT(elapsed_ms < 5000); + ASSERT_FALSE(stale_wording); + ASSERT_TRUE(message_names_failure); + ASSERT_TRUE(message_names_errno); + ASSERT_TRUE(message_names_path); + ASSERT_EQ(host.child_count, 1U); + ASSERT_EQ(nonzero_exits, 0); + bootstrap_endpoint_fixture_finish(&fixture); + PASS(); +} +#endif + SUITE(daemon_bootstrap) { RUN_TEST(daemon_bootstrap_classifies_default_and_ui_as_mcp_clients); RUN_TEST(daemon_bootstrap_classifies_stateless_commands_without_client); @@ -903,4 +1156,8 @@ SUITE(daemon_bootstrap) { #ifdef __APPLE__ RUN_TEST(daemon_bootstrap_darwin_launch_failure_is_synchronous); #endif +#ifndef _WIN32 + RUN_TEST(daemon_bootstrap_start_failure_record_round_trip); + RUN_TEST(daemon_bootstrap_fails_fast_when_daemon_dies_at_publication); +#endif } From 7b4eb4a00ac600060be4836ee4e1bbfa7bbedf24 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 8 Sep 2026 02:02:08 +0200 Subject: [PATCH 027/130] fix(pipeline): route a first index as no_existing_db, not invalid_existing_db The route probe opened whatever resolve_db_path returned, which inside the staged run is the run's own stage. On a first index nothing was copied into it, so the probe opened a 0-byte mkstemp placeholder, failed its integrity check, and warned "pipeline.route path=full reason=invalid_existing_db" -- on every first index of every project. Read back after an OOM kill it looked like a corrupted database and a daily "corrupt -> full rebuild -> OOM" loop (#1864); the loop is "no database -> full -> OOM", and the stale 0-byte stage never influenced the next run at all (each run mints a fresh name). cbm_pipeline_run now records whether the destination existed and was copied into the stage, and the route decision returns full early with reason=no_existing_db (or existing_db_backup_failed, after the backup_failed_full_rebuild warning) without probing a placeholder. "invalid_existing_db" stays reserved for a real copy that fails its integrity check. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- src/pipeline/pipeline.c | 21 ++++++++++++++ tests/test_pipeline.c | 61 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 3d600034e..5d21a1bde 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -196,6 +196,15 @@ struct cbm_pipeline { * full rebuild, so the MCP response can surface the migration. */ bool format_migration; + /* Recorded by cbm_pipeline_run for the staged run beneath it: whether + * the destination existed, and whether it was copied into the stage so + * that an incremental route has a real previous generation to work + * from. Without a copy the stage is the run's empty placeholder, and + * probing THAT for integrity is what reported every first index as + * "invalid_existing_db" (#1864). */ + bool final_existed; + bool existing_generation; + /* ADR (project_summaries) captured before a full-reindex DB delete, so it * can be restored after the rebuild. NULL when no ADR existed. Issue #516. */ char *saved_adr; @@ -1403,6 +1412,16 @@ static int capture_existing_adr(cbm_pipeline_t *p, const char *db_path) { static int try_incremental_or_delete_db(cbm_pipeline_t *p, cbm_file_info_t *files, int file_count, const cbm_file_hash_t *baseline_manifest, int baseline_count, bool force_full_on_mismatch) { + if (!p->existing_generation) { + /* Nothing to be incremental against: a first index, or a + * destination that could not be copied (already reported as + * backup_failed_full_rebuild). The stage is an empty placeholder, + * not a database, so it is not probed -- "invalid_existing_db" + * stays reserved for a real copy that fails its integrity check. */ + cbm_log_info("pipeline.route", "path", "full", "reason", + p->final_existed ? "existing_db_backup_failed" : "no_existing_db"); + return CBM_PIPELINE_FORCE_FULL_REINDEX; + } char *db_path = resolve_db_path(p); if (!db_path) { return CBM_PIPELINE_FORCE_FULL_REINDEX; @@ -2956,6 +2975,8 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { } } + p->final_existed = final_existed; + p->existing_generation = final_existed && backup_succeeded; char *configured_db_path = p->db_path; p->db_path = strdup(staging_path); if (!p->db_path) { diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 9ee54b41b..0892fd696 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -3707,6 +3707,66 @@ TEST(pipeline_sweep_removes_dead_writer_stage_keeps_locked_stage) { PASS(); } +static char g_route_log_capture[8192]; +static atomic_flag g_route_log_spin = ATOMIC_FLAG_INIT; + +/* Keeps only the route decisions; worker threads log too, and only the + * appends need serialising. */ +static void capture_route_log_sink(const char *line) { + if (!line || !strstr(line, "pipeline.route")) { + return; + } + while (atomic_flag_test_and_set_explicit(&g_route_log_spin, memory_order_acquire)) {} + size_t used = strlen(g_route_log_capture); + size_t avail = sizeof(g_route_log_capture) - used; + if (avail > 1) { + int n = snprintf(g_route_log_capture + used, avail, "%s\n", line); + if (n < 0 || (size_t)n >= avail) { + g_route_log_capture[sizeof(g_route_log_capture) - 1] = '\0'; + } + } + atomic_flag_clear_explicit(&g_route_log_spin, memory_order_release); +} + +/* #1864: a first index has no previous generation. The route probe used to + * open the run's own EMPTY stage placeholder, fail its integrity check, and + * warn "reason=invalid_existing_db" on every first index of every project, + * which the report read as a corrupted database and a crash loop. A fresh + * index says what it is. */ +TEST(pipeline_fresh_index_never_reports_invalid_existing_db) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_fresh_route_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + write_temp_file(tmp, "generation.py", "def StableGeneration():\n return 1\n"); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/generation.db", tmp); + + g_route_log_capture[0] = '\0'; + CBMLogLevel previous_level = cbm_log_get_level(); + cbm_log_set_level(CBM_LOG_DEBUG); + cbm_log_set_sink(capture_route_log_sink); + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *fresh = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + int fresh_rc = fresh ? cbm_pipeline_run(fresh) : -1; + cbm_pipeline_free(fresh); + cbm_log_set_sink(NULL); + cbm_log_set_level(previous_level); + + bool invalid_reported = strstr(g_route_log_capture, "invalid_existing_db") != NULL; + bool fresh_reported = strstr(g_route_log_capture, "reason=no_existing_db") != NULL; + bool db_present = path_exists(db_path); + int stage_count = count_generation_stage_artifacts(tmp, "generation.db"); + cbm_pipeline_incremental_test_reset_faults(); + th_rmtree(tmp); + + ASSERT_EQ(fresh_rc, 0); + ASSERT_TRUE(!invalid_reported); + ASSERT_TRUE(fresh_reported); + ASSERT_TRUE(db_present); + ASSERT_EQ(stage_count, 0); + PASS(); +} + /* Discovery and extraction must describe the same immutable generation. A * source file created after extraction is not present in the original file * list, so merely re-hashing that list cannot detect the race. Publication @@ -14413,6 +14473,7 @@ SUITE(pipeline_semantic_manifest_repro) { RUN_TEST(pipeline_minted_stage_is_owned_until_released); RUN_TEST(pipeline_stale_zero_byte_stage_beside_valid_db_routes_incremental_and_is_swept); RUN_TEST(pipeline_sweep_removes_dead_writer_stage_keeps_locked_stage); + RUN_TEST(pipeline_fresh_index_never_reports_invalid_existing_db); RUN_TEST(pipeline_source_mutation_before_publication_preserves_previous_generation); RUN_TEST(pipeline_source_addition_before_publication_preserves_previous_generation); RUN_TEST(pipeline_tsconfig_mutation_before_publication_preserves_previous_generation); From e5aabdf86ea86ee27b92477b21aa8d6c754a674d Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 8 Sep 2026 02:04:51 +0200 Subject: [PATCH 028/130] fix(cli): follow user-owned symlinked config files in the editors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every config editor (JSON-like, TOML, YAML, text) opens with O_NOFOLLOW and refuses symlinks, so a link planted at ~/.cursor/mcp.json cannot point a privileged writer at another file. Users keep exactly those files as symlinks into a dotfiles checkout; the blanket refusal made install and uninstall fail on them and, before the previous commit, turned uninstall into data loss (#1954). Decision C: a symlink the INVOKING user owns, resolving to a regular file the same user owns, is followed — as an explicit opt-in of the agent-configuration writers only. cbm_install_agent_configs and the uninstall activation register the user's configuration roots (HOME, XDG_CONFIG_HOME) for the duration of their work; a link whose own directory is outside every registered root, or any link seen by another editor caller, stays refused, so repository paths never follow. One shared helper (src/cli/config_edit_path.c) runs the race-safe sequence: lstat(link) → owner == invoking uid (a root process follows root-owned links only) → realpath → the target's parent directory opened as a dirfd (O_DIRECTORY|O_NOFOLLOW; owned by the invoking user or root, not writable by others unless sticky) → openat(dirfd, base, O_NOFOLLOW) → fstat: regular file, same owner, link count 1, and the same (dev, ino) as stat(realpath). Each editor reads through that validated descriptor, stages its temp file with openat(dirfd, …, O_CREAT|O_EXCL|O_NOFOLLOW), runs its pre-publish snapshot comparison against the resolved path, and commits with renameat on the pinned dirfd, so the link itself is never replaced. Every other link is refused with an observable reason (surfaced in the uninstall diagnostics): another owner, root over a non-root link, dangling, a directory or special target, a hard-linked or swapped target, a foreign or world-writable parent. Whether the target must also stay under a root is a pending user decision behind CBM_CONFIG_EDIT_TARGET_UNDER_ROOT (default: anywhere the user owns). Windows never follows reparse points. Tests: each editor suite gains a same-owner link edited in place under the opt-in (link preserved and still pointing at the same target, temp file staged beside the target, bytes identical to the same edit on a plain file, no stray temp files) and keeps its refusal cases; the JSON-like suite adds no-opt-in, outside-root, dangling, directory, identity-skew and world-writable-parent refusals. The foreign-owner and root-run refusals move the observer through a test seam (a test cannot create a foreign-owned link without root; a real root run demotes the link first). test_cli.c drives `uninstall -y` end to end through a symlinked ~/.cursor/mcp.json. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- Makefile.cbm | 3 +- src/cli/cli.c | 38 +++ src/cli/config_edit_path.c | 434 ++++++++++++++++++++++++++++++++++ src/cli/config_edit_path.h | 109 +++++++++ src/cli/config_json_like.c | 77 +++++- src/cli/config_text_edit.c | 78 +++++- src/cli/config_toml_edit.c | 96 ++++++-- src/cli/config_yaml_edit.c | 75 +++++- tests/test_cli.c | 94 ++++++++ tests/test_config_json_like.c | 411 +++++++++++++++++++++++++++++++- tests/test_config_text_edit.c | 74 +++++- tests/test_config_toml_edit.c | 64 ++++- tests/test_config_yaml_edit.c | 81 ++++++- 13 files changed, 1561 insertions(+), 73 deletions(-) create mode 100644 src/cli/config_edit_path.c create mode 100644 src/cli/config_edit_path.h diff --git a/Makefile.cbm b/Makefile.cbm index 4f1ab1371..f92647685 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -84,6 +84,7 @@ SANITIZE = -fsanitize=address,undefined -fno-omit-frame-pointer EDITOR_TEST_DEFINES = -DCBM_JSON_LIKE_ENABLE_TEST_API=1 \ -DCBM_TOML_EDIT_ENABLE_TEST_API=1 -DCBM_YAML_ENABLE_TEST_API=1 \ -DCBM_TEXT_EDIT_ENABLE_TEST_API=1 -DCBM_CLI_ENABLE_TEST_API=1 \ + -DCBM_CONFIG_EDIT_PATH_ENABLE_TEST_API=1 \ -DCBM_DIAGNOSTICS_ENABLE_TEST_API=1 -DCBM_ENABLE_TEST_SEAMS=1 # The build system is the single source of truth for "is this binary # instrumented": compiler-specific probes (__SANITIZE_ADDRESS__) miss @@ -409,7 +410,7 @@ GIT_SRCS = src/git/git_context.c CLI_SRCS = src/cli/cli.c src/cli/progress_sink.c src/cli/hook_augment.c src/cli/client_adapter.c \ src/cli/agent_clients.c src/cli/agent_profiles.c \ src/cli/config_json_like.c src/cli/config_toml_edit.c src/cli/config_yaml_edit.c \ - src/cli/config_text_edit.c src/cli/activation_transaction.c + src/cli/config_text_edit.c src/cli/config_edit_path.c src/cli/activation_transaction.c # UI module (graph visualization) UI_SRCS = \ diff --git a/src/cli/cli.c b/src/cli/cli.c index 29ddd7f4f..fc1b68e44 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -8,6 +8,7 @@ #include "cli/agent_profiles.h" #include "cli/cli.h" #include "cli/activation_transaction.h" +#include "cli/config_edit_path.h" #include "cli/config_json_like.h" #include "cli/config_text_edit.h" #include "cli/config_toml_edit.h" @@ -7776,6 +7777,21 @@ static void agent_uninstall_failure_record(const char *agent, const char *operat (void)snprintf(entry->detail, sizeof(entry->detail), "%s", detail ? detail : ""); } +/* The agent-configuration writers opt in to following user-owned symlinked + * config files under the user's configuration roots (#1954, decision C); + * every other caller of the config editors keeps refusing links. Cleared by + * the same command when its configuration work is done. */ +static void cli_config_follow_begin(const char *home) { + cbm_config_edit_path_follow_clear(); + if (home && home[0]) { + (void)cbm_config_edit_path_follow_add_root(home); + } + const char *xdg_config = getenv("XDG_CONFIG_HOME"); + if (xdg_config && xdg_config[0]) { + (void)cbm_config_edit_path_follow_add_root(xdg_config); + } +} + /* The closing list of what uninstall could not clean. Printed AFTER the * executable and the indexes are gone, so nothing in it is a reason to keep * the installation around — each line is one file the user removes an entry @@ -7853,6 +7869,14 @@ static void describe_agent_config_target(const char *path, char *out, size_t out : info.is_directory ? "directory" : info.is_regular ? "regular file" : "special file"; + /* A refused symlink names the rule that refused it (#1954): the user + * then knows whether to fix ownership, the target, or the parent. */ + char refusal[160]; + if (info.is_symlink && cbm_config_edit_path_refusal(path, refusal, sizeof(refusal))) { + (void)snprintf(out, out_size, " (target: symlink, %lld bytes; not followed: %s)", + (long long)info.size, refusal); + return; + } (void)snprintf(out, out_size, " (target: %s, %lld bytes)", kind, (long long)info.size); } @@ -9817,12 +9841,24 @@ static void install_additional_agent_configs(const cbm_detected_agents_t *agents } } +static int cbm_install_agent_configs_in_scope(const char *home, const char *binary_path, bool force, + bool dry_run, cbm_detected_agents_t *agents_in); + int cbm_install_agent_configs(const char *home, const char *binary_path, bool force, bool dry_run) { g_agent_install_errors = 0; cbm_detected_agents_t agents = cbm_detect_agents(home); if (g_client_selection && !cli_clients_apply_selection(g_client_selection, &agents)) { return CLI_ERR; } + cli_config_follow_begin(home); + int result = cbm_install_agent_configs_in_scope(home, binary_path, force, dry_run, &agents); + cbm_config_edit_path_follow_clear(); + return result; +} + +static int cbm_install_agent_configs_in_scope(const char *home, const char *binary_path, bool force, + bool dry_run, cbm_detected_agents_t *agents_in) { + cbm_detected_agents_t agents = *agents_in; if (!g_install_plan) { print_detected_agents(&agents, home); } @@ -12108,6 +12144,7 @@ static int cli_uninstall_activate(void *opaque) { return CLI_TRUE; } + cli_config_follow_begin(activation->home); if (activation->agents.claude_code) { uninstall_claude_code(activation->home, activation->bin_path, activation->dry_run); } @@ -12115,6 +12152,7 @@ static int cli_uninstall_activate(void *opaque) { uninstall_editor_agents(&activation->agents, activation->home, activation->dry_run); uninstall_additional_agents(&activation->agents, activation->home, activation->dry_run); uninstall_agent_client_registry(activation->home, activation->dry_run); + cbm_config_edit_path_follow_clear(); /* Agent-config failures are collected, never a gate: an entry the editors * refuse to touch (a symlinked config, a foreign file, a malformed diff --git a/src/cli/config_edit_path.c b/src/cli/config_edit_path.c new file mode 100644 index 000000000..b51c5c2a2 --- /dev/null +++ b/src/cli/config_edit_path.c @@ -0,0 +1,434 @@ +/* + * config_edit_path.c — follow a config symlink only when the agent-config + * writer opted in, the invoking user owns both the link and its regular-file + * target, and the target's parent directory can be pinned. See the header + * for the decision, the sequence and its boundaries (#1954). + */ +#include "cli/config_edit_path.h" + +#include "foundation/compat.h" +#include "foundation/compat_fs.h" + +#include +#include +#include + +#ifndef _WIN32 +#include +#include +#include +#include +#endif + +enum { EDIT_PATH_MAX_ROOTS = 4 }; + +static int edit_path_copy(const char *source, char *out, size_t out_size) { + int written = snprintf(out, out_size, "%s", source); + return written >= 0 && (size_t)written < out_size ? 0 : -1; +} + +static void edit_path_target_reset(cbm_config_edit_target_t *target) { + target->status = CBM_CONFIG_EDIT_PATH_DIRECT; + target->dirfd = -1; + target->fd = -1; + target->base[0] = '\0'; + target->path[0] = '\0'; +} + +static void edit_path_note(char *reason, size_t reason_size, const char *text) { + if (reason && reason_size > 0U) { + (void)snprintf(reason, reason_size, "%s", text); + } +} + +void cbm_config_edit_target_close(cbm_config_edit_target_t *target) { + if (!target) { + return; + } + int saved_errno = errno; +#ifndef _WIN32 + if (target->fd >= 0) { + (void)close(target->fd); + } + if (target->dirfd >= 0) { + (void)close(target->dirfd); + } +#endif + target->fd = -1; + target->dirfd = -1; + errno = saved_errno; +} + +#ifndef _WIN32 + +static CBM_TLS char edit_path_roots[EDIT_PATH_MAX_ROOTS][CBM_CONFIG_EDIT_PATH_MAX]; +static CBM_TLS size_t edit_path_root_count = 0U; + +#ifdef CBM_CONFIG_EDIT_PATH_ENABLE_TEST_API +static CBM_TLS int edit_path_uid_override_set = 0; +static CBM_TLS uid_t edit_path_uid_override = 0; +static CBM_TLS int edit_path_identity_skew = 0; + +void cbm_config_edit_path_set_invoking_uid_for_test(unsigned uid, int set) { + edit_path_uid_override_set = set != 0; + edit_path_uid_override = (uid_t)uid; +} + +void cbm_config_edit_path_set_identity_skew_for_test(int set) { + edit_path_identity_skew = set != 0; +} +#endif + +static uid_t edit_path_invoking_uid(void) { +#ifdef CBM_CONFIG_EDIT_PATH_ENABLE_TEST_API + if (edit_path_uid_override_set) { + return edit_path_uid_override; + } +#endif + return geteuid(); +} + +int cbm_config_edit_path_follow_add_root(const char *root) { + if (!root || !root[0] || edit_path_root_count >= EDIT_PATH_MAX_ROOTS) { + return -1; + } + char canonical[CBM_CONFIG_EDIT_PATH_MAX]; + if (!cbm_canonical_path(root, canonical, sizeof(canonical))) { + return -1; + } + if (edit_path_copy(canonical, edit_path_roots[edit_path_root_count], + sizeof(edit_path_roots[0])) != 0) { + return -1; + } + edit_path_root_count++; + return 0; +} + +void cbm_config_edit_path_follow_clear(void) { + edit_path_root_count = 0U; +} + +static int edit_path_under_root(const char *canonical) { + for (size_t i = 0U; i < edit_path_root_count; i++) { + const char *root = edit_path_roots[i]; + size_t root_length = strlen(root); + if (strcmp(canonical, root) == 0) { + return 1; + } + if (strncmp(canonical, root, root_length) == 0 && + (canonical[root_length] == '/' || root[root_length - 1U] == '/')) { + return 1; + } + } + return 0; +} + +/* Splits a canonical absolute path into its parent and its final name. */ +static int edit_path_split(const char *canonical, char *parent, size_t parent_size, char *base, + size_t base_size) { + const char *slash = strrchr(canonical, '/'); + if (!slash || slash[1] == '\0') { + return -1; + } + size_t parent_length = slash == canonical ? 1U : (size_t)(slash - canonical); + if (parent_length >= parent_size || edit_path_copy(slash + 1, base, base_size) != 0) { + return -1; + } + memcpy(parent, canonical, parent_length); + parent[parent_length] = '\0'; + return 0; +} + +/* The link's OWN directory must sit under a registered root: that is what + * makes the follow an opt-in of the agent-configuration writers rather than + * a property of every editor caller. */ +static int edit_path_link_dir_under_root(const char *path) { + if (edit_path_root_count == 0U) { + return 0; + } + char parent[CBM_CONFIG_EDIT_PATH_MAX]; + if (edit_path_copy(path, parent, sizeof(parent)) != 0) { + return 0; + } + char *slash = strrchr(parent, '/'); + if (!slash) { + parent[0] = '.'; + parent[1] = '\0'; + } else if (slash == parent) { + parent[1] = '\0'; + } else { + *slash = '\0'; + } + char canonical[CBM_CONFIG_EDIT_PATH_MAX]; + if (!cbm_canonical_path(parent, canonical, sizeof(canonical))) { + return 0; + } + return edit_path_under_root(canonical); +} + +static void edit_path_note_owner(char *reason, size_t reason_size, const char *what, + unsigned long owner, unsigned long self) { + if (reason && reason_size > 0U) { + (void)snprintf(reason, reason_size, "%s owned by uid %lu, not the invoking user (uid %lu)", + what, owner, self); + } +} + +/* One classification serves both the resolver and the diagnostic, so the + * reason a user reads is exactly the rule that refused the link. On REFUSED + * every descriptor opened along the way is closed again. */ +static int edit_path_classify(const char *path, cbm_config_edit_target_t *target, char *reason, + size_t reason_size) { + edit_path_target_reset(target); + if (reason && reason_size > 0U) { + reason[0] = '\0'; + } + if (!path || !path[0]) { + return CBM_CONFIG_EDIT_PATH_REFUSED; + } + struct stat link_state; + if (lstat(path, &link_state) != 0 || !S_ISLNK(link_state.st_mode)) { + return edit_path_copy(path, target->path, sizeof(target->path)) == 0 + ? CBM_CONFIG_EDIT_PATH_DIRECT + : CBM_CONFIG_EDIT_PATH_REFUSED; + } + if (!edit_path_link_dir_under_root(path)) { + edit_path_note(reason, reason_size, + edit_path_root_count == 0U + ? "symlinks are not followed by this command" + : "symlink outside the opted-in configuration roots"); + return CBM_CONFIG_EDIT_PATH_REFUSED; + } + uid_t self = edit_path_invoking_uid(); + if (self == 0 && link_state.st_uid != 0) { + if (reason && reason_size > 0U) { + (void)snprintf(reason, reason_size, + "symlink owned by uid %lu is not followed by a root process", + (unsigned long)link_state.st_uid); + } + return CBM_CONFIG_EDIT_PATH_REFUSED; + } + if (link_state.st_uid != self) { + edit_path_note_owner(reason, reason_size, "symlink", (unsigned long)link_state.st_uid, + (unsigned long)self); + return CBM_CONFIG_EDIT_PATH_REFUSED; + } + char resolved[CBM_CONFIG_EDIT_PATH_MAX]; + struct stat target_state; + if (!cbm_canonical_path(path, resolved, sizeof(resolved)) || + stat(resolved, &target_state) != 0) { + edit_path_note(reason, reason_size, "dangling symlink"); + return CBM_CONFIG_EDIT_PATH_REFUSED; + } + if (!S_ISREG(target_state.st_mode)) { + edit_path_note(reason, reason_size, + S_ISDIR(target_state.st_mode) ? "symlink to a directory" + : "symlink to a special file"); + return CBM_CONFIG_EDIT_PATH_REFUSED; + } + if (target_state.st_uid != self) { + edit_path_note_owner(reason, reason_size, "symlink target", + (unsigned long)target_state.st_uid, (unsigned long)self); + return CBM_CONFIG_EDIT_PATH_REFUSED; + } +#if CBM_CONFIG_EDIT_TARGET_UNDER_ROOT + if (!edit_path_under_root(resolved)) { + edit_path_note(reason, reason_size, "symlink target outside the configuration roots"); + return CBM_CONFIG_EDIT_PATH_REFUSED; + } +#endif + char parent[CBM_CONFIG_EDIT_PATH_MAX]; + if (edit_path_split(resolved, parent, sizeof(parent), target->base, sizeof(target->base)) != + 0 || + edit_path_copy(resolved, target->path, sizeof(target->path)) != 0) { + edit_path_note(reason, reason_size, "symlink target path is too long"); + edit_path_target_reset(target); + return CBM_CONFIG_EDIT_PATH_REFUSED; + } + + int dir_flags = O_RDONLY | O_NOFOLLOW; +#ifdef O_DIRECTORY + dir_flags |= O_DIRECTORY; +#endif +#ifdef O_CLOEXEC + dir_flags |= O_CLOEXEC; +#endif + target->dirfd = open(parent, dir_flags); + struct stat dir_state; + if (target->dirfd < 0 || fstat(target->dirfd, &dir_state) != 0 || !S_ISDIR(dir_state.st_mode)) { + edit_path_note(reason, reason_size, + "parent directory of the symlink target cannot be opened"); + cbm_config_edit_target_close(target); + edit_path_target_reset(target); + return CBM_CONFIG_EDIT_PATH_REFUSED; + } + if (dir_state.st_uid != self && dir_state.st_uid != 0) { + if (reason && reason_size > 0U) { + (void)snprintf(reason, reason_size, + "parent directory of the symlink target is owned by uid %lu", + (unsigned long)dir_state.st_uid); + } + cbm_config_edit_target_close(target); + edit_path_target_reset(target); + return CBM_CONFIG_EDIT_PATH_REFUSED; + } + if ((dir_state.st_mode & S_IWOTH) != 0 && (dir_state.st_mode & S_ISVTX) == 0) { + edit_path_note(reason, reason_size, + "parent directory of the symlink target is writable by others"); + cbm_config_edit_target_close(target); + edit_path_target_reset(target); + return CBM_CONFIG_EDIT_PATH_REFUSED; + } + + int file_flags = O_RDONLY | O_NOFOLLOW | O_NONBLOCK; +#ifdef O_CLOEXEC + file_flags |= O_CLOEXEC; +#endif + target->fd = openat(target->dirfd, target->base, file_flags); + struct stat opened_state; + if (target->fd < 0 || fstat(target->fd, &opened_state) != 0) { + edit_path_note(reason, reason_size, "symlink target cannot be opened beside its parent"); + cbm_config_edit_target_close(target); + edit_path_target_reset(target); + return CBM_CONFIG_EDIT_PATH_REFUSED; + } + const char *mismatch = NULL; + if (!S_ISREG(opened_state.st_mode)) { + mismatch = "symlink target is not a regular file"; + } else if (opened_state.st_uid != self) { + mismatch = "symlink target changed owner while it was being checked"; + } else if (opened_state.st_nlink != 1U) { + mismatch = "symlink target is hard-linked"; + } else if (opened_state.st_dev != target_state.st_dev || + opened_state.st_ino != target_state.st_ino) { + mismatch = "symlink target changed while it was being checked"; + } +#ifdef CBM_CONFIG_EDIT_PATH_ENABLE_TEST_API + if (!mismatch && edit_path_identity_skew) { + mismatch = "symlink target changed while it was being checked"; + } +#endif + if (mismatch) { + edit_path_note(reason, reason_size, mismatch); + cbm_config_edit_target_close(target); + edit_path_target_reset(target); + return CBM_CONFIG_EDIT_PATH_REFUSED; + } + target->status = CBM_CONFIG_EDIT_PATH_FOLLOWED; + return CBM_CONFIG_EDIT_PATH_FOLLOWED; +} + +int cbm_config_edit_target_create_temp(const cbm_config_edit_target_t *target, const char *name, + unsigned mode) { + if (!target || target->status != CBM_CONFIG_EDIT_PATH_FOLLOWED || target->dirfd < 0 || !name || + !name[0] || strchr(name, '/')) { + errno = EINVAL; + return -1; + } + int flags = O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW; +#ifdef O_CLOEXEC + flags |= O_CLOEXEC; +#endif + return openat(target->dirfd, name, flags, (mode_t)mode); +} + +int cbm_config_edit_target_commit(const cbm_config_edit_target_t *target, const char *name) { + if (!target || target->status != CBM_CONFIG_EDIT_PATH_FOLLOWED || target->dirfd < 0 || !name || + !name[0] || strchr(name, '/')) { + errno = EINVAL; + return -1; + } + if (renameat(target->dirfd, name, target->dirfd, target->base) != 0) { + return -1; + } + return fsync(target->dirfd) == 0 ? 0 : -1; +} + +int cbm_config_edit_target_unlink(const cbm_config_edit_target_t *target, const char *name) { + if (!target || target->status != CBM_CONFIG_EDIT_PATH_FOLLOWED || target->dirfd < 0 || !name || + !name[0] || strchr(name, '/')) { + errno = EINVAL; + return -1; + } + return unlinkat(target->dirfd, name, 0); +} + +#else /* _WIN32 */ + +#ifdef CBM_CONFIG_EDIT_PATH_ENABLE_TEST_API +void cbm_config_edit_path_set_invoking_uid_for_test(unsigned uid, int set) { + (void)uid; + (void)set; +} + +void cbm_config_edit_path_set_identity_skew_for_test(int set) { + (void)set; +} +#endif + +int cbm_config_edit_path_follow_add_root(const char *root) { + (void)root; + return 0; +} + +void cbm_config_edit_path_follow_clear(void) {} + +/* Reparse points are never followed by the editors on Windows; the path is + * handed back unchanged and FILE_FLAG_OPEN_REPARSE_POINT keeps refusing. */ +static int edit_path_classify(const char *path, cbm_config_edit_target_t *target, char *reason, + size_t reason_size) { + edit_path_target_reset(target); + if (reason && reason_size > 0U) { + reason[0] = '\0'; + } + if (!path || !path[0]) { + return CBM_CONFIG_EDIT_PATH_REFUSED; + } + return edit_path_copy(path, target->path, sizeof(target->path)) == 0 + ? CBM_CONFIG_EDIT_PATH_DIRECT + : CBM_CONFIG_EDIT_PATH_REFUSED; +} + +int cbm_config_edit_target_create_temp(const cbm_config_edit_target_t *target, const char *name, + unsigned mode) { + (void)target; + (void)name; + (void)mode; + errno = EINVAL; + return -1; +} + +int cbm_config_edit_target_commit(const cbm_config_edit_target_t *target, const char *name) { + (void)target; + (void)name; + errno = EINVAL; + return -1; +} + +int cbm_config_edit_target_unlink(const cbm_config_edit_target_t *target, const char *name) { + (void)target; + (void)name; + errno = EINVAL; + return -1; +} + +#endif /* _WIN32 */ + +int cbm_config_edit_target_open(const char *path, cbm_config_edit_target_t *target) { + if (!target) { + return CBM_CONFIG_EDIT_PATH_REFUSED; + } + int status = edit_path_classify(path, target, NULL, 0U); + target->status = status; + return status; +} + +int cbm_config_edit_path_refusal(const char *path, char *out, size_t out_size) { + if (!out || out_size == 0U) { + return 0; + } + cbm_config_edit_target_t target; + (void)edit_path_classify(path, &target, out, out_size); + cbm_config_edit_target_close(&target); + return out[0] != '\0' ? 1 : 0; +} diff --git a/src/cli/config_edit_path.h b/src/cli/config_edit_path.h new file mode 100644 index 000000000..abe3d0e4c --- /dev/null +++ b/src/cli/config_edit_path.h @@ -0,0 +1,109 @@ +/* + * config_edit_path.h — the path a configuration editor may operate on when + * the path it was given is a symlink. + * + * Every config editor (JSON-like, TOML, YAML, text) opens its file with + * O_NOFOLLOW and refuses symlinks: a link planted at ~/.cursor/mcp.json could + * otherwise point a privileged writer at any file its planter can name. Users, + * however, keep exactly those files as symlinks into a dotfiles checkout, and + * a blanket refusal turned `uninstall` into data loss (#1954). + * + * Decision C (2026-09-07): a symlink the INVOKING user owns, resolving to a + * regular file the same user owns, is followed — but only as an explicit + * OPT-IN of the agent-configuration writers, scoped to configuration roots + * (HOME / XDG dirs) they register with cbm_config_edit_path_follow_add_root(). + * With no root registered the editors behave exactly as before; a path whose + * link lives outside every registered root is never followed, so repository + * paths stay refused whatever the caller. + * + * Race-safe sequence for a followed link (POSIX): + * lstat(link) → link owner == invoking uid (a root process follows only + * root-owned links) → realpath → open the target's PARENT DIRECTORY as a + * dirfd (O_DIRECTORY|O_NOFOLLOW; owned by the invoking user or root, not + * writable by others unless sticky) → openat(dirfd, base, O_NOFOLLOW) → + * fstat: regular file, owned by the invoking user, link count 1, and the + * same (st_dev, st_ino) as stat(realpath). + * Everything else is refused with an observable reason + * (cbm_config_edit_path_refusal): another owner, a root process over a + * non-root link, a dangling link, a link to a directory or special file, a + * hard-linked or swapped target, a foreign or world-writable parent. The + * editor then reads through the validated descriptor, stages its temp file + * with openat(dirfd, …, O_CREAT|O_EXCL) and commits with renameat on the same + * dirfd, so the link itself is never replaced. Windows never follows reparse + * points here: the editors keep FILE_FLAG_OPEN_REPARSE_POINT and the helper + * hands the path back unchanged. + * + * The check runs at every open and before every commit, never cached: a link + * swapped mid-edit resolves to a different identity, which also fails the + * editors' own pre-publish snapshot comparison on the resolved path. + */ +#ifndef CBM_CONFIG_EDIT_PATH_H +#define CBM_CONFIG_EDIT_PATH_H + +#include + +enum { + /* Callers pass a buffer of at least this size (cbm_canonical_path's floor). */ + CBM_CONFIG_EDIT_PATH_MAX = 4096, + CBM_CONFIG_EDIT_NAME_MAX = 256, +}; + +/* Pending user decision (#1954): must the resolved TARGET also stay under a + * registered root (1), or may it live anywhere the invoking user owns, such + * as a dotfiles checkout on an external volume (0, the default)? One switch. */ +#ifndef CBM_CONFIG_EDIT_TARGET_UNDER_ROOT +#define CBM_CONFIG_EDIT_TARGET_UNDER_ROOT 0 +#endif + +typedef enum { + CBM_CONFIG_EDIT_PATH_REFUSED = -1, /* a link that must not be followed */ + CBM_CONFIG_EDIT_PATH_DIRECT = 0, /* not a link (or absent): path is used as given */ + CBM_CONFIG_EDIT_PATH_FOLLOWED = 1, /* user-owned link inside a root: the target is used */ +} cbm_config_edit_path_status_t; + +typedef struct { + int status; /* cbm_config_edit_path_status_t */ + int dirfd; /* FOLLOWED: the target's parent directory, pinned; else -1 */ + int fd; /* FOLLOWED: the validated target, O_RDONLY|O_NOFOLLOW; else -1 */ + char base[CBM_CONFIG_EDIT_NAME_MAX]; /* FOLLOWED: the target's name inside dirfd */ + char path[CBM_CONFIG_EDIT_PATH_MAX]; /* the path the editor operates on */ +} cbm_config_edit_target_t; + +/* Opt-in scope (per thread): links under these canonical roots may be + * followed. Up to four roots; returns 0 when registered. Clear when the + * agent-configuration work is done. */ +int cbm_config_edit_path_follow_add_root(const char *root); +void cbm_config_edit_path_follow_clear(void); + +/* Classifies path and, for a followed link, pins its parent directory and + * validates the target. Returns the status; on REFUSED the target holds no + * descriptors and an empty path, and the editor fails closed as before. + * Always pair with cbm_config_edit_target_close(). */ +int cbm_config_edit_target_open(const char *path, cbm_config_edit_target_t *target); +void cbm_config_edit_target_close(cbm_config_edit_target_t *target); + +/* FOLLOWED targets only (return -1 with errno = EINVAL otherwise): + * create a temp file `name` beside the target (openat O_CREAT|O_EXCL| + * O_NOFOLLOW; EEXIST when the name is taken), publish it over the target with + * renameat on the pinned directory (which is then fsync'ed), or discard it. */ +int cbm_config_edit_target_create_temp(const cbm_config_edit_target_t *target, const char *name, + unsigned mode); +int cbm_config_edit_target_commit(const cbm_config_edit_target_t *target, const char *name); +int cbm_config_edit_target_unlink(const cbm_config_edit_target_t *target, const char *name); + +/* Observed-fact reason a symlink at path is refused, for diagnostics + * ("symlink owned by uid 0, not the invoking user (uid 501)", "dangling + * symlink", …). Writes "" and returns 0 when path is not a refused link, 1 + * when a reason was written. Never reports errno — it may be stale (#1537). */ +int cbm_config_edit_path_refusal(const char *path, char *out, size_t out_size); + +#ifdef CBM_CONFIG_EDIT_PATH_ENABLE_TEST_API +/* Test seams (POSIX; no-ops on Windows). A test cannot create a link owned by + * another user without root, so the foreign-owner and root-run refusals move + * the observer instead of the file; the identity skew makes the validated + * descriptor disagree with stat(realpath), the swapped-target refusal. */ +void cbm_config_edit_path_set_invoking_uid_for_test(unsigned uid, int set); +void cbm_config_edit_path_set_identity_skew_for_test(int set); +#endif + +#endif /* CBM_CONFIG_EDIT_PATH_H */ diff --git a/src/cli/config_json_like.c b/src/cli/config_json_like.c index dc16c3c5c..4164fcca1 100644 --- a/src/cli/config_json_like.c +++ b/src/cli/config_json_like.c @@ -8,6 +8,7 @@ */ #include "cli/config_json_like.h" +#include "cli/config_edit_path.h" #include "foundation/compat.h" #include "foundation/compat_fs.h" @@ -1429,11 +1430,24 @@ static int jl_read_file(const char *path, char **content_out, size_t *length_out #ifdef O_CLOEXEC flags |= O_CLOEXEC; #endif - int descriptor = open(path, flags); + /* A symlink the invoking user owns, inside an opted-in configuration + * root, is read through the descriptor the helper validated on the + * target's pinned parent directory (#1954); everything else is opened by + * name with O_NOFOLLOW exactly as before. */ + cbm_config_edit_target_t target; + if (cbm_config_edit_target_open(path, &target) < 0) { + return -1; + } + int descriptor = target.fd; + target.fd = -1; + if (target.status != CBM_CONFIG_EDIT_PATH_FOLLOWED) { + descriptor = open(target.path, flags); + } + cbm_config_edit_target_close(&target); if (descriptor < 0) { if (errno == ENOENT) { struct stat path_state; - if (lstat(path, &path_state) == 0 || errno != ENOENT) { + if (lstat(target.path, &path_state) == 0 || errno != ENOENT) { return -1; } *missing_out = true; @@ -1593,10 +1607,31 @@ static int jl_replace_atomic(const char *temp_path, const char *path, bool desti #endif } -static int jl_write_atomic(const char *path, const char *content, size_t length, - const char *expected_content, size_t expected_length, - const jl_file_snapshot_t *expected_snapshot) { - if (jl_ensure_parent(path) != 0) { +static const char *jl_temp_name(const char *temp_path) { + const char *slash = strrchr(temp_path, '/'); + return slash ? slash + 1 : temp_path; +} + +/* Drop a staged temp file: through the pinned parent for a followed link, + * by name otherwise. */ +static void jl_discard_temp(const cbm_config_edit_target_t *target, const char *temp_path) { + if (target->status == CBM_CONFIG_EDIT_PATH_FOLLOWED) { + (void)cbm_config_edit_target_unlink(target, jl_temp_name(temp_path)); + } else { + (void)cbm_unlink(temp_path); + } +} + +/* For a followed link (#1954) the temp file is created with openat() beside + * the target and published with renameat() on the pinned parent, so the link + * itself is never replaced; every pre-publish comparison runs against the + * resolved path. A direct path keeps the by-name sequence unchanged. */ +static int jl_write_atomic_at(const cbm_config_edit_target_t *target, const char *content, + size_t length, const char *expected_content, size_t expected_length, + const jl_file_snapshot_t *expected_snapshot) { + const char *path = target->path; + bool followed = target->status == CBM_CONFIG_EDIT_PATH_FOLLOWED; + if (!followed && jl_ensure_parent(path) != 0) { return -1; } size_t path_length = strlen(path); @@ -1631,13 +1666,15 @@ static int jl_write_atomic(const char *path, const char *content, size_t length, #ifdef O_CLOEXEC flags |= O_CLOEXEC; #endif - int descriptor = open(temp_path, flags, 0600); + int descriptor = + followed ? cbm_config_edit_target_create_temp(target, jl_temp_name(temp_path), 0600U) + : open(temp_path, flags, 0600); if (descriptor >= 0) { file = fdopen(descriptor, "wb"); if (!file) { int saved_error = errno; close(descriptor); - (void)cbm_unlink(temp_path); + jl_discard_temp(target, temp_path); errno = saved_error; } } @@ -1680,7 +1717,7 @@ static int jl_write_atomic(const char *path, const char *content, size_t length, failed = true; } if (failed) { - cbm_unlink(temp_path); + jl_discard_temp(target, temp_path); free(temp_path); return -1; } @@ -1692,7 +1729,7 @@ static int jl_write_atomic(const char *path, const char *content, size_t length, temp_missing || temp_length != length || (length != 0U && memcmp(temp_content, content, length) != 0)) { free(temp_content); - cbm_unlink(temp_path); + jl_discard_temp(target, temp_path); free(temp_path); return -1; } @@ -1703,7 +1740,7 @@ static int jl_write_atomic(const char *path, const char *content, size_t length, } #endif if (jl_snapshot_matches_path(path, expected_content, expected_length, expected_snapshot) != 0) { - cbm_unlink(temp_path); + jl_discard_temp(target, temp_path); free(temp_path); return -1; } @@ -1714,8 +1751,9 @@ static int jl_write_atomic(const char *path, const char *content, size_t length, #endif if (jl_snapshot_matches_path(path, expected_content, expected_length, expected_snapshot) != 0 || jl_snapshot_matches_path(temp_path, content, length, &temp_snapshot) != 0 || - jl_replace_atomic(temp_path, path, expected_snapshot->exists) != 0) { - cbm_unlink(temp_path); + (followed ? cbm_config_edit_target_commit(target, jl_temp_name(temp_path)) + : jl_replace_atomic(temp_path, path, expected_snapshot->exists)) != 0) { + jl_discard_temp(target, temp_path); free(temp_path); return -1; } @@ -1723,6 +1761,19 @@ static int jl_write_atomic(const char *path, const char *content, size_t length, return 0; } +static int jl_write_atomic(const char *requested_path, const char *content, size_t length, + const char *expected_content, size_t expected_length, + const jl_file_snapshot_t *expected_snapshot) { + cbm_config_edit_target_t target; + if (cbm_config_edit_target_open(requested_path, &target) < 0) { + return -1; + } + int result = jl_write_atomic_at(&target, content, length, expected_content, expected_length, + expected_snapshot); + cbm_config_edit_target_close(&target); + return result; +} + static int jl_decode_utf8(const unsigned char *text, size_t remaining, uint32_t *codepoint, size_t *byte_count) { if (remaining == 0U) { diff --git a/src/cli/config_text_edit.c b/src/cli/config_text_edit.c index cee66bdfe..ad728f506 100644 --- a/src/cli/config_text_edit.c +++ b/src/cli/config_text_edit.c @@ -3,6 +3,7 @@ */ #include "cli/config_text_edit.h" +#include "cli/config_edit_path.h" #include "foundation/compat.h" #include "foundation/compat_fs.h" @@ -442,13 +443,26 @@ static int text_read_file(const char *path, char **data_out, size_t *len_out, #ifdef O_CLOEXEC flags |= O_CLOEXEC; #endif - int descriptor = open(path, flags); + /* A symlink the invoking user owns, inside an opted-in configuration + * root, is read through the descriptor the helper validated on the + * target's pinned parent directory (#1954); everything else is opened by + * name with O_NOFOLLOW exactly as before. */ + cbm_config_edit_target_t target; + if (cbm_config_edit_target_open(path, &target) < 0) { + return TEXT_ERROR; + } + int descriptor = target.fd; + target.fd = -1; + if (target.status != CBM_CONFIG_EDIT_PATH_FOLLOWED) { + descriptor = open(target.path, flags); + } + cbm_config_edit_target_close(&target); if (descriptor < 0) { if (errno != ENOENT) { return TEXT_ERROR; } struct stat path_state; - if (lstat(path, &path_state) == 0 || errno != ENOENT) { + if (lstat(target.path, &path_state) == 0 || errno != ENOENT) { return TEXT_ERROR; } char *empty = (char *)calloc(1U, 1U); @@ -589,14 +603,35 @@ static int text_replace_file(const char *temp_path, const char *path, int destin #endif } -static int text_write_atomic_mode(const char *path, const char *new_data, size_t new_len, - const char *old_data, size_t old_len, - const text_file_snapshot_t *snapshot, int override_mode, - unsigned int requested_mode) { +static const char *text_temp_name(const char *temp_path) { + const char *slash = strrchr(temp_path, '/'); + return slash ? slash + 1 : temp_path; +} + +/* Drop a staged temp file: through the pinned parent for a followed link, + * by name otherwise. */ +static void text_discard_temp(const cbm_config_edit_target_t *target, const char *temp_path) { + if (target->status == CBM_CONFIG_EDIT_PATH_FOLLOWED) { + (void)cbm_config_edit_target_unlink(target, text_temp_name(temp_path)); + } else { + (void)cbm_unlink(temp_path); + } +} + +/* For a followed link (#1954) the temp file is created with openat() beside + * the target and published with renameat() on the pinned parent, so the link + * itself is never replaced; every pre-publish comparison runs against the + * resolved path. A direct path keeps the by-name sequence unchanged. */ +static int text_write_atomic_mode_at(const cbm_config_edit_target_t *target, const char *new_data, + size_t new_len, const char *old_data, size_t old_len, + const text_file_snapshot_t *snapshot, int override_mode, + unsigned int requested_mode) { if (new_len > TEXT_MAX_BYTES || old_len > TEXT_MAX_BYTES || !text_requested_mode_valid(override_mode, requested_mode)) { return TEXT_ERROR; } + const char *path = target->path; + bool followed = target->status == CBM_CONFIG_EDIT_PATH_FOLLOWED; int content_same = new_len == old_len && (new_len == 0U || memcmp(new_data, old_data, new_len) == 0); #ifndef _WIN32 @@ -643,13 +678,15 @@ static int text_write_atomic_mode(const char *path, const char *new_data, size_t #ifdef O_CLOEXEC flags |= O_CLOEXEC; #endif - int descriptor = open(temp_path, flags, 0600); + int descriptor = + followed ? cbm_config_edit_target_create_temp(target, text_temp_name(temp_path), 0600U) + : open(temp_path, flags, 0600); if (descriptor >= 0) { file = text_fdopen(descriptor, "wb"); if (!file) { int saved_error = errno; text_close(descriptor); - (void)cbm_unlink(temp_path); + text_discard_temp(target, temp_path); errno = saved_error; } } @@ -702,7 +739,7 @@ static int text_write_atomic_mode(const char *path, const char *new_data, size_t failed = 1; } if (failed) { - (void)cbm_unlink(temp_path); + text_discard_temp(target, temp_path); free(temp_path); return TEXT_ERROR; } @@ -718,7 +755,7 @@ static int text_write_atomic_mode(const char *path, const char *new_data, size_t !text_snapshot_publication_equal(&trusted_temp_snapshot, &temp_snapshot) || temp_len != new_len || (new_len != 0U && memcmp(temp_data, new_data, new_len) != 0)) { free(temp_data); - (void)cbm_unlink(temp_path); + text_discard_temp(target, temp_path); free(temp_path); return TEXT_ERROR; } @@ -730,7 +767,7 @@ static int text_write_atomic_mode(const char *path, const char *new_data, size_t } #endif if (text_snapshot_matches_path(path, old_data, old_len, snapshot) != TEXT_OK) { - (void)cbm_unlink(temp_path); + text_discard_temp(target, temp_path); free(temp_path); return TEXT_ERROR; } @@ -741,8 +778,9 @@ static int text_write_atomic_mode(const char *path, const char *new_data, size_t #endif if (text_snapshot_matches_path(path, old_data, old_len, snapshot) != TEXT_OK || text_snapshot_matches_path(temp_path, new_data, new_len, &temp_snapshot) != TEXT_OK || - text_replace_file(temp_path, path, snapshot->exists) != TEXT_OK) { - (void)cbm_unlink(temp_path); + (followed ? cbm_config_edit_target_commit(target, text_temp_name(temp_path)) + : text_replace_file(temp_path, path, snapshot->exists)) != TEXT_OK) { + text_discard_temp(target, temp_path); free(temp_path); return TEXT_ERROR; } @@ -750,6 +788,20 @@ static int text_write_atomic_mode(const char *path, const char *new_data, size_t return TEXT_OK; } +static int text_write_atomic_mode(const char *requested_path, const char *new_data, size_t new_len, + const char *old_data, size_t old_len, + const text_file_snapshot_t *snapshot, int override_mode, + unsigned int requested_mode) { + cbm_config_edit_target_t target; + if (cbm_config_edit_target_open(requested_path, &target) < 0) { + return TEXT_ERROR; + } + int result = text_write_atomic_mode_at(&target, new_data, new_len, old_data, old_len, snapshot, + override_mode, requested_mode); + cbm_config_edit_target_close(&target); + return result; +} + static int text_write_atomic(const char *path, const char *new_data, size_t new_len, const char *old_data, size_t old_len, const text_file_snapshot_t *snapshot) { diff --git a/src/cli/config_toml_edit.c b/src/cli/config_toml_edit.c index ef73c66fc..745bdda0d 100644 --- a/src/cli/config_toml_edit.c +++ b/src/cli/config_toml_edit.c @@ -3,6 +3,7 @@ */ #include "cli/config_toml_edit.h" +#include "cli/config_edit_path.h" #include "foundation/compat.h" #include "foundation/compat_fs.h" @@ -465,13 +466,26 @@ static int toml_read_file(const char *path, char **out_data, size_t *out_len, #ifdef O_CLOEXEC flags |= O_CLOEXEC; #endif - int fd = open(path, flags); + /* A symlink the invoking user owns, inside an opted-in configuration + * root, is read through the descriptor the helper validated on the + * target's pinned parent directory (#1954); everything else is opened by + * name with O_NOFOLLOW exactly as before. */ + cbm_config_edit_target_t target; + if (cbm_config_edit_target_open(path, &target) < 0) { + return TOML_EDIT_ERR; + } + int fd = target.fd; + target.fd = -1; + if (target.status != CBM_CONFIG_EDIT_PATH_FOLLOWED) { + fd = open(target.path, flags); + } + cbm_config_edit_target_close(&target); if (fd < 0) { if (errno != ENOENT) { return TOML_EDIT_ERR; } struct stat path_state; - if (lstat(path, &path_state) == 0 || errno != ENOENT) { + if (lstat(target.path, &path_state) == 0 || errno != ENOENT) { return TOML_EDIT_ERR; } char *empty = (char *)malloc(1U); @@ -612,28 +626,65 @@ static int toml_replace_atomic(const char *temp_path, const char *path, int exis #endif } -static int toml_write_atomic(const char *path, const char *old_data, size_t old_len, - const char *new_data, size_t new_len, - const toml_file_snapshot_t *snapshot) { +static const char *toml_temp_name(const char *temp_path) { + const char *slash = strrchr(temp_path, '/'); + return slash ? slash + 1 : temp_path; +} + +/* Drop a staged temp file: through the pinned parent for a followed link, + * by name otherwise. */ +static void toml_discard_temp(const cbm_config_edit_target_t *target, const char *temp_path) { + if (target->status == CBM_CONFIG_EDIT_PATH_FOLLOWED) { + (void)cbm_config_edit_target_unlink(target, toml_temp_name(temp_path)); + } else { + (void)cbm_unlink(temp_path); + } +} + +/* For a followed link (#1954) the temp file is created with openat() beside + * the target and published with renameat() on the pinned parent, so the link + * itself is never replaced; every pre-publish comparison runs against the + * resolved path. A direct path keeps the mkstemp + rename sequence unchanged. */ +static int toml_write_atomic_at(const cbm_config_edit_target_t *target, const char *old_data, + size_t old_len, const char *new_data, size_t new_len, + const toml_file_snapshot_t *snapshot) { if (old_len > TOML_EDIT_MAX_BYTES || new_len > TOML_EDIT_MAX_BYTES) { return TOML_EDIT_ERR; } + const char *path = target->path; + bool followed = target->status == CBM_CONFIG_EDIT_PATH_FOLLOWED; if (old_len == new_len && (old_len == 0 || memcmp(old_data, new_data, old_len) == 0)) { return TOML_EDIT_OK; } size_t path_len = strlen(path); static const char suffix[] = ".XXXXXX"; - if (path_len > SIZE_MAX - sizeof(suffix)) { + enum { TOML_TEMP_SUFFIX_SPACE = 48, TOML_TEMP_ATTEMPTS = 64 }; + if (path_len > SIZE_MAX - TOML_TEMP_SUFFIX_SPACE) { return TOML_EDIT_ERR; } - char *temp_path = (char *)malloc(path_len + sizeof(suffix)); + size_t temp_capacity = path_len + TOML_TEMP_SUFFIX_SPACE; + char *temp_path = (char *)malloc(temp_capacity); if (!temp_path) { return TOML_EDIT_ERR; } - memcpy(temp_path, path, path_len); - memcpy(temp_path + path_len, suffix, sizeof(suffix)); - int fd = cbm_mkstemp(temp_path); + int fd = -1; + if (followed) { + for (unsigned attempt = 0U; attempt < TOML_TEMP_ATTEMPTS && fd < 0; attempt++) { + int written = snprintf(temp_path, temp_capacity, "%s.cbm-toml-%u.tmp", path, attempt); + if (written < 0 || (size_t)written >= temp_capacity) { + break; + } + fd = cbm_config_edit_target_create_temp(target, toml_temp_name(temp_path), 0600U); + if (fd < 0 && errno != EEXIST) { + break; + } + } + } else { + memcpy(temp_path, path, path_len); + memcpy(temp_path + path_len, suffix, sizeof(suffix)); + fd = cbm_mkstemp(temp_path); + } if (fd < 0) { free(temp_path); return TOML_EDIT_ERR; @@ -641,7 +692,7 @@ static int toml_write_atomic(const char *path, const char *old_data, size_t old_ FILE *file = toml_fdopen(fd, "wb"); if (!file) { (void)toml_close(fd); - (void)cbm_unlink(temp_path); + toml_discard_temp(target, temp_path); free(temp_path); return TOML_EDIT_ERR; } @@ -667,7 +718,7 @@ static int toml_write_atomic(const char *path, const char *old_data, size_t old_ failed = 1; } if (failed) { - (void)cbm_unlink(temp_path); + toml_discard_temp(target, temp_path); free(temp_path); return TOML_EDIT_ERR; } @@ -678,7 +729,7 @@ static int toml_write_atomic(const char *path, const char *old_data, size_t old_ !temp_snapshot.exists || temp_len != new_len || (new_len != 0U && memcmp(temp_data, new_data, new_len) != 0)) { free(temp_data); - (void)cbm_unlink(temp_path); + toml_discard_temp(target, temp_path); free(temp_path); return TOML_EDIT_ERR; } @@ -689,7 +740,7 @@ static int toml_write_atomic(const char *path, const char *old_data, size_t old_ } #endif if (toml_snapshot_matches_path(path, old_data, old_len, snapshot) != TOML_EDIT_OK) { - (void)cbm_unlink(temp_path); + toml_discard_temp(target, temp_path); free(temp_path); return TOML_EDIT_ERR; } @@ -700,8 +751,9 @@ static int toml_write_atomic(const char *path, const char *old_data, size_t old_ #endif if (toml_snapshot_matches_path(path, old_data, old_len, snapshot) != TOML_EDIT_OK || toml_snapshot_matches_path(temp_path, new_data, new_len, &temp_snapshot) != TOML_EDIT_OK || - toml_replace_atomic(temp_path, path, snapshot->exists) != TOML_EDIT_OK) { - (void)cbm_unlink(temp_path); + (followed ? cbm_config_edit_target_commit(target, toml_temp_name(temp_path)) + : toml_replace_atomic(temp_path, path, snapshot->exists)) != TOML_EDIT_OK) { + toml_discard_temp(target, temp_path); free(temp_path); return TOML_EDIT_ERR; } @@ -709,6 +761,18 @@ static int toml_write_atomic(const char *path, const char *old_data, size_t old_ return TOML_EDIT_OK; } +static int toml_write_atomic(const char *requested_path, const char *old_data, size_t old_len, + const char *new_data, size_t new_len, + const toml_file_snapshot_t *snapshot) { + cbm_config_edit_target_t target; + if (cbm_config_edit_target_open(requested_path, &target) < 0) { + return TOML_EDIT_ERR; + } + int result = toml_write_atomic_at(&target, old_data, old_len, new_data, new_len, snapshot); + cbm_config_edit_target_close(&target); + return result; +} + #ifdef CBM_TOML_EDIT_ENABLE_TEST_API void cbm_toml_set_precommit_hook_for_testing(cbm_toml_precommit_test_hook_t hook, void *context) { toml_precommit_test_hook = hook; diff --git a/src/cli/config_yaml_edit.c b/src/cli/config_yaml_edit.c index e084b59c1..e64827325 100644 --- a/src/cli/config_yaml_edit.c +++ b/src/cli/config_yaml_edit.c @@ -7,6 +7,7 @@ */ #include "cli/config_yaml_edit.h" +#include "cli/config_edit_path.h" #include "foundation/compat.h" #include "foundation/compat_fs.h" @@ -694,13 +695,26 @@ static int yaml_read_file(const char *path, char **out_data, size_t *out_len, #ifdef O_CLOEXEC flags |= O_CLOEXEC; #endif - int descriptor = open(path, flags); + /* A symlink the invoking user owns, inside an opted-in configuration + * root, is read through the descriptor the helper validated on the + * target's pinned parent directory (#1954); everything else is opened by + * name with O_NOFOLLOW exactly as before. */ + cbm_config_edit_target_t target; + if (cbm_config_edit_target_open(path, &target) < 0) { + return YAML_ERROR; + } + int descriptor = target.fd; + target.fd = -1; + if (target.status != CBM_CONFIG_EDIT_PATH_FOLLOWED) { + descriptor = open(target.path, flags); + } + cbm_config_edit_target_close(&target); if (descriptor < 0) { if (errno != ENOENT) { return YAML_ERROR; } struct stat path_state; - if (lstat(path, &path_state) == 0 || errno != ENOENT) { + if (lstat(target.path, &path_state) == 0 || errno != ENOENT) { return YAML_ERROR; } char *empty = (char *)calloc(YAML_UNIT, YAML_UNIT); @@ -869,9 +883,30 @@ static int yaml_replace_file(const char *temp_path, const char *path, bool desti #endif } -static int yaml_write_atomic(const char *path, const char *data, size_t len, - const char *expected_data, size_t expected_len, - const yaml_file_snapshot_t *expected_snapshot) { +static const char *yaml_temp_name(const char *temp_path) { + const char *slash = strrchr(temp_path, '/'); + return slash ? slash + 1 : temp_path; +} + +/* Drop a staged temp file: through the pinned parent for a followed link, + * by name otherwise. */ +static void yaml_discard_temp(const cbm_config_edit_target_t *target, const char *temp_path) { + if (target->status == CBM_CONFIG_EDIT_PATH_FOLLOWED) { + (void)cbm_config_edit_target_unlink(target, yaml_temp_name(temp_path)); + } else { + (void)cbm_unlink(temp_path); + } +} + +/* For a followed link (#1954) the temp file is created with openat() beside + * the target and published with renameat() on the pinned parent, so the link + * itself is never replaced; every pre-publish comparison runs against the + * resolved path. A direct path keeps the by-name sequence unchanged. */ +static int yaml_write_atomic_at(const cbm_config_edit_target_t *target, const char *data, + size_t len, const char *expected_data, size_t expected_len, + const yaml_file_snapshot_t *expected_snapshot) { + const char *path = target->path; + bool followed = target->status == CBM_CONFIG_EDIT_PATH_FOLLOWED; size_t path_len = 0U; if (yaml_bounded_strlen(path, YAML_OUTPUT_MAX, &path_len) != 0 || path_len > SIZE_MAX - YAML_TMP_SUFFIX_MAX - YAML_UNIT) { @@ -901,12 +936,14 @@ static int yaml_write_atomic(const char *path, const char *data, size_t len, #ifdef O_CLOEXEC flags |= O_CLOEXEC; #endif - int descriptor = open(temp_path, flags, YAML_NEW_FILE_MODE); + int descriptor = followed ? cbm_config_edit_target_create_temp( + target, yaml_temp_name(temp_path), YAML_NEW_FILE_MODE) + : open(temp_path, flags, YAML_NEW_FILE_MODE); if (descriptor >= 0) { file = fdopen(descriptor, "wb"); if (!file) { (void)close(descriptor); - (void)cbm_unlink(temp_path); + yaml_discard_temp(target, temp_path); free(temp_path); return YAML_ERROR; } @@ -947,7 +984,7 @@ static int yaml_write_atomic(const char *path, const char *data, size_t len, failed = true; } if (failed) { - (void)cbm_unlink(temp_path); + yaml_discard_temp(target, temp_path); free(temp_path); return YAML_ERROR; } @@ -958,7 +995,7 @@ static int yaml_write_atomic(const char *path, const char *data, size_t len, !temp_snapshot.exists || temp_len != len || (len != 0U && memcmp(temp_data, data, len) != 0)) { free(temp_data); - (void)cbm_unlink(temp_path); + yaml_discard_temp(target, temp_path); free(temp_path); return YAML_ERROR; } @@ -970,7 +1007,7 @@ static int yaml_write_atomic(const char *path, const char *data, size_t len, } #endif if (yaml_snapshot_matches_path(path, expected_data, expected_len, expected_snapshot) != 0) { - (void)cbm_unlink(temp_path); + yaml_discard_temp(target, temp_path); free(temp_path); return YAML_ERROR; } @@ -981,8 +1018,9 @@ static int yaml_write_atomic(const char *path, const char *data, size_t len, #endif if (yaml_snapshot_matches_path(path, expected_data, expected_len, expected_snapshot) != 0 || yaml_snapshot_matches_path(temp_path, data, len, &temp_snapshot) != 0 || - yaml_replace_file(temp_path, path, expected_snapshot->exists) != 0) { - (void)cbm_unlink(temp_path); + (followed ? cbm_config_edit_target_commit(target, yaml_temp_name(temp_path)) + : yaml_replace_file(temp_path, path, expected_snapshot->exists)) != 0) { + yaml_discard_temp(target, temp_path); free(temp_path); return YAML_ERROR; } @@ -990,6 +1028,19 @@ static int yaml_write_atomic(const char *path, const char *data, size_t len, return 0; } +static int yaml_write_atomic(const char *requested_path, const char *data, size_t len, + const char *expected_data, size_t expected_len, + const yaml_file_snapshot_t *expected_snapshot) { + cbm_config_edit_target_t target; + if (cbm_config_edit_target_open(requested_path, &target) < 0) { + return YAML_ERROR; + } + int result = + yaml_write_atomic_at(&target, data, len, expected_data, expected_len, expected_snapshot); + cbm_config_edit_target_close(&target); + return result; +} + #ifdef CBM_YAML_ENABLE_TEST_API void cbm_yaml_set_precommit_hook_for_testing(cbm_yaml_precommit_test_hook_t hook, void *context) { yaml_precommit_test_hook = hook; diff --git a/tests/test_cli.c b/tests/test_cli.c index aab397da4..3be2d7403 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -2402,6 +2402,99 @@ TEST(cli_uninstall_removes_binary_and_index_when_agent_config_cleanup_fails) { ASSERT_TRUE(failure_named); PASS(); } + +/* #1954 end to end: ~/.cursor/mcp.json is a user-owned symlink into a + * dotfiles checkout. The uninstall command opts in to following it, removes + * our entry THROUGH the link, leaves the link pointing where it did, and + * exits 0 with the binary and the index gone. */ +TEST(cli_uninstall_cleans_user_owned_symlinked_config) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-uninstall-symlinked-XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) { + FAIL("cbm_mkdtemp failed"); + } + char *old_home = NULL; + char *old_cache = NULL; + cli_activation_save_env(&old_home, &old_cache); + cbm_setenv("HOME", tmpdir, 1); + char *old_path = save_test_env("PATH"); + cbm_setenv("PATH", tmpdir, 1); + + char cache_dir[512]; + char index_path[640]; + snprintf(cache_dir, sizeof(cache_dir), "%s/cache", tmpdir); + cbm_setenv("CBM_CACHE_DIR", cache_dir, 1); + test_mkdirp(cache_dir); + snprintf(index_path, sizeof(index_path), "%s/project.db", cache_dir); + write_test_file(index_path, "index goes with the uninstall"); + + char bin_dir[512]; + char bin_target[640]; + snprintf(bin_dir, sizeof(bin_dir), "%s/.local/bin", tmpdir); + test_mkdirp(bin_dir); + snprintf(bin_target, sizeof(bin_target), "%s/codebase-memory-mcp", bin_dir); + write_test_file(bin_target, "binary goes with the uninstall"); + + char cursor_dir[512]; + char cursor_config[640]; + char dotfiles_dir[512]; + char dotfiles_config[640]; + snprintf(cursor_dir, sizeof(cursor_dir), "%s/.cursor", tmpdir); + snprintf(cursor_config, sizeof(cursor_config), "%s/mcp.json", cursor_dir); + snprintf(dotfiles_dir, sizeof(dotfiles_dir), "%s/.dotfiles/cursor", tmpdir); + snprintf(dotfiles_config, sizeof(dotfiles_config), "%s/mcp.json", dotfiles_dir); + test_mkdirp(cursor_dir); + test_mkdirp(dotfiles_dir); + write_test_file(dotfiles_config, + "{\n \"mcpServers\": {\n \"keep\": {\"command\": \"x\"}\n }\n}\n"); + int installed = cbm_install_editor_mcp(bin_target, dotfiles_config); + if (symlink(dotfiles_config, cursor_config) != 0) { + FAIL("symlink failed"); + } + + cli_activation_fake_t fake = {.mutation_reserve_result = 1}; + cbm_cli_activation_ops_t ops = cli_activation_fake_ops(&fake); + cbm_cli_set_activation_ops_for_test(&ops); + cli_fd_capture_t out_capture; + cli_fd_capture_t err_capture; + cli_fd_capture_begin(&out_capture, stdout, STDOUT_FILENO); + cli_fd_capture_begin(&err_capture, stderr, STDERR_FILENO); + char *argv[] = {"--yes"}; + int rc = cli_test_cmd_uninstall(1, argv); + char *err_text = cli_fd_capture_end(&err_capture); + char *out_text = cli_fd_capture_end(&out_capture); + cbm_cli_set_activation_ops_for_test(NULL); + cbm_set_auto_answer_for_test(0); + + struct stat state; + bool binary_gone = lstat(bin_target, &state) != 0 && errno == ENOENT; + bool index_gone = lstat(index_path, &state) != 0 && errno == ENOENT; + bool link_intact = lstat(cursor_config, &state) == 0 && S_ISLNK(state.st_mode); + char link_value[640] = {0}; + ssize_t link_length = readlink(cursor_config, link_value, sizeof(link_value) - 1U); + bool link_same = link_length > 0 && strcmp(link_value, dotfiles_config) == 0; + char *after = read_test_file_alloc(dotfiles_config); + bool entry_removed = + after && !strstr(after, "codebase-memory-mcp") && strstr(after, "\"keep\""); + bool no_error = err_text && !strstr(err_text, "error:"); + + cli_activation_restore_env(old_home, old_cache); + restore_test_env("PATH", old_path); + test_rmdir_r(tmpdir); + free(after); + free(err_text); + free(out_text); + + ASSERT_EQ(installed, 0); + ASSERT_EQ(rc, 0); + ASSERT_TRUE(binary_gone); + ASSERT_TRUE(index_gone); + ASSERT_TRUE(link_intact); + ASSERT_TRUE(link_same); + ASSERT_TRUE(entry_removed); + ASSERT_TRUE(no_error); + PASS(); +} #endif TEST(cli_activation_guard_is_bypassed_for_dry_run_and_plan) { @@ -14680,6 +14773,7 @@ SUITE(cli) { RUN_TEST(cli_uninstall_preserves_binary_and_index_when_cohort_does_not_drain); #ifndef _WIN32 RUN_TEST(cli_uninstall_removes_binary_and_index_when_agent_config_cleanup_fails); + RUN_TEST(cli_uninstall_cleans_user_owned_symlinked_config); #endif RUN_TEST(cli_activation_guard_is_bypassed_for_dry_run_and_plan); #ifdef _WIN32 diff --git a/tests/test_config_json_like.c b/tests/test_config_json_like.c index c6f29a387..08a070378 100644 --- a/tests/test_config_json_like.c +++ b/tests/test_config_json_like.c @@ -8,6 +8,8 @@ #define CBM_JSON_LIKE_ENABLE_TEST_API 1 #include "../src/cli/config_json_like.h" +#define CBM_CONFIG_EDIT_PATH_ENABLE_TEST_API 1 +#include "../src/cli/config_edit_path.h" #include "../src/foundation/compat.h" #include "../src/foundation/compat_fs.h" @@ -281,7 +283,28 @@ TEST(config_json_like_rejects_non_regular_path) { } #ifndef _WIN32 -TEST(config_json_like_rejects_symlink_without_touching_target) { +/* Any editor temp file left anywhere in the fixture directory, whatever + * document it was staged for: a link's target lives in the same directory. */ +static size_t jl_stray_temp_count(const jl_fixture_t *fixture) { + cbm_dir_t *directory = cbm_opendir(fixture->directory); + if (!directory) { + return SIZE_MAX; + } + size_t count = 0U; + cbm_dirent_t *entry = NULL; + while ((entry = cbm_readdir(directory)) != NULL) { + if (strstr(entry->name, ".cbm.tmp.")) { + count++; + } + } + cbm_closedir(directory); + return count; +} + +/* A link owned by someone else is never followed. A test cannot create a + * foreign-owned link without root, so the observer moves instead (test seam): + * the refusal touches neither the link nor its target and stages no temp file. */ +TEST(config_json_like_rejects_foreign_owned_symlink_without_touching_target) { jl_fixture_t fixture; ASSERT_EQ(jl_fixture_open(&fixture), 0); char target[sizeof(fixture.path) + 32U]; @@ -289,9 +312,18 @@ TEST(config_json_like_rejects_symlink_without_touching_target) { const char *original = "{\"target\":true}\n"; ASSERT_EQ(jl_write(target, original), 0); ASSERT_EQ(symlink(target, fixture.path), 0); + ASSERT_EQ(cbm_config_edit_path_follow_add_root(fixture.directory), 0); + cbm_config_edit_path_set_invoking_uid_for_test((unsigned)geteuid() + 1U, 1); const char *root[] = {NULL}; - ASSERT_EQ(cbm_json_like_upsert_entry(fixture.path, root, 0U, "owned", "true"), -1); + int upsert_rc = cbm_json_like_upsert_entry(fixture.path, root, 0U, "owned", "true"); + char reason[256]; + int refused = cbm_config_edit_path_refusal(fixture.path, reason, sizeof(reason)); + cbm_config_edit_path_set_invoking_uid_for_test(0U, 0); + cbm_config_edit_path_follow_clear(); + ASSERT_EQ(upsert_rc, -1); + ASSERT_EQ(refused, 1); + ASSERT_NOT_NULL(strstr(reason, "not the invoking user")); struct stat link_state; ASSERT_EQ(lstat(fixture.path, &link_state), 0); ASSERT(S_ISLNK(link_state.st_mode)); @@ -299,7 +331,7 @@ TEST(config_json_like_rejects_symlink_without_touching_target) { ASSERT_NOT_NULL(content); ASSERT_STR_EQ(content, original); free(content); - ASSERT_EQ(jl_temp_file_count(&fixture), 0U); + ASSERT_EQ(jl_stray_temp_count(&fixture), 0U); ASSERT_EQ(cbm_unlink(fixture.path), 0); ASSERT_EQ(cbm_unlink(target), 0); @@ -307,6 +339,350 @@ TEST(config_json_like_rejects_symlink_without_touching_target) { PASS(); } +/* A root process never follows a link a non-root user owns: a user-planted + * link must not redirect a privileged writer. A real root run (CI containers) + * demotes the link to an unprivileged owner first; elsewhere the observer moves. */ +TEST(config_json_like_root_refuses_user_owned_symlink) { + jl_fixture_t fixture; + ASSERT_EQ(jl_fixture_open(&fixture), 0); + char target[sizeof(fixture.path) + 32U]; + ASSERT(snprintf(target, sizeof(target), "%s/target.json", fixture.directory) > 0); + const char *original = "{\"target\":true}\n"; + ASSERT_EQ(jl_write(target, original), 0); + ASSERT_EQ(symlink(target, fixture.path), 0); + if (geteuid() == 0 && lchown(fixture.path, 65534, 65534) != 0) { + (void)cbm_unlink(fixture.path); + (void)cbm_unlink(target); + jl_fixture_close(&fixture); + FAIL("lchown to an unprivileged owner failed"); + } + ASSERT_EQ(cbm_config_edit_path_follow_add_root(fixture.directory), 0); + cbm_config_edit_path_set_invoking_uid_for_test(0U, 1); + + const char *root[] = {NULL}; + int upsert_rc = cbm_json_like_upsert_entry(fixture.path, root, 0U, "owned", "true"); + char reason[256]; + int refused = cbm_config_edit_path_refusal(fixture.path, reason, sizeof(reason)); + cbm_config_edit_path_set_invoking_uid_for_test(0U, 0); + cbm_config_edit_path_follow_clear(); + ASSERT_EQ(upsert_rc, -1); + ASSERT_EQ(refused, 1); + ASSERT_NOT_NULL(strstr(reason, "root process")); + struct stat link_state; + ASSERT_EQ(lstat(fixture.path, &link_state), 0); + ASSERT(S_ISLNK(link_state.st_mode)); + char *content = jl_read(target); + ASSERT_NOT_NULL(content); + ASSERT_STR_EQ(content, original); + free(content); + ASSERT_EQ(jl_stray_temp_count(&fixture), 0U); + + ASSERT_EQ(cbm_unlink(fixture.path), 0); + ASSERT_EQ(cbm_unlink(target), 0); + jl_fixture_close(&fixture); + PASS(); +} + +TEST(config_json_like_refuses_dangling_symlink) { + jl_fixture_t fixture; + ASSERT_EQ(jl_fixture_open(&fixture), 0); + char missing[sizeof(fixture.path) + 32U]; + ASSERT(snprintf(missing, sizeof(missing), "%s/missing.json", fixture.directory) > 0); + ASSERT_EQ(symlink(missing, fixture.path), 0); + ASSERT_EQ(cbm_config_edit_path_follow_add_root(fixture.directory), 0); + + const char *root[] = {NULL}; + int upsert_rc = cbm_json_like_upsert_entry(fixture.path, root, 0U, "owned", "true"); + char *document = NULL; + size_t length = 0U; + int read_rc = cbm_json_like_read_document(fixture.path, &document, &length); + free(document); + char reason[256]; + int refused = cbm_config_edit_path_refusal(fixture.path, reason, sizeof(reason)); + cbm_config_edit_path_follow_clear(); + ASSERT_EQ(upsert_rc, -1); + ASSERT_EQ(read_rc, -1); + ASSERT_EQ(refused, 1); + ASSERT_NOT_NULL(strstr(reason, "dangling")); + struct stat state; + ASSERT_EQ(lstat(fixture.path, &state), 0); + ASSERT(S_ISLNK(state.st_mode)); + ASSERT(lstat(missing, &state) != 0); + ASSERT_EQ(jl_stray_temp_count(&fixture), 0U); + + ASSERT_EQ(cbm_unlink(fixture.path), 0); + jl_fixture_close(&fixture); + PASS(); +} + +TEST(config_json_like_refuses_symlink_to_directory) { + jl_fixture_t fixture; + ASSERT_EQ(jl_fixture_open(&fixture), 0); + char sub[sizeof(fixture.path) + 32U]; + ASSERT(snprintf(sub, sizeof(sub), "%s/sub", fixture.directory) > 0); + ASSERT_EQ(cbm_mkdir(sub), 0); + ASSERT_EQ(symlink(sub, fixture.path), 0); + ASSERT_EQ(cbm_config_edit_path_follow_add_root(fixture.directory), 0); + + const char *root[] = {NULL}; + int upsert_rc = cbm_json_like_upsert_entry(fixture.path, root, 0U, "owned", "true"); + char reason[256]; + int refused = cbm_config_edit_path_refusal(fixture.path, reason, sizeof(reason)); + cbm_config_edit_path_follow_clear(); + ASSERT_EQ(upsert_rc, -1); + ASSERT_EQ(refused, 1); + ASSERT_NOT_NULL(strstr(reason, "directory")); + struct stat state; + ASSERT_EQ(lstat(fixture.path, &state), 0); + ASSERT(S_ISLNK(state.st_mode)); + ASSERT_EQ(jl_stray_temp_count(&fixture), 0U); + + ASSERT_EQ(cbm_unlink(fixture.path), 0); + ASSERT_EQ(cbm_rmdir(sub), 0); + jl_fixture_close(&fixture); + PASS(); +} + +/* Without the agent-config writers' opt-in, a same-owner link is still + * refused: the follow is never a property of every editor caller. */ +TEST(config_json_like_refuses_symlink_without_opt_in) { + jl_fixture_t fixture; + ASSERT_EQ(jl_fixture_open(&fixture), 0); + char target[sizeof(fixture.path) + 32U]; + ASSERT(snprintf(target, sizeof(target), "%s/target.json", fixture.directory) > 0); + const char *original = "{\"target\":true}\n"; + ASSERT_EQ(jl_write(target, original), 0); + ASSERT_EQ(symlink(target, fixture.path), 0); + cbm_config_edit_path_follow_clear(); + + const char *root[] = {NULL}; + ASSERT_EQ(cbm_json_like_upsert_entry(fixture.path, root, 0U, "owned", "true"), -1); + char reason[256]; + ASSERT_EQ(cbm_config_edit_path_refusal(fixture.path, reason, sizeof(reason)), 1); + ASSERT_NOT_NULL(strstr(reason, "not followed by this command")); + + /* An opt-in for a different root does not cover this link either. */ + char elsewhere[sizeof(fixture.path) + 32U]; + ASSERT(snprintf(elsewhere, sizeof(elsewhere), "%s/elsewhere", fixture.directory) > 0); + ASSERT_EQ(cbm_mkdir(elsewhere), 0); + ASSERT_EQ(cbm_config_edit_path_follow_add_root(elsewhere), 0); + int outside_rc = cbm_json_like_upsert_entry(fixture.path, root, 0U, "owned", "true"); + int outside_refused = cbm_config_edit_path_refusal(fixture.path, reason, sizeof(reason)); + cbm_config_edit_path_follow_clear(); + ASSERT_EQ(outside_rc, -1); + ASSERT_EQ(outside_refused, 1); + ASSERT_NOT_NULL(strstr(reason, "outside the opted-in configuration roots")); + + struct stat link_state; + ASSERT_EQ(lstat(fixture.path, &link_state), 0); + ASSERT(S_ISLNK(link_state.st_mode)); + char *content = jl_read(target); + ASSERT_NOT_NULL(content); + ASSERT_STR_EQ(content, original); + free(content); + ASSERT_EQ(jl_stray_temp_count(&fixture), 0U); + + ASSERT_EQ(cbm_rmdir(elsewhere), 0); + ASSERT_EQ(cbm_unlink(fixture.path), 0); + ASSERT_EQ(cbm_unlink(target), 0); + jl_fixture_close(&fixture); + PASS(); +} + +/* The validated descriptor must be the very file stat(realpath) saw: a + * target swapped between the two checks (seam) is refused, untouched. */ +TEST(config_json_like_refuses_symlink_when_target_identity_skews) { + jl_fixture_t fixture; + ASSERT_EQ(jl_fixture_open(&fixture), 0); + char target[sizeof(fixture.path) + 32U]; + ASSERT(snprintf(target, sizeof(target), "%s/target.json", fixture.directory) > 0); + const char *original = "{\"target\":true}\n"; + ASSERT_EQ(jl_write(target, original), 0); + ASSERT_EQ(symlink(target, fixture.path), 0); + ASSERT_EQ(cbm_config_edit_path_follow_add_root(fixture.directory), 0); + cbm_config_edit_path_set_identity_skew_for_test(1); + + const char *root[] = {NULL}; + int upsert_rc = cbm_json_like_upsert_entry(fixture.path, root, 0U, "owned", "true"); + char reason[256]; + int refused = cbm_config_edit_path_refusal(fixture.path, reason, sizeof(reason)); + cbm_config_edit_path_set_identity_skew_for_test(0); + cbm_config_edit_path_follow_clear(); + ASSERT_EQ(upsert_rc, -1); + ASSERT_EQ(refused, 1); + ASSERT_NOT_NULL(strstr(reason, "changed while it was being checked")); + char *content = jl_read(target); + ASSERT_NOT_NULL(content); + ASSERT_STR_EQ(content, original); + free(content); + ASSERT_EQ(jl_stray_temp_count(&fixture), 0U); + + ASSERT_EQ(cbm_unlink(fixture.path), 0); + ASSERT_EQ(cbm_unlink(target), 0); + jl_fixture_close(&fixture); + PASS(); +} + +/* The target's parent directory is pinned and must not be writable by others + * (a sticky directory is fine): otherwise anyone could stage a swap there. */ +TEST(config_json_like_refuses_symlink_with_world_writable_parent) { + jl_fixture_t fixture; + ASSERT_EQ(jl_fixture_open(&fixture), 0); + char dotfiles[sizeof(fixture.path) + 32U]; + char target[sizeof(fixture.path) + 64U]; + ASSERT(snprintf(dotfiles, sizeof(dotfiles), "%s/dotfiles", fixture.directory) > 0); + ASSERT(snprintf(target, sizeof(target), "%s/target.json", dotfiles) > 0); + ASSERT_EQ(cbm_mkdir(dotfiles), 0); + const char *original = "{\"target\":true}\n"; + ASSERT_EQ(jl_write(target, original), 0); + ASSERT_EQ(symlink(target, fixture.path), 0); + ASSERT_EQ(chmod(dotfiles, 0777), 0); + ASSERT_EQ(cbm_config_edit_path_follow_add_root(fixture.directory), 0); + + const char *root[] = {NULL}; + int loose_rc = cbm_json_like_upsert_entry(fixture.path, root, 0U, "owned", "true"); + char reason[256]; + int loose_refused = cbm_config_edit_path_refusal(fixture.path, reason, sizeof(reason)); + ASSERT_EQ(chmod(dotfiles, 0755), 0); + int tight_rc = cbm_json_like_upsert_entry(fixture.path, root, 0U, "owned", "true"); + cbm_config_edit_path_follow_clear(); + ASSERT_EQ(loose_rc, -1); + ASSERT_EQ(loose_refused, 1); + ASSERT_NOT_NULL(strstr(reason, "writable by others")); + ASSERT_EQ(tight_rc, 0); + char *content = jl_read(target); + ASSERT_NOT_NULL(content); + ASSERT_NOT_NULL(strstr(content, "\"owned\"")); + free(content); + + ASSERT_EQ(cbm_unlink(fixture.path), 0); + ASSERT_EQ(cbm_unlink(target), 0); + ASSERT_EQ(cbm_rmdir(dotfiles), 0); + jl_fixture_close(&fixture); + PASS(); +} + +typedef struct { + char seen_path[640]; + size_t temps_beside_target; + size_t calls; +} jl_follow_hook_state_t; + +/* Pre-commit hook: the editor hands over the path it is about to publish — + * for a followed link that must be the RESOLVED target, with the staged + * temp file sitting right beside it, never beside the link. */ +static void jl_follow_precommit_hook(const char *path, void *context) { + jl_follow_hook_state_t *state = context; + state->calls++; + (void)snprintf(state->seen_path, sizeof(state->seen_path), "%s", path); + char directory[640]; + (void)snprintf(directory, sizeof(directory), "%s", path); + char *slash = strrchr(directory, '/'); + if (!slash) { + return; + } + const char *base = slash + 1; + char prefix[320]; + (void)snprintf(prefix, sizeof(prefix), "%s.cbm.tmp.", base); + *slash = '\0'; + cbm_dir_t *handle = cbm_opendir(directory); + if (!handle) { + return; + } + cbm_dirent_t *entry = NULL; + while ((entry = cbm_readdir(handle)) != NULL) { + if (strncmp(entry->name, prefix, strlen(prefix)) == 0) { + state->temps_beside_target++; + } + } + cbm_closedir(handle); +} + +/* Decision C (#1954): a symlink the invoking user owns, pointing at a regular + * file the same user owns, is edited THROUGH once the writer opted in — the + * link survives and still points at the same target, the temp file is staged + * beside the target, and every edit is byte-identical to the same edit on a + * plain file. */ +TEST(config_json_like_follows_user_owned_symlink_in_place) { + jl_fixture_t fixture; + ASSERT_EQ(jl_fixture_open(&fixture), 0); + char target[sizeof(fixture.path) + 32U]; + char control[sizeof(fixture.path) + 32U]; + ASSERT(snprintf(target, sizeof(target), "%s/target.json", fixture.directory) > 0); + ASSERT(snprintf(control, sizeof(control), "%s/control.json", fixture.directory) > 0); + const char *original = "{\n // dotfiles\n \"mcpServers\": {\n" + " \"keep\": {\"command\": \"x\"}\n }\n}\n"; + ASSERT_EQ(jl_write(target, original), 0); + ASSERT_EQ(jl_write(control, original), 0); + /* Relative, the way a dotfiles checkout links its files. */ + ASSERT_EQ(symlink("target.json", fixture.path), 0); + ASSERT_EQ(cbm_config_edit_path_follow_add_root(fixture.directory), 0); + char target_real[CBM_CONFIG_EDIT_PATH_MAX]; + ASSERT_EQ(cbm_canonical_path(target, target_real, sizeof(target_real)), 1); + jl_follow_hook_state_t hook_state = {0}; + cbm_json_like_set_precommit_hook_for_testing(jl_follow_precommit_hook, &hook_state); + + const char *path[] = {"mcpServers"}; + int through_rc = + cbm_json_like_upsert_entry(fixture.path, path, 1U, "owned", "{\"command\":\"y\"}"); + cbm_json_like_set_precommit_hook_for_testing(NULL, NULL); + int plain_rc = cbm_json_like_upsert_entry(control, path, 1U, "owned", "{\"command\":\"y\"}"); + if (through_rc != 0 || plain_rc != 0) { + cbm_config_edit_path_follow_clear(); + } + ASSERT_EQ(through_rc, 0); + ASSERT_EQ(plain_rc, 0); + ASSERT_EQ(hook_state.calls, 1U); + ASSERT_STR_EQ(hook_state.seen_path, target_real); + ASSERT_EQ(hook_state.temps_beside_target, 1U); + struct stat link_state; + ASSERT_EQ(lstat(fixture.path, &link_state), 0); + ASSERT(S_ISLNK(link_state.st_mode)); + char link_value[256] = {0}; + ASSERT(readlink(fixture.path, link_value, sizeof(link_value) - 1U) > 0); + ASSERT_STR_EQ(link_value, "target.json"); + char *through = jl_read(target); + char *plain = jl_read(control); + ASSERT_NOT_NULL(through); + ASSERT_NOT_NULL(plain); + ASSERT_STR_EQ(through, plain); + ASSERT_NOT_NULL(strstr(through, "\"owned\"")); + ASSERT_NOT_NULL(strstr(through, "// dotfiles")); + + /* Reading through the link sees the target, and the compare-and-remove + * form uninstall relies on works through the link as well. */ + char *document = NULL; + size_t length = 0U; + ASSERT_EQ(cbm_json_like_read_document(fixture.path, &document, &length), 0); + ASSERT_NOT_NULL(document); + ASSERT_STR_EQ(document, through); + ASSERT_EQ( + cbm_json_like_remove_entry_if_unchanged(fixture.path, path, 1U, "owned", document, length), + 0); + ASSERT_EQ(cbm_json_like_remove_entry(control, path, 1U, "owned"), 0); + free(document); + free(through); + free(plain); + through = jl_read(target); + plain = jl_read(control); + ASSERT_NOT_NULL(through); + ASSERT_NOT_NULL(plain); + ASSERT_STR_EQ(through, plain); + ASSERT_NULL(strstr(through, "\"owned\"")); + free(through); + free(plain); + cbm_config_edit_path_follow_clear(); + ASSERT_EQ(lstat(fixture.path, &link_state), 0); + ASSERT(S_ISLNK(link_state.st_mode)); + ASSERT_EQ(jl_stray_temp_count(&fixture), 0U); + + ASSERT_EQ(cbm_unlink(fixture.path), 0); + ASSERT_EQ(cbm_unlink(target), 0); + ASSERT_EQ(cbm_unlink(control), 0); + jl_fixture_close(&fixture); + PASS(); +} + TEST(config_json_like_rejects_hard_link_without_splitting_identity) { jl_fixture_t fixture; ASSERT_EQ(jl_fixture_open(&fixture), 0); @@ -853,7 +1229,7 @@ TEST(config_json_like_nested_array_fails_closed_on_ambiguous_or_invalid_paths) { } #ifndef _WIN32 -TEST(config_json_like_nested_array_rejects_symlink_and_hardlink) { +TEST(config_json_like_nested_array_rejects_foreign_symlink_and_hardlink) { jl_fixture_t fixture; ASSERT_EQ(jl_fixture_open(&fixture), 0); const char *path[] = {"options"}; @@ -862,12 +1238,18 @@ TEST(config_json_like_nested_array_rejects_symlink_and_hardlink) { ASSERT(snprintf(target, sizeof(target), "%s/target.json", fixture.directory) > 0); ASSERT_EQ(jl_write(target, original), 0); ASSERT_EQ(symlink(target, fixture.path), 0); - ASSERT_EQ( - cbm_json_like_add_unique_string_at_path(fixture.path, path, 1U, "context_paths", "owned"), - -1); + /* Foreign-owned link (observer moved by the test seam): still refused. */ + ASSERT_EQ(cbm_config_edit_path_follow_add_root(fixture.directory), 0); + cbm_config_edit_path_set_invoking_uid_for_test((unsigned)geteuid() + 1U, 1); + int add_rc = + cbm_json_like_add_unique_string_at_path(fixture.path, path, 1U, "context_paths", "owned"); char *workspace = (char *)(uintptr_t)1U; - ASSERT_EQ(cbm_json_like_get_string_at_path(fixture.path, path, 1U, "context_paths", &workspace), - -1); + int get_rc = + cbm_json_like_get_string_at_path(fixture.path, path, 1U, "context_paths", &workspace); + cbm_config_edit_path_set_invoking_uid_for_test(0U, 0); + cbm_config_edit_path_follow_clear(); + ASSERT_EQ(add_rc, -1); + ASSERT_EQ(get_rc, -1); ASSERT_NULL(workspace); char *content = jl_read(target); ASSERT_NOT_NULL(content); @@ -1007,7 +1389,14 @@ SUITE(config_json_like) { RUN_TEST(config_json_like_existing_target_swap_after_check_preserves_winner); RUN_TEST(config_json_like_rejects_non_regular_path); #ifndef _WIN32 - RUN_TEST(config_json_like_rejects_symlink_without_touching_target); + RUN_TEST(config_json_like_rejects_foreign_owned_symlink_without_touching_target); + RUN_TEST(config_json_like_root_refuses_user_owned_symlink); + RUN_TEST(config_json_like_refuses_dangling_symlink); + RUN_TEST(config_json_like_refuses_symlink_to_directory); + RUN_TEST(config_json_like_refuses_symlink_without_opt_in); + RUN_TEST(config_json_like_refuses_symlink_when_target_identity_skews); + RUN_TEST(config_json_like_refuses_symlink_with_world_writable_parent); + RUN_TEST(config_json_like_follows_user_owned_symlink_in_place); RUN_TEST(config_json_like_rejects_hard_link_without_splitting_identity); RUN_TEST(config_json_like_preserves_owner_group_and_mode); #endif @@ -1030,7 +1419,7 @@ SUITE(config_json_like) { RUN_TEST(config_json_like_nested_array_preserves_jsonc_and_is_idempotent); RUN_TEST(config_json_like_nested_array_fails_closed_on_ambiguous_or_invalid_paths); #ifndef _WIN32 - RUN_TEST(config_json_like_nested_array_rejects_symlink_and_hardlink); + RUN_TEST(config_json_like_nested_array_rejects_foreign_symlink_and_hardlink); #endif RUN_TEST(config_json_like_nested_array_rejects_precommit_content_and_identity_races); RUN_TEST(config_json_like_nested_string_lookup_decodes_json5_workspace); diff --git a/tests/test_config_text_edit.c b/tests/test_config_text_edit.c index 47135be84..4ce474853 100644 --- a/tests/test_config_text_edit.c +++ b/tests/test_config_text_edit.c @@ -2,6 +2,8 @@ * test_config_text_edit.c — Hardened managed-text editor contracts. */ #include "../src/cli/config_text_edit.h" +#define CBM_CONFIG_EDIT_PATH_ENABLE_TEST_API 1 +#include "../src/cli/config_edit_path.h" #include "test_framework.h" #include "test_helpers.h" @@ -384,8 +386,16 @@ TEST(config_text_rejects_links_privileged_mode_and_preserves_metadata) { ASSERT(snprintf(alias, sizeof(alias), "%s/alias.md", dir) > 0); ASSERT_EQ(th_write_file(target, "target\n"), 0); ASSERT_EQ(symlink(target, path), 0); - ASSERT_EQ(cbm_text_write_owned_document(path, "owned\n"), -1); - ASSERT_EQ(cbm_text_migrate_owned_document_mode(path, "owned\n", NULL, 0U, 0755U), -1); + /* Foreign-owned link (observer moved by the test seam): still refused. */ + ASSERT_EQ(cbm_config_edit_path_follow_add_root(dir), 0); + cbm_config_edit_path_set_invoking_uid_for_test((unsigned)geteuid() + 1U, 1); + int foreign_write_rc = cbm_text_write_owned_document(path, "owned\n"); + int foreign_migrate_rc = cbm_text_migrate_owned_document_mode(path, "owned\n", NULL, 0U, 0755U); + cbm_config_edit_path_set_invoking_uid_for_test(0U, 0); + cbm_config_edit_path_follow_clear(); + ASSERT_EQ(foreign_write_rc, -1); + ASSERT_EQ(foreign_migrate_rc, -1); + ASSERT(cte_assert_bytes(target, "target\n", strlen("target\n"))); ASSERT_EQ(cbm_unlink(path), 0); ASSERT_EQ(th_write_file(path, "shared\n"), 0); @@ -414,6 +424,65 @@ TEST(config_text_rejects_links_privileged_mode_and_preserves_metadata) { PASS(); } +/* Decision C (#1954): a managed block behind a user-owned symlink is edited + * through the link — the link survives and the target carries the same bytes + * the same edit produces on a plain file. */ +TEST(config_text_follows_user_owned_symlink_in_place) { + char dir[CTE_PATH_CAP]; + char path[CTE_PATH_CAP]; + char target[CTE_PATH_CAP]; + char control[CTE_PATH_CAP]; + ASSERT_EQ(cte_fixture(dir, sizeof(dir), path, sizeof(path)), 0); + ASSERT(snprintf(target, sizeof(target), "%s/target.md", dir) > 0); + ASSERT(snprintf(control, sizeof(control), "%s/control.md", dir) > 0); + ASSERT_EQ(th_write_file(target, "# user notes\n"), 0); + ASSERT_EQ(th_write_file(control, "# user notes\n"), 0); + ASSERT_EQ(symlink("target.md", path), 0); + ASSERT_EQ(cbm_config_edit_path_follow_add_root(dir), 0); + + int through_rc = cbm_text_upsert_managed_block(path, CTE_BEGIN, CTE_END, "owned"); + int plain_rc = cbm_text_upsert_managed_block(control, CTE_BEGIN, CTE_END, "owned"); + if (through_rc != 0 || plain_rc != 0) { + cbm_config_edit_path_follow_clear(); + } + ASSERT_EQ(through_rc, 0); + ASSERT_EQ(plain_rc, 0); + struct stat link_state; + ASSERT_EQ(lstat(path, &link_state), 0); + ASSERT(S_ISLNK(link_state.st_mode)); + size_t through_len = 0U; + size_t plain_len = 0U; + char *through = cte_read_bytes(target, &through_len); + char *plain = cte_read_bytes(control, &plain_len); + ASSERT_NOT_NULL(through); + ASSERT_NOT_NULL(plain); + ASSERT_EQ(through_len, plain_len); + ASSERT_EQ(memcmp(through, plain, plain_len), 0); + ASSERT_NOT_NULL(strstr(through, "owned")); + free(through); + free(plain); + + int through_remove_rc = cbm_text_remove_managed_block(path, CTE_BEGIN, CTE_END); + int plain_remove_rc = cbm_text_remove_managed_block(control, CTE_BEGIN, CTE_END); + cbm_config_edit_path_follow_clear(); + ASSERT_EQ(through_remove_rc, 0); + ASSERT_EQ(plain_remove_rc, 0); + ASSERT_EQ(lstat(path, &link_state), 0); + ASSERT(S_ISLNK(link_state.st_mode)); + through = cte_read_bytes(target, &through_len); + plain = cte_read_bytes(control, &plain_len); + ASSERT_NOT_NULL(through); + ASSERT_NOT_NULL(plain); + ASSERT_EQ(through_len, plain_len); + ASSERT_EQ(memcmp(through, plain, plain_len), 0); + ASSERT_NULL(strstr(through, "owned")); + free(through); + free(plain); + ASSERT_EQ(cte_temp_count(dir), 0U); + th_cleanup(dir); + PASS(); +} + TEST(config_text_owned_document_mode_publishes_exact_bytes_atomically) { char dir[CTE_PATH_CAP]; char path[CTE_PATH_CAP]; @@ -670,6 +739,7 @@ SUITE(config_text_edit) { RUN_TEST(config_text_rejects_non_regular_paths); #ifndef _WIN32 RUN_TEST(config_text_rejects_links_privileged_mode_and_preserves_metadata); + RUN_TEST(config_text_follows_user_owned_symlink_in_place); RUN_TEST(config_text_owned_document_mode_publishes_exact_bytes_atomically); RUN_TEST(config_text_owned_document_mode_rejects_prepublish_replacement); RUN_TEST(config_text_owned_document_mode_rejects_closed_temp_replacement); diff --git a/tests/test_config_toml_edit.c b/tests/test_config_toml_edit.c index 1266556e0..cc64fd576 100644 --- a/tests/test_config_toml_edit.c +++ b/tests/test_config_toml_edit.c @@ -5,6 +5,8 @@ */ #define CBM_TOML_EDIT_ENABLE_TEST_API 1 #include "cli/config_toml_edit.h" +#define CBM_CONFIG_EDIT_PATH_ENABLE_TEST_API 1 +#include "cli/config_edit_path.h" #include "foundation/compat.h" #include "foundation/compat_fs.h" #include "test_framework.h" @@ -312,7 +314,13 @@ TEST(config_toml_rejects_symlink_hardlink_and_preserves_metadata) { ASSERT(snprintf(alias, sizeof(alias), "%s/alias.toml", dir) > 0); ASSERT_EQ(th_write_file(target, "target = true\n"), 0); ASSERT_EQ(symlink(target, path), 0); - ASSERT_EQ(cbm_toml_upsert_managed_block(path, CTE_BEGIN, CTE_END, "owned = true\n"), -1); + /* Foreign-owned link (observer moved by the test seam): still refused. */ + ASSERT_EQ(cbm_config_edit_path_follow_add_root(dir), 0); + cbm_config_edit_path_set_invoking_uid_for_test((unsigned)geteuid() + 1U, 1); + int foreign_rc = cbm_toml_upsert_managed_block(path, CTE_BEGIN, CTE_END, "owned = true\n"); + cbm_config_edit_path_set_invoking_uid_for_test(0U, 0); + cbm_config_edit_path_follow_clear(); + ASSERT_EQ(foreign_rc, -1); struct stat link_state; ASSERT_EQ(lstat(path, &link_state), 0); ASSERT(S_ISLNK(link_state.st_mode)); @@ -351,6 +359,59 @@ TEST(config_toml_rejects_symlink_hardlink_and_preserves_metadata) { } #endif +#ifndef _WIN32 +/* Decision C (#1954): a managed block behind a user-owned symlink is edited + * through the link — the link survives and the target carries the same bytes + * the same edit produces on a plain file. */ +TEST(config_toml_follows_user_owned_symlink_in_place) { + char dir[CTE_PATH_CAP]; + char path[CTE_PATH_CAP]; + char target[CTE_PATH_CAP]; + char control[CTE_PATH_CAP]; + char through[CTE_FILE_CAP]; + char plain[CTE_FILE_CAP]; + ASSERT_EQ(cte_fixture(dir, sizeof(dir), path, sizeof(path)), 0); + ASSERT(snprintf(target, sizeof(target), "%s/target.toml", dir) > 0); + ASSERT(snprintf(control, sizeof(control), "%s/control.toml", dir) > 0); + ASSERT_EQ(th_write_file(target, "target = true\n"), 0); + ASSERT_EQ(th_write_file(control, "target = true\n"), 0); + ASSERT_EQ(symlink("target.toml", path), 0); + ASSERT_EQ(cbm_config_edit_path_follow_add_root(dir), 0); + + int through_rc = cbm_toml_upsert_managed_block(path, CTE_BEGIN, CTE_END, "owned = true\n"); + int plain_rc = cbm_toml_upsert_managed_block(control, CTE_BEGIN, CTE_END, "owned = true\n"); + if (through_rc != 0 || plain_rc != 0) { + cbm_config_edit_path_follow_clear(); + } + ASSERT_EQ(through_rc, 0); + ASSERT_EQ(plain_rc, 0); + struct stat link_state; + ASSERT_EQ(lstat(path, &link_state), 0); + ASSERT(S_ISLNK(link_state.st_mode)); + ASSERT_EQ(cte_read(target, through, sizeof(through)), 0); + ASSERT_EQ(cte_read(control, plain, sizeof(plain)), 0); + ASSERT_STR_EQ(through, plain); + ASSERT_NOT_NULL(strstr(through, "owned = true")); + + int through_remove_rc = cbm_toml_remove_managed_block(path, CTE_BEGIN, CTE_END); + int plain_remove_rc = cbm_toml_remove_managed_block(control, CTE_BEGIN, CTE_END); + cbm_config_edit_path_follow_clear(); + ASSERT_EQ(through_remove_rc, 0); + ASSERT_EQ(plain_remove_rc, 0); + ASSERT_EQ(lstat(path, &link_state), 0); + ASSERT(S_ISLNK(link_state.st_mode)); + ASSERT_EQ(cte_read(target, through, sizeof(through)), 0); + ASSERT_EQ(cte_read(control, plain, sizeof(plain)), 0); + ASSERT_STR_EQ(through, plain); + ASSERT_NULL(strstr(through, "owned = true")); + ASSERT_EQ(cte_temp_count(dir), 0U); + ASSERT_EQ(cbm_unlink(target), 0); + ASSERT_EQ(cbm_unlink(control), 0); + th_cleanup(dir); + PASS(); +} +#endif + TEST(config_toml_managed_markers_ignore_multiline_strings) { char dir[CTE_PATH_CAP]; char path[CTE_PATH_CAP]; @@ -1411,6 +1472,7 @@ SUITE(config_toml_edit) { RUN_TEST(config_toml_rejects_non_regular_path); #ifndef _WIN32 RUN_TEST(config_toml_rejects_symlink_hardlink_and_preserves_metadata); + RUN_TEST(config_toml_follows_user_owned_symlink_in_place); #endif RUN_TEST(config_toml_managed_markers_ignore_multiline_strings); RUN_TEST(config_toml_managed_rejects_marker_in_block_and_unclosed_multiline); diff --git a/tests/test_config_yaml_edit.c b/tests/test_config_yaml_edit.c index 3e290240c..421c8b79c 100644 --- a/tests/test_config_yaml_edit.c +++ b/tests/test_config_yaml_edit.c @@ -8,6 +8,8 @@ #define CBM_YAML_ENABLE_TEST_API 1 #include +#define CBM_CONFIG_EDIT_PATH_ENABLE_TEST_API 1 +#include #include #include @@ -476,7 +478,13 @@ TEST(config_yaml_edit_rejects_symlinks_without_touching_target) { ASSERT_EQ(th_write_file(target, original), 0); ASSERT_EQ(symlink(target, fixture.path), 0); - ASSERT_EQ(cbm_yaml_upsert_string_list_item(fixture.path, "read", "AGENTS.md"), -1); + /* Foreign-owned link (observer moved by the test seam): still refused. */ + ASSERT_EQ(cbm_config_edit_path_follow_add_root(fixture.dir), 0); + cbm_config_edit_path_set_invoking_uid_for_test((unsigned)geteuid() + 1U, 1); + int foreign_rc = cbm_yaml_upsert_string_list_item(fixture.path, "read", "AGENTS.md"); + cbm_config_edit_path_set_invoking_uid_for_test(0U, 0); + cbm_config_edit_path_follow_clear(); + ASSERT_EQ(foreign_rc, -1); struct stat link_state; ASSERT_EQ(lstat(fixture.path, &link_state), 0); ASSERT(S_ISLNK(link_state.st_mode)); @@ -492,6 +500,64 @@ TEST(config_yaml_edit_rejects_symlinks_without_touching_target) { PASS(); } +/* Decision C (#1954): a user-owned symlink is edited through — the link + * survives and the target carries the bytes the same edit yields on a plain file. */ +TEST(config_yaml_edit_follows_user_owned_symlink_in_place) { + const char *original = "model: safe\n"; + yaml_fixture_t fixture; + ASSERT_EQ(yaml_fixture_init(&fixture, NULL), 0); + char target[sizeof(fixture.path) + 32U]; + char control[sizeof(fixture.path) + 32U]; + ASSERT(snprintf(target, sizeof(target), "%s/target.yaml", fixture.dir) > 0); + ASSERT(snprintf(control, sizeof(control), "%s/control.yaml", fixture.dir) > 0); + ASSERT_EQ(th_write_file(target, original), 0); + ASSERT_EQ(th_write_file(control, original), 0); + ASSERT_EQ(symlink("target.yaml", fixture.path), 0); + ASSERT_EQ(cbm_config_edit_path_follow_add_root(fixture.dir), 0); + + int through_rc = cbm_yaml_upsert_string_list_item(fixture.path, "read", "AGENTS.md"); + int plain_rc = cbm_yaml_upsert_string_list_item(control, "read", "AGENTS.md"); + if (through_rc != 0 || plain_rc != 0) { + cbm_config_edit_path_follow_clear(); + } + ASSERT_EQ(through_rc, 0); + ASSERT_EQ(plain_rc, 0); + struct stat link_state; + ASSERT_EQ(lstat(fixture.path, &link_state), 0); + ASSERT(S_ISLNK(link_state.st_mode)); + char *through = yaml_read_alloc(target); + char *plain = yaml_read_alloc(control); + ASSERT_NOT_NULL(through); + ASSERT_NOT_NULL(plain); + ASSERT_STR_EQ(through, plain); + ASSERT_NOT_NULL(strstr(through, "AGENTS.md")); + free(through); + free(plain); + + int through_remove_rc = cbm_yaml_remove_string_list_item(fixture.path, "read", "AGENTS.md"); + int plain_remove_rc = cbm_yaml_remove_string_list_item(control, "read", "AGENTS.md"); + cbm_config_edit_path_follow_clear(); + ASSERT_EQ(through_remove_rc, 0); + ASSERT_EQ(plain_remove_rc, 0); + ASSERT_EQ(lstat(fixture.path, &link_state), 0); + ASSERT(S_ISLNK(link_state.st_mode)); + through = yaml_read_alloc(target); + plain = yaml_read_alloc(control); + ASSERT_NOT_NULL(through); + ASSERT_NOT_NULL(plain); + ASSERT_STR_EQ(through, plain); + ASSERT_NULL(strstr(through, "AGENTS.md")); + free(through); + free(plain); + ASSERT_EQ(yaml_temp_file_count(&fixture), 0U); + + ASSERT_EQ(cbm_unlink(fixture.path), 0); + ASSERT_EQ(cbm_unlink(target), 0); + ASSERT_EQ(cbm_unlink(control), 0); + th_cleanup(fixture.dir); + PASS(); +} + TEST(config_yaml_edit_rejects_dangling_symlink) { yaml_fixture_t fixture; ASSERT_EQ(yaml_fixture_init(&fixture, NULL), 0); @@ -1935,9 +2001,15 @@ TEST(config_yaml_edit_nested_sequence_rejects_symlink_byte_identically) { ASSERT(snprintf(target, sizeof(target), "%s/target-hooks.yaml", fixture.dir) > 0); ASSERT_EQ(th_write_file(target, original), 0); ASSERT_EQ(symlink(target, fixture.path), 0); - ASSERT_EQ(cbm_yaml_upsert_mapping_sequence_item(fixture.path, yaml_hook_sequence_path, 2U, "id", - yaml_hook_identity, yaml_hook_canonical_item), - CBM_YAML_IDENTITY_EDIT_ERROR); + /* Foreign-owned link (observer moved by the test seam): still refused. */ + ASSERT_EQ(cbm_config_edit_path_follow_add_root(fixture.dir), 0); + cbm_config_edit_path_set_invoking_uid_for_test((unsigned)geteuid() + 1U, 1); + int foreign_rc = + cbm_yaml_upsert_mapping_sequence_item(fixture.path, yaml_hook_sequence_path, 2U, "id", + yaml_hook_identity, yaml_hook_canonical_item); + cbm_config_edit_path_set_invoking_uid_for_test(0U, 0); + cbm_config_edit_path_follow_clear(); + ASSERT_EQ(foreign_rc, CBM_YAML_IDENTITY_EDIT_ERROR); char *after = yaml_read_alloc(target); ASSERT_NOT_NULL(after); ASSERT_STR_EQ(after, original); @@ -1968,6 +2040,7 @@ SUITE(config_yaml_edit) { RUN_TEST(config_yaml_edit_rejects_non_regular_path); #ifndef _WIN32 RUN_TEST(config_yaml_edit_rejects_symlinks_without_touching_target); + RUN_TEST(config_yaml_edit_follows_user_owned_symlink_in_place); RUN_TEST(config_yaml_edit_rejects_dangling_symlink); RUN_TEST(config_yaml_edit_preserves_owner_group_and_mode); RUN_TEST(config_yaml_edit_rejects_hard_links_without_splitting_identity); From e8dcb28ba719e4272f605f3b961422bb1f3c3fad Mon Sep 17 00:00:00 2001 From: Brian McNaboe Date: Thu, 20 Aug 2026 12:26:03 -0400 Subject: [PATCH 029/130] test(hook-augment): pin worktree payload-cwd resolution The hook resolves a payload cwd inside a linked git worktree to that worktree own indexed project (and a subdir of it), never a sibling checkout or nothing, and an unindexed worktree gets a deliberate "no indexed graph project matched" notice rather than a silent 0 bytes. Main-checkout resolution is unchanged. These regression tests use a real linked worktree and lock the payload-cwd -> project invariant. Fixes #1752 Signed-off-by: Brian McNaboe --- tests/test_cli.c | 173 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) diff --git a/tests/test_cli.c b/tests/test_cli.c index 976280868..c13e27b99 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -9342,6 +9342,175 @@ TEST(cli_hook_session_exhausts_project_registry_pages) { PASS(); } +#ifndef _WIN32 +/* A linked git worktree is indexed as its OWN project, keyed by its checkout + * path. The SessionStart hook must return that project for a payload cwd inside + * the worktree (or a subdir of it) — never the main checkout's — and the main + * checkout's own resolution must be unchanged. Pins the payload-cwd → project + * resolution so a worktree can never regress to resolving a sibling checkout or + * to nothing (the historical silent-0-bytes worktree bug). */ +TEST(cli_hook_worktree_cwd_resolves_own_indexed_project) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-hook-worktree-XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) + FAIL("cbm_mkdtemp failed"); + char cache[512]; + char maindir[512]; + char wtdir[512]; + char subdir[600]; + snprintf(cache, sizeof(cache), "%s/cache", tmpdir); + snprintf(maindir, sizeof(maindir), "%s/checkout-main", tmpdir); + snprintf(wtdir, sizeof(wtdir), "%s/checkout-side", tmpdir); + snprintf(subdir, sizeof(subdir), "%s/nested", wtdir); + test_mkdirp(cache); + test_mkdirp(maindir); + + /* A real linked worktree so is_worktree/git_common_dir/canonical_root are + * exactly what the hook resolves against in the field. */ + char cmd[2048]; + snprintf(cmd, sizeof(cmd), + "cd \"%s\" && git init -q && git config user.email t@t && git config user.name t && " + ": > f.txt && git add f.txt && git commit -qm init && " + "git worktree add -q \"%s\" -b agl7branch", + maindir, wtdir); + if (system(cmd) != 0) { + test_rmdir_r(tmpdir); + FAIL("git worktree setup failed"); + } + test_mkdirp(subdir); + + /* Register each checkout as its own indexed project under the exact + * path-derived name the indexer uses (the db filename == that name). */ + char *main_name = cbm_project_name_from_path(maindir); + char *wt_name = cbm_project_name_from_path(wtdir); + ASSERT_NOT_NULL(main_name); + ASSERT_NOT_NULL(wt_name); + ASSERT_STR_NEQ(main_name, wt_name); + char db_main[900]; + char db_wt[900]; + snprintf(db_main, sizeof(db_main), "%s/%s.db", cache, main_name); + snprintf(db_wt, sizeof(db_wt), "%s/%s.db", cache, wt_name); + cbm_store_t *sm = cbm_store_open_path(db_main); + ASSERT_NOT_NULL(sm); + ASSERT_EQ(cbm_store_upsert_project(sm, main_name, maindir), CBM_STORE_OK); + cbm_store_close(sm); + cbm_store_t *sw = cbm_store_open_path(db_wt); + ASSERT_NOT_NULL(sw); + ASSERT_EQ(cbm_store_upsert_project(sw, wt_name, wtdir), CBM_STORE_OK); + cbm_store_close(sw); + + char *saved_cache = save_test_env("CBM_CACHE_DIR"); + cbm_setenv("CBM_CACHE_DIR", cache, 1); + + /* The two names share a prefix but diverge (checkout-main vs checkout-side), + * so a bare-name strstr for one never matches the other's context. */ + char input[1024]; + snprintf(input, sizeof(input), "{\"hook_event_name\":\"SessionStart\",\"cwd\":\"%s\"}", wtdir); + char *wt_out = cbm_hook_augment_lifecycle_json(input); + bool wt_ok = wt_out && strstr(wt_out, wt_name) && strstr(wt_out, "is indexed") && + !strstr(wt_out, main_name); + free(wt_out); + + snprintf(input, sizeof(input), "{\"hook_event_name\":\"SessionStart\",\"cwd\":\"%s\"}", subdir); + char *sub_out = cbm_hook_augment_lifecycle_json(input); + bool sub_ok = sub_out && strstr(sub_out, wt_name) && strstr(sub_out, "is indexed"); + free(sub_out); + + snprintf(input, sizeof(input), "{\"hook_event_name\":\"SessionStart\",\"cwd\":\"%s\"}", + maindir); + char *main_out = cbm_hook_augment_lifecycle_json(input); + bool main_ok = main_out && strstr(main_out, main_name) && strstr(main_out, "is indexed") && + !strstr(main_out, wt_name); + free(main_out); + + restore_test_env("CBM_CACHE_DIR", saved_cache); + free(main_name); + free(wt_name); + snprintf(cmd, sizeof(cmd), "cd \"%s\" && git worktree remove --force \"%s\" >/dev/null 2>&1", + maindir, wtdir); + (void)system(cmd); + test_rmdir_r(tmpdir); + + if (!wt_ok) + FAIL("worktree cwd must resolve the worktree's own indexed project"); + if (!sub_ok) + FAIL("a subdir of the worktree must resolve the worktree's own project"); + if (!main_ok) + FAIL("main-checkout resolution must be unchanged and not leak the worktree project"); + PASS(); +} + +/* Completes the payload-cwd matrix (with the indexed-worktree test above and the + * deadline/logging contract in cli_hook_augment_deadline_breadcrumb_issue858): a + * linked worktree that was never indexed must yield a deliberate, non-empty "no + * project matched — run index_repository" notice, never a silent 0-byte reply + * that an agent cannot distinguish from "no matches". */ +TEST(cli_hook_unindexed_worktree_reports_no_match_not_silent) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-hook-unindexed-wt-XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) + FAIL("cbm_mkdtemp failed"); + char cache[512]; + char maindir[512]; + char wtdir[512]; + snprintf(cache, sizeof(cache), "%s/cache", tmpdir); + snprintf(maindir, sizeof(maindir), "%s/checkout-main", tmpdir); + snprintf(wtdir, sizeof(wtdir), "%s/checkout-side", tmpdir); + test_mkdirp(cache); + test_mkdirp(maindir); + + char cmd[2048]; + snprintf(cmd, sizeof(cmd), + "cd \"%s\" && git init -q && git config user.email t@t && git config user.name t && " + ": > f.txt && git add f.txt && git commit -qm init && " + "git worktree add -q \"%s\" -b agl7unindexed", + maindir, wtdir); + if (system(cmd) != 0) { + test_rmdir_r(tmpdir); + FAIL("git worktree setup failed"); + } + + /* Index ONLY the main checkout; the worktree is deliberately left out. */ + char *main_name = cbm_project_name_from_path(maindir); + ASSERT_NOT_NULL(main_name); + char db_main[900]; + snprintf(db_main, sizeof(db_main), "%s/%s.db", cache, main_name); + cbm_store_t *sm = cbm_store_open_path(db_main); + ASSERT_NOT_NULL(sm); + ASSERT_EQ(cbm_store_upsert_project(sm, main_name, maindir), CBM_STORE_OK); + cbm_store_close(sm); + + char *saved_cache = save_test_env("CBM_CACHE_DIR"); + cbm_setenv("CBM_CACHE_DIR", cache, 1); + + char input[1024]; + snprintf(input, sizeof(input), "{\"hook_event_name\":\"SessionStart\",\"cwd\":\"%s\"}", wtdir); + char *wt_out = cbm_hook_augment_lifecycle_json(input); + bool deliberate = wt_out && wt_out[0] && strstr(wt_out, "no indexed graph project matched") && + !strstr(wt_out, "is indexed"); + free(wt_out); + + snprintf(input, sizeof(input), "{\"hook_event_name\":\"SessionStart\",\"cwd\":\"%s\"}", + maindir); + char *main_out = cbm_hook_augment_lifecycle_json(input); + bool main_ok = main_out && strstr(main_out, main_name) && strstr(main_out, "is indexed"); + free(main_out); + + restore_test_env("CBM_CACHE_DIR", saved_cache); + free(main_name); + snprintf(cmd, sizeof(cmd), "cd \"%s\" && git worktree remove --force \"%s\" >/dev/null 2>&1", + maindir, wtdir); + (void)system(cmd); + test_rmdir_r(tmpdir); + + if (!deliberate) + FAIL("unindexed worktree must get a deliberate no-match notice, never a silent 0 bytes"); + if (!main_ok) + FAIL("the indexed main checkout must still resolve in the same matrix"); + PASS(); +} +#endif + TEST(cli_hook_session_sanitizes_untrusted_project_metadata) { char tmpdir[256]; snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-hook-untrusted-project-XXXXXX"); @@ -14984,6 +15153,10 @@ SUITE(cli) { RUN_TEST(cli_augment_session_uses_workspace_roots); RUN_TEST(cli_hook_session_resolves_custom_named_index_by_root_path); RUN_TEST(cli_hook_session_exhausts_project_registry_pages); +#ifndef _WIN32 + RUN_TEST(cli_hook_worktree_cwd_resolves_own_indexed_project); + RUN_TEST(cli_hook_unindexed_worktree_reports_no_match_not_silent); +#endif RUN_TEST(cli_hook_session_sanitizes_untrusted_project_metadata); RUN_TEST(cli_hook_ownership_requires_exact_command_identity); RUN_TEST(cli_gemini_hook_upgrade_migrates_released_exact_commands); From 0567dc70a097f36a953368978d6728b0a412b229 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 8 Sep 2026 20:13:17 +0200 Subject: [PATCH 030/130] fix(daemon): guard the POSIX listen-failure recorder so Windows does not see it unused The rich errno+path recorder ipc_listen_failed (and ipc_listen_failure_reset) are defined before the POSIX '#ifndef _WIN32' listener block but only called inside it, so the Windows build compiled them dead and failed under -Werror,-Wunused-function (pr-smoke windows-latest). Wrap both in the same '#ifndef _WIN32'; the failure struct and its cross-platform accessor cbm_daemon_ipc_listen_failure_detail stay compiled on every platform. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- src/daemon/ipc.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/daemon/ipc.c b/src/daemon/ipc.c index b62a4616d..b44aff82a 100644 --- a/src/daemon/ipc.c +++ b/src/daemon/ipc.c @@ -56,6 +56,12 @@ const char *cbm_daemon_ipc_validation_detail(void) { static cbm_daemon_ipc_listen_failure_t ipc_listen_failure; +/* The failure recorder and its reset are only reached from the POSIX socket + * listener (all callers live in the `#ifndef _WIN32` block below); the Windows + * named-pipe listener does its own reporting. Guarding them keeps the always- + * compiled struct + accessor cross-platform while avoiding -Werror, + * -Wunused-function on the Windows build (#1828 CI). */ +#ifndef _WIN32 static void ipc_listen_failure_reset(void) { memset(&ipc_listen_failure, 0, sizeof(ipc_listen_failure)); } @@ -92,6 +98,7 @@ static void ipc_listen_failed(const char *runtime_dir, const char *stage, int er cbm_log_error("daemon.ipc.listen_failed", "stage", stage); } } +#endif /* !_WIN32 */ bool cbm_daemon_ipc_listen_failure_detail(cbm_daemon_ipc_listen_failure_t *out) { if (!out) { From 013e1f7ec7d4b981a1cd28bf35d5767c195f8447 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 8 Sep 2026 20:28:58 +0200 Subject: [PATCH 031/130] fix(cli): scope the activation guard to the install-target HOME (amendment) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rendezvous directory the CLI's activation guard talks to is per OS account (service.h), not per HOME/CBM_CACHE_DIR: every daemon owned by one user meets at the same endpoint, and only the cache fingerprint carried in the cohort identity separates one HOME/CBM_CACHE_DIR namespace from another. `cli_activation_production_reserve` used `cbm_version_cohort_reserve_for_mutation`, which treats any lifetime- lock holder at that shared endpoint as the daemon this activation must drain, with no cache-fingerprint awareness at all. An `install` run against a second HOME (a sandbox, a second profile, `--skip-binary` into an alternate CBM_CACHE_DIR) therefore reached the LIVE host daemon at the shared endpoint and drained it — disconnecting every one of the host's real MCP clients — even though nothing the sandboxed install touched belonged to the host's cache namespace. The fix adds `cli_activation_resolve_scope`, a non-blocking probe via the already cache-fingerprint-aware `cbm_version_cohort_acquire`, run before the blind drain path. It distinguishes three cases from the resolved target of THIS activation: - nothing to quiesce at all: a `--skip-binary` install that also resets no index publishes nothing, so no session anywhere needs to stop (`quiesce_required` on `cli_activation_guard_scoped`, plumbed from `cbm_cmd_install`'s own binary/index-reset decision); - the active cohort's cache fingerprint doesn't match this activation's target cache: a foreign namespace, left untouched; - same cache fingerprint (or no active cohort): this activation really does replace that daemon, so the existing drain/quiesce path runs unchanged, including the barrier wait when another activation already holds maintenance. Verified against a real forked daemon (a live runtime service with one committed client) standing in for the host: an install into a foreign HOME/CBM_CACHE_DIR (`--skip-binary` and a full binary install) leaves that daemon serving with its client still attached, while an install that targets the host's own namespace still drains it and the activation audit still names the client it disconnected. New/updated tests in tests/test_cli.c (cli suite): - cli_install_skip_binary_into_foreign_home_never_drains_host_cohort - cli_install_binary_into_foreign_home_never_drains_host_cohort - cli_install_into_host_namespace_still_drains_host_cohort (regression) - cli_install_skip_binary_unchanged_in_host_namespace_quiesces_nothing - cli_activation_quiesce_does_not_wait_on_bootstrap_startup updated to carry a real cache fingerprint for its participant, now that scope is fingerprint-gated src/mcp/index_supervisor.{c,h}: CBM_CLI_ENABLE_TEST_API-gated setter so the fixture's forked daemon can carry the same build fingerprint the guard captures from the supervisor. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- src/cli/cli.c | 163 +++++++++++++-- src/mcp/index_supervisor.c | 10 + src/mcp/index_supervisor.h | 10 + tests/test_cli.c | 407 ++++++++++++++++++++++++++++++++++++- 4 files changed, 576 insertions(+), 14 deletions(-) diff --git a/src/cli/cli.c b/src/cli/cli.c index d85ebf0b6..6cf9ddca8 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -214,6 +214,12 @@ typedef struct { bool cleanup_ok; bool original_cache_environment_present; bool cache_environment_overridden; + /* Scope: what THIS activation replaces decides whether any session must + * be quiesced at all, and the target cache namespace decides WHICH + * cohort. A skipped coordination holds no lock and drains nobody. */ + bool quiesce_required; + bool coordination_skipped; + char scope_detail[CBM_SZ_1K]; } cli_activation_production_context_t; static cbm_cli_activation_ops_t g_cli_activation_test_ops; @@ -528,6 +534,91 @@ static void cli_activation_release_cleanup_lease(cli_activation_production_conte } } +typedef enum { + CLI_ACTIVATION_SCOPE_ACTIVE = 0, + CLI_ACTIVATION_SCOPE_NOTHING_TO_REPLACE, + CLI_ACTIVATION_SCOPE_FOREIGN_COHORT, + CLI_ACTIVATION_SCOPE_ERROR, +} cli_activation_scope_t; + +static void cli_activation_log_guard_decision(const cli_activation_production_context_t *context, + const char *decision, const char *active_cache) { + char clients[32]; + (void)snprintf(clients, sizeof(clients), "%llu", + (unsigned long long)context->daemon_result.active_clients); + const char *scope = cbm_daemon_ipc_endpoint_runtime_dir(context->endpoint); + cbm_log_info("activation.guard", "scope", scope ? scope : "", "cache", + context->cache_fingerprint, "clients", clients, "decision", decision, + "active_cache", active_cache ? active_cache : ""); +} + +/* Whose sessions does this activation have to stop? The rendezvous directory + * is per OS account (service.h), so the host daemon of another HOME / + * CBM_CACHE_DIR meets this activation at the very same endpoint; only the + * cache fingerprint in the cohort identity separates the namespaces. An + * `install` into a sandbox HOME used to drain the host daemon and every MCP + * client behind it although nothing it touched belonged to the host. + * + * Two questions, answered from the activation's own target: does it replace + * anything at all (a --skip-binary install without an index reset publishes + * nothing), and whose cohort is active. Admission with an immediate deadline + * is the cheapest authoritative read of the active lifetime record: OK means + * the cohort is ours or empty, CONFLICT names the active identity (its cache + * fingerprint is filled for every conflict kind), BUSY means another + * activation holds maintenance and the barrier waits for it as before. */ +static cli_activation_scope_t cli_activation_resolve_scope( + cli_activation_production_context_t *context) { + const char *runtime_dir = cbm_daemon_ipc_endpoint_runtime_dir(context->endpoint); + const char *scope = runtime_dir ? runtime_dir : ""; + const char *action = cli_activation_action_text(context->action); + if (!context->quiesce_required) { + (void)snprintf(context->scope_detail, sizeof(context->scope_detail), + "nothing to quiesce: published binary and indexes unchanged; scope=%s", + scope); + cli_activation_log_guard_decision(context, "nothing_to_replace", NULL); + printf("No CBM session needs to stop for %s: the published binary and indexes are " + "unchanged.\n", + action); + (void)fflush(stdout); + return CLI_ACTIVATION_SCOPE_NOTHING_TO_REPLACE; + } + cbm_version_cohort_lease_t *lease = NULL; + cbm_daemon_conflict_t conflict; + memset(&conflict, 0, sizeof(conflict)); + cbm_version_cohort_status_t status = cbm_version_cohort_acquire( + context->cohort_manager, &context->identity, cbm_now_ms(), &lease, &conflict); + cli_activation_release_cleanup_lease(context, &lease); + if (lease) { + return CLI_ACTIVATION_SCOPE_ERROR; + } + switch (status) { + case CBM_VERSION_COHORT_OK: + case CBM_VERSION_COHORT_BUSY: + return CLI_ACTIVATION_SCOPE_ACTIVE; + case CBM_VERSION_COHORT_CONFLICT: + break; + default: + return CLI_ACTIVATION_SCOPE_ERROR; + } + if (!conflict.active_cache_fingerprint[0] || + strcmp(conflict.active_cache_fingerprint, context->cache_fingerprint) == 0) { + /* Same cache namespace, another build or version: that IS the daemon + * this activation replaces. An unreadable active cache stays in scope + * rather than silently exempting a same-namespace daemon. */ + return CLI_ACTIVATION_SCOPE_ACTIVE; + } + (void)snprintf(context->scope_detail, sizeof(context->scope_detail), + "active cohort serves another cache namespace (%.12s), this %s targets " + "%.12s; scope=%s; nothing stopped", + conflict.active_cache_fingerprint, action, context->cache_fingerprint, scope); + cli_activation_log_guard_decision(context, "foreign_cohort", conflict.active_cache_fingerprint); + printf("Leaving active CBM sessions untouched: they serve another cache namespace than " + "this %s targets.\n", + action); + (void)fflush(stdout); + return CLI_ACTIVATION_SCOPE_FOREIGN_COHORT; +} + static int cli_activation_production_reserve(void *opaque, cbm_cli_activation_lock_t *lease_out) { cli_activation_production_context_t *context = opaque; if (lease_out) { @@ -536,6 +627,26 @@ static int cli_activation_production_reserve(void *opaque, cbm_cli_activation_lo if (!context || !context->cohort_manager || !lease_out) { return CLI_ERR; } + cli_activation_scope_t scope = cli_activation_resolve_scope(context); + if (scope == CLI_ACTIVATION_SCOPE_ERROR) { + return CLI_ERR; + } + if (scope != CLI_ACTIVATION_SCOPE_ACTIVE) { + /* Nothing in the target namespace is being replaced, or the only + * active cohort serves another namespace: hold no maintenance, + * admission, lifetime or startup lock (each wakes or blocks the other + * namespace's sessions) and send no drain request. */ + if (!cli_activation_log_event(context, "quiesce_skipped", context->scope_detail)) { + return CLI_ERR; + } + context->coordination_skipped = true; + context->mutation_authorized = true; + *lease_out = context; + return 1; + } + printf("Stopping active CBM sessions and operations for %s...\n", + cli_activation_action_text(context->action)); + (void)fflush(stdout); cbm_version_cohort_quiesce_result_t quiesce = CBM_VERSION_COHORT_QUIESCE_NOT_NEEDED; cbm_version_cohort_lease_t *lease = NULL; context->control_deadline_ms = cli_activation_deadline_after(CLI_ACTIVATION_CONTROL_TIMEOUT_MS); @@ -582,6 +693,8 @@ static int cli_activation_production_reserve(void *opaque, cbm_cli_activation_lo } context->cohort_lease = lease; + cli_activation_log_guard_decision( + context, context->shutdown_requested ? "cohort_drained" : "no_active_cohort", NULL); if (!cli_activation_log_event(context, "daemon_stopped", context->shutdown_requested ? "cohort drained" : "no active cohort")) { @@ -600,6 +713,13 @@ static void cli_activation_production_release(void *opaque, cbm_cli_activation_l if (!context) { return; } + if (context->coordination_skipped) { + /* Nothing was held: the token is the context itself. */ + if (lease != (cbm_cli_activation_lock_t)context) { + context->cleanup_ok = false; + } + return; + } /* Global release order is the inverse of acquisition: startup first, * then lifetime/admission/maintenance through the cohort lease. */ if (context->startup_lock) { @@ -626,9 +746,11 @@ static void cli_activation_production_diagnostic(void *opaque, const char *messa static bool cli_activation_production_context_init(cli_activation_production_context_t *context, cbm_daemon_runtime_activation_action_t action, const char *target_version, - const char *target_build) { + const char *target_build, + bool quiesce_required) { memset(context, 0, sizeof(*context)); context->action = action; + context->quiesce_required = quiesce_required; context->target_version = target_version; context->target_build = target_build; context->cleanup_ok = true; @@ -735,23 +857,26 @@ static void cli_activation_production_context_close(cli_activation_production_co context->original_cache_environment = NULL; } -static int cli_activation_guard(cbm_daemon_runtime_activation_action_t action, - const char *target_version, const char *target_build, - cbm_cli_activation_mutation_fn mutation, void *mutation_context) { +/* quiesce_required: false when the activation publishes no binary and resets + * no index (a config-only install) — nothing running is then replaced, and + * no session is stopped for it. */ +static int cli_activation_guard_scoped(cbm_daemon_runtime_activation_action_t action, + const char *target_version, const char *target_build, + bool quiesce_required, + cbm_cli_activation_mutation_fn mutation, + void *mutation_context) { if (g_cli_activation_test_ops_set) { return cbm_cli_activation_guard_with_ops(&g_cli_activation_test_ops, mutation, mutation_context); } cli_activation_production_context_t context; - if (!cli_activation_production_context_init(&context, action, target_version, target_build)) { + if (!cli_activation_production_context_init(&context, action, target_version, target_build, + quiesce_required)) { cli_activation_production_context_close(&context); cli_activation_production_diagnostic(NULL, CLI_ACTIVATION_REFUSED_MESSAGE); return CLI_TRUE; } - printf("Stopping active CBM sessions and operations for %s...\n", - cli_activation_action_text(action)); - (void)fflush(stdout); if (!cli_activation_log_event(&context, "requested", NULL)) { cli_activation_production_context_close(&context); (void)fprintf(stderr, "error: activation request could not be recorded safely; " @@ -793,6 +918,13 @@ static int cli_activation_guard(cbm_daemon_runtime_activation_action_t action, return rc; } +static int cli_activation_guard(cbm_daemon_runtime_activation_action_t action, + const char *target_version, const char *target_build, + cbm_cli_activation_mutation_fn mutation, void *mutation_context) { + return cli_activation_guard_scoped(action, target_version, target_build, true, mutation, + mutation_context); +} + /* Tar header field offsets */ #define TAR_NAME_LEN 101 /* filename field: bytes 0-99 + NUL */ #define TAR_SIZE_OFFSET 124 /* octal size field offset */ @@ -10756,11 +10888,16 @@ int cbm_cmd_install(int argc, char **argv) { .force = force, .dry_run = dry_run, }; - int activation_rc = - dry_run ? cli_install_activate(&activation) - : cli_activation_guard(CBM_DAEMON_RUNTIME_ACTIVATION_INSTALL, CBM_VERSION, - has_binary_validator ? binary_validator.fingerprint : NULL, - cli_install_activate, &activation); + /* What this install replaces decides whether any session must stop: with + * the published binary untouched (--skip-binary, or an externally managed + * binary) and no index reset, agent configs are refreshed while every + * session stays up. */ + bool quiesce_required = has_binary_validator || delete_indexes; + int activation_rc = dry_run ? cli_install_activate(&activation) + : cli_activation_guard_scoped( + CBM_DAEMON_RUNTIME_ACTIVATION_INSTALL, CBM_VERSION, + has_binary_validator ? binary_validator.fingerprint : NULL, + quiesce_required, cli_install_activate, &activation); if (activation.binary_transaction) { (void)cli_activation_transaction_abort(&activation.binary_transaction); } diff --git a/src/mcp/index_supervisor.c b/src/mcp/index_supervisor.c index eed78f418..a035e3d6d 100644 --- a/src/mcp/index_supervisor.c +++ b/src/mcp/index_supervisor.c @@ -149,6 +149,16 @@ const char *cbm_index_supervisor_build_fingerprint(void) { return g_build_fingerprint[0] ? g_build_fingerprint : NULL; } +#if defined(CBM_CLI_ENABLE_TEST_API) +void cbm_index_supervisor_set_build_fingerprint_for_test(const char *fingerprint) { + if (!fingerprint || !worker_fingerprint_valid(fingerprint)) { + return; + } + g_build_fingerprint_capture_attempted = true; + (void)snprintf(g_build_fingerprint, sizeof(g_build_fingerprint), "%s", fingerprint); +} +#endif + static bool worker_fingerprint_valid(const char *fingerprint) { if (!fingerprint || strlen(fingerprint) != CBM_INDEX_WORKER_BUILD_FINGERPRINT_LENGTH) { return false; diff --git a/src/mcp/index_supervisor.h b/src/mcp/index_supervisor.h index 31259df90..cbc38f696 100644 --- a/src/mcp/index_supervisor.h +++ b/src/mcp/index_supervisor.h @@ -76,6 +76,16 @@ void cbm_index_worker_log_begin(const char *args_json, const char *repo_path); bool cbm_index_supervisor_capture_build_fingerprint(void); const char *cbm_index_supervisor_build_fingerprint(void); +#if defined(CBM_CLI_ENABLE_TEST_API) +/* Test-only: the runner captures a synthetic stub (see the capture seam) so + * spawned workers start instantly, but a daemon runtime service can only carry + * the REAL image hash. A fixture that runs a real daemon and drives the + * production activation guard against it aligns the captured value with that + * hash for its duration and restores the previous value afterwards. Invalid + * input is ignored. Never compiled into production. */ +void cbm_index_supervisor_set_build_fingerprint_for_test(const char *fingerprint); +#endif + typedef struct { const char *expected_build_fingerprint; const char *args_json; diff --git a/tests/test_cli.c b/tests/test_cli.c index 976280868..3a4b4a851 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -23,7 +23,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -39,6 +41,7 @@ #include #include #include +#include #endif #ifdef __APPLE__ #include @@ -47,6 +50,12 @@ #include #include +/* Same guarded fallback every product TU carries; CI injects the real value + * through CFLAGS_EXTRA for product objects and test objects alike. */ +#ifndef CBM_VERSION +#define CBM_VERSION "dev" +#endif + /* Internal prompt seam used to restore process-global state after command * tests that exercise --yes. */ void cbm_set_auto_answer_for_test(int value); @@ -1216,6 +1225,23 @@ TEST(cli_activation_quiesce_does_not_wait_on_bootstrap_startup) { test_rmdir_r(tmpdir); FAIL("runtime parent setup failed"); } + /* The participant models the daemon of THIS namespace: its cohort + * identity carries the fingerprint of the cache the install below + * targets, derived as main.c derives it. A foreign cache would (rightly) + * be left alone by the scoped guard. */ + char participant_cache[512]; + char participant_cache_canonical[1024]; + char participant_cache_fingerprint[CBM_SHA256_HEX_LEN + 1] = {0}; + snprintf(participant_cache, sizeof(participant_cache), "%s/cache", tmpdir); + if (!cbm_mkdir_p(participant_cache, 0700) || + !cbm_canonical_path(participant_cache, participant_cache_canonical, + sizeof(participant_cache_canonical))) { + test_rmdir_r(tmpdir); + FAIL("participant cache setup failed"); + } + cbm_normalize_path_sep(participant_cache_canonical); + cbm_sha256_hex(participant_cache_canonical, strlen(participant_cache_canonical), + participant_cache_fingerprint); int ready_pipe[2] = {-1, -1}; if (pipe(ready_pipe) != 0) { test_rmdir_r(tmpdir); @@ -1231,7 +1257,7 @@ TEST(cli_activation_quiesce_does_not_wait_on_bootstrap_startup) { cbm_daemon_build_identity_t identity = { .semantic_version = "cli-activation-test", .build_fingerprint = fingerprint, - .cache_fingerprint = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + .cache_fingerprint = participant_cache_fingerprint, .protocol_abi = CBM_DAEMON_RUNTIME_WIRE_ABI, .store_abi = 1, .feature_abi = 1, @@ -1467,6 +1493,381 @@ TEST(cli_install_recovers_markerless_stale_rendezvous) { ASSERT_TRUE(anchor_removed); PASS(); } +/* ── Activation-guard namespace scope (2026-09-08) ────────────────── + * The rendezvous directory is per OS ACCOUNT, never per HOME (service.h): + * every daemon of one user meets at one endpoint, and the cohort identity's + * cache fingerprint is what separates one HOME / CBM_CACHE_DIR namespace + * from another. An `install` into a second HOME (a sandbox, a second + * profile) therefore reaches the LIVE host daemon at the shared endpoint — + * and used to drain it, disconnecting every MCP client, although nothing + * that install touched belonged to the host namespace. + * + * The fixture models the host the way host.c builds it: a forked process + * that admits itself to the cohort with the host cache fingerprint, runs a + * real runtime service at the shared endpoint, and exits (releasing the + * cohort lease LAST, the real teardown order) once that service has been + * drained. The parent keeps one committed client connected to it across the + * install under test and asks the daemon itself afterwards. */ +#define CLI_SCOPE_HOST_DRAINED 3 +#define CLI_SCOPE_TIMEOUT_MS 5000U + +typedef struct { + char tmpdir[256]; + char runtime_parent[512]; + char host_home[512]; + char host_cache[512]; + char host_cache_fingerprint[CBM_SHA256_HEX_LEN + 1]; + char self_build[CBM_DAEMON_BUILD_FINGERPRINT_SIZE]; + char previous_supervisor_build[CBM_DAEMON_BUILD_FINGERPRINT_SIZE]; + char conflict_log[640]; + cbm_daemon_build_identity_t identity; + cbm_daemon_ipc_endpoint_t *endpoint; + cbm_daemon_runtime_client_t *client; + pid_t host; + int release_fd; + char *old_home; + char *old_cache; + char *old_shell; +} cli_scope_fixture_t; + +static _Noreturn void cli_scope_host_child(const cli_scope_fixture_t *fixture, int ready_fd, + int release_fd) { + cbm_daemon_ipc_endpoint_t *endpoint = + cbm_daemon_bootstrap_endpoint_new(fixture->runtime_parent); + cbm_version_cohort_manager_t *manager = + endpoint ? cbm_version_cohort_manager_new(endpoint) : NULL; + cbm_version_cohort_lease_t *lease = NULL; + cbm_daemon_conflict_t conflict; + cbm_daemon_runtime_service_t *service = NULL; + bool admitted = endpoint && manager && + cbm_version_cohort_acquire(manager, &fixture->identity, cbm_now_ms() + 45000U, + &lease, &conflict) == CBM_VERSION_COHORT_OK; + if (admitted) { + cbm_daemon_runtime_service_config_t config = { + .endpoint = endpoint, + .identity = fixture->identity, + .conflict_log_path = fixture->conflict_log, + .conflict_log_cap_bytes = 64U * 1024U, + .max_clients = 8, + .lease_timeout_ms = CLI_SCOPE_TIMEOUT_MS, + .request_timeout_ms = CLI_SCOPE_TIMEOUT_MS, + .shutdown_timeout_ms = CLI_SCOPE_TIMEOUT_MS, + /* Born permanent: only a drain/stop ends it, never the parent's + * client leaving, so "still alive" is a statement about the + * drain alone. */ + .permanent = true, + }; + service = cbm_daemon_runtime_service_start(&config); + } + bool ready_ok = + service && cbm_daemon_runtime_service_state(service) == CBM_DAEMON_RUNTIME_SERVICE_RUNNING; + char ready = ready_ok ? 'R' : 'E'; + (void)write(ready_fd, &ready, 1); + close(ready_fd); + bool drained = false; + uint64_t deadline = cbm_now_ms() + 120000U; + while (ready_ok && cbm_now_ms() < deadline) { + if (cbm_daemon_runtime_service_wait_exited(service, 50U)) { + drained = true; + break; + } + struct pollfd release_poll = {.fd = release_fd, .events = POLLIN, .revents = 0}; + if (poll(&release_poll, 1, 0) > 0) { + break; + } + } + uint64_t cleanup_deadline = cbm_now_ms() + 2U * CLI_SCOPE_TIMEOUT_MS; + if (service) { + if (!drained) { + (void)cbm_daemon_runtime_service_stop(service, CLI_SCOPE_TIMEOUT_MS); + } + while (!cbm_daemon_runtime_service_free(service) && cbm_now_ms() < cleanup_deadline) { + cbm_usleep(1000); + } + } + while (lease && cbm_version_cohort_lease_release(&lease) != CBM_PRIVATE_FILE_LOCK_OK && + cbm_now_ms() < cleanup_deadline) { + cbm_usleep(1000); + } + while (manager && cbm_version_cohort_manager_free(&manager) != CBM_PRIVATE_FILE_LOCK_OK && + cbm_now_ms() < cleanup_deadline) { + cbm_usleep(1000); + } + cbm_daemon_ipc_endpoint_free(endpoint); + close(release_fd); + _exit(!ready_ok ? 1 : drained ? CLI_SCOPE_HOST_DRAINED : 0); +} + +static bool cli_scope_fixture_start(cli_scope_fixture_t *fixture, const char *tag) { + memset(fixture, 0, sizeof(*fixture)); + fixture->host = -1; + fixture->release_fd = -1; + snprintf(fixture->tmpdir, sizeof(fixture->tmpdir), "/tmp/cli-guard-scope-%s-XXXXXX", tag); + if (!cbm_mkdtemp(fixture->tmpdir)) { + return false; + } + snprintf(fixture->runtime_parent, sizeof(fixture->runtime_parent), "%s/runtime", + fixture->tmpdir); + snprintf(fixture->host_home, sizeof(fixture->host_home), "%s/host", fixture->tmpdir); + snprintf(fixture->host_cache, sizeof(fixture->host_cache), "%s/cache", fixture->host_home); + snprintf(fixture->conflict_log, sizeof(fixture->conflict_log), "%s/conflicts.ndjson", + fixture->host_home); + /* The host identity's cache fingerprint is derived exactly as main.c does + * for a real daemon: resolved dir -> canonical path -> SHA-256. */ + char canonical_cache[1024]; + if (test_mkdirp(fixture->runtime_parent) != 0 || !cbm_mkdir_p(fixture->host_cache, 0700) || + !cbm_canonical_path(fixture->host_cache, canonical_cache, sizeof(canonical_cache))) { + return false; + } + cbm_normalize_path_sep(canonical_cache); + cbm_sha256_hex(canonical_cache, strlen(canonical_cache), fixture->host_cache_fingerprint); + if (!cbm_daemon_runtime_process_build_fingerprint((uint64_t)getpid(), fixture->self_build)) { + return false; + } + /* The guard claims the supervisor's captured build; the runner stubs that + * capture while a runtime service must carry the real image hash. Align + * the two for this fixture exactly as one production process has them. */ + const char *previous_supervisor_build = cbm_index_supervisor_build_fingerprint(); + snprintf(fixture->previous_supervisor_build, sizeof(fixture->previous_supervisor_build), "%s", + previous_supervisor_build ? previous_supervisor_build : ""); + cbm_index_supervisor_set_build_fingerprint_for_test(fixture->self_build); + fixture->identity = (cbm_daemon_build_identity_t){ + .semantic_version = CBM_VERSION, + .build_fingerprint = fixture->self_build, + .cache_fingerprint = fixture->host_cache_fingerprint, + .protocol_abi = CBM_DAEMON_RUNTIME_WIRE_ABI, + .store_abi = 1, + .feature_abi = 1, + }; + int ready_pipe[2] = {-1, -1}; + int release_pipe[2] = {-1, -1}; + if (pipe(ready_pipe) != 0) { + return false; + } + if (pipe(release_pipe) != 0) { + close(ready_pipe[0]); + close(ready_pipe[1]); + return false; + } + pid_t child = fork(); + if (child == 0) { + close(ready_pipe[0]); + close(release_pipe[1]); + cli_scope_host_child(fixture, ready_pipe[1], release_pipe[0]); + } + close(ready_pipe[1]); + close(release_pipe[0]); + char ready = 0; + bool host_ready = child > 0 && read(ready_pipe[0], &ready, 1) == 1 && ready == 'R'; + close(ready_pipe[0]); + fixture->host = child; + fixture->release_fd = release_pipe[1]; + cli_activation_save_env(&fixture->old_home, &fixture->old_cache); + const char *shell = getenv("SHELL"); + fixture->old_shell = shell ? strdup(shell) : NULL; + cbm_setenv("SHELL", "/bin/zsh", 1); + if (!host_ready) { + return false; + } + fixture->endpoint = cbm_daemon_bootstrap_endpoint_new(fixture->runtime_parent); + cbm_daemon_runtime_connect_result_t connect_result = {0}; + fixture->client = fixture->endpoint + ? cbm_daemon_runtime_client_connect(fixture->endpoint, &fixture->identity, + CLI_SCOPE_TIMEOUT_MS, &connect_result) + : NULL; + return fixture->client != NULL; +} + +/* Ask the host daemon itself: still running, not stopping, and the parent's + * committed client still admitted. */ +static bool cli_scope_host_serving(const cli_scope_fixture_t *fixture) { + cbm_daemon_runtime_status_t status = {0}; + return fixture->endpoint && + cbm_daemon_runtime_request_status(fixture->endpoint, &fixture->identity, + CLI_SCOPE_TIMEOUT_MS, &status) && + !status.stopping && status.committed_clients == 1; +} + +static int cli_scope_install(cli_scope_fixture_t *fixture, const char *home, const char *cache, + const char *bin_dir, bool skip_binary) { + cbm_setenv("HOME", home, 1); + cbm_setenv("CBM_CACHE_DIR", cache, 1); + cbm_cli_set_activation_runtime_parent_for_test(fixture->runtime_parent); + char dir_arg[704]; + snprintf(dir_arg, sizeof(dir_arg), "--dir=%s", bin_dir); + char *install_argv[] = {skip_binary ? "--skip-binary" : "--force", "--skip-config", "--yes", + dir_arg}; + int rc = cli_test_cmd_install(4, install_argv); + cbm_cli_set_activation_runtime_parent_for_test(g_cli_suite_runtime_parent); + cbm_set_auto_answer_for_test(0); + return rc; +} + +/* Returns the host child's exit status: 0 released intact, + * CLI_SCOPE_HOST_DRAINED when an activation drained it, -1 unknown. */ +static int cli_scope_fixture_finish(cli_scope_fixture_t *fixture) { + if (fixture->client) { + (void)cbm_daemon_runtime_client_close(fixture->client, CLI_SCOPE_TIMEOUT_MS); + fixture->client = NULL; + } + if (fixture->release_fd >= 0) { + /* Closing the write end is the signal (the child's poll sees POLLHUP); + * a write would raise SIGPIPE once a drained child is already gone. */ + close(fixture->release_fd); + fixture->release_fd = -1; + } + int host_exit = -1; + if (fixture->host > 0) { + int status = 0; + if (waitpid(fixture->host, &status, 0) == fixture->host && WIFEXITED(status)) { + host_exit = WEXITSTATUS(status); + } + fixture->host = -1; + } + cbm_daemon_ipc_endpoint_free(fixture->endpoint); + fixture->endpoint = NULL; + if (fixture->previous_supervisor_build[0]) { + cbm_index_supervisor_set_build_fingerprint_for_test(fixture->previous_supervisor_build); + } + if (fixture->old_shell) { + cbm_setenv("SHELL", fixture->old_shell, 1); + } else { + cbm_unsetenv("SHELL"); + } + free(fixture->old_shell); + fixture->old_shell = NULL; + cli_activation_restore_env(fixture->old_home, fixture->old_cache); + fixture->old_home = NULL; + fixture->old_cache = NULL; + test_rmdir_r(fixture->tmpdir); + return host_exit; +} + +static void cli_scope_foreign_paths(const cli_scope_fixture_t *fixture, char home[512], + char cache[576], char bin_dir[640], char activation_log[704]) { + snprintf(home, 512, "%s/sandbox", fixture->tmpdir); + snprintf(cache, 576, "%s/cache", home); + snprintf(bin_dir, 640, "%s/custom/bin", home); + snprintf(activation_log, 704, "%s/logs/activation-events.ndjson", cache); +} + +/* (a) `HOME= install --skip-binary` — the observed incident shape: + * the sandbox shares the account rendezvous, its cache namespace differs. */ +TEST(cli_install_skip_binary_into_foreign_home_never_drains_host_cohort) { + cli_scope_fixture_t fixture; + bool ready = cli_scope_fixture_start(&fixture, "skipbin"); + char foreign_home[512]; + char foreign_cache[576]; + char foreign_bin[640]; + char activation_log[704]; + cli_scope_foreign_paths(&fixture, foreign_home, foreign_cache, foreign_bin, activation_log); + bool prepared = ready && cbm_mkdir_p(foreign_home, 0700); + int install_rc = + prepared ? cli_scope_install(&fixture, foreign_home, foreign_cache, foreign_bin, true) : -1; + bool host_serving = prepared && cli_scope_host_serving(&fixture); + const char *events = read_test_file(activation_log); + bool completed = events && strstr(events, "\"phase\":\"completed\"") != NULL; + bool nothing_drained = events && strstr(events, "cohort drained") == NULL; + int host_exit = cli_scope_fixture_finish(&fixture); + + ASSERT_TRUE(ready); + ASSERT_EQ(install_rc, 0); + ASSERT_TRUE(host_serving); + ASSERT_EQ(host_exit, 0); + ASSERT_TRUE(completed); + ASSERT_TRUE(nothing_drained); + PASS(); +} + +/* (b) A full install (binary published into the sandbox bin dir) is scoped + * the same way: the host namespace is not what is being replaced. */ +TEST(cli_install_binary_into_foreign_home_never_drains_host_cohort) { + cli_scope_fixture_t fixture; + bool ready = cli_scope_fixture_start(&fixture, "binary"); + char foreign_home[512]; + char foreign_cache[576]; + char foreign_bin[640]; + char activation_log[704]; + cli_scope_foreign_paths(&fixture, foreign_home, foreign_cache, foreign_bin, activation_log); + char target_path[704]; + snprintf(target_path, sizeof(target_path), "%s/codebase-memory-mcp", foreign_bin); + bool prepared = ready && cbm_mkdir_p(foreign_home, 0700); + int install_rc = + prepared ? cli_scope_install(&fixture, foreign_home, foreign_cache, foreign_bin, false) + : -1; + bool host_serving = prepared && cli_scope_host_serving(&fixture); + struct stat target_status; + bool target_exists = stat(target_path, &target_status) == 0; + const char *events = read_test_file(activation_log); + bool completed = events && strstr(events, "\"phase\":\"completed\"") != NULL; + bool nothing_drained = events && strstr(events, "cohort drained") == NULL; + int host_exit = cli_scope_fixture_finish(&fixture); + + ASSERT_TRUE(ready); + ASSERT_EQ(install_rc, 0); + ASSERT_TRUE(target_exists); + ASSERT_TRUE(host_serving); + ASSERT_EQ(host_exit, 0); + ASSERT_TRUE(completed); + ASSERT_TRUE(nothing_drained); + PASS(); +} + +/* (c) Regression: an install that replaces the binary inside the host's own + * namespace still drains that cohort — and the audit names its client. */ +TEST(cli_install_into_host_namespace_still_drains_host_cohort) { + cli_scope_fixture_t fixture; + bool ready = cli_scope_fixture_start(&fixture, "host"); + char host_bin[640]; + char activation_log[704]; + snprintf(host_bin, sizeof(host_bin), "%s/custom/bin", fixture.host_home); + snprintf(activation_log, sizeof(activation_log), "%s/logs/activation-events.ndjson", + fixture.host_cache); + int install_rc = + ready ? cli_scope_install(&fixture, fixture.host_home, fixture.host_cache, host_bin, false) + : -1; + bool host_serving = ready && cli_scope_host_serving(&fixture); + const char *events = read_test_file(activation_log); + bool drained_in_log = events && strstr(events, "cohort drained") != NULL && + strstr(events, "\"daemon_active_clients\":1") != NULL; + int host_exit = cli_scope_fixture_finish(&fixture); + + ASSERT_TRUE(ready); + ASSERT_EQ(install_rc, 0); + ASSERT_FALSE(host_serving); + ASSERT_EQ(host_exit, CLI_SCOPE_HOST_DRAINED); + ASSERT_TRUE(drained_in_log); + PASS(); +} + +/* (d) `--skip-binary` with nothing to publish (no binary at the target, no + * index reset) replaces nothing, so even the host's own namespace has + * nothing to quiesce: agent configs are refreshed, sessions stay up. */ +TEST(cli_install_skip_binary_unchanged_in_host_namespace_quiesces_nothing) { + cli_scope_fixture_t fixture; + bool ready = cli_scope_fixture_start(&fixture, "unchanged"); + char absent_bin[640]; + char activation_log[704]; + snprintf(absent_bin, sizeof(absent_bin), "%s/absent/bin", fixture.host_home); + snprintf(activation_log, sizeof(activation_log), "%s/logs/activation-events.ndjson", + fixture.host_cache); + int install_rc = + ready ? cli_scope_install(&fixture, fixture.host_home, fixture.host_cache, absent_bin, true) + : -1; + bool host_serving = ready && cli_scope_host_serving(&fixture); + const char *events = read_test_file(activation_log); + bool completed = events && strstr(events, "\"phase\":\"completed\"") != NULL; + bool nothing_drained = events && strstr(events, "cohort drained") == NULL; + int host_exit = cli_scope_fixture_finish(&fixture); + + ASSERT_TRUE(ready); + ASSERT_EQ(install_rc, 0); + ASSERT_TRUE(host_serving); + ASSERT_EQ(host_exit, 0); + ASSERT_TRUE(completed); + ASSERT_TRUE(nothing_drained); + PASS(); +} #endif TEST(cli_install_force_quiesces_active_cohort_before_replacing_binary) { @@ -14774,6 +15175,10 @@ SUITE(cli) { RUN_TEST(cli_activation_cleanup_failure_fail_stops_before_lease_release); RUN_TEST(cli_activation_quiesce_does_not_wait_on_bootstrap_startup); RUN_TEST(cli_install_recovers_markerless_stale_rendezvous); + RUN_TEST(cli_install_skip_binary_into_foreign_home_never_drains_host_cohort); + RUN_TEST(cli_install_binary_into_foreign_home_never_drains_host_cohort); + RUN_TEST(cli_install_into_host_namespace_still_drains_host_cohort); + RUN_TEST(cli_install_skip_binary_unchanged_in_host_namespace_quiesces_nothing); #endif RUN_TEST(cli_install_force_quiesces_active_cohort_before_replacing_binary); RUN_TEST(cli_install_dir_and_skip_config_stage_first_install_safely); From 7bf366f63b3fd5211cec19e483ee2ed27145ca14 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 8 Sep 2026 20:30:17 +0200 Subject: [PATCH 032/130] fix(cli): guard the POSIX-only edit-path note recorder from the Windows build config_edit_path.c's edit_path_note is defined before the '#ifndef _WIN32' follow-resolver block but only called inside it, so the Windows build compiled it dead and failed under -Werror,-Wunused-function (pr-smoke windows-latest). Windows never follows a symlinked config, so guard the recorder with the same condition as its callers. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- src/cli/config_edit_path.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/cli/config_edit_path.c b/src/cli/config_edit_path.c index b51c5c2a2..ec26a5534 100644 --- a/src/cli/config_edit_path.c +++ b/src/cli/config_edit_path.c @@ -35,11 +35,16 @@ static void edit_path_target_reset(cbm_config_edit_target_t *target) { target->path[0] = '\0'; } +#ifndef _WIN32 +/* Only the POSIX symlink-follow resolver records a refusal reason; the Windows + * build never follows, so guard this with the same condition as its callers to + * avoid -Werror,-Wunused-function (#2110 CI). */ static void edit_path_note(char *reason, size_t reason_size, const char *text) { if (reason && reason_size > 0U) { (void)snprintf(reason, reason_size, "%s", text); } } +#endif /* !_WIN32 */ void cbm_config_edit_target_close(cbm_config_edit_target_t *target) { if (!target) { From 208ee99c86631d2af810b212d69f8a2db5f1c93e Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 8 Sep 2026 20:38:22 +0200 Subject: [PATCH 033/130] fix(mcp): bound the non-git auto_index guard by discovery count, not git ls-files (#713) auto_index_limit was bypassed for a non-git root because the admission guard counted `git ls-files`, which is 0 outside a repo, so the whole tree was walked into tens of GB of RAM. Bound the check by the discovery layer's bounded count with the limit as the ceiling; a non-git root over the limit is refused with the limit named, and git roots are unchanged. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- src/mcp/mcp.c | 2 +- tests/test_mcp.c | 144 ++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 126 insertions(+), 20 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 7aa40605c..87df93e60 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -17523,7 +17523,7 @@ static void maybe_auto_index(cbm_mcp_server_t *srv) { (void)snprintf(limit, sizeof(limit), "%d", file_limit); cbm_log_warn("autoindex.skip", "reason", file_count >= 0 ? "too_many_files" : "unsafe_or_unavailable_path", "files", - files, "limit", limit); + files, "limit", limit, "root", srv->session_root); return; } diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 6fbaaf66c..3234542c5 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -18943,22 +18943,72 @@ TEST(mcp_auto_watch_false_skips_watcher_on_connect) { } /* ══════════════════════════════════════════════════════════════════ - * #1466 — autoindex.skip must report the effective numeric limit + * #1466 / #713 — the auto_index_limit guard + * + * #1466: autoindex.skip must report the effective numeric limit. + * #713: the guard must bound NON-git roots. The 0.8.x guard counted + * `git ls-files | wc -l`, which is 0 outside a checkout, so a plain + * directory of 60k files was walked in full (tens of GB RSS). The + * bounded discovery count applies to every root, and the skip line + * names the limit AND the root so the reporter can act on it. * ══════════════════════════════════════════════════════════════════ */ static char autoindex_skip_log[1024]; +static bool autoindex_saw_done; -/* Keeps only the too_many_files skip line, so later lines cannot displace it. */ -static void autoindex_skip_capture_log(const char *line) { - if (line && strstr(line, "msg=autoindex.skip") && strstr(line, "reason=too_many_files")) { +/* Keeps only the too_many_files skip line, so later lines cannot displace it, + * and records whether an admitted auto-index actually ran to completion. */ +static void autoindex_limit_capture_log(const char *line) { + if (!line) { + return; + } + if (strstr(line, "msg=autoindex.skip") && strstr(line, "reason=too_many_files")) { snprintf(autoindex_skip_log, sizeof(autoindex_skip_log), "%s", line); } + if (strstr(line, "msg=autoindex.done")) { + autoindex_saw_done = true; + } } -/* Drive initialize → maybe_auto_index over a fresh project holding more tracked - * files than auto_index_limit, and capture the resulting skip warning. +typedef struct { + int files; /* indexable .py files written into the root */ + int limit; /* auto_index_limit */ + bool git_root; /* emulate a checkout via /.git/HEAD (no git binary) */ +} autoindex_limit_probe_t; + +static bool autoindex_limit_write_fixture(const char *repodir, + const autoindex_limit_probe_t *probe) { + if (th_mkdir_p(repodir) != 0) { + return false; + } + for (int i = 0; i < probe->files; i++) { + char path[640]; + char body[96]; + snprintf(path, sizeof(path), "%s/f%d.py", repodir, i); + snprintf(body, sizeof(body), "def f%d():\n return %d\n", i, i); + if (th_write_file(path, body) != 0) { + return false; + } + } + if (probe->git_root) { + char gitdir[640]; + char head[700]; + snprintf(gitdir, sizeof(gitdir), "%s/.git", repodir); + snprintf(head, sizeof(head), "%s/HEAD", gitdir); + if (th_mkdir_p(gitdir) != 0 || th_write_file(head, "ref: refs/heads/main\n") != 0) { + return false; + } + } + return true; +} + +/* Drive initialize → maybe_auto_index over a fresh root holding probe->files + * indexable files under auto_index_limit=probe->limit. skip_out receives the + * too_many_files warning (empty when admitted); *done_out reports whether an + * admitted auto-index ran to completion (the server free joins the thread). * Returns false on fixture setup failure. */ -static bool autoindex_skip_warning(char *out, size_t out_size) { +static bool autoindex_limit_probe(const autoindex_limit_probe_t *probe, char *skip_out, + size_t skip_size, bool *done_out) { char cache[256]; snprintf(cache, sizeof(cache), "%s/cbm-autoindex-limit-XXXXXX", cbm_tmpdir()); if (!cbm_mkdtemp(cache)) { @@ -18967,12 +19017,7 @@ static bool autoindex_skip_warning(char *out, size_t out_size) { char repodir[512]; snprintf(repodir, sizeof(repodir), "%s/repo", cache); - char file_a[640]; - char file_b[640]; - snprintf(file_a, sizeof(file_a), "%s/a.py", repodir); - snprintf(file_b, sizeof(file_b), "%s/b.py", repodir); - if (th_mkdir_p(repodir) != 0 || th_write_file(file_a, "def a():\n return 1\n") != 0 || - th_write_file(file_b, "def b():\n return 2\n") != 0) { + if (!autoindex_limit_write_fixture(repodir, probe)) { th_rmtree(cache); return false; } @@ -18992,27 +19037,31 @@ static bool autoindex_skip_warning(char *out, size_t out_size) { bool ok = false; cbm_config_t *cfg = cbm_config_open(cache); if (cfg) { + char limit[32]; + snprintf(limit, sizeof(limit), "%d", probe->limit); cbm_config_set(cfg, CBM_CONFIG_AUTO_INDEX, "true"); - cbm_config_set(cfg, CBM_CONFIG_AUTO_INDEX_LIMIT, "1"); + cbm_config_set(cfg, CBM_CONFIG_AUTO_INDEX_LIMIT, limit); cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); if (srv) { autoindex_skip_log[0] = '\0'; + autoindex_saw_done = false; CBMLogLevel prev_level = cbm_log_get_level(); - cbm_log_set_level(CBM_LOG_WARN); + cbm_log_set_level(CBM_LOG_INFO); cbm_log_set_format(CBM_LOG_FORMAT_TEXT); - cbm_log_set_sink_ex(autoindex_skip_capture_log, CBM_LOG_SINK_REPLACE); + cbm_log_set_sink_ex(autoindex_limit_capture_log, CBM_LOG_SINK_REPLACE); cbm_mcp_server_set_config(srv, cfg); char *resp = cbm_mcp_server_handle( srv, "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"initialize\",\"params\":{}}"); free(resp); - cbm_mcp_server_free(srv); + cbm_mcp_server_free(srv); /* joins an admitted autoindex thread */ cbm_log_set_sink(NULL); cbm_log_set_level(prev_level); - snprintf(out, out_size, "%s", autoindex_skip_log); + snprintf(skip_out, skip_size, "%s", autoindex_skip_log); + *done_out = autoindex_saw_done; ok = true; } cbm_config_close(cfg); @@ -19028,8 +19077,10 @@ static bool autoindex_skip_warning(char *out, size_t out_size) { /* RED before the fix: the warning carries `limit=auto_index_limit`, the config * key constant, instead of the configured value. */ TEST(autoindex_skip_reports_numeric_limit_issue1466) { + autoindex_limit_probe_t probe = {.files = 2, .limit = 1, .git_root = false}; char warning[1024]; - if (!autoindex_skip_warning(warning, sizeof(warning))) { + bool done = false; + if (!autoindex_limit_probe(&probe, warning, sizeof(warning), &done)) { PASS(); /* fixture setup failed (tmpdir/cwd unavailable) — skip */ } /* Not vacuous: the skip path must actually have been taken. */ @@ -19038,6 +19089,58 @@ TEST(autoindex_skip_reports_numeric_limit_issue1466) { ASSERT_NOT_NULL(strstr(warning, "files=2")); ASSERT_NOT_NULL(strstr(warning, "limit=1")); ASSERT_NULL(strstr(warning, "limit=auto_index_limit")); + ASSERT_FALSE(done); + PASS(); +} + +/* #713 at the reporter's shape scaled down (60 files vs limit 50, from + * 60k vs 50k): a PLAIN directory over the limit is refused, and the line + * names the limit and the root. RED on the `git ls-files` guard (count 0 → + * admitted → indexed) and RED while the line does not name the root. */ +TEST(autoindex_limit_guards_non_git_root_issue713) { + autoindex_limit_probe_t probe = {.files = 60, .limit = 50, .git_root = false}; + char warning[1024]; + bool done = false; + if (!autoindex_limit_probe(&probe, warning, sizeof(warning), &done)) { + PASS(); /* fixture setup failed (tmpdir/cwd unavailable) — skip */ + } + ASSERT_NOT_NULL(strstr(warning, "msg=autoindex.skip")); + ASSERT_NOT_NULL(strstr(warning, "reason=too_many_files")); + ASSERT_NOT_NULL(strstr(warning, "limit=50")); + const char *root = strstr(warning, "root="); + ASSERT_NOT_NULL(root); + ASSERT_NOT_NULL(strstr(root, "/repo")); + ASSERT_FALSE(done); + PASS(); +} + +/* The same plain directory one file UNDER the limit is admitted and indexed: + * the guard bounds, it does not fail closed on every non-git root. */ +TEST(autoindex_limit_admits_non_git_root_under_limit_issue713) { + autoindex_limit_probe_t probe = {.files = 49, .limit = 50, .git_root = false}; + char warning[1024]; + bool done = false; + if (!autoindex_limit_probe(&probe, warning, sizeof(warning), &done)) { + PASS(); /* fixture setup failed (tmpdir/cwd unavailable) — skip */ + } + ASSERT(warning[0] == '\0'); + ASSERT_TRUE(done); + PASS(); +} + +/* A checkout keeps its behaviour: one bounded count for both root kinds. */ +TEST(autoindex_limit_guards_git_root_issue713) { + autoindex_limit_probe_t probe = {.files = 60, .limit = 50, .git_root = true}; + char warning[1024]; + bool done = false; + if (!autoindex_limit_probe(&probe, warning, sizeof(warning), &done)) { + PASS(); /* fixture setup failed (tmpdir/cwd unavailable) — skip */ + } + ASSERT_NOT_NULL(strstr(warning, "msg=autoindex.skip")); + ASSERT_NOT_NULL(strstr(warning, "reason=too_many_files")); + ASSERT_NOT_NULL(strstr(warning, "limit=50")); + ASSERT_NOT_NULL(strstr(warning, "root=")); + ASSERT_FALSE(done); PASS(); } @@ -20320,6 +20423,9 @@ SUITE(mcp) { RUN_TEST(mcp_auto_watch_false_skips_watcher_on_connect); RUN_TEST(mcp_auto_watch_false_skips_supervised_autoindex_issue853); RUN_TEST(autoindex_skip_reports_numeric_limit_issue1466); + RUN_TEST(autoindex_limit_guards_non_git_root_issue713); + RUN_TEST(autoindex_limit_admits_non_git_root_under_limit_issue713); + RUN_TEST(autoindex_limit_guards_git_root_issue713); } /* Kept separate so daemon-coordination regressions can be iterated without From 3a4160b4610f83ea26daf69f1ffe7cb898bf48ad Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 8 Sep 2026 20:38:22 +0200 Subject: [PATCH 034/130] fix(cli): complete the OpenHands integration (settings.json mcp_config + agent-profile refs) (#1826) install wrote only part of the OpenHands config. Register the server globally under ~/.openhands/settings.json -> mcp_config {transport:stdio, command, enabled} (owned-entry, unrelated content preserved), and add "codebase-memory-mcp" to each ~/.openhands/agent-profiles/

.json mcp_server_refs array (null -> [it], existing -> appended once). uninstall removes exactly what it added. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- src/cli/cli.c | 209 ++++++++++++++++++++++++++++ src/cli/config_json_like.c | 66 ++++++++- src/cli/config_json_like.h | 5 + tests/test_cli.c | 248 ++++++++++++++++++++++++++++++++++ tests/test_config_json_like.c | 173 ++++++++++++++++++++++++ 5 files changed, 696 insertions(+), 5 deletions(-) diff --git a/src/cli/cli.c b/src/cli/cli.c index d85ebf0b6..21bb36d65 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -2282,6 +2282,141 @@ static int cbm_remove_openclaw_compaction(const char *config_path) { : CLI_ERR; } +/* ── OpenHands settings.json mcp_config (#1826) ─────────────── + * The mcpServers-style config installed above (cbm_install_editor_mcp into + * ~/.openhands/mcp.json) is not enough: OpenHands only loads a global MCP + * server it finds registered under settings.json -> mcp_config, in its own + * shape ({transport, command, enabled} — no args array). Agent profiles then + * opt in individually via mcp_server_refs (below). */ + +static size_t cbm_openhands_ownership_fields(cbm_json_like_object_field_t fields[3]) { + fields[0] = (cbm_json_like_object_field_t){ + .key = "transport", + .shape = CBM_JSON_LIKE_VALUE_STRING, + .expected_string = "stdio", + .flags = CBM_JSON_LIKE_FIELD_REQUIRED, + }; + fields[1] = (cbm_json_like_object_field_t){ + .key = "command", + .shape = CBM_JSON_LIKE_VALUE_STRING, + .expected_string = NULL, + .flags = CBM_JSON_LIKE_FIELD_REQUIRED | CBM_JSON_LIKE_FIELD_CAPTURE_STRING, + }; + fields[2] = (cbm_json_like_object_field_t){ + .key = "enabled", + .shape = CBM_JSON_LIKE_VALUE_LITERAL, + .expected_string = "true", + .flags = CBM_JSON_LIKE_FIELD_REQUIRED, + }; + return 3U; +} + +static char *cbm_build_openhands_mcp_entry(const char *binary_path) { + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + if (!doc) { + return NULL; + } + yyjson_mut_val *root = yyjson_mut_obj(doc); + bool ok = root && yyjson_mut_obj_add_strcpy(doc, root, "transport", "stdio") && + yyjson_mut_obj_add_strcpy(doc, root, "command", binary_path) && + yyjson_mut_obj_add_bool(doc, root, "enabled", true); + char *json = NULL; + if (ok) { + yyjson_mut_doc_set_root(doc, root); + json = yyjson_mut_write(doc, YYJSON_WRITE_NOFLAG, NULL); + } + yyjson_mut_doc_free(doc); + return json; +} + +/* Insert or leave alone: an already-owned entry (exact match, or annotated + * with extra keys the client added) needs no write. A same-named entry that + * does not match our shape is left untouched and reported as an error rather + * than overwritten, matching cbm_upsert_json_named_mcp's fail-closed rule for + * every other editor client. */ +static int cbm_upsert_openhands_settings_mcp(const char *binary_path, const char *settings_path) { + if (!binary_path || !settings_path) { + return CLI_ERR; + } + static const char *const path[] = {"mcp_config"}; + char *document = NULL; + size_t document_length = 0U; + int read_result = cbm_json_like_read_document(settings_path, &document, &document_length); + if (read_result < 0) { + return CLI_ERR; + } + if (read_result == 0) { + cbm_json_like_object_field_t fields[3]; + size_t field_count = cbm_openhands_ownership_fields(fields); + char *command = NULL; + int ownership = + cbm_json_like_match_object_entry(document, document_length, path, 1U, + CBM_DEFAULT_MCP_SERVER_NAME, fields, field_count, + &command); + free(command); + if (ownership == CBM_JSON_LIKE_OBJECT_MATCH || + ownership == CBM_JSON_LIKE_OBJECT_MATCH_WITH_EXTRAS) { + free(document); + return CLI_OK; + } + if (ownership != CBM_JSON_LIKE_OBJECT_MISSING) { + free(document); + return CLI_ERR; + } + } + char *entry = cbm_build_openhands_mcp_entry(binary_path); + if (!entry) { + free(document); + return CLI_ERR; + } + int edit_result = cbm_json_like_upsert_entry_if_unchanged( + settings_path, path, 1U, CBM_DEFAULT_MCP_SERVER_NAME, entry, + read_result == 1 ? NULL : document, document_length); + free(entry); + free(document); + return edit_result == 0 ? CLI_OK : CLI_ERR; +} + +/* Remove only an entry that is still recognisably ours (an annotated entry is + * left in place and reported, same rule as insertion above); a missing file, + * path, or entry is a successful no-op. */ +static int cbm_remove_openhands_settings_mcp(const char *settings_path) { + if (!settings_path) { + return CLI_ERR; + } + static const char *const path[] = {"mcp_config"}; + char *document = NULL; + size_t document_length = 0U; + int read_result = cbm_json_like_read_document(settings_path, &document, &document_length); + if (read_result == 1) { + free(document); + return CLI_OK; + } + if (read_result < 0) { + free(document); + return CLI_ERR; + } + cbm_json_like_object_field_t fields[3]; + size_t field_count = cbm_openhands_ownership_fields(fields); + char *command = NULL; + int ownership = cbm_json_like_match_object_entry(document, document_length, path, 1U, + CBM_DEFAULT_MCP_SERVER_NAME, fields, + field_count, &command); + free(command); + if (ownership == CBM_JSON_LIKE_OBJECT_MISSING || ownership == CBM_JSON_LIKE_OBJECT_MISMATCH) { + free(document); + return CLI_OK; + } + if (ownership != CBM_JSON_LIKE_OBJECT_MATCH) { + free(document); + return CLI_ERR; + } + int edit_result = cbm_json_like_remove_entry_if_unchanged( + settings_path, path, 1U, CBM_DEFAULT_MCP_SERVER_NAME, document, document_length); + free(document); + return edit_result == 0 ? CLI_OK : CLI_ERR; +} + /* ── VS Code MCP (servers key with type:stdio) ────────────────── */ int cbm_install_vscode_mcp(const char *binary_path, const char *config_path) { @@ -9228,6 +9363,49 @@ static void uninstall_vscode_profile_configs(const char *code_user, const char * cbm_closedir(directory); } +static bool cbm_filename_has_suffix(const char *name, const char *suffix) { + size_t name_len = strlen(name); + size_t suffix_len = strlen(suffix); + return name_len >= suffix_len && strcmp(name + (name_len - suffix_len), suffix) == 0; +} + +/* Register or unregister our server against every existing OpenHands agent + * profile's mcp_server_refs array (#1826). A missing agent-profiles/ + * directory is a silent no-op in both directions — install must never invent + * it, and uninstall has nothing to undo there. Only *.json entries are + * touched; a profile directory may hold arbitrary notes alongside profiles. */ +static void openhands_update_profile_refs(const char *profiles_dir, bool installing, + bool dry_run) { + cbm_dir_t *d = cbm_opendir(profiles_dir); + if (!d) { + return; + } + cbm_dirent_t *ent; + while ((ent = cbm_readdir(d)) != NULL) { + if (strcmp(ent->name, ".") == 0 || strcmp(ent->name, "..") == 0 || + !cbm_filename_has_suffix(ent->name, ".json")) { + continue; + } + char profile_path[CLI_BUF_1K]; + snprintf(profile_path, sizeof(profile_path), "%s/%s", profiles_dir, ent->name); + struct stat state; + if (stat(profile_path, &state) != 0 || !S_ISREG(state.st_mode) || dry_run) { + continue; + } + int result = installing ? cbm_json_like_add_unique_string( + profile_path, "mcp_server_refs", CBM_DEFAULT_MCP_SERVER_NAME) + : cbm_json_like_remove_string(profile_path, "mcp_server_refs", + CBM_DEFAULT_MCP_SERVER_NAME); + if (result != CLI_OK) { + record_agent_config_error(!installing, "OpenHands", + installing ? "profile_refs_install" + : "profile_refs_uninstall", + profile_path); + } + } + cbm_closedir(d); +} + /* Install MCP configs for editor-based agents (Zed, KiloCode, VS Code, OpenClaw). */ static void install_editor_agent_configs(const cbm_detected_agents_t *agents, const char *home, const char *binary_path, bool force, bool dry_run) { @@ -9490,11 +9668,32 @@ static void install_additional_agent_configs(const cbm_detected_agents_t *agents if (agents->openhands) { char cp[CLI_BUF_1K]; char skills_dir[CLI_BUF_1K]; + char settings_path[CLI_BUF_1K]; + char profiles_dir[CLI_BUF_1K]; snprintf(cp, sizeof(cp), "%s/.openhands/mcp.json", home); snprintf(skills_dir, sizeof(skills_dir), "%s/.agents/skills", home); + snprintf(settings_path, sizeof(settings_path), "%s/.openhands/settings.json", home); + snprintf(profiles_dir, sizeof(profiles_dir), "%s/.openhands/agent-profiles", home); install_generic_agent_config("OpenHands", binary_path, cp, NULL, dry_run, cbm_install_editor_mcp); install_agent_skill("OpenHands", skills_dir, force, dry_run); + /* #1826: the mcpServers-shaped mcp.json above is not enough — OpenHands + * only loads a server registered under settings.json -> mcp_config, and + * only for agent profiles that reference it. agent-profiles/ is never + * invented; a missing directory means nothing to register into yet. */ + if (g_install_plan) { + plan_record("OpenHands", "mcp_config", settings_path); + } else { + if (!dry_run) { + if (!prepare_config_parent(settings_path) || + cbm_upsert_openhands_settings_mcp(binary_path, settings_path) != CLI_OK) { + record_agent_config_error(false, "OpenHands", "settings_mcp_install", + settings_path); + } + } + printf(" settings mcp_config: %s\n", settings_path); + openhands_update_profile_refs(profiles_dir, true, dry_run); + } } if (agents->augment) { char cp[CLI_BUF_1K]; @@ -11750,11 +11949,21 @@ static void uninstall_additional_agents(const cbm_detected_agents_t *agents, con if (agents->openhands) { char cp[CLI_BUF_1K]; char skills_dir[CLI_BUF_1K]; + char settings_path[CLI_BUF_1K]; + char profiles_dir[CLI_BUF_1K]; snprintf(cp, sizeof(cp), "%s/.openhands/mcp.json", home); snprintf(skills_dir, sizeof(skills_dir), "%s/.agents/skills", home); + snprintf(settings_path, sizeof(settings_path), "%s/.openhands/settings.json", home); + snprintf(profiles_dir, sizeof(profiles_dir), "%s/.openhands/agent-profiles", home); uninstall_agent_mcp_instr((mcp_uninstall_args_t){"OpenHands", cp, NULL}, dry_run, cbm_remove_editor_mcp_owned); printf(" removed %d skill(s)\n", cbm_remove_skills(skills_dir, dry_run)); + /* #1826 counterpart: undo the settings.json registration and every + * agent profile's mcp_server_refs entry the install above added. */ + if (!dry_run && cbm_remove_openhands_settings_mcp(settings_path) != CLI_OK) { + record_agent_config_error(true, "OpenHands", "settings_mcp_uninstall", settings_path); + } + openhands_update_profile_refs(profiles_dir, false, dry_run); } if (agents->augment) { char cp[CLI_BUF_1K]; diff --git a/src/cli/config_json_like.c b/src/cli/config_json_like.c index dc16c3c5c..3ac7802a3 100644 --- a/src/cli/config_json_like.c +++ b/src/cli/config_json_like.c @@ -1223,8 +1223,19 @@ static int jl_make_insertion(const char *text, size_t length, size_t object_star return 0; } - if (object->close_pos == gap_start || - !jl_is_space((unsigned char)text[object->close_pos - 1U])) { + /* Zero-width insertion right at close_pos: whatever byte already sits at + * close_pos - 1 is untouched and stays in the output. When the original + * had a real gap there (close_pos != gap_start, e.g. "[ \"a\" ]"), that + * preserved byte already supplies a separator on the leading side, so + * mirroring it on the trailing side (space before the bracket) matches + * the array's own loose style. The fully tight case ("[\"a\"]", nothing + * at all between the last value and the bracket) has no such byte to + * lean on: manufacture ONE space so `,new` reads `, new` — the + * comma-spacing convention every other insertion path uses — and add + * nothing after, so `new]` stays `new]` rather than gaining a trailing + * space the original never had (#1826 byte-for-byte round-trip). */ + bool tight = object->close_pos == gap_start; + if (tight || !jl_is_space((unsigned char)text[object->close_pos - 1U])) { if (jl_buffer_char(insertion, ' ') != 0) { return -1; } @@ -1235,7 +1246,7 @@ static int jl_make_insertion(const char *text, size_t length, size_t object_star if (object->trailing_comma && jl_buffer_char(insertion, ',') != 0) { return -1; } - return jl_buffer_char(insertion, ' '); + return tight ? 0 : jl_buffer_char(insertion, ' '); } static int jl_apply_edits(const char *source, size_t source_length, jl_edit_t *edits, @@ -2406,6 +2417,13 @@ int cbm_json_like_remove_entry_if_unchanged(const char *file_path, const char *c expected_length); } +/* True when [start, end) is the bare 4-byte literal token null (never a + * quoted "null" string, which is 6 bytes with the quotes) — the documented + * OpenHands "no list yet" shape for mcp_server_refs (#1826). */ +static bool jl_is_null_literal(const char *text, size_t start, size_t end) { + return end - start == 4U && memcmp(text + start, "null", 4U) == 0; +} + int cbm_json_like_add_unique_string_at_path(const char *file_path, const char *const *object_path, size_t path_len, const char *array_key, const char *string_value) { @@ -2497,6 +2515,24 @@ int cbm_json_like_add_unique_string_at_path(const char *file_path, const char *c result = jl_insert_member(source, source_length, object_start, &object, object_path, path_len, SIZE_MAX, array_key, array_json.data, array_json.length, &updated, &updated_length); + } else if (jl_is_null_literal(source, object.match.value_start, object.match.value_end)) { + /* OpenHands profiles ship `"mcp_server_refs": null` as the documented + * "no list yet" shape (#1826). Splice the built one-element array over + * the bare null token in place, preserving every other byte (trailing + * comments included) the way the object-member and array-element + * edits below already do for their own value spans. */ + size_t head = object.match.value_start; + size_t tail = object.match.value_end; + updated_length = head + array_json.length + (source_length - tail); + updated = (char *)malloc(updated_length + 1U); + if (!updated) { + result = -1; + } else { + memcpy(updated, source, head); + memcpy(updated + head, array_json.data, array_json.length); + memcpy(updated + head + array_json.length, source + tail, source_length - tail); + updated[updated_length] = '\0'; + } } else if (source[object.match.value_start] != '[') { result = -1; } else { @@ -2759,6 +2795,24 @@ static int jl_decode_field_string(const char *text, size_t start, size_t end, } return jl_decode_string_value(text, start, end, value_out) == 0 ? 0 : 1; } + if (shape == CBM_JSON_LIKE_VALUE_LITERAL) { + /* The value's token boundaries already exclude surrounding trivia + * (jl_parse_value advances pos past exactly the token). Copy the raw + * bytes verbatim so a quoted "true" never equals the bare literal + * true, and the caller's expected_string comparison decides match. */ + if (start >= end) { + return 1; + } + size_t length = end - start; + char *copy = (char *)malloc(length + 1U); + if (!copy) { + return 1; + } + memcpy(copy, text + start, length); + copy[length] = '\0'; + *value_out = copy; + return 0; + } if (shape != CBM_JSON_LIKE_VALUE_SINGLE_STRING_ARRAY || start >= end || text[start] != '[') { return 1; } @@ -2827,11 +2881,13 @@ int cbm_json_like_match_object_entry(const char *document, size_t document_lengt size_t capture_count = 0U; for (size_t i = 0U; i < field_count; ++i) { if (!fields[i].key || fields[i].key[0] == '\0' || - fields[i].shape > CBM_JSON_LIKE_VALUE_SINGLE_STRING_ARRAY || + fields[i].shape > CBM_JSON_LIKE_VALUE_LITERAL || (fields[i].flags & ~(CBM_JSON_LIKE_FIELD_REQUIRED | CBM_JSON_LIKE_FIELD_CAPTURE_STRING)) != 0U || ((fields[i].flags & CBM_JSON_LIKE_FIELD_CAPTURE_STRING) != 0U && - fields[i].shape == CBM_JSON_LIKE_VALUE_EMPTY_ARRAY)) { + (fields[i].shape == CBM_JSON_LIKE_VALUE_EMPTY_ARRAY || + fields[i].shape == CBM_JSON_LIKE_VALUE_LITERAL)) || + (fields[i].shape == CBM_JSON_LIKE_VALUE_LITERAL && !fields[i].expected_string)) { return -1; } capture_count += (fields[i].flags & CBM_JSON_LIKE_FIELD_CAPTURE_STRING) != 0U ? 1U : 0U; diff --git a/src/cli/config_json_like.h b/src/cli/config_json_like.h index 23a8687e5..57cbfbfbf 100644 --- a/src/cli/config_json_like.h +++ b/src/cli/config_json_like.h @@ -47,6 +47,11 @@ typedef enum { CBM_JSON_LIKE_VALUE_STRING, CBM_JSON_LIKE_VALUE_EMPTY_ARRAY, CBM_JSON_LIKE_VALUE_SINGLE_STRING_ARRAY, + /* An exact bare token (true/false/null/a number) matched byte-for-byte + * against expected_string. Requires expected_string and may not be + * combined with CBM_JSON_LIKE_FIELD_CAPTURE_STRING — a fixed token has + * nothing meaningful to capture for the caller. */ + CBM_JSON_LIKE_VALUE_LITERAL, } cbm_json_like_value_shape_t; enum { diff --git a/tests/test_cli.c b/tests/test_cli.c index 976280868..a8d793644 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -3814,6 +3814,251 @@ TEST(cli_openclaw_uninstall_removes_compaction_when_workspace_is_ambiguous) { PASS(); } +/* ═══════════════════════════════════════════════════════════════════ + * OpenHands settings.json + agent-profiles registration (#1826) + * + * A bare ~/.openhands/mcp.json is not enough: OpenHands registers global MCP + * servers under settings.json → mcp_config, and every agent profile under + * agent-profiles/ must list the server in mcp_server_refs before that agent + * may use it. Foreign content survives byte-for-byte; uninstall removes + * exactly what install added. + * ═══════════════════════════════════════════════════════════════════ */ + +typedef struct { + char home[256]; + char binary[512]; /* the path `uninstall` expects: /.local/bin/... */ + char settings[640]; + char mcp[640]; + char profiles_dir[640]; + char default_profile[768]; + char custom_profile[768]; + char note[768]; + char *saved_home; + char *saved_path; + char *saved_cache; +} openhands_fixture_t; + +static bool openhands_fixture_open(openhands_fixture_t *fx, bool with_profiles) { + snprintf(fx->home, sizeof(fx->home), "/tmp/cli-openhands-XXXXXX"); + if (!cbm_mkdtemp(fx->home)) { + return false; + } + snprintf(fx->settings, sizeof(fx->settings), "%s/.openhands/settings.json", fx->home); +#ifdef _WIN32 + snprintf(fx->binary, sizeof(fx->binary), "%s/.local/bin/codebase-memory-mcp.exe", fx->home); +#else + snprintf(fx->binary, sizeof(fx->binary), "%s/.local/bin/codebase-memory-mcp", fx->home); +#endif + snprintf(fx->mcp, sizeof(fx->mcp), "%s/.openhands/mcp.json", fx->home); + snprintf(fx->profiles_dir, sizeof(fx->profiles_dir), "%s/.openhands/agent-profiles", fx->home); + snprintf(fx->default_profile, sizeof(fx->default_profile), "%s/default.json", fx->profiles_dir); + snprintf(fx->custom_profile, sizeof(fx->custom_profile), "%s/custom.json", fx->profiles_dir); + snprintf(fx->note, sizeof(fx->note), "%s/README.txt", fx->profiles_dir); + char openhands_dir[512]; + snprintf(openhands_dir, sizeof(openhands_dir), "%s/.openhands", fx->home); + test_mkdirp(with_profiles ? fx->profiles_dir : openhands_dir); + fx->saved_home = save_test_env("HOME"); + fx->saved_path = save_test_env("PATH"); + fx->saved_cache = save_test_env("CBM_CACHE_DIR"); + cbm_setenv("HOME", fx->home, 1); + cbm_setenv("PATH", fx->home, 1); + cbm_unsetenv("CBM_CACHE_DIR"); + return true; +} + +static void openhands_fixture_close(openhands_fixture_t *fx) { + restore_test_env("HOME", fx->saved_home); + restore_test_env("PATH", fx->saved_path); + restore_test_env("CBM_CACHE_DIR", fx->saved_cache); + test_rmdir_r(fx->home); +} + +static yyjson_doc *openhands_read_doc(const char *path) { + char *data = read_test_file_alloc(path); + if (!data) { + return NULL; + } + yyjson_doc *doc = yyjson_read(data, strlen(data), 0); + free(data); + return doc; +} + +/* settings.json → mcp_config. must be {transport:"stdio", command:, + * enabled:true}; returns false for any deviation. */ +static bool openhands_settings_entry_ok(const char *settings_path, const char *name, + const char *binary) { + yyjson_doc *doc = openhands_read_doc(settings_path); + yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; + yyjson_val *mcp_config = root ? yyjson_obj_get(root, "mcp_config") : NULL; + yyjson_val *entry = mcp_config ? yyjson_obj_get(mcp_config, name) : NULL; + bool ok = entry && yyjson_is_obj(entry) && yyjson_obj_size(entry) == 3U; + yyjson_val *transport = ok ? yyjson_obj_get(entry, "transport") : NULL; + yyjson_val *command = ok ? yyjson_obj_get(entry, "command") : NULL; + yyjson_val *enabled = ok ? yyjson_obj_get(entry, "enabled") : NULL; + ok = ok && transport && yyjson_is_str(transport) && + strcmp(yyjson_get_str(transport), "stdio") == 0; + ok = ok && command && yyjson_is_str(command) && strcmp(yyjson_get_str(command), binary) == 0; + ok = ok && enabled && yyjson_is_true(enabled); + yyjson_doc_free(doc); + return ok; +} + +static bool openhands_settings_entry_absent(const char *settings_path, const char *name) { + yyjson_doc *doc = openhands_read_doc(settings_path); + yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; + yyjson_val *mcp_config = root ? yyjson_obj_get(root, "mcp_config") : NULL; + bool absent = root && (!mcp_config || !yyjson_obj_get(mcp_config, name)); + yyjson_doc_free(doc); + return absent; +} + +/* profile → mcp_server_refs must be exactly the given strings in order. */ +static bool openhands_profile_refs_equal(const char *profile_path, const char *const *expected, + size_t expected_count) { + yyjson_doc *doc = openhands_read_doc(profile_path); + yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; + yyjson_val *refs = root ? yyjson_obj_get(root, "mcp_server_refs") : NULL; + bool ok = refs && yyjson_is_arr(refs) && yyjson_arr_size(refs) == expected_count; + for (size_t i = 0; ok && i < expected_count; i++) { + yyjson_val *item = yyjson_arr_get(refs, i); + ok = item && yyjson_is_str(item) && strcmp(yyjson_get_str(item), expected[i]) == 0; + } + yyjson_doc_free(doc); + return ok; +} + +static bool openhands_file_equals(const char *path, const char *expected) { + char *data = read_test_file_alloc(path); + bool equal = data && expected && strcmp(data, expected) == 0; + free(data); + return equal; +} + +static const char openhands_settings_before[] = "{\n" + " \"language\": \"en\",\n" + " \"mcp_config\": {\n" + " \"other-mcp\": {\"transport\": \"stdio\", " + "\"command\": \"/opt/other\", \"enabled\": false}\n" + " },\n" + " \"llm_model\": \"gpt-x\"\n" + "}\n"; +static const char openhands_default_profile_before[] = "{\n" + " \"name\": \"default\",\n" + " \"mcp_server_refs\": null,\n" + " \"tools\": [\"bash\"]\n" + "}\n"; +static const char openhands_custom_profile_before[] = + "{\"name\": \"custom\", \"mcp_server_refs\": [\"other-mcp\"]}\n"; +static const char openhands_note_before[] = "not a profile\n"; + +/* Foreign settings keys and a foreign mcp_config sibling survive verbatim; + * a null profile ref list becomes ours, an existing list is appended once; + * a second install is byte-idempotent; uninstall restores the foreign files + * byte-for-byte and removes only our ref. */ +TEST(cli_openhands_registers_settings_mcp_config_and_profile_refs_issue1826) { + openhands_fixture_t fx; + if (!openhands_fixture_open(&fx, true)) + FAIL("cbm_mkdtemp failed"); + write_test_file(fx.settings, openhands_settings_before); + write_test_file(fx.default_profile, openhands_default_profile_before); + write_test_file(fx.custom_profile, openhands_custom_profile_before); + write_test_file(fx.note, openhands_note_before); + + cbm_install_agent_configs(fx.home, fx.binary, false, false); + + const char *const only_ours[] = {"codebase-memory-mcp"}; + const char *const appended[] = {"other-mcp", "codebase-memory-mcp"}; + bool ours_ok = openhands_settings_entry_ok(fx.settings, "codebase-memory-mcp", fx.binary); + char *settings_after = read_test_file_alloc(fx.settings); + bool foreign_ok = + settings_after && strstr(settings_after, "\"language\": \"en\"") && + strstr(settings_after, "\"llm_model\": \"gpt-x\"") && + strstr(settings_after, "\"other-mcp\": {\"transport\": \"stdio\", \"command\": " + "\"/opt/other\", \"enabled\": false}"); + bool default_ok = openhands_profile_refs_equal(fx.default_profile, only_ours, 1U); + bool custom_ok = openhands_profile_refs_equal(fx.custom_profile, appended, 2U); + char *default_after = read_test_file_alloc(fx.default_profile); + bool default_foreign_ok = default_after && strstr(default_after, "\"name\": \"default\"") && + strstr(default_after, "\"tools\": [\"bash\"]"); + bool note_ok = openhands_file_equals(fx.note, openhands_note_before); + const char *const standard_json[] = {"mcpServers", "codebase-memory-mcp", fx.binary}; + bool mcp_json_ok = test_file_contains_all(fx.mcp, standard_json, 3); + char *custom_after = read_test_file_alloc(fx.custom_profile); + + /* Second install: byte-idempotent on every touched file. */ + cbm_install_agent_configs(fx.home, fx.binary, false, false); + bool idempotent = openhands_file_equals(fx.settings, settings_after) && + openhands_file_equals(fx.default_profile, default_after) && + openhands_file_equals(fx.custom_profile, custom_after); + free(settings_after); + free(default_after); + free(custom_after); + + char *argv[] = {"uninstall", "--yes"}; + int rc = cli_test_cmd_uninstall(2, argv); + bool settings_restored = openhands_file_equals(fx.settings, openhands_settings_before); + bool custom_restored = + openhands_file_equals(fx.custom_profile, openhands_custom_profile_before); + /* null → ["codebase-memory-mcp"] → [] : the ref is gone; an empty list is + * the minimal edit (the editor cannot know the list was null before). */ + bool default_cleared = openhands_profile_refs_equal(fx.default_profile, only_ours, 0U); + bool note_restored = openhands_file_equals(fx.note, openhands_note_before); + openhands_fixture_close(&fx); + + if (!ours_ok) + FAIL("settings.json must register mcp_config.codebase-memory-mcp {stdio, binary, enabled}"); + if (!foreign_ok) + FAIL("settings.json foreign keys and the foreign mcp_config sibling must survive verbatim"); + if (!default_ok) + FAIL("a null mcp_server_refs must become [\"codebase-memory-mcp\"]"); + if (!custom_ok) + FAIL("an existing mcp_server_refs list must gain codebase-memory-mcp exactly once"); + if (!default_foreign_ok) + FAIL("profile keys around mcp_server_refs must survive verbatim"); + if (!note_ok) + FAIL("non-JSON files under agent-profiles must not be touched"); + if (!mcp_json_ok) + FAIL("the existing ~/.openhands/mcp.json registration must be kept"); + if (!idempotent) + FAIL("a second install must be byte-idempotent"); + if (rc != 0 || !settings_restored) + FAIL("uninstall must restore settings.json byte-for-byte"); + if (!custom_restored) + FAIL("uninstall must restore a profile with foreign refs byte-for-byte"); + if (!default_cleared) + FAIL("uninstall must remove our ref from the null-origin profile"); + if (!note_restored) + FAIL("uninstall must not touch non-JSON files under agent-profiles"); + PASS(); +} + +/* Fresh ~/.openhands without settings.json or agent-profiles: settings.json is + * created with only our entry, no profile directory is invented, and uninstall + * removes the entry again. */ +TEST(cli_openhands_creates_settings_and_skips_missing_profiles_issue1826) { + openhands_fixture_t fx; + if (!openhands_fixture_open(&fx, false)) + FAIL("cbm_mkdtemp failed"); + + cbm_install_agent_configs(fx.home, fx.binary, false, false); + bool ours_ok = openhands_settings_entry_ok(fx.settings, "codebase-memory-mcp", fx.binary); + struct stat st; + bool no_profiles_invented = stat(fx.profiles_dir, &st) != 0; + + char *argv[] = {"uninstall", "--yes"}; + int rc = cli_test_cmd_uninstall(2, argv); + bool removed = openhands_settings_entry_absent(fx.settings, "codebase-memory-mcp"); + openhands_fixture_close(&fx); + + if (!ours_ok) + FAIL("a missing settings.json must be created with mcp_config.codebase-memory-mcp"); + if (!no_profiles_invented) + FAIL("install must never create ~/.openhands/agent-profiles"); + if (rc != 0 || !removed) + FAIL("uninstall must remove mcp_config.codebase-memory-mcp from settings.json"); + PASS(); +} + /* ═══════════════════════════════════════════════════════════════════ * VS Code MCP config tests * ═══════════════════════════════════════════════════════════════════ */ @@ -5168,6 +5413,7 @@ TEST(cli_new_agent_install_plans_use_documented_paths) { "/.hermes/skills/codebase-memory/SKILL.md", "\"openhands\"", "/.openhands/mcp.json", + "/.openhands/settings.json", "/.agents/skills/codebase-memory/SKILL.md", "\"cline\"", "/.cline/mcp.json", @@ -14858,6 +15104,8 @@ SUITE(cli) { RUN_TEST(cli_openclaw_mcp_preserves_valid_json5); RUN_TEST(cli_openclaw_mcp_uninstall_uses_nested_servers); RUN_TEST(cli_openclaw_compaction_preserves_user_owned_section); + RUN_TEST(cli_openhands_registers_settings_mcp_config_and_profile_refs_issue1826); + RUN_TEST(cli_openhands_creates_settings_and_skips_missing_profiles_issue1826); RUN_TEST(cli_openclaw_profile_uses_profile_state_and_default_workspace); RUN_TEST(cli_openclaw_uninstall_removes_compaction_when_workspace_is_ambiguous); diff --git a/tests/test_config_json_like.c b/tests/test_config_json_like.c index c6f29a387..8e6a2b28c 100644 --- a/tests/test_config_json_like.c +++ b/tests/test_config_json_like.c @@ -671,6 +671,176 @@ TEST(config_json_like_top_level_array_unique_string) { PASS(); } +/* OpenHands profiles ship `"mcp_server_refs": null` (#1826). A null value is + * the documented "no list yet" shape, so adding a unique string must replace + * exactly that token with a one-element array, preserving everything around + * it, and stay idempotent. Removal leaves the minimal edit: an empty array. */ +TEST(config_json_like_array_add_replaces_null_value_issue1826) { + jl_fixture_t fixture; + ASSERT_EQ(jl_fixture_open(&fixture), 0); + const char *original = "{\n" + " \"name\": \"default\",\n" + " \"mcp_server_refs\": null, // keep this user comment\n" + " \"tools\": [\"bash\"]\n" + "}\n"; + ASSERT_EQ(jl_write(fixture.path, original), 0); + ASSERT_EQ( + cbm_json_like_add_unique_string(fixture.path, "mcp_server_refs", "codebase-memory-mcp"), 0); + char *first = jl_read(fixture.path); + ASSERT_NOT_NULL(first); + ASSERT_STR_EQ(first, + "{\n" + " \"name\": \"default\",\n" + " \"mcp_server_refs\": [\"codebase-memory-mcp\"], // keep this user comment\n" + " \"tools\": [\"bash\"]\n" + "}\n"); + + ASSERT_EQ( + cbm_json_like_add_unique_string(fixture.path, "mcp_server_refs", "codebase-memory-mcp"), 0); + char *second = jl_read(fixture.path); + ASSERT_NOT_NULL(second); + ASSERT_STR_EQ(second, first); + free(first); + free(second); + + ASSERT_EQ(cbm_json_like_remove_string(fixture.path, "mcp_server_refs", "codebase-memory-mcp"), + 0); + char *removed = jl_read(fixture.path); + ASSERT_NOT_NULL(removed); + ASSERT_STR_EQ(removed, "{\n" + " \"name\": \"default\",\n" + " \"mcp_server_refs\": [], // keep this user comment\n" + " \"tools\": [\"bash\"]\n" + "}\n"); + free(removed); + + /* Only the literal null converts; any other non-array value still fails + * closed byte-identically. */ + const char *wrong_type = "{\"mcp_server_refs\": \"null\", \"keep\": true}\n"; + ASSERT_EQ(jl_write(fixture.path, wrong_type), 0); + ASSERT_EQ(cbm_json_like_add_unique_string(fixture.path, "mcp_server_refs", "x"), -1); + char *content = jl_read(fixture.path); + ASSERT_NOT_NULL(content); + ASSERT_STR_EQ(content, wrong_type); + free(content); + jl_fixture_close(&fixture); + PASS(); +} + +/* The OpenHands settings entry carries `enabled: true` (#1826). A literal shape + * lets ownership matching require that token exactly, so uninstall recognises + * our own entry instead of treating the boolean as a foreign annotation. */ +static int jl_match_openhands_entry(const char *document, + const cbm_json_like_object_field_t *fields, size_t field_count, + char **captured) { + static const char *const path[] = {"mcp_config"}; + return cbm_json_like_match_object_entry(document, strlen(document), path, 1U, + "codebase-memory-mcp", fields, field_count, captured); +} + +TEST(config_json_like_match_object_entry_literal_shape_issue1826) { + const cbm_json_like_object_field_t fields[] = { + {.key = "command", + .shape = CBM_JSON_LIKE_VALUE_STRING, + .expected_string = NULL, + .flags = CBM_JSON_LIKE_FIELD_REQUIRED | CBM_JSON_LIKE_FIELD_CAPTURE_STRING}, + {.key = "transport", + .shape = CBM_JSON_LIKE_VALUE_STRING, + .expected_string = "stdio", + .flags = CBM_JSON_LIKE_FIELD_REQUIRED}, + {.key = "enabled", + .shape = CBM_JSON_LIKE_VALUE_LITERAL, + .expected_string = "true", + .flags = CBM_JSON_LIKE_FIELD_REQUIRED}, + }; + char *captured = NULL; + const char *ours = "{\"mcp_config\": {\"codebase-memory-mcp\": {\"transport\": \"stdio\", " + "\"command\": \"/x/cbm\", \"enabled\": true}}}\n"; + ASSERT_EQ(jl_match_openhands_entry(ours, fields, 3U, &captured), CBM_JSON_LIKE_OBJECT_MATCH); + ASSERT_NOT_NULL(captured); + ASSERT_STR_EQ(captured, "/x/cbm"); + free(captured); + + /* Trivia around the token is not part of the value. */ + const char *spaced = "{\"mcp_config\": {\"codebase-memory-mcp\": {\"transport\": \"stdio\", " + "\"command\": \"/x/cbm\", \"enabled\" : true /* on */ }}}\n"; + ASSERT_EQ(jl_match_openhands_entry(spaced, fields, 3U, &captured), CBM_JSON_LIKE_OBJECT_MATCH); + free(captured); + + const char *disabled = "{\"mcp_config\": {\"codebase-memory-mcp\": {\"transport\": \"stdio\", " + "\"command\": \"/x/cbm\", \"enabled\": false}}}\n"; + ASSERT_EQ(jl_match_openhands_entry(disabled, fields, 3U, &captured), + CBM_JSON_LIKE_OBJECT_MISMATCH); + ASSERT_NULL(captured); + + const char *quoted = "{\"mcp_config\": {\"codebase-memory-mcp\": {\"transport\": \"stdio\", " + "\"command\": \"/x/cbm\", \"enabled\": \"true\"}}}\n"; + ASSERT_EQ(jl_match_openhands_entry(quoted, fields, 3U, &captured), + CBM_JSON_LIKE_OBJECT_MISMATCH); + ASSERT_NULL(captured); + + const char *missing = "{\"mcp_config\": {\"codebase-memory-mcp\": {\"transport\": \"stdio\", " + "\"command\": \"/x/cbm\"}}}\n"; + ASSERT_EQ(jl_match_openhands_entry(missing, fields, 3U, &captured), + CBM_JSON_LIKE_OBJECT_MISMATCH); + ASSERT_NULL(captured); + + /* A literal field can neither lack its token nor be captured. */ + const cbm_json_like_object_field_t no_token[] = { + fields[0], + {.key = "enabled", + .shape = CBM_JSON_LIKE_VALUE_LITERAL, + .expected_string = NULL, + .flags = CBM_JSON_LIKE_FIELD_REQUIRED}, + }; + ASSERT_EQ(jl_match_openhands_entry(ours, no_token, 2U, &captured), -1); + const cbm_json_like_object_field_t captured_literal[] = { + {.key = "enabled", + .shape = CBM_JSON_LIKE_VALUE_LITERAL, + .expected_string = "true", + .flags = CBM_JSON_LIKE_FIELD_REQUIRED | CBM_JSON_LIKE_FIELD_CAPTURE_STRING}, + }; + ASSERT_EQ(jl_match_openhands_entry(ours, captured_literal, 1U, &captured), -1); + PASS(); +} + +/* An OpenHands profile with an existing single-line list (#1826): the added + * name follows the list's own spacing, and removing it again restores the + * original bytes exactly — no stray space before the closing bracket. */ +static bool jl_round_trips(jl_fixture_t *fixture, const char *original, const char *expected) { + if (jl_write(fixture->path, original) != 0 || + cbm_json_like_add_unique_string(fixture->path, "mcp_server_refs", "codebase-memory-mcp") != + 0) { + return false; + } + char *added = jl_read(fixture->path); + bool ok = added && strcmp(added, expected) == 0; + free(added); + if (!ok || + cbm_json_like_remove_string(fixture->path, "mcp_server_refs", "codebase-memory-mcp") != 0) { + return false; + } + char *restored = jl_read(fixture->path); + ok = restored && strcmp(restored, original) == 0; + free(restored); + return ok; +} + +TEST(config_json_like_single_line_array_round_trips_byte_for_byte_issue1826) { + jl_fixture_t fixture; + ASSERT_EQ(jl_fixture_open(&fixture), 0); + ASSERT_TRUE(jl_round_trips( + &fixture, "{\"name\": \"custom\", \"mcp_server_refs\": [\"other-mcp\"]}\n", + "{\"name\": \"custom\", \"mcp_server_refs\": [\"other-mcp\", \"codebase-memory-mcp\"]}\n")); + ASSERT_TRUE( + jl_round_trips(&fixture, "{ \"mcp_server_refs\": [ \"other-mcp\" ] }\n", + "{ \"mcp_server_refs\": [ \"other-mcp\", \"codebase-memory-mcp\" ] }\n")); + ASSERT_TRUE(jl_round_trips(&fixture, "{\"mcp_server_refs\": [ ]}\n", + "{\"mcp_server_refs\": [ \"codebase-memory-mcp\" ]}\n")); + jl_fixture_close(&fixture); + PASS(); +} + TEST(config_json_like_top_level_array_create_escape_and_fail_closed) { jl_fixture_t fixture; ASSERT_EQ(jl_fixture_open(&fixture), 0); @@ -1025,6 +1195,9 @@ SUITE(config_json_like) { RUN_TEST(config_json_like_removes_first_middle_last_and_only); RUN_TEST(config_json_like_removal_preserves_comments_and_siblings); RUN_TEST(config_json_like_top_level_array_unique_string); + RUN_TEST(config_json_like_array_add_replaces_null_value_issue1826); + RUN_TEST(config_json_like_match_object_entry_literal_shape_issue1826); + RUN_TEST(config_json_like_single_line_array_round_trips_byte_for_byte_issue1826); RUN_TEST(config_json_like_top_level_array_create_escape_and_fail_closed); RUN_TEST(config_json_like_nested_array_creates_missing_path_and_escapes); RUN_TEST(config_json_like_nested_array_preserves_jsonc_and_is_idempotent); From 7e73b48ef155a0e14a6e058be56287232e3976ea Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 8 Sep 2026 22:27:41 +0200 Subject: [PATCH 035/130] style(cli): clang-format the OpenHands install helpers (#2116) CI's clang-format-20 flagged continuation-line alignment in the new OpenHands MCP install/uninstall helpers (cbm_build_openhands_mcp_entry, cbm_upsert_openhands_settings_mcp, cbm_remove_openhands_settings_mcp, openhands_update_profile_refs). Reformat only those regions with clang-format-20 to satisfy the lint gate; whitespace-only, no behaviour change. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- src/cli/cli.c | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/src/cli/cli.c b/src/cli/cli.c index 21bb36d65..010387adb 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -2318,8 +2318,8 @@ static char *cbm_build_openhands_mcp_entry(const char *binary_path) { } yyjson_mut_val *root = yyjson_mut_obj(doc); bool ok = root && yyjson_mut_obj_add_strcpy(doc, root, "transport", "stdio") && - yyjson_mut_obj_add_strcpy(doc, root, "command", binary_path) && - yyjson_mut_obj_add_bool(doc, root, "enabled", true); + yyjson_mut_obj_add_strcpy(doc, root, "command", binary_path) && + yyjson_mut_obj_add_bool(doc, root, "enabled", true); char *json = NULL; if (ok) { yyjson_mut_doc_set_root(doc, root); @@ -2349,10 +2349,9 @@ static int cbm_upsert_openhands_settings_mcp(const char *binary_path, const char cbm_json_like_object_field_t fields[3]; size_t field_count = cbm_openhands_ownership_fields(fields); char *command = NULL; - int ownership = - cbm_json_like_match_object_entry(document, document_length, path, 1U, - CBM_DEFAULT_MCP_SERVER_NAME, fields, field_count, - &command); + int ownership = cbm_json_like_match_object_entry(document, document_length, path, 1U, + CBM_DEFAULT_MCP_SERVER_NAME, fields, + field_count, &command); free(command); if (ownership == CBM_JSON_LIKE_OBJECT_MATCH || ownership == CBM_JSON_LIKE_OBJECT_MATCH_WITH_EXTRAS) { @@ -2400,8 +2399,8 @@ static int cbm_remove_openhands_settings_mcp(const char *settings_path) { size_t field_count = cbm_openhands_ownership_fields(fields); char *command = NULL; int ownership = cbm_json_like_match_object_entry(document, document_length, path, 1U, - CBM_DEFAULT_MCP_SERVER_NAME, fields, - field_count, &command); + CBM_DEFAULT_MCP_SERVER_NAME, fields, + field_count, &command); free(command); if (ownership == CBM_JSON_LIKE_OBJECT_MISSING || ownership == CBM_JSON_LIKE_OBJECT_MISMATCH) { free(document); @@ -9374,8 +9373,7 @@ static bool cbm_filename_has_suffix(const char *name, const char *suffix) { * directory is a silent no-op in both directions — install must never invent * it, and uninstall has nothing to undo there. Only *.json entries are * touched; a profile directory may hold arbitrary notes alongside profiles. */ -static void openhands_update_profile_refs(const char *profiles_dir, bool installing, - bool dry_run) { +static void openhands_update_profile_refs(const char *profiles_dir, bool installing, bool dry_run) { cbm_dir_t *d = cbm_opendir(profiles_dir); if (!d) { return; @@ -9392,15 +9390,14 @@ static void openhands_update_profile_refs(const char *profiles_dir, bool install if (stat(profile_path, &state) != 0 || !S_ISREG(state.st_mode) || dry_run) { continue; } - int result = installing ? cbm_json_like_add_unique_string( - profile_path, "mcp_server_refs", CBM_DEFAULT_MCP_SERVER_NAME) + int result = installing ? cbm_json_like_add_unique_string(profile_path, "mcp_server_refs", + CBM_DEFAULT_MCP_SERVER_NAME) : cbm_json_like_remove_string(profile_path, "mcp_server_refs", CBM_DEFAULT_MCP_SERVER_NAME); if (result != CLI_OK) { - record_agent_config_error(!installing, "OpenHands", - installing ? "profile_refs_install" - : "profile_refs_uninstall", - profile_path); + record_agent_config_error( + !installing, "OpenHands", + installing ? "profile_refs_install" : "profile_refs_uninstall", profile_path); } } cbm_closedir(d); From 91e31211df9dd61459617d857d46b71699e34df8 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 8 Sep 2026 22:30:34 +0200 Subject: [PATCH 036/130] =?UTF-8?q?fix(pipeline):=20take=20the=20stage=20l?= =?UTF-8?q?ock=20before=20the=20stage=20file=20is=20visible=20(close=20#21?= =?UTF-8?q?11=20create=E2=86=92register=20race)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit create_staging_path() minted the stage's main file with mkstemp and only then took its sidecar .lock via stage_owner_register(). A second, concurrent cbm_pipeline_run() against the same final_path (auto_index, or the stale-rendezvous-recovery retry path) could land its sweep_orphan_stages() in that window: the main file was visible but not yet locked, so the sweep treated it like a genuinely orphaned stage and deleted it out from under the live writer (POSIX), or collided on the not-yet-created sidecar and surfaced EACCES/errno=13 (Windows). That is the #2111 red on test-windows-guards and both test-windows CLANG64 x86_64 shards (index_format / simhash / index_resilience, with pipeline.stage action=lock_failed errno=13). Fix: mint the six-char alphanumeric suffix in create_staging_path() itself, create and hold the sidecar lock FIRST, then create the main file with O_EXCL. The stage's main file is therefore never present on disk without its lock already held, so a racing sweep can only ever find the stage lock-held: it reaches the stage through the .lock entry too (stage_entry_stage_length() matches it), probes the lock, sees a live holder, and keeps it. The create->register TOCTOU is closed by construction. #1839 is preserved: sweep_one_stage() still removes a stage whose sidecar is absent (ENOENT). With live stages now always carrying their sidecar, ENOENT is only ever a genuine pre-lock-era orphan minted by an older binary, which #1839 requires be swept. stage_owner_register() becomes stage_owner_adopt(), recording the already-held descriptor; re-taking the lock would self-conflict, since flock() and Windows _SH_DENYRW deny a second acquire of the same sidecar even from this process. A new after_stage_created test seam and pipeline_concurrent_sweep_must_not_- remove_inflight_stage drive a real concurrent sweep in that window; the test is RED without the lock-first ordering. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- src/pipeline/pipeline.c | 149 +++++++++++++++++++++++++------ src/pipeline/pipeline_internal.h | 10 +++ tests/test_pipeline.c | 125 ++++++++++++++++++++++++++ 3 files changed, 257 insertions(+), 27 deletions(-) diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 5d21a1bde..dba7fd629 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -37,6 +37,7 @@ enum { CBM_DIR_PERMS = 0755, PL_RING = 4, PL_RING_MASK = 3, PL_SEQ_PASSES = 6 }; #include "foundation/compat_thread.h" #include "foundation/profile.h" #include "foundation/mem.h" +#include "foundation/secure_random.h" #include #include @@ -73,6 +74,8 @@ static atomic_bool g_persist_test_cancel_after_destination_prepare = false; static atomic_bool g_persist_test_fail_adr_capture = false; static cbm_pipeline_test_hook_fn g_persist_test_before_final_manifest = NULL; static void *g_persist_test_before_final_manifest_userdata = NULL; +static cbm_pipeline_test_hook_fn g_persist_test_after_stage_created = NULL; +static void *g_persist_test_after_stage_created_userdata = NULL; void cbm_pipeline_incremental_test_fail_after_stage_dump_once(void) { atomic_store(&g_persist_test_fail_after_stage_dump, true); @@ -106,6 +109,29 @@ void cbm_pipeline_persist_test_run_before_final_manifest(void) { } } +void cbm_pipeline_incremental_test_after_stage_created_once(cbm_pipeline_test_hook_fn hook, + void *userdata) { + g_persist_test_after_stage_created = hook; + g_persist_test_after_stage_created_userdata = userdata; +} + +/* Fired by create_staging_path() right after the stage's main file is created + * with O_EXCL -- and, in the current lock-before-visible ordering, after its + * sidecar lock is already held. A test hook installed here can run a + * concurrent sweep (another cbm_pipeline_run() against the same final_path) at + * this instant to prove the just-created stage survives it. Under the OLD + * create-then-lock ordering this was the unlocked TOCTOU window, so the same + * hook binds RED if that ordering ever regresses. */ +void cbm_pipeline_persist_test_run_after_stage_created(void) { + cbm_pipeline_test_hook_fn hook = g_persist_test_after_stage_created; + void *userdata = g_persist_test_after_stage_created_userdata; + g_persist_test_after_stage_created = NULL; + g_persist_test_after_stage_created_userdata = NULL; + if (hook) { + hook(userdata); + } +} + bool cbm_pipeline_persist_test_take_failure_after_stage_dump(void) { return atomic_exchange(&g_persist_test_fail_after_stage_dump, false); } @@ -125,6 +151,8 @@ void cbm_pipeline_persist_test_reset_faults(void) { atomic_store(&g_persist_test_fail_adr_capture, false); g_persist_test_before_final_manifest = NULL; g_persist_test_before_final_manifest_userdata = NULL; + g_persist_test_after_stage_created = NULL; + g_persist_test_after_stage_created_userdata = NULL; } #endif @@ -1630,25 +1658,25 @@ void cbm_pipeline_stage_lock_drop(const char *stage_path, int lock_fd) { } } -static bool stage_owner_register(const char *stage_path) { - int fd = cbm_pipeline_stage_lock_hold(stage_path); - if (fd < 0) { - char errno_text[16]; - (void)snprintf(errno_text, sizeof(errno_text), "%d", errno); - cbm_log_warn("pipeline.stage", "action", "lock_failed", "errno", errno_text, "path", - stage_path); - return false; - } +/* Record an already-held stage lock in the per-process owner table, keyed by + * path so publish/finalize/discard can release it later. On success the table + * owns lock_fd; on failure the caller still does and must drop it. + * + * The lock must ALREADY be held: create_staging_path() takes it before the + * stage's main file is created (so the file is never visible on disk without + * its lock), then hands the descriptor here. Re-taking the lock in this helper + * would self-conflict -- both flock() and Windows _SH_DENYRW deny a second + * acquire of the same sidecar even from this same process. */ +static bool stage_owner_adopt(const char *stage_path, int lock_fd) { stage_owner_t *owner = (stage_owner_t *)malloc(sizeof(*owner)); char *path_copy = strdup(stage_path); if (!owner || !path_copy) { free(owner); free(path_copy); - cbm_pipeline_stage_lock_drop(stage_path, fd); return false; } owner->stage_path = path_copy; - owner->lock_fd = fd; + owner->lock_fd = lock_fd; stage_owners_lock(); owner->next = g_stage_owners; g_stage_owners = owner; @@ -2641,22 +2669,79 @@ static char *create_staging_path(const char *final_path) { } memcpy(path, final_path, final_len); memcpy(path + final_len, suffix, sizeof(suffix)); - int fd = cbm_mkstemp(path); - if (fd < 0) { - free(path); - return NULL; - } -#ifdef _WIN32 - _close(fd); -#else - close(fd); + /* The six random chars sit directly after the ".stage." marker; each + * attempt overwrites the "XXXXXX" template in place. Alphanumerics only, + * matching stage_suffix_at()/stage_entry_stage_length() so the sweep + * recognises the minted name and its sidecars. */ + char *random_at = path + final_len + (sizeof(cbm_stage_marker) - 1); + static const char alphabet[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + + /* Lock BEFORE the stage becomes visible: take the sidecar lock first, then + * create the stage's main file with O_EXCL. The main file is therefore + * never present on disk without its lock already held, so a concurrent + * run's sweep_orphan_stages() against the same final_path can only ever + * find this stage lock-held -- it reaches the stage through the .lock + * sidecar too (stage_entry_stage_length() matches it), probes the lock, + * sees a live holder, and keeps it. That closes the old create->register + * window that let a racing sweep delete a live-but-unlocked stage (POSIX) + * or collide on the sidecar with EACCES (Windows) -- #2111's windows-guards + * red. A pre-lock-era orphan minted by an OLDER binary still carries no + * lock, so the sweep's ENOENT path still removes it (#1839 preserved). + * + * A minted suffix collides with an existing stage only about 1 in 62^6; + * retry a bounded number of times, the way mkstemp/mkdtemp do, then fail. */ + for (int attempt = 0; attempt < 128; attempt++) { + unsigned char rnd[CBM_STAGE_SUFFIX_RANDOM_CHARS]; + if (!cbm_secure_random(rnd, sizeof(rnd))) { + free(path); + errno = EIO; + return NULL; + } + for (size_t i = 0; i < sizeof(rnd); i++) { + random_at[i] = alphabet[rnd[i] % (sizeof(alphabet) - 1)]; + } + errno = 0; + int lock_fd = cbm_pipeline_stage_lock_hold(path); + if (lock_fd < 0) { + /* A live twin already owns this exact suffix's sidecar (EAGAIN / + * EACCES), or the sidecar could not be created. Mint a fresh suffix + * and try again rather than contend for this one. */ + continue; + } + FILE *main_file = cbm_fopen(path, "wbx"); + if (!main_file) { + /* The suffix collided with a lock-less orphan's main file -- its + * sidecar was takeable, so it is not a live writer. Never inherit a + * stranger's bytes: drop the lock, remove the sidecar we just took, + * and mint a fresh suffix. The orphan's main file is left for a + * later sweep, which removes it as a pre-lock-era orphan. */ + cbm_pipeline_stage_lock_drop(path, lock_fd); + continue; + } + (void)fclose(main_file); +#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API + /* Main file now exists and its lock is already held. Under the OLD + * create-then-lock ordering this was the unlocked window; the + * concurrent-sweep test fires here to prove the stage now survives a + * racing sweep, and to bind RED if that ordering ever regresses. */ + cbm_pipeline_persist_test_run_after_stage_created(); #endif - if (!stage_owner_register(path)) { - (void)cbm_unlink(path); - free(path); - return NULL; + if (!stage_owner_adopt(path, lock_fd)) { + cbm_pipeline_stage_lock_drop(path, lock_fd); + (void)cbm_unlink(path); + free(path); + return NULL; + } + return path; } - return path; + /* Every attempt failed to take a lock -- keep the observability the old + * stage_owner_register() emitted for a lock failure. */ + char errno_text[16]; + (void)snprintf(errno_text, sizeof(errno_text), "%d", errno); + cbm_log_warn("pipeline.stage", "action", "lock_failed", "errno", errno_text, "path", path); + free(path); + errno = EEXIST; + return NULL; } /* A backup-failed destination may still have the only recoverable WAL or @@ -2876,8 +2961,18 @@ static void sweep_one_stage(const char *stage_path) { "errno", errno_text, "path", stage_path); return; } - /* No owner: absent sidecar (pre-ownership stage, or dropped) or a lock - * the kernel released with its writer. Ours now, from the lock down. */ + /* No owner: absent sidecar (an ENOENT probe) or a lock the kernel released + * with its writer. Ours now, from the lock down. + * + * The absent-sidecar (ENOENT) case is exactly a pre-lock-era orphan: a + * stage an OLDER binary minted with no sidecar at all (#1839 pins that + * these ARE swept). It is NOT an in-flight stage of a current run: since + * create_staging_path() now takes the sidecar lock BEFORE the stage's main + * file becomes visible on disk, a live stage always has its sidecar, so a + * concurrent sweep landing here for one would instead find the lock held + * above and keep it. Removing on ENOENT therefore reclaims genuine orphans + * without ever deleting a live stage (the create->register race behind + * #2111's windows-guards red is closed at the source). */ int64_t bytes = stage_bytes_on_disk(stage_path); remove_stage_files(stage_path); if (lock_fd >= 0) { diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 186b8ee66..36bd0cbf6 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -908,6 +908,15 @@ void cbm_pipeline_incremental_test_fail_adr_capture_once(void); typedef void (*cbm_pipeline_test_hook_fn)(void *userdata); void cbm_pipeline_incremental_test_before_final_manifest_once(cbm_pipeline_test_hook_fn hook, void *userdata); +/* Fires from create_staging_path(), right after the stage's main file is + * created with O_EXCL. In the current lock-before-visible ordering its sidecar + * lock is already held at this point, so a test hook installed here can run a + * concurrent sweep (via another cbm_pipeline_run() against the same + * final_path) and confirm the just-created stage survives it. Under the OLD + * create-then-lock ordering this was the unlocked window, so the hook also + * binds RED if that ordering regresses. */ +void cbm_pipeline_incremental_test_after_stage_created_once(cbm_pipeline_test_hook_fn hook, + void *userdata); cbm_incremental_route_t cbm_pipeline_incremental_test_last_route(void); void cbm_pipeline_incremental_test_reset_faults(void); @@ -917,6 +926,7 @@ bool cbm_pipeline_persist_test_take_failure_after_stage_dump(void); bool cbm_pipeline_persist_test_take_cancel_after_predump(void); bool cbm_pipeline_persist_test_take_cancel_after_destination_prepare(void); void cbm_pipeline_persist_test_run_before_final_manifest(void); +void cbm_pipeline_persist_test_run_after_stage_created(void); void cbm_pipeline_persist_test_reset_faults(void); #endif diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 0892fd696..54e2aa6f9 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -3707,6 +3707,130 @@ TEST(pipeline_sweep_removes_dead_writer_stage_keeps_locked_stage) { PASS(); } +/* #2111 create->register TOCTOU guard. The bug this pins: create_staging_path() + * used to make the stage's main file visible (via mkstemp) BEFORE taking its + * sidecar lock, and sweep_orphan_stages() treats an absent lock sidecar + * (ENOENT) as a confirmed-dead writer (kernel released the lock on death). So a + * second, concurrent cbm_pipeline_run() against the SAME final_path, landing + * its own sweep in that narrow unlocked window, removed the first run's + * in-flight stage out from under it: every extraction pass still completed + * (none touch the stage file on disk), but the publish that followed found its + * own stage gone. Two writers racing the same project is a real scenario this + * PR's own sweep exists to clean up after (auto_index; the recently-fixed + * stale-rendezvous-recovery retry path) -- not hypothetical, and exactly the + * shape of #2111's windows-guards red (every pass logged success, nothing was + * ever committed, "Pipeline failed" surfaced generic; on Windows the sidecar + * collision surfaced as EACCES/errno=13). + * + * The fix takes the sidecar lock BEFORE the main file becomes visible, so this + * hook -- fired the instant the main file exists -- finds the stage already + * lock-protected and the racing sweep keeps it. The hook fires at the same + * point under the old ordering, where the lock was NOT yet held, so this test + * goes RED if that ordering ever regresses. + * + * The racing run is cancelled immediately so it never reaches ITS OWN + * publish -- isolating the sweep's effect on the first run's stage from the + * separate question of two full runs both completing for the same project. */ +typedef struct { + const char *tmp_dir; + const char *db_path; + bool stage_survived; +} racing_sweep_arg_t; + +static bool find_sole_stage_path(const char *dir, const char *db_basename, char *out, + size_t out_sz) { + cbm_dir_t *d = cbm_opendir(dir); + if (!d) { + return false; + } + char prefix[256]; + snprintf(prefix, sizeof(prefix), "%s.stage.", db_basename); + size_t prefix_len = strlen(prefix); + bool found = false; + cbm_dirent_t *entry; + while ((entry = cbm_readdir(d)) != NULL) { + size_t name_len = strlen(entry->name); + enum { STAGE_SUFFIX_RANDOM_CHARS = 6 }; /* mirrors CBM_STAGE_SUFFIX_RANDOM_CHARS */ + if (strncmp(entry->name, prefix, prefix_len) == 0 && + name_len == prefix_len + STAGE_SUFFIX_RANDOM_CHARS) { + snprintf(out, out_sz, "%s/%s", dir, entry->name); + found = true; + break; + } + } + cbm_closedir(d); + return found; +} + +static void *racing_sweep_thread(void *arg) { + racing_sweep_arg_t *a = (racing_sweep_arg_t *)arg; + cbm_pipeline_t *p = cbm_pipeline_new(a->tmp_dir, a->db_path, CBM_MODE_FULL); + if (p) { + /* Its own sweep_orphan_stages() runs at the very start, before this + * run mints its own stage -- exactly like the first run's. Cancel + * immediately: this run must reach the sweep and nothing past it. */ + cbm_pipeline_cancel(p); + (void)cbm_pipeline_run(p); + cbm_pipeline_free(p); + } + return NULL; +} + +/* Fired from inside the FIRST run's create_staging_path(), the instant its + * stage main file exists (under the fix, with its sidecar lock already held; + * under the old create-then-lock ordering, before the lock was taken): run a + * second, cancelled cbm_pipeline_run() for the same project synchronously on + * another thread, so its sweep has every chance to reach the stage before + * control returns to the first run. */ +static void racing_sweep_hook(void *userdata) { + racing_sweep_arg_t *a = (racing_sweep_arg_t *)userdata; + char stage_path[600] = {0}; + bool had_stage = + find_sole_stage_path(a->tmp_dir, "generation.db", stage_path, sizeof(stage_path)); + cbm_thread_t tid; + if (cbm_thread_create(&tid, 0, racing_sweep_thread, a) == 0) { + cbm_thread_join(&tid); + } + a->stage_survived = had_stage && path_exists(stage_path); +} + +TEST(pipeline_concurrent_sweep_must_not_remove_inflight_stage) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_stage_race_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + write_temp_file(tmp, "generation.py", "def StableGeneration():\n return 1\n"); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/generation.db", tmp); + + cbm_pipeline_incremental_test_reset_faults(); + racing_sweep_arg_t race_arg = {.tmp_dir = tmp, .db_path = db_path, .stage_survived = false}; + cbm_pipeline_incremental_test_after_stage_created_once(racing_sweep_hook, &race_arg); + + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + char project[256]; + snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(p)); + int rc = cbm_pipeline_run(p); + cbm_pipeline_free(p); + + bool db_exists = path_exists(db_path); + int defined_count = -1; + int absent_count = -1; + observe_named_generation(db_path, project, "StableGeneration", "NeverDefined", &defined_count, + &absent_count); + int stage_count = count_generation_stage_artifacts(tmp, "generation.db"); + cbm_pipeline_incremental_test_reset_faults(); + th_rmtree(tmp); + + ASSERT_TRUE(race_arg.stage_survived); + ASSERT_EQ(rc, 0); + ASSERT_TRUE(db_exists); + ASSERT_EQ(defined_count, 1); + ASSERT_EQ(absent_count, 0); + ASSERT_EQ(stage_count, 0); + PASS(); +} + static char g_route_log_capture[8192]; static atomic_flag g_route_log_spin = ATOMIC_FLAG_INIT; @@ -14473,6 +14597,7 @@ SUITE(pipeline_semantic_manifest_repro) { RUN_TEST(pipeline_minted_stage_is_owned_until_released); RUN_TEST(pipeline_stale_zero_byte_stage_beside_valid_db_routes_incremental_and_is_swept); RUN_TEST(pipeline_sweep_removes_dead_writer_stage_keeps_locked_stage); + RUN_TEST(pipeline_concurrent_sweep_must_not_remove_inflight_stage); RUN_TEST(pipeline_fresh_index_never_reports_invalid_existing_db); RUN_TEST(pipeline_source_mutation_before_publication_preserves_previous_generation); RUN_TEST(pipeline_source_addition_before_publication_preserves_previous_generation); From a3c24a71b4c28e805b912f9f928b6b0b151f9495 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 8 Sep 2026 23:00:59 +0200 Subject: [PATCH 037/130] test(cli): guard POSIX-only fd-capture helpers under _WIN32 (#2110) cli_fd_capture_begin/end use POSIX fd redirection (dup/dup2/tmpfile) and are called only by the uninstall tests, which are themselves #ifndef _WIN32. Defined at top level, the helpers were compiled but unused on Windows, tripping -Werror,-Wunused-function on the test-windows CLANG64 leg. Wrap the struct and both helpers in #ifndef _WIN32 to match their callers. Test-only; no production change. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- tests/test_cli.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_cli.c b/tests/test_cli.c index 3316e7acd..38e2cb35f 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -722,6 +722,7 @@ static void test_rmdir_r(const char *path) { th_rmtree(path); } +#ifndef _WIN32 /* Capture everything a command writes to one fd (stdout or stderr) so a test * can assert on the transcript. Restores the fd on end and returns the text. */ typedef struct { @@ -774,6 +775,7 @@ static char *cli_fd_capture_end(cli_fd_capture_t *capture) { } return text; } +#endif /* !_WIN32 -- POSIX-only fd-capture helpers (#2110) */ /* Mandatory-daemon activation guard fixture. The production path uses the * stable per-account endpoint directly; these callbacks make every race From ecfb9642e153ffc656d0bc2344b17e9d454ff916 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 8 Sep 2026 23:45:24 +0200 Subject: [PATCH 038/130] test(daemon): assert fast-fail by mechanism, not wall-clock, in the bootstrap ENOSPC test (#2113, O9) The #1828 fail-fast test asserted `elapsed_ms < 5000`, a wall-clock budget that flaked on the ASan test-diag lane under CI contention while the fail-fast mechanism itself engaged correctly: the whole daemon_bootstrap suite ran in 11 s, so the test never reached the 30 s slow-timeout path -- it only overran the 5 s wall-clock budget. Per O9 a gate never asserts a transient timing window, and a rerun is not a fix. Drop the wall-clock measurement and rely on the deterministic mechanism the test already observes: the client surfaces the recorded ENOSPC cause verbatim ("failed to start" + errno + path), the slow "active or starting" timeout wording is absent, and exactly one daemon is spawned. That "failed to start" message is produced only on the fast-fail break (cbm_daemon_bootstrap_start_failure_format), never on the 30 s deadline path, so these assertions bind the fast-fail contract without any timing. Verified RED-on-revert: defeating the fast-fail detection makes the client hang to the 30 s deadline and report "active or starting", tripping ASSERT_FALSE(stale_wording). Full daemon_bootstrap suite green (28/28) under the ASan/UBSan build. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- tests/test_daemon_bootstrap.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/test_daemon_bootstrap.c b/tests/test_daemon_bootstrap.c index 20213ef5b..0114d6329 100644 --- a/tests/test_daemon_bootstrap.c +++ b/tests/test_daemon_bootstrap.c @@ -1029,7 +1029,9 @@ TEST(daemon_bootstrap_start_failure_record_round_trip) { * waiting the full 30 s and then reporting "active or starting" -- the * opposite of the truth. A real host is spawned against a runtime directory * whose record writes fail with ENOSPC; the client must report "failed to - * start" naming the errno and the path, within seconds, not the deadline. */ + * start" naming the errno and the path by ending the wait on the recorded + * cause, not by exhausting the deadline. The proof is the surfaced record and + * the single spawn (see the assertions), never a wall-clock measurement. */ TEST(daemon_bootstrap_fails_fast_when_daemon_dies_at_publication) { const char *old_cache = getenv("CBM_CACHE_DIR"); char *saved_cache = old_cache ? cbm_strdup(old_cache) : NULL; @@ -1071,13 +1073,10 @@ TEST(daemon_bootstrap_fails_fast_when_daemon_dies_at_publication) { cbm_daemon_bootstrap_result_t result; memset(&result, 0, sizeof(result)); cbm_daemon_bootstrap_status_t status = CBM_DAEMON_BOOTSTRAP_FAILED; - uint64_t elapsed_ms = UINT64_MAX; if (identity_ready) { cbm_daemon_ipc_posix_record_write_failure_set_for_test(ENOSPC); cbm_daemon_bootstrap_spawn_override_set_for_test(bootstrap_enospc_host_spawn, &host); - uint64_t started = cbm_now_ms(); status = cbm_daemon_bootstrap_execute(&config, &result); - elapsed_ms = cbm_now_ms() - started; cbm_daemon_bootstrap_spawn_override_set_for_test(NULL, NULL); cbm_daemon_ipc_posix_record_write_failure_set_for_test(0); } @@ -1115,7 +1114,14 @@ TEST(daemon_bootstrap_fails_fast_when_daemon_dies_at_publication) { ASSERT_TRUE(host.child_count >= 1); ASSERT_TRUE(log_read); ASSERT_TRUE(daemon_named_cause); - ASSERT(elapsed_ms < 5000); + /* Fast-fail is proven by the MECHANISM, never by wall-clock (O9: a gate + * never asserts a transient timing window). The recorded ENOSPC cause is + * surfaced verbatim ("failed to start" + errno + path) and the slow + * "active or starting" timeout wording is absent -- that message is emitted + * ONLY on the fast-fail break (cbm_daemon_bootstrap_start_failure_format), + * never on the 30 s deadline path -- and the client stopped after exactly + * one spawn instead of respawning a doomed daemon until the deadline. Any + * regression to the pre-#1828 30 s hang trips these deterministically. */ ASSERT_FALSE(stale_wording); ASSERT_TRUE(message_names_failure); ASSERT_TRUE(message_names_errno); From 92cb3b9a4bebbc79bd283e447104669ca068ea64 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 9 Sep 2026 00:56:55 +0200 Subject: [PATCH 039/130] fix(test): bound the cli activation-guard scope fixture's host waits (#2115) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The four namespace-scope tests fork a real host daemon and, in the parent, waited on it with two UNBOUNDED calls: read() for the child's one-byte readiness signal in cli_scope_fixture_start, and waitpid(host, 0) in cli_scope_fixture_finish. If the forked child stalls before it can signal — a fork-time sanitizer allocator stall is the classic cause, the same hazard the earlier posix_spawn work addressed — the parent blocks forever and the WHOLE cli suite hangs to the CI wall-clock kill (the observed rc=124 at 900 s on the ubuntu-24.04-arm gcc shard, which passed on every other #2115 shard). Attribution: the production drain path is already bounded (DRAIN 15 s, CONTROL 2 s) and a foreign-namespace install takes the non-blocking scope probe and skips draining entirely, so this is not a production deadlock — it is a test-fixture determinism gap, amplified by ASan on a loaded arm runner. Fix, test-only: - cli_scope_wait_ready(): poll the ready pipe to a generous, bounded deadline (90 s, comfortably past the child's own 45 s cohort-admission deadline plus service start), so a wedged child yields a clean ASSERT_TRUE(ready) failure instead of an unbounded read(). - cli_scope_reap_host(): reap with WNOHANG to a bound (60 s), then SIGKILL and reap, so a wedged child is force-killed rather than hanging finish(); a killed child leaves host_exit == -1, which fails the caller's assertion cleanly. This also prevents a stuck child from holding cohort locks that would poison later tests. - cli_scope_host_serving(): raise the status probe deadline from 5 s to 15 s so a slow-but-alive daemon on a loaded runner is not misread as "drained" (the flaky ASSERT(host_serving) failure this fixture showed); it still fails cleanly when the daemon is genuinely gone or reports stopping. None of these decides a healthy test — the child signals in well under a second on a healthy runner; the bounds only convert an otherwise unbounded hang into a deterministic pass/fail. The scoping contract still binds: with cli_activation_resolve_scope forced to always return ACTIVE, the three "must-not-drain" tests go RED at ASSERT(host_serving) and the host-namespace "must-drain" test stays GREEN. cli suite: 313 passed locally (macOS, twice); 30-round pre-fix and 8-round post-fix stress of the four tests were clean; a fault-injected never-ready child fails all four cleanly in ~37 s with no suite hang. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- tests/test_cli.c | 89 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 82 insertions(+), 7 deletions(-) diff --git a/tests/test_cli.c b/tests/test_cli.c index 3a4b4a851..43779e3bd 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -42,6 +42,7 @@ #include #include #include +#include #endif #ifdef __APPLE__ #include @@ -1510,6 +1511,16 @@ TEST(cli_install_recovers_markerless_stale_rendezvous) { * install under test and asks the daemon itself afterwards. */ #define CLI_SCOPE_HOST_DRAINED 3 #define CLI_SCOPE_TIMEOUT_MS 5000U +/* Generous, BOUNDED waits so a wedged host child (a fork-time sanitizer + * allocator stall is the classic cause — see the posix_spawn fix history) + * fails the test cleanly instead of hanging the whole suite to the CI + * wall-clock kill. Each comfortably exceeds the child's own 45 s cohort + * admission deadline plus a multi-second service start/teardown, so none trips + * for a merely slow-but-healthy runner; they only convert an otherwise + * unbounded hang into a deterministic pass/fail. */ +#define CLI_SCOPE_READY_TIMEOUT_MS 90000U +#define CLI_SCOPE_HOST_REAP_TIMEOUT_MS 60000U +#define CLI_SCOPE_HOST_SERVING_TIMEOUT_MS 15000U typedef struct { char tmpdir[256]; @@ -1598,6 +1609,68 @@ static _Noreturn void cli_scope_host_child(const cli_scope_fixture_t *fixture, i _exit(!ready_ok ? 1 : drained ? CLI_SCOPE_HOST_DRAINED : 0); } +/* Bounded read of the host child's one-byte readiness signal. A child that + * deadlocks before it can write (a fork-time allocator stall under a sanitizer + * is the classic cause) must never hang the whole suite on an unbounded read: + * poll to a generous deadline, then let the caller's ASSERT_TRUE(ready) fail + * cleanly. Returns the byte, or 0 when the child died, closed the pipe, or + * never answered in time. */ +static char cli_scope_wait_ready(int fd, uint32_t timeout_ms) { + uint64_t deadline = cbm_now_ms() + timeout_ms; + for (;;) { + int64_t remaining = (int64_t)deadline - (int64_t)cbm_now_ms(); + if (remaining <= 0) { + return 0; + } + struct pollfd pfd = {.fd = fd, .events = POLLIN, .revents = 0}; + int r = poll(&pfd, 1, (int)remaining); + if (r < 0) { + if (errno == EINTR) { + continue; + } + return 0; + } + if (r == 0) { + return 0; /* deadline reached with no signal */ + } + char ready = 0; + ssize_t got = read(fd, &ready, 1); + if (got == 1) { + return ready; + } + if (got < 0 && errno == EINTR) { + continue; + } + return 0; /* EOF (child gone) or error */ + } +} + +/* Reap the host child within a bound: the release signal makes a healthy child + * break within ~50 ms and finish teardown in a few seconds, so a child still + * alive past the deadline is wedged — SIGKILL it and reap so the suite always + * makes progress. Returns the child's exit code, or -1 when it had to be + * killed or did not exit cleanly; a -1 fails the caller's host_exit assertion + * cleanly rather than hanging. */ +static int cli_scope_reap_host(pid_t host, uint32_t timeout_ms) { + uint64_t deadline = cbm_now_ms() + timeout_ms; + int status = 0; + for (;;) { + pid_t reaped = waitpid(host, &status, WNOHANG); + if (reaped == host) { + return WIFEXITED(status) ? WEXITSTATUS(status) : -1; + } + if (reaped < 0 && errno != EINTR) { + return -1; + } + if (cbm_now_ms() >= deadline) { + (void)kill(host, SIGKILL); + (void)waitpid(host, &status, 0); + return -1; + } + cbm_usleep(2000); + } +} + static bool cli_scope_fixture_start(cli_scope_fixture_t *fixture, const char *tag) { memset(fixture, 0, sizeof(*fixture)); fixture->host = -1; @@ -1657,8 +1730,8 @@ static bool cli_scope_fixture_start(cli_scope_fixture_t *fixture, const char *ta } close(ready_pipe[1]); close(release_pipe[0]); - char ready = 0; - bool host_ready = child > 0 && read(ready_pipe[0], &ready, 1) == 1 && ready == 'R'; + char ready = child > 0 ? cli_scope_wait_ready(ready_pipe[0], CLI_SCOPE_READY_TIMEOUT_MS) : 0; + bool host_ready = ready == 'R'; close(ready_pipe[0]); fixture->host = child; fixture->release_fd = release_pipe[1]; @@ -1682,9 +1755,14 @@ static bool cli_scope_fixture_start(cli_scope_fixture_t *fixture, const char *ta * committed client still admitted. */ static bool cli_scope_host_serving(const cli_scope_fixture_t *fixture) { cbm_daemon_runtime_status_t status = {0}; + /* A generous, bounded status deadline: a foreign-namespace install leaves + * this daemon serving, so a slow response on a loaded runner must not be + * misread as "drained" (the flaky failure this fixture showed). The call + * still fails cleanly — a genuinely drained daemon is unreachable or + * reports stopping — it just no longer decides survival on a 5 s budget. */ return fixture->endpoint && cbm_daemon_runtime_request_status(fixture->endpoint, &fixture->identity, - CLI_SCOPE_TIMEOUT_MS, &status) && + CLI_SCOPE_HOST_SERVING_TIMEOUT_MS, &status) && !status.stopping && status.committed_clients == 1; } @@ -1718,10 +1796,7 @@ static int cli_scope_fixture_finish(cli_scope_fixture_t *fixture) { } int host_exit = -1; if (fixture->host > 0) { - int status = 0; - if (waitpid(fixture->host, &status, 0) == fixture->host && WIFEXITED(status)) { - host_exit = WEXITSTATUS(status); - } + host_exit = cli_scope_reap_host(fixture->host, CLI_SCOPE_HOST_REAP_TIMEOUT_MS); fixture->host = -1; } cbm_daemon_ipc_endpoint_free(fixture->endpoint); From e45d705105a63f2fd9b35ab9e37ffbc640be8231 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 9 Sep 2026 07:33:51 +0200 Subject: [PATCH 040/130] fix(mcp): free the extraction bitset cache on the in-process auto-index thread (LSan) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cbm_kind_in_set lazily builds a THREAD-LOCAL node-type bitset cache during extraction. That cache must be freed before a worker thread exits, or the calloc'd bitsets are orphaned when the thread's TLS is torn down and LeakSanitizer reports them at process exit. Parallel workers already free it (pass_parallel.c:993) and the main thread frees it at process teardown; the IN-PROCESS sequential auto-index runs extraction on autoindex_thread — a short-lived pthread joined by cbm_mcp_server_free — and was the one gap: it built the cache but never freed it (360 bytes in 9 allocations leaked, all from ks_build). This is a PRE-EXISTING latent leak: the extraction path and autoindex_thread are byte-identical to main. It is newly EXPOSED by #713's new autoindex_limit_admits_non_git_root_under_limit test, the first LSan-observed path that runs an in-process auto-index and joins the worker thread in-process — so the fix ships with the test that exposes it. Free the calling thread's bitset cache after cbm_pipeline_free(p), mirroring the established per-worker-thread teardown. Zero behavior change to the CLI/main path. macOS LSan lane, mcp suite: 360 B / 9 allocations -> 0. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- src/mcp/mcp.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 87df93e60..07ea0b698 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -17383,6 +17383,16 @@ static void register_watcher_if_enabled(cbm_mcp_server_t *srv) { } /* Background auto-index thread function */ +/* Extraction builds a THREAD-LOCAL node-type bitset cache (cbm_kind_in_set). + * Every worker thread that runs extraction must free that cache before it exits, + * or the calloc'd bitsets are orphaned when the thread's TLS is torn down and + * LeakSanitizer reports them at process exit. Parallel workers do this in + * pass_parallel.c; the in-process (sequential) auto-index runs extraction on + * THIS short-lived thread, so it must free its own cache too. Declared extern + * (not via internal/cbm/helpers.h) to avoid pulling the extraction layer's + * header into the MCP TU — the same pattern test_main.c uses for teardown. */ +extern void cbm_kind_in_set_free_cache(void); + static void *autoindex_thread(void *arg) { cbm_mcp_server_t *srv = (cbm_mcp_server_t *)arg; @@ -17422,6 +17432,7 @@ static void *autoindex_thread(void *arg) { cbm_pipeline_unlock(); cbm_pipeline_free(p); + cbm_kind_in_set_free_cache(); /* free THIS thread's extraction bitset cache (see above) */ cbm_mem_collect(); /* return mimalloc pages to OS after indexing (in-process only) */ if (rc == 0) { From 21ae02154a198265d7dbed13af4280aa2b1e5690 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=86=B2?= Date: Wed, 19 Aug 2026 14:39:38 +0800 Subject: [PATCH 041/130] feat(index): add opt-in discovery resource limits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Indexing accepts whatever a repository contains. A tree carrying a vendored monorepo, a generated dump, or a runaway build directory is discovered in full, and the first sign of trouble is a host under memory pressure with nothing that attributes it to indexing. Add two opt-in limits evaluated during discovery against accepted source files only: index_max_files and index_max_source_mb. Both default to off, so nothing changes until an operator sets one. Crossing a limit fails the whole attempt with a structured resource_limit_exceeded result naming the resource, the observed value and the limit; no partial graph is published, and an existing serving index keeps answering. Limits are read from the CLI-managed _config.db and are not MCP request arguments. A supervised parent replaces any caller-supplied policy before spawning its worker, and the worker rejects a missing or incomplete contract, so the CLI, the daemon and the supervised worker all enforce the same decision. Both keys reach an operator through the existing config get/set/list/reset with no new subcommand. `set` suppresses its own generic message for them because the policy writer names the precise reason -- so that writer speaks on every failure it can return, including a validated value whose write then fails on a database that cannot be written. Exiting non-zero in silence is not an acceptable answer from a CLI. The two shell regressions that hand-roll the supervisor's worker argv carry that contract as well. Without it the worker exits before either guard can observe anything, and the guard would go quietly vacuous. Signed-off-by: 刘冲 --- Makefile.cbm | 2 + README.md | 6 + docs/CONFIGURATION.md | 10 + docs/INDEX_RESOURCE_LIMITS.md | 68 +++++ scripts/test-runtime.sh | 10 + src/cli/cli.c | 56 +++- src/cli/cli.h | 7 + src/daemon/application.c | 35 ++- src/discover/discover.c | 51 +++- src/discover/discover.h | 9 +- src/foundation/index_policy.c | 153 +++++++++++ src/foundation/index_policy.h | 53 ++++ src/mcp/mcp.c | 135 +++++++++- src/mcp/mcp_internal.h | 8 + src/pipeline/pipeline.c | 47 +++- src/pipeline/pipeline.h | 9 + src/pipeline/pipeline_incremental.c | 28 +- src/pipeline/pipeline_internal.h | 4 +- tests/test_daemon_application.c | 86 +++++- tests/test_discover.c | 161 +++++++++++ tests/test_index_policy.c | 403 ++++++++++++++++++++++++++++ tests/test_main.c | 2 + tests/test_pipeline.c | 68 +++++ tests/test_worker_error_response.sh | 2 +- tests/test_worker_watchdog.sh | 2 +- 25 files changed, 1371 insertions(+), 44 deletions(-) create mode 100644 docs/INDEX_RESOURCE_LIMITS.md create mode 100644 src/foundation/index_policy.c create mode 100644 src/foundation/index_policy.h create mode 100644 tests/test_index_policy.c diff --git a/Makefile.cbm b/Makefile.cbm index 35844a529..ca9779156 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -251,6 +251,7 @@ FOUNDATION_SRCS = \ src/foundation/profile.c \ src/foundation/dump_verify.c \ src/foundation/limits.c \ + src/foundation/index_policy.c \ src/foundation/subprocess.c \ src/foundation/sha256.c \ src/foundation/secure_random.c \ @@ -532,6 +533,7 @@ TEST_FOUNDATION_SRCS = \ tests/test_str_intern.c \ tests/test_log.c \ tests/test_str_util.c \ + tests/test_index_policy.c \ tests/test_workspace.c \ tests/test_platform.c \ tests/test_diagnostics.c \ diff --git a/README.md b/README.md index c6ab67ee2..1a131d3c6 100644 --- a/README.md +++ b/README.md @@ -733,9 +733,15 @@ codebase-memory-mcp config set auto_index true # auto-index on session codebase-memory-mcp config set auto_index_limit 50000 # max files for auto-index codebase-memory-mcp config set auto_watch false # don't register background git watcher (default: true) codebase-memory-mcp config set watcher_enabled false # stop the watcher thread entirely (default: true) +codebase-memory-mcp config set index_max_files 250000 # optional per-index source-file limit +codebase-memory-mcp config set index_max_source_mb 16384 # optional per-index source-size limit codebase-memory-mcp config reset auto_index # reset to default ``` +The two `index_max_*` settings default to `off`. Exceeding one fails the complete +index attempt rather than publishing a partial graph; an existing serving index +is preserved. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md). + ### Environment Variables | Variable | Default | Description | diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 862d591c3..038256976 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -89,6 +89,8 @@ Current keys: | `auto_index_limit` | `50000` | Maximum file count allowed for automatic indexing of a new project. | | `auto_watch` | `true` | Register the session's project with the background git watcher on connect. Set `false` to keep a session from registering its project (the watcher still runs for other projects). | | `watcher_enabled` | `true` | Master switch for the background watcher subsystem. Set `false` to stop the watcher from starting at all — no poll thread and no project registration. Reindex manually with `index_repository` when disabled. | +| `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. | +| `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. | > **`watcher_enabled` vs `auto_watch`.** `watcher_enabled` controls whether the > watcher *subsystem* starts at all (the background poll thread). `auto_watch` is @@ -116,6 +118,14 @@ Current keys: > `auto_index` still runs, and `index_repository` stays available for manual > reindexing. +The two `index_max_*` settings are independent and disabled by default. They +apply to explicit indexing, automatic indexing, and watcher re-indexing, but not +to `cross-repo-intelligence`, which does not scan repository source files. +Equality is allowed; exceeding either setting fails the complete index request +and preserves any previously serving database. See +[Index resource limits](INDEX_RESOURCE_LIMITS.md) for counting, validation, and +error-response details. + ## 3. UI Settings The optional built-in graph UI stores its settings in: diff --git a/docs/INDEX_RESOURCE_LIMITS.md b/docs/INDEX_RESOURCE_LIMITS.md new file mode 100644 index 000000000..a3d57e015 --- /dev/null +++ b/docs/INDEX_RESOURCE_LIMITS.md @@ -0,0 +1,68 @@ +# Index resource limits + +Index resource limits are optional operator controls for repositories whose +discovery breadth is not known in advance. They are disabled by default so +existing large-repository workloads retain their current behavior. + +## Discovery settings + +| Key | Default | Accepted value | Protects | +|---|---:|---:|---| +| `index_max_files` | `off` | `off` or `1..10000000` | Accepted source-file count | +| `index_max_source_mb` | `off` | `off` or `1..1048576` | Accepted source-file bytes | + +Set or reset them with the normal configuration command: + +```bash +codebase-memory-mcp config set index_max_files 250000 +codebase-memory-mcp config set index_max_source_mb 16384 +codebase-memory-mcp config reset index_max_files +``` + +Values use base-10 integers. MiB means 1,048,576 bytes. Empty values, zero, +negative values, suffixes, trailing characters, and values outside the stated +ranges are rejected without changing the stored value. + +## Counting and failure semantics + +`index_max_files` counts a file only after it passes directory pruning, ignore +rules, filename and suffix filters, language detection, and the existing +per-file size rule. `index_max_source_mb` sums the filesystem sizes of that same +accepted set. + +Equality is allowed. The first file or byte that makes an observed value greater +than its limit stops discovery. CBM discards the partial file list and does not +publish a partial graph as a complete index. + +For an explicit MCP request the error payload contains: + +```json +{ + "status": "error", + "code": "resource_limit_exceeded", + "stage": "discovery", + "resource": "files", + "observed": 250001, + "limit": 250000, + "unit": "files", + "retryable": true, + "serving_index_preserved": true, + "message": "Index discovery exceeded index_max_files" +} +``` + +The previous database remains available because publication occurs only after a +complete discovery and successful staged build. If no previous database exists, +`serving_index_preserved` is false. + +## Trust and compatibility + +Limits are read from the CLI-managed `_config.db`; they are not MCP request +arguments. A supervised parent replaces any caller-supplied internal policy +before spawning its worker, and the worker rejects a missing or incomplete +parent policy. + +These settings do not replace or increase `auto_index_limit`, change the 512 MiB +single-file cap, alter workspace-root authorization, or affect +`cross-repo-intelligence`. With both settings `off`, discovery follows the +existing unbounded path. diff --git a/scripts/test-runtime.sh b/scripts/test-runtime.sh index cb69cadfb..5d2c0c3ae 100644 --- a/scripts/test-runtime.sh +++ b/scripts/test-runtime.sh @@ -105,6 +105,16 @@ _cbm_test_runtime_daemon() { CBM_CACHE_DIR="$_CBM_TEST_RUNTIME_PRODUCT_CACHE" "$1" daemon "$2" } +# The supervisor resolves one resource policy per index and hands it to the +# worker in argv; a worker that finds no complete policy refuses to start rather +# than index unbounded. A shell test that spawns `cli --index-worker` itself +# stands in for the supervisor and owes the worker the same object. Mirrors +# cbm_mcp_index_policy_add_to_args: every key in cbm_index_policy_key_at, and +# nothing else. +cbm_test_index_worker_policy_json() { + printf '%s' '"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"}' +} + cbm_test_runtime_cleanup() { local binary="${1:-}" root="$_CBM_TEST_RUNTIME_CREATED_ROOT" local name="${_CBM_TEST_RUNTIME_CREATED_ROOT##*/}" runtime_entry="" active=0 diff --git a/src/cli/cli.c b/src/cli/cli.c index 956489cd9..7770c0f47 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -7117,6 +7117,25 @@ bool cbm_config_watcher_enabled(cbm_config_t *cfg) { return cbm_config_get_bool(cfg, CBM_CONFIG_WATCHER_ENABLED, true); } +bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t *policy, + char *error, size_t error_size) { + if (!cfg || !policy) { + if (error && error_size > 0) { + (void)snprintf(error, error_size, "index resource configuration is unavailable"); + } + return false; + } + cbm_index_policy_init(policy); + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + const char *key = cbm_index_policy_key_at(index); + const char *value = cbm_config_get(cfg, key, cbm_index_policy_default_value(key)); + if (!cbm_index_policy_set(policy, key, value, error, error_size)) { + return false; + } + } + return true; +} + /* ── Config CLI subcommand ────────────────────────────────────── */ /* THE config-key table. list, get, help, and key validation all read this one @@ -7141,6 +7160,8 @@ static const config_key_def_t CONFIG_KEYS[] = { {CBM_CONFIG_UI_LANG, "auto", "Pin graph UI language: en, zh, or auto"}, {CBM_CONFIG_UI_ENABLED, "false", "Serve the graph UI on a loopback HTTP port"}, {CBM_CONFIG_UI_PORT, "9749", "Port for the graph UI listener when enabled"}, + {CBM_INDEX_CONFIG_MAX_FILES, "off", "Max accepted source files per index, or off"}, + {CBM_INDEX_CONFIG_MAX_SOURCE_MB, "off", "Max accepted source MiB per index, or off"}, }; /* #1558: ui_enabled and ui_port were reachable ONLY by hand-editing @@ -7166,6 +7187,31 @@ static bool config_key_is_ui(const char *key) { return key && (strcmp(key, CBM_CONFIG_UI_ENABLED) == 0 || strcmp(key, CBM_CONFIG_UI_PORT) == 0); } +static bool config_key_is_index_policy(const char *key) { + return key && (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0 || + strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0); +} + +static int config_index_policy_write(cbm_config_t *config, const char *key, const char *value) { + cbm_index_resource_policy_t candidate; + cbm_index_policy_init(&candidate); + char error[CLI_BUF_256]; + if (!cbm_index_policy_set(&candidate, key, value, error, sizeof(error))) { + (void)fprintf(stderr, "error: %s\n", error); + return CLI_ERR; + } + int rc = cbm_config_set(config, key, value); + if (rc != 0) { + /* The caller suppresses its own message for policy keys because this + * helper names the precise reason. That is only true if the helper + * speaks on every failure it can return: a validated value whose write + * then fails -- a locked or read-only _config.db -- used to exit + * non-zero having printed nothing at all. */ + (void)fprintf(stderr, "error: failed to set %s\n", key); + } + return rc; +} + static void config_ui_read(const char *key, char *out, size_t out_sz) { cbm_ui_config_t ui; cbm_ui_config_load(&ui); @@ -7303,10 +7349,16 @@ int cbm_cmd_config(int argc, char **argv) { rc = CLI_TRUE; } } else { - if (cbm_config_set(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]) == 0) { + int set_rc = + config_key_is_index_policy(argv[CLI_SKIP_ONE]) + ? config_index_policy_write(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]) + : cbm_config_set(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]); + if (set_rc == 0) { printf("%s = %s\n", argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]); } else { - (void)fprintf(stderr, "error: failed to set %s\n", argv[CLI_SKIP_ONE]); + if (!config_key_is_index_policy(argv[CLI_SKIP_ONE])) { + (void)fprintf(stderr, "error: failed to set %s\n", argv[CLI_SKIP_ONE]); + } rc = CLI_TRUE; } } diff --git a/src/cli/cli.h b/src/cli/cli.h index 139221f0a..cf7f6fa44 100644 --- a/src/cli/cli.h +++ b/src/cli/cli.h @@ -13,6 +13,8 @@ #include #include +#include "foundation/index_policy.h" + typedef struct cbm_mcp_server cbm_mcp_server_t; /* ── Version ──────────────────────────────────────────────────── */ @@ -420,6 +422,11 @@ int cbm_config_set(cbm_config_t *cfg, const char *key, const char *value); /* Delete a config key. Returns 0 on success. */ int cbm_config_delete(cbm_config_t *cfg, const char *key); +/* Load and validate the operator-controlled discovery policy. Invalid stored + * values fail closed instead of silently disabling a guard. */ +bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t *policy, + char *error, size_t error_size); + /* Well-known config keys */ #define CBM_CONFIG_AUTO_INDEX "auto_index" #define CBM_CONFIG_AUTO_INDEX_LIMIT "auto_index_limit" diff --git a/src/daemon/application.c b/src/daemon/application.c index 03ca80b37..39cc8b72c 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -208,7 +208,8 @@ static bool application_unique_recovery_file(char out[APPLICATION_PATH_CAP], con static bool application_update_reap(cbm_daemon_application_t *application, bool wait, uint32_t timeout_ms); static void *application_job_thread(void *opaque); -static char *application_auto_index_args(const char *root_path); +static char *application_auto_index_args(cbm_daemon_application_t *application, + const char *root_path); static cbm_daemon_application_job_t *application_job_subscribe_locked( cbm_daemon_application_t *application, const char *project_key, const char *root_path, const char *args_json, application_job_subscribe_status_t *status_out); @@ -1454,7 +1455,7 @@ static void application_auto_index_retry_pending_locked(cbm_daemon_application_t application_refresh_watch_locked(session); continue; } - char *args = application_auto_index_args(root_path); + char *args = application_auto_index_args(application, root_path); if (!args) { continue; } @@ -2004,7 +2005,27 @@ static bool application_update_reap(cbm_daemon_application_t *application, bool } } -static char *application_auto_index_args(const char *root_path) { +static bool application_index_args_add_policy(cbm_daemon_application_t *application, + yyjson_mut_doc *document, yyjson_mut_val *root) { + cbm_config_t *owned_config = NULL; + cbm_config_t *config = application ? application->config : NULL; + if (!config) { + owned_config = cbm_config_open(cbm_resolve_cache_dir()); + config = owned_config; + } + cbm_index_resource_policy_t policy; + char error[CBM_SZ_256] = {0}; + bool loaded = cbm_config_load_index_policy(config, &policy, error, sizeof(error)); + cbm_config_close(owned_config); + if (!loaded) { + cbm_log_error("daemon.index.policy", "error", error); + return false; + } + return cbm_mcp_index_policy_add_to_args(document, root, &policy); +} + +static char *application_auto_index_args(cbm_daemon_application_t *application, + const char *root_path) { yyjson_mut_doc *document = yyjson_mut_doc_new(NULL); yyjson_mut_val *root = document ? yyjson_mut_obj(document) : NULL; if (!document || !root) { @@ -2012,7 +2033,8 @@ static char *application_auto_index_args(const char *root_path) { return NULL; } yyjson_mut_doc_set_root(document, root); - char *args = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) + char *args = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) && + application_index_args_add_policy(application, document, root) ? yyjson_mut_write(document, 0, NULL) : NULL; yyjson_mut_doc_free(document); @@ -2059,7 +2081,7 @@ static void application_background_initialize_impl(cbm_daemon_application_sessio files); } bool args_required = auto_index_candidate && within_auto_index_limit; - char *args = args_required ? application_auto_index_args(root_path) : NULL; + char *args = args_required ? application_auto_index_args(application, root_path) : NULL; application_jobs_reap_completed(application); cbm_mutex_lock(&application->mutex); if (application->stopping || application_request_cancelled_locked(session)) { @@ -3457,7 +3479,8 @@ static int application_background_index(cbm_daemon_application_t *application, return -1; } yyjson_mut_doc_set_root(document, root); - bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root); + bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root) && + application_index_args_add_policy(application, document, root); char *default_project = cbm_project_name_from_path(canonical_root); bool custom_project = project_name[0] && (!default_project || strcmp(default_project, project_name) != 0); diff --git a/src/discover/discover.c b/src/discover/discover.c index 5cadaed75..fd4cea272 100644 --- a/src/discover/discover.c +++ b/src/discover/discover.c @@ -13,6 +13,7 @@ #include "foundation/constants.h" #include "foundation/compat_fs.h" +#include "foundation/limits.h" #include "foundation/workspace.h" #include "foundation/platform.h" #ifdef _WIN32 @@ -414,6 +415,10 @@ typedef struct { int capacity; int max_files; uint64_t deadline_ms; + const cbm_index_resource_policy_t *resource_policy; + cbm_index_resource_violation_t *resource_violation; + uint64_t source_files; + uint64_t source_bytes; bool count_only; bool collect_excluded; bool limit_exceeded; @@ -435,6 +440,21 @@ typedef struct { int ignored_total; } file_list_t; +static void file_list_resource_violation(file_list_t *fl, cbm_index_resource_t resource, + uint64_t observed, uint64_t limit) { + if (!fl || fl->limit_exceeded || fl->failed) { + return; + } + fl->limit_exceeded = true; + if (fl->resource_violation) { + *fl->resource_violation = (cbm_index_resource_violation_t){ + .resource = resource, + .observed = observed, + .limit = limit, + }; + } +} + static bool file_list_should_stop(file_list_t *fl) { if (!fl) { return true; @@ -504,6 +524,28 @@ static void fl_add(file_list_t *fl, const char *abs_path, const char *rel_path, fl->limit_exceeded = true; return; } + uint64_t source_size = size > 0 ? (uint64_t)size : 0; + bool resource_accepted = fl->resource_policy && source_size <= (uint64_t)cbm_max_file_bytes(); + if (resource_accepted) { + if (fl->resource_policy->max_files.enabled && + fl->source_files >= fl->resource_policy->max_files.value) { + file_list_resource_violation(fl, CBM_INDEX_RESOURCE_FILES, fl->source_files + 1U, + fl->resource_policy->max_files.value); + return; + } + if (fl->resource_policy->max_source_bytes.enabled) { + uint64_t limit = fl->resource_policy->max_source_bytes.value; + if (source_size > limit || fl->source_bytes > limit - source_size) { + uint64_t observed = source_size > UINT64_MAX - fl->source_bytes + ? UINT64_MAX + : fl->source_bytes + source_size; + file_list_resource_violation(fl, CBM_INDEX_RESOURCE_SOURCE_BYTES, observed, limit); + return; + } + fl->source_bytes += source_size; + } + fl->source_files++; + } if (fl->count_only) { fl->count++; return; @@ -1200,6 +1242,9 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc if (ignored_total_out) { *ignored_total_out = 0; } + if (opts && opts->resource_violation) { + *opts->resource_violation = (cbm_index_resource_violation_t){0}; + } if (!repo_path || !out || !count || (count_only && max_files < 0)) { return CBM_DISCOVER_ERROR; } @@ -1274,6 +1319,8 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc file_list_t fl = { .max_files = count_only ? max_files : -1, .deadline_ms = count_only ? deadline_ms : 0, + .resource_policy = opts ? opts->resource_policy : NULL, + .resource_violation = opts ? opts->resource_violation : NULL, .count_only = count_only, .collect_excluded = !count_only && excluded_out != NULL, .collect_ignored = !count_only && ignored_out != NULL, @@ -1296,11 +1343,11 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc } return fl.limit_exceeded ? CBM_DISCOVER_LIMIT_EXCEEDED : CBM_DISCOVER_OK; } - if (fl.failed) { + if (fl.failed || fl.limit_exceeded) { cbm_discover_free(fl.files, fl.count); cbm_discover_free_excluded(fl.excluded, fl.excluded_count); cbm_discover_free_ignored(fl.ignored, fl.ignored_count); - return CBM_DISCOVER_ERROR; + return fl.limit_exceeded ? CBM_DISCOVER_LIMIT_EXCEEDED : CBM_DISCOVER_ERROR; } *out = fl.files; diff --git a/src/discover/discover.h b/src/discover/discover.h index b15844340..fb8d7b5e6 100644 --- a/src/discover/discover.h +++ b/src/discover/discover.h @@ -17,6 +17,7 @@ /* Use the existing CBMLanguage enum from extraction layer */ #include "cbm.h" +#include "foundation/index_policy.h" /* ── Language detection ──────────────────────────────────────────── */ @@ -143,9 +144,11 @@ typedef struct { } cbm_file_info_t; typedef struct { - cbm_index_mode_t mode; /* CBM_MODE_FULL or CBM_MODE_FAST */ - const char *ignore_file; /* path to .cbmignore file, or NULL */ - int64_t max_file_size; /* 0 = no limit */ + cbm_index_mode_t mode; /* discovery filtering mode */ + const char *ignore_file; /* .cbmignore path, or NULL */ + int64_t max_file_size; /* 0 = no per-file limit */ + const cbm_index_resource_policy_t *resource_policy; /* NULL = no resource limits */ + cbm_index_resource_violation_t *resource_violation; /* optional exact diagnostic */ } cbm_discover_opts_t; typedef enum { diff --git a/src/foundation/index_policy.c b/src/foundation/index_policy.c new file mode 100644 index 000000000..0d2b986bf --- /dev/null +++ b/src/foundation/index_policy.c @@ -0,0 +1,153 @@ +#include "foundation/index_policy.h" + +#include +#include + +static const char *const INDEX_POLICY_KEYS[] = { + CBM_INDEX_CONFIG_MAX_FILES, + CBM_INDEX_CONFIG_MAX_SOURCE_MB, +}; + +static void set_error(char *error, size_t error_size, const char *key, uint64_t maximum) { + if (error && error_size > 0) { + (void)snprintf(error, error_size, "%s must be off or an integer from 1 to %llu", key, + (unsigned long long)maximum); + } +} + +static bool parse_bounded_uint64(const char *value, uint64_t maximum, uint64_t *parsed) { + if (!value || !value[0] || !parsed) { + return false; + } + uint64_t result = 0; + for (const unsigned char *cursor = (const unsigned char *)value; *cursor; cursor++) { + if (*cursor < '0' || *cursor > '9') { + return false; + } + uint64_t digit = (uint64_t)(*cursor - '0'); + if (result > (maximum - digit) / 10U) { + return false; + } + result = result * 10U + digit; + } + if (result == 0) { + return false; + } + *parsed = result; + return true; +} + +void cbm_index_policy_init(cbm_index_resource_policy_t *policy) { + if (policy) { + *policy = (cbm_index_resource_policy_t){0}; + } +} + +bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy) { + return policy && (policy->max_files.enabled || policy->max_source_bytes.enabled); +} + +size_t cbm_index_policy_key_count(void) { + return sizeof(INDEX_POLICY_KEYS) / sizeof(INDEX_POLICY_KEYS[0]); +} + +const char *cbm_index_policy_key_at(size_t index) { + return index < cbm_index_policy_key_count() ? INDEX_POLICY_KEYS[index] : NULL; +} + +const char *cbm_index_policy_default_value(const char *key) { + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + if (key && strcmp(key, INDEX_POLICY_KEYS[index]) == 0) { + return "off"; + } + } + return NULL; +} + +bool cbm_index_policy_set(cbm_index_resource_policy_t *policy, const char *key, const char *value, + char *error, size_t error_size) { + if (error && error_size > 0) { + error[0] = '\0'; + } + if (!policy || !key || !value) { + set_error(error, error_size, key ? key : "index resource limit", 0); + return false; + } + + cbm_index_limit_u64_t *target = NULL; + uint64_t maximum = 0; + uint64_t multiplier = 1; + if (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0) { + target = &policy->max_files; + maximum = CBM_INDEX_MAX_FILES_VALUE; + } else if (strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0) { + target = &policy->max_source_bytes; + maximum = CBM_INDEX_MAX_SOURCE_MB_VALUE; + multiplier = CBM_INDEX_MIB_BYTES; + } else { + set_error(error, error_size, key, 0); + return false; + } + + cbm_index_limit_u64_t candidate = {0}; + if (strcmp(value, "off") != 0) { + uint64_t parsed = 0; + if (!parse_bounded_uint64(value, maximum, &parsed)) { + set_error(error, error_size, key, maximum); + return false; + } + candidate.enabled = true; + candidate.value = parsed * multiplier; + } + *target = candidate; + return true; +} + +bool cbm_index_policy_format(const cbm_index_resource_policy_t *policy, const char *key, char *out, + size_t out_size) { + if (!policy || !key || !out || out_size == 0) { + return false; + } + const cbm_index_limit_u64_t *limit = NULL; + uint64_t divisor = 1; + if (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0) { + limit = &policy->max_files; + } else if (strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0) { + limit = &policy->max_source_bytes; + divisor = CBM_INDEX_MIB_BYTES; + } else { + return false; + } + int length = limit->enabled + ? snprintf(out, out_size, "%llu", (unsigned long long)(limit->value / divisor)) + : snprintf(out, out_size, "off"); + return length >= 0 && (size_t)length < out_size; +} + +const char *cbm_index_resource_name(cbm_index_resource_t resource) { + switch (resource) { + case CBM_INDEX_RESOURCE_FILES: + return "files"; + case CBM_INDEX_RESOURCE_SOURCE_BYTES: + return "source_bytes"; + case CBM_INDEX_RESOURCE_NONE: + default: + return "unknown"; + } +} + +const char *cbm_index_resource_unit(cbm_index_resource_t resource) { + return resource == CBM_INDEX_RESOURCE_FILES ? "files" : "bytes"; +} + +const char *cbm_index_resource_config_key(cbm_index_resource_t resource) { + switch (resource) { + case CBM_INDEX_RESOURCE_FILES: + return CBM_INDEX_CONFIG_MAX_FILES; + case CBM_INDEX_RESOURCE_SOURCE_BYTES: + return CBM_INDEX_CONFIG_MAX_SOURCE_MB; + case CBM_INDEX_RESOURCE_NONE: + default: + return "index_resource_limit"; + } +} diff --git a/src/foundation/index_policy.h b/src/foundation/index_policy.h new file mode 100644 index 000000000..6a6c499fc --- /dev/null +++ b/src/foundation/index_policy.h @@ -0,0 +1,53 @@ +#ifndef CBM_INDEX_POLICY_H +#define CBM_INDEX_POLICY_H + +#include +#include +#include + +#define CBM_INDEX_CONFIG_MAX_FILES "index_max_files" +#define CBM_INDEX_CONFIG_MAX_SOURCE_MB "index_max_source_mb" + +#define CBM_INDEX_MAX_FILES_VALUE UINT64_C(10000000) +#define CBM_INDEX_MAX_SOURCE_MB_VALUE UINT64_C(1048576) +#define CBM_INDEX_MIB_BYTES UINT64_C(1048576) + +typedef struct { + bool enabled; + uint64_t value; +} cbm_index_limit_u64_t; + +typedef struct { + cbm_index_limit_u64_t max_files; + cbm_index_limit_u64_t max_source_bytes; +} cbm_index_resource_policy_t; + +typedef enum { + CBM_INDEX_RESOURCE_NONE = 0, + CBM_INDEX_RESOURCE_FILES, + CBM_INDEX_RESOURCE_SOURCE_BYTES, +} cbm_index_resource_t; + +typedef struct { + cbm_index_resource_t resource; + uint64_t observed; + uint64_t limit; +} cbm_index_resource_violation_t; + +void cbm_index_policy_init(cbm_index_resource_policy_t *policy); +bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy); + +size_t cbm_index_policy_key_count(void); +const char *cbm_index_policy_key_at(size_t index); +const char *cbm_index_policy_default_value(const char *key); + +bool cbm_index_policy_set(cbm_index_resource_policy_t *policy, const char *key, const char *value, + char *error, size_t error_size); +bool cbm_index_policy_format(const cbm_index_resource_policy_t *policy, const char *key, char *out, + size_t out_size); + +const char *cbm_index_resource_name(cbm_index_resource_t resource); +const char *cbm_index_resource_unit(cbm_index_resource_t resource); +const char *cbm_index_resource_config_key(cbm_index_resource_t resource); + +#endif diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 1b0bce021..483c61bb1 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -10606,6 +10606,63 @@ cbm_mcp_supervised_result_disposition_t cbm_mcp_supervised_result_disposition( return CBM_MCP_SUPERVISED_RESULT_CONTAINED_FAILURE; } +static bool index_policy_from_worker_args(const char *args, cbm_index_resource_policy_t *policy, + char *error, size_t error_size) { + yyjson_doc *doc = args ? yyjson_read(args, strlen(args), 0) : NULL; + yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; + yyjson_val *encoded = + root && yyjson_is_obj(root) ? yyjson_obj_get(root, "_cbm_index_policy") : NULL; + if (!encoded || !yyjson_is_obj(encoded) || + yyjson_obj_size(encoded) != cbm_index_policy_key_count()) { + yyjson_doc_free(doc); + (void)snprintf(error, error_size, "missing or incomplete trusted worker policy"); + return false; + } + + cbm_index_policy_init(policy); + bool valid = true; + for (size_t index = 0; valid && index < cbm_index_policy_key_count(); index++) { + const char *key = cbm_index_policy_key_at(index); + yyjson_val *value = yyjson_obj_get(encoded, key); + valid = value && yyjson_is_str(value) && + cbm_index_policy_set(policy, key, yyjson_get_str(value), error, error_size); + } + yyjson_doc_free(doc); + if (!valid && error && error_size > 0 && error[0] == '\0') { + (void)snprintf(error, error_size, "invalid trusted worker policy"); + } + return valid; +} + +static bool load_index_policy(cbm_mcp_server_t *srv, const char *args, + cbm_index_resource_policy_t *policy, char *error, size_t error_size) { + if (cbm_index_worker_active()) { + return index_policy_from_worker_args(args, policy, error, error_size); + } + cbm_config_t *owned_config = NULL; + cbm_config_t *config = srv ? srv->config : NULL; + if (!config) { + owned_config = cbm_config_open(cbm_resolve_cache_dir()); + config = owned_config; + } + bool loaded = cbm_config_load_index_policy(config, policy, error, error_size); + cbm_config_close(owned_config); + return loaded; +} + +bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root, + const cbm_index_resource_policy_t *policy) { + yyjson_mut_val *encoded = yyjson_mut_obj(doc); + bool valid = encoded != NULL; + for (size_t index = 0; valid && index < cbm_index_policy_key_count(); index++) { + const char *key = cbm_index_policy_key_at(index); + char value[CBM_SZ_64]; + valid = cbm_index_policy_format(policy, key, value, sizeof(value)) && + yyjson_mut_obj_add_strcpy(doc, encoded, key, value); + } + return valid && yyjson_mut_obj_add_val(doc, root, "_cbm_index_policy", encoded); +} + /* Run index_repository in a supervised worker subprocess with skip-and-continue * (Stage 3c). Returns the response string (caller frees): * - the worker's own response on a clean first run (the common path); @@ -10871,10 +10928,20 @@ static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_p if (!root_path || !root_path[0]) { return NULL; } + cbm_index_resource_policy_t policy; + char policy_error[CBM_SZ_256] = {0}; + if (!load_index_policy(srv, NULL, &policy, policy_error, sizeof(policy_error))) { + cbm_log_error("index.policy", "error", policy_error); + return NULL; + } yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); yyjson_mut_val *root = yyjson_mut_obj(doc); yyjson_mut_doc_set_root(doc, root); - yyjson_mut_obj_add_strcpy(doc, root, "repo_path", root_path); + if (!yyjson_mut_obj_add_strcpy(doc, root, "repo_path", root_path) || + !cbm_mcp_index_policy_add_to_args(doc, root, &policy)) { + yyjson_mut_doc_free(doc); + return NULL; + } char *args = yy_doc_to_str(doc); yyjson_mut_doc_free(doc); if (!args) { @@ -10923,8 +10990,9 @@ static bool resolve_session_repo_path(cbm_mcp_server_t *srv, char **repo_path) { /* Preserve every index option while replacing all caller-supplied repo_path * keys with the one canonical path that was actually authorized. */ -static char *index_args_with_repo_path(const char *args, const char *canonical_repo_path) { - if (!args || !canonical_repo_path) { +static char *index_args_with_repo_path(const char *args, const char *canonical_repo_path, + const cbm_index_resource_policy_t *policy) { + if (!args || !canonical_repo_path || !policy) { return NULL; } yyjson_doc *source = yyjson_read(args, strlen(args), 0); @@ -10941,8 +11009,14 @@ static char *index_args_with_repo_path(const char *args, const char *canonical_r yyjson_mut_doc_free(copy); return NULL; } - (void)yyjson_mut_obj_remove_key(copy_root, "repo_path"); - if (!yyjson_mut_obj_add_strcpy(copy, copy_root, "repo_path", canonical_repo_path)) { + while (yyjson_mut_obj_get(copy_root, "repo_path")) { + (void)yyjson_mut_obj_remove_key(copy_root, "repo_path"); + } + while (yyjson_mut_obj_get(copy_root, "_cbm_index_policy")) { + (void)yyjson_mut_obj_remove_key(copy_root, "_cbm_index_policy"); + } + if (!yyjson_mut_obj_add_strcpy(copy, copy_root, "repo_path", canonical_repo_path) || + !cbm_mcp_index_policy_add_to_args(copy, copy_root, policy)) { yyjson_mut_doc_free(copy); return NULL; } @@ -10981,6 +11055,19 @@ static char *resolved_repo_path_from_project_arg(const char *args) { return root_path; } +static bool project_db_is_servable(const char *project, const char *db_path) { + cbm_store_t *store = db_path && db_path[0] ? cbm_store_open_path_query(db_path) : NULL; + if (!store) { + return false; + } + cbm_project_t stored_project = {0}; + bool servable = cbm_store_get_project(store, project, &stored_project) == CBM_STORE_OK && + stored_project.root_path && stored_project.root_path[0]; + cbm_project_free_fields(&stored_project); + cbm_store_close(store); + return servable; +} + static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { char *repo_path = cbm_mcp_get_string_arg(args, "repo_path"); char *mode_str = cbm_mcp_get_string_arg(args, "mode"); @@ -11035,10 +11122,19 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { return result; } + cbm_index_resource_policy_t resource_policy; + char policy_error[CBM_SZ_256] = {0}; + if (!load_index_policy(srv, args, &resource_policy, policy_error, sizeof(policy_error))) { + free(mode_str); + free(name_override); + free(repo_path); + return cbm_mcp_text_result(policy_error, true); + } + /* A daemon session delegates the one physical write to its shared job * registry only after path canonicalization and workspace authorization. */ if (srv->index_executor) { - char *worker_args = index_args_with_repo_path(args, repo_path); + char *worker_args = index_args_with_repo_path(args, repo_path, &resource_policy); char *coordinated = worker_args ? srv->index_executor(srv->index_executor_context, repo_path, worker_args) : NULL; @@ -11070,7 +11166,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { * installs the same guard before running the in-process pipeline. A marked * host fails closed if preparation or worker startup cannot complete. */ if (cbm_index_supervisor_should_wrap()) { - char *worker_args = index_args_with_repo_path(args, repo_path); + char *worker_args = index_args_with_repo_path(args, repo_path, &resource_policy); if (!worker_args) { free(mutation_project); free(repo_path); @@ -11142,11 +11238,15 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { } free(name_override); cbm_pipeline_set_persistence(p, persistence); + cbm_pipeline_set_resource_policy(p, &resource_policy); char *project_name = heap_strdup(cbm_pipeline_project_name(p)); /* Bootstrap from artifact if no local DB exists */ try_artifact_bootstrap(project_name, repo_path); + char serving_db_path[CBM_SZ_1K]; + project_db_path(project_name, serving_db_path, sizeof(serving_db_path)); + bool serving_index_was_servable = project_db_is_servable(project_name, serving_db_path); /* Close cached store — pipeline will delete + recreate the .db file */ if (srv->owns_store && srv->store) { @@ -11178,6 +11278,8 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { cbm_file_error_t *file_errors = NULL; int file_error_count = 0; cbm_pipeline_get_file_errors(p, &file_errors, &file_error_count); + cbm_index_resource_violation_t resource_violation = {0}; + cbm_pipeline_get_resource_violation(p, &resource_violation); cbm_mem_collect(); /* return mimalloc pages to OS after large indexing */ @@ -11252,6 +11354,25 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { "The validated staging database could not be published. Check " "free disk space and permissions on the cache directory; the " "previous index may have been rolled back."); + } else if (rc == CBM_PIPELINE_RESOURCE_LIMIT && + resource_violation.resource != CBM_INDEX_RESOURCE_NONE) { + const char *config_key = cbm_index_resource_config_key(resource_violation.resource); + char message[CBM_SZ_256]; + (void)snprintf(message, sizeof(message), "Index discovery exceeded %s", config_key); + yyjson_mut_obj_add_str(doc, root, "status", "error"); + yyjson_mut_obj_add_str(doc, root, "code", "resource_limit_exceeded"); + yyjson_mut_obj_add_str(doc, root, "stage", "discovery"); + yyjson_mut_obj_add_str(doc, root, "resource", + cbm_index_resource_name(resource_violation.resource)); + yyjson_mut_obj_add_uint(doc, root, "observed", resource_violation.observed); + yyjson_mut_obj_add_uint(doc, root, "limit", resource_violation.limit); + yyjson_mut_obj_add_str(doc, root, "unit", + cbm_index_resource_unit(resource_violation.resource)); + yyjson_mut_obj_add_bool(doc, root, "retryable", true); + yyjson_mut_obj_add_bool(doc, root, "serving_index_preserved", + serving_index_was_servable && + project_db_is_servable(project_name, serving_db_path)); + yyjson_mut_obj_add_strcpy(doc, root, "message", message); } else { yyjson_mut_obj_add_str(doc, root, "status", "error"); yyjson_mut_obj_add_str(doc, root, "hint", diff --git a/src/mcp/mcp_internal.h b/src/mcp/mcp_internal.h index 5e5c771a3..de170067a 100644 --- a/src/mcp/mcp_internal.h +++ b/src/mcp/mcp_internal.h @@ -1,10 +1,13 @@ #ifndef CBM_MCP_INTERNAL_H #define CBM_MCP_INTERNAL_H +#include "foundation/index_policy.h" #include "mcp/mcp.h" #include "pipeline/pipeline.h" /* cbm_changed_hunk_t */ #include "store/store.h" /* cbm_node_t */ +#include + /* White-box fault injection for deterministic cross-platform quarantine * safety tests. This header is internal and is not part of the MCP API. */ typedef bool (*cbm_mcp_quarantine_test_hook_fn)(void *context, const char *step); @@ -36,6 +39,11 @@ bool cbm_mcp_server_release_pristine_memory_store(cbm_mcp_server_t *srv); * On success replaces and frees *response_io; on failure it is unchanged. */ bool cbm_mcp_jsonrpc_response_prepend_notice(char **response_io, const char *notice); +/* Encode the complete trusted policy on an internal worker request. Callers + * must remove any untrusted field with the same name before invoking this. */ +bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root, + const cbm_index_resource_policy_t *policy); + enum { CBM_MCP_DEFAULT_AUTO_INDEX_LIMIT = 50000 }; /* Count indexable files with the pipeline's native full-mode discovery policy, diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 0ffb46c0b..187ca9cfd 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -157,6 +157,8 @@ struct cbm_pipeline { atomic_int cancelled_storage; atomic_int *cancelled; bool persistence; /* write .codebase-memory/graph.db.zst after indexing */ + cbm_index_resource_policy_t resource_policy; + cbm_index_resource_violation_t resource_violation; /* Indexing state (set during run) */ cbm_gbuf_t *gbuf; @@ -313,6 +315,20 @@ void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled) { } } +void cbm_pipeline_set_resource_policy(cbm_pipeline_t *p, + const cbm_index_resource_policy_t *policy) { + if (p && policy) { + p->resource_policy = *policy; + } +} + +void cbm_pipeline_get_resource_violation(const cbm_pipeline_t *p, + cbm_index_resource_violation_t *violation) { + if (violation) { + *violation = p ? p->resource_violation : (cbm_index_resource_violation_t){0}; + } +} + bool cbm_pipeline_set_project_name(cbm_pipeline_t *p, const char *name) { if (!p || !name || !name[0]) { return false; @@ -422,6 +438,14 @@ const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p) { return p ? p->repo_path : NULL; } +const cbm_index_resource_policy_t *cbm_pipeline_resource_policy(const cbm_pipeline_t *p) { + return p && cbm_index_policy_enabled(&p->resource_policy) ? &p->resource_policy : NULL; +} + +cbm_index_resource_violation_t *cbm_pipeline_resource_violation(cbm_pipeline_t *p) { + return p ? &p->resource_violation : NULL; +} + atomic_int *cbm_pipeline_cancelled_ptr(cbm_pipeline_t *p) { return p ? p->cancelled : NULL; } @@ -1964,15 +1988,17 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_hash_t *bas #if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API cbm_pipeline_persist_test_run_before_final_manifest(); #endif - if (cbm_pipeline_build_fresh_semantic_manifest(p->project_name, p->repo_path, p->mode, - &manifest, &manifest_count) != 0) { + int manifest_rc = + cbm_pipeline_build_fresh_semantic_manifest(p, p->project_name, &manifest, &manifest_count); + if (manifest_rc != 0) { cbm_log_error("pipeline.err", "phase", "semantic_manifest"); /* db_path and db_dir are this function's strdups; the success tail and * the publish-failure return release them, and these two aborts must * too -- LSan caught exactly these paths leaking both strings. */ free(db_dir); free(db_path); - return CBM_PIPELINE_ABORT_PRESERVE_DB; + return manifest_rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT + : CBM_PIPELINE_ABORT_PRESERVE_DB; } if (!cbm_pipeline_semantic_manifests_equal(baseline_manifest, baseline_count, manifest, manifest_count)) { @@ -2226,10 +2252,14 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { /* Phase 1: Discover files */ CBM_PROF_START(t_discover); + p->resource_violation = (cbm_index_resource_violation_t){0}; cbm_discover_opts_t opts = { .mode = p->requested_mode, .ignore_file = NULL, .max_file_size = 0, + .resource_policy = + cbm_index_policy_enabled(&p->resource_policy) ? &p->resource_policy : NULL, + .resource_violation = &p->resource_violation, }; cbm_file_info_t *files = NULL; int file_count = 0; @@ -2254,7 +2284,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { cbm_log_info("pipeline.discover", "files", itoa_buf(file_count), "elapsed_ms", itoa_buf((int)elapsed_ms(t0))); if (rc != 0 || check_cancel(p)) { - rc = CBM_NOT_FOUND; + rc = rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT : CBM_NOT_FOUND; goto cleanup; } @@ -2262,14 +2292,15 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { * bytes drive exact no-op comparison and are checked against a fresh * rediscovery immediately before any replacement is published. */ rc = mode_promoted - ? cbm_pipeline_build_fresh_semantic_manifest(p->project_name, p->repo_path, p->mode, - &baseline_manifest, &baseline_count) + ? cbm_pipeline_build_fresh_semantic_manifest(p, p->project_name, &baseline_manifest, + &baseline_count) : cbm_pipeline_build_semantic_manifest(p->project_name, p->repo_path, files, file_count, p->excluded_dirs, p->excluded_count, &p->git_ctx, p->userconfig, &baseline_manifest, &baseline_count); if (rc != 0) { - rc = CBM_PIPELINE_ABORT_PRESERVE_DB; + rc = rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT + : CBM_PIPELINE_ABORT_PRESERVE_DB; goto cleanup; } @@ -2313,7 +2344,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { cbm_log_info("pipeline.rediscover", "requested_mode", pipeline_mode_name(p->requested_mode), "effective_mode", pipeline_mode_name(p->mode), "files", itoa_buf(file_count)); if (rc != 0 || check_cancel(p)) { - rc = CBM_NOT_FOUND; + rc = rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT : CBM_NOT_FOUND; goto cleanup; } } diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index 954686849..733e7ea8a 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -53,6 +53,13 @@ cbm_pipeline_t *cbm_pipeline_new(const char *repo_path, const char *db_path, cbm * When enabled, the pipeline writes a compressed artifact after indexing. */ void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled); +/* Apply a validated discovery resource policy. The value is copied. */ +void cbm_pipeline_set_resource_policy(cbm_pipeline_t *p, const cbm_index_resource_policy_t *policy); + +/* Copy the exact discovery violation from the most recent run. */ +void cbm_pipeline_get_resource_violation(const cbm_pipeline_t *p, + cbm_index_resource_violation_t *violation); + /* Free a pipeline and all its internal state. NULL-safe. */ void cbm_pipeline_free(cbm_pipeline_t *p); @@ -71,6 +78,8 @@ void cbm_pipeline_free(cbm_pipeline_t *p); * written, the previous DB is intact (#1997 #832). Distinct from the cancel * sentinel so callers can name the cause instead of "pipeline failed". */ #define CBM_PIPELINE_ABORT_OVER_BUDGET (-5) +/* Opt-in discovery/resource policy breach: fail the attempt, keep serving DB. */ +#define CBM_PIPELINE_RESOURCE_LIMIT (-6) int cbm_pipeline_run(cbm_pipeline_t *p); /* Request cancellation of a running pipeline (thread-safe). */ diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index bf71a1f84..29ffe43d9 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -609,17 +609,20 @@ bool cbm_pipeline_semantic_manifests_equal(const cbm_file_hash_t *left, int left return equal; } -int cbm_pipeline_build_fresh_semantic_manifest(const char *project, const char *repo_path, int mode, +int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *project, cbm_file_hash_t **out, int *out_count) { - if (!project || !repo_path || !out || !out_count) { + const char *repo_path = cbm_pipeline_repo_path(p); + if (!p || !project || !repo_path || !out || !out_count) { return CBM_NOT_FOUND; } *out = NULL; *out_count = 0; cbm_discover_opts_t opts = { - .mode = (cbm_index_mode_t)mode, + .mode = (cbm_index_mode_t)cbm_pipeline_get_mode(p), .ignore_file = NULL, .max_file_size = 0, + .resource_policy = cbm_pipeline_resource_policy(p), + .resource_violation = cbm_pipeline_resource_violation(p), }; cbm_file_info_t *fresh_files = NULL; int fresh_file_count = 0; @@ -2262,13 +2265,13 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char #if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API cbm_pipeline_persist_test_run_before_final_manifest(); #endif - if (cbm_pipeline_build_fresh_semantic_manifest(project, cbm_pipeline_repo_path(p), - cbm_pipeline_get_mode(p), &manifest, - &manifest_count) != 0 || - !cbm_pipeline_semantic_manifests_equal(baseline_manifest, baseline_count, manifest, - manifest_count)) { + int manifest_rc = + cbm_pipeline_build_fresh_semantic_manifest(p, project, &manifest, &manifest_count); + if (manifest_rc != 0 || !cbm_pipeline_semantic_manifests_equal( + baseline_manifest, baseline_count, manifest, manifest_count)) { cbm_log_warn("delta.abort", "reason", "semantic_inputs_changed"); - result = CBM_PIPELINE_ABORT_PRESERVE_DB; + result = manifest_rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT + : CBM_PIPELINE_ABORT_PRESERVE_DB; goto out; } @@ -2854,8 +2857,8 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil #if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API cbm_pipeline_persist_test_run_before_final_manifest(); #endif - int manifest_rc = cbm_pipeline_build_fresh_semantic_manifest( - project, cbm_pipeline_repo_path(p), cbm_pipeline_get_mode(p), &manifest, &manifest_count); + int manifest_rc = + cbm_pipeline_build_fresh_semantic_manifest(p, project, &manifest, &manifest_count); if (manifest_rc != 0 || !cbm_pipeline_semantic_manifests_equal( baseline_manifest, baseline_count, manifest, manifest_count)) { cbm_log_warn("incremental.abort", "reason", "semantic_inputs_changed"); @@ -2865,7 +2868,8 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil free_mode_skipped(mode_skipped, mode_skipped_count); free(saved_adr); cbm_gbuf_free(existing); - return CBM_PIPELINE_ABORT_PRESERVE_DB; + return manifest_rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT + : CBM_PIPELINE_ABORT_PRESERVE_DB; } /* Step 7: atomically publish the complete staged generation. */ diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 3a6cc1e58..9be9aaaf7 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -741,7 +741,7 @@ bool cbm_pipeline_semantic_manifests_equal(const cbm_file_hash_t *left, int left const cbm_file_hash_t *right, int right_count); /* Re-run discovery and hash its exact semantic inputs. Used at the publication * boundary so late additions/deletions cannot escape a frozen file list. */ -int cbm_pipeline_build_fresh_semantic_manifest(const char *project, const char *repo_path, int mode, +int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *project, cbm_file_hash_t **out, int *out_count); /* Compatibility contract persisted in coverage metadata. Increment when a @@ -823,6 +823,8 @@ void cbm_pipeline_set_lsp_surfaces(cbm_pipeline_t *p, cbm_lsp_surface_row_t *row /* Pipeline accessors for incremental use */ const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p); +const cbm_index_resource_policy_t *cbm_pipeline_resource_policy(const cbm_pipeline_t *p); +cbm_index_resource_violation_t *cbm_pipeline_resource_violation(cbm_pipeline_t *p); atomic_int *cbm_pipeline_cancelled_ptr(cbm_pipeline_t *p); /* Record committed graph size (#334 gate axis) from the incremental path, * which cannot see the opaque cbm_pipeline struct. Call before the dump. */ diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index f587940c2..0108e8ec5 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -1348,6 +1348,7 @@ TEST(daemon_application_prune_clears_logical_watch_for_reregistration) { } enum { APP_FAKE_MAX_ATTEMPTS = 16 }; +enum { APP_FAKE_ARGS_CAP = 2048 }; typedef struct { atomic_int starts; @@ -1368,6 +1369,11 @@ typedef struct { char marker_paths[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP]; char quarantine_paths[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP]; char quarantine_seen[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP]; + char args_json[APP_FAKE_MAX_ATTEMPTS][APP_FAKE_ARGS_CAP]; + /* `starts` is incremented before the slot is filled, so waiting on it says + * a worker began, not that its arguments are readable. Each slot publishes + * itself instead, which is the edge a reader on another thread needs. */ + atomic_bool args_captured[APP_FAKE_MAX_ATTEMPTS]; size_t memory_budgets[APP_FAKE_MAX_ATTEMPTS]; } app_fake_worker_context_t; @@ -1432,6 +1438,9 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem atomic_init(&worker->cancelled, false); worker->result.exit_code = -1; if (worker->attempt < APP_FAKE_MAX_ATTEMPTS) { + (void)snprintf(context->args_json[worker->attempt], APP_FAKE_ARGS_CAP, "%s", + args_json ? args_json : ""); + atomic_store(&context->args_captured[worker->attempt], true); context->memory_budgets[worker->attempt] = memory_budget_bytes; if (marker_file) { (void)snprintf(context->marker_paths[worker->attempt], APP_TEST_PATH_CAP, "%s", @@ -1456,6 +1465,33 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem return 0; } +static bool app_wait_for_atomic_bool(atomic_bool *value, bool expected); + +static bool app_fake_worker_policy_equals(app_fake_worker_context_t *context, int attempt, + const char *max_files, const char *max_source_mb) { + if (!context || attempt < 0 || attempt >= APP_FAKE_MAX_ATTEMPTS) { + return false; + } + if (!app_wait_for_atomic_bool(&context->args_captured[attempt], true)) { + return false; + } + const char *args = context->args_json[attempt]; + yyjson_doc *document = yyjson_read(args, strlen(args), 0); + yyjson_val *root = document ? yyjson_doc_get_root(document) : NULL; + yyjson_val *policy = + root && yyjson_is_obj(root) ? yyjson_obj_get(root, "_cbm_index_policy") : NULL; + yyjson_val *files = + policy && yyjson_is_obj(policy) ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_FILES) : NULL; + yyjson_val *bytes = policy && yyjson_is_obj(policy) + ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB) + : NULL; + bool equal = files && bytes && yyjson_is_str(files) && yyjson_is_str(bytes) && + strcmp(yyjson_get_str(files), max_files) == 0 && + strcmp(yyjson_get_str(bytes), max_source_mb) == 0; + yyjson_doc_free(document); + return equal; +} + static cbm_index_worker_poll_t app_fake_worker_poll(void *opaque, cbm_daemon_application_worker_t handle, const cbm_index_worker_result_t **result_out) { @@ -2009,7 +2045,9 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { cbm_config_t *stored_config = cache_set ? cbm_config_open(cache) : NULL; bool config_ready = stored_config && cbm_config_set(stored_config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && - cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0; + cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "3") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "4") == 0; char canonical_root[APP_TEST_PATH_CAP] = {0}; bool canonical = dirs_ok && cbm_canonical_path(root, canonical_root, sizeof(canonical_root)); char *project = canonical ? cbm_project_name_from_path(canonical_root) : NULL; @@ -2054,6 +2092,7 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { bool first_owned = first_initialized && project && app_wait_for_subscribers(application, project, 1) && app_wait_for_atomic_int(&fake.starts, 1); + bool auto_policy_propagated = first_owned && app_fake_worker_policy_equals(&fake, 0, "3", "4"); bool second_initialized = app_test_initialize_profile(&callbacks, sessions[1], root, CBM_MCP_TOOL_PROFILE_ALL, NULL, NULL); bool coalesced = first_owned && second_initialized && project && @@ -2115,6 +2154,7 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { ASSERT_TRUE(restricted_started_nothing); ASSERT_TRUE(first_initialized); ASSERT_TRUE(first_owned); + ASSERT_TRUE(auto_policy_propagated); ASSERT_TRUE(second_initialized); ASSERT_TRUE(coalesced); ASSERT_TRUE(restricted_disconnect_kept_job); @@ -2389,6 +2429,46 @@ TEST(daemon_application_sensitive_root_blocks_watch_but_preserves_controls) { PASS(); } +TEST(daemon_application_programmatic_index_injects_resource_policy) { + char *root = th_mktempdir("cbm_app_policy_root"); + char *cache = th_mktempdir("cbm_app_policy_cache"); + ASSERT_NOT_NULL(root); + ASSERT_NOT_NULL(cache); + cbm_config_t *stored_config = cbm_config_open(cache); + bool configured = stored_config && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "5") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "6") == 0; + + app_fake_worker_context_t fake; + app_fake_worker_context_init(&fake); + atomic_store(&fake.allow_completion, true); + cbm_daemon_application_worker_ops_t worker_ops = { + .context = &fake, + .start = app_fake_worker_start, + .poll = app_fake_worker_poll, + .cancel = app_fake_worker_cancel, + .log_path = app_fake_worker_log_path, + .destroy = app_fake_worker_destroy, + }; + cbm_daemon_application_config_t config = { + .config = stored_config, + .worker_ops = &worker_ops, + }; + cbm_daemon_application_t *application = configured ? cbm_daemon_application_new(&config) : NULL; + int index_rc = + application ? cbm_daemon_application_index(application, "policy-programmatic", root) : -1; + bool policy_propagated = index_rc == 0 && app_fake_worker_policy_equals(&fake, 0, "5", "6"); + bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + + cbm_daemon_application_free(application); + cbm_config_close(stored_config); + th_cleanup(root); + th_cleanup(cache); + ASSERT_TRUE(configured); + ASSERT_TRUE(policy_propagated); + ASSERT_TRUE(stopped); + PASS(); +} TEST(daemon_application_auto_index_honors_tracked_file_limit) { app_env_backup_t cache_environment; bool cache_saved = app_env_backup_capture(&cache_environment, "CBM_CACHE_DIR"); @@ -2464,6 +2544,9 @@ TEST(daemon_application_auto_index_honors_tracked_file_limit) { PASS(); } +/* Native discovery never interprets a repository path as command text. Valid + * non-Git paths containing shell metacharacters remain countable literally. */ + /* Native discovery never interprets a repository path as command text. Valid * non-Git paths containing shell metacharacters remain countable literally. */ TEST(daemon_application_auto_index_file_count_handles_literal_metacharacter_path) { @@ -5570,6 +5653,7 @@ SUITE(daemon_application) { RUN_TEST(daemon_application_sensitive_root_blocks_auto_index_but_preserves_controls); RUN_TEST(daemon_application_sensitive_root_blocks_watch_but_preserves_controls); RUN_TEST(daemon_application_index_args_compare_repo_path_separator_equivalently); + RUN_TEST(daemon_application_programmatic_index_injects_resource_policy); RUN_TEST(daemon_application_auto_index_honors_tracked_file_limit); RUN_TEST(daemon_application_auto_index_file_count_handles_literal_metacharacter_path); RUN_TEST(daemon_application_auto_index_file_count_supports_non_git_roots); diff --git a/tests/test_discover.c b/tests/test_discover.c index 0f3dac7d2..5e9bdd8af 100644 --- a/tests/test_discover.c +++ b/tests/test_discover.c @@ -475,6 +475,163 @@ TEST(discover_bounded_count_matches_shebang_discovery) { PASS(); } +TEST(discover_resource_policy_off_matches_legacy_discovery) { + char *base = th_mktempdir("cbm_disc_policy_off"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "src/first.c"), "int first;\n"); + th_write_file(TH_PATH(base, "src/second.py"), "second = 2\n"); + th_write_file(TH_PATH(base, "src/ignored.png"), "not source\n"); + + cbm_file_info_t *legacy_files = NULL; + int legacy_count = 0; + cbm_discover_opts_t legacy_opts = {.mode = CBM_MODE_FULL}; + ASSERT_EQ(cbm_discover(base, &legacy_opts, &legacy_files, &legacy_count), CBM_DISCOVER_OK); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + cbm_index_resource_violation_t violation = {0}; + cbm_file_info_t *policy_files = NULL; + int policy_count = 0; + cbm_discover_opts_t policy_opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_discover_status_t status = cbm_discover(base, &policy_opts, &policy_files, &policy_count); + + ASSERT_EQ(status, CBM_DISCOVER_OK); + ASSERT_EQ(policy_count, legacy_count); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_NONE); + for (int index = 0; index < legacy_count; index++) { + ASSERT_STR_EQ(policy_files[index].rel_path, legacy_files[index].rel_path); + ASSERT_EQ(policy_files[index].size, legacy_files[index].size); + } + + cbm_discover_free(legacy_files, legacy_count); + cbm_discover_free(policy_files, policy_count); + th_cleanup(base); + PASS(); +} + +TEST(discover_resource_file_limit_is_exact_and_counts_only_accepted_sources) { + char *base = th_mktempdir("cbm_disc_policy_files"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, ".gitignore"), "ignored.c\n"); + th_write_file(TH_PATH(base, "accepted.c"), "int accepted;\n"); + th_write_file(TH_PATH(base, "ignored.c"), "int ignored;\n"); + th_write_file(TH_PATH(base, "unsupported.png"), "not source\n"); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_file_info_t *files = NULL; + int count = 0; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_OK); + ASSERT_EQ(count, 1); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_NONE); + cbm_discover_free(files, count); + + th_write_file(TH_PATH(base, "second.py"), "second = 2\n"); + files = NULL; + count = 99; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT(files == NULL); + ASSERT_EQ(count, 0); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FILES); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + + th_cleanup(base); + PASS(); +} + +TEST(discover_resource_source_bytes_allows_equality_and_rejects_one_more_byte) { + char *base = th_mktempdir("cbm_disc_policy_bytes"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "exact.c"), "1234567"); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_source_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 7}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_file_info_t *files = NULL; + int count = 0; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_OK); + ASSERT_EQ(count, 1); + cbm_discover_free(files, count); + + th_write_file(TH_PATH(base, "plus.py"), "x"); + files = NULL; + count = 99; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT(files == NULL); + ASSERT_EQ(count, 0); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_SOURCE_BYTES); + ASSERT_EQ(violation.observed, 8); + ASSERT_EQ(violation.limit, 7); + + th_cleanup(base); + PASS(); +} + +TEST(discover_resource_file_budget_excludes_existing_oversized_skip) { + char *base = th_mktempdir("cbm_disc_policy_oversized"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "accepted.c"), "x"); + th_write_file(TH_PATH(base, "oversized.py"), "123"); + const char *saved_limit = getenv("CBM_MAX_FILE_BYTES"); + char *saved_limit_copy = saved_limit ? strdup(saved_limit) : NULL; + cbm_setenv("CBM_MAX_FILE_BYTES", "2", 1); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_file_info_t *files = NULL; + int count = 0; + cbm_discover_status_t initial_status = cbm_discover(base, &opts, &files, &count); + bool oversized_did_not_consume_budget = initial_status == CBM_DISCOVER_OK && count == 2 && + violation.resource == CBM_INDEX_RESOURCE_NONE; + cbm_discover_free(files, count); + + th_write_file(TH_PATH(base, "second.c"), "y"); + files = NULL; + count = 0; + cbm_discover_status_t exceeded_status = cbm_discover(base, &opts, &files, &count); + bool accepted_sources_exceeded = exceeded_status == CBM_DISCOVER_LIMIT_EXCEEDED && + files == NULL && count == 0 && + violation.resource == CBM_INDEX_RESOURCE_FILES && + violation.observed == 2 && violation.limit == 1; + + if (saved_limit_copy) { + cbm_setenv("CBM_MAX_FILE_BYTES", saved_limit_copy, 1); + } else { + cbm_unsetenv("CBM_MAX_FILE_BYTES"); + } + free(saved_limit_copy); + th_cleanup(base); + + ASSERT_TRUE(oversized_did_not_consume_budget); + ASSERT_TRUE(accepted_sources_exceeded); + PASS(); +} + TEST(discover_skips_git_dir) { char *base = th_mktempdir("cbm_disc_git"); ASSERT(base != NULL); @@ -1931,6 +2088,10 @@ SUITE(discover) { RUN_TEST(discover_bounded_count_is_allocation_free_and_limit_exact); RUN_TEST(discover_bounded_count_fails_closed_after_deadline); RUN_TEST(discover_bounded_count_matches_shebang_discovery); + RUN_TEST(discover_resource_policy_off_matches_legacy_discovery); + RUN_TEST(discover_resource_file_limit_is_exact_and_counts_only_accepted_sources); + RUN_TEST(discover_resource_source_bytes_allows_equality_and_rejects_one_more_byte); + RUN_TEST(discover_resource_file_budget_excludes_existing_oversized_skip); RUN_TEST(discover_skips_git_dir); RUN_TEST(discover_with_gitignore); RUN_TEST(discover_with_global_xdg_ignore); diff --git a/tests/test_index_policy.c b/tests/test_index_policy.c new file mode 100644 index 000000000..d866a0631 --- /dev/null +++ b/tests/test_index_policy.c @@ -0,0 +1,403 @@ +#include "test_framework.h" +#include "test_helpers.h" + +#include "cli/cli.h" +#include "foundation/compat.h" +#include "foundation/index_policy.h" +#include "mcp/index_supervisor.h" +#include "mcp/mcp.h" +#include "store/store.h" + +#include +#include +#include +#include +#include +#include + +TEST(index_policy_defaults_are_disabled) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + + ASSERT_FALSE(policy.max_files.enabled); + ASSERT_FALSE(policy.max_source_bytes.enabled); + ASSERT_FALSE(cbm_index_policy_enabled(&policy)); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_FILES), "off"); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_SOURCE_MB), "off"); + PASS(); +} + +TEST(index_policy_file_limit_accepts_off_and_exact_range) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "1", error, sizeof(error))); + ASSERT_TRUE(policy.max_files.enabled); + ASSERT_EQ(policy.max_files.value, 1); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "10000000", error, + sizeof(error))); + ASSERT_EQ(policy.max_files.value, 10000000); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "off", error, sizeof(error))); + ASSERT_FALSE(policy.max_files.enabled); + PASS(); +} + +TEST(index_policy_source_limit_converts_mib_without_overflow) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "1", error, sizeof(error))); + ASSERT_TRUE(policy.max_source_bytes.enabled); + ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(1024) * UINT64_C(1024)); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "1048576", error, + sizeof(error))); + ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(1048576) * UINT64_C(1024) * UINT64_C(1024)); + PASS(); +} + +TEST(index_policy_invalid_value_is_rejected_atomically) { + static const char *const invalid[] = {"", "0", "-1", "1MB", + "1 ", "+1", "10000001", "18446744073709551616"}; + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "42", error, sizeof(error))); + + for (size_t index = 0; index < sizeof(invalid) / sizeof(invalid[0]); index++) { + cbm_index_resource_policy_t before = policy; + ASSERT_FALSE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, invalid[index], + error, sizeof(error))); + ASSERT_EQ(memcmp(&policy, &before, sizeof(policy)), 0); + ASSERT_TRUE(error[0] != '\0'); + } + ASSERT_FALSE(cbm_index_policy_set(&policy, "index_unknown_limit", "1", error, sizeof(error))); + PASS(); +} + +TEST(index_policy_format_round_trips_public_values) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + char value[64]; + + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "321", error, sizeof(error))); + ASSERT_TRUE(cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_FILES, value, sizeof(value))); + ASSERT_STR_EQ(value, "321"); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "7", error, sizeof(error))); + ASSERT_TRUE( + cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, value, sizeof(value))); + ASSERT_STR_EQ(value, "7"); + PASS(); +} + +TEST(index_policy_violation_metadata_is_stable) { + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_FILES), "files"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_SOURCE_BYTES), "source_bytes"); + ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_FILES), "files"); + ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_SOURCE_BYTES), "bytes"); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_FILES), + CBM_INDEX_CONFIG_MAX_FILES); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_SOURCE_BYTES), + CBM_INDEX_CONFIG_MAX_SOURCE_MB); + PASS(); +} + +TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) { + char *cache = th_mktempdir("cbm_index_policy_config"); + ASSERT_NOT_NULL(cache); + cbm_config_t *config = cbm_config_open(cache); + ASSERT_NOT_NULL(config); + cbm_index_resource_policy_t policy; + char error[256]; + + ASSERT_TRUE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); + ASSERT_FALSE(cbm_index_policy_enabled(&policy)); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "9"), 0); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "3"), 0); + ASSERT_TRUE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); + ASSERT_EQ(policy.max_files.value, 9); + ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(3) * CBM_INDEX_MIB_BYTES); + + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "corrupt"), 0); + ASSERT_FALSE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); + ASSERT_TRUE(strstr(error, CBM_INDEX_CONFIG_MAX_FILES) != NULL); + + cbm_config_close(config); + th_cleanup(cache); + PASS(); +} + +TEST(index_policy_cli_lists_both_operator_keys) { + bool files_found = false; + bool bytes_found = false; + for (size_t index = 0; index < cbm_cli_config_key_count_for_testing(); index++) { + const char *key = cbm_cli_config_key_at_for_testing(index); + files_found = files_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0); + bytes_found = bytes_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0); + } + ASSERT_TRUE(files_found); + ASSERT_TRUE(bytes_found); + PASS(); +} + +TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite) { + char *cache = th_mktempdir("cbm_index_policy_cli"); + ASSERT_NOT_NULL(cache); + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + (void)cbm_setenv("CBM_CACHE_DIR", cache, 1); + + char *set_valid[] = {"set", CBM_INDEX_CONFIG_MAX_FILES, "7"}; + char *set_invalid[] = {"set", CBM_INDEX_CONFIG_MAX_FILES, "0"}; + char *reset[] = {"reset", CBM_INDEX_CONFIG_MAX_FILES}; + int valid_rc = cbm_cmd_config(3, set_valid); + cbm_config_t *config = cbm_config_open(cache); + const char *stored = + config ? cbm_config_get(config, CBM_INDEX_CONFIG_MAX_FILES, "missing") : "missing"; + bool valid_stored = strcmp(stored, "7") == 0; + int invalid_rc = cbm_cmd_config(3, set_invalid); + stored = config ? cbm_config_get(config, CBM_INDEX_CONFIG_MAX_FILES, "missing") : "missing"; + bool invalid_preserved = strcmp(stored, "7") == 0; + int reset_rc = cbm_cmd_config(2, reset); + stored = config ? cbm_config_get(config, CBM_INDEX_CONFIG_MAX_FILES, "off") : "missing"; + bool reset_to_default = strcmp(stored, "off") == 0; + cbm_config_close(config); + + if (saved_cache_copy) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1); + } else { + (void)cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache_copy); + th_cleanup(cache); + + ASSERT_EQ(valid_rc, 0); + ASSERT_TRUE(valid_stored); + ASSERT_TRUE(invalid_rc != 0); + ASSERT_TRUE(invalid_preserved); + ASSERT_EQ(reset_rc, 0); + ASSERT_TRUE(reset_to_default); + PASS(); +} + +/* Capture stderr across one cbm_cmd_config invocation, mirroring the stdout + * idiom in test_cli.c: tmpfile+dup2+rewind is what works on the MinGW leg. */ +static int index_policy_config_stderr(int argc, char **argv, char *out, size_t cap) { + out[0] = '\0'; + FILE *capture = tmpfile(); + int saved = capture ? dup(fileno(stderr)) : -1; + if (!capture || saved < 0) { + if (capture) { + (void)fclose(capture); + } + if (saved >= 0) { + (void)close(saved); + } + return -1000; + } + (void)fflush(stderr); + if (dup2(fileno(capture), fileno(stderr)) < 0) { + (void)fclose(capture); + (void)close(saved); + return -1000; + } + int rc = cbm_cmd_config(argc, argv); + (void)fflush(stderr); + (void)dup2(saved, fileno(stderr)); + (void)close(saved); + rewind(capture); + size_t got = fread(out, 1, cap - 1, capture); + out[got] = '\0'; + (void)fclose(capture); + return rc; +} + +/* `config set` suppresses its own generic message for a policy key, trusting + * the policy writer to name the precise reason. That trust held only for a + * value the writer rejects. A value it accepts whose write then fails — a + * _config.db that cannot be written — exited non-zero having printed nothing + * at all, which is a CLI that failed in silence. + * + * The unwritable database here is a `config` that is a view: CREATE TABLE IF + * NOT EXISTS finds the name taken and succeeds, so the command gets past open + * with a healthy handle and fails at the INSERT. That is the exact path with + * no message, and it needs no file permissions, so it behaves the same on + * every leg and under a root CI container. */ +TEST(index_policy_cli_set_reports_a_failed_write) { + char *cache = th_mktempdir("cbm_index_policy_cli_write"); + ASSERT_NOT_NULL(cache); + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + (void)cbm_setenv("CBM_CACHE_DIR", cache, 1); + + char db_path[1024]; + (void)snprintf(db_path, sizeof(db_path), "%s/_config.db", cache); + sqlite3 *db = NULL; + bool fixture_ready = + sqlite3_open(db_path, &db) == SQLITE_OK && + sqlite3_exec(db, "CREATE VIEW config(key, value) AS SELECT 'pinned', 'pinned'", NULL, NULL, + NULL) == SQLITE_OK; + if (db) { + (void)sqlite3_close(db); + } + + char captured[1024]; + char *set_valid[] = {"set", CBM_INDEX_CONFIG_MAX_FILES, "7"}; + int rc = index_policy_config_stderr(3, set_valid, captured, sizeof(captured)); + + if (saved_cache_copy) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1); + } else { + (void)cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache_copy); + th_cleanup(cache); + + ASSERT_TRUE(fixture_ready); + ASSERT_TRUE(rc != 0); + ASSERT_TRUE(strstr(captured, CBM_INDEX_CONFIG_MAX_FILES) != NULL); + PASS(); +} + +TEST(index_policy_worker_rejects_missing_parent_policy) { + char *repo = th_mktempdir("cbm_index_policy_worker"); + ASSERT_NOT_NULL(repo); + char args[1024]; + (void)snprintf(args, sizeof(args), "{\"repo_path\":\"%s\",\"mode\":\"fast\"}", repo); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + + cbm_index_set_worker_role(true, NULL); + char *response = server ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL; + cbm_index_set_worker_role(false, NULL); + bool rejected = response && strstr(response, "missing or incomplete trusted worker policy"); + + free(response); + cbm_mcp_server_free(server); + th_cleanup(repo); + ASSERT_TRUE(rejected); + PASS(); +} + +TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index) { + char *repo = th_mktempdir("cbm_index_policy_mcp_repo"); + char *cache = th_mktempdir("cbm_index_policy_mcp_cache"); + ASSERT_NOT_NULL(repo); + ASSERT_NOT_NULL(cache); + ASSERT_EQ(th_write_file(TH_PATH(repo, "first.c"), "int first(void) { return 1; }\n"), 0); + + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + (void)cbm_setenv("CBM_CACHE_DIR", cache, 1); + cbm_config_t *config = cbm_config_open(cache); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + if (server && config) { + cbm_mcp_server_set_config(server, config); + } + + char args[2048]; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"ResourcePolicyFixture\"," + "\"mode\":\"fast\"}", + repo); + char *first_response = + server && config ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL; + bool first_indexed = first_response && strstr(first_response, "\\\"status\\\":\\\"indexed\\\""); + free(first_response); + + char db_path[2048]; + (void)snprintf(db_path, sizeof(db_path), "%s/ResourcePolicyFixture.db", cache); + cbm_store_t *before_store = cbm_store_open_path_query(db_path); + cbm_project_t before_project = {0}; + bool before_read = before_store && cbm_store_get_project(before_store, "ResourcePolicyFixture", + &before_project) == CBM_STORE_OK; + char *indexed_at_before = + before_read && before_project.indexed_at ? strdup(before_project.indexed_at) : NULL; + cbm_project_free_fields(&before_project); + cbm_store_close(before_store); + + bool configured = + config && cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "1") == 0 && + th_write_file(TH_PATH(repo, "second.py"), "def second():\n return 2\n") == 0; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"ResourcePolicyFixture\"," + "\"mode\":\"fast\",\"_cbm_index_policy\":{" + "\"index_max_files\":\"off\",\"index_max_source_mb\":\"off\"}}", + repo); + char *limited_response = + configured && server ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL; + bool contract_ok = limited_response && strstr(limited_response, "resource_limit_exceeded") && + strstr(limited_response, "\\\"stage\\\":\\\"discovery\\\"") && + strstr(limited_response, "\\\"resource\\\":\\\"files\\\"") && + strstr(limited_response, "\\\"observed\\\":2") && + strstr(limited_response, "\\\"limit\\\":1") && + strstr(limited_response, "\\\"unit\\\":\\\"files\\\"") && + strstr(limited_response, "\\\"retryable\\\":true") && + strstr(limited_response, "\\\"serving_index_preserved\\\":true"); + free(limited_response); + + cbm_store_t *after_store = cbm_store_open_path_query(db_path); + cbm_project_t after_project = {0}; + bool after_read = after_store && cbm_store_get_project(after_store, "ResourcePolicyFixture", + &after_project) == CBM_STORE_OK; + bool generation_preserved = indexed_at_before && after_read && after_project.indexed_at && + strcmp(indexed_at_before, after_project.indexed_at) == 0; + cbm_project_free_fields(&after_project); + cbm_store_close(after_store); + free(indexed_at_before); + + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"ResourcePolicyFixture\"," + "\"mode\":\"cross-repo-intelligence\"}", + repo); + char *cross_response = server ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL; + bool cross_repo_unaffected = + cross_response && strstr(cross_response, "resource_limit_exceeded") == NULL; + free(cross_response); + + cbm_mcp_server_free(server); + cbm_config_close(config); + (void)cbm_unlink(db_path); + char sidecar[2100]; + (void)snprintf(sidecar, sizeof(sidecar), "%s-wal", db_path); + (void)cbm_unlink(sidecar); + (void)snprintf(sidecar, sizeof(sidecar), "%s-shm", db_path); + (void)cbm_unlink(sidecar); + th_cleanup(repo); + th_cleanup(cache); + if (saved_cache_copy) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1); + } else { + (void)cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache_copy); + + ASSERT_TRUE(first_indexed); + ASSERT_TRUE(configured); + ASSERT_TRUE(contract_ok); + ASSERT_TRUE(generation_preserved); + ASSERT_TRUE(cross_repo_unaffected); + PASS(); +} + +SUITE(index_policy) { + RUN_TEST(index_policy_defaults_are_disabled); + RUN_TEST(index_policy_file_limit_accepts_off_and_exact_range); + RUN_TEST(index_policy_source_limit_converts_mib_without_overflow); + RUN_TEST(index_policy_invalid_value_is_rejected_atomically); + RUN_TEST(index_policy_format_round_trips_public_values); + RUN_TEST(index_policy_violation_metadata_is_stable); + RUN_TEST(index_policy_config_loads_defaults_values_and_rejects_corruption); + RUN_TEST(index_policy_cli_lists_both_operator_keys); + RUN_TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite); + RUN_TEST(index_policy_cli_set_reports_a_failed_write); + RUN_TEST(index_policy_worker_rejects_missing_parent_policy); + RUN_TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index); +} diff --git a/tests/test_main.c b/tests/test_main.c index 72d45be10..4340084b5 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -752,6 +752,7 @@ extern void suite_dyn_array(void); extern void suite_str_intern(void); extern void suite_log(void); extern void suite_str_util(void); +extern void suite_index_policy(void); extern void suite_workspace(void); extern void suite_platform(void); extern void suite_diagnostics(void); @@ -1005,6 +1006,7 @@ int main(int argc, char **argv) { RUN_SELECTED_SUITE(str_intern); RUN_SELECTED_SUITE(log); RUN_SELECTED_SUITE(str_util); + RUN_SELECTED_SUITE(index_policy); RUN_SELECTED_SUITE(workspace); RUN_SELECTED_SUITE(platform); RUN_SELECTED_SUITE(diagnostics); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 738445fdf..a01a4be70 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -151,6 +151,34 @@ TEST(pipeline_run_null) { PASS(); } +TEST(pipeline_discovery_limit_returns_exact_violation_without_publishing) { + char *repo = th_mktempdir("cbm_pipeline_discovery_limit"); + ASSERT_NOT_NULL(repo); + ASSERT_EQ(th_write_file(TH_PATH(repo, "first.c"), "int first;\n"), 0); + ASSERT_EQ(th_write_file(TH_PATH(repo, "second.py"), "second = 2\n"), 0); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/index.db", repo); + cbm_pipeline_t *pipeline = cbm_pipeline_new(repo, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(pipeline); + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + cbm_pipeline_set_resource_policy(pipeline, &policy); + + ASSERT_EQ(cbm_pipeline_run(pipeline), CBM_PIPELINE_RESOURCE_LIMIT); + cbm_index_resource_violation_t violation = {0}; + cbm_pipeline_get_resource_violation(pipeline, &violation); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FILES); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + ASSERT_FALSE(cbm_file_exists(db_path)); + + cbm_pipeline_free(pipeline); + th_cleanup(repo); + PASS(); +} + /* ── Focused: file-backed store persistence ─────────────────────── */ TEST(store_file_persistence) { @@ -3281,6 +3309,44 @@ static void mutate_semantic_input_before_final_manifest(void *userdata) { mutation->write_rc = th_write_file(mutation->path, mutation->replacement); } +TEST(pipeline_late_source_limit_preserves_previous_generation) { + char *repo = th_mktempdir("cbm_pipeline_late_limit"); + ASSERT_NOT_NULL(repo); + ASSERT_EQ(th_write_file(TH_PATH(repo, "first.c"), "int first;\n"), 0); + char db_path[512]; + char late_path[512]; + snprintf(db_path, sizeof(db_path), "%s/index.db", repo); + snprintf(late_path, sizeof(late_path), "%s/late.py", repo); + + manifest_race_mutation_t mutation = { + .path = late_path, + .replacement = "late = 2\n", + .write_rc = -1, + }; + cbm_pipeline_incremental_test_before_final_manifest_once( + mutate_semantic_input_before_final_manifest, &mutation); + cbm_pipeline_t *pipeline = cbm_pipeline_new(repo, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(pipeline); + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + cbm_pipeline_set_resource_policy(pipeline, &policy); + int rc = cbm_pipeline_run(pipeline); + cbm_index_resource_violation_t violation = {0}; + cbm_pipeline_get_resource_violation(pipeline, &violation); + cbm_pipeline_free(pipeline); + cbm_pipeline_incremental_test_reset_faults(); + + ASSERT_EQ(mutation.write_rc, 0); + ASSERT_EQ(rc, CBM_PIPELINE_RESOURCE_LIMIT); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FILES); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + ASSERT_FALSE(cbm_file_exists(db_path)); + th_cleanup(repo); + PASS(); +} + /* A graph and its exact manifest are one generation. If source bytes change * after extraction but before publication, the mixed generation must be * rejected and the previous live DB preserved. */ @@ -13931,6 +13997,8 @@ SUITE(pipeline) { RUN_TEST(pipeline_cancel); RUN_TEST(pipeline_cancel_null); RUN_TEST(pipeline_run_null); + RUN_TEST(pipeline_discovery_limit_returns_exact_violation_without_publishing); + RUN_TEST(pipeline_late_source_limit_preserves_previous_generation); /* Extraction back-pressure */ RUN_TEST(pipeline_backpressure_futile_nap_disengages); RUN_TEST(pipeline_over_budget_after_futility_fails_whole_and_preserves_db); diff --git a/tests/test_worker_error_response.sh b/tests/test_worker_error_response.sh index 33e2db115..500c57ff7 100755 --- a/tests/test_worker_error_response.sh +++ b/tests/test_worker_error_response.sh @@ -48,7 +48,7 @@ trap cleanup EXIT missing="${tmpdir}/repository-does-not-exist" response="${tmpdir}/worker.response" -args="{\"repo_path\":\"${missing}\",\"mode\":\"fast\"}" +args="{\"repo_path\":\"${missing}\",\"mode\":\"fast\",$(cbm_test_index_worker_policy_json)}" if ! CBM_CACHE_DIR="${tmpdir}/cache-worker" \ "${BINARY}" cli --index-worker \ diff --git a/tests/test_worker_watchdog.sh b/tests/test_worker_watchdog.sh index 7a52603a0..e38be10bf 100755 --- a/tests/test_worker_watchdog.sh +++ b/tests/test_worker_watchdog.sh @@ -109,7 +109,7 @@ SH chmod +x "${tmpdir}/wrapper.sh" CBM_BINARY="${BINARY}" BUILD_FINGERPRINT="${BUILD_FINGERPRINT}" TMPDIR_PATH="${tmpdir}" \ - ARGS_JSON="{\"repo_path\":\"${tmpdir}/repo\"}" \ + ARGS_JSON="{\"repo_path\":\"${tmpdir}/repo\",$(cbm_test_index_worker_policy_json)}" \ CBM_TEST_HANG_ON=hang_me \ CBM_TEST_WORKER_DESCENDANT_PID_FILE="${tmpdir}/descendant.pid" \ "${tmpdir}/wrapper.sh" & From 9ef7da8cbfba8de5526ccd05e7196f5fa73af8f1 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 9 Sep 2026 21:01:40 +0200 Subject: [PATCH 042/130] style(mcp): clang-format the auto-index boundary lines (#2116) The earlier leak-fix commit was formatted with Homebrew clang-format v22, which spaces the trailing comment on cbm_mem_collect() differently than CI's clang-format-20 (AlignTrailingComments). CI's lint / lint leg failed on the resulting drift. Reformat only that line with the exact CI binary. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- src/mcp/mcp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 07ea0b698..a1d1f48b5 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -17433,7 +17433,7 @@ static void *autoindex_thread(void *arg) { cbm_pipeline_free(p); cbm_kind_in_set_free_cache(); /* free THIS thread's extraction bitset cache (see above) */ - cbm_mem_collect(); /* return mimalloc pages to OS after indexing (in-process only) */ + cbm_mem_collect(); /* return mimalloc pages to OS after indexing (in-process only) */ if (rc == 0) { cbm_log_info("autoindex.done", "project", srv->session_project); From 91c63be5d4e41d44e0746ef533af4a0b1967d6b3 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 9 Sep 2026 21:03:01 +0200 Subject: [PATCH 043/130] fix(cli): keep config follow-roots off the static TLS block (#2110) #1954's config-symlink follow support stored its opted-in roots in a _Thread_local char[4][4096] -- 16 KiB in the binary's static TLS block. On glibc/x86-64 that block is carved out of every thread's stack, including the parent-death watchdog thread, created with a deliberately tiny 64 KiB stack (PARENT_WATCHDOG_STACK_SIZE, main.c). The extra 16 KiB pushed that thread's pthread_create past the static-TLS + TCB + guard budget, so it failed with EINVAL: client_start_parent_watchdog() returned false and the stdio server exited before answering initialize. The parent-death watchdog regression test (#406/#407, scripts/test.sh Step 5) went red on every Linux x86-64 leg while macOS/arm64 stayed green -- its TLS is not stack-carved. Move the buffer off the static TLS block: keep a _Thread_local POINTER (8 bytes) and calloc the 16 KiB lazily on first add, freeing it in follow_clear(). Per-thread isolation and the exact API/semantics are unchanged; the watchdog thread's static-TLS budget is restored. main.c and the watchdog path are untouched. Evidence (local, Linux x86-64 gcc under QEMU + Linux arm64): - PT_TLS MemSiz 59,889 B -> 43,505 B (-16,384 B, exactly the array). - parent-watchdog test 10/10 red -> 10/10 green; red again on revert. - origin/main 10/10 green on the same env (regression is PR-specific). - config editor + cli suites: 522/522 ASan/UBSan (macOS), 523/523 ASan/LeakSanitizer (Linux arm64), no leak. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- src/cli/config_edit_path.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/cli/config_edit_path.c b/src/cli/config_edit_path.c index ec26a5534..2217f3855 100644 --- a/src/cli/config_edit_path.c +++ b/src/cli/config_edit_path.c @@ -11,6 +11,7 @@ #include #include +#include #include #ifndef _WIN32 @@ -66,7 +67,16 @@ void cbm_config_edit_target_close(cbm_config_edit_target_t *target) { #ifndef _WIN32 -static CBM_TLS char edit_path_roots[EDIT_PATH_MAX_ROOTS][CBM_CONFIG_EDIT_PATH_MAX]; +/* The opted-in follow-roots are per-thread (test isolation), but the storage + * is a heap buffer behind a thread-local POINTER, not a thread-local array. + * A `_Thread_local char[4][4096]` sits in the binary's static TLS block, and + * glibc carves that block out of every thread's stack allocation — including + * the parent-death watchdog's deliberately tiny 64 KiB stack (main.c). The + * 16 KiB array pushed the watchdog's pthread_create over the static-TLS + + * guard budget, so it failed with EINVAL on glibc/x86-64 (green on macOS, + * whose TLS is not stack-carved): #2110 CI. The pointer keeps the static TLS + * block at 8 bytes; the buffer is allocated on first add and freed by clear. */ +static CBM_TLS char (*edit_path_roots)[CBM_CONFIG_EDIT_PATH_MAX] = NULL; static CBM_TLS size_t edit_path_root_count = 0U; #ifdef CBM_CONFIG_EDIT_PATH_ENABLE_TEST_API @@ -101,6 +111,12 @@ int cbm_config_edit_path_follow_add_root(const char *root) { if (!cbm_canonical_path(root, canonical, sizeof(canonical))) { return -1; } + if (!edit_path_roots) { + edit_path_roots = calloc(EDIT_PATH_MAX_ROOTS, sizeof(*edit_path_roots)); + if (!edit_path_roots) { + return -1; + } + } if (edit_path_copy(canonical, edit_path_roots[edit_path_root_count], sizeof(edit_path_roots[0])) != 0) { return -1; @@ -111,6 +127,8 @@ int cbm_config_edit_path_follow_add_root(const char *root) { void cbm_config_edit_path_follow_clear(void) { edit_path_root_count = 0U; + free(edit_path_roots); + edit_path_roots = NULL; } static int edit_path_under_root(const char *canonical) { From 4fa11d1ab0b5807a7eaed645a6e3675f0663def8 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 9 Sep 2026 21:06:27 +0200 Subject: [PATCH 044/130] fix(cli): remove orphan SQLite sidecars on index deletion (#2054) cbm_remove_indexes unlinked each *.db and *.db.tmp but never the SQLite sidecars (-wal/-shm/-journal), so `cbm update`, `cbm uninstall`, and `install --reset-indexes` (the four callers of the single deletion chokepoint) left orphan -wal/-shm files behind. Call the existing wide-path-safe cbm_remove_db_sidecars() helper on both the live and staged db paths after the .db unlink; it is idempotent and ENOENT-tolerant, so an orphan -wal that outlived its .db is still cleaned. Sidecars are not indexes, so the removed-count is unchanged. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- src/cli/cli.c | 6 ++++++ tests/test_cli.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/src/cli/cli.c b/src/cli/cli.c index 956489cd9..2783d2073 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -6961,6 +6961,12 @@ int cbm_remove_indexes(const char *home_dir) { if (cbm_unlink(path) == 0) { count++; } + /* Remove the SQLite sidecars (-wal/-shm/-journal) for both the + * live and staged DBs. Idempotent and ENOENT-tolerant, so it runs + * even when the .db unlink failed -- an orphan -wal can outlive + * its .db. Sidecars are not indexes, so count is unchanged. */ + cbm_remove_db_sidecars(path); + cbm_remove_db_sidecars(tmp_path); } } cbm_closedir(d); diff --git a/tests/test_cli.c b/tests/test_cli.c index ae1a5c5f4..155fef427 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -1670,6 +1670,53 @@ TEST(cli_install_reset_deletion_waits_for_final_activation_guard) { PASS(); } +/* Removing an index must also delete its SQLite sidecars (-wal/-shm/-journal), + * not just the .db. cbm_remove_indexes is the single deletion chokepoint for + * both explicit deleters (install --reset-indexes, uninstall), so an orphan + * -wal/-shm left behind here outlives the index everywhere. Reporter ask (c) + * on issue #2054. */ +TEST(cli_remove_indexes_deletes_orphan_sqlite_sidecars_issue2054) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-remove-indexes-sidecars-XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) { + FAIL("cbm_mkdtemp failed"); + } + char *old_home = NULL; + char *old_cache = NULL; + cli_activation_save_env(&old_home, &old_cache); + cbm_setenv("HOME", tmpdir, 1); + char cache_dir[512]; + snprintf(cache_dir, sizeof(cache_dir), "%s/cache", tmpdir); + cbm_setenv("CBM_CACHE_DIR", cache_dir, 1); + test_mkdirp(cache_dir); + + char db_path[640]; + char wal_path[640]; + char shm_path[640]; + snprintf(db_path, sizeof(db_path), "%s/proj.db", cache_dir); + snprintf(wal_path, sizeof(wal_path), "%s/proj.db-wal", cache_dir); + snprintf(shm_path, sizeof(shm_path), "%s/proj.db-shm", cache_dir); + write_test_file(db_path, "db"); + write_test_file(wal_path, "wal"); + write_test_file(shm_path, "shm"); + + int rc = cbm_remove_indexes(tmpdir); + + struct stat st; + bool db_absent = stat(db_path, &st) != 0; + bool wal_absent = stat(wal_path, &st) != 0; + bool shm_absent = stat(shm_path, &st) != 0; + cli_activation_restore_env(old_home, old_cache); + test_rmdir_r(tmpdir); + + /* One .db removed; sidecars are not indexes, so the count is unchanged. */ + ASSERT_EQ(rc, 1); + ASSERT_TRUE(db_absent); + ASSERT_TRUE(wal_absent); + ASSERT_TRUE(shm_absent); + PASS(); +} + TEST(cli_install_config_only_waits_for_cohort_drain) { char tmpdir[256]; snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-daemon-install-config-race-XXXXXX"); @@ -14884,6 +14931,7 @@ SUITE(cli) { RUN_TEST(cli_install_dir_and_skip_config_stage_first_install_safely); RUN_TEST(cli_activation_commands_reject_malformed_and_unknown_flags); RUN_TEST(cli_install_reset_deletion_waits_for_final_activation_guard); + RUN_TEST(cli_remove_indexes_deletes_orphan_sqlite_sidecars_issue2054); RUN_TEST(cli_install_config_only_waits_for_cohort_drain); RUN_TEST(cli_install_config_and_path_finish_before_guard_release); RUN_TEST(cli_install_config_failure_keeps_published_binary); From df579830f3941e5ebaef73dc87d183aee358fbcb Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 10 Sep 2026 00:40:49 +0200 Subject: [PATCH 045/130] fix(install): trust the local machine's built-in Administrator; name untrusted owners (#1705, #2023, #1686) Windows install/daemon runtime directories created by an elevated process are owned by the built-in Administrator ACCOUNT (RID 500 under the local machine account-domain SID), even when that account is renamed or disabled. cbm's directory-owner/ACL trust refused it, breaking install/activation on such machines. #1705: resolve THIS machine's built-in Administrator SID via LSA (LsaOpenPolicy(POLICY_VIEW_LOCAL_INFORMATION) + LsaQueryInformationPolicy(PolicyAccountDomainInformation) -> CreateWellKnownSid(WinAccountAdministratorSid, accountDomainSid)), cached once per process, and accept it as a trusted owner/grantee in the daemon (win_sid_trusted) and the CLI activation transaction. Resolution is deliberately NOT IsWellKnownSid(WinAccountAdministratorSid) and NOT a trailing-RID-500 match: both accept a FOREIGN S-1-5-21-*-500 (a domain administrator, or another machine's built-in Administrator), which would be a cross-machine trust escalation. Any LSA or synthesis failure caches "none" and grants no tolerance (fail closed). owner-not-current-user (the exact-owner target rule) is unchanged. #2023/#1686: an owner-trust refusal now names the offending owner SID (the path was already carried), so a terse "status -3, os 0" becomes actionable. Scope notes: on a domain controller -- which has no separate local SAM, so its account domain IS the AD domain -- this trusts the domain Administrator, by design (on a DC that account is the machine-equivalent top admin). An un-sysprepped cloned Windows image shares the machine account-domain SID, so a sibling clone's built-in Administrator would be non-foreign; this is inherent to machine-SID identity. Verified on a native Windows (arm64) VM: compiles under the sanitized build; the daemon_ipc SID-trust test passes (local RID-500 trusted, a fabricated foreign S-1-5-21-*-500 rejected, BUILTIN\Administrators trusted) at 35 passed / 0 failed with the fix, and 34 passed / 1 failed (ASSERT(local_ok)) with the trust clause reverted -- the test binds precisely to the fix. #1856 (stock ancestor ACEs): the Windows ancestor-ACL guards already pass; no defect reproduced, no code change. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- src/cli/activation_transaction.c | 109 ++++++++++++++++++++++++++++- src/daemon/ipc.c | 105 ++++++++++++++++++++++++++++ src/daemon/ipc.h | 7 ++ tests/test_daemon_ipc.c | 115 +++++++++++++++++++++++++++++++ 4 files changed, 333 insertions(+), 3 deletions(-) diff --git a/src/cli/activation_transaction.c b/src/cli/activation_transaction.c index 9bb669eef..03784ab4c 100644 --- a/src/cli/activation_transaction.c +++ b/src/cli/activation_transaction.c @@ -18,6 +18,7 @@ #define WIN32_LEAN_AND_MEAN #endif #include +#include #include #include #else @@ -424,6 +425,101 @@ static void activation_windows_security_destroy(activation_windows_security_t *s memset(security, 0, sizeof(*security)); } +/* #1705: THIS machine's built-in Administrator ACCOUNT (RID 500 under the local + * machine account-domain SID, S-1-5-21--500) is a trusted owner/grantee, + * mirroring the daemon's win_sid_trusted (src/daemon/ipc.c). It is resolved via + * LSA (the local machine account-domain SID) plus CreateWellKnownSid and compared + * with EqualSid. It is deliberately NOT IsWellKnownSid(WinAccountAdministratorSid) + * and NOT a trailing-RID-500 test: both accept a FOREIGN S-1-5-21-*-500 (a domain + * admin, or another machine's built-in Administrator), opening a cross-machine + * bypass. Resolved once and cached for the process; any LSA or synthesis failure + * caches "none" and grants NO tolerance (fail closed). advapi32 is already loaded + * (this file calls GetSecurityInfo etc.), so the functions are resolved from its + * module handle with no new import. CLI activation is single-threaded by + * contract, so the cache needs no lock. */ +typedef NTSTATUS(NTAPI *activation_lsa_open_policy_fn)(PLSA_UNICODE_STRING, PLSA_OBJECT_ATTRIBUTES, + ACCESS_MASK, PLSA_HANDLE); +typedef NTSTATUS(NTAPI *activation_lsa_query_information_policy_fn)(LSA_HANDLE, + POLICY_INFORMATION_CLASS, + PVOID *); +typedef NTSTATUS(NTAPI *activation_lsa_free_memory_fn)(PVOID); +typedef NTSTATUS(NTAPI *activation_lsa_close_fn)(LSA_HANDLE); +typedef BOOL(WINAPI *activation_create_well_known_sid_fn)(WELL_KNOWN_SID_TYPE, PSID, PSID, DWORD *); + +static PSID activation_windows_local_admin_sid(void) { + static bool resolved = false; + static PSID cached = NULL; + if (resolved) { + return cached; + } + resolved = true; + HMODULE advapi = GetModuleHandleW(L"advapi32.dll"); + if (!advapi) { + return NULL; + } + activation_lsa_open_policy_fn lsa_open = + (activation_lsa_open_policy_fn)(void (*)(void))GetProcAddress(advapi, "LsaOpenPolicy"); + activation_lsa_query_information_policy_fn lsa_query = + (activation_lsa_query_information_policy_fn)(void (*)(void))GetProcAddress( + advapi, "LsaQueryInformationPolicy"); + activation_lsa_free_memory_fn lsa_free = + (activation_lsa_free_memory_fn)(void (*)(void))GetProcAddress(advapi, "LsaFreeMemory"); + activation_lsa_close_fn lsa_close = + (activation_lsa_close_fn)(void (*)(void))GetProcAddress(advapi, "LsaClose"); + activation_create_well_known_sid_fn create_sid = + (activation_create_well_known_sid_fn)(void (*)(void))GetProcAddress(advapi, + "CreateWellKnownSid"); + if (!lsa_open || !lsa_query || !lsa_free || !lsa_close || !create_sid) { + return NULL; + } + LSA_OBJECT_ATTRIBUTES attributes; + memset(&attributes, 0, sizeof(attributes)); + LSA_HANDLE policy = NULL; + /* STATUS_SUCCESS is 0; any other status is treated as failure (fail closed). */ + if (lsa_open(NULL, &attributes, POLICY_VIEW_LOCAL_INFORMATION, &policy) != 0 || !policy) { + return NULL; + } + POLICY_ACCOUNT_DOMAIN_INFO *domain = NULL; + if (lsa_query(policy, PolicyAccountDomainInformation, (PVOID *)&domain) == 0 && domain && + domain->DomainSid && IsValidSid(domain->DomainSid)) { + DWORD needed = 0; + (void)create_sid(WinAccountAdministratorSid, domain->DomainSid, NULL, &needed); + if (needed > 0U) { + PSID admin = malloc(needed); + if (admin && create_sid(WinAccountAdministratorSid, domain->DomainSid, admin, &needed) && + IsValidSid(admin)) { + cached = admin; + } else { + free(admin); + } + } + } + if (domain) { + (void)lsa_free(domain); + } + (void)lsa_close(policy); + return cached; +} + +/* #2023/#1686: an owner refusal must name WHICH owner, not just "status -3, os 0". + * The path is already carried by g_activation_refusal_object; this appends the + * offending owner's SID string, mirroring the daemon's ACL/owner diagnostics, so + * the operator sees the exact identity to remove or the directory to move. */ +static void activation_windows_note_untrusted_owner(const char *predicate, PSID owner, + DWORD os_error) { + char label[192]; + LPSTR owner_text = NULL; + (void)snprintf( + label, sizeof(label), "%s; owner=%s", predicate, + (owner && IsValidSid(owner) && ConvertSidToStringSidA(owner, &owner_text) && owner_text) + ? owner_text + : "unresolved-sid"); + if (owner_text) { + (void)LocalFree(owner_text); + } + activation_note_refusal(label, os_error); +} + /* Trusted-owner acceptance for SOURCE-side objects: a downloaded release * bundle is owned by whatever the machine's default-owner policy dictates * (Administrators on GitHub-runner-class images). Its integrity is enforced @@ -440,11 +536,13 @@ static bool activation_windows_owner_is_trusted(HANDLE handle) { PSECURITY_DESCRIPTOR descriptor = NULL; DWORD result = GetSecurityInfo(handle, SE_FILE_OBJECT, OWNER_SECURITY_INFORMATION, &owner, NULL, NULL, NULL, &descriptor); + PSID local_admin = activation_windows_local_admin_sid(); bool trusted = result == ERROR_SUCCESS && owner && IsValidSid(owner) && (EqualSid(owner, user_sid) || IsWellKnownSid(owner, WinLocalSystemSid) || - IsWellKnownSid(owner, WinBuiltinAdministratorsSid)); + IsWellKnownSid(owner, WinBuiltinAdministratorsSid) || + (local_admin && EqualSid(owner, local_admin))); if (!trusted) { - activation_note_refusal("owner-not-trusted", result); + activation_windows_note_untrusted_owner("owner-not-trusted", owner, result); } if (descriptor) { (void)LocalFree(descriptor); @@ -522,7 +620,7 @@ static bool activation_windows_owner_is_current(HANDLE handle) { } } if (!same) { - activation_note_refusal("owner-not-current-user", result); + activation_windows_note_untrusted_owner("owner-not-current-user", owner, result); } if (descriptor) { (void)LocalFree(descriptor); @@ -596,10 +694,15 @@ static bool activation_windows_acl_check(HANDLE handle, DWORD tolerated_untruste PSID sid = (PSID)&ace->SidStart; size_t sid_capacity = (size_t)header->AceSize - sid_offset; DWORD sid_length = GetSidLengthRequired(((SID *)sid)->SubAuthorityCount); + PSID local_admin = activation_windows_local_admin_sid(); bool trusted = sid_length <= sid_capacity && IsValidSid(sid) && GetLengthSid(sid) == sid_length && (EqualSid(sid, user_sid) || IsWellKnownSid(sid, WinLocalSystemSid) || IsWellKnownSid(sid, WinBuiltinAdministratorsSid) || + /* #1705: THIS machine's built-in Administrator (RID-500), + * resolved via LSA — never a foreign S-1-5-21-*-500; same + * tolerance as the daemon's win_sid_trusted. */ + (local_admin && EqualSid(sid, local_admin)) || /* OWNER RIGHTS modulates whoever owns the object; the * owner is separately validated in every chain that * reaches here (same tolerance as the daemon IPC and diff --git a/src/daemon/ipc.c b/src/daemon/ipc.c index a706a7ca5..fed1a5ef4 100644 --- a/src/daemon/ipc.c +++ b/src/daemon/ipc.c @@ -3377,6 +3377,7 @@ bool cbm_daemon_ipc_local_transition_release(cbm_daemon_ipc_local_transition_t * #include #include #include +#include #include #include #include @@ -4034,17 +4035,121 @@ static bool win_sid_is_trusted_installer(const uint8_t *sid, size_t sid_length) return true; } +/* #1705: the built-in Administrator ACCOUNT of THIS machine — RID 500 under the + * local machine's own account-domain SID (S-1-5-21--500) — is a + * legitimate owner/grantee of directories an elevated install created, even when + * that account has been renamed or is disabled. It is resolved by asking LSA for + * the local machine account-domain SID and synthesizing its RID-500 SID with + * CreateWellKnownSid, then compared with EqualSid. + * + * It is deliberately NOT tested with IsWellKnownSid(sid, WinAccountAdministratorSid) + * and NOT by matching a trailing RID of 500: BOTH of those accept ANY domain's + * -500 — a domain administrator, or another machine's built-in Administrator — + * which is exactly the cross-machine trust escalation this must never open. Only + * THIS machine's -500 is trusted. + * + * Resolved once per process and cached; any LSA or synthesis failure leaves the + * cache NULL and therefore grants NO tolerance at all (fail closed). advapi32 is + * reached through the already-loaded module handle in win_security_t and the + * function pointers are resolved dynamically, matching this file's SID-API style + * and adding no static import. */ +typedef NTSTATUS(NTAPI *lsa_open_policy_fn)(PLSA_UNICODE_STRING, PLSA_OBJECT_ATTRIBUTES, ACCESS_MASK, + PLSA_HANDLE); +typedef NTSTATUS(NTAPI *lsa_query_information_policy_fn)(LSA_HANDLE, POLICY_INFORMATION_CLASS, + PVOID *); +typedef NTSTATUS(NTAPI *lsa_free_memory_fn)(PVOID); +typedef NTSTATUS(NTAPI *lsa_close_fn)(LSA_HANDLE); +typedef BOOL(WINAPI *create_well_known_sid_fn)(WELL_KNOWN_SID_TYPE, PSID, PSID, DWORD *); + +static INIT_ONCE g_local_admin_sid_once = INIT_ONCE_STATIC_INIT; +static PSID g_local_admin_sid = NULL; /* process-lifetime cache; NULL => no tolerance */ + +static BOOL CALLBACK win_resolve_local_admin_sid(PINIT_ONCE once, PVOID parameter, PVOID *context) { + (void)once; + (void)context; + win_security_t *security = (win_security_t *)parameter; + if (!security || !security->advapi) { + return TRUE; /* ran once; cache stays NULL (fail closed) */ + } + HMODULE advapi = security->advapi; + lsa_open_policy_fn lsa_open = + (lsa_open_policy_fn)(void (*)(void))GetProcAddress(advapi, "LsaOpenPolicy"); + lsa_query_information_policy_fn lsa_query = + (lsa_query_information_policy_fn)(void (*)(void))GetProcAddress(advapi, + "LsaQueryInformationPolicy"); + lsa_free_memory_fn lsa_free = + (lsa_free_memory_fn)(void (*)(void))GetProcAddress(advapi, "LsaFreeMemory"); + lsa_close_fn lsa_close = (lsa_close_fn)(void (*)(void))GetProcAddress(advapi, "LsaClose"); + create_well_known_sid_fn create_sid = + (create_well_known_sid_fn)(void (*)(void))GetProcAddress(advapi, "CreateWellKnownSid"); + if (!lsa_open || !lsa_query || !lsa_free || !lsa_close || !create_sid) { + return TRUE; + } + LSA_OBJECT_ATTRIBUTES attributes; + memset(&attributes, 0, sizeof(attributes)); + LSA_HANDLE policy = NULL; + /* STATUS_SUCCESS is 0; any other status (including informational positives) is + * treated as failure, keeping the outcome fail-closed. */ + if (lsa_open(NULL, &attributes, POLICY_VIEW_LOCAL_INFORMATION, &policy) != 0 || !policy) { + return TRUE; + } + POLICY_ACCOUNT_DOMAIN_INFO *domain = NULL; + if (lsa_query(policy, PolicyAccountDomainInformation, (PVOID *)&domain) == 0 && domain && + domain->DomainSid && security->is_valid_sid(domain->DomainSid)) { + DWORD needed = 0; + (void)create_sid(WinAccountAdministratorSid, domain->DomainSid, NULL, &needed); + if (needed > 0U) { + PSID resolved = malloc(needed); + if (resolved && + create_sid(WinAccountAdministratorSid, domain->DomainSid, resolved, &needed) && + security->is_valid_sid(resolved)) { + g_local_admin_sid = resolved; + } else { + free(resolved); + } + } + } + if (domain) { + (void)lsa_free(domain); + } + (void)lsa_close(policy); + return TRUE; +} + +static PSID win_local_admin_sid(win_security_t *security) { + if (!security || !security->advapi) { + return NULL; + } + (void)InitOnceExecuteOnce(&g_local_admin_sid_once, win_resolve_local_admin_sid, (PVOID)security, + NULL); + return g_local_admin_sid; +} + static bool win_sid_trusted(win_security_t *security, PSID sid) { if (!security || !sid || !security->is_valid_sid(sid)) { return false; } DWORD sid_length = security->get_length_sid(sid); + PSID local_admin = win_local_admin_sid(security); return (sid_length > 0U && security->equal_sid(sid, security->user_sid)) || security->is_well_known_sid(sid, WinLocalSystemSid) || security->is_well_known_sid(sid, WinBuiltinAdministratorsSid) || + (local_admin && security->equal_sid(sid, local_admin)) || win_sid_is_trusted_installer((const uint8_t *)sid, (size_t)sid_length); } +#ifdef CBM_ENABLE_TEST_SEAMS +bool cbm_daemon_ipc_win_sid_trusted_for_testing(void *sid) { + win_security_t security; + if (!win_security_init(&security)) { + return false; + } + bool trusted = win_sid_trusted(&security, (PSID)sid); + win_security_destroy(&security); + return trusted; +} +#endif + /* AppContainer identities: package SIDs (S-1-15-2-*) and capability SIDs * (S-1-15-3-*), under the APP_PACKAGE identifier authority (15). * diff --git a/src/daemon/ipc.h b/src/daemon/ipc.h index 2cdf792dd..b6f0e9634 100644 --- a/src/daemon/ipc.h +++ b/src/daemon/ipc.h @@ -94,6 +94,13 @@ const char *cbm_daemon_ipc_validation_detail(void); #ifdef CBM_ENABLE_TEST_SEAMS /* #1537: seed the detail so a test can prove the CLI refusal surfaces it. */ void cbm_daemon_ipc_set_validation_detail_for_testing(const char *detail); +#ifdef _WIN32 +/* #1705: run the daemon's directory-owner/ACE trust predicate against an + * arbitrary SID, so a test can assert THIS machine's built-in Administrator + * (RID-500) is trusted while a foreign S-1-5-21-*-500 is not. Returns false on + * any setup failure. Windows only. */ +bool cbm_daemon_ipc_win_sid_trusted_for_testing(void *sid); +#endif #endif /* Create/validate an owner-only directory and securely open one regular diff --git a/tests/test_daemon_ipc.c b/tests/test_daemon_ipc.c index 9bb550948..684c5e32b 100644 --- a/tests/test_daemon_ipc.c +++ b/tests/test_daemon_ipc.c @@ -920,6 +920,120 @@ TEST(daemon_ipc_windows_private_directory_allows_add_subdirectory_only_ancestor) PASS(); } +/* Derive THIS machine's account-domain SID (S-1-5-21-a-b-c) from the current + * process token (S-1-5-21-a-b-c-), independently of production's LSA-based + * resolution. Caller frees with FreeSid. */ +static PSID ipc_test_local_account_domain_sid(void) { + HANDLE token = NULL; + if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &token)) { + return NULL; + } + DWORD needed = 0; + (void)GetTokenInformation(token, TokenUser, NULL, 0, &needed); + PSID domain = NULL; + if (needed > 0) { + void *buffer = calloc(1, needed); + if (buffer && GetTokenInformation(token, TokenUser, buffer, needed, &needed)) { + PSID user = ((TOKEN_USER *)buffer)->User.Sid; + if (user && IsValidSid(user)) { + UCHAR count = *GetSidSubAuthorityCount(user); + SID_IDENTIFIER_AUTHORITY *authority = GetSidIdentifierAuthority(user); + /* Machine/domain account SID: NT authority (Value[5]==5), first + * sub-authority 21, at least the three domain sub-authorities plus + * the account RID. Drop the RID to recover the domain SID. */ + if (count >= 4 && authority->Value[5] == 5 && *GetSidSubAuthority(user, 0) == 21U) { + SID_IDENTIFIER_AUTHORITY nt = {SECURITY_NT_AUTHORITY}; + (void)AllocateAndInitializeSid( + &nt, 4, *GetSidSubAuthority(user, 0), *GetSidSubAuthority(user, 1), + *GetSidSubAuthority(user, 2), *GetSidSubAuthority(user, 3), 0, 0, 0, 0, + &domain); + } + } + } + free(buffer); + } + (void)CloseHandle(token); + return domain; +} + +/* Synthesize the RID-500 built-in Administrator SID for a given account-domain + * SID, exactly as production does. Caller frees with free(). */ +static PSID ipc_test_admin_sid_for_domain(PSID domain_sid) { + if (!domain_sid) { + return NULL; + } + DWORD needed = 0; + (void)CreateWellKnownSid(WinAccountAdministratorSid, domain_sid, NULL, &needed); + if (needed == 0) { + return NULL; + } + PSID admin = malloc(needed); + if (admin && CreateWellKnownSid(WinAccountAdministratorSid, domain_sid, admin, &needed) && + IsValidSid(admin)) { + return admin; + } + free(admin); + return NULL; +} + +/* #1705: THIS machine's built-in Administrator ACCOUNT (RID-500 under the local + * account-domain SID) is a trusted directory owner/grantee, while a FOREIGN + * S-1-5-21-*-500 (a domain administrator, or another machine's built-in + * Administrator) must never be. + * + * RED without the fix: win_sid_trusted rejects the local RID-500 — on the VM this + * process runs as `test`, whose RID is not 500, so the -500 SID matches neither + * the current user nor SYSTEM, the Administrators GROUP, or TrustedInstaller — so + * local_ok is false. GREEN with the fix. foreign_ok is false either way, which is + * the exact cross-machine bypass this must never open. */ +TEST(daemon_ipc_windows_sid_trust_accepts_local_admin_rejects_foreign_500) { + PSID local_domain = ipc_test_local_account_domain_sid(); + PSID local_admin = ipc_test_admin_sid_for_domain(local_domain); + + SID_IDENTIFIER_AUTHORITY nt = {SECURITY_NT_AUTHORITY}; + PSID foreign_domain = NULL; + (void)AllocateAndInitializeSid(&nt, 4, 21U, 111U, 222U, 333U, 0, 0, 0, 0, &foreign_domain); + PSID foreign_admin = ipc_test_admin_sid_for_domain(foreign_domain); + + PSID builtin_admins = NULL; + DWORD builtin_needed = 0; + (void)CreateWellKnownSid(WinBuiltinAdministratorsSid, NULL, NULL, &builtin_needed); + if (builtin_needed > 0) { + builtin_admins = malloc(builtin_needed); + if (builtin_admins && + !CreateWellKnownSid(WinBuiltinAdministratorsSid, NULL, builtin_admins, &builtin_needed)) { + free(builtin_admins); + builtin_admins = NULL; + } + } + + bool local_built = local_admin != NULL; + bool foreign_built = foreign_admin != NULL; + bool builtin_built = builtin_admins != NULL; + + bool local_ok = local_built && cbm_daemon_ipc_win_sid_trusted_for_testing(local_admin); + bool foreign_ok = foreign_built && cbm_daemon_ipc_win_sid_trusted_for_testing(foreign_admin); + bool builtin_ok = builtin_built && cbm_daemon_ipc_win_sid_trusted_for_testing(builtin_admins); + + free(local_admin); + free(foreign_admin); + free(builtin_admins); + if (local_domain) { + FreeSid(local_domain); + } + if (foreign_domain) { + FreeSid(foreign_domain); + } + + ASSERT_TRUE(local_built); + ASSERT_TRUE(foreign_built); + ASSERT_TRUE(builtin_built); + ASSERT_TRUE(builtin_ok); /* the seam works: an already-trusted SID passes */ + ASSERT_TRUE(local_ok); /* #1705 fix: THIS machine's RID-500 is trusted */ + ASSERT_FALSE(foreign_ok); /* safety: a foreign -500 is NEVER trusted */ + PASS(); +} + TEST(daemon_ipc_windows_legacy_bridge_covers_handoff_and_lifetime) { static const char key[] = "91a2b3c4d5e6f708"; char parent[TEST_PATH_CAP] = {0}; @@ -4964,6 +5078,7 @@ SUITE(daemon_ipc) { RUN_TEST(daemon_ipc_windows_private_directory_rejects_untrusted_ancestor_acl); RUN_TEST(daemon_ipc_windows_private_directory_ace_is_inheritable); RUN_TEST(daemon_ipc_windows_private_directory_allows_add_subdirectory_only_ancestor); + RUN_TEST(daemon_ipc_windows_sid_trust_accepts_local_admin_rejects_foreign_500); RUN_TEST(daemon_ipc_windows_legacy_bridge_covers_handoff_and_lifetime); RUN_TEST(daemon_ipc_windows_local_transition_atomically_reserves_legacy_pipe); RUN_TEST(daemon_ipc_windows_startup_retries_transient_rendezvous_reader); From 618849512d01a3d14fcb704532911f9d02c23436 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 10 Sep 2026 00:57:54 +0200 Subject: [PATCH 046/130] style(install): clang-format-20 the LSA helper declarations (#1705) The LSA typedefs and GetProcAddress casts in the #1705 local-Administrator resolver were formatted with a different clang-format; CI clang-format-20 (the lint leg) flagged the line-wrapping. Reformatted only those lines with the CI binary; no logic change. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Martin Vogel --- src/cli/activation_transaction.c | 9 +++++---- src/daemon/ipc.c | 9 ++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/cli/activation_transaction.c b/src/cli/activation_transaction.c index 03784ab4c..6185535f7 100644 --- a/src/cli/activation_transaction.c +++ b/src/cli/activation_transaction.c @@ -440,8 +440,8 @@ static void activation_windows_security_destroy(activation_windows_security_t *s typedef NTSTATUS(NTAPI *activation_lsa_open_policy_fn)(PLSA_UNICODE_STRING, PLSA_OBJECT_ATTRIBUTES, ACCESS_MASK, PLSA_HANDLE); typedef NTSTATUS(NTAPI *activation_lsa_query_information_policy_fn)(LSA_HANDLE, - POLICY_INFORMATION_CLASS, - PVOID *); + POLICY_INFORMATION_CLASS, + PVOID *); typedef NTSTATUS(NTAPI *activation_lsa_free_memory_fn)(PVOID); typedef NTSTATUS(NTAPI *activation_lsa_close_fn)(LSA_HANDLE); typedef BOOL(WINAPI *activation_create_well_known_sid_fn)(WELL_KNOWN_SID_TYPE, PSID, PSID, DWORD *); @@ -486,7 +486,8 @@ static PSID activation_windows_local_admin_sid(void) { (void)create_sid(WinAccountAdministratorSid, domain->DomainSid, NULL, &needed); if (needed > 0U) { PSID admin = malloc(needed); - if (admin && create_sid(WinAccountAdministratorSid, domain->DomainSid, admin, &needed) && + if (admin && + create_sid(WinAccountAdministratorSid, domain->DomainSid, admin, &needed) && IsValidSid(admin)) { cached = admin; } else { @@ -506,7 +507,7 @@ static PSID activation_windows_local_admin_sid(void) { * offending owner's SID string, mirroring the daemon's ACL/owner diagnostics, so * the operator sees the exact identity to remove or the directory to move. */ static void activation_windows_note_untrusted_owner(const char *predicate, PSID owner, - DWORD os_error) { + DWORD os_error) { char label[192]; LPSTR owner_text = NULL; (void)snprintf( diff --git a/src/daemon/ipc.c b/src/daemon/ipc.c index fed1a5ef4..cb30be8a7 100644 --- a/src/daemon/ipc.c +++ b/src/daemon/ipc.c @@ -4053,8 +4053,8 @@ static bool win_sid_is_trusted_installer(const uint8_t *sid, size_t sid_length) * reached through the already-loaded module handle in win_security_t and the * function pointers are resolved dynamically, matching this file's SID-API style * and adding no static import. */ -typedef NTSTATUS(NTAPI *lsa_open_policy_fn)(PLSA_UNICODE_STRING, PLSA_OBJECT_ATTRIBUTES, ACCESS_MASK, - PLSA_HANDLE); +typedef NTSTATUS(NTAPI *lsa_open_policy_fn)(PLSA_UNICODE_STRING, PLSA_OBJECT_ATTRIBUTES, + ACCESS_MASK, PLSA_HANDLE); typedef NTSTATUS(NTAPI *lsa_query_information_policy_fn)(LSA_HANDLE, POLICY_INFORMATION_CLASS, PVOID *); typedef NTSTATUS(NTAPI *lsa_free_memory_fn)(PVOID); @@ -4074,9 +4074,8 @@ static BOOL CALLBACK win_resolve_local_admin_sid(PINIT_ONCE once, PVOID paramete HMODULE advapi = security->advapi; lsa_open_policy_fn lsa_open = (lsa_open_policy_fn)(void (*)(void))GetProcAddress(advapi, "LsaOpenPolicy"); - lsa_query_information_policy_fn lsa_query = - (lsa_query_information_policy_fn)(void (*)(void))GetProcAddress(advapi, - "LsaQueryInformationPolicy"); + lsa_query_information_policy_fn lsa_query = (lsa_query_information_policy_fn)(void (*)( + void))GetProcAddress(advapi, "LsaQueryInformationPolicy"); lsa_free_memory_fn lsa_free = (lsa_free_memory_fn)(void (*)(void))GetProcAddress(advapi, "LsaFreeMemory"); lsa_close_fn lsa_close = (lsa_close_fn)(void (*)(void))GetProcAddress(advapi, "LsaClose"); From 262ab01217312938728d7176f587e45adca16f01 Mon Sep 17 00:00:00 2001 From: Subgurim Date: Fri, 11 Sep 2026 13:25:58 +0200 Subject: [PATCH 047/130] =?UTF-8?q?fix(windows):=20inicializa=20diagn?= =?UTF-8?q?=C3=B3sticos=20y=20verifica=20el=20l=C3=ADmite=20con=20control?= =?UTF-8?q?=20sin=20techo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Subgurim --- src/foundation/subprocess.c | 3 +++ src/foundation/subprocess.h | 4 ++-- src/mcp/index_supervisor.c | 5 +++-- tests/test_subprocess.c | 27 ++++++++++++++++++++++++--- 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/foundation/subprocess.c b/src/foundation/subprocess.c index 475fe6741..83f788937 100644 --- a/src/foundation/subprocess.c +++ b/src/foundation/subprocess.c @@ -445,6 +445,9 @@ static void cbm_subprocess_result_init(cbm_proc_result_t *result) { result->forced = false; result->tree_quiesced = false; result->supervision_failed = false; + result->job_memory_limit_bytes = 0; + result->peak_job_memory_bytes = 0; + result->job_memory_available = false; } static void cbm_subprocess_free_config(cbm_subprocess_t *process) { diff --git a/src/foundation/subprocess.h b/src/foundation/subprocess.h index 52267272c..54940de11 100644 --- a/src/foundation/subprocess.h +++ b/src/foundation/subprocess.h @@ -43,8 +43,8 @@ typedef struct { bool tree_quiesced; /* the owned process tree has no surviving processes */ bool supervision_failed; /* the bounded containment deadline expired; tree_quiesced is false */ size_t job_memory_limit_bytes; /* Windows Job Object commit cap; zero on POSIX or uncapped */ - size_t peak_job_memory_bytes; /* Windows peak commit, not RSS or proof of a limit hit */ - bool job_memory_available; /* terminal Job Object memory query succeeded */ + size_t peak_job_memory_bytes; /* Windows peak commit, not RSS or proof of a limit hit */ + bool job_memory_available; /* terminal Job Object memory query succeeded */ } cbm_proc_result_t; /* Called synchronously for each newly-completed log chunk while the child runs. diff --git a/src/mcp/index_supervisor.c b/src/mcp/index_supervisor.c index 4f4f3ac51..9aece3b3f 100644 --- a/src/mcp/index_supervisor.c +++ b/src/mcp/index_supervisor.c @@ -606,8 +606,9 @@ static void worker_terminal_log(cbm_index_worker_handle_t *handle) { handle->process_result.job_memory_limit_bytes); (void)snprintf(peak_text, sizeof(peak_text), "%zu", handle->process_result.peak_job_memory_bytes); - /* A rejected allocation can leave peak commit BELOW the cap. Report - * evidence, not an inferred OOM classification or a file to quarantine. */ + /* Peak accounting may include a denied allocation and exceed the cap, + * or remain below it. Report evidence, not an inferred OOM classification + * or a file to quarantine. */ cbm_log_warn("index.supervisor.worker_memory", "job_limit_bytes", limit_text, "peak_job_memory_bytes", handle->process_result.job_memory_available ? peak_text : "unavailable"); diff --git a/tests/test_subprocess.c b/tests/test_subprocess.c index 79d456376..308d6b584 100644 --- a/tests/test_subprocess.c +++ b/tests/test_subprocess.c @@ -280,10 +280,20 @@ TEST(subprocess_run_spawn_failure) { TEST(subprocess_run_null_bin_rejected) { cbm_proc_opts_t opts = {0}; opts.bin = NULL; - cbm_proc_result_t r; + /* Reusing a previous result must not expose stale Job Object diagnostics + * when validation rejects the next spawn before a process exists. */ + cbm_proc_result_t r = { + .job_memory_limit_bytes = 123, + .peak_job_memory_bytes = 456, + .job_memory_available = true, + }; int rc = cbm_subprocess_run(&opts, &r); ASSERT_EQ(rc, -1); ASSERT_EQ(r.outcome, CBM_PROC_SPAWN_FAILED); + ASSERT_EQ(r.exit_code, -1); + ASSERT_TRUE(r.job_memory_limit_bytes == 0); + ASSERT_TRUE(r.peak_job_memory_bytes == 0); + ASSERT_FALSE(r.job_memory_available); PASS(); } @@ -755,12 +765,22 @@ TEST(subprocess_windows_job_object_enforces_memory_limit) { cbm_proc_opts_t opts = {0}; opts.bin = self_path; opts.argv = argv; - opts.memory_limit_bytes = (size_t)1024U * 1024U * 1024U; opts.quiet_timeout_ms = 5000; + /* Without the cap the SAME allocation must succeed. A machine-wide commit + * shortage must fail this test, not masquerade as Job Object enforcement. */ + cbm_proc_result_t uncapped = {0}; + int uncapped_rc = cbm_subprocess_run(&opts, &uncapped); + opts.memory_limit_bytes = (size_t)1024U * 1024U * 1024U; cbm_proc_result_t result = {0}; int run_rc = cbm_subprocess_run(&opts, &result); free(self_path); + ASSERT_EQ(uncapped_rc, 0); + ASSERT_EQ(uncapped.outcome, CBM_PROC_CLEAN); + ASSERT_EQ(uncapped.exit_code, 0); + ASSERT_TRUE(uncapped.tree_quiesced); + ASSERT_FALSE(uncapped.supervision_failed); + ASSERT_TRUE(uncapped.job_memory_limit_bytes == 0); ASSERT_EQ(run_rc, 0); ASSERT_EQ(result.outcome, CBM_PROC_EXIT_NONZERO); ASSERT_EQ(result.exit_code, 73); @@ -769,7 +789,8 @@ TEST(subprocess_windows_job_object_enforces_memory_limit) { ASSERT_TRUE(result.job_memory_available); ASSERT_TRUE(result.job_memory_limit_bytes == opts.memory_limit_bytes); ASSERT_TRUE(result.peak_job_memory_bytes > 0); - ASSERT_TRUE(result.peak_job_memory_bytes <= result.job_memory_limit_bytes); + /* Windows may include the denied reservation in its peak counter, so peak + * can exceed the cap. The uncapped/capped exit codes prove enforcement. */ PASS(); #endif } From e65bc6b71e32d20aeafb42c00b6097a0e2ded2c0 Mon Sep 17 00:00:00 2001 From: Subgurim Date: Fri, 11 Sep 2026 13:43:13 +0200 Subject: [PATCH 048/130] docs(windows): aclara el alcance del margen de memoria Signed-off-by: Subgurim --- src/mcp/index_supervisor.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mcp/index_supervisor.c b/src/mcp/index_supervisor.c index 87cda7cbd..837f4a4dc 100644 --- a/src/mcp/index_supervisor.c +++ b/src/mcp/index_supervisor.c @@ -639,10 +639,10 @@ size_t cbm_index_worker_job_memory_limit(size_t memory_budget_bytes) { * needs memory). Keep their argv value but leave the OS cap disabled below * 512 MiB. KILL_ON_JOB_CLOSE remains in force regardless of the cap. * - * The cooperative RSS budget, including its fail-whole over-budget gate, - * gets first chance to recover/abort. Windows charges job-wide COMMIT, not - * RSS, so this 1.5x limit is a looser last-resort backstop for descendants - * too, not an exact enforcement of the worker's accounting budget. */ + * Windows charges job-wide COMMIT, not RSS. This 1.5x limit leaves room + * for the cooperative budget's recovery/abort gate, but an allocation can + * still be denied before the next cooperative check. It is a backstop for + * descendants too, not exact enforcement of the accounting budget. */ const size_t minimum_budget = (size_t)512U * 1024U * 1024U; if (memory_budget_bytes < minimum_budget) { return 0; From eeff3f19ff97d60af4f34298aa38a652fe0705b2 Mon Sep 17 00:00:00 2001 From: Subgurim Date: Fri, 11 Sep 2026 13:44:20 +0200 Subject: [PATCH 049/130] style(windows): ajusta el formato del comentario de memoria Signed-off-by: Subgurim --- src/mcp/index_supervisor.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/mcp/index_supervisor.c b/src/mcp/index_supervisor.c index 837f4a4dc..e238051e0 100644 --- a/src/mcp/index_supervisor.c +++ b/src/mcp/index_supervisor.c @@ -640,9 +640,12 @@ size_t cbm_index_worker_job_memory_limit(size_t memory_budget_bytes) { * 512 MiB. KILL_ON_JOB_CLOSE remains in force regardless of the cap. * * Windows charges job-wide COMMIT, not RSS. This 1.5x limit leaves room - * for the cooperative budget's recovery/abort gate, but an allocation can - * still be denied before the next cooperative check. It is a backstop for - * descendants too, not exact enforcement of the accounting budget. */ + * for the + * cooperative budget's recovery/abort gate, but an allocation can + * still be denied before + * the next cooperative check. It is a backstop for + * descendants too, not exact + * enforcement of the accounting budget. */ const size_t minimum_budget = (size_t)512U * 1024U * 1024U; if (memory_budget_bytes < minimum_budget) { return 0; From 6091db7104bcaa0b61566272290e04ce448851a7 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 11 Sep 2026 20:56:43 +0200 Subject: [PATCH 050/130] fix(daemon): keep an ephemeral generation for a mid-HELLO cold-storm racer A burst of one-shot clients (e.g. six parallel `cli list_projects`) shares one ephemeral daemon generation and races to connect. The generation retired the instant its last committed client disconnected, even when another one-shot's connection was already accepted and still mid-HELLO -- stranding that racer on a STOPPING daemon and forcing a full cold respawn (up to the 12s daemon-claim handoff each). Under load the retire/respawn churn could exhaust a client's 30s bootstrap budget, surfacing as the Windows guard's "cold-storm client failed (racing daemon spawn)" regression and, on POSIX, a startup that never reached "serving". When the last committed client of an ephemeral, non-permanent generation disconnects, linger a bounded window if a peer connection is already accepted but has not yet passed HELLO admission (in_use, connected, not admitted, not disconnecting -- runtime_has_pending_hello_peer_locked). The accept loop's reconcile retires the generation once that racer drains without committing or the bounded window (RUNTIME_EPHEMERAL_LINGER_MS, seam-overridable) elapses; a racer that does commit clears the linger. With no such racer -- the common single-client case -- the generation retires immediately, unchanged. The linger deliberately covers ONLY a pre-HELLO racer, never an already- admitted provisional session (HELLO done, application session opening): the contract that a provisional coordinator client cannot keep a retiring generation alive is preserved (daemon_runtime_final_disconnect_rejects_blocked_ provisional_session). The daemon cannot count cross-process cohort peers (advisory locks expose presence, not a holder count, and Windows LockFileEx has no non-owning probe), so this uses the one peer signal it has locally: accepted connections not yet admitted. The window is bounded, so a racer that never commits cannot wedge the generation (rules out the unbounded idle hang). Coordinator gains a transient `linger` hold (distinct from `permanent`) so its own last-client self-transition to STOPPING is suppressed while the racer is outstanding; clearing it while idle retires the coordinator at once. Verified (macOS, sanitized build): daemon suites 224 passed / 0 failed / 1 skipped (the skip is the pre-existing Linux-only userns case). New tests pin both directions -- daemon_runtime_ephemeral_lingers_while_peer_mid_hello (a raw accepted-but-unHELLO'd peer keeps the generation RUNNING after the committed client leaves) and daemon_runtime_ephemeral_linger_retires_within_bound (the bounded backstop retires even while the peer is still accepted). RED on revert: neutering the gate flips ASSERT(lingered) at test_daemon_runtime.c and nothing else. clang-format (Homebrew LLVM) + cppcheck + NOLINT clean. Signed-off-by: Martin Vogel --- src/daemon/daemon.c | 24 ++++++- src/daemon/daemon.h | 8 +++ src/daemon/host.c | 4 ++ src/daemon/runtime.c | 138 +++++++++++++++++++++++++++++++++++- src/daemon/runtime.h | 13 ++++ tests/test_daemon_runtime.c | 124 ++++++++++++++++++++++++++++++++ 6 files changed, 308 insertions(+), 3 deletions(-) diff --git a/src/daemon/daemon.c b/src/daemon/daemon.c index ae26ba36b..f6f4315b7 100644 --- a/src/daemon/daemon.c +++ b/src/daemon/daemon.c @@ -47,6 +47,11 @@ struct cbm_daemon_coordinator { size_t client_count; /* See cbm_daemon_coordinator_set_permanent. */ bool permanent; + /* See cbm_daemon_coordinator_set_linger. Distinct from `permanent`: a + * transient, runtime-driven hold that keeps an ephemeral coordinator + * admitting while cohort participants are still mid-bootstrap (cold-storm + * race, 2026-09). Clearing it while idle retires the coordinator at once. */ + bool linger; size_t job_count; size_t watch_count; size_t callback_count; @@ -291,7 +296,7 @@ static void release_client_locked(cbm_daemon_coordinator_t *coordinator, release_client_resources_locked(coordinator, client->id, batch); free(client); coordinator->client_count--; - if (coordinator->client_count == 0 && !coordinator->permanent) { + if (coordinator->client_count == 0 && !coordinator->permanent && !coordinator->linger) { coordinator->state = CBM_DAEMON_COORDINATOR_STOPPING; } } @@ -329,6 +334,23 @@ void cbm_daemon_coordinator_set_permanent(cbm_daemon_coordinator_t *coordinator, cbm_mutex_unlock(&coordinator->mutex); } +void cbm_daemon_coordinator_set_linger(cbm_daemon_coordinator_t *coordinator, bool linger) { + if (!coordinator) { + return; + } + cbm_mutex_lock(&coordinator->mutex); + coordinator->linger = linger; + /* Clearing the hold while the coordinator is already idle is the retirement + * signal: an ephemeral generation whose cohort participants have all + * connected and left (or whose linger window elapsed) transitions to + * STOPPING now, exactly as the last-client disconnect would have. */ + if (!linger && !coordinator->permanent && coordinator->client_count == 0 && + coordinator->state == CBM_DAEMON_COORDINATOR_RUNNING) { + coordinator->state = CBM_DAEMON_COORDINATOR_STOPPING; + } + cbm_mutex_unlock(&coordinator->mutex); +} + cbm_daemon_coordinator_t *cbm_daemon_coordinator_new(uint64_t lease_timeout_ms) { cbm_daemon_coordinator_t *coordinator = calloc(1, sizeof(*coordinator)); if (!coordinator) { diff --git a/src/daemon/daemon.h b/src/daemon/daemon.h index 6357cb151..09e6abd5b 100644 --- a/src/daemon/daemon.h +++ b/src/daemon/daemon.h @@ -74,6 +74,14 @@ cbm_daemon_coordinator_t *cbm_daemon_coordinator_new(uint64_t lease_timeout_ms); * self-transitions to STOPPING when its client count reaches zero; only the * explicit stop/drain paths end it. */ void cbm_daemon_coordinator_set_permanent(cbm_daemon_coordinator_t *coordinator, bool permanent); +/* Transient, runtime-driven hold (distinct from `permanent`) that keeps an + * ephemeral coordinator RUNNING — still admitting new clients — after its last + * client disconnects, while cohort participants are still mid-bootstrap racing + * connect() (cold-storm race, 2026-09). Setting it before the last disconnect + * suppresses the self-transition to STOPPING; clearing it while the coordinator + * is already idle transitions to STOPPING at once, so retirement stays prompt + * and bounded. Has no effect on a permanent coordinator. */ +void cbm_daemon_coordinator_set_linger(cbm_daemon_coordinator_t *coordinator, bool linger); /* The caller must first quiesce coordinator calls and hook invocations. */ void cbm_daemon_coordinator_free(cbm_daemon_coordinator_t *coordinator); diff --git a/src/daemon/host.c b/src/daemon/host.c index f475658b8..be0baba22 100644 --- a/src/daemon/host.c +++ b/src/daemon/host.c @@ -891,6 +891,10 @@ static bool host_wait_for_lifetime(cbm_daemon_runtime_service_t *service, return cbm_daemon_runtime_service_stop(service, HOST_RUNTIME_SHUTDOWN_MS); } host_http_reconcile_at(host, cbm_now_ms(), false); + /* Retire an ephemeral generation that lingered for cold-storm cohort + * participants once they drain, or once its bounded linger elapses. + * A no-op unless a cohort-participant hook armed a linger. */ + cbm_daemon_runtime_service_reconcile_lifetime(service); (void)cbm_daemon_runtime_service_wait_exited(service, HOST_WAIT_TICK_MS); } } diff --git a/src/daemon/runtime.c b/src/daemon/runtime.c index 7db0f0225..efa41a020 100644 --- a/src/daemon/runtime.c +++ b/src/daemon/runtime.c @@ -49,6 +49,16 @@ void cbm_daemon_runtime_set_containment_hook_for_testing( cbm_daemon_runtime_containment_hook_t hook) { atomic_store(&runtime_containment_hook_seam, hook); } + +/* Cold-storm ephemeral-linger seam (2026-09). Overrides the bounded linger + * window that ephemeral last-committed-client retirement grants while cohort + * participants are still mid-bootstrap, so a test can drive both the linger and + * its expiry backstop in test time. UINT32_MAX leaves the production constant; + * any other value (0 = expire immediately) overrides. */ +static _Atomic uint32_t runtime_ephemeral_linger_timeout_seam = UINT32_MAX; +void cbm_daemon_runtime_service_set_ephemeral_linger_timeout_for_testing(uint32_t timeout_ms) { + atomic_store(&runtime_ephemeral_linger_timeout_seam, timeout_ms); +} #endif #ifdef _WIN32 @@ -95,6 +105,14 @@ enum { * wedged behind it, and the dead generation held the endpoint pipes and * the UI port for nine hours with nothing logged. */ RUNTIME_ABANDONED_REQUEST_JOIN_TIMEOUT_MS = 30000, + /* Cold-storm race (2026-09): when the final committed client of an + * ephemeral generation disconnects while cohort participants are still + * admitted but mid-bootstrap (racing connect()), the generation lingers + * this long for them to connect instead of retiring out from under them. + * Mirrors the host initial-client window; the linger is always bounded so + * a participant that never connects cannot wedge the generation (the + * 900s host_serving hang). */ + RUNTIME_EPHEMERAL_LINGER_MS = 10000, RUNTIME_PATH_CAP = 4096, RENDEZVOUS_REQUEST_ABI_OFFSET = 0, @@ -231,6 +249,12 @@ struct cbm_daemon_runtime_service { /* Owned only by the convenience start() path. start_reserved() callers * retain their externally managed participant guard. */ cbm_daemon_ipc_participant_guard_t *owned_participant_guard; + /* Cold-storm ephemeral-retirement gate (2026-09). When the final committed + * client disconnects while cohort participants are still admitted but not + * yet committed, the generation lingers until this bounded deadline instead + * of retiring; reconcile_lifetime retires it once that window elapses with + * no new client committing. Zero means no linger is armed. */ + uint64_t ephemeral_linger_deadline_ms; }; struct cbm_daemon_runtime_worker { @@ -289,6 +313,16 @@ static uint64_t runtime_deadline_after(uint32_t timeout_ms) { return now_ms + (uint64_t)timeout_ms; } +static uint32_t runtime_ephemeral_linger_timeout_ms(void) { +#ifdef CBM_ENABLE_TEST_SEAMS + uint32_t seam = atomic_load(&runtime_ephemeral_linger_timeout_seam); + if (seam != UINT32_MAX) { + return seam; + } +#endif + return RUNTIME_EPHEMERAL_LINGER_MS; +} + static void runtime_wait_tick(uint64_t deadline_ms) { uint64_t now_ms = cbm_now_ms(); if (now_ms >= deadline_ms) { @@ -1174,10 +1208,32 @@ static void runtime_service_interrupt_connections(cbm_daemon_runtime_service_t * runtime_service_interrupt_connections_except(service, NULL, false); } +/* A cold-storm racer is a connection that has been ACCEPTED but has not yet + * passed HELLO admission (in_use with a live connection, not yet admitted, not + * tearing down) -- a one-shot client still mid-bootstrap. This deliberately + * does NOT count an already-admitted provisional session (HELLO done, app + * session opening): a provisional coordinator client must not keep a retiring + * generation alive (see runtime_worker_disconnect below and the + * daemon_runtime_final_disconnect_rejects_blocked_provisional_session guard). + * Caller holds service->mutex. `except` excludes the departing worker. */ +static bool runtime_has_pending_hello_peer_locked(const cbm_daemon_runtime_service_t *service, + const cbm_daemon_runtime_worker_t *except) { + for (size_t i = 0; i < service->worker_capacity; i++) { + const cbm_daemon_runtime_worker_t *worker = &service->workers[i]; + if (worker != except && worker->in_use && worker->connection && !worker->admitted && + !atomic_load_explicit(&worker->disconnecting, memory_order_acquire)) { + return true; + } + } + return false; +} + static void runtime_worker_disconnect(cbm_daemon_runtime_worker_t *worker) { cbm_daemon_runtime_service_t *service = worker->service; cbm_daemon_client_id_t client_id = CBM_DAEMON_CLIENT_ID_INVALID; uint64_t shutdown_deadline = runtime_deadline_after(service->shutdown_timeout_ms); + bool last_committed_left = false; + bool linger_armed = false; atomic_store_explicit(&worker->disconnecting, true, memory_order_release); cbm_mutex_lock(&service->mutex); if (worker->admitted) { @@ -1195,14 +1251,47 @@ static void runtime_worker_disconnect(cbm_daemon_runtime_worker_t *worker) { * alive after the final fully committed frontend disconnects. * A permanent generation (`daemon start`) deliberately survives * this: only the stop/drain ops or a process kill end it. */ - runtime_service_begin_stopping_locked(service, shutdown_deadline, false, - "last_committed_client_disconnected"); + last_committed_left = true; + /* Cold-storm race (2026-09): if another connection is already + * accepted and still mid-HELLO (not yet admitted) when the last + * committed client leaves, it is a one-shot peer racing to commit — + * retiring now strands it on a STOPPING daemon and forces a full + * cold respawn ("cold-storm client failed (racing daemon spawn)"). + * Linger a bounded window for it; the accept loop retires the + * generation once that racer drains without committing or the window + * elapses, and a racer that does commit clears the linger + * (runtime_worker_commit_admission). This deliberately lingers ONLY + * for a pre-HELLO racer, never for an already-admitted provisional + * session (HELLO done, app session opening): the contract that a + * provisional coordinator client cannot keep a retiring generation + * alive is preserved (the reject-blocked-provisional-session guard). + * With no such racer — the common single-client case — retire + * immediately, unchanged. (A cross-process cohort peer count is + * unavailable: advisory locks expose presence, not a holder count, + * and Windows LockFileEx has no non-owning probe.) */ + if (runtime_has_pending_hello_peer_locked(service, worker)) { + service->ephemeral_linger_deadline_ms = + runtime_deadline_after(runtime_ephemeral_linger_timeout_ms()); + linger_armed = true; + } else { + service->ephemeral_linger_deadline_ms = 0; + runtime_service_begin_stopping_locked(service, shutdown_deadline, false, + "last_committed_client_disconnected"); + } } } cbm_mutex_unlock(&service->mutex); if (client_id == CBM_DAEMON_CLIENT_ID_INVALID) { return; } + /* Set the coordinator hold BEFORE releasing this client so its last-client + * self-transition to STOPPING is suppressed while a racing peer is still + * mid-HELLO; when no peer is present the hold stays clear so the release + * retires the coordinator normally. last_committed_left implies a committed (hence + * admitted) client, so client_id is always valid here. */ + if (last_committed_left) { + cbm_daemon_coordinator_set_linger(service->coordinator, linger_armed); + } (void)cbm_daemon_client_disconnected(service->coordinator, client_id, cbm_now_ms()); if (worker->application_session_opened && !worker->application_cancelled) { worker->application_cancelled = true; @@ -1250,8 +1339,18 @@ static bool runtime_worker_commit_admission(cbm_daemon_runtime_worker_t *worker) if (committed) { worker->admission_committed = true; service->committed_clients++; + /* A freshly committed client ends any last-client linger window; the + * next drop to zero re-arms it against the participants outstanding + * then. */ + service->ephemeral_linger_deadline_ms = 0; } cbm_mutex_unlock(&service->mutex); + if (committed) { + /* Mirror the cleared linger onto the coordinator. client_count is + * already nonzero (admission incremented it), so this only resets the + * hold for the next idle window — it never retires a live coordinator. */ + cbm_daemon_coordinator_set_linger(service->coordinator, false); + } return committed; } @@ -2183,6 +2282,11 @@ static void *runtime_accept_loop(void *opaque) { continue; } + /* Self-retire a generation lingering for a cold-storm peer once the peer + * drains or the bounded window elapses, without an external driver. A + * no-op unless a last-committed-client linger is armed. */ + cbm_daemon_runtime_service_reconcile_lifetime(service); + cbm_daemon_ipc_connection_t *connection = NULL; int accepted = cbm_daemon_ipc_accept(service->listener, RUNTIME_ACCEPT_POLL_MS, &connection); @@ -2474,6 +2578,36 @@ size_t cbm_daemon_runtime_service_active_connections(cbm_daemon_runtime_service_ return count; } +void cbm_daemon_runtime_service_reconcile_lifetime(cbm_daemon_runtime_service_t *service) { + if (!service) { + return; + } + bool retired = false; + cbm_mutex_lock(&service->mutex); + if (service->state == CBM_DAEMON_RUNTIME_SERVICE_RUNNING && !service->permanent && + service->committed_clients == 0 && service->ephemeral_linger_deadline_ms != 0) { + bool peers_drained = !runtime_has_pending_hello_peer_locked(service, NULL); + bool window_elapsed = cbm_now_ms() >= service->ephemeral_linger_deadline_ms; + if (peers_drained || window_elapsed) { + /* The racing peer drained without committing (retire now), or never + * committed within the bounded window (the backstop that rules out + * an unbounded idle hang — the host_serving 900s mode). */ + service->ephemeral_linger_deadline_ms = 0; + runtime_service_begin_stopping_locked( + service, runtime_deadline_after(service->shutdown_timeout_ms), false, + peers_drained ? "ephemeral_peer_drained" : "ephemeral_linger_expired"); + retired = true; + } + } + cbm_mutex_unlock(&service->mutex); + if (retired) { + /* Release the coordinator hold now that the linger has resolved. With + * no client left this transitions the coordinator to STOPPING, so the + * service drains and exits cleanly rather than only on the deadline. */ + cbm_daemon_coordinator_set_linger(service->coordinator, false); + } +} + size_t cbm_daemon_runtime_service_job_subscribers(cbm_daemon_runtime_service_t *service, const char *project_key) { return service ? cbm_daemon_job_subscribers(service->coordinator, project_key) : 0; diff --git a/src/daemon/runtime.h b/src/daemon/runtime.h index 4d53da9a4..7a31dab61 100644 --- a/src/daemon/runtime.h +++ b/src/daemon/runtime.h @@ -329,6 +329,13 @@ uint64_t cbm_daemon_runtime_service_clients_admitted_total(cbm_daemon_runtime_se /* Includes accepted connections still waiting for HELLO. Never exceeds the * configured max_clients; over-cap peers receive REJECTED before close. */ size_t cbm_daemon_runtime_service_active_connections(cbm_daemon_runtime_service_t *service); + +/* Re-evaluate an ephemeral generation that is lingering after its last committed + * client disconnected (the bounded cold-storm debounce). Retires it once the + * linger window elapses with no new client committing; the bound also rules out + * an unbounded idle hang. The host lifetime loop calls this every tick; it is a + * no-op unless a last-committed-client linger is armed. */ +void cbm_daemon_runtime_service_reconcile_lifetime(cbm_daemon_runtime_service_t *service); size_t cbm_daemon_runtime_service_job_subscribers(cbm_daemon_runtime_service_t *service, const char *project_key); uint64_t cbm_daemon_runtime_service_client_process_id(cbm_daemon_runtime_service_t *service, @@ -441,6 +448,12 @@ typedef void (*cbm_daemon_runtime_containment_hook_t)(const char *component); void cbm_daemon_runtime_set_abandoned_request_join_timeout_for_testing(uint32_t timeout_ms); void cbm_daemon_runtime_set_containment_hook_for_testing( cbm_daemon_runtime_containment_hook_t hook); +/* Cold-storm ephemeral-linger seam (2026-09). Overrides the bounded linger + * window the last-committed-client retirement grants while cohort participants + * are mid-bootstrap, so a test drives both the linger and its expiry backstop + * in test time. UINT32_MAX restores the production constant; any other value + * (0 = expire immediately) overrides. Process-global; reset it after use. */ +void cbm_daemon_runtime_service_set_ephemeral_linger_timeout_for_testing(uint32_t timeout_ms); #endif #endif /* CBM_DAEMON_RUNTIME_H */ diff --git a/tests/test_daemon_runtime.c b/tests/test_daemon_runtime.c index 991506a74..827a8cf34 100644 --- a/tests/test_daemon_runtime.c +++ b/tests/test_daemon_runtime.c @@ -2617,6 +2617,126 @@ TEST(daemon_runtime_final_disconnect_automatically_exits_within_bound) { PASS(); } +/* Cold-storm ephemeral-retirement gate (2026-09). A one-shot client sharing an + * ephemeral generation and racing connect() must not be stranded when another + * one-shot's connection just left. The generation lingers while a peer is still + * accepted (mid-HELLO) at the moment the last committed client disconnects; the + * accept loop retires it once the peer drains or a bounded window elapses. A raw + * connection (accepted, no HELLO sent) is exactly that racing peer. */ +#if defined(CBM_ENABLE_TEST_SEAMS) +/* Direction 1 (the fix): with a peer accepted mid-HELLO, closing the last + * committed client leaves the generation RUNNING (lingering), not STOPPING, so + * the peer is not stranded. Reverting the active_connections gate flips + * `lingered` RED. A window far longer than the test keeps expiry from deciding + * the verdict. */ +TEST(daemon_runtime_ephemeral_lingers_while_peer_mid_hello) { + cbm_daemon_build_identity_t identity = + runtime_test_identity("2.4.0", runtime_test_self_build()); + cbm_daemon_runtime_service_set_ephemeral_linger_timeout_for_testing(600000); + runtime_test_fixture_t fixture; + bool started = runtime_test_fixture_start(&fixture, "storm-peer", &identity); + cbm_daemon_runtime_client_t *client = NULL; + cbm_daemon_ipc_connection_t *peer = NULL; + cbm_daemon_runtime_connect_result_t result = {0}; + bool peer_accepted = false; + bool lingered = false; + bool held_through_reconcile = false; + + if (started) { + client = cbm_daemon_runtime_client_connect(fixture.endpoint, &identity, + RUNTIME_TEST_TIMEOUT_MS, &result); + } + if (client) { + /* A racing peer: accepted (active_connections++), still mid-HELLO (no + * HELLO frame), so not a committed client. */ + peer = cbm_daemon_ipc_connect(fixture.endpoint, RUNTIME_TEST_TIMEOUT_MS); + for (int i = 0; i < 1000 && !peer_accepted; i++) { + if (cbm_daemon_runtime_service_active_connections(fixture.service) >= 2) { + peer_accepted = true; + break; + } + cbm_usleep(2000); + } + } + if (peer_accepted) { + (void)cbm_daemon_runtime_client_close(client, RUNTIME_TEST_TIMEOUT_MS); + client = NULL; + lingered = + cbm_daemon_runtime_service_state(fixture.service) == CBM_DAEMON_RUNTIME_SERVICE_RUNNING; + cbm_daemon_runtime_service_reconcile_lifetime(fixture.service); + held_through_reconcile = + cbm_daemon_runtime_service_state(fixture.service) == CBM_DAEMON_RUNTIME_SERVICE_RUNNING; + } + if (peer) { + cbm_daemon_ipc_connection_close(peer); + } + if (client) { + (void)cbm_daemon_runtime_client_close(client, RUNTIME_TEST_TIMEOUT_MS); + } + cbm_daemon_runtime_service_set_ephemeral_linger_timeout_for_testing(UINT32_MAX); + runtime_test_fixture_finish(&fixture); + + ASSERT_TRUE(started); + ASSERT_TRUE(peer_accepted); + ASSERT_TRUE(lingered); + ASSERT_TRUE(held_through_reconcile); + PASS(); +} + +/* Direction 2 (bounded, no hang): a peer that never commits must not wedge the + * lingering generation forever (the host_serving 900s hang). With the window + * collapsed to zero the accept loop reconcile reaches the backstop and retires + * even while the peer is still accepted. Removing the window backstop hangs + * this test. */ +TEST(daemon_runtime_ephemeral_linger_retires_within_bound) { + cbm_daemon_build_identity_t identity = + runtime_test_identity("2.4.0", runtime_test_self_build()); + cbm_daemon_runtime_service_set_ephemeral_linger_timeout_for_testing(0); + runtime_test_fixture_t fixture; + bool started = runtime_test_fixture_start(&fixture, "storm-bound", &identity); + cbm_daemon_runtime_client_t *client = NULL; + cbm_daemon_ipc_connection_t *peer = NULL; + cbm_daemon_runtime_connect_result_t result = {0}; + bool peer_accepted = false; + bool exited = false; + + if (started) { + client = cbm_daemon_runtime_client_connect(fixture.endpoint, &identity, + RUNTIME_TEST_TIMEOUT_MS, &result); + } + if (client) { + peer = cbm_daemon_ipc_connect(fixture.endpoint, RUNTIME_TEST_TIMEOUT_MS); + for (int i = 0; i < 1000 && !peer_accepted; i++) { + if (cbm_daemon_runtime_service_active_connections(fixture.service) >= 2) { + peer_accepted = true; + break; + } + cbm_usleep(2000); + } + } + if (peer_accepted) { + (void)cbm_daemon_runtime_client_close(client, RUNTIME_TEST_TIMEOUT_MS); + client = NULL; + /* Peer never commits + window already elapsed -> the bounded backstop in + * the accept loop reconcile must retire the generation, not hang. */ + exited = cbm_daemon_runtime_service_wait_exited(fixture.service, RUNTIME_TEST_TIMEOUT_MS); + } + if (peer) { + cbm_daemon_ipc_connection_close(peer); + } + if (client) { + (void)cbm_daemon_runtime_client_close(client, RUNTIME_TEST_TIMEOUT_MS); + } + cbm_daemon_runtime_service_set_ephemeral_linger_timeout_for_testing(UINT32_MAX); + runtime_test_fixture_finish(&fixture); + + ASSERT_TRUE(started); + ASSERT_TRUE(peer_accepted); + ASSERT_TRUE(exited); + PASS(); +} +#endif + TEST(daemon_runtime_authenticated_idle_connection_outlives_lease_interval) { cbm_daemon_build_identity_t identity = runtime_test_identity("2.4.0", runtime_test_self_build()); @@ -5002,6 +5122,10 @@ SUITE(daemon_runtime) { RUN_TEST(daemon_runtime_conflict_log_failure_uses_operation_log_fallback); RUN_TEST(daemon_runtime_disconnect_releases_only_connection_subscriptions); RUN_TEST(daemon_runtime_final_disconnect_automatically_exits_within_bound); +#if defined(CBM_ENABLE_TEST_SEAMS) + RUN_TEST(daemon_runtime_ephemeral_lingers_while_peer_mid_hello); + RUN_TEST(daemon_runtime_ephemeral_linger_retires_within_bound); +#endif RUN_TEST(daemon_runtime_authenticated_idle_connection_outlives_lease_interval); RUN_TEST(daemon_runtime_connection_cap_covers_slow_hello_and_stopping_is_terminal); RUN_TEST(daemon_runtime_rejects_forged_identity_extension); From f6a29270b72a5373e4d949ed3f4078a40830c57d Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 11 Sep 2026 21:01:17 +0200 Subject: [PATCH 051/130] fix(mcp): normalize the autoindex.skip root path in diagnostics (#713) The autoindex over-limit skip warning logged `root=` with the raw srv->session_root, which on Windows carries backslashes. The #713 guard test autoindex_limit_guards_non_git_root_issue713 asserts the warning names the skipped root via strstr(root, "/repo"), so it failed on the Windows CLANG64 (x86_64) leg where the path is `...\repo`. Normalize the logged root to forward slashes with cbm_normalize_path_sep() (a no-op on POSIX), matching cbm's forward-slash path convention used everywhere else in the graph and MCP output; the diagnostic is now identical across platforms. Production-side fix, not a test relaxation. Signed-off-by: Martin Vogel --- src/mcp/mcp.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index a1d1f48b5..d5935e775 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -17532,9 +17532,13 @@ static void maybe_auto_index(cbm_mcp_server_t *srv) { char limit[32]; (void)snprintf(files, sizeof(files), "%d", file_count); (void)snprintf(limit, sizeof(limit), "%d", file_limit); + char root_disp[CBM_SZ_1K]; + (void)snprintf(root_disp, sizeof(root_disp), "%s", srv->session_root); + cbm_normalize_path_sep( + root_disp); /* forward-slash paths in diagnostics (Windows \\ -> /) */ cbm_log_warn("autoindex.skip", "reason", file_count >= 0 ? "too_many_files" : "unsafe_or_unavailable_path", "files", - files, "limit", limit, "root", srv->session_root); + files, "limit", limit, "root", root_disp); return; } From 527016258e0da33ef883966f618609f4cf58abf8 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 11 Sep 2026 21:40:54 +0200 Subject: [PATCH 052/130] fix(extract): budget tree-sitter parses by thread CPU time, not wall clock The per-file tree-sitter parse was bounded by a 5s WALL-CLOCK budget. Under CI contention a worker thread can be descheduled long enough for 5s of wall time to elapse against near-zero CPU, so a small, perfectly parseable file was abandoned with a false "parse timeout" and yielded ZERO defs. When that file defined a symbol other files resolve against (e.g. celery.app.Celery.send_task backing an ASYNC_CALLS edge, or a userpb gRPC client backing GRPC_CALLS), the cross-file service edge then failed to classify -- surfacing as the flaky convergence_probe / edge_types_probe service-edge assertions (count 0), a verdict decided by wall-clock scheduling rather than the input (violates O9, "no timeout decides a test"). Gate the parse on per-thread CPU time instead, with a generous wall ceiling as a backstop: - New portable helper cbm_thread_cpu_time_ns() (compat.c/.h): POSIX/macOS clock_gettime(CLOCK_THREAD_CPUTIME_ID), Windows GetThreadTimes (kernel+user); returns 0 on clock failure, degrading safely to the wall ceiling. It is NOT cbm_clock_gettime, which is wall on every platform (QueryPerformanceCounter on Windows). - The progress callback's payload becomes {cpu_deadline_ns, wall_ceiling_ns}; it trips when this thread's CPU time passes the budget OR wall time passes t0 + budget * 12 (~60s for the 5s budget). Descheduling burns wall, not CPU, so a starved-but-parseable file survives; a CPU-spinning parse still trips the CPU deadline; a genuinely stuck parse still trips the wall ceiling (bounded, no hang). RED-repro (deterministic, CBM_ENABLE_TEST_SEAMS only): a thread-local wall-clock offset armed by CBM_TEST_WALL_STALL_ON emulates a descheduled worker (wall reads budget + 1s ahead while CPU is untouched -- above the old 1x wall budget, below the ceiling). async_calls_celery_wall_stall_seam arms it for the Celery DEFINITION file and asserts ASYNC_CALLS >= 1. With the fix the parse completes and the edge resolves; with the budget reverted to wall-only + a 1x ceiling the def file is dropped and the edge count is 0. Verified (macOS, sanitized): edge_types_probe + convergence_probe 107 passed / 0 failed with the fix; 58 passed / 1 failed on revert (only the seam test, at the ASYNC_CALLS assertion); 107 restored. Regression smoke (extraction parse_coverage pipeline lang_contract grammar_regression) 698 passed / 0 failed. lint-ci clean (cppcheck + clang-format Homebrew LLVM + NOLINT); the Windows GetThreadTimes path is compile-checked by CI (not runnable on macOS). Signed-off-by: Martin Vogel --- internal/cbm/cbm.c | 58 ++++++++++++++++++++++++++++++++--- src/foundation/compat.c | 24 +++++++++++++++ src/foundation/compat.h | 11 +++++++ tests/test_edge_types_probe.c | 32 +++++++++++++++++++ 4 files changed, 120 insertions(+), 5 deletions(-) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 0bd718f82..ff00a07f2 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -213,9 +213,42 @@ static const char *cbm_string_read(void *payload, uint32_t byte, TSPoint point, // --- Parse timeout callback --- +/* Budget for the tree-sitter progress callback. The PRIMARY gate is per-thread + * CPU time: a worker descheduled under CI contention burns WALL time but not + * CPU, so a starved-but-parseable file must NOT be abandoned merely because the + * 5 s budget elapsed in wall-clock against near-zero CPU. That false "parse + * timeout" silently dropped a file's defs and, with them, every cross-file edge + * those defs anchored (e.g. a Celery.send_task definition backing an ASYNC_CALLS + * edge). A generous WALL ceiling stays as a backstop so a genuinely + * stuck/spinning parse still terminates in bounded time. */ +#define CBM_PARSE_WALL_CEILING_FACTOR 12ULL /* ~60 s ceiling for the 5 s CPU budget */ + +typedef struct { + uint64_t cpu_deadline_ns; // trip once this thread's CPU time passes it + uint64_t wall_ceiling_ns; // hard wall backstop for a spinning/stuck parse +} CBMParseBudget; + +#ifdef CBM_ENABLE_TEST_SEAMS +/* Deterministic RED-repro seam (armed by CBM_TEST_WALL_STALL_ON in + * cbm_extract_file_ex): when set, the timeout callback reads the WALL clock this + * many ns ahead of reality while CPU time is untouched — emulating a worker + * descheduled long enough for the old wall-only budget to elapse against + * near-zero CPU. With the CPU-time budget the parse still completes; a wall-only + * / tight-ceiling budget drops the file. Thread-local so it cannot leak across + * worker threads. Compiled only into seam-enabled test artifacts. */ +static CBM_TLS uint64_t tl_parse_wall_seam_offset_ns = 0; +#endif + +/* tree-sitter's TSProgressCallback mandates a non-const TSParseState*; a const + * parameter here would not match the opts.progress_callback assignment below. */ +// cppcheck-suppress constParameterCallback static bool cbm_timeout_cb(TSParseState *state) { - uint64_t deadline = *(uint64_t *)state->payload; - return now_ns() > deadline; + const CBMParseBudget *budget = (const CBMParseBudget *)state->payload; + uint64_t wall = now_ns(); +#ifdef CBM_ENABLE_TEST_SEAMS + wall += tl_parse_wall_seam_offset_ns; +#endif + return cbm_thread_cpu_time_ns() > budget->cpu_deadline_ns || wall > budget->wall_ceiling_ns; } // --- Thread-local parser pool --- @@ -1620,11 +1653,26 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C }; TSParseOptions opts = {0}; - uint64_t deadline_ns = 0; // cppcheck-suppress unreadVariable + CBMParseBudget budget = {0}; // cppcheck-suppress unreadVariable if (timeout_micros > 0) { - deadline_ns = t0 + ((uint64_t)timeout_micros * USEC_TO_NSEC); - opts.payload = &deadline_ns; + uint64_t budget_ns = (uint64_t)timeout_micros * USEC_TO_NSEC; + // Descheduling burns wall time but not CPU: gate on this thread's CPU + // time so a starved-but-parseable file is not abandoned, with a generous + // wall ceiling as a backstop against a genuinely spinning/stuck parse. + budget.cpu_deadline_ns = cbm_thread_cpu_time_ns() + budget_ns; + budget.wall_ceiling_ns = t0 + budget_ns * CBM_PARSE_WALL_CEILING_FACTOR; + opts.payload = &budget; opts.progress_callback = cbm_timeout_cb; +#ifdef CBM_ENABLE_TEST_SEAMS + tl_parse_wall_seam_offset_ns = 0; + const char *stall_on = getenv("CBM_TEST_WALL_STALL_ON"); + if (stall_on && stall_on[0] && rel_path && strstr(rel_path, stall_on)) { + // Push the wall reading past the 1x budget (the old wall-only budget + // trips) but well under the generous ceiling (the CPU-time budget + // survives): budget + 1 s, deterministic, no real timing involved. + tl_parse_wall_seam_offset_ns = budget_ns + NSEC_PER_SEC; + } +#endif } TSTree *tree = ts_parser_parse_with_options(parser, NULL, ts_input, opts); diff --git a/src/foundation/compat.c b/src/foundation/compat.c index 74c25bd8e..94e7020bd 100644 --- a/src/foundation/compat.c +++ b/src/foundation/compat.c @@ -316,6 +316,30 @@ int cbm_clock_gettime(int clk_id, struct timespec *tp) { } #endif +/* ── Per-thread CPU time ──────────────────────────────────────── */ + +uint64_t cbm_thread_cpu_time_ns(void) { +#ifdef _WIN32 + FILETIME creation, exit_time, kernel, user; + if (!GetThreadTimes(GetCurrentThread(), &creation, &exit_time, &kernel, &user)) { + return 0; + } + /* kernel/user each carry a 64-bit count of 100-ns ticks split across a + * FILETIME's two DWORDs; recombine, sum, and scale to nanoseconds. The + * shift-combine avoids casting to ULARGE_INTEGER (alignment-unsafe per + * MSDN, and its union members trip cppcheck's unreadVariable). */ + uint64_t kernel_ticks = ((uint64_t)kernel.dwHighDateTime << 32) | kernel.dwLowDateTime; + uint64_t user_ticks = ((uint64_t)user.dwHighDateTime << 32) | user.dwLowDateTime; + return (kernel_ticks + user_ticks) * 100ULL; +#else + struct timespec ts; + if (clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts) != 0) { + return 0; + } + return (uint64_t)ts.tv_sec * 1000000000ULL + (uint64_t)ts.tv_nsec; +#endif +} + /* ── getline (Windows lacks it) ───────────────────────────────── */ #ifdef _WIN32 diff --git a/src/foundation/compat.h b/src/foundation/compat.h index 175d775e4..821d901f0 100644 --- a/src/foundation/compat.h +++ b/src/foundation/compat.h @@ -11,6 +11,7 @@ #include #include #include +#include #include /* stdlib.h declares getenv (cbm_tmpdir) and, on Windows, _putenv_s (cbm_setenv/ * cbm_unsetenv). The x86-64 mingw toolchain pulled it in transitively, but the @@ -99,6 +100,16 @@ static inline int cbm_nanosleep(const struct timespec *req, struct timespec *rem /* Sleeps for the full requested duration even when POSIX signals interrupt it. */ int cbm_nanosleep_full(const struct timespec *req); +/* Per-CALLING-THREAD CPU time in nanoseconds. Unlike cbm_clock_gettime — which + * measures WALL time on every platform (QueryPerformanceCounter on Windows) — + * this advances only while the calling thread is actually scheduled on a CPU, so + * a descheduled/starved thread accrues (almost) none. Use it to budget work + * against real CPU consumed rather than wall-clock, which a contended host + * inflates without the thread doing any work. POSIX (incl. macOS 10.12+): + * CLOCK_THREAD_CPUTIME_ID. Windows: GetThreadTimes kernel+user. Returns 0 if the + * platform clock is unavailable. Implemented in compat.c. */ +uint64_t cbm_thread_cpu_time_ns(void); + /* ── gmtime_r (Windows lacks it) ─────────────────────────────── */ #ifdef _WIN32 static inline struct tm *cbm_gmtime_r(const time_t *timep, struct tm *result) { diff --git a/tests/test_edge_types_probe.c b/tests/test_edge_types_probe.c index 84d7713ee..b330e05e8 100644 --- a/tests/test_edge_types_probe.c +++ b/tests/test_edge_types_probe.c @@ -842,6 +842,37 @@ TEST(async_calls_celery_python) { PASS(); } +/* RED-repro: a wall-clock / scheduling artifact must NOT drop a parseable + * file's defs. The CBM_TEST_WALL_STALL_ON seam forces the parse-timeout callback + * to read the WALL clock as (budget + 1 s) ahead while CPU time is untouched — + * emulating a worker descheduled under CI contention. celery/app.py DEFINES + * Celery.send_task; under the old wall-only 5 s budget that file is abandoned + * with zero defs, the "celery" QN the caller resolves against vanishes, and the + * ASYNC_CALLS edge count drops to 0 (the intermittent async_calls_celery_python + * failure). With the CPU-time budget the parse still completes (CPU under budget, + * wall under the generous ceiling) and the edge survives. Deterministic: the + * seam is a fixed offset, never real timing. Binds only under CBM_ENABLE_TEST_SEAMS + * (always set for the test-runner); a no-seam build exercises the plain edge. */ +TEST(async_calls_celery_wall_stall_seam) { + static const EtFile f[] = { + {"celery/app.py", + "class Celery:\n" + " def task(self):\n def decorator(fn): return fn\n return decorator\n\n" + " def send_task(self, name, args=None): return (name, args)\n\n" + "app = Celery()\n"}, + {"tasks/order_tasks.py", + "from celery.app import app\n\n\n" + "def dispatch_order_created(order_id):\n" + " return app.send_task('order_created', args=[order_id])\n\n\n" + "def dispatch_order_shipped(order_id):\n" + " return app.send_task('order_shipped', args=[order_id])\n"}}; + cbm_setenv("CBM_TEST_WALL_STALL_ON", "celery/app.py", 1); + int ok = et_edge_present(f, 2, "ASYNC_CALLS", 1); + cbm_unsetenv("CBM_TEST_WALL_STALL_ON"); + ASSERT_TRUE(ok); + PASS(); +} + /* Sidekiq (Ruby) — "Sidekiq" substring in resolved QN */ TEST(async_calls_sidekiq_ruby) { static const EtFile f[] = { @@ -1625,6 +1656,7 @@ SUITE(edge_types_probe) { /* ASYNC_CALLS — message queue dispatch (5 brokers × languages) */ RUN_TEST(async_calls_celery_python); + RUN_TEST(async_calls_celery_wall_stall_seam); RUN_TEST(async_calls_sidekiq_ruby); RUN_TEST(async_calls_kafkajs_ts); RUN_TEST(async_calls_sqs_go); From f6281874bba9a78ac7543564ff8f76b5262daeef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 12 Sep 2026 05:45:06 +0000 Subject: [PATCH 053/130] build(deps): bump github/codeql-action/upload-sarif Bumps [github/codeql-action/upload-sarif](https://github.com/github/codeql-action) from 4.37.9 to 4.38.0. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/cdf488f595d80d6e07e03d4674febd5ab45fa938...b96794f015dfd88f77b49b1c93e0fa7110f94c63) --- updated-dependencies: - dependency-name: github/codeql-action/upload-sarif dependency-version: 4.38.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index a208a62ca..f2d8bcb92 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -27,6 +27,6 @@ jobs: publish_results: true - name: Upload SARIF results - uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4 + uses: github/codeql-action/upload-sarif@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4 with: sarif_file: results.sarif From 9a460b389ce3002ac164605c3627bb86aa3b9be3 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 12 Sep 2026 15:38:20 +0200 Subject: [PATCH 054/130] fix(sha256): keep the message schedule out of the transform's frame sha256_transform held its 64-word message schedule as a 256-byte local. That is large enough for ASan's use-after-return fake stack, so an instrumented build heap-allocated it through __asan_stack_malloc on every 64-byte block -- millions of allocations to hash one large file. Under the sanitized CI settings (detect_stack_use_after_return=1) fingerprinting a freshly installed binary therefore took minutes, and the cli suite blew its 900s wall-clock budget and was killed as hung on the Linux arm64 leg. The suite passes on macOS, where the fake stack is not engaged the same way, which is why this only ever surfaced on one leg. Move the schedule into cbm_sha256_ctx so it is allocated once per hash instead of once per block. The computed values and the resulting digest are identical; no sanitizer option is relaxed and use-after-return detection stays enabled everywhere. Verified in the Linux arm64 sanitized container under the exact CI ASAN_OPTIONS (detect_stack_use_after_return=1:strict_string_checks=1: detect_stack_use_after_scope=1): cli 316 passed in 356s, previously killed at the 900s budget; daemon_version 10 passed, covering the build-fingerprint digests that would break first on a wrong hash. Signed-off-by: Martin Vogel --- src/foundation/sha256.c | 6 +++++- src/foundation/sha256.h | 8 ++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/foundation/sha256.c b/src/foundation/sha256.c index 937e363e6..84d37453e 100644 --- a/src/foundation/sha256.c +++ b/src/foundation/sha256.c @@ -25,7 +25,11 @@ static const uint32_t K[64] = { #define SIG1(x) (ROTR(x, 17) ^ ROTR(x, 19) ^ ((x) >> 10)) static void sha256_transform(cbm_sha256_ctx *c, const uint8_t *data) { - uint32_t m[64]; + /* Scratch comes from the context, not this frame: a 256-byte local here + * is fake-stacked by ASan's use-after-return mode on every 64-byte block + * (see the note on cbm_sha256_ctx::sched). Identical values, allocated + * once per hash instead of once per block. */ + uint32_t *m = c->sched; for (int i = 0, j = 0; i < 16; i++, j += 4) { m[i] = ((uint32_t)data[j] << 24) | ((uint32_t)data[j + 1] << 16) | ((uint32_t)data[j + 2] << 8) | (uint32_t)data[j + 3]; diff --git a/src/foundation/sha256.h b/src/foundation/sha256.h index a34a741d3..f3c7f9c55 100644 --- a/src/foundation/sha256.h +++ b/src/foundation/sha256.h @@ -17,6 +17,14 @@ typedef struct { uint64_t bitlen; uint8_t buf[64]; size_t buflen; + /* Message-schedule scratch for one compression round. It lives in the + * context rather than in sha256_transform's own frame because a 256-byte + * local there is large enough for ASan's use-after-return fake stack, + * which then heap-allocates it on EVERY 64-byte block — millions of + * __asan_stack_malloc calls to fingerprint one large file, turning + * sanitized binary fingerprinting into minutes. Here it is allocated once + * per hash instead of once per block. */ + uint32_t sched[64]; } cbm_sha256_ctx; void cbm_sha256_init(cbm_sha256_ctx *c); From aacf96a20e3b9c450ba968c8aae663da25598992 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 12 Sep 2026 16:21:30 +0200 Subject: [PATCH 055/130] fix(install): stop naming TLS 1.3 where schannel cannot negotiate it install.ps1 set its protocol bitmask to Tls12 -bor Tls13 unconditionally. Windows 10's schannel has no TLS 1.3 -- it arrives with Windows 11 and Server 2022 (build 20348) -- but .NET Framework 4.8 still DEFINES the enum value, so the assignment succeeds and nothing warns. The first HTTPS request then dies with "The request was aborted: Could not create SSL/TLS secure channel", because an unsupported flag in this bitmask is a hard failure rather than a downgrade. Every Windows 10 user was blocked before a single byte downloaded, on the documented `irm ... | iex` path, with an error that points at the network rather than at the installer. Compute the set instead: start at Tls12 and add Tls13 only on build 20348+. The enum-name probe alongside the build gate keeps the script parsing on .NET Framework 4.7, where the member does not exist at all. Verified on the Windows VM (build 26200) with the shipped parser: PARSE_OK, pure ASCII preserved, the guard computes and assigns "Tls12, Tls13" there, forcing build 19045 through the same expression yields "Tls12" alone, and an HTTPS HEAD to github.com under the assigned set returns 200. The contract test fails on the previous installer with all four assertions -- unconditional bit, missing build gate, missing enum probe, literal bitmask -- and passes with this change. Reported with a full ProcMon trace and a decoded SID by zaferavci1. Refs #1856 Signed-off-by: Martin Vogel --- install.ps1 | 18 +++++++++++++-- tests/test_windows_bundle_contract.sh | 32 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/install.ps1 b/install.ps1 index d63f1001d..3b087321b 100644 --- a/install.ps1 +++ b/install.ps1 @@ -7,8 +7,22 @@ $ErrorActionPreference = "Stop" -# Enforce TLS 1.2+ (older PowerShell defaults to TLS 1.0 which GitHub rejects) -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 -bor [Net.SecurityProtocolType]::Tls13 +# Enforce TLS 1.2+ (older PowerShell defaults to TLS 1.0 which GitHub rejects). +# +# TLS 1.3 is added ONLY where schannel can actually negotiate it: Windows 11 and +# Server 2022 (build 20348+). On Windows 10 the enum value still exists from +# .NET Framework 4.8 onward, so the assignment succeeds and nothing warns -- but +# the handshake then fails outright with "The request was aborted: Could not +# create SSL/TLS secure channel". An unsupported protocol flag in this bitmask +# is a hard failure, not a graceful downgrade, so every Windows 10 user was +# blocked at the first download (#1856). The enum-name probe additionally keeps +# the script parsing on .NET Framework 4.7, where Tls13 is not defined at all. +$CbmProtocols = [Net.SecurityProtocolType]::Tls12 +if ([Environment]::OSVersion.Version.Build -ge 20348 -and + ([enum]::GetNames([Net.SecurityProtocolType]) -contains 'Tls13')) { + $CbmProtocols = $CbmProtocols -bor [Net.SecurityProtocolType]::Tls13 +} +[Net.ServicePointManager]::SecurityProtocol = $CbmProtocols Add-Type -AssemblyName System.Net.Http $Repo = "DeusData/codebase-memory-mcp" diff --git a/tests/test_windows_bundle_contract.sh b/tests/test_windows_bundle_contract.sh index a40ac3083..4c4c681cd 100644 --- a/tests/test_windows_bundle_contract.sh +++ b/tests/test_windows_bundle_contract.sh @@ -242,6 +242,38 @@ require( "install.ps1 version probes must relax EAP with restore and pre-seed LASTEXITCODE (#1255)", ) +# ── 3c. The TLS bitmask must not name a protocol schannel cannot negotiate ─── +# Windows 10's schannel has no TLS 1.3 (it arrives with Windows 11 / Server +# 2022, build 20348). .NET Framework 4.8 still DEFINES SecurityProtocolType +# .Tls13, so setting the bit succeeds silently and nothing warns -- then the +# first HTTPS request dies with "Could not create SSL/TLS secure channel", +# because an unsupported flag in this bitmask is a hard failure rather than a +# downgrade. That blocked the installer on every Windows 10 machine before it +# downloaded a single byte (#1856). Pin both halves of the guard: the build +# gate, and the enum-name probe that keeps the script parsing on 4.7, where +# Tls13 does not exist. +require( + "[Net.SecurityProtocolType]::Tls12 -bor [Net.SecurityProtocolType]::Tls13" not in installer, + "install.ps1 must not set the TLS 1.3 bit unconditionally -- it hard-fails " + "the handshake on Windows 10, whose schannel cannot negotiate it (#1856)", +) +require( + re.search(r"\[Environment\]::OSVersion\.Version\.Build\s+-ge\s+20348", installer) is not None, + "install.ps1 must gate TLS 1.3 on build 20348+ (Windows 11 / Server 2022) (#1856)", +) +require( + re.search( + r"\[enum\]::GetNames\(\[Net\.SecurityProtocolType\]\)\s+-contains\s+'Tls13'", installer + ) + is not None, + "install.ps1 must probe the enum before naming Tls13, so it still parses on " + ".NET Framework 4.7 where the member is undefined (#1856)", +) +require( + "[Net.ServicePointManager]::SecurityProtocol = $CbmProtocols" in installer, + "install.ps1 must assign the computed protocol set, not a literal bitmask (#1856)", +) + # ── 4. Package-manager shims resolve the single Windows binary ─────────────── single_binary_contracts = { "pkg/npm/install.js": ( From 734906379f235c07cce960ce6a57746ea3be48bb Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 12 Sep 2026 16:50:42 +0200 Subject: [PATCH 056/130] fix(cli): name the check that refused, and a remedy that can remove it A reporter brought a full ProcMon trace to #1856 and still could not say which check refused their install. That is not their failing: the binary discards the answer before printing it. cli_activation_production_context_init() validates the cache, rendezvous and log directories -- it is the emitter MOST likely to hold a useful detail -- and it printed CLI_ACTIVATION_REFUSED_MESSAGE bare, through the raw sink rather than cli_activation_diagnostic(). So the reader got "Check the errors above" with nothing above. That is the exact dead end #1416 and #1537 already fixed; the property was repaired on the paths that had tests and left broken on this sibling, which had none. Route it through the attributing helper: the refusal now names the transaction refusal note or cbm_daemon_ipc_validation_detail(). The remedy was also wrong for the reported shape. `icacls

/remove:g ` cannot remove an INHERITED ACE, and the stock C:\ grant for Authenticated Users reaches every new child directory exactly that way -- so a reporter following our advice watches the command succeed and the refusal persist. The refusal now says whether the ACE was inherited, and the advice names `icacls /inheritance:r /grant:r "%USERNAME%":(OI)(CI)F` for that case. No safety check is relaxed. An install directory writable by other accounts is still refused; it is now refused in terms the reader can act on. The new contract test guards the CLASS rather than this one call site: any emitter of a refusal constant that bypasses the attributing helper fails it, which is what would have caught this. It fails on the previous tree naming src/cli/cli.c:750, plus the missing inherited-ACE report and the missing /inheritance:r remedy. Local: cli 314 passed; activation_transaction daemon_ipc daemon_bootstrap daemon_version 110 passed, 1 skipped (user namespaces are Linux-only, runs on the Linux leg); make -f Makefile.cbm lint-ci clean. Reported by zaferavci1. Refs #1856 Signed-off-by: Martin Vogel --- scripts/test.sh | 3 + src/cli/activation_transaction.c | 12 ++- src/cli/cli.c | 14 ++- tests/test_activation_diagnostic_contract.sh | 108 +++++++++++++++++++ 4 files changed, 133 insertions(+), 4 deletions(-) create mode 100644 tests/test_activation_diagnostic_contract.sh diff --git a/scripts/test.sh b/scripts/test.sh index 0aafe6c20..dc592ef80 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -215,6 +215,9 @@ bash "$ROOT/tests/test_soak_daemon_recovery_contract.sh" echo "=== Step 0e: Windows launcher bundle contract ===" bash "$ROOT/tests/test_windows_bundle_contract.sh" +echo "=== Step 0e2: activation refusal diagnostic contract ===" +bash "$ROOT/tests/test_activation_diagnostic_contract.sh" + echo "=== Step 0f: tree-sitter runtime Makefile dependencies ===" bash "$ROOT/tests/test_makefile_ts_runtime_dependencies.sh" diff --git a/src/cli/activation_transaction.c b/src/cli/activation_transaction.c index 6185535f7..75f5eeaaf 100644 --- a/src/cli/activation_transaction.c +++ b/src/cli/activation_transaction.c @@ -712,9 +712,17 @@ static bool activation_windows_acl_check(HANDLE handle, DWORD tolerated_untruste if (!trusted) { char label[128]; LPSTR sid_text = NULL; - (void)snprintf(label, sizeof(label), "acl-grants-cross-account-mutation to %s", + /* #1856: say whether the grant is INHERITED. The remedy differs and + * the wrong one silently does nothing: `icacls /remove:g ` + * cannot remove an inherited ACE -- that needs `/inheritance:r` -- + * and the stock `C:\` ACE for Authenticated Users reaches every new + * child directory exactly this way. A reporter following the + * generic advice sees the command succeed and the refusal persist. */ + bool inherited = (header->AceFlags & INHERITED_ACE) != 0; + (void)snprintf(label, sizeof(label), "acl-grants-cross-account-mutation to %s%s", ConvertSidToStringSidA(sid, &sid_text) && sid_text ? sid_text - : "unparsable-sid"); + : "unparsable-sid", + inherited ? " (inherited from a parent directory)" : ""); if (sid_text) { (void)LocalFree(sid_text); } diff --git a/src/cli/cli.c b/src/cli/cli.c index 5d71f6f65..511bebef6 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -240,7 +240,9 @@ static void cli_activation_diagnostic(const cbm_cli_activation_ops_t *ops, const "error: activation was refused by a filesystem safety check before any " "change was made: %s\n" "error: this is not a session problem. If the flagged directory is one you " - "trust, remove the flagged permission grant (icacls /remove:g ) " + "trust, remove the flagged permission grant (icacls /remove:g ; " + "an INHERITED grant needs icacls /inheritance:r /grant:r " + "\"%%USERNAME%%\":(OI)(CI)F instead, because /remove:g cannot delete one) " "or use an owner-private directory for --dir/CBM_CACHE_DIR, then retry.", note); diagnostic = attributed; @@ -747,7 +749,15 @@ static int cli_activation_guard(cbm_daemon_runtime_activation_action_t action, cli_activation_production_context_t context; if (!cli_activation_production_context_init(&context, action, target_version, target_build)) { cli_activation_production_context_close(&context); - cli_activation_production_diagnostic(NULL, CLI_ACTIVATION_REFUSED_MESSAGE); + /* #1856: this refusal used to print the generic text BARE. Every other + * emitter routes through cli_activation_diagnostic, which appends the + * transaction refusal note or cbm_daemon_ipc_validation_detail() and so + * names the check that actually refused. Here the reader got "Check the + * errors above" with nothing above -- exactly the dead end #1537/#1416 + * fixed on the other paths and missed on this one. Context init is + * where the cache, rendezvous and log directories are validated, so it + * is the emitter MOST likely to hold a detail worth showing. */ + cli_activation_diagnostic(NULL, CLI_ACTIVATION_REFUSED_MESSAGE); return CLI_TRUE; } printf("Stopping active CBM sessions and operations for %s...\n", diff --git a/tests/test_activation_diagnostic_contract.sh b/tests/test_activation_diagnostic_contract.sh new file mode 100644 index 000000000..ebc4f0f81 --- /dev/null +++ b/tests/test_activation_diagnostic_contract.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Activation-refusal diagnostics must never point at evidence they do not show. +# +# #1416 and #1537 both landed because a refusal printed generic text and told +# the reader to "check the errors above" when nothing was above. The fix was +# cli_activation_diagnostic(): it appends the transaction refusal note, or +# cbm_daemon_ipc_validation_detail(), so the message names the check that +# actually refused. +# +# #1856 showed the fix was incomplete in the way that matters: the property was +# repaired on the paths that had tests and left broken on a sibling call site +# that had none. cli_activation_production_context_init() -- the emitter that +# validates the cache, rendezvous and log directories, i.e. the one MOST likely +# to hold a useful detail -- printed the constant bare. A reporter with a full +# ProcMon trace still could not tell which check refused, because the binary +# discarded the answer before printing. +# +# So the guard here is the CLASS, not the one call site: every emitter of the +# refusal constants goes through the attributing helper. A future emitter that +# bypasses it fails this test instead of reaching a reporter. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" + +python3 - "$ROOT" <<'PY' +from __future__ import annotations + +import pathlib +import re +import sys + +root = pathlib.Path(sys.argv[1]) +failures: list[str] = [] + + +def require(condition: bool, message: str) -> None: + if not condition: + failures.append(message) + + +cli = (root / "src" / "cli" / "cli.c").read_text(encoding="utf-8") + +# 1. Every call that passes a refusal constant to a diagnostic sink must use the +# attributing helper. cli_activation_production_diagnostic is the raw sink -- +# legitimate as an ops->visible_diagnostic member (it receives the ALREADY +# attributed string), never as a direct emitter of the constant. +ATTRIBUTING = "cli_activation_diagnostic" +CONSTANTS = ("CLI_ACTIVATION_REFUSED_MESSAGE", "CLI_ACTIVATION_BUSY_MESSAGE") + +for call in re.finditer(r"(\w*diagnostic)\s*\(([^;]*?)\)\s*;", cli, re.S): + callee, arguments = call.group(1), call.group(2) + if not any(constant in arguments for constant in CONSTANTS): + continue + if callee == ATTRIBUTING: + continue + line = cli.count("\n", 0, call.start()) + 1 + require( + False, + f"src/cli/cli.c:{line}: {callee}() emits a refusal constant directly. " + f"Route it through {ATTRIBUTING}() so the message names the check that " + "refused instead of pointing at errors that were never printed (#1856)", + ) + +# 2. The helper must still be the one that appends both attribution channels -- +# deleting either turns the message back into a dead end. +helper = re.search( + r"static void cli_activation_diagnostic\((?:.|\n)*?\n\}\n", cli +) +require(helper is not None, "cli_activation_diagnostic() not found in src/cli/cli.c") +body = helper.group(0) if helper else "" +require( + "cbm_activation_transaction_refusal_note()" in body, + "cli_activation_diagnostic() must append the transaction refusal note (#1416)", +) +require( + "cbm_daemon_ipc_validation_detail()" in body, + "cli_activation_diagnostic() must append the daemon validation detail (#1537)", +) + +# 3. The remedy must be one that works. `icacls /remove:g` silently does nothing +# to an INHERITED ACE -- and the stock C:\ grant for Authenticated Users +# reaches every new child directory exactly that way, which is the shape +# #1856 was reported against. Advice that cannot fix the reported case is +# worse than none: the command succeeds and the refusal persists. +require( + "/inheritance:r" in body, + "the refusal remedy must name icacls /inheritance:r for an inherited grant " + "-- /remove:g cannot remove one, so the advice fails on the stock C:\\ ACE " + "that #1856 reported", +) + +# 4. And the refusal itself must say which of the two shapes it hit, or the +# reader cannot choose between those commands. +transaction = (root / "src" / "cli" / "activation_transaction.c").read_text(encoding="utf-8") +require( + re.search(r"AceFlags\s*&\s*INHERITED_ACE", transaction) is not None, + "src/cli/activation_transaction.c must report whether the refusing ACE was " + "inherited, so the printed remedy matches the grant (#1856)", +) + +if failures: + print("Activation diagnostic contract FAILED:") + for failure in failures: + print(f" - {failure}") + sys.exit(1) + +print("Activation diagnostic contract passed") +PY From fa99ff64eb6579fb15f3aae1aff7da349cb2999a Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 12 Sep 2026 18:01:46 +0200 Subject: [PATCH 057/130] test(cli): stop a negative probe from spending 15s proving silence The macos-15-intel leg killed the cli suite at its 900s wall clock while reporting 7537 passed, 0 failed. Two separate things were wrong. The suite was misclassified. cli spends 497s of the 900s default on macos-14, the FASTEST macOS runner, while macos-15-intel in the same matrix is 2.4-3.6x slower on comparable suites (daemon_runtime 842s vs 349s, stack_overflow_b 277s vs 76s). 497s at that ratio cannot fit. daemon_runtime, at 842s on that same runner, passes only because it was already in SLOW_SUITES. cli belongs there too -- it is deterministic and simply large, which is the only thing that list is allowed to mean. The drain test was also waiting on a clock instead of on the system. Profiling the suite per test (315 tests, 300s wall / 166s cpu) put cli_install_into_host_namespace_still_drains_host_cohort at 42s wall for 19s of cpu -- 23s idle, the largest single idle block, where the next worst was 11s. The existing cli_install_force_quiesces_active_cohort_before_replacing_binary drains a cohort too and idles 0.5s, which is what pointed at the difference. The cost was the host_serving probe. Its generous budget exists for the POSITIVE question -- is this daemon still up? -- where a slow reply on a loaded runner must not be misread as drained; that fixed a real flake and is kept. Asked in the negative it inverts: no reply is coming, so the full 15s is spent establishing silence and ASSERT_FALSE is decided by the timeout expiring rather than by the daemon's behaviour. The drain is already proven positively in that test -- install returns 0 only after the activation completed, and the host child exits CLI_SCOPE_HOST_DRAINED -- so the probe only has to confirm it. Also shortens the drained child's teardown budget, which retried service_free / lease_release / manager_free against 10s on a path where the activation had already torn the service down. Behaviour at the deadline is unchanged; it is reached sooner when nothing is wedged. Measured at only -3s on its own, kept because it is correct and free. Local, same machine, ASan+UBSan: the drain test 42.0s -> 25.8s (-38%), the suite 300s -> 279s, 315 passed before and after -- no test removed, no assertion weakened. PR #2188, which runs the same leg without these two tests, passed macos-15-intel in 37m34s, so the suite was not already over budget on main. Signed-off-by: Martin Vogel --- scripts/run-test-wave.py | 14 ++++++++++- tests/test_cli.c | 52 ++++++++++++++++++++++++++++++++++------ 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/scripts/run-test-wave.py b/scripts/run-test-wave.py index d24f80815..ce7e3d344 100755 --- a/scripts/run-test-wave.py +++ b/scripts/run-test-wave.py @@ -26,7 +26,19 @@ SUITE_NAME = re.compile(r"^[a-z0-9_]+$") SUMMARY = re.compile(r"^ (?P[0-9]+) passed") FAILED = re.compile(r"(?:^|, )(?P[0-9]+) failed") SKIPPED = re.compile(r"(?:^|, )(?P[0-9]+) skipped") -SLOW_SUITES = frozenset(("incremental", "store_arch", "daemon_runtime")) +# Suites whose honest runtime does not fit the default per-suite budget, and so +# get --slow-timeout instead. This is a statement about SIZE, never about +# flakiness: every suite here is deterministic and simply long, and a racy suite +# must be made deterministic rather than given more clock. +# +# `cli` joined the list because the classification had gone stale, not because +# anything regressed. It spends 497s of the 900s default on macos-14 -- the +# FASTEST macOS runner -- while the macos-15-intel runner in the same matrix is +# 2.4-3.6x slower on comparable suites (daemon_runtime 842s vs 349s, +# stack_overflow_b 277s vs 76s). 497s at that ratio cannot fit, so the suite was +# killed at 900s and reported as hung. daemon_runtime, at 842s on that same +# runner, survives only because it was already listed here. +SLOW_SUITES = frozenset(("incremental", "store_arch", "daemon_runtime", "cli")) POLL_SECONDS = 0.05 # WHY: the Windows descendant probe below is a cold `powershell.exe` + CIM diff --git a/tests/test_cli.c b/tests/test_cli.c index 4f427be3a..4b3234ad4 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -1522,6 +1522,27 @@ TEST(cli_install_recovers_markerless_stale_rendezvous) { #define CLI_SCOPE_READY_TIMEOUT_MS 90000U #define CLI_SCOPE_HOST_REAP_TIMEOUT_MS 60000U #define CLI_SCOPE_HOST_SERVING_TIMEOUT_MS 15000U +/* Teardown budget for a child the activation ALREADY drained: its service is + * gone, so each free/release succeeds immediately and this is only the + * give-up point if one unexpectedly does not. The undrained path keeps the + * full 2 x CLI_SCOPE_TIMEOUT_MS, which is where a wedged teardown is real. */ +#define CLI_SCOPE_CLEANUP_DRAINED_MS 1000U +/* Budget for probing that an ALREADY-drained host has stopped serving. + * + * The generous CLI_SCOPE_HOST_SERVING_TIMEOUT_MS exists for the POSITIVE + * question -- "is this daemon still up?" -- where a slow reply on a loaded + * runner must not be misread as drained. Asked in the negative it inverts: + * there is no reply coming, so the whole budget is spent proving silence, and + * the assertion is decided by a timeout expiring rather than by the system's + * own behaviour. That was 15 s of the drain test's wall clock and the largest + * single idle block in the cli suite. + * + * The drain is proven POSITIVELY elsewhere in that test: install returns 0 + * only after the activation completed, and the host child exits with + * CLI_SCOPE_HOST_DRAINED. By the time this probe runs the daemon is already + * gone, so a short budget confirms the same fact a long one would -- it just + * stops charging the suite for the wait. */ +#define CLI_SCOPE_HOST_DRAINED_PROBE_MS 2000U typedef struct { char tmpdir[256]; @@ -1588,7 +1609,16 @@ static _Noreturn void cli_scope_host_child(const cli_scope_fixture_t *fixture, i break; } } - uint64_t cleanup_deadline = cbm_now_ms() + 2U * CLI_SCOPE_TIMEOUT_MS; + /* A DRAINED child has already had its service torn down by the activation, + * so every teardown below succeeds on the first attempt; the long deadline + * exists for the wedged case, where we keep retrying before giving up. On + * the drained path that budget was pure wall clock -- the parent sits in + * cli_scope_reap_host waiting for this exit, and it was the single largest + * idle block in the whole cli suite. Behaviour at the deadline is + * unchanged (give up and _exit); it is simply reached sooner when there is + * nothing wedged to wait for. */ + uint64_t cleanup_deadline = + cbm_now_ms() + (drained ? CLI_SCOPE_CLEANUP_DRAINED_MS : 2U * CLI_SCOPE_TIMEOUT_MS); if (service) { if (!drained) { (void)cbm_daemon_runtime_service_stop(service, CLI_SCOPE_TIMEOUT_MS); @@ -1752,19 +1782,24 @@ static bool cli_scope_fixture_start(cli_scope_fixture_t *fixture, const char *ta return fixture->client != NULL; } +static bool cli_scope_host_serving_within(const cli_scope_fixture_t *fixture, + uint32_t timeout_ms) { + cbm_daemon_runtime_status_t status = {0}; + return fixture->endpoint && + cbm_daemon_runtime_request_status(fixture->endpoint, &fixture->identity, timeout_ms, + &status) && + !status.stopping && status.committed_clients == 1; +} + /* Ask the host daemon itself: still running, not stopping, and the parent's * committed client still admitted. */ static bool cli_scope_host_serving(const cli_scope_fixture_t *fixture) { - cbm_daemon_runtime_status_t status = {0}; /* A generous, bounded status deadline: a foreign-namespace install leaves * this daemon serving, so a slow response on a loaded runner must not be * misread as "drained" (the flaky failure this fixture showed). The call * still fails cleanly — a genuinely drained daemon is unreachable or * reports stopping — it just no longer decides survival on a 5 s budget. */ - return fixture->endpoint && - cbm_daemon_runtime_request_status(fixture->endpoint, &fixture->identity, - CLI_SCOPE_HOST_SERVING_TIMEOUT_MS, &status) && - !status.stopping && status.committed_clients == 1; + return cli_scope_host_serving_within(fixture, CLI_SCOPE_HOST_SERVING_TIMEOUT_MS); } static int cli_scope_install(cli_scope_fixture_t *fixture, const char *home, const char *cache, @@ -1902,7 +1937,10 @@ TEST(cli_install_into_host_namespace_still_drains_host_cohort) { int install_rc = ready ? cli_scope_install(&fixture, fixture.host_home, fixture.host_cache, host_bin, false) : -1; - bool host_serving = ready && cli_scope_host_serving(&fixture); + /* Negative probe: see CLI_SCOPE_HOST_DRAINED_PROBE_MS. The drain itself is + * asserted positively below via host_exit == CLI_SCOPE_HOST_DRAINED. */ + bool host_serving = ready && cli_scope_host_serving_within(&fixture, + CLI_SCOPE_HOST_DRAINED_PROBE_MS); const char *events = read_test_file(activation_log); bool drained_in_log = events && strstr(events, "cohort drained") != NULL && strstr(events, "\"daemon_active_clients\":1") != NULL; From 8c2692c1ee32284c813c05746e833dea5dfedfe0 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 12 Sep 2026 19:28:55 +0200 Subject: [PATCH 058/130] fix(graph-ui): bump vitest past the @vitest/mocker path traversal GHSA-82fw-gwwq-j7x9: path traversal / arbitrary file read via the @vitest/mocker redirect mock, CVSS 5.9, affecting >= 2.1.0 < 4.1.11. graph-ui resolved 4.1.8. Development scope, and graph-ui appears nowhere in scripts/package-release.sh, so no released artifact was ever exposed. It is fixed rather than reasoned around because it is the repository's only open code-scanning alert, and the gate that is supposed to act on it cannot do its job while it stands. The manifest floor moves ^4.1.0 -> ^4.1.11 as well as the lockfile, so a fresh npm install cannot resolve back into the vulnerable range. graph-ui: npm ci clean, 47 tests across 12 files pass on 4.1.11. Split from the code-scanning gate fix deliberately. That change makes the gate able to see open alerts and fail closed; merging it while this alert is still open would block it on the very condition it exists to detect. Landing the bump first lets Scorecard rescan main (it runs on push) and clear the alert, after which the gate fix merges against a clean slate. Signed-off-by: Martin Vogel --- graph-ui/package-lock.json | 96 +++++++++++++++++++------------------- graph-ui/package.json | 2 +- 2 files changed, 49 insertions(+), 49 deletions(-) diff --git a/graph-ui/package-lock.json b/graph-ui/package-lock.json index 1065f3783..b5441de62 100644 --- a/graph-ui/package-lock.json +++ b/graph-ui/package-lock.json @@ -33,7 +33,7 @@ "tailwindcss": "^4.1.0", "typescript": "^5.7.0", "vite": "^6.4.3", - "vitest": "^4.1.0" + "vitest": "^4.1.11" } }, "node_modules/@adobe/css-tools": { @@ -3603,16 +3603,16 @@ } }, "node_modules/@vitest/expect": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.8.tgz", - "integrity": "sha512-h3nDO677RDLEGlBxyQ5CW8RlMThSKSRLUePLOx09gNIWRL40edgA1GCZSZgf1W55MFAG6/Sw14KeaAnqv0NKdQ==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.11.tgz", + "integrity": "sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==", "dev": true, "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.8", - "@vitest/utils": "4.1.8", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", "chai": "^6.2.2", "tinyrainbow": "^3.1.0" }, @@ -3621,13 +3621,13 @@ } }, "node_modules/@vitest/mocker": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.8.tgz", - "integrity": "sha512-LEiN/xe4OSIbKe9HQIp5OC24agGD9J5CnmMgsLohVVoOPWL9a2sBoR6VBx43jQZb7Kr1l4RCuyCJzcAa0+dojw==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.11.tgz", + "integrity": "sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "4.1.8", + "@vitest/spy": "4.1.11", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, @@ -3648,9 +3648,9 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.8.tgz", - "integrity": "sha512-9GasEBxpZ1VYIpqHf/0+YGg121uSNwCKOJqIrTwWP/TB7DmFCiaBpNl3aPZzoLWfWkuqhbH8vJIVobZkvdo2cA==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.11.tgz", + "integrity": "sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==", "dev": true, "license": "MIT", "dependencies": { @@ -3661,13 +3661,13 @@ } }, "node_modules/@vitest/runner": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.8.tgz", - "integrity": "sha512-EmVxeBAfMJvycdjd6Hm+RbFBbA9fKvo0Kx37hNpBYoYeavH3RNsBXWDooR1mgD52dCrxIIuP7UotpfiwOikvcg==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.11.tgz", + "integrity": "sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "4.1.8", + "@vitest/utils": "4.1.11", "pathe": "^2.0.3" }, "funding": { @@ -3675,14 +3675,14 @@ } }, "node_modules/@vitest/snapshot": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.8.tgz", - "integrity": "sha512-acfZboRmAIf05DEKcBQy33VXojFJjtUdLyo7oOmV9kebb2xdU01UknNiPuPZoJZQyO7DF0gZdTGTpeAzET9QPQ==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.11.tgz", + "integrity": "sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.8", - "@vitest/utils": "4.1.8", + "@vitest/pretty-format": "4.1.11", + "@vitest/utils": "4.1.11", "magic-string": "^0.30.21", "pathe": "^2.0.3" }, @@ -3691,9 +3691,9 @@ } }, "node_modules/@vitest/spy": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.8.tgz", - "integrity": "sha512-6EevtBp6OZOPF7bmz36HrGMeP3txgVSrgebWxHOafDXGkhIzfXK14f8KF6MuFfgXXUeHxmpD3BQxkV00/3s5mA==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.11.tgz", + "integrity": "sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==", "dev": true, "license": "MIT", "funding": { @@ -3701,13 +3701,13 @@ } }, "node_modules/@vitest/utils": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.8.tgz", - "integrity": "sha512-uOJamYALNhfJ6iolExyQM40yIQwDqYnkKtQ5VCiSe17E33H0aQ/u+1GlRuz4LZBk6Mm3sg90G9hEbmEt37C1Zg==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.11.tgz", + "integrity": "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.8", + "@vitest/pretty-format": "4.1.11", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" }, @@ -5831,9 +5831,9 @@ } }, "node_modules/tinyrainbow": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", - "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.1.tgz", + "integrity": "sha512-yau8yJdTt989Mm0Bd/236QnzEiPf2xLLTqUZRUJOo/3CB078LSwzei343DgtJVmfJKJE3TMINY1u42SQsP6mXw==", "dev": true, "license": "MIT", "engines": { @@ -6141,19 +6141,19 @@ } }, "node_modules/vitest": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.8.tgz", - "integrity": "sha512-flY6ScbCIt9HThs+C5HS7jvGOB560DJtk/Z15IQROTA6zEy49Nh8T/dofWTQL+n3vswqn87sbJNiuqw1SDp5Ig==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.11.tgz", + "integrity": "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/expect": "4.1.8", - "@vitest/mocker": "4.1.8", - "@vitest/pretty-format": "4.1.8", - "@vitest/runner": "4.1.8", - "@vitest/snapshot": "4.1.8", - "@vitest/spy": "4.1.8", - "@vitest/utils": "4.1.8", + "@vitest/expect": "4.1.11", + "@vitest/mocker": "4.1.11", + "@vitest/pretty-format": "4.1.11", + "@vitest/runner": "4.1.11", + "@vitest/snapshot": "4.1.11", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", @@ -6181,12 +6181,12 @@ "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.8", - "@vitest/browser-preview": "4.1.8", - "@vitest/browser-webdriverio": "4.1.8", - "@vitest/coverage-istanbul": "4.1.8", - "@vitest/coverage-v8": "4.1.8", - "@vitest/ui": "4.1.8", + "@vitest/browser-playwright": "4.1.11", + "@vitest/browser-preview": "4.1.11", + "@vitest/browser-webdriverio": "4.1.11", + "@vitest/coverage-istanbul": "4.1.11", + "@vitest/coverage-v8": "4.1.11", + "@vitest/ui": "4.1.11", "happy-dom": "*", "jsdom": "*", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" diff --git a/graph-ui/package.json b/graph-ui/package.json index b1aa3998b..b023560e8 100644 --- a/graph-ui/package.json +++ b/graph-ui/package.json @@ -37,7 +37,7 @@ "tailwindcss": "^4.1.0", "typescript": "^5.7.0", "vite": "^6.4.3", - "vitest": "^4.1.0" + "vitest": "^4.1.11" }, "overrides": { "form-data": ">=4.0.6", From 1ad52f5fb1b3d96255756de01ce30912b1899e09 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 12 Sep 2026 19:26:18 +0200 Subject: [PATCH 059/130] fix(ci): make the code-scanning gate able to see alerts, and fail closed The codeql-gate has never checked anything. It declared no permissions block, so it inherited the workflow's contents:read and the code-scanning alert API answered 403. gh writes the API error BODY to stdout, so ALERTS=$(gh api '.../code-scanning/alerts?state=open' --jq 'length' 2>/dev/null || echo "0") left ALERTS holding {"message":"Resource not accessible by integration",...,"status":"403"}0 every [ ... -gt ... ] then failed as a non-integer comparison, the if took its false branch, and the step printed "CodeQL gate passed (0 alerts)" and exited 0. Observed in the log of a GREEN run (34700514102, job 103571322660) while the repository had an open high-severity alert. The `|| echo "0"` reads as defensive and is the exact opposite: it converts "I cannot see the alerts" into "there are no alerts", which is the one direction a security gate must never fail in. Grant the job security-events:read (and actions:read, which the CodeQL-run poll needs for the same reason), and replace the fallback with a helper that treats a non-zero gh exit or a non-numeric body as a hard failure. The helper returns rather than exits because it runs inside a command substitution, where an exit would end only the subshell and let the caller continue with an empty count -- the same silent pass in a new costume; both call sites check the status. tests/test_security_gate_fail_closed.sh pins both halves and fails with six assertions against the previous workflow. Also bumps graph-ui's vitest to ^4.1.11 (GHSA-82fw-gwwq-j7x9, path traversal via @vitest/mocker, CVSS 5.9). Development scope, and graph-ui ships in no release artifact, so nothing released was ever exposed -- but it was the sole open alert and the gate above is worthless while it stands unresolved. Raising the floor past the vulnerable range also stops a fresh install resolving back into it. graph-ui: npm ci clean, 47 tests in 12 files pass on 4.1.11. Signed-off-by: Martin Vogel --- .github/workflows/_security.yml | 44 +++++++- .github/workflows/pr.yml | 7 ++ .github/workflows/release.yml | 7 ++ graph-ui/package-lock.json | 96 ++++++++--------- graph-ui/package.json | 2 +- scripts/test.sh | 3 + tests/test_security_gate_fail_closed.sh | 133 ++++++++++++++++++++++++ 7 files changed, 241 insertions(+), 51 deletions(-) create mode 100644 tests/test_security_gate_fail_closed.sh diff --git a/.github/workflows/_security.yml b/.github/workflows/_security.yml index dd6f3400c..ce044eed2 100644 --- a/.github/workflows/_security.yml +++ b/.github/workflows/_security.yml @@ -41,6 +41,14 @@ jobs: codeql-gate: runs-on: ubuntu-latest timeout-minutes: 240 + # Without security-events:read this job inherits the workflow's + # contents:read, the code-scanning alert API answers 403, and the gate + # below reported "0 alerts" on every run it has ever made. actions:read is + # what the CodeQL-run poll above needs for the same reason. + permissions: + contents: read + security-events: read + actions: read steps: - name: Wait for CodeQL on current commit (max 150 min) env: @@ -77,11 +85,43 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | + # FAIL CLOSED. The previous form was `gh api ... 2>/dev/null || echo "0"`, + # which looks defensive and is the opposite. gh writes the API error + # BODY to stdout, so a 403 made ALERTS the string + # {"message":"Resource not accessible by integration",...,"status":"403"}0 + # every `[` comparison then failed as non-integer, the `if` took its + # false branch, and the step printed "CodeQL gate passed (0 alerts)" + # and exited 0. That is exactly how a security gate should NEVER fail: + # silently, in the direction of "everything is fine". An unreadable + # alert list is now an error, not a zero. + # NOTE: this runs in a command substitution, so it must `return`, not + # `exit` -- an `exit` here would only end the subshell and let the + # caller continue with an empty count, recreating the very bug this + # replaces. The callers below check the status explicitly. + count_open_alerts() { + local body status + body=$(gh api 'repos/${{ github.repository }}/code-scanning/alerts?state=open' \ + --jq 'length' 2>/dev/null) + status=$? + if [ $status -ne 0 ]; then + echo "BLOCKED: cannot read code scanning alerts (gh exit $status)." >&2 + echo " Refusing to treat an unreadable alert list as zero." >&2 + echo " Most likely cause: the job lacks security-events:read." >&2 + return 1 + fi + case "$body" in + ''|*[!0-9]*) + echo "BLOCKED: alert count was not a number: ${body:-}" >&2 + return 1 + ;; + esac + printf '%s' "$body" + } echo "Waiting 60s for alert API to settle..." sleep 60 - ALERTS=$(gh api 'repos/${{ github.repository }}/code-scanning/alerts?state=open' --jq 'length' 2>/dev/null || echo "0") + ALERTS=$(count_open_alerts) || exit 1 sleep 15 - ALERTS2=$(gh api 'repos/${{ github.repository }}/code-scanning/alerts?state=open' --jq 'length' 2>/dev/null || echo "0") + ALERTS2=$(count_open_alerts) || exit 1 [ "$ALERTS" -lt "$ALERTS2" ] && ALERTS=$ALERTS2 if [ "$ALERTS" -gt 0 ]; then echo "BLOCKED: $ALERTS open alert(s)" diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index c18a45d13..4fef6ea68 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -22,6 +22,13 @@ concurrency: jobs: security: uses: ./.github/workflows/_security.yml + # The called job requests security-events:read to count code-scanning + # alerts; a reusable workflow cannot request more than its caller grants, + # so withholding this here fails the whole run at startup. + permissions: + contents: read + security-events: read + actions: read secrets: inherit lint: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 03abb518e..15f19f41f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,6 +44,13 @@ jobs: # ── Security (starts immediately, blocks only verify) ─────────── security: uses: ./.github/workflows/_security.yml + # The called job requests security-events:read to count code-scanning + # alerts; a reusable workflow cannot request more than its caller grants, + # so withholding this here fails the whole run at startup. + permissions: + contents: read + security-events: read + actions: read secrets: inherit # ── 0. Preflight: refuse malformed dispatch inputs ────────────── diff --git a/graph-ui/package-lock.json b/graph-ui/package-lock.json index 1065f3783..b5441de62 100644 --- a/graph-ui/package-lock.json +++ b/graph-ui/package-lock.json @@ -33,7 +33,7 @@ "tailwindcss": "^4.1.0", "typescript": "^5.7.0", "vite": "^6.4.3", - "vitest": "^4.1.0" + "vitest": "^4.1.11" } }, "node_modules/@adobe/css-tools": { @@ -3603,16 +3603,16 @@ } }, "node_modules/@vitest/expect": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.8.tgz", - "integrity": "sha512-h3nDO677RDLEGlBxyQ5CW8RlMThSKSRLUePLOx09gNIWRL40edgA1GCZSZgf1W55MFAG6/Sw14KeaAnqv0NKdQ==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.11.tgz", + "integrity": "sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==", "dev": true, "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.8", - "@vitest/utils": "4.1.8", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", "chai": "^6.2.2", "tinyrainbow": "^3.1.0" }, @@ -3621,13 +3621,13 @@ } }, "node_modules/@vitest/mocker": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.8.tgz", - "integrity": "sha512-LEiN/xe4OSIbKe9HQIp5OC24agGD9J5CnmMgsLohVVoOPWL9a2sBoR6VBx43jQZb7Kr1l4RCuyCJzcAa0+dojw==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.11.tgz", + "integrity": "sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "4.1.8", + "@vitest/spy": "4.1.11", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, @@ -3648,9 +3648,9 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.8.tgz", - "integrity": "sha512-9GasEBxpZ1VYIpqHf/0+YGg121uSNwCKOJqIrTwWP/TB7DmFCiaBpNl3aPZzoLWfWkuqhbH8vJIVobZkvdo2cA==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.11.tgz", + "integrity": "sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==", "dev": true, "license": "MIT", "dependencies": { @@ -3661,13 +3661,13 @@ } }, "node_modules/@vitest/runner": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.8.tgz", - "integrity": "sha512-EmVxeBAfMJvycdjd6Hm+RbFBbA9fKvo0Kx37hNpBYoYeavH3RNsBXWDooR1mgD52dCrxIIuP7UotpfiwOikvcg==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.11.tgz", + "integrity": "sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "4.1.8", + "@vitest/utils": "4.1.11", "pathe": "^2.0.3" }, "funding": { @@ -3675,14 +3675,14 @@ } }, "node_modules/@vitest/snapshot": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.8.tgz", - "integrity": "sha512-acfZboRmAIf05DEKcBQy33VXojFJjtUdLyo7oOmV9kebb2xdU01UknNiPuPZoJZQyO7DF0gZdTGTpeAzET9QPQ==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.11.tgz", + "integrity": "sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.8", - "@vitest/utils": "4.1.8", + "@vitest/pretty-format": "4.1.11", + "@vitest/utils": "4.1.11", "magic-string": "^0.30.21", "pathe": "^2.0.3" }, @@ -3691,9 +3691,9 @@ } }, "node_modules/@vitest/spy": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.8.tgz", - "integrity": "sha512-6EevtBp6OZOPF7bmz36HrGMeP3txgVSrgebWxHOafDXGkhIzfXK14f8KF6MuFfgXXUeHxmpD3BQxkV00/3s5mA==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.11.tgz", + "integrity": "sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==", "dev": true, "license": "MIT", "funding": { @@ -3701,13 +3701,13 @@ } }, "node_modules/@vitest/utils": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.8.tgz", - "integrity": "sha512-uOJamYALNhfJ6iolExyQM40yIQwDqYnkKtQ5VCiSe17E33H0aQ/u+1GlRuz4LZBk6Mm3sg90G9hEbmEt37C1Zg==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.11.tgz", + "integrity": "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.8", + "@vitest/pretty-format": "4.1.11", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" }, @@ -5831,9 +5831,9 @@ } }, "node_modules/tinyrainbow": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", - "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.1.tgz", + "integrity": "sha512-yau8yJdTt989Mm0Bd/236QnzEiPf2xLLTqUZRUJOo/3CB078LSwzei343DgtJVmfJKJE3TMINY1u42SQsP6mXw==", "dev": true, "license": "MIT", "engines": { @@ -6141,19 +6141,19 @@ } }, "node_modules/vitest": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.8.tgz", - "integrity": "sha512-flY6ScbCIt9HThs+C5HS7jvGOB560DJtk/Z15IQROTA6zEy49Nh8T/dofWTQL+n3vswqn87sbJNiuqw1SDp5Ig==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.11.tgz", + "integrity": "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/expect": "4.1.8", - "@vitest/mocker": "4.1.8", - "@vitest/pretty-format": "4.1.8", - "@vitest/runner": "4.1.8", - "@vitest/snapshot": "4.1.8", - "@vitest/spy": "4.1.8", - "@vitest/utils": "4.1.8", + "@vitest/expect": "4.1.11", + "@vitest/mocker": "4.1.11", + "@vitest/pretty-format": "4.1.11", + "@vitest/runner": "4.1.11", + "@vitest/snapshot": "4.1.11", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", @@ -6181,12 +6181,12 @@ "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.8", - "@vitest/browser-preview": "4.1.8", - "@vitest/browser-webdriverio": "4.1.8", - "@vitest/coverage-istanbul": "4.1.8", - "@vitest/coverage-v8": "4.1.8", - "@vitest/ui": "4.1.8", + "@vitest/browser-playwright": "4.1.11", + "@vitest/browser-preview": "4.1.11", + "@vitest/browser-webdriverio": "4.1.11", + "@vitest/coverage-istanbul": "4.1.11", + "@vitest/coverage-v8": "4.1.11", + "@vitest/ui": "4.1.11", "happy-dom": "*", "jsdom": "*", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" diff --git a/graph-ui/package.json b/graph-ui/package.json index b1aa3998b..b023560e8 100644 --- a/graph-ui/package.json +++ b/graph-ui/package.json @@ -37,7 +37,7 @@ "tailwindcss": "^4.1.0", "typescript": "^5.7.0", "vite": "^6.4.3", - "vitest": "^4.1.0" + "vitest": "^4.1.11" }, "overrides": { "form-data": ">=4.0.6", diff --git a/scripts/test.sh b/scripts/test.sh index dc592ef80..b88707371 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -218,6 +218,9 @@ bash "$ROOT/tests/test_windows_bundle_contract.sh" echo "=== Step 0e2: activation refusal diagnostic contract ===" bash "$ROOT/tests/test_activation_diagnostic_contract.sh" +echo "=== Step 0e3: security gate fail-closed contract ===" +bash "$ROOT/tests/test_security_gate_fail_closed.sh" + echo "=== Step 0f: tree-sitter runtime Makefile dependencies ===" bash "$ROOT/tests/test_makefile_ts_runtime_dependencies.sh" diff --git a/tests/test_security_gate_fail_closed.sh b/tests/test_security_gate_fail_closed.sh new file mode 100644 index 000000000..114c16e5e --- /dev/null +++ b/tests/test_security_gate_fail_closed.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# The code-scanning gate must FAIL CLOSED. +# +# History. `codeql-gate` in .github/workflows/_security.yml counted open alerts +# with: +# +# ALERTS=$(gh api '.../code-scanning/alerts?state=open' --jq 'length' 2>/dev/null || echo "0") +# +# That reads as defensive and is the exact opposite. The job declared no +# `permissions:` block, so it inherited the workflow's `contents: read` and the +# alert API answered 403. `gh` writes the API error BODY to stdout, so `ALERTS` +# became the string +# +# {"message":"Resource not accessible by integration",...,"status":"403"}0 +# +# every `[ ... -gt ... ]` then failed as a non-integer comparison, the `if` took +# its false branch, and the step printed "CodeQL gate passed (0 alerts)" and +# exited 0. Observed live in a GREEN run (34700514102 / job 103571322660), while +# the repository had an open high-severity alert. +# +# A security gate that cannot read its input must stop the build, not wave it +# through. This test pins both halves of that: the permission that makes the +# read possible, and the fail-closed handling for when it still is not. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" + +python3 - "$ROOT" <<'PY' +from __future__ import annotations + +import pathlib +import re +import sys + +root = pathlib.Path(sys.argv[1]) +failures: list[str] = [] + + +def require(condition: bool, message: str) -> None: + if not condition: + failures.append(message) + + +workflow = (root / ".github" / "workflows" / "_security.yml").read_text(encoding="utf-8") + +# Isolate the codeql-gate job so a permission granted to some OTHER job cannot +# satisfy this test. +gate = re.search(r"\n codeql-gate:\n(?P(?: .*\n|\n)*)", workflow) +require(gate is not None, "_security.yml no longer defines a codeql-gate job") +body = gate.group("body") if gate else "" + +# 1. The read has to be possible at all. +require( + re.search(r"^ permissions:\s*$", body, re.M) is not None, + "codeql-gate must declare its own permissions block -- inheriting the " + "workflow default leaves it without security-events access and the alert " + "API answers 403", +) +require( + re.search(r"^ security-events:\s*read\s*$", body, re.M) is not None, + "codeql-gate must request security-events: read, or the code-scanning " + "alert query 403s and the gate cannot see any alert", +) + +# 2. The failure direction. `|| echo "0"` on the alert query is the specific +# construct that turned an API error into a clean pass. +require( + re.search(r"code-scanning/alerts[^\n]*\|\|\s*echo", body) is None, + "the alert count must not fall back to a literal on error -- an unreadable " + "alert list is not zero alerts", +) +require( + "BLOCKED: cannot read code scanning alerts" in body, + "codeql-gate must fail with a named error when the alert API cannot be read", +) +require( + re.search(r"alert count was not a number", body) is not None, + "codeql-gate must reject a non-numeric alert count instead of comparing it", +) + +# 3. The count is produced in a command substitution, so a bare `exit` inside +# the helper would end only the subshell and let the caller proceed with an +# empty value -- the same class of silent pass. The callers must check. +assignments = re.findall(r"^\s*ALERTS2?=\$\(count_open_alerts\)(.*)$", body, re.M) +require( + len(assignments) >= 2, + "expected both alert samples to go through the checked helper", +) +require( + all("|| exit 1" in tail for tail in assignments), + "every count_open_alerts call must be followed by || exit 1 -- a failure " + "inside a command substitution does not stop the caller on its own", +) + +# 4. Every caller must GRANT what the job requests. A reusable workflow cannot +# request more permission than its caller holds: asking for +# security-events:read from a caller that grants only contents:read does not +# downgrade, it fails the entire run with a startup_failure before a single +# job executes. Checking only the called workflow is how this test passed +# while the PR pipeline never started. +for caller_name in ("pr.yml", "release.yml"): + caller = (root / ".github" / "workflows" / caller_name).read_text(encoding="utf-8") + # The next sibling may be a comment line, not a key, so the lookahead has to + # accept any 2-space-indented non-space -- matching only ` \w` silently + # failed to find release.yml's job at all. + call = re.search( + r"\n security:\n(?P(?: .*\n|\n)*?)(?=\n? \S|\Z)", caller + ) + require(call is not None, f"{caller_name} no longer has a security: job") + call_body = call.group("body") if call else "" + require( + "uses: ./.github/workflows/_security.yml" in call_body, + f"{caller_name}'s security job must call _security.yml", + ) + require( + re.search(r"^ security-events:\s*read\s*$", call_body, re.M) is not None, + f"{caller_name} must grant security-events: read to the security job -- " + "a called workflow cannot request more than its caller grants, and the " + "run fails at startup rather than degrading", + ) + require( + re.search(r"^ actions:\s*read\s*$", call_body, re.M) is not None, + f"{caller_name} must grant actions: read to the security job", + ) + +if failures: + print("Security gate fail-closed contract FAILED:") + for failure in failures: + print(f" - {failure}") + sys.exit(1) + +print("Security gate fail-closed contract passed") +PY From 09c0e88a6484c5fb696d7fc7b5a5e3f32b8cba68 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 13 Sep 2026 01:27:43 +0200 Subject: [PATCH 060/130] fix(ci): grant the security permissions in dry-run.yml, and derive the caller list #2189 gave codeql-gate the security-events:read it needs to see code-scanning alerts, and granted it at the two callers I knew about: pr.yml and release.yml. There is a third. dry-run.yml calls the same reusable workflow and was left granting only the workflow default, so every dispatch of it now fails at startup -- before a single job runs, with no job output to explain why. Observed immediately: run 34725401182 on 7d74337e, startup_failure. The contract test did not catch it because it iterated a hardcoded ("pr.yml", "release.yml"). A test that checks only the callers you remembered cannot catch the one you forgot, and it reports green while doing so -- the same shape of false assurance as the 403-to-"0 alerts" bug this whole contract exists to prevent. So the list is now derived: every workflow containing `uses: ./.github/workflows/_security.yml` must grant security-events:read and actions:read, with a floor of three callers so a silent deletion is noticed too. Reverting only dry-run.yml's grant now fails the test by name. Signed-off-by: Martin Vogel --- .github/workflows/dry-run.yml | 7 +++++++ tests/test_security_gate_fail_closed.sh | 21 +++++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dry-run.yml b/.github/workflows/dry-run.yml index 43a6b6bd9..4cf2ec46c 100644 --- a/.github/workflows/dry-run.yml +++ b/.github/workflows/dry-run.yml @@ -58,6 +58,13 @@ jobs: # ── Security (independent island — does not block main pipeline) ── security: uses: ./.github/workflows/_security.yml + # The called job requests security-events:read to count code-scanning + # alerts; a reusable workflow cannot request more than its caller grants, + # so withholding this here fails the whole run at startup. + permissions: + contents: read + security-events: read + actions: read secrets: inherit # ── Lint (cppcheck + clang-format) ──────────────────────────── diff --git a/tests/test_security_gate_fail_closed.sh b/tests/test_security_gate_fail_closed.sh index 114c16e5e..0e79f2356 100644 --- a/tests/test_security_gate_fail_closed.sh +++ b/tests/test_security_gate_fail_closed.sh @@ -98,8 +98,25 @@ require( # downgrade, it fails the entire run with a startup_failure before a single # job executes. Checking only the called workflow is how this test passed # while the PR pipeline never started. -for caller_name in ("pr.yml", "release.yml"): - caller = (root / ".github" / "workflows" / caller_name).read_text(encoding="utf-8") +# The caller list is DERIVED, never hardcoded. An earlier revision listed +# ("pr.yml", "release.yml") by hand and stayed green while dry-run.yml -- a +# third caller nobody had listed -- failed every run at startup. A test that +# only checks the callers you remembered cannot catch the one you forgot. +workflow_dir = root / ".github" / "workflows" +callers = sorted( + path.name + for path in workflow_dir.glob("*.yml") + if "uses: ./.github/workflows/_security.yml" in path.read_text(encoding="utf-8") +) +require( + len(callers) >= 3, + "expected at least 3 callers of _security.yml, found: " + + (", ".join(callers) or "none") + + " -- if a caller was deliberately removed, lower this floor on purpose", +) + +for caller_name in callers: + caller = (workflow_dir / caller_name).read_text(encoding="utf-8") # The next sibling may be a comment line, not a key, so the lookahead has to # accept any 2-space-indented non-space -- matching only ` \w` silently # failed to find release.yml's job at all. From 469c3dd996f23cdb629f1bbd6a751bc0f29a99f2 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 13 Sep 2026 09:20:11 +0200 Subject: [PATCH 061/130] fix(test): exec the #1830 userns probe instead of forking a primed process The single-uid userns smoke test forked a child, unshared CLONE_NEWUSER, wrote a 1:1 uid_map, then called cbm_daemon_ipc_private_directory_secure() directly in that child. The call cannot give the right answer there. posix_ancestor_overflow_uid() in src/daemon/ipc.c derives the overflow uid once per process under pthread_once, and the comment above it calls the /proc state "immutable". unshare(CLONE_NEWUSER) is precisely what changes /proc/self/uid_map, and pthread_once state survives fork(). Seven earlier call sites in this suite run the derivation first, so by the time the test forks, the child has already inherited the HOST answer -- no overflow uid -- and refuses the directory no matter what its namespace says. The test therefore asserted a product behaviour it never executed, and failed as a product refusal (exit 1) while the product was correct: production daemons start inside their namespace and derive the right answer on first use. Re-exec /proc/self/exe into a new --userns-secure-probe mode so the decision is made by a process that STARTED in the namespace, which is also the only shape production ever takes. A failed exec gets its own exit code 3 and a named FAIL, so a broken harness can never be misread as a quiet refusal. Verified both directions on linux/arm64 in the container leg as uid 1001: reverted, 48 passed / 1 failed on "expected WEXITSTATUS(status) == 1"; with the fix, 49 passed, 0 skipped. The non-root half of that recipe is now written down at the test, because it is the part that fooled me first: as root the mapped range covers uid 0, so root-owned /tmp stays 1:1 inside the namespace and never turns overflow. The first run of this reproduction passed with the fix REVERTED for exactly that reason. Only a non-root uid reaches the branch the test exists to cover. Signed-off-by: Martin Vogel --- tests/test_daemon_ipc.c | 33 +++++++++++++++++++++++++++++---- tests/test_main.c | 16 ++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/tests/test_daemon_ipc.c b/tests/test_daemon_ipc.c index e9f0a9046..9a10f838c 100644 --- a/tests/test_daemon_ipc.c +++ b/tests/test_daemon_ipc.c @@ -5133,7 +5133,16 @@ TEST(daemon_ipc_posix_overflow_ancestor_tolerated_only_in_single_uid_ns_issue183 * blocks unshare(CLONE_NEWUSER) on the Colima container leg; some kernels ship * user namespaces disabled. WHAT WAS TRIED when it skips: fork + unshare * CLONE_NEWUSER + a 1:1 uid_map write, which the sandbox denied (EPERM). The - * deterministic decision coverage above is what binds the fix. */ + * deterministic decision coverage above is what binds the fix. + * + * TO REPRODUCE IT LOCALLY two things are needed, and the second is easy to + * miss: run the container with --security-opt seccomp=unconfined so unshare is + * permitted, AND run the suite as a NON-ROOT uid. As root the mapped range + * covers uid 0, so root-owned /tmp stays 1:1 inside the namespace, never turns + * overflow, and this test passes without ever reaching the branch it exists to + * cover -- it passes just as happily with the fix reverted. Under `su tester` + * (uid 1001, the shape CI's runner user has) the ancestors do go overflow and + * the assertion becomes real. */ TEST(daemon_ipc_posix_single_uid_userns_real_smoke_issue1830) { #if defined(__linux__) uid_t host_uid = geteuid(); @@ -5164,9 +5173,20 @@ TEST(daemon_ipc_posix_single_uid_userns_real_smoke_issue1830) { _exit(2); } /* Inside the ns / and /tmp now show the overflow uid. With #1830 the - * daemon can still build its private tree there; without it, refused. */ - bool secured = cbm_daemon_ipc_private_directory_secure(probe); - _exit(secured ? 0 : 1); + * daemon can still build its private tree there; without it, refused. + * + * EXEC, do not just call. The overflow uid is derived once per process + * (pthread_once) from /proc/self/uid_map, and that state survives + * fork(): seven earlier call sites in this suite prime it with the HOST + * answer, so a forked child keeps "no overflow uid" and refuses no + * matter what its namespace says. That made this test fail on the only + * platform where it actually runs (ubuntu-22.04; macOS compile-gates it + * out, Colima's seccomp blocks unshare, and 23.10+ restricts + * unprivileged userns) -- while looking like a product regression. + * Re-exec so the decision is made by a process that STARTED here, which + * is also the only shape production ever takes. */ + (void)execl("/proc/self/exe", "test-runner", "--userns-secure-probe", probe, (char *)NULL); + _exit(3); /* exec failed -- distinct from secure(0)/refused(1)/skip(2) */ } if (child < 0) { FAIL("fork failed for userns smoke"); @@ -5178,6 +5198,11 @@ TEST(daemon_ipc_posix_single_uid_userns_real_smoke_issue1830) { if (WIFEXITED(status) && WEXITSTATUS(status) == 2) { SKIP_PLATFORM("user namespaces unavailable (no CLONE_NEWUSER / seccomp-blocked)"); } + /* A failed re-exec must never read as a product refusal: 3 is its own code + * so a broken probe is a loud harness failure, not a quiet "refused". */ + if (WIFEXITED(status) && WEXITSTATUS(status) == 3) { + FAIL("userns smoke could not re-exec /proc/self/exe for the probe"); + } ASSERT_TRUE(WIFEXITED(status)); ASSERT_EQ(0, WEXITSTATUS(status)); PASS(); diff --git a/tests/test_main.c b/tests/test_main.c index 42848f991..463a41c0a 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -943,6 +943,22 @@ int main(int argc, char **argv) { (void)puts("codebase-memory-mcp test-runner"); return 0; } + /* #1830 userns smoke probe -- see the test that spawns it. + * + * The ancestor overflow uid is derived ONCE per process (pthread_once in + * src/daemon/ipc.c) from /proc/self/uid_map. That file is not immutable: + * unshare(CLONE_NEWUSER) is exactly what changes it, and pthread_once state + * survives fork(). A forked child therefore keeps the HOST answer and never + * re-derives inside its new namespace, so a fork-only smoke test refuses and + * cannot pass once anything earlier in the suite has primed the cache -- + * seven call sites do. Re-exec into this probe so the decision is made by a + * process that STARTED inside the namespace, which is the production shape + * the test means to cover. */ +#if defined(__linux__) + if (argc == 3 && strcmp(argv[1], "--userns-secure-probe") == 0) { + return cbm_daemon_ipc_private_directory_secure(argv[2]) ? 0 : 1; + } +#endif int mcp_idxfailclosed_rc = tf_maybe_run_mcp_idxfailclosed_probe(argc, argv); if (mcp_idxfailclosed_rc >= 0) { return mcp_idxfailclosed_rc; From f9038b84eb3bf82c2dff4cc4ea7f1ec1d8d13f01 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 13 Sep 2026 09:56:54 +0200 Subject: [PATCH 062/130] fix(test,daemon): make the userns probe revert-sensitive and stop a false invariant shipping Follow-up to the re-exec fix, from the challenger read the frozen scope requires for security-boundary units. Three findings, all addressed here; two direction questions go to the maintainer separately. 1. src/daemon/ipc.c stated an invariant that is FALSE. The comment above posix_ancestor_overflow_uid() read "Derived once from immutable /proc state". /proc/self/uid_map is not immutable -- unshare(CLONE_NEWUSER) is exactly what rewrites it -- and pthread_once state survives fork() already marked done. The hazard was written up in the tests, which is the one place an ipc.c maintainer will never look, while production shipped the opposite claim. The comment now states the real guarantee: the cache is safe only because every caller runs in a freshly exec'd process, and it names what would break it. An audit of every process-creation site in src/ backs that: the daemon double-fork execs (daemon/bootstrap.c), cbm_fork_with_retry execs (foundation/subprocess.c), index workers go through cbm_subprocess_spawn, worker_pool.c never forks, and src/ contains no setns or unshare at all. No production path forks without exec and then reaches this check. 2. The probe now _exit()s instead of returning. The runner is built with -fsanitize=address, so returning runs the atexit chain, and a future leak anywhere in the prologue would exit 23 -- which the parent would have read as a verdict on a security test that has none. 3. The parent now treats 0/1/2/3 as the closed protocol and names anything else with the code it actually saw. A signal or a sanitizer exit is a broken harness, not a refusal, and must not be able to look like one. The probe guard also now matches its caller (linux AND test seams) instead of compiling dead code in a non-seams build. VERIFICATION. The challenger was right that the earlier proof was incomplete: it showed the new test goes green, never that the test still binds the #1830 relaxation. Stubbing posix_compute_ancestor_overflow_uid() to POSIX_NO_OVERFLOW_UID -- the relaxation removed -- and running as uid 1001: tolerance off, new probe 48 passed, 1 failed (refused, exit 1) tolerance on, new probe 49 passed, 0 skipped so the smoke test is revert-sensitive to the production decision, not merely to the test change. Worth recording: both deterministic #1830 tests PASSED with the tolerance stubbed off, because they drive the test-seam override rather than the real derivation. They bind the decision table; only this smoke test binds the wiring. macOS daemon_ipc 53 passed 1 skipped; linux/arm64 49 passed 0 skipped; lint-ci clean. Signed-off-by: Martin Vogel --- src/daemon/ipc.c | 12 +++++++++++- tests/test_daemon_ipc.c | 14 +++++++++++++- tests/test_main.c | 8 ++++++-- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/daemon/ipc.c b/src/daemon/ipc.c index 1243405c7..cec3b60b4 100644 --- a/src/daemon/ipc.c +++ b/src/daemon/ipc.c @@ -1572,7 +1572,17 @@ static void posix_overflow_init_once(void) { #endif /* __linux__ */ /* The overflow uid tolerated for ancestors in this process, or - * POSIX_NO_OVERFLOW_UID when none. Derived once from immutable /proc state. */ + * POSIX_NO_OVERFLOW_UID when none. + * + * Derived once per process. NOT from immutable state, whatever an earlier + * revision of this comment claimed: unshare(CLONE_NEWUSER) rewrites + * /proc/self/uid_map, and pthread_once state survives fork() already marked + * done. A child that forks WITHOUT exec and then changes namespace therefore + * keeps the parent answer. That is safe here only because every caller runs in + * a freshly exec'd process -- the daemon double-fork execs + * (daemon/bootstrap.c), workers go through cbm_subprocess_spawn's fork+exec, + * and src/ contains no setns/unshare at all. If you ever add a fork-without-exec + * path that reaches this check, this cache is WRONG and must go. */ static uid_t posix_ancestor_overflow_uid(void) { #ifdef CBM_ENABLE_TEST_SEAMS if (g_posix_overflow_override_active) { diff --git a/tests/test_daemon_ipc.c b/tests/test_daemon_ipc.c index 9a10f838c..2da597add 100644 --- a/tests/test_daemon_ipc.c +++ b/tests/test_daemon_ipc.c @@ -5203,7 +5203,19 @@ TEST(daemon_ipc_posix_single_uid_userns_real_smoke_issue1830) { if (WIFEXITED(status) && WEXITSTATUS(status) == 3) { FAIL("userns smoke could not re-exec /proc/self/exe for the probe"); } - ASSERT_TRUE(WIFEXITED(status)); + /* {0,1,2,3} is the whole protocol. Any other code -- a sanitizer exit (LSan + * defaults to 23), an abort, a signal -- is a broken harness, not a refusal, + * and must say so with the code it actually saw. */ + if (!WIFEXITED(status)) { + FAIL("userns probe did not exit normally (signalled) -- not a security verdict"); + } + if (WEXITSTATUS(status) != 0 && WEXITSTATUS(status) != 1) { + char unexpected[128]; + (void)snprintf(unexpected, sizeof(unexpected), + "userns probe exited %d -- not a security verdict", + WEXITSTATUS(status)); + FAIL(unexpected); + } ASSERT_EQ(0, WEXITSTATUS(status)); PASS(); #else diff --git a/tests/test_main.c b/tests/test_main.c index 463a41c0a..e25de3026 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -954,9 +954,13 @@ int main(int argc, char **argv) { * seven call sites do. Re-exec into this probe so the decision is made by a * process that STARTED inside the namespace, which is the production shape * the test means to cover. */ -#if defined(__linux__) +#if defined(__linux__) && defined(CBM_ENABLE_TEST_SEAMS) if (argc == 3 && strcmp(argv[1], "--userns-secure-probe") == 0) { - return cbm_daemon_ipc_private_directory_secure(argv[2]) ? 0 : 1; + /* _exit, not return: this process exists to answer ONE boolean. A + * return runs the atexit chain, and the runner is built with + * -fsanitize=address, so a future leak anywhere in the prologue would + * exit 23 and read as a security verdict on a test that has none. */ + _exit(cbm_daemon_ipc_private_directory_secure(argv[2]) ? 0 : 1); } #endif int mcp_idxfailclosed_rc = tf_maybe_run_mcp_idxfailclosed_probe(argc, argv); From d21e6cc762a1816d8c274174c9bba01601b888ee Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 13 Sep 2026 10:21:16 +0200 Subject: [PATCH 063/130] fix(daemon,ci): stop caching the overflow uid, and run the userns guard on the core matrix Three maintainer decisions from the challenger read on the #1830 unit. 1. DELETE THE CACHE (not just fix its comment). posix_ancestor_overflow_uid() memoised via pthread_once behind a comment claiming "immutable /proc state". Both halves were false: unshare(CLONE_NEWUSER) rewrites /proc/self/uid_map, and pthread_once state survives fork() already marked done. A process that forked and then entered a namespace kept the parent verdict and refused a directory it should have accepted. It was harmless only because every caller today runs in a freshly exec'd process -- the daemon double-fork execs, workers go through cbm_subprocess_spawn's fork+exec, worker_pool.c never forks, and src/ contains no setns or unshare at all. That made a security decision depend on an invariant nothing enforced. The next fork-without-exec caller would have inherited a stale verdict silently. Now derived fresh per call: two small /proc reads against an openat + fstat + fchmod + ACL check per path component in the same walk. With a test that would have caught the original bug and needs no namespace. daemon_ipc_posix_overflow_uid_is_never_cached_issue1830 counts real derivations through the production entry point and asserts the count rises on every ancestor check. Re-introducing a cache fails it by name on every platform. Proven both ways on linux/arm64 as uid 1001: cache re-introduced 49 passed 1 failed; removed 50 passed 0 skipped. Worth knowing why that test is not redundant: with the re-exec probe in place, the userns smoke test PASSES even with the cache restored, because the probe is a fresh process and immune to it by construction. The two tests are complementary -- the smoke test binds the namespace wiring, this one binds the absence of the cache. 2. RUN THE GUARD ON THE CORE MATRIX. Ubuntu 23.10+ ships kernel.apparmor_restrict_unprivileged_userns=1, so the #1830 smoke test skipped on every 24.04 leg and its only real-namespace coverage sat on the two ubuntu-22.04 legs. Those are broad-matrix only, so the test ran in NO pull request, and GitHub retires those images on 2027-04-17 -- after which it would have become a permanent silent skip, and a skip is not a red. A deliberate relaxation of a security check was guarded by a test that ran nowhere a change would be reviewed, on a clock. _test.yml now lifts the restriction with one sysctl on every ubuntu leg (hosted runners have passwordless sudo), best-effort so a runner that refuses simply returns to skipping. The O10 skip-whitelist comment no longer presents the AppArmor restriction as immovable. 3. KEEP THE TOLERANCE, CORRECT THE SAFETY ARGUMENT. The comment claimed that in a single-uid map "nobody reachable can have created it or can mutate it". The overflow uid is what EVERY unmapped host uid maps to, not only host root (user_namespaces(7)), so overflow-owned does not prove root-created: on a shared host another local user's sticky directory is indistinguishable from root-owned /tmp from inside the namespace. The tolerance stays, because there is no in-namespace discriminator that could tell them apart -- and narrowing it would refuse the legitimate root-owned 01777 /tmp that #1830 exists to accept, i.e. it would revert the fix in practice. What changes is the claim: the real guarantee is that no principal reachable from inside the namespace can create or mutate such an ancestor, and the leaf is never tolerated as overflow (0700, euid-rechecked), so a hostile host-side ancestor owner is bounded to denial of service and socket-path control. macOS daemon_ipc 53 passed 2 skipped; linux/arm64 50 passed 0 skipped; lint-ci clean. Signed-off-by: Martin Vogel --- .github/workflows/_test.yml | 15 +++++++++ src/daemon/ipc.c | 63 +++++++++++++++++++++++++------------ src/daemon/ipc.h | 6 ++++ tests/test_daemon_ipc.c | 51 +++++++++++++++++++++++++++++- 4 files changed, 114 insertions(+), 21 deletions(-) diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index d915eb243..893cc45e3 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -107,6 +107,21 @@ jobs: if: startsWith(matrix.os, 'ubuntu') run: sudo apt-get update && sudo apt-get install -y zlib1g-dev ccache + # Ubuntu 23.10+ ships kernel.apparmor_restrict_unprivileged_userns=1, which + # blocks unshare(CLONE_NEWUSER) for unprivileged users. Without this, the + # #1830 single-uid-userns smoke test SKIPS on every 24.04 leg, and its only + # real-namespace coverage lived on the two ubuntu-22.04 legs -- which are + # broad-matrix only (never PR CI) and which GitHub retires on 2027-04-17. + # A deliberate relaxation of a security check was therefore guarded by a + # test that ran in no PR and was on a clock. Hosted runners have + # passwordless sudo, so one sysctl puts it on the CORE matrix instead. + # Best-effort: a runner that refuses simply returns to skipping. + - name: Permit unprivileged user namespaces (Ubuntu) + if: startsWith(matrix.os, 'ubuntu') + run: | + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || \ + echo "could not lift the userns restriction; namespace tests will skip" + - name: Install ccache (macOS) if: startsWith(matrix.os, 'macos') run: command -v ccache >/dev/null 2>&1 || brew install ccache diff --git a/src/daemon/ipc.c b/src/daemon/ipc.c index cec3b60b4..f8d7d70b4 100644 --- a/src/daemon/ipc.c +++ b/src/daemon/ipc.c @@ -1463,9 +1463,21 @@ static char *private_log_directory_path_copy(const char *directory_path) { * * The overflow owner is tolerated for ANCESTORS ONLY, and ONLY when * /proc/self/uid_map is a single-uid map whose sole inside id is our euid. In - * that shape no in-namespace principal can be the overflow owner, so an - * overflow-owned ancestor is exactly as safe as a root-owned one on the host: - * nobody reachable can have created it or can mutate it. A multi-uid map + * that shape no IN-NAMESPACE principal can be the overflow owner. + * + * Be precise about what that does and does not buy, because an earlier version + * of this comment overstated it. The overflow uid is what EVERY unmapped host + * uid maps to, not only host root (user_namespaces(7)), so "overflow-owned" + * does NOT prove "created by root". On a shared host, a directory owned by + * another local user is indistinguishable from root-owned /tmp once you are + * inside the namespace -- and there is no in-namespace discriminator that could + * tell them apart, which is precisely why the tolerance is scoped the way it + * is rather than made smarter. The real guarantee is narrower and still + * sufficient: no principal REACHABLE FROM INSIDE the namespace can create or + * mutate such an ancestor, and the private directory itself is never tolerated + * as overflow (see below), so a hostile host-side owner of an ancestor is + * bounded to denial of service and socket-path control. It cannot reach the + * leaf, which stays 0700 and euid-rechecked. A multi-uid map * (rootless podman with subuids) still shows host root as overflow and is * refused — conservative scope, not the safety argument. There is deliberately * no environment escape hatch. @@ -1545,7 +1557,20 @@ static bool posix_read_small_proc_file(const char *path, char *buffer, size_t ca return true; } +#ifdef CBM_ENABLE_TEST_SEAMS +/* Counts real derivations. A cache here was a security hazard once (see the + * note above posix_ancestor_overflow_uid); the contract test asserts this + * climbs on EVERY call so re-introducing one fails loudly. */ +static unsigned g_posix_overflow_compute_count; +unsigned cbm_daemon_ipc_posix_overflow_compute_count_for_test(void) { + return g_posix_overflow_compute_count; +} +#endif + static uid_t posix_compute_ancestor_overflow_uid(void) { +#ifdef CBM_ENABLE_TEST_SEAMS + g_posix_overflow_compute_count++; +#endif char map[256]; if (!posix_read_small_proc_file("/proc/self/uid_map", map, sizeof(map)) || !posix_uid_map_is_single_uid(map, geteuid())) { @@ -1564,25 +1589,24 @@ static uid_t posix_compute_ancestor_overflow_uid(void) { return (uid_t)overflow; } -static uid_t g_posix_overflow_cached = POSIX_NO_OVERFLOW_UID; -static pthread_once_t g_posix_overflow_once = PTHREAD_ONCE_INIT; -static void posix_overflow_init_once(void) { - g_posix_overflow_cached = posix_compute_ancestor_overflow_uid(); -} #endif /* __linux__ */ -/* The overflow uid tolerated for ancestors in this process, or - * POSIX_NO_OVERFLOW_UID when none. +/* The overflow uid tolerated for ancestors, or POSIX_NO_OVERFLOW_UID when none. * - * Derived once per process. NOT from immutable state, whatever an earlier - * revision of this comment claimed: unshare(CLONE_NEWUSER) rewrites + * DERIVED FRESH ON EVERY CALL, deliberately. This used to memoise via + * pthread_once behind a comment claiming "immutable /proc state". That claim + * was false in both halves: unshare(CLONE_NEWUSER) rewrites * /proc/self/uid_map, and pthread_once state survives fork() already marked - * done. A child that forks WITHOUT exec and then changes namespace therefore - * keeps the parent answer. That is safe here only because every caller runs in - * a freshly exec'd process -- the daemon double-fork execs - * (daemon/bootstrap.c), workers go through cbm_subprocess_spawn's fork+exec, - * and src/ contains no setns/unshare at all. If you ever add a fork-without-exec - * path that reaches this check, this cache is WRONG and must go. */ + * done -- so a process that forked and then changed namespace kept the parent + * answer and refused a directory it should have accepted. It happened to be + * harmless because every caller today runs in a freshly exec'd process, but + * that made a security decision depend on an invariant nothing enforced, and + * the next fork-without-exec caller would have silently inherited a stale + * verdict. + * + * The cost of not caching is two small /proc reads per ancestor check, against + * an openat + fstat + fchmod + ACL check per path component in the same walk. + * Do not re-introduce a cache here; the contract test counts derivations. */ static uid_t posix_ancestor_overflow_uid(void) { #ifdef CBM_ENABLE_TEST_SEAMS if (g_posix_overflow_override_active) { @@ -1590,8 +1614,7 @@ static uid_t posix_ancestor_overflow_uid(void) { } #endif #if defined(__linux__) - (void)pthread_once(&g_posix_overflow_once, posix_overflow_init_once); - return g_posix_overflow_cached; + return posix_compute_ancestor_overflow_uid(); #else return POSIX_NO_OVERFLOW_UID; #endif diff --git a/src/daemon/ipc.h b/src/daemon/ipc.h index 9bd8d8a54..545ecb357 100644 --- a/src/daemon/ipc.h +++ b/src/daemon/ipc.h @@ -131,6 +131,12 @@ bool cbm_daemon_ipc_posix_uid_map_is_single_uid_for_test(const char *uid_map, un bool cbm_daemon_ipc_posix_ancestor_stat_ok_for_test(unsigned long owner, unsigned int mode, unsigned long euid, bool overflow_active, unsigned long overflow_uid); +#if defined(__linux__) +/* Number of REAL overflow-uid derivations so far. The value is meaningless on + * its own; the point is that it must rise on every ancestor check, proving no + * cache has crept back in. */ +unsigned cbm_daemon_ipc_posix_overflow_compute_count_for_test(void); +#endif #endif #endif diff --git a/tests/test_daemon_ipc.c b/tests/test_daemon_ipc.c index 2da597add..b451f9bb3 100644 --- a/tests/test_daemon_ipc.c +++ b/tests/test_daemon_ipc.c @@ -5124,6 +5124,49 @@ TEST(daemon_ipc_posix_overflow_ancestor_tolerated_only_in_single_uid_ns_issue183 PASS(); } +/* #1830 regression guard, and the test that would have caught the original bug + * with no namespace at all. The overflow uid used to be memoised per process + * via pthread_once. That made a SECURITY decision sticky: pthread_once state + * survives fork(), unshare(CLONE_NEWUSER) rewrites /proc/self/uid_map, so a + * process that forked and then entered a namespace kept the parent verdict. + * Nothing in the suite could see it, because every deterministic #1830 test + * drives the override seam rather than the real derivation -- they bind the + * decision TABLE, not the WIRING. + * + * So assert the wiring directly: two ancestor checks must perform two real + * derivations. Re-introduce any cache and the second call derives zero times + * and this fails by name, on every platform, with no namespace required. */ +TEST(daemon_ipc_posix_overflow_uid_is_never_cached_issue1830) { +#if defined(__linux__) + char parent[TEST_PATH_CAP]; + if (!ipc_test_parent_new(parent, "overflow-nocache")) { + FAIL("could not create the probe parent directory"); + } + char probe[TEST_PATH_CAP]; + (void)snprintf(probe, sizeof(probe), "%s/x", parent); + + /* The override seam short-circuits derivation, so it must be OFF here or + * the test would measure nothing. */ + cbm_daemon_ipc_posix_set_ancestor_overflow_uid_for_test(false, 0); + + unsigned before = cbm_daemon_ipc_posix_overflow_compute_count_for_test(); + (void)cbm_daemon_ipc_private_directory_secure(probe); + unsigned after_first = cbm_daemon_ipc_posix_overflow_compute_count_for_test(); + (void)cbm_daemon_ipc_private_directory_secure(probe); + unsigned after_second = cbm_daemon_ipc_posix_overflow_compute_count_for_test(); + + (void)rmdir(probe); + ipc_test_remove_flat_dir(parent); + + ASSERT_TRUE(after_first > before); + /* The decisive half: the SECOND call must derive again. */ + ASSERT_TRUE(after_second > after_first); + PASS(); +#else + SKIP_PLATFORM("the overflow-uid derivation is Linux-only"); +#endif +} + /* #1830 real end-to-end smoke: inside a single-uid user namespace the * root-owned ancestors (/, /tmp) are overflow-owned, and the daemon must still * create its private directory there. An overflow-owned ancestor cannot be @@ -5131,7 +5174,12 @@ TEST(daemon_ipc_posix_overflow_ancestor_tolerated_only_in_single_uid_ns_issue183 * namespace is actually available. O10 whitelisted-skip everywhere it is not: * macOS has no user namespaces (compile-gated out); Docker's default seccomp * blocks unshare(CLONE_NEWUSER) on the Colima container leg; some kernels ship - * user namespaces disabled. WHAT WAS TRIED when it skips: fork + unshare + * user namespaces disabled. NOT on that list any more: Ubuntu 23.10+ restricts + * unprivileged userns via AppArmor, which used to confine this test to the two + * ubuntu-22.04 legs -- broad-matrix only, so it ran in no PR and sat on + * GitHub's 2027-04-17 retirement clock. _test.yml now lifts that with + * kernel.apparmor_restrict_unprivileged_userns=0 on every ubuntu leg, so this + * runs on the CORE matrix. WHAT WAS TRIED when it skips: fork + unshare * CLONE_NEWUSER + a 1:1 uid_map write, which the sandbox denied (EPERM). The * deterministic decision coverage above is what binds the fix. * @@ -5346,6 +5394,7 @@ SUITE(daemon_ipc) { #ifdef CBM_ENABLE_TEST_SEAMS RUN_TEST(daemon_ipc_posix_uid_map_single_uid_parse_issue1830); RUN_TEST(daemon_ipc_posix_overflow_ancestor_tolerated_only_in_single_uid_ns_issue1830); + RUN_TEST(daemon_ipc_posix_overflow_uid_is_never_cached_issue1830); RUN_TEST(daemon_ipc_posix_single_uid_userns_real_smoke_issue1830); #endif RUN_TEST(daemon_ipc_posix_startup_lock_is_cross_process); From 085017ab2172c2f3d4cd3e27251033ecbc5b814b Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 13 Sep 2026 10:31:45 +0200 Subject: [PATCH 064/130] fix(daemon): reword the overflow-uid note so the security audit stops blocking it scripts/security-audit.sh greps src/ for the fork call literal and blocks any file that is not on the allow-list. The note explaining why the overflow uid is no longer cached quoted that literal while describing the hazard, so the audit refused src/daemon/ipc.c -- on a COMMENT, in a file that performs no such call. Reworded rather than allow-listed. An allow-list entry would have silenced the check for the whole file, permanently, so that one sentence could keep its punctuation; the scanner is right to be blunt here and the comment is the cheaper thing to change. The reason is recorded inline so the next person does not reintroduce it. Signed-off-by: Martin Vogel --- src/daemon/ipc.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/daemon/ipc.c b/src/daemon/ipc.c index f8d7d70b4..a92befb08 100644 --- a/src/daemon/ipc.c +++ b/src/daemon/ipc.c @@ -1596,9 +1596,12 @@ static uid_t posix_compute_ancestor_overflow_uid(void) { * DERIVED FRESH ON EVERY CALL, deliberately. This used to memoise via * pthread_once behind a comment claiming "immutable /proc state". That claim * was false in both halves: unshare(CLONE_NEWUSER) rewrites - * /proc/self/uid_map, and pthread_once state survives fork() already marked - * done -- so a process that forked and then changed namespace kept the parent - * answer and refused a directory it should have accepted. It happened to be + * /proc/self/uid_map, and pthread_once state survives a forked child already + * marked done -- so a process that forked and then changed namespace kept the + * parent answer and refused a directory it should have accepted. + * (Spelled without the call syntax on purpose: scripts/security-audit.sh + * blocks that literal in src/, and an allow-list entry to let a COMMENT pass + * would weaken a real check on a real file.) It happened to be * harmless because every caller today runs in a freshly exec'd process, but * that made a security decision depend on an invariant nothing enforced, and * the next fork-without-exec caller would have silently inherited a stale From ac8f5b8a8137f15e95478be71ee6742e928ba5bc Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 13 Sep 2026 10:35:41 +0200 Subject: [PATCH 065/130] ci(test): report whether unprivileged userns is actually available The sysctl that lifts the AppArmor restriction is best-effort, and the test runner prints per-shard aggregates only -- 'N passed, M skipped', never individual test names. So if the sysctl stopped working, the #1830 namespace test would quietly return to skipping and the log would stay green and look identical. That is the same false assurance that let a security relaxation ship behind a test which never executed. Probe it the way the test does (unprivileged, no sudo) and print the answer, so there is one greppable line of evidence per ubuntu leg. Deliberately not a gate: a runner that cannot lift the restriction should still run everything else. Signed-off-by: Martin Vogel --- .github/workflows/_test.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 893cc45e3..7ccd56bf5 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -121,6 +121,17 @@ jobs: run: | sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || \ echo "could not lift the userns restriction; namespace tests will skip" + # Then SAY whether it worked. The runner prints per-shard aggregates + # ("N passed, M skipped") and never individual test names, so a + # namespace test that silently returns to skipping is invisible in a + # green log -- the same false assurance that let #1830 ship with a + # test that never executed. This line is the evidence. Probed exactly + # as the test does it: unprivileged, no sudo. + if unshare -U -r true 2>/dev/null; then + echo "CBM_USERNS_AVAILABLE=yes" + else + echo "CBM_USERNS_AVAILABLE=no (namespace-dependent tests will skip)" + fi - name: Install ccache (macOS) if: startsWith(matrix.os, 'macos') From 81a8a03c820cb8bccafd0ebac355eede21b5e8bf Mon Sep 17 00:00:00 2001 From: Anton Standrik Date: Sun, 13 Sep 2026 13:52:00 +0300 Subject: [PATCH 066/130] fix(test): isolate soak harness daemon runtime scripts/soak-test.sh claimed to isolate daemon coordination from interactive sessions through a private CBM_CACHE_DIR, but only CBM_RUNTIME_DIR moves the daemon rendezvous, so the soak shared the operator's account daemon and asserted that its own shutdown stopped it. Source scripts/test-runtime.sh for a harness-owned runtime and cache, and let the helper stop the private daemon before the root is removed. Add tests/test_soak_runtime_isolation_contract.sh, which fails before this change, and wire it into scripts/test.sh. Part of #1696. Signed-off-by: Anton Standrik --- scripts/soak-test.sh | 27 ++++-- scripts/test.sh | 3 + tests/test_soak_runtime_isolation_contract.sh | 82 +++++++++++++++++++ 3 files changed, 105 insertions(+), 7 deletions(-) create mode 100755 tests/test_soak_runtime_isolation_contract.sh diff --git a/scripts/soak-test.sh b/scripts/soak-test.sh index a12029a30..31a16259f 100755 --- a/scripts/soak-test.sh +++ b/scripts/soak-test.sh @@ -66,9 +66,17 @@ CBM_SOAK_MODE="${CBM_SOAK_MODE:-default}" RESULTS_DIR="${RESULTS_DIR:-soak-results}" mkdir -p "$RESULTS_DIR" -# Isolate daemon coordination from interactive CBM sessions and give this run -# a deterministic host-side daemon log. Wine needs a Windows-form cache path -# in the child environment while this Bash harness retains the host path. +# Every product process below must reach a daemon rendezvous this run owns. +# Only CBM_RUNTIME_DIR moves that rendezvous — a private CBM_CACHE_DIR alone +# still shares the operator's account daemon (#1691, #1696). +# shellcheck source=test-runtime.sh +source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh" +cbm_test_runtime_init +trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT + +# Give this run a deterministic host-side daemon log. Wine needs a Windows-form +# cache path in the child environment while this Bash harness retains the host +# path. # # On native Windows the cache CANNOT live under msys /tmp: the server's # cache-private executable-identity check walks the cache path's ancestors @@ -146,7 +154,7 @@ if [[ "$BINARY" == *.exe ]] && command -v cygpath >/dev/null 2>&1 && exit 1 fi else - SOAK_CACHE_DIR_HOST=$(mktemp -d "${TMPDIR:-/tmp}/cbm-soak-cache.XXXXXX") + SOAK_CACHE_DIR_HOST="$CBM_TEST_CACHE_DIR_HOST" fi SOAK_CACHE_DIR_VALUE="$SOAK_CACHE_DIR_HOST" if [[ "$BINARY" == *.exe ]] && command -v winepath >/dev/null 2>&1; then @@ -206,7 +214,12 @@ soak_cleanup() { if [ -f "$DAEMON_LOG" ]; then cp "$DAEMON_LOG" "$RESULTS_DIR/cbm-daemon.log" 2>/dev/null || true fi - rm -rf -- "$SOAK_PROJECT" "$SOAK_CACHE_DIR_HOST" + rm -rf -- "$SOAK_PROJECT" + # The helper stops this run's private daemon before removing its root and + # leaves the root behind for diagnosis when the daemon will not stop. The + # native-Windows cache sits under SOAK_WIN_ROOT, which goes only after the + # daemon check because the binary it probes with is the copy inside it. + cbm_test_runtime_cleanup "$BINARY" [ -z "${SOAK_WIN_ROOT:-}" ] || rm -rf -- "$SOAK_WIN_ROOT" } @@ -807,7 +820,7 @@ if [ "$SKIP_CRASH" != "--skip-crash-test" ] && [ "$CBM_SOAK_MODE" != "query-leak SERVER_PID="" exec 3>&- 4<&- if ! wait_for_daemon_stop "$DAEMON_STOP_COUNT"; then - echo "FAIL: last-session crash did not stop the shared daemon" + echo "FAIL: last-session crash did not stop the private daemon" exit 1 fi @@ -848,7 +861,7 @@ kill "$SERVER_PID" 2>/dev/null || true wait "$SERVER_PID" 2>/dev/null || true SERVER_PID="" if ! wait_for_daemon_stop "$FINAL_DAEMON_STOP_COUNT"; then - echo "FAIL: final frontend shutdown did not stop the shared daemon" + echo "FAIL: final frontend shutdown did not stop the private daemon" PASS=false fi diff --git a/scripts/test.sh b/scripts/test.sh index ecfc26998..7263dd616 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -212,6 +212,9 @@ bash "$ROOT/tests/test_ui_dev_proxy_security.sh" echo "=== Step 0d: daemon soak recovery contract ===" bash "$ROOT/tests/test_soak_daemon_recovery_contract.sh" +echo "=== Step 0d2: soak harness runtime isolation contract (#1696) ===" +bash "$ROOT/tests/test_soak_runtime_isolation_contract.sh" + echo "=== Step 0e: Windows launcher bundle contract ===" bash "$ROOT/tests/test_windows_bundle_contract.sh" diff --git a/tests/test_soak_runtime_isolation_contract.sh b/tests/test_soak_runtime_isolation_contract.sh new file mode 100755 index 000000000..340df7f0a --- /dev/null +++ b/tests/test_soak_runtime_isolation_contract.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Runtime-isolation contract for the soak harness (#1696, follow-up to #1691). +# +# scripts/soak-test.sh must be the only client of the daemon it measures: it +# asserts that a session crash and the final shutdown each stop "the daemon". +# It used to claim isolation from interactive sessions through a private +# CBM_CACHE_DIR, but only CBM_RUNTIME_DIR moves the daemon rendezvous +# (docs/CONFIGURATION.md), so the soak shared the operator's account daemon +# and either stopped it or was refused with a cache-root conflict. Drive the +# harness with an environment-probe fixture and require that no product +# process ever receives the caller's runtime or cache. + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +normalize_path() { + local path=${1%$'\r'} + if command -v cygpath >/dev/null 2>&1; then + cygpath -u "$path" 2>/dev/null && return 0 + fi + printf '%s\n' "${path//\\//}" +} + +ENV_PROBE="$WORKDIR/environment-probe" +cat > "$ENV_PROBE" <<'EOF' +#!/usr/bin/env bash +printf '%s\t%s\n' "${CBM_CACHE_DIR-}" "${CBM_RUNTIME_DIR-}" >> "$CBM_SOAK_ENV_PROBE" +[[ "${1-} ${2-}" == "daemon status" ]] && exit 1 +exit 0 +EOF +chmod +x "$ENV_PROBE" + +CALLER_CACHE="$WORKDIR/caller-cache" +CALLER_RUNTIME="$WORKDIR/caller-runtime" +ENV_LOG="$WORKDIR/environment.log" +mkdir -p "$CALLER_CACHE" "$CALLER_RUNTIME" + +# The fixture exits as soon as it has recorded its environment, so the soak +# fails at "server did not start"; only the environment it handed to the +# product is under test here. RESULTS_DIR keeps the soak's metrics out of cwd. +CBM_CACHE_DIR="$CALLER_CACHE" \ +CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ +CBM_SOAK_ENV_PROBE="$ENV_LOG" \ +RESULTS_DIR="$WORKDIR/results" \ + "$ROOT/scripts/soak-test.sh" "$ENV_PROBE" 1 --skip-crash-test \ + > "$WORKDIR/soak.out" 2>&1 || true + +[[ -s "$ENV_LOG" ]] || fail "soak-test did not execute the environment-probe fixture" + +CALLER_CACHE_NORMALIZED=$(normalize_path "$CALLER_CACHE") +CALLER_RUNTIME_NORMALIZED=$(normalize_path "$CALLER_RUNTIME") +private_root="" +while IFS=$'\t' read -r child_cache_raw child_runtime_raw; do + child_cache=$(normalize_path "$child_cache_raw") + child_runtime=$(normalize_path "$child_runtime_raw") + if [[ -z "$child_runtime" || "$child_runtime" == "$CALLER_RUNTIME_NORMALIZED" ]]; then + fail "soak-test exposed the caller CBM_RUNTIME_DIR to a product process" + fi + if [[ -z "$child_cache" || "$child_cache" == "$CALLER_CACHE_NORMALIZED" ]]; then + fail "soak-test exposed the caller CBM_CACHE_DIR to a product process" + fi + if [[ "${child_runtime%/*}" != "${child_cache%/*}" || + "${child_runtime##*/}" != "runtime" || "${child_cache##*/}" != "cache" ]]; then + fail "soak runtime/cache were not isolated beneath one private root" + fi + if [[ -n "$private_root" && "$private_root" != "${child_runtime%/*}" ]]; then + fail "soak-test switched private roots mid-run" + fi + private_root="${child_runtime%/*}" +done < "$ENV_LOG" + +[[ ! -e "$private_root" ]] || fail "soak-test left its private root behind: $private_root" + +echo "PASS: soak harness isolates its daemon runtime and cache from the caller" From 823e720982e1bce047a9243bd9b24d9688dadb72 Mon Sep 17 00:00:00 2001 From: Anton Standrik Date: Sun, 13 Sep 2026 13:52:08 +0300 Subject: [PATCH 067/130] fix(test): isolate memlab harness daemon runtime scripts/memlab.sh gave the profiled run a private CBM_CACHE_DIR only, so the process joined the operator's account daemon and cleanup removed the cache without stopping any daemon. Source scripts/test-runtime.sh for a harness-owned runtime and cache, and stop the private daemon before the work directory is removed. Add tests/test_memlab_runtime_isolation_contract.sh, which fails before this change, and wire it into scripts/test.sh. Part of #1696. Signed-off-by: Anton Standrik --- scripts/memlab.sh | 28 +++++- scripts/test.sh | 3 + .../test_memlab_runtime_isolation_contract.sh | 92 +++++++++++++++++++ 3 files changed, 118 insertions(+), 5 deletions(-) create mode 100755 tests/test_memlab_runtime_isolation_contract.sh diff --git a/scripts/memlab.sh b/scripts/memlab.sh index 373742556..4e441723a 100644 --- a/scripts/memlab.sh +++ b/scripts/memlab.sh @@ -24,6 +24,16 @@ if [ ! -x "$BINARY" ]; then exit 2 fi +# The profiled process must reach a daemon rendezvous and cache this run owns: +# only CBM_RUNTIME_DIR moves the rendezvous, so a private CBM_CACHE_DIR alone +# joined the operator's account daemon (#1691, #1696). The helper also stops +# that daemon before its cache is removed and leaves the root for diagnosis +# when it will not stop. +# shellcheck source=test-runtime.sh +source "$(dirname "$0")/test-runtime.sh" +cbm_test_runtime_init || exit 1 +trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT + WORK=$(mktemp -d 2>/dev/null || mktemp -d -t memlab) # Native Windows: the server walks the full ancestor chain of both the binary @@ -75,7 +85,12 @@ PROFILE_OUT="$PWD/memlab-${LABEL}.jsonl" RUN_LOG="$PWD/memlab-${LABEL}.log" rm -f "$PROFILE_OUT" "$RUN_LOG" -cleanup() { rm -rf "$WORK" 2>/dev/null || true; rm -rf "${WIN_ROOT:-}" 2>/dev/null || true; } +cleanup() { + # The helper probes with $BINARY, whose native-Windows copy lives in $WORK. + cbm_test_runtime_cleanup "$BINARY" + rm -rf "$WORK" 2>/dev/null || true + rm -rf "${WIN_ROOT:-}" 2>/dev/null || true +} trap cleanup EXIT # A fixed corpus: same file count and content on every platform, so a @@ -108,12 +123,15 @@ done echo "=== memlab: binary=$BINARY requests=$REQUESTS label=$LABEL ===" echo "corpus: $(find "$CORPUS" -name '*.py' | wc -l | tr -d ' ') files" -# The Windows binary needs a native path here; an msys /c/... path is not one. -if command -v cygpath >/dev/null 2>&1 && ! command -v winepath >/dev/null 2>&1; then +# The native Windows binary needs a native path here, under its stamped root; +# an msys /c/... path is not one. Everywhere else the helper's owner-only cache +# (already exported as CBM_CACHE_DIR) is the cache. +if [ -n "${WIN_ROOT:-}" ]; then mkdir -p "$WORK/cache" export CBM_CACHE_DIR="$(cygpath -w "$WORK/cache")" + CACHE_HOST="$WORK/cache" else - export CBM_CACHE_DIR="$WORK/cache" + CACHE_HOST="$CBM_TEST_CACHE_DIR_HOST" fi export CBM_MEM_PROFILE=1 export CBM_MEM_PROFILE_OUT="$PROFILE_OUT" @@ -139,7 +157,7 @@ python3 "$(dirname "$0")/memlab-drive.py" "$DRIVE_BINARY" "$DRIVE_CORPUS" "$REQU RC=$? # With CBM_CACHE_DIR set the process logs to its own file rather than stderr, # so fold that in or the census series is invisible. -cat "$WORK"/cache/logs/*.log >> "$RUN_LOG" 2>/dev/null || true +cat "$CACHE_HOST"/logs/*.log >> "$RUN_LOG" 2>/dev/null || true cat "$WORK/server-stderr.log" >> "$RUN_LOG" 2>/dev/null || true RESPONSES=$(sed -n "s/.*served=\\([0-9]*\\).*/\\1/p" "$WORK/drive.out" | head -1); RESPONSES=${RESPONSES:-0} CENSUS=$(grep -c "mem.census" "$RUN_LOG" 2>/dev/null | head -1); CENSUS=${CENSUS:-0} diff --git a/scripts/test.sh b/scripts/test.sh index ecfc26998..24902b39a 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -227,6 +227,9 @@ bash "$ROOT/tests/test_makefile_ts_runtime_dependencies.sh" echo "=== Step 0g: security fuzz harness self-test ===" bash "$ROOT/tests/test_security_fuzz_harness.sh" +echo "=== Step 0g2: memlab harness runtime isolation contract (#1696) ===" +bash "$ROOT/tests/test_memlab_runtime_isolation_contract.sh" + echo "=== Step 0h: smoke release-fixture contract ===" bash "$ROOT/tests/test_smoke_fixture_contract.sh" diff --git a/tests/test_memlab_runtime_isolation_contract.sh b/tests/test_memlab_runtime_isolation_contract.sh new file mode 100755 index 000000000..f414c41c5 --- /dev/null +++ b/tests/test_memlab_runtime_isolation_contract.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Runtime-isolation contract for the memlab harness (#1696, follow-up to #1691). +# +# scripts/memlab.sh starts the product over stdio to attribute retained memory +# and then removes its work directory. It used to give the run a private +# CBM_CACHE_DIR only, but only CBM_RUNTIME_DIR moves the daemon rendezvous +# (docs/CONFIGURATION.md), so the profiled process joined the operator's +# account daemon — refused with a cache-root conflict when one was live, or +# left as the account daemon with its cache deleted underneath it otherwise. +# +# memlab launches the product through its Python driver, and the driver hands +# the product the environment it inherited (memlab-drive.py passes no env= to +# Popen). Recording the environment at the driver boundary therefore observes +# exactly what the product receives, on every host — including Windows, whose +# native Python cannot exec a shell fixture — so a python3 shim stands in for +# the driver and no product process is started at all. + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +normalize_path() { + local path=${1%$'\r'} + if command -v cygpath >/dev/null 2>&1; then + cygpath -u "$path" 2>/dev/null && return 0 + fi + printf '%s\n' "${path//\\//}" +} + +if grep -q 'env=' "$ROOT/scripts/memlab-drive.py"; then + fail "memlab-drive.py no longer passes the harness environment through unchanged" +fi + +SHIM_DIR="$WORKDIR/bin" +mkdir -p "$SHIM_DIR" +cat > "$SHIM_DIR/python3" <<'EOF' +#!/usr/bin/env bash +printf '%s\t%s\n' "${CBM_CACHE_DIR-}" "${CBM_RUNTIME_DIR-}" >> "$CBM_MEMLAB_ENV_PROBE" +echo "served=0 failed=0" +exit 1 +EOF +DUMMY_BINARY="$WORKDIR/dummy-binary" +printf '#!/usr/bin/env bash\nexit 0\n' > "$DUMMY_BINARY" +chmod +x "$SHIM_DIR/python3" "$DUMMY_BINARY" + +CALLER_CACHE="$WORKDIR/caller-cache" +CALLER_RUNTIME="$WORKDIR/caller-runtime" +ENV_LOG="$WORKDIR/environment.log" +mkdir -p "$CALLER_CACHE" "$CALLER_RUNTIME" "$WORKDIR/cwd" + +# memlab writes its profile and log into $PWD, hence the cwd change. It is +# tracked without an executable bit, so run it through bash. +( + cd "$WORKDIR/cwd" + PATH="$SHIM_DIR:$PATH" \ + CBM_CACHE_DIR="$CALLER_CACHE" \ + CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ + CBM_MEMLAB_ENV_PROBE="$ENV_LOG" \ + bash "$ROOT/scripts/memlab.sh" "$DUMMY_BINARY" 1 probe > "$WORKDIR/memlab.out" 2>&1 || true +) + +[[ -s "$ENV_LOG" ]] || fail "memlab did not reach its driver" + +CALLER_CACHE_NORMALIZED=$(normalize_path "$CALLER_CACHE") +CALLER_RUNTIME_NORMALIZED=$(normalize_path "$CALLER_RUNTIME") +private_root="" +while IFS=$'\t' read -r child_cache_raw child_runtime_raw; do + child_cache=$(normalize_path "$child_cache_raw") + child_runtime=$(normalize_path "$child_runtime_raw") + if [[ -z "$child_runtime" || "$child_runtime" == "$CALLER_RUNTIME_NORMALIZED" ]]; then + fail "memlab exposed the caller CBM_RUNTIME_DIR to the product" + fi + if [[ -z "$child_cache" || "$child_cache" == "$CALLER_CACHE_NORMALIZED" ]]; then + fail "memlab exposed the caller CBM_CACHE_DIR to the product" + fi + if [[ "${child_runtime%/*}" != "${child_cache%/*}" || + "${child_runtime##*/}" != "runtime" || "${child_cache##*/}" != "cache" ]]; then + fail "memlab runtime/cache were not isolated beneath one private root" + fi + private_root="${child_runtime%/*}" +done < "$ENV_LOG" + +[[ ! -e "$private_root" ]] || fail "memlab left its private root behind: $private_root" + +echo "PASS: memlab harness isolates its daemon runtime and cache from the caller" From da3258f3c7e4bdad6ec23f9ba194729364bb1254 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 13 Sep 2026 16:45:33 +0200 Subject: [PATCH 068/130] feat(mem): a central allocation core, a linter that enforces it, and phase attribution in the index pipeline Memory in this project is now allocated through ONE core. This lands the core, the gate that keeps new code on it, and the instrumentation that lets an index run say which class of memory grew in which pass -- the three things the 2026-09-13 audit found missing. WHAT THE AUDIT FOUND There was no centralized allocation layer. foundation/mem.h was policy and measurement only (budget, RSS, over_budget, memory map, phase marks) with no alloc/free wrapper, so the budget could be observed after the fact and never enforced. Raw allocator calls in the tree: 4086 across 92 files. The extraction engine had already adopted arenas (1301 call sites); the graph buffer -- which holds the 35 GB peak of a kernel index -- used none, allocating every 64 B node, 48 B edge and every name/qualified_name/ properties_json individually. Two further facts made a self-accounting core the only workable design: 1. cbm_mem_map_collect() walks THIS thread's mimalloc heap only. Walking the process-wide heap is a data race TSan caught on macOS, so it is deliberately not done, and an 18-worker index attributes almost nothing -- the rest lands in residual. 2. The mimalloc global override is ON for Linux/MinGW and permanently OFF for macOS (two-level namespace: this binary's free becomes mi_free while system libraries keep allocating from the system zone, and a pointer crossing that boundary aborts). On macOS ordinary malloc is served by the system allocator; the startup audit reports owned_classes=0/6. Any accounting that assumes mimalloc owns the pointer is blind on an entire platform. THE CORE (src/foundation/mem_core.{h,c}) cbm_alloc / cbm_calloc / cbm_realloc / cbm_mem_strdup / cbm_free, each tagged with an allocation class (gbuf_node, gbuf_edge, gbuf_string, gbuf_index, extract, semantic, dump, store, other). Per-class live bytes, live blocks and peak, kept in atomics -- thread-safe by construction, identical on every platform, independent of which allocator serves malloc. libc semantics are preserved exactly (alloc(0) is freeable, free(NULL) is a no-op, realloc(NULL) allocates, a failed realloc leaves the block intact) so adoption is a mechanical rename and never a behaviour change. No per-allocation header: a {class,size} prefix costs 16 bytes on ~100M blocks at kernel scale, 1.6 GB of overhead to measure a memory problem. Sizes come from the platform usable-size query (malloc_size / malloc_usable_size / _msize), which is correct under either allocator and free. Where no query exists (BSD) the request size is charged, which understates -- the safe direction for a diagnostic. A mismatched class on free clamps at zero rather than wrapping, so a small drift can never masquerade as a colossal leak. Arena-backed allocators report in bulk through cbm_mem_class_add_external / remove_external so their memory appears in the same table; the extraction engine must not be rewritten to per-object allocation, that would undo the batching that keeps its allocation count low. THE LINTER (scripts/lint-memory-core.py, wired into make lint-ci) Raw malloc/calloc/realloc/free/strdup/strndup outside the core is a defect. 4086 sites cannot migrate at once, so the gate is a ratchet: a checked-in baseline (scripts/memory-core-baseline.txt) records each file's count and the build goes red when any file grows, or a new file appears with any. Files only ever go down; the baseline line is lowered in the change that migrates the file, and --strict fails a stale baseline. Comments and string literals are stripped first, so prose that mentions malloc( cannot trip it -- the security audit already bit on exactly that with fork(. Proven both ways: one appended malloc turns it red naming the file (83 -> 85), restoring turns it green. Scans src/ and internal/cbm/: cli, mcp, daemon, store, pipeline. PHASE ATTRIBUTION (src/pipeline/pipeline.c) cbm_mem_phase_mark and the new cbm_mem_class_log now fire at pipeline.begin and at every pass.timing site: parallel_extract, the six sequential passes, the eight predump passes, tests and dump_and_persist, with peaks reset per index. Both instruments existed in foundation/ and were wired only into MCP request handling, never into the index pipeline -- which is where the memory is. Locating the kernel peak this week required an external RSS sampler because nothing in-process could say which phase it was in. PRESSURE PRIMITIVES (system_info.c, mem.{c,h}, platform.h) cbm_system_available_ram() -- macOS host_statistics64 (free + inactive + purgeable), Linux MemAvailable, Windows GlobalMemoryStatusEx; 0 when unknown, never cached -- and cbm_mem_system_under_pressure(), true below 12.5% of RAM available and false when unknown so nothing ever aborts on a guess. Landed UNWIRED: an earlier draft used them to let the over-budget latch press on whenever the system was not under pressure, and that permitted the very overshoot this work exists to end (Decision A's tests at test_pipeline.c:13481 and :13583 went red under it, correctly). They are the backstop for the next step, admission control, which keeps a run under budget before an allocation rather than measuring it after. mem suite 10 new tests green; pipeline + mem 339 passed 0 failed; lint-ci clean including the new gate. Signed-off-by: Martin Vogel --- Makefile.cbm | 12 +- scripts/lint-memory-core.py | 222 ++++++++++++++++++++++++ scripts/memory-core-baseline.txt | 97 +++++++++++ src/foundation/mem.c | 16 ++ src/foundation/mem.h | 15 ++ src/foundation/mem_core.c | 289 +++++++++++++++++++++++++++++++ src/foundation/mem_core.h | 140 +++++++++++++++ src/foundation/platform.h | 4 + src/foundation/system_info.c | 62 +++++++ src/pipeline/pipeline.c | 29 ++++ tests/test_mem.c | 168 ++++++++++++++++++ 11 files changed, 1053 insertions(+), 1 deletion(-) create mode 100644 scripts/lint-memory-core.py create mode 100644 scripts/memory-core-baseline.txt create mode 100644 src/foundation/mem_core.c create mode 100644 src/foundation/mem_core.h diff --git a/Makefile.cbm b/Makefile.cbm index 2c2aeb193..92c31031f 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -248,6 +248,7 @@ FOUNDATION_SRCS = \ src/foundation/compat_fs.c \ src/foundation/compat_regex.c \ src/foundation/mem.c \ + src/foundation/mem_core.c \ src/foundation/diagnostics.c \ src/foundation/profile.c \ src/foundation/dump_verify.c \ @@ -1188,8 +1189,17 @@ lint-no-suppress: lint: lint-tidy lint-cppcheck lint-format lint-no-suppress @echo "=== All linters passed ===" +# Memory-core linter: memory is allocated through src/foundation/mem_core.h, +# not through raw malloc/calloc/realloc/free/strdup. A checked-in baseline +# (scripts/memory-core-baseline.txt) records each file's raw-site count and the +# gate goes red the moment any file grows. Files only ever go down; lower the +# baseline line in the same change that migrates the file. +lint-memory-core: + @echo "=== memory-core linter ===" + @python3 scripts/lint-memory-core.py + # CI linters (no clang-tidy — platform-dependent, enforced locally via pre-commit) -lint-ci: lint-cppcheck lint-format lint-no-suppress +lint-ci: lint-cppcheck lint-format lint-no-suppress lint-memory-core @echo "=== CI linters passed ===" # ── Local memory-diagnostic lanes (not PR-CI gates by decision: the diag diff --git a/scripts/lint-memory-core.py b/scripts/lint-memory-core.py new file mode 100644 index 000000000..a8f714ca0 --- /dev/null +++ b/scripts/lint-memory-core.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +""" +lint-memory-core.py — the memory-core linter. + +Memory in this project is allocated through ONE core (src/foundation/mem_core.h), +not through scattered raw malloc/calloc/realloc/free/strdup. This linter is the +gate that keeps that true. + +WHY A RATCHET, NOT A BAN. An audit on 2026-09-13 found ~800 raw allocation +sites in src/ and none of them can be migrated in one change. So the gate works +like every other honest debt gate: a checked-in baseline records how many raw +sites each file has TODAY, and the build goes red the moment any file has MORE +than its baseline, or a file not in the baseline grows one. Files can only ever +go down. When a file is migrated, its baseline line is lowered (or removed) in +the same change -- and --strict turns "below baseline" into a failure too, so +the baseline cannot silently rot behind the code. + +WHAT COUNTS AS RAW. A call to malloc/calloc/realloc/free/strdup/strndup that is +not part of a longer identifier. cbm_alloc, cbm_free, cbm_calloc, mi_malloc, +cbm_arena_alloc and heap_strdup are all NOT matches: the character before the +name is an identifier character. Comments and string literals are stripped +first, so prose that mentions malloc( does not trip the gate -- the security +audit already bit us once on exactly that with fork(. + +EXEMPT. The core itself and the allocator plumbing it sits on: + src/foundation/mem_core.c the route + src/foundation/arena.c bulk allocator that reports INTO the core + src/foundation/slab_alloc.c same + src/foundation/mem.c policy/measurement, probes with malloc + src/foundation/mem_override_*.c the --wrap / override shims + src/foundation/compat*.c libc replacement surface + internal/cbm/vendored/** not ours + vendored/** not ours + +Usage: + lint-memory-core.py check against the baseline (CI) + lint-memory-core.py --strict also fail when a file is BELOW baseline + lint-memory-core.py --write-baseline regenerate scripts/memory-core-baseline.txt + lint-memory-core.py --list print every raw site (file:line: call) + +Exit 0 = clean. Exit 1 = a file grew. Exit 2 = usage/IO error. +""" +from __future__ import annotations + +import argparse +import pathlib +import re +import sys + +ROOT = pathlib.Path(__file__).resolve().parent.parent +BASELINE = ROOT / "scripts" / "memory-core-baseline.txt" +SCAN_ROOTS = ("src", "internal/cbm") +RAW = re.compile(r"(? bool: + return rel in EXEMPT_EXACT or rel.startswith(EXEMPT_PREFIX) + + +def strip_comments_and_strings(text: str) -> str: + """Blank out comments and string literals, preserving line structure so + reported line numbers stay right. Character-class aware enough for C: + handles escapes inside strings and does not treat // inside a string as a + comment.""" + out = [] + i, n = 0, len(text) + while i < n: + c = text[i] + nxt = text[i + 1] if i + 1 < n else "" + if c == "/" and nxt == "*": + j = text.find("*/", i + 2) + j = n if j < 0 else j + 2 + out.append("".join(ch if ch == "\n" else " " for ch in text[i:j])) + i = j + elif c == "/" and nxt == "/": + j = text.find("\n", i) + j = n if j < 0 else j + out.append(" " * (j - i)) + i = j + elif c == '"' or c == "'": + q = c + j = i + 1 + while j < n and text[j] != q: + if text[j] == "\\": + j += 1 + if j < n and text[j] == "\n": + break + j += 1 + j = min(j + 1, n) + out.append(q + " " * max(0, j - i - 2) + (q if j - i >= 2 else "")) + i = j + else: + out.append(c) + i += 1 + return "".join(out) + + +def scan() -> dict[str, list[tuple[int, str]]]: + hits: dict[str, list[tuple[int, str]]] = {} + for root in SCAN_ROOTS: + base = ROOT / root + if not base.exists(): + continue + for path in sorted(base.rglob("*")): + if path.suffix not in (".c", ".h") or not path.is_file(): + continue + rel = path.relative_to(ROOT).as_posix() + if is_exempt(rel): + continue + text = strip_comments_and_strings(path.read_text(encoding="utf-8", errors="replace")) + for lineno, line in enumerate(text.splitlines(), 1): + for m in RAW.finditer(line): + hits.setdefault(rel, []).append((lineno, m.group(1))) + return hits + + +def read_baseline() -> dict[str, int]: + if not BASELINE.exists(): + return {} + out: dict[str, int] = {} + for raw in BASELINE.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + rel, _, count = line.rpartition("\t") + if not rel: + continue + try: + out[rel] = int(count) + except ValueError: + continue + return out + + +def write_baseline(hits: dict[str, list]) -> None: + lines = [ + "# memory-core-baseline.txt -- raw allocator call sites per file.", + "# Generated by scripts/lint-memory-core.py --write-baseline.", + "# A file may only ever go DOWN. Lower a line in the same change that", + "# migrates the file to src/foundation/mem_core.h; never raise one.", + "", + ] + for rel in sorted(hits): + lines.append(f"{rel}\t{len(hits[rel])}") + BASELINE.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--write-baseline", action="store_true") + ap.add_argument("--strict", action="store_true", help="fail when a file is BELOW its baseline") + ap.add_argument("--list", action="store_true", help="print every raw site") + args = ap.parse_args() + + hits = scan() + total = sum(len(v) for v in hits.values()) + + if args.write_baseline: + write_baseline(hits) + print(f"memory-core baseline written: {len(hits)} files, {total} raw sites") + return 0 + + if args.list: + for rel in sorted(hits): + for lineno, call in hits[rel]: + print(f"{rel}:{lineno}: {call}(") + print(f"-- {total} raw sites in {len(hits)} files") + return 0 + + baseline = read_baseline() + if not baseline: + print(f"ERROR: no baseline at {BASELINE.relative_to(ROOT)}; run --write-baseline", file=sys.stderr) + return 2 + + grew: list[str] = [] + shrank: list[str] = [] + for rel, sites in sorted(hits.items()): + now = len(sites) + allowed = baseline.get(rel, 0) + if now > allowed: + # Show the LAST sites: new code is usually appended, so these are the + # likeliest culprits. The linter cannot know which sites are new + # without a diff; --list prints them all. + delta = now - allowed + tail = ", ".join(f"{ln}:{c}" for ln, c in sites[-max(delta, 1):][-6:]) + grew.append(f" {rel}: grew by {delta} ({allowed} -> {now}); latest sites: {tail}" + f" [run --list for all]") + elif now < allowed: + shrank.append(f" {rel}: {now} (baseline {allowed}) -- lower the baseline") + for rel, allowed in sorted(baseline.items()): + if rel not in hits and allowed > 0: + shrank.append(f" {rel}: 0 (baseline {allowed}) -- remove the line") + + if grew: + print("memory-core linter FAILED: raw allocator use grew. Allocate through") + print("src/foundation/mem_core.h (cbm_alloc/cbm_calloc/cbm_realloc/cbm_free/") + print("cbm_mem_strdup) instead of malloc/calloc/realloc/free/strdup.") + print("\n".join(grew)) + if shrank: + print("memory-core ratchet: these files improved; tighten scripts/memory-core-baseline.txt:") + print("\n".join(shrank)) + if grew or (args.strict and shrank): + return 1 + print(f"memory-core linter passed: {total} raw sites across {len(hits)} files, none grew") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/memory-core-baseline.txt b/scripts/memory-core-baseline.txt new file mode 100644 index 000000000..67a2434d3 --- /dev/null +++ b/scripts/memory-core-baseline.txt @@ -0,0 +1,97 @@ +# memory-core-baseline.txt -- raw allocator call sites per file. +# Generated by scripts/lint-memory-core.py --write-baseline. +# A file may only ever go DOWN. Lower a line in the same change that +# migrates the file to src/foundation/mem_core.h; never raise one. + +internal/cbm/ac.c 20 +internal/cbm/arena.c 3 +internal/cbm/cbm.c 13 +internal/cbm/extract_defs.c 5 +internal/cbm/helpers.c 3 +internal/cbm/lsp/c_lsp.c 4 +internal/cbm/lsp/perl_lsp.c 21 +internal/cbm/lsp/py_lsp.c 2 +internal/cbm/macro_table.c 1 +internal/cbm/service_patterns.c 2 +internal/cbm/sqlite_writer.c 92 +src/cli/activation_transaction.c 80 +src/cli/agent_clients.c 73 +src/cli/agent_profiles.c 7 +src/cli/cli.c 173 +src/cli/client_adapter.c 3 +src/cli/config_edit_path.c 2 +src/cli/config_json_like.c 142 +src/cli/config_text_edit.c 72 +src/cli/config_toml_edit.c 85 +src/cli/config_yaml_edit.c 117 +src/cli/hook_augment.c 31 +src/cypher/cypher.c 132 +src/daemon/application.c 107 +src/daemon/bootstrap.c 13 +src/daemon/daemon.c 31 +src/daemon/frontend.c 12 +src/daemon/ipc.c 146 +src/daemon/project_lock.c 6 +src/daemon/runtime.c 45 +src/daemon/service.c 6 +src/daemon/version_cohort.c 9 +src/discover/discover.c 31 +src/discover/gitignore.c 12 +src/discover/userconfig.c 15 +src/foundation/diagnostics.c 7 +src/foundation/dyn_array.h 4 +src/foundation/hash_table.c 3 +src/foundation/lock_registry.c 18 +src/foundation/platform.c 11 +src/foundation/platform.h 5 +src/foundation/private_file_lock.c 56 +src/foundation/str_intern.c 7 +src/foundation/subprocess.c 24 +src/foundation/win_utf8.h 17 +src/foundation/workspace.c 3 +src/foundation/yaml.c 10 +src/git/git_context.c 18 +src/graph_buffer/graph_buffer.c 83 +src/main.c 32 +src/mcp/compact_out.c 39 +src/mcp/index_supervisor.c 8 +src/mcp/mcp.c 814 +src/pipeline/artifact.c 38 +src/pipeline/fqn.c 23 +src/pipeline/lsp_resolve.h 4 +src/pipeline/lsp_surface.c 5 +src/pipeline/pass_calls.c 23 +src/pipeline/pass_compile_commands.c 17 +src/pipeline/pass_complexity.c 15 +src/pipeline/pass_configlink.c 3 +src/pipeline/pass_cross_repo.c 8 +src/pipeline/pass_definitions.c 14 +src/pipeline/pass_enrichment.c 15 +src/pipeline/pass_ensemble_routing.c 10 +src/pipeline/pass_envscan.c 2 +src/pipeline/pass_githistory.c 27 +src/pipeline/pass_importance.c 4 +src/pipeline/pass_infrascan.c 2 +src/pipeline/pass_k8s.c 13 +src/pipeline/pass_lsp_cross.c 50 +src/pipeline/pass_parallel.c 31 +src/pipeline/pass_pkgmap.c 82 +src/pipeline/pass_semantic.c 13 +src/pipeline/pass_semantic_edges.c 48 +src/pipeline/pass_similarity.c 12 +src/pipeline/pass_tests.c 3 +src/pipeline/pass_usages.c 4 +src/pipeline/path_alias.c 42 +src/pipeline/pipeline.c 134 +src/pipeline/pipeline_delta.c 19 +src/pipeline/pipeline_incremental.c 144 +src/pipeline/registry.c 19 +src/pipeline/worker_pool.c 2 +src/semantic/semantic.c 54 +src/simhash/minhash.c 10 +src/store/store.c 358 +src/ui/config.c 12 +src/ui/http_server.c 31 +src/ui/httpd.c 21 +src/ui/layout3d.c 66 +src/watcher/watcher.c 33 diff --git a/src/foundation/mem.c b/src/foundation/mem.c index 61d13fe91..5f64bcb25 100644 --- a/src/foundation/mem.c +++ b/src/foundation/mem.c @@ -514,6 +514,22 @@ bool cbm_mem_over_budget(void) { return rss > g_budget; } +/* Reclaimable memory below this share of total RAM is where paging starts to + * hurt, so it is the point at which pressing on stops being reasonable. */ +enum { MEM_PRESSURE_AVAIL_DIVISOR = 8 }; /* 12.5% of total RAM */ + +bool cbm_mem_system_under_pressure(void) { + size_t available = cbm_system_available_ram(); + if (available == 0) { + return false; /* platform cannot answer - do not abort on a guess */ + } + cbm_system_info_t info = cbm_system_info(); + if (info.total_ram == 0) { + return false; + } + return available < info.total_ram / MEM_PRESSURE_AVAIL_DIVISOR; +} + size_t cbm_mem_worker_budget(int num_workers) { if (num_workers <= 0) { num_workers = SKIP_ONE; diff --git a/src/foundation/mem.h b/src/foundation/mem.h index bbd9d769f..826744789 100644 --- a/src/foundation/mem.h +++ b/src/foundation/mem.h @@ -75,6 +75,21 @@ bool cbm_mem_over_budget(void); /* Per-worker budget hint: budget / num_workers. */ size_t cbm_mem_worker_budget(int num_workers); +/* True only when the SYSTEM is genuinely short of memory, not merely when this + * process is over its advisory budget. + * + * The budget is a static fraction of TOTAL ram, so on a large host it refuses + * work the machine can plainly do: measured 2026-09-13, the linux kernel needs + * 31.75 GB of a 48 GB host against a 24 GB budget, and the previous release + * completed the very same index by overshooting to 33.56 GB. Aborting on the + * budget alone therefore turned a working index into a refusal. + * + * So the budget keeps its job as the BACKPRESSURE trigger (workers park, peers + * return transients) and this answers the different question of whether giving + * up is warranted. Returns false when availability is unknown: never abort on + * a guess. */ +bool cbm_mem_system_under_pressure(void); + /* Return unused pages to the OS. Call between files to bound per-file peak. */ void cbm_mem_collect(void); diff --git a/src/foundation/mem_core.c b/src/foundation/mem_core.c new file mode 100644 index 000000000..8120d7252 --- /dev/null +++ b/src/foundation/mem_core.c @@ -0,0 +1,289 @@ +/* + * mem_core.c — the allocation route. See mem_core.h for why it exists. + */ +#include "foundation/mem_core.h" + +#include "foundation/constants.h" +#include "foundation/log.h" + +#include +#include +#include +#include + +/* Usable-size query, per platform. + * + * Deliberately NOT mi_usable_size: the mimalloc global override is off on + * macOS (permanently — the two-level namespace aborts on cross-boundary + * frees), so a pointer from plain malloc there is not a mimalloc block and + * mi_usable_size would be undefined behaviour on it. Each platform's own query + * is correct under whichever allocator is actually installed, including when + * that allocator IS mimalloc via the Linux/MinGW override. */ +#if defined(__APPLE__) +#include /* malloc_size */ +#define CBM_USABLE_SIZE(p) malloc_size(p) +#elif defined(_WIN32) +#include /* _msize */ +#define CBM_USABLE_SIZE(p) _msize((void *)(p)) +#elif defined(__GLIBC__) || defined(__linux__) +#include /* malloc_usable_size */ +#define CBM_USABLE_SIZE(p) malloc_usable_size((void *)(p)) +#else +/* BSD and anything unknown: no portable query. Accounting then tracks the + * REQUESTED size, which understates by the rounding. Understating is the safe + * direction for a diagnostic (it never invents memory), and the alternative -- + * a per-block header -- costs 1.6 GB at kernel scale. */ +#define CBM_USABLE_SIZE_UNAVAILABLE 1 +#endif + +enum { MEM_CORE_REPORT_MIN = 64 }; + +typedef struct { + atomic_size_t live_bytes; + atomic_size_t live_blocks; + atomic_size_t peak_bytes; +} mem_class_stats_t; + +static mem_class_stats_t g_classes[CBM_MEM_CLASS_COUNT]; + +static const char *const g_class_names[CBM_MEM_CLASS_COUNT] = { + "other", "gbuf_node", "gbuf_edge", "gbuf_string", "gbuf_index", + "extract", "semantic", "dump", "store", +}; + +const char *cbm_mem_class_name(cbm_mem_class_t cls) { + if ((int)cls < 0 || (int)cls >= CBM_MEM_CLASS_COUNT) { + return "invalid"; + } + return g_class_names[cls]; +} + +/* Out-of-range classes are folded into OTHER rather than rejected: a + * mis-tagged allocation must still be freed correctly. Accounting accuracy is + * worth less than not corrupting the heap. */ +static mem_class_stats_t *class_slot(cbm_mem_class_t cls) { + if ((int)cls < 0 || (int)cls >= CBM_MEM_CLASS_COUNT) { + return &g_classes[CBM_MEM_CLASS_OTHER]; + } + return &g_classes[cls]; +} + +static void class_add(cbm_mem_class_t cls, size_t bytes, size_t blocks) { + mem_class_stats_t *st = class_slot(cls); + size_t now = atomic_fetch_add_explicit(&st->live_bytes, bytes, memory_order_relaxed) + bytes; + if (blocks) { + (void)atomic_fetch_add_explicit(&st->live_blocks, blocks, memory_order_relaxed); + } + /* Peak is best-effort under concurrency: a CAS loop here would serialise + * every allocation in the hot path to make a DIAGNOSTIC exact. Racing + * writers can leave the peak one increment low; that never changes a + * decision, and the cost of exactness would. */ + size_t seen = atomic_load_explicit(&st->peak_bytes, memory_order_relaxed); + while (now > seen) { + if (atomic_compare_exchange_weak_explicit(&st->peak_bytes, &seen, now, memory_order_relaxed, + memory_order_relaxed)) { + break; + } + } +} + +static void class_sub(cbm_mem_class_t cls, size_t bytes, size_t blocks) { + mem_class_stats_t *st = class_slot(cls); + /* Never wrap. A mismatched class on free (the one way a caller can get + * this wrong) would otherwise turn a small drift into a colossal bogus + * number that looks like a leak and sends someone hunting a phantom. */ + size_t seen = atomic_load_explicit(&st->live_bytes, memory_order_relaxed); + while (seen > 0) { + size_t want = bytes > seen ? 0 : seen - bytes; + if (atomic_compare_exchange_weak_explicit(&st->live_bytes, &seen, want, + memory_order_relaxed, memory_order_relaxed)) { + break; + } + } + if (blocks) { + size_t b = atomic_load_explicit(&st->live_blocks, memory_order_relaxed); + while (b > 0) { + if (atomic_compare_exchange_weak_explicit(&st->live_blocks, &b, b - 1, + memory_order_relaxed, memory_order_relaxed)) { + break; + } + } + } +} + +/* Charge what the allocator actually handed us where the platform can say, and + * fall back to the request otherwise. Called once per successful allocation. + * Two definitions rather than one with a constant-folded branch: on a platform + * with no usable-size query the fallback simply IS the request, and cppcheck + * rightly objects to a condition that can never be true. */ +#ifdef CBM_USABLE_SIZE_UNAVAILABLE +static size_t charge_size(const void *block, size_t requested) { + (void)block; + return requested; +} + +size_t cbm_mem_usable_size(const void *block) { + (void)block; + return 0; +} +#else +static size_t charge_size(const void *block, size_t requested) { + size_t usable = CBM_USABLE_SIZE(block); + return usable ? usable : requested; +} + +size_t cbm_mem_usable_size(const void *block) { + if (!block) { + return 0; + } + return CBM_USABLE_SIZE(block); +} +#endif + +void *cbm_alloc(cbm_mem_class_t cls, size_t bytes) { + void *block = malloc(bytes ? bytes : CBM_ALLOC_ONE); + if (!block) { + return NULL; + } + class_add(cls, charge_size(block, bytes), CBM_ALLOC_ONE); + return block; +} + +void *cbm_calloc(cbm_mem_class_t cls, size_t bytes) { + void *block = calloc(CBM_ALLOC_ONE, bytes ? bytes : CBM_ALLOC_ONE); + if (!block) { + return NULL; + } + class_add(cls, charge_size(block, bytes), CBM_ALLOC_ONE); + return block; +} + +void *cbm_realloc(cbm_mem_class_t cls, void *block, size_t bytes) { + if (!block) { + return cbm_alloc(cls, bytes); + } + /* Measure BEFORE: after realloc the old block is gone and its size is + * unknowable, so the decrement has to be computed first. */ + size_t old = charge_size(block, 0); + void *next = realloc(block, bytes ? bytes : CBM_ALLOC_ONE); + if (!next) { + return NULL; /* original intact and still charged - correct */ + } + class_sub(cls, old, 0); + class_add(cls, charge_size(next, bytes), 0); + return next; +} + +char *cbm_mem_strdup(cbm_mem_class_t cls, const char *s) { + if (!s) { + return NULL; + } + size_t len = strlen(s) + CBM_ALLOC_ONE; + char *copy = (char *)cbm_alloc(cls, len); + if (!copy) { + return NULL; + } + memcpy(copy, s, len); + return copy; +} + +void cbm_free(cbm_mem_class_t cls, void *block) { + if (!block) { + return; + } + class_sub(cls, charge_size(block, 0), CBM_ALLOC_ONE); + free(block); +} + +void cbm_mem_class_add_external(cbm_mem_class_t cls, size_t bytes) { + class_add(cls, bytes, 0); +} + +void cbm_mem_class_remove_external(cbm_mem_class_t cls, size_t bytes) { + class_sub(cls, bytes, 0); +} + +size_t cbm_mem_class_live_bytes(cbm_mem_class_t cls) { + return atomic_load_explicit(&class_slot(cls)->live_bytes, memory_order_relaxed); +} + +size_t cbm_mem_class_live_blocks(cbm_mem_class_t cls) { + return atomic_load_explicit(&class_slot(cls)->live_blocks, memory_order_relaxed); +} + +size_t cbm_mem_class_peak_bytes(cbm_mem_class_t cls) { + return atomic_load_explicit(&class_slot(cls)->peak_bytes, memory_order_relaxed); +} + +size_t cbm_mem_tracked_live_bytes(void) { + size_t total = 0; + for (int i = 0; i < CBM_MEM_CLASS_COUNT; i++) { + total += atomic_load_explicit(&g_classes[i].live_bytes, memory_order_relaxed); + } + return total; +} + +void cbm_mem_class_reset_peaks(void) { + for (int i = 0; i < CBM_MEM_CLASS_COUNT; i++) { + size_t live = atomic_load_explicit(&g_classes[i].live_bytes, memory_order_relaxed); + atomic_store_explicit(&g_classes[i].peak_bytes, live, memory_order_relaxed); + } +} + +int cbm_mem_class_report_json(char *out, size_t size) { + if (!out || size < MEM_CORE_REPORT_MIN) { + return 0; + } + int order[CBM_MEM_CLASS_COUNT]; + int n = 0; + for (int i = 0; i < CBM_MEM_CLASS_COUNT; i++) { + if (atomic_load_explicit(&g_classes[i].peak_bytes, memory_order_relaxed) > 0) { + order[n++] = i; + } + } + if (n == 0) { + return 0; + } + /* Insertion sort: n is at most CBM_MEM_CLASS_COUNT. */ + for (int i = 1; i < n; i++) { + int key = order[i]; + size_t kv = atomic_load_explicit(&g_classes[key].live_bytes, memory_order_relaxed); + int j = i - 1; + while (j >= 0 && + atomic_load_explicit(&g_classes[order[j]].live_bytes, memory_order_relaxed) < kv) { + order[j + 1] = order[j]; + j--; + } + order[j + 1] = key; + } + + int written = snprintf(out, size, "["); + for (int i = 0; i < n && written > 0 && (size_t)written < size; i++) { + int idx = order[i]; + int add = snprintf(out + written, size - (size_t)written, + "%s{\"class\":\"%s\",\"live_bytes\":%zu,\"live_blocks\":%zu," + "\"peak_bytes\":%zu}", + i ? "," : "", g_class_names[idx], + atomic_load_explicit(&g_classes[idx].live_bytes, memory_order_relaxed), + atomic_load_explicit(&g_classes[idx].live_blocks, memory_order_relaxed), + atomic_load_explicit(&g_classes[idx].peak_bytes, memory_order_relaxed)); + if (add < 0 || (size_t)(written + add) >= size) { + return 0; /* truncated: a partial JSON array is worse than none */ + } + written += add; + } + if ((size_t)written + CBM_ALLOC_ONE >= size) { + return 0; + } + out[written++] = ']'; + out[written] = '\0'; + return written; +} + +void cbm_mem_class_log(const char *tag) { + char report[CBM_SZ_1K]; + if (cbm_mem_class_report_json(report, sizeof(report)) <= 0) { + return; + } + cbm_log_info("mem.classes", "tag", tag ? tag : "-", "classes", report); +} diff --git a/src/foundation/mem_core.h b/src/foundation/mem_core.h new file mode 100644 index 000000000..042184ba8 --- /dev/null +++ b/src/foundation/mem_core.h @@ -0,0 +1,140 @@ +/* + * mem_core.h — THE allocation route. + * + * Memory in this project is allocated through one core, not through ~800 + * scattered malloc/calloc/strdup sites. This header is that core; + * foundation/mem.h remains policy and measurement (budget, RSS, pressure, + * phase marks) and deliberately owns no allocation. + * + * WHY THIS EXISTS (audited 2026-09-13) + * + * The budget could only ever be OBSERVED, never enforced. cbm_mem_over_budget() + * reads process RSS *after* the allocation that crossed the line already + * succeeded, and the only available response was to nap. A thermostat wired to + * a thermometer with no cooler attached. + * + * Worse, we could not even say WHERE the memory was. Two independent reasons: + * + * 1. cbm_mem_map_collect()'s live_bytes walks mi_theap_get_default() — THIS + * thread's mimalloc heap only. Walking the process-wide mi_heap_main() is + * a data race that TSan caught on macOS, so it is deliberately not done. + * An 18-worker index therefore attributes almost nothing; the rest lands + * in `residual`. + * 2. The mimalloc global override is ON for Linux/MinGW and permanently OFF + * for macOS (the two-level namespace turns this binary's free into mi_free + * while system libraries keep allocating from the system zone, and a + * pointer crossing that boundary aborts). On macOS ordinary malloc is + * served by the SYSTEM allocator: the startup audit reports + * owned_classes=0/6. Any accounting that assumes mimalloc owns the pointer + * is blind on an entire platform. + * + * So the core keeps its OWN counters. Atomic, per class, incremented at + * allocation and decremented at free. Thread-safe by construction, identical on + * every platform, and independent of which allocator actually serves malloc. + * + * NO PER-ALLOCATION HEADER. The obvious design — a {class,size} prefix — costs + * 16 bytes on every block, and the graph buffer makes ~100M of them at kernel + * scale: ~1.6 GB of pure overhead to measure a memory problem. Sizes come from + * the platform's usable-size query instead, which is exact-to-the-bucket, free, + * and correct under either allocator. + */ +#ifndef CBM_MEM_CORE_H +#define CBM_MEM_CORE_H + +#include +#include + +/* Allocation classes. + * + * A class is a BUDGETING bucket, not a type taxonomy: split only where the + * split would change a decision. These follow the measured phase profile of a + * kernel index (2026-09-13), where the peak was 35.20 GB and the two consumers + * behaved differently — extraction transients scale with worker count, the + * semantic plateau does not. Attribution that cannot separate those two cannot + * choose between "park workers" and "stream the vectors". */ +typedef enum { + CBM_MEM_CLASS_OTHER = 0, /* unclassified; the residual to drive down */ + CBM_MEM_CLASS_GBUF_NODE, /* node records (~64 B each) */ + CBM_MEM_CLASS_GBUF_EDGE, /* edge records (~48 B each) */ + CBM_MEM_CLASS_GBUF_STRING, /* name / qualified_name / properties_json */ + CBM_MEM_CLASS_GBUF_INDEX, /* the 8 lookup indexes (~3.77 GB at kernel scale) */ + CBM_MEM_CLASS_EXTRACT, /* per-file working set: arena, tree, source */ + CBM_MEM_CLASS_SEMANTIC, /* embedding vectors (~9.4 GB at kernel scale) */ + CBM_MEM_CLASS_DUMP, /* dump-time transients */ + CBM_MEM_CLASS_STORE, /* store batches and row buffers */ + CBM_MEM_CLASS_COUNT +} cbm_mem_class_t; + +/* Stable lowercase name, for logs and JSON. Never NULL, even out of range. */ +const char *cbm_mem_class_name(cbm_mem_class_t cls); + +/* ── The allocation route ────────────────────────────────────────────── + * + * Semantics match the C library exactly, so adoption is a mechanical rename and + * never a behaviour change: + * - cbm_alloc(cls, 0) returns a non-NULL pointer that is valid to free + * - cbm_free(cls, NULL) is a no-op + * - cbm_realloc(cls, NULL, n) behaves as cbm_alloc + * - a failed realloc leaves the original block intact and returns NULL + * + * The class passed to free/realloc MUST be the class the block was allocated + * with, or the counters drift. Pass the class through alongside the pointer, + * the same way a custom deleter would. */ +void *cbm_alloc(cbm_mem_class_t cls, size_t bytes); +void *cbm_calloc(cbm_mem_class_t cls, size_t bytes); +void *cbm_realloc(cbm_mem_class_t cls, void *block, size_t bytes); +char *cbm_mem_strdup(cbm_mem_class_t cls, const char *s); +void cbm_free(cbm_mem_class_t cls, void *block); + +/* ── Accounting ──────────────────────────────────────────────────────── + * + * live_bytes is what the ALLOCATOR handed us (usable size), so it exceeds the + * bytes requested by the per-block rounding and is the honest number for a + * memory budget: rounding is memory the process cannot use for anything else. + * + * These counters see only memory that went through this core. Everything still + * on raw malloc is invisible here — which is the point of + * cbm_mem_tracked_live_bytes() versus the process RSS in mem.h: the gap between + * them IS the unmigrated surface, and it should shrink as adoption spreads. An + * unmeasured allocation must never read as an absent one. */ +size_t cbm_mem_class_live_bytes(cbm_mem_class_t cls); +size_t cbm_mem_class_live_blocks(cbm_mem_class_t cls); +size_t cbm_mem_tracked_live_bytes(void); + +/* Peak live bytes for a class since process start (or the last reset). The + * budget question is always about the PEAK, never the value at the moment + * someone happened to look. */ +size_t cbm_mem_class_peak_bytes(cbm_mem_class_t cls); + +/* Drop all peaks to the current live values. For a measurement run that wants + * one phase, not the whole process history. Never resets live counters -- + * those track real outstanding blocks. */ +void cbm_mem_class_reset_peaks(void); + +/* JSON array of {class, live_bytes, live_blocks, peak_bytes}, biggest live + * first, classes with no activity omitted. Returns bytes written, 0 if none. */ +int cbm_mem_class_report_json(char *out, size_t size); + +/* Log the class table at info level under `tag`. For phase boundaries in the + * index pipeline, where the interesting question is which class grew. */ +void cbm_mem_class_log(const char *tag); + +/* Usable size of a block obtained from this core, or 0 when the platform + * cannot answer. Exposed because the same query is what makes header-free + * accounting possible, and callers doing their own bulk accounting (arenas) + * need the identical definition to stay consistent with these counters. */ +size_t cbm_mem_usable_size(const void *block); + +/* ── Bulk accounting, for allocators that are not this one ───────────── + * + * The extraction engine already allocates through arenas (1301 call sites) and + * must NOT be rewritten to per-object cbm_alloc — that would undo the very + * batching that keeps its allocation count low. Instead an arena reports its + * block acquisitions here, so arena-backed memory appears in the same table as + * heap memory and the totals stay comparable. + * + * Symmetric: every add must be matched by a remove of the same size. */ +void cbm_mem_class_add_external(cbm_mem_class_t cls, size_t bytes); +void cbm_mem_class_remove_external(cbm_mem_class_t cls, size_t bytes); + +#endif /* CBM_MEM_CORE_H */ diff --git a/src/foundation/platform.h b/src/foundation/platform.h index cc58e8429..7e9d3ff29 100644 --- a/src/foundation/platform.h +++ b/src/foundation/platform.h @@ -116,6 +116,10 @@ typedef struct { /* Query system information. Results are cached after first call. */ cbm_system_info_t cbm_system_info(void); +/* Physical memory the system could hand out right now, or 0 when the platform + * cannot answer. NOT cached - it changes during a run, which is the point. */ +size_t cbm_system_available_ram(void); + /* Recommended worker count for parallel indexing. * initial=true: all cores (user is waiting for initial index) * initial=false: max(1, perf_cores-1) (leave headroom for user apps) */ diff --git a/src/foundation/system_info.c b/src/foundation/system_info.c index 8ba237f6c..489a7dc65 100644 --- a/src/foundation/system_info.c +++ b/src/foundation/system_info.c @@ -26,6 +26,7 @@ enum { DEFAULT_CORES = 1, MIN_WORKERS = 1, CBM_WORKERS_MAX = 256 }; #endif #include #elif defined(__APPLE__) +#include /* host_statistics64 - reclaimable page accounting */ #include #elif defined(__NetBSD__) || defined(__FreeBSD__) || defined(__OpenBSD__) #include @@ -304,3 +305,64 @@ int cbm_default_worker_count(bool initial) { int workers = info.perf_cores - SKIP_ONE; return workers > 0 ? workers : MIN_WORKERS; } + +/* -- Available RAM -------------------------------------------------- + * + * Bytes the system could hand out right now, or 0 when the platform cannot + * answer. Deliberately NOT cached: core counts and total RAM are immutable + * hardware facts, but this changes continuously and the entire point is to + * observe it DURING an index. + * + * Why this exists: the indexer decided "out of memory" by comparing its own + * RSS against a static fraction of TOTAL ram. That refuses work a machine can + * plainly do -- measured 2026-09-13, the linux kernel needs 31.75 GB on a + * 48 GB host (66 percent) against a 0.5 default budget, and the same index + * completed on the previous release by overshooting to 33.56 GB. Whether + * memory is actually scarce is a property of the SYSTEM, not of a constant. */ +size_t cbm_system_available_ram(void) { +#ifdef _WIN32 + MEMORYSTATUSEX status; + status.dwLength = sizeof(status); + if (GlobalMemoryStatusEx(&status)) { + return (size_t)status.ullAvailPhys; + } + return 0; +#elif defined(__APPLE__) + /* free + inactive + purgeable. Counting only free_count would report + * pressure on any machine that is merely warm, because inactive and + * purgeable pages are reclaimed on demand. */ + mach_port_t host = mach_host_self(); + vm_size_t page_size = 0; + if (host_page_size(host, &page_size) != KERN_SUCCESS || page_size == 0) { + return 0; + } + vm_statistics64_data_t vm_stat = {0}; + mach_msg_type_number_t count = HOST_VM_INFO64_COUNT; + if (host_statistics64(host, HOST_VM_INFO64, (host_info64_t)&vm_stat, &count) != KERN_SUCCESS) { + return 0; + } + uint64_t pages = (uint64_t)vm_stat.free_count + (uint64_t)vm_stat.inactive_count + + (uint64_t)vm_stat.purgeable_count; + return (size_t)(pages * (uint64_t)page_size); +#elif !defined(__NetBSD__) && !defined(__FreeBSD__) && !defined(__OpenBSD__) + /* Linux: MemAvailable is the kernel estimate and accounts for reclaimable + * slab and page cache, which MemFree does not. */ + FILE *meminfo = fopen("/proc/meminfo", "re"); + if (!meminfo) { + return 0; + } + char line[CBM_SZ_256]; + size_t available = 0; + while (fgets(line, sizeof(line), meminfo) != NULL) { + unsigned long long kb = 0; + if (sscanf(line, "MemAvailable: %llu kB", &kb) == 1) { + available = (size_t)(kb * (unsigned long long)CBM_SZ_1K); + break; + } + } + (void)fclose(meminfo); + return available; +#else + return 0; /* BSD: unknown rather than guessed */ +#endif +} diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 25b1c2eb7..d87e5f90f 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -37,6 +37,7 @@ enum { CBM_DIR_PERMS = 0755, PL_RING = 4, PL_RING_MASK = 3, PL_SEQ_PASSES = 6 }; #include "foundation/compat_thread.h" #include "foundation/profile.h" #include "foundation/mem.h" +#include "foundation/mem_core.h" #include "foundation/secure_random.h" #include @@ -988,6 +989,23 @@ static void predump_importance(cbm_pipeline_ctx_t *ctx) { cbm_pipeline_pass_importance(ctx); } +/* Phase boundary for memory attribution. Two instruments, both already in + * foundation/, both previously wired ONLY into MCP request handling and never + * into the index pipeline -- which is where the memory is (a kernel index + * peaks at 35 GB in extraction, measured 2026-09-13 with an external sampler + * because nothing in-process could say which phase it was in): + * - cbm_mem_phase_mark attributes the committed-bytes delta since the last + * mark to the phase just ended. Off unless CBM_MEM_PHASES=1. + * - cbm_mem_class_log prints the mem_core class table, so the log answers + * WHICH class grew in WHICH pass. Logs nothing until a class has activity, + * so it is silent on a tree that has not migrated yet. + * Marks must bracket the whole path with no unlabelled gaps (mem.h), hence a + * mark at every pass.timing site plus pipeline.begin at the top. */ +static void pipeline_phase_mark(const char *pass) { + cbm_mem_phase_mark(pass); + cbm_mem_class_log(pass); +} + static void run_predump_passes(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) { static const struct { predump_pass_fn fn; @@ -1024,6 +1042,7 @@ static void run_predump_passes(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) { passes[i].fn(ctx); cbm_log_info("pass.timing", "pass", passes[i].name, "elapsed_ms", itoa_buf((int)elapsed_ms(t))); + pipeline_phase_mark(passes[i].name); } } @@ -1145,6 +1164,7 @@ static int run_sequential_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, } cbm_log_info("pass.timing", "pass", seq_passes[si].name, "elapsed_ms", itoa_buf((int)elapsed_ms(*t))); + pipeline_phase_mark(seq_passes[si].name); if (check_cancel(p)) { rc = CBM_NOT_FOUND; } @@ -1222,6 +1242,7 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, int rc = cbm_parallel_extract(ctx, files, file_count, cache, &shared_ids, worker_count); cbm_log_info("pass.timing", "pass", "parallel_extract", "elapsed_ms", itoa_buf((int)elapsed_ms(*t))); + pipeline_phase_mark("parallel_extract"); if (rc != 0 || check_cancel(p)) { for (int i = 0; i < file_count; i++) { cbm_free_result(cache[i]); @@ -2238,6 +2259,7 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_hash_t *bas } cbm_log_info("pass.timing", "pass", "dump_and_persist", "elapsed_ms", itoa_buf((int)elapsed_ms(*t)), "files", itoa_buf(manifest_count)); + pipeline_phase_mark("dump_and_persist"); if (p->ignored_total > p->ignored_count) { cbm_log_warn("index.ignored_capped", "stored", itoa_buf(p->ignored_count), "total", itoa_buf(p->ignored_total)); @@ -2302,6 +2324,7 @@ static int run_tests_and_history(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, int rc = cbm_pipeline_pass_tests(ctx, files, file_count); CBM_PROF_END_N("pipeline", "pass_tests", t_tests, file_count); cbm_log_info("pass.timing", "pass", "tests", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); + pipeline_phase_mark("tests"); if (rc == 0 && !check_cancel(p)) { CBM_PROF_START(t_gh); rc = run_githistory(p, ctx); @@ -3040,6 +3063,12 @@ static void sweep_orphan_stages(const char *final_path) { } int cbm_pipeline_run(cbm_pipeline_t *p) { + /* Per-index attribution: peaks and phase totals are about THIS index, not + * the process history, so they start clean here. The first mark opens + * the labelled path; every pass.timing site below closes a phase. */ + cbm_mem_class_reset_peaks(); + cbm_mem_phase_reset(); + cbm_mem_phase_mark("pipeline.begin"); if (!p) { return CBM_NOT_FOUND; } diff --git a/tests/test_mem.c b/tests/test_mem.c index 7db92b735..5ac8abe43 100644 --- a/tests/test_mem.c +++ b/tests/test_mem.c @@ -6,6 +6,7 @@ #include "test_framework.h" #include "test_helpers.h" #include "../src/foundation/mem.h" +#include "../src/foundation/mem_core.h" #include "../src/foundation/arena.h" #include "../src/foundation/slab_alloc.h" #include "../src/foundation/compat_thread.h" @@ -1336,6 +1337,163 @@ TEST(extract_traversal_stacks_come_from_ctx_scratch_issue2010) { PASS(); } +/* ── mem_core: the central allocation route ──────────────────────────── + * + * Every assertion below is a DELTA, never an absolute. Other code in this + * process may allocate through the core concurrently, so a test that pinned an + * absolute total would be measuring the rest of the suite. */ + +TEST(mem_core_accounts_alloc_and_free) { + size_t before = cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_NODE); + size_t blocks_before = cbm_mem_class_live_blocks(CBM_MEM_CLASS_GBUF_NODE); + + void *p = cbm_alloc(CBM_MEM_CLASS_GBUF_NODE, 4096); + ASSERT_TRUE(p != NULL); + size_t during = cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_NODE); + /* Charged at least what was asked for; usable size may round UP, never + * down, so a strict >= is the honest assertion. */ + ASSERT_TRUE(during >= before + 4096); + ASSERT_EQ((int)(cbm_mem_class_live_blocks(CBM_MEM_CLASS_GBUF_NODE) - blocks_before), 1); + + cbm_free(CBM_MEM_CLASS_GBUF_NODE, p); + ASSERT_EQ((int)(cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_NODE) - before), 0); + ASSERT_EQ((int)(cbm_mem_class_live_blocks(CBM_MEM_CLASS_GBUF_NODE) - blocks_before), 0); + PASS(); +} + +/* The whole point of classes: attribution. If a gbuf allocation could show up + * under semantic, the table could not choose between "park workers" and + * "stream the vectors" -- the decision this core exists to inform. */ +TEST(mem_core_classes_do_not_bleed) { + size_t node_before = cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_NODE); + size_t sem_before = cbm_mem_class_live_bytes(CBM_MEM_CLASS_SEMANTIC); + + void *p = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, 8192); + ASSERT_TRUE(p != NULL); + ASSERT_EQ((int)(cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_NODE) - node_before), 0); + ASSERT_TRUE(cbm_mem_class_live_bytes(CBM_MEM_CLASS_SEMANTIC) >= sem_before + 8192); + cbm_free(CBM_MEM_CLASS_SEMANTIC, p); + PASS(); +} + +TEST(mem_core_realloc_replaces_the_old_charge) { + size_t before = cbm_mem_class_live_bytes(CBM_MEM_CLASS_DUMP); + void *p = cbm_alloc(CBM_MEM_CLASS_DUMP, 1024); + ASSERT_TRUE(p != NULL); + p = cbm_realloc(CBM_MEM_CLASS_DUMP, p, 65536); + ASSERT_TRUE(p != NULL); + size_t grown = cbm_mem_class_live_bytes(CBM_MEM_CLASS_DUMP); + /* The old 1024 must be gone, not stacked on top: exactly one block is live, + * so the delta is bounded by the new size plus rounding, not by the sum. */ + ASSERT_TRUE(grown >= before + 65536); + ASSERT_TRUE(grown < before + 65536 + 65536); + cbm_free(CBM_MEM_CLASS_DUMP, p); + ASSERT_EQ((int)(cbm_mem_class_live_bytes(CBM_MEM_CLASS_DUMP) - before), 0); + PASS(); +} + +/* realloc(NULL) is alloc, and free(NULL) is a no-op: the core must match the C + * library exactly or adoption stops being a mechanical rename. */ +TEST(mem_core_matches_libc_null_semantics) { + size_t before = cbm_mem_class_live_bytes(CBM_MEM_CLASS_STORE); + cbm_free(CBM_MEM_CLASS_STORE, NULL); + ASSERT_EQ((int)(cbm_mem_class_live_bytes(CBM_MEM_CLASS_STORE) - before), 0); + + void *p = cbm_realloc(CBM_MEM_CLASS_STORE, NULL, 2048); + ASSERT_TRUE(p != NULL); + ASSERT_TRUE(cbm_mem_class_live_bytes(CBM_MEM_CLASS_STORE) >= before + 2048); + cbm_free(CBM_MEM_CLASS_STORE, p); + + /* A zero-size request still yields a freeable pointer. */ + void *z = cbm_alloc(CBM_MEM_CLASS_STORE, 0); + ASSERT_TRUE(z != NULL); + cbm_free(CBM_MEM_CLASS_STORE, z); + ASSERT_EQ((int)(cbm_mem_class_live_bytes(CBM_MEM_CLASS_STORE) - before), 0); + PASS(); +} + +TEST(mem_core_strdup_copies_and_accounts) { + size_t before = cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_STRING); + ASSERT_TRUE(cbm_mem_strdup(CBM_MEM_CLASS_GBUF_STRING, NULL) == NULL); + + const char *src = "qualified::name::example"; + char *copy = cbm_mem_strdup(CBM_MEM_CLASS_GBUF_STRING, src); + ASSERT_TRUE(copy != NULL); + ASSERT_TRUE(strcmp(copy, src) == 0); + ASSERT_TRUE(copy != src); + ASSERT_TRUE(cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_STRING) > before); + cbm_free(CBM_MEM_CLASS_GBUF_STRING, copy); + ASSERT_EQ((int)(cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_STRING) - before), 0); + PASS(); +} + +/* A budget decision is about the PEAK, not about whatever was live when + * someone looked. Peak must survive the free that follows it. */ +TEST(mem_core_peak_survives_the_free) { + cbm_mem_class_reset_peaks(); + size_t base = cbm_mem_class_peak_bytes(CBM_MEM_CLASS_EXTRACT); + void *p = cbm_alloc(CBM_MEM_CLASS_EXTRACT, 32768); + ASSERT_TRUE(p != NULL); + size_t peak_live = cbm_mem_class_peak_bytes(CBM_MEM_CLASS_EXTRACT); + ASSERT_TRUE(peak_live >= base + 32768); + cbm_free(CBM_MEM_CLASS_EXTRACT, p); + ASSERT_EQ((int)(cbm_mem_class_peak_bytes(CBM_MEM_CLASS_EXTRACT) - peak_live), 0); + PASS(); +} + +/* Arena-backed memory reports in bulk rather than per object: the extraction + * engine has 1301 arena call sites and rewriting them to per-object cbm_alloc + * would undo the batching that keeps its allocation count low. */ +TEST(mem_core_external_bulk_accounting_is_symmetric) { + size_t before = cbm_mem_class_live_bytes(CBM_MEM_CLASS_EXTRACT); + cbm_mem_class_add_external(CBM_MEM_CLASS_EXTRACT, 1024 * 1024); + ASSERT_EQ((int)(cbm_mem_class_live_bytes(CBM_MEM_CLASS_EXTRACT) - before), 1024 * 1024); + cbm_mem_class_remove_external(CBM_MEM_CLASS_EXTRACT, 1024 * 1024); + ASSERT_EQ((int)(cbm_mem_class_live_bytes(CBM_MEM_CLASS_EXTRACT) - before), 0); + PASS(); +} + +/* The one mistake a caller can actually make is freeing with the wrong class. + * That must never underflow the counter: an unsigned wrap would turn a small + * drift into a colossal bogus total that reads as a catastrophic leak and + * sends someone hunting a phantom. Clamp at zero instead. */ +TEST(mem_core_mismatched_class_never_wraps) { + void *p = cbm_alloc(CBM_MEM_CLASS_GBUF_EDGE, 4096); + ASSERT_TRUE(p != NULL); + /* Free against a class that was never charged for it. */ + size_t other_before = cbm_mem_class_live_bytes(CBM_MEM_CLASS_DUMP); + cbm_free(CBM_MEM_CLASS_DUMP, p); + size_t other_after = cbm_mem_class_live_bytes(CBM_MEM_CLASS_DUMP); + ASSERT_TRUE(other_after <= other_before); /* clamped, never wrapped */ + ASSERT_TRUE(other_after < (size_t)-1 / 2); + PASS(); +} + +TEST(mem_core_report_json_is_wellformed_or_empty) { + void *p = cbm_alloc(CBM_MEM_CLASS_GBUF_INDEX, 4096); + ASSERT_TRUE(p != NULL); + char buf[CBM_SZ_1K]; + int n = cbm_mem_class_report_json(buf, sizeof(buf)); + ASSERT_TRUE(n > 0); + ASSERT_TRUE(buf[0] == '['); + ASSERT_TRUE(buf[n - 1] == ']'); + ASSERT_TRUE(strstr(buf, "gbuf_index") != NULL); + /* A buffer too small must yield NOTHING, never a truncated array that a + * JSON reader would reject or, worse, silently mis-parse. */ + char tiny[8]; + ASSERT_EQ(cbm_mem_class_report_json(tiny, sizeof(tiny)), 0); + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, p); + PASS(); +} + +TEST(mem_core_class_names_are_total) { + ASSERT_TRUE(strcmp(cbm_mem_class_name(CBM_MEM_CLASS_SEMANTIC), "semantic") == 0); + /* Out of range must still answer, so a log line never takes a NULL. */ + ASSERT_TRUE(cbm_mem_class_name((cbm_mem_class_t)(CBM_MEM_CLASS_COUNT + 5)) != NULL); + ASSERT_TRUE(cbm_mem_class_name((cbm_mem_class_t)-1) != NULL); + PASS(); +} + SUITE(mem) { /* mem API */ RUN_TEST(mem_arena_eager_commit_follows_platform_commit_cost); @@ -1402,4 +1560,14 @@ SUITE(mem) { /* extraction scratch arena (#2010) */ RUN_TEST(extract_traversal_stacks_come_from_ctx_scratch_issue2010); + RUN_TEST(mem_core_accounts_alloc_and_free); + RUN_TEST(mem_core_classes_do_not_bleed); + RUN_TEST(mem_core_realloc_replaces_the_old_charge); + RUN_TEST(mem_core_matches_libc_null_semantics); + RUN_TEST(mem_core_strdup_copies_and_accounts); + RUN_TEST(mem_core_peak_survives_the_free); + RUN_TEST(mem_core_external_bulk_accounting_is_symmetric); + RUN_TEST(mem_core_mismatched_class_never_wraps); + RUN_TEST(mem_core_report_json_is_wellformed_or_empty); + RUN_TEST(mem_core_class_names_are_total); } From 7fedfe85c63446d3695b75d463a9b8767655730f Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 13 Sep 2026 16:56:14 +0200 Subject: [PATCH 069/130] lint(mem): scan all of src/ and internal/, exempt vendored/ by path segment The memory-core linter scanned src/ and internal/cbm/. Widen it to src/ and every subtree under internal/, so a new tree added there is covered without anyone remembering to list it, and the count is for the whole project -- cli, mcp, daemon, store, pipeline, the extraction engine. Vendored code is now excluded wherever a vendored/ segment appears in the path rather than by one hardcoded prefix, so that exemption cannot drift either. internal/ holds only cbm today, so the regenerated baseline is byte-identical: 92 files, 4086 raw sites. Widening lost no coverage and vendored stays out (0 sites listed). The red/green proof still holds under the new scope, and the failure message now names the NEW site -- one appended malloc reports grew by 2 (83 -> 85) at the appended line, instead of listing six pre-existing ones. Signed-off-by: Martin Vogel --- scripts/lint-memory-core.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/scripts/lint-memory-core.py b/scripts/lint-memory-core.py index a8f714ca0..59e8ba827 100644 --- a/scripts/lint-memory-core.py +++ b/scripts/lint-memory-core.py @@ -29,9 +29,13 @@ EXEMPT. The core itself and the allocator plumbing it sits on: src/foundation/mem.c policy/measurement, probes with malloc src/foundation/mem_override_*.c the --wrap / override shims src/foundation/compat*.c libc replacement surface - internal/cbm/vendored/** not ours + internal/**/vendored/** not ours vendored/** not ours +SCOPE. Everything under src/ and internal/ -- cli, mcp, daemon, store, +pipeline, the extraction engine -- so the count is for the whole project, not +one subsystem, and a leak shows up wherever it is. + Usage: lint-memory-core.py check against the baseline (CI) lint-memory-core.py --strict also fail when a file is BELOW baseline @@ -49,7 +53,7 @@ import sys ROOT = pathlib.Path(__file__).resolve().parent.parent BASELINE = ROOT / "scripts" / "memory-core-baseline.txt" -SCAN_ROOTS = ("src", "internal/cbm") +SCAN_ROOTS = ("src", "internal") RAW = re.compile(r"(? bool: + """Any vendored/ segment anywhere under internal/ is not ours.""" + return "/vendored/" in rel or rel.startswith("vendored/") + + def is_exempt(rel: str) -> bool: - return rel in EXEMPT_EXACT or rel.startswith(EXEMPT_PREFIX) + return rel in EXEMPT_EXACT or rel.startswith(EXEMPT_PREFIX) or is_vendored(rel) def strip_comments_and_strings(text: str) -> str: From b5f10c2214f7b4d34e71187214d51d4cfa7ba759 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 13 Sep 2026 14:51:00 +0200 Subject: [PATCH 070/130] fix(mcp): give the over-budget error a retry value, and say the peak is not the requirement v0.10.9 introduces Decision A: an over-budget index attempt fails whole with a named error instead of thrashing. The error told the caller to "Raise CBM_MEM_BUDGET_MB" but not to what, so the obvious next move is to read peak_rss_mb and retry just above it. That move fails, every time, by construction. The abort fires WHEN resident memory crosses the budget, so peak_rss_mb is pinned just above the budget -- it is where indexing was stopped, not what the repository needs. Measured on the linux kernel (94521 files, 43.1M LOC) while benching v0.10.9: default budget 24576 MB -> aborted, peak_rss_mb 25622 completing the same index actually took 31.75 GB So the reported peak understates the real requirement by 24%, and 1.32x the budget was needed. A caller retrying at peak+10% would have burned another 30 seconds to fail again with a nearly identical message. The response now carries suggested_budget_mb (1.5x the current budget, which would have cleared the kernel case) and the hint says plainly that peak_rss_mb is a floor, not a target. Computed as (3*b+1)/2 rather than b + b/2: integer division makes the latter degenerate to b at b == 1, so the suggestion would have repeated the budget that just failed -- the extended test caught exactly that. Not changed: the abort itself, the backpressure ladder, and worker cleanup. A clean test confirmed workers do NOT survive the abort (process count 2 before, 2 after, both idle at 0% CPU), so there was nothing to fix there. tests/test_mcp.c now pins both halves -- a concrete value strictly greater than the budget, and the hint stating the peak is where indexing STOPPED. Verified against the real pre-fix output captured during the bench, which contains neither. mcp suite 317 passed, 0 failed, 4 skipped; lint-ci clean. Signed-off-by: Martin Vogel --- src/mcp/mcp.c | 33 ++++++++++++++++++++++++++++----- tests/test_mcp.c | 10 ++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index f80d97c6e..e4d6e7eb2 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -11228,11 +11228,34 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { yyjson_mut_obj_add_str(doc, root, "previous_index", "preserved"); yyjson_mut_obj_add_int(doc, root, "budget_mb", budget_mb); yyjson_mut_obj_add_int(doc, root, "peak_rss_mb", peak_rss_mb); - yyjson_mut_obj_add_str(doc, root, "hint", - "Indexing stopped: resident memory stayed above the budget after " - "backpressure; no partial graph was published and the previous " - "index still serves. Raise CBM_MEM_BUDGET_MB, lower CBM_WORKERS, " - "or exclude large subtrees."); + /* A CONCRETE retry value, because "raise CBM_MEM_BUDGET_MB" alone makes + * the caller guess — and the obvious guess is wrong. peak_rss_mb is + * where the run was STOPPED (it is pinned just above the budget by + * construction), not what the repo needs, so retrying at peak+10% fails + * again. Measured 2026-09-13 on the linux kernel: aborted at 25622 MB + * against a 24576 MB budget, but completing it actually took 31.75 GB — + * 1.32x the budget, 1.24x the reported peak. Suggest 1.5x the budget so + * the first retry has a real chance, and say plainly that the peak is a + * floor rather than a requirement. */ + /* (3*b+1)/2 rather than b + b/2: integer division makes the latter + * degenerate to b for b == 1, so the "suggestion" would repeat the + * budget that just failed. Rounding up keeps it strictly larger for + * every positive budget. */ + int suggested_budget_mb = budget_mb > 0 ? (budget_mb * 3 + 1) / 2 : 0; + char hint_text[CBM_SZ_512]; + (void)snprintf(hint_text, sizeof(hint_text), + "Indexing stopped: resident memory stayed above the budget after " + "backpressure; no partial graph was published and the previous index " + "still serves. peak_rss_mb is where indexing was STOPPED, not what this " + "repo needs — the real requirement is higher, so retrying just above the " + "peak will fail again. Retry with CBM_MEM_BUDGET_MB=%d (1.5x the current " + "budget) if the machine has the RAM, or lower CBM_WORKERS, or exclude " + "large subtrees.", + suggested_budget_mb); + if (suggested_budget_mb > 0) { + yyjson_mut_obj_add_int(doc, root, "suggested_budget_mb", suggested_budget_mb); + } + yyjson_mut_obj_add_strcpy(doc, root, "hint", hint_text); } else if (rc == CBM_PIPELINE_ABORT_PRESERVE_DB) { /* The truthful abort message (#2020): the old generic "check repo_path" * hint sent people debugging a path that was fine, when the run diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 95f669852..974aad10e 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -17773,8 +17773,15 @@ TEST(index_repository_over_budget_reports_named_reason) { bool reason_named = second_reason && strcmp(second_reason, "over_memory_budget") == 0; bool previous_preserved = second_previous && strcmp(second_previous, "preserved") == 0; bool hint_names_knob = second_hint && strstr(second_hint, "CBM_MEM_BUDGET_MB") != NULL; + /* The hint must also say that peak_rss_mb is NOT the requirement. The abort + * fires when RSS crosses the budget, so the peak is pinned just above it by + * construction; a caller who retries at peak+10% fails again. Measured on + * the linux kernel 2026-09-13: aborted at 25622 MB against a 24576 MB + * budget, but completing it took 31.75 GB. */ + bool hint_warns_peak_is_not_need = second_hint && strstr(second_hint, "STOPPED") != NULL; int budget_mb = budget_doc_int(second_doc, "budget_mb", -1); int peak_rss_mb = budget_doc_int(second_doc, "peak_rss_mb", -1); + int suggested_mb = budget_doc_int(second_doc, "suggested_budget_mb", -1); yyjson_doc_free(second_doc); free(second); long db_size_after = (long)cbm_file_size(db_path); @@ -17817,8 +17824,11 @@ TEST(index_repository_over_budget_reports_named_reason) { ASSERT_TRUE(reason_named); ASSERT_TRUE(previous_preserved); ASSERT_TRUE(hint_names_knob); + ASSERT_TRUE(hint_warns_peak_is_not_need); ASSERT_EQ(budget_mb, 1); ASSERT_GT(peak_rss_mb, budget_mb); + /* A CONCRETE retry value, not just the knob name: 1.5x the budget. */ + ASSERT_GT(suggested_mb, budget_mb); /* Preserved on disk and still served: same file size, same node count. */ ASSERT_GT(db_size_before, 0L); ASSERT_EQ(db_size_after, db_size_before); From 00411ce7bc41995783bfb16aada673515585446d Mon Sep 17 00:00:00 2001 From: DavidHLP <144919470+DavidHLP@users.noreply.github.com> Date: Mon, 14 Sep 2026 00:52:09 +0800 Subject: [PATCH 071/130] fix(cli): handle trusted Linux system aliases during install activation Signed-off-by: DavidHLP <144919470+DavidHLP@users.noreply.github.com> --- src/cli/activation_transaction.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/cli/activation_transaction.c b/src/cli/activation_transaction.c index 75f5eeaaf..9494b70de 100644 --- a/src/cli/activation_transaction.c +++ b/src/cli/activation_transaction.c @@ -890,8 +890,12 @@ static bool activation_posix_acl_empty(int descriptor) { } static char *activation_posix_walk_path(const char *directory) { -#ifdef __APPLE__ - static const char *const aliases[] = {"/tmp", "/var"}; +#if defined(__APPLE__) || defined(__linux__) + /* macOS and immutable Linux layouts can expose writable trees through + * root-owned aliases (for example /tmp and /home on Atomic systems). + * Resolve only these trusted system aliases; arbitrary user symlinks must + * still fail the O_NOFOLLOW walk below. */ + static const char *const aliases[] = {"/tmp", "/var", "/home"}; for (size_t index = 0; index < sizeof(aliases) / sizeof(aliases[0]); index++) { const char *alias = aliases[index]; size_t alias_length = strlen(alias); @@ -901,8 +905,13 @@ static char *activation_posix_walk_path(const char *directory) { } struct stat alias_status; char resolved[4096]; - if (lstat(alias, &alias_status) != 0 || !S_ISLNK(alias_status.st_mode) || - alias_status.st_uid != 0 || !realpath(alias, resolved)) { + if (lstat(alias, &alias_status) != 0) { + return NULL; + } + if (!S_ISLNK(alias_status.st_mode)) { + continue; + } + if (alias_status.st_uid != 0 || !realpath(alias, resolved)) { return NULL; } struct stat resolved_status; From 581f0c74006beb3edd28babbebeda7e48b3fa1d6 Mon Sep 17 00:00:00 2001 From: DavidHLP Date: Mon, 14 Sep 2026 10:03:31 +0800 Subject: [PATCH 072/130] fix(cli): keep macOS alias policy unchanged Signed-off-by: DavidHLP --- src/cli/activation_transaction.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cli/activation_transaction.c b/src/cli/activation_transaction.c index 9494b70de..dcdb4a20c 100644 --- a/src/cli/activation_transaction.c +++ b/src/cli/activation_transaction.c @@ -895,7 +895,11 @@ static char *activation_posix_walk_path(const char *directory) { * root-owned aliases (for example /tmp and /home on Atomic systems). * Resolve only these trusted system aliases; arbitrary user symlinks must * still fail the O_NOFOLLOW walk below. */ +#ifdef __linux__ static const char *const aliases[] = {"/tmp", "/var", "/home"}; +#else + static const char *const aliases[] = {"/tmp", "/var"}; +#endif for (size_t index = 0; index < sizeof(aliases) / sizeof(aliases[0]); index++) { const char *alias = aliases[index]; size_t alias_length = strlen(alias); From 7b77fd4834afe355cc4c7326ac3a55f0502f7264 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 13 Sep 2026 14:51:00 +0200 Subject: [PATCH 073/130] feat(mem): memory on the core, waste cut, results spilled under budget Second increment on the memory core. The first landed the core, the linter and phase attribution with zero production call sites; this one migrates the memory that matters, proves where it goes, removes the waste the proof exposed, and makes the memory budget a promise: over budget, extraction results go to disk and the run completes at the floor instead of aborting. Every number below is from a real index on this machine (M5 Pro, 48 GB) with CBM_MEM_PHASES=1 and the release build. WHAT THE PROOF SAID (Go corpus: 21,875 files, 345 MB of source) The graph buffer, blamed for the peak since the 2026-07 research, is 3% of the memory: 0.55 GB peak. The per-file extraction results are the memory: 14.8 GB written into their arenas, of which 3.4 GB is reachable. The rest is temporaries -- cbm_node_text copies at 496 call sites, per-node QN strings, abandoned generations of every growable array -- that an arena cannot free and that the result kept alive until resolve ended, because the result owned the arena. The arenas were charged 28.8 GB of capacity for those 14.8 GB. On the kernel the same shape was 8.9 GB used by 89,731 results, and resolve grew it to 13.3 GB used / 17.2 GB capacity because the C walk allocated its working set in the result arena and stored arena pointers into the shared registry. After that, 7 GB of the kernel's post-resolve memory had no class at all. WHAT CHANGED Attribution. The graph buffer (87 raw sites -> 0), arena blocks, the tree-sitter slab allocator, the bound tree-sitter/SQLite allocators, the semantic path (pass_semantic_edges, pass_similarity, minhash: 80 sites), the hash table (Verstable, a class per table via CTX_TY: cbm_ht_create_in) and CBM_DYN_ARRAY allocate through the core. Classes arena, ts_tree, hash_table and dyn_array join the table. Two cross-boundary frees of buffer-owned strings (pass_importance, pass_complexity) go through a new setter, cbm_gbuf_node_set_properties_json. The linter baseline drops 4086 -> 3916. Result compaction (internal/cbm/result_compact.c). At the end of a file's extraction, everything reachable from its result is measured, copied into one exact-size arena (strings interned by content within the file) and the working arena is handed back. Later appends restart growth at the default block (cbm_arena_init_exact, arena grow_size). The stale duplicate internal/cbm/arena.h, which shared the foundation header's include guard and won or lost by include order, is a shim now. Record slimming. CBMCall 320 -> 72 bytes: the eight inline argument slots (256 bytes, mostly empty) are allocated on first capture. CBMUsage 48 -> 40. The per-file overlay contract (src/pipeline/pass_lsp_cross.c). One lifecycle for every language's cross-file resolve: a scratch arena, a per-file overlay registry chained to the sealed shared base, the walk against the overlay, resolved calls appended into the result, scratch destroyed. type_registry gains chain-aware iterators (every yielded index belongs to it->reg) and copy-on-write refinement (cbm_registry_func/type_for_update), and the C, C#, Go and Python walks use them. This is what an ASan heap-use-after-free in the lsp_resolution_probe suite demanded: the C walk stored per-call arena objects into the shared registry, and a kernel run with a naive scratch arena lost 1,045 edges to the resulting garbage reads. It also ends unsynchronized writes into the shared registry from parallel resolve workers. The 7 GB with no class, found and cleaned. gbuf_index tables and keys were 2.5 GB live and 7.1 GB at peak: the 18 worker buffers built the five secondary indexes they never query (cbm_gbuf_new_worker builds none), and 18M per-key index arrays started at 8 slots (cbm_da_push_min starts them at 2); edge property JSON is 81% "{}" (interned). The semantic pass allocated a 512-token stride per function up front (packed tokens with offsets: cbm_sem_corpus_add_docs_batch). Edge dedup keys are 128-bit hashes instead of sprintf'd strings (edge_key_map_t). Each worker keeps one reusable working arena between files (cbm_work_arena_take/give: rewind, not free) and every phase mark returns freed pages to the OS. Budget metric. The over-budget check compares the charged footprint -- max(phys_footprint, mimalloc commit) on macOS -- not resident_size: after extraction the kernel worker sat at 17.4 GB RSS with 5.5 GB charged, because pages mimalloc has purged stay resident until the OS wants them, and phys_footprint under-reports after MADV_FREE_REUSABLE cycles. Spill / admission control (internal/cbm/result_spill.c, the extract gate in pass_parallel.c). At 15/16 of the budget (or over it, or with CBM_MEM_SPILL=1) the gate latches spill mode -- early, because the gate sees the crossing per file pull and the workers' in-flight files carry the charge past the line before the first sweep lands: every compacted result is parked on disk as header + block in one append-only file per writer under /spill/, the results already cached are swept out by every worker, and registry build, def collection, surface rows, resolve and the infra-route passes load a result only for the moment they read it (relocated by base delta). The abort path stays closed while a sweep runs or a cached result remains; it fires only at the floor -- graph + registries + in-flight files. A result that owns sub-results stays in memory; a retained parse tree is dropped with the parked result and re-parsed on load. Every consumer of the result cache reads a slot through cbm_pipeline_result_acquire()/release() (pipeline_internal.h): a parked slot is NULL in the array, and the passes that indexed it directly lost every __route__infra__ node and its INFRA_MAPS edges before the contract. The other things the transient loads found: a def label that was the result's own pointer (SIGBUS in the Go cross registry builder), two JVM helpers that passed their input through, a dedup table whose keys borrowed sr->value (the insert spun on freed memory), and a latch that was visible before the store was open (a peer found nothing to park and failed the run while its neighbour wrote 9 GB). The arena rewind reuse branch had defeated the nblocks = CBM_ARENA_MAX_BLOCKS OOM seam the suites use; it reuses only blocks that exist. The incremental and probe routes still index the cache array themselves and therefore never spill (ctx->spill_allowed stays false there; follow-up). The semantic pass under a budget (src/pipeline/pass_semantic_edges.c). The pass held every function's tokens, the corpus's per-document token ids and the vectors being stored at once: 4.9 GB on the kernel, on top of the floor, and nothing sized it to the headroom. Now, when the charged footprint plus that transient would cross the budget, phases 2-4 run per batch of functions: tokenize -> count into the corpus -> free; finalize once; tokenize again -> vectorize -> store -> free, with pages returned to the OS after every batch. Tokenization is deterministic, so both passes see the same tokens and the graph does not change; the run pays with a second tokenize. CBM_SEM_BATCH= forces the batch size (the batched-equals-unbatched test). The corpus frees its per-document token ids at the end of finalize (the co-occurrence pass is their only reader), and mem.semantic.step lines (CBM_MEM_PHASES=1) put the class's live/peak bytes and the charged footprint at every sub-phase. The semantic corpus and tokenizer allocate through the core (54 raw sites -> 0). This is not attribution only: cbm_sem_tokenize handed out libc strdup blocks that pass_semantic_edges freed with cbm_free -- mi_free in the production build -- since the pass moved onto the core. No libc-backed test build can see a cross-allocator free, so the core now refuses a block it does not own (mi_is_in_heap_region) when CBM_MEM_PHASES=1 is set, fatally: every proof run in this PR ran with it armed, and the Go and kernel indexes report zero foreign blocks. Accounting without contention. The core charged every allocation and free to shared per-class atomics; with 18 workers on 18 cores every block bounced the same three cache lines, and the bench against the shipped v0.10.8 showed it as CPU time rising faster than wall time for the same graph: Kotlin 38 -> 121 s CPU (+58% wall), TypeScript 210 -> 596 s (+67%), Go 177 -> 312 s. The deltas are thread-local now and reach the shared counters once per 256 KB or 512 blocks of change per thread, when a parallel-for worker ends, and whenever a reader looks (its own thread first). Phase marks read after the join and are exact; a class peak is a diagnostic and can be low by threads x 256 KB. Result: Kotlin 10.9 s / 43 s CPU (the residual +22% wall against v0.10.8 tracks its +34% nodes), TypeScript 33.9 s (v0.10.8: 33.4), Go 48.4 s (v0.10.8: 52.6), Java 81.1 s (v0.10.8: 79.8); graphs unchanged. SQLite on a heap of its own. The same bench put the kernel's write step at 132 s where v0.10.8 needed 9.7 s, all of it in one coverage publish statement. A stack sample named the cost: mimalloc's free-page search and its periodic heap collect, walking the graph's pages -- millions of them once the graph lives on the core -- for every statement-journal chunk SQLite allocates. SQLite now allocates from a mimalloc heap of its own per thread (mi_heap_new; mi_free is heap-agnostic, so nothing else changes), and the periodic collect is set to the maximum mimalloc allows since the explicit release at phase marks and spill sweeps is where memory goes back. Coverage publish 132 -> 9.4 s; profiled kernel 376 -> 248 s (v0.10.8: 296 s). Formatting without a process lock. On macOS every vsnprintf takes the process locale under an unfair lock (localeconv_l inside __vfprintf); eighteen workers formatting type names on the C# corpus collapsed into it -- the 10 MB JIT test files took 68 s each instead of under a second -- and cbm_arena_sprintf paid it twice per string (size pass, write pass). Each thread now formats with a C locale object of its own (vsnprintf_l), and the common short string is formatted once into a stack buffer. Per-file budgets that cover the whole file. The 5 s per-file budget covered the parse only; the LSP walks after it had none. C# JIT stress files (a 23 MB single expression among them) parsed inside the budget -- the old wall-clock parse timeout used to drop them, which hid the rest -- and then held a worker for 346 s each in the usage walk (tree-sitter's ts_node_parent descends from the root: quadratic on a deep tree), and the cross-file resolve ran on the same trees. Three rules now, one site for every language (cbm_extract_file_ex, honoured by cbm_pxc_dispatch_file): a parse that used more than half the budget disqualifies the file from the LSP walks (lsp_skipped); the unified walk checks its thread CPU time every 1024 nodes against six budgets and stops there, keeping what it found (walk_truncated, implies lsp_skipped); a file whose parse plus walk spent the budget is skipped by the LSP walks as well. Each decision is logged with the path (extract.lsp.skipped, extract.walk.truncated). C# extraction 355 -> 39 s; the whole index 463 -> 128 s wall and 27 -> 11 GB peak against v0.10.8 with 299 more nodes. The walk cuts exactly the five JIT stress files (hugeexpr1, hugeSimpleExpr1, HugeArray1, HugeField1/2); the 15 MB System.Runtime.Intrinsics reference file walks to the end and, with four generic-nesting JIT regression tests, only skips the LSP walk under the parse-plus-walk rule. The crash behind the C# bench. dotnet/runtime is 42,555 files, and its resolve phase died with SIGBUS on every run of this branch. A libc-backed ASan build of the server named it: heap-use-after-free in c_adl_resolve, reading a type name that another worker had allocated in its per-file scratch arena and freed at the end of its file. The C++ class walk refines a method's return type when the declaration is more specific than the pre-registered one (NAMED -> pointer/reference/template), and it did that by casting the chained lookup result to non-const and writing a scratch-arena signature into it -- into the sealed shared base when the entry lived there. The overlay contract had exactly one bypass, and it was this cast; every lookup returns const, so a grep for the cast is the audit. The refinement now goes through cbm_registry_func_for_update: copy-on-write into the overlay, the base untouched. Test: clsp_method_return_refinement_is_copy_on_write (a sealed base with a NAMED method return, an in-class pointer declaration; the base signature pointer is unchanged and the overlay copy carries the refinement plus a marker only a copy has) -- RED on the cast, GREEN on the accessor. The fixed ASan build then indexed the whole corpus clean: 42,555 files, 1,224,981 nodes, 5,794,320 edges, worker exit 0, no report. Production build backs the core with mimalloc explicitly (never the global override, which macOS's two-level namespace forbids); the test build keeps libc so the sanitizers see every block -- the same split the tree-sitter and SQLite bindings already use. PROOF Go peak RSS 16.7 -> 4.0-5.2 GB; result arenas 14,804/28,817 -> 856/856 MB; wall 55 -> 52-60 s; graph inside the pre-existing run-to-run jitter (gRPC Route nodes: two identical runs differ by 21). Go CBM_MEM_SPILL=1: peak RSS 1.7-2.3 GB, 21,875 results parked (950 MB on disk), graph identical: 39 infra routes, 158 INFRA_MAPS, 21,875 surface rows, nodes/edges inside the Route jitter. Go CBM_MEM_BUDGET_MB=1200: spill latched at 1,204 MB, the first sweep took the charge to 768 MB, the run completed at 1.99 GB peak RSS. Kernel DEFAULT budget (24,576 MB): aborted before, completes now without spilling. Extraction 35.2 -> 16.5 GB RSS (5.6 GB footprint, 15.3 GB committed); results 8.9 GB = capacity; resolve tracked 24.6 -> 19.1 GB; peak RSS 27.3 -> 20.1 GB. Nodes identical (8,529,729); edges inside the +-400 run-to-run band the kernel shows between identical runs. Kernel CBM_MEM_BUDGET_MB=16000: aborted before, completes now through the spill path. Latched at 17.2 GB charged; 89,731 results parked (9.3 GB on disk, 370,188 loads); extraction end 6.2 GB RSS / 5.1 GB committed; resolve 8.6 GB RSS / 11.5 GB committed; peak RSS 16.6 GB (the semantic pass plateau, 13.3 GB RSS, is graph-derived and outside what spill can move). Nodes identical; edges 15,773,598. Kernel CBM_MEM_BUDGET_MB=15000, before the semantic batching: completes, every phase mark under 15,000 MB by every metric, but the process RSS high-water mark reached 16.9 GB inside the semantic pass (class peak 4.9 GB against 0.8 GB live). Kernel CBM_MEM_BUDGET_MB=15000, with it: completes; the charged footprint stays under 15,000 MB through the whole semantic pass (14,018 -> 14,381 MB across its sub-phases, 6 batches of 125,644 functions); every phase mark is under budget by every metric (max footprint 12.6 GB, max commit 11.5 GB); the process RSS high-water mark is 15.65 GB and is set in extraction, at the moment the gate latches (15,002 MB charged) and before the first sweep lands -- a 4% overshoot of in-flight work, no longer the semantic pass. Nodes identical, edges 15,772,913, zero foreign blocks. Kernel CBM_MEM_BUDGET_MB=15000, with the anticipatory latch: spill mode enters at 14,062 MB (near_budget); the charged footprint never exceeds 11.8 GB at any mark; the process RSS high-water mark is 15.1 GB in extraction and 15.3 GB during the final write-out -- 1-2% over, the in-flight window and the dump transient. Nodes identical, edges 15,772,543, zero foreign blocks. Go CBM_MEM_BUDGET_MB=1200 with it: latched at 1,131 MB, extraction peak 998 MB RSS, graph identical to the baseline. BENCH AGAINST THE SHIPPED v0.10.8 (same machine, same driver, 2026-09-14) corpus wall s peak RSS GB nodes / edges / CALLS perl 5.7 -> 5.3 0.09 -> 0.05 -3 / -130 / -3 php 6.3 -> 5.8 0.66 -> 0.20 +21.9% / +1.4% / +23.8% rust 6.8 -> 6.3 1.15 -> 0.67 +1 / +56 / -369 c 5.9 -> 5.7 0.16 -> 0.19 identical kotlin 8.9 -> 9.4 1.55 -> 0.96 +33.5% / -22.8% / +12.1% django 10.9 -> 10.2 3.13 -> 1.12 +6 / -14.5% / -28.4% go 52.6 -> 42.9 17.2 -> 3.9 +14.2% / -10.3% / +2.2% java 79.8 -> 73.8 27.6 -> 11.0 -121 / -1.9% / -0.6% csharp 463.3 -> 128.1 27.0 -> 11.2 +299 / -0.9% / -0.6% typescript 33.4 -> 33.6 12.0 -> 2.5 -186 / -3.5% / +130 kernel 294.8 -> 233.3 27.5 -> 25.1 -171 / -1.3% / -0.1% No corpus is slower beyond noise; CPU time follows wall (kernel 2,442 -> 1,403 s). Every graph delta against v0.10.8 is main between the release and the branch base (339b3f40), not this PR: a build of the base commit run through the same driver reproduces the final graphs exactly on django and kotlin, within the pre-existing gRPC Route jitter on go (30 nodes, 27 edges, CALLS identical), and on C# with the same node count. This PR's own C# difference against the base is 5 CALLS (the five truncated JIT stress files) plus one ijwhost swap and 563 edges of 5.79 M; the base takes 448 s / 29.7 GB for that corpus. Diagnostics that made this measurable stay in: the extract.arenas and extract.census lines (behind CBM_MEM_PHASES=1), the CBM_MEM_RELEASE=1 release-to-OS probe, and peak_charged_mb on every mem.phase line -- the high-water mark of the budget metric itself (cbm_mem_peak_charged), next to the RSS peak that counts pages already purged but not yet reclaimed. Tests: extract_compact_* and extract_spill_round_trip_keeps_every_field (extraction), parallel_spill_mode_builds_the_same_graph (parallel: every file parked, node count and every edge type equal to the in-memory run), pipeline_semantic_batched_matches_unbatched (pipeline: one repo indexed unbatched and with CBM_SEM_BATCH=5 -- byte-identical node vectors, token vectors and SEMANTICALLY_RELATED edges), extract_lsp_skipped_when_parse_used_ its_budget_share and extract_walk_truncated_at_its_cpu_budget (extraction: the seams mark a file / stop the walk, the LSP walk and the dispatcher skip it, the defs stay), registry_overlay_chain_iterates_ and_copies_on_write (c_lsp), arena exact-init/growth, mem charged/footprint/ pressure. Not in this increment: run-to-run edge jitter (+-400 on the kernel, Route class on Go) predates this work and has its own item; the incremental route spilling; a tighter result encoding (def ids instead of QN strings in resolved calls) is the next lever on the 9 GB the kernel's results still take. Signed-off-by: Martin Vogel --- Makefile.cbm | 2 + internal/cbm/arena.h | 48 +-- internal/cbm/cbm.c | 192 ++++++++++-- internal/cbm/cbm.h | 119 +++++-- internal/cbm/extract_calls.c | 13 + internal/cbm/extract_unified.c | 26 ++ internal/cbm/lsp/c_lsp.c | 88 +++--- internal/cbm/lsp/c_lsp.h | 11 +- internal/cbm/lsp/cs_lsp.c | 24 +- internal/cbm/lsp/go_lsp.c | 6 +- internal/cbm/lsp/py_lsp.c | 142 ++++----- internal/cbm/lsp/py_lsp.h | 4 + internal/cbm/lsp/type_registry.c | 239 ++++++++++++++- internal/cbm/lsp/type_registry.h | 46 +++ internal/cbm/result_compact.c | 460 ++++++++++++++++++++++++++++ internal/cbm/result_spill.c | 260 ++++++++++++++++ internal/cbm/result_spill.h | 65 ++++ scripts/lint-memory-core.py | 6 +- scripts/memory-core-baseline.txt | 9 +- src/foundation/arena.c | 92 +++++- src/foundation/arena.h | 18 ++ src/foundation/dyn_array.h | 84 +++-- src/foundation/hash_table.c | 28 +- src/foundation/hash_table.h | 6 + src/foundation/mem.c | 74 ++++- src/foundation/mem.h | 35 +++ src/foundation/mem_core.c | 176 ++++++++--- src/foundation/mem_core.h | 17 +- src/foundation/slab_alloc.c | 23 +- src/graph_buffer/graph_buffer.c | 456 +++++++++++++++++++-------- src/graph_buffer/graph_buffer.h | 13 + src/pipeline/lsp_surface.c | 13 +- src/pipeline/lsp_surface.h | 4 +- src/pipeline/pass_complexity.c | 4 +- src/pipeline/pass_importance.c | 4 +- src/pipeline/pass_lsp_cross.c | 260 ++++++++++------ src/pipeline/pass_lsp_cross.h | 8 +- src/pipeline/pass_parallel.c | 270 +++++++++++++++- src/pipeline/pass_semantic.c | 27 +- src/pipeline/pass_semantic_edges.c | 426 +++++++++++++++++++++----- src/pipeline/pass_similarity.c | 30 +- src/pipeline/pipeline.c | 315 +++++++++++++++++-- src/pipeline/pipeline_incremental.c | 18 +- src/pipeline/pipeline_internal.h | 33 ++ src/pipeline/worker_pool.c | 4 + src/semantic/semantic.c | 148 +++++---- src/semantic/semantic.h | 9 +- src/simhash/minhash.c | 26 +- tests/test_arena.c | 41 +++ tests/test_c_lsp.c | 163 ++++++++++ tests/test_extraction.c | 370 ++++++++++++++++++++++ tests/test_mem.c | 81 +++++ tests/test_parallel.c | 148 ++++++--- tests/test_pipeline.c | 146 +++++++++ 54 files changed, 4485 insertions(+), 845 deletions(-) create mode 100644 internal/cbm/result_compact.c create mode 100644 internal/cbm/result_spill.c create mode 100644 internal/cbm/result_spill.h diff --git a/Makefile.cbm b/Makefile.cbm index 92c31031f..066d3375c 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -276,6 +276,8 @@ EXTRACTION_SRCS = \ $(CBM_DIR)/extract_k8s.c \ $(CBM_DIR)/extract_dbt.c \ $(CBM_DIR)/helpers.c \ + $(CBM_DIR)/result_compact.c \ + $(CBM_DIR)/result_spill.c \ $(CBM_DIR)/lang_specs.c \ $(CBM_DIR)/macro_table.c \ $(CBM_DIR)/iris_export_xml.c \ diff --git a/internal/cbm/arena.h b/internal/cbm/arena.h index 5c6bef9f0..a451e4cf1 100644 --- a/internal/cbm/arena.h +++ b/internal/cbm/arena.h @@ -1,41 +1,7 @@ -#ifndef CBM_ARENA_H -#define CBM_ARENA_H - -#include - -// CBMArena is a simple bump allocator that allocates from fixed-size blocks. -// All memory is freed at once via cbm_arena_destroy(). Individual frees are not -// supported — this is by design for per-file extraction where all data has the -// same lifetime. -#define CBM_ARENA_MAX_BLOCKS 256 -#define CBM_ARENA_DEFAULT_BLOCK_SIZE (64 * 1024) // 64KB initial - -typedef struct { - char *blocks[CBM_ARENA_MAX_BLOCKS]; - size_t block_sizes[CBM_ARENA_MAX_BLOCKS]; // per-block sizes (for stats) - int nblocks; - size_t block_size; - size_t used; // bytes used in current block - size_t total_alloc; // cumulative bytes allocated (for stats) -} CBMArena; - -// Initialize an arena with the default block size. -void cbm_arena_init(CBMArena *a); - -// Allocate n bytes from the arena. Returns NULL on OOM or block exhaustion. -// All returned pointers are 8-byte aligned. -void *cbm_arena_alloc(CBMArena *a, size_t n); - -// Duplicate a string into arena memory. Returns arena-owned copy. -char *cbm_arena_strdup(CBMArena *a, const char *s); - -// Duplicate a string of known length into arena memory. NUL-terminates. -char *cbm_arena_strndup(CBMArena *a, const char *s, size_t len); - -// sprintf into arena memory. Returns arena-owned string. -char *cbm_arena_sprintf(CBMArena *a, const char *fmt, ...) __attribute__((format(printf, 2, 3))); - -// Free all blocks. Arena is invalid after this call. -void cbm_arena_destroy(CBMArena *a); - -#endif // CBM_ARENA_H +/* The arena lives in src/foundation/arena.h. This file used to be a stale + * copy with the same include guard and a shorter API; whichever header a + * translation unit included first won, silently. One definition now. */ +#ifndef CBM_INTERNAL_ARENA_SHIM_H +#define CBM_INTERNAL_ARENA_SHIM_H +#include "../../src/foundation/arena.h" /* relative: the lsp_all unit has no -Isrc */ +#endif /* CBM_INTERNAL_ARENA_SHIM_H */ diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index ff00a07f2..71bb4bb2f 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -1,7 +1,9 @@ /* Full declaration set for the same CBMArena, and it must precede cbm.h: * internal/cbm/arena.h declares a subset and the two share the CBM_ARENA_H * guard, so whichever is included first is the one this file sees. */ -#include "foundation/arena.h" // cbm_arena_init_sized +#include "foundation/arena.h" // cbm_arena_init_sized +#include "foundation/mem_core.h" // class accounting for the bound allocators +#include "foundation/log.h" // cbm_log_warn -- extract.lsp.skipped #include "cbm.h" #include "arena.h" // CBMArena, cbm_arena_init/alloc/strdup/destroy #include "helpers.h" @@ -222,6 +224,13 @@ static const char *cbm_string_read(void *payload, uint32_t byte, TSPoint point, * edge). A generous WALL ceiling stays as a backstop so a genuinely * stuck/spinning parse still terminates in bounded time. */ #define CBM_PARSE_WALL_CEILING_FACTOR 12ULL /* ~60 s ceiling for the 5 s CPU budget */ +/* A parse that used more than 1/N of the per-file budget disqualifies the file + * from the unbudgeted LSP walks (see cbm_extract_file_ex). */ +#define CBM_LSP_BUDGET_SHARE_DIV 2ULL +/* The unified walk may spend this many parse budgets of thread CPU time: wide + * enough for a 7,873-definition reference file (~10 s), tight enough to stop the + * generated JIT tests (65-350 s). */ +#define CBM_WALK_BUDGET_FACTOR 6ULL typedef struct { uint64_t cpu_deadline_ns; // trip once this thread's CPU time passes it @@ -303,16 +312,43 @@ static TSParser *get_thread_parser(const TSLanguage *ts_lang, CBMLanguage lang) * hook here the biggest per-request allocations in the process — SQLite's page * cache and its query working set — are invisible to the attribution profile * (#581). */ +/* SQLite allocates from a mimalloc heap of its own, one per thread. Its page + * queues then hold SQLite blocks only. Sharing the thread's default heap with + * the graph -- 40M+ blocks, millions of pages once the graph moved onto the + * core -- made every statement-journal chunk of the coverage publish step pay + * a walk over the graph's pages: 132 s on the kernel where v0.10.8, whose + * graph lived outside mimalloc, took 9.7 s (sampled 2026-09-14). mi_free + * works across heaps, so xFree and cross-thread frees are unchanged; a + * thread's heap is released with the thread. */ +static _Thread_local mi_heap_t *tl_sqlite_heap; + +static mi_heap_t *sqlite_heap(void) { + if (!tl_sqlite_heap) { + tl_sqlite_heap = mi_heap_new(); + } + return tl_sqlite_heap ? tl_sqlite_heap : mi_heap_main(); +} + static void *cbm_sqlite_malloc(int n) { - void *block = mi_malloc((size_t)n); + void *block = mi_heap_malloc(sqlite_heap(), (size_t)n); + if (block) { + cbm_mem_class_add_external(CBM_MEM_CLASS_STORE, mi_usable_size(block)); + } return block; } static void cbm_sqlite_free(void *p) { + if (p) { + cbm_mem_class_remove_external(CBM_MEM_CLASS_STORE, mi_usable_size(p)); + } mi_free(p); } static void *cbm_sqlite_realloc(void *p, int n) { - if (p) {} - void *grown = mi_realloc(p, (size_t)n); + size_t old_size = p ? mi_usable_size(p) : 0; + void *grown = mi_heap_realloc(sqlite_heap(), p, (size_t)n); + if (grown) { + cbm_mem_class_remove_external(CBM_MEM_CLASS_STORE, old_size); + cbm_mem_class_add_external(CBM_MEM_CLASS_STORE, mi_usable_size(grown)); + } return grown; } static int cbm_sqlite_size(void *p) { @@ -325,18 +361,31 @@ static int cbm_sqlite_roundup(int n) { * through these, and they too skip the interposer. */ static void *cbm_ts_malloc(size_t n) { void *block = mi_malloc(n); + if (block) { + cbm_mem_class_add_external(CBM_MEM_CLASS_TS_TREE, mi_usable_size(block)); + } return block; } static void *cbm_ts_calloc(size_t count, size_t size) { void *block = mi_calloc(count, size); + if (block) { + cbm_mem_class_add_external(CBM_MEM_CLASS_TS_TREE, mi_usable_size(block)); + } return block; } static void *cbm_ts_realloc(void *p, size_t n) { - if (p) {} + size_t old_size = p ? mi_usable_size(p) : 0; void *grown = mi_realloc(p, n); + if (grown) { + cbm_mem_class_remove_external(CBM_MEM_CLASS_TS_TREE, old_size); + cbm_mem_class_add_external(CBM_MEM_CLASS_TS_TREE, mi_usable_size(grown)); + } return grown; } static void cbm_ts_free(void *p) { + if (p) { + cbm_mem_class_remove_external(CBM_MEM_CLASS_TS_TREE, mi_usable_size(p)); + } mi_free(p); } @@ -1575,13 +1624,12 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C const CBMReturnTypeTable *return_type_table, CBMArena *scratch) { // Allocate result on heap (arena inside for all string data) - enum { SINGLE = 1 }; - CBMFileResult *result = (CBMFileResult *)calloc(SINGLE, sizeof(CBMFileResult)); + CBMFileResult *result = cbm_result_alloc(); if (!result) { return NULL; } - cbm_arena_init(&result->arena); + cbm_work_arena_take(&result->arena); CBMArena *a = &result->arena; /* Crash-quarantine hard guard (Stage 3c): a file the supervisor pinned as a @@ -1642,6 +1690,7 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C ts_parser_reset(parser); uint64_t t0 = now_ns(); + uint64_t cpu_start_ns = cbm_thread_cpu_time_ns(); // Build string input + timeout options for parse_with_options CBMStringInput str_input = {source, (uint32_t)source_len}; @@ -1654,8 +1703,9 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C TSParseOptions opts = {0}; CBMParseBudget budget = {0}; // cppcheck-suppress unreadVariable + uint64_t budget_ns = 0; if (timeout_micros > 0) { - uint64_t budget_ns = (uint64_t)timeout_micros * USEC_TO_NSEC; + budget_ns = (uint64_t)timeout_micros * USEC_TO_NSEC; // Descheduling burns wall time but not CPU: gate on this thread's CPU // time so a starved-but-parseable file is not abandoned, with a generous // wall ceiling as a backstop against a genuinely spinning/stuck parse. @@ -1688,6 +1738,33 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C TSNode root = ts_tree_root_node(tree); + /* Parse-budget share. A file whose parse alone consumed more than + * 1/CBM_LSP_BUDGET_SHARE_DIV of its budget is too large for the per-file + * LSP walk that follows: the walk is superlinear in expression size and + * has no budget of its own (C#, a 23 MB single-expression JIT test: 354 s + * in the walk, then a crash in the cross-file resolve on the same tree, + * 2026-09-14 -- the parse used to time out at 5 s and hide both). The + * unified extractor's defs stay; the LSP refinement here and the + * cross-file resolve (cbm_pxc_dispatch_file) skip the file, logged. The + * budget is the same for every parser, so the rule is too. */ + bool lsp_skipped = timeout_micros > 0 && (t1 - t0) * CBM_LSP_BUDGET_SHARE_DIV > budget_ns; +#ifdef CBM_ENABLE_TEST_SEAMS + { + const char *skip_on = getenv("CBM_TEST_LSP_SKIP_ON"); + if (skip_on && skip_on[0] && rel_path && strstr(rel_path, skip_on)) { + lsp_skipped = true; /* the test names the file; no real timing involved */ + } + } +#endif + if (lsp_skipped) { + char parse_ms[CBM_SZ_32]; + snprintf(parse_ms, sizeof(parse_ms), "%llu", + (unsigned long long)((t1 - t0) / CBM_NSEC_PER_MSEC)); + cbm_log_warn("extract.lsp.skipped", "reason", "parse_budget", "parse_ms", parse_ms, "path", + rel_path ? rel_path : ""); + result->lsp_skipped = true; + } + // Compute module QN. Java/Go derive the module from the CONTAINING // DIRECTORY (package semantics) rather than baking the filename stem in, // so def QNs, the LSP caller_qn, and the textual calls-enclosing QN all @@ -1710,6 +1787,8 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C .root = root, .macro_table = macro_table, .return_type_table = return_type_table, + .walk_deadline_cpu_ns = + timeout_micros > 0 ? cbm_thread_cpu_time_ns() + budget_ns * CBM_WALK_BUDGET_FACTOR : 0, }; // Run extractors: defs + imports use separate walks (unique recursion patterns), @@ -1717,6 +1796,21 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C cbm_extract_definitions(&ctx); cbm_extract_imports(&ctx); cbm_extract_unified(&ctx); + if (ctx.walk_budget_exhausted) { + result->walk_truncated = true; + result->lsp_skipped = true; + cbm_log_warn("extract.walk.truncated", "reason", "cpu_budget", "path", + rel_path ? rel_path : ""); + } + /* A file that spent the budget on parse plus walk is too heavy for the + * unbudgeted LSP walks as well (the C# JIT test files: 65-73 s each in + * the per-file walk after a parse under the share rule). */ + if (!result->lsp_skipped && timeout_micros > 0 && + cbm_thread_cpu_time_ns() - cpu_start_ns > budget_ns) { + result->lsp_skipped = true; + cbm_log_warn("extract.lsp.skipped", "reason", "file_budget", "path", + rel_path ? rel_path : ""); + } // Channel detection (Socket.IO / EventEmitter) — JS/TS only. cbm_extract_channels(&ctx); @@ -1736,7 +1830,7 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C // LSP type-aware call/usage resolution (per-file). Runs in every mode; // refines the tree-sitter + textual-resolution graph with type info. uint64_t lsp_start = now_ns(); - { + if (!result->lsp_skipped) { if (language == CBM_LANG_GO) { cbm_run_go_lsp(a, result, source, source_len, root); } @@ -1775,15 +1869,15 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C if (language == CBM_LANG_CSHARP) { cbm_run_cs_lsp(a, result, source, source_len, root); } - } - if (language == CBM_LANG_JAVA) { - cbm_run_java_lsp(a, result, source, source_len, root); - } - if (language == CBM_LANG_KOTLIN) { - cbm_run_kotlin_lsp(a, result, source, source_len, root); - } - if (language == CBM_LANG_RUST) { - cbm_run_rust_lsp(a, result, source, source_len, root); + if (language == CBM_LANG_JAVA) { + cbm_run_java_lsp(a, result, source, source_len, root); + } + if (language == CBM_LANG_KOTLIN) { + cbm_run_kotlin_lsp(a, result, source, source_len, root); + } + if (language == CBM_LANG_RUST) { + cbm_run_rust_lsp(a, result, source, source_len, root); + } } atomic_fetch_add(&total_lsp_ns, now_ns() - lsp_start); @@ -2156,6 +2250,40 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C return result; } +/* ── Per-worker working arena (see cbm.h) ───────────────────────────── */ +static CBM_TLS CBMArena tl_work_arena; +static CBM_TLS bool tl_work_arena_live = false; + +void cbm_work_arena_take(CBMArena *into) { + if (tl_work_arena_live) { + *into = tl_work_arena; + tl_work_arena_live = false; + cbm_arena_rewind(into); + return; + } + cbm_arena_init(into); +} + +void cbm_work_arena_release(void) { + if (tl_work_arena_live) { + cbm_arena_destroy(&tl_work_arena); + tl_work_arena_live = false; + } +} + +void cbm_work_arena_give(CBMArena *from) { + if (!from || from->nblocks == 0) { + return; + } + if (tl_work_arena_live || cbm_arena_capacity(from) > (size_t)CBM_WORK_ARENA_KEEP_BYTES) { + cbm_arena_destroy(from); + return; + } + tl_work_arena = *from; + tl_work_arena_live = true; + memset(from, 0, sizeof(*from)); +} + /* Public entry. Owns the traversal scratch arena for the whole of one file's * extraction: created here, handed to the body as ctx->scratch, destroyed on * the way out. The body has seven early returns, so bracketing it in a wrapper @@ -2176,6 +2304,25 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua return result; } +CBMFileResult *cbm_result_alloc(void) { + /* The one raw allocation of a result: cbm_free_result releases it with + * the matching free. Extraction and the spill loader both come here. */ + enum { SINGLE = 1 }; + return (CBMFileResult *)calloc(SINGLE, sizeof(CBMFileResult)); +} + +void cbm_result_release_owned(CBMFileResult *result) { + if (!result) { + return; + } + for (int i = 0; i < result->owned_result_count; i++) { + cbm_free_result(result->owned_results[i]); + } + free(result->owned_results); + result->owned_results = NULL; + result->owned_result_count = 0; +} + void cbm_free_result(CBMFileResult *result) { if (!result) { return; @@ -2184,12 +2331,7 @@ void cbm_free_result(CBMFileResult *result) { ts_tree_delete(result->cached_tree); result->cached_tree = NULL; } - for (int i = 0; i < result->owned_result_count; i++) { - cbm_free_result(result->owned_results[i]); - } - free(result->owned_results); - result->owned_results = NULL; - result->owned_result_count = 0; + cbm_result_release_owned(result); cbm_arena_destroy(&result->arena); free(result); } diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index a5bdba999..c62d3f281 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -248,31 +248,36 @@ typedef enum { } CBMSourceOrigin; typedef struct { - const char *callee_name; // raw callee text ("pkg.Func", "foo") - const char *enclosing_func_qn; // QN of enclosing function (or module QN) - const char *first_string_arg; // first string literal argument (URL, topic, key) or NULL - const char *second_arg_name; // second argument identifier (handler ref) or NULL - CBMCallArg args[CBM_MAX_CALL_ARGS]; // first N arguments with expressions - int arg_count; // number of captured arguments - int loop_depth; // enclosing loop nesting at the call site - int branch_depth; // enclosing branch nesting at the call site - int start_line; // 1-based source line of the call (for def range-match) - uint32_t site_start_byte; // exact AST occurrence span; end > start when present - uint32_t site_end_byte; // exclusive byte offset in the source file - CBMSourceOrigin source_origin; // raw source or C-family preprocessed buffer - bool is_method; // method/member call with an UNRESOLVED receiver. Perl: - // arrow/method call ($obj->m). TS/JS/TSX: member call - // x.foo() whose receiver is not this/super. Python: - // x.foo() where x is not self/cls/super() and is not - // rooted in an imported name. Read by the weak-member - // guard and by the pxc synthetic-carrier dedup key in - // pass_lsp_cross.c. Default false. - bool requires_lsp_resolution; // synthetic semantic candidate (for example an implicit - // C++ operator). Never fall back to textual resolution. - bool callee_is_locally_bound; // bare call foo() whose callee identifier is bound as a - // parameter of an enclosing function, so it cannot be the - // module-level foo. Python only today. Read by the - // weak-local-binding guard. Default false. + const char *callee_name; // raw callee text ("pkg.Func", "foo") + const char *enclosing_func_qn; // QN of enclosing function (or module QN) + const char *first_string_arg; // first string literal argument (URL, topic, key) or NULL + const char *second_arg_name; // second argument identifier (handler ref) or NULL + /* First arg_count captured arguments, arena-allocated on first capture; + * NULL when arg_count == 0. Was an inline args[CBM_MAX_CALL_ARGS] (256 of + * the record's 320 bytes) -- the Go corpus census (2026-09-13) put 825k + * calls at 251 MB with most of that empty slots. Readers index it exactly + * as before; only `sizeof` changed. */ + CBMCallArg *args; + int arg_count; // number of captured arguments (<= CBM_MAX_CALL_ARGS) + int loop_depth; // enclosing loop nesting at the call site + int branch_depth; // enclosing branch nesting at the call site + int start_line; // 1-based source line of the call (for def range-match) + uint32_t site_start_byte; // exact AST occurrence span; end > start when present + uint32_t site_end_byte; // exclusive byte offset in the source file + CBMSourceOrigin source_origin; // raw source or C-family preprocessed buffer + bool is_method; // method/member call with an UNRESOLVED receiver. Perl: + // arrow/method call ($obj->m). TS/JS/TSX: member call + // x.foo() whose receiver is not this/super. Python: + // x.foo() where x is not self/cls/super() and is not + // rooted in an imported name. Read by the weak-member + // guard and by the pxc synthetic-carrier dedup key in + // pass_lsp_cross.c. Default false. + bool requires_lsp_resolution; // synthetic semantic candidate (for example an implicit + // C++ operator). Never fall back to textual resolution. + bool callee_is_locally_bound; // bare call foo() whose callee identifier is bound as a + // parameter of an enclosing function, so it cannot be the + // module-level foo. Python only today. Read by the + // weak-local-binding guard. Default false. } CBMCall; typedef struct { @@ -286,16 +291,18 @@ typedef enum { } CBMUsageKind; typedef struct { - const char *ref_name; // referenced identifier - const char *enclosing_func_qn; // QN of enclosing function (or module QN) + const char *ref_name; // referenced identifier + const char *enclosing_func_qn; // QN of enclosing function (or module QN) + /* Fixed-width fields grouped so the record packs to 40 bytes (was 48; the + * Go corpus holds 4.68M of these). Field meanings unchanged. */ + uint32_t lexical_scope_id; // extraction-local scope instance; never graph identity + uint32_t site_start_byte; // exact reference-token span; end > start when present + uint32_t site_end_byte; // exclusive byte offset in the source file CBMUsageKind kind; // ordinary USAGE or explicit callable reference + CBMSourceOrigin source_origin; // raw source or C-family preprocessed buffer bool may_be_call_reference; // syntactic candidate; exact LSP proof may upgrade its edge bool semantic_reference_blocked; // lexical evidence blocks only unproven textual fallback bool semantic_reference_local_shadow; // blocker belongs to a non-module lexical scope - uint32_t lexical_scope_id; // extraction-local scope instance; never graph identity - uint32_t site_start_byte; // exact reference-token span; end > start when present - uint32_t site_end_byte; // exclusive byte offset in the source file - CBMSourceOrigin source_origin; // raw source or C-family preprocessed buffer bool is_member_access; // token is the member half of a selector/attribute // (Go x.f — field_identifier). The extractor strips // the receiver, so this is the only surviving record @@ -541,7 +548,16 @@ typedef struct CBMFileResult { int error_region_count; bool is_test_file; int imports_count; - TSTree *cached_tree; // retained parse tree (caller frees via cbm_free_tree) + TSTree *cached_tree; // retained parse tree (caller frees via cbm_free_tree) + /* The parse alone used more than its share of the per-file budget: the + * per-file LSP walk and the cross-file resolve skip this file (its + * unified-extractor defs stay). Set by cbm_extract_file_ex, honoured by + * cbm_pxc_dispatch_file -- one site for every language. */ + bool lsp_skipped; + /* The unified walk stopped at its CPU budget: defs/calls/usages found up + * to that point are kept, the rest of the file is not walked. Implies + * lsp_skipped. */ + bool walk_truncated; CBMLanguage cached_lang; // language of cached tree (for parser selection) // Retained source bytes — copied into `arena` by the parallel @@ -632,6 +648,14 @@ typedef struct { * class-body variable def records which class declares it (parent_class) * without changing its module-level qualified name. NULL elsewhere. */ const char *var_parent_class; + /* Per-file walk budget (thread CPU time, ns; 0 = unbounded). The unified + * cursor walk checks it every 1024 nodes and stops when it is spent, so + * no single file can hold a worker for minutes: a 23 MB single-expression + * C# test file cost 346 s in usage stamping alone (tree-sitter's + * ts_node_parent descends from the root, quadratic on a deep tree; + * 2026-09-14). What was extracted before the stop is kept. */ + uint64_t walk_deadline_cpu_ns; + bool walk_budget_exhausted; } CBMExtractCtx; // --- Public API --- @@ -675,6 +699,31 @@ void cbm_index_mark_done(const char *rel_path); // Extract all data from one file. Caller must call cbm_free_result(). // source must remain valid for the duration of the call. // timeout_micros: per-file parse timeout in microseconds (0 = no timeout). +/* Compact a finished result: copy everything reachable from it -- every + * record array at exact count, every string once (interned by content within + * the file), the retained source -- into one exact-size arena, and destroy the + * working arena the extractors wrote into. Measured on the Go corpus + * (2026-09-13): 14.8 GB written per index, 3.4 GB reachable; the rest was + * node-text copies and abandoned array generations no one could free because + * the result owned the arena. Call once, after the last per-file write and + * before the result is cached for later passes. Later appends into the arena + * still work (growth restarts at the default block). A composite's owned + * per-unit results are released: after the deep copy nothing points at them. + * On allocation failure the result is left exactly as it was. */ +void cbm_result_compact(CBMFileResult *result); + +/* The working arena the extractors write into, per worker thread. Extraction + * takes it (rewound, pages still mapped) instead of allocating a fresh arena + * per file; compaction returns it instead of destroying it. Reusing the same + * addresses directly is what stops the purge/re-commit churn that kept a + * kernel worker at 15 GB resident with 4-5 GB charged. An arena that grew past + * CBM_WORK_ARENA_KEEP_BYTES (one giant file) is destroyed, not kept. */ +enum { CBM_WORK_ARENA_KEEP_BYTES = 16 * 1024 * 1024 }; +void cbm_work_arena_take(CBMArena *into); +void cbm_work_arena_give(CBMArena *from); +/* Drop this thread's kept working arena (end of an extraction pass). */ +void cbm_work_arena_release(void); + CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage language, const char *project, const char *rel_path, int64_t timeout_micros, const char **extra_defines, // NULL-terminated, or NULL @@ -696,6 +745,12 @@ CBMFileResult *cbm_extract_file_ex( // Free all memory associated with a result. void cbm_free_result(CBMFileResult *result); +/* Allocate an empty result; cbm_free_result releases it. */ +CBMFileResult *cbm_result_alloc(void); + +/* Release a composite result's per-unit results (the owner of that array). */ +void cbm_result_release_owned(CBMFileResult *result); + // Free only the cached tree from a result (caller retained it for reuse). void cbm_free_tree(CBMFileResult *result); diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index c22a11e01..47250203d 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -2059,6 +2059,12 @@ static void extract_call_args(CBMExtractCtx *ctx, TSNode args, CBMCall *call) { for (uint32_t ai = 0; ai < argc && call->arg_count < CBM_MAX_CALL_ARGS; ai++) { TSNode arg_node = ts_node_named_child(args, ai); const char *ak = ts_node_type(arg_node); + if (!call->args) { + call->args = cbm_arena_calloc(ctx->arena, CBM_MAX_CALL_ARGS * sizeof(CBMCallArg)); + if (!call->args) { + return; + } + } CBMCallArg *ca = &call->args[call->arg_count]; memset(ca, 0, sizeof(*ca)); @@ -3768,6 +3774,13 @@ CBMInvocationDescriptor handle_calls(CBMExtractCtx *ctx, TSNode node, const CBML if (strcmp(ack, "method_arg") != 0) { continue; } + if (!call.args) { + call.args = cbm_arena_calloc(ctx->arena, + CBM_MAX_CALL_ARGS * sizeof(CBMCallArg)); + if (!call.args) { + break; + } + } CBMCallArg *ca = &call.args[call.arg_count]; memset(ca, 0, sizeof(*ca)); ca->index = call.arg_count; diff --git a/internal/cbm/extract_unified.c b/internal/cbm/extract_unified.c index 5c2905f9c..cd7f3bd28 100644 --- a/internal/cbm/extract_unified.c +++ b/internal/cbm/extract_unified.c @@ -5,6 +5,8 @@ #include "lang_specs.h" // CBMLangSpec, cbm_lang_spec, CBM_LANG_* #include "tree_sitter/api.h" // TSNode, TSTreeCursor, ts_tree_cursor_*, ts_node_* #include "foundation/constants.h" +#include "foundation/compat.h" // cbm_thread_cpu_time_ns +#include enum { MAX_INFRA_BINDINGS = 8 }; @@ -2596,9 +2598,33 @@ void cbm_extract_unified(CBMExtractCtx *ctx) { state.branch_depth = 0; uint32_t depth = 0; + uint32_t visited = 0; +#ifdef CBM_ENABLE_TEST_SEAMS + /* CBM_TEST_WALK_BUDGET_NODES=: the budget is "spent" after n nodes, + * no real timing involved. */ + uint32_t seam_budget_nodes = 0; + { + const char *seam = getenv("CBM_TEST_WALK_BUDGET_NODES"); + if (seam && seam[0]) { + seam_budget_nodes = (uint32_t)strtoul(seam, NULL, 10); + } + } +#endif for (;;) { TSNode node = ts_tree_cursor_current_node(&cursor); + visited++; + if (ctx->walk_deadline_cpu_ns != 0 && (visited & 1023u) == 0 && + cbm_thread_cpu_time_ns() > ctx->walk_deadline_cpu_ns) { + ctx->walk_budget_exhausted = true; + break; + } +#ifdef CBM_ENABLE_TEST_SEAMS + if (seam_budget_nodes != 0 && visited > seam_budget_nodes) { + ctx->walk_budget_exhausted = true; + break; + } +#endif bool trivia = is_unified_trivia_node(node); if (!trivia) { /* Trivia consumes no semantic state. Scope expiry may be deferred diff --git a/internal/cbm/lsp/c_lsp.c b/internal/cbm/lsp/c_lsp.c index c5991561e..374f6f404 100644 --- a/internal/cbm/lsp/c_lsp.c +++ b/internal/cbm/lsp/c_lsp.c @@ -63,6 +63,7 @@ void c_lsp_init(CLSPContext *ctx, CBMArena *arena, const char *source, int sourc ctx->source = source; ctx->source_len = source_len; ctx->registry = registry; + ctx->registry_head = (CBMTypeRegistry *)registry; ctx->module_qn = module_qn; ctx->module_qn_len = module_qn ? strlen(module_qn) : 0; ctx->enclosing_func_qn = module_qn; @@ -2819,10 +2820,10 @@ static const CBMRegisteredFunc *c_lookup_member_depth(CLSPContext *ctx, const ch size_t slen = strlen(shortn); const char *best_qn = NULL; CBMTypeShortIter it; - cbm_registry_types_by_short_name(ctx->registry, shortn, &it); + cbm_registry_types_by_short_name_chain(ctx->registry, shortn, &it); int i; while ((i = cbm_type_short_iter_next(&it)) >= 0) { - const char *q = ctx->registry->types[i].qualified_name; + const char *q = it.reg->types[i].qualified_name; if (!q) { continue; } @@ -3670,12 +3671,12 @@ static const CBMRegisteredFunc *c_lookup_free_operator(CLSPContext *ctx, const c const char *right_ns = extract_namespace_from_qn(ctx->arena, right_qn); const CBMRegisteredFunc *match = NULL; CBMFreeFuncIter it; - cbm_registry_free_funcs_by_short_name(ctx->registry, operator_name, &it); + cbm_registry_free_funcs_by_short_name_chain(ctx->registry, operator_name, &it); for (int index = cbm_free_func_iter_next(&it); index >= 0; index = cbm_free_func_iter_next(&it)) { - if (index >= ctx->registry->func_count) + if (index >= it.reg->func_count) continue; - const CBMRegisteredFunc *candidate = &ctx->registry->funcs[index]; + const CBMRegisteredFunc *candidate = &it.reg->funcs[index]; if (candidate->receiver_type || !candidate->short_name || strcmp(candidate->short_name, operator_name) != 0 || !candidate->qualified_name) { continue; @@ -3730,13 +3731,13 @@ static const CBMRegisteredFunc *c_lookup_free_unary_operator(CLSPContext *ctx, const CBMRegisteredFunc *match = NULL; const CBMType *actuals[1] = {operand_type}; CBMFreeFuncIter it; - cbm_registry_free_funcs_by_short_name(ctx->registry, operator_name, &it); + cbm_registry_free_funcs_by_short_name_chain(ctx->registry, operator_name, &it); for (int index = cbm_free_func_iter_next(&it); index >= 0; index = cbm_free_func_iter_next(&it)) { - if (index >= ctx->registry->func_count) { + if (index >= it.reg->func_count) { continue; } - const CBMRegisteredFunc *candidate = &ctx->registry->funcs[index]; + const CBMRegisteredFunc *candidate = &it.reg->funcs[index]; if (candidate->receiver_type || !candidate->short_name || strcmp(candidate->short_name, operator_name) != 0 || !candidate->qualified_name) { continue; @@ -4882,18 +4883,16 @@ static void c_process_function(CLSPContext *ctx, TSNode func_node) { // indexing bitcoin). Cross-phase template deduction then relies on the // positional fallback, which is graceful degradation. if (ctx->in_template && ctx->template_param_count > 0 && !ctx->registry_shared) { - // Find the registered function and set type_param_names - for (int ri = 0; ri < ((CBMTypeRegistry *)ctx->registry)->func_count; ri++) { - CBMRegisteredFunc *rf = &((CBMTypeRegistry *)ctx->registry)->funcs[ri]; - if (strcmp(rf->qualified_name, func_qn) == 0 && !rf->type_param_names) { - const char **tpn = (const char **)cbm_arena_alloc( - ctx->arena, (ctx->template_param_count + 1) * sizeof(const char *)); - for (int ti = 0; ti < ctx->template_param_count; ti++) - tpn[ti] = ctx->template_param_names[ti]; - tpn[ctx->template_param_count] = NULL; - rf->type_param_names = tpn; - break; - } + // Set type_param_names on the registered function -- on the head's + // own copy of it (copy-on-write), never on a shared base entry. + CBMRegisteredFunc *rf = cbm_registry_func_for_update(ctx->registry_head, func_qn); + if (rf && !rf->type_param_names) { + const char **tpn = (const char **)cbm_arena_alloc( + ctx->arena, (ctx->template_param_count + 1) * sizeof(const char *)); + for (int ti = 0; ti < ctx->template_param_count; ti++) + tpn[ti] = ctx->template_param_names[ti]; + tpn[ctx->template_param_count] = NULL; + rf->type_param_names = tpn; } } @@ -4929,12 +4928,9 @@ static void c_process_function(CLSPContext *ctx, TSNode func_node) { } // Set min_params on the registered function (for default-arg overload matching) if (total_params > 0 && defaulted_params > 0) { - for (int ri = 0; ri < ((CBMTypeRegistry *)ctx->registry)->func_count; ri++) { - CBMRegisteredFunc *rf = &((CBMTypeRegistry *)ctx->registry)->funcs[ri]; - if (strcmp(rf->qualified_name, func_qn) == 0 && rf->min_params < 0) { - rf->min_params = total_params - defaulted_params; - break; - } + CBMRegisteredFunc *rf = cbm_registry_func_for_update(ctx->registry_head, func_qn); + if (rf && rf->min_params < 0) { + rf->min_params = total_params - defaulted_params; } } @@ -5059,7 +5055,7 @@ static void c_process_body_child(CLSPContext *ctx, TSNode child) { tpn[ctx->template_param_count] = NULL; rf.type_param_names = tpn; } - cbm_registry_add_func((CBMTypeRegistry *)ctx->registry, rf); + cbm_registry_add_func(ctx->registry_head, rf); } } } @@ -5161,14 +5157,7 @@ static void c_process_class(CLSPContext *ctx, TSNode class_node) { // Store template param names on the registered type (for substitution) if (ctx->in_template && ctx->template_param_names && ctx->template_param_count > 0) { - CBMRegisteredType *rt = NULL; - for (int ri = 0; ri < ((CBMTypeRegistry *)ctx->registry)->type_count; ri++) { - if (strcmp(((CBMTypeRegistry *)ctx->registry)->types[ri].qualified_name, - class_qn) == 0) { - rt = &((CBMTypeRegistry *)ctx->registry)->types[ri]; - break; - } - } + CBMRegisteredType *rt = cbm_registry_type_for_update(ctx->registry_head, class_qn); if (rt && !rt->type_param_names) { const char **tpn = (const char **)cbm_arena_alloc( ctx->arena, (ctx->template_param_count + 1) * sizeof(const char *)); @@ -5341,14 +5330,25 @@ static void c_process_class(CLSPContext *ctx, TSNode class_node) { should_upgrade = true; } if (should_upgrade) { - // Update existing entry's signature return type - const CBMType **new_rets = (const CBMType **)cbm_arena_alloc( - ctx->arena, 2 * sizeof(const CBMType *)); - new_rets[0] = actual_ret; - new_rets[1] = NULL; - CBMRegisteredFunc *mut = (CBMRegisteredFunc *)existing; - mut->signature = cbm_type_func_replace_returns( - ctx->arena, existing->signature, new_rets); + /* Refine the return type on the HEAD copy only + * (copy-on-write). `existing` may live in the + * sealed shared base; writing a scratch-arena + * signature into it was read by other workers + * after this file's arena died (ASan heap-use- + * after-free in c_adl_resolve on dotnet/runtime, + * 2026-09-14). NULL = the head is sealed too: + * skip the refinement. */ + CBMRegisteredFunc *mut = cbm_registry_func_for_update( + ctx->registry_head, existing->qualified_name); + if (mut) { + const CBMType **new_rets = + (const CBMType **)cbm_arena_alloc( + ctx->arena, 2 * sizeof(const CBMType *)); + new_rets[0] = actual_ret; + new_rets[1] = NULL; + mut->signature = cbm_type_func_replace_returns( + ctx->arena, mut->signature, new_rets); + } } break; } @@ -5364,7 +5364,7 @@ static void c_process_class(CLSPContext *ctx, TSNode class_node) { rf.receiver_type = ctx->enclosing_class_qn; rf.signature = cbm_type_func(ctx->arena, NULL, NULL, rets); rf.min_params = -1; - cbm_registry_add_func((CBMTypeRegistry *)ctx->registry, rf); + cbm_registry_add_func(ctx->registry_head, rf); break; } else if (strcmp(dk, "reference_declarator") == 0) { actual_ret = cbm_type_reference(ctx->arena, actual_ret); diff --git a/internal/cbm/lsp/c_lsp.h b/internal/cbm/lsp/c_lsp.h index 9aa936960..af81d5bbd 100644 --- a/internal/cbm/lsp/c_lsp.h +++ b/internal/cbm/lsp/c_lsp.h @@ -13,6 +13,11 @@ typedef struct { const char *source; int source_len; const CBMTypeRegistry *registry; + /* The writable head of the registry chain: the per-file overlay the + * dispatcher hands in, or the per-file registry on the non-cross path. + * Every refinement (lazy add, min_params, template params) goes here + * via cbm_registry_*_for_update; nothing behind ->fallback is written. */ + CBMTypeRegistry *registry_head; CBMScope *current_scope; // Include map: header_path -> namespace QN prefix @@ -94,9 +99,9 @@ typedef struct { // READ-ONLY across resolve workers — never mutate it // (and never store per-worker arena pointers into it) bool debug; - int eval_depth; // recursion depth for c_eval_expr_type (crash guard) - int eval_steps; // total expression eval calls for current file (hang guard) - int walk_depth; // c_resolve_calls_in_node self-recursion (AST nesting) + int eval_depth; // recursion depth for c_eval_expr_type (crash guard) + int eval_steps; // total expression eval calls for current file (hang guard) + int walk_depth; // c_resolve_calls_in_node self-recursion (AST nesting) int control_flow_depth; // if/loop/switch/catch nesting; assignments merge fail-closed } CLSPContext; diff --git a/internal/cbm/lsp/cs_lsp.c b/internal/cbm/lsp/cs_lsp.c index 0beed0a56..fc3b548ef 100644 --- a/internal/cbm/lsp/cs_lsp.c +++ b/internal/cbm/lsp/cs_lsp.c @@ -492,10 +492,10 @@ const char *cs_resolve_type_name(CSLSPContext *ctx, const char *raw) { CBMTypeShortIter ts_it; int i; if (ctx->registry) { - cbm_registry_types_by_short_name(ctx->registry, the_short, &ts_it); + cbm_registry_types_by_short_name_chain(ctx->registry, the_short, &ts_it); } while (ctx->registry && (i = cbm_type_short_iter_next(&ts_it)) >= 0) { - const CBMRegisteredType *cand = &ctx->registry->types[i]; + const CBMRegisteredType *cand = &ts_it.reg->types[i]; if (!cand->short_name || strcmp(cand->short_name, the_short) != 0) continue; int score = 0; @@ -643,12 +643,12 @@ static const CBMRegisteredFunc *cs_lookup_extension(CSLSPContext *ctx, const cha * registration-order tie-break exactly (bucket chains are not in funcs[] * order). */ CBMFreeFuncIter ext_it; - cbm_registry_free_funcs_by_short_name(ctx->registry, method_name, &ext_it); + cbm_registry_free_funcs_by_short_name_chain(ctx->registry, method_name, &ext_it); const CBMRegisteredFunc *best = NULL; int best_idx = -1; int i; while ((i = cbm_free_func_iter_next(&ext_it)) >= 0) { - const CBMRegisteredFunc *cand = &ctx->registry->funcs[i]; + const CBMRegisteredFunc *cand = &ext_it.reg->funcs[i]; if (!cand->short_name || strcmp(cand->short_name, method_name) != 0) continue; if (cand->receiver_type) @@ -1522,8 +1522,12 @@ static const char *cs_resolve_callable_name(CSLSPContext *ctx, const char *name) * generated fixtures can register receiverless callables. Only accept a * unique short-name match; ambiguity is intentionally not guessed. */ const CBMRegisteredFunc *only = NULL; - for (int i = 0; ctx->registry && i < ctx->registry->func_count; i++) { - const CBMRegisteredFunc *candidate = &ctx->registry->funcs[i]; + CBMFreeFuncIter all_funcs; + if (ctx->registry) { + cbm_registry_all_funcs_chain(ctx->registry, &all_funcs); + } + for (int i = -1; ctx->registry && (i = cbm_free_func_iter_next(&all_funcs)) >= 0;) { + const CBMRegisteredFunc *candidate = &all_funcs.reg->funcs[i]; if (candidate->receiver_type || !candidate->short_name || strcmp(candidate->short_name, name) != 0) { continue; @@ -2142,8 +2146,12 @@ static void cs_resolve_invocation(CSLSPContext *ctx, TSNode call) { /* Last resort: any free function with this short name in registry. */ const CBMRegisteredFunc *best = NULL; int best_score = -1; - for (int i = 0; ctx->registry && i < ctx->registry->func_count; i++) { - const CBMRegisteredFunc *cand = &ctx->registry->funcs[i]; + CBMFreeFuncIter all_funcs; + if (ctx->registry) { + cbm_registry_all_funcs_chain(ctx->registry, &all_funcs); + } + for (int i = -1; ctx->registry && (i = cbm_free_func_iter_next(&all_funcs)) >= 0;) { + const CBMRegisteredFunc *cand = &all_funcs.reg->funcs[i]; if (cand->receiver_type) continue; if (!cand->short_name || strcmp(cand->short_name, bare) != 0) diff --git a/internal/cbm/lsp/go_lsp.c b/internal/cbm/lsp/go_lsp.c index 7efa273e1..7fec20a9f 100644 --- a/internal/cbm/lsp/go_lsp.c +++ b/internal/cbm/lsp/go_lsp.c @@ -1551,8 +1551,10 @@ static void resolve_calls_in_node_inner(GoLSPContext* ctx, TSNode node) { int impl_count = 0; // Skip stdlib types when interface is from a project package bool iface_is_project = iface_qn && strchr(iface_qn, '/') != NULL; - for (int ti = 0; ti < ctx->registry->type_count && impl_count < 2; ti++) { - const CBMRegisteredType* cand = &ctx->registry->types[ti]; + CBMTypeShortIter all_types; + cbm_registry_all_types_chain(ctx->registry, &all_types); + for (int ti = -1; impl_count < 2 && (ti = cbm_type_short_iter_next(&all_types)) >= 0;) { + const CBMRegisteredType* cand = &all_types.reg->types[ti]; if (cand->is_interface) continue; if (!cand->qualified_name) continue; if (cand->alias_of) continue; diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.c index 6dbc126e1..8a0396115 100644 --- a/internal/cbm/lsp/py_lsp.c +++ b/internal/cbm/lsp/py_lsp.c @@ -238,6 +238,7 @@ void py_lsp_init(PyLSPContext *ctx, CBMArena *arena, const char *source, int sou ctx->source = source; ctx->source_len = source_len; ctx->registry = registry; + ctx->registry_head = (CBMTypeRegistry *)registry; ctx->module_qn = module_qn; ctx->resolved_calls = out; ctx->current_scope = py_scope_push_checked(ctx); @@ -394,7 +395,7 @@ static void py_import_syntax_match_add(PyImportSyntaxMatch *match, PyDirectImpor * registry then retains responsibility for proving that the exact target is * an undecorated, unambiguous callable. */ static const char *py_canonical_from_import_qn(PyLSPContext *ctx, TSNode statement, - TSNode imported_name, const char *incoming_qn) { + TSNode imported_name, const char *incoming_qn) { if (!ctx || !incoming_qn || ts_node_is_null(statement) || ts_node_is_null(imported_name)) return NULL; TSNode module = py_from_import_module_node(statement); @@ -407,10 +408,9 @@ static const char *py_canonical_from_import_qn(PyLSPContext *ctx, TSNode stateme while (module_suffix && *module_suffix == '.') module_suffix++; - const char *full_suffix = - module_suffix && module_suffix[0] - ? cbm_arena_sprintf(ctx->arena, "%s.%s", module_suffix, member) - : member; + const char *full_suffix = module_suffix && module_suffix[0] + ? cbm_arena_sprintf(ctx->arena, "%s.%s", module_suffix, member) + : member; if (!full_suffix) return NULL; if (py_qn_has_boundary_suffix(incoming_qn, full_suffix)) @@ -445,17 +445,16 @@ static void py_import_match_statement(PyLSPContext *ctx, TSNode stmt, const char TSNode alias = ts_node_child_by_field_name(item, "alias", 5); if (py_import_node_text_equals(ctx, alias, local)) { const char *canonical = py_canonical_from_import_qn(ctx, stmt, name, qn); - py_import_syntax_match_add( - match, canonical ? PY_FROM_IMPORT : PY_IMPORT_UNCLASSIFIED, - canonical ? canonical : qn); + py_import_syntax_match_add(match, + canonical ? PY_FROM_IMPORT : PY_IMPORT_UNCLASSIFIED, + canonical ? canonical : qn); } - } else if ((strcmp(kind, "identifier") == 0 || - strcmp(kind, "dotted_name") == 0) && + } else if ((strcmp(kind, "identifier") == 0 || strcmp(kind, "dotted_name") == 0) && py_import_node_text_equals(ctx, item, local)) { const char *canonical = py_canonical_from_import_qn(ctx, stmt, item, qn); - py_import_syntax_match_add( - match, canonical ? PY_FROM_IMPORT : PY_IMPORT_UNCLASSIFIED, - canonical ? canonical : qn); + py_import_syntax_match_add(match, + canonical ? PY_FROM_IMPORT : PY_IMPORT_UNCLASSIFIED, + canonical ? canonical : qn); } } return; @@ -489,8 +488,7 @@ static void py_import_match_statement(PyLSPContext *ctx, TSNode stmt, const char } } -static PyDirectImportKind py_import_match_result(PyImportSyntaxMatch *match, - const char **qn_io) { +static PyDirectImportKind py_import_match_result(PyImportSyntaxMatch *match, const char **qn_io) { if (!match || !qn_io) return PY_DIRECT_IMPORT_UNKNOWN; if (match->count == 0) @@ -503,8 +501,7 @@ static PyDirectImportKind py_import_match_result(PyImportSyntaxMatch *match, } static PyDirectImportKind py_import_kind_from_statement(PyLSPContext *ctx, TSNode stmt, - const char *local, - const char **qn_io) { + const char *local, const char **qn_io) { const char *qn = qn_io ? *qn_io : NULL; if (!ctx || !local || !qn || !qn_io || ts_node_is_null(stmt)) return PY_DIRECT_IMPORT_UNKNOWN; @@ -513,8 +510,8 @@ static PyDirectImportKind py_import_kind_from_statement(PyLSPContext *ctx, TSNod return py_import_match_result(&match, qn_io); } -static PyDirectImportKind py_import_kind_from_ast(PyLSPContext *ctx, TSNode root, - const char *local, const char **qn_io) { +static PyDirectImportKind py_import_kind_from_ast(PyLSPContext *ctx, TSNode root, const char *local, + const char **qn_io) { const char *qn = qn_io ? *qn_io : NULL; if (!ctx || !local || !qn || !qn_io || ts_node_is_null(root)) return PY_DIRECT_IMPORT_UNKNOWN; @@ -589,36 +586,34 @@ static void py_bind_import_index(PyLSPContext *ctx, int index, bool synthetic_fa if (!local || !qn || strcmp(local, "*") == 0) return; - PyDirectImportKind direct_kind = ctx->import_kinds - ? (PyDirectImportKind)ctx->import_kinds[index] - : PY_DIRECT_IMPORT_UNKNOWN; + PyDirectImportKind direct_kind = + ctx->import_kinds ? (PyDirectImportKind)ctx->import_kinds[index] : PY_DIRECT_IMPORT_UNKNOWN; bool from_style = direct_kind == PY_FROM_IMPORT || - (direct_kind == PY_DIRECT_IMPORT_UNKNOWN && - import_is_from_style(local, qn)); + (direct_kind == PY_DIRECT_IMPORT_UNKNOWN && import_is_from_style(local, qn)); const CBMType *t; if (direct_kind == PY_DIRECT_IMPORT_UNALIASED) { - // Python binds only the root of an unaliased dotted import. + // Python binds only the root of an unaliased dotted import. t = cbm_type_module(ctx->arena, local); } else if (direct_kind == PY_DIRECT_IMPORT_ALIASED) { t = cbm_type_module(ctx->arena, qn); } else if (from_style) { - // `from X import Y` — bind Y to NAMED(X.Y). Phase 6 attribute - // resolution checks the registry to upgrade to MODULE / class - // / function as appropriate. + // `from X import Y` — bind Y to NAMED(X.Y). Phase 6 attribute + // resolution checks the registry to upgrade to MODULE / class + // / function as appropriate. t = cbm_type_named(ctx->arena, qn); } else if (strchr(qn, '.') != NULL) { - // Dotted path whose tail does NOT match the local name: an - // ALIASED binding — `from X import Y as Z` (Z names function/ - // class X.Y) or `import a.b as z` (z names module a.b). The - // CBMImport shape cannot distinguish the two, but NAMED(qn) - // covers both: phase 6 upgrades it to the registered function/ - // class for the from-import and to MODULE for the module alias. - // Binding MODULE here made `g()` calls on `from m import f as g` - // resolve as calls on a module — lsp=MISS, and the whole CALLS - // edge was lost (#988). + // Dotted path whose tail does NOT match the local name: an + // ALIASED binding — `from X import Y as Z` (Z names function/ + // class X.Y) or `import a.b as z` (z names module a.b). The + // CBMImport shape cannot distinguish the two, but NAMED(qn) + // covers both: phase 6 upgrades it to the registered function/ + // class for the from-import and to MODULE for the module alias. + // Binding MODULE here made `g()` calls on `from m import f as g` + // resolve as calls on a module — lsp=MISS, and the whole CALLS + // edge was lost (#988). t = cbm_type_named(ctx->arena, qn); } else { - // `import X` / `import X as Y` (single segment) — MODULE(X). + // `import X` / `import X as Y` (single segment) — MODULE(X). t = cbm_type_module(ctx->arena, qn); } const CBMRegisteredFunc *imported = @@ -841,8 +836,7 @@ static void py_emit_resolved_call(PyLSPContext *ctx, const char *callee_qn, cons } static void py_emit_resolved_reference(PyLSPContext *ctx, const char *callee_qn, - const char *source_name, TSNode site, - const char *strategy) { + const char *source_name, TSNode site, const char *strategy) { if (!ctx || !ctx->resolved_calls || !callee_qn || !ctx->enclosing_func_qn || ts_node_is_null(site)) { return; @@ -1027,8 +1021,7 @@ static void py_resolve_value_references_at(PyLSPContext *ctx, TSNode call) { if (candidate && cbm_scope_contains(ctx->current_scope, source_name)) { const CBMType *binding = cbm_type_resolve_alias(cbm_scope_lookup(ctx->current_scope, source_name)); - if (binding && binding->kind == CBM_TYPE_NAMED && - binding->data.named.qualified_name && + if (binding && binding->kind == CBM_TYPE_NAMED && binding->data.named.qualified_name && strcmp(binding->data.named.qualified_name, candidate) == 0) { py_emit_unresolved_reference(ctx, candidate, arg); } @@ -1195,16 +1188,10 @@ static void py_register_instance_field(PyLSPContext *ctx, const char *class_qn, return; } - // Find the type entry. cbm_registry_lookup_type returns a const pointer; - // we need a mutable pointer into the registry's array. - CBMRegisteredType *rt = NULL; - for (int i = 0; i < ctx->registry->type_count; i++) { - const char *qn = ctx->registry->types[i].qualified_name; - if (qn && strcmp(qn, class_qn) == 0) { - rt = &ctx->registry->types[i]; - break; - } - } + // The writable entry for the class: the head's own, or a copy of the + // base entry made in the head (copy-on-write). The shared base is never + // written -- it used to be, from every parallel resolve worker. + CBMRegisteredType *rt = cbm_registry_type_for_update(ctx->registry_head, class_qn); if (!rt) return; @@ -1570,11 +1557,12 @@ static const CBMType *py_eval_expr_type_uncached(PyLSPContext *ctx, TSNode node) const char *prefix = cbm_arena_sprintf(ctx->arena, "%s.", qn); size_t prefix_len = strlen(prefix); bool is_submodule = false; - for (int i = 0; i < ctx->registry->func_count; i++) { - const char *fqn = ctx->registry->funcs[i].qualified_name; + CBMFreeFuncIter all_funcs; + cbm_registry_all_funcs_chain(ctx->registry, &all_funcs); + for (int i = -1; !is_submodule && (i = cbm_free_func_iter_next(&all_funcs)) >= 0;) { + const char *fqn = all_funcs.reg->funcs[i].qualified_name; if (fqn && strncmp(fqn, prefix, prefix_len) == 0) { is_submodule = true; - break; } } if (is_submodule) @@ -2600,8 +2588,7 @@ static void py_emit_call_for(PyLSPContext *ctx, TSNode call_node) { const char *local = ctx->import_local_names[i]; const char *import_qn = ctx->import_module_qns[i]; if (local && import_qn && strcmp(local, fname) == 0 && - strcmp(import_qn, qn) == 0 && - py_import_index_is_from_binding(ctx, i)) { + strcmp(import_qn, qn) == 0 && py_import_index_is_from_binding(ctx, i)) { imported_alias = true; break; } @@ -2758,8 +2745,8 @@ static void py_emit_call_for(PyLSPContext *ctx, TSNode call_node) { // Skip if mod is already rooted under the project to avoid // "..mod". if (!(strncmp(mod, ctx->module_qn, root_len) == 0 && mod[root_len] == '.')) { - char *qual_mod = (char *)cbm_arena_alloc(ctx->arena, root_len + 1 + - strlen(mod) + 1); + char *qual_mod = + (char *)cbm_arena_alloc(ctx->arena, root_len + 1 + strlen(mod) + 1); if (qual_mod) { memcpy(qual_mod, ctx->module_qn, root_len); qual_mod[root_len] = '.'; @@ -4047,10 +4034,8 @@ static void py_process_function(PyLSPContext *ctx, TSNode func_node, const char // For methods, bind `self`/`cls` AFTER param walk so the receiver type // wins over the unannotated `self` / `cls` parameter declaration. if (ctx->enclosing_class_qn) { - py_scope_bind(ctx, "self", - cbm_type_named(ctx->arena, ctx->enclosing_class_qn)); - py_scope_bind(ctx, "cls", - cbm_type_named(ctx->arena, ctx->enclosing_class_qn)); + py_scope_bind(ctx, "self", cbm_type_named(ctx->arena, ctx->enclosing_class_qn)); + py_scope_bind(ctx, "cls", cbm_type_named(ctx->arena, ctx->enclosing_class_qn)); } TSNode body = ts_node_child_by_field_name(func_node, "body", 4); @@ -4197,12 +4182,14 @@ static void py_bind_external_module_classes(PyLSPContext *ctx, TSNode root) { if (!ctx || !ctx->registry || !ctx->module_qn) return; size_t prefix_len = strlen(ctx->module_qn); - for (int i = 0; i < ctx->registry->type_count; i++) { - const CBMRegisteredType *type = &ctx->registry->types[i]; + CBMTypeShortIter all_types; + cbm_registry_all_types_chain(ctx->registry, &all_types); + for (int i = -1; (i = cbm_type_short_iter_next(&all_types)) >= 0;) { + const CBMRegisteredType *type = &all_types.reg->types[i]; const char *qn = type->qualified_name; const char *name = type->short_name; - if (!qn || !name || strncmp(qn, ctx->module_qn, prefix_len) != 0 || - qn[prefix_len] != '.' || py_root_defines_class_named(ctx, root, name)) { + if (!qn || !name || strncmp(qn, ctx->module_qn, prefix_len) != 0 || qn[prefix_len] != '.' || + py_root_defines_class_named(ctx, root, name)) { continue; } py_scope_bind(ctx, name, cbm_type_named(ctx->arena, qn)); @@ -4502,8 +4489,7 @@ static void py_invalidate_possible_bindings(PyLSPContext *ctx, TSNode node, int py_invalidate_possible_bindings(ctx, definition, depth + 1); return; } - if (strcmp(kind, "import_statement") == 0 || - strcmp(kind, "import_from_statement") == 0) { + if (strcmp(kind, "import_statement") == 0 || strcmp(kind, "import_from_statement") == 0) { py_invalidate_import_bindings(ctx, node); return; } @@ -4511,8 +4497,7 @@ static void py_invalidate_possible_bindings(PyLSPContext *ctx, TSNode node, int TSNode right = ts_node_child_by_field_name(node, "right", 5); if (ts_node_is_null(right)) return; - py_invalidate_binding_target(ctx, ts_node_child_by_field_name(node, "left", 4), - depth + 1); + py_invalidate_binding_target(ctx, ts_node_child_by_field_name(node, "left", 4), depth + 1); py_invalidate_possible_bindings(ctx, right, depth + 1); return; } @@ -4523,8 +4508,7 @@ static void py_invalidate_possible_bindings(PyLSPContext *ctx, TSNode node, int py_invalidate_possible_bindings(ctx, right, depth + 1); return; } - if (strcmp(kind, "named_expression") == 0 || - strcmp(kind, "assignment_expression") == 0) { + if (strcmp(kind, "named_expression") == 0 || strcmp(kind, "assignment_expression") == 0) { TSNode name = ts_node_child_by_field_name(node, "name", 4); if (ts_node_is_null(name)) name = ts_node_child_by_field_name(node, "left", 4); @@ -4619,8 +4603,7 @@ static void py_replay_import_local(PyLSPContext *ctx, TSNode stmt, const char *l continue; } const char *candidate_qn = ctx->import_module_qns[i]; - PyDirectImportKind kind = - py_import_kind_from_statement(ctx, stmt, local, &candidate_qn); + PyDirectImportKind kind = py_import_kind_from_statement(ctx, stmt, local, &candidate_qn); if (!py_replayable_import_kind(kind)) continue; if (!chosen_qn) { @@ -4642,8 +4625,7 @@ static void py_replay_import_local(PyLSPContext *ctx, TSNode stmt, const char *l py_bind_import_index(ctx, chosen, false); } -static void py_replay_import_statement(PyLSPContext *ctx, TSNode stmt, - unsigned char *consumed) { +static void py_replay_import_statement(PyLSPContext *ctx, TSNode stmt, unsigned char *consumed) { if (py_import_statement_is_wildcard(stmt)) { py_invalidate_module_bindings_for_wildcard(ctx); return; @@ -4686,8 +4668,7 @@ void py_lsp_process_file(PyLSPContext *ctx, TSNode root) { py_bind_external_module_classes(ctx, root); unsigned char *consumed_imports = NULL; if (ctx->import_count > 0) { - consumed_imports = - (unsigned char *)cbm_arena_alloc(ctx->arena, (size_t)ctx->import_count); + consumed_imports = (unsigned char *)cbm_arena_alloc(ctx->arena, (size_t)ctx->import_count); if (consumed_imports) memset(consumed_imports, 0, (size_t)ctx->import_count); } @@ -4700,8 +4681,7 @@ void py_lsp_process_file(PyLSPContext *ctx, TSNode root) { for (uint32_t i = 0; i < nc; i++) { TSNode c = ts_node_named_child(root, i); const char *ck = ts_node_type(c); - if (strcmp(ck, "import_statement") == 0 || - strcmp(ck, "import_from_statement") == 0) { + if (strcmp(ck, "import_statement") == 0 || strcmp(ck, "import_from_statement") == 0) { py_replay_import_statement(ctx, c, consumed_imports); } else if (strcmp(ck, "function_definition") == 0) { py_bind_module_function(ctx, c); diff --git a/internal/cbm/lsp/py_lsp.h b/internal/cbm/lsp/py_lsp.h index 96a85dc5d..939025100 100644 --- a/internal/cbm/lsp/py_lsp.h +++ b/internal/cbm/lsp/py_lsp.h @@ -44,6 +44,10 @@ typedef struct { const char *source; int source_len; const CBMTypeRegistry *registry; + /* Writable head of the registry chain (the per-file overlay, or the + * per-file registry on the non-cross path); refinements go here via + * cbm_registry_*_for_update, never into anything behind ->fallback. */ + CBMTypeRegistry *registry_head; CBMScope *current_scope; // Import map: local_name -> module_qn (arena-allocated, NULL-terminated). diff --git a/internal/cbm/lsp/type_registry.c b/internal/cbm/lsp/type_registry.c index 30c551899..4e2f02012 100644 --- a/internal/cbm/lsp/type_registry.c +++ b/internal/cbm/lsp/type_registry.c @@ -301,6 +301,9 @@ static void build_ffunc_short_index(CBMTypeRegistry *reg, CBMArena *idx_arena) { void cbm_registry_types_by_short_name(const CBMTypeRegistry *reg, const char *short_name, CBMTypeShortIter *out) { out->reg = reg; + out->chain = false; + out->key = short_name; + out->shadow = NULL; out->hash = fnv1a(short_name); if (reg->type_qn_buckets && reg->type_qn_bucket_count > 0) { if (reg->type_short_buckets && reg->type_short_bucket_count > 0) { @@ -319,7 +322,7 @@ void cbm_registry_types_by_short_name(const CBMTypeRegistry *reg, const char *sh } } -int cbm_type_short_iter_next(CBMTypeShortIter *it) { +static int type_short_next_local(CBMTypeShortIter *it) { const CBMTypeRegistry *reg = it->reg; while (it->chain_idx >= 0) { const CBMRegistryHashEntry *e = ®->type_short_entries[it->chain_idx]; @@ -335,9 +338,79 @@ int cbm_type_short_iter_next(CBMTypeShortIter *it) { return -1; } +static const CBMRegisteredType *lookup_type_self(const CBMTypeRegistry *reg, + const char *qualified_name); +static const CBMRegisteredFunc *lookup_func_self(const CBMTypeRegistry *reg, + const char *qualified_name); + +/* A base entry is shadowed when the head holds an entry with the same QN. */ +static bool type_shadowed(const CBMTypeRegistry *shadow, const CBMTypeRegistry *reg, int i) { + if (!shadow || shadow == reg || !reg->types[i].qualified_name) { + return false; + } + return lookup_type_self(shadow, reg->types[i].qualified_name) != NULL; +} +static bool func_shadowed(const CBMTypeRegistry *shadow, const CBMTypeRegistry *reg, int i) { + if (!shadow || shadow == reg || !reg->funcs[i].qualified_name) { + return false; + } + return lookup_func_self(shadow, reg->funcs[i].qualified_name) != NULL; +} + +static void type_short_open_linear(const CBMTypeRegistry *reg, CBMTypeShortIter *out) { + out->reg = reg; + out->hash = 0; + out->chain_idx = -1; + out->tail_i = 0; + out->tail_end = reg->type_count; +} + +int cbm_type_short_iter_next(CBMTypeShortIter *it) { + for (;;) { + int p = type_short_next_local(it); + if (p >= 0) { + if (it->chain && type_shadowed(it->shadow, it->reg, p)) { + continue; + } + return p; + } + if (!it->chain || !it->reg->fallback) { + return -1; + } + const CBMTypeRegistry *next = it->reg->fallback; + const char *key = it->key; + const CBMTypeRegistry *shadow = it->shadow; + if (key) { + cbm_registry_types_by_short_name(next, key, it); + } else { + type_short_open_linear(next, it); + } + it->chain = true; + it->key = key; + it->shadow = shadow; + } +} + +void cbm_registry_types_by_short_name_chain(const CBMTypeRegistry *head, const char *short_name, + CBMTypeShortIter *out) { + cbm_registry_types_by_short_name(head, short_name, out); + out->chain = true; + out->shadow = head; +} + +void cbm_registry_all_types_chain(const CBMTypeRegistry *head, CBMTypeShortIter *out) { + type_short_open_linear(head, out); + out->chain = true; + out->key = NULL; + out->shadow = head; +} + void cbm_registry_types_by_embedded_bare(const CBMTypeRegistry *reg, const char *bare, CBMTypeEmbedIter *out) { out->reg = reg; + out->chain = false; + out->key = bare; + out->shadow = NULL; out->hash = fnv1a(bare); out->prev_type = -1; if (reg->type_qn_buckets && reg->type_qn_bucket_count > 0) { @@ -358,7 +431,7 @@ void cbm_registry_types_by_embedded_bare(const CBMTypeRegistry *reg, const char } } -int cbm_type_embed_iter_next(CBMTypeEmbedIter *it) { +static int type_embed_next_local(CBMTypeEmbedIter *it) { const CBMTypeRegistry *reg = it->reg; while (it->chain_idx >= 0) { const CBMRegistryHashEntry *e = ®->type_embed_entries[it->chain_idx]; @@ -377,9 +450,40 @@ int cbm_type_embed_iter_next(CBMTypeEmbedIter *it) { return -1; } +int cbm_type_embed_iter_next(CBMTypeEmbedIter *it) { + for (;;) { + int p = type_embed_next_local(it); + if (p >= 0) { + if (it->chain && type_shadowed(it->shadow, it->reg, p)) { + continue; + } + return p; + } + if (!it->chain || !it->reg->fallback) { + return -1; + } + const CBMTypeRegistry *next = it->reg->fallback; + const char *key = it->key; + const CBMTypeRegistry *shadow = it->shadow; + cbm_registry_types_by_embedded_bare(next, key, it); + it->chain = true; + it->shadow = shadow; + } +} + +void cbm_registry_types_by_embedded_bare_chain(const CBMTypeRegistry *head, const char *bare, + CBMTypeEmbedIter *out) { + cbm_registry_types_by_embedded_bare(head, bare, out); + out->chain = true; + out->shadow = head; +} + void cbm_registry_free_funcs_by_short_name(const CBMTypeRegistry *reg, const char *short_name, CBMFreeFuncIter *out) { out->reg = reg; + out->chain = false; + out->key = short_name; + out->shadow = NULL; out->hash = fnv1a(short_name); if (reg->func_qn_buckets && reg->func_qn_bucket_count > 0) { if (reg->ffunc_short_buckets && reg->ffunc_short_bucket_count > 0) { @@ -397,7 +501,7 @@ void cbm_registry_free_funcs_by_short_name(const CBMTypeRegistry *reg, const cha } } -int cbm_free_func_iter_next(CBMFreeFuncIter *it) { +static int free_func_next_local(CBMFreeFuncIter *it) { const CBMTypeRegistry *reg = it->reg; while (it->chain_idx >= 0) { const CBMRegistryHashEntry *e = ®->ffunc_short_entries[it->chain_idx]; @@ -413,9 +517,57 @@ int cbm_free_func_iter_next(CBMFreeFuncIter *it) { return -1; } +static void free_func_open_linear(const CBMTypeRegistry *reg, CBMFreeFuncIter *out) { + out->reg = reg; + out->hash = 0; + out->chain_idx = -1; + out->tail_i = 0; + out->tail_end = reg->func_count; +} + +int cbm_free_func_iter_next(CBMFreeFuncIter *it) { + for (;;) { + int p = free_func_next_local(it); + if (p >= 0) { + if (it->chain && func_shadowed(it->shadow, it->reg, p)) { + continue; + } + return p; + } + if (!it->chain || !it->reg->fallback) { + return -1; + } + const CBMTypeRegistry *next = it->reg->fallback; + const char *key = it->key; + const CBMTypeRegistry *shadow = it->shadow; + if (key) { + cbm_registry_free_funcs_by_short_name(next, key, it); + } else { + free_func_open_linear(next, it); + } + it->chain = true; + it->key = key; + it->shadow = shadow; + } +} + +void cbm_registry_free_funcs_by_short_name_chain(const CBMTypeRegistry *head, + const char *short_name, CBMFreeFuncIter *out) { + cbm_registry_free_funcs_by_short_name(head, short_name, out); + out->chain = true; + out->shadow = head; +} + +void cbm_registry_all_funcs_chain(const CBMTypeRegistry *head, CBMFreeFuncIter *out) { + free_func_open_linear(head, out); + out->chain = true; + out->key = NULL; + out->shadow = head; +} + void cbm_registry_methods(const CBMTypeRegistry *reg, const char *receiver_qn, const char *method_name, CBMMethodIter *out) { - memset(out, 0, sizeof(*out)); + memset(out, 0, sizeof(*out)); /* chain=false, shadow=NULL */ out->reg = reg; out->receiver_qn = receiver_qn; out->method_name = method_name; @@ -435,10 +587,7 @@ void cbm_registry_methods(const CBMTypeRegistry *reg, const char *receiver_qn, } } -int cbm_method_iter_next(CBMMethodIter *it) { - if (!it || !it->reg || !it->receiver_qn || !it->method_name) { - return -1; - } +static int method_next_local(CBMMethodIter *it) { const CBMTypeRegistry *reg = it->reg; while (it->chain_idx >= 0) { const CBMRegistryHashEntry *e = ®->method_entries[it->chain_idx]; @@ -465,6 +614,80 @@ int cbm_method_iter_next(CBMMethodIter *it) { return -1; } +int cbm_method_iter_next(CBMMethodIter *it) { + if (!it || !it->reg || !it->receiver_qn || !it->method_name) { + return -1; + } + for (;;) { + int p = method_next_local(it); + if (p >= 0) { + if (it->chain && func_shadowed(it->shadow, it->reg, p)) { + continue; + } + return p; + } + if (!it->chain || !it->reg->fallback) { + return -1; + } + const CBMTypeRegistry *next = it->reg->fallback; + const CBMTypeRegistry *shadow = it->shadow; + cbm_registry_methods(next, it->receiver_qn, it->method_name, it); + it->chain = true; + it->shadow = shadow; + } +} + +void cbm_registry_methods_chain(const CBMTypeRegistry *head, const char *receiver_qn, + const char *method_name, CBMMethodIter *out) { + cbm_registry_methods(head, receiver_qn, method_name, out); + out->chain = true; + out->shadow = head; +} + +CBMRegisteredFunc *cbm_registry_func_for_update(CBMTypeRegistry *head, const char *qualified_name) { + if (!head || !qualified_name || head->read_only) { + return NULL; + } + const CBMRegisteredFunc *own = lookup_func_self(head, qualified_name); + if (own) { + return (CBMRegisteredFunc *)own; + } + for (const CBMTypeRegistry *r = head->fallback; r; r = r->fallback) { + const CBMRegisteredFunc *base = lookup_func_self(r, qualified_name); + if (base) { + int before = head->func_count; + cbm_registry_add_func(head, *base); + if (head->func_count == before + 1) { + return &head->funcs[before]; + } + return NULL; + } + } + return NULL; +} + +CBMRegisteredType *cbm_registry_type_for_update(CBMTypeRegistry *head, const char *qualified_name) { + if (!head || !qualified_name || head->read_only) { + return NULL; + } + const CBMRegisteredType *own = lookup_type_self(head, qualified_name); + if (own) { + return (CBMRegisteredType *)own; + } + for (const CBMTypeRegistry *r = head->fallback; r; r = r->fallback) { + const CBMRegisteredType *base = lookup_type_self(r, qualified_name); + if (base) { + int before = head->type_count; + cbm_registry_add_type(head, *base); + if (head->type_count == before + 1) { + return &head->types[before]; + } + return NULL; + } + } + return NULL; +} + void cbm_registry_finalize_into(CBMTypeRegistry *reg, CBMArena *idx_arena) { if (!reg || !idx_arena) return; diff --git a/internal/cbm/lsp/type_registry.h b/internal/cbm/lsp/type_registry.h index 7f7737448..c752350b1 100644 --- a/internal/cbm/lsp/type_registry.h +++ b/internal/cbm/lsp/type_registry.h @@ -242,6 +242,16 @@ typedef struct { int chain_idx; int tail_i; int tail_end; + /* Chain mode (see cbm_registry_*_chain): when this registry is exhausted + * the iterator re-opens on reg->fallback; `reg` then names the registry + * the yielded index belongs to, so callers deref it->reg->types[i], never + * the registry they started from. `key` re-opens the same query on the + * next link; NULL = linear scan. `shadow` is the head: fallback entries + * whose qualified_name the head also holds are skipped (an overlay copy + * hides its base original). */ + bool chain; + const char *key; + const CBMTypeRegistry *shadow; } CBMTypeShortIter; void cbm_registry_types_by_short_name(const CBMTypeRegistry *reg, const char *short_name, CBMTypeShortIter *out); @@ -263,6 +273,9 @@ typedef struct { int tail_i; // next tail/linear type index int tail_end; // reg->type_count snapshot int prev_type; // last yielded type index (adjacent-dedup); -1 = none + bool chain; + const char *key; + const CBMTypeRegistry *shadow; } CBMTypeEmbedIter; void cbm_registry_types_by_embedded_bare(const CBMTypeRegistry *reg, const char *bare, CBMTypeEmbedIter *out); @@ -277,6 +290,9 @@ typedef struct { int chain_idx; int tail_i; int tail_end; + bool chain; + const char *key; + const CBMTypeRegistry *shadow; } CBMFreeFuncIter; void cbm_registry_free_funcs_by_short_name(const CBMTypeRegistry *reg, const char *short_name, @@ -296,11 +312,41 @@ typedef struct { int chain_idx; int tail_i; int tail_end; + bool chain; + const CBMTypeRegistry *shadow; } CBMMethodIter; void cbm_registry_methods(const CBMTypeRegistry *reg, const char *receiver_qn, const char *method_name, CBMMethodIter *out); int cbm_method_iter_next(CBMMethodIter *it); +/* ── Chain-aware iteration and copy-on-write (per-file overlay contract) ── + * + * A resolve walk is handed a per-file OVERLAY registry chained (`fallback`) to + * the immutable shared base. Lookups already chain; these make the index + * iterators and the whole-registry scans chain too, and give a walk one way + * to refine an entry: copy it into the head first. Nothing behind `fallback` + * is ever written. Every yielded index belongs to it->reg at that moment. */ +void cbm_registry_types_by_short_name_chain(const CBMTypeRegistry *head, const char *short_name, + CBMTypeShortIter *out); +void cbm_registry_types_by_embedded_bare_chain(const CBMTypeRegistry *head, const char *bare, + CBMTypeEmbedIter *out); +void cbm_registry_free_funcs_by_short_name_chain(const CBMTypeRegistry *head, + const char *short_name, CBMFreeFuncIter *out); +void cbm_registry_methods_chain(const CBMTypeRegistry *head, const char *receiver_qn, + const char *method_name, CBMMethodIter *out); +/* Linear scans over every func / type in the chain (head first, shadowed + * base entries skipped). Same iterator types, same it->reg contract. */ +void cbm_registry_all_funcs_chain(const CBMTypeRegistry *head, CBMFreeFuncIter *out); +void cbm_registry_all_types_chain(const CBMTypeRegistry *head, CBMTypeShortIter *out); + +/* The writable entry for qualified_name: the head's own entry if it has one, + * else a copy of the first fallback entry added to the head (copy-on-write), + * else NULL. NULL also when the head is sealed (read_only) -- a walk must never + * mutate a shared registry, so the caller skips the refinement, exactly as the + * sealed-registry no-op in cbm_registry_add_func does today. */ +CBMRegisteredFunc *cbm_registry_func_for_update(CBMTypeRegistry *head, const char *qualified_name); +CBMRegisteredType *cbm_registry_type_for_update(CBMTypeRegistry *head, const char *qualified_name); + // --- TS-specific helpers (return NULL for types without these signatures) --- // If the type has a call signature (e.g., `interface F { (x:number): string }`), return diff --git a/internal/cbm/result_compact.c b/internal/cbm/result_compact.c new file mode 100644 index 000000000..05470f6e1 --- /dev/null +++ b/internal/cbm/result_compact.c @@ -0,0 +1,460 @@ +/* + * result_compact.c — copy a finished CBMFileResult into one exact-size arena. + * + * Why this exists (Go corpus census, 2026-09-13): extraction writes every + * temporary into the result arena — cbm_node_text copies at 496 call sites, + * per-node QN sprintf, enclosing-QN strings — and GROW_ARRAY leaves each + * previous generation of every record array dead behind it. 14.8 GB written, + * 3.4 GB reachable, and all of it retained until after resolve because the + * result owned the arena. Compaction walks what is reachable, measures it, + * copies it into a single block of exactly that size (strings interned by + * content within the file), and destroys the working arena. + * + * Three passes over one traversal: + * COUNT — number of string references, to size the intern table + * MEASURE — bytes every allocation will take (aligned like the arena does) + * COPY — the same allocations, for real, into the fresh arena + * MEASURE and COPY issue identical allocation sequences, so the block fits + * exactly; any failure leaves the result untouched. + */ + +#include "cbm.h" +#include "foundation/arena.h" +#include "foundation/constants.h" +#include "foundation/mem_core.h" +#include "result_spill.h" /* cbm_result_relocate */ + +#include +#include +#include + +enum { CR_ALIGN = 7, CR_MIN_TABLE = 64, CR_TABLE_LOAD = 2 }; + +typedef enum { CR_COUNT = 0, CR_MEASURE, CR_COPY, CR_RELOCATE } cr_phase_t; + +typedef struct { + const char *src; /* NULL = empty slot */ + char *dst; /* copy in the new arena; NULL until COPY reaches it */ + uint64_t hash; + size_t len; /* strlen(src) */ +} cr_slot_t; + +typedef struct { + cr_phase_t phase; + size_t refs; /* COUNT: string references seen */ + size_t bytes; /* MEASURE: total arena bytes */ + CBMArena *dst; /* COPY: the fresh arena */ + bool failed; + cr_slot_t *slots; + size_t cap; /* power of two */ + /* RELOCATE: pointers in [old_base, old_base + old_len) move by delta. */ + const char *old_base; + size_t old_len; + ptrdiff_t delta; +} cr_ctx_t; + +static bool cr_in_old_block(const cr_ctx_t *c, const void *p) { + const char *cp = (const char *)p; + return cp >= c->old_base && cp < c->old_base + c->old_len; +} + +static size_t cr_aligned(size_t n) { + return (n + CR_ALIGN) & ~(size_t)CR_ALIGN; +} + +static uint64_t cr_hash(const char *s, size_t len) { + uint64_t h = 1469598103934665603ULL; + for (size_t i = 0; i < len; i++) { + h ^= (unsigned char)s[i]; + h *= 1099511628211ULL; + } + return h; +} + +/* Find or insert the slot for s. Never fails: the table is sized from COUNT + * at half load, and the same references are presented again in COPY. */ +static cr_slot_t *cr_slot(cr_ctx_t *c, const char *s) { + size_t len = strlen(s); + uint64_t h = cr_hash(s, len); + size_t mask = c->cap - SKIP_ONE; + size_t i = (size_t)h & mask; + for (;;) { + cr_slot_t *slot = &c->slots[i]; + if (!slot->src) { + slot->src = s; + slot->hash = h; + slot->len = len; + slot->dst = NULL; + return slot; + } + if (slot->hash == h && slot->len == len && + (slot->src == s || memcmp(slot->src, s, len) == 0)) { + return slot; + } + i = (i + SKIP_ONE) & mask; + } +} + +/* A raw allocation of `bytes` in the new arena: MEASURE books it, COPY makes + * it. Zero bytes is no allocation (the arena returns NULL for it too). */ +static void *cr_alloc(cr_ctx_t *c, size_t bytes) { + if (bytes == 0) { + return NULL; + } + if (c->phase == CR_MEASURE) { + c->bytes += cr_aligned(bytes); + return NULL; + } + if (c->phase == CR_COPY) { + void *p = cbm_arena_alloc(c->dst, bytes); + if (!p) { + c->failed = true; + } + return p; + } + return NULL; +} + +/* A string field: interned by content. COPY rewrites the field. */ +static void cr_str(cr_ctx_t *c, const char **field) { + const char *s = *field; + if (!s) { + return; + } + if (c->phase == CR_RELOCATE) { + if (cr_in_old_block(c, s)) { + *field = s + c->delta; + } + return; + } + if (c->phase == CR_COUNT) { + c->refs++; + return; + } + cr_slot_t *slot = cr_slot(c, s); + if (c->phase == CR_MEASURE) { + if (!slot->dst) { + slot->dst = (char *)s; /* mark as booked; reset before COPY */ + c->bytes += cr_aligned(slot->len + SKIP_ONE); + } + return; + } + if (!slot->dst) { + char *copy = (char *)cr_alloc(c, slot->len + SKIP_ONE); + if (!copy) { + return; + } + memcpy(copy, slot->src, slot->len + SKIP_ONE); + slot->dst = copy; + } + *field = slot->dst; +} + +/* A blob (fingerprint, retained source): copied verbatim. */ +static void cr_blob(cr_ctx_t *c, const void **field, size_t bytes) { + if (!*field || bytes == 0) { + return; + } + if (c->phase == CR_RELOCATE) { + if (cr_in_old_block(c, *field)) { + *field = (const char *)*field + c->delta; + } + return; + } + if (c->phase == CR_COPY) { + void *copy = cr_alloc(c, bytes); + if (!copy) { + return; + } + memcpy(copy, *field, bytes); + *field = copy; + } else { + (void)cr_alloc(c, bytes); + } +} + +/* A record array: copied at exact count, then the caller walks its fields. */ +static void cr_array(cr_ctx_t *c, void **items, int count, size_t elem) { + if (!*items || count <= 0) { + if (c->phase == CR_COPY) { + *items = NULL; + } + return; + } + if (c->phase == CR_RELOCATE) { + if (cr_in_old_block(c, *items)) { + *items = (char *)*items + c->delta; + } + return; + } + cr_blob(c, (const void **)items, (size_t)count * elem); +} + +/* A NULL-terminated list of strings: the pointer array plus each string. */ +static void cr_list(cr_ctx_t *c, const char ***field) { + const char **list = *field; + if (!list) { + return; + } + int n = 0; + if (c->phase == CR_RELOCATE) { + /* The array may live in a block we can no longer read at its old + * address: relocate the pointer first, then count and walk it. */ + cr_blob(c, (const void **)field, sizeof(char *)); + list = *field; + while (list[n]) { + n++; + } + for (int i = 0; i < n; i++) { + cr_str(c, &list[i]); + } + return; + } + while (list[n]) { + n++; + } + cr_blob(c, (const void **)field, (size_t)(n + SKIP_ONE) * sizeof(char *)); + const char **walk = *field; /* the copy in COPY, the original otherwise */ + for (int i = 0; i < n && walk; i++) { + cr_str(c, &walk[i]); + } +} + +/* A counted list of strings (signature_param_types). */ +static void cr_counted_list(cr_ctx_t *c, const char ***field, int count) { + if (!*field || count <= 0) { + return; + } + cr_blob(c, (const void **)field, (size_t)count * sizeof(char *)); + /* (RELOCATE: the pointer moved; the walk below now reads the new array.) */ + const char **walk = *field; + for (int i = 0; i < count && walk; i++) { + cr_str(c, &walk[i]); + } +} + +static void cr_walk_def(cr_ctx_t *c, CBMDefinition *d) { + cr_str(c, &d->name); + cr_str(c, &d->qualified_name); + cr_str(c, &d->label); + cr_str(c, &d->file_path); + cr_str(c, &d->signature); + cr_str(c, &d->return_type); + cr_str(c, &d->receiver); + cr_str(c, &d->docstring); + cr_str(c, &d->parent_class); + cr_list(c, &d->decorators); + cr_list(c, &d->base_classes); + cr_list(c, &d->param_names); + cr_list(c, &d->param_types); + cr_counted_list(c, &d->signature_param_types, d->signature_param_count); + cr_list(c, &d->return_types); + cr_str(c, &d->route_path); + cr_str(c, &d->route_method); + cr_blob(c, (const void **)&d->fingerprint, + d->fingerprint_k > 0 ? (size_t)d->fingerprint_k * sizeof(uint32_t) : 0); + cr_str(c, &d->structural_profile); + cr_str(c, &d->body_tokens); + cr_str(c, &d->impl_trait); +} + +static void cr_walk_call(cr_ctx_t *c, CBMCall *call) { + cr_str(c, &call->callee_name); + cr_str(c, &call->enclosing_func_qn); + cr_str(c, &call->first_string_arg); + cr_str(c, &call->second_arg_name); + int argc = call->arg_count; + if (argc > CBM_MAX_CALL_ARGS) { + argc = CBM_MAX_CALL_ARGS; + } + if (call->args && argc > 0) { + cr_array(c, (void **)&call->args, argc, sizeof(CBMCallArg)); + for (int i = 0; i < argc && call->args; i++) { + cr_str(c, &call->args[i].expr); + cr_str(c, &call->args[i].value); + cr_str(c, &call->args[i].keyword); + } + } else if (c->phase == CR_COPY) { + call->args = NULL; + call->arg_count = 0; + } +} + +static void cr_walk(cr_ctx_t *c, CBMFileResult *r) { + cr_array(c, (void **)&r->defs.items, r->defs.count, sizeof(CBMDefinition)); + for (int i = 0; i < r->defs.count && r->defs.items; i++) { + cr_walk_def(c, &r->defs.items[i]); + } + cr_array(c, (void **)&r->calls.items, r->calls.count, sizeof(CBMCall)); + for (int i = 0; i < r->calls.count && r->calls.items; i++) { + cr_walk_call(c, &r->calls.items[i]); + } + cr_array(c, (void **)&r->imports.items, r->imports.count, sizeof(CBMImport)); + for (int i = 0; i < r->imports.count && r->imports.items; i++) { + cr_str(c, &r->imports.items[i].local_name); + cr_str(c, &r->imports.items[i].module_path); + } + cr_array(c, (void **)&r->usages.items, r->usages.count, sizeof(CBMUsage)); + for (int i = 0; i < r->usages.count && r->usages.items; i++) { + cr_str(c, &r->usages.items[i].ref_name); + cr_str(c, &r->usages.items[i].enclosing_func_qn); + } + cr_array(c, (void **)&r->throws.items, r->throws.count, sizeof(CBMThrow)); + for (int i = 0; i < r->throws.count && r->throws.items; i++) { + cr_str(c, &r->throws.items[i].exception_name); + cr_str(c, &r->throws.items[i].enclosing_func_qn); + } + cr_array(c, (void **)&r->rw.items, r->rw.count, sizeof(CBMReadWrite)); + for (int i = 0; i < r->rw.count && r->rw.items; i++) { + cr_str(c, &r->rw.items[i].var_name); + cr_str(c, &r->rw.items[i].enclosing_func_qn); + } + cr_array(c, (void **)&r->type_refs.items, r->type_refs.count, sizeof(CBMTypeRef)); + for (int i = 0; i < r->type_refs.count && r->type_refs.items; i++) { + cr_str(c, &r->type_refs.items[i].type_name); + cr_str(c, &r->type_refs.items[i].enclosing_func_qn); + } + cr_array(c, (void **)&r->env_accesses.items, r->env_accesses.count, sizeof(CBMEnvAccess)); + for (int i = 0; i < r->env_accesses.count && r->env_accesses.items; i++) { + cr_str(c, &r->env_accesses.items[i].env_key); + cr_str(c, &r->env_accesses.items[i].enclosing_func_qn); + } + cr_array(c, (void **)&r->type_assigns.items, r->type_assigns.count, sizeof(CBMTypeAssign)); + for (int i = 0; i < r->type_assigns.count && r->type_assigns.items; i++) { + cr_str(c, &r->type_assigns.items[i].var_name); + cr_str(c, &r->type_assigns.items[i].type_name); + cr_str(c, &r->type_assigns.items[i].enclosing_func_qn); + } + cr_array(c, (void **)&r->impl_traits.items, r->impl_traits.count, sizeof(CBMImplTrait)); + for (int i = 0; i < r->impl_traits.count && r->impl_traits.items; i++) { + cr_str(c, &r->impl_traits.items[i].trait_name); + cr_str(c, &r->impl_traits.items[i].struct_name); + cr_str(c, &r->impl_traits.items[i].struct_qn); + } + cr_array(c, (void **)&r->resolved_calls.items, r->resolved_calls.count, + sizeof(CBMResolvedCall)); + for (int i = 0; i < r->resolved_calls.count && r->resolved_calls.items; i++) { + cr_str(c, &r->resolved_calls.items[i].caller_qn); + cr_str(c, &r->resolved_calls.items[i].callee_qn); + cr_str(c, &r->resolved_calls.items[i].strategy); + cr_str(c, &r->resolved_calls.items[i].reason); + } + cr_array(c, (void **)&r->string_refs.items, r->string_refs.count, sizeof(CBMStringRef)); + for (int i = 0; i < r->string_refs.count && r->string_refs.items; i++) { + cr_str(c, &r->string_refs.items[i].value); + cr_str(c, &r->string_refs.items[i].enclosing_func_qn); + cr_str(c, &r->string_refs.items[i].key_path); + } + cr_array(c, (void **)&r->infra_bindings.items, r->infra_bindings.count, + sizeof(CBMInfraBinding)); + for (int i = 0; i < r->infra_bindings.count && r->infra_bindings.items; i++) { + cr_str(c, &r->infra_bindings.items[i].source_name); + cr_str(c, &r->infra_bindings.items[i].target_url); + cr_str(c, &r->infra_bindings.items[i].broker); + } + cr_array(c, (void **)&r->channels.items, r->channels.count, sizeof(CBMChannel)); + for (int i = 0; i < r->channels.count && r->channels.items; i++) { + cr_str(c, &r->channels.items[i].channel_name); + cr_str(c, &r->channels.items[i].transport); + cr_str(c, &r->channels.items[i].enclosing_func_qn); + } + cr_str(c, &r->module_qn); + cr_str(c, &r->namespace_name); + cr_list(c, &r->exports); + cr_list(c, &r->constants); + cr_list(c, &r->global_vars); + cr_list(c, &r->macros); + cr_str(c, &r->error_msg); + cr_str(c, &r->error_ranges); + cr_blob(c, (const void **)&r->source, r->source ? (size_t)r->source_len + SKIP_ONE : 0); +} + +void cbm_result_relocate(CBMFileResult *result, const char *old_base, size_t len, char *new_base) { + if (!result || !old_base || !new_base || len == 0 || old_base == new_base) { + return; + } + cr_ctx_t c; + memset(&c, 0, sizeof(c)); + c.phase = CR_RELOCATE; + c.old_base = old_base; + c.old_len = len; + c.delta = new_base - old_base; + cr_walk(&c, result); +} + +static size_t cr_pow2_at_least(size_t n) { + size_t cap = CR_MIN_TABLE; + while (cap < n) { + cap *= PAIR_LEN; + } + return cap; +} + +void cbm_result_compact(CBMFileResult *result) { + if (!result || result->arena.nblocks == 0) { + return; + } + cr_ctx_t c; + memset(&c, 0, sizeof(c)); + + /* Work on a copy of the header: every pointer rewrite lands here and the + * caller's result is replaced only once everything succeeded. */ + CBMFileResult tmp = *result; + + c.phase = CR_COUNT; + cr_walk(&c, &tmp); + + c.cap = cr_pow2_at_least(c.refs * CR_TABLE_LOAD + CR_MIN_TABLE); + c.slots = (cr_slot_t *)cbm_calloc(CBM_MEM_CLASS_EXTRACT, c.cap * sizeof(cr_slot_t)); + if (!c.slots) { + return; + } + + c.phase = CR_MEASURE; + cr_walk(&c, &tmp); + for (size_t i = 0; i < c.cap; i++) { + c.slots[i].dst = NULL; /* MEASURE used dst as a booked marker */ + } + + CBMArena fresh; + cbm_arena_init_exact(&fresh, c.bytes); + if (fresh.nblocks == 0) { + cbm_free(CBM_MEM_CLASS_EXTRACT, c.slots); + return; + } + + c.phase = CR_COPY; + c.dst = &fresh; + cr_walk(&c, &tmp); + cbm_free(CBM_MEM_CLASS_EXTRACT, c.slots); + if (c.failed) { + cbm_arena_destroy(&fresh); + return; + } + + /* Exact-count arrays: nothing may append into the dead headroom. */ + tmp.defs.cap = tmp.defs.count; + tmp.calls.cap = tmp.calls.count; + tmp.imports.cap = tmp.imports.count; + tmp.usages.cap = tmp.usages.count; + tmp.throws.cap = tmp.throws.count; + tmp.rw.cap = tmp.rw.count; + tmp.type_refs.cap = tmp.type_refs.count; + tmp.env_accesses.cap = tmp.env_accesses.count; + tmp.type_assigns.cap = tmp.type_assigns.count; + tmp.impl_traits.cap = tmp.impl_traits.count; + tmp.resolved_calls.cap = tmp.resolved_calls.count; + tmp.string_refs.cap = tmp.string_refs.count; + tmp.infra_bindings.cap = tmp.infra_bindings.count; + tmp.channels.cap = tmp.channels.count; + + /* A composite kept its per-unit results only so shallow-copied strings + * stayed valid; every string is now a copy of its own. */ + cbm_result_release_owned(result); + tmp.owned_results = NULL; + tmp.owned_result_count = 0; + + cbm_work_arena_give(&result->arena); /* kept for this thread's next file */ + tmp.arena = fresh; + *result = tmp; +} diff --git a/internal/cbm/result_spill.c b/internal/cbm/result_spill.c new file mode 100644 index 000000000..b75f06fb5 --- /dev/null +++ b/internal/cbm/result_spill.c @@ -0,0 +1,260 @@ +/* + * result_spill.c — see result_spill.h. + * + * Layout on disk, per parked result: + * spill_rec_hdr_t (magic, block length, block base address at park time, + * the CBMFileResult header as it was in memory) + * block bytes (the single compacted arena block) + * + * The header's pointers are meaningless on disk; the loader rebuilds the + * arena at a new address and shifts every pointer by the delta + * (cbm_result_relocate, implemented on the compaction traversal so it sees + * exactly the fields compaction copied). + */ + +#include "result_spill.h" + +#include "foundation/arena.h" +#include "foundation/compat_fs.h" +#include "foundation/compat_thread.h" +#include "foundation/constants.h" +#include "foundation/log.h" +#include "foundation/mem_core.h" + +#include +#include +#include +#include +#ifdef _WIN32 +#include +#define SPILL_PID() ((long)_getpid()) +#define SPILL_SEEK(fp, off) _fseeki64((fp), (long long)(off), SEEK_SET) +#else +#include +#define SPILL_PID() ((long)getpid()) +#define SPILL_SEEK(fp, off) fseeko((fp), (off_t)(off), SEEK_SET) +#endif + +static const uint64_t SPILL_MAGIC = 0x5350494C4C524553ULL; /* "SPILLRES" */ + +typedef struct { + uint64_t magic; + uint64_t block_len; + uint64_t old_base; /* (uintptr_t) of the block when parked */ + CBMFileResult header; +} spill_rec_hdr_t; + +typedef struct { + FILE *fp; + cbm_mutex_t mu; /* reads seek; writes append -- one lock per file */ + char path[CBM_SZ_1K]; + uint64_t end; /* bytes written so far (append offset) */ +} spill_file_t; + +typedef struct { + int writer; /* -1 = empty */ + uint64_t offset; /* record start in that writer's file */ + uint64_t rec_len; /* header + block */ +} spill_slot_t; + +struct cbm_result_spill { + spill_file_t *files; + int writers; + spill_slot_t *slots; + int slot_count; + _Atomic int64_t parked; + _Atomic int64_t bytes; + _Atomic int64_t loads; +}; + +cbm_result_spill_t *cbm_result_spill_open(const char *dir, int writers, int slots) { + if (!dir || !dir[0] || writers <= 0 || slots <= 0) { + return NULL; + } + char spill_dir[CBM_SZ_1K]; + if (snprintf(spill_dir, sizeof(spill_dir), "%s/spill", dir) >= (int)sizeof(spill_dir)) { + return NULL; + } + if (!cbm_mkdir_p(spill_dir, 0700)) { + cbm_log_warn("mem.spill.open_failed", "dir", spill_dir, "reason", "mkdir"); + return NULL; + } + cbm_result_spill_t *sp = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*sp)); + if (!sp) { + return NULL; + } + sp->files = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)writers * sizeof(spill_file_t)); + sp->slots = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)slots * sizeof(spill_slot_t)); + if (!sp->files || !sp->slots) { + cbm_result_spill_close(sp); + return NULL; + } + sp->writers = writers; + sp->slot_count = slots; + for (int i = 0; i < slots; i++) { + sp->slots[i].writer = -1; + } + for (int w = 0; w < writers; w++) { + spill_file_t *f = &sp->files[w]; + snprintf(f->path, sizeof(f->path), "%s/spill-%ld-%d.bin", spill_dir, SPILL_PID(), w); + f->fp = cbm_fopen(f->path, "w+b"); + if (!f->fp) { + cbm_log_warn("mem.spill.open_failed", "path", f->path, "reason", "fopen"); + cbm_result_spill_close(sp); + return NULL; + } + cbm_mutex_init(&f->mu); + } + cbm_log_info("mem.spill.open", "dir", spill_dir, "writers", writers > 0 ? "yes" : "no"); + return sp; +} + +bool cbm_result_spill_has(const cbm_result_spill_t *sp, int slot) { + return sp && slot >= 0 && slot < sp->slot_count && sp->slots[slot].writer >= 0; +} + +bool cbm_result_spill_park(cbm_result_spill_t *sp, int writer, int slot, CBMFileResult *result) { + if (!sp || !result || writer < 0 || writer >= sp->writers || slot < 0 || + slot >= sp->slot_count || sp->slots[slot].writer >= 0) { + return false; + } + /* Only a compacted result is one block with every pointer inside it; a + * result that owns sub-results (embedded languages) has more arenas than + * that and stays in memory. A retained parse tree is a re-parse cache, + * not data: it is dropped with the in-memory result below. */ + if (result->arena.nblocks != 1 || result->owned_result_count != 0) { + return false; + } + spill_file_t *f = &sp->files[writer]; + spill_rec_hdr_t hdr; + memset(&hdr, 0, sizeof(hdr)); + hdr.magic = SPILL_MAGIC; + hdr.block_len = result->arena.used; /* bytes actually written into the block */ + hdr.old_base = (uint64_t)(uintptr_t)result->arena.blocks[0]; + hdr.header = *result; + hdr.header.cached_tree = NULL; /* never on disk: the loader gets no tree */ + cbm_mutex_lock(&f->mu); + uint64_t offset = f->end; + bool ok = SPILL_SEEK(f->fp, offset) == 0 && fwrite(&hdr, sizeof(hdr), 1, f->fp) == 1 && + (hdr.block_len == 0 || fwrite(result->arena.blocks[0], hdr.block_len, 1, f->fp) == 1); + if (ok) { + f->end = offset + sizeof(hdr) + hdr.block_len; + } + cbm_mutex_unlock(&f->mu); + if (!ok) { + cbm_log_warn("mem.spill.write_failed", "path", f->path); + return false; + } + sp->slots[slot].writer = writer; + sp->slots[slot].offset = offset; + sp->slots[slot].rec_len = sizeof(hdr) + hdr.block_len; + atomic_fetch_add_explicit(&sp->parked, 1, memory_order_relaxed); + atomic_fetch_add_explicit(&sp->bytes, (int64_t)(sizeof(hdr) + hdr.block_len), + memory_order_relaxed); + cbm_free_result(result); + return true; +} + +CBMFileResult *cbm_result_spill_load(const cbm_result_spill_t *sp, int slot) { + if (!cbm_result_spill_has(sp, slot)) { + return NULL; + } + const spill_slot_t *s = &sp->slots[slot]; + spill_file_t *f = &sp->files[s->writer]; + spill_rec_hdr_t hdr; + cbm_mutex_lock(&f->mu); + bool ok = SPILL_SEEK(f->fp, s->offset) == 0 && fread(&hdr, sizeof(hdr), 1, f->fp) == 1 && + hdr.magic == SPILL_MAGIC; + CBMFileResult *r = NULL; + if (ok) { + r = cbm_result_alloc(); + if (r) { + *r = hdr.header; + r->cached_tree = NULL; + memset(&r->arena, 0, sizeof(r->arena)); + cbm_arena_init_exact(&r->arena, hdr.block_len ? (size_t)hdr.block_len : 1); + if (r->arena.nblocks == 1 && + (hdr.block_len == 0 || + fread(r->arena.blocks[0], (size_t)hdr.block_len, 1, f->fp) == 1)) { + r->arena.used = (size_t)hdr.block_len; + r->arena.total_alloc = (size_t)hdr.block_len; + } else { + ok = false; + } + } else { + ok = false; + } + } + cbm_mutex_unlock(&f->mu); + if (!ok) { + cbm_log_warn("mem.spill.read_failed", "path", f->path); + if (r) { + cbm_free_result(r); + } + return NULL; + } + cbm_result_relocate(r, (const char *)(uintptr_t)hdr.old_base, (size_t)hdr.block_len, + r->arena.blocks[0]); + atomic_fetch_add_explicit((_Atomic int64_t *)&sp->loads, 1, memory_order_relaxed); + return r; +} + +bool cbm_result_spill_peek_header(const cbm_result_spill_t *sp, int slot, CBMFileResult *out) { + if (!out || !cbm_result_spill_has(sp, slot)) { + return false; + } + const spill_slot_t *s = &sp->slots[slot]; + spill_file_t *f = &sp->files[s->writer]; + spill_rec_hdr_t hdr; + cbm_mutex_lock(&f->mu); + bool ok = SPILL_SEEK(f->fp, s->offset) == 0 && fread(&hdr, sizeof(hdr), 1, f->fp) == 1 && + hdr.magic == SPILL_MAGIC; + cbm_mutex_unlock(&f->mu); + if (ok) { + *out = hdr.header; + } + return ok; +} + +void cbm_result_spill_peek_counts(const cbm_result_spill_t *sp, int slot, int *defs, int *impls) { + CBMFileResult hdr; + bool ok = cbm_result_spill_peek_header(sp, slot, &hdr); + if (defs) { + *defs = ok ? hdr.defs.count : 0; + } + if (impls) { + *impls = ok ? hdr.impl_traits.count : 0; + } +} + +void cbm_result_spill_stats(const cbm_result_spill_t *sp, int64_t *parked, int64_t *bytes, + int64_t *loads) { + if (parked) { + *parked = sp ? atomic_load_explicit(&sp->parked, memory_order_relaxed) : 0; + } + if (bytes) { + *bytes = sp ? atomic_load_explicit(&sp->bytes, memory_order_relaxed) : 0; + } + if (loads) { + *loads = sp ? atomic_load_explicit(&sp->loads, memory_order_relaxed) : 0; + } +} + +void cbm_result_spill_close(cbm_result_spill_t *sp) { + if (!sp) { + return; + } + if (sp->files) { + for (int w = 0; w < sp->writers; w++) { + spill_file_t *f = &sp->files[w]; + if (f->fp) { + (void)fclose(f->fp); + (void)cbm_unlink(f->path); + cbm_mutex_destroy(&f->mu); + } + } + cbm_free(CBM_MEM_CLASS_OTHER, sp->files); + } + cbm_free(CBM_MEM_CLASS_OTHER, sp->slots); + cbm_free(CBM_MEM_CLASS_OTHER, sp); +} diff --git a/internal/cbm/result_spill.h b/internal/cbm/result_spill.h new file mode 100644 index 000000000..8738870aa --- /dev/null +++ b/internal/cbm/result_spill.h @@ -0,0 +1,65 @@ +/* + * result_spill.h — per-file extraction results parked on disk when the + * memory budget is hit. + * + * A compacted CBMFileResult is one exact-size arena block plus a header whose + * pointers all point into that block (cbm_result_compact guarantees it). That + * makes a result a relocatable blob: written as header + block, read back at + * any address by shifting every pointer by the base delta. The store is one + * append-only file per writer thread under the cache directory; loads use + * pread and are safe from any thread. Nothing here decides WHEN to spill -- + * the pipeline does, on the budget signal (see cbm_pipeline_ctx_t.spill). + */ +#ifndef CBM_RESULT_SPILL_H +#define CBM_RESULT_SPILL_H + +#include "cbm.h" + +#include +#include +#include + +typedef struct cbm_result_spill cbm_result_spill_t; + +/* One store for one pipeline run: `writers` append-only files under + * /spill--.bin, `slots` result slots (one per input file). + * NULL on failure (no directory, no space): the caller keeps results in + * memory as before. */ +cbm_result_spill_t *cbm_result_spill_open(const char *dir, int writers, int slots); + +/* Park a compacted result in slot `slot` via writer `writer`, then free the + * in-memory result (a retained parse tree goes with it; a loaded result has + * none). Returns false (and leaves the result untouched) when the result is + * not a single-block compaction, owns sub-results, or the write fails. */ +bool cbm_result_spill_park(cbm_result_spill_t *sp, int writer, int slot, CBMFileResult *result); + +/* True when slot `slot` holds a parked result. */ +bool cbm_result_spill_has(const cbm_result_spill_t *sp, int slot); + +/* Load slot `slot` back into memory: a fresh CBMFileResult the caller owns + * (cbm_free_result). NULL when the slot is empty or the read fails. Safe from + * several threads for different slots. */ +CBMFileResult *cbm_result_spill_load(const cbm_result_spill_t *sp, int slot); + +/* Read only the parked header of slot `slot` into *out: every count in it + * is valid, every pointer meaningless. False when the slot is empty or the + * read fails. Lets a consumer skip the load when there is nothing to read. */ +bool cbm_result_spill_peek_header(const cbm_result_spill_t *sp, int slot, CBMFileResult *out); + +/* Read only the parked header of slot `slot`: how many defs and impl + * relations it holds (the collector sizes its array before loading). */ +void cbm_result_spill_peek_counts(const cbm_result_spill_t *sp, int slot, int *defs, int *impls); + +/* Counters for the log: results parked, bytes on disk, loads served. */ +void cbm_result_spill_stats(const cbm_result_spill_t *sp, int64_t *parked, int64_t *bytes, + int64_t *loads); + +/* Close and delete the files. */ +void cbm_result_spill_close(cbm_result_spill_t *sp); + +/* Relocation primitive used by the loader: every pointer inside `result` + * that points into [old_base, old_base + len) is shifted to the block now at + * `new_base`. Exposed for tests. */ +void cbm_result_relocate(CBMFileResult *result, const char *old_base, size_t len, char *new_base); + +#endif /* CBM_RESULT_SPILL_H */ diff --git a/scripts/lint-memory-core.py b/scripts/lint-memory-core.py index 59e8ba827..18c329383 100644 --- a/scripts/lint-memory-core.py +++ b/scripts/lint-memory-core.py @@ -24,7 +24,7 @@ audit already bit us once on exactly that with fork(. EXEMPT. The core itself and the allocator plumbing it sits on: src/foundation/mem_core.c the route - src/foundation/arena.c bulk allocator that reports INTO the core + (arena.c allocates its blocks THROUGH the core -- class arena -- and is scanned) src/foundation/slab_alloc.c same src/foundation/mem.c policy/measurement, probes with malloc src/foundation/mem_override_*.c the --wrap / override shims @@ -58,9 +58,7 @@ RAW = re.compile(r"(? #include #include +#if defined(__APPLE__) +#include +#endif #include void cbm_arena_init(CBMArena *a) { @@ -28,27 +32,52 @@ void cbm_arena_init_sized(CBMArena *a, size_t block_size) { block_size = CBM_SZ_64; /* minimum sanity */ } a->block_size = block_size; - a->blocks[0] = (char *)malloc(block_size); + a->grow_size = block_size * PAIR_LEN; + a->blocks[0] = (char *)cbm_alloc(CBM_MEM_CLASS_ARENA, block_size); if (a->blocks[0]) { a->block_sizes[0] = block_size; a->nblocks = SKIP_ONE; } } +void cbm_arena_init_exact(CBMArena *a, size_t bytes) { + size_t block = (bytes + ARENA_ALIGN) & ~(size_t)ARENA_ALIGN; + cbm_arena_init_sized(a, block); + a->grow_size = CBM_ARENA_DEFAULT_BLOCK_SIZE; +} + static int arena_grow(CBMArena *a, size_t min_size) { + /* A rewound arena still owns blocks past the cursor: use the next one if + * it fits, otherwise drop it and everything after it and grow fresh. */ + if (a->cur + SKIP_ONE < a->nblocks && a->blocks[a->cur + SKIP_ONE]) { + if (a->block_sizes[a->cur + SKIP_ONE] >= min_size) { + a->cur++; + a->block_size = a->block_sizes[a->cur]; + a->used = 0; + return ARENA_GROW_OK; + } + for (int i = a->cur + SKIP_ONE; i < a->nblocks; i++) { + cbm_free(CBM_MEM_CLASS_ARENA, a->blocks[i]); + a->blocks[i] = NULL; + a->block_sizes[i] = 0; + } + a->nblocks = a->cur + SKIP_ONE; + } if (a->nblocks >= CBM_ARENA_MAX_BLOCKS) { return 0; } - size_t new_size = a->block_size * PAIR_LEN; + size_t new_size = a->grow_size; if (new_size < min_size) { new_size = min_size; } - char *block = (char *)malloc(new_size); + a->grow_size = new_size * PAIR_LEN; + char *block = (char *)cbm_alloc(CBM_MEM_CLASS_ARENA, new_size); if (!block) { return 0; } a->blocks[a->nblocks] = block; a->block_sizes[a->nblocks] = new_size; + a->cur = a->nblocks; a->nblocks++; a->block_size = new_size; a->used = 0; @@ -69,7 +98,7 @@ void *cbm_arena_alloc(CBMArena *a, size_t n) { return NULL; } } - char *ptr = a->blocks[a->nblocks - SKIP_ONE] + a->used; + char *ptr = a->blocks[a->cur] + a->used; a->used += n; a->total_alloc += n; return ptr; @@ -107,10 +136,33 @@ char *cbm_arena_strndup(CBMArena *a, const char *s, size_t len) { return dst; } +/* On macOS every vsnprintf consults the process locale under an unfair lock + * (localeconv_l inside __vfprintf). Eighteen workers formatting type names + * collapsed into that lock on the C# corpus -- the 10 MB JIT test files took + * 68 s each instead of under a second (sampled 2026-09-14) -- and the old + * two-pass form paid it twice per string. Each thread formats with a C locale + * object of its own, so no thread ever waits for another, and the common + * short string is formatted once into a stack buffer. */ +#if defined(__APPLE__) +static _Thread_local locale_t tl_c_locale; +static locale_t arena_c_locale(void) { + if (!tl_c_locale) { + tl_c_locale = newlocale(LC_ALL_MASK, "C", NULL); + } + return tl_c_locale; /* NULL = the global locale, the pre-fix behaviour */ +} +#define ARENA_VSNPRINTF(buf, n, fmt, ap) vsnprintf_l((buf), (n), arena_c_locale(), (fmt), (ap)) +#else +#define ARENA_VSNPRINTF(buf, n, fmt, ap) vsnprintf((buf), (n), (fmt), (ap)) +#endif + +enum { ARENA_SPRINTF_LOCAL = 512 }; + char *cbm_arena_sprintf(CBMArena *a, const char *fmt, ...) { + char local[ARENA_SPRINTF_LOCAL]; va_list args; va_start(args, fmt); - int needed = vsnprintf(NULL, 0, fmt, args); + int needed = ARENA_VSNPRINTF(local, sizeof(local), fmt, args); va_end(args); if (needed < 0) { return NULL; @@ -120,35 +172,59 @@ char *cbm_arena_sprintf(CBMArena *a, const char *fmt, ...) { if (!dst) { return NULL; } + if ((size_t)needed < sizeof(local)) { + memcpy(dst, local, (size_t)needed + SKIP_ONE); + return dst; + } va_start(args, fmt); - vsnprintf(dst, (size_t)needed + SKIP_ONE, fmt, args); + ARENA_VSNPRINTF(dst, (size_t)needed + SKIP_ONE, fmt, args); va_end(args); return dst; } +void cbm_arena_rewind(CBMArena *a) { + if (!a || a->nblocks == 0) { + return; + } + a->cur = 0; + a->block_size = a->block_sizes[0]; + a->used = 0; + a->total_alloc = 0; +} + +size_t cbm_arena_capacity(const CBMArena *a) { + size_t total = 0; + for (int i = 0; a && i < a->nblocks; i++) { + total += a->block_sizes[i]; + } + return total; +} + void cbm_arena_reset(CBMArena *a) { /* Keep first block, free the rest */ for (int i = SKIP_ONE; i < a->nblocks; i++) { - free(a->blocks[i]); + cbm_free(CBM_MEM_CLASS_ARENA, a->blocks[i]); a->blocks[i] = NULL; a->block_sizes[i] = 0; } if (a->nblocks > SKIP_ONE) { a->nblocks = SKIP_ONE; } + a->cur = 0; a->used = 0; a->total_alloc = 0; /* Reset block_size to match surviving block — prevents overflow if * block_size grew during previous allocations (e.g., CBM_SZ_128 → CBM_SZ_256). */ if (a->nblocks == SKIP_ONE) { a->block_size = a->block_sizes[0]; + a->grow_size = a->block_size * PAIR_LEN; } } void cbm_arena_destroy(CBMArena *a) { for (int i = 0; i < a->nblocks; i++) { - free(a->blocks[i]); + cbm_free(CBM_MEM_CLASS_ARENA, a->blocks[i]); } memset(a, 0, sizeof(*a)); } diff --git a/src/foundation/arena.h b/src/foundation/arena.h index 663f07459..8bac689ff 100644 --- a/src/foundation/arena.h +++ b/src/foundation/arena.h @@ -24,6 +24,8 @@ typedef struct { size_t block_size; /* current block capacity */ size_t used; /* bytes used in current block */ size_t total_alloc; /* cumulative bytes allocated (for stats) */ + size_t grow_size; /* size of the NEXT block added; doubles per growth */ + int cur; /* index of the block allocations come from (rewind sets 0) */ } CBMArena; /* Initialize arena with default block size. */ @@ -32,6 +34,14 @@ void cbm_arena_init(CBMArena *a); /* Initialize arena with a custom initial block size. */ void cbm_arena_init_sized(CBMArena *a, size_t block_size); +/* Initialize an arena as ONE block of exactly `bytes` (rounded to alignment), + * with later growth restarting at the default block size rather than doubling + * the exact block. This is the compaction target: a result whose reachable + * data measures N bytes lands in one N-byte block with no tail, and a later + * append (the cross-file LSP pass adds resolved calls) costs one small block, + * not 2N. */ +void cbm_arena_init_exact(CBMArena *a, size_t bytes); + /* Allocate n bytes (8-byte aligned). Returns NULL on OOM. */ void *cbm_arena_alloc(CBMArena *a, size_t n); @@ -50,6 +60,14 @@ char *cbm_arena_sprintf(CBMArena *a, const char *fmt, ...) __attribute__((format /* Reset arena for reuse: keeps first block, frees the rest. */ void cbm_arena_reset(CBMArena *a); +/* Rewind: keep EVERY block, start allocating from the first one again. The + * pages stay mapped and are overwritten by the next use -- no free, no + * purge, no re-commit. This is what a per-worker working arena wants between + * files: the same addresses reused directly. cbm_arena_capacity() says how + * much such an arena holds, so a caller can drop an outsized one. */ +void cbm_arena_rewind(CBMArena *a); +size_t cbm_arena_capacity(const CBMArena *a); + /* Free all blocks. Arena is zeroed after this. */ void cbm_arena_destroy(CBMArena *a); diff --git a/src/foundation/dyn_array.h b/src/foundation/dyn_array.h index 002b285c5..0f11ed01a 100644 --- a/src/foundation/dyn_array.h +++ b/src/foundation/dyn_array.h @@ -20,6 +20,7 @@ #include #include +#include "mem_core.h" /* same directory: units without -Isrc include this header relatively */ /* Declare a dynamic array type for a given element type. */ #define CBM_DYN_ARRAY(T) \ @@ -30,25 +31,45 @@ } /* Push an element. Grows by 2x when full. */ -#define cbm_da_push(da, item) \ - do { \ - if ((da)->count >= (da)->cap) { \ - int _new_cap = (da)->cap ? (da)->cap * 2 : 8; \ - void *_new = realloc((da)->items, (size_t)_new_cap * sizeof(*(da)->items)); \ - if (!_new) \ - break; \ - (da)->items = _new; \ - (da)->cap = _new_cap; \ - } \ - (da)->items[(da)->count++] = (item); \ +#define cbm_da_push(da, item) \ + do { \ + if ((da)->count >= (da)->cap) { \ + int _new_cap = (da)->cap ? (da)->cap * 2 : 8; \ + void *_new = cbm_realloc(CBM_MEM_CLASS_DYN_ARRAY, (da)->items, \ + (size_t)_new_cap * sizeof(*(da)->items)); \ + if (!_new) \ + break; \ + (da)->items = _new; \ + (da)->cap = _new_cap; \ + } \ + (da)->items[(da)->count++] = (item); \ + } while (0) + +/* Push with a caller-chosen first capacity. cbm_da_push starts at 8 slots, + * which is 64 bytes of headroom on every array; an index that keeps one + * small array per key (millions of keys, most holding one or two entries) + * pays that on every key. Growth stays 2x. */ +#define cbm_da_push_min(da, item, min_cap) \ + do { \ + if ((da)->count >= (da)->cap) { \ + int _new_cap = (da)->cap ? (da)->cap * 2 : (min_cap); \ + void *_new = cbm_realloc(CBM_MEM_CLASS_DYN_ARRAY, (da)->items, \ + (size_t)_new_cap * sizeof(*(da)->items)); \ + if (!_new) \ + break; \ + (da)->items = _new; \ + (da)->cap = _new_cap; \ + } \ + (da)->items[(da)->count++] = (item); \ } while (0) /* Push an element with a pointer return (for in-place init). */ -#define cbm_da_push_ptr(da) \ - (((da)->count >= (da)->cap \ - ? ((void)((da)->cap = (da)->cap ? (da)->cap * 2 : 8), \ - (void)((da)->items = realloc((da)->items, (size_t)(da)->cap * sizeof(*(da)->items)))) \ - : (void)0), \ +#define cbm_da_push_ptr(da) \ + (((da)->count >= (da)->cap \ + ? ((void)((da)->cap = (da)->cap ? (da)->cap * 2 : 8), \ + (void)((da)->items = cbm_realloc(CBM_MEM_CLASS_DYN_ARRAY, (da)->items, \ + (size_t)(da)->cap * sizeof(*(da)->items)))) \ + : (void)0), \ &(da)->items[(da)->count++]) /* Pop last element. Returns the element. Undefined if empty. */ @@ -61,24 +82,25 @@ #define cbm_da_clear(da) ((da)->count = 0) /* Free all memory. */ -#define cbm_da_free(da) \ - do { \ - free((da)->items); \ - (da)->items = NULL; \ - (da)->count = 0; \ - (da)->cap = 0; \ +#define cbm_da_free(da) \ + do { \ + cbm_free(CBM_MEM_CLASS_DYN_ARRAY, (da)->items); \ + (da)->items = NULL; \ + (da)->count = 0; \ + (da)->cap = 0; \ } while (0) /* Reserve capacity (grow if needed, never shrink). */ -#define cbm_da_reserve(da, n) \ - do { \ - if ((n) > (da)->cap) { \ - void *_new = realloc((da)->items, (size_t)(n) * sizeof(*(da)->items)); \ - if (_new) { \ - (da)->items = _new; \ - (da)->cap = (n); \ - } \ - } \ +#define cbm_da_reserve(da, n) \ + do { \ + if ((n) > (da)->cap) { \ + void *_new = cbm_realloc(CBM_MEM_CLASS_DYN_ARRAY, (da)->items, \ + (size_t)(n) * sizeof(*(da)->items)); \ + if (_new) { \ + (da)->items = _new; \ + (da)->cap = (n); \ + } \ + } \ } while (0) /* Insert at index, shifting elements right. */ diff --git a/src/foundation/hash_table.c b/src/foundation/hash_table.c index 1ee33e32e..58ff6ac54 100644 --- a/src/foundation/hash_table.c +++ b/src/foundation/hash_table.c @@ -27,11 +27,26 @@ * include below generates static inline functions named cbm_vt_init, * cbm_vt_cleanup, cbm_vt_get, cbm_vt_insert, etc., plus the cbm_vt * struct itself. */ +/* Verstable allocates its bucket/entry blocks through the memory core; the + * table's ctx is the class those blocks are charged to (Verstable hands the + * ctx and the block size to both hooks). Without this the graph buffer's + * 8.5M node keys and 15.8M edge keys on the kernel were memory no class + * could see. */ +static void *ht_alloc_in(size_t size, cbm_mem_class_t *cls) { + return cbm_alloc(*cls, size); +} +static void ht_free_in(void *ptr, size_t size, cbm_mem_class_t *cls) { + (void)size; + cbm_free(*cls, ptr); +} #define NAME cbm_vt #define KEY_TY const char * #define VAL_TY void * #define HASH_FN vt_hash_string #define CMPR_FN vt_cmpr_string +#define CTX_TY cbm_mem_class_t +#define MALLOC_FN ht_alloc_in +#define FREE_FN ht_free_in #include "../../internal/cbm/vendored/verstable/verstable.h" /* The opaque CBMHashTable struct holds the Verstable instance + a @@ -42,16 +57,20 @@ struct CBMHashTable { }; CBMHashTable *cbm_ht_create(uint32_t initial_capacity) { - CBMHashTable *ht = (CBMHashTable *)calloc(CBM_ALLOC_ONE, sizeof(*ht)); + return cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, initial_capacity); +} + +CBMHashTable *cbm_ht_create_in(cbm_mem_class_t cls, uint32_t initial_capacity) { + CBMHashTable *ht = (CBMHashTable *)cbm_calloc(cls, sizeof(*ht)); if (!ht) return NULL; - cbm_vt_init(&ht->vt); + cbm_vt_init(&ht->vt, cls); if (initial_capacity > 0) { /* Reserve enough buckets for the requested entries. Verstable * computes the minimum bucket count internally. */ if (!cbm_vt_reserve(&ht->vt, (size_t)initial_capacity)) { cbm_vt_cleanup(&ht->vt); - free(ht); + cbm_free(cls, ht); return NULL; } } @@ -61,8 +80,9 @@ CBMHashTable *cbm_ht_create(uint32_t initial_capacity) { void cbm_ht_free(CBMHashTable *ht) { if (!ht) return; + cbm_mem_class_t cls = ht->vt.ctx; cbm_vt_cleanup(&ht->vt); - free(ht); + cbm_free(cls, ht); } void *cbm_ht_set(CBMHashTable *ht, const char *key, void *value) { diff --git a/src/foundation/hash_table.h b/src/foundation/hash_table.h index a4a0646c4..17e270924 100644 --- a/src/foundation/hash_table.h +++ b/src/foundation/hash_table.h @@ -15,6 +15,7 @@ #include #include +#include "mem_core.h" /* cbm_mem_class_t; same directory: the lsp_all unit has no -Isrc */ #include /* Opaque — full definition lives in hash_table.c. */ @@ -24,6 +25,11 @@ typedef struct CBMHashTable CBMHashTable; * buckets and avoid early growth; 0 = library default). */ CBMHashTable *cbm_ht_create(uint32_t initial_capacity); +/* Same, with the memory class the table's buckets and entries are charged + * to. cbm_ht_create charges CBM_MEM_CLASS_HASH_TABLE; an owner that wants + * its indexes attributed (the graph buffer: gbuf_index) names its class. */ +CBMHashTable *cbm_ht_create_in(cbm_mem_class_t cls, uint32_t initial_capacity); + /* Free the hash table (does NOT free keys or values). */ void cbm_ht_free(CBMHashTable *ht); diff --git a/src/foundation/mem.c b/src/foundation/mem.c index 5f64bcb25..1c044c071 100644 --- a/src/foundation/mem.c +++ b/src/foundation/mem.c @@ -32,8 +32,12 @@ #include #elif defined(__APPLE__) #include +#include /* malloc_zone_pressure_relief */ #else #include +#if defined(__GLIBC__) +#include /* malloc_trim */ +#endif #endif /* Does THIS build ask mimalloc to replace ordinary malloc process-wide? @@ -508,10 +512,48 @@ void cbm_mem_set_budget_for_tests(size_t bytes) { g_budget = bytes; } +size_t cbm_mem_allocator_committed(void) { + size_t commit = 0; + mi_process_info(NULL, NULL, NULL, NULL, NULL, &commit, NULL, NULL); + return commit; +} + +static _Atomic size_t g_peak_charged; +size_t cbm_mem_charged(void) { + /* The OS number (phys_footprint on macOS, RSS elsewhere) is the charge + * for everything the process maps; the allocator's committed bytes are + * the floor for the memory we hold through mimalloc. macOS was measured + * under-reporting the former after MADV_FREE_REUSABLE cycles (kernel + * extraction: 4.1 GB charged, 15.6 GB committed, 13.4 GB tracked live), + * so the larger of the two is the honest reading. */ +#if defined(__APPLE__) + size_t os_charge = cbm_mem_footprint(); + if (os_charge == 0) { + os_charge = cbm_mem_rss(); /* footprint unavailable: fall back */ + } +#else + size_t os_charge = cbm_mem_rss(); +#endif + size_t committed = cbm_mem_allocator_committed(); + size_t charged = committed > os_charge ? committed : os_charge; + /* High-water mark of the charge itself, at the granularity of the gate + * that reads it (every file pull, every phase mark). RSS high-water + * counts pages already purged to the OS but not yet reclaimed + * (MADV_FREE); this is the number the budget is measured against. */ + size_t seen = atomic_load_explicit(&g_peak_charged, memory_order_relaxed); + while (charged > seen && + !atomic_compare_exchange_weak_explicit(&g_peak_charged, &seen, charged, + memory_order_relaxed, memory_order_relaxed)) {} + return charged; +} +size_t cbm_mem_peak_charged(void) { + return atomic_load_explicit(&g_peak_charged, memory_order_relaxed); +} + bool cbm_mem_over_budget(void) { - size_t rss = cbm_mem_rss(); - check_pressure(rss); - return rss > g_budget; + size_t charged = cbm_mem_charged(); + check_pressure(charged); + return charged > g_budget; } /* Reclaimable memory below this share of total RAM is where paging starts to @@ -541,6 +583,28 @@ void cbm_mem_collect(void) { mi_collect(true); } +size_t cbm_mem_footprint(void) { +#if defined(__APPLE__) + task_vm_info_data_t vm = {0}; + mach_msg_type_number_t count = TASK_VM_INFO_COUNT; + if (task_info(mach_task_self(), TASK_VM_INFO, (task_info_t)&vm, &count) == KERN_SUCCESS) { + return (size_t)vm.phys_footprint; + } + return 0; +#else + return os_rss(); +#endif +} + +void cbm_mem_release_to_os(void) { + mi_collect(true); +#if defined(__APPLE__) + (void)malloc_zone_pressure_relief(NULL, 0); +#elif defined(__GLIBC__) + (void)malloc_trim(0); +#endif +} + /* ── Memory map (see mem.h for how to read the triple) ─────────────── */ static const size_t MEM_MAP_BUCKET_LIMITS[CBM_MEM_MAP_BUCKETS] = { @@ -744,6 +808,10 @@ static bool mem_phase_enabled(void) { return on; } +bool cbm_mem_phases_enabled(void) { + return mem_phase_enabled(); +} + void cbm_mem_phase_mark(const char *label) { if (!mem_phase_enabled()) { return; diff --git a/src/foundation/mem.h b/src/foundation/mem.h index 826744789..9b318fc9e 100644 --- a/src/foundation/mem.h +++ b/src/foundation/mem.h @@ -56,6 +56,24 @@ size_t cbm_mem_rss(void); /* Peak RSS in bytes. */ size_t cbm_mem_peak_rss(void); +/* High-water mark of cbm_mem_charged() as seen by its callers (the extract + * gate reads it per file pull, the phase marks per pass): the peak of the + * budget metric itself, next to the RSS peak that counts reclaimable pages. */ +size_t cbm_mem_peak_charged(void); + +/* The number the budget is enforced against: what the OS charges this + * process. On macOS that is phys_footprint -- resident_size keeps every page + * mimalloc has purged until the kernel reclaims it, and the kernel proof + * (2026-09-13) had the worker at 17.4 GB RSS with 5.5 GB charged after + * extraction. Elsewhere the two are the same number. Falls back to RSS when + * the footprint is unavailable. */ +size_t cbm_mem_charged(void); + +/* Bytes mimalloc currently has committed (its own counter). With the core + * backed by mimalloc this is the allocator-level truth: tracked live bytes + * plus fragmentation and metadata, minus nothing the OS may or may not have + * reclaimed yet. 0 when unavailable. */ +size_t cbm_mem_allocator_committed(void); /* Total budget in bytes. */ size_t cbm_mem_budget(void); @@ -93,6 +111,19 @@ bool cbm_mem_system_under_pressure(void); /* Return unused pages to the OS. Call between files to bound per-file peak. */ void cbm_mem_collect(void); +/* What the OS charges this process for memory-pressure purposes. On macOS + * that is phys_footprint, which EXCLUDES pages the allocator has already + * marked reusable (MADV_FREE_REUSABLE) -- resident_size still counts them, so + * after a large free the two can differ by gigabytes. Elsewhere equals RSS. + * 0 when unavailable. */ +size_t cbm_mem_footprint(void); + +/* Hand freed memory back to the OS on every allocator this process uses: + * mimalloc (mi_collect), the macOS system zones (malloc_zone_pressure_relief) + * and glibc (malloc_trim). Costs a few ms; call at phase boundaries after a + * bulk release, never per allocation. */ +void cbm_mem_release_to_os(void); + /* ── Memory map: where does the process's memory actually live? ────── * * A growth diagnostic that is honest about what it cannot see. Walking the @@ -223,6 +254,10 @@ void cbm_mem_phase_mark(const char *label); /* Drop all accumulated phase totals (call once at the start of a measurement). */ void cbm_mem_phase_reset(void); +/* True when CBM_MEM_PHASES=1 turned phase attribution on for this process. + * Instruments that walk large structures (the result census) gate on it. */ +bool cbm_mem_phases_enabled(void); + /* Write the phase table as a JSON array of {label, bytes, hits}, biggest total * first. Returns bytes written (0 when disabled or empty). */ int cbm_mem_phase_report_json(char *out, size_t size); diff --git a/src/foundation/mem_core.c b/src/foundation/mem_core.c index 8120d7252..dee1fb759 100644 --- a/src/foundation/mem_core.c +++ b/src/foundation/mem_core.c @@ -2,6 +2,10 @@ * mem_core.c — the allocation route. See mem_core.h for why it exists. */ #include "foundation/mem_core.h" +#include "foundation/mem.h" + +/* Ownership check for blocks handed back to the core (defined with cbm_free). */ +static void check_owned(const void *block, const char *op); #include "foundation/constants.h" #include "foundation/log.h" @@ -19,7 +23,14 @@ * mi_usable_size would be undefined behaviour on it. Each platform's own query * is correct under whichever allocator is actually installed, including when * that allocator IS mimalloc via the Linux/MinGW override. */ -#if defined(__APPLE__) +#if defined(CBM_BIND_TS_ALLOCATOR) && CBM_BIND_TS_ALLOCATOR +#include +#define CBM_BACKING_MALLOC(n) mi_malloc(n) +#define CBM_BACKING_CALLOC(n) mi_calloc(CBM_ALLOC_ONE, n) +#define CBM_BACKING_REALLOC(p, n) mi_realloc(p, n) +#define CBM_BACKING_FREE(p) mi_free(p) +#define CBM_USABLE_SIZE(p) mi_usable_size((void *)(p)) +#elif defined(__APPLE__) #include /* malloc_size */ #define CBM_USABLE_SIZE(p) malloc_size(p) #elif defined(_WIN32) @@ -36,6 +47,13 @@ #define CBM_USABLE_SIZE_UNAVAILABLE 1 #endif +#ifndef CBM_BACKING_MALLOC +#define CBM_BACKING_MALLOC(n) malloc(n) +#define CBM_BACKING_CALLOC(n) calloc(CBM_ALLOC_ONE, n) +#define CBM_BACKING_REALLOC(p, n) realloc(p, n) +#define CBM_BACKING_FREE(p) free(p) +#endif + enum { MEM_CORE_REPORT_MIN = 64 }; typedef struct { @@ -47,8 +65,8 @@ typedef struct { static mem_class_stats_t g_classes[CBM_MEM_CLASS_COUNT]; static const char *const g_class_names[CBM_MEM_CLASS_COUNT] = { - "other", "gbuf_node", "gbuf_edge", "gbuf_string", "gbuf_index", - "extract", "semantic", "dump", "store", + "other", "gbuf_node", "gbuf_edge", "gbuf_string", "gbuf_index", "extract", "arena", + "ts_tree", "semantic", "dump", "store", "hash_table", "dyn_array", }; const char *cbm_mem_class_name(cbm_mem_class_t cls) { @@ -68,42 +86,64 @@ static mem_class_stats_t *class_slot(cbm_mem_class_t cls) { return &g_classes[cls]; } -static void class_add(cbm_mem_class_t cls, size_t bytes, size_t blocks) { - mem_class_stats_t *st = class_slot(cls); - size_t now = atomic_fetch_add_explicit(&st->live_bytes, bytes, memory_order_relaxed) + bytes; - if (blocks) { - (void)atomic_fetch_add_explicit(&st->live_blocks, blocks, memory_order_relaxed); +/* ── Accounting: thread-local deltas, shared atomics on flush ────────── + * The hot path (every allocation and free on every worker) touches only + * thread-local memory. The shared per-class counters see one flush per + * MEM_FLUSH_BYTES / MEM_FLUSH_BLOCKS of change per thread, or an explicit + * cbm_mem_class_flush_thread() -- which every parallel-for worker calls when + * its work item ends and every reader calls for its own thread first. With + * one atomic per allocation, 18 workers on 18 cores bounced the same three + * cache lines on every block: Kotlin CPU 38 -> 121 s for a smaller graph, + * Go 177 -> 312 s (bench vs v0.10.8, 2026-09-14). A class's live figure can + * lag a running worker by at most MEM_FLUSH_BYTES; the phase marks read + * after the workers joined, so they are exact. Peaks are recorded at flush + * and are low by at most threads x MEM_FLUSH_BYTES -- a diagnostic. */ +enum { MEM_FLUSH_BYTES = 256 * 1024, MEM_FLUSH_BLOCKS = 512 }; + +typedef struct { + long bytes; /* signed: allocations add, frees subtract */ + long blocks; +} mem_delta_t; + +static _Thread_local mem_delta_t tl_delta[CBM_MEM_CLASS_COUNT]; + +/* live += delta, never wrapping below zero: a mismatched class on free (the + * one way a caller can get this wrong) must not turn a small drift into a + * colossal bogus number that looks like a leak. Returns the new value. */ +static size_t apply_signed(atomic_size_t *counter, long delta) { + if (delta >= 0) { + return atomic_fetch_add_explicit(counter, (size_t)delta, memory_order_relaxed) + + (size_t)delta; } - /* Peak is best-effort under concurrency: a CAS loop here would serialise - * every allocation in the hot path to make a DIAGNOSTIC exact. Racing - * writers can leave the peak one increment low; that never changes a - * decision, and the cost of exactness would. */ - size_t seen = atomic_load_explicit(&st->peak_bytes, memory_order_relaxed); - while (now > seen) { - if (atomic_compare_exchange_weak_explicit(&st->peak_bytes, &seen, now, memory_order_relaxed, + size_t sub = (size_t)(-delta); + size_t seen = atomic_load_explicit(counter, memory_order_relaxed); + while (true) { + size_t want = sub > seen ? 0 : seen - sub; + if (atomic_compare_exchange_weak_explicit(counter, &seen, want, memory_order_relaxed, memory_order_relaxed)) { - break; + return want; } } } -static void class_sub(cbm_mem_class_t cls, size_t bytes, size_t blocks) { - mem_class_stats_t *st = class_slot(cls); - /* Never wrap. A mismatched class on free (the one way a caller can get - * this wrong) would otherwise turn a small drift into a colossal bogus - * number that looks like a leak and sends someone hunting a phantom. */ - size_t seen = atomic_load_explicit(&st->live_bytes, memory_order_relaxed); - while (seen > 0) { - size_t want = bytes > seen ? 0 : seen - bytes; - if (atomic_compare_exchange_weak_explicit(&st->live_bytes, &seen, want, - memory_order_relaxed, memory_order_relaxed)) { - break; - } +static void class_flush_one(cbm_mem_class_t cls) { + mem_delta_t *d = &tl_delta[cls]; + if (d->bytes == 0 && d->blocks == 0) { + return; } - if (blocks) { - size_t b = atomic_load_explicit(&st->live_blocks, memory_order_relaxed); - while (b > 0) { - if (atomic_compare_exchange_weak_explicit(&st->live_blocks, &b, b - 1, + long bytes = d->bytes; + long blocks = d->blocks; + d->bytes = 0; + d->blocks = 0; + mem_class_stats_t *st = &g_classes[cls]; + size_t now = apply_signed(&st->live_bytes, bytes); + (void)apply_signed(&st->live_blocks, blocks); + if (bytes > 0) { + /* Peak is best-effort under concurrency: racing writers can leave it + * one flush low; that never changes a decision. */ + size_t seen = atomic_load_explicit(&st->peak_bytes, memory_order_relaxed); + while (now > seen) { + if (atomic_compare_exchange_weak_explicit(&st->peak_bytes, &seen, now, memory_order_relaxed, memory_order_relaxed)) { break; } @@ -111,11 +151,36 @@ static void class_sub(cbm_mem_class_t cls, size_t bytes, size_t blocks) { } } -/* Charge what the allocator actually handed us where the platform can say, and - * fall back to the request otherwise. Called once per successful allocation. - * Two definitions rather than one with a constant-folded branch: on a platform - * with no usable-size query the fallback simply IS the request, and cppcheck - * rightly objects to a condition that can never be true. */ +void cbm_mem_class_flush_thread(void) { + for (int i = 0; i < CBM_MEM_CLASS_COUNT; i++) { + class_flush_one((cbm_mem_class_t)i); + } +} + +static cbm_mem_class_t class_index(cbm_mem_class_t cls) { + return ((int)cls < 0 || (int)cls >= CBM_MEM_CLASS_COUNT) ? CBM_MEM_CLASS_OTHER : cls; +} + +static void class_add(cbm_mem_class_t cls, size_t bytes, size_t blocks) { + cls = class_index(cls); + mem_delta_t *d = &tl_delta[cls]; + d->bytes += (long)bytes; + d->blocks += (long)blocks; + if (d->bytes >= MEM_FLUSH_BYTES || d->blocks >= MEM_FLUSH_BLOCKS) { + class_flush_one(cls); + } +} + +static void class_sub(cbm_mem_class_t cls, size_t bytes, size_t blocks) { + cls = class_index(cls); + mem_delta_t *d = &tl_delta[cls]; + d->bytes -= (long)bytes; + d->blocks -= (long)blocks; + if (d->bytes <= -MEM_FLUSH_BYTES || d->blocks <= -MEM_FLUSH_BLOCKS) { + class_flush_one(cls); + } +} + #ifdef CBM_USABLE_SIZE_UNAVAILABLE static size_t charge_size(const void *block, size_t requested) { (void)block; @@ -141,7 +206,7 @@ size_t cbm_mem_usable_size(const void *block) { #endif void *cbm_alloc(cbm_mem_class_t cls, size_t bytes) { - void *block = malloc(bytes ? bytes : CBM_ALLOC_ONE); + void *block = CBM_BACKING_MALLOC(bytes ? bytes : CBM_ALLOC_ONE); if (!block) { return NULL; } @@ -150,7 +215,7 @@ void *cbm_alloc(cbm_mem_class_t cls, size_t bytes) { } void *cbm_calloc(cbm_mem_class_t cls, size_t bytes) { - void *block = calloc(CBM_ALLOC_ONE, bytes ? bytes : CBM_ALLOC_ONE); + void *block = CBM_BACKING_CALLOC(bytes ? bytes : CBM_ALLOC_ONE); if (!block) { return NULL; } @@ -162,10 +227,11 @@ void *cbm_realloc(cbm_mem_class_t cls, void *block, size_t bytes) { if (!block) { return cbm_alloc(cls, bytes); } + check_owned(block, "realloc"); /* Measure BEFORE: after realloc the old block is gone and its size is * unknowable, so the decrement has to be computed first. */ size_t old = charge_size(block, 0); - void *next = realloc(block, bytes ? bytes : CBM_ALLOC_ONE); + void *next = CBM_BACKING_REALLOC(block, bytes ? bytes : CBM_ALLOC_ONE); if (!next) { return NULL; /* original intact and still charged - correct */ } @@ -187,12 +253,33 @@ char *cbm_mem_strdup(cbm_mem_class_t cls, const char *s) { return copy; } +/* A block that did not come from the backing allocator reached the core: + * a cross-allocator free (a libc strdup handed to cbm_free, which is mi_free + * in the production build) that no libc-backed test build can see. Checked + * where the backing is mimalloc and only under CBM_MEM_PHASES=1 -- the proof + * runs -- and fatal there: silent heap corruption is the alternative. */ +#if defined(CBM_BIND_TS_ALLOCATOR) && CBM_BIND_TS_ALLOCATOR +static void check_owned(const void *block, const char *op) { + if (!cbm_mem_phases_enabled() || mi_is_in_heap_region(block)) { + return; + } + cbm_log_error("mem.core.foreign_block", "op", op); + abort(); +} +#else +static void check_owned(const void *block, const char *op) { + (void)block; + (void)op; +} +#endif + void cbm_free(cbm_mem_class_t cls, void *block) { if (!block) { return; } + check_owned(block, "free"); class_sub(cls, charge_size(block, 0), CBM_ALLOC_ONE); - free(block); + CBM_BACKING_FREE(block); } void cbm_mem_class_add_external(cbm_mem_class_t cls, size_t bytes) { @@ -204,18 +291,22 @@ void cbm_mem_class_remove_external(cbm_mem_class_t cls, size_t bytes) { } size_t cbm_mem_class_live_bytes(cbm_mem_class_t cls) { + cbm_mem_class_flush_thread(); return atomic_load_explicit(&class_slot(cls)->live_bytes, memory_order_relaxed); } size_t cbm_mem_class_live_blocks(cbm_mem_class_t cls) { + cbm_mem_class_flush_thread(); return atomic_load_explicit(&class_slot(cls)->live_blocks, memory_order_relaxed); } size_t cbm_mem_class_peak_bytes(cbm_mem_class_t cls) { + cbm_mem_class_flush_thread(); return atomic_load_explicit(&class_slot(cls)->peak_bytes, memory_order_relaxed); } size_t cbm_mem_tracked_live_bytes(void) { + cbm_mem_class_flush_thread(); size_t total = 0; for (int i = 0; i < CBM_MEM_CLASS_COUNT; i++) { total += atomic_load_explicit(&g_classes[i].live_bytes, memory_order_relaxed); @@ -224,6 +315,7 @@ size_t cbm_mem_tracked_live_bytes(void) { } void cbm_mem_class_reset_peaks(void) { + cbm_mem_class_flush_thread(); for (int i = 0; i < CBM_MEM_CLASS_COUNT; i++) { size_t live = atomic_load_explicit(&g_classes[i].live_bytes, memory_order_relaxed); atomic_store_explicit(&g_classes[i].peak_bytes, live, memory_order_relaxed); @@ -231,6 +323,7 @@ void cbm_mem_class_reset_peaks(void) { } int cbm_mem_class_report_json(char *out, size_t size) { + cbm_mem_class_flush_thread(); if (!out || size < MEM_CORE_REPORT_MIN) { return 0; } @@ -281,6 +374,7 @@ int cbm_mem_class_report_json(char *out, size_t size) { } void cbm_mem_class_log(const char *tag) { + cbm_mem_class_flush_thread(); char report[CBM_SZ_1K]; if (cbm_mem_class_report_json(report, sizeof(report)) <= 0) { return; diff --git a/src/foundation/mem_core.h b/src/foundation/mem_core.h index 042184ba8..1514c3c68 100644 --- a/src/foundation/mem_core.h +++ b/src/foundation/mem_core.h @@ -57,11 +57,15 @@ typedef enum { CBM_MEM_CLASS_GBUF_NODE, /* node records (~64 B each) */ CBM_MEM_CLASS_GBUF_EDGE, /* edge records (~48 B each) */ CBM_MEM_CLASS_GBUF_STRING, /* name / qualified_name / properties_json */ - CBM_MEM_CLASS_GBUF_INDEX, /* the 8 lookup indexes (~3.77 GB at kernel scale) */ - CBM_MEM_CLASS_EXTRACT, /* per-file working set: arena, tree, source */ - CBM_MEM_CLASS_SEMANTIC, /* embedding vectors (~9.4 GB at kernel scale) */ + CBM_MEM_CLASS_GBUF_INDEX, /* the 8 lookup indexes (383 MB peak on the Go corpus) */ + CBM_MEM_CLASS_EXTRACT, /* per-file working set: source text, extraction scratch */ + CBM_MEM_CLASS_ARENA, /* CBMArena blocks -- every arena, whoever owns it */ + CBM_MEM_CLASS_TS_TREE, /* tree-sitter: parse trees + parser state (bound allocator) */ + CBM_MEM_CLASS_SEMANTIC, /* semantic pass: vectors, token pools, LSH (87 MB on Go) */ CBM_MEM_CLASS_DUMP, /* dump-time transients */ - CBM_MEM_CLASS_STORE, /* store batches and row buffers */ + CBM_MEM_CLASS_STORE, /* SQLite (bound mem methods) + store batches and row buffers */ + CBM_MEM_CLASS_HASH_TABLE, /* CBMHashTable buckets/entries not claimed by an owner class */ + CBM_MEM_CLASS_DYN_ARRAY, /* CBM_DYN_ARRAY item storage (every cbm_da_* user) */ CBM_MEM_CLASS_COUNT } cbm_mem_class_t; @@ -97,6 +101,11 @@ void cbm_free(cbm_mem_class_t cls, void *block); * cbm_mem_tracked_live_bytes() versus the process RSS in mem.h: the gap between * them IS the unmigrated surface, and it should shrink as adoption spreads. An * unmeasured allocation must never read as an absent one. */ +/* Push this thread's pending accounting deltas to the shared counters. Every + * reader does it for its own thread; a parallel-for worker does it when its + * work item ends, so the phase marks (read after the join) are exact. */ +void cbm_mem_class_flush_thread(void); + size_t cbm_mem_class_live_bytes(cbm_mem_class_t cls); size_t cbm_mem_class_live_blocks(cbm_mem_class_t cls); size_t cbm_mem_tracked_live_bytes(void); diff --git a/src/foundation/slab_alloc.c b/src/foundation/slab_alloc.c index 95d23587e..684aa1838 100644 --- a/src/foundation/slab_alloc.c +++ b/src/foundation/slab_alloc.c @@ -39,6 +39,7 @@ #include "foundation/slab_alloc.h" #include "foundation/compat.h" #include "foundation/compat_thread.h" +#include "foundation/mem_core.h" #include #include @@ -177,7 +178,7 @@ static bool slab_map_set(uintptr_t base, slab_page_t *val) { slab_map_unlock(); return true; /* nothing to unregister */ } - l2 = (slab_map_l2_t *)calloc(1, sizeof(*l2)); + l2 = (slab_map_l2_t *)cbm_calloc(CBM_MEM_CLASS_TS_TREE, sizeof(*l2)); if (!l2) { slab_map_unlock(); return false; @@ -190,7 +191,7 @@ static bool slab_map_set(uintptr_t base, slab_page_t *val) { slab_map_unlock(); return true; } - l3 = (slab_map_l3_t *)calloc(1, sizeof(*l3)); + l3 = (slab_map_l3_t *)cbm_calloc(CBM_MEM_CLASS_TS_TREE, sizeof(*l3)); if (!l3) { slab_map_unlock(); return false; @@ -217,6 +218,7 @@ static bool slab_grow(slab_state_t *s) { if (cbm_aligned_alloc(&mem, SLAB_PAGE_SIZE, SLAB_PAGE_SIZE) != 0 || !mem) { return false; } + cbm_mem_class_add_external(CBM_MEM_CLASS_TS_TREE, SLAB_PAGE_SIZE); slab_page_t *page = (slab_page_t *)mem; page->next = s->pages; atomic_init(&page->owner, s); @@ -224,6 +226,7 @@ static bool slab_grow(slab_state_t *s) { atomic_init(&page->refcount, 1u); /* owner guard */ if (!slab_map_register_page(page)) { + cbm_mem_class_remove_external(CBM_MEM_CLASS_TS_TREE, SLAB_PAGE_SIZE); cbm_aligned_free(page); return false; } @@ -269,6 +272,7 @@ static void slab_reclaim_pages(slab_state_t *s, bool clear_installed) { unsigned prev = atomic_fetch_sub_explicit(&p->refcount, 1u, memory_order_acq_rel); if (prev == 1u) { slab_map_unregister_page(p); + cbm_mem_class_remove_external(CBM_MEM_CLASS_TS_TREE, SLAB_PAGE_SIZE); cbm_aligned_free(p); } p = next; @@ -292,7 +296,7 @@ static void *slab_malloc(size_t size) { if (!s->freelist) { slab_refill(s); if (!s->freelist) { - return malloc(size); /* grow failed → heap fallback */ + return cbm_alloc(CBM_MEM_CLASS_TS_TREE, size); /* grow failed → heap fallback */ } } slab_free_node_t *node = s->freelist; @@ -302,8 +306,8 @@ static void *slab_malloc(size_t size) { return node; } - /* >64B: straight to malloc (= mimalloc in production) */ - return malloc(size); + /* >64B: straight to the core (= mimalloc in production) */ + return cbm_alloc(CBM_MEM_CLASS_TS_TREE, size); } static void *slab_calloc(size_t count, size_t size) { @@ -338,7 +342,7 @@ static void *slab_realloc(void *ptr, size_t new_size) { return ptr; } /* Promote slab → heap */ - void *new_ptr = malloc(new_size); + void *new_ptr = cbm_alloc(CBM_MEM_CLASS_TS_TREE, new_size); if (!new_ptr) { return NULL; } @@ -347,8 +351,8 @@ static void *slab_realloc(void *ptr, size_t new_size) { return new_ptr; } - /* Case 2: heap pointer (from malloc) */ - return realloc(ptr, new_size); + /* Case 2: heap pointer (from the core) */ + return cbm_realloc(CBM_MEM_CLASS_TS_TREE, ptr, new_size); } static void slab_free(void *ptr) { @@ -359,7 +363,7 @@ static void slab_free(void *ptr) { slab_page_t *page = slab_map_lookup(base); if (!page) { /* Not a slab chunk → plain heap pointer (>64B or grow-fallback). */ - free(ptr); + cbm_free(CBM_MEM_CLASS_TS_TREE, ptr); return; } @@ -386,6 +390,7 @@ static void slab_free(void *ptr) { if (prev == 1u) { /* We returned the final chunk of a retired page → release it. */ slab_map_unregister_page(page); + cbm_mem_class_remove_external(CBM_MEM_CLASS_TS_TREE, SLAB_PAGE_SIZE); cbm_aligned_free(page); } } diff --git a/src/graph_buffer/graph_buffer.c b/src/graph_buffer/graph_buffer.c index 939cc8cdb..50ef7fc58 100644 --- a/src/graph_buffer/graph_buffer.c +++ b/src/graph_buffer/graph_buffer.c @@ -36,6 +36,7 @@ enum { #include "foundation/dyn_array.h" #include "foundation/profile.h" #include "foundation/mem.h" +#include "foundation/mem_core.h" #include #include @@ -60,6 +61,18 @@ static inline void *intptr_to_ptr(intptr_t v) { #define EDGE_KEY_BUF CBM_SZ_256 /* Per-type or per-key edge list stored in hash tables as values */ +typedef struct { + uint64_t h0; + uint64_t h1; + cbm_gbuf_edge_t *edge; /* NULL = empty slot */ +} edge_key_slot_t; + +typedef struct { + edge_key_slot_t *slots; + size_t cap; /* power of two, 0 = not allocated */ + size_t count; +} edge_key_map_t; + typedef CBM_DYN_ARRAY(const cbm_gbuf_edge_t *) edge_ptr_array_t; /* Per-label or per-name node list */ @@ -95,7 +108,7 @@ struct cbm_gbuf { CBM_DYN_ARRAY(cbm_gbuf_edge_t *) edges; /* Edge dedup index: "srcID:tgtID:type" → cbm_gbuf_edge_t* */ - CBMHashTable *edge_by_key; + edge_key_map_t edge_by_key; /* dedup: 128-bit key hash -> edge */ /* Edge secondary indexes: composite keys → edge_ptr_array_t */ CBMHashTable *edges_by_source_type; /* "srcID:type" → edge_ptr_array_t* */ @@ -124,7 +137,7 @@ struct cbm_gbuf { /* ── Helpers ─────────────────────────────────────────────────────── */ static char *heap_strdup(const char *s) { - return s ? strdup(s) : strdup("{}"); + return cbm_mem_strdup(CBM_MEM_CLASS_GBUF_STRING, s ? s : "{}"); } /* Intern a repetitive string into the buffer's pool: identical content collapses @@ -137,7 +150,7 @@ static const char *gb_intern(cbm_gbuf_t *gb, const char *s) { if (found) { return found; } - char *copy = strdup(key); + char *copy = cbm_mem_strdup(CBM_MEM_CLASS_GBUF_STRING, key); if (copy) { cbm_ht_set(gb->intern_pool, copy, copy); /* key == value == owned copy */ } @@ -201,22 +214,142 @@ static void make_src_type_key(char *buf, size_t bufsz, int64_t src, const char * snprintf(buf, bufsz, "%lld:%s", (long long)src, type); } +/* ── Edge dedup index: 128-bit key hash -> edge, open addressing ────── + * + * The dedup key is "src:tgt:type:props" (make_edge_key). Keeping it as a + * string per edge cost the kernel 15.8M heap strings (~1.2 GB live, and the + * same again transiently in worker buffers during merge). Two independent + * 64-bit FNV-1a hashes of the composed key take 16 bytes; a false match needs + * both to collide, ~n^2 / 2^129 for n edges -- below any other failure mode of + * the index. Linear probing, backward-shift deletion (no tombstones), load + * kept under 1/2. Values are the edge pointers (stable heap records). */ + +static void edge_key_hash(const char *key, uint64_t *h0, uint64_t *h1) { + uint64_t a = 1469598103934665603ULL; + uint64_t b = 0x9E3779B97F4A7C15ULL; + for (const unsigned char *p = (const unsigned char *)key; *p; p++) { + a ^= *p; + a *= 1099511628211ULL; + b = (b ^ *p) * 0xFF51AFD7ED558CCDULL; + b ^= b >> 29; + } + *h0 = a; + *h1 = b ^ (a >> 17); +} + +static void edge_key_map_free(edge_key_map_t *m) { + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, m->slots); + m->slots = NULL; + m->cap = 0; + m->count = 0; +} + +static edge_key_slot_t *edge_key_map_find(const edge_key_map_t *m, uint64_t h0, uint64_t h1) { + if (!m->slots) { + return NULL; + } + size_t mask = m->cap - SKIP_ONE; + for (size_t i = (size_t)h0 & mask;; i = (i + SKIP_ONE) & mask) { + edge_key_slot_t *s = &m->slots[i]; + if (!s->edge) { + return NULL; + } + if (s->h0 == h0 && s->h1 == h1) { + return s; + } + } +} + +static bool edge_key_map_grow(edge_key_map_t *m) { + size_t new_cap = m->cap ? m->cap * PAIR_LEN : CBM_SZ_1K; + edge_key_slot_t *slots = cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, new_cap * sizeof(*slots)); + if (!slots) { + return false; + } + size_t mask = new_cap - SKIP_ONE; + for (size_t i = 0; i < m->cap; i++) { + edge_key_slot_t *s = &m->slots[i]; + if (!s->edge) { + continue; + } + size_t j = (size_t)s->h0 & mask; + while (slots[j].edge) { + j = (j + SKIP_ONE) & mask; + } + slots[j] = *s; + } + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, m->slots); + m->slots = slots; + m->cap = new_cap; + return true; +} + +static bool edge_key_map_set(edge_key_map_t *m, uint64_t h0, uint64_t h1, cbm_gbuf_edge_t *edge) { + if ((m->count + SKIP_ONE) * PAIR_LEN > m->cap && !edge_key_map_grow(m)) { + return false; + } + size_t mask = m->cap - SKIP_ONE; + size_t i = (size_t)h0 & mask; + while (m->slots[i].edge) { + if (m->slots[i].h0 == h0 && m->slots[i].h1 == h1) { + m->slots[i].edge = edge; + return true; + } + i = (i + SKIP_ONE) & mask; + } + m->slots[i].h0 = h0; + m->slots[i].h1 = h1; + m->slots[i].edge = edge; + m->count++; + return true; +} + +static void edge_key_map_delete(edge_key_map_t *m, uint64_t h0, uint64_t h1) { + edge_key_slot_t *s = edge_key_map_find(m, h0, h1); + if (!s) { + return; + } + size_t mask = m->cap - SKIP_ONE; + size_t i = (size_t)(s - m->slots); + m->slots[i].edge = NULL; + m->count--; + /* Backward shift: pull later entries of the same probe run into the hole. */ + for (size_t j = (i + SKIP_ONE) & mask; m->slots[j].edge; j = (j + SKIP_ONE) & mask) { + size_t home = (size_t)m->slots[j].h0 & mask; + /* entry at j may move to i if its home is not in (i, j] cyclically */ + bool movable = (i <= j) ? (home <= i || home > j) : (home <= i && home > j); + if (movable) { + m->slots[i] = m->slots[j]; + m->slots[j].edge = NULL; + i = j; + } + } +} + /* Get or create a node_ptr_array_t in a hash table */ +static void node_array_push(node_ptr_array_t *arr, const cbm_gbuf_node_t *node); + static node_ptr_array_t *get_or_create_node_array(CBMHashTable *ht, const char *key) { + if (!ht) { + return NULL; /* worker buffer: no secondary indexes (cbm_gbuf_new_worker) */ + } node_ptr_array_t *arr = cbm_ht_get(ht, key); if (!arr) { - arr = calloc(CBM_ALLOC_ONE, sizeof(node_ptr_array_t)); - cbm_ht_set(ht, strdup(key), arr); + arr = cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, sizeof(node_ptr_array_t)); + cbm_ht_set(ht, cbm_mem_strdup(CBM_MEM_CLASS_GBUF_INDEX, key), arr); } return arr; } /* Get or create an edge_ptr_array_t in a hash table */ static edge_ptr_array_t *get_or_create_edge_array(CBMHashTable *ht, const char *key) { + if (!ht) { + return NULL; + } edge_ptr_array_t *arr = cbm_ht_get(ht, key); if (!arr) { - arr = calloc(CBM_ALLOC_ONE, sizeof(edge_ptr_array_t)); - cbm_ht_set(ht, strdup(key), arr); + arr = cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, sizeof(edge_ptr_array_t)); + cbm_ht_set(ht, cbm_mem_strdup(CBM_MEM_CLASS_GBUF_INDEX, key), arr); } return arr; } @@ -227,9 +360,9 @@ static void free_node_array(const char *key, void *value, void *ud) { node_ptr_array_t *arr = value; if (arr) { cbm_da_free(arr); - free(arr); + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, arr); } - free((void *)key); + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, (void *)key); } /* Free an edge_ptr_array_t (callback) */ @@ -238,30 +371,40 @@ static void free_edge_array(const char *key, void *value, void *ud) { edge_ptr_array_t *arr = value; if (arr) { cbm_da_free(arr); - free(arr); + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, arr); } - free((void *)key); + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, (void *)key); } -/* Free keys only (for edge_by_key, deleted_set) */ -static void free_key_only(const char *key, void *value, void *ud) { +/* Free index keys only (deleted_set id keys). */ +static void free_index_key(const char *key, void *value, void *ud) { (void)value; (void)ud; - free((void *)key); + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, (void *)key); +} + +/* Free one intern-pool entry: key == value == the single owned string copy + * that nodes/edges borrowed for label/file_path/type. */ +static void free_intern_entry(const char *key, void *value, void *ud) { + (void)value; + (void)ud; + cbm_free(CBM_MEM_CLASS_GBUF_STRING, (void *)key); } /* Free a single node's owned strings. label and file_path are interned * (pool-owned) — NOT freed here; the pool frees them once in cbm_gbuf_free. */ static void free_node_strings(cbm_gbuf_node_t *n) { - free(n->name); - free(n->qualified_name); - free(n->properties_json); + cbm_free(CBM_MEM_CLASS_GBUF_STRING, n->name); + cbm_free(CBM_MEM_CLASS_GBUF_STRING, n->qualified_name); + cbm_free(CBM_MEM_CLASS_GBUF_STRING, n->properties_json); } -/* Free a single edge's owned strings. type is interned (pool-owned) — NOT - * freed here; the pool frees it once in cbm_gbuf_free. */ +/* An edge owns no strings: type AND properties_json are interned (pool-owned; + * the pool frees each once in cbm_gbuf_free). Edge properties repeat massively + * -- on the Go corpus 1.8M edges carry 343k distinct property strings and 854k + * are exactly "{}" -- and an edge's properties are never mutated in place. */ static void free_edge_strings(cbm_gbuf_edge_t *e) { - free(e->properties_json); + (void)e; } /* Allocate the next buffer-local or shared-atomic ID. */ @@ -303,9 +446,10 @@ static void unindex_edge(cbm_gbuf_t *gb, const cbm_gbuf_edge_t *e) { char key[EDGE_KEY_BUF]; make_edge_key(key, sizeof(key), e->source_id, e->target_id, e->type, e->properties_json); - const char *ekey = cbm_ht_get_key(gb->edge_by_key, key); - cbm_ht_delete(gb->edge_by_key, key); - free((void *)ekey); + uint64_t h0; + uint64_t h1; + edge_key_hash(key, &h0, &h1); + edge_key_map_delete(&gb->edge_by_key, h0, h1); make_src_type_key(key, sizeof(key), e->source_id, e->type); remove_edge_from_ptr_array(cbm_ht_get(gb->edges_by_source_type, key), e->id); @@ -329,7 +473,7 @@ static void cascade_delete_edges(cbm_gbuf_t *gb, CBMHashTable *deleted_set) { if (cbm_ht_get(deleted_set, src_id) || cbm_ht_get(deleted_set, tgt_id)) { unindex_edge(gb, e); free_edge_strings(e); - free(e); + cbm_free(CBM_MEM_CLASS_GBUF_EDGE, e); } else { gb->edges.items[write_idx++] = gb->edges.items[i]; } @@ -346,7 +490,8 @@ static void register_node_in_indexes(cbm_gbuf_t *gb, cbm_gbuf_node_t *node) { while (nc <= node->id) { nc *= 2; } - cbm_gbuf_node_t **grown = realloc(gb->by_id, (size_t)nc * sizeof(*grown)); + cbm_gbuf_node_t **grown = + cbm_realloc(CBM_MEM_CLASS_GBUF_INDEX, gb->by_id, (size_t)nc * sizeof(*grown)); if (grown) { memset(grown + gb->by_id_cap, 0, (size_t)(nc - gb->by_id_cap) * sizeof(*grown)); gb->by_id = grown; @@ -359,16 +504,29 @@ static void register_node_in_indexes(cbm_gbuf_t *gb, cbm_gbuf_node_t *node) { node_ptr_array_t *by_label = get_or_create_node_array(gb->nodes_by_label, node->label ? node->label : ""); - cbm_da_push(by_label, (const cbm_gbuf_node_t *)node); + node_array_push(by_label, node); node_ptr_array_t *by_name = get_or_create_node_array(gb->nodes_by_name, node->name ? node->name : ""); - cbm_da_push(by_name, (const cbm_gbuf_node_t *)node); + node_array_push(by_name, node); } -/* Push an edge pointer into a dynamic array (wraps macro to reduce CC contribution). */ +/* Per-key index arrays start at two slots: on the kernel the secondary + * indexes hold ~18M of these arrays and most keys have one or two entries, + * so the default 8-slot start was 1 GB of empty headroom. NULL = the buffer + * has no secondary indexes (worker buffers). */ +enum { GB_INDEX_ARRAY_FIRST_CAP = 2 }; static void edge_array_push(edge_ptr_array_t *arr, const cbm_gbuf_edge_t *edge) { - cbm_da_push(arr, edge); + if (!arr) { + return; + } + cbm_da_push_min(arr, edge, GB_INDEX_ARRAY_FIRST_CAP); +} +static void node_array_push(node_ptr_array_t *arr, const cbm_gbuf_node_t *node) { + if (!arr) { + return; + } + cbm_da_push_min(arr, node, GB_INDEX_ARRAY_FIRST_CAP); } /* Index an edge by one key into a hash table bucket. */ @@ -392,6 +550,9 @@ static void register_edge_in_indexes(cbm_gbuf_t *gb, cbm_gbuf_edge_t *edge) { /* Rebuild edge secondary indexes from scratch (after bulk deletion). */ static void rebuild_edge_secondary_indexes(cbm_gbuf_t *gb) { + if (!gb->edges_by_type) { + return; /* worker buffer: nothing to rebuild */ + } cbm_ht_foreach(gb->edges_by_source_type, free_edge_array, NULL); cbm_ht_free(gb->edges_by_source_type); cbm_ht_foreach(gb->edges_by_target_type, free_edge_array, NULL); @@ -399,9 +560,9 @@ static void rebuild_edge_secondary_indexes(cbm_gbuf_t *gb) { cbm_ht_foreach(gb->edges_by_type, free_edge_array, NULL); cbm_ht_free(gb->edges_by_type); - gb->edges_by_source_type = cbm_ht_create(CBM_SZ_256); - gb->edges_by_target_type = cbm_ht_create(CBM_SZ_256); - gb->edges_by_type = cbm_ht_create(CBM_SZ_32); + gb->edges_by_source_type = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_256); + gb->edges_by_target_type = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_256); + gb->edges_by_type = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_32); for (int i = 0; i < gb->edges.count; i++) { register_edge_in_indexes(gb, gb->edges.items[i]); @@ -412,7 +573,7 @@ static void rebuild_edge_secondary_indexes(cbm_gbuf_t *gb) { static void release_gbuf_indexes(cbm_gbuf_t *gb) { cbm_ht_free(gb->node_by_qn); gb->node_by_qn = NULL; - free(gb->by_id); + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, gb->by_id); gb->by_id = NULL; gb->by_id_cap = 0; cbm_ht_foreach(gb->nodes_by_label, free_node_array, NULL); @@ -421,9 +582,7 @@ static void release_gbuf_indexes(cbm_gbuf_t *gb) { cbm_ht_foreach(gb->nodes_by_name, free_node_array, NULL); cbm_ht_free(gb->nodes_by_name); gb->nodes_by_name = NULL; - cbm_ht_foreach(gb->edge_by_key, free_key_only, NULL); - cbm_ht_free(gb->edge_by_key); - gb->edge_by_key = NULL; + edge_key_map_free(&gb->edge_by_key); cbm_ht_foreach(gb->edges_by_source_type, free_edge_array, NULL); cbm_ht_free(gb->edges_by_source_type); gb->edges_by_source_type = NULL; @@ -438,28 +597,27 @@ static void release_gbuf_indexes(cbm_gbuf_t *gb) { /* ── Lifecycle ──────────────────────────────────────────────────── */ cbm_gbuf_t *cbm_gbuf_new(const char *project, const char *root_path) { - cbm_gbuf_t *gb = calloc(CBM_ALLOC_ONE, sizeof(cbm_gbuf_t)); + cbm_gbuf_t *gb = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(cbm_gbuf_t)); if (!gb) { return NULL; } - gb->project = strdup(project ? project : ""); - gb->root_path = strdup(root_path ? root_path : ""); + gb->project = cbm_mem_strdup(CBM_MEM_CLASS_GBUF_STRING, project ? project : ""); + gb->root_path = cbm_mem_strdup(CBM_MEM_CLASS_GBUF_STRING, root_path ? root_path : ""); gb->next_id = SKIP_ONE; gb->shared_ids = NULL; - gb->node_by_qn = cbm_ht_create(CBM_SZ_256); + gb->node_by_qn = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_256); gb->by_id = NULL; gb->by_id_cap = 0; - gb->nodes_by_label = cbm_ht_create(CBM_SZ_32); - gb->nodes_by_name = cbm_ht_create(CBM_SZ_256); + gb->nodes_by_label = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_32); + gb->nodes_by_name = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_256); - gb->edge_by_key = cbm_ht_create(CBM_SZ_512); - gb->edges_by_source_type = cbm_ht_create(CBM_SZ_256); - gb->edges_by_target_type = cbm_ht_create(CBM_SZ_256); - gb->edges_by_type = cbm_ht_create(CBM_SZ_32); + gb->edges_by_source_type = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_256); + gb->edges_by_target_type = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_256); + gb->edges_by_type = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_32); - gb->intern_pool = cbm_ht_create(CBM_SZ_1K); + gb->intern_pool = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_STRING, CBM_SZ_1K); return gb; } @@ -473,6 +631,30 @@ cbm_gbuf_t *cbm_gbuf_new_shared_ids(const char *project, const char *root_path, return gb; } +cbm_gbuf_t *cbm_gbuf_new_worker(const char *project, const char *root_path, + _Atomic int64_t *id_source) { + cbm_gbuf_t *gb = cbm_gbuf_new_shared_ids(project, root_path, id_source); + if (!gb) { + return NULL; + } + /* A worker buffer is appended to and merged; it is never asked by label, + * name or edge type. Its secondary indexes were pure cost: on the kernel + * the index class peaked at 7.1 GB during resolve against 2.5 GB live in + * the main buffer, the difference being 18 workers' throwaway indexes. + * node_by_qn and edge_by_key stay: upsert and dedup need them. */ + cbm_ht_free(gb->nodes_by_label); + cbm_ht_free(gb->nodes_by_name); + cbm_ht_free(gb->edges_by_source_type); + cbm_ht_free(gb->edges_by_target_type); + cbm_ht_free(gb->edges_by_type); + gb->nodes_by_label = NULL; + gb->nodes_by_name = NULL; + gb->edges_by_source_type = NULL; + gb->edges_by_target_type = NULL; + gb->edges_by_type = NULL; + return gb; +} + void cbm_gbuf_free(cbm_gbuf_t *gb) { if (!gb) { return; @@ -482,7 +664,7 @@ void cbm_gbuf_free(cbm_gbuf_t *gb) { for (int i = 0; i < gb->nodes.count; i++) { cbm_gbuf_node_t *n = gb->nodes.items[i]; free_node_strings(n); - free(n); + cbm_free(CBM_MEM_CLASS_GBUF_NODE, n); } cbm_da_free(&gb->nodes); @@ -490,7 +672,7 @@ void cbm_gbuf_free(cbm_gbuf_t *gb) { for (int i = 0; i < gb->edges.count; i++) { cbm_gbuf_edge_t *e = gb->edges.items[i]; free_edge_strings(e); - free(e); + cbm_free(CBM_MEM_CLASS_GBUF_EDGE, e); } cbm_da_free(&gb->edges); @@ -498,7 +680,7 @@ void cbm_gbuf_free(cbm_gbuf_t *gb) { if (gb->node_by_qn) { cbm_ht_free(gb->node_by_qn); } - free(gb->by_id); + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, gb->by_id); if (gb->nodes_by_label) { cbm_ht_foreach(gb->nodes_by_label, free_node_array, NULL); cbm_ht_free(gb->nodes_by_label); @@ -507,10 +689,7 @@ void cbm_gbuf_free(cbm_gbuf_t *gb) { cbm_ht_foreach(gb->nodes_by_name, free_node_array, NULL); cbm_ht_free(gb->nodes_by_name); } - if (gb->edge_by_key) { - cbm_ht_foreach(gb->edge_by_key, free_key_only, NULL); - cbm_ht_free(gb->edge_by_key); - } + edge_key_map_free(&gb->edge_by_key); if (gb->edges_by_source_type) { cbm_ht_foreach(gb->edges_by_source_type, free_edge_array, NULL); cbm_ht_free(gb->edges_by_source_type); @@ -526,28 +705,28 @@ void cbm_gbuf_free(cbm_gbuf_t *gb) { /* Free vector storage */ for (int i = 0; i < gb->dump_vector_count; i++) { - free((void *)gb->dump_vectors[i].vector); + cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)gb->dump_vectors[i].vector); } - free(gb->dump_vectors); + cbm_free(CBM_MEM_CLASS_SEMANTIC, gb->dump_vectors); /* Free token vector storage */ for (int i = 0; i < gb->dump_token_vec_count; i++) { - free((void *)gb->dump_token_vecs[i].token); - free((void *)gb->dump_token_vecs[i].vector); + cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)gb->dump_token_vecs[i].token); + cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)gb->dump_token_vecs[i].vector); } - free(gb->dump_token_vecs); + cbm_free(CBM_MEM_CLASS_SEMANTIC, gb->dump_token_vecs); /* Free interned strings (node label/file_path, edge type) — pool owns one - * copy each (key == value), freed exactly once via free_key_only. Done after - * nodes/edges since they borrowed these pointers. */ + * copy each (key == value), freed exactly once via free_intern_entry. Done + * after nodes/edges since they borrowed these pointers. */ if (gb->intern_pool) { - cbm_ht_foreach(gb->intern_pool, free_key_only, NULL); + cbm_ht_foreach(gb->intern_pool, free_intern_entry, NULL); cbm_ht_free(gb->intern_pool); } - free(gb->project); - free(gb->root_path); - free(gb); + cbm_free(CBM_MEM_CLASS_GBUF_STRING, gb->project); + cbm_free(CBM_MEM_CLASS_GBUF_STRING, gb->root_path); + cbm_free(CBM_MEM_CLASS_OTHER, gb); } /* ── Vector storage ──────────────────────────────────────────────── */ @@ -560,7 +739,8 @@ int cbm_gbuf_store_vector(cbm_gbuf_t *gb, int64_t node_id, const uint8_t *vector if (gb->dump_vector_count >= gb->dump_vector_cap) { int new_cap = gb->dump_vector_cap < VEC_INIT_CAP ? VEC_INIT_CAP : gb->dump_vector_cap * VEC_GROW; - CBMDumpVector *grown = realloc(gb->dump_vectors, (size_t)new_cap * sizeof(CBMDumpVector)); + CBMDumpVector *grown = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, gb->dump_vectors, + (size_t)new_cap * sizeof(CBMDumpVector)); if (!grown) { return GB_ERR; } @@ -568,7 +748,7 @@ int cbm_gbuf_store_vector(cbm_gbuf_t *gb, int64_t node_id, const uint8_t *vector gb->dump_vector_cap = new_cap; } /* Copy vector data */ - uint8_t *vec_copy = malloc((size_t)vector_len); + uint8_t *vec_copy = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)vector_len); if (!vec_copy) { return GB_ERR; } @@ -592,15 +772,15 @@ int cbm_gbuf_store_token_vector(cbm_gbuf_t *gb, const char *token, const uint8_t if (gb->dump_token_vec_count >= gb->dump_token_vec_cap) { int new_cap = gb->dump_token_vec_cap < TV_INIT_CAP ? TV_INIT_CAP : gb->dump_token_vec_cap * TV_GROW; - CBMDumpTokenVec *grown = - realloc(gb->dump_token_vecs, (size_t)new_cap * sizeof(CBMDumpTokenVec)); + CBMDumpTokenVec *grown = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, gb->dump_token_vecs, + (size_t)new_cap * sizeof(CBMDumpTokenVec)); if (!grown) { return GB_ERR; } gb->dump_token_vecs = grown; gb->dump_token_vec_cap = new_cap; } - uint8_t *vec_copy = malloc((size_t)vector_len); + uint8_t *vec_copy = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)vector_len); if (!vec_copy) { return GB_ERR; } @@ -610,7 +790,7 @@ int cbm_gbuf_store_token_vector(cbm_gbuf_t *gb, const char *token, const uint8_t gb->dump_token_vecs[idx] = (CBMDumpTokenVec){ .id = idx + SKIP_ONE, /* 1-based sequential ID */ .project = gb->project, - .token = strdup(token), + .token = cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, token), .vector = vec_copy, .vector_len = vector_len, .idf = idf, @@ -721,30 +901,30 @@ int64_t cbm_gbuf_upsert_node(cbm_gbuf_t *gb, const char *label, const char *name cbm_ht_get(gb->nodes_by_name, existing->name ? existing->name : ""), existing->id); } existing->label = (char *)new_label_interned; - free(existing->name); + cbm_free(CBM_MEM_CLASS_GBUF_STRING, existing->name); existing->name = new_name; existing->file_path = (char *)gb_intern(gb, file_path); existing->start_line = start_line; existing->end_line = end_line; if (new_props) { - free(existing->properties_json); + cbm_free(CBM_MEM_CLASS_GBUF_STRING, existing->properties_json); existing->properties_json = new_props; } if (label_changed) { node_ptr_array_t *by_label = get_or_create_node_array( gb->nodes_by_label, existing->label ? existing->label : ""); - cbm_da_push(by_label, (const cbm_gbuf_node_t *)existing); + node_array_push(by_label, existing); } if (name_changed) { node_ptr_array_t *by_name = get_or_create_node_array(gb->nodes_by_name, existing->name ? existing->name : ""); - cbm_da_push(by_name, (const cbm_gbuf_node_t *)existing); + node_array_push(by_name, existing); } return existing->id; } /* Heap-allocate a new node (pointer stays stable across array growth) */ - cbm_gbuf_node_t *node = calloc(CBM_ALLOC_ONE, sizeof(cbm_gbuf_node_t)); + cbm_gbuf_node_t *node = cbm_calloc(CBM_MEM_CLASS_GBUF_NODE, sizeof(cbm_gbuf_node_t)); if (!node) { return 0; } @@ -766,6 +946,19 @@ int64_t cbm_gbuf_upsert_node(cbm_gbuf_t *gb, const char *label, const char *name return id; } +int cbm_gbuf_node_set_properties_json(cbm_gbuf_node_t *node, const char *json) { + if (!node) { + return GB_ERR; + } + char *copy = heap_strdup(json); + if (!copy) { + return GB_ERR; + } + cbm_free(CBM_MEM_CLASS_GBUF_STRING, node->properties_json); + node->properties_json = copy; + return 0; +} + const cbm_gbuf_node_t *cbm_gbuf_find_by_qn(const cbm_gbuf_t *gb, const char *qn) { if (!gb || !qn) { return NULL; @@ -834,7 +1027,8 @@ int cbm_gbuf_delete_by_label(cbm_gbuf_t *gb, const char *label) { char id_buf[CBM_SZ_32]; make_id_key(id_buf, sizeof(id_buf), n->id); - cbm_ht_set(deleted_set, strdup(id_buf), intptr_to_ptr(SKIP_ONE)); + cbm_ht_set(deleted_set, cbm_mem_strdup(CBM_MEM_CLASS_GBUF_INDEX, id_buf), + intptr_to_ptr(SKIP_ONE)); /* Remove from primary indexes */ cbm_ht_delete(gb->node_by_qn, n->qualified_name); @@ -849,7 +1043,7 @@ int cbm_gbuf_delete_by_label(cbm_gbuf_t *gb, const char *label) { /* Cascade-delete edges referencing deleted nodes */ cascade_delete_edges(gb, deleted_set); - cbm_ht_foreach(deleted_set, free_key_only, NULL); + cbm_ht_foreach(deleted_set, free_index_key, NULL); cbm_ht_free(deleted_set); return 0; } @@ -876,7 +1070,8 @@ int cbm_gbuf_delete_by_file(cbm_gbuf_t *gb, const char *file_path) { char id_buf[CBM_SZ_32]; make_id_key(id_buf, sizeof(id_buf), n->id); - cbm_ht_set(deleted_set, strdup(id_buf), intptr_to_ptr(SKIP_ONE)); + cbm_ht_set(deleted_set, cbm_mem_strdup(CBM_MEM_CLASS_GBUF_INDEX, id_buf), + intptr_to_ptr(SKIP_ONE)); /* Remove from secondary indexes */ remove_node_from_ptr_array(cbm_ht_get(gb->nodes_by_label, n->label), n->id); @@ -890,7 +1085,7 @@ int cbm_gbuf_delete_by_file(cbm_gbuf_t *gb, const char *file_path) { /* NULL out QN so dump's liveness check (cbm_ht_get by QN) fails * even if a new node with the same QN is inserted later via merge. */ - free(n->qualified_name); + cbm_free(CBM_MEM_CLASS_GBUF_STRING, n->qualified_name); n->qualified_name = NULL; deleted_count++; } @@ -903,7 +1098,7 @@ int cbm_gbuf_delete_by_file(cbm_gbuf_t *gb, const char *file_path) { /* Cascade-delete edges referencing deleted nodes */ cascade_delete_edges(gb, deleted_set); - cbm_ht_foreach(deleted_set, free_key_only, NULL); + cbm_ht_foreach(deleted_set, free_index_key, NULL); cbm_ht_free(deleted_set); { char s_buf[CBM_SZ_16]; @@ -945,7 +1140,8 @@ int cbm_gbuf_load_from_db(cbm_gbuf_t *gb, const char *db_path, const char *proje } sqlite3_finalize(stmt); - int64_t *old_to_new = calloc((size_t)(max_old_id + SKIP_ONE), sizeof(int64_t)); + int64_t *old_to_new = + cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, (size_t)(max_old_id + SKIP_ONE) * sizeof(int64_t)); if (!old_to_new) { cbm_store_close(store); return CBM_NOT_FOUND; @@ -957,7 +1153,7 @@ int cbm_gbuf_load_from_db(cbm_gbuf_t *gb, const char *db_path, const char *proje "SELECT id, label, name, qualified_name, file_path, start_line, end_line, properties " "FROM nodes WHERE project = ? ORDER BY id", CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { - free(old_to_new); + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, old_to_new); cbm_store_close(store); return CBM_NOT_FOUND; } @@ -985,7 +1181,7 @@ int cbm_gbuf_load_from_db(cbm_gbuf_t *gb, const char *db_path, const char *proje "SELECT source_id, target_id, type, properties " "FROM edges WHERE project = ?", CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { - free(old_to_new); + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, old_to_new); cbm_store_close(store); return CBM_NOT_FOUND; } @@ -1005,7 +1201,7 @@ int cbm_gbuf_load_from_db(cbm_gbuf_t *gb, const char *db_path, const char *proje } sqlite3_finalize(stmt); - free(old_to_new); + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, old_to_new); cbm_store_close(store); return 0; } @@ -1101,17 +1297,20 @@ int64_t cbm_gbuf_insert_edge(cbm_gbuf_t *gb, int64_t source_id, int64_t target_i char key[EDGE_KEY_BUF]; make_edge_key(key, sizeof(key), source_id, target_id, type, properties_json); - cbm_gbuf_edge_t *existing = cbm_ht_get(gb->edge_by_key, key); + uint64_t h0; + uint64_t h1; + edge_key_hash(key, &h0, &h1); + edge_key_slot_t *hit = edge_key_map_find(&gb->edge_by_key, h0, h1); + cbm_gbuf_edge_t *existing = hit ? hit->edge : NULL; if (existing) { if (edge_props_should_replace(existing->properties_json, properties_json)) { - free(existing->properties_json); - existing->properties_json = heap_strdup(properties_json); + existing->properties_json = (char *)gb_intern(gb, properties_json); } return existing->id; } /* Heap-allocate a new edge (pointer stays stable) */ - cbm_gbuf_edge_t *edge = calloc(CBM_ALLOC_ONE, sizeof(cbm_gbuf_edge_t)); + cbm_gbuf_edge_t *edge = cbm_calloc(CBM_MEM_CLASS_GBUF_EDGE, sizeof(cbm_gbuf_edge_t)); if (!edge) { return 0; } @@ -1121,13 +1320,13 @@ int64_t cbm_gbuf_insert_edge(cbm_gbuf_t *gb, int64_t source_id, int64_t target_i edge->source_id = source_id; edge->target_id = target_id; edge->type = (char *)gb_intern(gb, type); - edge->properties_json = heap_strdup(properties_json); + edge->properties_json = (char *)gb_intern(gb, properties_json); /* Store pointer in array */ cbm_da_push(&gb->edges, edge); /* Dedup index */ - cbm_ht_set(gb->edge_by_key, strdup(key), edge); + (void)edge_key_map_set(&gb->edge_by_key, h0, h1, edge); /* Secondary indexes */ register_edge_in_indexes(gb, edge); @@ -1212,11 +1411,12 @@ int cbm_gbuf_delete_edges_by_type(cbm_gbuf_t *gb, const char *type) { char key[EDGE_KEY_BUF]; make_edge_key(key, sizeof(key), e->source_id, e->target_id, e->type, e->properties_json); - const char *ekey = cbm_ht_get_key(gb->edge_by_key, key); - cbm_ht_delete(gb->edge_by_key, key); - free((void *)ekey); + uint64_t h0; + uint64_t h1; + edge_key_hash(key, &h0, &h1); + edge_key_map_delete(&gb->edge_by_key, h0, h1); free_edge_strings(e); - free(e); + cbm_free(CBM_MEM_CLASS_GBUF_EDGE, e); } else { gb->edges.items[write_idx++] = gb->edges.items[i]; } @@ -1234,8 +1434,8 @@ int cbm_gbuf_delete_edges_by_type(cbm_gbuf_t *gb, const char *type) { /* Free remap hash table entries (key = heap string, value = heap int64_t*) */ static void free_remap_entry(const char *key, void *val, void *ud) { (void)ud; - free((void *)key); - free(val); + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, (void *)key); + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, val); } /* Handle QN collision: update dst node fields (src wins), record remap if IDs differ. @@ -1296,24 +1496,24 @@ static void merge_update_existing(cbm_gbuf_t *dst, cbm_gbuf_node_t *existing, existing->id); } existing->label = (char *)new_label; - free(existing->name); + cbm_free(CBM_MEM_CLASS_GBUF_STRING, existing->name); existing->name = heap_strdup(sn->name); existing->file_path = (char *)gb_intern(dst, sn->file_path); existing->start_line = sn->start_line; existing->end_line = sn->end_line; if (sn->properties_json) { - free(existing->properties_json); + cbm_free(CBM_MEM_CLASS_GBUF_STRING, existing->properties_json); existing->properties_json = heap_strdup(sn->properties_json); } if (label_changed) { node_ptr_array_t *by_label = get_or_create_node_array( dst->nodes_by_label, existing->label ? existing->label : ""); - cbm_da_push(by_label, (const cbm_gbuf_node_t *)existing); + node_array_push(by_label, existing); } if (name_changed) { node_ptr_array_t *by_name = get_or_create_node_array( dst->nodes_by_name, existing->name ? existing->name : ""); - cbm_da_push(by_name, (const cbm_gbuf_node_t *)existing); + node_array_push(by_name, existing); } } } @@ -1324,15 +1524,15 @@ static void merge_update_existing(cbm_gbuf_t *dst, cbm_gbuf_node_t *existing, } char key[CBM_SZ_32]; make_id_key(key, sizeof(key), sn->id); - int64_t *val = malloc(sizeof(int64_t)); + int64_t *val = cbm_alloc(CBM_MEM_CLASS_GBUF_INDEX, sizeof(int64_t)); *val = existing->id; - cbm_ht_set(*remap, strdup(key), val); + cbm_ht_set(*remap, cbm_mem_strdup(CBM_MEM_CLASS_GBUF_INDEX, key), val); } } /* Copy a non-colliding src node into dst with its original ID. */ static void merge_copy_new_node(cbm_gbuf_t *dst, const cbm_gbuf_node_t *sn) { - cbm_gbuf_node_t *node = calloc(CBM_ALLOC_ONE, sizeof(cbm_gbuf_node_t)); + cbm_gbuf_node_t *node = cbm_calloc(CBM_MEM_CLASS_GBUF_NODE, sizeof(cbm_gbuf_node_t)); if (!node) { return; } @@ -1445,7 +1645,7 @@ static char *extract_prop_string(const char *props, const char *key_quoted, cons yyjson_val *v = yyjson_obj_get(yyjson_doc_get_root(doc), key); if (v && yyjson_is_str(v)) { const char *sv = yyjson_get_str(v); - out = cbm_strndup(sv, strlen(sv)); + out = cbm_mem_strdup(CBM_MEM_CLASS_DUMP, sv); } yyjson_doc_free(doc); return out; @@ -1477,11 +1677,11 @@ static CBMDumpNode *build_dump_nodes(cbm_gbuf_t *gb, int live_count, int64_t *te int64_t max_temp_id, int *out_count, cbm_gbuf_node_t ***src_out) { size_t cap = (size_t)(live_count > 0 ? live_count : SKIP_ONE); - CBMDumpNode *dump_nodes = malloc(cap * sizeof(CBMDumpNode)); + CBMDumpNode *dump_nodes = cbm_alloc(CBM_MEM_CLASS_DUMP, cap * sizeof(CBMDumpNode)); /* Parallel gbuf-node pointers so a streamed partition can free its heavy * properties_json after the rows are persisted. NULL on OOM disables the * per-partition free (the dump still succeeds). */ - cbm_gbuf_node_t **src = malloc(cap * sizeof(cbm_gbuf_node_t *)); + cbm_gbuf_node_t **src = cbm_alloc(CBM_MEM_CLASS_DUMP, cap * sizeof(cbm_gbuf_node_t *)); int idx = 0; for (int i = 0; i < gb->nodes.count; i++) { @@ -1534,10 +1734,10 @@ static CBMDumpEdge *build_dump_edges(cbm_gbuf_t *gb, const int64_t *temp_to_fina } } - CBMDumpEdge *dump_edges = - malloc((size_t)(valid_edges > 0 ? valid_edges : SKIP_ONE) * sizeof(CBMDumpEdge)); - char **url_paths = calloc((size_t)(valid_edges > 0 ? valid_edges : SKIP_ONE), sizeof(char *)); - char **local_names = calloc((size_t)(valid_edges > 0 ? valid_edges : SKIP_ONE), sizeof(char *)); + size_t edge_cap = (size_t)(valid_edges > 0 ? valid_edges : SKIP_ONE); + CBMDumpEdge *dump_edges = cbm_alloc(CBM_MEM_CLASS_DUMP, edge_cap * sizeof(CBMDumpEdge)); + char **url_paths = cbm_calloc(CBM_MEM_CLASS_DUMP, edge_cap * sizeof(char *)); + char **local_names = cbm_calloc(CBM_MEM_CLASS_DUMP, edge_cap * sizeof(char *)); int idx = 0; for (int i = 0; i < gb->edges.count; i++) { @@ -1629,14 +1829,14 @@ static void free_dump_resources(char **url_paths, char **local_names, int edge_c CBMDumpEdge *dump_edges, CBMDumpNode *dump_nodes, int64_t *temp_to_final) { for (int i = 0; i < edge_count; i++) { - free(url_paths[i]); - free(local_names[i]); + cbm_free(CBM_MEM_CLASS_DUMP, url_paths[i]); + cbm_free(CBM_MEM_CLASS_DUMP, local_names[i]); } - free(url_paths); - free(local_names); - free(dump_edges); - free(dump_nodes); - free(temp_to_final); + cbm_free(CBM_MEM_CLASS_DUMP, url_paths); + cbm_free(CBM_MEM_CLASS_DUMP, local_names); + cbm_free(CBM_MEM_CLASS_DUMP, dump_edges); + cbm_free(CBM_MEM_CLASS_DUMP, dump_nodes); + cbm_free(CBM_MEM_CLASS_DUMP, temp_to_final); } static int count_live_nodes(cbm_gbuf_t *gb) { @@ -1678,7 +1878,7 @@ int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path) { CBM_PROF_START(t_build_nodes); int64_t max_temp_id = gb->next_id; - int64_t *temp_to_final = calloc((size_t)max_temp_id, sizeof(int64_t)); + int64_t *temp_to_final = cbm_calloc(CBM_MEM_CLASS_DUMP, (size_t)max_temp_id * sizeof(int64_t)); if (!temp_to_final) { return CBM_NOT_FOUND; } @@ -1710,9 +1910,9 @@ int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path) { * uninitialized budget from ever triggering the free). */ cbm_db_writer_t *w = cbm_writer_open(path); if (!w) { - free(src_nodes); - free(dump_nodes); - free(temp_to_final); + cbm_free(CBM_MEM_CLASS_DUMP, src_nodes); + cbm_free(CBM_MEM_CLASS_DUMP, dump_nodes); + cbm_free(CBM_MEM_CLASS_DUMP, temp_to_final); return CBM_NOT_FOUND; } @@ -1732,7 +1932,7 @@ int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path) { free_heavy = free_heavy || (cbm_mem_budget() > 0 && cbm_mem_over_budget()); if (free_heavy && src_nodes) { for (int j = off; j < off + chunk; j++) { - free(src_nodes[j]->properties_json); + cbm_free(CBM_MEM_CLASS_GBUF_STRING, src_nodes[j]->properties_json); src_nodes[j]->properties_json = NULL; dump_nodes[j].properties = NULL; } @@ -1766,7 +1966,7 @@ int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path) { log_dump_summary(node_idx, edge_idx); free_dump_resources(url_paths, local_names, edge_idx, dump_edges, dump_nodes, temp_to_final); - free(src_nodes); + cbm_free(CBM_MEM_CLASS_DUMP, src_nodes); return rc; } @@ -1791,7 +1991,7 @@ int cbm_gbuf_flush_to_store(cbm_gbuf_t *gb, cbm_store_t *store) { * Temp IDs start at 1 and are sequential, but can have gaps from edge inserts. * Use max_id as size. */ int64_t max_temp_id = gb->next_id; - int64_t *temp_to_real = calloc(max_temp_id, sizeof(int64_t)); + int64_t *temp_to_real = cbm_calloc(CBM_MEM_CLASS_DUMP, (size_t)max_temp_id * sizeof(int64_t)); for (int i = 0; i < gb->nodes.count; i++) { cbm_gbuf_node_t *n = gb->nodes.items[i]; @@ -1840,7 +2040,7 @@ int cbm_gbuf_flush_to_store(cbm_gbuf_t *gb, cbm_store_t *store) { cbm_store_create_indexes(store); cbm_store_end_bulk(store); - free(temp_to_real); + cbm_free(CBM_MEM_CLASS_DUMP, temp_to_real); return 0; } @@ -1854,7 +2054,7 @@ int cbm_gbuf_merge_into_store(cbm_gbuf_t *gb, cbm_store_t *store) { /* Build temp_id → real_id map */ int64_t max_temp_id = gb->next_id; - int64_t *temp_to_real = calloc(max_temp_id, sizeof(int64_t)); + int64_t *temp_to_real = cbm_calloc(CBM_MEM_CLASS_DUMP, (size_t)max_temp_id * sizeof(int64_t)); for (int i = 0; i < gb->nodes.count; i++) { cbm_gbuf_node_t *n = gb->nodes.items[i]; @@ -1899,6 +2099,6 @@ int cbm_gbuf_merge_into_store(cbm_gbuf_t *gb, cbm_store_t *store) { cbm_store_commit(store); - free(temp_to_real); + cbm_free(CBM_MEM_CLASS_DUMP, temp_to_real); return 0; } diff --git a/src/graph_buffer/graph_buffer.h b/src/graph_buffer/graph_buffer.h index 6c8c6babe..ffbff0469 100644 --- a/src/graph_buffer/graph_buffer.h +++ b/src/graph_buffer/graph_buffer.h @@ -50,6 +50,12 @@ cbm_gbuf_t *cbm_gbuf_new(const char *project, const char *root_path); * IDs are allocated via atomic_fetch_add on *id_source. * Used for parallel extraction where multiple gbufs need unique IDs. * If id_source is NULL, behaves like cbm_gbuf_new(). */ +/* A worker buffer: shared ids, no secondary indexes (by label / name / edge + * type). It is filled by one worker and merged into the main buffer; the + * finders that need those indexes return nothing on it. */ +cbm_gbuf_t *cbm_gbuf_new_worker(const char *project, const char *root_path, + _Atomic int64_t *id_source); + cbm_gbuf_t *cbm_gbuf_new_shared_ids(const char *project, const char *root_path, _Atomic int64_t *id_source); @@ -175,4 +181,11 @@ int cbm_gbuf_flush_to_store(cbm_gbuf_t *gb, cbm_store_t *store); * Returns 0 on success. */ int cbm_gbuf_merge_into_store(cbm_gbuf_t *gb, cbm_store_t *store); +/* Replace a node's properties_json with a buffer-owned copy of json (NULL + * maps to "{}"). The buffer allocates and frees every string it owns through + * the memory core; passes that rewrite properties MUST go through here rather + * than freeing node->properties_json themselves. Returns 0, or -1 on OOM + * (the old value is kept). */ +int cbm_gbuf_node_set_properties_json(cbm_gbuf_node_t *node, const char *json); + #endif /* CBM_GRAPH_BUFFER_H */ diff --git a/src/pipeline/lsp_surface.c b/src/pipeline/lsp_surface.c index 369f7f7e5..41d4e4d13 100644 --- a/src/pipeline/lsp_surface.c +++ b/src/pipeline/lsp_surface.c @@ -17,6 +17,7 @@ * bytes is the early-cutoff key: a body edit reserializes identically. */ #include "pipeline/lsp_surface.h" +#include "pipeline/pipeline_internal.h" #include #include @@ -126,8 +127,8 @@ static char *surface_file_to_json(const CBMFileResult *result, const CBMLSPDef * return json; } -int cbm_lsp_surface_build_rows(const char *project, CBMFileResult **cache, - const cbm_file_info_t *files, int file_count, +int cbm_lsp_surface_build_rows(const cbm_pipeline_ctx_t *ctx, const char *project, + CBMFileResult **cache, const cbm_file_info_t *files, int file_count, const CBMLSPDef *all_defs, const int *def_starts, cbm_lsp_surface_row_t **out_rows, int *out_count) { *out_rows = NULL; @@ -141,7 +142,9 @@ int cbm_lsp_surface_build_rows(const char *project, CBMFileResult **cache, } int n = 0; for (int i = 0; i < file_count; i++) { - if (!cache[i]) { + bool loaded = false; + CBMFileResult *fr = cbm_pipeline_result_acquire(ctx, cache, i, NULL, &loaded); + if (!fr) { /* Never parsed this run (read/extract skip): no surface claim. * The routing layer treats a missing row as "must full-rebuild * before this file can be reasoned about", which is the correct @@ -150,8 +153,8 @@ int cbm_lsp_surface_build_rows(const char *project, CBMFileResult **cache, } int start = def_starts ? def_starts[i] : 0; int end = def_starts ? def_starts[i + 1] : 0; - char *json = - surface_file_to_json(cache[i], all_defs ? all_defs + start : NULL, end - start); + char *json = surface_file_to_json(fr, all_defs ? all_defs + start : NULL, end - start); + cbm_pipeline_result_release(fr, loaded); if (!json) { cbm_store_free_lsp_surfaces(rows, n); return -1; diff --git a/src/pipeline/lsp_surface.h b/src/pipeline/lsp_surface.h index a1d826158..de3390d85 100644 --- a/src/pipeline/lsp_surface.h +++ b/src/pipeline/lsp_surface.h @@ -29,8 +29,8 @@ * heap strings; release with cbm_store_free_lsp_surfaces. Files with an * empty surface still get a row (empty arrays hash too — "no defs" must be * distinguishable from "no data"). Returns 0, or -1 on allocation failure. */ -int cbm_lsp_surface_build_rows(const char *project, CBMFileResult **cache, - const cbm_file_info_t *files, int file_count, +int cbm_lsp_surface_build_rows(const cbm_pipeline_ctx_t *ctx, const char *project, + CBMFileResult **cache, const cbm_file_info_t *files, int file_count, const CBMLSPDef *all_defs, const int *def_starts, cbm_lsp_surface_row_t **out_rows, int *out_count); diff --git a/src/pipeline/pass_complexity.c b/src/pipeline/pass_complexity.c index 420237b81..ab1574566 100644 --- a/src/pipeline/pass_complexity.c +++ b/src/pipeline/pass_complexity.c @@ -95,8 +95,8 @@ static void append_complexity_props(cbm_gbuf_node_t *node, int tld, bool recursi free(neu); return; } - free(node->properties_json); - node->properties_json = neu; + (void)cbm_gbuf_node_set_properties_json(node, neu); + free(neu); } /* Content-only node order: qualified_name, then file path and start line. diff --git a/src/pipeline/pass_importance.c b/src/pipeline/pass_importance.c index c544d0a10..e331dce90 100644 --- a/src/pipeline/pass_importance.c +++ b/src/pipeline/pass_importance.c @@ -292,8 +292,8 @@ void cbm_pipeline_importance_append_prop(cbm_gbuf_node_t *node, double score) { if (!neu) { return; } - free(node->properties_json); - node->properties_json = neu; + (void)cbm_gbuf_node_set_properties_json(node, neu); + free(neu); } /* ── The scoring rule ───────────────────────────────────────────────── diff --git a/src/pipeline/pass_lsp_cross.c b/src/pipeline/pass_lsp_cross.c index 704a73d41..3450d2c4f 100644 --- a/src/pipeline/pass_lsp_cross.c +++ b/src/pipeline/pass_lsp_cross.c @@ -16,6 +16,7 @@ #include "pipeline/pass_lsp_cross.h" #include "pipeline/lsp_surface.h" +#include "result_spill.h" #include "pipeline/pipeline_internal.h" #include "pipeline/lsp_resolve.h" #include "lsp/go_lsp.h" @@ -104,7 +105,11 @@ static char *pxc_read_file(const char *path, int *out_len) { * plus Protocol/Function/Method — variables, modules, decorators, etc. are * skipped. Struct passes through so Rust/Go struct type-registration via the * cross-file LSP path is not dropped. */ -static const char *pxc_map_label(const char *label) { +/* The label a cross def keeps is a static spelling, never the result's own + * pointer: in spill mode the CBMFileResult is freed (parked on disk) the + * moment the collector is done with it, and the defs outlive it. A kept + * spelling outside the table falls back to an arena copy. */ +static const char *pxc_map_label(CBMArena *arena, const char *label) { if (!label) return NULL; if (cbm_label_is_type_like(label) || strcmp(label, "Protocol") == 0 || @@ -115,7 +120,14 @@ static const char *pxc_map_label(const char *label) { * name-only fallback). Every registrar filters by explicit label, so * the other languages ignore Variable defs untouched. */ strcmp(label, "Variable") == 0) { - return label; + static const char *const canon[] = {"Class", "Struct", "Interface", "Enum", "Type", + "Trait", "Protocol", "Function", "Method", "Variable"}; + for (size_t i = 0; i < sizeof(canon) / sizeof(canon[0]); i++) { + if (strcmp(label, canon[i]) == 0) { + return canon[i]; + } + } + return arena ? cbm_arena_strdup(arena, label) : NULL; } return NULL; } @@ -268,22 +280,28 @@ static const char *pxc_last_component(const char *qn) { return dot ? dot + 1 : qn; } +/* Every return is arena-owned: the fallback spelling is a copy, never the + * result's pointer (the result may be parked on disk before the def is read). */ static const char *pxc_jvm_type_qn(CBMArena *arena, const char *namespace_name, const char *type_qn_or_name) { - if (!arena || !namespace_name || !namespace_name[0] || !type_qn_or_name) { - return type_qn_or_name; + if (!arena || !type_qn_or_name) { + return NULL; } - const char *short_name = pxc_last_component(type_qn_or_name); + const char *short_name = + (namespace_name && namespace_name[0]) ? pxc_last_component(type_qn_or_name) : NULL; if (!short_name || !short_name[0]) { - return type_qn_or_name; + return cbm_arena_strdup(arena, type_qn_or_name); } return cbm_arena_sprintf(arena, "%s.%s", namespace_name, short_name); } static const char *pxc_jvm_def_qn(CBMArena *arena, const CBMDefinition *src, const char *namespace_name, const char *label) { - if (!arena || !src || !namespace_name || !namespace_name[0]) { - return src ? src->qualified_name : NULL; + if (!arena || !src) { + return NULL; + } + if (!namespace_name || !namespace_name[0]) { + return cbm_arena_strdup(arena, src->qualified_name); /* a copy, see above */ } if (strcmp(label, "Method") == 0 || strcmp(label, "Function") == 0 || strcmp(label, "Constructor") == 0) { @@ -373,7 +391,7 @@ static int pxc_build_lsp_def(CBMArena *arena, const CBMDefinition *src, const ch const char *namespace_name, CBMLanguage lang, CBMLSPDef *dst, const cbm_registry_t *reg, const char **imp_keys, const char **imp_vals, int imp_count) { - const char *label = pxc_map_label(src->label); + const char *label = pxc_map_label(arena, src->label); if (!label || !src->qualified_name || !src->name) return -1; memset(dst, 0, sizeof(*dst)); @@ -381,18 +399,20 @@ static int pxc_build_lsp_def(CBMArena *arena, const CBMDefinition *src, const ch dst->qualified_name = pxc_jvm_def_qn(arena, src, namespace_name, label); dst->receiver_type = pxc_jvm_type_qn(arena, namespace_name, src->parent_class); } else { - dst->qualified_name = src->qualified_name; - dst->receiver_type = src->parent_class; + dst->qualified_name = cbm_arena_strdup(arena, src->qualified_name); + dst->receiver_type = src->parent_class ? cbm_arena_strdup(arena, src->parent_class) : NULL; } - dst->short_name = src->name; + /* Copies, not borrows: a result may be parked on disk (spill mode) the + * moment the collector is done with it; the def list outlives it. */ + dst->short_name = cbm_arena_strdup(arena, src->name); dst->label = label; dst->def_module_qn = module_qn; - dst->namespace_name = namespace_name; + dst->namespace_name = namespace_name ? cbm_arena_strdup(arena, namespace_name) : NULL; dst->is_interface = (strcmp(label, "Interface") == 0 || strcmp(label, "Protocol") == 0); /* Single return-type string. The per-language registrars split on '|' * for multi-return languages (Go); single-return languages just see one * piece, which is what's already stored. */ - dst->return_types = src->return_type; + dst->return_types = src->return_type ? cbm_arena_strdup(arena, src->return_type) : NULL; /* Languages whose cross registrars read embedded_types as QNs get their * bases resolved against the project registry; everyone else keeps the * raw source spelling their own registrar already knows how to handle. */ @@ -400,14 +420,42 @@ static int pxc_build_lsp_def(CBMArena *arena, const CBMDefinition *src, const ch ? pxc_join_base_qns(arena, src->base_classes, reg, module_qn, imp_keys, imp_vals, imp_count) : pxc_join_pipe(arena, src->base_classes); - dst->signature_param_types = src->signature_param_types; - dst->signature_param_count = src->signature_param_count; + dst->signature_param_types = NULL; + dst->signature_param_count = 0; + if (src->signature_param_types && src->signature_param_count > 0) { + const char **types = (const char **)cbm_arena_alloc( + arena, (size_t)src->signature_param_count * sizeof(const char *)); + if (types) { + for (int i = 0; i < src->signature_param_count; i++) { + types[i] = src->signature_param_types[i] + ? cbm_arena_strdup(arena, src->signature_param_types[i]) + : NULL; + } + dst->signature_param_types = types; + dst->signature_param_count = src->signature_param_count; + } + } dst->lang = lang; - dst->decorators = src->decorators; + dst->decorators = NULL; + if (src->decorators) { + int n = 0; + while (src->decorators[n]) { + n++; + } + const char **decos = + (const char **)cbm_arena_alloc(arena, (size_t)(n + 1) * sizeof(const char *)); + if (decos) { + for (int i = 0; i < n; i++) { + decos[i] = cbm_arena_strdup(arena, src->decorators[i]); + } + decos[n] = NULL; + dst->decorators = decos; + } + } if (lang == CBM_LANG_RUST) { /* Exact impl-block provenance is captured while the Rust impl node is * still on hand. Do not reconstruct it later from leaf names. */ - dst->trait_qn = src->impl_trait; + dst->trait_qn = src->impl_trait ? cbm_arena_strdup(arena, src->impl_trait) : NULL; dst->is_abstract = src->is_abstract; } return 0; @@ -490,14 +538,14 @@ static int pxc_build_rust_impl_relation(CBMArena *arena, const CBMImplTrait *imp if (!arena || !impl || !impl->trait_name || !impl->struct_name || !impl->struct_qn) { return -1; } - const char *receiver_qn = impl->struct_qn; + const char *receiver_qn = cbm_arena_strdup(arena, impl->struct_qn); memset(dst, 0, sizeof(*dst)); dst->qualified_name = receiver_qn; dst->short_name = pxc_qn_leaf(receiver_qn); dst->label = "RustImpl"; dst->receiver_type = receiver_qn; dst->def_module_qn = module_qn; - dst->trait_qn = impl->trait_name; /* raw; canonicalized by Rust registry */ + dst->trait_qn = cbm_arena_strdup(arena, impl->trait_name); /* raw; canonicalized later */ dst->lang = CBM_LANG_RUST; dst->is_rust_impl_relation = true; return 0; @@ -506,17 +554,24 @@ static int pxc_build_rust_impl_relation(CBMArena *arena, const CBMImplTrait *imp /* Collect a project-wide CBMLSPDef[] from all cached results. Returns a * malloc'd array (caller frees) of length *out_count. String fields are * borrowed from cache[i]->arena and from def_modules[i] (also borrowed). */ -CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMFileResult **cache, - const cbm_file_info_t *files, int file_count, - const char *project_name, char **def_modules, int *out_count, - int *out_def_starts) { +CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMArena *arena, + CBMFileResult **cache, const cbm_file_info_t *files, + int file_count, const char *project_name, char **def_modules, + int *out_count, int *out_def_starts) { + const cbm_result_spill_t *spill = ctx ? ctx->spill : NULL; int total = 0; for (int i = 0; i < file_count; i++) { + int defs = 0; + int impls = 0; if (cache[i]) { - total += cache[i]->defs.count; - if (files[i].language == CBM_LANG_RUST) { - total += cache[i]->impl_traits.count; - } + defs = cache[i]->defs.count; + impls = cache[i]->impl_traits.count; + } else if (spill && cbm_result_spill_has(spill, i)) { + cbm_result_spill_peek_counts(spill, i, &defs, &impls); + } + total += defs; + if (files[i].language == CBM_LANG_RUST) { + total += impls; } } if (total == 0) { @@ -540,19 +595,21 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMFileResult out_def_starts[fi] = idx; } const int file_start = idx; - if (!cache[fi]) + CBMFileResult *fr = cache[fi]; + bool fr_loaded = false; + if (!fr && spill && cbm_result_spill_has(spill, fi)) { + fr = cbm_result_spill_load(spill, fi); + fr_loaded = fr != NULL; + } + if (!fr) continue; if (!def_modules[fi]) { def_modules[fi] = cbm_pipeline_fqn_module_dir(project_name, files[fi].rel_path, pxc_module_is_dir(files[fi].language)); } - const char *namespace_name = cache[fi]->namespace_name; + const char *namespace_name = fr->namespace_name; if ((!namespace_name || !namespace_name[0]) && files[fi].rel_path) { - namespace_name = - pxc_infer_jvm_namespace(&cache[fi]->arena, files[fi].rel_path, files[fi].language); - if (namespace_name && namespace_name[0]) { - cache[fi]->namespace_name = namespace_name; - } + namespace_name = pxc_infer_jvm_namespace(arena, files[fi].rel_path, files[fi].language); } /* One import map per FILE (not per def, and not per base name): the * cross-file base-class resolution below needs the same local-name → @@ -567,29 +624,31 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMFileResult if (ctx && ctx->registry && pxc_lang_resolves_base_qns(files[fi].language)) { base_reg = ctx->registry; cbm_pxc_build_import_map(ctx->gbuf, project_name, files[fi].rel_path, - files[fi].language, cache[fi], &imp_keys, &imp_vals, - &imp_count); + files[fi].language, fr, &imp_keys, &imp_vals, &imp_count); } - for (int di = 0; di < cache[fi]->defs.count; di++) { - if (pxc_build_lsp_def(&cache[fi]->arena, &cache[fi]->defs.items[di], def_modules[fi], - namespace_name, files[fi].language, &defs[idx], base_reg, - imp_keys, imp_vals, imp_count) == 0) { + for (int di = 0; di < fr->defs.count; di++) { + if (pxc_build_lsp_def(arena, &fr->defs.items[di], def_modules[fi], namespace_name, + files[fi].language, &defs[idx], base_reg, imp_keys, imp_vals, + imp_count) == 0) { idx++; } } cbm_pxc_free_import_map(imp_keys, imp_vals, imp_count); /* NULL-safe */ if (files[fi].language == CBM_LANG_GO) { - pxc_fold_go_struct_fields(&cache[fi]->arena, cache[fi], defs, file_start, idx); + pxc_fold_go_struct_fields(arena, fr, defs, file_start, idx); } if (files[fi].language == CBM_LANG_RUST) { - for (int ii = 0; ii < cache[fi]->impl_traits.count; ii++) { - if (pxc_build_rust_impl_relation( - &cache[fi]->arena, &cache[fi]->impl_traits.items[ii], project_name, - files[fi].rel_path, def_modules[fi], &defs[idx]) == 0) { + for (int ii = 0; ii < fr->impl_traits.count; ii++) { + if (pxc_build_rust_impl_relation(arena, &fr->impl_traits.items[ii], project_name, + files[fi].rel_path, def_modules[fi], + &defs[idx]) == 0) { idx++; } } } + if (fr_loaded) { + cbm_free_result(fr); + } } if (out_def_starts) { out_def_starts[file_count] = idx; @@ -1065,7 +1124,15 @@ static void pxc_append_synthetic_calls(CBMArena *dst_arena, CBMCallArray *dst_ca src->first_string_arg ? cbm_arena_strdup(dst_arena, src->first_string_arg) : NULL; dst.second_arg_name = src->second_arg_name ? cbm_arena_strdup(dst_arena, src->second_arg_name) : NULL; - for (int ai = 0; ai < CBM_MAX_CALL_ARGS; ai++) { + dst.args = NULL; + if (src->args && src->arg_count > 0) { + dst.args = cbm_arena_calloc(dst_arena, (size_t)src->arg_count * sizeof(CBMCallArg)); + if (!dst.args) { + dst.arg_count = 0; + } + } + for (int ai = 0; dst.args && ai < src->arg_count; ai++) { + dst.args[ai].index = src->args[ai].index; dst.args[ai].expr = src->args[ai].expr ? cbm_arena_strdup(dst_arena, src->args[ai].expr) : NULL; dst.args[ai].value = @@ -1272,6 +1339,10 @@ void cbm_pxc_filter_stats(uint64_t *defs_registered, uint64_t *build_files, uint *filter_failed = atomic_load_explicit(&g_pxc_filter_failed, memory_order_relaxed); } +/* A file the extractor marked lsp_skipped (its parse alone used more than its + * share of the per-file budget) takes no cross-file resolve either: the same + * tree the per-file walk could not afford. One check for every language and + * both drivers. */ void cbm_pxc_dispatch_file(CBMLanguage lang, CBMFileResult *result, const char *source, int source_len, const char *rel, const char *def_module, const CBMCrossLspRegistries *cross_registries, @@ -1279,6 +1350,9 @@ void cbm_pxc_dispatch_file(CBMLanguage lang, CBMFileResult *result, const char * int all_def_count, const char **imp_keys, const char **imp_vals, int imp_count, CBMTypeRegistry *(*rust_shared_get)(void *), void *rust_shared_ctx) { + if (result && result->lsp_skipped) { + return; + } if (!result) { return; } @@ -1286,63 +1360,64 @@ void cbm_pxc_dispatch_file(CBMLanguage lang, CBMFileResult *result, const char * CBMTypeRegistry *prebuilt = cross_registries ? cbm_pxc_registry_for_lang(cross_registries, lang) : NULL; if (prebuilt) { + /* One lifecycle for every language (2026-09-13): the walk runs in a + * scratch arena against a per-file OVERLAY registry chained to the + * immutable shared base; whatever it registers or refines lands in + * the overlay and dies with the file; only the resolved calls it + * produces are copied into the result. Before this, Go/C/C#/Java/TS + * were handed the RESULT arena (and C/C#/TS the shared base to write + * into): the kernel proof measured that as 4.4 GB of resolve-time + * growth in retained results plus arena pointers stored into the + * shared registry -- a use-after-free class the moment the arena was + * not the result arena (ASan, lsp_resolution_probe). */ + CBMArena scratch; + cbm_arena_init(&scratch); + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &scratch); + overlay.fallback = prebuilt; + CBMResolvedCallArray out = {0}; + CBMCallArray synthetic_calls = {0}; switch (lang) { case CBM_LANG_GO: /* Tier 3 (metadata-driven): pure lookup over the Tier-1 - * lsp_unresolved entries — no parse, no AST walk. Then the - * AST walk on the shared Tier-2 registry (mirroring every - * other language) so NAMED receivers evaluated against - * project-wide defs also resolve. The walk variant below is - * read-only — the sealed registry is safe for parallel - * workers. */ + * lsp_unresolved entries -- no parse, no AST walk -- then the + * AST walk against the overlay (chained to the sealed base). */ cbm_go_fast_resolve_qualified_calls(result, prebuilt, imp_keys, imp_vals, imp_count); - cbm_run_go_lsp_cross_with_registry(&result->arena, source, source_len, def_module, - prebuilt, imp_keys, imp_vals, imp_count, - result->cached_tree, &result->resolved_calls); + cbm_run_go_lsp_cross_with_registry(&scratch, source, source_len, def_module, &overlay, + imp_keys, imp_vals, imp_count, result->cached_tree, + &out); used_prebuilt = true; break; - case CBM_LANG_PYTHON: { - CBMArena scratch; - cbm_arena_init(&scratch); - CBMResolvedCallArray out = {0}; - CBMCallArray synthetic_calls = {0}; - cbm_run_py_lsp_cross_with_registry(&scratch, source, source_len, def_module, prebuilt, + case CBM_LANG_PYTHON: + cbm_run_py_lsp_cross_with_registry(&scratch, source, source_len, def_module, &overlay, imp_keys, imp_vals, imp_count, result->cached_tree, &out, &synthetic_calls); - pxc_append_results(&result->arena, &result->resolved_calls, &out); - pxc_append_synthetic_calls(&result->arena, &result->calls, &synthetic_calls); - cbm_arena_destroy(&scratch); used_prebuilt = true; break; - } case CBM_LANG_C: case CBM_LANG_CPP: case CBM_LANG_CUDA: - cbm_run_c_lsp_cross_with_registry( - &result->arena, source, source_len, def_module, (lang != CBM_LANG_C), prebuilt, - imp_keys, imp_vals, imp_count, result->cached_tree, &result->resolved_calls); + cbm_run_c_lsp_cross_with_registry(&scratch, source, source_len, def_module, + (lang != CBM_LANG_C), &overlay, imp_keys, imp_vals, + imp_count, result->cached_tree, &out); used_prebuilt = true; break; case CBM_LANG_CSHARP: - cbm_run_cs_lsp_cross_with_registry(&result->arena, source, source_len, def_module, - prebuilt, imp_vals, imp_count, result->cached_tree, - &result->resolved_calls); + cbm_run_cs_lsp_cross_with_registry(&scratch, source, source_len, def_module, &overlay, + imp_vals, imp_count, result->cached_tree, &out); used_prebuilt = true; break; case CBM_LANG_JAVA: - /* Own-module defs go into a per-file overlay; imports and stdlib - * resolve through the shared base (#1669). */ - cbm_run_java_lsp_cross_with_registry( - &result->arena, result, source, source_len, def_module, prebuilt, imp_keys, - imp_vals, imp_count, result->cached_tree, &result->resolved_calls); + /* Java builds its own-module overlay on top of the one it is + * handed; both live in scratch now. */ + cbm_run_java_lsp_cross_with_registry(&scratch, result, source, source_len, def_module, + &overlay, imp_keys, imp_vals, imp_count, + result->cached_tree, &out); used_prebuilt = true; break; case CBM_LANG_JAVASCRIPT: case CBM_LANG_TYPESCRIPT: case CBM_LANG_TSX: { - /* TS: per-file OVERLAY chained to the shared base. Filter to - * own+imports so the overlay builder can pick out own-module - * defs without scanning the whole project. */ bool js; bool jsx; bool dts; @@ -1361,10 +1436,9 @@ void cbm_pxc_dispatch_file(CBMLanguage lang, CBMFileResult *result, const char * ts_def_count = fc; } } - cbm_run_ts_lsp_cross_with_registry(&result->arena, source, source_len, def_module, js, - jsx, dts, prebuilt, ts_defs, ts_def_count, imp_keys, - imp_vals, imp_count, result->cached_tree, - &result->resolved_calls); + cbm_run_ts_lsp_cross_with_registry(&scratch, source, source_len, def_module, js, jsx, + dts, &overlay, ts_defs, ts_def_count, imp_keys, + imp_vals, imp_count, result->cached_tree, &out); free(ts_filtered); used_prebuilt = true; break; @@ -1374,6 +1448,11 @@ void cbm_pxc_dispatch_file(CBMLanguage lang, CBMFileResult *result, const char * default: break; } + if (used_prebuilt) { + pxc_append_results(&result->arena, &result->resolved_calls, &out); + pxc_append_synthetic_calls(&result->arena, &result->calls, &synthetic_calls); + } + cbm_arena_destroy(&scratch); } if (used_prebuilt) { @@ -1498,15 +1577,22 @@ int cbm_pipeline_pass_lsp_cross(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t * int def_count = 0; int *def_starts = (int *)calloc((size_t)file_count + 1, sizeof(int)); - CBMLSPDef *all_defs = cbm_pxc_collect_all_defs(ctx, cache, files, file_count, ctx->project_name, - def_modules, &def_count, def_starts); + /* The defs own their strings in the caller-owned seq_cross_arena, which + * the registries and later passes already borrow from (see below). */ + if (!ctx->seq_cross_arena_live) { + cbm_arena_init(&ctx->seq_cross_arena); + ctx->seq_cross_arena_live = true; + } + CBMLSPDef *all_defs = + cbm_pxc_collect_all_defs(ctx, &ctx->seq_cross_arena, cache, files, file_count, + ctx->project_name, def_modules, &def_count, def_starts); /* Same seam as the parallel driver: serialize per-file surfaces while the * result cache is alive. Failure only degrades to a full rebuild on the * next incremental run. */ if (ctx->pipeline && all_defs && def_starts) { cbm_lsp_surface_row_t *surface_rows = NULL; int surface_count = 0; - if (cbm_lsp_surface_build_rows(ctx->project_name, cache, files, file_count, all_defs, + if (cbm_lsp_surface_build_rows(ctx, ctx->project_name, cache, files, file_count, all_defs, def_starts, &surface_rows, &surface_count) == 0) { cbm_pipeline_set_lsp_surfaces(ctx->pipeline, surface_rows, surface_count); } diff --git a/src/pipeline/pass_lsp_cross.h b/src/pipeline/pass_lsp_cross.h index 705b804cc..04a65f2ba 100644 --- a/src/pipeline/pass_lsp_cross.h +++ b/src/pipeline/pass_lsp_cross.h @@ -63,10 +63,10 @@ bool cbm_pxc_has_cross_lsp(CBMLanguage lang); * inputs pass_semantic uses to draw its INHERITS edge, so the LSP's * inheritance view and the graph's cannot diverge. Pass NULL to keep the raw * source spelling (surface-probe paths that build no registry). */ -CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMFileResult **cache, - const cbm_file_info_t *files, int file_count, - const char *project_name, char **def_modules, int *out_count, - int *out_def_starts); +CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMArena *arena, + CBMFileResult **cache, const cbm_file_info_t *files, + int file_count, const char *project_name, char **def_modules, + int *out_count, int *out_def_starts); /* Detect TS dialect flags from a relative path. */ void cbm_pxc_ts_modes(CBMLanguage lang, const char *rel_path, bool *out_js, bool *out_jsx, diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 759c199f6..e3439f293 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -70,6 +70,8 @@ enum { PP_CSHARP_M_PREFIX_LEN = 2 }; #define PP_RETAIN_PER_FILE_HARD_MAX_BYTES (32ULL * 1024 * 1024) /* 32 MiB per file */ #include "pipeline/pipeline.h" #include "pipeline/pipeline_internal.h" +#include "result_spill.h" +#include "foundation/platform.h" /* cbm_resolve_cache_dir */ #include "pipeline/pass_lsp_cross.h" /* cbm_pxc_* helpers for fused cross-file LSP */ #include "pipeline/lsp_resolve.h" #include "lsp/rust_cargo.h" @@ -649,6 +651,12 @@ typedef struct { _Atomic int64_t *shared_ids; _Atomic int *cancelled; _Atomic int next_file_idx; + cbm_pipeline_ctx_t *pctx; /* spill store + mode latch live here */ + _Atomic uint8_t *slot_state; /* per file: 0 in progress, 1 cached, 2 parked */ + _Atomic int spill_sweeps_running; /* workers inside pp_spill_sweep right now */ + int spill_env; /* CBM_MEM_SPILL for this run: 1 force, 0 off, 2 budget decides */ + cbm_mutex_t spill_mu; /* serializes the one store open */ + _Atomic int spill_unavailable; /* store could not open: never retry */ cbm_pkg_entries_t *pkg_entries; /* per-worker manifest arrays (separate allocation) */ @@ -762,13 +770,174 @@ static void pp_fail_whole_over_budget(extract_ctx_t *ec) { } } +/* ── Spill / admission control ──────────────────────────────────────── */ + +/* Spill mode is entered this fraction short of the budget: 1/16 = 6.25%, + * ~940 MB at 15 GB, above the 4% the kernel's in-flight files were measured + * to carry past the line. */ +enum { PP_SPILL_EARLY_DIV = 16 }; + +/* CBM_MEM_SPILL=1 forces spill mode from the first file (tests, small + * machines); CBM_MEM_SPILL=0 keeps results in memory even over budget. Read + * once per extraction run, never cached per process: a test toggles it. */ +static int pp_spill_env_read(void) { + char buf[CBM_SZ_16]; + if (cbm_safe_getenv("CBM_MEM_SPILL", buf, sizeof(buf), NULL)) { + return buf[0] == '1' ? 1 : 0; + } + return 2; /* unset: the budget decides */ +} +static bool pp_spill_forced_by_env(const extract_ctx_t *ec) { + return ec->spill_env == 1; +} +static bool pp_spill_allowed(const extract_ctx_t *ec) { + return ec->spill_env != 0; +} + +/* Latch spill mode (once) and open the store. The store is open BEFORE the + * latch is visible: a peer that reads the latch must also find the store, or + * it sees "nothing to park" while this worker is still opening files and runs + * the futility cycle instead of joining the sweep (kernel, 16 GB budget, + * 2026-09-13: the abort fired before the first sweep had logged). Never fails + * the run: without a store the results simply stay in memory and the nap / + * futility path applies as before. */ +static void pp_spill_enter(extract_ctx_t *ec, const char *reason) { + if (!ec->pctx || !ec->pctx->spill_allowed || !pp_spill_allowed(ec)) { + return; + } + if (atomic_load_explicit(&ec->pctx->spill_mode, memory_order_acquire) != 0 || + atomic_load_explicit(&ec->spill_unavailable, memory_order_relaxed) != 0) { + return; + } + cbm_mutex_lock(&ec->spill_mu); + if (atomic_load_explicit(&ec->pctx->spill_mode, memory_order_acquire) == 0 && + atomic_load_explicit(&ec->spill_unavailable, memory_order_relaxed) == 0) { + if (!ec->pctx->spill) { + ec->pctx->spill = + cbm_result_spill_open(cbm_resolve_cache_dir(), ec->max_workers, ec->file_count); + } + size_t mb = (size_t)1024 * 1024; + cbm_log_warn("mem.spill.on", "reason", reason, "charged_mb", + itoa_log((int)(cbm_mem_charged() / mb)), "budget_mb", + itoa_log((int)(cbm_mem_budget() / mb)), "store", + ec->pctx->spill ? "open" : "unavailable"); + if (ec->pctx->spill) { + atomic_store_explicit(&ec->pctx->spill_mode, 1, memory_order_release); + } else { + atomic_store_explicit(&ec->spill_unavailable, 1, memory_order_relaxed); + } + } + cbm_mutex_unlock(&ec->spill_mu); +} + +static bool pp_spill_active(const extract_ctx_t *ec) { + return ec->pctx && ec->pctx->spill && + atomic_load_explicit(&ec->pctx->spill_mode, memory_order_acquire) != 0; +} + +CBMFileResult *cbm_pipeline_result_acquire(const cbm_pipeline_ctx_t *ctx, CBMFileResult **cache, + int i, cbm_result_want_fn want, bool *loaded) { + *loaded = false; + if (cache && cache[i]) { + return (!want || want(cache[i])) ? cache[i] : NULL; + } + if (!ctx || !ctx->spill || !cbm_result_spill_has(ctx->spill, i)) { + return NULL; + } + if (want) { + CBMFileResult hdr; + if (!cbm_result_spill_peek_header(ctx->spill, i, &hdr) || !want(&hdr)) { + return NULL; + } + } + CBMFileResult *r = cbm_result_spill_load(ctx->spill, i); + *loaded = r != NULL; + return r; +} + +void cbm_pipeline_result_release(CBMFileResult *r, bool loaded) { + if (r && loaded) { + cbm_free_result(r); + } +} + +void cbm_pipeline_spill_close(cbm_pipeline_ctx_t *ctx) { + if (!ctx || !ctx->spill) { + return; + } + int64_t parked = 0; + int64_t bytes = 0; + int64_t loads = 0; + cbm_result_spill_stats(ctx->spill, &parked, &bytes, &loads); + cbm_log_info("mem.spill.done", "parked", itoa_log((int)parked), "mb", + itoa_log((int)(bytes / (1024 * 1024))), "loads", itoa_log((int)loads)); + cbm_result_spill_close(ctx->spill); + ctx->spill = NULL; + atomic_store_explicit(&ctx->spill_mode, 0, memory_order_release); + cbm_work_arena_release(); +} + +/* Park every cached result that is complete. Slots are claimed 1 -> 2 so two + * workers never park the same result; a failed park hands the slot back. */ +/* True while a cached (not yet parked) result exists or a sweep is running: + * memory can still come down without anyone napping. */ +static bool pp_spill_work_remains(const extract_ctx_t *ec) { + if (!ec->slot_state) { + return false; + } + if (atomic_load_explicit(&ec->spill_sweeps_running, memory_order_acquire) > 0) { + return true; + } + for (int i = 0; i < ec->file_count; i++) { + if (atomic_load_explicit(&ec->slot_state[i], memory_order_relaxed) == 1) { + return true; + } + } + return false; +} + +static int pp_spill_sweep(extract_ctx_t *ec, int worker_id) { + if (!pp_spill_active(ec) || !ec->slot_state) { + return 0; + } + atomic_fetch_add_explicit(&ec->spill_sweeps_running, 1, memory_order_acq_rel); + int parked = 0; + for (int i = 0; i < ec->file_count; i++) { + if (atomic_load_explicit(&ec->over_budget_abort, memory_order_relaxed)) { + break; + } + uint8_t expected = 1; + if (!atomic_compare_exchange_strong_explicit(&ec->slot_state[i], &expected, (uint8_t)2, + memory_order_acq_rel, memory_order_relaxed)) { + continue; + } + CBMFileResult *r = ec->result_cache[i]; + if (r && cbm_result_spill_park(ec->pctx->spill, worker_id, i, r)) { + ec->result_cache[i] = NULL; + parked++; + if ((parked & 255) == 0) { + cbm_mem_release_to_os(); + } + } else { + atomic_store_explicit(&ec->slot_state[i], (uint8_t)1, memory_order_release); + } + } + if (parked > 0) { + cbm_mem_release_to_os(); + cbm_log_info("mem.spill.sweep", "parked", itoa_log(parked), "charged_mb", + itoa_log((int)(cbm_mem_charged() / ((size_t)1024 * 1024)))); + } + atomic_fetch_sub_explicit(&ec->spill_sweeps_running, 1, memory_order_acq_rel); + return parked; +} + static void extract_worker(int worker_id, void *ctx_ptr) { extract_ctx_t *ec = ctx_ptr; extract_worker_state_t *ws = &ec->workers[worker_id]; /* Lazy gbuf creation */ if (!ws->local_gbuf) { - ws->local_gbuf = cbm_gbuf_new_shared_ids(ec->project_name, ec->repo_path, ec->shared_ids); + ws->local_gbuf = cbm_gbuf_new_worker(ec->project_name, ec->repo_path, ec->shared_ids); } /* Pull files from shared atomic counter */ @@ -805,6 +974,32 @@ static void extract_worker(int worker_id, void *ctx_ptr) { * and the previously serving index keeps answering. */ if (cbm_mem_budget() > 0) { bool over = cbm_mem_over_budget(); + /* Anticipation: the gate sees the crossing per file pull, and the + * workers' in-flight files carry the charge past the line before + * the first sweep lands (kernel, 15 GB budget: high-water 15.65 + * GB, 4% over, all of it set in that window). Spill mode is + * entered PP_SPILL_EARLY_DIV-th short of the budget, so results + * stop accumulating in memory while the headroom still covers the + * in-flight work. The nap / futility path still keys on the + * budget itself. */ + if (!over && !pp_spill_active(ec)) { + size_t budget = cbm_mem_budget(); + if (cbm_mem_charged() > budget - budget / PP_SPILL_EARLY_DIV) { + pp_spill_enter(ec, "near_budget"); + } + } + if (over) { + /* Admission control, first response: park what can be parked. + * Only what is still over budget after that -- the floor -- + * reaches the nap / futility / abort path below, and only + * once nothing is left to park anywhere. */ + pp_spill_enter(ec, "over_budget"); + int parked = pp_spill_sweep(ec, worker_id); + over = cbm_mem_over_budget(); + if (over && pp_spill_active(ec) && (parked > 0 || pp_spill_work_remains(ec))) { + over = false; /* memory is still on its way down */ + } + } bool futile = atomic_load_explicit(&ec->bp_futile, memory_order_relaxed) != 0; if (over && !futile) { /* Act only on the 0→1 transition: all workers race into the @@ -1005,8 +1200,24 @@ static void extract_worker(int worker_id, void *ctx_ptr) { * and the retention copy (if any) lives in result->arena. */ free_source(source); - /* Cache result (arena + extracted data, no tree) for Phase 3B and Phase 4 */ - ec->result_cache[file_idx] = result; + /* Everything this file will ever contribute has been written; drop + * the working arena (node-text copies, abandoned array generations) + * and keep only the reachable result. See cbm_result_compact. */ + cbm_result_compact(result); + + /* Cache result (arena + extracted data, no tree) for Phase 3B and + * Phase 4 -- or park it straight to disk in spill mode. */ + if (pp_spill_active(ec) && + cbm_result_spill_park(ec->pctx->spill, worker_id, file_idx, result)) { + if (ec->slot_state) { + atomic_store_explicit(&ec->slot_state[file_idx], (uint8_t)2, memory_order_release); + } + } else { + ec->result_cache[file_idx] = result; + if (ec->slot_state) { + atomic_store_explicit(&ec->slot_state[file_idx], (uint8_t)1, memory_order_release); + } + } /* Progress logging: log every 10 files (atomic read, no contention) */ if ((sort_pos + SKIP_ONE) % PP_LOG_INTERVAL == 0 || sort_pos + SKIP_ONE == ec->file_count) { @@ -1032,6 +1243,7 @@ static void extract_worker(int worker_id, void *ctx_ptr) { } /* Final cleanup (parser already destroyed in loop, just slab state) */ + cbm_work_arena_release(); /* the working arena kept between files */ cbm_slab_destroy_thread(); cbm_kind_in_set_free_cache(); /* free this worker thread's node-type bitset cache */ } @@ -1158,6 +1370,8 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file .result_cache = result_cache, .shared_ids = shared_ids, .cancelled = ctx->cancelled, + .pctx = ctx, + .slot_state = NULL, .pkg_entries = pkg_entries, .err_lists = err_lists, .retain_sources = resolved_opts.retain_sources, @@ -1173,12 +1387,30 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file atomic_init(&ec.oversized_warned, 0); atomic_init(&ec.bp_futile, 0); atomic_init(&ec.over_budget_abort, 0); + atomic_init(&ec.spill_sweeps_running, 0); + atomic_init(&ec.spill_unavailable, 0); + cbm_mutex_init(&ec.spill_mu); + ec.slot_state = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)file_count * sizeof(_Atomic uint8_t)); + ec.spill_env = pp_spill_env_read(); + if (pp_spill_forced_by_env(&ec)) { + pp_spill_enter(&ec, "env"); + } /* Sub-phase: Dispatch workers (parse + extract per file, PARALLEL) */ CBM_PROF_START(t_dispatch); cbm_parallel_for_opts_t parallel_opts = {.max_workers = worker_count, .force_pthreads = false}; cbm_scale_begin(&ec.scale, "parallel_extract", (long)file_count); cbm_parallel_for(worker_count, extract_worker, &ec, parallel_opts); + if (ctx->spill) { + int64_t parked = 0; + int64_t bytes = 0; + cbm_result_spill_stats(ctx->spill, &parked, &bytes, NULL); + cbm_log_info("mem.spill.extract_done", "parked", itoa_log((int)parked), "mb", + itoa_log((int)(bytes / (1024 * 1024)))); + } + cbm_mutex_destroy(&ec.spill_mu); + cbm_free(CBM_MEM_CLASS_OTHER, ec.slot_state); + ec.slot_state = NULL; cbm_scale_end(&ec.scale); CBM_PROF_END_N("parallel_extract", "3_dispatch_workers_parallel", t_dispatch, file_count); @@ -1378,6 +1610,11 @@ int cbm_build_registry_from_cache(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t } CBMFileResult *result = result_cache[i]; + bool loaded = false; + if (!result && ctx->spill && cbm_result_spill_has(ctx->spill, i)) { + result = cbm_result_spill_load(ctx->spill, i); + loaded = result != NULL; + } if (!result) { continue; } @@ -1392,6 +1629,9 @@ int cbm_build_registry_from_cache(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t imports_edges += create_imports_edges(ctx, result, rel, namespace_map); create_channel_edges(ctx, result, rel); cbm_pipeline_create_env_configures_for_file(ctx, result, rel); + if (loaded) { + cbm_free_result(result); + } } cbm_pipeline_namespace_map_free(namespace_map); @@ -1414,7 +1654,8 @@ typedef struct __attribute__((aligned(CBM_CACHE_LINE))) { * registry's textual matcher. Surfaced in the parallel.resolve.done * log line so divergence between pipelines becomes observable. */ int lsp_overrides; - char _pad[CBM_CACHE_LINE - sizeof(cbm_gbuf_t *) - ((PP_RING + 1) * sizeof(int))]; + CBMFileResult *loaded; /* spill-loaded result this worker is using */ + char _pad[CBM_CACHE_LINE - 2 * sizeof(void *) - ((PP_RING + 1) * sizeof(int))]; } resolve_worker_state_t; typedef struct { @@ -1502,6 +1743,7 @@ typedef struct { * necessary (#1669: 87% of a Java index), so it is the one that must never * again grow superlinear without saying so. */ cbm_scale_probe_t scale; + cbm_pipeline_ctx_t *pctx; /* spill store access */ } resolve_ctx_t; /* Minimum buffer space needed per arg JSON object */ @@ -3048,8 +3290,7 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { cbm_pxc_set_rust_manifest(rc->rust_manifest); if (!ws->local_edge_buf) { - ws->local_edge_buf = - cbm_gbuf_new_shared_ids(rc->project_name, rc->repo_path, rc->shared_ids); + ws->local_edge_buf = cbm_gbuf_new_worker(rc->project_name, rc->repo_path, rc->shared_ids); } /* Per-worker service-pattern result cache. The same resolved QN @@ -3073,7 +3314,16 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { uint64_t _loop_t0 = extract_now_ns(); + if (ws->loaded) { + cbm_free_result(ws->loaded); + ws->loaded = NULL; + } CBMFileResult *result = rc->result_cache[file_idx]; + if (!result && rc->pctx && rc->pctx->spill && + cbm_result_spill_has(rc->pctx->spill, file_idx)) { + result = cbm_result_spill_load(rc->pctx->spill, file_idx); + ws->loaded = result; + } if (!result) { atomic_fetch_add_explicit(&rc->time_ns_total_loop, extract_now_ns() - _loop_t0, memory_order_relaxed); @@ -3293,6 +3543,10 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { * into dead TLS and are never retired, so a later cross-thread free can * never bring their refcount to zero (leak). Retiring them here releases * each page as its final chunk returns. */ + if (ws->loaded) { + cbm_free_result(ws->loaded); + ws->loaded = NULL; + } cbm_pxc_set_rust_manifest(NULL); cbm_destroy_thread_parser(); cbm_slab_destroy_thread(); @@ -3322,7 +3576,8 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, bool have_rust = false; for (int i = 0; i < file_count; i++) { - if (result_cache[i] && files[i].language == CBM_LANG_RUST) { + if (files[i].language == CBM_LANG_RUST && + (result_cache[i] || (ctx->spill && cbm_result_spill_has(ctx->spill, i)))) { have_rust = true; break; } @@ -3340,6 +3595,7 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, } resolve_ctx_t rc = { + .pctx = ctx, .files = files, .file_count = file_count, .project_name = ctx->project_name, diff --git a/src/pipeline/pass_semantic.c b/src/pipeline/pass_semantic.c index 8afc53958..58821beb0 100644 --- a/src/pipeline/pass_semantic.c +++ b/src/pipeline/pass_semantic.c @@ -12,6 +12,7 @@ * Depends on: pass_definitions having populated the registry and graph buffer */ #include "foundation/constants.h" +#include "foundation/mem_core.h" #include "foundation/str_util.h" // cbm_json_escape #include "pipeline/pipeline.h" #include @@ -49,7 +50,7 @@ static char *read_file(const char *path, int *out_len) { } /* +pad: tree-sitter lexer lookahead reads past EOF; keep it in-bounds */ enum { CBM_TS_LOOKAHEAD_PAD = 16 }; - char *buf = malloc((size_t)size + CBM_TS_LOOKAHEAD_PAD); + char *buf = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)size + CBM_TS_LOOKAHEAD_PAD); if (!buf) { (void)fclose(f); return NULL; @@ -84,8 +85,12 @@ static int build_import_map(cbm_pipeline_ctx_t *ctx, const char *rel_path, /* Fast path: build from cached extraction result (no JSON parsing) */ if (result && result->imports.count > 0) { - const char **keys = calloc((size_t)result->imports.count, sizeof(const char *)); - const char **vals = calloc((size_t)result->imports.count, sizeof(const char *)); + const char **keys = + cbm_calloc(CBM_MEM_CLASS_SEMANTIC, + (size_t)((size_t)result->imports.count) * (sizeof(const char *))); + const char **vals = + cbm_calloc(CBM_MEM_CLASS_SEMANTIC, + (size_t)((size_t)result->imports.count) * (sizeof(const char *))); int count = 0; for (int i = 0; i < result->imports.count; i++) { @@ -99,7 +104,7 @@ static int build_import_map(cbm_pipeline_ctx_t *ctx, const char *rel_path, if (!target) { continue; } - keys[count] = strdup(imp->local_name); + keys[count] = cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, imp->local_name); vals[count] = target->qualified_name; count++; } @@ -126,8 +131,10 @@ static int build_import_map(cbm_pipeline_ctx_t *ctx, const char *rel_path, return 0; } - const char **keys = calloc(edge_count, sizeof(const char *)); - const char **vals = calloc(edge_count, sizeof(const char *)); + const char **keys = + cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)(edge_count) * (sizeof(const char *))); + const char **vals = + cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)(edge_count) * (sizeof(const char *))); int count = 0; for (int i = 0; i < edge_count; i++) { @@ -158,12 +165,12 @@ static int build_import_map(cbm_pipeline_ctx_t *ctx, const char *rel_path, static void free_import_map(const char **keys, const char **vals, int count) { if (keys) { for (int i = 0; i < count; i++) { - free((void *)keys[i]); + cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)keys[i]); } - free((void *)keys); + cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)keys); } if (vals) { - free((void *)vals); + cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)vals); } } @@ -496,7 +503,7 @@ static CBMFileResult *sem_get_or_extract(cbm_pipeline_ctx_t *ctx, int file_idx, } CBMFileResult *r = cbm_extract_file(source, source_len, fi->language, ctx->project_name, fi->rel_path, CBM_EXTRACT_BUDGET, NULL, NULL); - free(source); + cbm_free(CBM_MEM_CLASS_SEMANTIC, source); if (r) { *owned = true; } diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.c index 2b69385da..b71c85a83 100644 --- a/src/pipeline/pass_semantic_edges.c +++ b/src/pipeline/pass_semantic_edges.c @@ -10,6 +10,8 @@ * Runs in moderate and full modes (not fast). Controlled by pipeline mode. */ #include "foundation/constants.h" +#include "foundation/mem.h" +#include "foundation/mem_core.h" #include "pipeline/pipeline.h" #include #include "pipeline/pipeline_internal.h" @@ -93,7 +95,8 @@ static void deferred_buf_push(deferred_edge_buf_t *buf, int64_t src, int64_t tgt bool same_file, int i, int j, int c) { if (buf->count >= buf->cap) { int nc = buf->cap < CBM_SZ_256 ? CBM_SZ_256 : buf->cap * GROW; - deferred_edge_t *grown = realloc(buf->edges, (size_t)nc * sizeof(deferred_edge_t)); + deferred_edge_t *grown = + cbm_realloc(CBM_MEM_CLASS_SEMANTIC, buf->edges, (size_t)nc * sizeof(deferred_edge_t)); if (!grown) { return; } @@ -110,7 +113,7 @@ static void deferred_buf_push(deferred_edge_buf_t *buf, int64_t src, int64_t tgt } static void deferred_buf_free(deferred_edge_buf_t *buf) { - free(buf->edges); + cbm_free(CBM_MEM_CLASS_SEMANTIC, buf->edges); buf->edges = NULL; buf->count = buf->cap = 0; } @@ -128,7 +131,7 @@ static int push_pattern_token(char **tokens, int count, int max_tokens, const ch if (count >= max_tokens) { return count; } - tokens[count] = strdup(text); + tokens[count] = cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, text); return tokens[count] ? count + SKIP_ONE : count; } @@ -357,7 +360,7 @@ static int json_str_array(const char *json, const char *key, char **out, int max break; } int len = (int)(end - start); - out[count] = malloc((size_t)len + SKIP_ONE); + out[count] = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)len + SKIP_ONE); memcpy(out[count], start, (size_t)len); out[count][len] = '\0'; count++; @@ -398,7 +401,7 @@ static int tokenize_json_array_field(const char *json, const char *key, char **t if (count < max_tokens) { count += cbm_sem_tokenize(arr[p], tokens + count, max_tokens - count); } - free(arr[p]); + cbm_free(CBM_MEM_CLASS_SEMANTIC, arr[p]); } return count; } @@ -428,7 +431,7 @@ static const char **collect_sorted_call_neighbors(const cbm_gbuf_t *gbuf, int64_ if (rc != 0 || ec <= 0) { return NULL; } - const char **names = malloc((size_t)ec * sizeof(char *)); + const char **names = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)ec * sizeof(char *)); if (!names) { return NULL; } @@ -458,7 +461,7 @@ static int tokenize_call_neighbors(const cbm_gbuf_node_t *n, const cbm_gbuf_t *g for (int e = 0; e < nn && e < MAX_CALLEES && count < max_tokens; e++) { count += cbm_sem_tokenize(names[e], tokens + count, max_tokens - count); } - free((void *)names); + cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)names); return count; } @@ -511,7 +514,7 @@ static void build_api_vec(const cbm_gbuf_t *gbuf, int64_t node_id, cbm_sem_vec_t cbm_sem_random_index(names[i], &callee_ri); cbm_sem_vec_add_scaled(out, &callee_ri, PSE_UNIT_POS); } - free((void *)names); + cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)names); cbm_sem_normalize(out); } @@ -533,7 +536,7 @@ static void build_type_vec(const char *props_json, cbm_sem_vec_t *out) { cbm_sem_vec_t ri; cbm_sem_random_index(ptypes[i], &ri); cbm_sem_vec_add_scaled(out, &ri, PSE_UNIT_POS); - free(ptypes[i]); + cbm_free(CBM_MEM_CLASS_SEMANTIC, ptypes[i]); } cbm_sem_normalize(out); } @@ -549,7 +552,7 @@ static void build_deco_vec(const char *props_json, cbm_sem_vec_t *out) { cbm_sem_vec_t ri; cbm_sem_random_index(decos[i], &ri); cbm_sem_vec_add_scaled(out, &ri, PSE_UNIT_POS); - free(decos[i]); + cbm_free(CBM_MEM_CLASS_SEMANTIC, decos[i]); } cbm_sem_normalize(out); } @@ -594,10 +597,41 @@ static void decode_minhash(const char *props_json, cbm_sem_func_t *func) { /* ── Parallel Phase 2: Tokenize nodes ────────────────────────────── */ +/* One growable token-pointer buffer per worker. A function's tokens are + * appended contiguously; where they landed is recorded per function so a + * single pass after the parallel phase can pack everything into one array + * with offsets. This replaces a fixed CBM_SEM_MAX_TOKENS-slot stride per + * function (4 KB each, 7.4 GB on the kernel). */ +typedef struct { + char **items; + size_t count; + size_t cap; +} tok_buf_t; + +static bool tok_buf_append(tok_buf_t *b, char **tokens, int count) { + if (b->count + (size_t)count > b->cap) { + size_t new_cap = b->cap ? b->cap * 2 : (size_t)CBM_SZ_4K; + while (new_cap < b->count + (size_t)count) { + new_cap *= 2; + } + char **grown = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, b->items, new_cap * sizeof(char *)); + if (!grown) { + return false; + } + b->items = grown; + b->cap = new_cap; + } + memcpy(b->items + b->count, tokens, (size_t)count * sizeof(char *)); + b->count += (size_t)count; + return true; +} + typedef struct { const cbm_gbuf_node_t **node_ptrs; /* node pointer per function index */ cbm_gbuf_t *gbuf; /* read-only during tokenization */ - char **all_tokens; /* output: all_tokens[f * MAX + t] */ + tok_buf_t *bufs; /* per worker: appended token pointers */ + int *tok_worker; /* per function: which worker's buffer */ + size_t *tok_off; /* per function: offset inside that buffer */ int *token_counts; /* output: token count per function */ int func_count; _Atomic int next_idx; @@ -617,11 +651,10 @@ static void tokenize_worker(int worker_id, void *ctx_ptr) { } const cbm_gbuf_node_t *n = tc->node_ptrs[f]; - /* Write directly into the shared buffer slice for this function — the - * strdup'd tokens are owned by all_tokens[] from the moment they land - * in this slot, which avoids a spurious analyzer "leak" diagnostic on - * the previous stack-local relay pattern. */ - char **dst = &tc->all_tokens[(ptrdiff_t)f * CBM_SEM_MAX_TOKENS]; + /* Tokenize into a per-call scratch slice, intern, then append the + * surviving pointers to this worker's buffer. The strings are owned by + * the worker's intern pool (or by the buffer until interned). */ + char *dst[CBM_SEM_MAX_TOKENS]; int count = tokenize_node(n, tc->gbuf, dst, CBM_SEM_MAX_TOKENS); count = inject_pattern_tokens(n, tc->gbuf, dst, count, CBM_SEM_MAX_TOKENS); if (tc->pools && tc->pools[worker_id]) { @@ -629,13 +662,19 @@ static void tokenize_worker(int worker_id, void *ctx_ptr) { for (int t = 0; t < count; t++) { char *canon = cbm_ht_get(pool, dst[t]); if (canon) { - free(dst[t]); + cbm_free(CBM_MEM_CLASS_SEMANTIC, dst[t]); dst[t] = canon; } else { cbm_ht_set(pool, dst[t], dst[t]); /* key borrows the value */ } } } + tok_buf_t *b = &tc->bufs[worker_id]; + tc->tok_worker[f] = worker_id; + tc->tok_off[f] = b->count; + if (!tok_buf_append(b, dst, count)) { + count = 0; /* OOM: the function contributes no tokens, never garbage */ + } tc->token_counts[f] = count; } } @@ -645,6 +684,7 @@ static void tokenize_worker(int worker_id, void *ctx_ptr) { typedef struct { cbm_sem_func_t *funcs; char **all_tokens; + const size_t *offsets; /* function f = all_tokens[offsets[f] ..] */ int *token_counts; cbm_sem_corpus_t *corpus; uint8_t *qvecs; /* output: pre-quantized int8 vectors [func_count * CBM_SEM_DIM] */ @@ -662,11 +702,11 @@ static void vec_build_worker(int worker_id, void *ctx_ptr) { } int tc = vc->token_counts[f]; - char **tokens = &vc->all_tokens[(ptrdiff_t)f * CBM_SEM_MAX_TOKENS]; + char **tokens = &vc->all_tokens[vc->offsets[f]]; /* TF-IDF weights */ - int *indices = malloc((size_t)tc * sizeof(int)); - float *weights = malloc((size_t)tc * sizeof(float)); + int *indices = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)tc * sizeof(int)); + float *weights = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)tc * sizeof(float)); int tfidf_len = 0; for (int t = 0; t < tc; t++) { float idf = cbm_sem_corpus_idf(vc->corpus, tokens[t]); @@ -964,13 +1004,14 @@ static int phase1_scan_functions(cbm_gbuf_t *gbuf, cbm_sem_func_t **out_funcs, for (int i = 0; i < node_count; i++) { if (func_count >= func_cap) { int new_cap = func_cap < MAX_FUNCS_INIT ? MAX_FUNCS_INIT : func_cap * GROW; - cbm_sem_func_t *grown = realloc(funcs, (size_t)new_cap * sizeof(cbm_sem_func_t)); + cbm_sem_func_t *grown = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, funcs, + (size_t)new_cap * sizeof(cbm_sem_func_t)); if (!grown) { break; } funcs = grown; - const cbm_gbuf_node_t **np_grown = - realloc(node_ptrs, (size_t)new_cap * sizeof(cbm_gbuf_node_t *)); + const cbm_gbuf_node_t **np_grown = cbm_realloc( + CBM_MEM_CLASS_SEMANTIC, node_ptrs, (size_t)new_cap * sizeof(cbm_gbuf_node_t *)); if (!np_grown) { break; } @@ -1023,7 +1064,8 @@ static void phase5c_build_lsh_buckets(const uint64_t *signatures, int func_count if (bucket->count >= bucket->cap) { int nc = bucket->cap < SEM_BUCKET_CAP_INIT ? SEM_BUCKET_CAP_INIT : bucket->cap * GROW; - int *ni = realloc(bucket->items, (size_t)nc * sizeof(int)); + int *ni = + cbm_realloc(CBM_MEM_CLASS_SEMANTIC, bucket->items, (size_t)nc * sizeof(int)); if (!ni) { continue; } @@ -1065,7 +1107,7 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_buf } deferred_edge_t *pairs = NULL; if (total_pairs > 0) { - pairs = malloc((size_t)total_pairs * sizeof(deferred_edge_t)); + pairs = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)total_pairs * sizeof(deferred_edge_t)); } if (!pairs) { for (int w = 0; w < worker_count; w++) { @@ -1096,7 +1138,7 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_buf edge_counts[de->j]++; total_edges++; } - free(pairs); + cbm_free(CBM_MEM_CLASS_SEMANTIC, pairs); return total_edges; } @@ -1130,7 +1172,8 @@ static void phase3c_export_token_vectors(cbm_gbuf_t *gbuf, cbm_sem_corpus_t *cor /* Phase 5a: generate NUM_HYPERPLANES × CBM_SEM_DIM deterministic random * float hyperplanes seeded from XXH3 so signatures are reproducible. */ static hyperplane_row_t *phase5a_build_hyperplanes(void) { - hyperplane_row_t *hyperplanes = malloc(sizeof(hyperplane_row_t) * NUM_HYPERPLANES); + hyperplane_row_t *hyperplanes = + cbm_alloc(CBM_MEM_CLASS_SEMANTIC, sizeof(hyperplane_row_t) * NUM_HYPERPLANES); if (!hyperplanes) { return NULL; } @@ -1164,13 +1207,31 @@ static void phase1b_decode_and_build(cbm_sem_func_t *funcs, const cbm_gbuf_node_ /* Phase 2: tokenize each function's metadata in parallel, filling * all_tokens[] and token_counts[]. Caller allocates the arrays. */ -static void phase2_tokenize(const cbm_gbuf_node_t **node_ptrs, cbm_gbuf_t *gbuf, char **all_tokens, - int *token_counts, int func_count, int worker_count, - CBMHashTable **pools) { +static void phase2_tokenize(const cbm_gbuf_node_t **node_ptrs, cbm_gbuf_t *gbuf, char ***out_tokens, + size_t **out_offsets, int *token_counts, int func_count, + int worker_count, CBMHashTable **pools) { + *out_tokens = NULL; + *out_offsets = NULL; + tok_buf_t *bufs = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)worker_count * sizeof(tok_buf_t)); + int *tok_worker = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)func_count * sizeof(int)); + size_t *tok_off = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)func_count * sizeof(size_t)); + size_t *offsets = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)func_count * sizeof(size_t)); + if (!bufs || !tok_worker || !tok_off || !offsets) { + cbm_free(CBM_MEM_CLASS_SEMANTIC, bufs); + cbm_free(CBM_MEM_CLASS_SEMANTIC, tok_worker); + cbm_free(CBM_MEM_CLASS_SEMANTIC, tok_off); + cbm_free(CBM_MEM_CLASS_SEMANTIC, offsets); + for (int f = 0; f < func_count; f++) { + token_counts[f] = 0; + } + return; + } tokenize_ctx_t tc = { .node_ptrs = node_ptrs, .gbuf = gbuf, - .all_tokens = all_tokens, + .bufs = bufs, + .tok_worker = tok_worker, + .tok_off = tok_off, .token_counts = token_counts, .func_count = func_count, .pools = pools, @@ -1178,22 +1239,60 @@ static void phase2_tokenize(const cbm_gbuf_node_t **node_ptrs, cbm_gbuf_t *gbuf, atomic_init(&tc.next_idx, 0); cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; cbm_parallel_for(worker_count, tokenize_worker, &tc, opts); + + /* Pack: one array, functions addressed by offset. Worker w's buffer lands + * at base[w]; a function's tokens sit at base[worker] + its offset. */ + size_t total = 0; + for (int w = 0; w < worker_count; w++) { + total += bufs[w].count; + } + char **packed = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (total ? total : 1) * sizeof(char *)); + if (packed) { + size_t base = 0; + for (int w = 0; w < worker_count; w++) { + if (bufs[w].count) { + memcpy(packed + base, bufs[w].items, bufs[w].count * sizeof(*bufs[w].items)); + } + /* Copied: give the worker buffer back now, so the packing transient + * is the packed array plus ONE worker buffer, not plus all of them. */ + cbm_free(CBM_MEM_CLASS_SEMANTIC, bufs[w].items); + bufs[w].items = NULL; + bufs[w].cap = base; /* reuse: base offset of this worker's tokens */ + base += bufs[w].count; + } + for (int f = 0; f < func_count; f++) { + offsets[f] = bufs[tok_worker[f]].cap + tok_off[f]; + } + } else { + for (int f = 0; f < func_count; f++) { + token_counts[f] = 0; + } + } + for (int w = 0; w < worker_count; w++) { + cbm_free(CBM_MEM_CLASS_SEMANTIC, bufs[w].items); + } + cbm_free(CBM_MEM_CLASS_SEMANTIC, bufs); + cbm_free(CBM_MEM_CLASS_SEMANTIC, tok_worker); + cbm_free(CBM_MEM_CLASS_SEMANTIC, tok_off); + *out_tokens = packed; + *out_offsets = offsets; } /* Phase 4a: build per-function TF-IDF + RI vectors in parallel, producing * int8-quantized qvecs for subsequent storage. Phase 4b runs sequentially * to store them in gbuf because gbuf is not thread-safe. */ static void phase4_build_and_store_vectors(cbm_gbuf_t *gbuf, cbm_sem_func_t *funcs, - char **all_tokens, int *token_counts, - cbm_sem_corpus_t *corpus, int func_count, - int worker_count) { - uint8_t *qvecs = malloc((size_t)func_count * CBM_SEM_DIM); + char **all_tokens, const size_t *offsets, + int *token_counts, cbm_sem_corpus_t *corpus, + int func_count, int worker_count) { + uint8_t *qvecs = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)func_count * CBM_SEM_DIM); if (!qvecs) { return; } vec_build_ctx_t vc = { .funcs = funcs, .all_tokens = all_tokens, + .offsets = offsets, .token_counts = token_counts, .corpus = corpus, .qvecs = qvecs, @@ -1206,7 +1305,7 @@ static void phase4_build_and_store_vectors(cbm_gbuf_t *gbuf, cbm_sem_func_t *fun cbm_gbuf_store_vector(gbuf, funcs[f].node_id, &qvecs[(ptrdiff_t)f * CBM_SEM_DIM], CBM_SEM_DIM); } - free(qvecs); + cbm_free(CBM_MEM_CLASS_SEMANTIC, qvecs); } /* Phase 5: hyperplane generation → signatures → LSH bucket population. @@ -1215,7 +1314,8 @@ static void phase4_build_and_store_vectors(cbm_gbuf_t *gbuf, cbm_sem_func_t *fun static void phase5_lsh_build(cbm_sem_func_t *funcs, int func_count, int worker_count, uint64_t **out_signatures, sem_bucket_t ***out_buckets) { hyperplane_row_t *hyperplanes = phase5a_build_hyperplanes(); - uint64_t *signatures = calloc((size_t)func_count, sizeof(uint64_t)); + uint64_t *signatures = + cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)((size_t)func_count) * (sizeof(uint64_t))); if (hyperplanes && signatures) { sig_build_ctx_t sc = { .funcs = funcs, @@ -1227,12 +1327,14 @@ static void phase5_lsh_build(cbm_sem_func_t *funcs, int func_count, int worker_c cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; cbm_parallel_for(worker_count, sig_build_worker, &sc, opts); } - free(hyperplanes); + cbm_free(CBM_MEM_CLASS_SEMANTIC, hyperplanes); - sem_bucket_t **band_buckets = calloc(SEM_LSH_BANDS, sizeof(sem_bucket_t *)); + sem_bucket_t **band_buckets = + cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)(SEM_LSH_BANDS) * (sizeof(sem_bucket_t *))); if (band_buckets) { for (int b = 0; b < SEM_LSH_BANDS; b++) { - band_buckets[b] = calloc(SEM_BUCKET_COUNT, sizeof(sem_bucket_t)); + band_buckets[b] = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, + (size_t)(SEM_BUCKET_COUNT) * (sizeof(sem_bucket_t))); } phase5c_build_lsh_buckets(signatures, func_count, band_buckets); } @@ -1270,21 +1372,22 @@ static void free_lsh_buckets(sem_bucket_t **band_buckets) { continue; } for (int h = 0; h < SEM_BUCKET_COUNT; h++) { - free(band_buckets[b][h].items); + cbm_free(CBM_MEM_CLASS_SEMANTIC, band_buckets[b][h].items); } - free(band_buckets[b]); + cbm_free(CBM_MEM_CLASS_SEMANTIC, band_buckets[b]); } - free(band_buckets); + cbm_free(CBM_MEM_CLASS_SEMANTIC, band_buckets); } /* Phases 3a/3b/3c bundled: create corpus, batch-add docs, finalize, export * enriched token vectors to the graph buffer. Returns the new corpus, which * the caller must cbm_sem_corpus_free() later. */ -static cbm_sem_corpus_t *run_corpus_phase(cbm_gbuf_t *gbuf, char **all_tokens, int *token_counts, +static cbm_sem_corpus_t *run_corpus_phase(cbm_gbuf_t *gbuf, char **all_tokens, + const size_t *offsets, int *token_counts, int func_count) { CBM_PROF_START(t_phase3a); cbm_sem_corpus_t *corpus = cbm_sem_corpus_new(); - cbm_sem_corpus_add_docs_batch(corpus, all_tokens, token_counts, func_count, CBM_SEM_MAX_TOKENS); + cbm_sem_corpus_add_docs_batch(corpus, all_tokens, offsets, token_counts, func_count); CBM_PROF_END_N("semantic_edges", "3a_corpus_batch", t_phase3a, func_count); CBM_PROF_START(t_phase3b); @@ -1304,11 +1407,13 @@ static cbm_sem_corpus_t *run_corpus_phase(cbm_gbuf_t *gbuf, char **all_tokens, i static int run_scoring_phase(cbm_gbuf_t *gbuf, cbm_sem_func_t *funcs, uint64_t *signatures, sem_bucket_t **band_buckets, cbm_sem_config_t cfg, int func_count, int worker_count) { - int *edge_counts = calloc((size_t)func_count, sizeof(int)); - deferred_edge_buf_t *worker_bufs = calloc((size_t)worker_count, sizeof(deferred_edge_buf_t)); + int *edge_counts = + cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)((size_t)func_count) * (sizeof(int))); + deferred_edge_buf_t *worker_bufs = cbm_calloc( + CBM_MEM_CLASS_SEMANTIC, (size_t)((size_t)worker_count) * (sizeof(deferred_edge_buf_t))); if (!edge_counts || !worker_bufs) { - free(edge_counts); - free(worker_bufs); + cbm_free(CBM_MEM_CLASS_SEMANTIC, edge_counts); + cbm_free(CBM_MEM_CLASS_SEMANTIC, worker_bufs); return 0; } for (int w = 0; w < worker_count; w++) { @@ -1324,8 +1429,8 @@ static int run_scoring_phase(cbm_gbuf_t *gbuf, cbm_sem_func_t *funcs, uint64_t * int total = phase6b_merge_edges(gbuf, worker_bufs, worker_count, edge_counts, cfg.max_edges); CBM_PROF_END_N("semantic_edges", "6b_edge_merge_seq", t_phase6b, total); - free(worker_bufs); - free(edge_counts); + cbm_free(CBM_MEM_CLASS_SEMANTIC, worker_bufs); + cbm_free(CBM_MEM_CLASS_SEMANTIC, edge_counts); return total; } @@ -1333,20 +1438,22 @@ static int run_scoring_phase(cbm_gbuf_t *gbuf, cbm_sem_func_t *funcs, uint64_t * static void free_token_pool_entry(const char *key, void *value, void *ud) { (void)key; /* key == value: one owned string per unique token */ (void)ud; - free(value); + cbm_free(CBM_MEM_CLASS_SEMANTIC, value); } /* all_tokens slots BORROW their strings from the per-worker intern pools; * the pools own exactly one copy per unique token per worker. */ static void free_funcs_and_tokens(cbm_sem_func_t *funcs, int func_count, char **all_tokens, - const int *token_counts, CBMHashTable **pools, int worker_count) { + size_t *offsets, const int *token_counts, CBMHashTable **pools, + int worker_count) { (void)token_counts; + cbm_free(CBM_MEM_CLASS_SEMANTIC, offsets); for (int f = 0; f < func_count; f++) { - free(funcs[f].tfidf_indices); - free(funcs[f].tfidf_weights); + cbm_free(CBM_MEM_CLASS_SEMANTIC, funcs[f].tfidf_indices); + cbm_free(CBM_MEM_CLASS_SEMANTIC, funcs[f].tfidf_weights); } - free(all_tokens); - free(funcs); + cbm_free(CBM_MEM_CLASS_SEMANTIC, all_tokens); + cbm_free(CBM_MEM_CLASS_SEMANTIC, funcs); if (pools) { for (int w = 0; w < worker_count; w++) { if (pools[w]) { @@ -1354,10 +1461,153 @@ static void free_funcs_and_tokens(cbm_sem_func_t *funcs, int func_count, char ** cbm_ht_free(pools[w]); } } - free(pools); + cbm_free(CBM_MEM_CLASS_SEMANTIC, pools); } } +/* ── Memory checkpoints inside the pass (CBM_MEM_PHASES=1) ─────────── */ + +/* The pass is one phase mark to the pipeline; its transient lives between + * marks. These lines put the semantic class's live/peak bytes and the charged + * footprint at every sub-phase boundary, so the peak is attributable. */ +static void sem_mem_mark(const char *step) { + if (!cbm_mem_phases_enabled()) { + return; + } + enum { MB = 1024 * 1024 }; + char live[CBM_SZ_16]; + char peak[CBM_SZ_16]; + char charged[CBM_SZ_16]; + snprintf(live, sizeof(live), "%zu", cbm_mem_class_live_bytes(CBM_MEM_CLASS_SEMANTIC) / MB); + snprintf(peak, sizeof(peak), "%zu", cbm_mem_class_peak_bytes(CBM_MEM_CLASS_SEMANTIC) / MB); + snprintf(charged, sizeof(charged), "%zu", cbm_mem_charged() / MB); + cbm_log_info("mem.semantic.step", "step", step, "sem_live_mb", live, "sem_peak_mb", peak, + "charged_mb", charged); +} + +/* ── Headroom-sized batches ───────────────────────────────────────── */ + +/* Phases 2-4 hold, per function, its token pointers, the corpus's per-doc + * token ids and the quantized vector being stored: ~3 KB per function on the + * kernel (783k functions -> the 4.9 GB transient the 15 GB budget run + * overshot with, 2026-09-13). The retained part -- the funcs array and the + * corpus entries -- is what scoring needs whole and stays. When the charged + * footprint plus that transient would cross the budget, the token phases run + * per batch of functions: tokenize -> count into the corpus -> free, then, + * after finalize, tokenize again -> vectorize -> store -> free. Tokenization + * is deterministic, so both passes see the same tokens and the graph does + * not change; the run pays with the second tokenize. */ +enum { + SEM_BATCH_BYTES_PER_FUNC = 4096, /* measured 3 KB, rounded up */ + SEM_BATCH_MIN_FUNCS = 4096, +}; + +static int sem_batch_size(int func_count) { + /* CBM_SEM_BATCH= forces the batch size regardless of budget (the + * batched-equals-unbatched test, small-machine iteration). */ + char forced_buf[CBM_SZ_16]; + if (cbm_safe_getenv("CBM_SEM_BATCH", forced_buf, sizeof(forced_buf), NULL)) { + int forced = atoi(forced_buf); + if (forced > 0 && forced < func_count) { + cbm_log_info("mem.semantic.batches", "functions", itoa_log(func_count), "batch", + itoa_log(forced), "reason", "env"); + return forced; + } + } + size_t budget = cbm_mem_budget(); + if (budget == 0 || func_count <= SEM_BATCH_MIN_FUNCS) { + return func_count; + } + size_t charged = cbm_mem_charged(); + size_t headroom = budget > charged ? budget - charged : 0; + size_t transient = (size_t)func_count * SEM_BATCH_BYTES_PER_FUNC; + /* Half the headroom is the transient's share; the other half is what the + * later phases (signatures, buckets, deferred edges) and mimalloc keep. */ + size_t share = headroom / 2; + if (transient <= share) { + return func_count; + } + size_t batch = share / SEM_BATCH_BYTES_PER_FUNC; + if (batch < SEM_BATCH_MIN_FUNCS) { + batch = SEM_BATCH_MIN_FUNCS; + } + if (batch > (size_t)func_count) { + batch = (size_t)func_count; + } + enum { MB = 1024 * 1024 }; + cbm_log_info("mem.semantic.batches", "functions", itoa_log(func_count), "batch", + itoa_log((int)batch), "headroom_mb", itoa_log((int)(headroom / MB)), + "transient_mb", itoa_log((int)(transient / MB))); + return (int)batch; +} + +/* One batch of the token phases: tokenize functions [first, first+count) into + * a fresh packed array the caller frees with sem_batch_free_tokens(). */ +static void sem_batch_tokenize(const cbm_gbuf_node_t **node_ptrs, cbm_gbuf_t *gbuf, int first, + int count, int worker_count, CBMHashTable **pools, int *token_counts, + char ***out_tokens, size_t **out_offsets) { + phase2_tokenize(node_ptrs + first, gbuf, out_tokens, out_offsets, token_counts + first, count, + worker_count, pools); +} + +static void sem_batch_free_tokens(char **tokens, size_t *offsets) { + cbm_free(CBM_MEM_CLASS_SEMANTIC, tokens); + cbm_free(CBM_MEM_CLASS_SEMANTIC, offsets); +} + +/* Phases 2-4 in headroom-sized batches. The corpus is fed batch by batch + * (its per-doc token ids are ints, kept for co-occurrence), finalized once, + * then vectors are built batch by batch from a second tokenization. */ +static cbm_sem_corpus_t *run_token_phases_batched(cbm_gbuf_t *gbuf, cbm_sem_func_t *funcs, + const cbm_gbuf_node_t **node_ptrs, + int *token_counts, int func_count, int batch, + int worker_count, CBMHashTable **token_pools) { + CBM_PROF_START(t_phase2); + cbm_sem_corpus_t *corpus = cbm_sem_corpus_new(); + for (int first = 0; first < func_count; first += batch) { + int count = func_count - first < batch ? func_count - first : batch; + char **tokens = NULL; + size_t *offsets = NULL; + sem_batch_tokenize(node_ptrs, gbuf, first, count, worker_count, token_pools, token_counts, + &tokens, &offsets); + if (tokens && offsets) { + cbm_sem_corpus_add_docs_batch(corpus, tokens, offsets, token_counts + first, count); + } + sem_batch_free_tokens(tokens, offsets); + cbm_mem_release_to_os(); /* the batch's pages leave the charge now, not at the mark */ + } + CBM_PROF_END_N("semantic_edges", "2+3a_tokenize_count_batched", t_phase2, func_count); + sem_mem_mark("3a_corpus_batched"); + + CBM_PROF_START(t_phase3b); + cbm_sem_corpus_finalize(corpus); + CBM_PROF_END_N("semantic_edges", "3b_corpus_finalize_seq", t_phase3b, + cbm_sem_corpus_token_count(corpus)); + CBM_PROF_START(t_phase3c); + phase3c_export_token_vectors(gbuf, corpus); + CBM_PROF_END_N("semantic_edges", "3c_token_vec_export_seq", t_phase3c, + cbm_sem_corpus_token_count(corpus)); + sem_mem_mark("3c_export"); + + CBM_PROF_START(t_phase4); + for (int first = 0; first < func_count; first += batch) { + int count = func_count - first < batch ? func_count - first : batch; + char **tokens = NULL; + size_t *offsets = NULL; + sem_batch_tokenize(node_ptrs, gbuf, first, count, worker_count, token_pools, token_counts, + &tokens, &offsets); + if (tokens && offsets) { + phase4_build_and_store_vectors(gbuf, funcs + first, tokens, offsets, + token_counts + first, corpus, count, worker_count); + } + sem_batch_free_tokens(tokens, offsets); + cbm_mem_release_to_os(); + } + CBM_PROF_END_N("semantic_edges", "2+4_tokenize_vectorize_batched", t_phase4, func_count); + sem_mem_mark("4_vectors"); + return corpus; +} + /* ── Pass entry point ────────────────────────────────────────────── */ int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx) { @@ -1381,37 +1631,53 @@ int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx) { cbm_log_info("pass.semantic.collected", "functions", itoa_log(func_count)); if (func_count < PSE_MIN_FUNCS_FOR_PAIR) { - free(funcs); - free(node_ptrs); + cbm_free(CBM_MEM_CLASS_SEMANTIC, funcs); + cbm_free(CBM_MEM_CLASS_SEMANTIC, node_ptrs); cbm_log_info("pass.done", "pass", "semantic_edges", "edges", "0"); return 0; } + sem_mem_mark("1b_decode_build"); + /* Phase 2: Tokenize all nodes (PARALLEL) */ int worker_count = cbm_default_worker_count(false); - char **all_tokens = malloc((size_t)func_count * sizeof(char *) * CBM_SEM_MAX_TOKENS); - int *token_counts = calloc((size_t)func_count, sizeof(int)); + char **all_tokens = NULL; /* packed by phase2_tokenize */ + size_t *tok_offsets = NULL; /* per function: index into all_tokens */ + int *token_counts = + cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)((size_t)func_count) * (sizeof(int))); - CBM_PROF_START(t_phase2); - CBMHashTable **token_pools = calloc((size_t)worker_count, sizeof(CBMHashTable *)); + CBMHashTable **token_pools = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)((size_t)worker_count) * + (sizeof(CBMHashTable *))); if (token_pools) { for (int w = 0; w < worker_count; w++) { token_pools[w] = cbm_ht_create(CBM_SZ_1K); } } - phase2_tokenize(node_ptrs, gbuf, all_tokens, token_counts, func_count, worker_count, - token_pools); - CBM_PROF_END_N("semantic_edges", "2_tokenize_parallel", t_phase2, func_count); - free(node_ptrs); + cbm_sem_corpus_t *corpus = NULL; + int batch = sem_batch_size(func_count); + if (batch < func_count) { + corpus = run_token_phases_batched(gbuf, funcs, node_ptrs, token_counts, func_count, batch, + worker_count, token_pools); + cbm_free(CBM_MEM_CLASS_SEMANTIC, node_ptrs); + } else { + CBM_PROF_START(t_phase2); + phase2_tokenize(node_ptrs, gbuf, &all_tokens, &tok_offsets, token_counts, func_count, + worker_count, token_pools); + CBM_PROF_END_N("semantic_edges", "2_tokenize_parallel", t_phase2, func_count); + cbm_free(CBM_MEM_CLASS_SEMANTIC, node_ptrs); + sem_mem_mark("2_tokenize"); - /* Phase 3: Build corpus (batch add), finalize, export enriched token vectors. */ - cbm_sem_corpus_t *corpus = run_corpus_phase(gbuf, all_tokens, token_counts, func_count); + /* Phase 3: Build corpus (batch add), finalize, export enriched token vectors. */ + corpus = run_corpus_phase(gbuf, all_tokens, tok_offsets, token_counts, func_count); + sem_mem_mark("3_corpus"); - /* Phase 4: Build per-function TF-IDF + RI vectors (PARALLEL) and store them. */ - CBM_PROF_START(t_phase4); - phase4_build_and_store_vectors(gbuf, funcs, all_tokens, token_counts, corpus, func_count, - worker_count); - CBM_PROF_END_N("semantic_edges", "4_build_and_store_vec", t_phase4, func_count); + /* Phase 4: Build per-function TF-IDF + RI vectors (PARALLEL) and store them. */ + CBM_PROF_START(t_phase4); + phase4_build_and_store_vectors(gbuf, funcs, all_tokens, tok_offsets, token_counts, corpus, + func_count, worker_count); + CBM_PROF_END_N("semantic_edges", "4_build_and_store_vec", t_phase4, func_count); + sem_mem_mark("4_vectors"); + } cbm_log_info("pass.semantic.vectors_stored", "count", itoa_log(func_count)); @@ -1424,18 +1690,22 @@ int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx) { cbm_log_info("pass.semantic.lsh_built", "functions", itoa_log(func_count), "bands", itoa_log(SEM_LSH_BANDS)); + sem_mem_mark("5_lsh"); /* Phase 6: Parallel scoring + sequential edge merge. */ int total_edges = run_scoring_phase(gbuf, funcs, signatures, band_buckets, cfg, func_count, worker_count); + sem_mem_mark("6_score"); + /* Phase 7: Cleanup */ CBM_PROF_START(t_phase7); free_lsh_buckets(band_buckets); - free(signatures); + cbm_free(CBM_MEM_CLASS_SEMANTIC, signatures); cbm_log_info("pass.done", "pass", "semantic_edges", "edges", itoa_log(total_edges)); - free_funcs_and_tokens(funcs, func_count, all_tokens, token_counts, token_pools, worker_count); - free(token_counts); + free_funcs_and_tokens(funcs, func_count, all_tokens, tok_offsets, token_counts, token_pools, + worker_count); + cbm_free(CBM_MEM_CLASS_SEMANTIC, token_counts); cbm_sem_corpus_free(corpus); CBM_PROF_END("semantic_edges", "7_cleanup", t_phase7); diff --git a/src/pipeline/pass_similarity.c b/src/pipeline/pass_similarity.c index db4297202..8b456e100 100644 --- a/src/pipeline/pass_similarity.c +++ b/src/pipeline/pass_similarity.c @@ -8,6 +8,7 @@ * Runs as a post-pass after enrichment (both full and incremental). */ #include "foundation/constants.h" +#include "foundation/mem_core.h" #include "pipeline/pipeline.h" #include #include "pipeline/pipeline_internal.h" @@ -132,7 +133,8 @@ static int collect_fp_entries(cbm_gbuf_t *gbuf, fp_entry_t **out_entries) { } if (count >= cap) { int new_cap = cap < FP_ENTRY_INIT_CAP ? FP_ENTRY_INIT_CAP : cap * FP_ENTRY_GROW; - fp_entry_t *grown = realloc(entries, (size_t)new_cap * sizeof(fp_entry_t)); + fp_entry_t *grown = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, entries, + (size_t)new_cap * sizeof(fp_entry_t)); if (!grown) { break; } @@ -178,7 +180,8 @@ static void sim_edge_buf_push(sim_edge_buf_t *buf, int64_t src, int64_t tgt, dou bool same_file) { if (buf->count >= buf->cap) { int nc = buf->cap < SIM_EDGE_INIT_CAP ? SIM_EDGE_INIT_CAP : buf->cap * SIM_EDGE_GROW; - sim_deferred_edge_t *grown = realloc(buf->edges, (size_t)nc * sizeof(sim_deferred_edge_t)); + sim_deferred_edge_t *grown = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, buf->edges, + (size_t)nc * sizeof(sim_deferred_edge_t)); if (!grown) { return; } @@ -271,9 +274,9 @@ static int merge_sim_edges(cbm_gbuf_t *gbuf, sim_edge_buf_t *worker_bufs, int wo cbm_gbuf_insert_edge(gbuf, de->source_id, de->target_id, "SIMILAR_TO", props); total++; } - free(worker_bufs[w].edges); + cbm_free(CBM_MEM_CLASS_SEMANTIC, worker_bufs[w].edges); } - free(worker_bufs); + cbm_free(CBM_MEM_CLASS_SEMANTIC, worker_bufs); return total; } @@ -293,7 +296,7 @@ int cbm_pipeline_pass_similarity(cbm_pipeline_ctx_t *ctx) { cbm_log_info("pass.similarity.collected", "nodes_with_fp", itoa_log(entry_count)); if (entry_count < MIN_FP_ENTRIES) { - free(entries); + cbm_free(CBM_MEM_CLASS_SEMANTIC, entries); cbm_log_info("pass.done", "pass", "similarity", "edges", "0"); return 0; } @@ -301,9 +304,10 @@ int cbm_pipeline_pass_similarity(cbm_pipeline_ctx_t *ctx) { /* Phase 2: Build LSH index (sequential — cbm_lsh_insert mutates shared state) */ CBM_PROF_START(t_lsh_build); cbm_lsh_index_t *lsh = cbm_lsh_new(); - cbm_lsh_entry_t *lsh_entries = malloc((size_t)entry_count * sizeof(cbm_lsh_entry_t)); + cbm_lsh_entry_t *lsh_entries = + cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)entry_count * sizeof(cbm_lsh_entry_t)); if (!lsh_entries) { - free(entries); + cbm_free(CBM_MEM_CLASS_SEMANTIC, entries); cbm_lsh_free(lsh); return CBM_NOT_FOUND; } @@ -325,9 +329,11 @@ int cbm_pipeline_pass_similarity(cbm_pipeline_ctx_t *ctx) { * in its own deferred buffer. Shared edge_counts is atomic. * Final merge into gbuf is sequential (gbuf not thread-safe). */ CBM_PROF_START(t_query_emit); - _Atomic int *edge_counts = calloc((size_t)entry_count, sizeof(_Atomic int)); + _Atomic int *edge_counts = + cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)((size_t)entry_count) * (sizeof(_Atomic int))); int worker_count = cbm_default_worker_count(false); - sim_edge_buf_t *worker_bufs = calloc((size_t)worker_count, sizeof(sim_edge_buf_t)); + sim_edge_buf_t *worker_bufs = cbm_calloc( + CBM_MEM_CLASS_SEMANTIC, (size_t)((size_t)worker_count) * (sizeof(sim_edge_buf_t))); { sim_query_ctx_t sc = { @@ -349,9 +355,9 @@ int cbm_pipeline_pass_similarity(cbm_pipeline_ctx_t *ctx) { cbm_log_info("pass.done", "pass", "similarity", "edges", itoa_log(total_edges)); - free(edge_counts); - free(lsh_entries); - free(entries); + cbm_free(CBM_MEM_CLASS_SEMANTIC, edge_counts); + cbm_free(CBM_MEM_CLASS_SEMANTIC, lsh_entries); + cbm_free(CBM_MEM_CLASS_SEMANTIC, entries); cbm_lsh_free(lsh); return 0; } diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index d87e5f90f..1d9b72eac 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -10,6 +10,8 @@ * 6. Post-passes: tests, communities, HTTP links, git history * 7. Dump graph buffer to SQLite */ +#include "foundation/arena.h" // FIRST: internal/cbm/arena.h shares the CBM_ARENA_H guard and lacks cbm_arena_total + #include "foundation/constants.h" enum { CBM_DIR_PERMS = 0755, PL_RING = 4, PL_RING_MASK = 3, PL_SEQ_PASSES = 6 }; @@ -38,6 +40,7 @@ enum { CBM_DIR_PERMS = 0755, PL_RING = 4, PL_RING_MASK = 3, PL_SEQ_PASSES = 6 }; #include "foundation/profile.h" #include "foundation/mem.h" #include "foundation/mem_core.h" +#include "result_spill.h" #include "foundation/secure_random.h" #include @@ -294,9 +297,26 @@ static const char *itoa_buf(int val) { /* Log current + peak RSS at a pipeline phase boundary (memory profiling). */ static void log_phase_mem(const char *phase) { enum { PL_BYTES_PER_MB = 1024 * 1024 }; - cbm_log_info("mem.phase", "phase", phase, "rss_mb", - itoa_buf((int)(cbm_mem_rss() / PL_BYTES_PER_MB)), "peak_mb", - itoa_buf((int)(cbm_mem_peak_rss() / PL_BYTES_PER_MB))); + /* tracked_mb is what the memory core can account for; rss_mb - tracked_mb + * is the part of the process no class explains yet. */ + /* itoa_buf is a 4-slot ring: two calls per line, never more. */ + char rss_mb[CBM_SZ_32]; + char footprint_mb[CBM_SZ_32]; + char commit_mb[CBM_SZ_32]; + char tracked_mb[CBM_SZ_32]; + char peak_mb[CBM_SZ_32]; + char peak_charged_mb[CBM_SZ_32]; + snprintf(rss_mb, sizeof(rss_mb), "%zu", cbm_mem_rss() / PL_BYTES_PER_MB); + snprintf(footprint_mb, sizeof(footprint_mb), "%zu", cbm_mem_footprint() / PL_BYTES_PER_MB); + snprintf(commit_mb, sizeof(commit_mb), "%zu", cbm_mem_allocator_committed() / PL_BYTES_PER_MB); + snprintf(tracked_mb, sizeof(tracked_mb), "%zu", cbm_mem_tracked_live_bytes() / PL_BYTES_PER_MB); + snprintf(peak_mb, sizeof(peak_mb), "%zu", cbm_mem_peak_rss() / PL_BYTES_PER_MB); + (void)cbm_mem_charged(); /* fold this mark into the high-water mark */ + snprintf(peak_charged_mb, sizeof(peak_charged_mb), "%zu", + cbm_mem_peak_charged() / PL_BYTES_PER_MB); + cbm_log_info("mem.phase", "phase", phase, "rss_mb", rss_mb, "footprint_mb", footprint_mb, + "commit_mb", commit_mb, "tracked_mb", tracked_mb, "peak_mb", peak_mb, + "peak_charged_mb", peak_charged_mb); } /* ── Lifecycle ──────────────────────────────────────────────────── */ @@ -804,19 +824,32 @@ static int process_one_infra_binding(cbm_gbuf_t *gbuf, const CBMInfraBinding *ib return SKIP_ONE; } -static void cbm_pipeline_process_infra_bindings(cbm_gbuf_t *gbuf, const cbm_file_info_t *files, +static bool want_infra_bindings(const CBMFileResult *header) { + return header->infra_bindings.count > 0; +} + +static bool want_string_refs(const CBMFileResult *header) { + return header->string_refs.count > 0; +} + +static void cbm_pipeline_process_infra_bindings(const cbm_pipeline_ctx_t *ctx, cbm_gbuf_t *gbuf, + const cbm_file_info_t *files, CBMFileResult **result_cache, int file_count) { int bindings = 0; for (int i = 0; i < file_count; i++) { - if (!result_cache[i]) { + bool loaded = false; + const CBMFileResult *r = + cbm_pipeline_result_acquire(ctx, result_cache, i, want_infra_bindings, &loaded); + if (!r) { continue; } - for (int bi = 0; bi < result_cache[i]->infra_bindings.count; bi++) { - const CBMInfraBinding *ib = &result_cache[i]->infra_bindings.items[bi]; + for (int bi = 0; bi < r->infra_bindings.count; bi++) { + const CBMInfraBinding *ib = &r->infra_bindings.items[bi]; if (ib->source_name && ib->target_url) { bindings += process_one_infra_binding(gbuf, ib, files[i].rel_path); } } + cbm_pipeline_result_release((CBMFileResult *)r, loaded); } if (bindings > 0) { char buf[CBM_SZ_16]; @@ -929,7 +962,8 @@ static bool route_sr_denied(const CBMStringRef *sr) { return is_upstream_config_key(sr->key_path); } -static void cbm_pipeline_extract_infra_routes(cbm_gbuf_t *gbuf, const cbm_file_info_t *files, +static void cbm_pipeline_extract_infra_routes(const cbm_pipeline_ctx_t *ctx, cbm_gbuf_t *gbuf, + const cbm_file_info_t *files, CBMFileResult **result_cache, int file_count) { /* DENY-WINS-BY-VALUE: the same URL is often extracted as several string_refs * at different key_path granularities (full path, leaf key, flat). The Route @@ -937,29 +971,45 @@ static void cbm_pipeline_extract_infra_routes(cbm_gbuf_t *gbuf, const cbm_file_i * per-ref guard — e.g. a denied full path `registries.terraform-registry.url` * is defeated by a sibling leaf `url`. So pass 1 collects every URL value * denied under ANY of its refs; pass 2 mints only values never denied. (#521) */ + /* The table borrows nothing: a key is copied into `denied_keys`, because + * the result that holds sr->value is released after its file (spill + * mode loads it only for that moment) while the table spans both + * passes. A borrowed key hashed freed memory and the insert spun. */ + CBMArena denied_keys; + cbm_arena_init(&denied_keys); CBMHashTable *denied = cbm_ht_create(16); for (int pass = 0; pass < 2; pass++) { for (int i = 0; i < file_count; i++) { - if (!result_cache[i] || !is_infra_file(files[i].rel_path) || - is_ci_tooling_config(files[i].rel_path)) { + if (!is_infra_file(files[i].rel_path) || is_ci_tooling_config(files[i].rel_path)) { continue; } - for (int si = 0; si < result_cache[i]->string_refs.count; si++) { - const CBMStringRef *sr = &result_cache[i]->string_refs.items[si]; + bool loaded = false; + const CBMFileResult *r = + cbm_pipeline_result_acquire(ctx, result_cache, i, want_string_refs, &loaded); + if (!r) { + continue; + } + for (int si = 0; si < r->string_refs.count; si++) { + const CBMStringRef *sr = &r->string_refs.items[si]; if (sr->kind != CBM_STRREF_URL || !sr->value || !strstr(sr->value, "://")) { continue; } if (pass == 0) { if (denied && route_sr_denied(sr)) { - cbm_ht_set(denied, sr->value, (void *)1); + const char *key = cbm_arena_strdup(&denied_keys, sr->value); + if (key && !cbm_ht_has(denied, key)) { + cbm_ht_set(denied, key, (void *)1); + } } } else if (!denied || !cbm_ht_has(denied, sr->value)) { try_upsert_infra_route(gbuf, sr, files[i].rel_path); } } + cbm_pipeline_result_release((CBMFileResult *)r, loaded); } } cbm_ht_free(denied); + cbm_arena_destroy(&denied_keys); } /* Run decorator_tags, configlink, and route matching passes. */ @@ -1003,9 +1053,208 @@ static void predump_importance(cbm_pipeline_ctx_t *ctx) { * mark at every pass.timing site plus pipeline.begin at the top. */ static void pipeline_phase_mark(const char *pass) { cbm_mem_phase_mark(pass); + /* A phase boundary is where memory is genuinely idle (the results after + * resolve, the semantic transient): hand it back before reading the + * numbers. Measured 2026-09-13 with the mimalloc-backed core: the Go worker + * floor went 15.8 -> 1.0 GB RSS. Milliseconds per phase. */ + cbm_mem_release_to_os(); + log_phase_mem(pass); cbm_mem_class_log(pass); } +/* Research census (2026-09-13): what the retained per-file results are made + * of. Records: count x sizeof per kind, plus the capacity the growable arrays + * reserved (each growth leaves the previous generation dead in the arena). + * Strings: bytes per field family, counted once per record (a pointer that + * is shared between records is charged every time it appears, so a family + * that is really shared shows up LARGER than its arena bytes -- that is the + * signal that interning would win). Measurement only. */ +static size_t census_len(const char *sv) { + return sv ? strlen(sv) + SKIP_ONE : 0; +} +static size_t census_list(const char **list) { + size_t n = 0; + if (!list) { + return 0; + } + for (int i = 0; list[i]; i++) { + n += census_len(list[i]) + sizeof(char *); + } + return n + sizeof(char *); +} +static void log_result_census(const char *tag, CBMFileResult **cache, int file_count) { + enum { PL_BYTES_PER_MB = 1024 * 1024 }; + size_t rec_defs = 0, rec_calls = 0, rec_usages = 0, rec_rw = 0, rec_typerefs = 0; + size_t rec_imports = 0, rec_resolved = 0, rec_other = 0; + size_t cap_defs = 0, cap_calls = 0, cap_usages = 0, cap_rw = 0, cap_typerefs = 0; + size_t cap_other = 0; + size_t n_defs = 0, n_calls = 0, n_usages = 0, n_rw = 0, n_typerefs = 0, n_resolved = 0; + size_t str_def_names = 0, str_def_sig = 0, str_def_doc = 0, str_def_tokens = 0; + size_t str_def_profile = 0, str_def_lists = 0, str_def_fp = 0, str_def_misc = 0; + size_t str_call_names = 0, str_call_enclosing = 0, str_call_args = 0; + size_t str_usage_names = 0, str_usage_enclosing = 0, str_rw = 0, str_typeref = 0; + size_t str_resolved = 0, str_source = 0, str_module = 0; + for (int i = 0; i < file_count; i++) { + const CBMFileResult *r = cache ? cache[i] : NULL; + if (!r) { + continue; + } + rec_defs += (size_t)r->defs.count * sizeof(CBMDefinition); + cap_defs += (size_t)r->defs.cap * sizeof(CBMDefinition); + rec_calls += (size_t)r->calls.count * sizeof(CBMCall); + cap_calls += (size_t)r->calls.cap * sizeof(CBMCall); + rec_usages += (size_t)r->usages.count * sizeof(CBMUsage); + cap_usages += (size_t)r->usages.cap * sizeof(CBMUsage); + rec_rw += (size_t)r->rw.count * sizeof(CBMReadWrite); + cap_rw += (size_t)r->rw.cap * sizeof(CBMReadWrite); + rec_typerefs += (size_t)r->type_refs.count * sizeof(CBMTypeRef); + cap_typerefs += (size_t)r->type_refs.cap * sizeof(CBMTypeRef); + rec_imports += (size_t)r->imports.count * sizeof(CBMImport); + rec_resolved += (size_t)r->resolved_calls.count * sizeof(CBMResolvedCall); + rec_other += (size_t)r->throws.count * sizeof(CBMThrow) + + (size_t)r->env_accesses.count * sizeof(CBMEnvAccess) + + (size_t)r->type_assigns.count * sizeof(CBMTypeAssign) + + (size_t)r->string_refs.count * sizeof(CBMStringRef) + + (size_t)r->impl_traits.count * sizeof(CBMImplTrait) + + (size_t)r->infra_bindings.count * sizeof(CBMInfraBinding) + + (size_t)r->channels.count * sizeof(CBMChannel); + cap_other += (size_t)r->imports.cap * sizeof(CBMImport) + + (size_t)r->resolved_calls.cap * sizeof(CBMResolvedCall) + + (size_t)r->throws.cap * sizeof(CBMThrow) + + (size_t)r->env_accesses.cap * sizeof(CBMEnvAccess) + + (size_t)r->type_assigns.cap * sizeof(CBMTypeAssign) + + (size_t)r->string_refs.cap * sizeof(CBMStringRef) + + (size_t)r->impl_traits.cap * sizeof(CBMImplTrait) + + (size_t)r->infra_bindings.cap * sizeof(CBMInfraBinding) + + (size_t)r->channels.cap * sizeof(CBMChannel); + n_defs += (size_t)r->defs.count; + n_calls += (size_t)r->calls.count; + n_usages += (size_t)r->usages.count; + n_rw += (size_t)r->rw.count; + n_typerefs += (size_t)r->type_refs.count; + n_resolved += (size_t)r->resolved_calls.count; + str_source += (size_t)(r->source ? r->source_len + 1 : 0); + str_module += census_len(r->module_qn) + census_len(r->namespace_name) + + census_list(r->exports) + census_list(r->constants) + + census_list(r->global_vars) + census_list(r->macros); + for (int d = 0; d < r->defs.count; d++) { + const CBMDefinition *def = &r->defs.items[d]; + str_def_names += census_len(def->name) + census_len(def->qualified_name) + + census_len(def->label) + census_len(def->file_path) + + census_len(def->parent_class); + str_def_sig += census_len(def->signature) + census_len(def->return_type) + + census_len(def->receiver); + str_def_doc += census_len(def->docstring); + str_def_tokens += census_len(def->body_tokens); + str_def_profile += census_len(def->structural_profile); + str_def_lists += census_list(def->decorators) + census_list(def->base_classes) + + census_list(def->param_names) + census_list(def->param_types) + + census_list(def->return_types); + for (int k = 0; k < def->signature_param_count; k++) { + str_def_lists += census_len(def->signature_param_types[k]) + sizeof(char *); + } + str_def_fp += def->fingerprint ? (size_t)def->fingerprint_k * sizeof(uint32_t) : 0; + str_def_misc += census_len(def->route_path) + census_len(def->route_method) + + census_len(def->impl_trait); + } + for (int c = 0; c < r->calls.count; c++) { + const CBMCall *call = &r->calls.items[c]; + str_call_names += census_len(call->callee_name) + census_len(call->first_string_arg) + + census_len(call->second_arg_name); + str_call_enclosing += census_len(call->enclosing_func_qn); + for (int a = 0; a < call->arg_count && a < CBM_MAX_CALL_ARGS; a++) { + str_call_args += census_len(call->args[a].expr) + census_len(call->args[a].value) + + census_len(call->args[a].keyword); + } + } + for (int u = 0; u < r->usages.count; u++) { + str_usage_names += census_len(r->usages.items[u].ref_name); + str_usage_enclosing += census_len(r->usages.items[u].enclosing_func_qn); + } + for (int w = 0; w < r->rw.count; w++) { + str_rw += + census_len(r->rw.items[w].var_name) + census_len(r->rw.items[w].enclosing_func_qn); + } + for (int t = 0; t < r->type_refs.count; t++) { + str_typeref += census_len(r->type_refs.items[t].type_name) + + census_len(r->type_refs.items[t].enclosing_func_qn); + } + for (int q = 0; q < r->resolved_calls.count; q++) { + const CBMResolvedCall *rc = &r->resolved_calls.items[q]; + str_resolved += census_len(rc->caller_qn) + census_len(rc->callee_qn) + + census_len(rc->strategy) + census_len(rc->reason); + } + } + /* One snprintf per line: itoa_buf is a small TLS ring and a line with a + * dozen values would overwrite its own earlier fields. */ + char line[CBM_SZ_1K]; +#define MB(x) ((unsigned long)((x) / PL_BYTES_PER_MB)) + snprintf(line, sizeof(line), "defs=%lu calls=%lu usages=%lu rw=%lu type_refs=%lu resolved=%lu", + (unsigned long)n_defs, (unsigned long)n_calls, (unsigned long)n_usages, + (unsigned long)n_rw, (unsigned long)n_typerefs, (unsigned long)n_resolved); + cbm_log_info("extract.census.records", "tag", tag, "v", line); + snprintf( + line, sizeof(line), + "defs=%lu calls=%lu usages=%lu rw=%lu type_refs=%lu imports=%lu resolved=%lu other=%lu", + MB(rec_defs), MB(rec_calls), MB(rec_usages), MB(rec_rw), MB(rec_typerefs), MB(rec_imports), + MB(rec_resolved), MB(rec_other)); + cbm_log_info("extract.census.record_mb", "tag", tag, "v", line); + snprintf(line, sizeof(line), "defs=%lu calls=%lu usages=%lu rw=%lu type_refs=%lu other=%lu", + MB(cap_defs), MB(cap_calls), MB(cap_usages), MB(cap_rw), MB(cap_typerefs), + MB(cap_other)); + cbm_log_info("extract.census.array_cap_mb", "tag", tag, "v", line); + snprintf(line, sizeof(line), + "names=%lu signature=%lu docstring=%lu body_tokens=%lu structural_profile=%lu " + "lists=%lu fingerprint=%lu misc=%lu", + MB(str_def_names), MB(str_def_sig), MB(str_def_doc), MB(str_def_tokens), + MB(str_def_profile), MB(str_def_lists), MB(str_def_fp), MB(str_def_misc)); + cbm_log_info("extract.census.def_strings_mb", "tag", tag, "v", line); + snprintf(line, sizeof(line), + "call_names=%lu call_enclosing=%lu call_args=%lu usage_names=%lu " + "usage_enclosing=%lu rw=%lu type_refs=%lu resolved=%lu source=%lu module=%lu", + MB(str_call_names), MB(str_call_enclosing), MB(str_call_args), MB(str_usage_names), + MB(str_usage_enclosing), MB(str_rw), MB(str_typeref), MB(str_resolved), MB(str_source), + MB(str_module)); + cbm_log_info("extract.census.ref_strings_mb", "tag", tag, "v", line); +#undef MB +} + +/* The per-file result arenas are the largest retained structure of an index + * (Go corpus, 2026-09-13: 28.8 GB of arena capacity live at the end of + * extraction against 15 GB resident). Capacity is what the core charges + * (block sizes); used is what extraction wrote; trees counts results still + * holding a tree-sitter tree. The three numbers together say whether the cost + * is the data, the block-doubling headroom, or retained trees. */ +static void log_result_arenas(const char *tag, CBMFileResult **cache, int file_count) { + enum { PL_BYTES_PER_MB = 1024 * 1024 }; + if (!cbm_mem_phases_enabled()) { + return; /* a walk over every result: diagnostics only (CBM_MEM_PHASES=1) */ + } + size_t used = 0; + size_t capacity = 0; + int results = 0; + int trees = 0; + for (int i = 0; i < file_count; i++) { + const CBMFileResult *r = cache ? cache[i] : NULL; + if (!r) { + continue; + } + results++; + used += cbm_arena_total(&r->arena); + for (int b = 0; b < r->arena.nblocks; b++) { + capacity += r->arena.block_sizes[b]; + } + if (r->cached_tree) { + trees++; + } + } + cbm_log_info("extract.arenas", "tag", tag, "results", itoa_buf(results), "used_mb", + itoa_buf((int)(used / PL_BYTES_PER_MB)), "capacity_mb", + itoa_buf((int)(capacity / PL_BYTES_PER_MB)), "trees", itoa_buf(trees)); + log_result_census(tag, cache, file_count); +} + +/* Results are gone after resolve; so is the store. Logs what spill did. */ static void run_predump_passes(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) { static const struct { predump_pass_fn fn; @@ -1174,8 +1423,8 @@ static int run_sequential_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, * one. process_one_infra_binding self-creates the topic Route node when no * code-side dispatch created it (e.g. a standalone scheduler manifest). */ if (seq_cache && rc == 0) { - cbm_pipeline_extract_infra_routes(p->gbuf, files, seq_cache, file_count); - cbm_pipeline_process_infra_bindings(p->gbuf, files, seq_cache, file_count); + cbm_pipeline_extract_infra_routes(ctx, p->gbuf, files, seq_cache, file_count); + cbm_pipeline_process_infra_bindings(ctx, p->gbuf, files, seq_cache, file_count); } if (seq_cache) { for (int i = 0; i < file_count; i++) { @@ -1239,15 +1488,20 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, return CBM_NOT_FOUND; } cbm_clock_gettime(CLOCK_MONOTONIC, t); + /* This driver is the spill owner: every consumer below reads the cache + * through cbm_pipeline_result_acquire() and the store is closed here. */ + ctx->spill_allowed = true; int rc = cbm_parallel_extract(ctx, files, file_count, cache, &shared_ids, worker_count); cbm_log_info("pass.timing", "pass", "parallel_extract", "elapsed_ms", itoa_buf((int)elapsed_ms(*t))); pipeline_phase_mark("parallel_extract"); + log_result_arenas("post_extract", cache, file_count); if (rc != 0 || check_cancel(p)) { for (int i = 0; i < file_count; i++) { cbm_free_result(cache[i]); } free(cache); + cbm_pipeline_spill_close(ctx); return rc != 0 ? rc : CBM_NOT_FOUND; } cbm_gbuf_set_next_id(p->gbuf, atomic_load(&shared_ids)); @@ -1263,7 +1517,7 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, rc = cbm_build_registry_from_cache(ctx, files, file_count, cache); cbm_log_info("pass.timing", "pass", "registry_build", "elapsed_ms", itoa_buf((int)elapsed_ms(*t))); - log_phase_mem("registry_build"); + pipeline_phase_mark("registry_build"); if (rc != 0 || check_cancel(p)) { for (int i = 0; i < file_count; i++) { if (cache[i]) { @@ -1271,6 +1525,7 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, } } free(cache); + cbm_pipeline_spill_close(ctx); return rc != 0 ? rc : CBM_NOT_FOUND; } /* Registry consumers may materialize serial nodes (Channel, EnvVar, and @@ -1308,13 +1563,17 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, int def_count = 0; CBMLSPDef *all_defs = NULL; int *def_starts = NULL; + /* The collected defs own their strings in this arena (results may be on + * disk by now); the per-language cross registries share it. */ + CBMArena cross_lsp_arena; + cbm_arena_init(&cross_lsp_arena); if (run_cross_lsp) { def_modules = (char **)calloc((size_t)file_count, sizeof(char *)); def_starts = (int *)calloc((size_t)file_count + 1, sizeof(int)); - all_defs = def_modules - ? cbm_pxc_collect_all_defs(ctx, cache, files, file_count, ctx->project_name, - def_modules, &def_count, def_starts) - : NULL; + all_defs = def_modules ? cbm_pxc_collect_all_defs(ctx, &cross_lsp_arena, cache, files, + file_count, ctx->project_name, + def_modules, &def_count, def_starts) + : NULL; } /* Serialize per-file LSP surfaces NOW — the result cache dies with this * pass, and the rows are what lets an incremental run detect body-only @@ -1323,7 +1582,7 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, if (ctx->pipeline && all_defs && def_starts) { cbm_lsp_surface_row_t *surface_rows = NULL; int surface_count = 0; - if (cbm_lsp_surface_build_rows(ctx->project_name, cache, files, file_count, all_defs, + if (cbm_lsp_surface_build_rows(ctx, ctx->project_name, cache, files, file_count, all_defs, def_starts, &surface_rows, &surface_count) == 0) { cbm_pipeline_set_lsp_surfaces(ctx->pipeline, surface_rows, surface_count); } else { @@ -1344,8 +1603,6 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, * during resolve. Per-file work is then: parse + AST walk + O(1) lookups * — no registry build, no Phase 1b mutations. Languages added so far: * Go, Python, C/C++, C#, TS/JS, Java. Others (Kotlin, PHP) fall back to per-file. */ - CBMArena cross_lsp_arena; - cbm_arena_init(&cross_lsp_arena); CBMCrossLspRegistries cross_registries = {0}; if (all_defs) { /* Per-builder split of lsp_cross_prepare — attributes a slow prepare to @@ -1387,13 +1644,14 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, } cbm_log_info("pass.timing", "pass", "lsp_cross_prepare", "elapsed_ms", itoa_buf((int)elapsed_ms(*t))); - log_phase_mem("lsp_cross_prepare"); + pipeline_phase_mark("lsp_cross_prepare"); cbm_clock_gettime(CLOCK_MONOTONIC, t); rc = cbm_parallel_resolve(ctx, files, file_count, cache, &shared_ids, worker_count, all_defs, def_count, def_modules, module_def_index, &cross_registries); cbm_log_info("pass.timing", "pass", "parallel_resolve", "elapsed_ms", itoa_buf((int)elapsed_ms(*t))); - log_phase_mem("parallel_resolve"); + pipeline_phase_mark("parallel_resolve"); + log_result_arenas("post_resolve", cache, file_count); cbm_pxc_free_module_def_index(module_def_index); cbm_arena_destroy(&cross_lsp_arena); /* releases all per-lang registries */ free(all_defs); @@ -1404,20 +1662,22 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, free(def_modules); } cbm_gbuf_set_next_id(p->gbuf, atomic_load(&shared_ids)); - cbm_pipeline_extract_infra_routes(p->gbuf, files, cache, file_count); - cbm_pipeline_process_infra_bindings(p->gbuf, files, cache, file_count); + cbm_pipeline_extract_infra_routes(ctx, p->gbuf, files, cache, file_count); + cbm_pipeline_process_infra_bindings(ctx, p->gbuf, files, cache, file_count); for (int i = 0; i < file_count; i++) { if (cache[i]) { cbm_free_result(cache[i]); } } free(cache); + cbm_pipeline_spill_close(ctx); if (rc != 0) { return rc; } cbm_clock_gettime(CLOCK_MONOTONIC, t); cbm_pipeline_pass_k8s(ctx, files, file_count); cbm_log_info("pass.timing", "pass", "k8s", "elapsed_ms", itoa_buf((int)elapsed_ms(*t))); + pipeline_phase_mark("k8s"); return check_cancel(p) ? CBM_NOT_FOUND : 0; } @@ -2377,6 +2637,7 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, pass_structure(p, files, file_count); CBM_PROF_END_N("pipeline", "pass_structure", t_struct, file_count); cbm_log_info("pass.timing", "pass", "structure", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); + pipeline_phase_mark("structure"); if (check_cancel(p)) { return CBM_NOT_FOUND; } diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index bf71a1f84..7c84726e2 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1285,12 +1285,13 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed int fresh_count = 0; CBMLSPDef *fresh_defs = def_modules && def_starts - ? cbm_pxc_collect_all_defs(ctx, cache, changed_files, ci, ctx->project_name, - def_modules, &fresh_count, def_starts) + ? cbm_pxc_collect_all_defs(ctx, &closure->arena, cache, changed_files, ci, + ctx->project_name, def_modules, &fresh_count, + def_starts) : NULL; if ((fresh_defs || fresh_count == 0) && def_starts && - cbm_lsp_surface_build_rows(ctx->project_name, cache, changed_files, ci, fresh_defs, - def_starts, &closure->fresh_rows, + cbm_lsp_surface_build_rows(ctx, ctx->project_name, cache, changed_files, ci, + fresh_defs, def_starts, &closure->fresh_rows, &closure->fresh_count) != 0) { closure->fresh_rows = NULL; closure->fresh_count = 0; @@ -1587,15 +1588,18 @@ static int closure_probe_surfaces(cbm_pipeline_t *p, const char *project, int *def_starts = (int *)calloc((size_t)probe_count + 1, sizeof(int)); int def_count = 0; CBMLSPDef *defs = NULL; + CBMArena probe_arena; + cbm_arena_init(&probe_arena); if (def_modules && def_starts) { - defs = cbm_pxc_collect_all_defs(NULL, cache, probe_files, probe_count, project, - def_modules, &def_count, def_starts); - rc = cbm_lsp_surface_build_rows(project, cache, probe_files, probe_count, defs, + defs = cbm_pxc_collect_all_defs(NULL, &probe_arena, cache, probe_files, probe_count, + project, def_modules, &def_count, def_starts); + rc = cbm_lsp_surface_build_rows(NULL, project, cache, probe_files, probe_count, defs, def_starts, out_rows, out_count); } else { rc = -1; } free(defs); + cbm_arena_destroy(&probe_arena); free(def_starts); if (def_modules) { for (int i = 0; i < probe_count; i++) { diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 8deb38aba..0a3820142 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -136,8 +136,41 @@ typedef struct { /* ObjectScript method-return-type table built from extracted definitions * (NULL until pass_calls builds it). Owned by pipeline.c. */ const CBMReturnTypeTable *return_type_table; + + /* Spill / admission control (2026-09-13). spill_mode latches on the first + * over-budget observation in the extract gate (or on CBM_MEM_SPILL=1): + * from then on every compacted result is parked on disk instead of held + * in the cache, results already cached are swept out, and every later + * consumer (registry build, def collection, resolve) loads a result only + * for the moment it reads it. Memory then sits at the floor -- graph + + * registries + in-flight files -- and the run pays with disk reads. + * NULL/0 = results stay in memory as always. Owned by pipeline.c. */ + struct cbm_result_spill *spill; + _Atomic int spill_mode; + /* Set by the ONE owner whose every result-cache consumer goes through + * cbm_pipeline_result_acquire()/release() and that closes the store + * (run_parallel_pipeline). An owner that leaves it false never spills: + * the incremental and probe routes still hand the cache array to passes + * that index it directly, so they keep results in memory (follow-up). */ + bool spill_allowed; } cbm_pipeline_ctx_t; +/* ── Result-cache access contract (spill mode) ──────────────────────── + * After extraction a slot of the result cache is either the in-memory result + * or NULL with the result parked on disk (ctx->spill). Every consumer reads a + * slot through this pair; a pass that indexes the array itself is blind to + * parked results (the infra-route passes lost every __route__infra__ node + * that way, 2026-09-13). `want` (NULL = always) sees the parked HEADER first + * -- counts are valid, pointers are not -- and can veto the load, so a pass + * after one rare list does not read every parked result back from disk. */ +typedef bool (*cbm_result_want_fn)(const CBMFileResult *header); +CBMFileResult *cbm_pipeline_result_acquire(const cbm_pipeline_ctx_t *ctx, CBMFileResult **cache, + int i, cbm_result_want_fn want, bool *loaded); +void cbm_pipeline_result_release(CBMFileResult *r, bool loaded); + +/* Log the store counters, close and delete the store, drop the latch. */ +void cbm_pipeline_spill_close(cbm_pipeline_ctx_t *ctx); + /* Transcode an ObjectScript Studio Export XML file and compose every generated * UDL class into one cacheable result. The returned result owns all child * extraction arenas and is released with the ordinary cbm_free_result(). */ diff --git a/src/pipeline/worker_pool.c b/src/pipeline/worker_pool.c index 9cde541a4..ac0ab9552 100644 --- a/src/pipeline/worker_pool.c +++ b/src/pipeline/worker_pool.c @@ -14,6 +14,7 @@ enum { WP_TRUE = 1, WP_MIN = 1, WP_STEP = 1 }; #include "foundation/platform.h" #include "foundation/compat_thread.h" +#include "foundation/mem_core.h" #include #include @@ -48,6 +49,9 @@ static void *pthread_worker(void *arg) { } wa->fn(idx, wa->ctx); } + /* This thread ends here: hand its pending memory-class deltas to the + * shared counters, so the phase mark that follows the join is exact. */ + cbm_mem_class_flush_thread(); return NULL; } diff --git a/src/semantic/semantic.c b/src/semantic/semantic.c index 8023e70cd..4592f4874 100644 --- a/src/semantic/semantic.c +++ b/src/semantic/semantic.c @@ -6,6 +6,7 @@ * Uses xxHash for deterministic random vectors. Pure C, zero dependencies. */ #include "semantic/semantic.h" +#include "foundation/mem_core.h" #include "foundation/constants.h" #include "foundation/hash_table.h" #include "foundation/log.h" @@ -158,7 +159,7 @@ static bool is_camel_break(const char *name, int i) { static void flush_token(char *buf, int *blen, char **out, int *count, int max_out) { if (*blen > 0 && *count < max_out) { buf[*blen] = '\0'; - out[(*count)++] = strdup(buf); + out[(*count)++] = cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, buf); } *blen = 0; } @@ -358,7 +359,7 @@ int cbm_sem_tokenize(const char *name, char **out, int max_out) { for (int t = 0; t < orig_count && count < max_out; t++) { for (int a = 0; abbrevs[a].abbrev; a++) { if (strcmp(out[t], abbrevs[a].abbrev) == 0) { - out[count++] = strdup(abbrevs[a].expanded); + out[count++] = cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, abbrevs[a].expanded); break; } } @@ -422,7 +423,8 @@ static void ensure_pretrained_map(void) { const char *tok = PRETRAINED_TOKENS[i]; if (tok && tok[0]) { snprintf(idx_buf, sizeof(idx_buf), "%d", i); - cbm_ht_set(g_pretrained_map, strdup(tok), strdup(idx_buf)); + cbm_ht_set(g_pretrained_map, cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, tok), + cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, idx_buf)); } } atomic_store_explicit(&g_pretrained_ready, MAP_READY, memory_order_release); @@ -524,7 +526,8 @@ static int corpus_get_or_add(cbm_sem_corpus_t *c, const char *token) { } if (c->entry_count >= c->entry_cap) { int new_cap = c->entry_cap < CORPUS_INIT_CAP ? CORPUS_INIT_CAP : c->entry_cap * PAIR_LEN; - corpus_entry_t *grown = realloc(c->entries, (size_t)new_cap * sizeof(corpus_entry_t)); + corpus_entry_t *grown = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, c->entries, + (size_t)new_cap * sizeof(corpus_entry_t)); if (!grown) { return CBM_NOT_FOUND; } @@ -532,16 +535,17 @@ static int corpus_get_or_add(cbm_sem_corpus_t *c, const char *token) { c->entry_cap = new_cap; } int idx = c->entry_count++; - c->entries[idx].token = strdup(token); + c->entries[idx].token = cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, token); c->entries[idx].doc_freq = 0; memset(&c->entries[idx].enriched_vec, 0, sizeof(cbm_sem_vec_t)); snprintf(idx_buf, sizeof(idx_buf), "%d", idx); - cbm_ht_set(c->token_map, strdup(token), strdup(idx_buf)); + cbm_ht_set(c->token_map, cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, token), + cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, idx_buf)); return idx; } cbm_sem_corpus_t *cbm_sem_corpus_new(void) { - cbm_sem_corpus_t *c = calloc(SKIP_ONE, sizeof(cbm_sem_corpus_t)); + cbm_sem_corpus_t *c = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, sizeof(cbm_sem_corpus_t)); if (c) { c->token_map = cbm_ht_create(CORPUS_INIT_CAP); } @@ -556,11 +560,13 @@ void cbm_sem_corpus_add_doc(cbm_sem_corpus_t *corpus, const char **tokens, int c if (corpus->doc_count >= corpus->doc_cap) { int new_cap = corpus->doc_cap < DOC_TOKENS_INIT ? DOC_TOKENS_INIT : corpus->doc_cap * PAIR_LEN; - int **grown_ids = realloc(corpus->doc_token_ids, (size_t)new_cap * sizeof(int *)); - int *grown_counts = realloc(corpus->doc_token_counts, (size_t)new_cap * sizeof(int)); + int **grown_ids = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_ids, + (size_t)new_cap * sizeof(int *)); + int *grown_counts = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_counts, + (size_t)new_cap * sizeof(int)); if (!grown_ids || !grown_counts) { - free(grown_ids); - free(grown_counts); + cbm_free(CBM_MEM_CLASS_SEMANTIC, grown_ids); + cbm_free(CBM_MEM_CLASS_SEMANTIC, grown_counts); return; } corpus->doc_token_ids = grown_ids; @@ -568,11 +574,13 @@ void cbm_sem_corpus_add_doc(cbm_sem_corpus_t *corpus, const char **tokens, int c corpus->doc_cap = new_cap; } int doc_idx = corpus->doc_count++; - corpus->doc_token_ids[doc_idx] = malloc((size_t)count * sizeof(int)); + corpus->doc_token_ids[doc_idx] = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)count * sizeof(int)); corpus->doc_token_counts[doc_idx] = count; /* Per-doc unique set for IDF */ - int *seen = calloc((size_t)corpus->entry_cap + (size_t)count + CORPUS_INIT_CAP, sizeof(int)); + int *seen = + cbm_calloc(CBM_MEM_CLASS_SEMANTIC, + ((size_t)corpus->entry_cap + (size_t)count + CORPUS_INIT_CAP) * sizeof(int)); int seen_count = 0; for (int i = 0; i < count; i++) { @@ -594,7 +602,7 @@ void cbm_sem_corpus_add_doc(cbm_sem_corpus_t *corpus, const char **tokens, int c corpus->entries[tid].doc_freq++; } } - free(seen); + cbm_free(CBM_MEM_CLASS_SEMANTIC, seen); } /* ── Parallel corpus batch build ──────────────────────────────────── */ @@ -612,8 +620,8 @@ void cbm_sem_corpus_add_doc(cbm_sem_corpus_t *corpus, const char **tokens, int c typedef struct { cbm_sem_corpus_t *corpus; char **all_tokens; + const size_t *offsets; /* document d = all_tokens[offsets[d] ..] */ const int *token_counts; - int max_tokens; int doc_count; _Atomic int *doc_freq_atomic; /* per-entry atomic counter (entry_count long) */ _Atomic int next_idx; @@ -630,12 +638,12 @@ static void batch_resolve_one_doc(batch_resolve_ctx_t *bc, int doc_index, int *s bc->corpus->doc_token_counts[doc_index] = 0; return; } - int *ids = malloc((size_t)count * sizeof(int)); + int *ids = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)count * sizeof(int)); bc->corpus->doc_token_ids[doc_index] = ids; bc->corpus->doc_token_counts[doc_index] = count; int seen_count = 0; - char **tokens = &bc->all_tokens[(ptrdiff_t)doc_index * bc->max_tokens]; + char **tokens = &bc->all_tokens[bc->offsets[doc_index]]; for (int i = 0; i < count; i++) { const char *idx_str = cbm_ht_get(bc->corpus->token_map, tokens[i]); int tid = CBM_NOT_FOUND; @@ -671,7 +679,7 @@ static void batch_resolve_worker(int worker_id, void *ctx_ptr) { batch_resolve_ctx_t *bc = ctx_ptr; /* Per-worker scratch for unique-per-doc tracking */ int local_seen_cap = CBM_SEM_SEEN_INIT_CAP; - int *seen = malloc((size_t)local_seen_cap * sizeof(int)); + int *seen = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)local_seen_cap * sizeof(int)); if (!seen) { return; } @@ -689,7 +697,7 @@ static void batch_resolve_worker(int worker_id, void *ctx_ptr) { for (int d = start; d < end; d++) { int count = bc->token_counts[d]; if (count > local_seen_cap) { - int *grown = realloc(seen, (size_t)count * sizeof(int)); + int *grown = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, seen, (size_t)count * sizeof(int)); if (!grown) { continue; } @@ -699,12 +707,12 @@ static void batch_resolve_worker(int worker_id, void *ctx_ptr) { batch_resolve_one_doc(bc, d, seen); } } - free(seen); + cbm_free(CBM_MEM_CLASS_SEMANTIC, seen); } void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens, - const int *token_counts, int doc_count, int max_tokens_per_doc) { - if (!corpus || !all_tokens || !token_counts || doc_count <= 0) { + const size_t *offsets, const int *token_counts, int doc_count) { + if (!corpus || !all_tokens || !offsets || !token_counts || doc_count <= 0) { return; } @@ -712,11 +720,13 @@ void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens, * Hash table mutation can't be parallelized; strdup+insert is the cost. */ if (corpus->doc_cap < corpus->doc_count + doc_count) { int new_cap = corpus->doc_count + doc_count; - int **grown_ids = realloc(corpus->doc_token_ids, (size_t)new_cap * sizeof(int *)); - int *grown_counts = realloc(corpus->doc_token_counts, (size_t)new_cap * sizeof(int)); + int **grown_ids = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_ids, + (size_t)new_cap * sizeof(int *)); + int *grown_counts = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_counts, + (size_t)new_cap * sizeof(int)); if (!grown_ids || !grown_counts) { - free(grown_ids); - free(grown_counts); + cbm_free(CBM_MEM_CLASS_SEMANTIC, grown_ids); + cbm_free(CBM_MEM_CLASS_SEMANTIC, grown_counts); return; } corpus->doc_token_ids = grown_ids; @@ -728,7 +738,7 @@ void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens, for (int d = 0; d < doc_count; d++) { int count = token_counts[d]; - char **tokens = &all_tokens[(ptrdiff_t)d * max_tokens_per_doc]; + char **tokens = &all_tokens[offsets[d]]; for (int i = 0; i < count; i++) { /* Inserts token into token_map if new; we discard return here — * Phase B will re-lookup in read-only mode to get the ID. */ @@ -739,14 +749,15 @@ void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens, /* Phase B (PARALLEL): Resolve tokens → IDs and count doc_freq per entry. * token_map is now read-only; each worker owns its doc range (no writes * to shared state except atomic doc_freq counters). */ - _Atomic int *doc_freq_atomic = calloc((size_t)corpus->entry_count, sizeof(_Atomic int)); + _Atomic int *doc_freq_atomic = + cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)corpus->entry_count * sizeof(_Atomic int)); if (!doc_freq_atomic) { /* OOM fallback: sequential path. Roll back doc_count first since * add_doc increments it itself. */ corpus->doc_count = base_doc; for (int d = 0; d < doc_count; d++) { int count = token_counts[d]; - char **tokens = &all_tokens[(ptrdiff_t)d * max_tokens_per_doc]; + char **tokens = &all_tokens[offsets[d]]; cbm_sem_corpus_add_doc(corpus, (const char **)tokens, count); } return; @@ -757,7 +768,7 @@ void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens, .corpus = corpus, .all_tokens = all_tokens, .token_counts = token_counts, - .max_tokens = max_tokens_per_doc, + .offsets = offsets, .doc_count = doc_count, .doc_freq_atomic = doc_freq_atomic, }; @@ -775,7 +786,7 @@ void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens, corpus->entries[i].doc_freq += atomic_load_explicit(&doc_freq_atomic[i], memory_order_relaxed); } - free(doc_freq_atomic); + cbm_free(CBM_MEM_CLASS_SEMANTIC, doc_freq_atomic); } /* ── Parallel corpus_finalize ─────────────────────────────────────── */ @@ -1205,14 +1216,15 @@ static void pass1_quantize_worker(int worker_id, void *ctx_ptr) { /* Build reverse index: token_id → list of (doc_id, position) pairs. * SEQUENTIAL (fast: just pointer arithmetic + flat array fill). */ static reverse_index_t *build_reverse_index(cbm_sem_corpus_t *corpus) { - reverse_index_t *rev = calloc(SKIP_ONE, sizeof(reverse_index_t)); + reverse_index_t *rev = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, sizeof(reverse_index_t)); if (!rev) { return NULL; } /* Phase A: count occurrences per token */ - int *counts = calloc((size_t)corpus->entry_count + SKIP_ONE, sizeof(int)); + int *counts = + cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)corpus->entry_count + SKIP_ONE) * sizeof(int)); if (!counts) { - free(rev); + cbm_free(CBM_MEM_CLASS_SEMANTIC, rev); return NULL; } long total = 0; @@ -1228,10 +1240,11 @@ static reverse_index_t *build_reverse_index(cbm_sem_corpus_t *corpus) { } } /* Phase B: exclusive prefix sum → offsets[] */ - rev->offsets = malloc(((size_t)corpus->entry_count + SKIP_ONE) * sizeof(int)); + rev->offsets = + cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)corpus->entry_count + SKIP_ONE) * sizeof(int)); if (!rev->offsets) { - free(counts); - free(rev); + cbm_free(CBM_MEM_CLASS_SEMANTIC, counts); + cbm_free(CBM_MEM_CLASS_SEMANTIC, rev); return NULL; } int running = 0; @@ -1242,13 +1255,13 @@ static reverse_index_t *build_reverse_index(cbm_sem_corpus_t *corpus) { } rev->offsets[corpus->entry_count] = running; /* Phase C: fill flat array. Ensure allocation size > 0 even for empty - * corpora (avoids malloc(0) which is implementation-defined). */ + * corpora (avoids cbm_alloc(CBM_MEM_CLASS_SEMANTIC, 0) which is implementation-defined). */ size_t flat_bytes = (total > 0 ? (size_t)total : SKIP_ONE) * sizeof(cooccur_pos_t); - rev->flat = malloc(flat_bytes); + rev->flat = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, flat_bytes); if (!rev->flat) { - free(rev->offsets); - free(counts); - free(rev); + cbm_free(CBM_MEM_CLASS_SEMANTIC, rev->offsets); + cbm_free(CBM_MEM_CLASS_SEMANTIC, counts); + cbm_free(CBM_MEM_CLASS_SEMANTIC, rev); return NULL; } for (int d = 0; d < corpus->doc_count; d++) { @@ -1263,7 +1276,7 @@ static reverse_index_t *build_reverse_index(cbm_sem_corpus_t *corpus) { } } } - free(counts); + cbm_free(CBM_MEM_CLASS_SEMANTIC, counts); return rev; } @@ -1271,9 +1284,9 @@ static void free_reverse_index(reverse_index_t *rev) { if (!rev) { return; } - free(rev->offsets); - free(rev->flat); - free(rev); + cbm_free(CBM_MEM_CLASS_SEMANTIC, rev->offsets); + cbm_free(CBM_MEM_CLASS_SEMANTIC, rev->flat); + cbm_free(CBM_MEM_CLASS_SEMANTIC, rev); } /* Bundle of parameters shared by the finalize sub-phases. */ @@ -1323,7 +1336,8 @@ static void finalize_pass1(finalize_params_t *p) { /* Sub-phases 4+5: quantize pass1 to int8, run RRI pass 2, blend + normalize. */ static void finalize_pass2(finalize_params_t *p) { - int8_t *pass1_q = malloc((size_t)p->corpus->entry_count * CBM_SEM_DIM * sizeof(int8_t)); + int8_t *pass1_q = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, + (size_t)p->corpus->entry_count * CBM_SEM_DIM * sizeof(int8_t)); if (pass1_q) { pass1_quant_ctx_t qc = { .entries = p->corpus->entries, @@ -1334,7 +1348,8 @@ static void finalize_pass2(finalize_params_t *p) { cbm_parallel_for(p->worker_count, pass1_quantize_worker, &qc, p->opts); } - cbm_sem_vec_t *pass1 = malloc((size_t)p->corpus->entry_count * sizeof(cbm_sem_vec_t)); + cbm_sem_vec_t *pass1 = + cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)p->corpus->entry_count * sizeof(cbm_sem_vec_t)); if (pass1) { for (int i = 0; i < p->corpus->entry_count; i++) { pass1[i] = p->corpus->entries[i].enriched_vec; @@ -1366,15 +1381,25 @@ static void finalize_pass2(finalize_params_t *p) { }; atomic_init(&bc.next_idx, 0); cbm_parallel_for(p->worker_count, blend_worker, &bc, p->opts); - free(pass1); + cbm_free(CBM_MEM_CLASS_SEMANTIC, pass1); } - free(pass1_q); + cbm_free(CBM_MEM_CLASS_SEMANTIC, pass1_q); norm_ctx_t nc = {.entries = p->corpus->entries, .entry_count = p->corpus->entry_count}; atomic_init(&nc.next_idx, 0); cbm_parallel_for(p->worker_count, normalize_worker, &nc, p->opts); } +/* The per-document token id lists exist for the co-occurrence pass only; + * released at the end of finalize so the vector phase runs without them + * (783k small blocks on the kernel). */ +static void corpus_release_docs(cbm_sem_corpus_t *corpus) { + for (int d = 0; d < corpus->doc_count; d++) { + cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_ids[d]); + corpus->doc_token_ids[d] = NULL; + } +} + void cbm_sem_corpus_finalize(cbm_sem_corpus_t *corpus) { if (!corpus || corpus->finalized) { return; @@ -1401,8 +1426,8 @@ void cbm_sem_corpus_finalize(cbm_sem_corpus_t *corpus) { corpus->finalized = true; return; } - cbm_sem_src_entry_t *src_entries = - calloc((size_t)corpus->entry_count, sizeof(cbm_sem_src_entry_t)); + cbm_sem_src_entry_t *src_entries = cbm_calloc( + CBM_MEM_CLASS_SEMANTIC, (size_t)corpus->entry_count * sizeof(cbm_sem_src_entry_t)); if (!src_entries) { free_reverse_index(rev); corpus->finalized = true; @@ -1423,8 +1448,9 @@ void cbm_sem_corpus_finalize(cbm_sem_corpus_t *corpus) { finalize_pass1(¶ms); finalize_pass2(¶ms); - free(src_entries); + cbm_free(CBM_MEM_CLASS_SEMANTIC, src_entries); free_reverse_index(rev); + corpus_release_docs(corpus); corpus->finalized = true; } @@ -1490,8 +1516,8 @@ const char *cbm_sem_corpus_token_at(const cbm_sem_corpus_t *corpus, int index, static void free_ht_kv(const char *key, void *value, void *userdata) { (void)userdata; - free((void *)key); - free(value); + cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)key); + cbm_free(CBM_MEM_CLASS_SEMANTIC, value); } void cbm_sem_corpus_free(cbm_sem_corpus_t *corpus) { @@ -1499,19 +1525,19 @@ void cbm_sem_corpus_free(cbm_sem_corpus_t *corpus) { return; } for (int i = 0; i < corpus->entry_count; i++) { - free(corpus->entries[i].token); + cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->entries[i].token); } - free(corpus->entries); + cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->entries); for (int d = 0; d < corpus->doc_count; d++) { - free(corpus->doc_token_ids[d]); + cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_ids[d]); } - free(corpus->doc_token_ids); - free(corpus->doc_token_counts); + cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_ids); + cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_counts); if (corpus->token_map) { cbm_ht_foreach(corpus->token_map, free_ht_kv, NULL); cbm_ht_free(corpus->token_map); } - free(corpus); + cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus); } /* ── Combined scoring ────────────────────────────────────────────── */ diff --git a/src/semantic/semantic.h b/src/semantic/semantic.h index ddb5f5896..335c38f0e 100644 --- a/src/semantic/semantic.h +++ b/src/semantic/semantic.h @@ -24,6 +24,7 @@ #ifndef CBM_SEMANTIC_H #define CBM_SEMANTIC_H +#include /* size_t */ #include #include @@ -166,11 +167,13 @@ cbm_sem_corpus_t *cbm_sem_corpus_new(void); void cbm_sem_corpus_add_doc(cbm_sem_corpus_t *corpus, const char **tokens, int count); /* Batch-build the corpus from pre-tokenized documents (PARALLEL variant). - * `all_tokens` layout: all_tokens[f * max_tokens_per_doc + t] = token pointer. - * `token_counts[f]` = number of tokens in document f. + * Document f's tokens are all_tokens[offsets[f] .. offsets[f] + token_counts[f]). + * Packed, not strided: a fixed CBM_SEM_MAX_TOKENS (512) slots per document + * was 4 KB per function up front -- 7.4 GB on the kernel for tokens that + * average a few dozen per function. * This replaces a loop of cbm_sem_corpus_add_doc() calls. */ void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens, - const int *token_counts, int doc_count, int max_tokens_per_doc); + const size_t *offsets, const int *token_counts, int doc_count); /* Finalize: compute IDF, build enriched token vectors via co-occurrence. */ void cbm_sem_corpus_finalize(cbm_sem_corpus_t *corpus); diff --git a/src/simhash/minhash.c b/src/simhash/minhash.c index dc42ef4f5..168a38e74 100644 --- a/src/simhash/minhash.c +++ b/src/simhash/minhash.c @@ -9,6 +9,7 @@ */ #include "simhash/minhash.h" #include "foundation/constants.h" +#include "foundation/mem_core.h" #include "foundation/log.h" /* Inline all xxHash functions — avoids separate compilation unit. */ #define XXH_INLINE_ALL @@ -355,7 +356,8 @@ static uint32_t band_hash(const cbm_minhash_t *fp, int band) { static void bucket_push(lsh_bucket_t *bucket, int entry_index) { if (bucket->count >= bucket->cap) { int new_cap = bucket->cap < BUCKET_INIT_CAP ? BUCKET_INIT_CAP : bucket->cap * GROW_FACTOR; - int *new_items = realloc(bucket->items, (size_t)new_cap * sizeof(int)); + int *new_items = + cbm_realloc(CBM_MEM_CLASS_SEMANTIC, bucket->items, (size_t)new_cap * sizeof(int)); if (!new_items) { return; } @@ -366,7 +368,8 @@ static void bucket_push(lsh_bucket_t *bucket, int entry_index) { } cbm_lsh_index_t *cbm_lsh_new(void) { - cbm_lsh_index_t *idx = calloc(SKIP_ONE, sizeof(cbm_lsh_index_t)); + cbm_lsh_index_t *idx = + cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)(SKIP_ONE) * (sizeof(cbm_lsh_index_t))); return idx; } @@ -379,8 +382,8 @@ void cbm_lsh_insert(cbm_lsh_index_t *idx, const cbm_lsh_entry_t *entry) { if (idx->entry_count >= idx->entry_cap) { int new_cap = idx->entry_cap < ENTRY_INIT_CAP ? ENTRY_INIT_CAP : idx->entry_cap * GROW_FACTOR; - cbm_lsh_entry_t *new_entries = - realloc(idx->entries, (size_t)new_cap * sizeof(cbm_lsh_entry_t)); + cbm_lsh_entry_t *new_entries = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, idx->entries, + (size_t)new_cap * sizeof(cbm_lsh_entry_t)); if (!new_entries) { return; } @@ -405,7 +408,7 @@ typedef struct { } seen_set_t; static void seen_set_init(seen_set_t *s) { - s->slots = calloc(SEEN_SET_SIZE, sizeof(int64_t)); + s->slots = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)(SEEN_SET_SIZE) * (sizeof(int64_t))); s->cap = SEEN_SET_SIZE; /* 0 means empty — node_ids are always > 0 */ } @@ -429,7 +432,7 @@ static bool seen_set_insert(seen_set_t *s, int64_t node_id) { } static void seen_set_free(seen_set_t *s) { - free(s->slots); + cbm_free(CBM_MEM_CLASS_SEMANTIC, s->slots); s->slots = NULL; } @@ -439,7 +442,8 @@ static bool result_push(cbm_lsh_index_t *idx, const cbm_lsh_entry_t *candidate) int new_cap = idx->result_cap < RESULT_INIT_CAP ? RESULT_INIT_CAP : idx->result_cap * GROW_FACTOR; const cbm_lsh_entry_t **new_buf = - realloc(idx->result_buf, (size_t)new_cap * sizeof(const cbm_lsh_entry_t *)); + cbm_realloc(CBM_MEM_CLASS_SEMANTIC, idx->result_buf, + (size_t)new_cap * sizeof(const cbm_lsh_entry_t *)); if (!new_buf) { return false; } @@ -529,10 +533,10 @@ void cbm_lsh_free(cbm_lsh_index_t *idx) { /* Free bucket arrays */ for (int b = 0; b < CBM_LSH_BANDS; b++) { for (int h = 0; h < LSH_BUCKET_COUNT; h++) { - free(idx->bands[b][h].items); + cbm_free(CBM_MEM_CLASS_SEMANTIC, idx->bands[b][h].items); } } - free(idx->entries); - free(idx->result_buf); - free(idx); + cbm_free(CBM_MEM_CLASS_SEMANTIC, idx->entries); + cbm_free(CBM_MEM_CLASS_SEMANTIC, idx->result_buf); + cbm_free(CBM_MEM_CLASS_SEMANTIC, idx); } diff --git a/tests/test_arena.c b/tests/test_arena.c index 788c1eb9c..20021a4be 100644 --- a/tests/test_arena.c +++ b/tests/test_arena.c @@ -5,6 +5,45 @@ #include "../src/foundation/arena.h" #include +TEST(arena_init_exact_one_block_then_default_growth) { + CBMArena a; + cbm_arena_init_exact(&a, 100); + ASSERT_EQ(a.nblocks, 1); + ASSERT_EQ(a.block_sizes[0], 104); /* rounded to the 8-byte allocation grain */ + ASSERT_EQ(a.grow_size, CBM_ARENA_DEFAULT_BLOCK_SIZE); + void *p = cbm_arena_alloc(&a, 100); + ASSERT_NOT_NULL(p); + ASSERT_EQ(a.used, 104); + ASSERT_EQ(a.nblocks, 1); /* the exact block is full, nothing wasted */ + void *q = cbm_arena_alloc(&a, 8); + ASSERT_NOT_NULL(q); + ASSERT_EQ(a.nblocks, 2); + ASSERT_EQ(a.block_sizes[1], CBM_ARENA_DEFAULT_BLOCK_SIZE); /* not 2 x 104 */ + ASSERT_EQ(a.grow_size, 2 * CBM_ARENA_DEFAULT_BLOCK_SIZE); + cbm_arena_destroy(&a); + PASS(); +} + +TEST(arena_growth_doubles_grow_size_and_reset_restores_it) { + CBMArena a; + cbm_arena_init_sized(&a, 64); + ASSERT_EQ(a.grow_size, 128); + ASSERT_NOT_NULL(cbm_arena_alloc(&a, 64)); + ASSERT_NOT_NULL(cbm_arena_alloc(&a, 8)); /* -> block 128 */ + ASSERT_EQ(a.nblocks, 2); + ASSERT_EQ(a.block_sizes[1], 128); + ASSERT_EQ(a.grow_size, 256); + ASSERT_NOT_NULL(cbm_arena_alloc(&a, 1000)); /* larger than grow_size: exact */ + ASSERT_EQ(a.nblocks, 3); + ASSERT_EQ(a.block_sizes[2], 1000); + ASSERT_EQ(a.grow_size, 2000); + cbm_arena_reset(&a); + ASSERT_EQ(a.nblocks, 1); + ASSERT_EQ(a.grow_size, 128); + cbm_arena_destroy(&a); + PASS(); +} + TEST(arena_init_default) { CBMArena a; cbm_arena_init(&a); @@ -428,6 +467,8 @@ TEST(arena_strndup_zero_len) { } SUITE(arena) { + RUN_TEST(arena_init_exact_one_block_then_default_growth); + RUN_TEST(arena_growth_doubles_grow_size_and_reset_restores_it); RUN_TEST(arena_init_default); RUN_TEST(arena_init_sized); RUN_TEST(arena_alloc_basic); diff --git a/tests/test_c_lsp.c b/tests/test_c_lsp.c index 0ba216ce0..4172fcc7f 100644 --- a/tests/test_c_lsp.c +++ b/tests/test_c_lsp.c @@ -15250,6 +15250,167 @@ TEST(clsp_tier2_shared_registry_readonly_c) { PASS(); } +/* The method-return refinement in the C++ class walk used to cast the chained + * lookup result and write a scratch-arena signature into it. With the entry in + * the sealed shared base, every other worker read that signature after the + * arena of this file had died (ASan heap-use-after-free in c_adl_resolve on + * dotnet/runtime, 2026-09-14). Contract: the refinement is copy-on-write into + * the overlay; the base signature pointer never changes. min_params = 7 is a + * marker only a COPY of the base entry carries (a fresh registration sets -1), + * so the assertion proves the upgrade path ran, not a re-registration. */ +TEST(clsp_method_return_refinement_is_copy_on_write) { + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry base; + cbm_registry_init(&base, &arena); + const CBMType *rets[2] = {cbm_type_named(&arena, "test.mod.Item"), NULL}; + CBMRegisteredFunc f; + memset(&f, 0, sizeof(f)); + f.qualified_name = "test.mod.Box.items"; + f.short_name = "items"; + f.receiver_type = "test.mod.Box"; + f.signature = cbm_type_func(&arena, NULL, NULL, rets); + f.min_params = 7; + cbm_registry_add_func(&base, f); + cbm_registry_finalize(&base); + base.read_only = true; + const CBMRegisteredFunc *base_entry = cbm_registry_lookup_func(&base, "test.mod.Box.items"); + ASSERT_NOT_NULL(base_entry); + const CBMType *base_sig = base_entry->signature; + ASSERT_EQ(base_sig->data.func.return_types[0]->kind, CBM_TYPE_NAMED); + + CBMArena scratch; + cbm_arena_init(&scratch); + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &scratch); + overlay.fallback = &base; + /* The in-class declaration refines the NAMED return to a POINTER. */ + const char *src = "struct Item { int v; };\n" + "struct Box {\n" + " Item *items();\n" + "};\n"; + CBMResolvedCallArray out = {0}; + cbm_run_c_lsp_cross_with_registry(&scratch, src, (int)strlen(src), "test.mod", + /*cpp_mode=*/true, &overlay, NULL, NULL, 0, NULL, &out); + + ASSERT(base_entry->signature == base_sig); + ASSERT_EQ(base_sig->data.func.return_types[0]->kind, CBM_TYPE_NAMED); + const CBMRegisteredFunc *refined = cbm_registry_lookup_func(&overlay, "test.mod.Box.items"); + ASSERT_NOT_NULL(refined); + ASSERT(refined >= overlay.funcs && refined < overlay.funcs + overlay.func_count); + ASSERT_EQ(refined->min_params, 7); + ASSERT_EQ(refined->signature->data.func.return_types[0]->kind, CBM_TYPE_POINTER); + cbm_arena_destroy(&scratch); + cbm_arena_destroy(&arena); + PASS(); +} + +/* The per-file overlay contract (type_registry.c chain API): a walk is handed + * an overlay chained to a sealed base. Lookups and iterators see the base + * through the overlay, every yielded index belongs to it.reg, a refinement is + * copy-on-write into the overlay (the base never changes), and an overlay copy + * shadows its base original in chained iteration. */ +TEST(registry_overlay_chain_iterates_and_copies_on_write) { + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry base; + cbm_registry_init(&base, &arena); + CBMRegisteredFunc f; + memset(&f, 0, sizeof(f)); + f.qualified_name = "pkg.alpha"; + f.short_name = "alpha"; + f.min_params = -1; + cbm_registry_add_func(&base, f); + f.qualified_name = "pkg.beta"; + f.short_name = "beta"; + cbm_registry_add_func(&base, f); + CBMRegisteredType t; + memset(&t, 0, sizeof(t)); + t.qualified_name = "pkg.T"; + t.short_name = "T"; + cbm_registry_add_type(&base, t); + cbm_registry_finalize(&base); + base.read_only = true; + + CBMArena scratch; + cbm_arena_init(&scratch); + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &scratch); + overlay.fallback = &base; + + /* Lookups chain through the empty overlay. */ + ASSERT_NOT_NULL(cbm_registry_lookup_func(&overlay, "pkg.alpha")); + ASSERT_NOT_NULL(cbm_registry_lookup_type(&overlay, "pkg.T")); + + /* Chained iteration reaches the base; it.reg names where the index lives. */ + CBMFreeFuncIter it; + cbm_registry_free_funcs_by_short_name_chain(&overlay, "alpha", &it); + int i = cbm_free_func_iter_next(&it); + ASSERT(i >= 0); + ASSERT(it.reg == &base); + ASSERT(strcmp(it.reg->funcs[i].qualified_name, "pkg.alpha") == 0); + ASSERT_EQ(cbm_free_func_iter_next(&it), -1); + /* The plain iterator on the overlay alone sees nothing -- unchanged. */ + cbm_registry_free_funcs_by_short_name(&overlay, "alpha", &it); + ASSERT_EQ(cbm_free_func_iter_next(&it), -1); + + /* Copy-on-write: the refinement lands in the overlay, the base is untouched, + * and chained lookup now returns the refined copy. */ + CBMRegisteredFunc *w = cbm_registry_func_for_update(&overlay, "pkg.alpha"); + ASSERT_NOT_NULL(w); + ASSERT(w >= overlay.funcs && w < overlay.funcs + overlay.func_count); + w->min_params = 2; + ASSERT_EQ(base.funcs[0].min_params, -1); + ASSERT_EQ(cbm_registry_lookup_func(&overlay, "pkg.alpha")->min_params, 2); + ASSERT(cbm_registry_func_for_update(&overlay, "pkg.alpha") == + w); /* own entry, no second copy */ + ASSERT_EQ(overlay.func_count, 1); + + /* The base original is shadowed: chained iteration yields exactly one alpha, + * from the overlay. */ + cbm_registry_free_funcs_by_short_name_chain(&overlay, "alpha", &it); + int seen = 0; + while (cbm_free_func_iter_next(&it) >= 0) { + ASSERT(it.reg == &overlay); + seen++; + } + ASSERT_EQ(seen, 1); + /* Linear chain over everything: alpha (overlay copy) + beta (base). */ + cbm_registry_all_funcs_chain(&overlay, &it); + seen = 0; + while (cbm_free_func_iter_next(&it) >= 0) { + seen++; + } + ASSERT_EQ(seen, 2); + + /* Types: same contract. */ + CBMTypeShortIter ti; + cbm_registry_types_by_short_name_chain(&overlay, "T", &ti); + int k = cbm_type_short_iter_next(&ti); + ASSERT(k >= 0); + ASSERT(ti.reg == &base); + CBMRegisteredType *wt = cbm_registry_type_for_update(&overlay, "pkg.T"); + ASSERT_NOT_NULL(wt); + ASSERT_EQ(overlay.type_count, 1); + cbm_registry_all_types_chain(&overlay, &ti); + seen = 0; + while (cbm_type_short_iter_next(&ti) >= 0) { + seen++; + } + ASSERT_EQ(seen, 1); + + /* A sealed head refuses refinement; an unknown QN yields nothing. */ + overlay.read_only = true; + ASSERT(cbm_registry_func_for_update(&overlay, "pkg.beta") == NULL); + overlay.read_only = false; + ASSERT(cbm_registry_func_for_update(&overlay, "pkg.nope") == NULL); + ASSERT_EQ(base.func_count, 2); + + cbm_arena_destroy(&scratch); + cbm_arena_destroy(&arena); + PASS(); +} + /* Direct guard for the type-name / embedded-type / free-function registry * indexes and their iterators (type_registry.c). Verifies that every iterator * preserves ascending registry order, which is part of resolver tie-breaking. */ @@ -16216,6 +16377,8 @@ SUITE(c_lsp) { RUN_TEST(clsp_tier2_shared_registry_readonly_c); RUN_TEST(clsp_tier2_shared_registry_readonly_cpp); RUN_TEST(seal_py_shared_registry_readonly); + RUN_TEST(registry_overlay_chain_iterates_and_copies_on_write); + RUN_TEST(clsp_method_return_refinement_is_copy_on_write); RUN_TEST(seal_py_shared_registry_readonly_fields); RUN_TEST(seal_cs_shared_registry_readonly); RUN_TEST(seal_ts_shared_registry_readonly); diff --git a/tests/test_extraction.c b/tests/test_extraction.c index cb01dcf14..14d75bf8b 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -12,6 +12,8 @@ #include "../src/foundation/compat_fs.h" #include #include "macro_table.h" +#include "result_spill.h" +#include "pipeline/pass_lsp_cross.h" #include "iris_export_xml.h" /* ── Helpers ───────────────────────────────────────────────────── */ @@ -7120,7 +7122,375 @@ TEST(non_config_language_module_has_no_promoted_description_issue519) { PASS(); } +/* ── Result compaction (cbm_result_compact) ────────────────────────────── */ + +static const char *COMPACT_PY_SRC = "import os\n" + "from typing import List\n" + "\n" + "@app.route(\"/items\")\n" + "def list_items(limit: int, offset: int = 0) -> List[str]:\n" + " \"\"\"Return items.\"\"\"\n" + " rows = fetch(limit, offset=offset)\n" + " total = len(rows)\n" + " for r in rows:\n" + " print(r, total)\n" + " return rows\n" + "\n" + "class Store(Base):\n" + " def get(self, key):\n" + " return self.data.get(key, None)\n" + "\n" + " def put(self, key, value):\n" + " self.data[key] = value\n" + " return fetch(key, value)\n"; + +static bool cmp_str_eq(const char *a, const char *b) { + if (!a || !b) { + return a == b; + } + return strcmp(a, b) == 0; +} + +static bool cmp_list_eq(const char **a, const char **b) { + if (!a || !b) { + return a == b; + } + int i = 0; + for (; a[i] && b[i]; i++) { + if (strcmp(a[i], b[i]) != 0) { + return false; + } + } + return a[i] == NULL && b[i] == NULL; +} + +static size_t arena_capacity(const CBMArena *a) { + size_t total = 0; + for (int i = 0; i < a->nblocks; i++) { + total += a->block_sizes[i]; + } + return total; +} + +TEST(extract_compact_keeps_every_field_and_shrinks_the_arena) { + CBMFileResult *r = extract(COMPACT_PY_SRC, CBM_LANG_PYTHON, "t", "svc.py"); + CBMFileResult *ref = extract(COMPACT_PY_SRC, CBM_LANG_PYTHON, "t", "svc.py"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(ref); + ASSERT_FALSE(r->has_error); + ASSERT(r->defs.count >= 4); /* list_items, Store, get, put (+ module) */ + ASSERT(r->calls.count >= 4); /* fetch x2, len, print, get */ + ASSERT(r->usages.count >= 1); + ASSERT(r->imports.count >= 2); + bool saw_args = false; + for (int i = 0; i < ref->calls.count; i++) { + saw_args = saw_args || ref->calls.items[i].arg_count > 0; + } + ASSERT(saw_args); + + size_t used_before = cbm_arena_total(&r->arena); + size_t cap_before = arena_capacity(&r->arena); + cbm_result_compact(r); + + /* One exact block: capacity == bytes used, no dead headroom. */ + ASSERT_EQ(r->arena.nblocks, 1); + ASSERT_EQ(arena_capacity(&r->arena), cbm_arena_total(&r->arena)); + ASSERT(cbm_arena_total(&r->arena) < used_before); + ASSERT(arena_capacity(&r->arena) < cap_before); + ASSERT_EQ(r->defs.cap, r->defs.count); + ASSERT_EQ(r->calls.cap, r->calls.count); + ASSERT_EQ(r->usages.cap, r->usages.count); + + /* Every field survives, by value. */ + ASSERT_EQ(r->defs.count, ref->defs.count); + for (int i = 0; i < r->defs.count; i++) { + const CBMDefinition *a = &r->defs.items[i]; + const CBMDefinition *b = &ref->defs.items[i]; + ASSERT(cmp_str_eq(a->name, b->name)); + ASSERT(cmp_str_eq(a->qualified_name, b->qualified_name)); + ASSERT(cmp_str_eq(a->label, b->label)); + ASSERT(cmp_str_eq(a->file_path, b->file_path)); + ASSERT(cmp_str_eq(a->signature, b->signature)); + ASSERT(cmp_str_eq(a->return_type, b->return_type)); + ASSERT(cmp_str_eq(a->docstring, b->docstring)); + ASSERT(cmp_str_eq(a->parent_class, b->parent_class)); + ASSERT(cmp_str_eq(a->route_path, b->route_path)); + ASSERT(cmp_str_eq(a->body_tokens, b->body_tokens)); + ASSERT(cmp_str_eq(a->structural_profile, b->structural_profile)); + ASSERT(cmp_list_eq(a->decorators, b->decorators)); + ASSERT(cmp_list_eq(a->base_classes, b->base_classes)); + ASSERT(cmp_list_eq(a->param_names, b->param_names)); + ASSERT(cmp_list_eq(a->param_types, b->param_types)); + ASSERT(cmp_list_eq(a->return_types, b->return_types)); + ASSERT_EQ(a->signature_param_count, b->signature_param_count); + for (int k = 0; k < a->signature_param_count; k++) { + ASSERT(cmp_str_eq(a->signature_param_types[k], b->signature_param_types[k])); + } + ASSERT_EQ(a->start_line, b->start_line); + ASSERT_EQ(a->end_line, b->end_line); + ASSERT_EQ(a->complexity, b->complexity); + ASSERT_EQ(a->lines, b->lines); + ASSERT_EQ(a->is_exported, b->is_exported); + ASSERT_EQ(a->fingerprint_k, b->fingerprint_k); + if (a->fingerprint_k > 0) { + ASSERT_NOT_NULL(a->fingerprint); + ASSERT(memcmp(a->fingerprint, b->fingerprint, + (size_t)a->fingerprint_k * sizeof(uint32_t)) == 0); + } + } + ASSERT_EQ(r->calls.count, ref->calls.count); + for (int i = 0; i < r->calls.count; i++) { + const CBMCall *a = &r->calls.items[i]; + const CBMCall *b = &ref->calls.items[i]; + ASSERT(cmp_str_eq(a->callee_name, b->callee_name)); + ASSERT(cmp_str_eq(a->enclosing_func_qn, b->enclosing_func_qn)); + ASSERT(cmp_str_eq(a->first_string_arg, b->first_string_arg)); + ASSERT_EQ(a->arg_count, b->arg_count); + ASSERT_EQ(a->start_line, b->start_line); + ASSERT_EQ(a->site_start_byte, b->site_start_byte); + ASSERT_EQ(a->site_end_byte, b->site_end_byte); + ASSERT_EQ(a->is_method, b->is_method); + for (int k = 0; k < a->arg_count; k++) { + ASSERT(cmp_str_eq(a->args[k].expr, b->args[k].expr)); + ASSERT(cmp_str_eq(a->args[k].value, b->args[k].value)); + ASSERT(cmp_str_eq(a->args[k].keyword, b->args[k].keyword)); + ASSERT_EQ(a->args[k].index, b->args[k].index); + } + } + ASSERT_EQ(r->usages.count, ref->usages.count); + for (int i = 0; i < r->usages.count; i++) { + ASSERT(cmp_str_eq(r->usages.items[i].ref_name, ref->usages.items[i].ref_name)); + ASSERT(cmp_str_eq(r->usages.items[i].enclosing_func_qn, + ref->usages.items[i].enclosing_func_qn)); + ASSERT_EQ(r->usages.items[i].kind, ref->usages.items[i].kind); + ASSERT_EQ(r->usages.items[i].site_start_byte, ref->usages.items[i].site_start_byte); + ASSERT_EQ(r->usages.items[i].is_member_access, ref->usages.items[i].is_member_access); + } + ASSERT_EQ(r->imports.count, ref->imports.count); + for (int i = 0; i < r->imports.count; i++) { + ASSERT(cmp_str_eq(r->imports.items[i].local_name, ref->imports.items[i].local_name)); + ASSERT(cmp_str_eq(r->imports.items[i].module_path, ref->imports.items[i].module_path)); + } + ASSERT_EQ(r->rw.count, ref->rw.count); + ASSERT_EQ(r->type_refs.count, ref->type_refs.count); + ASSERT(cmp_str_eq(r->module_qn, ref->module_qn)); + ASSERT(cmp_list_eq(r->exports, ref->exports)); + + /* Interned by content: two records with the same enclosing QN share one + * copy after compaction. */ + bool shared = false; + for (int i = 0; i < r->calls.count && !shared; i++) { + for (int j = i + 1; j < r->calls.count && !shared; j++) { + if (r->calls.items[i].enclosing_func_qn && r->calls.items[j].enclosing_func_qn && + strcmp(r->calls.items[i].enclosing_func_qn, r->calls.items[j].enclosing_func_qn) == + 0) { + shared = r->calls.items[i].enclosing_func_qn == r->calls.items[j].enclosing_func_qn; + } + } + } + ASSERT(shared); + + /* The arena stays usable for the cross-file pass: growth restarts at the + * default block, never at twice the compact block. */ + char *later = cbm_arena_strdup(&r->arena, "appended after compaction"); + ASSERT_NOT_NULL(later); + ASSERT_EQ(r->arena.nblocks, 2); + ASSERT_EQ(r->arena.block_sizes[1], CBM_ARENA_DEFAULT_BLOCK_SIZE); + + cbm_free_result(r); + cbm_free_result(ref); + PASS(); +} + +TEST(extract_compact_is_idempotent_and_survives_empty_results) { + CBMFileResult *r = extract(COMPACT_PY_SRC, CBM_LANG_PYTHON, "t", "svc.py"); + ASSERT_NOT_NULL(r); + cbm_result_compact(r); + size_t once = cbm_arena_total(&r->arena); + int defs = r->defs.count; + cbm_result_compact(r); + ASSERT_EQ(cbm_arena_total(&r->arena), once); + ASSERT_EQ(r->defs.count, defs); + ASSERT_EQ(r->arena.nblocks, 1); + cbm_free_result(r); + + CBMFileResult *empty = extract("", CBM_LANG_PYTHON, "t", "empty.py"); + ASSERT_NOT_NULL(empty); + cbm_result_compact(empty); + ASSERT_EQ(empty->defs.count + empty->calls.count, empty->defs.count + empty->calls.count); + ASSERT(empty->arena.nblocks >= 1); + cbm_free_result(empty); + + cbm_result_compact(NULL); /* no-op */ + PASS(); +} + +/* ── Result spill (result_spill.c): park -> load is a faithful round trip ── */ + +TEST(extract_spill_round_trip_keeps_every_field) { + CBMFileResult *r = extract(COMPACT_PY_SRC, CBM_LANG_PYTHON, "t", "svc.py"); + CBMFileResult *ref = extract(COMPACT_PY_SRC, CBM_LANG_PYTHON, "t", "svc.py"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(ref); + cbm_result_compact(r); + cbm_result_compact(ref); + + char dir[512]; + snprintf(dir, sizeof(dir), "%s/cbm_spill_XXXXXX", cbm_tmpdir()); + ASSERT_NOT_NULL(cbm_mkdtemp(dir)); + cbm_result_spill_t *sp = cbm_result_spill_open(dir, 2, 3); + ASSERT_NOT_NULL(sp); + ASSERT_FALSE(cbm_result_spill_has(sp, 1)); + + /* A result that is not compacted (two blocks) is refused, untouched. */ + CBMFileResult *raw = extract(COMPACT_PY_SRC, CBM_LANG_PYTHON, "t", "raw.py"); + ASSERT_NOT_NULL(raw); + if (raw->arena.nblocks > 1) { + ASSERT_FALSE(cbm_result_spill_park(sp, 0, 2, raw)); + ASSERT_FALSE(cbm_result_spill_has(sp, 2)); + } + cbm_free_result(raw); + + /* Park frees the in-memory result; the slot is then on disk. Each + * precondition is named so a refusal says which one it was. */ + int defs_before = r->defs.count; + int calls_before = r->calls.count; + ASSERT_EQ(r->arena.nblocks, 1); + ASSERT_EQ(r->owned_result_count, 0); + ASSERT_NOT_NULL(r->cached_tree); /* the extraction helper keeps the tree: park drops it */ + ASSERT_TRUE(cbm_result_spill_park(sp, 1, 1, r)); + r = NULL; + ASSERT_TRUE(cbm_result_spill_has(sp, 1)); + int peek_defs = -1; + int peek_impls = -1; + cbm_result_spill_peek_counts(sp, 1, &peek_defs, &peek_impls); + ASSERT_EQ(peek_defs, defs_before); + ASSERT_EQ(peek_impls, 0); + + CBMFileResult *back = cbm_result_spill_load(sp, 1); + ASSERT_NOT_NULL(back); + ASSERT_NULL(back->cached_tree); + ASSERT_EQ(back->arena.nblocks, 1); + ASSERT_EQ(back->defs.count, defs_before); + ASSERT_EQ(back->calls.count, calls_before); + for (int i = 0; i < back->defs.count; i++) { + const CBMDefinition *a = &back->defs.items[i]; + const CBMDefinition *b = &ref->defs.items[i]; + ASSERT(cmp_str_eq(a->name, b->name)); + ASSERT(cmp_str_eq(a->qualified_name, b->qualified_name)); + ASSERT(cmp_str_eq(a->label, b->label)); + ASSERT(cmp_str_eq(a->file_path, b->file_path)); + ASSERT(cmp_str_eq(a->signature, b->signature)); + ASSERT(cmp_str_eq(a->docstring, b->docstring)); + ASSERT(cmp_list_eq(a->decorators, b->decorators)); + ASSERT(cmp_list_eq(a->param_names, b->param_names)); + ASSERT_EQ(a->start_line, b->start_line); + ASSERT_EQ(a->fingerprint_k, b->fingerprint_k); + if (a->fingerprint_k > 0) { + ASSERT(memcmp(a->fingerprint, b->fingerprint, + (size_t)a->fingerprint_k * sizeof(uint32_t)) == 0); + } + /* Every pointer now lives in the loaded block, none in the old one. */ + const char *lo = back->arena.blocks[0]; + const char *hi = lo + back->arena.used; + ASSERT(a->name >= lo && a->name < hi); + ASSERT(a->qualified_name >= lo && a->qualified_name < hi); + } + for (int i = 0; i < back->calls.count; i++) { + const CBMCall *a = &back->calls.items[i]; + const CBMCall *b = &ref->calls.items[i]; + ASSERT(cmp_str_eq(a->callee_name, b->callee_name)); + ASSERT(cmp_str_eq(a->enclosing_func_qn, b->enclosing_func_qn)); + ASSERT_EQ(a->arg_count, b->arg_count); + for (int k = 0; k < a->arg_count; k++) { + ASSERT(cmp_str_eq(a->args[k].expr, b->args[k].expr)); + } + } + ASSERT_EQ(back->usages.count, ref->usages.count); + for (int i = 0; i < back->usages.count; i++) { + ASSERT(cmp_str_eq(back->usages.items[i].ref_name, ref->usages.items[i].ref_name)); + } + ASSERT(cmp_str_eq(back->module_qn, ref->module_qn)); + ASSERT(cmp_list_eq(back->exports, ref->exports)); + + /* Loading twice yields two independent copies. */ + CBMFileResult *again = cbm_result_spill_load(sp, 1); + ASSERT_NOT_NULL(again); + ASSERT(again->arena.blocks[0] != back->arena.blocks[0]); + ASSERT_EQ(again->defs.count, defs_before); + int64_t parked = 0; + int64_t bytes = 0; + int64_t loads = 0; + cbm_result_spill_stats(sp, &parked, &bytes, &loads); + ASSERT_EQ(parked, 1); + ASSERT(bytes > 0); + ASSERT_EQ(loads, 2); + + cbm_free_result(again); + cbm_free_result(back); + cbm_free_result(ref); + cbm_result_spill_close(sp); + cbm_rmdir(dir); + PASS(); +} + +/* ── LSP budget share: an oversized parse disqualifies the file from the walks ── */ + +/* CBM_TEST_LSP_SKIP_ON names the file (no real timing): the result carries + * lsp_skipped, the per-file LSP walk did not run (no LSP-resolved calls), + * and the shared cross-file dispatcher returns without touching it. The + * unified extractor definitions are still there. */ +TEST(extract_lsp_skipped_when_parse_used_its_budget_share) { + cbm_setenv("CBM_TEST_LSP_SKIP_ON", "budget_share.py", 1); + CBMFileResult *skipped = extract(COMPACT_PY_SRC, CBM_LANG_PYTHON, "t", "budget_share.py"); + cbm_unsetenv("CBM_TEST_LSP_SKIP_ON"); + CBMFileResult *walked = extract(COMPACT_PY_SRC, CBM_LANG_PYTHON, "t", "walked.py"); + ASSERT_NOT_NULL(skipped); + ASSERT_NOT_NULL(walked); + ASSERT_TRUE(skipped->lsp_skipped); + ASSERT_FALSE(walked->lsp_skipped); + ASSERT_GT(skipped->defs.count, 0); /* the unified extractor still ran */ + ASSERT_TRUE(skipped->defs.count <= walked->defs.count); /* the LSP walk adds its own defs */ + ASSERT_EQ(skipped->resolved_calls.count, 0); + + /* The dispatcher is the one gate for every language and both drivers. */ + int calls_before = skipped->calls.count; + cbm_pxc_dispatch_file(CBM_LANG_PYTHON, skipped, COMPACT_PY_SRC, (int)strlen(COMPACT_PY_SRC), + "budget_share.py", "t", NULL, NULL, NULL, 0, NULL, NULL, 0, NULL, NULL); + ASSERT_EQ(skipped->calls.count, calls_before); + ASSERT_EQ(skipped->resolved_calls.count, 0); + + cbm_free_result(skipped); + cbm_free_result(walked); + PASS(); +} + +/* CBM_TEST_WALK_BUDGET_NODES stops the unified walk after that many nodes + * (no real timing): the result is walk_truncated, therefore lsp_skipped, and + * the definitions the walk had not reached are the only loss. */ +TEST(extract_walk_truncated_at_its_cpu_budget) { + cbm_setenv("CBM_TEST_WALK_BUDGET_NODES", "8", 1); + CBMFileResult *cut = extract(COMPACT_PY_SRC, CBM_LANG_PYTHON, "t", "walk_budget.py"); + cbm_unsetenv("CBM_TEST_WALK_BUDGET_NODES"); + CBMFileResult *full = extract(COMPACT_PY_SRC, CBM_LANG_PYTHON, "t", "walk_full.py"); + ASSERT_NOT_NULL(cut); + ASSERT_NOT_NULL(full); + ASSERT_TRUE(cut->walk_truncated); + ASSERT_TRUE(cut->lsp_skipped); + ASSERT_FALSE(full->walk_truncated); + ASSERT_TRUE(cut->usages.count <= full->usages.count); + ASSERT_TRUE(cut->calls.count <= full->calls.count); + cbm_free_result(cut); + cbm_free_result(full); + PASS(); +} + SUITE(extraction) { + RUN_TEST(extract_compact_keeps_every_field_and_shrinks_the_arena); + RUN_TEST(extract_compact_is_idempotent_and_survives_empty_results); + RUN_TEST(extract_spill_round_trip_keeps_every_field); + RUN_TEST(extract_lsp_skipped_when_parse_used_its_budget_share); + RUN_TEST(extract_walk_truncated_at_its_cpu_budget); /* Initialize extraction library */ cbm_init(); diff --git a/tests/test_mem.c b/tests/test_mem.c index 5ac8abe43..a943282e4 100644 --- a/tests/test_mem.c +++ b/tests/test_mem.c @@ -6,6 +6,7 @@ #include "test_framework.h" #include "test_helpers.h" #include "../src/foundation/mem.h" +#include "../src/foundation/platform.h" /* cbm_system_info, cbm_system_available_ram */ #include "../src/foundation/mem_core.h" #include "../src/foundation/arena.h" #include "../src/foundation/slab_alloc.h" @@ -1486,8 +1487,82 @@ TEST(mem_core_report_json_is_wellformed_or_empty) { PASS(); } +/* ── Pressure primitives and the charged reading ─────────────────────── */ + +TEST(mem_charged_is_positive_and_consistent_with_rss) { + size_t charged = cbm_mem_charged(); + size_t rss = cbm_mem_rss(); + ASSERT(charged > 0); + ASSERT(rss > 0); + /* Same order of magnitude as RSS on every platform: the charged value + * may sit below RSS (purged-but-resident pages) or slightly above it + * (compressed pages), never at zero or at a multiple. */ + ASSERT(charged < rss * 4); + ASSERT(rss < charged * 4 + (size_t)64 * 1024 * 1024); + PASS(); +} + +/* The charged high-water mark never reads below a charge just taken, and a + * later, larger charge lifts it: it is a max over every reading. */ +TEST(mem_peak_charged_is_the_high_water_of_charged) { + size_t charged = cbm_mem_charged(); + ASSERT_TRUE(charged > 0); + ASSERT_TRUE(cbm_mem_peak_charged() >= charged); + size_t peak_before = cbm_mem_peak_charged(); + (void)cbm_mem_charged(); + ASSERT_TRUE(cbm_mem_peak_charged() >= peak_before); + PASS(); +} + +TEST(mem_footprint_zero_or_plausible) { + size_t fp = cbm_mem_footprint(); + if (fp > 0) { + ASSERT(fp >= (size_t)1024 * 1024); /* a live test process is more than 1 MB */ + } + PASS(); +} + +TEST(mem_system_available_ram_is_within_total) { + size_t avail = cbm_system_available_ram(); + cbm_system_info_t info = cbm_system_info(); + if (avail == 0 || info.total_ram == 0) { + PASS(); /* platform cannot answer; the caller treats that as unknown */ + } + ASSERT(avail <= info.total_ram); + PASS(); +} + +TEST(mem_system_under_pressure_is_a_pure_threshold) { + size_t avail = cbm_system_available_ram(); + cbm_system_info_t info = cbm_system_info(); + bool under = cbm_mem_system_under_pressure(); + if (avail == 0 || info.total_ram == 0) { + ASSERT_FALSE(under); /* never abort on a guess */ + PASS(); + } + ASSERT_EQ(under, avail < info.total_ram / 8); + PASS(); +} + +TEST(mem_over_budget_follows_the_charged_reading) { + size_t saved = cbm_mem_budget(); + size_t charged = cbm_mem_charged(); + ASSERT(charged > 0); + cbm_mem_set_budget_for_tests(charged * 4); + ASSERT_FALSE(cbm_mem_over_budget()); + cbm_mem_set_budget_for_tests(charged / 4 + 1); + ASSERT_TRUE(cbm_mem_over_budget()); + cbm_mem_set_budget_for_tests(saved); + PASS(); +} + TEST(mem_core_class_names_are_total) { ASSERT_TRUE(strcmp(cbm_mem_class_name(CBM_MEM_CLASS_SEMANTIC), "semantic") == 0); + ASSERT_TRUE(strcmp(cbm_mem_class_name(CBM_MEM_CLASS_ARENA), "arena") == 0); + ASSERT_TRUE(strcmp(cbm_mem_class_name(CBM_MEM_CLASS_TS_TREE), "ts_tree") == 0); + ASSERT_TRUE(strcmp(cbm_mem_class_name(CBM_MEM_CLASS_STORE), "store") == 0); + ASSERT_TRUE(strcmp(cbm_mem_class_name(CBM_MEM_CLASS_HASH_TABLE), "hash_table") == 0); + ASSERT_TRUE(strcmp(cbm_mem_class_name(CBM_MEM_CLASS_DYN_ARRAY), "dyn_array") == 0); /* Out of range must still answer, so a log line never takes a NULL. */ ASSERT_TRUE(cbm_mem_class_name((cbm_mem_class_t)(CBM_MEM_CLASS_COUNT + 5)) != NULL); ASSERT_TRUE(cbm_mem_class_name((cbm_mem_class_t)-1) != NULL); @@ -1570,4 +1645,10 @@ SUITE(mem) { RUN_TEST(mem_core_mismatched_class_never_wraps); RUN_TEST(mem_core_report_json_is_wellformed_or_empty); RUN_TEST(mem_core_class_names_are_total); + RUN_TEST(mem_charged_is_positive_and_consistent_with_rss); + RUN_TEST(mem_peak_charged_is_the_high_water_of_charged); + RUN_TEST(mem_footprint_zero_or_plausible); + RUN_TEST(mem_system_available_ram_is_within_total); + RUN_TEST(mem_system_under_pressure_is_a_pure_threshold); + RUN_TEST(mem_over_budget_follows_the_charged_reading); } diff --git a/tests/test_parallel.c b/tests/test_parallel.c index 631446f9c..13477b67c 100644 --- a/tests/test_parallel.c +++ b/tests/test_parallel.c @@ -19,6 +19,7 @@ #include "foundation/platform.h" #include "foundation/log.h" #include "cbm.h" +#include "result_spill.h" #include #include @@ -271,6 +272,11 @@ static cbm_gbuf_t *run_sequential_with_lsp_cross(const char *project, const char /* ── Run parallel pipeline on files, returning gbuf ───────────────── */ +/* Spill mode for the harness: the run below opts its context in (the way + * run_parallel_pipeline does) and records how many results the store parked. */ +static bool g_harness_spill = false; +static int64_t g_harness_parked = -1; + static cbm_gbuf_t *run_parallel_with_extract_opts_and_mutator( const char *project, const char *repo_path, cbm_file_info_t *files, int file_count, int worker_count, const cbm_parallel_extract_opts_t *extract_opts, @@ -286,6 +292,7 @@ static cbm_gbuf_t *run_parallel_with_extract_opts_and_mutator( .gbuf = gbuf, .registry = reg, .cancelled = &cancelled, + .spill_allowed = g_harness_spill, }; if (seed_structure) { @@ -306,6 +313,14 @@ static cbm_gbuf_t *run_parallel_with_extract_opts_and_mutator( cbm_parallel_extract(&ctx, files, file_count, result_cache, &shared_ids, worker_count); } cbm_gbuf_set_next_id(gbuf, atomic_load(&shared_ids)); + if (g_harness_spill) { + int64_t bytes = 0; + int64_t loads = 0; + g_harness_parked = -1; + if (ctx.spill) { + cbm_result_spill_stats(ctx.spill, &g_harness_parked, &bytes, &loads); + } + } if (mutator) { mutator(result_cache, file_count, mutator_ud); @@ -323,8 +338,10 @@ static cbm_gbuf_t *run_parallel_with_extract_opts_and_mutator( * cbm_pxc_run_one(_ts) per file BEFORE materializing CALLS edges. */ char **def_modules = (char **)calloc((size_t)file_count, sizeof(char *)); int def_count = 0; + CBMArena cross_arena; + cbm_arena_init(&cross_arena); CBMLSPDef *all_defs = - def_modules ? cbm_pxc_collect_all_defs(&ctx, result_cache, files, file_count, + def_modules ? cbm_pxc_collect_all_defs(&ctx, &cross_arena, result_cache, files, file_count, ctx.project_name, def_modules, &def_count, NULL) : NULL; CBMModuleDefIndex *module_def_index = @@ -337,6 +354,7 @@ static cbm_gbuf_t *run_parallel_with_extract_opts_and_mutator( cbm_pxc_free_module_def_index(module_def_index); free(all_defs); + cbm_arena_destroy(&cross_arena); if (def_modules) { for (int i = 0; i < file_count; i++) { free(def_modules[i]); @@ -348,6 +366,7 @@ static cbm_gbuf_t *run_parallel_with_extract_opts_and_mutator( if (result_cache[i]) cbm_free_result(result_cache[i]); free(result_cache); + cbm_pipeline_spill_close(&ctx); harness_ctx_free_tables(&ctx); cbm_registry_free(reg); @@ -528,6 +547,48 @@ TEST(parallel_total_edges) { PASS(); } +/* ── Spill mode: the graph is the in-memory graph ─────────────────── */ + +/* CBM_MEM_SPILL=1 parks every compacted result on disk the moment it is + * extracted; registry build, def collection, surfaces, resolve and the infra + * passes read each one back only for the moment they need it. The graph must + * not be able to tell: same nodes, same edges per type as the in-memory run + * of the same repo -- and every file must actually have gone through the + * store (a store that failed to open would silently test nothing). */ +TEST(parallel_spill_mode_builds_the_same_graph) { + if (ensure_parity_setup() != 0) + FAIL("setup failed"); + cbm_discover_opts_t opts = {.mode = CBM_MODE_FULL}; + cbm_file_info_t *files = NULL; + int file_count = 0; + ASSERT_EQ(cbm_discover(g_par_tmpdir, &opts, &files, &file_count), 0); + ASSERT_GT(file_count, 0); + + cbm_setenv("CBM_MEM_SPILL", "1", 1); + g_harness_spill = true; + cbm_gbuf_t *spilled = run_parallel("par-test", g_par_tmpdir, files, file_count, 2); + g_harness_spill = false; + cbm_unsetenv("CBM_MEM_SPILL"); + cbm_discover_free(files, file_count); + ASSERT(spilled != NULL); + + ASSERT_EQ((int)g_harness_parked, file_count); + ASSERT_EQ(cbm_gbuf_node_count(spilled), cbm_gbuf_node_count(g_par_gbuf)); + ASSERT_EQ(cbm_gbuf_edge_count(spilled), cbm_gbuf_edge_count(g_par_gbuf)); + static const char *const types[] = {"CALLS", "DEFINES", "DEFINES_METHOD", "IMPORTS", + "USES", "INHERITS", "IMPLEMENTS"}; + for (size_t t = 0; t < sizeof(types) / sizeof(types[0]); t++) { + int in_memory = cbm_gbuf_edge_count_by_type(g_par_gbuf, types[t]); + int on_disk = cbm_gbuf_edge_count_by_type(spilled, types[t]); + if (in_memory != on_disk) { + printf(" FAIL: %s edges: in_memory=%d spilled=%d\n", types[t], in_memory, on_disk); + } + ASSERT_EQ(in_memory, on_disk); + } + cbm_gbuf_free(spilled); + PASS(); +} + /* ── Empty file list ──────────────────────────────────────────────── */ TEST(parallel_empty_files) { @@ -2583,10 +2644,9 @@ TEST(parallel_kotlin_nonbinary_operator_carriers_reach_graph) { * the test green. The optional crate_b-local helper pins the second failure * mode where a confident in-file resolution used to suppress cross-LSP. */ static cbm_gbuf_t *run_issue56_parallel_workspace(bool local_decoy) { - static const char workspace_toml[] = - "[workspace]\n" - "members = [\"crate_a\", \"crate_b\"]\n" - "resolver = \"2\"\n"; + static const char workspace_toml[] = "[workspace]\n" + "members = [\"crate_a\", \"crate_b\"]\n" + "resolver = \"2\"\n"; static const char crate_a_toml[] = "[package]\nname = \"crate_a\"\nversion = \"0.1.0\"\nedition = \"2021\"\n"; static const char crate_b_toml[] = @@ -2594,13 +2654,11 @@ static cbm_gbuf_t *run_issue56_parallel_workspace(bool local_decoy) { "\n[dependencies]\ncrate_a = { path = \"../crate_a\" }\n"; static const char crate_a_source[] = "pub fn helper() {}\n"; static const char unlisted_source[] = "pub fn helper() {}\n"; - static const char caller_without_local[] = - "fn run() { crate_a::helper(); }\n" - "fn main() { run(); }\n"; - static const char caller_with_local[] = - "fn helper() {}\n" - "fn run() { crate_a::helper(); }\n" - "fn main() { run(); }\n"; + static const char caller_without_local[] = "fn run() { crate_a::helper(); }\n" + "fn main() { run(); }\n"; + static const char caller_with_local[] = "fn helper() {}\n" + "fn run() { crate_a::helper(); }\n" + "fn main() { run(); }\n"; char tmpdir[256]; snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_issue56_parallel_XXXXXX"); @@ -2669,13 +2727,12 @@ TEST(parallel_rust_cross_crate_worker_receives_workspace_manifest) { const cbm_gbuf_edge_t *correct = find_call_edge_to_target_fragment(gbuf, "main.run", ".crate_a."); const bool correct_found = correct != NULL; - const bool unlisted = - callable_has_call_target_fragment(gbuf, "main.run", ".unlisted."); + const bool unlisted = callable_has_call_target_fragment(gbuf, "main.run", ".unlisted."); const bool manifest_strategy = correct && correct->properties_json && strstr(correct->properties_json, "lsp_cross_crate"); if (!correct || unlisted || !manifest_strategy) { - printf(" issue56 manifest diagnostic: correct=%d unlisted=%d strategy=%d\n", - correct_found, unlisted, manifest_strategy); + printf(" issue56 manifest diagnostic: correct=%d unlisted=%d strategy=%d\n", correct_found, + unlisted, manifest_strategy); } cbm_gbuf_free(gbuf); @@ -2692,8 +2749,7 @@ TEST(parallel_rust_cross_crate_manifest_beats_confident_local_resolution) { const cbm_gbuf_edge_t *correct = find_call_edge_to_target_fragment(gbuf, "main.run", ".crate_a."); const bool correct_found = correct != NULL; - const bool wrong_local = - callable_has_call_target_fragment(gbuf, "main.run", ".crate_b."); + const bool wrong_local = callable_has_call_target_fragment(gbuf, "main.run", ".crate_b."); const bool manifest_strategy = correct && correct->properties_json && strstr(correct->properties_json, "lsp_cross_crate"); if (!correct || wrong_local || !manifest_strategy) { @@ -3259,7 +3315,8 @@ static cbm_gbuf_t *run_go_field_chain_sequential(const char *project, const char r->defs.count, r->imports.count, r->calls.count, r->resolved_calls.count); for (int j = 0; j < r->resolved_calls.count; j++) { const CBMResolvedCall *rc = &r->resolved_calls.items[j]; - printf(" [diag] rc caller=%s callee=%s strategy=%s conf=%.2f kind=%d span=[%u,%u)\n", + printf(" [diag] rc caller=%s callee=%s strategy=%s conf=%.2f kind=%d " + "span=[%u,%u)\n", rc->caller_qn ? rc->caller_qn : "?", rc->callee_qn ? rc->callee_qn : "?", rc->strategy ? rc->strategy : "?", rc->confidence, (int)rc->kind, (unsigned)rc->site_start_byte, (unsigned)rc->site_end_byte); @@ -3267,13 +3324,16 @@ static cbm_gbuf_t *run_go_field_chain_sequential(const char *project, const char for (int j = 0; j < r->calls.count; j++) { const CBMCall *c = &r->calls.items[j]; printf(" [diag] call callee=%s enclosing=%s span=[%u,%u) req=%d\n", - c->callee_name ? c->callee_name : "?", c->enclosing_func_qn ? c->enclosing_func_qn : "?", - (unsigned)c->site_start_byte, (unsigned)c->site_end_byte, (int)c->requires_lsp_resolution); + c->callee_name ? c->callee_name : "?", + c->enclosing_func_qn ? c->enclosing_func_qn : "?", + (unsigned)c->site_start_byte, (unsigned)c->site_end_byte, + (int)c->requires_lsp_resolution); } for (int j = 0; j < r->defs.count; j++) { const CBMDefinition *d = &r->defs.items[j]; - printf(" [diag] def label=%s qn=%s parent=%s ret=%s\n", d->label ? d->label : "?", - d->qualified_name ? d->qualified_name : "?", d->parent_class ? d->parent_class : "?", + printf(" [diag] def label=%s qn=%s parent=%s ret=%s\n", + d->label ? d->label : "?", d->qualified_name ? d->qualified_name : "?", + d->parent_class ? d->parent_class : "?", d->return_type ? d->return_type : "?"); } } @@ -3324,11 +3384,13 @@ TEST(parallel_go_cross_package_field_chain_resolves) { "\n" "type OrderService struct{}\n" "\n" - "func (s *OrderService) PlaceOrder(ctx context.Context, req *pb.PlaceOrderReq) (*pb.PlaceOrderRsp, error) {\n" + "func (s *OrderService) PlaceOrder(ctx context.Context, req " + "*pb.PlaceOrderReq) (*pb.PlaceOrderRsp, error) {\n" " return nil, nil\n" "}\n" "\n" - "func (s *OrderService) ListOrders(ctx context.Context, req *pb.ListOrdersReq) (*pb.ListOrdersRsp, error) {\n" + "func (s *OrderService) ListOrders(ctx context.Context, req " + "*pb.ListOrdersReq) (*pb.ListOrdersRsp, error) {\n" " return nil, nil\n" "}\n") != 0 || th_write_file(app_path, "package handler\n" @@ -3353,15 +3415,18 @@ TEST(parallel_go_cross_package_field_chain_resolves) { " searchSvc *service.SearchService\n" "}\n" "\n" - "func (h *OrderHandler) ListOrders(ctx context.Context, req *pb.ListOrdersReq) (*pb.ListOrdersRsp, error) {\n" + "func (h *OrderHandler) ListOrders(ctx context.Context, req " + "*pb.ListOrdersReq) (*pb.ListOrdersRsp, error) {\n" " return h.orderSvc.ListOrders(ctx, req)\n" "}\n" "\n" - "func (h *OrderHandler) PlaceOrder(ctx context.Context, req *pb.PlaceOrderReq) (*pb.PlaceOrderRsp, error) {\n" + "func (h *OrderHandler) PlaceOrder(ctx context.Context, req " + "*pb.PlaceOrderReq) (*pb.PlaceOrderRsp, error) {\n" " return h.orderSvc.PlaceOrder(ctx, req)\n" "}\n" "\n" - "func (h *OrderHandler) UpdateCart(ctx context.Context, req *pb.UpdateCartReq) (*pb.UpdateCartRsp, error) {\n" + "func (h *OrderHandler) UpdateCart(ctx context.Context, req " + "*pb.UpdateCartReq) (*pb.UpdateCartRsp, error) {\n" " return h.cartSvc.UpdateCart(ctx, req)\n" "}\n") != 0) { th_rmtree(tmpdir); @@ -3379,11 +3444,11 @@ TEST(parallel_go_cross_package_field_chain_resolves) { cbm_gbuf_t *gbuf = run_go_field_chain_sequential("go_field_fold", tmpdir, files, 2); ASSERT_NOT_NULL(gbuf); - const cbm_gbuf_edge_t *edge = find_call_edge_to_target_fragment(gbuf, "handler.PlaceOrder", ".service.PlaceOrder"); + const cbm_gbuf_edge_t *edge = + find_call_edge_to_target_fragment(gbuf, "handler.PlaceOrder", ".service.PlaceOrder"); const bool found = edge != NULL; - const bool dispatch = - edge && edge->properties_json && - strstr(edge->properties_json, "\"strategy\":\"lsp_type_dispatch\""); + const bool dispatch = edge && edge->properties_json && + strstr(edge->properties_json, "\"strategy\":\"lsp_type_dispatch\""); if (!found || !dispatch) { printf(" go field chain diagnostic: found=%d dispatch=%d\n", found, dispatch); if (edge && edge->properties_json) { @@ -3414,8 +3479,7 @@ TEST(parallel_go_cross_package_field_chain_resolves) { all[i] ? cbm_gbuf_find_by_id(gbuf, all[i]->source_id) : NULL; const cbm_gbuf_node_t *dst = all[i] ? cbm_gbuf_find_by_id(gbuf, all[i]->target_id) : NULL; - printf(" CALLS edge: %s -> %s props=%s\n", - src ? src->qualified_name : "?", + printf(" CALLS edge: %s -> %s props=%s\n", src ? src->qualified_name : "?", dst ? dst->qualified_name : "?", all[i]->properties_json ? all[i]->properties_json : "{}"); } @@ -3797,8 +3861,8 @@ TEST(lsp_resolve_project_prefixed_duplicate_is_not_ambiguous) { const char *project = "proj"; cbm_gbuf_t *gbuf = cbm_gbuf_new(project, "/tmp"); ASSERT_NOT_NULL(gbuf); - int64_t target_id = cbm_gbuf_upsert_node(gbuf, "Function", "handler", - "proj.mod.Target.handler", "target.py", 1, 1, "{}"); + int64_t target_id = cbm_gbuf_upsert_node(gbuf, "Function", "handler", "proj.mod.Target.handler", + "target.py", 1, 1, "{}"); ASSERT_GT(target_id, 0); CBMCall call = make_call("proj.mod.Caller.run", "handler"); @@ -3807,8 +3871,7 @@ TEST(lsp_resolve_project_prefixed_duplicate_is_not_ambiguous) { CBMResolvedCall raw = make_rc("proj.mod.Caller.run", "mod.Target.handler", 0.75f); raw.site_start_byte = call.site_start_byte; raw.site_end_byte = call.site_end_byte; - CBMResolvedCall prefixed = - make_rc("proj.mod.Caller.run", "proj.mod.Target.handler", 0.90f); + CBMResolvedCall prefixed = make_rc("proj.mod.Caller.run", "proj.mod.Target.handler", 0.90f); prefixed.site_start_byte = call.site_start_byte; prefixed.site_end_byte = call.site_end_byte; CBMResolvedCall items[] = {raw, prefixed}; @@ -3827,9 +3890,8 @@ TEST(lsp_resolve_project_prefix_spellings_stay_ambiguous_when_both_nodes_exist) ASSERT_NOT_NULL(gbuf); int64_t raw_id = cbm_gbuf_upsert_node(gbuf, "Function", "handler", "mod.Target.handler", "raw.py", 1, 1, "{}"); - int64_t prefixed_id = cbm_gbuf_upsert_node(gbuf, "Function", "handler", - "proj.mod.Target.handler", "prefixed.py", 1, 1, - "{}"); + int64_t prefixed_id = cbm_gbuf_upsert_node( + gbuf, "Function", "handler", "proj.mod.Target.handler", "prefixed.py", 1, 1, "{}"); ASSERT_GT(raw_id, 0); ASSERT_GT(prefixed_id, 0); ASSERT(raw_id != prefixed_id); @@ -3840,8 +3902,7 @@ TEST(lsp_resolve_project_prefix_spellings_stay_ambiguous_when_both_nodes_exist) CBMResolvedCall raw = make_rc("proj.mod.Caller.run", "mod.Target.handler", 0.75f); raw.site_start_byte = call.site_start_byte; raw.site_end_byte = call.site_end_byte; - CBMResolvedCall prefixed = - make_rc("proj.mod.Caller.run", "proj.mod.Target.handler", 0.90f); + CBMResolvedCall prefixed = make_rc("proj.mod.Caller.run", "proj.mod.Target.handler", 0.90f); prefixed.site_start_byte = call.site_start_byte; prefixed.site_end_byte = call.site_end_byte; CBMResolvedCall items[] = {raw, prefixed}; @@ -4348,6 +4409,7 @@ SUITE(parallel) { RUN_TEST(parallel_implements_parity); RUN_TEST(parallel_semantic_fixture_expected_counts); RUN_TEST(parallel_total_edges); + RUN_TEST(parallel_spill_mode_builds_the_same_graph); RUN_TEST(parallel_empty_files); RUN_TEST(parallel_args_json_no_overflow); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 62c7104b8..565c72719 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -14269,6 +14269,151 @@ TEST(pipeline_markdown_and_config_prose_reaches_fts_body) { * attribute, so the sanitizer is silent there. The label counts pin the * fixture to what it claims: at least one Struct and zero Function/Method, * i.e. the scan genuinely finds nothing to sort. */ +/* ── Semantic pass, batched == unbatched ─────────────────────────────── */ + +/* Under memory pressure the semantic pass tokenizes, counts and vectorizes + * in headroom-sized batches of functions (tokenizing twice) instead of + * holding every function's tokens at once. The graph must not be able to + * tell: the same repo indexed with CBM_SEM_BATCH=5 (forced batches, 30 + * functions -> 6 batches) yields byte-identical node vectors, token vectors + * and SEMANTICALLY_RELATED edges. Rows are compared by qualified name, never + * by node id: parallel extraction hands out ids in worker order. */ +static void write_sem_family(const char *base, const char *file, const char *subject) { + char path[512]; + snprintf(path, sizeof(path), "%s/%s", base, file); + char body[4096]; + snprintf(body, sizeof(body), + "package main\n\n" + "// Parse%sConfig reads the %s config file and returns the parsed %s config.\n" + "func Parse%sConfig(path string) (*%sConfig, error) {\n" + "\treturn Load%sConfig(path)\n}\n\n" + "// Load%sConfig loads the %s config from disk and validates it.\n" + "func Load%sConfig(path string) (*%sConfig, error) {\n" + "\tcfg := &%sConfig{}\n\tValidate%sConfig(cfg)\n\treturn cfg, nil\n}\n\n" + "// Validate%sConfig checks the %s config for missing fields.\n" + "func Validate%sConfig(cfg *%sConfig) bool {\n\treturn cfg != nil\n}\n\n" + "// Write%sConfig serializes the %s config back to disk.\n" + "func Write%sConfig(path string, cfg *%sConfig) error {\n" + "\tValidate%sConfig(cfg)\n\treturn nil\n}\n\n" + /* A near-duplicate of Load: same doc, same body, same calls -- the + * pair the pass must relate, in every venue. */ + "// Load%sConfigFile loads the %s config from disk and validates it.\n" + "func Load%sConfigFile(path string) (*%sConfig, error) {\n" + "\tcfg := &%sConfig{}\n\tValidate%sConfig(cfg)\n\treturn cfg, nil\n}\n\n" + "type %sConfig struct{ Name string }\n", + subject, subject, subject, subject, subject, subject, subject, subject, subject, + subject, subject, subject, subject, subject, subject, subject, subject, subject, + subject, subject, subject, subject, subject, subject, subject, subject, subject, + subject); + th_write_file(path, body); +} + +/* Step both statements in lockstep; every column of every row must match. */ +static bool sem_same_rows(sqlite3 *a, sqlite3 *b, const char *sql, int *rows) { + sqlite3_stmt *sa = NULL; + sqlite3_stmt *sb = NULL; + bool same = sqlite3_prepare_v2(a, sql, -1, &sa, NULL) == SQLITE_OK && + sqlite3_prepare_v2(b, sql, -1, &sb, NULL) == SQLITE_OK; + *rows = 0; + while (same) { + int ra = sqlite3_step(sa); + int rb = sqlite3_step(sb); + if (ra != rb) { + same = false; + break; + } + if (ra != SQLITE_ROW) { + break; + } + int cols = sqlite3_column_count(sa); + for (int c = 0; c < cols && same; c++) { + const unsigned char *va = sqlite3_column_text(sa, c); + const unsigned char *vb = sqlite3_column_text(sb, c); + same = (va == NULL) == (vb == NULL) && + (!va || strcmp((const char *)va, (const char *)vb) == 0); + } + (*rows)++; + } + sqlite3_finalize(sa); + sqlite3_finalize(sb); + return same; +} + +TEST(pipeline_semantic_batched_matches_unbatched) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_sembatch_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + static const char *const subjects[] = {"User", "Server", "Client", "Cache", "Queue", "Mail"}; + for (size_t i = 0; i < sizeof(subjects) / sizeof(subjects[0]); i++) { + char file[64]; + snprintf(file, sizeof(file), "%s_config.go", subjects[i]); + write_sem_family(tmp, file, subjects[i]); + } + + char db_plain[512]; + char db_batched[512]; + snprintf(db_plain, sizeof(db_plain), "%s/plain.db", tmp); + snprintf(db_batched, sizeof(db_batched), "%s/batched.db", tmp); + + /* The default threshold (0.75) admits no pair on a 30-function fixture; + * 0.3 admits the near-duplicates. The test is about equality, not the bar. */ + cbm_setenv("CBM_SEMANTIC_THRESHOLD", "0.3", 1); + cbm_unsetenv("CBM_SEM_BATCH"); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_plain, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_pipeline_free(p); + + p = cbm_pipeline_new(tmp, db_batched, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + cbm_setenv("CBM_SEM_BATCH", "5", 1); /* read by the pass, once per run */ + int rc = cbm_pipeline_run(p); + cbm_unsetenv("CBM_SEM_BATCH"); + cbm_unsetenv("CBM_SEMANTIC_THRESHOLD"); + cbm_pipeline_free(p); + ASSERT_EQ(rc, 0); + + cbm_store_t *sp = cbm_store_open_path(db_plain); + cbm_store_t *sb = cbm_store_open_path(db_batched); + ASSERT_NOT_NULL(sp); + ASSERT_NOT_NULL(sb); + sqlite3 *a = cbm_store_get_db(sp); + sqlite3 *b = cbm_store_get_db(sb); + + /* The fixture must exercise batching: more functions than the forced + * batch, and a semantic pass that actually stored vectors and edges. */ + sqlite3_stmt *st = NULL; + ASSERT_EQ( + sqlite3_prepare_v2(a, "SELECT COUNT(*) FROM nodes WHERE label = 'Function'", -1, &st, NULL), + SQLITE_OK); + ASSERT_EQ(sqlite3_step(st), SQLITE_ROW); + int functions = sqlite3_column_int(st, 0); + sqlite3_finalize(st); + ASSERT_GT(functions, 5); + + int rows = 0; + ASSERT_TRUE(sem_same_rows(a, b, + "SELECT n.qualified_name, hex(v.vector) FROM node_vectors v " + "JOIN nodes n ON n.id = v.node_id ORDER BY n.qualified_name", + &rows)); + ASSERT_EQ(rows, functions); + ASSERT_TRUE(sem_same_rows( + a, b, "SELECT token, hex(vector), idf FROM token_vectors ORDER BY token", &rows)); + ASSERT_GT(rows, 0); + ASSERT_TRUE( + sem_same_rows(a, b, + "SELECT s.qualified_name, t.qualified_name, e.properties FROM edges e " + "JOIN nodes s ON s.id = e.source_id JOIN nodes t ON t.id = e.target_id " + "WHERE e.type = 'SEMANTICALLY_RELATED' ORDER BY 1, 2", + &rows)); + ASSERT_GT(rows, 0); + + cbm_store_close(sp); + cbm_store_close(sb); + th_rmtree(tmp); + PASS(); +} + TEST(pipeline_semantic_edges_no_functions) { char tmp[256]; snprintf(tmp, sizeof(tmp), "/tmp/cbm_nofunc_XXXXXX"); @@ -14730,6 +14875,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_delta_patch_indexes_docstring_into_fts_body); RUN_TEST(pipeline_markdown_and_config_prose_reaches_fts_body); RUN_TEST(pipeline_semantic_edges_no_functions); + RUN_TEST(pipeline_semantic_batched_matches_unbatched); } /* Focused semantic-manifest and publication contracts. Kept separate from the From 3b7f2559213ee80f5cfe0e2329c418211e09749e Mon Sep 17 00:00:00 2001 From: DavidHLP Date: Mon, 14 Sep 2026 18:32:09 +0800 Subject: [PATCH 074/130] fix(cli): ignore absent system aliases during activation walk Signed-off-by: DavidHLP --- src/cli/activation_transaction.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/activation_transaction.c b/src/cli/activation_transaction.c index dcdb4a20c..a56052775 100644 --- a/src/cli/activation_transaction.c +++ b/src/cli/activation_transaction.c @@ -910,7 +910,7 @@ static char *activation_posix_walk_path(const char *directory) { struct stat alias_status; char resolved[4096]; if (lstat(alias, &alias_status) != 0) { - return NULL; + continue; } if (!S_ISLNK(alias_status.st_mode)) { continue; From a2f29b3fe73927d547fed0fe2e0f20d4738ad7c2 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 14 Sep 2026 19:17:40 +0200 Subject: [PATCH 075/130] fix(mem): the 15 GB budget is met by its own metric, end to end The 15 GB kernel run completed with every phase mark under budget, but the budget metric itself -- the charged footprint the extract gate reads on every file pull -- reached 16.8 GB. Two defects, found with a probe at the gate (one line per 256 MB of charge after the spill latch, with the class table; mem.charge.probe, CBM_MEM_PHASES=1, stays in). Worker graph buffers kept a dense id array over the global id space. The charge climbed gradually from the 14.06 GB latch to 14.96 GB, then jumped about 1 GB between two consecutive gate reads, and the only class that moved in that step was gbuf_index. Worker buffers draw ids from the shared counter, so each of the 18 kept an id -> node array spanning the whole global id space -- 8.5M ids x 8 B, next power of two -- almost entirely empty, and when the global id crossed 2^23 all eighteen doubled in the same instant. Nothing asks a worker buffer by id before the merge (every cbm_gbuf_find_by_id caller runs on the main buffer); the worker constructor now drops the array the way it already drops the five secondary indexes, and the main buffer answers by id after the merge. Spill mode did not park what was already cached. The sweeps ran only on an over-budget observation; a run that latched early (near_budget) and then stayed under budget through extraction -- which is exactly what the first fix produced: 14,949 MB at the extract mark -- carried the 44,797 results cached before the latch (8 GB) into registry build and resolve, which cannot park, and resolve ran at a charged 21.8 GB against the 15 GB budget with nothing to stop it. The earlier runs only fitted because the id-array burst tripped the sweeps by accident. Extraction now ends with a final sweep whenever spill mode is on: results belong on disk before the phases that cannot park inherit them. Kernel, CBM_MEM_BUDGET_MB=15000, before -> after: charged high-water 16,771-16,818 MB -> 14,582 and 14,732 MB in two runs (97-98% of the budget); the final sweep parks ~45k results and takes the charge from 14.6 to 4.3 GB before resolve, which peaks at 11.5 GB committed; wall 234 -> 224-230 s; nodes identical (8,529,729), edges inside the run-to-run band. RSS high-water 18.7 GB is pages already purged and not yet reclaimed: the footprint never exceeds 7.3 GB at a mark. peak_charged_mb (7b77fd48) is what made this measurable: the kernel at the default budget reads charged 22.3 GB against 24.6 GB while ps reports 29 GB RSS, and the 15 GB run read 16.8 GB where every mark was under. Tests: gbuf_worker_buffer_keeps_no_by_id_array (graph_buffer: one node at id 2M in a worker buffer must not grow the index class by even 1 MB where the dense array costs 16 MB; the worker answers by QN, the main buffer by id after the merge) -- RED on the dense array, GREEN without it. Signed-off-by: Martin Vogel --- src/graph_buffer/graph_buffer.c | 42 ++++++++++++++++++++---------- src/pipeline/pass_parallel.c | 45 +++++++++++++++++++++++++++++++++ tests/test_graph_buffer.c | 33 ++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 13 deletions(-) diff --git a/src/graph_buffer/graph_buffer.c b/src/graph_buffer/graph_buffer.c index 50ef7fc58..3ef86bcdd 100644 --- a/src/graph_buffer/graph_buffer.c +++ b/src/graph_buffer/graph_buffer.c @@ -98,6 +98,10 @@ struct cbm_gbuf { * strings at kernel scale, plus a snprintf+strdup+hash on every one of * the ~18 hot find_by_id call sites. */ cbm_gbuf_node_t **by_id; + /* Worker buffers (cbm_gbuf_new_worker) keep no by_id array: their ids + * come from the shared counter, so a dense array would span the whole + * global id space in every worker and double in lockstep. */ + bool by_id_off; int64_t by_id_cap; /* Secondary node indexes */ @@ -485,22 +489,24 @@ static void cascade_delete_edges(cbm_gbuf_t *gb, CBMHashTable *deleted_set) { static void register_node_in_indexes(cbm_gbuf_t *gb, cbm_gbuf_node_t *node) { cbm_ht_set(gb->node_by_qn, node->qualified_name, node); - if (node->id >= gb->by_id_cap) { - int64_t nc = gb->by_id_cap > 0 ? gb->by_id_cap : CBM_SZ_1K; - while (nc <= node->id) { - nc *= 2; + if (!gb->by_id_off) { + if (node->id >= gb->by_id_cap) { + int64_t nc = gb->by_id_cap > 0 ? gb->by_id_cap : CBM_SZ_1K; + while (nc <= node->id) { + nc *= 2; + } + cbm_gbuf_node_t **grown = + cbm_realloc(CBM_MEM_CLASS_GBUF_INDEX, gb->by_id, (size_t)nc * sizeof(*grown)); + if (grown) { + memset(grown + gb->by_id_cap, 0, (size_t)(nc - gb->by_id_cap) * sizeof(*grown)); + gb->by_id = grown; + gb->by_id_cap = nc; + } } - cbm_gbuf_node_t **grown = - cbm_realloc(CBM_MEM_CLASS_GBUF_INDEX, gb->by_id, (size_t)nc * sizeof(*grown)); - if (grown) { - memset(grown + gb->by_id_cap, 0, (size_t)(nc - gb->by_id_cap) * sizeof(*grown)); - gb->by_id = grown; - gb->by_id_cap = nc; + if (node->id >= 0 && node->id < gb->by_id_cap) { + gb->by_id[node->id] = node; } } - if (node->id >= 0 && node->id < gb->by_id_cap) { - gb->by_id[node->id] = node; - } node_ptr_array_t *by_label = get_or_create_node_array(gb->nodes_by_label, node->label ? node->label : ""); @@ -652,6 +658,16 @@ cbm_gbuf_t *cbm_gbuf_new_worker(const char *project, const char *root_path, gb->edges_by_source_type = NULL; gb->edges_by_target_type = NULL; gb->edges_by_type = NULL; + /* Nor by id: with ids from the shared counter, a dense id -> node array + * in every worker spans the whole global id space (8.5M ids x 8 B x 18 + * workers on the kernel) and all of them double in the same instant -- + * a 1 GB step between two reads of the memory gate, which is where the + * 15 GB budget was missed (2026-09-14). Nothing asks a worker buffer by + * id before the merge; the main buffer answers after it. */ + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, gb->by_id); + gb->by_id = NULL; + gb->by_id_cap = 0; + gb->by_id_off = true; return gb; } diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index e3439f293..0cacf8148 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -80,6 +80,7 @@ enum { PP_CSHARP_M_PREFIX_LEN = 2 }; #include "foundation/compat.h" #include "foundation/compat_fs.h" #include "foundation/compat_thread.h" +#include "foundation/mem_core.h" #include "graph_buffer/graph_buffer.h" #include "service_patterns.h" #include "foundation/platform.h" @@ -931,6 +932,36 @@ static int pp_spill_sweep(extract_ctx_t *ec, int worker_id) { return parked; } +/* Diagnostic (CBM_MEM_PHASES=1): where does the charge go between the + * near-budget latch and the first over-budget observation? One line per + * 256 MB step of the charge above its last logged value while spill mode is + * on, and one at the first over-budget observation, each with the + * footprint / commit / tracked breakdown and the class table. */ +static _Atomic size_t g_probe_last_mb = 0; +static _Atomic int g_probe_over_logged = 0; +static void pp_charge_probe(extract_ctx_t *ec, bool over) { + if (!cbm_mem_phases_enabled() || !pp_spill_active(ec)) { + return; + } + enum { PROBE_STEP_MB = 256, PROBE_MB = 1024 * 1024 }; + size_t charged_mb = cbm_mem_charged() / PROBE_MB; + size_t last = atomic_load_explicit(&g_probe_last_mb, memory_order_relaxed); + bool step = charged_mb >= last + PROBE_STEP_MB && + atomic_compare_exchange_strong_explicit(&g_probe_last_mb, &last, charged_mb, + memory_order_relaxed, memory_order_relaxed); + bool first_over = + over && atomic_exchange_explicit(&g_probe_over_logged, 1, memory_order_relaxed) == 0; + if (!step && !first_over) { + return; + } + cbm_log_info("mem.charge.probe", "event", first_over ? "first_over" : "step", "charged_mb", + itoa_log((int)charged_mb), "footprint_mb", + itoa_log((int)(cbm_mem_footprint() / PROBE_MB)), "commit_mb", + itoa_log((int)(cbm_mem_allocator_committed() / PROBE_MB)), "tracked_mb", + itoa_log((int)(cbm_mem_tracked_live_bytes() / PROBE_MB))); + cbm_mem_class_log(first_over ? "charge.first_over" : "charge.step"); +} + static void extract_worker(int worker_id, void *ctx_ptr) { extract_ctx_t *ec = ctx_ptr; extract_worker_state_t *ws = &ec->workers[worker_id]; @@ -974,6 +1005,7 @@ static void extract_worker(int worker_id, void *ctx_ptr) { * and the previously serving index keeps answering. */ if (cbm_mem_budget() > 0) { bool over = cbm_mem_over_budget(); + pp_charge_probe(ec, over); /* Anticipation: the gate sees the crossing per file pull, and the * workers' in-flight files carry the charge past the line before * the first sweep lands (kernel, 15 GB budget: high-water 15.65 @@ -1401,6 +1433,19 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file cbm_parallel_for_opts_t parallel_opts = {.max_workers = worker_count, .force_pthreads = false}; cbm_scale_begin(&ec.scale, "parallel_extract", (long)file_count); cbm_parallel_for(worker_count, extract_worker, &ec, parallel_opts); + if (pp_spill_active(&ec) && + !atomic_load_explicit(&ec.over_budget_abort, memory_order_relaxed)) { + /* Spill mode was entered, so results belong on disk: park every + * result still cached before the phases that cannot park (registry + * build, resolve, the semantic pass) inherit them. The sweeps above + * run only on an over-budget observation; a run that latched early + * and then stayed under budget through extraction (kernel, 15 GB, + * 2026-09-14: 14,949 MB at this point, 44,797 results = 8 GB still + * cached) reached resolve with no headroom and aborted there. */ + int parked = pp_spill_sweep(&ec, 0); + cbm_log_info("mem.spill.final_sweep", "parked", itoa_log(parked), "charged_mb", + itoa_log((int)(cbm_mem_charged() / ((size_t)1024 * 1024)))); + } if (ctx->spill) { int64_t parked = 0; int64_t bytes = 0; diff --git a/tests/test_graph_buffer.c b/tests/test_graph_buffer.c index 232286045..256fc0cec 100644 --- a/tests/test_graph_buffer.c +++ b/tests/test_graph_buffer.c @@ -6,6 +6,8 @@ */ #include "test_framework.h" #include "graph_buffer/graph_buffer.h" +#include "foundation/mem_core.h" +#include #include "store/store.h" #include @@ -1120,7 +1122,38 @@ TEST(gbuf_flush_skips_orphan_edges) { /* ── Suite ─────────────────────────────────────────────────────── */ +/* A worker buffer draws ids from the shared counter, so a dense id -> node + * array in it spans the whole global id space: 18 workers x (next power of + * two above the highest id) x 8 B, doubling in lockstep -- a 1 GB step + * inside one gate interval on the kernel at 8M ids (2026-09-14). A worker + * buffer is never asked by id before the merge, so it keeps no such array; + * the main buffer it merges into still answers by id. One node at id 2M + * would cost a 16 MB array; the index class must not grow by even 1 MB. */ +TEST(gbuf_worker_buffer_keeps_no_by_id_array) { + _Atomic int64_t ids; + atomic_init(&ids, (int64_t)1 << 21); + size_t before = cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_INDEX); + cbm_gbuf_t *w = cbm_gbuf_new_worker("p", "/r", &ids); + ASSERT_NOT_NULL(w); + int64_t id = cbm_gbuf_upsert_node(w, "Function", "f", "p.f", "a.c", 1, 2, "{}"); + ASSERT_TRUE(id >= ((int64_t)1 << 21)); + size_t after = cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_INDEX); + size_t grown = after > before ? after - before : 0; + ASSERT_TRUE(grown < ((size_t)1 << 20)); + ASSERT_TRUE(cbm_gbuf_find_by_id(w, id) == NULL); + ASSERT_NOT_NULL(cbm_gbuf_find_by_qn(w, "p.f")); + + cbm_gbuf_t *main_gb = cbm_gbuf_new("p", "/r"); + ASSERT_NOT_NULL(main_gb); + cbm_gbuf_merge(main_gb, w); + ASSERT_NOT_NULL(cbm_gbuf_find_by_id(main_gb, id)); + cbm_gbuf_free(w); + cbm_gbuf_free(main_gb); + PASS(); +} + SUITE(graph_buffer) { + RUN_TEST(gbuf_worker_buffer_keeps_no_by_id_array); /* Original tests */ RUN_TEST(gbuf_create_free); RUN_TEST(gbuf_free_null); From 9724d903046626df0640ced218259cb58070e4e8 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 14 Sep 2026 19:17:40 +0200 Subject: [PATCH 076/130] fix(mem): dedicated SQLite heap only in the worker; guard the committed stat The daemon leaked 180 KB per query. The Linux soak's quick leg went from 11 to 144 MB of RSS in ten minutes (13.1x, 628 MB/hr), mimalloc's committed bytes tracking RSS almost byte for byte and the growth linear in the query count, file descriptors flat -- where the same leg had been flat at 15 MB in July. The cause is the previous commit's SQLite heap: one mimalloc heap per thread, created on first use. The daemon runs a thread per connection, so every query ran on a fresh thread with a fresh heap, and every block the shared connection keeps beyond the request -- page cache pages, the statement cache, schema objects -- was allocated in that heap's pages. When the thread exited, its heap was deleted and those pages were left behind holding a few live blocks each: committed for as long as the connection lives, a few pages per query, never reused by anyone. The dedicated heap exists for one reason: the index worker's default heap holds the graph (40M+ blocks), and SQLite churn on that heap paid a page walk per allocation during the coverage publish (132 s vs 9.7 s on the kernel). That process has long-lived threads and exits when the index is done. So the dedicated heap is now a switch, off by default, that the index worker turns on at its entry (main.c, --index-worker); everywhere else SQLite allocates from the calling thread's default heap exactly as before the previous commit. The switch exists in every build and changes nothing where the allocator binds are compiled out. Linux soak, quick leg, before -> after: RSS 11 -> 144 MB in ten minutes (13.1x, 628 MB/hr) -> 10.6 -> 11.5 MB and flat from the first minute over 564 queries with reindexes every two minutes; query-leak leg: 11.6 MB flat over 1,376 queries; committed heap 6.8 MB flat. The same soak showed a second thing. From its second sample on, the query-leak leg reported a committed heap of 2^64 - 121 MB: mimalloc's committed statistic is a signed counter merged per thread at thread exit, and a process whose long-lived thread commits what its short-lived threads free reads it negative -- cast to size_t on the way out. That figure is one of the two the budget takes the larger of. cbm_mem_allocator_committed now reports 0 for a reading above SIZE_MAX/2, so the charge falls back to the OS number instead of a 16 EB budget breach, and the diagnostics report the committed pair through it (the peak never undercuts the current value and never carries a wrapped one), the way they already did for RSS. Detector: the soak's quick and query-leak legs (scripts/soak-legs.sh), the release-gating sequence in every venue -- RED on the previous tree (13.1x), GREEN on this one. A unit test cannot see this class: the test build keeps SQLite on libc so the sanitizers see every block. Signed-off-by: Martin Vogel --- internal/cbm/cbm.c | 26 +++++++++++++++++++++++--- internal/cbm/cbm.h | 6 ++++++ src/foundation/diagnostics.c | 15 +++++++++++++++ src/foundation/mem.c | 9 +++++++++ src/main.c | 5 +++++ 5 files changed, 58 insertions(+), 3 deletions(-) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 71bb4bb2f..52614d74c 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -300,6 +300,20 @@ static TSParser *get_thread_parser(const TSLanguage *ts_lang, CBMLanguage lang) * to mimalloc would mismatch ASan/CRT frees — there these binds compile to * no-ops and the build stays unchanged. */ +/* SQLite on a dedicated mimalloc heap per thread: ON only in the index worker, + * whose default heap holds the graph (SQLite churn on that heap paid a page + * walk per allocation: 132 s vs 9.7 s on the kernel's coverage publish). OFF + * everywhere else: the daemon runs a thread per connection, and a heap + * created per such thread pins every SQLite block the shared connection + * keeps (page cache, statement cache) to pages nobody's heap owns any more -- + * the Linux soak grew 180 KB per query, 11 -> 144 MB in ten minutes, where + * the default thread heap had been flat (2026-09-14). */ +static _Atomic int g_sqlite_dedicated_heap; + +void cbm_sqlite_dedicated_heap(bool on) { + atomic_store_explicit(&g_sqlite_dedicated_heap, on ? 1 : 0, memory_order_relaxed); +} + #if defined(CBM_BIND_TS_ALLOCATOR) && CBM_BIND_TS_ALLOCATOR #include @@ -322,15 +336,20 @@ static TSParser *get_thread_parser(const TSLanguage *ts_lang, CBMLanguage lang) * thread's heap is released with the thread. */ static _Thread_local mi_heap_t *tl_sqlite_heap; +/* NULL = the calling thread's default heap (mi_malloc); see the switch above. */ static mi_heap_t *sqlite_heap(void) { + if (!atomic_load_explicit(&g_sqlite_dedicated_heap, memory_order_relaxed)) { + return NULL; + } if (!tl_sqlite_heap) { tl_sqlite_heap = mi_heap_new(); } - return tl_sqlite_heap ? tl_sqlite_heap : mi_heap_main(); + return tl_sqlite_heap; } static void *cbm_sqlite_malloc(int n) { - void *block = mi_heap_malloc(sqlite_heap(), (size_t)n); + mi_heap_t *heap = sqlite_heap(); + void *block = heap ? mi_heap_malloc(heap, (size_t)n) : mi_malloc((size_t)n); if (block) { cbm_mem_class_add_external(CBM_MEM_CLASS_STORE, mi_usable_size(block)); } @@ -344,7 +363,8 @@ static void cbm_sqlite_free(void *p) { } static void *cbm_sqlite_realloc(void *p, int n) { size_t old_size = p ? mi_usable_size(p) : 0; - void *grown = mi_heap_realloc(sqlite_heap(), p, (size_t)n); + mi_heap_t *heap = sqlite_heap(); + void *grown = heap ? mi_heap_realloc(heap, p, (size_t)n) : mi_realloc(p, (size_t)n); if (grown) { cbm_mem_class_remove_external(CBM_MEM_CLASS_STORE, old_size); cbm_mem_class_add_external(CBM_MEM_CLASS_STORE, mi_usable_size(grown)); diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index c62d3f281..d875459c1 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -669,6 +669,12 @@ typedef struct { // also calls it so non-main entry points (pipeline passes) still get the binds. // In the test build (no CBM_BIND_TS_ALLOCATOR) this is a no-op. void cbm_alloc_init(void); +/* SQLite allocates from a dedicated mimalloc heap per thread while on; the + * index worker turns it on (its default heap holds the graph). Off elsewhere: + * a thread-per-connection daemon would pin connection-lifetime blocks to + * dead threads. The switch exists in every build; it changes nothing where + * the allocator binds are compiled out. */ +void cbm_sqlite_dedicated_heap(bool on); // Initialize the library. Call once at startup. Returns 0 on success. int cbm_init(void); diff --git a/src/foundation/diagnostics.c b/src/foundation/diagnostics.c index 5b2ee2cb8..77f16c5bc 100644 --- a/src/foundation/diagnostics.c +++ b/src/foundation/diagnostics.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -538,6 +539,20 @@ static void write_diagnostics(void) { * macOS/Windows, and peak is reconciled to never undercut current. */ current_rss = cbm_mem_rss(); peak_rss = cbm_mem_peak_rss(); + /* The committed pair has the same failure: the counter is signed and + * merged per thread at thread exit, so a thread-per-connection daemon + * reads it negative once short-lived threads have freed what a long-lived + * one committed (query-leak soak, Linux 2026-09-14: heap_committed_bytes + * printed 2^64 - 121 MB from the second sample on). cbm_mem_allocator_ + * committed reports 0 for a negative reading; the peak never undercuts + * the current value and never carries a wrapped one. */ + current_commit = cbm_mem_allocator_committed(); + if (peak_commit > (SIZE_MAX >> 1)) { + peak_commit = 0; + } + if (peak_commit < current_commit) { + peak_commit = current_commit; + } int fds = count_open_fds(); time_t now = time(NULL); diff --git a/src/foundation/mem.c b/src/foundation/mem.c index 1c044c071..753e101f9 100644 --- a/src/foundation/mem.c +++ b/src/foundation/mem.c @@ -515,6 +515,15 @@ void cbm_mem_set_budget_for_tests(size_t bytes) { size_t cbm_mem_allocator_committed(void) { size_t commit = 0; mi_process_info(NULL, NULL, NULL, NULL, NULL, &commit, NULL, NULL); + /* The statistic behind this is a signed counter merged per thread at + * thread exit; a process whose long-lived thread commits what its + * short-lived threads free reads it NEGATIVE, cast to size_t here. The + * daemon's query-leak soak reported 2^64 - 121 MB from the second sample + * on (Linux, 2026-09-14). A negative reading is no reading: report 0 so + * the charge falls back to the OS number instead of a 16 EB budget breach. */ + if (commit > (SIZE_MAX >> 1)) { + return 0; + } return commit; } diff --git a/src/main.c b/src/main.c index 284a899cb..202aa6a18 100644 --- a/src/main.c +++ b/src/main.c @@ -854,6 +854,11 @@ static int run_cli(int argc, char **argv, cbm_project_lock_manager_t *project_lo const char *worker_marker = cli_strip_flag_value(&argc, argv, CBM_INDEX_WORKER_MARKER_ARG); const char *worker_quarantine = cli_strip_flag_value(&argc, argv, CBM_INDEX_WORKER_QUARANTINE_ARG); + if (index_worker) { + /* The graph lives on this process's heaps: SQLite gets heaps of its + * own here, and only here (cbm_sqlite_dedicated_heap). */ + cbm_sqlite_dedicated_heap(true); + } cbm_index_set_worker_role_options(index_worker, response_out, worker_single_thread, worker_marker, worker_quarantine, cbm_index_worker_memory_budget_bytes()); From 9e69cb7d9b7ad4b40fa5f8cbbb653dc70e108130 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 14 Sep 2026 21:04:16 +0200 Subject: [PATCH 077/130] fix(mem): free the thread locale at exit; a settling gate keeps its latch Two CI-only lanes went red on 9724d903, both this branch's own doing. The macOS leak lane: every test passed and LeakSanitizer reported 16 to 101 objects of 1,472 bytes per test process, allocated from cbm_arena_sprintf. The formatter creates one C locale per thread on macOS (the vsnprintf locale lock fix) and kept it in a thread-local pointer only, so every worker thread that ever formatted a name leaked its locale at exit. A pthread key destructor now frees it when the thread exits; the main thread keeps its locale until process exit. The TSan lane on Ubuntu arm: pipeline_backpressure_futile_nap_disengages counted 8 nap cycles against a bound of 7, the "gate re-paid per pull" shape the test exists for. The spill shortcut reports "not over" while a sweep is still bringing memory down, and that reading re-armed the futility latch, so the next over-budget pull paid a full nap cycle again. The latch now re-arms only on a genuine under-budget reading; a reading the shortcut produced is not one. Verification: the local macOS leak lane (make -f Makefile.cbm test-lsan) reports zero leaks, 8050 passed / 10 skipped (the CI lane had 16-101 leaked objects per process); the local TSan lane (scripts/test.sh --tsan) passes 1157 / 9 skipped with the nap-cycle test green (the CI lane had it at 8 cycles); pipeline, parallel and arena suites twice and lint-ci on the fixed tree. Signed-off-by: Martin Vogel --- src/foundation/arena.c | 21 +++++++++++++++++++++ src/pipeline/pass_parallel.c | 10 +++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/foundation/arena.c b/src/foundation/arena.c index 0d809ef1d..20d604de0 100644 --- a/src/foundation/arena.c +++ b/src/foundation/arena.c @@ -18,6 +18,7 @@ enum { ARENA_ALIGN = 7, ARENA_GROW_OK = 1 }; #include #include #if defined(__APPLE__) +#include #include #endif #include @@ -144,10 +145,30 @@ char *cbm_arena_strndup(CBMArena *a, const char *s, size_t len) { * object of its own, so no thread ever waits for another, and the common * short string is formatted once into a stack buffer. */ #if defined(__APPLE__) +/* One locale object per thread, freed when the thread exits: the object is + * heap memory that the thread-local pointer alone kept, so every worker + * thread that ever formatted a name leaked 1,472 bytes at exit (the macOS + * LSan lane on PR #2202, 16-101 objects per test process). A pthread key + * destructor is the one hook that runs at thread exit for a TLS-held + * resource; the main thread keeps its locale until process exit. */ static _Thread_local locale_t tl_c_locale; +static pthread_key_t tl_c_locale_key; +static pthread_once_t tl_c_locale_once = PTHREAD_ONCE_INIT; +static void arena_c_locale_free(void *loc) { + if (loc) { + freelocale((locale_t)loc); + } +} +static void arena_c_locale_key_init(void) { + (void)pthread_key_create(&tl_c_locale_key, arena_c_locale_free); +} static locale_t arena_c_locale(void) { if (!tl_c_locale) { tl_c_locale = newlocale(LC_ALL_MASK, "C", NULL); + if (tl_c_locale) { + pthread_once(&tl_c_locale_once, arena_c_locale_key_init); + (void)pthread_setspecific(tl_c_locale_key, tl_c_locale); + } } return tl_c_locale; /* NULL = the global locale, the pre-fix behaviour */ } diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 0cacf8148..f5e4ab4d2 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -1020,6 +1020,7 @@ static void extract_worker(int worker_id, void *ctx_ptr) { pp_spill_enter(ec, "near_budget"); } } + bool settling = false; if (over) { /* Admission control, first response: park what can be parked. * Only what is still over budget after that -- the floor -- @@ -1030,6 +1031,7 @@ static void extract_worker(int worker_id, void *ctx_ptr) { over = cbm_mem_over_budget(); if (over && pp_spill_active(ec) && (parked > 0 || pp_spill_work_remains(ec))) { over = false; /* memory is still on its way down */ + settling = true; } } bool futile = atomic_load_explicit(&ec->bp_futile, memory_order_relaxed) != 0; @@ -1046,7 +1048,13 @@ static void extract_worker(int worker_id, void *ctx_ptr) { atomic_store_explicit(&ec->bp_futile, 0, memory_order_relaxed); } } - } else if (!over && futile) { + } else if (!over && !settling && futile) { + /* Re-arm only on a genuine under-budget reading. A reading the + * spill shortcut produced ("still on its way down") is not + * one: re-arming on it made the next over-budget pull pay a + * full nap cycle again -- the gate re-paid per pull that + * pipeline_backpressure_futile_nap_disengages guards against + * (TSan lane on PR #2202: 8 cycles against a bound of 7). */ atomic_store_explicit(&ec->bp_futile, 0, memory_order_relaxed); } } From d89ccabef064661794269c0de726021736d72b51 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 14 Sep 2026 21:26:06 +0200 Subject: [PATCH 078/130] fix(mem): the parked result image is declared defined for MemorySanitizer The MSan lane on 9724d903 stopped in the extraction suite's spill round-trip test: uninitialized bytes in fwrite at offset 4087 of a 6,952-byte region, the parked result's arena block (the stack overflow that followed is MemorySanitizer's own report path failing on the nested bug). The park writes a result as an opaque image: the record header, a struct copy, and the compacted arena block. Both carry padding bytes no code ever wrote, inside structs and between objects, and MSan tracks that mark through the compaction's memcpy, so zeroing the destination block only moved the report (offset 6714 in the local MSan container). The image is read back whole and never interpreted byte by byte; the padding content is irrelevant on disk. Under MSan the header and the block are declared defined right before the write (__msan_unpoison, compiled in only when the build is instrumented); every other build compiles the annotation to nothing. The exact block is also zeroed once when it is created, so the gaps between objects are deterministic in the spill file. Verification: extraction, arena and parallel suites under ASan on the fixed tree; the extraction suite in the local MSan container image passes 348 / 0 with no sanitizer report (it reproduced the CI report exactly before the fix). Signed-off-by: Martin Vogel --- internal/cbm/result_spill.c | 20 ++++++++++++++++++++ src/foundation/arena.c | 9 +++++++++ 2 files changed, 29 insertions(+) diff --git a/internal/cbm/result_spill.c b/internal/cbm/result_spill.c index b75f06fb5..b79dc0e6c 100644 --- a/internal/cbm/result_spill.c +++ b/internal/cbm/result_spill.c @@ -21,6 +21,24 @@ #include "foundation/log.h" #include "foundation/mem_core.h" +/* The park writes a result as an opaque image: the record header (a struct + * copy) and the compacted arena block. Both carry padding bytes no code ever + * wrote -- inside structs, between objects -- and MemorySanitizer tracks + * that mark through the compaction's memcpy, so it refuses the fwrite of an + * image that is read back whole and never interpreted byte by byte (CI MSan + * lane on #2202: offset 4087, then 6714, of a 6,952-byte block). Under MSan + * the image is declared defined right before the write; every other build + * compiles this to nothing. */ +#if defined(__has_feature) +#if __has_feature(memory_sanitizer) +#include +#define SPILL_IMAGE_DEFINED(p, n) __msan_unpoison((p), (n)) +#endif +#endif +#ifndef SPILL_IMAGE_DEFINED +#define SPILL_IMAGE_DEFINED(p, n) ((void)0) +#endif + #include #include #include @@ -133,6 +151,8 @@ bool cbm_result_spill_park(cbm_result_spill_t *sp, int writer, int slot, CBMFile hdr.old_base = (uint64_t)(uintptr_t)result->arena.blocks[0]; hdr.header = *result; hdr.header.cached_tree = NULL; /* never on disk: the loader gets no tree */ + SPILL_IMAGE_DEFINED(&hdr, sizeof(hdr)); + SPILL_IMAGE_DEFINED(result->arena.blocks[0], hdr.block_len); cbm_mutex_lock(&f->mu); uint64_t offset = f->end; bool ok = SPILL_SEEK(f->fp, offset) == 0 && fwrite(&hdr, sizeof(hdr), 1, f->fp) == 1 && diff --git a/src/foundation/arena.c b/src/foundation/arena.c index 20d604de0..82a2f2af6 100644 --- a/src/foundation/arena.c +++ b/src/foundation/arena.c @@ -45,6 +45,15 @@ void cbm_arena_init_exact(CBMArena *a, size_t bytes) { size_t block = (bytes + ARENA_ALIGN) & ~(size_t)ARENA_ALIGN; cbm_arena_init_sized(a, block); a->grow_size = CBM_ARENA_DEFAULT_BLOCK_SIZE; + /* An exact block is a compacted result's image and may be written to + * disk whole (result_spill): the alignment gaps between objects and the + * padding inside structs are never written by the copy, and MemorySanitizer + * refuses an fwrite of uninitialized bytes (CI MSan lane on #2202, offset + * 4087 of a 6,952-byte block). Zeroed once here, every byte of the image + * is defined; the cost is one pass over memory the copy is about to touch. */ + if (a->blocks[0]) { + memset(a->blocks[0], 0, a->block_sizes[0]); + } } static int arena_grow(CBMArena *a, size_t min_size) { From d70b79417978a9f3ab0d1aa849879dbf049793cf Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 14 Sep 2026 22:12:28 +0200 Subject: [PATCH 079/130] fix(mem): the MSan guard in the spill store goes through sanitized.h cppcheck cannot evaluate `#if __has_feature(memory_sanitizer)` even behind `#if defined(__has_feature)` ("undefined function-like macro invocation", the lint lane on d89ccabe), and foundation/sanitized.h already exists for exactly that: it defines __has_feature away where the preprocessor lacks it. The spill store's image annotation now includes that header and tests the feature directly; the annotation itself is unchanged. Signed-off-by: Martin Vogel --- internal/cbm/result_spill.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/cbm/result_spill.c b/internal/cbm/result_spill.c index b79dc0e6c..89716398a 100644 --- a/internal/cbm/result_spill.c +++ b/internal/cbm/result_spill.c @@ -29,13 +29,11 @@ * lane on #2202: offset 4087, then 6714, of a 6,952-byte block). Under MSan * the image is declared defined right before the write; every other build * compiles this to nothing. */ -#if defined(__has_feature) +#include "foundation/sanitized.h" /* __has_feature exists everywhere, cppcheck included */ #if __has_feature(memory_sanitizer) #include #define SPILL_IMAGE_DEFINED(p, n) __msan_unpoison((p), (n)) -#endif -#endif -#ifndef SPILL_IMAGE_DEFINED +#else #define SPILL_IMAGE_DEFINED(p, n) ((void)0) #endif From 172787de9c18352bb29545da2426d9d61b260fc3 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 15 Sep 2026 18:08:36 +0200 Subject: [PATCH 080/130] fix(mem): no glibc malloc_trim; the static Linux release links again The v0.10.9 release run (34948714902) failed in both Linux portable builds: the static link found malloc, free, realloc, calloc, memalign, valloc, pvalloc and posix_memalign defined twice, once by mimalloc's override and once by libc.a(malloc.o). Nothing in the build flags had changed since v0.10.8; what changed was one call. cbm_mem_release_to_os called glibc's malloc_trim(0), and that symbol lives only in malloc.o, so the reference pulled the whole object into the link next to mimalloc's definitions. Dynamic builds never noticed: interposition resolves the duplicates at load time, which is why the whole test matrix was green. The call was pointless where it broke things: on Linux and Windows the override routes every malloc to mimalloc, so glibc's heap holds nothing to trim and mi_collect(true) is the release. The branch and its include are gone; macOS keeps its pressure-relief call, since the system heap still serves everything outside the core there. Verification: the static build (scripts/build.sh CC=gcc CXX=g++ STATIC=1) in the local arm64 container reproduced the CI failure exactly before the change and links after it, the binary reporting its version; mem and parallel suites green on the host. Signed-off-by: Martin Vogel --- src/foundation/mem.c | 11 ++++++----- src/foundation/mem.h | 8 +++++--- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/foundation/mem.c b/src/foundation/mem.c index 753e101f9..48f0eeeff 100644 --- a/src/foundation/mem.c +++ b/src/foundation/mem.c @@ -35,9 +35,6 @@ #include /* malloc_zone_pressure_relief */ #else #include -#if defined(__GLIBC__) -#include /* malloc_trim */ -#endif #endif /* Does THIS build ask mimalloc to replace ordinary malloc process-wide? @@ -606,11 +603,15 @@ size_t cbm_mem_footprint(void) { } void cbm_mem_release_to_os(void) { + /* mi_collect is the release: on Linux and Windows mimalloc owns malloc, so + * there is no libc heap to trim. glibc's malloc_trim was called here once + * and cost the static Linux release its link: the reference pulls + * libc.a(malloc.o) in beside mimalloc's malloc/free (multiple definition, + * release run 34948714902, 2026-09-15). macOS keeps the system heap for + * everything outside the core, hence the pressure-relief call there. */ mi_collect(true); #if defined(__APPLE__) (void)malloc_zone_pressure_relief(NULL, 0); -#elif defined(__GLIBC__) - (void)malloc_trim(0); #endif } diff --git a/src/foundation/mem.h b/src/foundation/mem.h index 9b318fc9e..8fe909f0e 100644 --- a/src/foundation/mem.h +++ b/src/foundation/mem.h @@ -119,9 +119,11 @@ void cbm_mem_collect(void); size_t cbm_mem_footprint(void); /* Hand freed memory back to the OS on every allocator this process uses: - * mimalloc (mi_collect), the macOS system zones (malloc_zone_pressure_relief) - * and glibc (malloc_trim). Costs a few ms; call at phase boundaries after a - * bulk release, never per allocation. */ + * mimalloc (mi_collect) and, on macOS, the system zones + * (malloc_zone_pressure_relief). Never glibc's malloc_trim: on Linux mimalloc + * owns malloc, and the reference alone breaks the static release link. Costs + * a few ms; call at phase boundaries after a bulk release, never per + * allocation. */ void cbm_mem_release_to_os(void); /* ── Memory map: where does the process's memory actually live? ────── From 62fd80b842f00bd72e655d23c6afbfe12e199b6b Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 16 Sep 2026 01:45:40 +0200 Subject: [PATCH 081/130] chore(pkg): sync packaging metadata to v0.11.0 Every packaging surface states the newest v-prefixed tag, as the version-metadata contract requires: the Chocolatey nuspec and installer (checksum re-pinned to the released windows-amd64 archive), the Go installer, the Nix flake, server.json, the npm and PyPI wrappers. Signed-off-by: Martin Vogel --- flake.nix | 2 +- pkg/chocolatey/codebase-memory-mcp.nuspec | 4 ++-- pkg/chocolatey/tools/chocolateyInstall.ps1 | 4 ++-- pkg/go/cmd/codebase-memory-mcp/main.go | 2 +- pkg/npm/package.json | 2 +- pkg/pypi/pyproject.toml | 2 +- pkg/pypi/src/codebase_memory_mcp/_cli.py | 2 +- server.json | 6 +++--- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/flake.nix b/flake.nix index 50f6319ba..5a0afbb98 100644 --- a/flake.nix +++ b/flake.nix @@ -63,7 +63,7 @@ packages = forAllSystems (pkgs: rec { default = pkgs.stdenv.mkDerivation { pname = "codebase-memory-mcp"; - version = "0.10.8"; + version = "0.11.0"; src = ./.; diff --git a/pkg/chocolatey/codebase-memory-mcp.nuspec b/pkg/chocolatey/codebase-memory-mcp.nuspec index 5ce1deacc..7b6f6cabe 100644 --- a/pkg/chocolatey/codebase-memory-mcp.nuspec +++ b/pkg/chocolatey/codebase-memory-mcp.nuspec @@ -2,7 +2,7 @@ codebase-memory-mcp - 0.10.8 + 0.11.0 codebase-memory-mcp DeusData https://github.com/DeusData/codebase-memory-mcp @@ -28,7 +28,7 @@ After installation, run: to configure your coding agents automatically. - https://github.com/DeusData/codebase-memory-mcp/releases/tag/v0.10.8 + https://github.com/DeusData/codebase-memory-mcp/releases/tag/v0.11.0 diff --git a/pkg/chocolatey/tools/chocolateyInstall.ps1 b/pkg/chocolatey/tools/chocolateyInstall.ps1 index 132b86d90..031ac9e86 100644 --- a/pkg/chocolatey/tools/chocolateyInstall.ps1 +++ b/pkg/chocolatey/tools/chocolateyInstall.ps1 @@ -1,9 +1,9 @@ $ErrorActionPreference = 'Stop' $packageName = 'codebase-memory-mcp' -$version = '0.10.8' +$version = '0.11.0' $url64 = "https://github.com/DeusData/codebase-memory-mcp/releases/download/v${version}/codebase-memory-mcp-windows-amd64.zip" -$checksum64 = 'b43ad982994c4d829670749e08d3b622a74bb20041fc0a7d02bef6113f81c34d' +$checksum64 = '6eb6beaf261b19e419766e78baf93cbc3cf1c6338cff8fb7c0234859f96d1685' $installDir = Join-Path $env:ChocolateyBinRoot $packageName Install-ChocolateyZipPackage ` diff --git a/pkg/go/cmd/codebase-memory-mcp/main.go b/pkg/go/cmd/codebase-memory-mcp/main.go index 91f1adc33..65f6e131a 100644 --- a/pkg/go/cmd/codebase-memory-mcp/main.go +++ b/pkg/go/cmd/codebase-memory-mcp/main.go @@ -36,7 +36,7 @@ import ( const ( repo = "DeusData/codebase-memory-mcp" - version = "0.10.8" + version = "0.11.0" windowsBinaryName = "codebase-memory-mcp.exe" maxRedirects = 5 diff --git a/pkg/npm/package.json b/pkg/npm/package.json index f2b122d84..d19488f7f 100644 --- a/pkg/npm/package.json +++ b/pkg/npm/package.json @@ -1,6 +1,6 @@ { "name": "codebase-memory-mcp", - "version": "0.10.8", + "version": "0.11.0", "description": "Fast code intelligence for AI coding agents — installs a verified native runtime set", "mcpName": "io.github.DeusData/codebase-memory-mcp", "license": "MIT", diff --git a/pkg/pypi/pyproject.toml b/pkg/pypi/pyproject.toml index 5821d2a4b..f848e7cd8 100644 --- a/pkg/pypi/pyproject.toml +++ b/pkg/pypi/pyproject.toml @@ -8,7 +8,7 @@ build-backend = "hatchling.build" [project] name = "codebase-memory-mcp" -version = "0.10.8" +version = "0.11.0" description = "Fast code intelligence for AI coding agents — installs a verified native runtime set" readme = "README.md" license = { text = "MIT" } diff --git a/pkg/pypi/src/codebase_memory_mcp/_cli.py b/pkg/pypi/src/codebase_memory_mcp/_cli.py index 8a4aa112c..458b8c350 100644 --- a/pkg/pypi/src/codebase_memory_mcp/_cli.py +++ b/pkg/pypi/src/codebase_memory_mcp/_cli.py @@ -338,7 +338,7 @@ def _version() -> str: from importlib.metadata import version return version("codebase-memory-mcp") except Exception: - return "0.10.8" + return "0.11.0" def _os_name() -> str: diff --git a/server.json b/server.json index 3ab583af4..9d0bd68b7 100644 --- a/server.json +++ b/server.json @@ -8,12 +8,12 @@ "source": "github" }, "websiteUrl": "https://deusdata.github.io/codebase-memory-mcp/", - "version": "0.10.8", + "version": "0.11.0", "packages": [ { "registryType": "npm", "identifier": "codebase-memory-mcp", - "version": "0.10.8", + "version": "0.11.0", "runtimeHint": "npx", "transport": { "type": "stdio" @@ -22,7 +22,7 @@ { "registryType": "pypi", "identifier": "codebase-memory-mcp", - "version": "0.10.8", + "version": "0.11.0", "runtimeHint": "uvx", "transport": { "type": "stdio" From 22ce867efd88dff9c219f789967ed2612329ee6d Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 16 Sep 2026 01:55:26 +0200 Subject: [PATCH 082/130] =?UTF-8?q?docs(readme):=20current=20test=20count?= =?UTF-8?q?=20=E2=80=94=208,060=20tests=20across=20141=20suites?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Badge 6768 -> 8050 passing, the macOS full-suite count on v0.11.0's main (8,050 passed, 10 platform-skipped, 141 suites, run 35003197455); the build-from-source section states the suite total and suite count. Signed-off-by: Martin Vogel --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c6ab67ee2..3911bacf8 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![GitHub Release](https://img.shields.io/github/v/release/DeusData/codebase-memory-mcp?style=flat&color=blue)](https://github.com/DeusData/codebase-memory-mcp/releases/latest) [![License](https://img.shields.io/badge/license-MIT-green)](LICENSE) [![CI](https://img.shields.io/github/actions/workflow/status/DeusData/codebase-memory-mcp/dry-run.yml?label=CI)](https://github.com/DeusData/codebase-memory-mcp/actions/workflows/dry-run.yml) -[![Tests](https://img.shields.io/badge/tests-6768_passing-brightgreen)](https://github.com/DeusData/codebase-memory-mcp) +[![Tests](https://img.shields.io/badge/tests-8050_passing-brightgreen)](https://github.com/DeusData/codebase-memory-mcp) [![Languages](https://img.shields.io/badge/languages-162-orange)](https://github.com/DeusData/codebase-memory-mcp) [![Hybrid LSP](https://img.shields.io/badge/Hybrid_LSP-10_languages-blue)](#hybrid-lsp) [![Agents](https://img.shields.io/badge/agent_surfaces-45-purple)](https://github.com/DeusData/codebase-memory-mcp) @@ -425,7 +425,7 @@ scripts/build.sh # without the UI (development only) Every platform ships **one self-contained executable**: the graph UI and the agent integration templates are linked into the binary, so an extracted archive is immediately complete. -Run the test suite (6,768 tests across 120 suites): +Run the test suite (8,060 tests across 141 suites): ```bash scripts/test.sh # full: clean sanitizer build + all suites + guards From 58c1c0caa7061ae868305dbe59a6c5e67004849d Mon Sep 17 00:00:00 2001 From: Anton Dziatkovskii <194927794+tonydzi@users.noreply.github.com> Date: Wed, 16 Sep 2026 10:38:31 -0700 Subject: [PATCH 083/130] docs: MCP tool count is 17; document the three missing tools The TOOLS[] registry in src/mcp/mcp.c and the v0.11.0 --help output list 17 tools, but README, the npm README, CONTRIBUTING and the Pages site still say 15 (or 14), and the README tool table has no row for get_file_outline, compare_graphs or check_index_coverage. Refs #1297 Assisted-by: Claude Code / claude-opus-5 Machine: MacBook-Anton Account: a@gmail Operator: robot:git-s3-docs-fix-lane Signed-off-by: Anton Dziatkovskii <194927794+tonydzi@users.noreply.github.com> --- CONTRIBUTING.md | 2 +- README.md | 11 +++++++---- docs/index.html | 4 ++-- pkg/npm/README.md | 4 ++-- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 96ce8f861..12567ac82 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,7 +55,7 @@ src/ foundation/ Arena allocator, hash table, string utils, platform compat store/ SQLite graph storage (WAL mode, FTS5) cypher/ Cypher query → SQL translation - mcp/ MCP server (JSON-RPC 2.0 over stdio, 14 tools) + mcp/ MCP server (JSON-RPC 2.0 over stdio, 17 tools) pipeline/ Multi-pass indexing pipeline pass_*.c Individual pipeline passes (definitions, calls, usages, etc.) httplink.c HTTP route extraction (Go/Express/Laravel/Ktor/Python) diff --git a/README.md b/README.md index 3911bacf8..cc94d5194 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ **The fastest and most efficient code intelligence engine for AI coding agents.** Full-indexes an average repository in milliseconds, the Linux kernel (28M LOC, 75K files) in 3 minutes. Answers structural queries in under 1ms. Ships as a native executable with a small verified runtime-asset set for macOS, Linux, and Windows — download, run `install`, done. -High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across all 162 languages, enhanced with [**Hybrid LSP** semantic type resolution](#hybrid-lsp) for Python, TypeScript / JavaScript / JSX / TSX, PHP, C#, Go, C, C++, Java, Kotlin, Rust, and Perl — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 15 MCP tools. No language runtime, hosted service, or API key. Plug and play across 45 supported automatic/conditional client surfaces. +High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across all 162 languages, enhanced with [**Hybrid LSP** semantic type resolution](#hybrid-lsp) for Python, TypeScript / JavaScript / JSX / TSX, PHP, C#, Go, C, C++, Java, Kotlin, Rust, and Perl — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 17 MCP tools. No language runtime, hosted service, or API key. Plug and play across 45 supported automatic/conditional client surfaces. > **Research** — The design and benchmarks behind this project are described in the preprint [*Codebase-Memory: Tree-Sitter-Based Knowledge Graphs for LLM Code Exploration via MCP*](https://arxiv.org/abs/2603.27277) (arXiv:2603.27277). Evaluated across 31 real-world repositories: 83% answer quality, 10× fewer tokens, 2.1× fewer tool calls vs. file-by-file exploration. @@ -37,7 +37,7 @@ High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-si - **45 supported automatic/conditional client surfaces** — `install` configures detected clients and safely activates conditional clients only when their documented platform, marker, or explicit existing config path is present. See [Multi-Agent Support](#multi-agent-support) for the complete matrix and manual/UI-only boundaries. - **Built-in graph visualization** — 3D interactive UI at `localhost:9749`, served from the binary itself. - **Infrastructure-as-code indexing** — Dockerfiles, Kubernetes manifests, and Kustomize overlays indexed as graph nodes with cross-references. `Resource` nodes for K8s kinds, `Module` nodes for Kustomize overlays with `IMPORTS` edges to referenced resources. -- **15 MCP tools** — search, trace, architecture, impact analysis, targeted index-coverage checks, Cypher queries, dead code detection, cross-service HTTP linking, ADR management, and more. +- **17 MCP tools** — search, trace, architecture, impact analysis, targeted index-coverage checks, Cypher queries, dead code detection, cross-service HTTP linking, ADR management, and more. ## Quick Start @@ -461,7 +461,7 @@ Add to `~/.claude.json` (user scope) or project `.mcp.json`: } ``` -Restart your agent. Verify with `/mcp` — you should see `codebase-memory-mcp` with 15 tools. +Restart your agent. Verify with `/mcp` — you should see `codebase-memory-mcp` with 17 tools. @@ -673,6 +673,7 @@ JSON arguments can also be piped on stdin, for tools that take arguments. A tool | `list_projects` | List all indexed projects with node/edge counts. | | `delete_project` | Remove a project and all its graph data. | | `index_status` | Check indexing status of a project. | +| `check_index_coverage` | Check whether exact paths or a scope are indexed and fresh. A clean result means no recorded gap, not proof of completeness. | ### Querying @@ -683,7 +684,9 @@ JSON arguments can also be piped on stdin, for tools that take arguments. A tool | `detect_changes` | Map git diff to affected symbols + blast radius with risk classification. | | `query_graph` | Execute Cypher-like graph queries (read-only). | | `get_graph_schema` | Node/edge counts, relationship patterns, property definitions per label. Run this first. | +| `compare_graphs` | Compare two indexed snapshots: node/edge additions and removals between a base and a target. | | `get_code_snippet` | Read source code for a function by qualified name. | +| `get_file_outline` | Declaration outline of one repository-relative file in source order, with optional label filter and paging. | | `get_architecture` | Codebase overview: languages, packages, routes, hotspots, clusters, ADR. | | `search_code` | Grep-like text search within indexed project files. | | `manage_adr` | CRUD for Architecture Decision Records (`get` reads, `update` replaces the whole document, `set_sections` rewrites only the named sections and leaves every other byte untouched, `sections` lists headings). Query modes do not wait behind a same-project reindex; writes remain serialized. | @@ -843,7 +846,7 @@ Also supported (not yet benchmarked): Ada, Agda, Apex, Assembly (NASM), Astro, A src/ main.c Entry point (MCP stdio server + CLI + install/update/config) daemon/ Per-account session coordination, IPC, lifecycle, shared jobs/watchers - mcp/ MCP server (15 tools, JSON-RPC 2.0, session detection, auto-index) + mcp/ MCP server (17 tools, JSON-RPC 2.0, session detection, auto-index) cli/ Install/uninstall/update/config (45 client surfaces, hooks, instructions) store/ SQLite graph storage (nodes, edges, traversal, search, Louvain) pipeline/ Multi-pass indexing (structure → definitions → calls → HTTP links → config → tests) diff --git a/docs/index.html b/docs/index.html index 56496d5d8..fdc1501a5 100644 --- a/docs/index.html +++ b/docs/index.html @@ -56,7 +56,7 @@ "featureList": [ "Indexes 158 programming languages via vendored tree-sitter grammars", "Hybrid LSP semantic type resolution for Python, TypeScript/JavaScript, PHP, C#, Go, C/C++, Java, Kotlin, and Rust", - "15 MCP tools for structural search, call-path tracing, targeted coverage checks, and Cypher graph queries", + "17 MCP tools for structural search, call-path tracing, targeted coverage checks, and Cypher graph queries", "Semantic vector code search via bundled nomic-embed-code embeddings (no API key, fully local)", "Semantic graph edges (SEMANTICALLY_RELATED) and near-clone detection (SIMILAR_TO, MinHash + LSH)", "Cross-service linking for HTTP, gRPC, GraphQL, tRPC, and pub/sub channels with confidence scoring", @@ -739,7 +739,7 @@

The optional UI variant adds one verified, content-addressed frontend pack and serves an interactive 3D graph at localhost:9749 to explore nodes, edges, and clusters visually.

-

15 MCP tools

+

17 MCP tools

search_graph, trace_path, detect_changes, query_graph (Cypher), get_architecture, get_code_snippet, check_index_coverage, manage_adr, and 7 more.

diff --git a/pkg/npm/README.md b/pkg/npm/README.md index 614b81b1c..e1e5abe3e 100644 --- a/pkg/npm/README.md +++ b/pkg/npm/README.md @@ -7,7 +7,7 @@ **The fastest and most efficient code intelligence engine for AI coding agents.** Full-indexes an average repository in milliseconds, the Linux kernel (28M LOC, 75K files) in 3 minutes. Answers structural queries in under 1ms. This npm wrapper downloads, verifies, and caches the selected native runtime set: the executable, its authenticated integration asset, and—when requested—the content-addressed UI pack. -High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across 162 languages — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 15 MCP tools. No hosted service or API key; this package requires Node.js to manage and launch the cached native runtime set. Plug and play across 45 automatic/conditional client surfaces. +High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across 162 languages — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 17 MCP tools. No hosted service or API key; this package requires Node.js to manage and launch the cached native runtime set. Plug and play across 45 automatic/conditional client surfaces. ## Installation @@ -38,7 +38,7 @@ Restart your agent. Say **"Index this project"** — done. - **Lifecycle hooks stay conservative** — Kimi uses `UserPromptSubmit`; on macOS/Linux, GitLab Duo gets a fail-open user `SessionStart`, while Devin gets `UserPromptSubmit`, `PostCompaction`, and a deduplicated `SessionStart` when Claude does not already provide it. Qoder, GitLab Duo, Devin, and Factory hooks are withheld on Windows without a documented shell/executor contract. Cline's auto-activating file hooks are withheld because their context output is not reliably consumed, CodeBuddy beta hooks are not auto-installed, and Cursor context hooks remain withheld. - **Subagent access is explicit** — Claude, Gemini, Kiro, Qwen, CodeBuddy, KiloCode, Mistral Vibe, Grok Build, Qoder, Junie, and Factory get documented graph profiles with the narrowest tool/server filters their schemas support. KiloCode and Vibe enumerate read-only query tools rather than using server wildcards. Cursor, Rovo, Pochi, and Cline use explicit parent handoff where child MCP is unavailable or unsafe; IBM Bob receives no invented hook or agent. - **Manual, UI, cloud, or repository-managed (not counted)** — Qodo, Warp MCP, JetBrains AI/ACP, GitHub Copilot coding agent, Jules, CodeRabbit, Replit, BLACKBOX AI, Plandex, and SWE-agent. Warp is counted above for its detected skill installation; its MCP connection remains manual. -- **15 MCP tools** — search, trace, architecture, impact analysis, Cypher queries, dead code detection, cross-service HTTP linking, ADR management, and more. +- **17 MCP tools** — search, trace, architecture, impact analysis, Cypher queries, dead code detection, cross-service HTTP linking, ADR management, and more. ## Supported Platforms From e0becdeb094dd5e898997a2add826a9f84befbd7 Mon Sep 17 00:00:00 2001 From: yasmoradi Date: Thu, 17 Sep 2026 22:25:14 +0200 Subject: [PATCH 084/130] fix(discover): map MSBuild, resx, XAML and app-manifest files to XML The XML documents that make up a .NET repository project system were absent from EXT_TABLE, so .csproj, .props, .targets, .resx, .plist, .xaml and the app manifests were never indexed and their content was unreachable through search_code. tree_sitter_xml is already wired up, so these only needed rows in the table. Fixes #2229 Signed-off-by: yasmoradi --- src/discover/language.c | 22 ++++++++++++++++++++++ tests/test_language.c | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/src/discover/language.c b/src/discover/language.c index e61a07c43..c9d92719a 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -321,6 +321,28 @@ static const ext_entry_t EXT_TABLE[] = { {".xsd", CBM_LANG_XML}, {".xsl", CBM_LANG_XML}, {".svg", CBM_LANG_XML}, + /* MSBuild project system. Plain XML documents, and the files that carry a + * .NET repository's package references, target frameworks, build hooks and + * project layout — none of which were reachable while these were unmapped. */ + {".csproj", CBM_LANG_XML}, + {".vbproj", CBM_LANG_XML}, + {".fsproj", CBM_LANG_XML}, + {".props", CBM_LANG_XML}, + {".targets", CBM_LANG_XML}, + {".nuspec", CBM_LANG_XML}, + {".slnx", CBM_LANG_XML}, + {".runsettings", CBM_LANG_XML}, + /* .NET resource files: the localized strings a UI reads back by key. */ + {".resx", CBM_LANG_XML}, + /* XAML views: WPF, WinUI, MAUI (.xaml) and Avalonia (.axaml). */ + {".xaml", CBM_LANG_XML}, + {".axaml", CBM_LANG_XML}, + /* Application manifests: Apple property lists and privacy manifests, Win32 + * side-by-side manifests and MSIX packages. */ + {".plist", CBM_LANG_XML}, + {".xcprivacy", CBM_LANG_XML}, + {".manifest", CBM_LANG_XML}, + {".appxmanifest", CBM_LANG_XML}, /* YAML */ {".yaml", CBM_LANG_YAML}, diff --git a/tests/test_language.c b/tests/test_language.c index 6dbc9b49c..18ad09600 100644 --- a/tests/test_language.c +++ b/tests/test_language.c @@ -413,6 +413,36 @@ TEST(lang_ext_svg) { ASSERT_EQ(cbm_language_for_extension(".svg"), CBM_LANG_XML); PASS(); } +/* Issue #2229: the XML documents that make up a .NET repository's project + * system were unmapped, so a solution's package references, target frameworks, + * localized strings and app manifests were never indexed or searchable. */ +TEST(lang_ext_msbuild_projects) { + ASSERT_EQ(cbm_language_for_extension(".csproj"), CBM_LANG_XML); + ASSERT_EQ(cbm_language_for_extension(".vbproj"), CBM_LANG_XML); + ASSERT_EQ(cbm_language_for_extension(".fsproj"), CBM_LANG_XML); + ASSERT_EQ(cbm_language_for_extension(".props"), CBM_LANG_XML); + ASSERT_EQ(cbm_language_for_extension(".targets"), CBM_LANG_XML); + ASSERT_EQ(cbm_language_for_extension(".nuspec"), CBM_LANG_XML); + ASSERT_EQ(cbm_language_for_extension(".slnx"), CBM_LANG_XML); + ASSERT_EQ(cbm_language_for_extension(".runsettings"), CBM_LANG_XML); + PASS(); +} +TEST(lang_ext_resx) { + ASSERT_EQ(cbm_language_for_extension(".resx"), CBM_LANG_XML); + PASS(); +} +TEST(lang_ext_xaml) { + ASSERT_EQ(cbm_language_for_extension(".xaml"), CBM_LANG_XML); + ASSERT_EQ(cbm_language_for_extension(".axaml"), CBM_LANG_XML); + PASS(); +} +TEST(lang_ext_app_manifests) { + ASSERT_EQ(cbm_language_for_extension(".plist"), CBM_LANG_XML); + ASSERT_EQ(cbm_language_for_extension(".xcprivacy"), CBM_LANG_XML); + ASSERT_EQ(cbm_language_for_extension(".manifest"), CBM_LANG_XML); + ASSERT_EQ(cbm_language_for_extension(".appxmanifest"), CBM_LANG_XML); + PASS(); +} TEST(lang_ext_markdown) { ASSERT_EQ(cbm_language_for_extension(".md"), CBM_LANG_MARKDOWN); PASS(); @@ -1361,6 +1391,10 @@ SUITE(language) { RUN_TEST(lang_ext_xsl); RUN_TEST(lang_ext_xsd); RUN_TEST(lang_ext_svg); + RUN_TEST(lang_ext_msbuild_projects); + RUN_TEST(lang_ext_resx); + RUN_TEST(lang_ext_xaml); + RUN_TEST(lang_ext_app_manifests); RUN_TEST(lang_ext_markdown); RUN_TEST(lang_ext_mdx); RUN_TEST(lang_ext_makefile); From 81264814f162ebfbecfc609a951afa93aa43f8c9 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 18 Sep 2026 03:30:39 +0200 Subject: [PATCH 085/130] feat(qa): memory + CPU waste sanitizer over the memory core MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answers a question the existing tooling could not: of everything we allocate, how much is never used? Lanes, all driven by scripts/memwaste.sh: event every allocation observed through the memory core — size, age, churn, and whether the bytes were ever written access SanitizerCoverage load/store instrumentation, so "allocated and never touched" is measured rather than inferred scaling k vs 2k replicas through clang profile counters, to catch super-linear work per file padding allocator slack per size class ratchet the numbers recorded per venue so a regression is visible The event lane needed a foreign-free counter: on Windows the override sees frees for blocks it never allocated (they come from the CRT), and counting those as untracked tripped the soundness gate. mem_events now separates them (14 daemon / 5 worker on the Go corpus). Two portability fixes the lanes surfaced, both real: - the enable decision must not be cached before `environ` exists. Under -fsanitize-coverage the UBSan preinit runs dlsym -> malloc -> our wrapper before glibc sets up environ, so the lane silently disabled itself and produced no dump at all on Linux. - the access map needs 48 bits: arm64 Linux hands out addresses above 2^47, which a 47-bit map dropped on the floor. mem_profile is retired into this: it measured a strict subset and had its own sampling story. Local-only by decision — not a PR-CI gate. _memwaste.yml runs the lanes on all three platforms report-only, and pr.yml/release.yml carry the wiring. Signed-off-by: Martin Vogel --- .github/workflows/_memwaste.yml | 163 ++ .github/workflows/pr.yml | 11 + .github/workflows/release.yml | 9 + Makefile.cbm | 132 +- scripts/memlab-drive.py | 24 +- scripts/memlab-report.py | 95 +- scripts/memlab.sh | 23 +- scripts/memwaste-padding.py | 174 ++ scripts/memwaste-ratchet.py | 105 ++ scripts/memwaste-report.py | 392 +++++ scripts/memwaste-scaling.py | 262 ++++ scripts/memwaste.sh | 182 +++ scripts/sanitizer-coverage-map.py | 306 ++++ scripts/sanitizer-map.json | 126 ++ src/foundation/mem_core.c | 64 + src/foundation/mem_events.c | 2178 ++++++++++++++++++++++++++ src/foundation/mem_events.h | 359 +++++ src/foundation/mem_override_darwin.c | 267 ++++ src/foundation/mem_override_libc.c | 738 +++++++++ src/foundation/mem_override_posix.c | 84 +- src/foundation/mem_override_win.c | 61 +- src/foundation/mem_profile.c | 325 ---- src/foundation/mem_profile.h | 66 - src/main.c | 8 + tests/test_mem_events.c | 510 ++++++ 25 files changed, 6192 insertions(+), 472 deletions(-) create mode 100644 .github/workflows/_memwaste.yml create mode 100644 scripts/memwaste-padding.py create mode 100644 scripts/memwaste-ratchet.py create mode 100644 scripts/memwaste-report.py create mode 100644 scripts/memwaste-scaling.py create mode 100644 scripts/memwaste.sh create mode 100644 scripts/sanitizer-coverage-map.py create mode 100644 scripts/sanitizer-map.json create mode 100644 src/foundation/mem_events.c create mode 100644 src/foundation/mem_events.h create mode 100644 src/foundation/mem_override_darwin.c create mode 100644 src/foundation/mem_override_libc.c delete mode 100644 src/foundation/mem_profile.c delete mode 100644 src/foundation/mem_profile.h create mode 100644 tests/test_mem_events.c diff --git a/.github/workflows/_memwaste.yml b/.github/workflows/_memwaste.yml new file mode 100644 index 000000000..ab9e1e3c6 --- /dev/null +++ b/.github/workflows/_memwaste.yml @@ -0,0 +1,163 @@ +# Reusable: the waste sanitizer (memory + CPU waste), REPORT-ONLY. +# +# What runs (scripts/memwaste.sh, memwaste-scaling.py, memwaste-ratchet.py): +# mode=pr Linux: the event lane (memory waste + CPU work) indexing this +# repository at the checked-out commit, and the CPU scaling lane +# (src/ replicated 2x against 4x, per-function counter ratios). +# mode=full the same, plus the event lane on macOS and Windows and the access +# lane (read/write verdicts per allocation) on Linux. +# +# The corpus is the repository itself: pinned by the commit, no network, and a +# real mix of C, C++, Python, JavaScript and vendored code. +# +# REPORT-ONLY: every job uploads its report and prints the ratchet comparison +# against scripts/memwaste-baseline.json, but nothing here can turn a PR red -- +# the ratchet runs without --enforce and the PR's ci-ok gate does not depend on +# these jobs. A job fails only when the MEASUREMENT is unsound (a table +# overflowed, frees the layer never saw): that is a defect of the tool, and a +# red job is how it gets noticed. Switching the ratchet to --enforce is a +# separate, deliberate change once the numbers have been stable across reruns. +name: Memwaste + +on: + workflow_call: + inputs: + mode: + description: "pr (Linux event + scaling) or full (all platforms, access lane)" + type: string + default: pr + +permissions: + contents: read + +jobs: + event-linux: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install deps + run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 clang llvm + - name: Event lane on this repository + run: | + mkdir -p "$RUNNER_TEMP/memwaste-event" + scripts/memwaste.sh "$GITHUB_WORKSPACE" --top 40 --out "$RUNNER_TEMP/memwaste-event" \ + | tee "$RUNNER_TEMP/memwaste-event/report.txt" + - name: Ratchet (report-only) + run: python3 scripts/memwaste-ratchet.py --out "$RUNNER_TEMP/memwaste-event" --venue ${{ inputs.mode }}-linux + - name: Upload report + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: memwaste-event-linux-${{ inputs.mode }} + path: | + ${{ runner.temp }}/memwaste-event/report.txt + ${{ runner.temp }}/memwaste-event/memwaste.json + ${{ runner.temp }}/memwaste-event/index.json + if-no-files-found: warn + + scaling-linux: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install deps + run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 clang llvm + - name: CPU scaling lane (src/ at 2x against 4x) + # Report-only: the script exits 1 when it flags a super-linear function. + continue-on-error: true + run: | + mkdir -p "$RUNNER_TEMP/memwaste-scaling" + python3 scripts/memwaste-scaling.py --corpus src --replicas 2 \ + --out "$RUNNER_TEMP/memwaste-scaling" | tee "$RUNNER_TEMP/memwaste-scaling/report.txt" + - name: Upload report + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: memwaste-scaling-linux-${{ inputs.mode }} + path: ${{ runner.temp }}/memwaste-scaling/report.txt + if-no-files-found: warn + + access-linux: + if: ${{ inputs.mode == 'full' }} + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install deps + run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 clang llvm + - name: Access lane on this repository + run: | + mkdir -p "$RUNNER_TEMP/memwaste-access" + scripts/memwaste.sh "$GITHUB_WORKSPACE" --lane access --top 40 --sort untouched \ + --out "$RUNNER_TEMP/memwaste-access" | tee "$RUNNER_TEMP/memwaste-access/report.txt" + - name: Upload report + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: memwaste-access-linux-${{ inputs.mode }} + path: | + ${{ runner.temp }}/memwaste-access/report.txt + ${{ runner.temp }}/memwaste-access/memwaste.json + if-no-files-found: warn + + event-macos: + if: ${{ inputs.mode == 'full' }} + runs-on: macos-14 + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Event lane on this repository + run: | + mkdir -p "$RUNNER_TEMP/memwaste-event" + scripts/memwaste.sh "$GITHUB_WORKSPACE" --top 40 --out "$RUNNER_TEMP/memwaste-event" \ + | tee "$RUNNER_TEMP/memwaste-event/report.txt" + - name: Ratchet (report-only) + run: python3 scripts/memwaste-ratchet.py --out "$RUNNER_TEMP/memwaste-event" --venue full-macos + - name: Upload report + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: memwaste-event-macos-${{ inputs.mode }} + path: | + ${{ runner.temp }}/memwaste-event/report.txt + ${{ runner.temp }}/memwaste-event/memwaste.json + if-no-files-found: warn + + event-windows: + if: ${{ inputs.mode == 'full' }} + runs-on: windows-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # v2 + with: + msystem: CLANG64 + path-type: inherit + install: >- + mingw-w64-clang-x86_64-clang + mingw-w64-clang-x86_64-llvm + mingw-w64-clang-x86_64-zlib + mingw-w64-clang-x86_64-python3 + make + git + coreutils + - name: Event lane on this repository + shell: msys2 {0} + run: | + out="$(cygpath -u "$RUNNER_TEMP")/memwaste-event" + mkdir -p "$out" + CC=clang CXX=clang++ scripts/memwaste.sh "$(cygpath -u "$GITHUB_WORKSPACE")" --top 40 --out "$out" \ + | tee "$out/report.txt" + - name: Ratchet (report-only) + shell: msys2 {0} + run: python3 scripts/memwaste-ratchet.py --out "$(cygpath -u "$RUNNER_TEMP")/memwaste-event" --venue full-windows + - name: Upload report + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: memwaste-event-windows-${{ inputs.mode }} + path: | + ${{ runner.temp }}/memwaste-event/report.txt + ${{ runner.temp }}/memwaste-event/memwaste.json + if-no-files-found: warn diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 4fef6ea68..0cfc796be 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -154,6 +154,17 @@ jobs: CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 1000M + # ── Waste sanitizer, REPORT-ONLY: memory + CPU waste on this repository and + # the per-function scaling lane (Linux). Deliberately NOT in ci-ok's needs: + # it uploads evidence and prints the ratchet, it never blocks a merge. Only + # product-code PRs pay for it. ── + memwaste: + needs: [changes] + if: ${{ !cancelled() && needs.changes.outputs.product == 'true' }} + uses: ./.github/workflows/_memwaste.yml + with: + mode: pr + ci-ok: # The one required context (besides dco) — fails unless every PR stage # succeeded, so matrix renames can never silently deadlock merges. `skipped` diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 15f19f41f..c0e796364 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -145,6 +145,15 @@ jobs: version: ${{ inputs.version }} use_release_artifacts: true + # ── Waste sanitizer, REPORT-ONLY ──────────────────────────────── + # The same full run as the dry run, on the release commit. Not in + # release-draft's needs: the report is evidence for the release notes, never + # a publication gate. + memwaste: + uses: ./.github/workflows/_memwaste.yml + with: + mode: full + # ── 6. Create DRAFT release ──────────────────────────────────── # Requires smoke to have actually SUCCEEDED, not merely "not failed". The bare # !cancelled() && !failure() form is fail-OPEN: a skipped smoke is neither diff --git a/Makefile.cbm b/Makefile.cbm index 066d3375c..317b51411 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -76,8 +76,8 @@ endif # the prod build uses mimalloc (MI_OVERRIDE=1); the test build is CRT+ASan, where # binding would create an alloc/free mismatch, so the guard is prod-only. CFLAGS_PROD = $(CFLAGS_COMMON) -O2 -DCBM_BIND_TS_ALLOCATOR=1 $(TEST_SEAM_DEFINE) \ - $(GLOBAL_OVERRIDE_DEFINE) $(CFLAGS_EXTRA) -CXXFLAGS_PROD = $(CXXFLAGS_COMMON) -O2 + $(GLOBAL_OVERRIDE_DEFINE) $(MEMWASTE_CFLAGS) $(CFLAGS_EXTRA) +CXXFLAGS_PROD = $(CXXFLAGS_COMMON) -O2 $(MEMWASTE_VENDOR_CFLAGS) # Test flags: debug + sanitizers (override SANITIZE= to disable on Windows) SANITIZE = -fsanitize=address,undefined -fno-omit-frame-pointer @@ -216,6 +216,75 @@ ELF_HARDENING_FLAGS := -Wl,-z,noexecstack -Wl,-z,separate-code \ endif endif +# MEMWASTE=1: the waste-sanitizer flavour (src/foundation/mem_events.h). Built +# like production -- -O2, mimalloc, the real allocator routing -- plus the event +# layer and its observers, because waste measured under ASan's allocator would +# describe ASan. Frame pointers stay so the macOS observer can walk to the +# caller. On Linux this is also what finally LINKS the --wrap observation shim: +# its flags and source were defined here but never reached a link line, so the +# Linux profiler had been reporting nothing. +# +# Every object is built this way, VENDORED ones included (grammars, the +# tree-sitter runtime, SQLite, tre, lz4, zstd, the C++ preprocessor): waste in +# a vendored parser is waste in this program. mimalloc alone is left as it is -- +# its reads and writes are the allocator's own bookkeeping, not program data. +# +# MEMWASTE_ACCESS=1 adds the access lane (reads and writes of every tracked +# block through the compiler's load/store callbacks; clang only). It builds at +# -O1 without vectorisation: at -O2 the callbacks describe the OPTIMISED program, +# and reads the optimiser removed would be reported as never read. +# MEMWASTE_PROFILE=1 adds clang's execution counters (-fprofile-instr-generate) +# for the CPU scaling lane (scripts/memwaste-scaling.py). +# Each flavour needs its own BUILD_DIR: vendored objects are not rebuilt when +# only the flavour changes. +# +# The libc wrappers (mem_override_libc.c) need --wrap on Linux and Windows; +# macOS defines the symbols instead. STATIC=1 (musl) is not a memwaste target. +MEMWASTE ?= 0 +MEMWASTE_ACCESS ?= 0 +MEMWASTE_PROFILE ?= 0 +ifeq ($(MEMWASTE_ACCESS),1) +override MEMWASTE := 1 +endif +ifeq ($(MEMWASTE_PROFILE),1) +override MEMWASTE := 1 +endif +MEMWASTE_CFLAGS := +MEMWASTE_VENDOR_CFLAGS := +MEMWASTE_LDFLAGS := +MEMWASTE_SRCS := +MEMWASTE_ALLOC_SYMS_LINUX := malloc calloc realloc free strdup strndup posix_memalign \ + aligned_alloc memalign +MEMWASTE_LIBC_SYMS_LINUX := memcpy memmove memset memcmp strlen strcmp strncmp \ + __memcpy_chk __memmove_chk __memset_chk read __read_chk write \ + pread pwrite pread64 pwrite64 open open64 __open_2 __open64_2 \ + stat lstat fopen fopen64 fread __fread_chk fwrite opendir readdir \ + pthread_mutex_lock +MEMWASTE_LIBC_SYMS_WIN := memcpy memmove memset memcmp strlen strcmp strncmp _wfopen fread fwrite +ifeq ($(MEMWASTE),1) +MEMWASTE_CFLAGS := -DCBM_MEMWASTE=1 -fno-omit-frame-pointer +MEMWASTE_VENDOR_CFLAGS := -DCBM_MEMWASTE=1 -fno-omit-frame-pointer +ifeq ($(IS_LINUX),yes) +MEMWASTE_LDFLAGS := $(foreach sym,$(MEMWASTE_ALLOC_SYMS_LINUX) $(MEMWASTE_LIBC_SYMS_LINUX),-Wl,--wrap=$(sym)) +MEMWASTE_SRCS := src/foundation/mem_override_posix.c +endif +ifeq ($(IS_MINGW),yes) +MEMWASTE_LDFLAGS := $(foreach sym,$(MEMWASTE_LIBC_SYMS_WIN),-Wl,--wrap=$(sym)) +endif +endif +ifeq ($(MEMWASTE_ACCESS),1) +MEMWASTE_ACCESS_FLAGS := -DCBM_MEMWASTE_ACCESS=1 -O1 -fno-vectorize -fno-slp-vectorize \ + -fsanitize-coverage=inline-bool-flag,trace-loads,trace-stores +MEMWASTE_CFLAGS += $(MEMWASTE_ACCESS_FLAGS) +MEMWASTE_VENDOR_CFLAGS += $(MEMWASTE_ACCESS_FLAGS) +endif +ifeq ($(MEMWASTE_PROFILE),1) +MEMWASTE_PROFILE_FLAGS := -DCBM_MEMWASTE_PROFILE=1 -fprofile-instr-generate -fprofile-update=atomic +MEMWASTE_CFLAGS += $(MEMWASTE_PROFILE_FLAGS) +MEMWASTE_VENDOR_CFLAGS += $(MEMWASTE_PROFILE_FLAGS) +MEMWASTE_LDFLAGS += $(MEMWASTE_PROFILE_FLAGS) +endif + # The POSIX wrap shim exists only so the profiler can observe allocations. It # must never reach a sanitized build: the Linux/macOS test builds are CRT+ASan, # and redirecting malloc into mimalloc underneath ASan's own interception mixes @@ -225,7 +294,7 @@ endif # instead of the system libstdc++ (scripts/msan.sh overrides both vars). CXX_STDLIB ?= -lstdc++ CXX_STDLIB_FLAGS ?= -LDFLAGS = -lm $(CXX_STDLIB) -lpthread -lz $(WIN32_LIBS) $(STATIC_FLAGS) $(MIMALLOC_WRAP_FLAGS) $(ELF_HARDENING_FLAGS) +LDFLAGS = -lm $(CXX_STDLIB) -lpthread -lz $(WIN32_LIBS) $(STATIC_FLAGS) $(MIMALLOC_WRAP_FLAGS) $(MEMWASTE_LDFLAGS) $(ELF_HARDENING_FLAGS) LDFLAGS_TEST = -lm $(CXX_STDLIB) -lpthread -lz $(SANITIZE) $(WIN32_LIBS) $(MIMALLOC_WRAP_FLAGS) $(ELF_HARDENING_FLAGS) LDFLAGS_TSAN = -lm $(CXX_STDLIB) -lpthread -lz $(TSAN_SANITIZE) $(WIN32_LIBS) $(MIMALLOC_WRAP_FLAGS) $(ELF_HARDENING_FLAGS) @@ -249,6 +318,9 @@ FOUNDATION_SRCS = \ src/foundation/compat_regex.c \ src/foundation/mem.c \ src/foundation/mem_core.c \ + src/foundation/mem_events.c \ + src/foundation/mem_override_darwin.c \ + src/foundation/mem_override_libc.c \ src/foundation/diagnostics.c \ src/foundation/profile.c \ src/foundation/dump_verify.c \ @@ -472,10 +544,19 @@ ifneq ($(filter yes,$(IS_MINGW) $(IS_LINUX)),) MIMALLOC_OVERRIDE_DEFINE := -DMI_MALLOC_OVERRIDE=1 GLOBAL_OVERRIDE_DEFINE := -DCBM_MEM_GLOBAL_OVERRIDE=1 endif +# MI_DEBUG=0: the production allocator is mimalloc's RELEASE build. Its header +# defaults to MI_DEBUG=2 unless NDEBUG is defined, and nothing here defined it, +# so every shipped binary up to v0.11.0 ran mimalloc's debug build: internal +# assertions, fine-grained statistics, padding on every block and a memset of +# every freed block. The waste sanitizer measured it on the Go corpus as 92 GB +# (macOS) and 444 GB (Linux, where mimalloc also serves libc's malloc) of byte +# work inside the allocator (2026-09-17). Committed-bytes accounting, which the +# memory budget reads, is kept in every build (subproc stats, not MI_STAT). +# The TEST build below keeps mimalloc's debug checks. MIMALLOC_CFLAGS = -std=c11 -O2 -w \ -Ivendored/mimalloc/include \ -Ivendored/mimalloc/src \ - -DMI_OVERRIDE=1 \ + -DMI_OVERRIDE=1 -DMI_DEBUG=0 \ $(MIMALLOC_OVERRIDE_DEFINE) MIMALLOC_CFLAGS_TEST = -std=c11 -g -O1 -w \ -Ivendored/mimalloc/include \ @@ -496,7 +577,7 @@ MIMALLOC_CFLAGS_TEST = -std=c11 -g -O1 -w \ SQLITE3_OMIT_FLAGS = -DSQLITE_OMIT_LOAD_EXTENSION SQLITE3_SRC = vendored/sqlite3/sqlite3.c SQLITE3_CFLAGS = -std=c11 -O2 -w -DSQLITE_DQS=0 -DSQLITE_THREADSAFE=1 -DSQLITE_ENABLE_FTS5 \ - $(SQLITE3_OMIT_FLAGS) + $(SQLITE3_OMIT_FLAGS) $(MEMWASTE_VENDOR_CFLAGS) SQLITE3_CFLAGS_TEST = -std=c11 -g -O1 -w -DSQLITE_DQS=0 -DSQLITE_THREADSAFE=1 -DSQLITE_ENABLE_FTS5 \ $(SQLITE3_OMIT_FLAGS) @@ -508,7 +589,7 @@ TRE_CFLAGS = -std=c11 -g -O1 -w -Ivendored/tre YYJSON_SRC = vendored/yyjson/yyjson.c # All production sources -PROD_SRCS = $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) $(DAEMON_SRCS) $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(SEMANTIC_SRCS) $(TRACES_SRCS) $(WATCHER_SRCS) $(GIT_SRCS) $(CLI_SRCS) $(UI_SRCS) $(YYJSON_SRC) +PROD_SRCS = $(MEMWASTE_SRCS) $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) $(DAEMON_SRCS) $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(SEMANTIC_SRCS) $(TRACES_SRCS) $(WATCHER_SRCS) $(GIT_SRCS) $(CLI_SRCS) $(UI_SRCS) $(YYJSON_SRC) EXISTING_C_SRCS = $(EXTRACTION_SRCS) $(LSP_SRCS) $(TS_RUNTIME_SRC) \ $(GRAMMAR_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) @@ -641,7 +722,7 @@ TEST_CLI_SRCS = tests/test_cli.c tests/test_agent_clients.c tests/test_agent_pro tests/test_config_toml_edit.c tests/test_config_yaml_edit.c tests/test_config_text_edit.c \ tests/test_activation_transaction.c -TEST_MEM_SRCS = tests/test_mem.c +TEST_MEM_SRCS = tests/test_mem.c tests/test_mem_events.c TEST_UI_SRCS = tests/test_ui.c TEST_HTTPD_SRCS = tests/test_httpd.c @@ -746,7 +827,8 @@ BUILD_DIR = build/c # ── Object file compilation (grammars need relaxed warnings) ───── # Grammar + tree-sitter runtime: compiled without -Werror (upstream code has warnings) -GRAMMAR_CFLAGS = -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) +GRAMMAR_CFLAGS = -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \ + $(MEMWASTE_VENDOR_CFLAGS) GRAMMAR_CFLAGS_TEST = -std=c11 -D_DEFAULT_SOURCE -g -O1 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \ -DCBM_ENABLE_TEST_SEAMS=1 $(SANITIZED_DEFINE) $(SANITIZE) GRAMMAR_CFLAGS_TSAN = -std=c11 -D_DEFAULT_SOURCE -g -O1 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \ @@ -764,7 +846,7 @@ PP_OBJ_TSAN = $(BUILD_DIR)/tsan_preprocessor.o # ── Targets ────────────────────────────────────────────────────── -.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-tidy-diff lint-cppcheck lint-format security +.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-tidy-diff lint-cppcheck lint-format security fuzz $(BUILD_DIR): mkdir -p $(BUILD_DIR) @@ -868,7 +950,7 @@ $(BUILD_DIR)/tsan_tre.o: $(TRE_SRC) | $(BUILD_DIR) $(CC) $(TRE_CFLAGS) $(TSAN_SANITIZE) -c -o $@ $< $(BUILD_DIR)/prod_tre.o: $(TRE_SRC) | $(BUILD_DIR) - $(CC) $(TRE_CFLAGS) -O2 -c -o $@ $< + $(CC) $(TRE_CFLAGS) -O2 $(MEMWASTE_VENDOR_CFLAGS) -c -o $@ $< endif # Vendored LZ4 (test build) @@ -1036,14 +1118,14 @@ $(BUILD_DIR)/prod_preprocessor.o: $(CBM_DIR)/preprocessor.cpp | $(BUILD_DIR) # Vendored LZ4 (compiled separately, not unity-built via lz4_store.c) LZ4_OBJ_PROD = $(BUILD_DIR)/prod_lz4.o $(BUILD_DIR)/prod_lz4hc.o $(BUILD_DIR)/prod_lz4.o: $(CBM_DIR)/vendored/lz4/lz4.c | $(BUILD_DIR) - $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR) -c -o $@ $< + $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w $(MEMWASTE_VENDOR_CFLAGS) -I$(CBM_DIR) -c -o $@ $< $(BUILD_DIR)/prod_lz4hc.o: $(CBM_DIR)/vendored/lz4/lz4hc.c | $(BUILD_DIR) - $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR)/vendored/lz4 -c -o $@ $< + $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w $(MEMWASTE_VENDOR_CFLAGS) -I$(CBM_DIR)/vendored/lz4 -c -o $@ $< # Vendored zstd (compiled separately, not unity-built via zstd_store.c) ZSTD_OBJ_PROD = $(BUILD_DIR)/prod_zstd.o $(BUILD_DIR)/prod_zstd.o: $(CBM_DIR)/vendored/zstd/zstd.c | $(BUILD_DIR) - $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR)/vendored/zstd -c -o $@ $< + $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w $(MEMWASTE_VENDOR_CFLAGS) -I$(CBM_DIR)/vendored/zstd -c -o $@ $< OBJS_VENDORED_PROD = $(MIMALLOC_OBJ_PROD) $(SQLITE3_OBJ_PROD) $(TRE_OBJ_PROD) $(GRAMMAR_OBJS_PROD) $(TS_RUNTIME_OBJ_PROD) $(LSP_OBJ_PROD) $(PP_OBJ_PROD) $(LZ4_OBJ_PROD) $(ZSTD_OBJ_PROD) $(UNIXCODER_OBJ) @@ -1065,7 +1147,7 @@ MAIN_SRC = src/main.c # for trap debugging -- silently re-run the previously instrumented binary, # which during this bisect produced a "crashes without sanitizers too" result # that was pure artifact and sent the investigation down the wrong path. -BUILD_CONFIG_SIG := TEST_SEAMS=$(TEST_SEAMS)|CFLAGS_EXTRA=$(CFLAGS_EXTRA)|SANITIZE=$(SANITIZE) +BUILD_CONFIG_SIG := TEST_SEAMS=$(TEST_SEAMS)|CFLAGS_EXTRA=$(CFLAGS_EXTRA)|SANITIZE=$(SANITIZE)|MEMWASTE=$(MEMWASTE)|MEMWASTE_ACCESS=$(MEMWASTE_ACCESS)|MEMWASTE_PROFILE=$(MEMWASTE_PROFILE) .PHONY: build-config-check build-config-check: $(BUILD_DIR)/.build-config: build-config-check | $(BUILD_DIR) @@ -1261,6 +1343,28 @@ test-lsan: cd $(CURDIR) && ASAN_OPTIONS="detect_leaks=1:halt_on_error=1" \ ./build/lsan/test-runner $(LSAN_SUITES) +# ── Coverage-guided fuzzing (libFuzzer) ───────────────────────── +# One binary, three targets (tests/fuzz/cbm_fuzz.c): the per-file extractor, +# the Cypher parser, the TOML/YAML/JSON config editors. ASan+UBSan as in the +# test lane, plus fuzzer-no-link on EVERY object -- the vendored grammars and +# the tree-sitter runtime feed coverage back, which is where the parser state +# space lives. Needs a clang that ships libFuzzer (Homebrew LLVM on macOS; Apple +# clang does not). Run through scripts/fuzz.sh. +FUZZ_CC ?= clang +FUZZ_CXX ?= clang++ +FUZZ_SANITIZE = -fsanitize=address,undefined,fuzzer-no-link -fno-omit-frame-pointer +FUZZ_SRC = tests/fuzz/cbm_fuzz.c +fuzz: + $(MAKE) -f Makefile.cbm build/fuzz/cbm-fuzz BUILD_DIR=build/fuzz \ + CC=$(FUZZ_CC) CXX=$(FUZZ_CXX) SANITIZE="$(FUZZ_SANITIZE)" + +$(BUILD_DIR)/cbm-fuzz: $(FUZZ_SRC) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TEST) $(PROJECT_HDRS) | $(BUILD_DIR) + $(CC) $(CFLAGS_TEST) -fsanitize=fuzzer -o $@ \ + $(FUZZ_SRC) $(PROD_SRCS) \ + $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) \ + $(OBJS_VENDORED_TEST) \ + $(LDFLAGS_TEST) + # Path-sensitive memory analysis: leak paths, null derefs, uninitialized reads. # Non-gating: triage findings against the code (this lane's first run produced # 9 real fixes and 6 recorded false positives — see the analyzer batch commit). diff --git a/scripts/memlab-drive.py b/scripts/memlab-drive.py index cef508810..bac474126 100644 --- a/scripts/memlab-drive.py +++ b/scripts/memlab-drive.py @@ -32,6 +32,16 @@ def rpc(proc, payload, timeout_note): return message +def tool_failed(reply): + """A JSON-RPC error, or a tool result the server marked isError: MCP reports + a failed tool call as a normal result, so checking only "error" counts every + failure as served (a lab measuring nothing looked healthy).""" + if "error" in reply: + return True + result = reply.get("result") + return isinstance(result, dict) and bool(result.get("isError")) + + def main(): parser = argparse.ArgumentParser() parser.add_argument("binary") @@ -63,14 +73,16 @@ def main(): "params": {"protocolVersion": "2024-11-05", "capabilities": {}, "clientInfo": {"name": "memlab", "version": "1"}}}, "initialize") + project = None if not args.skip_index: indexed = rpc(proc, {"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": "index_repository", - "arguments": {"path": args.corpus}}}, + "arguments": {"repo_path": args.corpus}}}, "index_repository") - if "error" in indexed: - print(f"index failed: {indexed['error']}", file=sys.stderr) + if tool_failed(indexed): + print(f"index failed: {indexed.get('error') or indexed.get('result')}", file=sys.stderr) return 3 + project = (indexed.get("result", {}).get("structuredContent") or {}).get("project") halfway = 2 + args.requests // 2 for i in range(2, args.requests + 2): @@ -84,11 +96,15 @@ def main(): "list_projects": {}, "get_graph_schema": {}, "search_code": {"pattern": "Widget"}}.get(args.tool, {}) + if project and args.tool != "list_projects": + arguments = dict(arguments, project=project) reply = rpc(proc, {"jsonrpc": "2.0", "id": i, "method": "tools/call", "params": {"name": args.tool, "arguments": arguments}}, f"request {i}") served += 1 - if "error" in reply: + if tool_failed(reply): + if failures == 0: + print(f"first failure (request {i}): {json.dumps(reply)[:600]}", file=sys.stderr, flush=True) failures += 1 finally: try: diff --git a/scripts/memlab-report.py b/scripts/memlab-report.py index cf538fa7d..eae4bccf8 100644 --- a/scripts/memlab-report.py +++ b/scripts/memlab-report.py @@ -1,19 +1,20 @@ #!/usr/bin/env python3 """Analyse #581 memory-lab output: attribute growth to allocation sites. -Reads the JSONL the profiler appends (CBM_MEM_PROFILE_OUT) and the mem.census -lines from a daemon log, and prints: +Reads the waste layer's dumps (CBM_MEMWASTE_OUT, one at every request-stage phase +mark; see scripts/memlab.sh) and the mem.census lines from a daemon log, and +prints: * retention per request, as a linear fit rather than first-vs-last, so noise does not masquerade as a trend * the call sites whose live bytes grow across the run, ranked by growth - * an explicit unattributed remainder: process growth the profiler could not - account for. That number is the honesty check — a report that attributed - everything while tracking only >=32 KiB allocations would be lying. + * an explicit unattributed remainder: process growth the layer could not + account for (allocator metadata, fragmentation, memory outside malloc). + That number is the honesty check. Usage: - memlab-report.py [--census ] [--top N] - memlab-report.py --diff # platform comparison + memlab-report.py [--census ] [--binary BIN] [--top N] + memlab-report.py --diff [--binary BIN] # platform comparison """ import argparse import json @@ -23,9 +24,11 @@ from collections import defaultdict def load_profile(path): - """Return (summaries, site_series). Records are appended over time, so the - Nth occurrence of a site is its Nth sample.""" - summaries, series = [], defaultdict(list) + """Return (summaries, site_series) for the process with the most dumps. + Every dump lists every site, so the Nth occurrence of a site is its Nth + sample; a site first seen late is padded with zeros so series align.""" + procs = defaultdict(lambda: {"summaries": [], "series": defaultdict(list)}) + cur = None with open(path, "r", errors="replace") as handle: for line in handle: line = line.strip() @@ -35,11 +38,31 @@ def load_profile(path): record = json.loads(line) except json.JSONDecodeError: continue - if "site" in record: - series[record["site"]].append(record) - else: - summaries.append(record) - return summaries, series + if "memwaste" in record: + cur = procs[record.get("pid", 0)] + cur["summaries"].append(record) + elif cur is not None and "site" in record and "work" not in record: + samples = cur["series"][record["site"]] + while len(samples) < len(cur["summaries"]) - 1: + samples.append({"live_bytes": 0}) + samples.append(record) + if not procs: + return [], {} + best = max(procs.values(), key=lambda p: len(p["summaries"])) + return best["summaries"], best["series"] + + +def load_symbols(binary, summaries, sites): + """Symbolise site addresses with the waste report's own symboliser.""" + if not binary or not summaries: + return {} + import importlib.util + import os + spec = importlib.util.spec_from_file_location( + "memwaste_report", os.path.join(os.path.dirname(os.path.abspath(__file__)), "memwaste-report.py")) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod.symbolise(binary, summaries[-1].get("image_base", "0x0"), list(sites)) def linear_slope(values): @@ -57,7 +80,7 @@ def linear_slope(values): def parse_census(path): """Pull the pool totals out of mem.census log lines.""" - fields = ("priv_kb", "rss_kb", "crt_kb", "biggest_kb", "prof_live_kb") + fields = ("priv_kb", "rss_kb", "crt_kb", "biggest_kb") rows = [] pattern = re.compile(r"(\w+)=(\S+)") with open(path, "r", errors="replace") as handle: @@ -71,16 +94,16 @@ def parse_census(path): return rows -def report(path, census_path, top): +def report(path, census_path, top, binary=None): summaries, series = load_profile(path) if not summaries and not series: - print(f"{path}: no profiler records — was CBM_MEM_PROFILE=1 set?") + print(f"{path}: no waste-layer records -- was the memwaste flavour run with CBM_MEMWASTE=1?") return + names = load_symbols(binary, summaries, series.keys()) last = summaries[-1] if summaries else {} print(f"=== {path} ===") if last: - print(f"threshold : {last.get('threshold', 0)} bytes " - f"(allocations below this are invisible by design)") + print(f"samples : {len(summaries)} dumps (every allocation size is tracked)") print(f"sites seen : {last.get('sites', 0)}") print(f"attributed live: {last.get('live_bytes', 0)/1048576:.1f} MB " f"in {last.get('live_blocks', 0)} blocks") @@ -94,23 +117,23 @@ def report(path, census_path, top): slope = linear_slope(live) if slope <= 0 and live[-1] <= live[0]: continue - growth.append((slope, live[-1] - live[0], live[-1], samples[-1])) + growth.append((slope, live[-1] - live[0], live[-1], names.get(site, site))) growth.sort(key=lambda row: row[0], reverse=True) print(f"\n--- sites whose retained bytes GROW (top {top}) ---") if not growth: print("none — no tracked site retains more over time.") - for slope, delta, live, sample in growth[:top]: - frames = " <- ".join(sample.get("frames", [])[:4]) + for slope, delta, live, name in growth[:top]: print(f" +{slope/1024:8.1f} KB/sample total +{delta/1048576:6.2f} MB " f"live {live/1048576:6.2f} MB") - print(f" {frames}") + print(f" {name}") if census_path: rows = parse_census(census_path) if rows: priv = [r["priv_kb"] for r in rows if "priv_kb" in r] - prof = [r["prof_live_kb"] for r in rows if "prof_live_kb" in r] + # attributed = what the waste layer holds live, sampled at its own dumps + prof = [int(h.get("live_bytes", 0)) // 1024 for h in summaries] print(f"\n--- pools vs attribution ({len(rows)} samples) ---") if priv: print(f" private committed : {priv[0]/1024:.1f} -> {priv[-1]/1024:.1f} MB " @@ -119,24 +142,27 @@ def report(path, census_path, top): unattributed = (priv[-1] - priv[0]) - (prof[-1] - prof[0]) print(f" attributed growth : {(prof[-1]-prof[0])/1024:+.1f} MB") print(f" UNATTRIBUTED : {unattributed/1024:+.1f} MB " - "<- below-threshold allocations, allocator metadata, or fragmentation") + "<- allocator metadata, fragmentation, or memory outside malloc (mmap, stacks)") -def diff(path_a, path_b, top): - """Same sites, two platforms: what does one retain that the other does not?""" - _, series_a = load_profile(path_a) +def diff(path_a, path_b, top, binary=None): + """Same sites, two platforms: what does one retain that the other does not? + Raw addresses differ between binaries, so sites are matched by symbol name + when --binary is given (the same binary for both runs) and by address otherwise.""" + summaries_a, series_a = load_profile(path_a) _, series_b = load_profile(path_b) + names = load_symbols(binary, summaries_a, set(series_a) | set(series_b)) rows = [] for site, samples in series_a.items(): a_live = samples[-1]["live_bytes"] b_live = series_b[site][-1]["live_bytes"] if site in series_b else 0 if a_live - b_live > 0: - rows.append((a_live - b_live, a_live, b_live, samples[-1])) + rows.append((a_live - b_live, a_live, b_live, site)) rows.sort(key=lambda row: row[0], reverse=True) print(f"=== {path_a} minus {path_b} (top {top}) ===") - for delta, a_live, b_live, sample in rows[:top]: + for delta, a_live, b_live, site in rows[:top]: print(f" +{delta/1048576:7.2f} MB A={a_live/1048576:.2f} MB B={b_live/1048576:.2f} MB") - print(f" {' <- '.join(sample.get('frames', [])[:4])}") + print(f" {names.get(site, site)}") def main(): @@ -145,11 +171,12 @@ def main(): parser.add_argument("--census") parser.add_argument("--diff") parser.add_argument("--top", type=int, default=12) + parser.add_argument("--binary", help="the memwaste binary, for symbol names") args = parser.parse_args() if args.diff: - diff(args.profile, args.diff, args.top) + diff(args.profile, args.diff, args.top, args.binary) else: - report(args.profile, args.census, args.top) + report(args.profile, args.census, args.top, args.binary) return 0 diff --git a/scripts/memlab.sh b/scripts/memlab.sh index 373742556..537738d60 100644 --- a/scripts/memlab.sh +++ b/scripts/memlab.sh @@ -10,9 +10,14 @@ # # Usage: scripts/memlab.sh [requests] [label] # -# Output: memlab-

17 MCP tools

-

search_graph, trace_path, detect_changes, query_graph (Cypher), get_architecture, get_code_snippet, check_index_coverage, manage_adr, and 7 more.

+

search_graph, trace_path, detect_changes, query_graph (Cypher), get_architecture, get_code_snippet, check_index_coverage, manage_adr, and 9 more.

Cypher graph queries

From 6166934defd9bc5b687ab9c8de3779e71a844395 Mon Sep 17 00:00:00 2001 From: lorenzozanee Date: Sun, 20 Sep 2026 07:17:12 +0800 Subject: [PATCH 103/130] fix(cypher): ignore unbound optional variables in count Signed-off-by: lorenzozanee --- src/cypher/cypher.c | 22 ++++++++++++++++++++++ tests/test_cypher.c | 30 ++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/src/cypher/cypher.c b/src/cypher/cypher.c index 9dfe97c76..239768e27 100644 --- a/src/cypher/cypher.c +++ b/src/cypher/cypher.c @@ -4450,11 +4450,33 @@ static void ret_agg_init_group(ret_agg_entry_t *entry, const char *key, int item } /* Accumulate a binding into RETURN aggregation */ +static bool binding_has_value(binding_t *b, const char *var, const char *prop) { + if (!var || strcmp(var, "*") == 0) { + return true; + } + char full[CBM_SZ_256]; + if (prop) { + snprintf(full, sizeof(full), "%s.%s", var, prop); + } else { + snprintf(full, sizeof(full), "%s", var); + } + for (int i = 0; i < b->var_count; i++) { + if (strcmp(b->var_names[i], full) == 0) { + return true; + } + } + return binding_get_edge(b, var) != NULL || binding_get(b, var) != NULL; +} + static void ret_agg_accumulate(ret_agg_entry_t *entry, cbm_return_clause_t *ret, binding_t *b) { for (int ci = 0; ci < ret->count; ci++) { if (!is_aggregate_func(ret->items[ci].func)) { continue; } + if (strcmp(ret->items[ci].func, "COUNT") == 0 && + !binding_has_value(b, ret->items[ci].variable, ret->items[ci].property)) { + continue; + } entry->counts[ci]++; const char *raw = binding_get_virtual(b, ret->items[ci].variable, ret->items[ci].property); double dv = strtod(raw, NULL); diff --git a/tests/test_cypher.c b/tests/test_cypher.c index fbd9d36bf..3f4771480 100644 --- a/tests/test_cypher.c +++ b/tests/test_cypher.c @@ -1744,6 +1744,35 @@ TEST(cypher_exec_count) { PASS(); } +TEST(cypher_exec_optional_bound_terminal_count_ignores_unbound) { + cbm_store_t *s = setup_cypher_store(); + cbm_cypher_result_t r = {0}; + + int rc = cbm_cypher_execute(s, + "MATCH (f:Function) OPTIONAL MATCH (t)-[:CALLS]->(f) " + "RETURN f.name, COUNT(t) AS cnt", + "test", 0, &r); + ASSERT_EQ(rc, 0); + ASSERT_EQ(r.row_count, 4); + + for (int i = 0; i < r.row_count; i++) { + const char *name = r.rows[i][0]; + const char *count = r.rows[i][1]; + if (strcmp(name, "HandleOrder") == 0) { + ASSERT_STR_EQ(count, "0"); + } else if (strcmp(name, "ValidateOrder") == 0 || strcmp(name, "SubmitOrder") == 0 || + strcmp(name, "LogError") == 0) { + ASSERT_STR_EQ(count, "1"); + } else { + ASSERT_TRUE(false); + } + } + + cbm_cypher_result_free(&r); + cbm_store_close(s); + PASS(); +} + TEST(cypher_exec_limit) { cbm_store_t *s = setup_cypher_store(); cbm_cypher_result_t r = {0}; @@ -4578,6 +4607,7 @@ SUITE(cypher) { RUN_TEST(cypher_exec_calls_with_where); RUN_TEST(cypher_exec_inbound); RUN_TEST(cypher_exec_count); + RUN_TEST(cypher_exec_optional_bound_terminal_count_ignores_unbound); RUN_TEST(cypher_exec_limit); RUN_TEST(cypher_exec_order_by); RUN_TEST(cypher_exec_variable_length); From e8a46c16e233bff111092dff18985cca94890bcc Mon Sep 17 00:00:00 2001 From: lorenzozanee Date: Sun, 20 Sep 2026 07:18:15 +0800 Subject: [PATCH 104/130] fix(typescript): preserve awaited generic call edges Signed-off-by: lorenzozanee --- internal/cbm/extract_calls.c | 10 ++++++++++ internal/cbm/lsp/ts_lsp.c | 9 +++++++++ tests/test_extraction.c | 13 +++++++++++++ tests/test_ts_lsp.c | 13 +++++++++++++ 4 files changed, 45 insertions(+) diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index 47250203d..728059c71 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -258,9 +258,18 @@ static char *extract_constructor_callee(CBMArena *a, TSNode node, const char *so } // Try common field-based callee resolution (function, name, method fields). +static TSNode unwrap_await_callee(TSNode node) { + if (ts_node_is_null(node) || strcmp(ts_node_type(node), "await_expression") != 0 || + ts_node_named_child_count(node) == 0) { + return node; + } + return ts_node_named_child(node, 0); +} + static char *extract_callee_from_fields(CBMArena *a, TSNode node, const char *source) { // Try "function" field TSNode func_node = ts_node_child_by_field_name(node, TS_FIELD("function")); + func_node = unwrap_await_callee(func_node); if (!ts_node_is_null(func_node)) { const char *fk = ts_node_type(func_node); if (strcmp(fk, "selector_expression") == 0) { @@ -3521,6 +3530,7 @@ static CBMPrimaryCalleeSelection select_primary_callee(CBMExtractCtx *ctx, TSNod } selection.expr = language_specific_callee_expr(ctx->language, node); + selection.expr = unwrap_await_callee(selection.expr); if (is_dynamic_callee_expr(ctx, selection.expr)) { selection.expr = (TSNode){0}; return selection; diff --git a/internal/cbm/lsp/ts_lsp.c b/internal/cbm/lsp/ts_lsp.c index fc8b8911f..0072aa999 100644 --- a/internal/cbm/lsp/ts_lsp.c +++ b/internal/cbm/lsp/ts_lsp.c @@ -2800,8 +2800,17 @@ static void resolve_value_references_at(TSLSPContext *ctx, TSNode call_node) { } } +static TSNode unwrap_await_callee(TSNode node) { + if (ts_node_is_null(node) || strcmp(ts_node_type(node), "await_expression") != 0 || + ts_node_named_child_count(node) == 0) { + return node; + } + return ts_node_named_child(node, 0); +} + static void resolve_call_at(TSLSPContext *ctx, TSNode call_node) { TSNode fn = ts_node_child_by_field_name(call_node, "function", TS_LSP_FIELD_LEN("function")); + fn = unwrap_await_callee(fn); if (ts_node_is_null(fn)) return; diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 14d75bf8b..f4939517e 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -4362,6 +4362,18 @@ TEST(swift_labeled_call_string_arg_issue1892) { * literal, consumed as client(buildPath(id)). The builder's URL is recorded in * the per-file constant map and resolved at the call site, for both return * statements and arrow expression bodies. */ +TEST(extract_ts_await_generic_call_issue2210) { + CBMFileResult *r = extract("function parseJsonBody() { return {} as T; }\n" + "async function plain() { return await parseJsonBody(); }\n" + "async function generic() { return await parseJsonBody(); }\n", + CBM_LANG_TYPESCRIPT, "t", "await.ts"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT_EQ(count_calls_named(r, "parseJsonBody"), 2); + cbm_free_result(r); + PASS(); +} + TEST(extract_ts_url_builder_issue1009) { CBMFileResult *r = extract("function thingDetail(id: string): string {\n" " return `/api/v1/things/${id}/detail`;\n" @@ -7831,6 +7843,7 @@ SUITE(extraction) { RUN_TEST(extract_go_binary_concat_url_issue1249); RUN_TEST(extract_go_binary_concat_url_no_literal_suffix_issue1249); RUN_TEST(extract_ts_url_builder_issue1009); + RUN_TEST(extract_ts_await_generic_call_issue2210); RUN_TEST(extract_ts_route_handler_after_named_middleware); RUN_TEST(extract_ts_route_handler_after_inline_middleware); RUN_TEST(extract_ts_url_builder_composed_issue1009); diff --git a/tests/test_ts_lsp.c b/tests/test_ts_lsp.c index 52c3bbacb..b5e38b592 100644 --- a/tests/test_ts_lsp.c +++ b/tests/test_ts_lsp.c @@ -729,6 +729,17 @@ TEST(tslsp_nocrash_megasource) { /* ── Category 11: generics (deeper) ─────────────────────────────────────────── */ +TEST(tslsp_await_generic_call_issue2210) { + CBMFileResult *r = extract_ts("function parseJsonBody(): T { return null as T; }\n" + "async function plain() { return await parseJsonBody(); }\n" + "async function generic() { return await parseJsonBody(); }\n"); + ASSERT_NOT_NULL(r); + ASSERT_TRUE(find_resolved(r, "plain", "parseJsonBody") >= 0); + ASSERT_TRUE(find_resolved(r, "generic", "parseJsonBody") >= 0); + cbm_free_result(r); + PASS(); +} + TEST(tslsp_generic_identity_inference) { CBMFileResult *r = extract_ts("function identity(x: T): T { return x; }\n" "class Box { use(): void {} }\n" @@ -4316,6 +4327,8 @@ SUITE(ts_lsp) { RUN_TEST(tslsp_nocrash_using_decl); RUN_TEST(tslsp_nocrash_megasource); + RUN_TEST(tslsp_await_generic_call_issue2210); + /* Category 11: generics deeper */ RUN_TEST(tslsp_generic_identity_inference); RUN_TEST(tslsp_generic_array_map_chain); From 243b405afeb3998707ea4d556f3178c8e8419cab Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 20 Sep 2026 04:18:37 +0200 Subject: [PATCH 105/130] build(deps): bump codeql-action init and analyze together to v4.38.0 Dependabot split one upgrade across three PRs -- init (#2185), analyze (#2187) and upload-sarif (#2186) -- but init and analyze share a configuration file and CodeQL refuses to read one written by a different version: Loaded a configuration file for version '4.38.0', but running version '4.37.9' So #2185 and #2187 each go red on their own and neither can reach a green required check, which means they cannot be merged one after the other either: whichever lands first leaves the pair mismatched. The action also warns about it directly -- "Not all workflow steps that use github/codeql-action actions use the same version". Both refs in codeql.yml therefore move in one commit, to the same pinned SHA b96794f015dfd88f77b49b1c93e0fa7110f94c63 that both bot PRs target. scorecard.yml's upload-sarif is deliberately not touched here: it runs in a separate workflow that shares no configuration with init/analyze, which is why #2186 is green on its own and can land as its own PR. Supersedes #2185 and #2187. Signed-off-by: Martin Vogel --- .github/workflows/codeql.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index dbaa52a6c..e83a82d06 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -29,7 +29,7 @@ jobs: run: sudo apt-get update && sudo apt-get install -y zlib1g-dev - name: Initialize CodeQL - uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 + uses: github/codeql-action/init@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0 with: languages: c-cpp build-mode: manual @@ -38,6 +38,6 @@ jobs: run: scripts/build.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 + uses: github/codeql-action/analyze@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0 with: category: "/language:c-cpp" From 96a2fc35dfcf14662cee491430366f4e8d8b8149 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 20 Sep 2026 14:07:12 +0200 Subject: [PATCH 106/130] test(mcp): ask for JSON where the coverage test asserts a JSON field #1787 landed correct production code and a test that cannot pass on today's main. The test is the only thing wrong, and only in how it asks. check_index_coverage answers in the compact table by default, where a field name never appears: main.go main.go coverage_unavailable metadata_match read_source_and_reindex [] The test asserts on the JSON field spelling: strstr(response, "\"freshness\":\"metadata_match\"") Both halves are right; they were simply written against different response formats. The branch was cut on 2026-09-02, before that default changed, and was never rebased -- its CI green is from a main that no longer exists. The neighbouring coverage tests already pass "format":"json" for exactly this reason; this one predates them. So the request gains format=json and the assertions are untouched. Asserting the field rather than a bare strstr for "metadata_match" is what keeps the check exact: the value also appears in the table's recommended_action column and would be matched by any other path's row. Verified: build/c/test-runner mcp -> 318 passed, 0 failed, 4 skipped. Before this change the same suite reported 317 passed, 1 failed on identical production code. What this is NOT: a production defect. coverage_path_freshness on main uses cbm_path_info_utf8, exactly as #1787 intended, and the Windows second-vs- nanosecond truncation it fixes is gone. I checked that by reading origin/main rather than the local checkout, which was stale at 59a05eb1 and still carried the old helper -- that misreading briefly convinced me the merge had dropped the fix, and it had not. Signed-off-by: Martin Vogel --- tests/test_mcp.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 4a02211b2..7d51c9c6d 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -5800,8 +5800,14 @@ TEST(tool_check_index_coverage_freshness_uses_indexer_mtime_source_issue1714) { info.size), CBM_STORE_OK); - char *response = cbm_mcp_handle_tool( - srv, "check_index_coverage", "{\"project\":\"test-project\",\"paths\":[\"main.go\"]}"); + /* format=json, like every other coverage test here: the DEFAULT response is + * the compact table, in which a field name never appears. Keeping the + * assertion on the JSON field is what makes it exact — a bare strstr for + * "metadata_match" would also be satisfied by a neighbouring column or by + * another path's row. */ + char *response = cbm_mcp_handle_tool(srv, "check_index_coverage", + "{\"project\":\"test-project\",\"paths\":[\"main.go\"]," + "\"format\":\"json\"}"); ASSERT_NOT_NULL(response); ASSERT_NOT_NULL(strstr(response, "\"freshness\":\"metadata_match\"")); free(response); @@ -5815,8 +5821,9 @@ TEST(tool_check_index_coverage_freshness_uses_indexer_mtime_source_issue1714) { ASSERT_EQ(cbm_store_upsert_file_hash(store, "test-project", "main.go", "", seconds_mtime_ns, info.size), CBM_STORE_OK); - response = cbm_mcp_handle_tool( - srv, "check_index_coverage", "{\"project\":\"test-project\",\"paths\":[\"main.go\"]}"); + response = cbm_mcp_handle_tool(srv, "check_index_coverage", + "{\"project\":\"test-project\",\"paths\":[\"main.go\"]," + "\"format\":\"json\"}"); ASSERT_NOT_NULL(response); ASSERT_NOT_NULL(strstr(response, "\"freshness\":\"metadata_changed\"")); free(response); From fcf451c6a6dda768dfec5a8f61a01ac1e1cfa6bf Mon Sep 17 00:00:00 2001 From: XIYBHK Date: Wed, 2 Sep 2026 15:57:28 +0800 Subject: [PATCH 107/130] fix(parsing): recover export-macro-hidden type definitions (#1989) Collect a bounded set of conventional export macro candidates and inject empty definitions into the existing C/C++ preprocessing second pass without overriding explicit caller definitions. Conservatively reconcile remapped definitions to recover hidden classes, structs, enums, free functions, and inline methods while suppressing matching base-class phantom callables. Add focused regression coverage for supported suffixes, ordinary all-caps negative controls, candidate limits, comments, strings, raw strings, overlong names, explicit define priority, and C/C++ extraction. Rebase onto main 92abefa3 and route the temporary superseded-definition bitmap through the extraction memory class to comply with the current allocation contract. Preserve the error-region dropped counter initialization. Signed-off-by: XIYBHK --- internal/cbm/cbm.c | 251 +++++++++++++++++++++- internal/cbm/preprocessor.cpp | 305 ++++++++++++++++++++++++++- internal/cbm/preprocessor.h | 15 ++ tests/test_extraction.c | 384 ++++++++++++++++++++++++++++++++++ 4 files changed, 945 insertions(+), 10 deletions(-) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 51c14eca1..2f475229a 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -1514,6 +1514,63 @@ static bool cbm_span_contains_callable_def(const char *src, int src_len, uint32_ return false; } +/* #1989: type-def counterpart of cbm_span_contains_callable_def. A rescued + * class/struct/enum definition has its name followed by a base clause (':') or + * a body ('{'), not a parameter list, so the callable validator would reject + * every type def pulled from the expanded tree. */ +static bool cbm_span_contains_type_def(const char *src, int src_len, uint32_t start_line, + uint32_t end_line, const char *name) { + if (!src || src_len <= 0 || !name || !name[0] || start_line == 0 || end_line < start_line) { + return false; + } + int span_start = 0; + uint32_t line = 1; + while (span_start < src_len && line < start_line) { + if (src[span_start++] == '\n') { + line++; + } + } + if (line != start_line) { + return false; + } + int span_end = span_start; + while (span_end < src_len && line <= end_line) { + if (src[span_end++] == '\n') { + line++; + } + } + size_t name_len = strlen(name); + for (int pos = span_start; pos + (int)name_len <= span_end; pos++) { + if (strncmp(src + pos, name, name_len) != 0 || + (pos > 0 && cbm_identifier_char(src[pos - 1])) || + (pos + (int)name_len < src_len && cbm_identifier_char(src[pos + name_len]))) { + continue; + } + int open = pos + (int)name_len; + while (open < span_end && isspace((unsigned char)src[open])) { + open++; + } + // Base clause (": public B"), body ("{"), or template args ("<..."). + if (open < span_end && (src[open] == ':' || src[open] == '{' || src[open] == '<')) { + return true; + } + } + return false; +} + +/* #1989: label buckets for the misparse-correction gates. A raw def extracted + * from a broken `class MOD_API Foo` shape lands under a callable label while + * the expanded tree yields a proper type label for the same name. */ +static bool cbm_def_label_is_type(const char *label) { + return label != NULL && (strcmp(label, "Class") == 0 || strcmp(label, "Enum") == 0 || + strcmp(label, "Interface") == 0 || strcmp(label, "Type") == 0 || + strcmp(label, "Struct") == 0 || strcmp(label, "Union") == 0); +} + +static bool cbm_def_label_is_callable(const char *label) { + return label != NULL && (strcmp(label, "Function") == 0 || strcmp(label, "Method") == 0); +} + /* Remap an expanded-source definition back to the original input file. Every * line in the definition must be attributable to the main file; generated * macro bodies, included headers, and ambiguous spans fail closed. */ @@ -2133,6 +2190,14 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C // Defs keep original-source line numbers; only CALLS are extracted from expanded source. if (language == CBM_LANG_C || language == CBM_LANG_CPP || language == CBM_LANG_CUDA) { uint64_t pp_start = now_ns(); + /* #1989: collect build-system export-macro candidates from the raw + * source. They are predefined empty in the expanded buffer (see + * preprocessor.cpp) so `class MOD_API Foo` parses with its real name, + * and the rescue loop below can adopt corrected type defs. Bounded: + * at most CBM_EXPORT_MACRO_MAX names per file. */ + char export_cands[CBM_EXPORT_MACRO_MAX][CBM_EXPORT_MACRO_NAME_MAX]; + int export_cand_count = + cbm_export_macro_candidates(source, source_len, export_cands, CBM_EXPORT_MACRO_MAX); CBMPreprocessedSource *preprocessed = cbm_preprocess_with_map( source, source_len, rel_path, extra_defines, include_paths, language != CBM_LANG_C); if (preprocessed && preprocessed->source) { @@ -2213,17 +2278,52 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C * recover defs from it — adopting ONLY those that * intersect a raw ERROR region, whose name is visible on * the raw source line, and whose QN the raw pass did not - * already extract. */ - if (ts_node_has_error(root)) { + * already extract. + * + * #1989: build-system export macros (_API, + * _EXPORT, ...) are empty on the real compile line but + * opaque to tree-sitter, so `class MOD_API Foo` misparses + * with the macro as the type name — often with NO raw ERROR + * region at all (the class_specifier "succeeds" with the + * wrong name; enums and free functions do error out). When + * export-macro candidates were collected for this file, also + * adopt expanded defs that replace a raw def literally named + * as a candidate, or that correct a raw misparse label (the + * raw pass extracted the same name as Function/Method from + * the broken function_definition shape; the expanded tree + * yields a proper type def). Superseded raw defs are + * dropped so the corrected definition is not shadowed by + * the qualified-name dedup. With no candidates the block + * reduces to the original #961 behavior. */ + if (ts_node_has_error(root) || export_cand_count > 0) { cbm_error_regions_t raw_regs = {{0}, {0}, 0, 0}; cbm_collect_error_regions(root, &raw_regs, source, source_len); - if (raw_regs.count > 0) { + if (raw_regs.count > 0 || export_cand_count > 0) { int defs_before = result->defs.count; cbm_extract_definitions(&pp_ctx); int w = defs_before; + char *superseded = NULL; + if (export_cand_count > 0 && defs_before > 0) { + superseded = + (char *)cbm_calloc(CBM_MEM_CLASS_EXTRACT, (size_t)defs_before); + } + /* #1989: spans of defs already adopted from the + * expanded tree. A def nested inside one of these + * (an inline method of a rescued class) was dropped + * by the raw pass — the misparsed + * `class MOD_API Foo {...}` parsed as a + * function_definition, and nested defs in its body + * are never walked. Bounded: one span per adopted + * def, capped at 64. */ + struct { + uint32_t start; + uint32_t end; + } rescued_spans[64]; + int rescued_count = 0; for (int i = defs_before; i < result->defs.count; i++) { CBMDefinition *d = &result->defs.items[i]; bool adopt = false; + int supersede_j = -1; if (cbm_remap_preprocessed_def(d, preprocessed)) { for (int rj = 0; rj < raw_regs.count && !adopt; rj++) { if (d->start_line <= raw_regs.ends[rj] && @@ -2231,26 +2331,159 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C adopt = true; } } + if (!adopt && export_cand_count > 0 && d->name) { + for (int ci = 0; ci < export_cand_count && !adopt; ci++) { + for (int j = 0; j < defs_before && !adopt; j++) { + CBMDefinition *r = &result->defs.items[j]; + if (!r->name || + strcmp(r->name, export_cands[ci]) != 0) { + continue; + } + if (r->start_line <= d->end_line && + d->start_line <= r->end_line && + cbm_span_contains_type_def( + source, source_len, d->start_line, + d->end_line, d->name)) { + adopt = true; + supersede_j = j; + } + } + } + if (!adopt && d->label && cbm_def_label_is_type(d->label)) { + for (int j = 0; j < defs_before && !adopt; j++) { + CBMDefinition *r = &result->defs.items[j]; + /* Widened past callables (#1989): the + * misparse also mints value-label raw + * defs for types (`class X_API FEmpty {}` + * -> Variable FEmpty). Same name, + * overlapping span, raw label is NOT a + * type => raw is the misparse artifact. */ + if (!r->name || !r->label || !d->name || + strcmp(r->name, d->name) != 0 || + cbm_def_label_is_type(r->label)) { + continue; + } + if (r->start_line <= d->end_line && + d->start_line <= r->end_line) { + adopt = true; + supersede_j = j; + } + } + } + /* Nested rescue (#1989): a def nested inside + * an already-adopted def's span — the inline + * method of a rescued class, dropped by the + * raw pass because the class parsed as a + * function body. The same validation gates + * (name-on-line, shape, QN dedup) still run + * below before it lands. */ + if (!adopt && rescued_count > 0) { + for (int k = 0; k < rescued_count && !adopt; k++) { + if (d->start_line >= rescued_spans[k].start && + d->end_line <= rescued_spans[k].end) { + adopt = true; + } + } + } + } } - if (adopt && (!d->name || - !cbm_line_contains(source, source_len, d->start_line, - d->name) || - !cbm_span_contains_callable_def( - source, source_len, d->start_line, d->end_line, - d->name))) { + if (adopt && + (!d->name || + !cbm_line_contains(source, source_len, d->start_line, + d->name) || + (!cbm_span_contains_callable_def(source, source_len, + d->start_line, d->end_line, + d->name) && + !cbm_span_contains_type_def(source, source_len, d->start_line, + d->end_line, d->name)))) { adopt = false; } for (int j = 0; j < defs_before && adopt; j++) { + if (j == supersede_j || (superseded && superseded[j])) { + continue; // being replaced by this very def + } const char *q = result->defs.items[j].qualified_name; if (q && d->qualified_name && strcmp(q, d->qualified_name) == 0) { + /* #1989: a raw def holding the same QN is + * usually a reject — unless it is the + * misparse artifact this very def came to + * replace (raw "Function FCalc" from the + * broken `class MOD_API FCalc` shape vs the + * rescued "Class FCalc"). Same name, same + * span, raw label is not a type: supersede + * instead of dropping the correction. */ + CBMDefinition *r = &result->defs.items[j]; + if (d->label && cbm_def_label_is_type(d->label) && + r->label && r->name && d->name && + strcmp(r->name, d->name) == 0 && + !cbm_def_label_is_type(r->label) && + r->start_line == d->start_line && + r->end_line == d->end_line && superseded) { + superseded[j] = 1; + continue; + } adopt = false; } } if (adopt) { + if (supersede_j >= 0 && supersede_j < defs_before && + superseded) { + superseded[supersede_j] = 1; + } + /* Phantom-artifact suppression (#1989): on the + * raw tree `class MOD_API Foo : public Base + * {...}` parses as a function_definition whose + * declarator is the BASE-CLASS name, so the raw + * pass mints a bogus Function def named after + * the base spanning the whole class. Fingerprint: + * a raw callable def with an IDENTICAL span to + * the adopted type def whose name is one of the + * adopted def's base classes. A real method is + * a strict subset of the class span and never + * carries a base-class name. */ + if (d->label && cbm_def_label_is_type(d->label) && superseded && + d->base_classes) { + for (int j = 0; j < defs_before; j++) { + CBMDefinition *r = &result->defs.items[j]; + if (!r->label || !r->name || + !cbm_def_label_is_callable(r->label) || + r->start_line != d->start_line || + r->end_line != d->end_line) { + continue; + } + for (const char **b = d->base_classes; *b; b++) { + if (strcmp(*b, r->name) == 0) { + superseded[j] = 1; + break; + } + } + } + } + if (rescued_count < 64) { + rescued_spans[rescued_count].start = d->start_line; + rescued_spans[rescued_count].end = d->end_line; + rescued_count++; + } result->defs.items[w++] = *d; } } + if (superseded) { + int rw = 0; + for (int j = 0; j < defs_before; j++) { + if (!superseded[j]) { + result->defs.items[rw++] = result->defs.items[j]; + } + } + int shift = defs_before - rw; + if (shift > 0 && w > defs_before) { + memmove(&result->defs.items[rw], + &result->defs.items[defs_before], + (size_t)(w - defs_before) * sizeof(CBMDefinition)); + } + w -= shift; + cbm_free(CBM_MEM_CLASS_EXTRACT, superseded); + } result->defs.count = w; } } diff --git a/internal/cbm/preprocessor.cpp b/internal/cbm/preprocessor.cpp index 54cb073fe..9be6298f3 100644 --- a/internal/cbm/preprocessor.cpp +++ b/internal/cbm/preprocessor.cpp @@ -42,6 +42,283 @@ static bool has_preprocessor_work(const char *source, int source_len) { return false; } +// ── Export-macro candidates (#1989) ───────────────────────────────────────── +// Build systems define symbol-export macros empty on the compiler command line +// (UE's UBT: `/D "MODULE_API="`; CMake generate_export_header: `_EXPORT`), +// so they never appear as #define lines in the source. tree-sitter has no +// preprocessor state either, and `class MODULE_API Foo` misparses with the +// macro token as the type name (worse: enums and free functions are lost to +// ERROR regions entirely). To mirror the real compile line, the preprocessed +// second pass predefines the conventional export-macro-shaped identifiers found +// in the file as empty. The shape is deliberately narrow — ALL_CAPS identifier +// ending in a known export suffix, with a non-trivial prefix — and the list is +// capped, so ordinary all-caps identifiers cannot be swept in wholesale. +static const char *kExportMacroSuffixes[] = {"_API", "_EXPORT", "_IMPORT", + "_DLLEXPORT", "_DEPRECATED", NULL}; + +static bool is_export_macro_shape(const char *id, size_t len) { + if (id[0] < 'A' || id[0] > 'Z') { + return false; + } + for (size_t i = 0; i < len; i++) { + char c = id[i]; + bool upper = c >= 'A' && c <= 'Z'; + bool digit = c >= '0' && c <= '9'; + if (!upper && !digit && c != '_') { + return false; + } + } + for (int s = 0; kExportMacroSuffixes[s]; s++) { + size_t slen = strlen(kExportMacroSuffixes[s]); + // Require at least two prefix characters before the suffix so a bare + // "X_API"-style token (single-letter, easily a genuine symbol) stays out. + if (len >= slen + 2 && strncmp(id + len - slen, kExportMacroSuffixes[s], slen) == 0) { + return true; + } + } + return false; +} + +static bool is_identifier_start(char c) { + return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_'; +} + +static bool is_identifier_char(char c) { + return is_identifier_start(c) || (c >= '0' && c <= '9'); +} + +// C++ raw string literals (R"delim(...)delim"). Returns the index one past +// the closing quote, or -1 when the form cannot be confidently recognized +// (unterminated, malformed delimiter) — the caller then STOPS collecting so +// the mis-modeled literal cannot poison the scan state of the rest of the +// file (#1989 review). +static int skip_raw_string(const char *source, int source_len, int quote) { + int j = quote + 1; + int delim_start = j; + while (j < source_len) { + char c = source[j]; + if (c == '(') { + break; + } + // Delimiter chars: the standard d-char set excludes space, parens, + // backslash, and the line breaks — a double quote IS a legal d-char + // (R"""(...)""" compiles), so it must not end the delimiter (#1989 + // review round 3). + if (c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == ')' || c == '\\' || + j - delim_start >= 16) { + return -1; + } + j++; + } + if (j >= source_len || source[j] != '(') { + return -1; + } + int delim_len = j - delim_start; + j++; // past the opening "(" + while (j < source_len) { + if (source[j] == ')') { + int k = j + 1; + int m = 0; + while (m < delim_len && k + m < source_len && + source[k + m] == source[delim_start + m]) { + m++; + } + if (m == delim_len && k + m < source_len && source[k + m] == '"') { + return k + m + 1; + } + } + j++; + } + return -1; // unterminated raw string: uncertain +} + +// If the identifier just scanned is a raw-string prefix (R, LR, uR, UR, u8R) +// immediately followed by a quote, return skip_raw_string(...). Returns 0 for +// an ordinary identifier (including an identifier merely ENDING in R — only a +// standalone prefix token starts a raw string) and -1 on an unrecognizable +// raw form. +static int raw_string_after_prefix(const char *source, int source_len, int start, int end) { + static const char *prefixes[] = {"R", "LR", "uR", "UR", "u8R", NULL}; + size_t idlen = (size_t)(end - start); + bool is_prefix = false; + for (int p = 0; prefixes[p]; p++) { + if (idlen == strlen(prefixes[p]) && strncmp(source + start, prefixes[p], idlen) == 0) { + is_prefix = true; + break; + } + } + if (!is_prefix || end >= source_len || source[end] != '"') { + return 0; + } + return skip_raw_string(source, source_len, end); +} + +// Advance past comments and string/char literals so only real code tokens +// consume the candidate budget (#1989 review). Handles line-spliced // +// comments (a backslash before the newline continues the comment — line +// splicing happens before comment recognition) and, via the identifier-scan +// hook in the callers, C++ raw string literals. A digit separator (1'000'000) +// can still over-skip — that fails safe, toward NOT collecting a candidate. +static int skip_non_code(const char *source, int source_len, int i) { + while (i < source_len) { + char c = source[i]; + if (c == '/' && i + 1 < source_len && source[i + 1] == '/') { + i += 2; + for (;;) { + while (i < source_len && source[i] != '\n') { + i++; + } + if (i >= source_len) { + break; + } + // Line splice: a backslash right before the newline (allowing + // \r\n) means the comment continues on the next line. + int back = i - 1; + if (back >= 0 && source[back] == '\r') { + back--; + } + if (back >= 0 && source[back] == '\\') { + i++; // spliced: keep consuming inside the comment + continue; + } + i++; // real end-of-comment newline + break; + } + } else if (c == '/' && i + 1 < source_len && source[i + 1] == '*') { + i += 2; + while (i + 1 < source_len && !(source[i] == '*' && source[i + 1] == '/')) { + i++; + } + if (i + 1 < source_len) { + i += 2; // past the closing "*/" + } else { + i = source_len; // unterminated comment runs to EOF + } + } else if (c == '"' || c == '\'') { + char quote = c; + i++; + while (i < source_len) { + if (source[i] == '\\') { + i += 2; + continue; + } + if (source[i] == quote) { + i++; + break; + } + i++; + } + } else { + return i; + } + } + return i; +} + +// Count-only scan used by the early gate (no storage). +static bool has_export_macro_candidates(const char *source, int source_len) { + if (!source || source_len <= 0) { + return false; + } + int i = 0; + while (i < source_len) { + i = skip_non_code(source, source_len, i); + if (i >= source_len) { + break; + } + if (!is_identifier_start(source[i])) { + i++; + continue; + } + int start = i; + while (i < source_len && is_identifier_char(source[i])) { + i++; + } + // A raw-string prefix consumes everything up to its closing quote; + // an unrecognizable raw form poisons the scan state, so fail toward + // NOT running the second pass (raw behavior preserved). + int raw = raw_string_after_prefix(source, source_len, start, i); + if (raw < 0) { + return false; + } + if (raw > 0) { + i = raw; + continue; + } + if (is_export_macro_shape(source + start, (size_t)(i - start))) { + return true; + } + } + return false; +} + +static int collect_export_macro_candidates(const char *source, int source_len, + char (*out)[CBM_EXPORT_MACRO_NAME_MAX], int max_out) { + if (!source || source_len <= 0 || !out || max_out <= 0) { + return 0; + } + int stored = 0; + int i = 0; + while (i < source_len) { + i = skip_non_code(source, source_len, i); + if (i >= source_len) { + break; + } + if (!is_identifier_start(source[i])) { + i++; + continue; + } + int start = i; + while (i < source_len && is_identifier_char(source[i])) { + i++; + } + // Raw string (see has_export_macro_candidates): skip its body, or stop + // collecting entirely when the form cannot be confidently recognized — + // continuing would mis-tokenize the rest of the file and could both + // mint phantom candidates and mask real ones (#1989 review). + int raw = raw_string_after_prefix(source, source_len, start, i); + if (raw < 0) { + break; + } + if (raw > 0) { + i = raw; + continue; + } + size_t len = (size_t)(i - start); + /* Length gate FIRST: the dedup below reads out[k][0..len] against rows + * of CBM_EXPORT_MACRO_NAME_MAX bytes — an over-long candidate must be + * rejected before any compare touches them. */ + if (len >= CBM_EXPORT_MACRO_NAME_MAX) { + continue; + } + if (!is_export_macro_shape(source + start, len)) { + continue; + } + bool dup = false; + for (int k = 0; k < stored; k++) { + if (strncmp(out[k], source + start, len) == 0 && out[k][len] == '\0') { + dup = true; + break; + } + } + if (dup) { + continue; + } + if (stored >= max_out) { + break; // bounded: stop collecting once the cap is reached + } + memcpy(out[stored], source + start, len); + out[stored][len] = '\0'; + stored++; + } + return stored; +} + +int cbm_export_macro_candidates(const char *source, int source_len, + char (*out)[CBM_EXPORT_MACRO_NAME_MAX], int max_out) { + return collect_export_macro_candidates(source, source_len, out, max_out); +} + static int count_expanded_lines(const std::string &text) { int count = 1; for (char c : text) { @@ -146,7 +423,11 @@ static bool build_line_map(const std::string &expanded, const std::string &main_ CBMPreprocessedSource *cbm_preprocess_with_map(const char *source, int source_len, const char *filename, const char **extra_defines, const char **include_paths, int cpp_mode) { - if (!has_preprocessor_work(source, source_len)) { + // Run the second pass when there are directives to evaluate OR when the file + // carries export-macro-shaped identifiers to predefine empty (#1989) — a UE + // plugin header with only `#pragma once` + `#include` lines still needs it. + if (!has_preprocessor_work(source, source_len) && + !has_export_macro_candidates(source, source_len)) { return NULL; // NULL = no expansion needed, use original } @@ -156,6 +437,28 @@ CBMPreprocessedSource *cbm_preprocess_with_map(const char *source, int source_le for (int i = 0; extra_defines[i]; i++) dui.defines.push_back(extra_defines[i]); } + // Predefine collected export-macro candidates as empty, mirroring the + // real compile command line (`/D "MODULE_API="`). Names already provided + // by the caller win — never override an explicit define. + char export_cands[CBM_EXPORT_MACRO_MAX][CBM_EXPORT_MACRO_NAME_MAX]; + int export_cand_count = + collect_export_macro_candidates(source, source_len, export_cands, CBM_EXPORT_MACRO_MAX); + for (int i = 0; i < export_cand_count; i++) { + bool provided = false; + for (std::list::const_iterator it = dui.defines.begin(); + it != dui.defines.end(); ++it) { + const std::string &def = *it; + size_t eq = def.find('='); + std::string name = (eq == std::string::npos) ? def : def.substr(0, eq); + if (name == export_cands[i]) { + provided = true; + break; + } + } + if (!provided) { + dui.defines.push_back(std::string(export_cands[i]) + "="); + } + } if (include_paths) { for (int i = 0; include_paths[i]; i++) dui.includePaths.push_back(include_paths[i]); diff --git a/internal/cbm/preprocessor.h b/internal/cbm/preprocessor.h index 72828c012..1b108b335 100644 --- a/internal/cbm/preprocessor.h +++ b/internal/cbm/preprocessor.h @@ -4,6 +4,15 @@ #include #include +/* #1989: build-system export macros (UE "_API", CMake generate_export_header + * "_EXPORT", ...) are defined empty on the real compile command line but are + * invisible to tree-sitter, so `class MOD_API Foo` misparses with the macro token + * as the type name. The preprocessed second pass predefines the conventional + * export-macro-shaped identifiers found in a source file as empty, mirroring the + * real build line. The candidate list is bounded. */ +#define CBM_EXPORT_MACRO_MAX 32 +#define CBM_EXPORT_MACRO_NAME_MAX 96 + #ifdef __cplusplus extern "C" { #endif @@ -15,6 +24,12 @@ typedef struct { int expanded_line_count; } CBMPreprocessedSource; +// Collect conventional export-macro-shaped identifiers (ALL_CAPS ending in +// _API/_EXPORT/_IMPORT/_DLLEXPORT/_DEPRECATED) from the source, up to max_out +// unique names. Returns the number of names stored in out. +int cbm_export_macro_candidates(const char *source, int source_len, + char (*out)[CBM_EXPORT_MACRO_NAME_MAX], int max_out); + // Preprocess C/C++ source: expand macros, evaluate #ifdef, resolve #include. // Returns malloc-allocated expanded source, or NULL if no expansion needed/on failure. // extra_defines: NULL-terminated array of "NAME=VALUE" strings (can be NULL). diff --git a/tests/test_extraction.c b/tests/test_extraction.c index a61083755..e5800f8da 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -8,6 +8,7 @@ #include "test_framework.h" #include "cbm.h" #include "foundation/constants.h" /* CBM_SZ_* */ +#include "preprocessor.h" /* cbm_export_macro_candidates (#1989) */ #include "../src/foundation/compat.h" /* cbm_clock_gettime (wide-flat scaling guard) */ #include "../src/foundation/compat_fs.h" #include @@ -5751,6 +5752,370 @@ TEST(extract_c_clean_file_no_recovery_duplicates_issue961) { PASS(); } +/* #1989: build-system export macros (UE "_API", CMake generate_export_header + * "_EXPORT") are empty on the real compile line but opaque to tree-sitter, + * so `class MOD_API Foo` misparses. The preprocessed second pass predefines the + * conventional export-macro-shaped identifiers found in the file as empty, and + * the recovery loop adopts the corrected type defs (superseding the raw + * misparse artifacts). */ +TEST(extract_cpp_export_macro_class_recovery_issue1989) { + CBMFileResult *r = extract("#pragma once\n" + "UCLASS()\n" + "class DUMMY_API UFoo : public UObject\n" + "{\n" + " GENERATED_BODY()\n" + "public:\n" + " int32 X = 0;\n" + "};\n", + CBM_LANG_CPP, "p", "ue.h"); + ASSERT_NOT_NULL(r); + const CBMDefinition *cls = find_def(r, "UFoo"); + ASSERT_NOT_NULL(cls); /* was missing entirely before the fix */ + ASSERT_EQ(cls->start_line, 3u); + ASSERT_NULL(find_def(r, "DUMMY_API")); /* macro-as-name artifact superseded */ + /* The raw misparse also mints a phantom Function def NAMED AFTER THE BASE + * CLASS spanning the whole class ("UObject" from the declarator of the + * broken function_definition shape) — it must be suppressed, not just the + * macro-named artifact. */ + ASSERT_NULL(find_def(r, "UObject")); + cbm_free_result(r); + PASS(); +} + +TEST(extract_cpp_export_macro_struct_recovery_issue1989) { + /* The struct case is the SILENT failure: the raw tree parses "successfully" + * with the macro token as the name and raises no error flag at all. */ + CBMFileResult *r = extract("#pragma once\n" + "USTRUCT()\n" + "struct DUMMY_API FBar\n" + "{\n" + " int32 Y;\n" + "};\n", + CBM_LANG_CPP, "p", "ue_struct.h"); + ASSERT_NOT_NULL(r); + const CBMDefinition *st = find_def(r, "FBar"); + ASSERT_NOT_NULL(st); + ASSERT(st->label && strcmp(st->label, "Class") == 0); /* struct_specifier -> Class */ + ASSERT_NULL(find_def(r, "DUMMY_API")); + cbm_free_result(r); + PASS(); +} + +TEST(extract_cpp_export_macro_enum_recovery_issue1989) { + /* `enum class MOD_API EKind` is lost to an ERROR region entirely. */ + CBMFileResult *r = extract("#pragma once\n" + "UENUM()\n" + "enum class DUMMY_API EKind : uint8\n" + "{\n" + " A,\n" + " B\n" + "};\n", + CBM_LANG_CPP, "p", "ue_enum.h"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(find_def(r, "EKind")); + cbm_free_result(r); + PASS(); +} + +TEST(extract_cpp_export_macro_free_function_recovery_issue1989) { + /* A free function DEFINED with the export macro is lost to an ERROR region + * on the raw tree (prototypes never mint defs — same as clean code). */ + CBMFileResult *r = extract("#pragma once\n" + "DUMMY_API int Add(int a, int b)\n" + "{\n" + " return a + b;\n" + "}\n", + CBM_LANG_CPP, "p", "ue_fn.h"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(find_def(r, "Add")); + cbm_free_result(r); + PASS(); +} + +TEST(extract_cpp_export_macro_inline_method_recovery_issue1989) { + /* The misparsed class parsed as a function body on the raw tree, so the + * raw walk never extracted its inline methods — the rescued class def must + * carry them in via the nested-span adoption. */ + CBMFileResult *r = extract("#pragma once\n" + "class DUMMY_API FCalc\n" + "{\n" + "public:\n" + " int Add(int a, int b) { return a + b; }\n" + "};\n", + CBM_LANG_CPP, "p", "ue_inline.h"); + ASSERT_NOT_NULL(r); + const CBMDefinition *cls = find_def(r, "FCalc"); + ASSERT_NOT_NULL(cls); + ASSERT(cls->label && strcmp(cls->label, "Class") == 0); /* not the raw "Function" mislabel */ + const CBMDefinition *m = find_def(r, "Add"); + ASSERT_NOT_NULL(m); + ASSERT(m->label && strcmp(m->label, "Method") == 0); + cbm_free_result(r); + PASS(); +} + +/* Negative control the maintainer asked for: ordinary ALL_CAPS identifiers + * (constants, enum values) must NOT be swept in as export-macro candidates and + * stripped from the graph. Only the narrow _API/_EXPORT/... suffix shape is. */ +TEST(extract_cpp_export_macro_negative_control_ordinary_caps_issue1989) { + CBMFileResult *r = extract("#pragma once\n" + "#define MAX_CONNECTIONS 16\n" + "const int TIMEOUT_MS = 250;\n" + "enum class State { IDLE, RUNNING };\n" + "struct Config { int MAX_RETRIES; };\n", + CBM_LANG_CPP, "p", "caps.h"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(find_def(r, "Config")); + ASSERT_TRUE(has_def(r, "Variable", "TIMEOUT_MS")); + cbm_free_result(r); + PASS(); +} + +/* C files get the same rescue (SQLITE_API-style prefixes are C, not C++). */ +TEST(extract_c_export_macro_recovery_issue1989) { + CBMFileResult *r = extract("typedef struct sqlite3 sqlite3;\n" + "SQLITE_API int sqlite3_open(const char *path)\n" + "{\n" + " return 0;\n" + "}\n", + CBM_LANG_C, "p", "db.c"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(find_def(r, "sqlite3_open")); + cbm_free_result(r); + PASS(); +} + +/* #1989 review (P1): an over-long candidate-shaped identifier (>= 96 chars) + * arriving AFTER a stored candidate must be rejected BEFORE the dedup + * compares — the pre-fix dedup ran strncmp(out[k], src, len) with + * len >= CBM_EXPORT_MACRO_NAME_MAX against 96-byte rows (out-of-bounds read; + * sanitizer lanes crash). Locally unsanitized, so this pins the behavior + * (normal candidate still recovers, no crash); CI's sanitized lanes guard + * the memory error itself. */ +TEST(extract_cpp_export_macro_overlong_candidate_safe_issue1989) { + char src[4096]; + int off = snprintf(src, sizeof(src), "class MOD_API First { int v; };\nclass "); + ASSERT_GTE(off, 0); + memset(src + off, 'A', 100); + off += 100; + ASSERT_GTE(snprintf(src + off, sizeof(src) - (size_t)off, "_API Long { int w; };\n"), 0); + CBMFileResult *r = extract(src, CBM_LANG_CPP, "p", "overlong.h"); + ASSERT_NOT_NULL(r); + const CBMDefinition *first = find_def(r, "First"); + ASSERT_NOT_NULL(first); + ASSERT(first->label && strcmp(first->label, "Class") == 0); + cbm_free_result(r); + PASS(); +} + +/* #1989 review (P2): candidate-shaped tokens inside comments and string + * literals must NOT consume the bounded budget — 32 of them ahead of the real + * class would otherwise exhaust the cap and leave the real macro undefined. */ +TEST(extract_cpp_export_macro_comment_string_budget_issue1989) { + char src[8192]; + int off = 0; + for (int n = 0; n < 31; n++) { + ASSERT_GTE(snprintf(src + off, sizeof(src) - (size_t)off, "// see DOC%02d_API notes\n", n), + 0); + off += (int)strlen(src + off); + } + ASSERT_GTE(snprintf(src + off, sizeof(src) - (size_t)off, + "static const char *kRef = \"STRING_DOC_API\";\n" + "class REALMOD_API Real { int v; };\n"), + 0); + CBMFileResult *r = extract(src, CBM_LANG_CPP, "p", "budget.h"); + ASSERT_NOT_NULL(r); + const CBMDefinition *cls = find_def(r, "Real"); + ASSERT_NOT_NULL(cls); + ASSERT(cls->label && strcmp(cls->label, "Class") == 0); + cbm_free_result(r); + PASS(); +} + +/* #1989 review: the bounded-budget contract — 33 distinct candidates collect + * only the first CBM_EXPORT_MACRO_MAX (32, in source order); the 33rd stays + * unpredefined and its class keeps the raw misparse (value label, not Class). */ +TEST(extract_cpp_export_macro_candidate_cap_issue1989) { + char src[8192]; + int off = 0; + for (int n = 0; n < 33; n++) { + ASSERT_GTE( + snprintf(src + off, sizeof(src) - (size_t)off, "class CAP%02d_API K%02d {};\n", n, n), + 0); + off += (int)strlen(src + off); + } + CBMFileResult *r = extract(src, CBM_LANG_CPP, "p", "cap.h"); + ASSERT_NOT_NULL(r); + int recovered = 0; + for (int n = 0; n < 33; n++) { + char name[8]; + snprintf(name, sizeof(name), "K%02d", n); + const CBMDefinition *d = find_def(r, name); + if (d && d->label && strcmp(d->label, "Class") == 0) { + recovered++; + } + } + ASSERT_EQ(recovered, 32); + /* The cap contract's other half: the 33rd candidate (K32) must STILL be + * present with its raw misparse — dropping the def entirely would be a + * regression, not a graceful cap. */ + const CBMDefinition *cap_tail = find_def(r, "K32"); + ASSERT_NOT_NULL(cap_tail); + ASSERT_TRUE(!cap_tail->label || strcmp(cap_tail->label, "Class") != 0); + cbm_free_result(r); + PASS(); +} + +/* #1989 review round 2: line-spliced // comments. A backslash before the + * newline continues the comment (splicing happens before comment + * recognition), so "FAKE_API" below is comment text and must NOT consume + * budget. 32 spliced fake comments would otherwise exhaust the cap and leave + * the real class unrecovered. */ +TEST(extract_cpp_export_macro_spliced_comment_budget_issue1989) { + char src[8192]; + int off = 0; + for (int n = 0; n < 31; n++) { + ASSERT_GTE( + snprintf(src + off, sizeof(src) - (size_t)off, "// fake \\\nFAKE%02d_API notes\n", n), + 0); + off += (int)strlen(src + off); + } + ASSERT_GTE(snprintf(src + off, sizeof(src) - (size_t)off, + "// last \\\r\nSPLICEFAKE_API notes\n" + "class REALMOD_API Real { int v; };\n"), + 0); + CBMFileResult *r = extract(src, CBM_LANG_CPP, "p", "spliced.h"); + ASSERT_NOT_NULL(r); + const CBMDefinition *cls = find_def(r, "Real"); + ASSERT_NOT_NULL(cls); + ASSERT(cls->label && strcmp(cls->label, "Class") == 0); + cbm_free_result(r); + PASS(); +} + +/* #1989 review round 2: C++ raw string literals. Tokens inside R"(...)" + * must not be collected, and code after the literal must still be scanned — + * an unmodeled raw string previously let "BAR_API" leak in AND masked the + * real candidate after the literal's closing quote. An unrecognizable raw + * form (unterminated/malformed delimiter) stops collection entirely so scan + * state cannot be poisoned. */ +TEST(extract_cpp_export_macro_raw_string_issue1989) { + CBMFileResult *r = extract("const char *s = R\"(foo \" BAR_API)\";\n" + "class REALMOD_API Real { int v; };\n", + CBM_LANG_CPP, "p", "raw.h"); + ASSERT_NOT_NULL(r); + const CBMDefinition *cls = find_def(r, "Real"); + ASSERT_NOT_NULL(cls); + ASSERT(cls->label && strcmp(cls->label, "Class") == 0); + cbm_free_result(r); + PASS(); +} + +TEST(extract_cpp_export_macro_raw_string_delimited_issue1989) { + CBMFileResult *r = extract("auto log = R\"log(FOO_API \"quote\")log\";\n" + "class DELIMMOD_API Delim { int v; };\n", + CBM_LANG_CPP, "p", "raw_delim.h"); + ASSERT_NOT_NULL(r); + const CBMDefinition *cls = find_def(r, "Delim"); + ASSERT_NOT_NULL(cls); + ASSERT(cls->label && strcmp(cls->label, "Class") == 0); + cbm_free_result(r); + PASS(); +} + +/* #1989 review: every conventional suffix variant must be collected and + * recovered, not just _API. */ +TEST(extract_cpp_export_macro_suffix_variants_issue1989) { + CBMFileResult *r = extract("class LIB_EXPORT A1 { int v; };\n" + "class LIB_IMPORT B2 { int v; };\n" + "class LIB_DLLEXPORT C3 { int v; };\n" + "class LIB_DEPRECATED D4 { int v; };\n", + CBM_LANG_CPP, "p", "suffix.h"); + ASSERT_NOT_NULL(r); + const char *names[] = {"A1", "B2", "C3", "D4"}; + for (int n = 0; n < 4; n++) { + const CBMDefinition *d = find_def(r, names[n]); + ASSERT_NOT_NULL(d); + ASSERT(d->label && strcmp(d->label, "Class") == 0); + } + cbm_free_result(r); + PASS(); +} + +/* #1989 review: caller-provided defines win — the collector must not push a + * define for a name the caller already provided (a duplicate could flip the + * expansion and silently change what parses). With an explicit empty define + * the class still recovers; with a caller define that corrupts the header, + * the injected empty define must NOT clobber it back into parsing. */ +TEST(extract_cpp_export_macro_explicit_define_priority_issue1989) { + const char *src = "class MYMOD_API Foo { int v; };\n"; + + const char *empty_defines[] = {"MYMOD_API=", NULL}; + CBMFileResult *r = cbm_extract_file(src, (int)strlen(src), CBM_LANG_CPP, "p", "prio_empty.h", 0, + empty_defines, NULL); + ASSERT_NOT_NULL(r); + const CBMDefinition *cls = find_def(r, "Foo"); + ASSERT_NOT_NULL(cls); + ASSERT(cls->label && strcmp(cls->label, "Class") == 0); + cbm_free_result(r); + + const char *body_defines[] = {"MYMOD_API=Junk", NULL}; + CBMFileResult *r2 = cbm_extract_file(src, (int)strlen(src), CBM_LANG_CPP, "p", "prio_body.h", 0, + body_defines, NULL); + ASSERT_NOT_NULL(r2); + const CBMDefinition *foo2 = find_def(r2, "Foo"); + ASSERT_TRUE(foo2 == NULL || (foo2->label && strcmp(foo2->label, "Class") != 0)); + cbm_free_result(r2); + PASS(); +} + +/* #1989 review round 2: the collector CONTRACT itself, called directly (the + * reviewer's method): the raw string's inner token is not a candidate and the + * code after the literal IS. */ +TEST(extract_cpp_export_macro_collector_raw_string_contract_issue1989) { + const char *src = "const char *s = R\"(foo \" BAR_API)\";\n" + "class REALMOD_API Real { int v; };\n"; + char out[CBM_EXPORT_MACRO_MAX][CBM_EXPORT_MACRO_NAME_MAX]; + int n = cbm_export_macro_candidates(src, (int)strlen(src), out, CBM_EXPORT_MACRO_MAX); + ASSERT_EQ(n, 1); + ASSERT(strcmp(out[0], "REALMOD_API") == 0); + PASS(); +} + +/* Delimiter form R"log(...)log" works the same way. */ +TEST(extract_cpp_export_macro_collector_raw_string_delim_contract_issue1989) { + const char *src = "auto log = R\"log(FOO_API \"q\")log\";\n" + "class DELIMMOD_API Delim { int v; };\n"; + char out[CBM_EXPORT_MACRO_MAX][CBM_EXPORT_MACRO_NAME_MAX]; + int n = cbm_export_macro_candidates(src, (int)strlen(src), out, CBM_EXPORT_MACRO_MAX); + ASSERT_EQ(n, 1); + ASSERT(strcmp(out[0], "DELIMMOD_API") == 0); + PASS(); +} + +/* A double quote is a legal raw-string delimiter character. Keep scanning + * after R"""(...)""" so a later export macro is still collected. */ +TEST(extract_cpp_export_macro_collector_raw_string_quote_delim_issue1989) { + const char *src = "const char *s = R\"\"\"(FOO_API)\"\"\";\n" + "class QUOTEMOD_API Quoted { int v; };\n"; + char out[CBM_EXPORT_MACRO_MAX][CBM_EXPORT_MACRO_NAME_MAX]; + int n = cbm_export_macro_candidates(src, (int)strlen(src), out, CBM_EXPORT_MACRO_MAX); + ASSERT_EQ(n, 1); + ASSERT(strcmp(out[0], "QUOTEMOD_API") == 0); + PASS(); +} + +/* Unterminated raw literal: the scan stops rather than mis-tokenizing the + * rest of the file (uncertain state -> fail toward raw behavior). */ +TEST(extract_cpp_export_macro_collector_raw_string_unterminated_issue1989) { + const char *src = "const char *s = R\"(never closed BAR_API\n" + "class REALMOD_API Real { int v; };\n"; + char out[CBM_EXPORT_MACRO_MAX][CBM_EXPORT_MACRO_NAME_MAX]; + int n = cbm_export_macro_candidates(src, (int)strlen(src), out, CBM_EXPORT_MACRO_MAX); + ASSERT_EQ(n, 0); + PASS(); +} + /* #668: walk_defs used a fixed `walk_defs_frame_t stack[4096]` — a ~160 KB * C-stack frame that overflowed small thread stacks (the reporter's crash was in * the "definitions pass" on a large SQL file), and whose `top < 4096` push guards @@ -7974,6 +8339,25 @@ SUITE(extraction) { RUN_TEST(extract_cpp_preproc_macro_generated_callable_skipped_issue949); RUN_TEST(extract_c_ifdef_split_brace_after_include_remapped_issue949); RUN_TEST(extract_c_clean_file_no_recovery_duplicates_issue961); + RUN_TEST(extract_cpp_export_macro_class_recovery_issue1989); + RUN_TEST(extract_cpp_export_macro_struct_recovery_issue1989); + RUN_TEST(extract_cpp_export_macro_enum_recovery_issue1989); + RUN_TEST(extract_cpp_export_macro_free_function_recovery_issue1989); + RUN_TEST(extract_cpp_export_macro_inline_method_recovery_issue1989); + RUN_TEST(extract_cpp_export_macro_negative_control_ordinary_caps_issue1989); + RUN_TEST(extract_c_export_macro_recovery_issue1989); + RUN_TEST(extract_cpp_export_macro_overlong_candidate_safe_issue1989); + RUN_TEST(extract_cpp_export_macro_comment_string_budget_issue1989); + RUN_TEST(extract_cpp_export_macro_candidate_cap_issue1989); + RUN_TEST(extract_cpp_export_macro_spliced_comment_budget_issue1989); + RUN_TEST(extract_cpp_export_macro_raw_string_issue1989); + RUN_TEST(extract_cpp_export_macro_raw_string_delimited_issue1989); + RUN_TEST(extract_cpp_export_macro_collector_raw_string_contract_issue1989); + RUN_TEST(extract_cpp_export_macro_collector_raw_string_delim_contract_issue1989); + RUN_TEST(extract_cpp_export_macro_collector_raw_string_quote_delim_issue1989); + RUN_TEST(extract_cpp_export_macro_collector_raw_string_unterminated_issue1989); + RUN_TEST(extract_cpp_export_macro_suffix_variants_issue1989); + RUN_TEST(extract_cpp_export_macro_explicit_define_priority_issue1989); RUN_TEST(walk_defs_no_truncation_over_4096_issue668); RUN_TEST(extract_rust_test_attr_marks_is_test_issue855); RUN_TEST(extract_c_test_dir_marks_is_test_issue1294); From 460ebefc87df4dd5142bdc339702edd4feec4696 Mon Sep 17 00:00:00 2001 From: juan Carlos Estrada Montoya Date: Fri, 4 Sep 2026 14:20:11 -0500 Subject: [PATCH 108/130] fix(graph-buffer): log why an atomic publish failed instead of a successful dump MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cbm_gbuf_dump_to_sqlite() emitted gbuf.dump regardless of the writer's result, so a run that published nothing still logged node and edge counts as if it had worked, and the failure reached the user only as the generic "Pipeline failed. Check repo_path exists and contains source files." The writer's temp -> final rename in publish_writer_output() returned a bare ERR_WRITE_FAILED. #1628 taught cbm_rename_replace() to translate the platform error into errno so callers could report why, and wired up the stage -> final rename in pipeline.c. This is the other rename on that path, and it is the one that runs first. Preserve errno across the cleanup in every publish error branch. That includes cbm_writer_open(), where the cleanup unlinks a file that was never created: without the save it leaves ENOENT behind, so a directory that denied the create is reported as a missing path — the #1620 case, stated wrongly. Read the value in graph_buffer.c before the profiling macro can clobber it, and report gbuf.dump_failed with the return code and that errno. A sticky append failure clears errno rather than attach a reason it does not have, and the field is omitted when there is none. No logging is added inside internal/cbm/, which has none today. Tests cover both halves: the dump reports a failure instead of a summary, the writer names its truncation reason, and the publish rename's errno survives a cleanup that itself fails. Refs #1620, #2001. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012FxkVF773gDGNmBmsLnzN7 Signed-off-by: juan Carlos Estrada Montoya --- internal/cbm/sqlite_writer.c | 26 +++++++++ src/graph_buffer/graph_buffer.c | 38 ++++++++++++- tests/test_graph_buffer.c | 73 +++++++++++++++++++++++++ tests/test_sqlite_writer.c | 97 +++++++++++++++++++++++++++++++++ 4 files changed, 233 insertions(+), 1 deletion(-) diff --git a/internal/cbm/sqlite_writer.c b/internal/cbm/sqlite_writer.c index 55ab937d4..da8aba06c 100644 --- a/internal/cbm/sqlite_writer.c +++ b/internal/cbm/sqlite_writer.c @@ -28,6 +28,7 @@ #include #include #include +#include #ifdef _WIN32 #include @@ -1777,6 +1778,10 @@ static int sync_writer_output(FILE *fp) { } static int discard_writer_output(write_db_ctx_t *w, int rc) { + /* Cleanup runs after the failure that brought us here, and a library + * call may set errno even when it succeeds. Carry the reason across it + * so the caller can report WHY the publish failed. */ + int failure_errno = errno; if (w->fp) { (void)fclose(w->fp); w->fp = NULL; @@ -1784,6 +1789,7 @@ static int discard_writer_output(write_db_ctx_t *w, int rc) { if (w->temp_path[0]) { (void)cbm_unlink(w->temp_path); } + errno = failure_errno; return rc; } @@ -1792,10 +1798,12 @@ static int publish_writer_output(write_db_ctx_t *w) { return discard_writer_output(w, ERR_WRITE_FAILED); } if (fclose(w->fp) != 0) { + int failure_errno = errno; w->fp = NULL; if (w->temp_path[0]) { (void)cbm_unlink(w->temp_path); } + errno = failure_errno; return ERR_WRITE_FAILED; } w->fp = NULL; @@ -1803,7 +1811,12 @@ static int publish_writer_output(write_db_ctx_t *w) { return 0; } if (cbm_rename_replace(w->temp_path, w->final_path) != 0) { + /* cbm_rename_replace translated the platform error into errno so the + * caller can say what denied the publish (#1620). Preserve it across + * the cleanup unlink. */ + int rename_errno = errno; (void)cbm_unlink(w->temp_path); + errno = rename_errno; return ERR_WRITE_FAILED; } /* Sidecars are removed only after the replacement succeeds. On POSIX, @@ -2303,13 +2316,22 @@ cbm_db_writer_t *cbm_writer_open(const char *path) { int n = snprintf(w->wc.final_path, sizeof(w->wc.final_path), "%s", path); if (n < 0 || (size_t)n >= sizeof(w->wc.final_path) || make_writer_temp_path(path, w, w->wc.temp_path, sizeof(w->wc.temp_path)) != 0) { + /* Both conditions are truncation and neither sets errno; name the + * reason rather than let the caller report a stale one. */ free(w); + errno = ENAMETOOLONG; return NULL; } FILE *fp = cbm_fopen(w->wc.temp_path, "wb"); if (!fp) { + /* The cleanup below unlinks a file that was never created, so it + * fails and leaves ENOENT behind — which reads as a missing path + * when the real answer is that the directory refused the create. + * That is the #1620 case, so carry the open's reason across it. */ + int open_errno = errno; (void)cbm_unlink(w->wc.temp_path); free(w); + errno = open_errno; return NULL; } w->wc.fp = fp; @@ -2377,6 +2399,10 @@ int cbm_writer_finalize(cbm_db_writer_t *w, const char *project, const char *roo write_db_ctx_t wc = w->wc; /* value copy survives free(w) */ free(w); if (err != 0) { + /* A sticky append failure: errno belongs to whatever call failed + * many calls ago, not to the publish. Clear it so the caller does + * not attach a reason this path does not have. */ + errno = 0; return discard_writer_output(&wc, err); } return write_db_after_nodes(&wc, nodes_root); diff --git a/src/graph_buffer/graph_buffer.c b/src/graph_buffer/graph_buffer.c index 24553657d..e55eb28f0 100644 --- a/src/graph_buffer/graph_buffer.c +++ b/src/graph_buffer/graph_buffer.c @@ -39,6 +39,7 @@ enum { #include "foundation/mem_core.h" #include +#include #include #include // int64_t #include @@ -1841,6 +1842,32 @@ static void log_dump_summary(int node_count, int edge_count) { cbm_log_info("gbuf.dump", "nodes", b1, "edges", b2); } +/* A dump that published nothing gets exactly one signature in the log. + * Without it the writer's return code is the only evidence, and it reaches + * the user as a generic pipeline failure that blames their repository + * (#1620, #2001). + * + * failure_errno is read by the caller at the point of failure: the value is + * only meaningful before the next library call. Zero means the failure did + * not originate at the publish boundary — an append that failed many calls + * earlier — and the field is omitted rather than claim a reason this record + * does not have. */ +static void log_dump_failed(int rc, int failure_errno, int node_count, int edge_count) { + char b1[CBM_SZ_16]; + char b3[CBM_SZ_16]; + char b4[CBM_SZ_16]; + snprintf(b1, sizeof(b1), "%d", rc); + snprintf(b3, sizeof(b3), "%d", node_count); + snprintf(b4, sizeof(b4), "%d", edge_count); + if (failure_errno == 0) { + cbm_log_error("gbuf.dump_failed", "rc", b1, "nodes", b3, "edges", b4); + return; + } + char b2[CBM_SZ_16]; + snprintf(b2, sizeof(b2), "%d", failure_errno); + cbm_log_error("gbuf.dump_failed", "rc", b1, "errno", b2, "nodes", b3, "edges", b4); +} + static void free_dump_resources(char **url_paths, char **local_names, int edge_count, CBMDumpEdge *dump_edges, CBMDumpNode *dump_nodes, int64_t *temp_to_final) { @@ -1896,6 +1923,7 @@ int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path) { int64_t max_temp_id = gb->next_id; int64_t *temp_to_final = cbm_calloc(CBM_MEM_CLASS_DUMP, (size_t)max_temp_id * sizeof(int64_t)); if (!temp_to_final) { + log_dump_failed(CBM_NOT_FOUND, errno, 0, 0); return CBM_NOT_FOUND; } @@ -1926,6 +1954,7 @@ int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path) { * uninitialized budget from ever triggering the free). */ cbm_db_writer_t *w = cbm_writer_open(path); if (!w) { + log_dump_failed(CBM_NOT_FOUND, errno, node_idx, 0); cbm_free(CBM_MEM_CLASS_DUMP, src_nodes); cbm_free(CBM_MEM_CLASS_DUMP, dump_nodes); cbm_free(CBM_MEM_CLASS_DUMP, temp_to_final); @@ -1980,12 +2009,19 @@ int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path) { int frc = cbm_writer_finalize(w, gb->project, gb->root_path, indexed_at, dump_nodes, node_idx, dump_edges, edge_idx, gb->dump_vectors, gb->dump_vector_count, gb->dump_token_vecs, gb->dump_token_vec_count); + /* Read errno before anything else can touch it: the profiling macro + * below is already one library call away from losing the reason. */ + int finalize_errno = errno; CBM_PROF_END_N("dump", "6_write_db_finalize", t_finalize, node_idx + edge_idx); if (rc == 0) { rc = frc; } - log_dump_summary(node_idx, edge_idx); + if (rc != 0) { + log_dump_failed(rc, finalize_errno, node_idx, edge_idx); + } else { + log_dump_summary(node_idx, edge_idx); + } free_dump_resources(url_paths, local_names, edge_idx, dump_edges, dump_nodes, temp_to_final); cbm_free(CBM_MEM_CLASS_DUMP, src_nodes); return rc; diff --git a/tests/test_graph_buffer.c b/tests/test_graph_buffer.c index 256fc0cec..8a32d5a34 100644 --- a/tests/test_graph_buffer.c +++ b/tests/test_graph_buffer.c @@ -9,6 +9,10 @@ #include "foundation/mem_core.h" #include #include "store/store.h" +#include "../src/foundation/compat.h" +#include "foundation/compat_fs.h" +#include "foundation/log.h" +#include #include /* ── Node operations ───────────────────────────────────────────── */ @@ -1120,6 +1124,72 @@ TEST(gbuf_flush_skips_orphan_edges) { PASS(); } +/* ── Publish failure reporting ───────────────────────────────── */ + +static char g_log_capture[4096]; +static CBMLogLevel g_prev_log_level; +static CBMLogFormat g_prev_log_format; + +static void capture_log_sink(const char *line) { + size_t used = strlen(g_log_capture); + size_t avail = sizeof(g_log_capture) - used; + if (avail <= 1) { + return; + } + int n = snprintf(g_log_capture + used, avail, "%s\n", line); + if (n < 0 || (size_t)n >= avail) { + g_log_capture[sizeof(g_log_capture) - 1] = '\0'; + } +} + +static void capture_logs_start(void) { + g_log_capture[0] = '\0'; + g_prev_log_level = cbm_log_get_level(); + g_prev_log_format = cbm_log_get_format(); + cbm_log_set_level(CBM_LOG_DEBUG); + /* The assertions below read the text encoding, so pin it rather than + * inherit whatever CBM_LOG_FORMAT left set. */ + cbm_log_set_format(CBM_LOG_FORMAT_TEXT); + cbm_log_set_sink(capture_log_sink); +} + +static const char *capture_logs_end(void) { + cbm_log_set_sink(NULL); + cbm_log_set_level(g_prev_log_level); + cbm_log_set_format(g_prev_log_format); + return g_log_capture; +} + +/* A dump that publishes nothing has to say so, and say why. Renaming onto an + * existing directory is how test_sqlite_writer already forces the publish to + * fail; here it stands in for any host that denies the rename (#1620). */ +TEST(gbuf_dump_failure_logs_reason) { + char dir[256]; + snprintf(dir, sizeof(dir), "/tmp/cbm_gbuf_pub_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(dir)); + + cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/repo"); + ASSERT_NOT_NULL(gb); + int64_t id = cbm_gbuf_upsert_node(gb, "Function", "main", "pkg.main", "main.go", 1, 10, "{}"); + ASSERT_GT(id, 0); + + capture_logs_start(); + int rc = cbm_gbuf_dump_to_sqlite(gb, dir); + const char *logs = capture_logs_end(); + + ASSERT(rc != 0); + ASSERT_NOT_NULL(strstr(logs, "gbuf.dump_failed")); + /* The reason survived the cleanup unlink. */ + ASSERT_NOT_NULL(strstr(logs, "errno=")); + ASSERT(strstr(logs, "errno=0 ") == NULL); + /* And the run is not also reported as a successful dump. */ + ASSERT(strstr(logs, "msg=gbuf.dump ") == NULL); + + cbm_gbuf_free(gb); + cbm_rmdir(dir); + PASS(); +} + /* ── Suite ─────────────────────────────────────────────────────── */ /* A worker buffer draws ids from the shared counter, so a dense id -> node @@ -1221,4 +1291,7 @@ SUITE(graph_buffer) { RUN_TEST(gbuf_shared_ids_null_fallback); RUN_TEST(gbuf_next_id_set_next_id_roundtrip); RUN_TEST(gbuf_next_id_null_safe); + + /* Publish failure reporting */ + RUN_TEST(gbuf_dump_failure_logs_reason); } diff --git a/tests/test_sqlite_writer.c b/tests/test_sqlite_writer.c index 1084d0529..a80fee059 100644 --- a/tests/test_sqlite_writer.c +++ b/tests/test_sqlite_writer.c @@ -13,6 +13,7 @@ /* sqlite_writer.h is at internal/cbm/ — Makefile adds -Iinternal/cbm */ #include "sqlite_writer.h" /* CBMDumpNode, CBMDumpEdge, cbm_write_db */ #include "sqlite3.h" /* vendored/sqlite3/ via -Ivendored/sqlite3 */ +#include #include /* ── Helper: create temp file path ─────────────────────────────── */ @@ -101,6 +102,45 @@ static int count_temp_outputs_for(const char *path) { return count; } +/* Locate the writer's temp output for `path`. Same naming rule as + * count_temp_outputs_for, but returns the full path of the first match. */ +static int find_temp_output_for(const char *path, char *out, size_t out_size) { + char dir[256]; + char base[256]; + const char *slash = strrchr(path, '/'); + if (slash) { + size_t dir_len = (size_t)(slash - path); + if (dir_len == 0 || dir_len >= sizeof(dir)) { + return -1; + } + memcpy(dir, path, dir_len); + dir[dir_len] = '\0'; + snprintf(base, sizeof(base), "%s", slash + 1); + } else { + snprintf(dir, sizeof(dir), "."); + snprintf(base, sizeof(base), "%s", path); + } + + cbm_dir_t *d = cbm_opendir(dir); + if (!d) { + return -1; + } + size_t base_len = strlen(base); + int found = -1; + cbm_dirent_t *ent; + while ((ent = cbm_readdir(d)) != NULL) { + size_t name_len = strlen(ent->name); + if (name_len > base_len + 5 && strncmp(ent->name, base, base_len) == 0 && + strncmp(ent->name + base_len, ".tmp.", 5) == 0) { + snprintf(out, out_size, "%s/%s", dir, ent->name); + found = 0; + break; + } + } + cbm_closedir(d); + return found; +} + /* ── Tests ─────────────────────────────────────────────────────── */ TEST(sw_minimal_data) { @@ -877,6 +917,61 @@ TEST(sw_publish_preserves_live_reader) { /* ── Suite ─────────────────────────────────────────────────────── */ +TEST(sw_open_truncated_path_names_its_reason) { + /* final_path is a fixed 4K buffer and make_writer_temp_path only fails by + * truncation. Neither sets an errno of its own, so without naming one the + * caller reports whatever was left behind by an unrelated call. */ + char longpath[5000]; + memset(longpath, 'a', sizeof(longpath) - 1); + longpath[sizeof(longpath) - 1] = '\0'; + longpath[0] = '/'; + + errno = 0; + cbm_db_writer_t *w = cbm_writer_open(longpath); + ASSERT(w == NULL); + ASSERT_EQ(errno, ENAMETOOLONG); + PASS(); +} + +TEST(sw_publish_failure_reports_the_rename_not_the_cleanup) { +#ifdef _WIN32 + SKIP_PLATFORM("removing a still-open file to make the cleanup unlink fail"); +#endif + char dir[256]; + snprintf(dir, sizeof(dir), "/tmp/cbm_sw_cleanup_XXXXXX"); + ASSERT(cbm_mkdtemp(dir) != NULL); + + char final_path[320]; + snprintf(final_path, sizeof(final_path), "%s/db.sqlite", dir); + ASSERT_EQ(write_fixture_file(final_path, "destination"), 0); + + cbm_db_writer_t *w = cbm_writer_open(final_path); + ASSERT(w != NULL); + + /* Turn the still-open temp into a directory. The publish rename then fails + * with ENOTDIR (directory onto a file), and the cleanup unlink fails too + * (EISDIR on Linux, EPERM on macOS) — so only a saved errno still carries + * the rename's reason out to the caller. */ + char temp[512]; + ASSERT_EQ(find_temp_output_for(final_path, temp, sizeof(temp)), 0); + ASSERT_EQ(cbm_unlink(temp), 0); + ASSERT(cbm_mkdir_p(temp, 0700)); + + errno = 0; + int rc = cbm_writer_finalize(w, "test", "/tmp/test", "2026-07-07T00:00:00Z", NULL, 0, NULL, 0, + NULL, 0, NULL, 0); + int publish_errno = errno; + ASSERT(rc != 0); + ASSERT_EQ(publish_errno, ENOTDIR); + /* The destination is intact: a failed publish never replaces it. */ + ASSERT(fixture_file_equals(final_path, "destination")); + + cbm_rmdir(temp); + cbm_unlink(final_path); + cbm_rmdir(dir); + PASS(); +} + SUITE(sqlite_writer) { RUN_TEST(sw_minimal_data); RUN_TEST(sw_imports_local_name_unique); @@ -890,4 +985,6 @@ SUITE(sqlite_writer) { RUN_TEST(sw_publish_failure_preserves_destination_sidecars); RUN_TEST(sw_publish_supports_non_ascii_path); RUN_TEST(sw_publish_preserves_live_reader); + RUN_TEST(sw_open_truncated_path_names_its_reason); + RUN_TEST(sw_publish_failure_reports_the_rename_not_the_cleanup); } From af3e003ad8e67cd1fe3b20dafaf4d33a4b351852 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 20 Sep 2026 18:15:33 +0200 Subject: [PATCH 109/130] fix(mcp): one release point for handle_index_repository's argument strings main fails the memory-core linter: src/mcp/mcp.c: grew by 3 (814 -> 817) PR #1723 (merged as 92abefa3) added a new early-return path to handle_index_repository and, following the function's convention, freed its three argument strings there with raw free(). Its CI lint was green on 2026-09-09. On 2026-09-19 the memwaste merge cut this file's raw sites and ratcheted the baseline down to 814, so by the time #1723 landed its green was measured against a rule that no longer existed. The linter is right and the merge-result check that let it through was incomplete: it built and ran the suites and did not run the linter. It does now. The fix is not cbm_free. The three strings come from cbm_mcp_get_string_arg and resolved_repo_path_from_project_arg and are raw heap memory, released raw by every other path in the function; routing three of them through the core would free raw memory through the accounting layer. The actual defect is that the function had 41 raw free() calls of the same three arguments across a dozen early returns, and #1723 added one more block in the same shape. One helper, index_args_free(repo_path, mode_str, name_override), now replaces twelve of those blocks. free(NULL) is a no-op, so the one path where repo_path is still NULL passes NULL. On the cross-repo branch mode_str used to be freed before handle_cross_repo_mode and the other two after it; mode_str is neither passed to nor read by that call, so releasing all three after it is not observable. Five sites on the normal-completion path are left as they were: they free the strings at different points as ownership is handed to the pipeline, and collapsing them would change that order for no gain. Raw sites in this file: 817 -> 785. The baseline follows the improvement, 814 -> 785, as the linter asks; it can only tighten. Build: clean, -Werror, ASan/UBSan. mcp cli daemon_application: 692 passed, 4 skipped, 0 failed -- index_repository_honors_allowed_root, which runs through a replaced early return, included. Signed-off-by: Martin Vogel --- scripts/memory-core-baseline.txt | 2 +- src/mcp/mcp.c | 57 ++++++++++++-------------------- 2 files changed, 23 insertions(+), 36 deletions(-) diff --git a/scripts/memory-core-baseline.txt b/scripts/memory-core-baseline.txt index 8bc450f7f..19fdd363d 100644 --- a/scripts/memory-core-baseline.txt +++ b/scripts/memory-core-baseline.txt @@ -52,7 +52,7 @@ src/git/git_context.c 18 src/main.c 32 src/mcp/compact_out.c 39 src/mcp/index_supervisor.c 8 -src/mcp/mcp.c 814 +src/mcp/mcp.c 785 src/pipeline/artifact.c 38 src/pipeline/fqn.c 23 src/pipeline/lsp_resolve.h 4 diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 39aaf0dcc..7bfca8d7a 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -11062,6 +11062,16 @@ static bool project_db_is_servable(const char *project, const char *db_path) { return servable; } +/* The three heap strings handle_index_repository owns from + * cbm_mcp_get_string_arg / resolved_repo_path_from_project_arg. One release + * point keeps the dozen early-return paths in step; free(NULL) is a no-op, so + * a path may pass NULL for an argument it never obtained. */ +static void index_args_free(char *repo_path, char *mode_str, char *name_override) { + free(repo_path); + free(mode_str); + free(name_override); +} + static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { char *repo_path = cbm_mcp_get_string_arg(args, "repo_path"); char *mode_str = cbm_mcp_get_string_arg(args, "mode"); @@ -11074,15 +11084,12 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { } if (!repo_path) { - free(mode_str); - free(name_override); + index_args_free(NULL, mode_str, name_override); return cbm_mcp_text_result("repo_path is required", true); } if (!resolve_session_repo_path(srv, &repo_path)) { - free(mode_str); - free(name_override); - free(repo_path); + index_args_free(repo_path, mode_str, name_override); return cbm_mcp_text_result("failed to resolve repo_path", true); } @@ -11102,26 +11109,20 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { if (repo_path && repo_path[0] && !cbm_workspace_root_allowed(repo_path, cbm_workspace_home_dir(), cbm_workspace_cache_dir(), allowed_root, boundary_err, sizeof(boundary_err))) { - free(mode_str); - free(name_override); - free(repo_path); + index_args_free(repo_path, mode_str, name_override); return cbm_mcp_text_result(boundary_err, true); } if (mode_str && strcmp(mode_str, "cross-repo-intelligence") == 0) { - free(mode_str); char *result = handle_cross_repo_mode(srv, repo_path, name_override, args); - free(name_override); - free(repo_path); + index_args_free(repo_path, mode_str, name_override); return result; } cbm_index_resource_policy_t resource_policy; char policy_error[CBM_SZ_256] = {0}; if (!load_index_policy(srv, args, &resource_policy, policy_error, sizeof(policy_error))) { - free(mode_str); - free(name_override); - free(repo_path); + index_args_free(repo_path, mode_str, name_override); return cbm_mcp_text_result(policy_error, true); } @@ -11133,9 +11134,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { worker_args ? srv->index_executor(srv->index_executor_context, repo_path, worker_args) : NULL; free(worker_args); - free(repo_path); - free(mode_str); - free(name_override); + index_args_free(repo_path, mode_str, name_override); return coordinated ? coordinated : cbm_mcp_text_result( "daemon index coordinator could not start the operation", true); @@ -11148,9 +11147,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { char *mutation_project = cbm_project_name_from_path(name_override && name_override[0] ? name_override : repo_path); if (!mutation_project) { - free(repo_path); - free(mode_str); - free(name_override); + index_args_free(repo_path, mode_str, name_override); return cbm_mcp_text_result("could not resolve index project name", true); } @@ -11163,24 +11160,18 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { char *worker_args = index_args_with_repo_path(args, repo_path, &resource_policy); if (!worker_args) { free(mutation_project); - free(repo_path); - free(mode_str); - free(name_override); + index_args_free(repo_path, mode_str, name_override); return cbm_mcp_text_result("failed to prepare supervised index request", true); } char *supervised = index_run_supervised(srv, worker_args); free(worker_args); if (supervised) { free(mutation_project); - free(repo_path); - free(mode_str); - free(name_override); + index_args_free(repo_path, mode_str, name_override); return supervised; } free(mutation_project); - free(repo_path); - free(mode_str); - free(name_override); + index_args_free(repo_path, mode_str, name_override); return cbm_mcp_text_result( "index supervision failed before a contained worker could start; no " "in-process fallback was attempted", @@ -11189,18 +11180,14 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { if (!mcp_project_mutation_begin(srv, mutation_project)) { free(mutation_project); - free(repo_path); - free(mode_str); - free(name_override); + index_args_free(repo_path, mode_str, name_override); return cbm_mcp_text_result("index operation blocked by another mutation for this project", true); } if (mcp_request_cancelled(srv)) { mcp_project_mutation_end(srv, mutation_project); free(mutation_project); - free(repo_path); - free(mode_str); - free(name_override); + index_args_free(repo_path, mode_str, name_override); return cbm_mcp_text_result("index operation cancelled for this request", true); } From 25d08258e567ec417bf7b18b14a63e30162fc7da Mon Sep 17 00:00:00 2001 From: News Date: Mon, 21 Sep 2026 01:37:51 +0900 Subject: [PATCH 110/130] fix(windows): keep aligned allocations on mimalloc Signed-off-by: News --- src/foundation/compat_thread.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/foundation/compat_thread.c b/src/foundation/compat_thread.c index 94af9f812..b86a7f3fb 100644 --- a/src/foundation/compat_thread.c +++ b/src/foundation/compat_thread.c @@ -2,7 +2,7 @@ * compat_thread.c — Portable thread, mutex, and aligned allocation. * * POSIX: thin wrappers around pthreads and posix_memalign. - * Windows: CreateThread, CRITICAL_SECTION, _aligned_malloc. + * Windows: CreateThread, CRITICAL_SECTION, aligned allocation. */ #include "foundation/mem_events.h" #include "foundation/constants.h" @@ -279,12 +279,20 @@ void cbm_mutex_destroy(cbm_mutex_t *m) { #ifdef _WIN32 int cbm_aligned_alloc(void **ptr, size_t alignment, size_t size) { +#if defined(CBM_MEM_GLOBAL_OVERRIDE) && CBM_MEM_GLOBAL_OVERRIDE + *ptr = mi_malloc_aligned(size, alignment); +#else *ptr = _aligned_malloc(size, alignment); +#endif return *ptr ? 0 : -1; } void cbm_aligned_free(void *ptr) { +#if defined(CBM_MEM_GLOBAL_OVERRIDE) && CBM_MEM_GLOBAL_OVERRIDE + mi_free(ptr); +#else _aligned_free(ptr); +#endif } #else /* POSIX */ From dadc57dbfa8615ce897c338f6af8d6c6eb1674b1 Mon Sep 17 00:00:00 2001 From: Tamer Date: Sun, 20 Sep 2026 21:37:41 +0200 Subject: [PATCH 111/130] fix(smoke): keep MCP tool inventory exact Signed-off-by: Tamer --- scripts/smoke-invariants.sh | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/scripts/smoke-invariants.sh b/scripts/smoke-invariants.sh index 3a4f9ce09..175a59903 100755 --- a/scripts/smoke-invariants.sh +++ b/scripts/smoke-invariants.sh @@ -520,10 +520,10 @@ inv_mcp_initialize() { return 0 } -# ── Invariant 4: tools/list returns all expected tools ───────────────────── -# Cross-check against the canonical 14-tool list (TOOLS[] in src/mcp/mcp.c). -EXPECTED_TOOLS="index_repository search_graph query_graph trace_path get_code_snippet get_graph_schema get_architecture search_code list_projects delete_project index_status detect_changes manage_adr ingest_traces" -EXPECTED_TOOL_COUNT=14 +# ── Invariant 4: tools/list returns exactly the expected tools ───────────── +# Cross-check against the canonical registry (TOOLS[] in src/mcp/mcp.c). +EXPECTED_TOOLS="index_repository search_graph query_graph trace_path get_code_snippet get_file_outline get_graph_schema compare_graphs get_architecture search_code list_projects delete_project index_status check_index_coverage detect_changes manage_adr ingest_traces" +EXPECTED_TOOL_COUNT="$(printf '%s\n' $EXPECTED_TOOLS | grep -c .)" inv_tools_list() { if ! mcp_alive; then fail "tools-list" "server not alive" @@ -573,7 +573,7 @@ print(r.get("nextCursor") or "")' 2>/dev/null)" fail "tools-list" "got $got_count tools across $page page(s), expected $EXPECTED_TOOL_COUNT; names=[$got_names]" return fi - local missing="" + local missing="" extra="" duplicates="" local t for t in $EXPECTED_TOOLS; do case " $got_names " in @@ -581,10 +581,21 @@ print(r.get("nextCursor") or "")' 2>/dev/null)" *) missing="$missing $t" ;; esac done + for t in $got_names; do + case " $EXPECTED_TOOLS " in + *" $t "*) ;; + *) extra="$extra $t" ;; + esac + done + duplicates="$(printf '%s\n' $got_names | sort | uniq -d | tr '\n' ' ')" if [ -n "$missing" ]; then fail "tools-list" "missing tools:$missing" + elif [ -n "$extra" ]; then + fail "tools-list" "unexpected tools:$extra" + elif [ -n "$duplicates" ]; then + fail "tools-list" "duplicate tools: $duplicates" else - pass "tools-list (all $EXPECTED_TOOL_COUNT tools present)" + pass "tools-list (exactly $EXPECTED_TOOL_COUNT unique tools present)" fi } @@ -626,7 +637,7 @@ inv_every_tool() { return fi - # name|minimal-args (JSON object) for the remaining 13 tools. + # name|minimal-args (JSON object) for the remaining 16 tools. # Args chosen to be minimally valid per TOOLS[] required fields. local p="$PROJ_NAME" local -a CALLS @@ -635,11 +646,14 @@ inv_every_tool() { "query_graph|{\"project\":\"$p\",\"query\":\"MATCH (n) RETURN n.name LIMIT 5\"}" "trace_path|{\"project\":\"$p\",\"function_name\":\"compute\",\"direction\":\"both\"}" "get_code_snippet|{\"project\":\"$p\",\"qualified_name\":\"compute\"}" + "get_file_outline|{\"project\":\"$p\",\"file_path\":\"src/main.py\"}" "get_graph_schema|{\"project\":\"$p\"}" + "compare_graphs|{\"base_project\":\"$p\",\"target_project\":\"$p\"}" "get_architecture|{\"project\":\"$p\"}" "search_code|{\"project\":\"$p\",\"pattern\":\"def \"}" "list_projects|{}" "index_status|{\"project\":\"$p\"}" + "check_index_coverage|{\"project\":\"$p\",\"paths\":[\"src/main.py\"]}" "detect_changes|{\"project\":\"$p\"}" "manage_adr|{\"project\":\"$p\",\"mode\":\"get\"}" "ingest_traces|{\"project\":\"$p\",\"traces\":[]}" From cda5e3c9e5c4d2251467ced00ff305205c368d17 Mon Sep 17 00:00:00 2001 From: Tamer Date: Sun, 20 Sep 2026 21:37:43 +0200 Subject: [PATCH 112/130] docs(ci): avoid hard-coded MCP tool count Signed-off-by: Tamer --- .github/workflows/smoke.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index aa91f91c3..e9b4f9357 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -1,7 +1,7 @@ # Smoke invariants across the WIDEST set of GitHub-hosted runners. Builds the # production source path with explicit fault-injection test seams and runs # scripts/smoke-invariants.sh -# (version/help, MCP initialize handshake [#513], all 14 tools invocable, malformed- +# (version/help, MCP initialize handshake [#513], all registered tools invocable, malformed- # input resilience, clean EOF exit, shared-lib resolution, install dry-run). # # Maximizing platforms is the point: ubuntu-22.04 (older glibc → AlmaLinux/#182 From c1128db8187db2dac27a7ad14120fc09802c8018 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 20 Sep 2026 21:40:37 +0200 Subject: [PATCH 113/130] fix(test): the worker-scope script sends the policy a supervisor sends main fails scripts/test.sh at Step 5f on every platform: === Step 5f: worker request-scope regression === worker did not index the admitted repository {"content":[{"type":"text","text":"missing or incomplete trusted worker policy"}], ... "isError":true} Second collision from the #1723 merge (92abefa3), and the same kind as the first. #1723 made an index worker refuse any request that does not carry `_cbm_index_policy`: a worker never resolves the resource policy from config or environment, the supervisor does and sends it along. tests/test_worker_session_scope.sh, added on 2026-09-06 by a08b9ecb, plays the supervisor by hand -- it runs `cli --index-worker index_repository '{"repo_path":...,"mode":"fast"}'` directly -- and #1723's branch never contained that script, so its CI never ran the two together. The merge was textually clean and the result was wrong. It stayed hidden behind the lint failure the previous commit fixes, because the test stage only runs once lint is green. The worker is right and the script was incomplete: it now sends what the supervisor sends, both limits off, which is the default (`"index_max_files":"off","index_max_source_mb":"off"`, the same object tests/test_index_policy.c uses). The fail-closed case carries it too, so that a request without repo_path is still refused for its missing scope ("request workspace scope invalid") and not, by accident, for a missing policy. No assertion changed. RED, production binary built from this branch, script unmodified: worker did not index the admitted repository ... "missing or incomplete trusted worker policy" ... rc=1 GREEN: ok: index worker is scoped to the admitted request, not the daemon environment rc=0 On the failed CI run of this PR, 12 of the 13 red test jobs carry exactly that signature and nothing else. The 13th, test-windows-guards, is test_daemon_stability.py section_cold_storm ("secure CLI coordination could not be created (endpoint)", a racing cold start) -- a known nondeterministic guard, unrelated to either commit here. Step 6, which Step 5f's failure kept CI from reaching, passes locally: security-strings allow-list 4 passed, destructive-ordering contract PASS. Signed-off-by: Martin Vogel --- tests/test_worker_session_scope.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_worker_session_scope.sh b/tests/test_worker_session_scope.sh index fed32e97e..ab018ba76 100644 --- a/tests/test_worker_session_scope.sh +++ b/tests/test_worker_session_scope.sh @@ -70,8 +70,15 @@ run_worker() { --response-out "${response}" >"${out}" 2>"${err}" } +# A worker never reads the resource policy from config or environment: the +# supervisor resolves it and sends it inside the request, and a request +# without one is refused ("missing or incomplete trusted worker policy"). +# This script plays the supervisor, so it sends what the supervisor sends -- +# both limits off, the default. +policy='"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"}' + response="${tmpdir}/scoped.response" -if ! run_worker "{\"repo_path\":\"${repo}\",\"mode\":\"fast\"}" "${response}" \ +if ! run_worker "{\"repo_path\":\"${repo}\",\"mode\":\"fast\",${policy}}" "${response}" \ "${tmpdir}/scoped.out" "${tmpdir}/scoped.err"; then echo "worker exited nonzero for an admitted request" >&2 cat "${tmpdir}/scoped.err" >&2 @@ -94,7 +101,7 @@ fi # Fail closed: no repo_path means no request scope, so no indexing at all. unscoped="${tmpdir}/unscoped.response" -if run_worker '{"mode":"fast"}' "${unscoped}" "${tmpdir}/unscoped.out" "${tmpdir}/unscoped.err"; then +if run_worker "{\"mode\":\"fast\",${policy}}" "${unscoped}" "${tmpdir}/unscoped.out" "${tmpdir}/unscoped.err"; then echo "worker ran without a request workspace scope" >&2 exit 1 fi From 183cd727be5968fec3ce43176ebb1fa255f8976b Mon Sep 17 00:00:00 2001 From: Anton Standrik Date: Sun, 13 Sep 2026 13:53:45 +0300 Subject: [PATCH 114/130] fix(test): isolate smoke harness daemon runtime scripts/smoke-test.sh retires the account daemon from seven call sites, but its wrappers sandbox only HOME/TMPDIR/CBM_CACHE_DIR and only CBM_RUNTIME_DIR moves the daemon rendezvous, so every retirement reached the operator's live daemon. Source scripts/test-runtime.sh so every product process runs under a harness-owned runtime and cache, and clean that root up from the EXIT trap. Add tests/test_smoke_runtime_isolation_contract.sh, which fails before this change, and wire it into scripts/test.sh. Part of #1696. Signed-off-by: Anton Standrik --- scripts/smoke-test.sh | 17 +++- scripts/test.sh | 3 + .../test_smoke_runtime_isolation_contract.sh | 82 +++++++++++++++++++ 3 files changed, 99 insertions(+), 3 deletions(-) create mode 100755 tests/test_smoke_runtime_isolation_contract.sh diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index 3371fc424..e996da847 100755 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -22,6 +22,8 @@ they stage the release fixture, start the fixture server, and sandbox HOME/TEMP/agent-config destinations. Called bare, the download/checksum/ install-script phases (12-13) SKIP for lack of a fixture server, and the run mutates the REAL profile — the venue-parity contract forbids that in any venue. +The daemon runtime and cache are private to the run either way: every product +process is started under a CBM_RUNTIME_DIR/CBM_CACHE_DIR this harness owns. Arguments: product binary to smoke @@ -41,6 +43,15 @@ if [ -n "$SMOKE_MODE" ] && [ "$SMOKE_MODE" != "--agent-config-only" ]; then fi REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")/.." && pwd)" +# Every product process below — the phases, the install/update E2E and the +# daemon retirements — must reach a daemon rendezvous this run owns. Only +# CBM_RUNTIME_DIR moves that rendezvous; the wrappers' HOME/TMPDIR/CBM_CACHE_DIR +# sandbox does not, so without this the retirements land on the operator's live +# account daemon (#1691, #1696). +# shellcheck source=test-runtime.sh +source "$REPO_ROOT/scripts/test-runtime.sh" +cbm_test_runtime_init + smoke_mktemp_file() { if [ -n "${SMOKE_TEMP_ROOT:-}" ]; then mktemp "$SMOKE_TEMP_ROOT/cbm-smoke.XXXXXX" @@ -100,8 +111,8 @@ copy_smoke_binary() { cp "$BINARY" "$destination" } -# Retire the shared account daemon (if one is running) and wait until it -# reports not-running. Install/uninstall flows leave an ephemeral daemon +# Retire this run's private account daemon (if one is running) and wait until +# it reports not-running. Install/uninstall flows leave an ephemeral daemon # draining asynchronously whose mapped generation backing and open logs # block rm on Windows (POSIX rm doesn't care) — so every cleanup of a # fixture HOME that received an install, and the final cache removal, must @@ -149,7 +160,7 @@ CODEX_LIFECYCLE_HOME="" if command -v cygpath &>/dev/null; then TMPDIR=$(cygpath -m "$TMPDIR") fi -trap 'smoke_rmtree "$TMPDIR" "${DRYRUN_HOME:-}" "${CODEX_LIFECYCLE_HOME:-}"' EXIT +trap 'smoke_rmtree "$TMPDIR" "${DRYRUN_HOME:-}" "${CODEX_LIFECYCLE_HOME:-}"; cbm_test_runtime_cleanup "$BINARY"' EXIT CLI_STDERR=$(smoke_mktemp_file) # 10 of the cli call sites assign directly (VAR=$(cli ...)). Under diff --git a/scripts/test.sh b/scripts/test.sh index 50d65643f..0845a2e77 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -272,6 +272,9 @@ bash "$ROOT/tests/test_release_gate_chain_contract.sh" echo "=== Step 0t: test runtime isolation contract (#1691) ===" bash "$ROOT/tests/test_runtime_isolation_contract.sh" +echo "=== Step 0t2: smoke harness runtime isolation contract (#1696) ===" +bash "$ROOT/tests/test_smoke_runtime_isolation_contract.sh" + echo "=== Step 0u: shell line-ending contract ===" bash "$ROOT/tests/test_shell_line_endings.sh" diff --git a/tests/test_smoke_runtime_isolation_contract.sh b/tests/test_smoke_runtime_isolation_contract.sh new file mode 100755 index 000000000..e91e1d38d --- /dev/null +++ b/tests/test_smoke_runtime_isolation_contract.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Runtime-isolation contract for the smoke harness (#1696, follow-up to #1691). +# +# scripts/smoke-test.sh is the process that actually starts the product during +# a smoke run, and it retires "the account daemon" seven times through +# `daemon stop`. Its wrappers sandbox HOME/XDG/TMPDIR and CBM_CACHE_DIR, but +# only CBM_RUNTIME_DIR moves the daemon rendezvous (docs/CONFIGURATION.md), so +# without a private runtime every one of those stops lands on the operator's +# live daemon. Drive the harness with an environment-probe fixture and require +# that no product process ever receives the caller's runtime or cache. + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +normalize_path() { + local path=${1%$'\r'} + if command -v cygpath >/dev/null 2>&1; then + cygpath -u "$path" 2>/dev/null && return 0 + fi + printf '%s\n' "${path//\\//}" +} + +ENV_PROBE="$WORKDIR/environment-probe" +cat > "$ENV_PROBE" <<'EOF' +#!/usr/bin/env bash +printf '%s\t%s\n' "${CBM_CACHE_DIR-}" "${CBM_RUNTIME_DIR-}" >> "$CBM_SMOKE_ENV_PROBE" +case "${1-}" in + --version) echo "v0.0.0-probe"; exit 0 ;; + daemon) [[ "${2-}" == status ]] && exit 1; exit 0 ;; +esac +echo '{}' +exit 0 +EOF +chmod +x "$ENV_PROBE" + +CALLER_CACHE="$WORKDIR/caller-cache" +CALLER_RUNTIME="$WORKDIR/caller-runtime" +ENV_LOG="$WORKDIR/environment.log" +mkdir -p "$CALLER_CACHE" "$CALLER_RUNTIME" + +# The fixture answers nothing beyond --version, so the smoke fails early; only +# the environment it handed to the product is under test here. +CBM_CACHE_DIR="$CALLER_CACHE" \ +CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ +CBM_SMOKE_ENV_PROBE="$ENV_LOG" \ + "$ROOT/scripts/smoke-test.sh" "$ENV_PROBE" > "$WORKDIR/smoke.out" 2>&1 || true + +[[ -s "$ENV_LOG" ]] || fail "smoke-test did not execute the environment-probe fixture" + +CALLER_CACHE_NORMALIZED=$(normalize_path "$CALLER_CACHE") +CALLER_RUNTIME_NORMALIZED=$(normalize_path "$CALLER_RUNTIME") +private_root="" +while IFS=$'\t' read -r child_cache_raw child_runtime_raw; do + child_cache=$(normalize_path "$child_cache_raw") + child_runtime=$(normalize_path "$child_runtime_raw") + if [[ -z "$child_runtime" || "$child_runtime" == "$CALLER_RUNTIME_NORMALIZED" ]]; then + fail "smoke-test exposed the caller CBM_RUNTIME_DIR to a product process" + fi + if [[ -z "$child_cache" || "$child_cache" == "$CALLER_CACHE_NORMALIZED" ]]; then + fail "smoke-test exposed the caller CBM_CACHE_DIR to a product process" + fi + if [[ "${child_runtime%/*}" != "${child_cache%/*}" || + "${child_runtime##*/}" != "runtime" || "${child_cache##*/}" != "cache" ]]; then + fail "smoke runtime/cache were not isolated beneath one private root" + fi + if [[ -n "$private_root" && "$private_root" != "${child_runtime%/*}" ]]; then + fail "smoke-test switched private roots mid-run" + fi + private_root="${child_runtime%/*}" +done < "$ENV_LOG" + +[[ ! -e "$private_root" ]] || fail "smoke-test left its private root behind: $private_root" + +echo "PASS: smoke harness isolates its daemon runtime and cache from the caller" From fa138acf44525ef94cfd6e16772d601da5623aef Mon Sep 17 00:00:00 2001 From: Anton Standrik Date: Mon, 21 Sep 2026 11:29:04 +0300 Subject: [PATCH 115/130] fix(test): arm smoke runtime cleanup at init, retire the daemon first Review on #2199: cbm_test_runtime_init ran more than a hundred lines before the only EXIT trap, so under `set -e` a failure in the fixture mktemp or its cygpath conversion left the private root behind, and the fixture trap ran smoke_rmtree ahead of the runtime cleanup. Arm `trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT` immediately after a successful init, as soak-test.sh and memlab.sh do. The fixture trap that replaces it now retires the private daemon first and removes the fixtures second, so no earlier cleanup step stands between the exit and the runtime cleanup; on Windows the retirement is also what unblocks the fixture rm. tests/test_smoke_runtime_isolation_contract.sh pins the early trap to the window between the init call and the first fixture; the pin fails against the previous head. Part of #1696. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Anton Standrik --- scripts/smoke-test.sh | 11 +++++++++- .../test_smoke_runtime_isolation_contract.sh | 21 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index e996da847..d1c64c6b7 100755 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -51,6 +51,11 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")/.." && pwd)" # shellcheck source=test-runtime.sh source "$REPO_ROOT/scripts/test-runtime.sh" cbm_test_runtime_init +# Armed here rather than only with the fixture trap below: the fixture mktemp +# and its cygpath conversion sit between the two, and under `set -e` a failure +# there would otherwise leave the private root behind. The fixture trap +# replaces this one and keeps the same cleanup as its first step. +trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT smoke_mktemp_file() { if [ -n "${SMOKE_TEMP_ROOT:-}" ]; then @@ -160,7 +165,11 @@ CODEX_LIFECYCLE_HOME="" if command -v cygpath &>/dev/null; then TMPDIR=$(cygpath -m "$TMPDIR") fi -trap 'smoke_rmtree "$TMPDIR" "${DRYRUN_HOME:-}" "${CODEX_LIFECYCLE_HOME:-}"; cbm_test_runtime_cleanup "$BINARY"' EXIT +# Runtime cleanup first, so no earlier cleanup step stands between the exit +# and the private daemon's retirement; on Windows that retirement is also what +# unblocks the fixture rm (mapped binary, open logs). smoke_rmtree never fails, +# so the fixture removal still runs after it. +trap 'cbm_test_runtime_cleanup "$BINARY"; smoke_rmtree "$TMPDIR" "${DRYRUN_HOME:-}" "${CODEX_LIFECYCLE_HOME:-}"' EXIT CLI_STDERR=$(smoke_mktemp_file) # 10 of the cli call sites assign directly (VAR=$(cli ...)). Under diff --git a/tests/test_smoke_runtime_isolation_contract.sh b/tests/test_smoke_runtime_isolation_contract.sh index e91e1d38d..31ba7a7b6 100755 --- a/tests/test_smoke_runtime_isolation_contract.sh +++ b/tests/test_smoke_runtime_isolation_contract.sh @@ -79,4 +79,25 @@ done < "$ENV_LOG" [[ ! -e "$private_root" ]] || fail "smoke-test left its private root behind: $private_root" +# The run above exits through the fixture trap, so it cannot show what happens +# to the private root when the harness dies before that trap exists — the +# fixture mktemp and its cygpath conversion are in that window, and under +# `set -e` either can end the run. Reproducing that failure would mean scanning +# the shared /tmp parent for orphaned roots, which races every other harness +# test in the same suite, so pin the ordering instead: the cleanup trap is +# armed between the init call and the first fixture work. +smoke="$ROOT/scripts/smoke-test.sh" +smoke_line_of() { + # A missing pattern is the failure this check reports, not a reason to end + # the test silently under `set -e`. + grep -n "$1" "$smoke" | head -1 | cut -d: -f1 || true +} +init_line=$(smoke_line_of '^cbm_test_runtime_init$') +early_trap_line=$(smoke_line_of "^trap 'cbm_test_runtime_cleanup \"\$BINARY\"' EXIT\$") +fixture_line=$(smoke_line_of '^TMPDIR=\$(smoke_mktemp_dir)$') +if [[ -z "$init_line" || -z "$early_trap_line" || -z "$fixture_line" ]] || + ((early_trap_line < init_line || early_trap_line > fixture_line)); then + fail "smoke-test must arm the runtime cleanup trap between cbm_test_runtime_init and its first fixture" +fi + echo "PASS: smoke harness isolates its daemon runtime and cache from the caller" From 1a3425536be95f0d2db7b932edd6d5f19143aa09 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 16 Sep 2026 20:02:31 +0200 Subject: [PATCH 116/130] =?UTF-8?q?fix(graph):=20quality=20round=20after?= =?UTF-8?q?=20the=20v0.11.0=20probe=20=E2=80=94=20deterministic=20resoluti?= =?UTF-8?q?on,=20findability,=20coverage,=20names?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nine findings from the 2026-09-16 graph-quality probe over the eleven bench corpora, each with a test that fails without its fix and a before/after measurement on the real corpus. Deterministic resolution (O9). Two indexes of one kernel tree differed in 5,821 IMPORTS and 632 CALLS edges. Three order/load dependencies carried it: the `#include` target was the first by-name hit whose path ends with the include path — a bucket in worker merge order (include/linux/device.h vs tools/virtio/linux/device.h); the registry broke suffix_match score ties by bucket position; the LSP budget-share rule measured wall time. Include targets now prefer the File node, the least nested path, then the lexicographic path; registry ties prefer the least nested QN, then the smaller QN; the budget share is measured in thread CPU time like the budget itself. Two kernel indexes are now identical (8,529,723 nodes, 1,974,940 CALLS, zero churn), as are two elasticsearch indexes. Findability. search_graph's BM25 path honours the `label` filter and ranks an exact-name definition above test methods that repeat the token (15/15 probe queries). Coverage. The parse-coverage gap tree subtracts definitions the walker recovered inside an error region (multi-line only); kernel/sched/core.c reports 476 flagged lines instead of 7,674. Precision. Config-language callers (Makefile, CMake, YAML) no longer bind unique_name into code (kernel Makefile -> C CALLS 195 -> 0). JS-family token names ({}, 1, (x: T) => U) are not definitions; computed, quoted and escaped member keys stay (TypeScript 204 junk -> 0, 1,782 real names kept). Names are cut at their first line (elasticsearch YAML 8,306 -> 0); a comment inside an argument list is not an argument, so no Route node is named by a Java block comment; the wildcard route `/*` is still a route. HTML/Vue/Svelte/Astro embedded member calls stay unbound. Recall. A Python call through a self- or cls-rooted attribute keeps a specific unique_name member target (Django +751, elasticsearch +3,288 CALLS, no new callee names). Robustness. Edge properties never carry a split UTF-8 sequence (the 120-byte expression cut and the JSON escaper both trim a partial code point). CLI array flags accept a JSON-array literal. Cypher seeds a single-hop pattern from its selective end (kernel `MATCH (a)-[:CALLS]->(b) WHERE b.name = '_printk'`: execution limit -> 7.9 s). Signed-off-by: Martin Vogel --- internal/cbm/cbm.c | 220 ++++++++++++++++++++++++++---- internal/cbm/cbm.h | 45 ++++--- internal/cbm/extract_calls.c | 31 +++++ internal/cbm/service_patterns.c | 22 +++ internal/cbm/service_patterns.h | 6 + src/cli/cli.c | 26 ++++ src/cypher/cypher.c | 96 +++++++++++++ src/foundation/str_util.c | 36 ++++- src/foundation/str_util.h | 9 +- src/mcp/mcp.c | 40 +++++- src/pipeline/pass_calls.c | 18 ++- src/pipeline/pass_parallel.c | 27 +++- src/pipeline/pass_pkgmap.c | 53 ++++++-- src/pipeline/pipeline.h | 14 ++ src/pipeline/registry.c | 163 ++++++++++++++++++++-- tests/test_cli.c | 75 +++++++---- tests/test_cypher.c | 58 ++++++++ tests/test_extraction.c | 134 ++++++++++++++++++ tests/test_infrascan.c | 13 ++ tests/test_mcp.c | 138 +++++++++++++++++-- tests/test_parse_coverage.c | 154 +++++++++++++++++---- tests/test_pipeline.c | 231 ++++++++++++++++++++++++++++++++ tests/test_registry.c | 141 +++++++++++++++++++ tests/test_str_util.c | 25 ++++ 24 files changed, 1635 insertions(+), 140 deletions(-) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 2f475229a..8b319adda 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -125,7 +125,79 @@ void cbm_reset_profile(void) { } \ } while (0) +/* A definition name never spans lines. Config extractors hand over a node's + * whole text — a YAML block key, a Makefile recipe (`endif\n\n$(obj)/x.h`), a + * ktest .conf directive — and the line break rode into the graph (2026-09-16 + * probe: 8,301 Field nodes on elastic/elasticsearch, 91 kernel nodes). Cut at + * the first line break and drop trailing blanks; the qualified name carries + * the same text as its last segment and is cut the same way. */ +static const char *cbm_first_line(CBMArena *a, const char *text) { + if (!text) { + return text; + } + const char *nl = strpbrk(text, "\r\n"); + if (!nl) { + return text; + } + size_t n = (size_t)(nl - text); + while (n > 0 && (text[n - 1] == ' ' || text[n - 1] == '\t')) { + n--; + } + char *cut = (char *)cbm_arena_alloc(a, n + 1); + if (!cut) { + return text; + } + memcpy(cut, text, n); + cut[n] = '\0'; + return cut; +} + +/* JS/TS files: a definition named by a literal token is the walker naming a + * function or variable from an object key or a numeric pattern element — + * `{}` (1,534 incoming CALLS on microsoft/TypeScript's test baselines), `1` + * (38 nodes), a quoted string. An identifier starts with a letter, `_`, `$`, + * `#` (private members) or a non-ASCII byte; anything else is not a name. */ +static bool cbm_js_family_path(const char *path) { + if (!path) { + return false; + } + const char *dot = strrchr(path, '.'); + if (!dot) { + return false; + } + static const char *const exts[] = {".js", ".mjs", ".cjs", ".jsx", ".ts", + ".mts", ".cts", ".tsx", ".ets", NULL}; + for (int i = 0; exts[i]; i++) { + if (strcmp(dot, exts[i]) == 0) { + return true; + } + } + return false; +} + +static bool cbm_js_name_is_junk(const char *name) { + if (!name || !name[0]) { + return false; /* empty names are handled by the callers' own rules */ + } + unsigned char c = (unsigned char)name[0]; + bool identifier_start = (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' || + c == '$' || c == '#' || c >= 0x80; + /* Member names JS spells without an identifier start are still names: + * computed keys (`[Symbol.iterator]`), string-literal keys (`"my-key"`, + * `'x'`) and escaped identifiers (`А`). Measured on the TypeScript + * corpus: 1,531 `[…]` members, 97 quoted members, 154 escaped + * identifiers — all real definitions. What remains ({…} patterns, numeric + * literals, parenthesised types, `...rest`) is a token, not a name. */ + bool member_key_start = c == '[' || c == '"' || c == '\'' || c == '\\'; + return !identifier_start && !member_key_start; +} + void cbm_defs_push(CBMDefArray *arr, CBMArena *a, CBMDefinition def) { + def.name = cbm_first_line(a, def.name); + def.qualified_name = cbm_first_line(a, def.qualified_name); + if (cbm_js_family_path(def.file_path) && cbm_js_name_is_junk(def.name)) { + return; + } GROW_ARRAY(arr, a); arr->items[arr->count++] = def; } @@ -1358,14 +1430,88 @@ static void cbm_mark_pp_error_rows(TSNode n, uint8_t *rows, uint32_t row_count, /* Recovery subtraction (#963): tree-sitter error recovery plus the * ERROR-descending def walker often still extract constructs INSIDE a failed * region (verified: a function in an #ifdef-split ERROR region and even a - * `def broken(:` both came back as defs). A region whose every line is - * covered by definitions that START inside it is definitely recovered — its - * constructs ARE in the graph — so flagging it would be a false miss. - * Container defs (Module/Package) are ignored: a file-spanning Module node is - * not evidence the region's constructs survived. Conservative: partially - * covered regions stay flagged. */ -static bool cbm_region_is_recovered(uint32_t rs, uint32_t re, const CBMDefArray *defs) { - enum { MAX_COVER_DEFS = 256 }; + * `def broken(:` both came back as defs). The lines a definition that STARTS + * inside the region covers are therefore in the graph, and only the lines no + * definition covers can honestly be called missed. Container defs + * (Module/Package) are ignored: a file-spanning Module node is not evidence + * the region's constructs survived. + * + * This used to be all-or-nothing — a region stayed flagged whole unless every + * line was covered. On torvalds/linux (2026-09-16 probe) one ERROR node that + * swallowed the second half of kernel/sched/core.c (lines 5522–11284) survived + * because of the comment and macro lines between its 286 extracted functions, + * and the coverage report told a reader 68 % of the scheduler was unindexed. + * Now the uncovered gaps are reported instead, and a gap holding only blank, + * comment or preprocessor lines is not a miss at all. */ +static uint32_t *cbm_line_offsets(const char *src, int src_len, uint32_t *out_lines) { + uint32_t lines = 1; + for (int i = 0; i < src_len; i++) { + if (src[i] == '\n') { + lines++; + } + } + uint32_t *offs = (uint32_t *)malloc((size_t)(lines + 1) * sizeof(uint32_t)); + if (!offs) { + *out_lines = 0; + return NULL; + } + uint32_t n = 0; + offs[n++] = 0; + for (int i = 0; i < src_len && n < lines; i++) { + if (src[i] == '\n') { + offs[n++] = (uint32_t)i + 1; + } + } + offs[n] = (uint32_t)src_len; + *out_lines = n; + return offs; +} + +/* A 1-based line that holds nothing a reader would call missed code: blank, + * a comment line, or a preprocessor directive (the import pass owns #include; + * #if/#endif carry no construct of their own). */ +static bool cbm_line_is_inert(const char *src, const uint32_t *offs, uint32_t nlines, + uint32_t line) { + if (!src || !offs || line == 0 || line > nlines) { + return true; + } + const char *p = src + offs[line - 1]; + const char *end = src + offs[line]; + while (p < end && (*p == ' ' || *p == '\t' || *p == '\r')) { + p++; + } + if (p >= end || *p == '\n') { + return true; + } + if (p[0] == '/' && p + 1 < end && (p[1] == '/' || p[1] == '*')) { + return true; + } + return p[0] == '*' || p[0] == '#'; +} + +static void cbm_regions_emit_gap(cbm_error_regions_t *out, uint32_t gs, uint32_t ge, + const char *src, const uint32_t *offs, uint32_t nlines) { + bool live = false; + for (uint32_t l = gs; l <= ge && !live; l++) { + live = !cbm_line_is_inert(src, offs, nlines, l); + } + if (!live) { + return; + } + if (out->count < CBM_MAX_ERROR_REGIONS) { + out->starts[out->count] = gs; + out->ends[out->count] = ge; + out->count++; + } else { + out->dropped++; + } +} + +/* Replace [rs, re] by the sub-ranges no in-region definition covers. */ +static void cbm_region_uncovered_gaps(uint32_t rs, uint32_t re, const CBMDefArray *defs, + const char *src, const uint32_t *offs, uint32_t nlines, + cbm_error_regions_t *out) { + enum { MAX_COVER_DEFS = 1024 }; uint32_t starts[MAX_COVER_DEFS]; uint32_t ends[MAX_COVER_DEFS]; int n = 0; @@ -1377,14 +1523,25 @@ static bool cbm_region_is_recovered(uint32_t rs, uint32_t re, const CBMDefArray if (d->start_line < rs || d->start_line > re) { continue; /* recovery evidence must originate inside the region */ } + /* A one-line definition salvaged from inside an error region is the + * walker's guess at broken text (`def x(:` comes back as a def, a + * mis-parsed `static _Thread_local int x` as a Variable named `int`, + * and Python's synthetic builtins all sit on line 1); only a construct + * whose body spans lines proves it was understood. Conservative: real + * one-line variables and macros inside a failed region keep their line + * flagged. */ + if (d->end_line <= d->start_line) { + continue; + } starts[n] = d->start_line; - ends[n] = d->end_line < d->start_line ? d->start_line : d->end_line; + ends[n] = d->end_line; n++; } if (n == 0) { - return false; + cbm_regions_emit_gap(out, rs, re, src, offs, nlines); + return; } - /* Insertion-sort by start, then sweep for gaps in [rs, re]. */ + /* Insertion-sort by start, then sweep and emit the gaps in [rs, re]. */ for (int i = 1; i < n; i++) { uint32_t s = starts[i]; uint32_t e = ends[i]; @@ -1397,16 +1554,18 @@ static bool cbm_region_is_recovered(uint32_t rs, uint32_t re, const CBMDefArray starts[j + 1] = s; ends[j + 1] = e; } - uint32_t covered_to = rs - 1; + uint32_t cursor = rs; for (int i = 0; i < n; i++) { - if (starts[i] > covered_to + 1) { - return false; /* uncovered gap */ + if (starts[i] > cursor) { + cbm_regions_emit_gap(out, cursor, starts[i] - 1, src, offs, nlines); } - if (ends[i] > covered_to) { - covered_to = ends[i]; + if (ends[i] + 1 > cursor) { + cursor = ends[i] + 1; } } - return covered_to >= re; + if (cursor <= re) { + cbm_regions_emit_gap(out, cursor, re, src, offs, nlines); + } } /* #961: true when 1-based `line` of `src` contains `name` (used to verify a @@ -1598,16 +1757,19 @@ static bool cbm_remap_preprocessed_def(CBMDefinition *def, const CBMPreprocessed return true; } -static void cbm_subtract_recovered_regions(cbm_error_regions_t *regs, const CBMDefArray *defs) { - int kept = 0; - for (int i = 0; i < regs->count; i++) { - if (!cbm_region_is_recovered(regs->starts[i], regs->ends[i], defs)) { - regs->starts[kept] = regs->starts[i]; - regs->ends[kept] = regs->ends[i]; - kept++; - } +static void cbm_subtract_recovered_regions(cbm_error_regions_t *regs, const CBMDefArray *defs, + const char *src, int src_len) { + if (regs->count <= 0) { + return; } - regs->count = kept; + uint32_t nlines = 0; + uint32_t *offs = cbm_line_offsets(src, src_len, &nlines); + cbm_error_regions_t out = {{0}, {0}, 0, regs->dropped}; + for (int i = 0; i < regs->count; i++) { + cbm_region_uncovered_gaps(regs->starts[i], regs->ends[i], defs, src, offs, nlines, &out); + } + free(offs); + *regs = out; } /* #1071: a function-like macro invocation whose argument is a type token @@ -2019,6 +2181,10 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C TSTree *tree = ts_parser_parse_with_options(parser, NULL, ts_input, opts); uint64_t t1 = now_ns(); + uint64_t parse_cpu_ns = cbm_thread_cpu_time_ns() - cpu_start_ns; +#ifdef CBM_ENABLE_TEST_SEAMS + t1 += tl_parse_wall_seam_offset_ns; /* the stall seam inflates every wall reading */ +#endif if (!tree) { result->has_error = true; @@ -2669,7 +2835,7 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C * splitting a brace inside a recovered function looks unrecovered: * the refinement keeps only the discarded branch, the function starts * above it, and the evidence falls outside the range. */ - cbm_subtract_recovered_regions(®s, &result->defs); + cbm_subtract_recovered_regions(®s, &result->defs, source, source_len); /* Phase 2: cut what is left down to the lines the preprocessed parse * could not explain. */ if (pp_line_map) { diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index a7907289a..a00192a18 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -269,26 +269,31 @@ typedef struct { * calls at 251 MB with most of that empty slots. Readers index it exactly * as before; only `sizeof` changed. */ CBMCallArg *args; - int arg_count; // number of captured arguments (<= CBM_MAX_CALL_ARGS) - int loop_depth; // enclosing loop nesting at the call site - int branch_depth; // enclosing branch nesting at the call site - int start_line; // 1-based source line of the call (for def range-match) - uint32_t site_start_byte; // exact AST occurrence span; end > start when present - uint32_t site_end_byte; // exclusive byte offset in the source file - CBMSourceOrigin source_origin; // raw source or C-family preprocessed buffer - bool is_method; // method/member call with an UNRESOLVED receiver. Perl: - // arrow/method call ($obj->m). TS/JS/TSX: member call - // x.foo() whose receiver is not this/super. Python: - // x.foo() where x is not self/cls/super() and is not - // rooted in an imported name. Read by the weak-member - // guard and by the pxc synthetic-carrier dedup key in - // pass_lsp_cross.c. Default false. - bool requires_lsp_resolution; // synthetic semantic candidate (for example an implicit - // C++ operator). Never fall back to textual resolution. - bool callee_is_locally_bound; // bare call foo() whose callee identifier is bound as a - // parameter of an enclosing function, so it cannot be the - // module-level foo. Python only today. Read by the - // weak-local-binding guard. Default false. + int arg_count; // number of captured arguments (<= CBM_MAX_CALL_ARGS) + int loop_depth; // enclosing loop nesting at the call site + int branch_depth; // enclosing branch nesting at the call site + int start_line; // 1-based source line of the call (for def range-match) + uint32_t site_start_byte; // exact AST occurrence span; end > start when present + uint32_t site_end_byte; // exclusive byte offset in the source file + CBMSourceOrigin source_origin; // raw source or C-family preprocessed buffer + bool is_method; // method/member call with an UNRESOLVED receiver. Perl: + // arrow/method call ($obj->m). TS/JS/TSX: member call + // x.foo() whose receiver is not this/super. Python: + // x.foo() where x is not self/cls/super() and is not + // rooted in an imported name. Read by the weak-member + // guard and by the pxc synthetic-carrier dedup key in + // pass_lsp_cross.c. Default false. + bool requires_lsp_resolution; // synthetic semantic candidate (for example an implicit + // C++ operator). Never fall back to textual resolution. + bool callee_is_locally_bound; // bare call foo() whose callee identifier is bound as a + // parameter of an enclosing function, so it cannot be the + // module-level foo. Python only today. Read by the + // weak-local-binding guard. Default false. + bool receiver_is_self_attribute; // Python member call whose receiver is an attribute + // chain rooted at self/cls but not self/cls itself + // (self.compiler.apply_converters()). An object the + // class owns, not a parameter: read by the weak-member + // guard's unique-name exemption. Default false. } CBMCall; typedef struct { diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index 728059c71..83820fdc6 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -2068,6 +2068,14 @@ static void extract_call_args(CBMExtractCtx *ctx, TSNode args, CBMCall *call) { for (uint32_t ai = 0; ai < argc && call->arg_count < CBM_MAX_CALL_ARGS; ai++) { TSNode arg_node = ts_node_named_child(args, ai); const char *ak = ts_node_type(arg_node); + /* tree-sitter lists a comment inside the argument list as a named + * child (Java block_comment/line_comment, C-family comment). It is + * not an argument: taken as one, a leading block comment became + * args[0] and the Route pass read its text as a URL. */ + if (strcmp(ak, "comment") == 0 || strcmp(ak, "line_comment") == 0 || + strcmp(ak, "block_comment") == 0) { + continue; + } if (!call->args) { call->args = cbm_arena_calloc(ctx->arena, CBM_MAX_CALL_ARGS * sizeof(CBMCallArg)); if (!call->args) { @@ -2965,6 +2973,28 @@ static char *resolve_objectscript_instance_call(CBMArena *a, TSNode node, const * statically-known type here, so the call must not bind by short name alone. * Note `self.client.send()` is NOT exempt: the receiver is `self.client`, an * attribute of unknown type, not `self` itself. */ +/* True when a Python attribute-call receiver is an attribute chain ROOTED at + * self/cls but is not self/cls itself: `self.compiler.apply_converters()` has + * receiver `self.compiler`, an object the class owns. The weak-member guard's + * unique-name exemption keys on this shape (see cbm_weak_member_unique_name_ + * exempt): a bare parameter (`accelerator.backward()`) carries no ownership + * evidence and stays suppressed. Direct `self.m()` is already exempt via + * python_receiver_is_exempt and is deliberately NOT flagged here. */ +static bool python_receiver_rooted_at_self(CBMExtractCtx *ctx, TSNode receiver) { + if (ts_node_is_null(receiver) || strcmp(ts_node_type(receiver), "attribute") != 0) { + return false; + } + TSNode root = receiver; + while (!ts_node_is_null(root) && strcmp(ts_node_type(root), "attribute") == 0) { + root = ts_node_child_by_field_name(root, TS_FIELD("object")); + } + if (ts_node_is_null(root) || strcmp(ts_node_type(root), "identifier") != 0) { + return false; + } + char *name = cbm_node_text(ctx->arena, root, ctx->source); + return name && (strcmp(name, "self") == 0 || strcmp(name, "cls") == 0); +} + static bool python_receiver_is_exempt(CBMExtractCtx *ctx, TSNode receiver) { if (ts_node_is_null(receiver)) { return false; @@ -3695,6 +3725,7 @@ CBMInvocationDescriptor handle_calls(CBMExtractCtx *ctx, TSNode node, const CBML if (!ts_node_is_null(fn) && strcmp(ts_node_type(fn), "attribute") == 0) { TSNode obj = ts_node_child_by_field_name(fn, TS_FIELD("object")); call.is_method = !python_receiver_is_exempt(ctx, obj); + call.receiver_is_self_attribute = python_receiver_rooted_at_self(ctx, obj); } else if (!ts_node_is_null(fn) && strcmp(ts_node_type(fn), "identifier") == 0) { call.callee_is_locally_bound = python_callee_is_bound_parameter(ctx, state, fn); } diff --git a/internal/cbm/service_patterns.c b/internal/cbm/service_patterns.c index e3740f5aa..0dd404877 100644 --- a/internal/cbm/service_patterns.c +++ b/internal/cbm/service_patterns.c @@ -714,6 +714,25 @@ static const char *strip_string_delimiters(const char *literal, char *buf, size_ return buf; } +/* A comment opens with the slash a route opens with, and an argument list + * that starts with one handed three Java block comments to the Route pass as + * URLs (elasticsearch, 2026-09-16). The wildcard route (slash-star alone) is a + * path, so the block-comment shape needs its closing star-slash; a route + * literal never holds a line break. */ +bool cbm_service_pattern_is_comment_text(const char *text) { + if (!text || text[0] != '/') { + return false; + } + if (strchr(text, '\n') != NULL || strchr(text, '\r') != NULL) { + return true; + } + size_t n = strlen(text); + if (text[1] == '*' && n >= 4 && text[n - 2] == '*' && text[n - 1] == '/') { + return true; + } + return text[1] == '/' && (text[2] == ' ' || text[2] == '\t'); +} + bool cbm_service_pattern_is_http_route_literal(const char *literal, const char *callee_name) { char path_buf[1024]; const char *path = strip_string_delimiters(literal, path_buf, sizeof(path_buf)); @@ -729,6 +748,9 @@ bool cbm_service_pattern_is_http_route_literal(const char *literal, const char * if (path[0] != '/') { return false; } + if (cbm_service_pattern_is_comment_text(path)) { + return false; + } if (callee_is_delimiter_or_filesystem_builder(callee_name)) { return false; } diff --git a/internal/cbm/service_patterns.h b/internal/cbm/service_patterns.h index 4642a08de..d9a93c251 100644 --- a/internal/cbm/service_patterns.h +++ b/internal/cbm/service_patterns.h @@ -48,6 +48,12 @@ bool cbm_service_pattern_is_global_fetch(const char *callee_name); * Rejects filesystem paths and non-HTTP string consumers. */ bool cbm_service_pattern_is_http_route_literal(const char *literal, const char *callee_name); +/* True when text that reached a path slot is a source comment rather than a + * path: a block comment (slash-star … star-slash), a line comment (slash-slash + * followed by a space), or any text with a line break. The wildcard route + * (slash-star alone) is a path. */ +bool cbm_service_pattern_is_comment_text(const char *text); + /* Per-worker TLS cache for cbm_service_pattern_match results. The * pattern matcher runs once per resolved CALL edge in emit_service_ * edge — that's 6 pattern lists × ~30 patterns × strstr per call ≈ diff --git a/src/cli/cli.c b/src/cli/cli.c index 8645fbe56..b59a894a6 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -13752,6 +13752,32 @@ static void cli_add_typed(yyjson_mut_doc *out, yyjson_mut_val *obj, const char * arr = yyjson_mut_arr(out); yyjson_mut_obj_add(obj, yyjson_mut_strcpy(out, key), arr); } + /* The help prints `--paths `, and a caller who writes the + * array literally — `--paths '["lib","t"]'` — used to get ONE element + * holding that literal text (2026-09-16 probe: check_index_coverage + * reported the fake path `["lib","t"]`). A value that parses as a JSON + * array contributes its elements; anything else is one element. */ + if (have_value && value && value[0] == '[') { + yyjson_doc *lit = yyjson_read(value, strlen(value), 0); + yyjson_val *lit_root = lit ? yyjson_doc_get_root(lit) : NULL; + if (lit_root && yyjson_is_arr(lit_root)) { + size_t idx; + size_t max; + yyjson_val *elem; + yyjson_arr_foreach(lit_root, idx, max, elem) { + if (yyjson_is_str(elem)) { + yyjson_mut_arr_add_strcpy(out, arr, yyjson_get_str(elem)); + } else { + yyjson_mut_arr_add_val(arr, yyjson_val_mut_copy(out, elem)); + } + } + yyjson_doc_free(lit); + return; + } + if (lit) { + yyjson_doc_free(lit); + } + } yyjson_mut_arr_add_strcpy(out, arr, have_value ? value : ""); return; } diff --git a/src/cypher/cypher.c b/src/cypher/cypher.c index 73592a4b8..dd2544fd0 100644 --- a/src/cypher/cypher.c +++ b/src/cypher/cypher.c @@ -5014,8 +5014,104 @@ static void execute_return_clause(cbm_query_t *q, cbm_return_clause_t *ret, bind limit_is_engine_budget); } +/* ── Planner: seed a single-hop pattern from its selective end ────────────── + * + * The executor scans pattern node 0 and expands from there. Written as + * `MATCH (a)-[:CALLS]->(b) WHERE b.name = '_printk'`, that scans every node + * in the store before the filter on `b` can discard any — on an 8.5 M node + * kernel graph the query hit the execution-time limit while the anchored + * spelling `MATCH (b:Function {name:'_printk'})<-[:CALLS]-(a)` answered in + * seconds (2026-09-16 probe). A reader should not have to know which end to + * write first: when the far node carries an equality on name/qualified_name, + * an inline property, or a label and the near node carries less, the pattern + * is walked from the far end with the relationship direction inverted. The + * result set is identical; only the enumeration order changes. Restricted to + * one single-relationship pattern with an explicit RETURN, and never to an + * OPTIONAL MATCH, whose anchor is part of its meaning. */ +static int cypher_cond_selectivity(const cbm_condition_t *c, const char *var) { + if (!c || !c->variable || !var || strcmp(c->variable, var) != 0 || c->negated) { + return 0; + } + if (!c->op || strcmp(c->op, "=") != 0 || !c->property) { + return 0; + } + return (strcmp(c->property, "name") == 0 || strcmp(c->property, "qualified_name") == 0) ? 3 : 2; +} + +static int cypher_expr_selectivity(const cbm_expr_t *e, const char *var) { + if (!e) { + return 0; + } + if (e->type == EXPR_CONDITION) { + return cypher_cond_selectivity(&e->cond, var); + } + if (e->type == EXPR_AND) { + int l = cypher_expr_selectivity(e->left, var); + int r = cypher_expr_selectivity(e->right, var); + return l > r ? l : r; + } + return 0; /* OR / NOT / XOR: no single equality to seed from */ +} + +static int cypher_node_selectivity(const cbm_node_pattern_t *n, const cbm_where_clause_t *w) { + int s = 0; + for (int i = 0; i < n->prop_count; i++) { + const char *k = n->props[i].key; + int v = (k && (strcmp(k, "name") == 0 || strcmp(k, "qualified_name") == 0)) ? 3 : 2; + if (v > s) { + s = v; + } + } + if (w && n->variable) { + int v = 0; + if (w->root) { + v = cypher_expr_selectivity(w->root, n->variable); + } else if (!w->op || strcmp(w->op, "AND") == 0) { + for (int i = 0; i < w->count; i++) { + int c = cypher_cond_selectivity(&w->conditions[i], n->variable); + if (c > v) { + v = c; + } + } + } + if (v > s) { + s = v; + } + } + if (s == 0 && n->label) { + s = 1; + } + return s; +} + +static void cypher_plan_seed_from_selective_end(cbm_query_t *q) { + if (!q || q->pattern_count != 1 || !q->ret || (q->pattern_optional && q->pattern_optional[0])) { + return; + } + cbm_pattern_t *p = &q->patterns[0]; + if (p->rel_count != 1 || p->node_count != 2) { + return; + } + if (cypher_node_selectivity(&p->nodes[1], q->where) <= + cypher_node_selectivity(&p->nodes[0], q->where)) { + return; + } + cbm_node_pattern_t tmp = p->nodes[0]; + p->nodes[0] = p->nodes[1]; + p->nodes[1] = tmp; + cbm_rel_pattern_t *r = &p->rels[0]; + if (r->direction && strcmp(r->direction, "outbound") == 0) { + free((void *)r->direction); + r->direction = heap_strdup("inbound"); + } else if (r->direction && strcmp(r->direction, "inbound") == 0) { + free((void *)r->direction); + r->direction = heap_strdup("outbound"); + } +} + static int execute_single(cbm_store_t *store, cbm_query_t *q, const char *project, int max_rows, result_builder_t *rb) { + cypher_plan_seed_from_selective_end(q); cbm_pattern_t *pat0 = &q->patterns[0]; /* Step 1: Scan initial nodes */ diff --git a/src/foundation/str_util.c b/src/foundation/str_util.c index c8aec01af..9f65c079a 100644 --- a/src/foundation/str_util.c +++ b/src/foundation/str_util.c @@ -310,6 +310,33 @@ bool cbm_validate_project_name(const char *name) { return true; } +int cbm_utf8_trim_partial(char *buf) { + if (!buf) { + return 0; + } + int pos = (int)strlen(buf); + if (pos == 0) { + return 0; + } + /* Walk back over continuation bytes to the lead byte; drop the sequence + * when fewer continuation bytes follow it than its lead byte announces. */ + int back = pos - 1; + int cont = 0; + while (back >= 0 && ((unsigned char)buf[back] & 0xC0) == 0x80) { + back--; + cont++; + } + if (back >= 0) { + unsigned char lead = (unsigned char)buf[back]; + int need = lead >= 0xF0 ? 3 : lead >= 0xE0 ? 2 : lead >= 0xC0 ? 1 : 0; + if (need > cont) { + pos = back; + buf[pos] = '\0'; + } + } + return pos; +} + int cbm_json_escape(char *buf, int bufsize, const char *src) { if (!buf || bufsize <= 0) { return 0; @@ -319,7 +346,8 @@ int cbm_json_escape(char *buf, int bufsize, const char *src) { return 0; } int pos = 0; - for (int i = 0; src[i] && pos < bufsize - JSON_NUL_RESERVE; i++) { + int i = 0; + for (; src[i] && pos < bufsize - JSON_NUL_RESERVE; i++) { unsigned char c = (unsigned char)src[i]; if (c == '"' || c == '\\') { if (pos + JSON_ESC_LEN > bufsize - JSON_NUL_RESERVE) { @@ -356,5 +384,11 @@ int cbm_json_escape(char *buf, int bufsize, const char *src) { } } buf[pos] = '\0'; + /* A truncated output must never end inside a multibyte UTF-8 sequence: a + * callee cut mid-character persisted bytes SQLite cannot decode as text + * (2026-09-16 probe: nine CALLS rows across rust, java and typescript). */ + if (src[i] != '\0') { + pos = cbm_utf8_trim_partial(buf); + } return pos; } diff --git a/src/foundation/str_util.h b/src/foundation/str_util.h index e0ff46057..afaeb6ba0 100644 --- a/src/foundation/str_util.h +++ b/src/foundation/str_util.h @@ -91,9 +91,16 @@ bool cbm_validate_project_name(const char *name); } \ } while (0) +/* Drop a trailing INCOMPLETE UTF-8 sequence from a NUL-terminated buffer in + * place (a lead byte whose continuation bytes were cut off). Returns the new + * length. Complete sequences and ASCII are left untouched. Use after any + * byte-count truncation of text that will be stored or serialised. */ +int cbm_utf8_trim_partial(char *buf); + /* Escape a string for safe embedding in JSON: escapes " \ and control chars. * Writes into buf (including NUL). Returns number of chars written (excl NUL). - * If buf is too small, output is truncated but always NUL-terminated. */ + * If buf is too small, output is truncated but always NUL-terminated, and it + * never ends inside a multibyte UTF-8 sequence. */ int cbm_json_escape(char *buf, int bufsize, const char *src); #endif /* CBM_STR_UTIL_H */ diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 7bfca8d7a..ca88bdb52 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -3762,6 +3762,8 @@ enum { BM25_BIND_OFFSET = 4, BM25_BIND_INNER = 5, BM25_BIND_FILE = 6, + BM25_BIND_LABEL = 7, + BM25_BIND_EXACT = 8, BM25_SQL_AUTO_LEN = -1, /* Inner FTS5 candidate cap. SQLite can early-terminate a plain FTS5 query * (no JOIN/WHERE on outer table) of the form: @@ -4019,8 +4021,8 @@ static char *bm25_render(const bm25_output_row_t *rows, int returned, int total, * Returns NULL if FTS5 is unavailable or the query produced no usable tokens, * in which case the caller falls back to the regex-based search path. */ static char *bm25_search(cbm_store_t *store, const char *project, const char *query, - const char *file_pattern, int limit, int offset, bool tree_format, - size_t max_output_bytes) { + const char *file_pattern, const char *label, int limit, int offset, + bool tree_format, size_t max_output_bytes) { sqlite3 *db = cbm_store_get_db(store); if (!db) { return NULL; @@ -4046,9 +4048,18 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu * because no outer predicate blocks it. We fetch BM25_INNER_LIMIT top candidates * from the FTS5 index, then join/filter/boost only those rows. bm25() returns a * NEGATIVE score (lower = more relevant). */ + /* Exact-name tier (2026-09-16 probe): a definition whose NAME is the query + * outranks every partial hit. BM25 term frequency otherwise rewards a long + * test-method name that repeats the token — `table references table with + * same name` — over the `Table` class itself, and the label tiers below + * then push the class's own methods above it (Method 10 > Class 5). The + * definition the reader asked for by name comes first; case-insensitive + * exact spelling comes next; everything else keeps its BM25 order. */ const char *sql = "SELECT n.id, n.label, n.name, n.qualified_name, n.file_path, n.start_line, n.end_line, " " (fts.base_rank " + " - CASE WHEN n.name = ?8 THEN 30.0 " + " WHEN lower(n.name) = lower(?8) THEN 20.0 ELSE 0.0 END " " - CASE WHEN n.label IN ('Function','Method') THEN 10.0 " " WHEN n.label = 'Route' THEN 8.0 " " WHEN n.label IN (" CBM_SQL_TYPE_LIKE_LABELS ") THEN 5.0 " @@ -4070,6 +4081,10 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu * must be changed together or results desynchronise from counts. */ " AND n.label NOT IN ('File','Folder','Variable','Project') " " AND (?6 IS NULL OR n.file_path LIKE ?6) " + /* The caller's label filter applies in query mode exactly as it does + * in the structural mode (2026-09-16 probe: `label=Class` was ignored + * here and five Methods came back). MIRRORED in the count query. */ + " AND (?7 IS NULL OR n.label = ?7) " /* rank ties are common (boosted floats) — the id tie-break makes * offset pages contractually stable across calls. */ "ORDER BY rank, n.id " @@ -4090,6 +4105,12 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu } else { sqlite3_bind_null(stmt, BM25_BIND_FILE); } + if (label && label[0]) { + sqlite3_bind_text(stmt, BM25_BIND_LABEL, label, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT); + } else { + sqlite3_bind_null(stmt, BM25_BIND_LABEL); + } + sqlite3_bind_text(stmt, BM25_BIND_EXACT, query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT); /* Count hits within the same inner-limit window — capped at BM25_INNER_LIMIT. * Uses the identical subquery structure so the FTS5 early-exit applies here too. */ @@ -4107,6 +4128,7 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu * not describe the rows returned. */ " AND n.label NOT IN ('File','Folder','Variable','Project')" " AND (?6 IS NULL OR n.file_path LIKE ?6)" + " AND (?7 IS NULL OR n.label = ?7)" ")"; sqlite3_stmt *cs = NULL; if (sqlite3_prepare_v2(db, count_sql, BM25_SQL_AUTO_LEN, &cs, NULL) == SQLITE_OK) { @@ -4121,6 +4143,12 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu } else { sqlite3_bind_null(cs, BM25_BIND_FILE); } + if (label && label[0]) { + sqlite3_bind_text(cs, BM25_BIND_LABEL, label, BM25_SQL_AUTO_LEN, + MCP_SQLITE_TRANSIENT); + } else { + sqlite3_bind_null(cs, BM25_BIND_LABEL); + } if (sqlite3_step(cs) == SQLITE_ROW) { total = sqlite3_column_int(cs, 0); } @@ -4153,10 +4181,10 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu int row_count = 0; while (rows && row_count < row_cap && sqlite3_step(stmt) == SQLITE_ROW) { const char *qn = (const char *)sqlite3_column_text(stmt, BM25_COL_QN); - const char *label = (const char *)sqlite3_column_text(stmt, BM25_COL_LABEL); + const char *row_label = (const char *)sqlite3_column_text(stmt, BM25_COL_LABEL); const char *file = (const char *)sqlite3_column_text(stmt, BM25_COL_FILE); rows[row_count].qualified_name = heap_strdup(qn ? qn : ""); - rows[row_count].label = heap_strdup(label ? label : ""); + rows[row_count].label = heap_strdup(row_label ? row_label : ""); rows[row_count].file_path = heap_strdup(file ? file : ""); rows[row_count].start_line = sqlite3_column_int(stmt, BM25_COL_START); rows[row_count].end_line = sqlite3_column_int(stmt, BM25_COL_END); @@ -5089,9 +5117,11 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { } if (query && query[0]) { char *q_file_pattern = cbm_mcp_get_string_arg(args, "file_pattern"); - char *bm25_json = bm25_search(store, project, query, q_file_pattern, limit, offset, + char *q_label = cbm_mcp_get_string_arg(args, "label"); + char *bm25_json = bm25_search(store, project, query, q_file_pattern, q_label, limit, offset, !json_format, max_output_bytes); free(q_file_pattern); + free(q_label); if (bm25_json) { free(query); free(project); diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index c9744dda2..84958954c 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -626,15 +626,29 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, * reintroduce the #592/#606 false-edge class for .ets files. */ bool suppress_weak_member = lang == CBM_LANG_PYTHON || lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX || - lang == CBM_LANG_ARKTS; + lang == CBM_LANG_ARKTS || + /* Files whose calls are embedded JS/TS (\n"); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/html_member.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + const char *project = cbm_pipeline_project_name(p); + + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + + /* NEGATIVE: localStorage's own method must not bind the docs bundle. */ + ASSERT_FALSE(cross_file_call_exists(s, project, "index", "getItem")); + /* POSITIVE: the bare project call in storage.js resolves as before. */ + ASSERT_TRUE(cross_file_call_exists(s, project, "loadTheme", "getItem")); + + cbm_store_close(s); + cbm_pipeline_free(p); + th_rmtree(tmp); + PASS(); +} + /* Fixture for the #1928 cross-language reference-guard probes (sequential and * parallel twins). pad_files > 0 adds filler files to push the index over the * parallel-pipeline threshold, since USAGE/WRITES/READS have one resolver per @@ -7589,6 +7706,60 @@ TEST(pipeline_python_project) { PASS(); } +/* `#include ` from arch/x/bugs.c: two headers end with the + * include path (include/linux/device.h, tools/virtio/linux/device.h) and each + * declares `struct device`. The exact-file lookup returned the first matching + * node in by-name bucket order — the workers' merge order, different run to + * run: 5,821 kernel IMPORTS edges moved between two indexes of one tree and + * the CALLS resolved through their import maps moved with them. The target is + * the least nested File node, whichever order the nodes were registered in. */ +static void add_device_header(cbm_gbuf_t *gb, const char *rel, const char *qn_stem) { + char fqn[256]; + snprintf(fqn, sizeof(fqn), "%s.h.__file__", qn_stem); + cbm_gbuf_upsert_node(gb, "File", "device.h", fqn, rel, 0, 0, "{}"); + char cqn[256]; + snprintf(cqn, sizeof(cqn), "%s.device", qn_stem); + cbm_gbuf_upsert_node(gb, "Class", "device", cqn, rel, 10, 40, "{}"); +} + +static const cbm_gbuf_node_t *resolve_device_h_from(cbm_gbuf_t *gb) { + atomic_int cancelled = 0; + cbm_pipeline_ctx_t ctx = { + .project_name = "p", + .repo_path = "/tmp/p", + .gbuf = gb, + .registry = NULL, + .cancelled = &cancelled, + }; + CBMImport imp = {0}; + imp.module_path = "linux/device.h"; + imp.local_name = "h"; + return cbm_pipeline_resolve_import_node(&ctx, "arch/x/bugs.c", "p.arch.x.bugs.c.__file__", &imp, + NULL); +} + +TEST(pipeline_header_include_target_is_independent_of_registration_order) { + cbm_gbuf_t *a = cbm_gbuf_new("p", "/tmp/cbm_include_order_a"); + cbm_gbuf_t *b = cbm_gbuf_new("p", "/tmp/cbm_include_order_b"); + ASSERT_NOT_NULL(a); + ASSERT_NOT_NULL(b); + add_device_header(a, "include/linux/device.h", "p.include.linux.device"); + add_device_header(a, "tools/virtio/linux/device.h", "p.tools.virtio.linux.device"); + add_device_header(b, "tools/virtio/linux/device.h", "p.tools.virtio.linux.device"); + add_device_header(b, "include/linux/device.h", "p.include.linux.device"); + + const cbm_gbuf_node_t *ta = resolve_device_h_from(a); + const cbm_gbuf_node_t *tb = resolve_device_h_from(b); + ASSERT_NOT_NULL(ta); + ASSERT_NOT_NULL(tb); + ASSERT_STR_EQ(ta->qualified_name, "p.include.linux.device.h.__file__"); + ASSERT_STR_EQ(tb->qualified_name, "p.include.linux.device.h.__file__"); + + cbm_gbuf_free(a); + cbm_gbuf_free(b); + PASS(); +} + /* #768 end-to-end: `import { A, B } from './lib'` must survive the REAL * pipeline (extract -> gbuf dedup -> raw SQLite dump) as TWO IMPORTS edges * with distinct local_name — and the dumped DB must satisfy its own schema. @@ -10517,6 +10688,61 @@ TEST(registry_receiver_chain_keeps_project_extension_issue1893) { PASS(); } +/* A factory chain into a nested type: Settings.builder().putList names the only + * putList in the project, Settings.Builder.putList, and Settings — the chain's + * root — is in the candidate's ancestry even though the candidate's immediate + * parent (Builder) is not spelled in the chain. The parent-only rule refused + * 9,473 such unique-name calls on elastic/elasticsearch (v0.10.8 → v0.11.0 A/B, + * 2026-09-16). */ +TEST(registry_receiver_chain_admits_factory_chain_into_nested_type) { + cbm_registry_t *reg = cbm_registry_new(); + cbm_registry_add(reg, "putList", "org.elasticsearch.common.settings.Settings.Builder.putList", + "Method"); + + cbm_resolution_t r = cbm_registry_resolve( + reg, "Settings.builder().putList", "org.elasticsearch.index.IndexSettings", NULL, NULL, 0); + ASSERT_STR_EQ(r.qualified_name, "org.elasticsearch.common.settings.Settings.Builder.putList"); + ASSERT_STR_EQ(r.strategy, "unique_name"); + + /* The same shape through a value in between: RequestOptions.DEFAULT + * .toBuilder().setWarningsHandler -> RequestOptions.Builder.setWarningsHandler. */ + cbm_registry_add(reg, "setWarningsHandler", + "org.elasticsearch.client.RequestOptions.Builder.setWarningsHandler", + "Method"); + r = cbm_registry_resolve(reg, "RequestOptions.DEFAULT.toBuilder().setWarningsHandler", + "org.elasticsearch.client.Rest", NULL, NULL, 0); + ASSERT_STR_EQ(r.qualified_name, + "org.elasticsearch.client.RequestOptions.Builder.setWarningsHandler"); + + cbm_registry_free(reg); + PASS(); +} + +/* The ancestry rule must not widen the gate to foreign roots: Base64 is the + * JDK's, so the project's DocOffsetsCodec.getEncoder stays refused, exactly as + * #1893's URLSession case above. */ +TEST(registry_receiver_chain_still_refuses_foreign_root_with_ancestry_rule) { + cbm_registry_t *reg = cbm_registry_new(); + cbm_registry_add(reg, "getEncoder", "org.elasticsearch.index.codec.DocOffsetsCodec.getEncoder", + "Method"); + + cbm_resolution_t r = cbm_registry_resolve( + reg, "Base64.getEncoder", "org.elasticsearch.index.IndexSettings", NULL, NULL, 0); + ASSERT_NULL(r.qualified_name); + + /* Nor may a package segment admit it: Math.toIntExact must not bind + * org.elasticsearch.common.Math.toIntExact through "Math" appearing as an + * ancestry segment ONLY IF that segment really is the type — here it is, and + * that is the correct outcome (the project's own Math.toIntExact). */ + cbm_registry_add(reg, "toIntExact", "org.elasticsearch.common.Math.toIntExact", "Method"); + r = cbm_registry_resolve(reg, "Math.toIntExact", "org.elasticsearch.index.IndexSettings", NULL, + NULL, 0); + ASSERT_STR_EQ(r.qualified_name, "org.elasticsearch.common.Math.toIntExact"); + + cbm_registry_free(reg); + PASS(); +} + /* A lower-case root names a value, whose type the chain does not show. The gate * must not look at it, or every ordinary vm.load style call would be refused. */ TEST(registry_receiver_chain_ignores_lowercase_root_issue1893) { @@ -14841,6 +15067,8 @@ SUITE(pipeline) { #endif RUN_TEST(pipeline_tsjs_receiver_suppresses_weak_method_edge); RUN_TEST(pipeline_python_receiver_suppresses_weak_method_edge); + RUN_TEST(pipeline_python_receiver_keeps_specific_unique_name_member_call); + RUN_TEST(pipeline_html_embedded_member_call_stays_unbound); RUN_TEST(pipeline_go_rw_usage_never_cross_into_c); RUN_TEST(pipeline_go_rw_usage_never_cross_into_c_parallel); RUN_TEST(pipeline_go_bare_ref_never_binds_field); @@ -14876,6 +15104,7 @@ SUITE(pipeline) { RUN_TEST(usages_kotlin_no_duplicate_calls); /* Language integration tests */ RUN_TEST(pipeline_python_project); + RUN_TEST(pipeline_header_include_target_is_independent_of_registration_order); RUN_TEST(pipeline_imports_multi_symbol_edges); RUN_TEST(pipeline_go_cross_package_call); RUN_TEST(pipeline_swift_cross_package_import); @@ -15019,6 +15248,8 @@ SUITE(pipeline) { RUN_TEST(registry_receiver_chain_refuses_library_unique_name_issue1893); RUN_TEST(registry_receiver_chain_refuses_library_suffix_match_issue1893); RUN_TEST(registry_receiver_chain_keeps_project_extension_issue1893); + RUN_TEST(registry_receiver_chain_admits_factory_chain_into_nested_type); + RUN_TEST(registry_receiver_chain_still_refuses_foreign_root_with_ancestry_rule); RUN_TEST(registry_receiver_chain_ignores_lowercase_root_issue1893); RUN_TEST(registry_receiver_chain_ignores_bare_name_issue1893); RUN_TEST(registry_fuzzy_confidence_single); diff --git a/tests/test_registry.c b/tests/test_registry.c index 2674e7b1d..5f751a899 100644 --- a/tests/test_registry.c +++ b/tests/test_registry.c @@ -820,6 +820,96 @@ TEST(perl_suppress_keeps_high_confidence_and_genuine_calls) { PASS(); } +/* 2026-09-16 probe on torvalds/linux: a Makefile target bound the C function + * `sk_psock.eval` through unique_name. Build and configuration languages have + * no cross-language call semantics, so for them a unique match into another + * language is a collision by construction; a code caller keeps #1572. */ +TEST(cross_language_config_caller_drops_unique_name_too) { + ASSERT_TRUE(cbm_suppress_cross_language_suffix_match(CBM_LANG_MAKEFILE, "include/linux/skmsg.h", + "unique_name")); + ASSERT_TRUE( + cbm_suppress_cross_language_suffix_match(CBM_LANG_CMAKE, "src/main.c", "unique_name")); + ASSERT_TRUE( + cbm_suppress_cross_language_suffix_match(CBM_LANG_YAML, "app/models.py", "unique_name")); + /* A code caller's unique_name into another language is still #1572. */ + ASSERT_FALSE(cbm_suppress_cross_language_suffix_match( + CBM_LANG_PYTHON, "web/src/pages/Editor.js", "unique_name")); + /* Same-language config targets are untouched. */ + ASSERT_FALSE( + cbm_suppress_cross_language_suffix_match(CBM_LANG_MAKEFILE, "lib/Makefile", "unique_name")); + /* Receiver-aware strategies are never this guard's business. */ + ASSERT_FALSE( + cbm_suppress_cross_language_suffix_match(CBM_LANG_MAKEFILE, "src/main.c", "same_module")); + PASS(); +} + +/* Same-named candidates that tie on test-status and namespace proximity used + * to be settled by bucket position, i.e. by the order files were registered: + * two indexes of one kernel tree differed by 632 CALLS edges, `dev_name` + * landing on any of twenty same-named struct fields or on nothing at all. + * The tie is now a function of the candidate set: the least nested + * definition, then the smaller QN — whichever order the registry was built + * in (O9). */ +TEST(registry_tie_break_is_independent_of_registration_order) { + const char *cands[] = { + "proj.drivers.media.cec.i2c.ch7322.ch7322_conn_match.dev_name", /* struct field */ + "proj.sound.soc.codecs.tas2783-sdw.tas2783_prv.dev_name", /* struct field */ + "proj.include.linux.device.dev_name", /* the function */ + "proj.arch.um.drivers.pty.pty_chan.dev_name", /* struct field */ + }; + const char *labels[] = {"Field", "Field", "Function", "Field"}; + const int n = 4; + + cbm_registry_t *forward = cbm_registry_new(); + cbm_registry_t *backward = cbm_registry_new(); + for (int i = 0; i < n; i++) { + cbm_registry_add(forward, "dev_name", cands[i], labels[i]); + cbm_registry_add(backward, "dev_name", cands[n - 1 - i], labels[n - 1 - i]); + } + + /* A far-away caller with no imports: every candidate scores the same. */ + cbm_resolution_t f = + cbm_registry_resolve(forward, "dev_name", "proj.kernel.irq.msi", NULL, NULL, 0); + cbm_resolution_t b = + cbm_registry_resolve(backward, "dev_name", "proj.kernel.irq.msi", NULL, NULL, 0); + ASSERT_NOT_NULL(f.qualified_name); + ASSERT_NOT_NULL(b.qualified_name); + ASSERT_STR_EQ(f.qualified_name, "proj.include.linux.device.dev_name"); + ASSERT_STR_EQ(b.qualified_name, "proj.include.linux.device.dev_name"); + ASSERT_STR_EQ(f.strategy, "suffix_match"); + ASSERT_STR_EQ(b.strategy, "suffix_match"); + + /* Equal depth: the smaller QN, from either order. */ + cbm_registry_t *lex_a = cbm_registry_new(); + cbm_registry_t *lex_b = cbm_registry_new(); + cbm_registry_add(lex_a, "sg_next", "proj.tools.virtio.scatterlist.sg_next", "Function"); + cbm_registry_add(lex_a, "sg_next", "proj.include.linux.scatterlist.sg_next", "Function"); + cbm_registry_add(lex_b, "sg_next", "proj.include.linux.scatterlist.sg_next", "Function"); + cbm_registry_add(lex_b, "sg_next", "proj.tools.virtio.scatterlist.sg_next", "Function"); + cbm_resolution_t la = + cbm_registry_resolve(lex_a, "sg_next", "proj.drivers.scsi.arm_scsi", NULL, NULL, 0); + cbm_resolution_t lb = + cbm_registry_resolve(lex_b, "sg_next", "proj.drivers.scsi.arm_scsi", NULL, NULL, 0); + ASSERT_STR_EQ(la.qualified_name, "proj.include.linux.scatterlist.sg_next"); + ASSERT_STR_EQ(lb.qualified_name, "proj.include.linux.scatterlist.sg_next"); + + /* Proximity still outranks depth: the sibling wins over a shallower stranger. */ + cbm_registry_t *near = cbm_registry_new(); + cbm_registry_add(near, "vnic_rq_free", "proj.lib.vnic_rq_free", "Function"); + cbm_registry_add(near, "vnic_rq_free", "proj.drivers.scsi.fnic.vnic_rq.vnic_rq_free", + "Function"); + cbm_resolution_t nr = cbm_registry_resolve(near, "vnic_rq_free", + "proj.drivers.scsi.fnic.fnic_res", NULL, NULL, 0); + ASSERT_STR_EQ(nr.qualified_name, "proj.drivers.scsi.fnic.vnic_rq.vnic_rq_free"); + + cbm_registry_free(forward); + cbm_registry_free(backward); + cbm_registry_free(lex_a); + cbm_registry_free(lex_b); + cbm_registry_free(near); + PASS(); +} + TEST(cross_language_suffix_match_drops_py_vs_js) { /* #725: two same-named symbols in different languages. suffix_match is the * strategy that collapses them; unique_name is #1572 and must stay. */ @@ -939,6 +1029,53 @@ TEST(dynamic_suppress_keeps_high_confidence_and_non_methods) { PASS(); } +TEST(python_builtin_member_table_matches_builtin_type_methods) { + /* Ends and middle of the sorted table, so a mis-sorted insert shows up. */ + ASSERT_TRUE(cbm_python_is_builtin_member("add")); + ASSERT_TRUE(cbm_python_is_builtin_member("extend")); + ASSERT_TRUE(cbm_python_is_builtin_member("items")); + ASSERT_TRUE(cbm_python_is_builtin_member("print")); + ASSERT_TRUE(cbm_python_is_builtin_member("startswith")); + ASSERT_TRUE(cbm_python_is_builtin_member("zfill")); + /* Project-specific spellings are not builtin members. */ + ASSERT_FALSE(cbm_python_is_builtin_member("apply_converters")); + ASSERT_FALSE(cbm_python_is_builtin_member("lazy_model_operation")); + ASSERT_FALSE(cbm_python_is_builtin_member("describe")); + ASSERT_FALSE(cbm_python_is_builtin_member(NULL)); + ASSERT_FALSE(cbm_python_is_builtin_member("")); + PASS(); +} + +TEST(weak_member_unique_name_exempt_is_python_self_rooted_unique_and_specific_only) { + /* The exemption: Python, self/cls-rooted receiver, unique_name, callee not + * a builtin member. */ + ASSERT_TRUE(cbm_weak_member_unique_name_exempt(true, true, "self.compiler.apply_converters", + "unique_name")); + ASSERT_TRUE(cbm_weak_member_unique_name_exempt(true, true, "cls.registry.lazy_model_operation", + "unique_name")); + /* A bare parameter/local receiver carries no ownership evidence: #1276's + * accelerator.backward() and c.describe() stay suppressed. */ + ASSERT_FALSE( + cbm_weak_member_unique_name_exempt(true, false, "accelerator.backward", "unique_name")); + /* A builtin type's own method stays suppressed even on self. */ + ASSERT_FALSE( + cbm_weak_member_unique_name_exempt(true, true, "self.parts.extend", "unique_name")); + ASSERT_FALSE(cbm_weak_member_unique_name_exempt(true, true, "self.out.print", "unique_name")); + /* Only unique_name carries the one-definition evidence. */ + ASSERT_FALSE(cbm_weak_member_unique_name_exempt(true, true, "self.compiler.apply_converters", + "suffix_match")); + ASSERT_FALSE(cbm_weak_member_unique_name_exempt(true, true, "self.compiler.apply_converters", + "field_type_hint")); + /* Python only: the JS/TS family keeps its recorded trade. */ + ASSERT_FALSE( + cbm_weak_member_unique_name_exempt(false, true, "this.compiler.apply", "unique_name")); + /* Degenerate inputs never exempt. */ + ASSERT_FALSE(cbm_weak_member_unique_name_exempt(true, true, NULL, "unique_name")); + ASSERT_FALSE(cbm_weak_member_unique_name_exempt(true, true, "", "unique_name")); + ASSERT_FALSE(cbm_weak_member_unique_name_exempt(true, true, "self.x.y", NULL)); + PASS(); +} + TEST(local_binding_suppress_drops_weak_shadowed_bare_calls) { /* A bare `run()` whose callee is a parameter of an enclosing scope cannot be * the module-level `run`, so a weak short-name match fabricates the edge. */ @@ -1103,10 +1240,14 @@ SUITE(registry) { RUN_TEST(perl_suppress_drops_weak_builtin_and_method_matches); RUN_TEST(perl_suppress_keeps_high_confidence_and_genuine_calls); RUN_TEST(cross_language_suffix_match_drops_py_vs_js); + RUN_TEST(cross_language_config_caller_drops_unique_name_too); + RUN_TEST(registry_tie_break_is_independent_of_registration_order); RUN_TEST(cross_language_ref_drops_go_vs_c); RUN_TEST(go_bare_ref_never_binds_field); RUN_TEST(dynamic_suppress_drops_weak_method_matches); RUN_TEST(dynamic_suppress_keeps_high_confidence_and_non_methods); + RUN_TEST(python_builtin_member_table_matches_builtin_type_methods); + RUN_TEST(weak_member_unique_name_exempt_is_python_self_rooted_unique_and_specific_only); RUN_TEST(local_binding_suppress_drops_weak_shadowed_bare_calls); RUN_TEST(local_binding_suppress_keeps_unshadowed_and_strong_strategies); RUN_TEST(weak_call_guards_share_one_drop_list); diff --git a/tests/test_str_util.c b/tests/test_str_util.c index cd6f54210..7ad729df9 100644 --- a/tests/test_str_util.c +++ b/tests/test_str_util.c @@ -436,6 +436,30 @@ TEST(json_escape_control_chars) { PASS(); } +/* A truncated escape must never end inside a multibyte UTF-8 sequence: the + * callee text of a CALLS edge is escaped into a fixed buffer, and a cut umlaut + * persisted bytes SQLite could not decode as text (2026-09-16 probe: rust, + * java and typescript stores). "aé" is a, 0xC3, 0xA9. */ +TEST(json_escape_never_splits_utf8) { + char buf[3]; /* room for two payload bytes + NUL: the lead byte would fit, its continuation not + */ + int len = cbm_json_escape(buf, sizeof(buf), "a\xC3\xA9"); + ASSERT_STR_EQ(buf, "a"); + ASSERT_EQ(len, 1); + + char whole[4]; + len = cbm_json_escape(whole, sizeof(whole), "a\xC3\xA9"); + ASSERT_STR_EQ(whole, "a\xC3\xA9"); + ASSERT_EQ(len, 3); + + /* A four-byte sequence cut after two bytes is dropped whole. */ + char emoji[3]; + len = cbm_json_escape(emoji, sizeof(emoji), "\xF0\x9F\x98\x80z"); + ASSERT_STR_EQ(emoji, ""); + ASSERT_EQ(len, 0); + PASS(); +} + /* ── SNPRINTF_APPEND tests ────────────────────────────────────── */ TEST(snprintf_append_basic) { @@ -547,6 +571,7 @@ SUITE(str_util) { RUN_TEST(validate_shell_arg_spaces); /* JSON Escaping */ RUN_TEST(json_escape_control_chars); + RUN_TEST(json_escape_never_splits_utf8); /* SNPRINTF_APPEND */ RUN_TEST(snprintf_append_basic); RUN_TEST(snprintf_append_fills_exactly); From 24afb562dc6551072a611fc7ab7922ab2a8f6ab3 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 16 Sep 2026 22:11:18 +0200 Subject: [PATCH 117/130] fix(graph): route the new scratch through the memory core The lint chain stops at its first red target, so a clang-format failure hid the memory-core linter until PR CI: the coverage line-offset table, the Cypher direction swap and the search label each added a raw allocator site. The table now comes from cbm_alloc/cbm_free (extract class), the swap frees through safe_str_free, and the BM25 row array moves to cbm_calloc/cbm_free so mcp.c's raw-site count goes down (baseline 814 -> 813) rather than up. Signed-off-by: Martin Vogel --- internal/cbm/cbm.c | 5 +++-- scripts/memory-core-baseline.txt | 2 +- src/cypher/cypher.c | 11 +++++++---- src/mcp/mcp.c | 6 ++++-- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 8b319adda..538d940c7 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -1450,7 +1450,8 @@ static uint32_t *cbm_line_offsets(const char *src, int src_len, uint32_t *out_li lines++; } } - uint32_t *offs = (uint32_t *)malloc((size_t)(lines + 1) * sizeof(uint32_t)); + uint32_t *offs = + (uint32_t *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)(lines + 1) * sizeof(uint32_t)); if (!offs) { *out_lines = 0; return NULL; @@ -1768,7 +1769,7 @@ static void cbm_subtract_recovered_regions(cbm_error_regions_t *regs, const CBMD for (int i = 0; i < regs->count; i++) { cbm_region_uncovered_gaps(regs->starts[i], regs->ends[i], defs, src, offs, nlines, &out); } - free(offs); + cbm_free(CBM_MEM_CLASS_EXTRACT, offs); *regs = out; } diff --git a/scripts/memory-core-baseline.txt b/scripts/memory-core-baseline.txt index 19fdd363d..ee54b498b 100644 --- a/scripts/memory-core-baseline.txt +++ b/scripts/memory-core-baseline.txt @@ -52,7 +52,7 @@ src/git/git_context.c 18 src/main.c 32 src/mcp/compact_out.c 39 src/mcp/index_supervisor.c 8 -src/mcp/mcp.c 785 +src/mcp/mcp.c 784 src/pipeline/artifact.c 38 src/pipeline/fqn.c 23 src/pipeline/lsp_resolve.h 4 diff --git a/src/cypher/cypher.c b/src/cypher/cypher.c index dd2544fd0..234f49087 100644 --- a/src/cypher/cypher.c +++ b/src/cypher/cypher.c @@ -5100,12 +5100,15 @@ static void cypher_plan_seed_from_selective_end(cbm_query_t *q) { p->nodes[0] = p->nodes[1]; p->nodes[1] = tmp; cbm_rel_pattern_t *r = &p->rels[0]; + const char *inverted = NULL; if (r->direction && strcmp(r->direction, "outbound") == 0) { - free((void *)r->direction); - r->direction = heap_strdup("inbound"); + inverted = "inbound"; } else if (r->direction && strcmp(r->direction, "inbound") == 0) { - free((void *)r->direction); - r->direction = heap_strdup("outbound"); + inverted = "outbound"; + } + if (inverted) { + safe_str_free(&r->direction); + r->direction = heap_strdup(inverted); } } diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index ca88bdb52..75ff8e396 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -63,6 +63,7 @@ enum { #include "cli/cli.h" #include "watcher/watcher.h" #include "foundation/mem.h" +#include "foundation/mem_core.h" #include "foundation/diagnostics.h" #include "foundation/platform.h" #include "foundation/compat.h" @@ -3880,7 +3881,7 @@ static void bm25_output_rows_free(bm25_output_row_t *rows, int count) { free(rows[i].label); free(rows[i].file_path); } - free(rows); + cbm_free(CBM_MEM_CLASS_OTHER, rows); } static void bm25_lines_str(char *out, size_t size, int start, int end) { @@ -4177,7 +4178,8 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu } int row_cap = limit > 0 ? limit : BM25_DEFAULT_LIMIT; - bm25_output_row_t *rows = calloc((size_t)row_cap, sizeof(*rows)); + bm25_output_row_t *rows = + (bm25_output_row_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)row_cap * sizeof(*rows)); int row_count = 0; while (rows && row_count < row_cap && sqlite3_step(stmt) == SQLITE_ROW) { const char *qn = (const char *)sqlite3_column_text(stmt, BM25_COL_QN); From df83d25ace9885e252363cb04be25a17da85c9e8 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 20 Sep 2026 10:03:48 +0200 Subject: [PATCH 118/130] fix(extract): drop the parse CPU reading nothing reads any more Fallout of rebasing this branch onto main. This branch was written while the LSP walk could be skipped for a file whose parse had used more than its share of the budget, and that share was measured in the thread's CPU time. Main has since removed the rule entirely -- no clock decides what ends up in the graph -- so cpu_start_ns is gone from the function and the only line still reading it was computing a value with no remaining consumer. The conflict resolution kept main's side in cbm.c, which is what leaves this line stranded: it is dead by then, and the build says so. error: use of undeclared identifier 'cpu_start_ns' Signed-off-by: Martin Vogel --- internal/cbm/cbm.c | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 538d940c7..b54fb54a7 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -2182,7 +2182,6 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C TSTree *tree = ts_parser_parse_with_options(parser, NULL, ts_input, opts); uint64_t t1 = now_ns(); - uint64_t parse_cpu_ns = cbm_thread_cpu_time_ns() - cpu_start_ns; #ifdef CBM_ENABLE_TEST_SEAMS t1 += tl_parse_wall_seam_offset_ns; /* the stall seam inflates every wall reading */ #endif From f428d583008ce87281bac138c02bd4375c56d735 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 20 Sep 2026 19:56:36 +0200 Subject: [PATCH 119/130] docs(contributing): state the terms for AI-assisted and AI-authored contributions CONTRIBUTING.md said nothing about contributions written by or with an AI agent, and the project now receives both kinds: PRs from accounts that openly identify as an agent acting for a person (#2225 is one, and a good one), and, in September, an account that posted approvals and change requests styled as maintainer decisions across nineteen threads. The first deserves a clear welcome; the second needs a rule that was until now only enforced, never written down. The new section says: disclosed AI authorship is welcome and never held against a change; the DCO sign-off is a legal certification only a person can make, so a human is accountable for every commit; the evidence bar is the same for everyone -- a claim is something a reviewer can check. Co-authored-by trailers naming a tool are fine, links to private agent sessions are not. High-volume low-effort submissions are closed whoever produced them. A second subsection states who speaks for the project -- the maintainer account only -- and that impersonating a maintainer, by a person or an agent, gets the reviews dismissed, the comments hidden, a public correction in the affected threads, and the account reported and blocked. That is what was done in September; it is now written where a contributor can read it first. Signed-off-by: Martin Vogel --- CONTRIBUTING.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 96ce8f861..ff969c483 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -149,6 +149,25 @@ If in doubt, open an issue and ask. - Run `scripts/test.sh` and `scripts/lint.sh` before submitting - Keep PRs focused — avoid unrelated reformatting or refactoring +## AI-assisted and AI-authored contributions + +Contributions written with the help of an AI tool, or written by an AI agent acting for a person, are welcome here on the same terms as any other. What we review is the change, not who or what typed it. Three things make that work: + +- **Say so.** If an agent wrote the PR or is answering in the thread, disclose it — a line in the PR description or at the top of a comment is enough ("written with X", "this account is operated by an AI agent on behalf of Y"). Disclosure is never held against a contribution. Finding out later is. +- **A person is accountable.** The [DCO sign-off](#license-and-sign-off-dco--required-on-every-commit) is a legal certification, and only a person can make it. The human behind the account certifies every commit, answers for it, and is who we talk to if something goes wrong. An agent may carry the conversation; it cannot carry the responsibility. +- **The bar does not move.** Every claim in a PR must be something a reviewer can check: the failing test, the command that was run, the numbers and the machine they came from. "The model says it works" is not evidence, and a confident description does not replace a test that fails without the fix. We read AI-written PRs exactly as carefully as human-written ones — no more suspiciously, and no less. + +Two practical notes: + +- `Co-authored-by:` trailers naming a tool are fine. Links to private chat or agent sessions are not — they are dead to everyone else and do not belong in permanent history; we will ask you to remove them. +- High-volume, low-effort submissions — many near-identical PRs, changes nobody ran, replies that do not engage with the review — will be closed, whoever or whatever produced them. One careful PR is worth more than twenty generated ones. + +### Who speaks for the project + +Only the maintainer account ([@DeusData](https://github.com/DeusData)) speaks for this project. A review, an approval, a "this will be merged" or a "this is a duplicate" from any other account is that person's opinion, however official it sounds — helpful reviews from the community are very welcome, and that is what they are. + +**Impersonating a maintainer is not acceptable**, by a person or by an agent: presenting yourself as part of the maintainer team, posting approvals or change requests styled as maintainer decisions, or telling other contributors what the project has decided. We dismiss such reviews, hide the comments, say publicly in the affected threads that they were not ours, and report and block the account. If you are unsure whether a response you received is from the project, check the account name — and ask. + ## Security We take security seriously. All PRs go through: From a262a35724b3246353f07b5de96ec4731681dd4b Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 20 Sep 2026 19:16:19 +0200 Subject: [PATCH 120/130] ci(pr): classify changed files from the merge ref when the files API 422s The `changes` job decides whether a PR touches product code, and ci-ok requires it. It asks GitHub for the PR's file list. An earlier fix moved it off the .diff endpoint, which rejects PRs over 20k lines, onto the paginated pulls/files endpoint -- but that endpoint renders the diff server-side as well, and for a PR that regenerates a vendored parser it answers HTTP 422: Sorry, this diff is taking too long to generate. Observed on #2246 (a 42 MB sql/parser.c): `changes` red, therefore ci-ok red, for a reason that has nothing to do with the contribution. The compare endpoint fails the same way, so there is no API route to the file list for such a PR. Every grammar refresh would hit this. When the API call fails, the job now takes the list from git instead. The PR merge ref's first parent is the base, so a name-only diff across that one merge commit is exactly the PR's changed files. The fetch is depth 2 with --filter=blob:none into a bare repository under RUNNER_TEMP: commits and trees only, no blobs, nothing checked out and nothing from the PR executed. No new action, no new permission, and the API path and the classification regex are untouched, so ordinary PRs behave as before. A ::notice:: line records when the fallback was used. If the fetch fails too, the step exits non-zero and writes no output -- the gate fails closed, as it did before. Proof, by executing the workflow's own run block under `bash -eo pipefail` with a stub gh: API path, docs-only list rc 0 product=false fallback, real #2246 rc 0 product=true, the same five files `git diff --numstat` reports for the PR; 0.85 s, 188 KB fetched fallback, nonexistent merge ref rc 128 no output written Contract tests that read pr.yml -- security_gate_fail_closed, smoke_fixture_contract, windows_bundle_contract, venue_parity_contract -- pass. Not addressed here, noted for a follow-up: the classification pipes the list into `grep -q` under pipefail, which can report a match as a failure once the list outgrows the pipe buffer. Signed-off-by: Martin Vogel --- .github/workflows/pr.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 0cfc796be..6e5691423 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -64,7 +64,23 @@ jobs: run: | # The full .diff endpoint rejects large-but-valid PRs at 20k lines. # The paginated files endpoint remains filename-only for this gate. - FILES=$(gh api --paginate "repos/$REPO/pulls/$PR/files?per_page=100" --jq '.[].filename') + if FILES=$(gh api --paginate "repos/$REPO/pulls/$PR/files?per_page=100" --jq '.[].filename'); then + echo "file list: pulls/files API" + else + # The files endpoint renders the diff server-side too, and answers + # 422 "this diff is taking too long to generate" for a PR that + # regenerates a vendored parser (#2246: a 42 MB sql/parser.c) -- + # so does the compare endpoint. The PR merge ref's first parent is + # the base, so a name-only diff across it is the same file list. + # depth 2 + blob:none fetches commits and trees only (well under + # 1 MB, under a second); nothing from the PR is checked out or run. + echo "::notice::pulls/files API failed; file list taken from refs/pull/$PR/merge" + CHANGES_GIT="$RUNNER_TEMP/changes.git" + git init -q --bare "$CHANGES_GIT" + git --git-dir="$CHANGES_GIT" fetch -q --depth=2 --filter=blob:none \ + "https://github.com/$REPO" "refs/pull/$PR/merge" + FILES=$(git --git-dir="$CHANGES_GIT" diff --name-only --no-renames FETCH_HEAD^1 FETCH_HEAD) + fi printf '%s\n' "$FILES" if printf '%s\n' "$FILES" | grep -qE '^(src/|internal/|install\.(sh|ps1)|scripts/build\.sh|scripts/smoke-test\.sh|scripts/smoke-local\.sh|scripts/smoke-fixture-server\.py|scripts/gen-third-party-notices\.sh|scripts/env\.sh|test-infrastructure/vm/(vm-smoke\.sh|windows-user-path-guard\.ps1)|Makefile\.cbm)'; then echo "product=true" >> "$GITHUB_OUTPUT" From 65d60e72f5b86f5982b15f0d88826ba79a085aa2 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 20 Sep 2026 19:34:37 +0200 Subject: [PATCH 121/130] ci(pr): the changes classifier no longer fails open on a large file list Found while proving the previous commit, in the same step, and the worse of the two defects because it fails OPEN. The classification was if printf '%s\n' "$FILES" | grep -qE '^(src/|internal/|...)'; then under the runner's `bash -eo pipefail`. grep -q exits on its first match; once the list is larger than the pipe buffer printf is still writing, dies of SIGPIPE, and pipefail turns the MATCH into a failed pipeline. The else branch then writes product=false, and pr-smoke and memwaste are skipped -- on exactly the PRs that change the most files. The previous commit's message flagged this as a follow-up; it is fixed here instead. The list now reaches grep through a here-string, so there is no pipe to break. The regex is unchanged. Proof, by executing the workflow's own run block under `bash -eo pipefail` with a stub gh, ten runs each: 849 KB list, first line src/a.c before: product=false 10/10 after: product=true 10/10 849 KB list, docs only product=false before and after small docs-only list product=false 422 fallback on the real #2246 product=true fallback, nonexistent merge ref rc 128, no output written It takes roughly 1,500 changed files to cross the buffer, which is why ordinary PRs never showed it. Signed-off-by: Martin Vogel --- .github/workflows/pr.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 6e5691423..8cb464b19 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -82,7 +82,11 @@ jobs: FILES=$(git --git-dir="$CHANGES_GIT" diff --name-only --no-renames FETCH_HEAD^1 FETCH_HEAD) fi printf '%s\n' "$FILES" - if printf '%s\n' "$FILES" | grep -qE '^(src/|internal/|install\.(sh|ps1)|scripts/build\.sh|scripts/smoke-test\.sh|scripts/smoke-local\.sh|scripts/smoke-fixture-server\.py|scripts/gen-third-party-notices\.sh|scripts/env\.sh|test-infrastructure/vm/(vm-smoke\.sh|windows-user-path-guard\.ps1)|Makefile\.cbm)'; then + # A here-string, not a pipe: under this shell's pipefail, grep -q + # exits on its first match, printf dies of SIGPIPE once the list + # outgrows the pipe buffer, and the MATCH is then reported as a + # failure -- product=false, smoke skipped, on the largest PRs. + if grep -qE '^(src/|internal/|install\.(sh|ps1)|scripts/build\.sh|scripts/smoke-test\.sh|scripts/smoke-local\.sh|scripts/smoke-fixture-server\.py|scripts/gen-third-party-notices\.sh|scripts/env\.sh|test-infrastructure/vm/(vm-smoke\.sh|windows-user-path-guard\.ps1)|Makefile\.cbm)' <<<"$FILES"; then echo "product=true" >> "$GITHUB_OUTPUT" else echo "product=false" >> "$GITHUB_OUTPUT" From da4bd25a01b612517512ee628bf0ec652000e80a Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 21 Sep 2026 22:46:04 +0200 Subject: [PATCH 122/130] fix(test): the ENOSPC bootstrap test no longer lets a deadline decide its verdict daemon_bootstrap_fails_fast_when_daemon_dies_at_publication went red on the MSan lane of three PRs that touch no daemon code at all (#2245, #2158, #2140) with tests/test_daemon_bootstrap.c:1116: ASSERT(daemon_named_cause) and green on a rerun of the identical commit. Cause, attributed 2026-09-20. The test forks a real daemon host whose record publication fails with ENOSPC, and asserts that the client names that cause instead of waiting out its 30 s startup deadline. But before it listens, the host SHA-256s its own executable image, and the test runner is ~450 MB: ~3 s natively, 20-32 s under MSan (48 CI samples, median 25.8 s). The spawn override returned as soon as it had forked, so the client's 30 s clock raced the host's hash. On a slow run the deadline fired first, the reaper SIGKILLed a host that had not failed yet, and the log had no listen_failed line to find. Nothing was wrong with the product: with a 28 s artificial delay the client does learn the recorded ENOSPC cause. The spawn override now reaps the forked host before it returns. The host writes its start-failure record and only then releases its lifetime reservation and exits, so "record on disk, reservation released" is a stable state pinned by construction when the client takes its first look -- and that first look is guaranteed: the client's post-spawn wait loop runs at least once even past the deadline, and its first failure check after a spawn is unthrottled. How long the host takes no longer matters. No production code changes; the assertions are unchanged. A regression to the pre-#1828 behaviour (a client that ignores the record) still fails the unchanged assertions: its message no longer names the failure, the errno and the path. Proof, one build with temporary env-switched instrumentation that is not part of this commit (a sleep in the forked host before it starts, and a switch back to the old fire-and-forget spawn): spawn forked host delayed daemon_bootstrap suite synchronous no 28 passed old (async) 31 s 27 passed, 1 failed: ASSERT(log_read) synchronous 31 s 28 passed The old spawn fails under the delay one assertion earlier than CI did (log_read, not daemon_named_cause): the injected sleep sits before the host opens its log, whereas in CI the host was killed mid-hash with the log already open. Same mechanism -- the client's deadline beating the host -- and the 2026-09-20 attribution reproduced the CI message itself with a delay placed after the log is opened. The final tree, without the instrumentation: daemon_bootstrap 28 passed, three runs in a row. Signed-off-by: Martin Vogel --- tests/test_daemon_bootstrap.c | 37 ++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/tests/test_daemon_bootstrap.c b/tests/test_daemon_bootstrap.c index 0114d6329..c2243cbea 100644 --- a/tests/test_daemon_bootstrap.c +++ b/tests/test_daemon_bootstrap.c @@ -893,6 +893,8 @@ typedef struct { char parent[BOOTSTRAP_TEST_PATH_CAP]; cbm_daemon_build_identity_t identity; pid_t children[BOOTSTRAP_ENOSPC_MAX_CHILDREN]; + int exit_status[BOOTSTRAP_ENOSPC_MAX_CHILDREN]; + bool reaped[BOOTSTRAP_ENOSPC_MAX_CHILDREN]; size_t child_count; size_t spawn_calls; } bootstrap_enospc_host_t; @@ -928,19 +930,40 @@ static bool bootstrap_enospc_host_spawn(void *opaque, int run_result = endpoint ? cbm_daemon_host_run(&config) : 0; _exit(run_result == -1 ? 0 : 50); } - state->children[state->child_count++] = child; + /* SYNCHRONOUS on purpose. The daemon host writes its start-failure record + * and only then lets go of its lifetime reservation and exits, so once it + * has been reaped "record on disk, reservation released" is a STABLE state + * pinned by construction. The client's first look after the spawn (the + * wait loop always runs once and its first failure check is unthrottled) + * therefore finds the record however long the host took to get there. + * + * Returning while the host was still running made the verdict ride on the + * client's 30 s startup deadline: before it listens the host SHA-256s its + * own image, which for the ~450 MB test runner is ~3 s natively and ~26 s + * under MSan (48 CI samples: 20-32 s). The deadline won that race on slow + * runs and the reaper then SIGKILLed a host that had not failed yet -- + * `ASSERT(daemon_named_cause)` red on PRs that touch no C at all (#2245, + * #2158, #2140). A deadline must never decide a test (O9). */ + size_t slot = state->child_count++; + state->children[slot] = child; + int status = 0; + pid_t waited; + do { + waited = waitpid(child, &status, 0); + } while (waited < 0 && errno == EINTR); + state->reaped[slot] = waited == child; + state->exit_status[slot] = status; return true; } static void bootstrap_enospc_reap(bootstrap_enospc_host_t *state, int *nonzero_exits) { *nonzero_exits = 0; for (size_t i = 0; i < state->child_count; i++) { - int status = 0; - pid_t waited; - do { - waited = waitpid(state->children[i], &status, WNOHANG); - } while (waited < 0 && errno == EINTR); - if (waited == 0) { + int status = state->exit_status[i]; + if (!state->reaped[i]) { + /* Only reachable when the synchronous wait in the spawn itself + * failed; never leave a host behind. */ + pid_t waited; (void)kill(state->children[i], SIGKILL); do { waited = waitpid(state->children[i], &status, 0); From c314dea4c1409a3a68fb33706b49e272809b5079 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 20 Sep 2026 19:29:00 +0200 Subject: [PATCH 123/130] fix(service-patterns): match library ids on identifier boundaries Distilled from #1245 by Andrew Hundt (d15071c9, the match_qn half). match_qn was a raw strstr(qn, library_id). Short ids fired inside unrelated words, and a service kind REPLACES the plain CALLS edge, so a false match both invents a service edge and deletes a real call: "gin." in plugin. -> route registration; any "/"-prefixed first argument then minted a Route node "nconf" in encryptionconfig -> CONFIGURES for every call into the package "tonic" in Monotonic -> gRPC; unparseable, so NO edge at all "express" in expression -> route registration "get_env" in get_envelope, get_environ -> CONFIGURES "dio" "surf" "curl" "resty" "phin" "hyper" "treq" "rocket" in studio, surface, curly, restyle, dolphin, hyperbolic, streq, rocketmq The rule (qn_hit_on_boundary; every occurrence of an id is tried, not only the first). A separator is any character that is not an ASCII letter or digit. before: start of string | separator | the id itself starts with a separator ("@trpc/server") | the id starts with an uppercase letter -- a capital opens a new CamelCase word whatever precedes it (AsyncHttpClient, IHttpClientFactory, NSURLSession) after: end of string | separator | the id itself ends in a separator ("gin.") | uppercase letter (GuzzleHttp, FeignClient, KafkaProducer) | digit (urllib2, Mint.HTTP2, amqp091-go) Rejected: a lowercase- or digit-initial id glued to a preceding letter or digit, and any id continued by a lowercase letter. Two deliberate differences from the upstream hunk: upstream's before-rule needed a GuzzleHttp table entry and would have dropped IHttpClientFactory and NSURLSession, and without the digit after-boundary urllib2 and amqp091-go stop matching. All three match on main today. No recall is traded for this. No boundary rule can tell "grequests" from "myrequests", so a library whose own name glues a prefix or suffix onto another id gets an explicit entry with the kind (and broker) it had on main -- 47 of them, found mechanically by running main's matcher and the new one over 173 candidate names and listing every one that went from a kind to NONE: HTTP grequests txrequests redaxios gaxios libcurl curlpp curlcpp hyperlocal guzzlehttp ASYNC aiokafka pykafka rskafka librdkafka rdkafkacpp aioamqp amqpstorm pamqp pyamqp amqprs amqpcpp pynats jnats aiomqtt amqtt hbmqtt umqtt mqttools emqtt rumqtt gomqtt libmosquitto mosquittopp pubsublite gocelery CONFIG wgetenv getenvb qgetenv dotenvx phpdotenv ROUTE_REG apiflask honox fasthttprouter GRAPHQL gqlparser gqlgenc aiogqlc Left at NONE on purpose: names that are not clients of the id they contain and were misclassified on main -- hypercorn, hyperlink, hyperopt, openresty, gqlalchemy, celeryconfig, kafkacat, gnatsd, pypubsub. One visible reclassification: hyperium/tonic was HTTP through "hyper" inside "hyperium" and is now gRPC through "tonic". RED, production reverted and the final tests kept: infrascan_service_pattern_match_rejects_ids_inside_words FAIL tests/test_infrascan.c:105: svc_case_mismatches(cases) == 28, expected 0 == 0 4 passed, 1 failed All 28 negatives mismatch on main. The 47 glued-library positives are RED the other way round: against the boundary rule without the entries. Each positive uses a name containing no second id, so it binds to its own entry -- "aiokafka.AIOKafkaProducer" would have passed through "KafkaProducer" and proven nothing. GREEN: infrascan 5, pipeline 281, parallel 74, extraction 350, edge_types_probe 59, route_canon 11, lang_contract 41, cross_repo 8, registry 64, mcp 318 (4 Windows-only skips). 0 failed. Effect on real code -- production binaries, main 92abefa3 against this change. main against main differs by 0 edges and 0 nodes on all three corpora, so every delta below is the change: django CONFIGURES 76 -> 74; both calls return as CALLS (geom.get_envelope, WSGIRequestHandler.get_environ) typescript CONFIGURES 12,026 -> 12,024; one bogus Route "/*type*/" gone with its CALLS and HANDLES; 3 calls return as CALLS kubernetes CONFIGURES 7,962 -> 7,609: all 353 are "nconf" inside encryptionconfig / authorizationconfig / authenticationconfig, and all 353 return as CALLS. 8 bogus Routes gone -- os.ReadFile of the serviceaccount namespace file, t.Run("/TwoWay"), "/tmp/test" -- with 26 CALLS, 2 HANDLES and 21 TESTS edges that pointed at them. 374 plain CALLS restored; 8 of them (validate.Monotonic and friends) had NO edge at all on main. GRPC_CALLS 60 -> 60. HTTP_CALLS 398 -> 407: nine calls main swallowed as route registrations now reach the existing arg_url heuristic like every other package. 88 of the 408 removed service-derived edges were read at the call site -- all of django and typescript, 82 of 402 in kubernetes: 88 false positives, 0 true positives lost. None of the three corpora uses a glued-name library, so the 47 entries are covered by the unit test only. Known limits: "_" is a separator, so optical_fiber.len still matches "fiber." exactly as requests_get matches "requests" by design; a lowercase id followed by a capital matches (kafkaProducer), which also admits a dioXide-style name; a user wrapper glued without a separator (mygetenv) is no longer classified; a glued-name library missing from the list is NONE until someone adds it. Co-authored-by: Andrew Hundt Signed-off-by: Martin Vogel --- internal/cbm/service_patterns.c | 124 ++++++++++++++++++++-- tests/test_infrascan.c | 183 +++++++++++++++++++++++++++++++- 2 files changed, 295 insertions(+), 12 deletions(-) diff --git a/internal/cbm/service_patterns.c b/internal/cbm/service_patterns.c index 0dd404877..c2425e09e 100644 --- a/internal/cbm/service_patterns.c +++ b/internal/cbm/service_patterns.c @@ -140,6 +140,20 @@ static const lib_pattern_t http_libraries[] = { {"socket.http", CBM_SVC_HTTP, NULL}, {"resty.http", CBM_SVC_HTTP, NULL}, + /* Glued-name libraries. match_qn needs an identifier boundary around an + * id, so a library whose own name glues a lowercase prefix/suffix onto + * another id ("grequests", "curlpp") is listed explicitly: no boundary + * rule can tell "grequests" from "myrequests". */ + {"grequests", CBM_SVC_HTTP, NULL}, /* Python: gevent + requests */ + {"txrequests", CBM_SVC_HTTP, NULL}, /* Python: Twisted + requests */ + {"redaxios", CBM_SVC_HTTP, NULL}, /* JS: axios-compatible fetch client */ + {"gaxios", CBM_SVC_HTTP, NULL}, /* JS: Google's axios-style client */ + {"libcurl", CBM_SVC_HTTP, NULL}, /* C, node-libcurl */ + {"curlpp", CBM_SVC_HTTP, NULL}, /* C++ libcurl wrapper */ + {"curlcpp", CBM_SVC_HTTP, NULL}, /* C++ libcurl wrapper */ + {"hyperlocal", CBM_SVC_HTTP, NULL}, /* Rust: hyper over unix sockets */ + {"guzzlehttp", CBM_SVC_HTTP, NULL}, /* PHP: Guzzle's composer vendor */ + {NULL, CBM_SVC_NONE, NULL}, }; @@ -263,6 +277,33 @@ static const lib_pattern_t async_libraries[] = { {"dapr.clients.grpc", CBM_SVC_ASYNC, "dapr"}, {"DaprClient", CBM_SVC_ASYNC, "dapr"}, + /* Glued-name libraries (see the note in http_libraries). */ + {"aiokafka", CBM_SVC_ASYNC, "kafka"}, + {"pykafka", CBM_SVC_ASYNC, "kafka"}, + {"librdkafka", CBM_SVC_ASYNC, "kafka"}, + {"rdkafkacpp", CBM_SVC_ASYNC, "kafka"}, + {"rskafka", CBM_SVC_ASYNC, "kafka"}, + {"aioamqp", CBM_SVC_ASYNC, "rabbitmq"}, + {"amqpstorm", CBM_SVC_ASYNC, "rabbitmq"}, + {"pamqp", CBM_SVC_ASYNC, "rabbitmq"}, + {"pyamqp", CBM_SVC_ASYNC, "rabbitmq"}, + {"amqprs", CBM_SVC_ASYNC, "rabbitmq"}, + {"amqpcpp", CBM_SVC_ASYNC, "rabbitmq"}, + {"pynats", CBM_SVC_ASYNC, "nats"}, + {"jnats", CBM_SVC_ASYNC, "nats"}, + {"aiomqtt", CBM_SVC_ASYNC, "mqtt"}, + {"amqtt", CBM_SVC_ASYNC, "mqtt"}, + {"hbmqtt", CBM_SVC_ASYNC, "mqtt"}, + {"umqtt", CBM_SVC_ASYNC, "mqtt"}, + {"mqttools", CBM_SVC_ASYNC, "mqtt"}, + {"emqtt", CBM_SVC_ASYNC, "mqtt"}, + {"rumqtt", CBM_SVC_ASYNC, "mqtt"}, + {"gomqtt", CBM_SVC_ASYNC, "mqtt"}, + {"libmosquitto", CBM_SVC_ASYNC, "mqtt"}, + {"mosquittopp", CBM_SVC_ASYNC, "mqtt"}, + {"pubsublite", CBM_SVC_ASYNC, "pubsub"}, + {"gocelery", CBM_SVC_ASYNC, "celery"}, + {NULL, CBM_SVC_NONE, NULL}, }; @@ -308,6 +349,13 @@ static const lib_pattern_t config_libraries[] = { {"Application.get_env", CBM_SVC_CONFIG, NULL}, {"Application.fetch_env", CBM_SVC_CONFIG, NULL}, + /* Glued-name accessors (see the note in http_libraries). */ + {"wgetenv", CBM_SVC_CONFIG, NULL}, /* Windows CRT: _wgetenv, _wgetenv_s */ + {"getenvb", CBM_SVC_CONFIG, NULL}, /* Python: os.getenvb */ + {"qgetenv", CBM_SVC_CONFIG, NULL}, /* Qt */ + {"dotenvx", CBM_SVC_CONFIG, NULL}, /* JS: @dotenvx/dotenvx */ + {"phpdotenv", CBM_SVC_CONFIG, NULL}, /* PHP: vlucas/phpdotenv */ + {NULL, CBM_SVC_NONE, NULL}, }; @@ -377,6 +425,11 @@ static const lib_pattern_t route_reg_libraries[] = { {"akka.http.scaladsl.server", CBM_SVC_ROUTE_REG, NULL}, {"play.api.routing", CBM_SVC_ROUTE_REG, NULL}, + /* Glued-name frameworks (see the note in http_libraries). */ + {"apiflask", CBM_SVC_ROUTE_REG, NULL}, /* Python: Flask-based */ + {"honox", CBM_SVC_ROUTE_REG, NULL}, /* JS: Hono meta-framework */ + {"fasthttprouter", CBM_SVC_ROUTE_REG, NULL}, /* Go: httprouter for fasthttp */ + {NULL, CBM_SVC_NONE, NULL}, }; @@ -448,6 +501,11 @@ static const lib_pattern_t graphql_libraries[] = { {"async-graphql", CBM_SVC_GRAPHQL, NULL}, {"juniper", CBM_SVC_GRAPHQL, NULL}, + /* Glued-name libraries (see the note in http_libraries). */ + {"gqlparser", CBM_SVC_GRAPHQL, NULL}, /* Go: vektah/gqlparser */ + {"gqlgenc", CBM_SVC_GRAPHQL, NULL}, /* Go: gqlgen client generator */ + {"aiogqlc", CBM_SVC_GRAPHQL, NULL}, /* Python: asyncio GraphQL client */ + {NULL, CBM_SVC_NONE, NULL}, }; @@ -534,17 +592,67 @@ static const method_suffix_t method_suffixes[] = { /* ── Matching implementation ───────────────────────────────────── */ -/* Check if any library identifier appears as a substring in the QN. - * Case-sensitive: "requests" matches "project.venv.requests.api.get" - * but not "Requests". Library names are specific enough to avoid - * false positives even with substring matching. */ +static bool qn_is_lower(char ch) { + return ch >= 'a' && ch <= 'z'; +} + +static bool is_digit_char(char ch) { + return ch >= '0' && ch <= '9'; +} + +static bool qn_is_alnum(char ch) { + return qn_is_lower(ch) || is_digit_char(ch) || (ch >= 'A' && ch <= 'Z'); +} + +/* True when the occurrence of `id` at `hit` sits on identifier boundaries. + * + * A raw substring match let short ids fire inside unrelated words: "gin." + * in "plugin.", "dio" in "studio", "surf" in "surface", "express" in + * "expression". Every character that is not an ASCII letter or digit is a + * separator ('.', '/', '\\', ':', '_', '-', '$', '@', ...). + * + * BEFORE: start of string, a separator, an id that itself starts with a + * separator ("@trpc/server"), or an id that starts with an + * uppercase letter — a capital opens a new CamelCase word whatever + * precedes it ("AsyncHttpClient", "IHttpClientFactory", + * "NSURLSession"). Rejected: a lowercase/digit-initial id glued to + * a letter or digit ("plugin." / "studio" / "myrequests"). + * AFTER: end of string, a separator, an id that itself ends in a + * separator ("gin."), an uppercase letter (CamelCase glue: + * "GuzzleHttp", "FeignClient", "KafkaProducer", "kafkaProducer") + * or a digit (version suffix: "urllib2", "amqp091-go"). + * Rejected: a lowercase letter continuing the word ("surface", + * "curly", "expression"). */ +static bool qn_hit_on_boundary(const char *qn, const char *hit, const char *id, size_t id_len) { + char first = id[0]; + char last = id[id_len - 1]; + if (hit > qn && qn_is_alnum(hit[-1]) && (qn_is_lower(first) || is_digit_char(first))) { + return false; + } + if (qn_is_alnum(last) && qn_is_lower(hit[id_len])) { + return false; + } + return true; +} + +/* Check if any library identifier appears in the QN on identifier + * boundaries (see qn_hit_on_boundary). Case-sensitive: "requests" matches + * "project.venv.requests.api.get" and "svc.requests_get" but neither + * "Requests" nor "myrequests". */ static const lib_pattern_t *match_qn(const char *qn, const lib_pattern_t *patterns) { if (!qn || !qn[0]) { return NULL; } for (int i = 0; patterns[i].library_id != NULL; i++) { - if (strstr(qn, patterns[i].library_id) != NULL) { - return &patterns[i]; + const char *id = patterns[i].library_id; + size_t id_len = strlen(id); + if (id_len == 0) { + continue; + } + for (const char *hit = strstr(qn, id); hit != NULL; hit = strstr(hit + 1, id)) { + if (qn_hit_on_boundary(qn, hit, id, id_len)) { + return &patterns[i]; + } } } return NULL; @@ -574,10 +682,6 @@ static bool contains_segment(const char *path, const char *segment) { return false; } -static bool is_digit_char(char ch) { - return ch >= '0' && ch <= '9'; -} - static bool has_http_route_marker(const char *path) { if (starts_with_segment(path, "api") || starts_with_segment(path, "apis") || starts_with_segment(path, "graphql") || starts_with_segment(path, "health") || diff --git a/tests/test_infrascan.c b/tests/test_infrascan.c index cd7df945d..a9bf1aef9 100644 --- a/tests/test_infrascan.c +++ b/tests/test_infrascan.c @@ -1,11 +1,11 @@ #include "test_framework.h" #include "graph_buffer/graph_buffer.h" #include "pipeline/pipeline_internal.h" +#include "service_patterns.h" #include #include - -bool cbm_service_pattern_is_http_route_literal(const char *literal, const char *callee_name); +#include static int has_data_flow(cbm_gbuf_t *gb, int64_t source_id, int64_t target_id) { const cbm_gbuf_edge_t **edges = NULL; @@ -48,6 +48,183 @@ TEST(infrascan_http_route_literal_guard_rejects_filesystem_paths) { PASS(); } +/* ── Service-pattern QN boundaries (distilled from PR #1245) ───────── + * A library id must sit on an identifier boundary inside the QN. A raw + * substring match classified "proj.plugin.loader" as gin ("gin.") and + * "proj.studio.x" as Dio ("dio"), which minted bogus Route / HTTP_CALLS + * edges. The rule must stay open for CamelCase glue ("GuzzleHttp", + * "FeignClient", "KafkaProducer", "IHttpClientFactory") and for version + * suffixes ("urllib2", "amqp091-go"). */ +typedef struct { + const char *qn; + cbm_svc_kind_t want; +} svc_case_t; + +static int svc_case_mismatches(const svc_case_t *cases) { + int bad = 0; + for (int i = 0; cases[i].qn != NULL; i++) { + cbm_svc_kind_t got = cbm_service_pattern_match(cases[i].qn); + if (got != cases[i].want) { + printf(" service-pattern mismatch: \"%s\" -> %d, expected %d\n", cases[i].qn, (int)got, + (int)cases[i].want); + bad++; + } + } + return bad; +} + +TEST(infrascan_service_pattern_match_rejects_ids_inside_words) { + static const svc_case_t cases[] = { + /* "gin." inside plugin. */ + {"proj.plugins.autorun.tests.test_plugin._dispatch", CBM_SVC_NONE}, + {"proj.plugin.loader", CBM_SVC_NONE}, + /* "chi." / "echo." / "fiber." */ + {"proj.hibachi.grill", CBM_SVC_NONE}, + {"sound.reecho.apply", CBM_SVC_NONE}, + {"textile.microfiber.weave", CBM_SVC_NONE}, + /* "dio" inside studio / radio */ + {"proj.studio.x", CBM_SVC_NONE}, + {"radio.tune", CBM_SVC_NONE}, + /* "surf" / "resty" / "curl" / "phin" / "hyper" / "sling" / "needle" / "treq" */ + {"app.surface.draw", CBM_SVC_NONE}, + {"ui.restyle.apply", CBM_SVC_NONE}, + /* control: "forestry" spells "restr", never contained "resty" */ + {"pkg.forestry.trees", CBM_SVC_NONE}, + {"util.curly.brace", CBM_SVC_NONE}, + {"sea.dolphin.swim", CBM_SVC_NONE}, + {"math.hyperbolic.tanh", CBM_SVC_NONE}, + {"farm.gosling.walk", CBM_SVC_NONE}, + {"sewing.needles.sort", CBM_SVC_NONE}, + {"util.streq", CBM_SVC_NONE}, + /* "requests" glued to a lowercase prefix (upstream's case) */ + {"proj.myrequests.client.get", CBM_SVC_NONE}, + /* "express" / "rocket" / "tonic" / "nconf" */ + {"ast.expression.eval", CBM_SVC_NONE}, + {"mq.rocketmq.send", CBM_SVC_NONE}, + {"geo.tectonic.plates", CBM_SVC_NONE}, + {"app.unconfigured.warn", CBM_SVC_NONE}, + {"cli.runconfig.load", CBM_SVC_NONE}, + {"aws.s3express.session", CBM_SVC_NONE}, + /* not clients of the id they contain: ASGI server, URL library, + * nginx platform, Memgraph OGM, Celery settings module, a word */ + {"hypercorn.run.serve", CBM_SVC_NONE}, + {"hyperlink.URL.from_text", CBM_SVC_NONE}, + {"openresty.core.request", CBM_SVC_NONE}, + {"gqlalchemy.Memgraph.execute", CBM_SVC_NONE}, + {"celeryconfig.broker_url", CBM_SVC_NONE}, + {"kafkaesque.story.tell", CBM_SVC_NONE}, + {NULL, CBM_SVC_NONE}, + }; + ASSERT_EQ(svc_case_mismatches(cases), 0); + PASS(); +} + +TEST(infrascan_service_pattern_match_keeps_real_library_qns) { + static const svc_case_t cases[] = { + /* Route registration: ids that end in a separator, own segment */ + {"github.com/gin-gonic/gin.Default", CBM_SVC_ROUTE_REG}, + {"proj.gin.router.GET", CBM_SVC_ROUTE_REG}, + {"gin.GET", CBM_SVC_ROUTE_REG}, + {"chi.NewRouter", CBM_SVC_ROUTE_REG}, + {"github.com/labstack/echo.New", CBM_SVC_ROUTE_REG}, + {"fiber.New", CBM_SVC_ROUTE_REG}, + {"proj.express.router.get", CBM_SVC_ROUTE_REG}, + {"@hapi/hapi.server.route", CBM_SVC_ROUTE_REG}, + /* HTTP: own segment / separator-joined wrapper */ + {"proj.venv.requests.api.get", CBM_SVC_HTTP}, + {"proj.service.requests_get", CBM_SVC_HTTP}, + {"package:dio/dio.dart", CBM_SVC_HTTP}, + {"dio.Dio.get", CBM_SVC_HTTP}, + {"pkg.net.curl.get", CBM_SVC_HTTP}, + {"curl_exec", CBM_SVC_HTTP}, + {"surf.get", CBM_SVC_HTTP}, + {"hyper.Client.request", CBM_SVC_HTTP}, + {"phin.promisified", CBM_SVC_HTTP}, + {"needle.get", CBM_SVC_HTTP}, + {"github.com/go-resty/resty.New", CBM_SVC_HTTP}, + {"github.com/dghubble/sling.New", CBM_SVC_HTTP}, + /* HTTP: CamelCase glue after the id */ + {"GuzzleHttp\\Client", CBM_SVC_HTTP}, + {"proj.GuzzleHttp.Client.get", CBM_SVC_HTTP}, + {"org.springframework.cloud.openfeign.FeignClientBuilder.build", CBM_SVC_HTTP}, + {"com.acme.net.HttpClientFactory.create", CBM_SVC_HTTP}, + /* HTTP: CamelCase glue before the id */ + {"System.Net.Http.IHttpClientFactory.CreateClient", CBM_SVC_HTTP}, + {"org.asynchttpclient.AsyncHttpClient.prepareGet", CBM_SVC_HTTP}, + {"Foundation.NSURLSession.dataTask", CBM_SVC_HTTP}, + /* HTTP: version suffix after the id */ + {"urllib2.urlopen", CBM_SVC_HTTP}, + {"Mint.HTTP2.request", CBM_SVC_HTTP}, + /* Async */ + {"com.acme.events.KafkaProducer.send", CBM_SVC_ASYNC}, + {"this.kafkaProducer.send", CBM_SVC_ASYNC}, + {"org.apache.kafka.clients.producer.KafkaProducer.send", CBM_SVC_ASYNC}, + {"github.com/rabbitmq/amqp091-go.Channel.Publish", CBM_SVC_ASYNC}, + {"github.com/nats-io/nats.go.Conn.Publish", CBM_SVC_ASYNC}, + /* Config */ + {"os.getenv", CBM_SVC_CONFIG}, + {"os.Getenv", CBM_SVC_CONFIG}, + /* Glued-name libraries: the library's own name glues a lowercase + * prefix/suffix onto another id, so each has an explicit table entry. + * Every QN below contains NO other id — only the entry can match. */ + {"grequests.map", CBM_SVC_HTTP}, + {"txrequests.Session.send", CBM_SVC_HTTP}, + {"redaxios.create", CBM_SVC_HTTP}, + {"gaxios.request", CBM_SVC_HTTP}, + {"libcurl.version", CBM_SVC_HTTP}, + {"node-libcurl.Easy.perform", CBM_SVC_HTTP}, + {"curlpp.Easy.perform", CBM_SVC_HTTP}, + {"curlcpp.easy.perform", CBM_SVC_HTTP}, + {"hyperlocal.UnixConnector.call", CBM_SVC_HTTP}, + {"guzzlehttp/psr7.Utils.streamFor", CBM_SVC_HTTP}, + {"aiokafka.helpers.create_ssl_context", CBM_SVC_ASYNC}, + {"pykafka.topic.Topic.get_producer", CBM_SVC_ASYNC}, + {"librdkafka.rd_version", CBM_SVC_ASYNC}, + {"rdkafkacpp.version", CBM_SVC_ASYNC}, + {"rskafka.client.partition.write", CBM_SVC_ASYNC}, + {"aioamqp.connect", CBM_SVC_ASYNC}, + {"amqpstorm.Connection.channel", CBM_SVC_ASYNC}, + {"pamqp.frame.marshal", CBM_SVC_ASYNC}, + {"kombu.transport.pyamqp.Transport.establish_connection", CBM_SVC_ASYNC}, + {"amqprs.channel.basic_publish", CBM_SVC_ASYNC}, + {"amqpcpp.TcpChannel.publish", CBM_SVC_ASYNC}, + {"pynats.client.send", CBM_SVC_ASYNC}, + {"jnats.options.build", CBM_SVC_ASYNC}, + {"aiomqtt.client.publish", CBM_SVC_ASYNC}, + {"amqtt.session.deliver", CBM_SVC_ASYNC}, + {"hbmqtt.session.deliver", CBM_SVC_ASYNC}, + {"umqtt.simple.publish", CBM_SVC_ASYNC}, + {"mqttools.client.publish", CBM_SVC_ASYNC}, + {"emqtt.publish", CBM_SVC_ASYNC}, + {"rumqtt.client.publish", CBM_SVC_ASYNC}, + {"github.com/256dpi/gomqtt/client.Publish", CBM_SVC_ASYNC}, + {"libmosquitto.publish", CBM_SVC_ASYNC}, + {"mosquittopp.publish", CBM_SVC_ASYNC}, + {"google.cloud.pubsublite.admin.create_topic", CBM_SVC_ASYNC}, + {"gocelery.client.delay", CBM_SVC_ASYNC}, + {"_wgetenv", CBM_SVC_CONFIG}, + {"_wgetenv_s", CBM_SVC_CONFIG}, + {"os.getenvb", CBM_SVC_CONFIG}, + {"qgetenv", CBM_SVC_CONFIG}, + {"@dotenvx/dotenvx.config", CBM_SVC_CONFIG}, + {"vlucas/phpdotenv.load", CBM_SVC_CONFIG}, + {"apiflask.scaffold.get", CBM_SVC_ROUTE_REG}, + {"honox.factory.createRoute", CBM_SVC_ROUTE_REG}, + {"fasthttprouter.Router.GET", CBM_SVC_ROUTE_REG}, + {"github.com/vektah/gqlparser/v2.LoadSchema", CBM_SVC_GRAPHQL}, + {"github.com/Yamashou/gqlgenc.client.Post", CBM_SVC_GRAPHQL}, + {"aiogqlc.client.execute", CBM_SVC_GRAPHQL}, + {NULL, CBM_SVC_NONE}, + }; + ASSERT_EQ(svc_case_mismatches(cases), 0); + ASSERT_STR_EQ(cbm_service_pattern_broker("aiokafka.helpers.create_ssl_context"), "kafka"); + ASSERT_STR_EQ(cbm_service_pattern_broker("umqtt.simple.publish"), "mqtt"); + ASSERT_STR_EQ(cbm_service_pattern_broker("com.acme.events.KafkaProducer.send"), "kafka"); + ASSERT_STR_EQ(cbm_service_pattern_broker("github.com/rabbitmq/amqp091-go.Channel.Publish"), + "rabbitmq"); + PASS(); +} + TEST(infrascan_route_nodes_skip_bad_http_url_paths) { cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/cbm_infrascan_route_guard"); ASSERT_NOT_NULL(gb); @@ -128,6 +305,8 @@ TEST(infrascan_http_calls_join_matching_handler_route) { SUITE(infrascan) { RUN_TEST(infrascan_http_route_literal_guard_rejects_filesystem_paths); + RUN_TEST(infrascan_service_pattern_match_rejects_ids_inside_words); + RUN_TEST(infrascan_service_pattern_match_keeps_real_library_qns); RUN_TEST(infrascan_route_nodes_skip_bad_http_url_paths); RUN_TEST(infrascan_http_calls_join_matching_handler_route); } From 343d7dd5e47b817999098605370f0eab706675f0 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 20 Sep 2026 19:45:31 +0200 Subject: [PATCH 124/130] fix(extract): a C-family function's return type keeps its pointer and qualifiers Distilled from #1245 by Andrew Hundt (c51e5cb0, the extract_defs.c half; the mcp.c and token-reduction parts are not taken). For C-family functions return_type was the text of the declaration's `type` node alone. In these grammars the `*`, `&` and `&&` live on the declarator and cv-qualifiers are sibling nodes, so const char *get_name(void) published char robj **lookupKeys(...) published robj Text &operator=(const Text &) published Text and no C or C++ function in any index had a pointer return type at all. The C/C++ resolver's return-type parser already strips a leading const/volatile and handles trailing `*`, `&`, `&&` -- branches that were unreachable for functions, so every `Foo *f()` was registered as returning Foo by value. c_declared_return_type() renders the declared type at both sites (functions and methods) for C, C++, CUDA, GLSL, HLSL, ISPC, Slang and Objective-C -- the list resolve_func_name_c_family uses: so `const char *`, `char **`, `Text &`, `Text &&`, `Text *&`, `volatile unsigned long *`; a pointer-level qualifier follows its star (`char *const *`); east-const `char const *` normalises to `const char *`. With no qualifier and no marker the old text is returned unchanged. It measures, then writes into one exact-size arena allocation; the declarator walk is a plain child chain, no recursion. def.return_types is untouched. Two deliberate differences from the upstream hunk. Upstream copied every type_qualifier node, which in these grammars includes constexpr, _Noreturn, mutable and __extension__ -- `constexpr int f()` would have become `constexpr int`. Only const, volatile, restrict, __restrict, __restrict__ and _Atomic are kept, and both cases are asserted unchanged. And the walk stops at the first declarator that is neither pointer nor reference, so `int (*f(void))(int)` stays `int` rather than becoming a wrong `int *`. RED, production reverted and the final tests kept: c_function_return_type_preserves_pointer_and_qualifier FAIL tests/test_extraction.c:1005: "char" != "const char *" cpp_method_return_type_preserves_pointer_and_qualifier FAIL tests/test_extraction.c:1072: "char" != "const char *" 351 passed, 2 failed c_function_return_type_plain_unchanged is the control: it passes before and after. GREEN: extraction 353, c_lsp 762, lang_contract 41, pipeline 281. Effect on real code -- Redis, production binaries, old against new. 39,132 nodes in both. 1,442 Function nodes change, in return_type only: void -> void * (270), char -> const char * (101), unsigned char -> unsigned char * (91), char -> char * (81), robj -> robj * (57), ... Functions with a `*` in return_type: 0 -> 1,440. With one worker the complete edge set apart from SEMANTICALLY_RELATED is byte-identical old against new across six runs (three each) -- CALLS, USAGE, WRITES, IMPORTS and the rest do not move. SEMANTICALLY_RELATED goes 350 -> 325 because that pass tokenises the return-type string into its type vector. One thing measured rather than assumed. With several workers, which `dict.h` Redis' src/dict.c imports (src/ or deps/hiredis/) depends on worker merge order. That is main's defect, not this change's: unmodified main picked src/dict.h in 2 of 10 runs and the hiredis header in 8; with one worker both binaries always pick src/dict.h. This change picked src/dict.h in 6 of 10. At ten runs a side that gap is not separable from chance (Fisher exact, two-sided p = 0.17), and a shifted rate would be unsurprising for a timing-dependent pick; it is recorded here rather than explained. The include-target fix is a separate change (#2227). Known limits, all pre-existing and out of scope: struct and class FIELDS have the same defect (`const char *data;` publishes char); C prototypes and declaration-only C++ methods are not extracted as definitions, so they are not reached; where a macro is misparsed as the type (`LUA_API const char *f()`), the base was already wrong and now reads `const LUA_API *`; the resolver's parser does not yet understand a pointer-level qualifier or a leading _Atomic/restrict and falls back to a named-type lookup for those, as it effectively did for every pointer return before. Co-authored-by: Andrew Hundt Signed-off-by: Martin Vogel --- internal/cbm/extract_defs.c | 157 +++++++++++++++++++++++++++++++++++- tests/test_extraction.c | 98 ++++++++++++++++++++++ 2 files changed, 253 insertions(+), 2 deletions(-) diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 93da084be..916f1136f 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -3477,6 +3477,159 @@ static TSNode find_function_params(TSNode func_node, CBMLanguage lang) { return params; } +/* ── C-family declared return type ────────────────────────────────── + * The C-family grammars split a declared return type three ways: the `type` + * field (`char`), sibling type_qualifier nodes (`const`), and the + * pointer/reference declarators wrapping the function declarator (`*`). Taking + * only the `type` field published `const char *get_name(void)` as "char". + * + * Canonical spelling: leading cv-qualifiers in source order, the base type text + * verbatim, then one space and the declarator markers outermost-first with no + * space between them — `const char *`, `char **`, `Text &`, `Text *&`. A + * qualifier on a pointer level follows its `*` and is separated from the next + * marker by a space: `char *const *`. A qualifier written after the base type + * (`char const *`) is normalized to the leading position. */ + +/* Output sink: measures when buf is NULL, writes otherwise. Rendering twice + * sizes the arena allocation exactly without a second copy of the logic. */ +typedef struct { + char *buf; + size_t len; +} c_rt_out_t; + +static void c_rt_put(c_rt_out_t *out, const char *text, size_t n) { + if (out->buf) { + memcpy(out->buf + out->len, text, n); + } + out->len += n; +} + +static void c_rt_put_str(c_rt_out_t *out, const char *text) { + c_rt_put(out, text, strlen(text)); +} + +static void c_rt_put_node(c_rt_out_t *out, TSNode node, const char *source) { + uint32_t start = ts_node_start_byte(node); + uint32_t end = ts_node_end_byte(node); + if (end > start) { + c_rt_put(out, source + start, (size_t)(end - start)); + } +} + +/* type_qualifier also covers keywords that are not part of the type + * (`constexpr`, `_Noreturn`, `mutable`, `__extension__`, …). Keep only the ones + * that are, so `constexpr int f()` still returns "int". */ +static bool is_c_return_cv_qualifier(TSNode node, const char *source) { + static const char *const kept[] = {"const", "volatile", "restrict", "__restrict", + "__restrict__", "_Atomic", NULL}; + if (strcmp(ts_node_type(node), "type_qualifier") != 0) { + return false; + } + uint32_t start = ts_node_start_byte(node); + uint32_t end = ts_node_end_byte(node); + size_t len = end > start ? (size_t)(end - start) : 0; + for (const char *const *k = kept; *k; k++) { + if (strlen(*k) == len && memcmp(source + start, *k, len) == 0) { + return true; + } + } + return false; +} + +static bool is_c_declarator_lang(CBMLanguage lang) { + return lang == CBM_LANG_C || lang == CBM_LANG_CPP || lang == CBM_LANG_CUDA || + lang == CBM_LANG_GLSL || lang == CBM_LANG_HLSL || lang == CBM_LANG_ISPC || + lang == CBM_LANG_SLANG || lang == CBM_LANG_OBJC; +} + +/* Render the canonical return type into `out`; returns how many qualifiers and + * markers were added around the base type (0 = the base text alone is already + * the whole type). The declarator walk is one strict child chain, so it is + * O(depth) with no recursion and needs no depth cap. It stops at the first node + * that is neither a pointer nor a reference declarator: for a function returning + * a function pointer (`int (*f(void))(int)`) that is the outer + * function_declarator, which leaves the base type as it was. */ +static size_t c_rt_render(c_rt_out_t *out, TSNode func_node, TSNode type_node, TSNode declarator, + const char *source) { + size_t added = 0; + uint32_t decl_start = ts_node_start_byte(declarator); + uint32_t nc = ts_node_named_child_count(func_node); + for (uint32_t i = 0; i < nc; i++) { + TSNode ch = ts_node_named_child(func_node, i); + if (ts_node_start_byte(ch) >= decl_start) { + break; + } + if (is_c_return_cv_qualifier(ch, source)) { + c_rt_put_node(out, ch, source); + c_rt_put_str(out, " "); + added++; + } + } + c_rt_put_node(out, type_node, source); + + bool need_space = true; + TSNode decl = declarator; + while (!ts_node_is_null(decl)) { + const char *dk = ts_node_type(decl); + bool is_ref = strcmp(dk, "reference_declarator") == 0; + if (!is_ref && strcmp(dk, "pointer_declarator") != 0) { + break; + } + if (need_space) { + c_rt_put_str(out, " "); + need_space = false; + } + /* A reference_declarator opens with its `&` / `&&` token. */ + TSNode marker = ts_node_child(decl, 0); + if (is_ref && !ts_node_is_null(marker) && !ts_node_is_named(marker)) { + c_rt_put_node(out, marker, source); + } else { + c_rt_put_str(out, is_ref ? "&" : "*"); + } + added++; + uint32_t dn = ts_node_named_child_count(decl); + for (uint32_t i = 0; i < dn; i++) { + TSNode q = ts_node_named_child(decl, i); + if (is_c_return_cv_qualifier(q, source)) { + c_rt_put_node(out, q, source); + need_space = true; + added++; + } + } + /* tree-sitter-cpp/-cuda give a reference_declarator's inner declarator no + * `declarator` field (see find_c_params); it is the one named child. */ + TSNode inner = ts_node_child_by_field_name(decl, TS_FIELD("declarator")); + if (ts_node_is_null(inner) && is_ref && dn > 0) { + inner = ts_node_named_child(decl, 0); + } + decl = inner; + } + return added; +} + +/* Declared return type of a C-family function/method node whose `type` field is + * `type_node`. Any other language, and any type with nothing around its base + * type, gets the base type text exactly as before. */ +static char *c_declared_return_type(CBMExtractCtx *ctx, TSNode func_node, TSNode type_node) { + CBMArena *a = ctx->arena; + TSNode declarator = ts_node_child_by_field_name(func_node, TS_FIELD("declarator")); + if (!is_c_declarator_lang(ctx->language) || ts_node_is_null(declarator)) { + return cbm_node_text(a, type_node, ctx->source); + } + c_rt_out_t out = {NULL, 0}; + if (c_rt_render(&out, func_node, type_node, declarator, ctx->source) == 0) { + return cbm_node_text(a, type_node, ctx->source); + } + out.buf = (char *)cbm_arena_alloc(a, out.len + NULL_TERM); + if (!out.buf) { + return cbm_node_text(a, type_node, ctx->source); + } + out.len = 0; + (void)c_rt_render(&out, func_node, type_node, declarator, ctx->source); + out.buf[out.len] = '\0'; + return out.buf; +} + // C++: resolve trailing return type (auto f() -> Type) on a declarator node. // Updates def->return_type and def->return_types if trailing type found. static void resolve_cpp_trailing_return(CBMArena *a, TSNode func_node, const char *source, @@ -3712,7 +3865,7 @@ static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec for (const char **f = rt_fields; *f; f++) { TSNode rt = ts_node_child_by_field_name(func_node, *f, (uint32_t)strlen(*f)); if (!ts_node_is_null(rt)) { - def.return_type = cbm_node_text(a, rt, ctx->source); + def.return_type = c_declared_return_type(ctx, func_node, rt); def.return_types = extract_return_types(a, rt, ctx->source, ctx->language); break; } @@ -4964,7 +5117,7 @@ static void push_method_def(CBMExtractCtx *ctx, TSNode child, TSNode class_node, for (const char **f = rt_fields; *f; f++) { TSNode rt = ts_node_child_by_field_name(child, *f, (uint32_t)strlen(*f)); if (!ts_node_is_null(rt)) { - def.return_type = cbm_node_text(a, rt, ctx->source); + def.return_type = c_declared_return_type(ctx, child, rt); break; } } diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 8d9bf742b..331aa54f3 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -977,6 +977,65 @@ TEST(c_struct) { PASS(); } +/* return_type of the first definition named `name`; NULL when there is no such + * definition or it carries no return type. */ +static const char *def_return_type(CBMFileResult *r, const char *name) { + for (int i = 0; i < r->defs.count; i++) { + if (strcmp(r->defs.items[i].name, name) == 0) { + return r->defs.items[i].return_type; + } + } + return NULL; +} + +/* PR #1245: the C grammar splits a declared return type across the `type` node, + * sibling type_qualifier nodes and the pointer_declarator chain wrapping the + * function declarator. Taking only the `type` node published `const char *` as + * "char". Canonical spelling: qualifiers, base type, one space, then the + * declarator markers unspaced (`const char *`, `char **`). */ +TEST(c_function_return_type_preserves_pointer_and_qualifier) { + CBMFileResult *r = extract("static const char *text_end(const char *text) { return text; }\n" + "char **table(void) { return 0; }\n" + "char const *east_const(void) { return 0; }\n" + "const struct Point *find_point(void) { return 0; }\n" + "volatile unsigned long *reg(void) { return 0; }\n" + "char *const *frozen(void) { return 0; }\n", + CBM_LANG_C, "t", "returns.c"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT_STR_EQ(def_return_type(r, "text_end"), "const char *"); + ASSERT_STR_EQ(def_return_type(r, "table"), "char **"); + ASSERT_STR_EQ(def_return_type(r, "east_const"), "const char *"); + ASSERT_STR_EQ(def_return_type(r, "find_point"), "const struct Point *"); + ASSERT_STR_EQ(def_return_type(r, "reg"), "volatile unsigned long *"); + ASSERT_STR_EQ(def_return_type(r, "frozen"), "char *const *"); + cbm_free_result(r); + PASS(); +} + +/* Guard for the fix above: a return type with no qualifier and no declarator + * marker is already correct and must come out byte-identical. */ +TEST(c_function_return_type_plain_unchanged) { + CBMFileResult *r = extract("int scalar(void) { return 0; }\n" + "void nothing(void) {}\n" + "unsigned long wide(void) { return 0; }\n" + "struct Point make_point(void) { struct Point p; return p; }\n" + "static inline size_t count(void) { return 0; }\n" + "_Noreturn void die(void) { for (;;) {} }\n", + CBM_LANG_C, "t", "plain.c"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT_STR_EQ(def_return_type(r, "scalar"), "int"); + ASSERT_STR_EQ(def_return_type(r, "nothing"), "void"); + ASSERT_STR_EQ(def_return_type(r, "wide"), "unsigned long"); + ASSERT_STR_EQ(def_return_type(r, "make_point"), "struct Point"); + ASSERT_STR_EQ(def_return_type(r, "count"), "size_t"); + /* _Noreturn parses as a type_qualifier but is not part of the type. */ + ASSERT_STR_EQ(def_return_type(r, "die"), "void"); + cbm_free_result(r); + PASS(); +} + /* --- C++ --- */ TEST(cpp_class) { CBMFileResult *r = extract( @@ -990,6 +1049,42 @@ TEST(cpp_class) { PASS(); } +/* PR #1245, C++ side: in-class methods go through a separate extraction path + * from free functions, and C++ adds reference markers (`&`, `&&`) whose + * reference_declarator carries no `declarator` field. */ +TEST(cpp_method_return_type_preserves_pointer_and_qualifier) { + CBMFileResult *r = extract("class Text {\n" + "public:\n" + " const char *end() { return nullptr; }\n" + " char **table() { return nullptr; }\n" + " Text &self() { return *this; }\n" + " const Text &cself() const { return *this; }\n" + " Text *&slot() { return next_; }\n" + " Text *next_;\n" + " int width() const { return 0; }\n" + " constexpr int square(int x) const { return x * x; }\n" + "};\n" + "const Text &shared() { static Text t; return t; }\n" + "Text &&moved(Text &t) { return static_cast(t); }\n" + "const char *Text::c_str() const { return nullptr; }\n", + CBM_LANG_CPP, "t", "text.cpp"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT_STR_EQ(def_return_type(r, "end"), "const char *"); + ASSERT_STR_EQ(def_return_type(r, "table"), "char **"); + ASSERT_STR_EQ(def_return_type(r, "self"), "Text &"); + ASSERT_STR_EQ(def_return_type(r, "cself"), "const Text &"); + ASSERT_STR_EQ(def_return_type(r, "slot"), "Text *&"); + ASSERT_STR_EQ(def_return_type(r, "width"), "int"); + /* constexpr parses as a type_qualifier but is not part of the type. */ + ASSERT_STR_EQ(def_return_type(r, "square"), "int"); + ASSERT_STR_EQ(def_return_type(r, "shared"), "const Text &"); + ASSERT_STR_EQ(def_return_type(r, "moved"), "Text &&"); + ASSERT_STR_EQ(def_return_type(r, "c_str"), "const char *"); + cbm_free_result(r); + PASS(); +} + /* ═══════════════════════════════════════════════════════════════════ * Group C: Scripting / Dynamic Languages * ═══════════════════════════════════════════════════════════════════ */ @@ -8206,8 +8301,11 @@ SUITE(extraction) { RUN_TEST(go_interface); RUN_TEST(zig_function); RUN_TEST(c_function); + RUN_TEST(c_function_return_type_preserves_pointer_and_qualifier); + RUN_TEST(c_function_return_type_plain_unchanged); RUN_TEST(c_struct); RUN_TEST(cpp_class); + RUN_TEST(cpp_method_return_type_preserves_pointer_and_qualifier); /* Scripting */ RUN_TEST(python_function); From a041794f8e51ddf42342899be7b2ac26f951d777 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 22 Sep 2026 01:12:51 +0200 Subject: [PATCH 125/130] fix(daemon): losing the runtime-directory creation race is not a failure on Windows Several processes first-starting together against a runtime directory that does not exist yet -- test-windows-guards' section_cold_storm, or a host that launches more than one MCP server on its very first run -- all observe the final path component absent. One CreateDirectoryW wins; the others get ERROR_ALREADY_EXISTS. win_private_directory_tree_secure() treated that as a failure and recorded no validation detail, so every loser exited with a bare secure CLI coordination could not be created (endpoint) The function called right after the walk, win_runtime_directory_secure(), has always tolerated ERROR_ALREADY_EXISTS for the same directory, and the POSIX walk tolerates EEXIST at the same point; only the Windows walk in front of them did not. Seen four times on 2026-09-21 on unrelated PRs (#1768 three attempts in a row, #2140, #1999, #808). The racy lines date from July; the guard began exercising them on 2026-09-03, when each guard section was given its own empty CBM_RUNTIME_DIR, so the final component is now absent at storm time on every run. The walk now treats ERROR_ALREADY_EXISTS from its own CreateDirectoryW as "the directory I wanted exists". Nothing is trusted because of that: an ancestor still goes through win_directory_component_secure(), and the final component through win_runtime_directory_secure(), which refuses a non-directory or reparse point and enforces owner and DACL. Every refusal on this path now names its component and its rule. The walk reports the Windows error when it can neither create nor inspect a component, and win_runtime_directory_secure() says whether the path could not be created, cannot be inspected, exists but is not a directory, or is a reparse point. One helper owns the wide-to-UTF-8 conversion for these messages and the existing ancestor message now uses it too, so src/daemon/ipc.c stays at its memory-core baseline. Deterministic reproduction, no threads and no timing: a test seam fires in the walk between "component observed absent" and CreateDirectoryW, and the test plays the process that wins the creation. daemon_ipc_windows_private_directory_survives_lost_creation_race daemon_ipc_windows_private_directory_refuses_and_names_a_planted_file Verification. macOS arm64 (ASan+UBSan): build clean, daemon_ipc 53 passed (2 Windows-only skips), daemon_bootstrap 28 passed, lint-memory-core unchanged (ipc.c stays at 146 raw sites). The two new tests are Windows-only and have NOT been run locally: the Windows VM was down when this was written, so the RED run on the seam-only tree and the GREEN run with this fix are delegated to CI's windows-latest legs (test-windows shards, test-windows-guards) by an explicit maintainer decision. If the guard suite's cold storm still fails with this in place, the attribution above is wrong and this reverts. Signed-off-by: Martin Vogel --- src/daemon/ipc.c | 88 ++++++++++++++++++++++++--- src/daemon/ipc_internal.h | 11 ++++ tests/test_daemon_ipc.c | 125 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 217 insertions(+), 7 deletions(-) diff --git a/src/daemon/ipc.c b/src/daemon/ipc.c index a92befb08..3917a7a05 100644 --- a/src/daemon/ipc.c +++ b/src/daemon/ipc.c @@ -214,6 +214,31 @@ static void ipc_startup_gate_run(void) { } #endif +#ifdef _WIN32 +static cbm_daemon_ipc_win_directory_create_hook_fn g_win_directory_create_hook_for_test; +static void *g_win_directory_create_hook_context_for_test; +#endif + +void cbm_daemon_ipc_win_directory_create_hook_set_for_test( + cbm_daemon_ipc_win_directory_create_hook_fn hook, void *context) { +#ifdef _WIN32 + g_win_directory_create_hook_context_for_test = context; + g_win_directory_create_hook_for_test = hook; +#else + (void)hook; + (void)context; +#endif +} + +#ifdef _WIN32 +static void ipc_win_directory_create_hook_run(const wchar_t *path) { + cbm_daemon_ipc_win_directory_create_hook_fn hook = g_win_directory_create_hook_for_test; + if (hook) { + hook(path, g_win_directory_create_hook_context_for_test); + } +} +#endif + bool cbm_daemon_ipc_windows_legacy_names(const char *canonical_runtime_parent, const char *instance_key, char pipe_out[CBM_DAEMON_IPC_WINDOWS_NAME_CAP], @@ -3994,6 +4019,16 @@ static char *wide_to_utf8(const wchar_t *value) { return utf8; } +/* Record ": " as the validation detail, with the wide path + * rendered as UTF-8. One place owns the conversion buffer, so every refusal on + * the directory walk can name its component without each call site carrying + * its own allocation. */ +static void ipc_validation_detail_set_for_path(const wchar_t *path, const char *message) { + char *path_utf8 = wide_to_utf8(path); + ipc_validation_detail_set("%s: %s", path_utf8 ? path_utf8 : "", message); + free(path_utf8); +} + static wchar_t *wide_copy(const wchar_t *value) { if (!value) { return NULL; @@ -4861,13 +4896,28 @@ static bool win_runtime_directory_secure(const wchar_t *runtime_dir) { return false; } bool created = CreateDirectoryW(runtime_dir, &security.directory_attributes) != 0; - if (!created && GetLastError() != ERROR_ALREADY_EXISTS) { + DWORD create_error = created ? ERROR_SUCCESS : GetLastError(); + if (!created && create_error != ERROR_ALREADY_EXISTS) { + char message[96]; + (void)snprintf(message, sizeof(message), + "could not create the directory (Windows error %lu)", + (unsigned long)create_error); + ipc_validation_detail_set_for_path(runtime_dir, message); win_security_destroy(&security); return false; } DWORD attributes = GetFileAttributesW(runtime_dir); if (attributes == INVALID_FILE_ATTRIBUTES || (attributes & FILE_ATTRIBUTE_DIRECTORY) == 0 || (attributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0) { + /* What already exists at this path is adopted only if it is a plain + * directory. Say which rule refused it: the walk in front of this + * function tolerates losing a creation race, so this check is what + * stands between that tolerance and a planted file or junction. */ + const char *rule = attributes == INVALID_FILE_ATTRIBUTES ? "cannot be inspected" + : (attributes & FILE_ATTRIBUTE_DIRECTORY) == 0 + ? "exists but is not a directory" + : "is a reparse point (junction or symlink)"; + ipc_validation_detail_set_for_path(runtime_dir, rule); win_security_destroy(&security); return false; } @@ -5021,8 +5071,35 @@ static bool win_private_directory_tree_secure(const wchar_t *directory_path) { DWORD attributes = GetFileAttributesW(path); if (attributes == INVALID_FILE_ATTRIBUTES) { DWORD error = GetLastError(); - ok = (error == ERROR_FILE_NOT_FOUND || error == ERROR_PATH_NOT_FOUND) && - CreateDirectoryW(path, &security.directory_attributes) != 0; + bool absent = error == ERROR_FILE_NOT_FOUND || error == ERROR_PATH_NOT_FOUND; + if (absent) { + ipc_win_directory_create_hook_run(path); + /* Several processes first-starting together all observe + * this component absent; one CreateDirectoryW wins and + * the rest get ERROR_ALREADY_EXISTS. Losing that race is + * not a failure -- the directory this process wanted now + * exists, exactly as if it had been there all along. It + * is NOT trusted for that: an ancestor still goes through + * win_directory_component_secure() below, and the final + * component through win_runtime_directory_secure(), which + * refuses a non-directory or reparse point and enforces + * owner and DACL. The POSIX walk tolerates EEXIST at the + * same point for the same reason. */ + if (CreateDirectoryW(path, &security.directory_attributes) == 0) { + error = GetLastError(); + absent = error == ERROR_ALREADY_EXISTS; + } + } + ok = absent; + if (!ok) { + /* Never a bare "(endpoint)": name the component and the + * Windows error that refused it. */ + char message[96]; + (void)snprintf(message, sizeof(message), + "could not create or inspect the directory (Windows error %lu)", + (unsigned long)error); + ipc_validation_detail_set_for_path(path, message); + } } /* Ancestors are observe-only and must already be secure. The * final current-user directory is intentionally handled below by @@ -5034,10 +5111,7 @@ static bool win_private_directory_tree_secure(const wchar_t *directory_path) { * at this walk position; the helper set the inner rule. */ char inner[384]; (void)snprintf(inner, sizeof(inner), "%s", ipc_validation_detail_buffer); - char *component_utf8 = wide_to_utf8(path); - ipc_validation_detail_set( - "%s: %s", component_utf8 ? component_utf8 : "", inner); - free(component_utf8); + ipc_validation_detail_set_for_path(path, inner); } } } diff --git a/src/daemon/ipc_internal.h b/src/daemon/ipc_internal.h index 4b7cf1fd5..f481b7378 100644 --- a/src/daemon/ipc_internal.h +++ b/src/daemon/ipc_internal.h @@ -110,4 +110,15 @@ void cbm_daemon_ipc_posix_record_write_failure_set_for_test(int errno_value); typedef void (*cbm_daemon_ipc_startup_gate_fn)(void *context); void cbm_daemon_ipc_startup_gate_set_for_test(cbm_daemon_ipc_startup_gate_fn gate, void *context); +/* Deterministic-interleaving seam for the Windows private-directory walk: + * fires after a path component was observed ABSENT and before this process + * tries to create it. A test plays the concurrent process that wins that + * creation, which pins the cold-start race (N processes first-starting + * against a runtime directory that does not exist yet) by construction + * instead of by thread timing. `path` is the NUL-terminated component path at + * the walk position. No-op off Windows. */ +typedef void (*cbm_daemon_ipc_win_directory_create_hook_fn)(const wchar_t *path, void *context); +void cbm_daemon_ipc_win_directory_create_hook_set_for_test( + cbm_daemon_ipc_win_directory_create_hook_fn hook, void *context); + #endif /* CBM_DAEMON_IPC_INTERNAL_H */ diff --git a/tests/test_daemon_ipc.c b/tests/test_daemon_ipc.c index b451f9bb3..815f630a6 100644 --- a/tests/test_daemon_ipc.c +++ b/tests/test_daemon_ipc.c @@ -882,6 +882,129 @@ TEST(daemon_ipc_windows_private_directory_ace_is_inheritable) { PASS(); } +typedef struct { + int calls; + bool plant_file; /* false: play a racer that creates the DIRECTORY first */ + bool planted; +} ipc_test_win_create_racer_t; + +/* Plays the concurrent process that wins the creation of a path component the + * walk has just observed absent. Runs inside the walk, at the exact point the + * race lives, so the interleaving is pinned by construction: no threads, no + * timing. */ +static void ipc_test_win_create_racer(const wchar_t *path, void *context) { + ipc_test_win_create_racer_t *racer = context; + racer->calls++; + if (racer->calls != 1) { + return; + } + if (racer->plant_file) { + HANDLE file = + CreateFileW(path, GENERIC_WRITE, 0, NULL, CREATE_NEW, FILE_ATTRIBUTE_NORMAL, NULL); + racer->planted = file != INVALID_HANDLE_VALUE; + if (racer->planted) { + (void)CloseHandle(file); + } + } else { + racer->planted = CreateDirectoryW(path, NULL) != 0; + } +} + +/* Cold-start race (test-windows-guards section_cold_storm, 2026-09-21; the + * same thing a user gets when a host launches several MCP servers on its very + * first run): N processes first-start against a runtime directory that does + * not exist yet. All of them observe the final component absent, one + * CreateDirectoryW wins, and every other process gets ERROR_ALREADY_EXISTS. + * The walk treated that as a failure and recorded no detail, so the losers + * exited with a bare "secure CLI coordination could not be created + * (endpoint)". Losing the creation race is not a failure: the directory this + * process wanted now exists, and the owner / DACL / not-a-reparse-point checks + * that follow are exactly the ones a pre-existing directory gets. The POSIX + * walk has always tolerated EEXIST here. */ +TEST(daemon_ipc_windows_private_directory_survives_lost_creation_race) { + char parent[TEST_PATH_CAP] = {0}; + char cache[TEST_PATH_CAP] = {0}; + bool paths_ok = false; + bool secured = false; + bool owner_only = false; + ipc_test_win_create_racer_t racer = {0}; + + if (ipc_test_parent_new(parent, "win-create-race")) { + int written = snprintf(cache, sizeof(cache), "%s/cache", parent); + paths_ok = written > 0 && written < (int)sizeof(cache); + } + if (paths_ok) { + cbm_daemon_ipc_win_directory_create_hook_set_for_test(ipc_test_win_create_racer, &racer); + secured = cbm_daemon_ipc_private_directory_secure(cache); + cbm_daemon_ipc_win_directory_create_hook_set_for_test(NULL, NULL); + } + if (secured) { + /* The racer created the directory with an inherited DACL; the winner's + * directory must still end up owner-only, like any adopted one. */ + PACL dacl = NULL; + PSECURITY_DESCRIPTOR descriptor = NULL; + ACL_SIZE_INFORMATION information; + memset(&information, 0, sizeof(information)); + if (GetNamedSecurityInfoA(cache, SE_FILE_OBJECT, DACL_SECURITY_INFORMATION, NULL, NULL, + &dacl, NULL, &descriptor) == ERROR_SUCCESS && + dacl && + GetAclInformation(dacl, &information, sizeof(information), AclSizeInformation)) { + owner_only = information.AceCount == 1; + } + if (descriptor) { + (void)LocalFree(descriptor); + } + } + + (void)RemoveDirectoryA(cache); + ipc_test_remove_flat_dir(parent); + + ASSERT_TRUE(paths_ok); + /* The seam sat on the path and the racer really won: without these two the + * test could pass without ever exercising the race. */ + ASSERT_EQ(racer.calls, 1); + ASSERT_TRUE(racer.planted); + ASSERT_TRUE(secured); + ASSERT_TRUE(owner_only); + PASS(); +} + +/* The tolerance above must not admit anything: when what appeared at the path + * is NOT a directory, the walk still refuses -- and it says why. A bare + * "(endpoint)" is what four reporters in #1533/#1574 were left with; every + * refusal on this path names the component and the rule. */ +TEST(daemon_ipc_windows_private_directory_refuses_and_names_a_planted_file) { + char parent[TEST_PATH_CAP] = {0}; + char cache[TEST_PATH_CAP] = {0}; + char detail[512] = {0}; + bool paths_ok = false; + bool refused = false; + ipc_test_win_create_racer_t racer = {.plant_file = true}; + + if (ipc_test_parent_new(parent, "win-create-race-file")) { + int written = snprintf(cache, sizeof(cache), "%s/cache", parent); + paths_ok = written > 0 && written < (int)sizeof(cache); + } + if (paths_ok) { + cbm_daemon_ipc_win_directory_create_hook_set_for_test(ipc_test_win_create_racer, &racer); + refused = !cbm_daemon_ipc_private_directory_secure(cache); + cbm_daemon_ipc_win_directory_create_hook_set_for_test(NULL, NULL); + const char *why = cbm_daemon_ipc_validation_detail(); + (void)snprintf(detail, sizeof(detail), "%s", why ? why : ""); + } + + (void)DeleteFileA(cache); + ipc_test_remove_flat_dir(parent); + + ASSERT_TRUE(paths_ok); + ASSERT_EQ(racer.calls, 1); + ASSERT_TRUE(racer.planted); + ASSERT_TRUE(refused); + ASSERT_NOT_NULL(strstr(detail, "cache")); + ASSERT_NOT_NULL(strstr(detail, "not a directory")); + PASS(); +} + TEST(daemon_ipc_windows_private_directory_allows_add_subdirectory_only_ancestor) { char parent[TEST_PATH_CAP] = {0}; char add_only[TEST_PATH_CAP] = {0}; @@ -5361,6 +5484,8 @@ SUITE(daemon_ipc) { RUN_TEST(daemon_ipc_windows_default_endpoint_ignores_temp_environment); RUN_TEST(daemon_ipc_windows_private_directory_rejects_untrusted_ancestor_acl); RUN_TEST(daemon_ipc_windows_private_directory_ace_is_inheritable); + RUN_TEST(daemon_ipc_windows_private_directory_survives_lost_creation_race); + RUN_TEST(daemon_ipc_windows_private_directory_refuses_and_names_a_planted_file); RUN_TEST(daemon_ipc_windows_private_directory_allows_add_subdirectory_only_ancestor); RUN_TEST(daemon_ipc_windows_sid_trust_accepts_local_admin_rejects_foreign_500); RUN_TEST(daemon_ipc_windows_legacy_bridge_covers_handoff_and_lifetime); From aaa72fbe60ad341325e9c944e32cf67d98eca387 Mon Sep 17 00:00:00 2001 From: Lowpower <3133734+Lowpower@users.noreply.github.com> Date: Thu, 10 Sep 2026 03:06:03 +0000 Subject: [PATCH 126/130] feat(cli): point top-level cli --help at --format json CLI_USAGE described outer --json but did not mention tool-level --format json. Add a help-only pointer that tools which accept format support --format tree|json, and distinguish payload JSON from the full MCP envelope. No new flags, env vars, or output-default changes. Fixes #2102 Signed-off-by: Lowpower <3133734+Lowpower@users.noreply.github.com> --- src/cli/cli.h | 12 ++++++++++++ src/main.c | 8 +------- tests/test_cli.c | 16 ++++++++++++++++ 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/cli/cli.h b/src/cli/cli.h index cf7f6fa44..a1b749bb0 100644 --- a/src/cli/cli.h +++ b/src/cli/cli.h @@ -27,6 +27,18 @@ const char *cbm_cli_get_version(void); /* ── CLI tool arguments (flags / --args-file / --help) ────────── */ +/* Top-level `cli --help` text printed by run_cli() in src/main.c. + * Documents tool-level --format without adding a session-wide flag (#2102). */ +#define CBM_CLI_USAGE \ + "Usage: codebase-memory-mcp cli [--quiet] [--progress] [--verbose] [--json] " \ + "[json_args]\n" \ + " --quiet Show errors only; cannot combine with --progress or outer --verbose\n" \ + " --progress Show lifecycle progress even when stderr is redirected\n" \ + " --verbose Include informational logs (preserves CBM_LOG_LEVEL=debug)\n" \ + " --json Print the raw MCP result envelope\n" \ + " Tools that accept format support --format tree|json (default: tree).\n" \ + " --format json prints payload JSON; outer --json prints the full MCP envelope.\n" + /* Convert `--flag value` / `--flag=value` / bare-boolean `--flag` arguments for * a tool into a JSON arguments object string, using the tool's input_schema to * type values (string/integer/boolean) and to collect repeated flags into diff --git a/src/main.c b/src/main.c index 9b91f205c..65e041217 100644 --- a/src/main.c +++ b/src/main.c @@ -675,13 +675,7 @@ static bool client_start_parent_watchdog(DWORD initial_ppid) { /* ── CLI mode ───────────────────────────────────────────────────── */ -#define CLI_USAGE \ - "Usage: codebase-memory-mcp cli [--quiet] [--progress] [--verbose] [--json] " \ - "[json_args]\n" \ - " --quiet Show errors only; cannot combine with --progress or outer --verbose\n" \ - " --progress Show lifecycle progress even when stderr is redirected\n" \ - " --verbose Include informational logs (preserves CBM_LOG_LEVEL=debug)\n" \ - " --json Print the raw MCP result envelope\n" +#define CLI_USAGE CBM_CLI_USAGE /* `cli --help`; wording lives in cli.h (#2102) */ /* Extract text content from MCP tool result envelope and print it. * MCP results: {"content":[{"type":"text","text":"..."}],"isError":...} diff --git a/tests/test_cli.c b/tests/test_cli.c index 211ec6355..2ca51ab2f 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -15336,6 +15336,21 @@ TEST(cli_print_tool_help_issue680) { PASS(); } +/* #2102: top-level `cli --help` (CLI_USAGE) must point at tool-level + * `--format json` and distinguish it from outer `--json`. This is help-only: + * the usage synopsis still has no session-wide --format / CBM_CLI_FORMAT. */ +TEST(cli_usage_points_to_tool_format_json_issue2102) { + ASSERT_NOT_NULL(strstr(CBM_CLI_USAGE, "--format tree|json")); + ASSERT_NOT_NULL(strstr(CBM_CLI_USAGE, "payload JSON")); + ASSERT_NOT_NULL(strstr(CBM_CLI_USAGE, "full MCP envelope")); + ASSERT_NOT_NULL(strstr(CBM_CLI_USAGE, "--json Print the raw MCP result envelope")); + ASSERT_NOT_NULL(strstr(CBM_CLI_USAGE, + "cli [--quiet] [--progress] [--verbose] [--json] ")); + ASSERT_NULL(strstr(CBM_CLI_USAGE, "[--format")); + ASSERT_NULL(strstr(CBM_CLI_USAGE, "CBM_CLI_FORMAT")); + PASS(); +} + /* #1359: `cli ` with no argument-bearing token used to slurp stdin to EOF * for ANY non-terminal stdin. The ordinary automation caller never sends that * EOF — Node's child_process.spawn defaults to stdio:['pipe','pipe','pipe'] and @@ -16376,6 +16391,7 @@ SUITE(cli) { RUN_TEST(cli_build_args_json_key_equals_value_issue680); RUN_TEST(cli_build_args_json_bad_positional_errors_issue680); RUN_TEST(cli_print_tool_help_issue680); + RUN_TEST(cli_usage_points_to_tool_format_json_issue2102); /* Stdin argument gate (#1359) */ RUN_TEST(cli_zero_argument_tool_never_reads_stdin_issue1359); From 2f557f53986976b2df1eb2a1a540c9d48cf91a8e Mon Sep 17 00:00:00 2001 From: Lowpower <3133734+Lowpower@users.noreply.github.com> Date: Thu, 10 Sep 2026 03:17:17 +0000 Subject: [PATCH 127/130] style(cli): clang-format CBM_CLI_USAGE and help-text test Signed-off-by: Lowpower <3133734+Lowpower@users.noreply.github.com> --- src/cli/cli.h | 14 +++++++------- tests/test_cli.c | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/cli/cli.h b/src/cli/cli.h index a1b749bb0..5522edef0 100644 --- a/src/cli/cli.h +++ b/src/cli/cli.h @@ -29,14 +29,14 @@ const char *cbm_cli_get_version(void); /* Top-level `cli --help` text printed by run_cli() in src/main.c. * Documents tool-level --format without adding a session-wide flag (#2102). */ -#define CBM_CLI_USAGE \ +#define CBM_CLI_USAGE \ "Usage: codebase-memory-mcp cli [--quiet] [--progress] [--verbose] [--json] " \ - "[json_args]\n" \ - " --quiet Show errors only; cannot combine with --progress or outer --verbose\n" \ - " --progress Show lifecycle progress even when stderr is redirected\n" \ - " --verbose Include informational logs (preserves CBM_LOG_LEVEL=debug)\n" \ - " --json Print the raw MCP result envelope\n" \ - " Tools that accept format support --format tree|json (default: tree).\n" \ + "[json_args]\n" \ + " --quiet Show errors only; cannot combine with --progress or outer --verbose\n" \ + " --progress Show lifecycle progress even when stderr is redirected\n" \ + " --verbose Include informational logs (preserves CBM_LOG_LEVEL=debug)\n" \ + " --json Print the raw MCP result envelope\n" \ + " Tools that accept format support --format tree|json (default: tree).\n" \ " --format json prints payload JSON; outer --json prints the full MCP envelope.\n" /* Convert `--flag value` / `--flag=value` / bare-boolean `--flag` arguments for diff --git a/tests/test_cli.c b/tests/test_cli.c index 2ca51ab2f..01fb79ec8 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -15344,8 +15344,8 @@ TEST(cli_usage_points_to_tool_format_json_issue2102) { ASSERT_NOT_NULL(strstr(CBM_CLI_USAGE, "payload JSON")); ASSERT_NOT_NULL(strstr(CBM_CLI_USAGE, "full MCP envelope")); ASSERT_NOT_NULL(strstr(CBM_CLI_USAGE, "--json Print the raw MCP result envelope")); - ASSERT_NOT_NULL(strstr(CBM_CLI_USAGE, - "cli [--quiet] [--progress] [--verbose] [--json] ")); + ASSERT_NOT_NULL(strstr(CBM_CLI_USAGE, "cli [--quiet] [--progress] [--verbose] [--json] " + "")); ASSERT_NULL(strstr(CBM_CLI_USAGE, "[--format")); ASSERT_NULL(strstr(CBM_CLI_USAGE, "CBM_CLI_FORMAT")); PASS(); From f3bdf11a039b8ed8d685b04ddecedaac1e409167 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Deust?= Date: Tue, 25 Aug 2026 10:53:40 +0000 Subject: [PATCH 128/130] feat(pipeline): link markdown file references into the graph MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Markdown docs reference other repo files constantly (a coding-standards doc links the modules it governs, a README points at entry points) but none of that surfaced as edges, so fan-in queries were blind to documentation hubs. In an A/B retrieval benchmark I ran in August on a docs-heavy repo, the graph's top fan-in answer was off by roughly 17x: the most-referenced file was a standards doc with 173 references and zero inbound edges. New pre-dump pass, pass_doclinks.c, modeled on pass_configlink.c: three strategies emit REFERENCES_FILE edges between existing File nodes only (unresolvable targets are dropped, the pass never invents nodes): inline links, backtick paths, bare path mentions. Targets resolve against the referencing file's directory and the repo root; repeated references collapse into one edge carrying strategy, confidence and count. Registered in the pre-dump sequence after configlink and in the incremental post-passes; REFERENCES_FILE added to the skill's edge-type list and the structural/language contract tests. Tests mirror test_configlink.c (real files in a tmpdir, File nodes in a gbuf, run the pass, assert edges): inline link, backtick, bare mention, http/anchor ignored, anchor-suffixed file link, dedupe with count, relative-vs-root resolution, unresolvable-target guard, NULL repo_path skip. A companion change adds the same linking for shell files (source lines and script invocations); split out to keep each change reviewable. Validation: scripts/build.sh clean; focused serial runner (doclinks, configlink, pipeline, edge_structural, lang_contract) 340 passed under ASan/UBSan; cppcheck clean on the new files with the repo's flags. Signed-off-by: Clément Deust Co-Authored-By: Claude Fable 5 --- Makefile.cbm | 3 +- src/cli/cli.c | 4 +- src/pipeline/pass_doclinks.c | 495 ++++++++++++++++++++++++++++ src/pipeline/pipeline.c | 4 + src/pipeline/pipeline_incremental.c | 8 + src/pipeline/pipeline_internal.h | 3 + tests/test_doclinks.c | 312 ++++++++++++++++++ tests/test_edge_structural.c | 1 + tests/test_lang_contract.c | 1 + tests/test_main.c | 4 + 10 files changed, 832 insertions(+), 3 deletions(-) create mode 100644 src/pipeline/pass_doclinks.c create mode 100644 tests/test_doclinks.c diff --git a/Makefile.cbm b/Makefile.cbm index 305cb6937..7a3cd31e1 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -449,6 +449,7 @@ PIPELINE_SRCS = \ src/pipeline/pass_gitdiff.c \ src/pipeline/pass_configures.c \ src/pipeline/pass_configlink.c \ + src/pipeline/pass_doclinks.c \ src/pipeline/pass_route_nodes.c \ src/pipeline/pass_enrichment.c \ src/pipeline/pass_envscan.c \ @@ -675,7 +676,7 @@ TEST_DISCOVER_SRCS = \ TEST_GRAPH_BUFFER_SRCS = tests/test_graph_buffer.c -TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c +TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c TEST_WATCHER_SRCS = tests/test_watcher.c diff --git a/src/cli/cli.c b/src/cli/cli.c index b59a894a6..b868fdca2 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -1513,8 +1513,8 @@ static const char skill_content[] = "\n" "## Edge Types\n" "CALLS, HTTP_CALLS, ASYNC_CALLS, DATA_FLOWS, IMPORTS, DEFINES, DEFINES_METHOD,\n" - "HANDLES, IMPLEMENTS, OVERRIDE, USAGE, CALL_REFERENCE, CONFIGURES, FILE_CHANGES_WITH,\n" - "SIMILAR_TO, SEMANTICALLY_RELATED, CONTAINS_FILE, CONTAINS_FOLDER,\n" + "HANDLES, IMPLEMENTS, OVERRIDE, USAGE, CALL_REFERENCE, CONFIGURES, REFERENCES_FILE,\n" + "FILE_CHANGES_WITH, SIMILAR_TO, SEMANTICALLY_RELATED, CONTAINS_FILE, CONTAINS_FOLDER,\n" "CONTAINS_PACKAGE\n" "\n" "## Cypher Examples (for query_graph)\n" diff --git a/src/pipeline/pass_doclinks.c b/src/pipeline/pass_doclinks.c new file mode 100644 index 000000000..d27a22ce0 --- /dev/null +++ b/src/pipeline/pass_doclinks.c @@ -0,0 +1,495 @@ +/* + * pass_doclinks.c — Documentation → file reference linking (pre-dump pass). + * + * Markdown docs reference other repo files constantly — a coding-standards + * doc links to the modules it governs, a README points at entry points — + * but none of that surfaced as graph edges, so fan-in queries were blind to + * documentation hubs: on a docs-heavy repo the top fan-in answer was off by + * an order of magnitude because the most-referenced doc had zero inbound + * edges. + * + * Three strategies emit REFERENCES_FILE edges between EXISTING File nodes + * (targets that don't resolve to an indexed file are dropped — the pass + * never invents nodes): + * MD 1. Inline link: [text](relative/path.ext) (not http/mailto/#anchor) + * MD 2. Backtick path: `path/with/slash.ext` or `file.ext` + * MD 3. Bare mention: relative/path.ext (slash + extension) + * + * Targets resolve relative to the referencing file's directory AND the repo + * root (docs are written both ways). Repeated references between the same + * file pair are collapsed to one edge carrying a "count" property; the edge + * keeps the highest-confidence strategy that matched. + * + * Operates on the graph buffer before dump to .db file. + */ +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" +#include "graph_buffer/graph_buffer.h" +#include "foundation/constants.h" +#include "foundation/hash_table.h" +#include "foundation/log.h" +#include "foundation/compat_fs.h" +#include "foundation/limits.h" + +#include +#include +#include +#include +#include +#include + +#define SLEN(s) (sizeof(s) - SKIP_ONE) + +/* ── Doc link confidence scores ──────────────────────────────────── */ +/* Markdown strategies */ +#define DOCLINK_MD_INLINE 0.95 +#define DOCLINK_MD_BACKTICK 0.85 +#define DOCLINK_MD_BARE 0.70 + +/* Edge type emitted by this pass. */ +#define DOCLINK_EDGE_TYPE "REFERENCES_FILE" + +enum { + DOCLINK_MAX_REFS = CBM_SZ_256, /* distinct targets per referencing file */ + DOCLINK_MAX_SEGS = CBM_SZ_64, /* path segments during normalization */ +}; + +/* ── Path classification ─────────────────────────────────────────── */ + +/* Extension of the path's basename (including the dot), or NULL. */ +static const char *doclink_path_ext(const char *path) { + if (!path) { + return NULL; + } + const char *base = strrchr(path, '/'); + base = base ? base + SKIP_ONE : path; + return strrchr(base, '.'); +} + +static bool doclink_is_markdown_path(const char *path) { + const char *ext = doclink_path_ext(path); + return ext && (strcmp(ext, ".md") == 0 || strcmp(ext, ".mdx") == 0); +} + +/* ── File reading (mirrors pass_semantic.c read_file, minus TS pad) ── */ + +static char *doclink_read_file(const char *path) { + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + return NULL; + } + (void)fseek(f, 0, SEEK_END); + long size = ftell(f); + (void)fseek(f, 0, SEEK_SET); + if (size <= 0 || size > cbm_max_file_bytes()) { + (void)fclose(f); + return NULL; + } + char *buf = malloc((size_t)size + SKIP_ONE); + if (!buf) { + (void)fclose(f); + return NULL; + } + size_t nread = fread(buf, SKIP_ONE, (size_t)size, f); + (void)fclose(f); + if (nread > (size_t)size) { + nread = (size_t)size; + } + buf[nread] = '\0'; + return buf; +} + +/* ── Path normalization + resolution ─────────────────────────────── */ + +/* Normalize "a/./b/../c" into "a/c". Rejects paths that escape the repo + * root (leading ".."), empty results, and over-long/over-deep inputs. */ +static bool doclink_normalize(const char *in, char *out, size_t out_sz) { + size_t seg_starts[DOCLINK_MAX_SEGS]; + int depth = 0; + size_t out_len = 0; + const char *p = in; + out[0] = '\0'; + while (*p) { + const char *seg = p; + const char *slash = strchr(p, '/'); + size_t seg_len = slash ? (size_t)(slash - p) : strlen(p); + p = slash ? slash + SKIP_ONE : p + seg_len; + if (seg_len == 0 || (seg_len == SKIP_ONE && seg[0] == '.')) { + continue; + } + if (seg_len == PAIR_LEN && seg[0] == '.' && seg[SKIP_ONE] == '.') { + if (depth == 0) { + return false; /* escapes the repo root */ + } + depth--; + out_len = seg_starts[depth]; + out[out_len] = '\0'; + continue; + } + if (depth >= DOCLINK_MAX_SEGS || out_len + seg_len + PAIR_LEN >= out_sz) { + return false; + } + seg_starts[depth++] = out_len; + if (out_len > 0) { + out[out_len++] = '/'; + } + memcpy(out + out_len, seg, seg_len); + out_len += seg_len; + out[out_len] = '\0'; + } + return out_len > 0; +} + +/* Per-file accumulator: dedupes repeated references to the same target. */ +typedef struct { + int64_t target_id; + int count; + double confidence; + const char *strategy; /* static string literal */ +} doclink_ref_t; + +typedef struct { + cbm_gbuf_t *gb; + CBMHashTable *files_by_path; /* rel_path → cbm_gbuf_node_t* (borrowed) */ + const cbm_gbuf_node_t *src; /* referencing File node */ + char src_dir[CBM_SZ_512]; /* its directory ("" at repo root) */ + doclink_ref_t refs[DOCLINK_MAX_REFS]; + int ref_count; + bool truncated; +} doclink_ctx_t; + +/* Resolve a reference against the referencing file's directory, then the + * repo root, mirroring how humans write doc links. Returns the already- + * indexed File node or NULL — unresolvable references are dropped. */ +static const cbm_gbuf_node_t *doclink_resolve(doclink_ctx_t *dc, const char *ref) { + const char *r = ref; + while (r[0] == '.' && r[SKIP_ONE] == '/') { + r += PAIR_LEN; + } + if (r[0] == '/') { + r++; /* "/docs/x.md" is repo-root-relative by doc convention */ + } + if (r[0] == '\0') { + return NULL; + } + + char norm[CBM_SZ_512]; + if (dc->src_dir[0] != '\0') { + char joined[CBM_SZ_512]; + int n = snprintf(joined, sizeof(joined), "%s/%s", dc->src_dir, r); + if (n > 0 && (size_t)n < sizeof(joined) && doclink_normalize(joined, norm, sizeof(norm))) { + const cbm_gbuf_node_t *node = cbm_ht_get(dc->files_by_path, norm); + if (node) { + return node; + } + } + } + if (doclink_normalize(r, norm, sizeof(norm))) { + return cbm_ht_get(dc->files_by_path, norm); + } + return NULL; +} + +/* Record one match. Same-pair repeats bump the count; a higher-confidence + * strategy upgrades the edge's confidence + strategy label. */ +static void doclink_record(doclink_ctx_t *dc, const cbm_gbuf_node_t *target, double confidence, + const char *strategy) { + if (!target || target->id == dc->src->id) { + return; /* never self-reference */ + } + for (int i = 0; i < dc->ref_count; i++) { + if (dc->refs[i].target_id == target->id) { + dc->refs[i].count++; + if (confidence > dc->refs[i].confidence) { + dc->refs[i].confidence = confidence; + dc->refs[i].strategy = strategy; + } + return; + } + } + if (dc->ref_count >= DOCLINK_MAX_REFS) { + dc->truncated = true; + return; + } + dc->refs[dc->ref_count].target_id = target->id; + dc->refs[dc->ref_count].count = SKIP_ONE; + dc->refs[dc->ref_count].confidence = confidence; + dc->refs[dc->ref_count].strategy = strategy; + dc->ref_count++; +} + +/* Emit accumulated references as REFERENCES_FILE edges. Returns edge count. */ +static int doclink_flush(doclink_ctx_t *dc) { + int emitted = 0; + for (int i = 0; i < dc->ref_count; i++) { + char props[CBM_SZ_256]; + (void)snprintf(props, sizeof(props), + "{\"strategy\":\"%s\",\"confidence\":%.2f,\"count\":%d}", + dc->refs[i].strategy, dc->refs[i].confidence, dc->refs[i].count); + if (cbm_gbuf_insert_edge(dc->gb, dc->src->id, dc->refs[i].target_id, DOCLINK_EDGE_TYPE, + props) > 0) { + emitted++; + } + } + if (dc->truncated) { + char cap_buf[CBM_SZ_16]; + (void)snprintf(cap_buf, sizeof(cap_buf), "%d", DOCLINK_MAX_REFS); + cbm_log_info("doclinks.truncated", "file", dc->src->file_path ? dc->src->file_path : "", + "cap", cap_buf); + } + dc->ref_count = 0; + dc->truncated = false; + return emitted; +} + +/* ── Markdown scanning ───────────────────────────────────────────── */ + +/* Characters allowed in a path-shaped token (backtick / bare mention). */ +static bool doclink_token_pathlike(const char *tok) { + bool last_seg_has_dot = false; + for (const char *p = tok; *p; p++) { + unsigned char c = (unsigned char)*p; + if (c == '/') { + last_seg_has_dot = false; + continue; + } + if (c == '.') { + last_seg_has_dot = true; + continue; + } + if (!isalnum(c) && c != '_' && c != '-' && c != '+' && c != '@' && c != '~') { + return false; + } + } + /* the basename must carry an extension — bare words are not paths */ + return last_seg_has_dot; +} + +/* A markdown link target worth resolving: not a URL, mailto, or pure anchor. */ +static bool doclink_md_target_ok(const char *target) { + if (target[0] == '\0' || target[0] == '#') { + return false; + } + if (strstr(target, "://") != NULL || strncmp(target, "mailto:", SLEN("mailto:")) == 0) { + return false; + } + return true; +} + +/* MD 1: inline links [text](target). Consumed spans are blanked so the + * backtick / bare-mention scans below cannot re-match the same path. */ +static void doclink_scan_md_links(doclink_ctx_t *dc, char *line) { + char *p = line; + while ((p = strstr(p, "](")) != NULL) { + char *close = strchr(p + PAIR_LEN, ')'); + if (!close) { + return; + } + char target[CBM_SZ_512]; + size_t tlen = (size_t)(close - (p + PAIR_LEN)); + if (tlen < sizeof(target)) { + memcpy(target, p + PAIR_LEN, tlen); + target[tlen] = '\0'; + char *cut = strchr(target, ' '); /* [t](path "title") */ + if (cut) { + *cut = '\0'; + } + cut = strchr(target, '#'); /* [t](path#anchor) */ + if (cut) { + *cut = '\0'; + } + if (doclink_md_target_ok(target)) { + doclink_record(dc, doclink_resolve(dc, target), DOCLINK_MD_INLINE, + "md_inline_link"); + } + } + /* blank the whole [text](target) span, link text included, so a + * path-shaped link text is not re-counted as a bare mention */ + char *open = p; + while (open > line && *open != '[') { + open--; + } + if (*open != '[') { + open = p; + } + memset(open, ' ', (size_t)(close - open) + SKIP_ONE); + p = close + SKIP_ONE; + } +} + +/* MD 2: backtick-quoted paths `src/foo.c` / `build.sh`. */ +static void doclink_scan_md_backticks(doclink_ctx_t *dc, char *line) { + char *p = line; + while ((p = strchr(p, '`')) != NULL) { + char *end = strchr(p + SKIP_ONE, '`'); + if (!end) { + return; + } + char tok[CBM_SZ_512]; + size_t tlen = (size_t)(end - (p + SKIP_ONE)); + if (tlen > 0 && tlen < sizeof(tok)) { + memcpy(tok, p + SKIP_ONE, tlen); + tok[tlen] = '\0'; + if (doclink_token_pathlike(tok)) { + doclink_record(dc, doclink_resolve(dc, tok), DOCLINK_MD_BACKTICK, + "md_backtick_path"); + } + } + memset(p, ' ', (size_t)(end - p) + SKIP_ONE); + p = end + SKIP_ONE; + } +} + +static bool doclink_md_delim(char c) { + return isspace((unsigned char)c) || strchr("()[]{}<>\"',;:`*|", c) != NULL; +} + +/* MD 3: bare relative path mentions — must contain a slash AND an extension + * (and, via doclink_resolve, an indexed file) to count. */ +static void doclink_scan_md_bare(doclink_ctx_t *dc, const char *line) { + const char *p = line; + while (*p) { + while (*p && doclink_md_delim(*p)) { + p++; + } + const char *start = p; + while (*p && !doclink_md_delim(*p)) { + p++; + } + size_t tlen = (size_t)(p - start); + char tok[CBM_SZ_512]; + if (tlen == 0 || tlen >= sizeof(tok)) { + continue; + } + memcpy(tok, start, tlen); + tok[tlen] = '\0'; + while (tlen > 0 && tok[tlen - SKIP_ONE] == '.') { + tok[--tlen] = '\0'; /* sentence-ending period */ + } + if (strchr(tok, '/') != NULL && doclink_token_pathlike(tok)) { + doclink_record(dc, doclink_resolve(dc, tok), DOCLINK_MD_BARE, "md_bare_mention"); + } + } +} + +static void doclink_scan_md_line(doclink_ctx_t *dc, char *line) { + doclink_scan_md_links(dc, line); + doclink_scan_md_backticks(dc, line); + doclink_scan_md_bare(dc, line); +} + +/* ── Per-file driver ─────────────────────────────────────────────── */ + +/* Scan one referencing file's content line by line and emit its edges. */ +static int doclink_scan_file(doclink_ctx_t *dc, const cbm_gbuf_node_t *node, const char *source) { + dc->src = node; + dc->ref_count = 0; + dc->truncated = false; + dc->src_dir[0] = '\0'; + const char *slash = strrchr(node->file_path, '/'); + if (slash) { + size_t dlen = (size_t)(slash - node->file_path); + if (dlen >= sizeof(dc->src_dir)) { + return 0; + } + memcpy(dc->src_dir, node->file_path, dlen); + dc->src_dir[dlen] = '\0'; + } + + const char *p = source; + char line[CBM_SZ_4K]; + while (*p) { + const char *eol = strchr(p, '\n'); + size_t line_len = eol ? (size_t)(eol - p) : strlen(p); + if (line_len >= sizeof(line)) { + line_len = sizeof(line) - SKIP_ONE; + } + memcpy(line, p, line_len); + line[line_len] = '\0'; + p = eol ? eol + SKIP_ONE : p + line_len; + + doclink_scan_md_line(dc, line); + } + return doclink_flush(dc); +} + +/* ── Pass entry point ────────────────────────────────────────────── */ + +/* True when at least one File node is a markdown file. */ +static bool doclink_has_doc_files(const cbm_gbuf_node_t *const *files, int file_count) { + for (int i = 0; i < file_count; i++) { + if (doclink_is_markdown_path(files[i]->file_path)) { + return true; + } + } + return false; +} + +/* Scan every markdown File node's on-disk content, emitting edges. + * md_edges receives the emitted edge count. */ +static void doclink_scan_repo(doclink_ctx_t *dc, const char *repo_path, + const cbm_gbuf_node_t *const *files, int file_count, int *md_edges) { + for (int i = 0; i < file_count; i++) { + if (!files[i]->file_path || !doclink_is_markdown_path(files[i]->file_path)) { + continue; + } + + char abs_path[CBM_PATH_MAX]; + int n = snprintf(abs_path, sizeof(abs_path), "%s/%s", repo_path, files[i]->file_path); + if (n <= 0 || (size_t)n >= sizeof(abs_path)) { + continue; + } + char *source = doclink_read_file(abs_path); + if (!source) { + continue; + } + int emitted = doclink_scan_file(dc, files[i], source); + free(source); + *md_edges += emitted; + } +} + +int cbm_pipeline_pass_doclinks(cbm_pipeline_ctx_t *ctx) { + cbm_gbuf_t *gb = ctx->gbuf; + + const cbm_gbuf_node_t **files = NULL; + int file_count = 0; + if (cbm_gbuf_find_by_label(gb, "File", &files, &file_count) != 0 || file_count == 0) { + return 0; + } + + /* Early exit: no markdown/shell files means nothing to scan. */ + if (!doclink_has_doc_files(files, file_count)) { + cbm_log_info("doclinks.skip", "reason", "no_doc_files"); + return 0; + } + if (!ctx->repo_path) { + cbm_log_info("doclinks.skip", "reason", "no_repo_path"); + return 0; + } + + doclink_ctx_t dc; + memset(&dc, 0, sizeof(dc)); + dc.gb = gb; + dc.files_by_path = cbm_ht_create((uint32_t)file_count); + if (!dc.files_by_path) { + return 0; + } + for (int i = 0; i < file_count; i++) { + if (files[i]->file_path) { + /* key borrowed from the node (owned by gbuf, outlives the pass) */ + cbm_ht_set(dc.files_by_path, files[i]->file_path, (void *)files[i]); + } + } + + int md_edges = 0; + doclink_scan_repo(&dc, ctx->repo_path, files, file_count, &md_edges); + cbm_ht_free(dc.files_by_path); + + char buf1[CBM_SZ_16]; + (void)snprintf(buf1, sizeof(buf1), "%d", md_edges); + cbm_log_info("doclinks.strategy", "name", "markdown", "edges", buf1); + cbm_log_info("doclinks.done", "total", buf1); + + return md_edges; +} diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 6bc0310dd..fa62441a1 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -1054,6 +1054,9 @@ static void predump_sem(cbm_pipeline_ctx_t *ctx) { static void predump_cfg(cbm_pipeline_ctx_t *ctx) { cbm_pipeline_pass_configlink(ctx); } +static void predump_doclinks(cbm_pipeline_ctx_t *ctx) { + cbm_pipeline_pass_doclinks(ctx); +} static void predump_complexity(cbm_pipeline_ctx_t *ctx) { cbm_pipeline_pass_complexity(ctx); } @@ -1288,6 +1291,7 @@ static void run_predump_passes(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) { } passes[] = { {predump_deco, "decorator_tags", false}, {predump_cfg, "configlink", false}, + {predump_doclinks, "doclinks", false}, {predump_route, "route_match", false}, {predump_ensemble, "ensemble_routing", false}, {predump_sim, "similarity", true}, diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 894b73257..154518658 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1446,6 +1446,14 @@ static int run_postpasses(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_file return rc < 0 ? rc : CBM_NOT_FOUND; } + cbm_clock_gettime(CLOCK_MONOTONIC, &t); + rc = cbm_pipeline_pass_doclinks(ctx); + cbm_log_info("pass.timing", "pass", "incr_doclinks", "elapsed_ms", + itoa_buf((int)elapsed_ms(t))); + if (rc < 0 || cbm_pipeline_check_cancel(ctx)) { + return rc < 0 ? rc : CBM_NOT_FOUND; + } + /* SIMILAR_TO + SEMANTICALLY_RELATED edges only in moderate/full modes */ if (ctx->mode <= CBM_MODE_MODERATE) { cbm_clock_gettime(CLOCK_MONOTONIC, &t); diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 78ce486a0..275094731 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -665,6 +665,9 @@ int cbm_pipeline_pass_decorator_tags(cbm_gbuf_t *gbuf, const char *project); /* Pre-dump pass: config ↔ code linking. */ int cbm_pipeline_pass_configlink(cbm_pipeline_ctx_t *ctx); +/* Pre-dump pass: markdown/shell → file REFERENCES_FILE linking. */ +int cbm_pipeline_pass_doclinks(cbm_pipeline_ctx_t *ctx); + /* Pre-dump pass: SIMILAR_TO edges via MinHash fingerprinting. */ int cbm_pipeline_pass_similarity(cbm_pipeline_ctx_t *ctx); diff --git a/tests/test_doclinks.c b/tests/test_doclinks.c new file mode 100644 index 000000000..a03542d9b --- /dev/null +++ b/tests/test_doclinks.c @@ -0,0 +1,312 @@ +/* + * test_doclinks.c — Tests for markdown/shell → file reference linking. + * + * Unit-test approach mirrors test_configlink.c: create real files in a + * tmpdir (the pass reads content from disk), set up File nodes in a gbuf, + * run the pass, check REFERENCES_FILE edges. + */ +#include "../src/foundation/compat.h" +#include "test_framework.h" +#include "test_helpers.h" +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" +#include "graph_buffer/graph_buffer.h" + +#include +#include +#include +#include + +/* ── Helpers ─────────────────────────────────────────────────────── */ + +/* Run the pass with a minimal ctx (same shape as run_configlink). */ +static int run_doclinks(cbm_gbuf_t *gb, const char *project, const char *repo_path) { + atomic_int cancelled; + atomic_init(&cancelled, 0); + cbm_pipeline_ctx_t ctx = { + .project_name = project, + .repo_path = repo_path, + .gbuf = gb, + .cancelled = &cancelled, + }; + return cbm_pipeline_pass_doclinks(&ctx); +} + +/* Create a File node the way pass_structure does: QN via fqn_compute with + * "__file__", file_path = repo-relative path. Returns the node id. */ +static int64_t add_file_node(cbm_gbuf_t *gb, const char *project, const char *rel) { + char *qn = cbm_pipeline_fqn_compute(project, rel, "__file__"); + const char *slash = strrchr(rel, '/'); + const char *basename = slash ? slash + 1 : rel; + int64_t id = cbm_gbuf_upsert_node(gb, "File", basename, qn, rel, 0, 0, NULL); + free(qn); + return id; +} + +/* Find the REFERENCES_FILE edge between two node ids. NULL if absent. */ +static const cbm_gbuf_edge_t *find_ref_edge(cbm_gbuf_t *gb, int64_t src, int64_t tgt) { + const cbm_gbuf_edge_t **edges = NULL; + int count = 0; + cbm_gbuf_find_edges_by_type(gb, "REFERENCES_FILE", &edges, &count); + for (int i = 0; i < count; i++) { + if (edges[i]->source_id == src && edges[i]->target_id == tgt) { + return edges[i]; + } + } + return NULL; +} + +/* True when the edge's props JSON carries the given strategy. */ +static bool edge_has_strategy(const cbm_gbuf_edge_t *e, const char *strategy) { + char needle[64]; + snprintf(needle, sizeof(needle), "\"strategy\":\"%s\"", strategy); + return e && e->properties_json && strstr(e->properties_json, needle) != NULL; +} + +/* Total REFERENCES_FILE edge count. */ +static int ref_edge_count(cbm_gbuf_t *gb) { + return cbm_gbuf_edge_count_by_type(gb, "REFERENCES_FILE"); +} + +/* Fixture: tmpdir + project + gbuf. */ +typedef struct { + char tmpdir[256]; + char *project; + cbm_gbuf_t *gb; +} dl_fix_t; + +static bool dl_fix_init(dl_fix_t *fx) { + snprintf(fx->tmpdir, sizeof(fx->tmpdir), "/tmp/cbm_doclinks_XXXXXX"); + if (!cbm_mkdtemp(fx->tmpdir)) { + return false; + } + fx->project = cbm_project_name_from_path(fx->tmpdir); + fx->gb = cbm_gbuf_new(fx->project, fx->tmpdir); + return fx->gb != NULL; +} + +static void dl_fix_free(dl_fix_t *fx) { + cbm_gbuf_free(fx->gb); + free(fx->project); + th_rmtree(fx->tmpdir); +} + +/* ── Markdown: inline link ───────────────────────────────────────── */ + +TEST(doclinks_md_inline_link) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + th_write_file(TH_PATH(fx.tmpdir, "docs/guide.md"), + "# Guide\n\nSee [the build script](../scripts/build.sh) for details.\n"); + th_write_file(TH_PATH(fx.tmpdir, "scripts/build.sh"), "#!/bin/sh\necho build\n"); + + int64_t doc_id = add_file_node(fx.gb, fx.project, "docs/guide.md"); + int64_t script_id = add_file_node(fx.gb, fx.project, "scripts/build.sh"); + + int n = run_doclinks(fx.gb, fx.project, fx.tmpdir); + ASSERT_GT(n, 0); + + const cbm_gbuf_edge_t *e = find_ref_edge(fx.gb, doc_id, script_id); + ASSERT_NOT_NULL(e); + ASSERT_TRUE(edge_has_strategy(e, "md_inline_link")); + ASSERT_NOT_NULL(strstr(e->properties_json, "\"confidence\":0.95")); + + dl_fix_free(&fx); + PASS(); +} + +/* ── Markdown: backtick path ─────────────────────────────────────── */ + +TEST(doclinks_md_backtick_path) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + th_write_file(TH_PATH(fx.tmpdir, "README.md"), + "Run `scripts/build.sh` before pushing. `not_a_file.xyz` is unknown.\n"); + th_write_file(TH_PATH(fx.tmpdir, "scripts/build.sh"), "#!/bin/sh\n"); + + int64_t doc_id = add_file_node(fx.gb, fx.project, "README.md"); + int64_t script_id = add_file_node(fx.gb, fx.project, "scripts/build.sh"); + + run_doclinks(fx.gb, fx.project, fx.tmpdir); + + const cbm_gbuf_edge_t *e = find_ref_edge(fx.gb, doc_id, script_id); + ASSERT_NOT_NULL(e); + ASSERT_TRUE(edge_has_strategy(e, "md_backtick_path")); + /* `not_a_file.xyz` has no File node → no invented target */ + ASSERT_EQ(ref_edge_count(fx.gb), 1); + + dl_fix_free(&fx); + PASS(); +} + +/* ── Markdown: bare mention ──────────────────────────────────────── */ + +TEST(doclinks_md_bare_mention) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + th_write_file(TH_PATH(fx.tmpdir, "STANDARDS.md"), + "All handlers live in src/handlers.c and follow the pattern there.\n"); + th_write_file(TH_PATH(fx.tmpdir, "src/handlers.c"), "int h(void) { return 0; }\n"); + + int64_t doc_id = add_file_node(fx.gb, fx.project, "STANDARDS.md"); + int64_t code_id = add_file_node(fx.gb, fx.project, "src/handlers.c"); + + run_doclinks(fx.gb, fx.project, fx.tmpdir); + + const cbm_gbuf_edge_t *e = find_ref_edge(fx.gb, doc_id, code_id); + ASSERT_NOT_NULL(e); + ASSERT_TRUE(edge_has_strategy(e, "md_bare_mention")); + ASSERT_NOT_NULL(strstr(e->properties_json, "\"confidence\":0.70")); + + dl_fix_free(&fx); + PASS(); +} + +/* ── Markdown: non-file links ignored (http / mailto / #anchor) ──── */ + +TEST(doclinks_md_non_file_link_ignored) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + th_write_file(TH_PATH(fx.tmpdir, "README.md"), + "See [the site](https://example.com/scripts/build.sh) or\n" + "[mail us](mailto:dev@example.com) or [below](#usage).\n"); + th_write_file(TH_PATH(fx.tmpdir, "scripts/build.sh"), "#!/bin/sh\n"); + + add_file_node(fx.gb, fx.project, "README.md"); + add_file_node(fx.gb, fx.project, "scripts/build.sh"); + + int n = run_doclinks(fx.gb, fx.project, fx.tmpdir); + ASSERT_EQ(n, 0); + ASSERT_EQ(ref_edge_count(fx.gb), 0); + + dl_fix_free(&fx); + PASS(); +} + +/* ── Markdown: anchor suffix on a file link still resolves ───────── */ + +TEST(doclinks_md_link_with_anchor_resolves_file) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + th_write_file(TH_PATH(fx.tmpdir, "docs/a.md"), "See [setup](../INSTALL.md#quick-start).\n"); + th_write_file(TH_PATH(fx.tmpdir, "INSTALL.md"), "# Install\n"); + + int64_t doc_id = add_file_node(fx.gb, fx.project, "docs/a.md"); + int64_t tgt_id = add_file_node(fx.gb, fx.project, "INSTALL.md"); + + run_doclinks(fx.gb, fx.project, fx.tmpdir); + ASSERT_NOT_NULL(find_ref_edge(fx.gb, doc_id, tgt_id)); + + dl_fix_free(&fx); + PASS(); +} + +/* ── Dedupe: repeated references collapse to one counted edge ────── */ + +TEST(doclinks_dedupe_keeps_count) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + th_write_file(TH_PATH(fx.tmpdir, "README.md"), + "Use [build](scripts/build.sh) daily.\n" + "Run `scripts/build.sh` first, then scripts/build.sh again.\n"); + th_write_file(TH_PATH(fx.tmpdir, "scripts/build.sh"), "#!/bin/sh\n"); + + int64_t doc_id = add_file_node(fx.gb, fx.project, "README.md"); + int64_t script_id = add_file_node(fx.gb, fx.project, "scripts/build.sh"); + + int n = run_doclinks(fx.gb, fx.project, fx.tmpdir); + /* three matches, ONE edge */ + ASSERT_EQ(n, 1); + ASSERT_EQ(ref_edge_count(fx.gb), 1); + + const cbm_gbuf_edge_t *e = find_ref_edge(fx.gb, doc_id, script_id); + ASSERT_NOT_NULL(e); + ASSERT_NOT_NULL(strstr(e->properties_json, "\"count\":3")); + /* highest-confidence match kind wins the edge label */ + ASSERT_TRUE(edge_has_strategy(e, "md_inline_link")); + ASSERT_NOT_NULL(strstr(e->properties_json, "\"confidence\":0.95")); + + dl_fix_free(&fx); + PASS(); +} + +/* ── Path resolution: relative to the referencing file's directory ── */ + +TEST(doclinks_resolves_relative_to_file_dir) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + /* docs/deep/page.md links an upward "../../src/main.c" path and a + * sibling-relative "notes.md" path, neither repo-root-relative. */ + th_write_file(TH_PATH(fx.tmpdir, "docs/deep/page.md"), + "See [main](../../src/main.c) and [notes](notes.md).\n"); + th_write_file(TH_PATH(fx.tmpdir, "docs/deep/notes.md"), "# notes\n"); + th_write_file(TH_PATH(fx.tmpdir, "src/main.c"), "int main(void) { return 0; }\n"); + + int64_t page_id = add_file_node(fx.gb, fx.project, "docs/deep/page.md"); + int64_t notes_id = add_file_node(fx.gb, fx.project, "docs/deep/notes.md"); + int64_t main_id = add_file_node(fx.gb, fx.project, "src/main.c"); + + run_doclinks(fx.gb, fx.project, fx.tmpdir); + + ASSERT_NOT_NULL(find_ref_edge(fx.gb, page_id, main_id)); + ASSERT_NOT_NULL(find_ref_edge(fx.gb, page_id, notes_id)); + + dl_fix_free(&fx); + PASS(); +} + +/* ── Never invent nodes: unresolvable targets produce nothing ────── */ + +TEST(doclinks_unresolvable_target_no_edge) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + th_write_file(TH_PATH(fx.tmpdir, "README.md"), + "See [gone](docs/removed.md) and `also/missing.sh`.\n"); + + add_file_node(fx.gb, fx.project, "README.md"); + + int node_count_before = cbm_gbuf_node_count(fx.gb); + int n = run_doclinks(fx.gb, fx.project, fx.tmpdir); + ASSERT_EQ(n, 0); + ASSERT_EQ(ref_edge_count(fx.gb), 0); + ASSERT_EQ(cbm_gbuf_node_count(fx.gb), node_count_before); + + dl_fix_free(&fx); + PASS(); +} + +/* ── NULL repo_path (configlink-style unit setups) is a clean skip ── */ + +TEST(doclinks_null_repo_path_skips) { + cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/test"); + add_file_node(gb, "test", "README.md"); + int n = run_doclinks(gb, "test", NULL); + ASSERT_EQ(n, 0); + cbm_gbuf_free(gb); + PASS(); +} + +/* ── Suite ───────────────────────────────────────────────────────── */ + +SUITE(doclinks) { + /* Markdown strategies */ + RUN_TEST(doclinks_md_inline_link); + RUN_TEST(doclinks_md_backtick_path); + RUN_TEST(doclinks_md_bare_mention); + RUN_TEST(doclinks_md_non_file_link_ignored); + RUN_TEST(doclinks_md_link_with_anchor_resolves_file); + + /* Dedupe + resolution + guards */ + RUN_TEST(doclinks_dedupe_keeps_count); + RUN_TEST(doclinks_resolves_relative_to_file_dir); + RUN_TEST(doclinks_unresolvable_target_no_edge); + RUN_TEST(doclinks_null_repo_path_skips); +} diff --git a/tests/test_edge_structural.c b/tests/test_edge_structural.c index 3f43628fa..798fef3d0 100644 --- a/tests/test_edge_structural.c +++ b/tests/test_edge_structural.c @@ -258,6 +258,7 @@ static const char *ES_ALL_EDGE_TYPES[] = {"CALLS", "INHERITS", "INFRA_MAPS", "OVERRIDE", + "REFERENCES_FILE", "SEMANTICALLY_RELATED", "SIMILAR_TO", "TESTS_FILE", diff --git a/tests/test_lang_contract.c b/tests/test_lang_contract.c index 5a32bcda0..6eb993564 100644 --- a/tests/test_lang_contract.c +++ b/tests/test_lang_contract.c @@ -1070,6 +1070,7 @@ static const char *ALL_EDGE_TYPES[] = {"CALLS", "INHERITS", "INFRA_MAPS", "OVERRIDE", + "REFERENCES_FILE", "SEMANTICALLY_RELATED", "SIMILAR_TO", "TESTS_FILE", diff --git a/tests/test_main.c b/tests/test_main.c index 31cceac60..1278c1c70 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -852,6 +852,7 @@ extern void suite_store_pragmas(void); extern void suite_store_checkpoint(void); extern void suite_traces(void); extern void suite_configlink(void); +extern void suite_doclinks(void); extern void suite_infrascan(void); extern void suite_cli(void); extern void suite_agent_clients(void); @@ -1181,6 +1182,9 @@ int main(int argc, char **argv) { /* Config link */ RUN_SELECTED_SUITE(configlink); + /* Doc/shell file reference link */ + RUN_SELECTED_SUITE(doclinks); + /* Infrastructure scanning */ RUN_SELECTED_SUITE(infrascan); From 6ae6cd4c3f280cb8e3ffec2858269055e390ffec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Deust?= Date: Tue, 22 Sep 2026 08:48:33 +0200 Subject: [PATCH 129/130] fix(pipeline): address doclinks review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace raw malloc/free with cbm_alloc/cbm_free (CBM_MEM_CLASS_EXTRACT) and a growable CBM_DYN_ARRAY for matches. Remove the 4KiB line-copy cap; lines are split in place instead of copied. Remove the DOCLINK_MAX_REFS(256) cap; the ref vector now grows, is sorted by target id, then collapsed. Track reference kind (bare, relative, rooted) so /x never attempts a directory join and ./x never falls back to root. Report per-file skips via cbm_pipeline_add_file_error and cbm_log_warn, and check cbm_pipeline_check_cancel in the per-file loop. Add negative-control tests for all of the above, plus CRLF handling, fenced-code-block pinning, bare URLs, backtick-basename exactness, and ".." escaping. Co-Authored-By: Claude Sonnet 5 Signed-off-by: Clément Deust --- src/pipeline/pass_doclinks.c | 231 +++++++++++++++++++---------- src/pipeline/pipeline_internal.h | 2 +- tests/test_doclinks.c | 246 ++++++++++++++++++++++++++++++- tests/test_main.c | 2 +- tests/test_pipeline.c | 31 ++++ 5 files changed, 432 insertions(+), 80 deletions(-) diff --git a/src/pipeline/pass_doclinks.c b/src/pipeline/pass_doclinks.c index d27a22ce0..4b75357e5 100644 --- a/src/pipeline/pass_doclinks.c +++ b/src/pipeline/pass_doclinks.c @@ -30,6 +30,8 @@ #include "foundation/log.h" #include "foundation/compat_fs.h" #include "foundation/limits.h" +#include "foundation/mem_core.h" +#include "foundation/dyn_array.h" #include #include @@ -50,8 +52,7 @@ #define DOCLINK_EDGE_TYPE "REFERENCES_FILE" enum { - DOCLINK_MAX_REFS = CBM_SZ_256, /* distinct targets per referencing file */ - DOCLINK_MAX_SEGS = CBM_SZ_64, /* path segments during normalization */ + DOCLINK_MAX_SEGS = CBM_SZ_64, /* path segments during normalization */ }; /* ── Path classification ─────────────────────────────────────────── */ @@ -71,23 +72,51 @@ static bool doclink_is_markdown_path(const char *path) { return ext && (strcmp(ext, ".md") == 0 || strcmp(ext, ".mdx") == 0); } -/* ── File reading (mirrors pass_semantic.c read_file, minus TS pad) ── */ - -static char *doclink_read_file(const char *path) { +/* ── File reading (mirrors pass_definitions.c read_file, minus TS pad) ── + * + * Reports *out_status so the caller can attribute a skip to the right + * reason (open failure vs. oversized vs. OOM) instead of a silent drop — + * hitting a limit here degrades to a REPORTED skip, per limits.h. */ +static char *doclink_read_file(const char *path, long *out_size, cbm_read_status_t *out_status) { + if (out_size) { + *out_size = 0; + } + if (out_status) { + *out_status = CBM_READ_OK; + } FILE *f = cbm_fopen(path, "rb"); if (!f) { + if (out_status) { + *out_status = CBM_READ_OPEN_FAIL; + } return NULL; } (void)fseek(f, 0, SEEK_END); long size = ftell(f); (void)fseek(f, 0, SEEK_SET); - if (size <= 0 || size > cbm_max_file_bytes()) { + if (out_size) { + *out_size = size; + } + if (size <= 0) { (void)fclose(f); + if (out_status) { + *out_status = CBM_READ_EMPTY; + } return NULL; } - char *buf = malloc((size_t)size + SKIP_ONE); + if (size > cbm_max_file_bytes()) { + (void)fclose(f); + if (out_status) { + *out_status = CBM_READ_OVERSIZED; + } + return NULL; + } + char *buf = cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)size + SKIP_ONE); if (!buf) { (void)fclose(f); + if (out_status) { + *out_status = CBM_READ_OOM; + } return NULL; } size_t nread = fread(buf, SKIP_ONE, (size_t)size, f); @@ -140,33 +169,45 @@ static bool doclink_normalize(const char *in, char *out, size_t out_sz) { return out_len > 0; } -/* Per-file accumulator: dedupes repeated references to the same target. */ +/* One raw match before dedup. Collected in a growable vector (no per-file + * cap) and collapsed by doclink_flush(). */ typedef struct { int64_t target_id; - int count; double confidence; const char *strategy; /* static string literal */ -} doclink_ref_t; +} doclink_match_t; typedef struct { cbm_gbuf_t *gb; CBMHashTable *files_by_path; /* rel_path → cbm_gbuf_node_t* (borrowed) */ const cbm_gbuf_node_t *src; /* referencing File node */ char src_dir[CBM_SZ_512]; /* its directory ("" at repo root) */ - doclink_ref_t refs[DOCLINK_MAX_REFS]; - int ref_count; - bool truncated; + CBM_DYN_ARRAY(doclink_match_t) matches; } doclink_ctx_t; -/* Resolve a reference against the referencing file's directory, then the - * repo root, mirroring how humans write doc links. Returns the already- - * indexed File node or NULL — unresolvable references are dropped. */ +/* How a reference was written, which decides where it may resolve. Mixing + * these up let a rooted "/x" still match a same-named file in the + * referencing directory, and let an explicit "./x" fall back to an + * unrelated same-named file at the repo root. */ +typedef enum { + DOCLINK_REF_BARE = 0, /* "x": referencing dir, then repo root */ + DOCLINK_REF_RELATIVE, /* "./x": referencing dir only */ + DOCLINK_REF_ROOTED, /* "/x": repo root only */ +} doclink_ref_kind_t; + +/* Resolve a reference against the referencing file's directory and/or the + * repo root, per its kind. Returns the already-indexed File node or NULL — + * unresolvable references are dropped. */ static const cbm_gbuf_node_t *doclink_resolve(doclink_ctx_t *dc, const char *ref) { const char *r = ref; - while (r[0] == '.' && r[SKIP_ONE] == '/') { - r += PAIR_LEN; - } - if (r[0] == '/') { + doclink_ref_kind_t kind = DOCLINK_REF_BARE; + if (r[0] == '.' && r[SKIP_ONE] == '/') { + kind = DOCLINK_REF_RELATIVE; + while (r[0] == '.' && r[SKIP_ONE] == '/') { + r += PAIR_LEN; + } + } else if (r[0] == '/') { + kind = DOCLINK_REF_ROOTED; r++; /* "/docs/x.md" is repo-root-relative by doc convention */ } if (r[0] == '\0') { @@ -174,7 +215,7 @@ static const cbm_gbuf_node_t *doclink_resolve(doclink_ctx_t *dc, const char *ref } char norm[CBM_SZ_512]; - if (dc->src_dir[0] != '\0') { + if (kind != DOCLINK_REF_ROOTED && dc->src_dir[0] != '\0') { char joined[CBM_SZ_512]; int n = snprintf(joined, sizeof(joined), "%s/%s", dc->src_dir, r); if (n > 0 && (size_t)n < sizeof(joined) && doclink_normalize(joined, norm, sizeof(norm))) { @@ -184,61 +225,67 @@ static const cbm_gbuf_node_t *doclink_resolve(doclink_ctx_t *dc, const char *ref } } } - if (doclink_normalize(r, norm, sizeof(norm))) { + if (kind != DOCLINK_REF_RELATIVE && doclink_normalize(r, norm, sizeof(norm))) { return cbm_ht_get(dc->files_by_path, norm); } return NULL; } -/* Record one match. Same-pair repeats bump the count; a higher-confidence - * strategy upgrades the edge's confidence + strategy label. */ +/* Record one match. No cap, no dedup here — dc->matches is a plain append + * log; doclink_flush() sorts and collapses same-target matches so a + * generated file with hundreds of distinct targets loses nothing. */ static void doclink_record(doclink_ctx_t *dc, const cbm_gbuf_node_t *target, double confidence, const char *strategy) { if (!target || target->id == dc->src->id) { return; /* never self-reference */ } - for (int i = 0; i < dc->ref_count; i++) { - if (dc->refs[i].target_id == target->id) { - dc->refs[i].count++; - if (confidence > dc->refs[i].confidence) { - dc->refs[i].confidence = confidence; - dc->refs[i].strategy = strategy; - } - return; - } - } - if (dc->ref_count >= DOCLINK_MAX_REFS) { - dc->truncated = true; - return; - } - dc->refs[dc->ref_count].target_id = target->id; - dc->refs[dc->ref_count].count = SKIP_ONE; - dc->refs[dc->ref_count].confidence = confidence; - dc->refs[dc->ref_count].strategy = strategy; - dc->ref_count++; + doclink_match_t m = {.target_id = target->id, .confidence = confidence, .strategy = strategy}; + cbm_da_push(&dc->matches, m); } -/* Emit accumulated references as REFERENCES_FILE edges. Returns edge count. */ +static int doclink_match_cmp(const void *a, const void *b) { + int64_t ta = ((const doclink_match_t *)a)->target_id; + int64_t tb = ((const doclink_match_t *)b)->target_id; + return (ta > tb) - (ta < tb); +} + +/* Emit accumulated references as REFERENCES_FILE edges. Sorts by target id + * (O(M log M)) and collapses same-target runs: count = occurrences, the + * highest-confidence match in the run wins strategy + confidence. Returns + * edge count. */ static int doclink_flush(doclink_ctx_t *dc) { + if (dc->matches.count > 1) { + qsort(dc->matches.items, (size_t)dc->matches.count, sizeof(dc->matches.items[0]), + doclink_match_cmp); + } + int emitted = 0; - for (int i = 0; i < dc->ref_count; i++) { + int i = 0; + while (i < dc->matches.count) { + int64_t target_id = dc->matches.items[i].target_id; + double confidence = dc->matches.items[i].confidence; + const char *strategy = dc->matches.items[i].strategy; + int count = SKIP_ONE; + int j = i + SKIP_ONE; + while (j < dc->matches.count && dc->matches.items[j].target_id == target_id) { + count++; + if (dc->matches.items[j].confidence > confidence) { + confidence = dc->matches.items[j].confidence; + strategy = dc->matches.items[j].strategy; + } + j++; + } + char props[CBM_SZ_256]; (void)snprintf(props, sizeof(props), - "{\"strategy\":\"%s\",\"confidence\":%.2f,\"count\":%d}", - dc->refs[i].strategy, dc->refs[i].confidence, dc->refs[i].count); - if (cbm_gbuf_insert_edge(dc->gb, dc->src->id, dc->refs[i].target_id, DOCLINK_EDGE_TYPE, - props) > 0) { + "{\"strategy\":\"%s\",\"confidence\":%.2f,\"count\":%d}", strategy, + confidence, count); + if (cbm_gbuf_insert_edge(dc->gb, dc->src->id, target_id, DOCLINK_EDGE_TYPE, props) > 0) { emitted++; } + i = j; } - if (dc->truncated) { - char cap_buf[CBM_SZ_16]; - (void)snprintf(cap_buf, sizeof(cap_buf), "%d", DOCLINK_MAX_REFS); - cbm_log_info("doclinks.truncated", "file", dc->src->file_path ? dc->src->file_path : "", - "cap", cap_buf); - } - dc->ref_count = 0; - dc->truncated = false; + cbm_da_clear(&dc->matches); return emitted; } @@ -380,11 +427,15 @@ static void doclink_scan_md_line(doclink_ctx_t *dc, char *line) { /* ── Per-file driver ─────────────────────────────────────────────── */ -/* Scan one referencing file's content line by line and emit its edges. */ -static int doclink_scan_file(doclink_ctx_t *dc, const cbm_gbuf_node_t *node, const char *source) { +/* Scan one referencing file's content line by line and emit its edges. + * `source` is the caller's private mutable buffer (freshly read for this + * file, never reused): lines are terminated in place ('\n' -> '\0') and + * scanned by pointer, so there is no fixed-size copy and no line-length + * cap to straddle a token across (a prior CBM_SZ_4K copy buffer could cut + * "src/foo.cpp" to "src/foo.c" at the boundary and bind to the wrong + * file). A trailing '\r' (CRLF) is trimmed the same way. */ +static int doclink_scan_file(doclink_ctx_t *dc, const cbm_gbuf_node_t *node, char *source) { dc->src = node; - dc->ref_count = 0; - dc->truncated = false; dc->src_dir[0] = '\0'; const char *slash = strrchr(node->file_path, '/'); if (slash) { @@ -396,18 +447,20 @@ static int doclink_scan_file(doclink_ctx_t *dc, const cbm_gbuf_node_t *node, con dc->src_dir[dlen] = '\0'; } - const char *p = source; - char line[CBM_SZ_4K]; + char *p = source; while (*p) { - const char *eol = strchr(p, '\n'); - size_t line_len = eol ? (size_t)(eol - p) : strlen(p); - if (line_len >= sizeof(line)) { - line_len = sizeof(line) - SKIP_ONE; + char *line = p; + char *eol = strchr(p, '\n'); + if (eol) { + *eol = '\0'; + p = eol + SKIP_ONE; + } else { + p += strlen(p); + } + size_t line_len = strlen(line); + if (line_len > 0 && line[line_len - SKIP_ONE] == '\r') { + line[line_len - SKIP_ONE] = '\0'; } - memcpy(line, p, line_len); - line[line_len] = '\0'; - p = eol ? eol + SKIP_ONE : p + line_len; - doclink_scan_md_line(dc, line); } return doclink_flush(dc); @@ -426,10 +479,16 @@ static bool doclink_has_doc_files(const cbm_gbuf_node_t *const *files, int file_ } /* Scan every markdown File node's on-disk content, emitting edges. - * md_edges receives the emitted edge count. */ -static void doclink_scan_repo(doclink_ctx_t *dc, const char *repo_path, + * md_edges receives the emitted edge count. Checks cancellation once per + * file — on a docs-heavy repo this is the whole pass's cancel latency — + * and every skip is reported via cbm_pipeline_add_file_error, never + * silent. */ +static void doclink_scan_repo(cbm_pipeline_ctx_t *ctx, doclink_ctx_t *dc, const char *repo_path, const cbm_gbuf_node_t *const *files, int file_count, int *md_edges) { for (int i = 0; i < file_count; i++) { + if (cbm_pipeline_check_cancel(ctx)) { + return; + } if (!files[i]->file_path || !doclink_is_markdown_path(files[i]->file_path)) { continue; } @@ -439,12 +498,29 @@ static void doclink_scan_repo(doclink_ctx_t *dc, const char *repo_path, if (n <= 0 || (size_t)n >= sizeof(abs_path)) { continue; } - char *source = doclink_read_file(abs_path); + long file_size = 0; + cbm_read_status_t rst = CBM_READ_OK; + char *source = doclink_read_file(abs_path, &file_size, &rst); if (!source) { + if (rst == CBM_READ_OVERSIZED) { + long cap = cbm_max_file_bytes(); + char reason[96]; + (void)snprintf(reason, sizeof(reason), "oversized (%lld MB > %lld MB)", + (long long)(file_size / (CBM_SZ_1K * CBM_SZ_1K)), + (long long)(cap / (CBM_SZ_1K * CBM_SZ_1K))); + cbm_pipeline_add_file_error(ctx->pipeline, files[i]->file_path, reason, + "oversized"); + cbm_log_warn("doclinks.file_oversized", "path", files[i]->file_path); + } else if (rst == CBM_READ_OPEN_FAIL || rst == CBM_READ_OOM) { + cbm_pipeline_add_file_error(ctx->pipeline, files[i]->file_path, "read failed", + "read"); + cbm_log_warn("doclinks.file_unreadable", "path", files[i]->file_path); + } + /* CBM_READ_EMPTY: benign 0-byte file, nothing to index, not reported. */ continue; } int emitted = doclink_scan_file(dc, files[i], source); - free(source); + cbm_free(CBM_MEM_CLASS_EXTRACT, source); *md_edges += emitted; } } @@ -458,7 +534,7 @@ int cbm_pipeline_pass_doclinks(cbm_pipeline_ctx_t *ctx) { return 0; } - /* Early exit: no markdown/shell files means nothing to scan. */ + /* Early exit: no markdown files means nothing to scan. */ if (!doclink_has_doc_files(files, file_count)) { cbm_log_info("doclinks.skip", "reason", "no_doc_files"); return 0; @@ -483,7 +559,8 @@ int cbm_pipeline_pass_doclinks(cbm_pipeline_ctx_t *ctx) { } int md_edges = 0; - doclink_scan_repo(&dc, ctx->repo_path, files, file_count, &md_edges); + doclink_scan_repo(ctx, &dc, ctx->repo_path, files, file_count, &md_edges); + cbm_da_free(&dc.matches); cbm_ht_free(dc.files_by_path); char buf1[CBM_SZ_16]; diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 275094731..e7782adef 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -665,7 +665,7 @@ int cbm_pipeline_pass_decorator_tags(cbm_gbuf_t *gbuf, const char *project); /* Pre-dump pass: config ↔ code linking. */ int cbm_pipeline_pass_configlink(cbm_pipeline_ctx_t *ctx); -/* Pre-dump pass: markdown/shell → file REFERENCES_FILE linking. */ +/* Pre-dump pass: markdown → file REFERENCES_FILE linking. */ int cbm_pipeline_pass_doclinks(cbm_pipeline_ctx_t *ctx); /* Pre-dump pass: SIMILAR_TO edges via MinHash fingerprinting. */ diff --git a/tests/test_doclinks.c b/tests/test_doclinks.c index a03542d9b..af24c2447 100644 --- a/tests/test_doclinks.c +++ b/tests/test_doclinks.c @@ -1,5 +1,5 @@ /* - * test_doclinks.c — Tests for markdown/shell → file reference linking. + * test_doclinks.c — Tests for markdown → file reference linking. * * Unit-test approach mirrors test_configlink.c: create real files in a * tmpdir (the pass reads content from disk), set up File nodes in a gbuf, @@ -294,6 +294,237 @@ TEST(doclinks_null_repo_path_skips) { PASS(); } +/* ── Rooted "/x" resolves against the repo root ONLY ─────────────── */ + +TEST(doclinks_rooted_reference_skips_directory_join) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + /* docs/guide.md links "/README.md" (repo-root-relative by doc + * convention). Both docs/README.md and the root README.md exist: a + * rooted reference must resolve to the root file, never to a + * same-named file in the referencing directory. */ + th_write_file(TH_PATH(fx.tmpdir, "docs/guide.md"), "See [home](/README.md).\n"); + th_write_file(TH_PATH(fx.tmpdir, "docs/README.md"), "# Wrong target\n"); + th_write_file(TH_PATH(fx.tmpdir, "README.md"), "# Root\n"); + + int64_t doc_id = add_file_node(fx.gb, fx.project, "docs/guide.md"); + int64_t docs_readme_id = add_file_node(fx.gb, fx.project, "docs/README.md"); + int64_t root_readme_id = add_file_node(fx.gb, fx.project, "README.md"); + + run_doclinks(fx.gb, fx.project, fx.tmpdir); + + ASSERT_NOT_NULL(find_ref_edge(fx.gb, doc_id, root_readme_id)); + ASSERT_NULL(find_ref_edge(fx.gb, doc_id, docs_readme_id)); + + dl_fix_free(&fx); + PASS(); +} + +/* ── Explicit "./x" resolves against the referencing dir ONLY ────── */ + +TEST(doclinks_explicit_relative_no_root_fallback) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + /* docs/a.md links "./config.md". docs/config.md does not exist, but a + * root config.md does — an explicit "./" reference must not fall back + * to an unrelated same-named file at the repo root. */ + th_write_file(TH_PATH(fx.tmpdir, "docs/a.md"), "See [cfg](./config.md).\n"); + th_write_file(TH_PATH(fx.tmpdir, "config.md"), "# Root config\n"); + + int64_t doc_id = add_file_node(fx.gb, fx.project, "docs/a.md"); + add_file_node(fx.gb, fx.project, "config.md"); + + int n = run_doclinks(fx.gb, fx.project, fx.tmpdir); + ASSERT_EQ(n, 0); + ASSERT_EQ(ref_edge_count(fx.gb), 0); + (void)doc_id; + + dl_fix_free(&fx); + PASS(); +} + +/* ── A line past the old 4 KiB copy cap is scanned in full ───────── */ + +TEST(doclinks_long_line_no_truncation) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + /* Padding + "src/foo.cpp" (no trailing newline) straddles the old + * CBM_SZ_4K line-copy cap so that, under the old bug, the copy cut + * exactly after "src/foo.c" and dropped "pp": a bare mention of + * src/foo.cpp silently became a reference to the wrong file. Both + * files exist so a wrong match is observable. */ + enum { FILLER_WORDS = 681 }; /* 681 * strlen("lorem ") == 4086 */ + char content[CBM_SZ_4K * 2]; + size_t pos = 0; + for (int i = 0; i < FILLER_WORDS; i++) { + memcpy(content + pos, "lorem ", 6); + pos += 6; + } + ASSERT_EQ((int)pos, 4086); + memcpy(content + pos, "src/foo.cpp", 11); + pos += 11; + content[pos] = '\0'; + + th_write_file(TH_PATH(fx.tmpdir, "README.md"), content); + th_write_file(TH_PATH(fx.tmpdir, "src/foo.c"), "// wrong target\n"); + th_write_file(TH_PATH(fx.tmpdir, "src/foo.cpp"), "// right target\n"); + + int64_t doc_id = add_file_node(fx.gb, fx.project, "README.md"); + int64_t short_id = add_file_node(fx.gb, fx.project, "src/foo.c"); + int64_t long_id = add_file_node(fx.gb, fx.project, "src/foo.cpp"); + + run_doclinks(fx.gb, fx.project, fx.tmpdir); + + ASSERT_NOT_NULL(find_ref_edge(fx.gb, doc_id, long_id)); + ASSERT_NULL(find_ref_edge(fx.gb, doc_id, short_id)); + + dl_fix_free(&fx); + PASS(); +} + +/* ── More than the old 256-target cap: nothing is dropped ────────── */ + +TEST(doclinks_many_distinct_targets_no_cap) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + enum { TARGETS = 300 }; /* > the old DOCLINK_MAX_REFS (256) */ + char content[TARGETS * 24]; + size_t pos = 0; + int64_t doc_id = add_file_node(fx.gb, fx.project, "MANY.md"); + for (int i = 0; i < TARGETS; i++) { + char rel[32]; + char line[40]; + snprintf(rel, sizeof(rel), "gen/f%03d.txt", i); + th_write_file(TH_PATH(fx.tmpdir, rel), "x\n"); + add_file_node(fx.gb, fx.project, rel); + int n = snprintf(line, sizeof(line), "%s\n", rel); + memcpy(content + pos, line, (size_t)n); + pos += (size_t)n; + } + content[pos] = '\0'; + th_write_file(TH_PATH(fx.tmpdir, "MANY.md"), content); + + int n = run_doclinks(fx.gb, fx.project, fx.tmpdir); + ASSERT_EQ(n, TARGETS); + ASSERT_EQ(ref_edge_count(fx.gb), TARGETS); + (void)doc_id; + + dl_fix_free(&fx); + PASS(); +} + +/* ── ".." past the repo root is dropped, not resolved ────────────── */ + +TEST(doclinks_dotdot_escaping_root_dropped) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + /* README.md sits at the repo root (depth 0): "../outside.md" has + * nowhere to go up from and must be dropped, not resolved. */ + th_write_file(TH_PATH(fx.tmpdir, "README.md"), "See [x](../outside.md).\n"); + + add_file_node(fx.gb, fx.project, "README.md"); + + int n = run_doclinks(fx.gb, fx.project, fx.tmpdir); + ASSERT_EQ(n, 0); + ASSERT_EQ(ref_edge_count(fx.gb), 0); + + dl_fix_free(&fx); + PASS(); +} + +/* ── A bare URL in prose is not mistaken for a repo file path ────── */ + +TEST(doclinks_bare_url_in_prose_no_edge) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + th_write_file(TH_PATH(fx.tmpdir, "README.md"), + "Fetch it from http://host/path/file.c directly.\n"); + th_write_file(TH_PATH(fx.tmpdir, "path/file.c"), "int x;\n"); + + add_file_node(fx.gb, fx.project, "README.md"); + add_file_node(fx.gb, fx.project, "path/file.c"); + + int n = run_doclinks(fx.gb, fx.project, fx.tmpdir); + ASSERT_EQ(n, 0); + ASSERT_EQ(ref_edge_count(fx.gb), 0); + + dl_fix_free(&fx); + PASS(); +} + +/* ── Exact-path matching: a bare basename never fuzzy-binds ──────── */ + +TEST(doclinks_backtick_basename_does_not_fuzzy_match) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + /* `main.c` in docs/ must resolve against docs/main.c or the repo-root + * main.c ONLY (exact path match) — never against src/main.c just + * because the basename matches. */ + th_write_file(TH_PATH(fx.tmpdir, "docs/a.md"), "See `main.c` for the entry point.\n"); + th_write_file(TH_PATH(fx.tmpdir, "src/main.c"), "int main(void) { return 0; }\n"); + + add_file_node(fx.gb, fx.project, "docs/a.md"); + add_file_node(fx.gb, fx.project, "src/main.c"); + + int n = run_doclinks(fx.gb, fx.project, fx.tmpdir); + ASSERT_EQ(n, 0); + ASSERT_EQ(ref_edge_count(fx.gb), 0); + + dl_fix_free(&fx); + PASS(); +} + +/* ── CRLF line endings do not corrupt a matched target ───────────── */ + +TEST(doclinks_crlf_line_endings) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + th_write_file(TH_PATH(fx.tmpdir, "README.md"), + "# Title\r\nSee [build](scripts/build.sh) for details.\r\n"); + th_write_file(TH_PATH(fx.tmpdir, "scripts/build.sh"), "#!/bin/sh\n"); + + int64_t doc_id = add_file_node(fx.gb, fx.project, "README.md"); + int64_t script_id = add_file_node(fx.gb, fx.project, "scripts/build.sh"); + + run_doclinks(fx.gb, fx.project, fx.tmpdir); + ASSERT_NOT_NULL(find_ref_edge(fx.gb, doc_id, script_id)); + + dl_fix_free(&fx); + PASS(); +} + +/* ── Fenced code blocks are scanned like prose (pinned, not fixed) ── */ + +TEST(doclinks_fenced_code_block_bare_mention_pinned) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + th_write_file(TH_PATH(fx.tmpdir, "README.md"), + "Build it:\n\n```\ngcc src/main.c -o out\n```\n"); + th_write_file(TH_PATH(fx.tmpdir, "src/main.c"), "int main(void) { return 0; }\n"); + + int64_t doc_id = add_file_node(fx.gb, fx.project, "README.md"); + int64_t code_id = add_file_node(fx.gb, fx.project, "src/main.c"); + + run_doclinks(fx.gb, fx.project, fx.tmpdir); + + const cbm_gbuf_edge_t *e = find_ref_edge(fx.gb, doc_id, code_id); + ASSERT_NOT_NULL(e); + ASSERT_TRUE(edge_has_strategy(e, "md_bare_mention")); + ASSERT_NOT_NULL(strstr(e->properties_json, "\"confidence\":0.70")); + + dl_fix_free(&fx); + PASS(); +} + /* ── Suite ───────────────────────────────────────────────────────── */ SUITE(doclinks) { @@ -309,4 +540,17 @@ SUITE(doclinks) { RUN_TEST(doclinks_resolves_relative_to_file_dir); RUN_TEST(doclinks_unresolvable_target_no_edge); RUN_TEST(doclinks_null_repo_path_skips); + + /* Reference-kind precedence (rooted / explicit-relative / bare) */ + RUN_TEST(doclinks_rooted_reference_skips_directory_join); + RUN_TEST(doclinks_explicit_relative_no_root_fallback); + + /* Negative controls / regression guards */ + RUN_TEST(doclinks_long_line_no_truncation); + RUN_TEST(doclinks_many_distinct_targets_no_cap); + RUN_TEST(doclinks_dotdot_escaping_root_dropped); + RUN_TEST(doclinks_bare_url_in_prose_no_edge); + RUN_TEST(doclinks_backtick_basename_does_not_fuzzy_match); + RUN_TEST(doclinks_crlf_line_endings); + RUN_TEST(doclinks_fenced_code_block_bare_mention_pinned); } diff --git a/tests/test_main.c b/tests/test_main.c index 1278c1c70..db3748288 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -1182,7 +1182,7 @@ int main(int argc, char **argv) { /* Config link */ RUN_SELECTED_SUITE(configlink); - /* Doc/shell file reference link */ + /* Markdown file reference link */ RUN_SELECTED_SUITE(doclinks); /* Infrastructure scanning */ diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index d32ab3538..6ef16a89b 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -314,6 +314,36 @@ TEST(pipeline_grpc_routes_cover_every_service_past_the_old_cap) { PASS(); } +/* Pipeline-level: the doclinks pass runs as part of a real index and its + * REFERENCES_FILE edges land in the store, not just in a unit test that + * calls cbm_pipeline_pass_doclinks() directly. Registration-list gap: every + * doclinks unit test calls the pass function by hand, so deleting its + * {predump_doclinks, "doclinks", false} entry in pipeline.c would leave + * every one of them green. */ +TEST(pipeline_doclinks_edge_lands_in_store) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_doclinks_pipeline_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + + write_temp_file(tmp, "README.md", "See [main](src/main.go) for the entry point.\n"); + write_temp_file(tmp, "src/main.go", "package main\n\nfunc main() {}\n"); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/doclinks.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + const char *project = cbm_pipeline_project_name(p); + ASSERT_GT(cbm_store_count_edges_by_type(s, project, "REFERENCES_FILE"), 0); + cbm_store_close(s); + cbm_pipeline_free(p); + th_rmtree(tmp); + PASS(); +} + /* Spilling must be invisible in the OUTPUT: the same repository indexed with * results parked on disk must produce the same graph as one indexed entirely in * memory. It did not. The namespace map that `use`/`using`/package imports @@ -15079,6 +15109,7 @@ SUITE(pipeline) { RUN_TEST(store_bulk_persistence); /* Integration: structure pass */ RUN_TEST(pipeline_grpc_routes_cover_every_service_past_the_old_cap); + RUN_TEST(pipeline_doclinks_edge_lands_in_store); RUN_TEST(pipeline_spill_resolves_namespace_imports_like_memory); RUN_TEST(pipeline_structure_nodes); RUN_TEST(pipeline_committed_counts_match_persisted); From 9a9534cbc45d40afa4e8bf070030a806acea1f5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Deust?= Date: Tue, 22 Sep 2026 08:54:40 +0200 Subject: [PATCH 130/130] docs: list REFERENCES_FILE in the README edge types table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The full Edge Types list near the graph schema section still missed the new REFERENCES_FILE type added by the doclinks pass. The "selected" list earlier in the file is explicitly a subset and is left as is. Co-Authored-By: Claude Sonnet 5 Signed-off-by: Clément Deust --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 66b50fe08..9c82c18ae 100644 --- a/README.md +++ b/README.md @@ -704,7 +704,7 @@ JSON arguments can also be piped on stdin, for tools that take arguments. A tool ### Edge Types -`CONTAINS_PACKAGE`, `CONTAINS_FOLDER`, `CONTAINS_FILE`, `DEFINES`, `DEFINES_METHOD`, `IMPORTS`, `CALLS`, `CALL_REFERENCE`, `HTTP_CALLS`, `ASYNC_CALLS`, `IMPLEMENTS`, `HANDLES`, `USAGE`, `CONFIGURES`, `WRITES`, `MEMBER_OF`, `TESTS`, `USES_TYPE`, `FILE_CHANGES_WITH` +`CONTAINS_PACKAGE`, `CONTAINS_FOLDER`, `CONTAINS_FILE`, `DEFINES`, `DEFINES_METHOD`, `IMPORTS`, `CALLS`, `CALL_REFERENCE`, `HTTP_CALLS`, `ASYNC_CALLS`, `IMPLEMENTS`, `HANDLES`, `USAGE`, `CONFIGURES`, `REFERENCES_FILE`, `WRITES`, `MEMBER_OF`, `TESTS`, `USES_TYPE`, `FILE_CHANGES_WITH` ### Qualified Names