From 902f85b0f9566dea5a73a5dc2a724a282dc25fcb Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 27 Aug 2026 04:05:48 +0200 Subject: [PATCH 01/31] fix(install): forward clients selector Signed-off-by: Martin Vogel --- install.sh | 55 ++++++--- tests/test_smoke_fixture_contract.sh | 168 +++++++++++++++++++++++++++ 2 files changed, 208 insertions(+), 15 deletions(-) diff --git a/install.sh b/install.sh index 094c04a43..9f4b4ec99 100755 --- a/install.sh +++ b/install.sh @@ -19,6 +19,8 @@ main() { REPO="DeusData/codebase-memory-mcp" INSTALL_DIR="$HOME/.local/bin" SKIP_CONFIG=false +CLIENTS_SET=false +CLIENTS="" CBM_DOWNLOAD_URL="${CBM_DOWNLOAD_URL:-https://github.com/${REPO}/releases/latest/download}" # Security: every remote hop must remain HTTPS. Plain HTTP is accepted only @@ -79,26 +81,46 @@ download_file() { fi } -for arg in "$@"; do - case "$arg" in - --dir=*) INSTALL_DIR="${arg#--dir=}" ;; - --skip-config) SKIP_CONFIG=true ;; +while [ "$#" -gt 0 ]; do + case "$1" in + --dir=*) + INSTALL_DIR="${1#--dir=}" + shift + ;; + --dir) + if [ "$#" -lt 2 ] || [[ "$2" == -* ]]; then + echo "install.sh: '--dir' needs a value. Please consult --help." >&2 + exit 2 + fi + INSTALL_DIR="$2" + shift 2 + ;; + --clients=*) + CLIENTS_SET=true + CLIENTS="${1#--clients=}" + shift + ;; + --skip-config) + SKIP_CONFIG=true + shift + ;; --help|-h) - echo "Usage: install.sh [--dir=] [--skip-config]" - echo " --dir PATH Install directory (default: ~/.local/bin)" - echo " --skip-config Skip automatic agent configuration" + echo "Usage: install.sh [--dir=] [--clients=] [--skip-config]" + echo " --dir PATH Install directory (default: ~/.local/bin)" + echo " --clients LIST Configure only comma-separated clients" + echo " --skip-config Skip automatic agent configuration" exit 0 ;; + -*) + echo "install.sh: unknown option '$1'. Please consult --help." >&2 + exit 2 + ;; + *) + echo "install.sh: unexpected argument '$1'. Please consult --help." >&2 + exit 2 + ;; esac done -# Handle --dir (space-separated) -prev="" -for arg in "$@"; do - if [ "$prev" = "--dir" ]; then - INSTALL_DIR="$arg" - fi - prev="$arg" -done detect_os() { case "$(uname -s)" in @@ -314,6 +336,9 @@ echo "Verified candidate: $CANDIDATE_VERSION" DEST="$INSTALL_DIR/codebase-memory-mcp" INSTALL_ARGS=(-y --force "--dir=$INSTALL_DIR") +if [ "$CLIENTS_SET" = true ]; then + INSTALL_ARGS+=("--clients=$CLIENTS") +fi if [ "$SKIP_CONFIG" = true ]; then INSTALL_ARGS+=(--skip-config) fi diff --git a/tests/test_smoke_fixture_contract.sh b/tests/test_smoke_fixture_contract.sh index 43fb248fd..9bd88d941 100755 --- a/tests/test_smoke_fixture_contract.sh +++ b/tests/test_smoke_fixture_contract.sh @@ -8,10 +8,13 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd)" python3 - "$ROOT" <<'PY' from __future__ import annotations +import hashlib +import os import pathlib import re import subprocess import sys +import tarfile import tempfile import time import urllib.request @@ -413,6 +416,171 @@ require( "scripts/test.sh must run the smoke fixture contract", ) +# Functional wrapper contract: install.sh must pass its wrapper-only selectors +# to the downloaded candidate, while retaining the existing dir/skip controls. +# Native Windows ships and exercises install.ps1; the Unix wrapper is covered +# on both macOS and Linux venue legs. +if sys.platform != "win32": + with tempfile.TemporaryDirectory(prefix="cbm-install-wrapper-") as temp: + temp_path = pathlib.Path(temp) + fixture = temp_path / "fixture" + payload = temp_path / "payload" + fixture.mkdir() + payload.mkdir() + + uname_s = subprocess.check_output(["uname", "-s"], text=True).strip() + uname_m = subprocess.check_output(["uname", "-m"], text=True).strip() + os_name = "darwin" if uname_s == "Darwin" else "linux" + if uname_m in ("arm64", "aarch64"): + arch_name = "arm64" + else: + arch_name = "amd64" + portable = "-portable" if os_name == "linux" else "" + archive_name = ( + f"codebase-memory-mcp-{os_name}-{arch_name}{portable}.tar.gz" + ) + archive = fixture / archive_name + + candidate = payload / "codebase-memory-mcp" + candidate.write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + "if [ \"${1:-}\" = --version ]; then echo 'cbm fixture 0'; exit 0; fi\n" + "if [ \"${1:-}\" != install ]; then exit 64; fi\n" + "printf '%s\\n' \"$@\" > \"$CBM_INSTALL_ARG_LOG\"\n" + "target=''\n" + "for arg in \"$@\"; do case \"$arg\" in --dir=*) target=${arg#--dir=} ;; esac; done\n" + "[ -n \"$target\" ] || exit 65\n" + "mkdir -p \"$target\"\n" + "cp \"$0\" \"$target/codebase-memory-mcp\"\n", + encoding="utf-8", + ) + candidate.chmod(0o755) + (payload / "LICENSE").write_text("fixture license\n", encoding="utf-8") + (payload / "install.sh").write_text(install_script, encoding="utf-8") + (payload / "THIRD_PARTY_NOTICES.md").write_text( + "fixture notices\n", encoding="utf-8" + ) + with tarfile.open(archive, "w:gz") as bundle: + for name in ( + "codebase-memory-mcp", + "LICENSE", + "install.sh", + "THIRD_PARTY_NOTICES.md", + ): + bundle.add(payload / name, arcname=name) + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + (fixture / "checksums.txt").write_text( + f"{digest} {archive_name}\n", encoding="ascii" + ) + + # Replace curl only inside the subprocess environment. The wrapper still + # performs its real archive/checksum/extraction flow, but all bytes come + # from this generated fixture and no socket is opened. + fake_bin = temp_path / "fake-bin" + fake_bin.mkdir() + fake_curl = fake_bin / "curl" + fake_curl.write_text( + "#!/usr/bin/env python3\n" + "import os, pathlib, shutil, sys, urllib.parse\n" + "args = sys.argv[1:]\n" + "target = pathlib.Path(args[args.index('-o') + 1])\n" + "name = pathlib.PurePosixPath(urllib.parse.urlparse(args[-1]).path).name\n" + "shutil.copyfile(pathlib.Path(os.environ['CBM_INSTALL_FIXTURE']) / name, target)\n", + encoding="utf-8", + ) + fake_curl.chmod(0o755) + + wrapper = root / "install.sh" + help_result = subprocess.run( + [str(wrapper), "--help"], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=10, + check=False, + ) + require( + help_result.returncode == 0 + and "--dir" in help_result.stdout + and "--skip-config" in help_result.stdout + and "--clients" in help_result.stdout, + "install.sh --help must document dir, skip-config, and clients", + ) + + base_env = dict(os.environ) + base_env.update( + CBM_DOWNLOAD_URL="http://127.0.0.1:9", + CBM_INSTALL_FIXTURE=str(fixture), + HOME=str(temp_path / "home"), + PATH=f"{fake_bin}{os.pathsep}{base_env['PATH']}", + ) + pathlib.Path(base_env["HOME"]).mkdir() + + def run_wrapper( + log_name: str, *arguments: str + ) -> tuple[subprocess.CompletedProcess[str], list[str]]: + argument_log = temp_path / log_name + env = dict(base_env) + env["CBM_INSTALL_ARG_LOG"] = str(argument_log) + result = subprocess.run( + [str(wrapper), *arguments], + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=20, + check=False, + ) + logged = ( + argument_log.read_text(encoding="utf-8").splitlines() + if argument_log.is_file() + else [] + ) + return result, logged + + unknown, unknown_args = run_wrapper( + "unknown-args.log", "--definitely-not-a-wrapper-flag" + ) + require( + unknown.returncode == 2 + and "Please consult --help." in unknown.stdout + and not unknown_args, + "install.sh must reject unknown flags before executing the candidate", + ) + + selected_dir = temp_path / "selected-bin" + selected, selected_args = run_wrapper( + "selected-args.log", + f"--dir={selected_dir}", + "--clients=claude,codex", + "--skip-config", + ) + require( + selected.returncode == 0 + and selected_args + == [ + "install", + "-y", + "--force", + f"--dir={selected_dir}", + "--clients=claude,codex", + "--skip-config", + ], + "install.sh must forward the explicit clients selector with dir and skip-config", + ) + + ordinary_dir = temp_path / "ordinary-bin" + ordinary, ordinary_args = run_wrapper( + "ordinary-args.log", f"--dir={ordinary_dir}" + ) + require( + ordinary.returncode == 0 + and ordinary_args + == ["install", "-y", "--force", f"--dir={ordinary_dir}"], + "install.sh without selectors must preserve the ordinary install arguments", + ) + # Functional check: the helper must publish a live kernel-assigned port and # serve the exact expected artifact. This is intentionally build-free. if helper.is_file(): From 787f87d81c47b8f9355c69638c0f3a473d899c7d Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 26 Aug 2026 18:15:09 +0200 Subject: [PATCH 02/31] fix: block sensitive Windows program roots Signed-off-by: Martin Vogel --- src/foundation/workspace.c | 44 ++++++++- src/mcp/mcp.c | 35 +++++++ src/mcp/mcp_internal.h | 8 ++ tests/repro/repro_issue403.c | 177 +++++++---------------------------- tests/test_mcp.c | 82 ++++++++++++++++ tests/test_workspace.c | 118 +++++++++++++++++++++++ 6 files changed, 315 insertions(+), 149 deletions(-) diff --git a/src/foundation/workspace.c b/src/foundation/workspace.c index a85e17822..cb254f3d9 100644 --- a/src/foundation/workspace.c +++ b/src/foundation/workspace.c @@ -211,6 +211,34 @@ static bool ws_any_component_matches(const char *path, const char *const *names, return false; } +/* The default per-user Windows application prefix. Match from the drive root, + * by whole segments, so a project merely containing the same sequence (or a + * sibling such as Programs-src) is not captured. The second segment is the one + * and only user name; its spelling is deliberately unrestricted. */ +static bool ws_is_windows_user_programs_tree(const char *path) { + if (!ws_is_windows_style(path)) { + return false; + } + static const char *const expected[] = {"Users", NULL, "AppData", "Local", "Programs"}; + const char *p = path + ws_volume_prefix_len(path); + for (size_t i = 0; i < sizeof(expected) / sizeof(expected[0]); i++) { + while (ws_is_sep(*p)) { + p++; + } + if (!*p) { + return false; + } + const char *start = p; + while (*p && !ws_is_sep(*p)) { + p++; + } + if (expected[i] && !ws_component_equals(start, (size_t)(p - start), expected[i], true)) { + return false; + } + } + return true; +} + /* True when b is a or lives under a. Compares on a separator boundary so * "/a/bc" is not treated as living under "/a/b". */ static bool ws_is_ancestor_or_equal(const char *a, const char *b) { @@ -297,6 +325,10 @@ cbm_ws_verdict_t cbm_workspace_classify_root(const char *canonical_path, const c return CBM_WS_DENY_SENSITIVE; } + if (ws_is_windows_user_programs_tree(canonical_path)) { + return CBM_WS_DENY_SENSITIVE; + } + /* "C:/Users" is the user tree itself — refuse it as a root, but leave the * projects below it alone; that is where Windows users actually work. */ if (windows_style && depth == 1 && @@ -316,7 +348,7 @@ const char *cbm_workspace_verdict_reason(cbm_ws_verdict_t verdict) { case CBM_WS_DENY_ABSOLUTE: return "path is a volume root or holds the codebase-memory cache; it cannot be indexed"; case CBM_WS_DENY_SENSITIVE: - return "path is a home or credential directory"; + return "path is a home, credential, system, or application-install directory"; default: break; } @@ -455,10 +487,15 @@ bool cbm_workspace_grant_add(const char *cache_dir, const char *home_dir, } } - /* Already present is success, not a duplicate error. */ + bool sensitive = verdict == CBM_WS_DENY_SENSITIVE; + + /* Already present is success, not a duplicate error. A sensitive approval + * is narrower: an old ordinary grant (or an ordinary ancestor) establishes + * containment, but does not record the exact human-approved exception that + * cbm_workspace_root_allowed requires. Append that exact marked entry once. */ ws_match_t existing = {canonical_path, false, false}; (void)ws_grant_walk(cache_dir, ws_match_visit, &existing); - if (existing.contained) { + if (existing.contained && (!sensitive || existing.exact_sensitive)) { return true; } @@ -476,7 +513,6 @@ bool cbm_workspace_grant_add(const char *cache_dir, const char *home_dir, } return false; } - bool sensitive = verdict == CBM_WS_DENY_SENSITIVE; (void)fprintf(f, "%s%s\n", sensitive ? "!" : "", canonical_path); bool ok = fclose(f) == 0; if (!ok && err) { diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 04316627b..4c4982929 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -1600,6 +1600,10 @@ struct cbm_mcp_server { void *quarantine_test_context; cbm_mcp_command_test_hook_fn command_test_hook; void *command_test_context; +#ifdef CBM_ENABLE_TEST_SEAMS + cbm_mcp_auto_index_count_test_hook_fn auto_index_count_test_hook; + void *auto_index_count_test_context; +#endif size_t search_output_limit_override; cbm_thread_t autoindex_tid; bool autoindex_active; /* true if auto-index thread was started */ @@ -1669,6 +1673,17 @@ void cbm_mcp_server_set_config(cbm_mcp_server_t *srv, struct cbm_config *cfg) { } } +#ifdef CBM_ENABLE_TEST_SEAMS +void cbm_mcp_server_set_auto_index_count_test_hook(cbm_mcp_server_t *srv, + cbm_mcp_auto_index_count_test_hook_fn hook, + void *context) { + if (srv) { + srv->auto_index_count_test_hook = hook; + srv->auto_index_count_test_context = context; + } +} +#endif + bool cbm_mcp_server_set_session_context(cbm_mcp_server_t *srv, const char *session_root, const char *allowed_root) { if (!srv || !session_root || session_root[0] == '\0' || @@ -11624,6 +11639,21 @@ static void maybe_auto_index(cbm_mcp_server_t *srv) { return; /* no session root detected */ } + /* Automatic work must honor the same shared workspace boundary as the + * explicit index_repository worker. Do this before the existing-DB watcher + * branch and before bounded discovery, so a refused session root begins no + * automatic observation or indexing. An exact sensitive-root grant remains + * the shared policy's authenticated override. */ + const char *allowed_root = + srv->allowed_root_policy_set ? srv->allowed_root : getenv("CBM_ALLOWED_ROOT"); + char boundary_err[CBM_SZ_1K]; + if (!cbm_workspace_root_allowed(srv->session_root, cbm_workspace_home_dir(), + cbm_workspace_cache_dir(), allowed_root, boundary_err, + sizeof(boundary_err))) { + cbm_log_warn("autoindex.skip", "reason", "workspace_boundary", "detail", boundary_err); + return; + } + /* Check if project already has a DB */ const char *home = cbm_get_home_dir(); if (home) { @@ -11656,6 +11686,11 @@ static void maybe_auto_index(cbm_mcp_server_t *srv) { /* Quick tracked-file count check to avoid OOM on massive repos. */ int file_count = -1; +#ifdef CBM_ENABLE_TEST_SEAMS + if (srv->auto_index_count_test_hook) { + srv->auto_index_count_test_hook(srv->auto_index_count_test_context); + } +#endif if (!cbm_mcp_auto_index_within_file_limit(srv->session_root, file_limit, &file_count)) { char files[32]; (void)snprintf(files, sizeof(files), "%d", file_count); diff --git a/src/mcp/mcp_internal.h b/src/mcp/mcp_internal.h index d9321c595..61b9383c9 100644 --- a/src/mcp/mcp_internal.h +++ b/src/mcp/mcp_internal.h @@ -9,12 +9,20 @@ * safety tests. This header is internal and is not part of the MCP API. */ typedef bool (*cbm_mcp_quarantine_test_hook_fn)(void *context, const char *step); typedef bool (*cbm_mcp_command_test_hook_fn)(void *context, const char *command); +#ifdef CBM_ENABLE_TEST_SEAMS +typedef void (*cbm_mcp_auto_index_count_test_hook_fn)(void *context); +#endif void cbm_mcp_server_set_quarantine_test_hook(cbm_mcp_server_t *srv, cbm_mcp_quarantine_test_hook_fn hook, void *context); void cbm_mcp_server_set_command_test_hook(cbm_mcp_server_t *srv, cbm_mcp_command_test_hook_fn hook, void *context); void cbm_mcp_server_set_search_output_limit_for_test(cbm_mcp_server_t *srv, size_t limit); +#ifdef CBM_ENABLE_TEST_SEAMS +void cbm_mcp_server_set_auto_index_count_test_hook(cbm_mcp_server_t *srv, + cbm_mcp_auto_index_count_test_hook_fn hook, + void *context); +#endif /* Release only the constructor-created pristine in-memory store. Public * cbm_mcp_server_new(NULL) semantics remain unchanged; daemon sessions use diff --git a/tests/repro/repro_issue403.c b/tests/repro/repro_issue403.c index 09f4e4bbe..c2451b9ea 100644 --- a/tests/repro/repro_issue403.c +++ b/tests/repro/repro_issue403.c @@ -1,159 +1,46 @@ /* - * repro_issue403.c -- Reproduce-first case for OPEN bug #403. + * repro_issue403.c -- Reproduce-first boundary case for issue #403. * - * Issue: #403 -- "The IDE's installation directory is unnecessarily indexed" - * https://github.com/DeusData/codebase-memory-mcp/issues/403 - * - * Wrongly-indexed directory: AppData/Local/Programs/Antigravity - * (the Antigravity IDE install tree; reported name confirmed in issue comments) - * - * Root cause (src/discover/discover.c): - * cbm_should_skip_dir() (line 339) tests only the BARE directory name - * (entry->name, the last path component) against ALWAYS_SKIP_DIRS and - * FAST_SKIP_DIRS. None of "AppData", "Local", "Programs", or "Antigravity" - * appears in either list. Therefore cbm_discover() walks straight into the - * IDE install tree and indexes every source-like file it contains. - * - * There is no install-directory guard at ANY layer: - * - ALWAYS_SKIP_DIRS covers VCS, build tools, and caches -- not IDE - * install prefixes (Programs, AppData/Local/Programs, etc.). - * - The .gitignore path is only loaded when a .git directory is present - * (is_git_repo gate, line 777 of discover.c). An IDE install dir does - * not contain .git, so .gitignore exclusions never fire. - * - The cbmignore path (opts->ignore_file or .cbmignore at root) is - * similarly absent from an install dir by default. - * Result: any source-extension file found under Antigravity/ is returned - * as a discovered file, bloating the graph with IDE internals. - * - * Expected (correct) behaviour: - * When cbm_discover() is called on a directory that contains an - * "Antigravity" subdirectory (or more generally any IDE install subtree), - * files under that subdirectory must NOT appear in the discovered file list. - * The correct fix (per the issue owner's comment) is to add "Antigravity" - * (and the broader "Programs" / install-dir pattern) to the exclusion layer, - * OR to extend the exclusion to root-path patterns so auto-index never picks - * an install dir as a project root in the first place. - * - * Actual (buggy) behaviour: - * cbm_discover() returns files under Antigravity/ as normal discovered - * files because the bare dirname "Antigravity" is absent from ALWAYS_SKIP_DIRS. - * - * Why RED on current code: - * The fixture creates a temp dir with: - * normal.py -- a legitimate source file (control: MUST appear) - * Antigravity/ide.py -- sentinel inside the IDE install dir (MUST NOT appear) - * cbm_discover() is called on the temp dir. The loop below asserts that - * ide.py is NOT in the result. On current code "Antigravity" is not skipped, - * so ide.py IS discovered and the ASSERT_FALSE fires RED. - * - * Fix location (not implemented here): - * src/discover/discover.c, ALWAYS_SKIP_DIRS array: - * Add "Antigravity" (and any other IDE install dir names to be excluded) - * to the NULL-terminated list. The broader fix is to extend the list with - * install-path components ("Programs", "AppData") or, per the issue owner, - * to implement a root-path exclusion in the auto-index root-selection logic - * so directories under AppData/Local/Programs are never chosen as repo roots. - * - * Exclusion is NOT config-driven in the current code. The closest knob is a - * .cbmignore file at the repo root (loaded unconditionally, unlike .gitignore - * which requires .git/). Passing opts->ignore_file also works. However, - * neither is set in this test -- we assert on the default behaviour, which is - * what the bug reporter experiences. + * Automatic session discovery must not treat the default per-user Windows + * application install tree as a workspace. This is a root-selection policy, + * not a vendor-name exclusion: ordinary projects elsewhere remain eligible. */ -#include #include "test_framework.h" -#include "test_helpers.h" -#include "discover/discover.h" +#include "foundation/workspace.h" -#include -#include -#include +TEST(repro_issue403_windows_user_programs_root_is_sensitive) { + static const char *const roots[] = { + "C:/Users/dev/AppData/Local/Programs", + "C:/Users/dev/AppData/Local/Programs/Antigravity", + "D:/Users/runneradmin/AppData/Local/Programs/Editor", + "c:\\uSeRs\\Dev\\aPpDaTa\\LoCaL\\pRoGrAmS\\IDE", + }; -/* ── Fixture ──────────────────────────────────────────────────────────────── - * - * Directory layout (NOT a git repo -- no .git/ subdir): - * - * / - * normal.py <- legitimate source file; MUST be discovered - * Antigravity/ - * ide.py <- sentinel inside IDE install dir; must NOT appear - * - * cbm_discover() is called on with no opts (NULL) so all default - * exclusions apply and no extra ignore file is consulted. - * - * Control assertion (expected GREEN even on buggy code): - * normal.py IS in the result -- proves discovery ran at all. - * - * Primary assertion (RED on buggy code): - * ide.py is NOT in the result -- the Antigravity subtree was skipped. - */ - -TEST(repro_issue403_install_dir_excluded) { - /* --- set up temp directory --- */ - char tmpdir[256]; - snprintf(tmpdir, sizeof(tmpdir), "%s/cbm_repro403_XXXXXX", cbm_tmpdir()); - ASSERT_NOT_NULL(cbm_mkdtemp(tmpdir)); - - /* Control file: a normal Python source at the repo root. */ - ASSERT_EQ(0, th_write_file(TH_PATH(tmpdir, "normal.py"), - "def hello(): return 1\n")); - - /* Sentinel file: a Python source inside the Antigravity install dir. - * This is the file that MUST be absent from discovery results. - * th_write_file creates intermediate directories automatically. */ - ASSERT_EQ(0, th_write_file(TH_PATH(tmpdir, "Antigravity/ide.py"), - "# Antigravity IDE internal module\ndef _internal(): pass\n")); - - /* --- Run discovery (default opts: no .git, no .cbmignore, no opts) --- */ - cbm_file_info_t *files = NULL; - int count = 0; - int rc = cbm_discover(tmpdir, NULL, &files, &count); - ASSERT_EQ(0, rc); - - /* --- Scan results --- */ - bool normal_found = false; - bool ide_file_found = false; - for (int i = 0; i < count; i++) { - if (strcmp(files[i].rel_path, "normal.py") == 0) { - normal_found = true; - } - /* Match any path that descends into the Antigravity directory. */ - if (strncmp(files[i].rel_path, "Antigravity/", 12) == 0 || - strcmp(files[i].rel_path, "Antigravity") == 0) { - ide_file_found = true; - printf(" BUG #403 reproduced: IDE install-dir file indexed: %s\n", - files[i].rel_path); - } + for (size_t i = 0; i < sizeof(roots) / sizeof(roots[0]); i++) { + ASSERT_EQ(cbm_workspace_classify_root(roots[i], NULL, NULL), CBM_WS_DENY_SENSITIVE); } - - cbm_discover_free(files, count); - th_rmtree(tmpdir); - - /* Control: normal.py must be discovered -- discovery ran correctly. */ - ASSERT_TRUE(normal_found); - - /* - * PRIMARY assertion (RED on buggy code): - * - * No file under Antigravity/ may appear in the discovered set. - * On current code, "Antigravity" is absent from ALWAYS_SKIP_DIRS so - * cbm_should_skip_dir("Antigravity", ...) returns false and the walk - * descends into it. ide.py is discovered, ide_file_found is true, and - * this ASSERT_FALSE fires RED. - * - * After the fix -- "Antigravity" added to ALWAYS_SKIP_DIRS (or an - * equivalent install-path exclusion applied) -- cbm_should_skip_dir - * returns true, the subtree is skipped, ide_file_found stays false, - * and this assertion passes GREEN. - */ - ASSERT_FALSE(ide_file_found); - + ASSERT_TRUE(cbm_workspace_verdict_is_overridable(CBM_WS_DENY_SENSITIVE)); PASS(); } -/* ── Suite ──────────────────────────────────────────────────────────────── */ +TEST(repro_issue403_nearby_projects_are_not_false_positives) { + static const char *const roots[] = { + "C:/Users/dev/projects/app", + "C:/Users/dev/AppData/Local", + "C:/Users/dev/AppData/Local/Programs-src", + "C:/Users/dev/AppData/Local/ProgramsBackup", + "C:/workspace/Users/dev/AppData/Local/Programs", + "C:/Users/dev/team/AppData/Local/Programs", + }; + + for (size_t i = 0; i < sizeof(roots) / sizeof(roots[0]); i++) { + ASSERT_EQ(cbm_workspace_classify_root(roots[i], NULL, NULL), CBM_WS_ALLOW); + } + PASS(); +} SUITE(repro_issue403) { - RUN_TEST(repro_issue403_install_dir_excluded); + RUN_TEST(repro_issue403_windows_user_programs_root_is_sensitive); + RUN_TEST(repro_issue403_nearby_projects_are_not_false_positives); } diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 278d33e68..df19badef 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -10,6 +10,7 @@ #include "../src/foundation/log.h" #include "../src/foundation/platform.h" /* cbm_file_size */ #include "../src/foundation/subprocess.h" +#include "../src/foundation/workspace.h" #include "../src/mcp/compact_out.h" #include "test_framework.h" #include "test_helpers.h" @@ -1293,6 +1294,83 @@ TEST(server_handle_initialized_notification) { PASS(); } +#ifdef CBM_ENABLE_TEST_SEAMS +static void issue403_count_started(void *context) { + int *calls = context; + (*calls)++; +} + +static int issue403_initialize_count_calls(const char *session_root, bool approve_sensitive) { + char *cache = th_mktempdir("cbm_mcp_403"); + if (!cache) { + return -1; + } + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + cbm_setenv("CBM_CACHE_DIR", cache, 1); + + char err[1024]; + bool approved = !approve_sensitive || + cbm_workspace_grant_add(cache, cbm_workspace_home_dir(), session_root, true, + err, sizeof(err)); + cbm_config_t *cfg = approved ? cbm_config_open(cache) : NULL; + cbm_mcp_server_t *srv = cfg ? cbm_mcp_server_new(NULL) : NULL; + int calls = -2; + if (srv) { + cbm_config_set(cfg, CBM_CONFIG_AUTO_INDEX, "true"); + /* The hook proves entry to the count path; keep the actual discovery + * call fail-closed so this unit test can never launch an index thread. */ + cbm_config_set(cfg, CBM_CONFIG_AUTO_INDEX_LIMIT, "-1"); + if (cbm_mcp_server_set_session_context(srv, session_root, NULL)) { + calls = 0; + cbm_mcp_server_set_config(srv, cfg); + cbm_mcp_server_set_auto_index_count_test_hook(srv, issue403_count_started, &calls); + char *response = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":403,\"method\":\"initialize\",\"params\":{}}"); + free(response); + } + cbm_mcp_server_free(srv); + } + if (cfg) { + cbm_config_close(cfg); + } + + restore_cache_dir(saved_cache_copy); + free(saved_cache_copy); + th_cleanup(cache); + return calls; +} + +TEST(mcp_issue403_sensitive_root_stops_before_discovery_count) { + int sensitive = issue403_initialize_count_calls( + "C:/Users/dev/AppData/Local/Programs/Antigravity", false); + int ordinary = issue403_initialize_count_calls("C:/Users/dev/projects/app", false); + ASSERT_EQ(sensitive, 0); + ASSERT_EQ(ordinary, 1); + PASS(); +} + +TEST(mcp_issue403_explicit_approval_preserves_auto_index) { + char *sensitive_home = th_mktempdir("cbm_mcp_403_home"); + ASSERT_NOT_NULL(sensitive_home); + const char *saved_home = getenv("HOME"); + char *saved_home_copy = saved_home ? strdup(saved_home) : NULL; + cbm_setenv("HOME", sensitive_home, 1); + + int approved = issue403_initialize_count_calls(sensitive_home, true); + + if (saved_home_copy) { + cbm_setenv("HOME", saved_home_copy, 1); + } else { + cbm_unsetenv("HOME"); + } + free(saved_home_copy); + th_cleanup(sensitive_home); + ASSERT_EQ(approved, 1); + PASS(); +} +#endif + TEST(server_handle_tools_list) { cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); @@ -11291,6 +11369,10 @@ SUITE(mcp) { /* Server protocol handling */ RUN_TEST(server_handle_initialize); RUN_TEST(server_handle_initialized_notification); +#ifdef CBM_ENABLE_TEST_SEAMS + RUN_TEST(mcp_issue403_sensitive_root_stops_before_discovery_count); + RUN_TEST(mcp_issue403_explicit_approval_preserves_auto_index); +#endif RUN_TEST(server_handle_tools_list); RUN_TEST(server_handle_tools_list_defaults_to_all_tools_and_accepts_cursor); RUN_TEST(server_handle_analysis_profile_filters_and_rejects_mutators); diff --git a/tests/test_workspace.c b/tests/test_workspace.c index 736b4477f..faf936e1b 100644 --- a/tests/test_workspace.c +++ b/tests/test_workspace.c @@ -132,6 +132,120 @@ TEST(ws_windows_system_trees_are_sensitive) { PASS(); } +TEST(ws_windows_user_programs_tree_has_exact_sensitive_boundaries) { + static const char *const sensitive[] = { + "C:/Users/dev/AppData/Local/Programs", + "C:/Users/dev/AppData/Local/Programs/Antigravity", + "D:/Users/runneradmin/AppData/Local/Programs/Editor", + "c:\\uSeRs\\Dev\\aPpDaTa\\LoCaL\\pRoGrAmS\\IDE", + }; + for (size_t i = 0; i < sizeof(sensitive) / sizeof(sensitive[0]); i++) { + ASSERT_EQ(cbm_workspace_classify_root(sensitive[i], HOME, CACHE), + CBM_WS_DENY_SENSITIVE); + } + + static const char *const allowed[] = { + "C:/Users/dev/projects/app", + "C:/Users/dev/AppData/Local", + "C:/Users/dev/AppData/Local/Programs-src", + "C:/Users/dev/AppData/Local/ProgramsBackup", + "C:/workspace/Users/dev/AppData/Local/Programs", + "C:/Users/dev/team/AppData/Local/Programs", + }; + for (size_t i = 0; i < sizeof(allowed) / sizeof(allowed[0]); i++) { + ASSERT_EQ(cbm_workspace_classify_root(allowed[i], HOME, CACHE), CBM_WS_ALLOW); + } + PASS(); +} + +TEST(ws_sensitive_root_explicit_approval_is_preserved) { + char root[256]; + char *created = th_mktempdir("cbm_ws_403_home"); + ASSERT_NOT_NULL(created); + snprintf(root, sizeof(root), "%s", created); + + char cache[256]; + created = th_mktempdir("cbm_ws_403"); + ASSERT_NOT_NULL(created); + snprintf(cache, sizeof(cache), "%s", created); + + char err[1024]; + ASSERT_FALSE(cbm_workspace_root_allowed(root, root, cache, NULL, err, sizeof(err))); + ASSERT_NOT_NULL(strstr(err, "--approve-sensitive")); + ASSERT_TRUE(cbm_workspace_grant_add(cache, root, root, true, err, sizeof(err))); + ASSERT_TRUE(cbm_workspace_root_allowed(root, root, cache, NULL, err, sizeof(err))); + + th_cleanup(root); + th_cleanup(cache); + PASS(); +} + +TEST(ws_sensitive_approval_upgrades_existing_ordinary_exact_grant) { + char root[256]; + char *created = th_mktempdir("cbm_ws_403_exact"); + ASSERT_NOT_NULL(created); + snprintf(root, sizeof(root), "%s", created); + + char cache[256]; + created = th_mktempdir("cbm_ws_403_exact_cache"); + ASSERT_NOT_NULL(created); + snprintf(cache, sizeof(cache), "%s", created); + + char err[1024]; + ASSERT_TRUE(cbm_workspace_grant_add(cache, NULL, root, false, err, sizeof(err))); + ASSERT_FALSE(cbm_workspace_root_allowed(root, root, cache, NULL, err, sizeof(err))); + + ASSERT_TRUE(cbm_workspace_grant_add(cache, root, root, true, err, sizeof(err))); + ASSERT_TRUE(cbm_workspace_root_allowed(root, root, cache, NULL, err, sizeof(err))); + /* A repeated explicit approval must recognize the marked exact grant rather + * than append another exception. */ + ASSERT_TRUE(cbm_workspace_grant_add(cache, root, root, true, err, sizeof(err))); + + char listed[4096]; + char expected[4096]; + ASSERT_TRUE(cbm_workspace_grant_list(cache, listed, sizeof(listed))); + snprintf(expected, sizeof(expected), "%s\n(approved) %s\n", root, root); + ASSERT_STR_EQ(listed, expected); + + th_cleanup(root); + th_cleanup(cache); + PASS(); +} + +TEST(ws_sensitive_approval_adds_exact_exception_under_ordinary_ancestor) { + char ancestor[256]; + char *created = th_mktempdir("cbm_ws_403_ancestor"); + ASSERT_NOT_NULL(created); + snprintf(ancestor, sizeof(ancestor), "%s", created); + + char cache[256]; + created = th_mktempdir("cbm_ws_403_ancestor_cache"); + ASSERT_NOT_NULL(created); + snprintf(cache, sizeof(cache), "%s", created); + + char sensitive[1024]; + snprintf(sensitive, sizeof(sensitive), "%s/private-project", ancestor); + ASSERT_EQ(cbm_mkdir(sensitive), 0); + char err[1024]; + ASSERT_TRUE(cbm_workspace_grant_add(cache, NULL, ancestor, false, err, sizeof(err))); + ASSERT_FALSE(cbm_workspace_root_allowed(sensitive, sensitive, cache, NULL, err, sizeof(err))); + + ASSERT_TRUE(cbm_workspace_grant_add(cache, sensitive, sensitive, true, err, sizeof(err))); + ASSERT_TRUE( + cbm_workspace_root_allowed(sensitive, sensitive, cache, NULL, err, sizeof(err))); + ASSERT_TRUE(cbm_workspace_grant_add(cache, sensitive, sensitive, true, err, sizeof(err))); + + char listed[4096]; + char expected[4096]; + ASSERT_TRUE(cbm_workspace_grant_list(cache, listed, sizeof(listed))); + snprintf(expected, sizeof(expected), "%s\n(approved) %s\n", ancestor, sensitive); + ASSERT_STR_EQ(listed, expected); + + th_cleanup(ancestor); + th_cleanup(cache); + PASS(); +} + /* POSIX is case-sensitive, so a differently-cased directory is a different one * and must not be refused. */ TEST(ws_posix_matching_is_case_sensitive) { @@ -275,6 +389,10 @@ SUITE(workspace) { RUN_TEST(ws_home_itself_is_sensitive_but_subdirs_are_fine); RUN_TEST(ws_credential_directories_are_sensitive_at_any_depth); RUN_TEST(ws_windows_system_trees_are_sensitive); + RUN_TEST(ws_windows_user_programs_tree_has_exact_sensitive_boundaries); + RUN_TEST(ws_sensitive_root_explicit_approval_is_preserved); + RUN_TEST(ws_sensitive_approval_upgrades_existing_ordinary_exact_grant); + RUN_TEST(ws_sensitive_approval_adds_exact_exception_under_ordinary_ancestor); RUN_TEST(ws_posix_matching_is_case_sensitive); RUN_TEST(ws_null_context_disables_dependent_checks); RUN_TEST(ws_every_verdict_has_a_reason); From 31724178b2623b074f1ab87b1588f91a0dd5760d Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 26 Aug 2026 18:25:31 +0200 Subject: [PATCH 03/31] docs: preserve savings measurement guide Signed-off-by: Martin Vogel --- README.md | 2 + docs/MEASURING_SAVINGS.md | 296 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 298 insertions(+) create mode 100644 docs/MEASURING_SAVINGS.md diff --git a/README.md b/README.md index ca39ef418..ef766813e 100644 --- a/README.md +++ b/README.md @@ -287,6 +287,8 @@ Benchmarked on Apple M3 Pro: **Token efficiency**: Five structural queries consumed ~3,400 tokens via codebase-memory-mcp versus ~412,000 tokens via file-by-file grep exploration — a **99.2% reduction**. +To measure comparable quality, latency, and agent-efficiency metrics on your own workload, see [Measuring quality, latency, and agent savings](docs/MEASURING_SAVINGS.md). Exact reproduction of the figures above requires the original inputs and raw artifacts. + ## Troubleshooting & Diagnostics codebase-memory-mcp runs **100% locally and collects no telemetry** — your code, queries, environment, and usage never leave your machine. That privacy guarantee also means that when you hit something we can't reproduce on our side (a slow memory climb over hours, a performance regression, a leak that only appears after days of real use), **we have no data at all unless you choose to send it.** Here is how to capture it yourself. diff --git a/docs/MEASURING_SAVINGS.md b/docs/MEASURING_SAVINGS.md new file mode 100644 index 000000000..60c0fdbdd --- /dev/null +++ b/docs/MEASURING_SAVINGS.md @@ -0,0 +1,296 @@ +# Measuring quality, latency, and agent savings + +This guide describes how to measure codebase-memory-mcp (CBM) on your own +repository without mixing three different questions: + +1. Does the agent give a better answer? +2. Is CBM fast and stable under the workload? +3. Does graph-assisted exploration use fewer model tokens or tool calls than + file-by-file exploration? + +Measure and report those classes separately. A fast graph query does not prove +that the final answer is correct, and CBM's query counters do not reveal the +agent's model-token or total tool-call consumption. The README's published +[performance figures](../README.md#performance), the +[language benchmark](BENCHMARK.md) and the more detailed +[evaluation plan](EVALUATION_PLAN.md) provide the grading concepts and the +larger comparative methodology behind this smaller recipe. + +## Freeze the experiment + +The shell examples in this guide require a POSIX-compatible shell on macOS or +Linux, or Git Bash on Windows. They are not native PowerShell syntax. +Run the command blocks in one dedicated shell: the setup and preflight blocks +enable fail-fast handling, and they capture Git command output before testing it +so a failed inner command cannot be mistaken for a clean worktree. + +Before either condition runs, record: + +- the repository URL or local identity and exact Git commit SHA; +- the question set and expected scope of each answer; +- the CBM version, index mode, operating system, and machine details; +- the agent model/version, system prompt, tool instructions, context limits, + and per-question budget; +- which condition runs first, plus any warm-up policy; +- the client or evaluation harness used to record tokens and tool calls. + +Use the same repository SHA, questions, model, prompt, budget, and clean-session +policy for both conditions. Do not let the second condition see the first +condition's answers or tool results. If runs are repeated, choose the repeat +count and aggregation method before looking at results and retain every run, +including failures and zero-result queries. + +Do not run either condition in a merely clean-looking working copy. A frozen +experiment includes untracked files, so create separate clean detached +worktrees for the two conditions: + +```bash +set -eu +SOURCE_REPO=/absolute/path/to/source-repository +COMMITISH=main +SHA=$(git -C "$SOURCE_REPO" rev-parse "$COMMITISH^{commit}") || exit 1 +MODE=full +RUN_ROOT=$(mktemp -d) || exit 1 +GRAPH_REPO="$RUN_ROOT/graph" +BASELINE_REPO="$RUN_ROOT/baseline" +ARTIFACT_ROOT="$RUN_ROOT/artifacts" + +git -C "$SOURCE_REPO" worktree add --detach "$GRAPH_REPO" "$SHA" || exit 1 +git -C "$SOURCE_REPO" worktree add --detach "$BASELINE_REPO" "$SHA" || exit 1 +mkdir "$ARTIFACT_ROOT" || exit 1 + +GRAPH_SHA=$(git -C "$GRAPH_REPO" rev-parse HEAD) || exit 1 +GRAPH_STATUS=$(git -C "$GRAPH_REPO" status --porcelain=v1 --untracked-files=all) || exit 1 +BASELINE_SHA=$(git -C "$BASELINE_REPO" rev-parse HEAD) || exit 1 +BASELINE_STATUS=$(git -C "$BASELINE_REPO" status --porcelain=v1 --untracked-files=all) || exit 1 +test "$GRAPH_SHA" = "$SHA" || exit 1 +test -z "$GRAPH_STATUS" || exit 1 +test "$BASELINE_SHA" = "$SHA" || exit 1 +test -z "$BASELINE_STATUS" || exit 1 +``` + +Repeat the matching SHA and cleanliness assertions immediately before indexing +and immediately before each condition. If either assertion fails, stop and +create a new detached worktree; do not erase unknown files to make a reused +checkout appear clean. Write run artifacts under `ARTIFACT_ROOT`, outside both +condition worktrees. + +## Preflight the graph + +Immediately before every Graph-condition measurement, repeat its clean-worktree +assertions and require a successful fresh index of that exact worktree and mode: + +```bash +set -eu +GRAPH_SHA=$(git -C "$GRAPH_REPO" rev-parse HEAD) || exit 1 +GRAPH_STATUS=$(git -C "$GRAPH_REPO" status --porcelain=v1 --untracked-files=all) || exit 1 +test "$GRAPH_SHA" = "$SHA" || exit 1 +test -z "$GRAPH_STATUS" || exit 1 +codebase-memory-mcp cli index_repository \ + --repo-path "$GRAPH_REPO" \ + --mode "$MODE" || exit 1 +GRAPH_SHA_AFTER=$(git -C "$GRAPH_REPO" rev-parse HEAD) || exit 1 +GRAPH_STATUS_AFTER=$(git -C "$GRAPH_REPO" status --porcelain=v1 --untracked-files=all) || exit 1 +test "$GRAPH_SHA_AFTER" = "$SHA" || exit 1 +test -z "$GRAPH_STATUS_AFTER" || exit 1 +``` + +Take `PROJECT_NAME` from the successful indexing response. A verbose status call +is useful only as a root/current-HEAD cross-check: + +```bash +codebase-memory-mcp cli list_projects +codebase-memory-mcp cli index_status --project PROJECT_NAME --verbose +``` + +Confirm that `root_path` is `GRAPH_REPO`, `git.head_sha` is `SHA`, and the Git +context is detached. `index_status` describes the project root and its current +filesystem Git context; it does **not** by itself prove that indexed records +were produced from that revision. The immediately preceding successful fresh +`index_repository` call is the freshness requirement. + +Finally, run one or more representative queries whose expected symbols you +have verified directly at the recorded SHA: + +```bash +codebase-memory-mcp cli search_graph \ + --project PROJECT_NAME \ + --name-pattern 'KNOWN_SYMBOL_PATTERN' \ + --limit 10 +``` + +Do not start measurement until the fresh index succeeds, the status cross-check +names the intended detached checkout and current SHA, and the representative +queries return the expected symbols. Save the indexing response, status, and +query outputs with the run artifacts. Immediately before the file-by-file +condition, repeat the corresponding SHA and cleanliness assertions for +`BASELINE_REPO`. + +## 1. Measure answer quality + +Create a fixed set of real developer questions. Include a mix of definition +discovery, relationships or call paths, targeted source retrieval, +architecture, and cross-cutting questions where those dimensions apply. Record +the expected scope or an independently derived ground truth for each question. + +Run two isolated conditions: + +| Condition | Allowed exploration tools | +|-----------|---------------------------| +| Graph | CBM graph tools such as `search_graph`, `trace_path`, `query_graph`, `get_code_snippet`, `get_architecture`, and `search_code` | +| File-by-file baseline | File listing, text search, and targeted file reads only | + +Grade the final answers against the source at the frozen SHA, not against how +plausible they sound. The compact rubric in [BENCHMARK.md](BENCHMARK.md) uses +PASS (1.0), PARTIAL (0.5), and FAIL (0.0), excluding truly inapplicable +questions from the denominator. For a finer comparison, follow +[EVALUATION_PLAN.md](EVALUATION_PLAN.md): score correctness, completeness, and +specificity separately, blind the grader to which condition produced each +answer, randomize A/B order, and require source evidence for the grade. + +Keep quality scores beside, but separate from, efficiency metrics. A token +reduction is only useful when the answer still meets the chosen quality bar. + +## 2. Measure latency and stability + +Record indexing time separately from query latency, and classify the index run +as full-source, artifact-assisted, or incremental. When no local project +database exists, `index_repository` can import a compatible +`.codebase-memory/graph.db.zst` and then take the incremental-manifest route. +Before timing, record whether that artifact exists in `GRAPH_REPO`, retain the +index log, and require the later successful `artifact.import` record containing +`db` and `size_mb` before classifying the run as artifact-assisted. A bootstrap +attempt alone, or an `artifact.import` record containing `skip` or `err`, does +not prove that the artifact was used. For queries, use the same fixed workload +in the same order, identify warm-up calls in advance, and retain per-call +durations and exit status rather than only a single average. Report the machine, +OS, CBM version, repository SHA, index mode, question set, index-run class, and +whether query results are cold or warm. + +### Built-in diagnostics + +For a daemon-backed run, enable diagnostics before the first session starts. +The daemon captures its environment at startup; if it is already running, +close all daemon-backed sessions before changing the setting. See +[Configuration](CONFIGURATION.md#4-environment-variables) for the complete +environment contract. + +```bash +export CBM_DIAGNOSTICS=1 +``` + +CBM creates a fresh randomized, owner-private diagnostics directory below the +system temporary directory. Do not assume or construct an old predictable +`/tmp` filename. Discover the exact `snapshot` and `trajectory` paths from the +`diagnostics.start` JSON record in +`${CBM_CACHE_DIR}/logs/cbm-daemon.log` (the default cache directory is +`~/.cache/codebase-memory-mcp`). The record is emitted even when the configured +log level suppresses ordinary logging. The README's +[diagnostics section](../README.md#troubleshooting--diagnostics) explains the +files and retention behavior. + +The live `snapshot.json` includes CBM-side `query_count`, `query_errors`, +`query_total_us`, `query_avg_us`, and `query_max_us`, along with process resource +counters. The retained `trajectory.ndjson` provides the resource and query-count +trend over time. These are useful for CBM latency and stability analysis, but +they are not a record of an agent's model usage or of non-CBM tools. +Because a daemon can be shared by multiple sessions, use an otherwise idle +daemon and record before/after values (or start a dedicated run) when attributing +its counters to one workload. + +### Canonical soak workload + +From a source checkout, the canonical endurance entry point is: + +```bash +scripts/soak-legs.sh build/c/codebase-memory-mcp 10 +``` + +It runs the quick mixed workload and the read-only query-leak workload, checks +for a valid completion summary, and writes per-call latency plus resource +results. The quick leg writes `soak-results/`; the query-leak leg writes +`soak-results-query-leak/`. Each directory contains `latency.csv`, `metrics.csv`, +and `summary.txt`. The two CSV files begin with these exact headers: + +```text +timestamp,tool,duration_ms,exit_code +timestamp,uptime_s,rss_bytes,heap_committed,fd_count,query_count,query_max_us +``` + +Do not invoke `scripts/soak-test.sh` directly; `soak-legs.sh` owns the +release-gating sequence. Treat these artifacts as stability and CBM latency +evidence, not as answer-quality or model-token evidence. + +## 3. Measure token and tool-call savings + +Run the Graph and file-by-file conditions on the same frozen inputs described +above. The MCP client or evaluation harness must capture agent usage because +CBM cannot know the final model input/output token count or the agent's total +tool-call consumption. + +Define each `run_id` as one paired experimental replicate. Each +`(run_id, condition)` is exactly one isolated client session that answers one +question. Capture one row for every usage window the client directly measures: + +```text +run_id,condition,repo_sha,question_id,window,input_tokens,output_tokens,total_tokens,tool_calls,wall_time_ms,answer_artifact,quality_score +``` + +In each row, `tool_calls` is the count of every client tool invocation inside +that same window, including orchestration calls and the graph or file tools +permitted for that condition, plus retries, errors, and zero-result calls. Do +not count only CBM calls. Candidate windows are: + +- **Answering tokens:** input plus output tokens between fixed markers around + the question-answering phase. +- **Full-session tokens:** the entire isolated session, including orientation, + initial probes, dead ends, and answer formatting. + +The full-session value best represents an adopter's total cost; the answering +window helps explain where a difference arose. Do not substitute CBM's +`query_count` for either `tool_calls` value: it cannot see file searches, file +reads, or other client tools. + +For each Graph/baseline pair, compare only a window that both clients report +directly, with at most one row per `(run_id, condition, window)`. If either +client cannot expose a window, omit that window or record its usage fields as +N/A and exclude it from the comparison. Never infer a missing window, partition +a session total across questions, or duplicate one session total into multiple +question rows. + +Select one window `W` and calculate from the Graph and baseline rows for that +same window only: + +```text +token reduction (%) = 100 * (baseline tokens - Graph tokens) / baseline tokens +tool-call reduction (%) = 100 * (baseline calls - Graph calls) / baseline calls +token ratio = baseline tokens / Graph tokens +tool-call ratio = baseline calls / Graph calls +``` + +Calculate and label every common measured window separately; never mix their +numerators or denominators. If baseline tokens or calls are zero, the +corresponding reduction percentage is N/A. If Graph tokens or calls are zero, +the corresponding ratio is N/A. Do not add pseudocounts. Publish the raw paired +counts alongside ratios and percentages. Also publish the quality result for +each pair, the number of runs, aggregation method, failures, and experimental +controls. Never turn a result from one repository, question set, model, or +machine into a universal savings claim. + +## Reproducibility checklist + +- Repository identity and exact SHA are recorded; each condition uses a clean, + detached worktree with no tracked or untracked changes. +- CBM version, index mode, project name, and preflight outputs are retained. +- Every Graph run follows a successful fresh index of the frozen worktree and + mode; verbose status is only a root/current-HEAD cross-check. +- Questions, ground truth, prompts, budgets, and condition order are frozen. +- Graph and file-by-file runs use isolated sessions and the same controls. +- Quality, CBM latency/stability, and agent savings are reported separately. +- Usage counts come directly from the client or evaluation harness; each + condition run is one isolated question/session, common windows use separate + rows, and unsupported windows are omitted or N/A rather than inferred. +- Diagnostics paths come from `diagnostics.start`, not a guessed temp path. +- Raw outputs, errors, zero results, and calculation inputs are retained. +- Claims identify their repository, SHA, question set, model, machine, and run + count, with no invented or extrapolated benchmark numbers. From 3a8c0d9758e5337d3878b2099dad3e2001bb81d9 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 27 Aug 2026 05:38:15 +0200 Subject: [PATCH 04/31] fix: enforce workspace policy in daemon background work Signed-off-by: Martin Vogel --- src/daemon/application.c | 30 ++++ tests/test_daemon_application.c | 263 ++++++++++++++++++++++++++++++++ 2 files changed, 293 insertions(+) diff --git a/src/daemon/application.c b/src/daemon/application.c index 7e4a35be7..12543808e 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -14,6 +14,7 @@ #include "foundation/secure_random.h" #include "foundation/sha256.h" #include "foundation/subprocess.h" +#include "foundation/workspace.h" #include "mcp/index_supervisor.h" #include "mcp/mcp.h" #include "mcp/mcp_internal.h" @@ -429,6 +430,22 @@ static void application_release_session_watch_locked(cbm_daemon_application_sess } } +static bool application_session_workspace_allowed(const cbm_daemon_application_session_t *session, + const char *operation) { + const char *root = session ? cbm_mcp_server_session_root(session->mcp) : NULL; + char boundary_error[CBM_SZ_1K]; + bool allowed = + root && root[0] && + cbm_workspace_root_allowed(root, cbm_workspace_home_dir(), cbm_workspace_cache_dir(), + cbm_mcp_server_allowed_root(session->mcp), boundary_error, + sizeof(boundary_error)); + if (!allowed) { + cbm_log_warn("daemon.workspace.skipped", "operation", operation, "detail", + root && root[0] ? boundary_error : "session root is unavailable"); + } + return allowed; +} + /* Caller holds application->mutex. */ static void application_refresh_watch_locked(cbm_daemon_application_session_t *session) { cbm_daemon_application_t *application = session->application; @@ -442,6 +459,10 @@ static void application_refresh_watch_locked(cbm_daemon_application_session_t *s if (!project || !project[0] || !root || !root[0]) { return; } + if (!application_session_workspace_allowed(session, "watch")) { + application_release_session_watch_locked(session); + return; + } bool enabled = !application->config || cbm_config_get_bool(application->config, CBM_CONFIG_AUTO_WATCH, true); bool db_exists = application_regular_db_exists(project); @@ -1394,6 +1415,11 @@ static void application_auto_index_retry_pending_locked(cbm_daemon_application_t session->auto_index_retry_pending = false; continue; } + if (!application_session_workspace_allowed(session, "auto_index_retry")) { + session->auto_index_retry_pending = false; + application_refresh_watch_locked(session); + continue; + } if (application_regular_db_exists(project)) { session->auto_index_retry_pending = false; application_refresh_watch_locked(session); @@ -1940,6 +1966,10 @@ static void application_background_initialize_impl(cbm_daemon_application_sessio : CBM_MCP_DEFAULT_AUTO_INDEX_LIMIT; int tracked_files = -1; bool auto_index_candidate = auto_index && !db_exists; + if (auto_index_candidate && + !application_session_workspace_allowed(session, "auto_index_discovery")) { + auto_index_candidate = false; + } bool within_auto_index_limit = !auto_index_candidate || cbm_mcp_auto_index_within_file_limit(root_path, auto_index_limit, &tracked_files); diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index c639ace2f..e0ef6b459 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -15,6 +15,7 @@ #include "foundation/platform.h" #include "foundation/sha256.h" #include "foundation/subprocess.h" +#include "foundation/workspace.h" #include "mcp/mcp.h" #include "mcp/mcp_internal.h" #include "pipeline/pipeline.h" @@ -2089,6 +2090,266 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { PASS(); } +TEST(daemon_application_sensitive_root_blocks_auto_index_but_preserves_controls) { + app_env_backup_t cache_environment; + app_env_backup_t home_environment; + bool environments_saved = app_env_backup_capture(&cache_environment, "CBM_CACHE_DIR") && + app_env_backup_capture(&home_environment, "HOME"); + char sensitive_raw[APP_TEST_PATH_CAP]; + char ordinary_raw[APP_TEST_PATH_CAP]; + char cache_raw[APP_TEST_PATH_CAP]; + (void)snprintf(sensitive_raw, sizeof(sensitive_raw), "%s/cbm-app-sensitive-auto-home-XXXXXX", + cbm_tmpdir()); + (void)snprintf(ordinary_raw, sizeof(ordinary_raw), "%s/cbm-app-sensitive-auto-project-XXXXXX", + cbm_tmpdir()); + (void)snprintf(cache_raw, sizeof(cache_raw), "%s/cbm-app-sensitive-auto-cache-XXXXXX", + cbm_tmpdir()); + bool dirs_ok = cbm_mkdtemp(sensitive_raw) != NULL && cbm_mkdtemp(ordinary_raw) != NULL && + cbm_mkdtemp(cache_raw) != NULL; + char sensitive[APP_TEST_PATH_CAP] = {0}; + char ordinary[APP_TEST_PATH_CAP] = {0}; + char cache[APP_TEST_PATH_CAP] = {0}; + bool canonical = dirs_ok && cbm_canonical_path(sensitive_raw, sensitive, sizeof(sensitive)) && + cbm_canonical_path(ordinary_raw, ordinary, sizeof(ordinary)) && + cbm_canonical_path(cache_raw, cache, sizeof(cache)); + char sensitive_source[APP_TEST_PATH_CAP]; + char ordinary_source[APP_TEST_PATH_CAP]; + (void)snprintf(sensitive_source, sizeof(sensitive_source), "%s/tracked.c", sensitive); + (void)snprintf(ordinary_source, sizeof(ordinary_source), "%s/tracked.c", ordinary); + bool files_ok = canonical && app_test_create_empty_file(sensitive_source) && + app_test_create_empty_file(ordinary_source); + bool environments_set = environments_saved && files_ok && + cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0 && + cbm_setenv("HOME", sensitive, 1) == 0; + cbm_config_t *stored_config = environments_set ? cbm_config_open(cache) : NULL; + bool config_ready = stored_config && + cbm_config_set(stored_config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && + cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0; + + app_fake_worker_context_t fake; + app_fake_worker_context_init(&fake); + cbm_daemon_application_worker_ops_t worker_ops = { + .context = &fake, + .start = app_fake_worker_start, + .poll = app_fake_worker_poll, + .cancel = app_fake_worker_cancel, + .log_path = app_fake_worker_log_path, + .destroy = app_fake_worker_destroy, + }; + cbm_daemon_application_config_t config = { + .config = stored_config, + .worker_ops = &worker_ops, + }; + cbm_daemon_application_t *application = + config_ready ? cbm_daemon_application_new(&config) : NULL; + cbm_daemon_runtime_application_callbacks_t callbacks = + cbm_daemon_application_runtime_callbacks(application); + cbm_daemon_runtime_application_session_t *sensitive_session = + application ? app_test_open(&callbacks, 4030) : NULL; + cbm_daemon_runtime_application_session_t *ordinary_session = + application ? app_test_open(&callbacks, 4031) : NULL; + cbm_daemon_runtime_application_session_t *approved_session = + application ? app_test_open(&callbacks, 4032) : NULL; + + int background_baseline = cbm_daemon_application_background_initializes_for_test(); + bool sensitive_initialized = app_test_initialize_profile( + &callbacks, sensitive_session, sensitive, CBM_MCP_TOOL_PROFILE_ALL, NULL, NULL); + bool sensitive_evaluated = app_wait_for_background_initializes(background_baseline + 1); + bool sensitive_blocked = sensitive_initialized && sensitive_evaluated && + atomic_load(&fake.starts) == 0 && + cbm_daemon_application_active_jobs(application) == 0; + + bool ordinary_initialized = app_test_initialize_profile(&callbacks, ordinary_session, ordinary, + CBM_MCP_TOOL_PROFILE_ALL, NULL, NULL); + bool ordinary_admitted = ordinary_initialized && app_wait_for_atomic_int(&fake.starts, 1) && + cbm_daemon_application_active_jobs(application) == 1; + if (ordinary_session) { + callbacks.session_cancel(callbacks.context, ordinary_session); + callbacks.session_close(callbacks.context, ordinary_session); + ordinary_session = NULL; + } + bool ordinary_reaped = ordinary_admitted && app_wait_for_atomic_int(&fake.cancels, 1) && + app_wait_for_atomic_int(&fake.destroys, 1) && + app_wait_for_active_jobs(application, 0); + + char grant_error[APP_TEST_PATH_CAP]; + bool approved = ordinary_reaped && cbm_workspace_grant_add(cache, sensitive, sensitive, true, + grant_error, sizeof(grant_error)); + bool approved_initialized = + approved && app_test_initialize_profile(&callbacks, approved_session, sensitive, + CBM_MCP_TOOL_PROFILE_ALL, NULL, NULL); + bool approved_admitted = approved_initialized && app_wait_for_atomic_int(&fake.starts, 2) && + cbm_daemon_application_active_jobs(application) == 1; + + if (sensitive_session) { + callbacks.session_cancel(callbacks.context, sensitive_session); + callbacks.session_close(callbacks.context, sensitive_session); + } + if (approved_session) { + callbacks.session_cancel(callbacks.context, approved_session); + callbacks.session_close(callbacks.context, approved_session); + } + if (ordinary_session) { + callbacks.session_cancel(callbacks.context, ordinary_session); + callbacks.session_close(callbacks.context, ordinary_session); + } + bool approved_reaped = approved_admitted && app_wait_for_atomic_int(&fake.cancels, 2) && + app_wait_for_atomic_int(&fake.destroys, 2) && + app_wait_for_active_jobs(application, 0); + if (!approved_reaped) { + atomic_store(&fake.allow_completion, true); + } + bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + cbm_daemon_application_free(application); + cbm_config_close(stored_config); + (void)th_rmtree(sensitive); + (void)th_rmtree(ordinary); + (void)th_rmtree(cache); + bool cache_restored = app_env_backup_restore(&cache_environment); + bool home_restored = app_env_backup_restore(&home_environment); + + ASSERT_TRUE(environments_saved); + ASSERT_TRUE(dirs_ok); + ASSERT_TRUE(canonical); + ASSERT_TRUE(files_ok); + ASSERT_TRUE(environments_set); + ASSERT_TRUE(config_ready); + ASSERT_TRUE(sensitive_blocked); + ASSERT_TRUE(ordinary_admitted); + ASSERT_TRUE(ordinary_reaped); + ASSERT_TRUE(approved); + ASSERT_TRUE(approved_admitted); + ASSERT_TRUE(approved_reaped); + ASSERT_TRUE(stopped); + ASSERT_TRUE(cache_restored); + ASSERT_TRUE(home_restored); + PASS(); +} + +TEST(daemon_application_sensitive_root_blocks_watch_but_preserves_controls) { + app_env_backup_t cache_environment; + app_env_backup_t home_environment; + bool environments_saved = app_env_backup_capture(&cache_environment, "CBM_CACHE_DIR") && + app_env_backup_capture(&home_environment, "HOME"); + char sensitive_raw[APP_TEST_PATH_CAP]; + char ordinary_raw[APP_TEST_PATH_CAP]; + char cache_raw[APP_TEST_PATH_CAP]; + (void)snprintf(sensitive_raw, sizeof(sensitive_raw), "%s/cbm-app-sensitive-watch-home-XXXXXX", + cbm_tmpdir()); + (void)snprintf(ordinary_raw, sizeof(ordinary_raw), "%s/cbm-app-sensitive-watch-project-XXXXXX", + cbm_tmpdir()); + (void)snprintf(cache_raw, sizeof(cache_raw), "%s/cbm-app-sensitive-watch-cache-XXXXXX", + cbm_tmpdir()); + bool dirs_ok = cbm_mkdtemp(sensitive_raw) != NULL && cbm_mkdtemp(ordinary_raw) != NULL && + cbm_mkdtemp(cache_raw) != NULL; + char sensitive[APP_TEST_PATH_CAP] = {0}; + char ordinary[APP_TEST_PATH_CAP] = {0}; + char cache[APP_TEST_PATH_CAP] = {0}; + bool canonical = dirs_ok && cbm_canonical_path(sensitive_raw, sensitive, sizeof(sensitive)) && + cbm_canonical_path(ordinary_raw, ordinary, sizeof(ordinary)) && + cbm_canonical_path(cache_raw, cache, sizeof(cache)); + bool environments_set = environments_saved && canonical && + cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0 && + cbm_setenv("HOME", sensitive, 1) == 0; + cbm_config_t *stored_config = environments_set ? cbm_config_open(cache) : NULL; + bool config_ready = stored_config && + cbm_config_set(stored_config, CBM_CONFIG_AUTO_INDEX, "false") == 0 && + cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "true") == 0; + char *sensitive_project = canonical ? cbm_project_name_from_path(sensitive) : NULL; + char *ordinary_project = canonical ? cbm_project_name_from_path(ordinary) : NULL; + char sensitive_db[APP_TEST_PATH_CAP] = {0}; + char ordinary_db[APP_TEST_PATH_CAP] = {0}; + if (sensitive_project) { + (void)snprintf(sensitive_db, sizeof(sensitive_db), "%s/%s.db", cache, sensitive_project); + } + if (ordinary_project) { + (void)snprintf(ordinary_db, sizeof(ordinary_db), "%s/%s.db", cache, ordinary_project); + } + bool db_ready = config_ready && sensitive_project && ordinary_project && + app_test_create_empty_file(sensitive_db) && + app_test_create_empty_file(ordinary_db); + + cbm_store_t *store = cbm_store_open_memory(); + cbm_watcher_t *watcher = cbm_watcher_new(store, app_test_index_noop, NULL); + cbm_daemon_application_config_t config = { + .watcher = watcher, + .config = stored_config, + }; + cbm_daemon_application_t *application = + db_ready && watcher ? cbm_daemon_application_new(&config) : NULL; + cbm_daemon_runtime_application_callbacks_t callbacks = + cbm_daemon_application_runtime_callbacks(application); + cbm_daemon_runtime_application_session_t *sensitive_session = + application ? app_test_open(&callbacks, 4033) : NULL; + cbm_daemon_runtime_application_session_t *ordinary_session = + application ? app_test_open(&callbacks, 4034) : NULL; + cbm_daemon_runtime_application_session_t *approved_session = + application ? app_test_open(&callbacks, 4035) : NULL; + + bool sensitive_initialized = app_test_initialize_profile( + &callbacks, sensitive_session, sensitive, CBM_MCP_TOOL_PROFILE_ALL, NULL, NULL); + bool sensitive_blocked = sensitive_initialized && cbm_watcher_watch_count(watcher) == 0; + bool ordinary_initialized = app_test_initialize_profile(&callbacks, ordinary_session, ordinary, + CBM_MCP_TOOL_PROFILE_ALL, NULL, NULL); + bool ordinary_watched = ordinary_initialized && cbm_watcher_watch_count(watcher) == 1; + if (ordinary_session) { + callbacks.session_close(callbacks.context, ordinary_session); + ordinary_session = NULL; + } + bool ordinary_released = ordinary_watched && cbm_watcher_watch_count(watcher) == 0; + + char grant_error[APP_TEST_PATH_CAP]; + bool approved = ordinary_released && cbm_workspace_grant_add(cache, sensitive, sensitive, true, + grant_error, sizeof(grant_error)); + bool approved_initialized = + approved && app_test_initialize_profile(&callbacks, approved_session, sensitive, + CBM_MCP_TOOL_PROFILE_ALL, NULL, NULL); + bool approved_watched = approved_initialized && cbm_watcher_watch_count(watcher) == 1; + + if (sensitive_session) { + callbacks.session_close(callbacks.context, sensitive_session); + } + if (approved_session) { + callbacks.session_close(callbacks.context, approved_session); + } + if (ordinary_session) { + callbacks.session_close(callbacks.context, ordinary_session); + } + bool watches_released = watcher && cbm_watcher_watch_count(watcher) == 0; + bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + cbm_daemon_application_free(application); + if (watcher) { + cbm_watcher_stop(watcher); + cbm_watcher_free(watcher); + } + cbm_store_close(store); + cbm_config_close(stored_config); + free(sensitive_project); + free(ordinary_project); + (void)th_rmtree(sensitive); + (void)th_rmtree(ordinary); + (void)th_rmtree(cache); + bool cache_restored = app_env_backup_restore(&cache_environment); + bool home_restored = app_env_backup_restore(&home_environment); + + ASSERT_TRUE(environments_saved); + ASSERT_TRUE(dirs_ok); + ASSERT_TRUE(canonical); + ASSERT_TRUE(environments_set); + ASSERT_TRUE(config_ready); + ASSERT_TRUE(db_ready); + ASSERT_TRUE(sensitive_blocked); + ASSERT_TRUE(ordinary_watched); + ASSERT_TRUE(ordinary_released); + ASSERT_TRUE(approved); + ASSERT_TRUE(approved_watched); + ASSERT_TRUE(watches_released); + ASSERT_TRUE(stopped); + ASSERT_TRUE(cache_restored); + ASSERT_TRUE(home_restored); + PASS(); +} + TEST(daemon_application_auto_index_honors_tracked_file_limit) { app_env_backup_t cache_environment; bool cache_saved = app_env_backup_capture(&cache_environment, "CBM_CACHE_DIR"); @@ -5097,6 +5358,8 @@ SUITE(daemon_application) { RUN_TEST(daemon_application_free_releases_live_watch_once); RUN_TEST(daemon_application_prune_clears_logical_watch_for_reregistration); RUN_TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions); + RUN_TEST(daemon_application_sensitive_root_blocks_auto_index_but_preserves_controls); + RUN_TEST(daemon_application_sensitive_root_blocks_watch_but_preserves_controls); RUN_TEST(daemon_application_auto_index_honors_tracked_file_limit); RUN_TEST(daemon_application_auto_index_file_count_handles_literal_metacharacter_path); RUN_TEST(daemon_application_auto_index_file_count_supports_non_git_roots); From fde1695de2849862fff267fd5d6a3f144b26b07c Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 27 Aug 2026 12:39:30 +0200 Subject: [PATCH 05/31] fix: ignore trailing Dockerfile whitespace at EOF Signed-off-by: Martin Vogel --- internal/cbm/cbm.c | 36 +++++++++++++++++++------ tests/test_parse_coverage.c | 53 +++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 8 deletions(-) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 6c46d7c91..85c0ba97b 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -810,17 +810,37 @@ static void cbm_error_regions_push(cbm_error_regions_t *acc, TSNode n) { * Deliberately narrow — ZERO-WIDTH AT EOF ONLY. A MISSING or ERROR node with * WIDTH still counts even at EOF (a Makefile whose last recipe line is * unterminated really does lose the recipe), and anything before EOF is - * untouched. */ -static bool cbm_is_eof_terminator_miss(TSNode n, int source_len) { + * untouched. + * + * #1746: the Dockerfile grammar places that zero-width missing newline before + * trailing horizontal whitespace rather than at raw EOF. Preserve the broad + * exact-EOF rule above; only extend it past spaces/tabs when the missing token + * is specifically a newline. */ +static bool cbm_is_eof_terminator_miss(TSNode n, const char *source, int source_len) { if (!ts_node_is_missing(n) || source_len < 0) { return false; } uint32_t start = ts_node_start_byte(n); uint32_t end = ts_node_end_byte(n); - return start == end && end == (uint32_t)source_len; + if (start != end || end > (uint32_t)source_len) { + return false; + } + if (end == (uint32_t)source_len) { + return true; + } + if (!source || strcmp(ts_node_type(n), "\n") != 0) { + return false; + } + for (uint32_t i = end; i < (uint32_t)source_len; i++) { + if (source[i] != ' ' && source[i] != '\t') { + return false; + } + } + return true; } -static void cbm_collect_error_regions(TSNode n, cbm_error_regions_t *acc, int source_len) { +static void cbm_collect_error_regions(TSNode n, cbm_error_regions_t *acc, const char *source, + int source_len) { if (acc->count >= CBM_MAX_ERROR_REGIONS) { return; } @@ -828,12 +848,12 @@ static void cbm_collect_error_regions(TSNode n, cbm_error_regions_t *acc, int so for (uint32_t i = 0; i < k && acc->count < CBM_MAX_ERROR_REGIONS; i++) { TSNode c = ts_node_child(n, i); if (ts_node_is_missing(c) || strcmp(ts_node_type(c), "ERROR") == 0) { - if (cbm_is_eof_terminator_miss(c, source_len)) { + if (cbm_is_eof_terminator_miss(c, source, source_len)) { continue; /* absent final newline only — nothing was dropped */ } cbm_error_regions_push(acc, c); /* top-most region; do not descend */ } else if (ts_node_has_error(c)) { - cbm_collect_error_regions(c, acc, source_len); + cbm_collect_error_regions(c, acc, source, source_len); } } } @@ -1456,7 +1476,7 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua * already extract. */ if (ts_node_has_error(root)) { cbm_error_regions_t raw_regs = {{0}, {0}, 0}; - cbm_collect_error_regions(root, &raw_regs, source_len); + cbm_collect_error_regions(root, &raw_regs, source, source_len); if (raw_regs.count > 0) { int defs_before = result->defs.count; cbm_extract_definitions(&pp_ctx); @@ -1614,7 +1634,7 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua if (strcmp(ts_node_type(root), "ERROR") == 0) { cbm_error_regions_push(®s, root); /* whole file unparseable */ } else { - cbm_collect_error_regions(root, ®s, source_len); + cbm_collect_error_regions(root, ®s, source, source_len); } cbm_subtract_recovered_regions(®s, &result->defs); /* #1071: don't flag a benign function-like-macro call (defined in-file) diff --git a/tests/test_parse_coverage.c b/tests/test_parse_coverage.c index 1c0720c94..423406612 100644 --- a/tests/test_parse_coverage.c +++ b/tests/test_parse_coverage.c @@ -295,6 +295,55 @@ TEST(dockerfile_with_final_newline_still_clean_issue1610) { PASS(); } +/* #1746: on Windows, a Dockerfile whose final instruction is followed by one + * ASCII space and then EOF was reported as parse_partial. */ +TEST(dockerfile_trailing_space_at_eof_not_flagged_issue1746) { + const char *src = "FROM scratch\nENTRYPOINT [\"a\"] "; + CBMFileResult *r = do_extract(src, CBM_LANG_DOCKERFILE, "Dockerfile"); + ASSERT_NOT_NULL(r); + bool flagged = r->parse_incomplete; + if (flagged) { + fprintf(stderr, " exact issue #1746 fixture flagged: ranges=%s\n", + r->error_ranges ? r->error_ranges : "(none)"); + } + cbm_free_result(r); + if (flagged) { + FAIL("trailing horizontal whitespace at Dockerfile EOF must not be parse_partial"); + } + PASS(); +} + +/* The same bytes WITH the LF must stay clean. This is a separate test so the + * control executes even while the exact affected fixture is RED. */ +TEST(dockerfile_trailing_space_with_final_newline_clean_issue1746) { + const char *src = "FROM scratch\nENTRYPOINT [\"a\"] \n"; + CBMFileResult *r = do_extract(src, CBM_LANG_DOCKERFILE, "Dockerfile"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->parse_incomplete); + cbm_free_result(r); + PASS(); +} + +/* Removing the trailing space while retaining EOF must also stay clean. */ +TEST(dockerfile_without_trailing_space_at_eof_clean_issue1746) { + const char *src = "FROM scratch\nENTRYPOINT [\"a\"]"; + CBMFileResult *r = do_extract(src, CBM_LANG_DOCKERFILE, "Dockerfile"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->parse_incomplete); + cbm_free_result(r); + PASS(); +} + +/* The reporter also observed the same failure when the first line uses CRLF. */ +TEST(dockerfile_crlf_trailing_space_at_eof_not_flagged_issue1746) { + const char *src = "FROM scratch\r\nENTRYPOINT [\"a\"] "; + CBMFileResult *r = do_extract(src, CBM_LANG_DOCKERFILE, "Dockerfile"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->parse_incomplete); + cbm_free_result(r); + PASS(); +} + /* Language-general, not a Dockerfile patch: these four were each proven to flag * on a stripped trailing newline. */ TEST(missing_final_newline_not_flagged_across_grammars_issue1610) { @@ -381,6 +430,10 @@ SUITE(parse_coverage) { RUN_TEST(c_trailing_recovered_defs_keep_flag); RUN_TEST(dockerfile_missing_final_newline_not_flagged_issue1610); RUN_TEST(dockerfile_with_final_newline_still_clean_issue1610); + RUN_TEST(dockerfile_trailing_space_at_eof_not_flagged_issue1746); + RUN_TEST(dockerfile_trailing_space_with_final_newline_clean_issue1746); + RUN_TEST(dockerfile_without_trailing_space_at_eof_clean_issue1746); + RUN_TEST(dockerfile_crlf_trailing_space_at_eof_not_flagged_issue1746); RUN_TEST(missing_final_newline_not_flagged_across_grammars_issue1610); RUN_TEST(real_error_before_eof_still_flagged_without_final_newline_issue1610); RUN_TEST(width_bearing_error_at_eof_still_flagged_issue1610); From ee32816e3ce59855262df3d738b96e13931da131 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 27 Aug 2026 14:03:22 +0200 Subject: [PATCH 06/31] fix(pi): send tool arguments over stdin Signed-off-by: Martin Vogel --- scripts/smoke-test.sh | 100 +++++++++++++++++++++++++++++++++---- src/cli/client_adapter.c | 10 +++- src/cli/client_adapter.h | 5 +- tests/test_agent_clients.c | 39 +++++++++++++++ 4 files changed, 141 insertions(+), 13 deletions(-) diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index 53dc130ae..7eb24864c 100755 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -838,15 +838,16 @@ else echo; echo "-- B3 first bytes (od) --"; echo "$IM_ARR" | od -c | head -6; exit 1 fi -# B4: STDIN — piped JSON resolves; this path must NOT emit a deprecation warning. -IM_STDIN=$(echo "{\"project\":\"$PROJECT\"}" | "$BINARY" cli get_graph_schema 2>"$CLI_STDERR") -if ! echo "$IM_STDIN" | python3 -c "import json,sys; d=json.loads(sys.stdin.read()); sys.exit(0 if 'node_labels' in d else 1)" 2>/dev/null; then - echo "FAIL B4: stdin get_graph_schema did not resolve"; echo "$IM_STDIN" | head -c 300; cat "$CLI_STDERR"; exit 1 +# B4: STDIN + --json is the generated-client transport. It must return the +# complete MCP result envelope and must NOT emit a deprecation warning. +IM_STDIN=$(printf '%s' "{\"project\":\"$PROJECT\"}" | "$BINARY" cli --json get_graph_schema 2>"$CLI_STDERR") +if ! printf '%s' "$IM_STDIN" | python3 -c "import json,sys; d=json.loads(sys.stdin.read()); c=d.get('content'); p=json.loads(c[0].get('text','')) if isinstance(c,list) and c and c[0].get('type') == 'text' else None; sys.exit(0 if d.get('isError') is not True and isinstance(p,dict) and isinstance(p.get('node_labels'),list) else 1)" 2>/dev/null; then + echo "FAIL B4: compact stdin + --json did not return a successful get_graph_schema MCP payload"; echo "$IM_STDIN" | head -c 300; cat "$CLI_STDERR"; exit 1 fi if grep -qi 'deprecated' "$CLI_STDERR"; then echo "FAIL B4: stdin path wrongly emitted a deprecation warning"; cat "$CLI_STDERR"; exit 1 fi -echo "OK B4: STDIN input resolves, no deprecation warning" +echo "OK B4: compact STDIN + --json returns a successful schema MCP envelope, no deprecation warning" # B5: --args-file — JSON read from a file resolves; must NOT warn deprecated. IM_ARGS_FILE=$(smoke_mktemp_file) @@ -890,7 +891,8 @@ else echo "FAIL B6c: 'notatool --help' did not report 'unknown tool'"; cat "$CLI_STDERR"; exit 1 fi -# B7: DEPRECATION guard — one raw-JSON call MUST warn on stderr; flag form must NOT. +# B7: DEPRECATION control — positional raw JSON MUST warn on stderr; this is +# retained only to prove B4 is exercising the non-deprecated stdin transport. cli search_graph "{\"project\":\"$PROJECT\",\"name_pattern\":\"compute\"}" >/dev/null || true if grep -qi 'deprecated' "$CLI_STDERR"; then echo "OK B7a: raw-JSON cli emits deprecation warning on stderr" @@ -2302,16 +2304,96 @@ if ! path_match "$CMD" "$SELF_PATH" || fi echo "OK 8ak: custom KIMI_CODE_HOME MCP + durable context + UserPromptSubmit hook" -# 8al: Pi has documented instructions and skill, but no invented MCP config. +# 8al: Pi has documented instructions and skill, no invented MCP config, and +# an installed generated extension that uses the non-deprecated stdin bridge. PI_INSTRUCTIONS="$FAKE_HOME/.pi/agent/AGENTS.md" PI_SKILL="$FAKE_HOME/.pi/agent/skills/codebase-memory/SKILL.md" +PI_EXTENSION="$FAKE_HOME/.pi/agent/extensions/cbmem.ts" if ! grep -q 'search_graph' "$PI_INSTRUCTIONS" 2>/dev/null || ! grep -q 'Sessions and Subagents' "$PI_SKILL" 2>/dev/null || + ! grep -Fq "spawn(BIN, ['cli', '--json', tool], {" "$PI_EXTENSION" 2>/dev/null || + ! grep -Fq "stdio: ['pipe', 'pipe', 'pipe']" "$PI_EXTENSION" 2>/dev/null || + ! grep -Fq "child.stdin.on('error'" "$PI_EXTENSION" 2>/dev/null || + ! grep -Fq 'child.stdin.end(JSON.stringify(args ?? {}));' "$PI_EXTENSION" 2>/dev/null || + grep -Fq 'tool, JSON.stringify(args ?? {})]' "$PI_EXTENSION" 2>/dev/null || + grep -Fq "stdio: ['ignore', 'pipe', 'pipe']" "$PI_EXTENSION" 2>/dev/null || [ -e "$FAKE_HOME/.pi/agent/mcp.json" ]; then - echo "FAIL 8al: Pi durable context missing or unsupported MCP config created" + echo "FAIL 8al: Pi durable context or stdin client bridge missing, or unsupported MCP config created" exit 1 fi -echo "OK 8al: Pi durable context only (no MCP config)" +echo "OK 8al: Pi durable context + stdin client bridge (no MCP config)" + +# 8al-node: execute the generated extension against a child that exits without +# consuming a deliberately over-pipe-capacity payload. This is the lifecycle +# Node implements: without an stdin error listener, the late EPIPE is an +# unhandled EventEmitter error and crashes the Pi host. A second call emits a +# valid MCP envelope before the same early exit, proving parsed JSON remains +# authoritative over a retained stdin transport error. +# +# SKIP_WHITELIST: the minimal C-only Linux image intentionally has no Node. +# What was tried: making Node a universal core-suite prerequisite would widen +# the product's C build dependencies. The exact generated-source assertions +# above still run there; this live probe gates every Node-equipped smoke venue. +if command -v node >/dev/null 2>&1; then + PI_NODE=$(command -v node) + PI_PROBE_DIR="$TMPDIR/pi-node-probe" + mkdir -p "$PI_PROBE_DIR" + python3 - "$PI_EXTENSION" "$PI_PROBE_DIR/cbmem.mjs" <<'PYPIADAPTER' +import pathlib +import sys + +source, destination = map(pathlib.Path, sys.argv[1:]) +text = source.read_text(encoding="utf-8") +lines = text.splitlines() +matches = [i for i, line in enumerate(lines) if line.startswith("const BIN = ")] +if len(matches) != 1: + raise SystemExit("generated Pi extension has no unique BIN declaration") +lines[matches[0]] = "const BIN = process.execPath;" +destination.write_text("\n".join(lines) + "\n", encoding="utf-8") +PYPIADAPTER + cat >"$PI_PROBE_DIR/cli" <<'PICHILD' +const fs = require('node:fs'); +if (process.argv[3] === 'get_graph_schema') { + fs.writeSync(1, JSON.stringify({ content: [{ type: 'text', text: 'schema-ok' }] }) + '\n'); +} +process.exit(0); +PICHILD + if ! grep -Fxq 'const BIN = process.execPath;' "$PI_PROBE_DIR/cbmem.mjs" || + ! grep -Fxq "const fs = require('node:fs');" "$PI_PROBE_DIR/cli"; then + echo "FAIL 8al-node: generated lifecycle probe is not portable across Node launch environments" + exit 1 + fi + cat >"$PI_PROBE_DIR/probe.mjs" <<'PIPROBE' +import extension from './cbmem.mjs'; + +const tools = []; +extension({ registerTool: (definition) => tools.push(definition) }); +const byName = (name) => tools.find((tool) => tool.name === name); +const args = { padding: 'x'.repeat(16 * 1024 * 1024) }; + +const successful = await byName('get_graph_schema').execute('probe-ok', args); +if (successful?.content?.[0]?.text !== 'schema-ok') { + throw new Error('valid child JSON was not authoritative over stdin EPIPE'); +} + +let transportError = ''; +try { + await byName('search_graph').execute('probe-error', args); +} catch (error) { + transportError = String(error?.message ?? error); +} +if (!/(EPIPE|broken pipe|write)/i.test(transportError)) { + throw new Error(`missing surfaced stdin transport error: ${transportError || ''}`); +} +PIPROBE + if ! (cd "$PI_PROBE_DIR" && "$PI_NODE" probe.mjs); then + echo "FAIL 8al-node: generated Pi extension did not contain early-exit stdin errors" + exit 1 + fi + echo "OK 8al-node: generated Pi extension contains EPIPE and keeps valid JSON authoritative" +else + echo "SKIP 8al-node: Node unavailable in this C-only smoke venue (whitelisted above)" +fi # 8am: Warp receives the documented shared skill; MCP remains user/UI-managed. WARP_SKILL="$FAKE_HOME/.agents/skills/codebase-memory/SKILL.md" diff --git a/src/cli/client_adapter.c b/src/cli/client_adapter.c index cc7454793..4d136bb5e 100644 --- a/src/cli/client_adapter.c +++ b/src/cli/client_adapter.c @@ -168,14 +168,16 @@ char *cbm_client_adapter_pi(const char *binary_path) { sb_append( &sb, "async function call(tool, args, signal) {\n" " return new Promise((resolve) => {\n" - " const child = spawn(BIN, ['cli', '--json', tool, JSON.stringify(args ?? {})], {\n" - " stdio: ['ignore', 'pipe', 'pipe'],\n" + " const child = spawn(BIN, ['cli', '--json', tool], {\n" + " stdio: ['pipe', 'pipe', 'pipe'],\n" " env: { ...process.env, CBM_LOG_LEVEL: 'error' },\n" " });\n" " let out = '';\n" + " let stdinError;\n" " const onAbort = () => { if (!child.killed) child.kill(); };\n" " signal?.addEventListener('abort', onAbort, { once: true });\n" " child.stdout.on('data', (d) => (out += d.toString()));\n" + " child.stdin.on('error', (e) => { stdinError = e; });\n" " child.on('error', (e) => {\n" " signal?.removeEventListener('abort', onAbort);\n" " resolve({ error: String(e && e.message ? e.message : e) });\n" @@ -186,8 +188,12 @@ char *cbm_client_adapter_pi(const char *binary_path) { " for (let i = lines.length - 1; i >= 0; i--) {\n" " try { return resolve(JSON.parse(lines[i])); } catch { /* keep scanning */ }\n" " }\n" + " if (stdinError) {\n" + " return resolve({ error: String(stdinError.message || stdinError) });\n" + " }\n" " resolve({ error: 'no JSON response from codebase-memory-mcp' });\n" " });\n" + " child.stdin.end(JSON.stringify(args ?? {}));\n" " });\n" "}\n\n"); diff --git a/src/cli/client_adapter.h b/src/cli/client_adapter.h index f0c812e62..c1bfb652e 100644 --- a/src/cli/client_adapter.h +++ b/src/cli/client_adapter.h @@ -30,8 +30,9 @@ /* Emit a pi extension registering every tool in the MCP registry. * * pi has no MCP client, so this bridge is the only route to the graph for it. - * Each registered tool shells out to ` cli ''`, which - * is an existing public entry point rather than a private one. + * Each registered tool shells out to ` cli --json ` and writes + * its JSON arguments to stdin, using an existing public entry point without + * the deprecated positional-JSON transport. * * `binary_path` is embedded as a JS string literal and is escaped; a Windows * path such as C:\Users\x\bin\cbm.exe must not corrupt the module or, worse, diff --git a/tests/test_agent_clients.c b/tests/test_agent_clients.c index ceb717e18..519cbed93 100644 --- a/tests/test_agent_clients.c +++ b/tests/test_agent_clients.c @@ -1139,6 +1139,44 @@ TEST(client_adapter_pi_emits_parameters_and_execute) { PASS(); } +/* #1834: raw JSON in argv selects the deprecated compatibility path. The Pi + * bridge must keep arguments out of process listings and feed the existing + * stdin transport only after all child handlers are attached. */ +TEST(client_adapter_pi_sends_arguments_over_stdin_issue1834) { + char *js = cbm_client_adapter_pi("/usr/local/bin/codebase-memory-mcp"); + ASSERT_NOT_NULL(js); + + const char *spawn = strstr(js, "spawn(BIN, ['cli', '--json', tool], {"); + const char *stdio = strstr(js, "stdio: ['pipe', 'pipe', 'pipe']"); + const char *stdout_handler = strstr(js, "child.stdout.on('data'"); + const char *stdin_error_handler = strstr(js, "child.stdin.on('error'"); + const char *error_handler = strstr(js, "child.on('error'"); + const char *close_handler = strstr(js, "child.on('close'"); + const char *stdin_end = strstr(js, "child.stdin.end(JSON.stringify(args ?? {}));"); + + ASSERT_NOT_NULL(spawn); + ASSERT_NOT_NULL(stdio); + ASSERT_NOT_NULL(stdout_handler); + ASSERT_NOT_NULL(stdin_error_handler); + ASSERT_NOT_NULL(error_handler); + ASSERT_NOT_NULL(close_handler); + ASSERT_NOT_NULL(stdin_end); + ASSERT(stdout_handler < stdin_end); + ASSERT(stdin_error_handler < stdin_end); + ASSERT(error_handler < stdin_end); + ASSERT(close_handler < stdin_end); + ASSERT_NOT_NULL(strstr(js, "let stdinError;")); + ASSERT_NOT_NULL(strstr(js, "stdinError = e;")); + ASSERT_NOT_NULL(strstr(js, "if (stdinError)")); + + /* These are the exact deprecated transport shapes being retired. */ + ASSERT_NULL(strstr(js, "tool, JSON.stringify(args ?? {})]")); + ASSERT_NULL(strstr(js, "stdio: ['ignore', 'pipe', 'pipe']")); + + free(js); + PASS(); +} + /* Result wrap + abort: Pi's TUI reads result.content; a spawn/parse failure * must throw rather than return a result without content; AbortSignal must * kill the cli child. String-shape only; a live Pi process is not gated. */ @@ -1248,6 +1286,7 @@ SUITE(agent_clients) { RUN_TEST(client_adapter_pi_registers_every_registry_tool); RUN_TEST(client_adapter_pi_default_exports_its_factory_issue1550); RUN_TEST(client_adapter_pi_emits_parameters_and_execute); + RUN_TEST(client_adapter_pi_sends_arguments_over_stdin_issue1834); RUN_TEST(client_adapter_pi_wraps_result_and_honors_abort); RUN_TEST(client_adapter_escapes_windows_paths_and_quotes); RUN_TEST(client_adapter_opencode_sends_the_required_hook_event); From 7710f86e927c5221c395ecbdab9d0f3b75edcb71 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 27 Aug 2026 02:49:11 +0200 Subject: [PATCH 07/31] fix(daemon): avoid duplicate idle maintenance polling Signed-off-by: Martin Vogel --- src/daemon/frontend.c | 109 ++++++++++++----- src/daemon/frontend.h | 12 ++ tests/test_daemon_frontend.c | 222 +++++++++++++++++++++++++++++++++++ 3 files changed, 316 insertions(+), 27 deletions(-) diff --git a/src/daemon/frontend.c b/src/daemon/frontend.c index f5116934c..17e0a8d2f 100644 --- a/src/daemon/frontend.c +++ b/src/daemon/frontend.c @@ -93,23 +93,83 @@ struct cbm_daemon_maintenance_monitor { cbm_daemon_maintenance_cancel_fn cancel; void *cancel_context; atomic_bool stopping; + bool exit_when_cancel_not_needed; int exit_code; char participant[FRONTEND_PARTICIPANT_NAME_CAP]; }; +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS +static atomic_bool frontend_test_hold_monitor = ATOMIC_VAR_INIT(false); +static atomic_bool frontend_test_monitor_is_waiting = ATOMIC_VAR_INIT(false); +static atomic_uint_fast64_t frontend_test_monitor_observation_count = ATOMIC_VAR_INIT(0); +static atomic_uint_fast64_t frontend_test_worker_observation_count = ATOMIC_VAR_INIT(0); +static atomic_uint_fast64_t frontend_test_worker_idle_count = ATOMIC_VAR_INIT(0); + +void cbm_daemon_frontend_test_observer_reset(bool hold_monitor) { + atomic_store_explicit(&frontend_test_monitor_observation_count, 0, memory_order_release); + atomic_store_explicit(&frontend_test_worker_observation_count, 0, memory_order_release); + atomic_store_explicit(&frontend_test_worker_idle_count, 0, memory_order_release); + atomic_store_explicit(&frontend_test_monitor_is_waiting, false, memory_order_release); + atomic_store_explicit(&frontend_test_hold_monitor, hold_monitor, memory_order_release); +} + +void cbm_daemon_frontend_test_observer_release(void) { + atomic_store_explicit(&frontend_test_hold_monitor, false, memory_order_release); +} + +bool cbm_daemon_frontend_test_monitor_waiting(void) { + return atomic_load_explicit(&frontend_test_monitor_is_waiting, memory_order_acquire); +} + +uint64_t cbm_daemon_frontend_test_monitor_observations(void) { + return atomic_load_explicit(&frontend_test_monitor_observation_count, memory_order_acquire); +} + +uint64_t cbm_daemon_frontend_test_worker_observations(void) { + return atomic_load_explicit(&frontend_test_worker_observation_count, memory_order_acquire); +} + +uint64_t cbm_daemon_frontend_test_worker_idle_cycles(void) { + return atomic_load_explicit(&frontend_test_worker_idle_count, memory_order_acquire); +} +#endif + +static cbm_version_cohort_maintenance_presence_t frontend_observe_maintenance( + cbm_version_cohort_manager_t *manager, bool independent_monitor) { +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS + if (independent_monitor) { + atomic_fetch_add_explicit(&frontend_test_monitor_observation_count, 1, + memory_order_release); + if (atomic_load_explicit(&frontend_test_hold_monitor, memory_order_acquire)) { + atomic_store_explicit(&frontend_test_monitor_is_waiting, true, memory_order_release); + while (atomic_load_explicit(&frontend_test_hold_monitor, memory_order_acquire)) { + cbm_usleep(FRONTEND_WAIT_US); + } + atomic_store_explicit(&frontend_test_monitor_is_waiting, false, memory_order_release); + } + } else { + atomic_fetch_add_explicit(&frontend_test_worker_observation_count, 1, memory_order_release); + } +#else + (void)independent_monitor; +#endif + return cbm_version_cohort_maintenance_presence_terminal(manager); +} + static void *frontend_maintenance_monitor_worker(void *opaque) { cbm_daemon_maintenance_monitor_t *monitor = opaque; while (!atomic_load_explicit(&monitor->stopping, memory_order_acquire)) { cbm_version_cohort_maintenance_presence_t presence = - cbm_version_cohort_maintenance_presence_terminal(monitor->manager); + frontend_observe_maintenance(monitor->manager, true); if (presence == CBM_VERSION_COHORT_MAINTENANCE_ABSENT) { cbm_usleep(FRONTEND_MAINTENANCE_POLL_MS * 1000U); continue; } if (presence == CBM_VERSION_COHORT_MAINTENANCE_REQUESTED) { + bool cancel_requested = false; if (monitor->cancel) { - (void)monitor->cancel(monitor->cancel_context); + cancel_requested = monitor->cancel(monitor->cancel_context); } /* Never log, write to, or flush agent stdio from this monitor. * Structured logging itself writes to stderr, and this thread's @@ -117,6 +177,9 @@ static void *frontend_maintenance_monitor_worker(void *opaque) { * thread is blocked on a full stdout/stderr pipe. The activation * owner records the maintenance event durably. */ + if (monitor->exit_when_cancel_not_needed && !cancel_requested) { + _Exit(monitor->exit_code); + } uint64_t now = cbm_now_ms(); uint64_t deadline = now > UINT64_MAX - FRONTEND_MAINTENANCE_GRACE_MS ? UINT64_MAX @@ -139,9 +202,10 @@ static void *frontend_maintenance_monitor_worker(void *opaque) { return NULL; } -cbm_daemon_maintenance_monitor_t *cbm_daemon_maintenance_monitor_start( +static cbm_daemon_maintenance_monitor_t *frontend_maintenance_monitor_start( cbm_version_cohort_manager_t *manager, cbm_daemon_maintenance_cancel_fn cancel, - void *cancel_context, int exit_code, const char *participant) { + void *cancel_context, int exit_code, const char *participant, + bool exit_when_cancel_not_needed) { if (!manager || exit_code < 0 || !participant || !participant[0]) { return NULL; } @@ -152,6 +216,7 @@ cbm_daemon_maintenance_monitor_t *cbm_daemon_maintenance_monitor_start( monitor->manager = manager; monitor->cancel = cancel; monitor->cancel_context = cancel_context; + monitor->exit_when_cancel_not_needed = exit_when_cancel_not_needed; monitor->exit_code = exit_code; atomic_init(&monitor->stopping, false); int written = snprintf(monitor->participant, sizeof(monitor->participant), "%s", participant); @@ -163,6 +228,13 @@ cbm_daemon_maintenance_monitor_t *cbm_daemon_maintenance_monitor_start( return monitor; } +cbm_daemon_maintenance_monitor_t *cbm_daemon_maintenance_monitor_start( + cbm_version_cohort_manager_t *manager, cbm_daemon_maintenance_cancel_fn cancel, + void *cancel_context, int exit_code, const char *participant) { + return frontend_maintenance_monitor_start(manager, cancel, cancel_context, exit_code, + participant, false); +} + bool cbm_daemon_maintenance_monitor_stop(cbm_daemon_maintenance_monitor_t **monitor_io) { if (!monitor_io || !*monitor_io) { return false; @@ -177,19 +249,6 @@ bool cbm_daemon_maintenance_monitor_stop(cbm_daemon_maintenance_monitor_t **moni return true; } -static void frontend_exit_for_maintenance(frontend_state_t *state) { - cbm_version_cohort_maintenance_presence_t presence = - cbm_version_cohort_maintenance_presence_terminal(state->cohort_manager); - if (presence == CBM_VERSION_COHORT_MAINTENANCE_ABSENT) { - return; - } - /* Do not fclose stdin across threads. Process exit closes the authenticated - * kernel IPC handle, and daemon ownership then cancels only this session. - * Agent stdout/stderr may both be backpressured, so terminal paths must not - * log, write, or flush before fail-stop. */ - _Exit(presence == CBM_VERSION_COHORT_MAINTENANCE_REQUESTED ? EXIT_SUCCESS : EXIT_FAILURE); -} - static void frontend_item_free(frontend_item_t *item) { if (item) { free(item->message); @@ -414,20 +473,15 @@ static frontend_cancellation_route_t frontend_route_cancellation( static void *frontend_worker(void *opaque) { frontend_state_t *state = opaque; - uint64_t next_maintenance_check_ms = 0; for (;;) { - uint64_t now_ms = cbm_now_ms(); - if (now_ms >= next_maintenance_check_ms) { - frontend_exit_for_maintenance(state); - next_maintenance_check_ms = now_ms > UINT64_MAX - FRONTEND_MAINTENANCE_POLL_MS - ? UINT64_MAX - : now_ms + FRONTEND_MAINTENANCE_POLL_MS; - } frontend_item_t item = {0}; if (!frontend_pop_begin(state, &item)) { if (frontend_should_stop(state)) { break; } +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS + atomic_fetch_add_explicit(&frontend_test_worker_idle_count, 1, memory_order_release); +#endif cbm_usleep(FRONTEND_IDLE_WAIT_US); continue; } @@ -583,8 +637,9 @@ int cbm_daemon_frontend_mcp_run(cbm_daemon_runtime_client_t *client, }; cbm_mutex_init(&state.mutex); atomic_init(&state.completed_items, 0); - cbm_daemon_maintenance_monitor_t *maintenance_monitor = cbm_daemon_maintenance_monitor_start( - cohort_manager, frontend_cancel_for_maintenance, &state, EXIT_SUCCESS, "MCP frontend"); + cbm_daemon_maintenance_monitor_t *maintenance_monitor = + frontend_maintenance_monitor_start(cohort_manager, frontend_cancel_for_maintenance, &state, + EXIT_SUCCESS, "MCP frontend", true); if (!maintenance_monitor) { cbm_mutex_destroy(&state.mutex); (void)cbm_daemon_runtime_client_close(client, FRONTEND_CLOSE_TIMEOUT_MS); diff --git a/src/daemon/frontend.h b/src/daemon/frontend.h index d114c8f9c..f23395774 100644 --- a/src/daemon/frontend.h +++ b/src/daemon/frontend.h @@ -42,6 +42,18 @@ cbm_daemon_maintenance_monitor_t *cbm_daemon_maintenance_monitor_start( void *cancel_context, int exit_code, const char *participant); bool cbm_daemon_maintenance_monitor_stop(cbm_daemon_maintenance_monitor_t **monitor_io); +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS +/* Deterministic observer accounting for the idle-frontend regression test. + * The hold is inert until reset(true), and production builds contain neither + * these controls nor their counters. */ +void cbm_daemon_frontend_test_observer_reset(bool hold_monitor); +void cbm_daemon_frontend_test_observer_release(void); +bool cbm_daemon_frontend_test_monitor_waiting(void); +uint64_t cbm_daemon_frontend_test_monitor_observations(void); +uint64_t cbm_daemon_frontend_test_worker_observations(void); +uint64_t cbm_daemon_frontend_test_worker_idle_cycles(void); +#endif + /* Takes ownership of client and borrows cohort_manager for the complete call. * A dedicated reader keeps observing stdin while one joinable worker performs * daemon requests. An independent maintenance monitor remains runnable when diff --git a/tests/test_daemon_frontend.c b/tests/test_daemon_frontend.c index d6b066b3b..ed0e868e9 100644 --- a/tests/test_daemon_frontend.c +++ b/tests/test_daemon_frontend.c @@ -34,6 +34,23 @@ typedef struct { cbm_version_cohort_manager_t *manager; } frontend_maintenance_fixture_t; +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS +typedef struct { + frontend_maintenance_fixture_t maintenance; + char conflict_log[FRONTEND_TEST_PATH_CAP]; + cbm_daemon_runtime_service_t *service; + cbm_daemon_runtime_client_t *client; +} frontend_idle_fixture_t; + +typedef struct { + cbm_daemon_runtime_client_t *client; + cbm_version_cohort_manager_t *manager; + FILE *input; + FILE *output; + int result; +} frontend_idle_run_t; +#endif + static bool frontend_maintenance_fixture_start(frontend_maintenance_fixture_t *fixture, const char *tag) { memset(fixture, 0, sizeof(*fixture)); @@ -59,6 +76,148 @@ static void frontend_maintenance_fixture_finish(frontend_maintenance_fixture_t * memset(fixture, 0, sizeof(*fixture)); } +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS +static cbm_daemon_runtime_application_session_t *frontend_idle_session_open( + void *context, cbm_daemon_client_id_t client_id, uint64_t authenticated_process_id) { + return context && client_id != CBM_DAEMON_CLIENT_ID_INVALID && authenticated_process_id != 0 + ? (cbm_daemon_runtime_application_session_t *)context + : NULL; +} + +static cbm_daemon_runtime_application_status_t frontend_idle_request( + void *context, cbm_daemon_runtime_application_session_t *session, + cbm_daemon_runtime_application_token_t request_token, const uint8_t *request, + uint32_t request_length, uint8_t **response_out, uint32_t *response_length_out) { + (void)request_token; + if (!context || session != (cbm_daemon_runtime_application_session_t *)context || + !response_out || !response_length_out || (request_length > 0 && !request)) { + return CBM_DAEMON_RUNTIME_APPLICATION_HANDLER_ERROR; + } + *response_out = NULL; + *response_length_out = 0; + return CBM_DAEMON_RUNTIME_APPLICATION_OK; +} + +static void frontend_idle_request_cancel( + void *context, cbm_daemon_runtime_application_session_t *session, + cbm_daemon_runtime_application_token_t request_token) { + (void)context; + (void)session; + (void)request_token; +} + +static void frontend_idle_session_cancel( + void *context, cbm_daemon_runtime_application_session_t *session) { + (void)context; + (void)session; +} + +static void frontend_idle_session_close( + void *context, cbm_daemon_runtime_application_session_t *session) { + (void)context; + (void)session; +} + +static uint64_t frontend_test_process_id(void) { +#ifdef _WIN32 + return (uint64_t)GetCurrentProcessId(); +#else + return (uint64_t)getpid(); +#endif +} + +static bool frontend_idle_fixture_start(frontend_idle_fixture_t *fixture) { + static const char cache[] = + "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"; + memset(fixture, 0, sizeof(*fixture)); + if (!frontend_maintenance_fixture_start(&fixture->maintenance, "idle-observer")) { + return false; + } + char build[CBM_DAEMON_BUILD_FINGERPRINT_SIZE]; + int log_written = snprintf(fixture->conflict_log, sizeof(fixture->conflict_log), + "%s/conflicts.ndjson", fixture->maintenance.parent); + if (log_written <= 0 || log_written >= (int)sizeof(fixture->conflict_log) || + !cbm_daemon_runtime_process_build_fingerprint(frontend_test_process_id(), build)) { + return false; + } + cbm_daemon_build_identity_t identity = { + .semantic_version = "2.4.0", + .build_fingerprint = build, + .cache_fingerprint = cache, + .protocol_abi = 3, + .store_abi = 11, + .feature_abi = 7, + }; + cbm_daemon_runtime_application_callbacks_t callbacks = { + .context = fixture, + .session_open = frontend_idle_session_open, + .request = frontend_idle_request, + .request_cancel = frontend_idle_request_cancel, + .session_cancel = frontend_idle_session_cancel, + .session_close = frontend_idle_session_close, + }; + cbm_daemon_runtime_service_config_t config = { + .endpoint = fixture->maintenance.endpoint, + .identity = identity, + .conflict_log_path = fixture->conflict_log, + .conflict_log_cap_bytes = 64U * 1024U, + .max_clients = 2, + .lease_timeout_ms = 5000, + .request_timeout_ms = 30000, + .shutdown_timeout_ms = 30000, + .application = callbacks, + }; + fixture->service = cbm_daemon_runtime_service_start(&config); + cbm_daemon_runtime_connect_result_t connect_result = {0}; + fixture->client = fixture->service + ? cbm_daemon_runtime_client_connect(fixture->maintenance.endpoint, &identity, + 30000, &connect_result) + : NULL; + return fixture->client && connect_result.status == CBM_DAEMON_RUNTIME_CONNECT_ACCEPTED; +} + +static bool frontend_idle_fixture_finish(frontend_idle_fixture_t *fixture) { + bool ok = true; + if (fixture->client) { + ok = cbm_daemon_runtime_client_close(fixture->client, 30000) && ok; + fixture->client = NULL; + } + if (fixture->service) { + if (cbm_daemon_runtime_service_state(fixture->service) != + CBM_DAEMON_RUNTIME_SERVICE_EXITED) { + ok = cbm_daemon_runtime_service_stop(fixture->service, 30000) && ok; + } + ok = cbm_daemon_runtime_service_free(fixture->service) && ok; + fixture->service = NULL; + } + frontend_maintenance_fixture_finish(&fixture->maintenance); + return ok; +} + +static FILE *frontend_test_fdopen_read(int fd) { +#ifdef _WIN32 + return _fdopen(fd, "rb"); +#else + return fdopen(fd, "rb"); +#endif +} + +static int frontend_test_close_fd(int fd) { +#ifdef _WIN32 + return _close(fd); +#else + return close(fd); +#endif +} + +static void *frontend_idle_run(void *opaque) { + frontend_idle_run_t *run = opaque; + run->result = + cbm_daemon_frontend_mcp_run(run->client, run->manager, run->input, run->output); + return NULL; +} +#endif + #ifndef _WIN32 static const char FRONTEND_TEST_BUILD[] = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; @@ -1142,6 +1301,66 @@ TEST(daemon_frontend_rejects_non_notification_cancellation_shapes) { PASS(); } +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS +/* An idle frontend has one independent maintenance observer. Hold that + * observer before its first secure presence read, then require the real worker + * to complete an idle cycle. Any worker-side presence read is therefore an + * exact duplicate, independent of elapsed time or scheduler cadence. */ +TEST(daemon_frontend_idle_uses_one_maintenance_observer) { + frontend_idle_fixture_t fixture; + bool fixture_ready = frontend_idle_fixture_start(&fixture); + int input_pipe[2] = {-1, -1}; + bool pipe_ready = fixture_ready && cbm_pipe(input_pipe) == 0; + FILE *input = pipe_ready ? frontend_test_fdopen_read(input_pipe[0]) : NULL; + FILE *output = input ? tmpfile() : NULL; + frontend_idle_run_t run = { + .client = fixture.client, + .manager = fixture.maintenance.manager, + .input = input, + .output = output, + .result = -1, + }; + if (output) { + fixture.client = NULL; /* cbm_daemon_frontend_mcp_run consumes it. */ + } + cbm_daemon_frontend_test_observer_reset(true); + cbm_thread_t frontend_thread; + bool thread_started = + output && cbm_thread_create(&frontend_thread, 0, frontend_idle_run, &run) == 0; + uint64_t deadline = cbm_now_ms() + 10000U; + while (thread_started && cbm_now_ms() < deadline && + (!cbm_daemon_frontend_test_monitor_waiting() || + cbm_daemon_frontend_test_worker_idle_cycles() == 0)) { + cbm_usleep(1000); + } + bool monitor_waiting = cbm_daemon_frontend_test_monitor_waiting(); + uint64_t monitor_observations = cbm_daemon_frontend_test_monitor_observations(); + uint64_t worker_observations = cbm_daemon_frontend_test_worker_observations(); + uint64_t idle_cycles = cbm_daemon_frontend_test_worker_idle_cycles(); + cbm_daemon_frontend_test_observer_release(); + bool input_closed = !pipe_ready || frontend_test_close_fd(input_pipe[1]) == 0; + bool joined = thread_started && cbm_thread_join(&frontend_thread) == 0; + bool input_stream_closed = !input || fclose(input) == 0; + bool output_closed = !output || fclose(output) == 0; + bool fixture_closed = frontend_idle_fixture_finish(&fixture); + + ASSERT_TRUE(fixture_ready); + ASSERT_TRUE(pipe_ready); + ASSERT_TRUE(thread_started); + ASSERT_TRUE(monitor_waiting); + ASSERT_EQ(monitor_observations, 1); + ASSERT_TRUE(idle_cycles > 0); + ASSERT_EQ(worker_observations, 0); + ASSERT_TRUE(input_closed); + ASSERT_TRUE(joined); + ASSERT_EQ(run.result, 0); + ASSERT_TRUE(input_stream_closed); + ASSERT_TRUE(output_closed); + ASSERT_TRUE(fixture_closed); + PASS(); +} +#endif + #ifndef _WIN32 /* RED: an MCP frontend's main thread can remain blocked forever in stdio while * install/update/uninstall owns maintenance intent. The already-present @@ -1442,6 +1661,9 @@ SUITE(daemon_frontend) { RUN_TEST(daemon_frontend_correlates_cancellation_to_exact_request); RUN_TEST(daemon_frontend_ignores_cancellation_text_in_string_content); RUN_TEST(daemon_frontend_rejects_non_notification_cancellation_shapes); +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS + RUN_TEST(daemon_frontend_idle_uses_one_maintenance_observer); +#endif #ifndef _WIN32 RUN_TEST(daemon_frontend_maintenance_exits_while_stdio_reader_is_blocked); RUN_TEST(daemon_local_participant_monitor_cancels_then_bounds_active_operation); From 0f1e65d625bb0b87fbea96251caaef0b10deebd5 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 27 Aug 2026 08:52:02 +0200 Subject: [PATCH 08/31] fix: preserve Cargo context in parallel Rust resolution Signed-off-by: Martin Vogel --- src/pipeline/pass_lsp_cross.c | 44 +++++++++- src/pipeline/pass_lsp_cross.h | 10 ++- src/pipeline/pass_parallel.c | 37 +++++++- tests/repro/repro_issue56.c | 112 +++--------------------- tests/test_parallel.c | 159 ++++++++++++++++++++++++++++++++++ 5 files changed, 255 insertions(+), 107 deletions(-) diff --git a/src/pipeline/pass_lsp_cross.c b/src/pipeline/pass_lsp_cross.c index 17491a9ea..684554455 100644 --- a/src/pipeline/pass_lsp_cross.c +++ b/src/pipeline/pass_lsp_cross.c @@ -693,7 +693,10 @@ bool cbm_pxc_has_cross_lsp(CBMLanguage lang) { /* Append cross-file results from `src_out` (allocated in a scratch arena * about to be destroyed) into `dst_calls` (lives in cache_entry->arena), - * copying every string field into dst_arena. Skips entries whose + * copying every string field into dst_arena. A manifest-qualified Rust + * cross-crate result supersedes any earlier result for the exact same source + * occurrence: the Cargo member path is stronger evidence than a same-named + * local fallback. Skips entries whose * (kind, caller_qn, callee_qn, source span) is already present — avoids * inflating the array with cross-file duplicates without conflating distinct * same-named occurrences. For an exact duplicate, retain the higher-confidence @@ -715,6 +718,39 @@ static void pxc_append_results(CBMArena *dst_arena, CBMResolvedCallArray *dst_ca cbm_arena_init(&keys); CBMHashTable *seen = cbm_ht_create((uint32_t)(dst_calls->count + src_out->count + 1)); + /* Per-file Rust resolution may already have confidently matched the tail + * of `member::call()` to a same-named local function. Once the cross pass + * proves that `member` is a declared Cargo workspace member, replace only + * records for that exact parser occurrence before constructing the dedup + * table. This preserves distinct same-named calls at other spans while + * preventing the stale local result from out-ranking manifest evidence. */ + for (int j = 0; j < src_out->count; j++) { + const CBMResolvedCall *src = &src_out->items[j]; + if (!src->strategy || strcmp(src->strategy, "lsp_cross_crate") != 0 || !src->caller_qn || + !src->callee_qn || src->site_end_byte <= src->site_start_byte) { + continue; + } + for (int i = 0; i < dst_calls->count; i++) { + CBMResolvedCall *dst = &dst_calls->items[i]; + if (!dst->caller_qn || dst->kind != src->kind || + dst->site_start_byte != src->site_start_byte || + dst->site_end_byte != src->site_end_byte || + dst->source_origin != src->source_origin || + strcmp(dst->caller_qn, src->caller_qn) != 0) { + continue; + } + dst->caller_qn = cbm_arena_strdup(dst_arena, src->caller_qn); + dst->callee_qn = cbm_arena_strdup(dst_arena, src->callee_qn); + dst->strategy = cbm_arena_strdup(dst_arena, src->strategy); + dst->confidence = src->confidence; + dst->reason = src->reason ? cbm_arena_strdup(dst_arena, src->reason) : NULL; + dst->kind = src->kind; + dst->site_start_byte = src->site_start_byte; + dst->site_end_byte = src->site_end_byte; + dst->source_origin = src->source_origin; + } + } + for (int i = 0; i < dst_calls->count; i++) { const CBMResolvedCall *rc = &dst_calls->items[i]; if (rc->caller_qn && rc->callee_qn) { @@ -1191,8 +1227,8 @@ void cbm_pxc_dispatch_file(CBMLanguage lang, CBMFileResult *result, const char * free(filtered); } -static bool pxc_build_rust_manifest(const cbm_pipeline_ctx_t *ctx, CBMArena *marena, - CBMCargoManifest *out_m) { +bool cbm_pxc_build_rust_manifest(const cbm_pipeline_ctx_t *ctx, CBMArena *marena, + CBMCargoManifest *out_m) { if (!ctx || !ctx->repo_path || !marena || !out_m) return false; char path[1024]; @@ -1234,7 +1270,7 @@ int cbm_pipeline_pass_lsp_cross(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t * bool have_manifest = false; if (have_rust) { cbm_arena_init(&cargo_arena); - have_manifest = pxc_build_rust_manifest(ctx, &cargo_arena, &cargo_manifest); + have_manifest = cbm_pxc_build_rust_manifest(ctx, &cargo_arena, &cargo_manifest); cbm_pxc_set_rust_manifest(have_manifest ? &cargo_manifest : NULL); } diff --git a/src/pipeline/pass_lsp_cross.h b/src/pipeline/pass_lsp_cross.h index 149959bce..3198d79d4 100644 --- a/src/pipeline/pass_lsp_cross.h +++ b/src/pipeline/pass_lsp_cross.h @@ -169,10 +169,14 @@ static inline CBMTypeRegistry *cbm_pxc_registry_for_lang(const CBMCrossLspRegist } } -/* Borrow the (thread-local) Rust Cargo manifest the cross-file LSP pass set for - * cross-crate (#56) routing. The Tier-2 prebuilt Rust resolve reads it so it sees - * exactly what the per-file fallback (cbm_pxc_run_one) would on the same thread. */ +/* Build and borrow the Rust Cargo manifest used for cross-crate (#56) routing. + * The manifest owns strings in manifest_arena; callers keep that arena alive + * until every resolver worker has joined. Each worker must install the shared + * immutable pointer in its own TLS slot before dispatch and clear it afterward. */ struct CBMCargoManifest; +bool cbm_pxc_build_rust_manifest(const cbm_pipeline_ctx_t *ctx, CBMArena *manifest_arena, + struct CBMCargoManifest *out_manifest); +void cbm_pxc_set_rust_manifest(const struct CBMCargoManifest *manifest); const struct CBMCargoManifest *cbm_pxc_get_rust_manifest(void); /* Run the cross-file LSP resolver for non-TS languages. Appends diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 9d61a1e82..716f26995 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -70,6 +70,7 @@ enum { PP_CSHARP_M_PREFIX_LEN = 2 }; #include "pipeline/pipeline_internal.h" #include "pipeline/pass_lsp_cross.h" /* cbm_pxc_* helpers for fused cross-file LSP */ #include "pipeline/lsp_resolve.h" +#include "lsp/rust_cargo.h" #include "helpers.h" /* cbm_kind_in_set_free_cache — per-worker-thread cache teardown */ #include "pipeline/worker_pool.h" #include "foundation/compat.h" @@ -1398,6 +1399,11 @@ typedef struct { * Stored as CBMCrossLspRegistries* (typedef from pass_lsp_cross.h). */ CBMCrossLspRegistries *cross_registries; + /* Parsed once on the coordinator thread and borrowed read-only by resolve + * workers. The pointer is installed into each worker's TLS slot so Rust's + * manifest-aware cross-file resolver sees the same Cargo context. */ + const CBMCargoManifest *rust_manifest; + /* F4: LAZILY-built shared Rust registry (built ONCE, on the first NULL-filter * rust file — the ~all_defs amplifier files). Not eager: repos whose rust files * all filter to subsets never pay the O(all_defs) build + multi-GB RSS. Built @@ -2944,6 +2950,8 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { resolve_ctx_t *rc = ctx_ptr; resolve_worker_state_t *ws = &rc->workers[worker_id]; + cbm_pxc_set_rust_manifest(rc->rust_manifest); + if (!ws->local_edge_buf) { ws->local_edge_buf = cbm_gbuf_new_shared_ids(rc->project_name, rc->repo_path, rc->shared_ids); @@ -3009,6 +3017,8 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { * a mixed-source-root Java↔Kotlin site remains unresolved, so JVM * callers run whenever either kind of site exists. */ bool jvm_cross_lsp = (lang == CBM_LANG_JAVA || lang == CBM_LANG_KOTLIN); + bool rust_workspace_cross_lsp = + (lang == CBM_LANG_RUST && rc->rust_manifest && rc->rust_manifest->member_count > 0); int call_reference_sites = pp_call_reference_site_count(result, lang); int semantic_sites = result->calls.count + call_reference_sites; int qualified_lsp_sites = pp_qualified_lsp_site_count(result); @@ -3016,7 +3026,8 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { bool cross_lsp_eligible = (rc->all_defs && rc->def_count > 0 && cbm_pxc_has_cross_lsp(lang) && (semantic_sites > 0 || pending_lsp_site) && - (jvm_cross_lsp || pending_lsp_site || qualified_lsp_sites < semantic_sites) && + (jvm_cross_lsp || rust_workspace_cross_lsp || pending_lsp_site || + qualified_lsp_sites < semantic_sites) && !is_generated); /* Skip files with nothing else to resolve and no cross-LSP work. */ @@ -3187,6 +3198,7 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { * into dead TLS and are never retired, so a later cross-thread free can * never bring their refcount to zero (leak). Retiring them here releases * each page as its final chunk returns. */ + cbm_pxc_set_rust_manifest(NULL); cbm_destroy_thread_parser(); cbm_slab_destroy_thread(); cbm_service_pattern_cache_end(); @@ -3213,6 +3225,25 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, } memset(workers, 0, (size_t)worker_count * sizeof(resolve_worker_state_t)); + bool have_rust = false; + for (int i = 0; i < file_count; i++) { + if (result_cache[i] && files[i].language == CBM_LANG_RUST) { + have_rust = true; + break; + } + } + CBMArena rust_manifest_arena; + CBMCargoManifest rust_manifest; + bool rust_manifest_arena_live = false; + const CBMCargoManifest *rust_manifest_ptr = NULL; + if (have_rust) { + cbm_arena_init(&rust_manifest_arena); + rust_manifest_arena_live = true; + if (cbm_pxc_build_rust_manifest(ctx, &rust_manifest_arena, &rust_manifest)) { + rust_manifest_ptr = &rust_manifest; + } + } + resolve_ctx_t rc = { .files = files, .file_count = file_count, @@ -3230,6 +3261,7 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, .def_modules = def_modules, .module_def_index = module_def_index, .cross_registries = cross_registries, + .rust_manifest = rust_manifest_ptr, }; atomic_init(&rc.next_file_idx, 0); atomic_init(&rc.lsp_cross_processed, 0); @@ -3255,6 +3287,9 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, rc.rust_shared_arena_live = false; } cbm_mutex_destroy(&rc.rust_shared_mu); + if (rust_manifest_arena_live) { + cbm_arena_destroy(&rust_manifest_arena); + } /* Sub-phase: Merge all local edge bufs into main gbuf (SEQUENTIAL) */ CBM_PROF_START(t_resolve_merge); diff --git a/tests/repro/repro_issue56.c b/tests/repro/repro_issue56.c index c5cbf596b..414756746 100644 --- a/tests/repro/repro_issue56.c +++ b/tests/repro/repro_issue56.c @@ -1,92 +1,17 @@ /* - * repro_issue56.c — Reproduce-first case for OPEN bug #56. + * repro_issue56.c — production-pipeline regression for Rust cross-crate calls. * - * Bug #56: "Cross-crate call graphs stop at boundaries" (Rust) + * The fixture is a two-member Cargo workspace. crate_b calls + * `crate_a::helper()` and also declares a same-named local helper, so generic + * bare-name resolution cannot accidentally satisfy the assertion. The test + * requires a persisted CALLS edge into crate_a's namespace. * - * ROOT CAUSE (pipeline / Rust LSP path): - * The tree-sitter-only Rust extractor has no access to Cargo metadata - * at extraction time, so when it sees `crate_a::helper()` inside - * crate_b, it records a raw call-site for the path but has no registry - * entry for `crate_a::helper` — only the definitions in the *same file* - * were seeded. The LSP resolver therefore cannot match the call-site to - * a callee QN across the crate boundary, and the resulting - * CBMResolvedCall is either absent or marked with low confidence and - * discarded. When the pipeline writes graph edges for this project, no - * CALLS edge is minted for the cross-crate call — the call graph stops - * at the crate edge. - * - * v0.8.1 added a hybrid-LSP Rust path that "materially improves" this - * (issue comment, maintainer 2026-06-25), but the reporter was asked to - * retest; the issue remains OPEN because no retest confirming resolution - * was provided. The workspace-member wiring test - * (rustlsp_extra_cargo_wires_workspace_member in test_rust_lsp.c) only - * exercises the *single-file LSP* layer with a manually-parsed manifest; - * it does NOT verify that the full production pipeline (rh_index_files → - * cbm_pipeline → graph store) persists a cross-crate CALLS edge for a - * real multi-file Cargo workspace fixture. That gap is what this test - * fills. - * - * FIXTURE: - * A minimal Cargo workspace with two crates: - * - * [workspace Cargo.toml] — workspace root, declares members - * crate_a/Cargo.toml — library crate "crate_a" - * crate_a/src/lib.rs — exposes `pub fn helper() {}` - * crate_b/Cargo.toml — binary crate "crate_b", depends on crate_a - * crate_b/src/main.rs — calls `crate_a::helper()` from `fn run()`; - * also defines a LOCAL `fn helper()` to break - * bare-name uniqueness (see note below) - * - * The only meaningful cross-crate CALLS edge is: - * crate_b::run → crate_a::helper - * - * EXPECTED (correct) behaviour: - * After indexing the workspace through the production MCP pipeline, the - * graph store must contain at least one CALLS edge whose TARGET node's - * qualified_name contains "crate_a" (i.e. routes into the crate_a - * namespace, not into crate_b's local helper). - * - * ACTUAL (buggy) behaviour: - * The pipeline extracts both files, but the cross-crate path - * `crate_a::helper` in crate_b/src/main.rs is not resolved to a graph - * node in crate_a because Cargo workspace member metadata is not - * plumbed into the per-file extraction phase. Result: zero CALLS edges - * to the crate_a namespace. - * - * WHY THIS IS RED ON CURRENT CODE (even post-v0.8.1): - * The rustlsp_extra_cargo_wires_workspace_member unit test exercises only - * the LSP layer (cbm_run_rust_lsp_with_manifest called with a parsed - * CBMCargoManifest) and confirms the resolver *can* route - * `engine::boot()` to `engine.boot` when given the manifest explicitly. - * BUT: the production pipeline's per-file extraction path - * (cbm_extract_file → cbm_run_rust_lsp) does NOT receive a pre-parsed - * workspace manifest — it only gets the individual file's content. - * Additionally, cbm_pxc_has_cross_lsp() returns false for CBM_LANG_RUST - * (pass_lsp_cross.c), so the cross-file LSP pass is never invoked for - * Rust. Therefore a real workspace indexed through index_repository - * produces no CALLS edges crossing into crate_a, and this test is RED. - * - * WHY THE OLD >= 2 COUNT TEST FALSE-PASSED: - * With a unique `helper` name in the project (one definition in crate_a, - * no other `helper` anywhere), the generic pipeline name resolver - * (registry.c, resolve_name_lookup) resolves `crate_a::helper` to the - * sole `helper` candidate by bare-name suffix scoring — WITHOUT needing - * any cross-crate workspace metadata. This produced calls >= 2 (the - * intra-file main→run plus the bare-name-resolved run→helper), making - * the old ASSERT_GTE(calls, 2) GREEN even though the bug was not fixed. - * - * Fix: add a LOCAL `fn helper()` in crate_b/src/main.rs so there are - * now TWO `helper` candidates in the project registry. The generic - * resolver either picks the wrong one (crate_b-local) or abstains - * (ambiguous). Only a correctly crate-qualified resolver routes - * `crate_a::helper` specifically to crate_a's node. The assertion then - * checks the TARGET node's qualified_name contains "crate_a" — a count - * check is no longer sufficient because the local helper also contributes - * a CALLS edge (run_local→helper). - * - * UNCERTAINTY: - * If a future version plumbs workspace metadata or wires Rust lsp_cross - * correctly, this test will go GREEN — that is the intended outcome. + * The direct parallel regression lives in test_parallel.c because the bug was + * specific to worker-local pipeline context: the root Cargo manifest was + * parsed by the sequential cross-LSP driver but was not propagated into + * parallel resolve workers. A confident local result could also suppress the + * manifest-aware cross pass. This repro remains as an end-to-end guard for the + * same graph contract through the production harness. */ #include "test_framework.h" @@ -108,19 +33,8 @@ * CALLS edge's TARGET node has a qualified_name containing "crate_a", * proving the cross-crate boundary was traversed. * - * RED condition: - * No CALLS edge whose target QN contains "crate_a" exists in the store. - * - * This test is RED on current code because: - * 1. cbm_run_rust_lsp is called with NULL manifest (cbm.c:645), so no - * workspace metadata is available at extraction time. - * 2. cbm_pxc_has_cross_lsp returns false for CBM_LANG_RUST - * (pass_lsp_cross.c:281), so the cross-file LSP pass never runs for - * Rust and cannot seed crate_a defs into crate_b's resolver context. - * 3. With two `helper` candidates (crate_a and crate_b-local), the - * generic resolver's qualified_suffix_match fails (neither QN ends - * with ".crate_a.helper") and bare-name scoring picks the crate_b- - * local one or abstains, never routing to crate_a. + * Regression condition: no CALLS edge whose target QN contains "crate_a" + * exists in the store. */ TEST(repro_issue56_cross_crate_calls) { /* diff --git a/tests/test_parallel.c b/tests/test_parallel.c index c33d26690..1a1c42ce3 100644 --- a/tests/test_parallel.c +++ b/tests/test_parallel.c @@ -351,6 +351,12 @@ static cbm_gbuf_t *run_parallel_with_extract_opts_and_mutator( harness_ctx_free_tables(&ctx); cbm_registry_free(reg); + /* cbm_parallel_extract installs a process-global package map whose + * production owner is the surrounding pipeline lifecycle. This direct + * pass harness owns that lifecycle itself, so mirror production teardown + * before returning the graph to the test. */ + cbm_pkgmap_free(cbm_pipeline_get_pkgmap()); + cbm_pipeline_set_pkgmap(NULL); return gbuf; } @@ -925,6 +931,25 @@ static bool callable_has_call_target_fragment(const cbm_gbuf_t *gbuf, const char return false; } +static const cbm_gbuf_edge_t *find_call_edge_to_target_fragment(const cbm_gbuf_t *gbuf, + const char *source_tail, + const char *target_fragment) { + const cbm_gbuf_node_t *source = find_unique_callable_node_by_tail(gbuf, source_tail); + if (!source || !target_fragment) + return NULL; + const cbm_gbuf_edge_t **edges = NULL; + int count = 0; + if (cbm_gbuf_find_edges_by_source_type(gbuf, source->id, "CALLS", &edges, &count) != 0) + return NULL; + for (int i = 0; i < count; i++) { + const cbm_gbuf_node_t *target = + edges[i] ? cbm_gbuf_find_by_id(gbuf, edges[i]->target_id) : NULL; + if (target && target->qualified_name && strstr(target->qualified_name, target_fragment)) + return edges[i]; + } + return NULL; +} + static int count_calls_edges_to_tail(const cbm_gbuf_t *gbuf, const char *target_tail) { const cbm_gbuf_node_t *target = find_unique_callable_node_by_tail(gbuf, target_tail); if (!target) @@ -2534,6 +2559,138 @@ TEST(parallel_kotlin_nonbinary_operator_carriers_reach_graph) { PASS(); } +/* Build the smallest real Cargo workspace that exercises the direct parallel + * resolve API. crate_a is a declared workspace member, while unlisted is a + * graph-visible decoy with the same helper name. That decoy is intentional: + * without manifest-aware routing, a bare-name match cannot accidentally make + * the test green. The optional crate_b-local helper pins the second failure + * mode where a confident in-file resolution used to suppress cross-LSP. */ +static cbm_gbuf_t *run_issue56_parallel_workspace(bool local_decoy) { + static const char workspace_toml[] = + "[workspace]\n" + "members = [\"crate_a\", \"crate_b\"]\n" + "resolver = \"2\"\n"; + static const char crate_a_toml[] = + "[package]\nname = \"crate_a\"\nversion = \"0.1.0\"\nedition = \"2021\"\n"; + static const char crate_b_toml[] = + "[package]\nname = \"crate_b\"\nversion = \"0.1.0\"\nedition = \"2021\"\n" + "\n[dependencies]\ncrate_a = { path = \"../crate_a\" }\n"; + static const char crate_a_source[] = "pub fn helper() {}\n"; + static const char unlisted_source[] = "pub fn helper() {}\n"; + static const char caller_without_local[] = + "fn run() { crate_a::helper(); }\n" + "fn main() { run(); }\n"; + static const char caller_with_local[] = + "fn helper() {}\n" + "fn run() { crate_a::helper(); }\n" + "fn main() { run(); }\n"; + + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_issue56_parallel_XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) + return NULL; + + char crate_a[512]; + char crate_a_src[512]; + char crate_b[512]; + char crate_b_src[512]; + char unlisted[512]; + char unlisted_src[512]; + snprintf(crate_a, sizeof(crate_a), "%s/crate_a", tmpdir); + snprintf(crate_a_src, sizeof(crate_a_src), "%s/src", crate_a); + snprintf(crate_b, sizeof(crate_b), "%s/crate_b", tmpdir); + snprintf(crate_b_src, sizeof(crate_b_src), "%s/src", crate_b); + snprintf(unlisted, sizeof(unlisted), "%s/unlisted", tmpdir); + snprintf(unlisted_src, sizeof(unlisted_src), "%s/src", unlisted); + if (cbm_mkdir(crate_a) != 0 || cbm_mkdir(crate_a_src) != 0 || cbm_mkdir(crate_b) != 0 || + cbm_mkdir(crate_b_src) != 0 || cbm_mkdir(unlisted) != 0 || cbm_mkdir(unlisted_src) != 0) { + th_rmtree(tmpdir); + return NULL; + } + + char root_manifest[512]; + char crate_a_manifest[512]; + char crate_b_manifest[512]; + char crate_a_file[512]; + char crate_b_file[512]; + char unlisted_file[512]; + snprintf(root_manifest, sizeof(root_manifest), "%s/Cargo.toml", tmpdir); + snprintf(crate_a_manifest, sizeof(crate_a_manifest), "%s/Cargo.toml", crate_a); + snprintf(crate_b_manifest, sizeof(crate_b_manifest), "%s/Cargo.toml", crate_b); + snprintf(crate_a_file, sizeof(crate_a_file), "%s/lib.rs", crate_a_src); + snprintf(crate_b_file, sizeof(crate_b_file), "%s/main.rs", crate_b_src); + snprintf(unlisted_file, sizeof(unlisted_file), "%s/lib.rs", unlisted_src); + if (th_write_file(root_manifest, workspace_toml) != 0 || + th_write_file(crate_a_manifest, crate_a_toml) != 0 || + th_write_file(crate_b_manifest, crate_b_toml) != 0 || + th_write_file(crate_a_file, crate_a_source) != 0 || + th_write_file(crate_b_file, local_decoy ? caller_with_local : caller_without_local) != 0 || + th_write_file(unlisted_file, unlisted_source) != 0) { + th_rmtree(tmpdir); + return NULL; + } + + cbm_discover_opts_t opts = {.mode = CBM_MODE_FULL}; + cbm_file_info_t *files = NULL; + int file_count = 0; + if (cbm_discover(tmpdir, &opts, &files, &file_count) != 0 || file_count < 3) { + cbm_discover_free(files, file_count); + th_rmtree(tmpdir); + return NULL; + } + cbm_gbuf_t *gbuf = + run_parallel("cbm_issue56_parallel", tmpdir, files, file_count, 2 /* real workers */); + cbm_discover_free(files, file_count); + th_rmtree(tmpdir); + return gbuf; +} + +TEST(parallel_rust_cross_crate_worker_receives_workspace_manifest) { + cbm_gbuf_t *gbuf = run_issue56_parallel_workspace(false); + ASSERT_NOT_NULL(gbuf); + + const cbm_gbuf_edge_t *correct = + find_call_edge_to_target_fragment(gbuf, "main.run", ".crate_a."); + const bool correct_found = correct != NULL; + const bool unlisted = + callable_has_call_target_fragment(gbuf, "main.run", ".unlisted."); + const bool manifest_strategy = + correct && correct->properties_json && strstr(correct->properties_json, "lsp_cross_crate"); + if (!correct || unlisted || !manifest_strategy) { + printf(" issue56 manifest diagnostic: correct=%d unlisted=%d strategy=%d\n", + correct_found, unlisted, manifest_strategy); + } + cbm_gbuf_free(gbuf); + + ASSERT_TRUE(correct_found); + ASSERT_FALSE(unlisted); + ASSERT_TRUE(manifest_strategy); + PASS(); +} + +TEST(parallel_rust_cross_crate_manifest_beats_confident_local_resolution) { + cbm_gbuf_t *gbuf = run_issue56_parallel_workspace(true); + ASSERT_NOT_NULL(gbuf); + + const cbm_gbuf_edge_t *correct = + find_call_edge_to_target_fragment(gbuf, "main.run", ".crate_a."); + const bool correct_found = correct != NULL; + const bool wrong_local = + callable_has_call_target_fragment(gbuf, "main.run", ".crate_b."); + const bool manifest_strategy = + correct && correct->properties_json && strstr(correct->properties_json, "lsp_cross_crate"); + if (!correct || wrong_local || !manifest_strategy) { + printf(" issue56 local-shadow diagnostic: correct=%d wrong_local=%d strategy=%d\n", + correct_found, wrong_local, manifest_strategy); + } + cbm_gbuf_free(gbuf); + + ASSERT_TRUE(correct_found); + ASSERT_FALSE(wrong_local); + ASSERT_TRUE(manifest_strategy); + PASS(); +} + /* A known external macro semantic owns its parser occurrence even when the * external target is not materialized as a graph node. It must not fall back * to a same-named local function in either pipeline. */ @@ -3926,6 +4083,8 @@ SUITE(parallel) { RUN_TEST(parallel_tsx_import_namespace_exact_parity); RUN_TEST(parallel_kotlin_external_protocol_does_not_use_project_class_method_tail); RUN_TEST(parallel_kotlin_nonbinary_operator_carriers_reach_graph); + RUN_TEST(parallel_rust_cross_crate_worker_receives_workspace_manifest); + RUN_TEST(parallel_rust_cross_crate_manifest_beats_confident_local_resolution); RUN_TEST(parallel_rust_known_macro_does_not_fallback_to_local_function); RUN_TEST(parallel_rust_proc_macros_are_decorates_and_usage_only); RUN_TEST(parallel_c_preprocessed_coordinate_collision_preserves_hidden_target); From cbd241d7f1b33248501010cbaac50b07785373b8 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 28 Aug 2026 16:41:38 +0200 Subject: [PATCH 09/31] fix(tests): venue-parity interface probes must not pipe into grep -q MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two layer-5 interface probes tested for a substring by piping the captured output into `grep -q`. Under `set -o pipefail` (line 32) that turns a SATISFIED contract into a probe failure whenever the payload is larger than the pipe buffer: `grep -q` exits at the first match, and the writer takes EPIPE on the remainder, so the pipeline reports the writer's non-zero status even though the match succeeded. Observed on macos-15-intel: test_venue_parity_contract.sh: line 404: printf: write error: Broken pipe INTERFACE CONTRACT: scripts/ci/smoke-artifact.sh --help must exit 0 and print a Usage: block (rc=0) VENUE PARITY CONTRACT VIOLATED — interface probes failed (layer 5) rc=0 and the Usage: block was present — the match is exactly what made grep exit early. `smoke-artifact.sh --help` is 698 bytes against a 512-byte PIPE_BUF, and prints `Usage:` on its first line, so the writer is killed at the earliest possible moment. Whether the remaining write lands before grep exits is a scheduling race, which is why this only ever showed up on one runner. Replace both pipelines with in-shell `case` matching: no subprocess, no pipe, no race, and a strictly literal match (the second probe's "--help." was previously a regex whose `.` matched any character). Verified: the contract passes (18 --help entries, 8 strict-flag entries), and the matcher still returns 0 for output with no Usage: block, so the probe can still fail. Signed-off-by: Martin Vogel --- tests/test_venue_parity_contract.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/test_venue_parity_contract.sh b/tests/test_venue_parity_contract.sh index b609a1493..4bcd694d7 100755 --- a/tests/test_venue_parity_contract.sh +++ b/tests/test_venue_parity_contract.sh @@ -401,7 +401,16 @@ test-infrastructure/vm/win.sh case "$entry" in *.py) out=$(cd "$ROOT" && python3 "$entry" --help 2>&1) && rc=0 || rc=$? ;; esac - if [ "$rc" -ne 0 ] || ! printf '%s' "$out" | grep -q "Usage:"; then + # Substring test in-shell: piping into `grep -q` is unsafe under + # `set -o pipefail` because grep exits at the first match and the + # writer takes EPIPE on any payload larger than the pipe buffer, + # turning a SATISFIED contract into a probe failure (macOS, 698-byte + # --help vs a 512-byte PIPE_BUF). + case "$out" in + *"Usage:"*) has_usage=1 ;; + *) has_usage=0 ;; + esac + if [ "$rc" -ne 0 ] || [ "$has_usage" -eq 0 ]; then echo "INTERFACE CONTRACT: $entry --help must exit 0 and print a Usage: block (rc=$rc)" >&2 probe_failures=1 fi @@ -422,7 +431,11 @@ scripts/smoke-invariants.sh " for entry in $STRICT_ENTRIES; do out=$(cd "$ROOT" && bash "$entry" --definitely-not-a-flag 2>&1) && rc=0 || rc=$? - if [ "$rc" -ne 2 ] || ! printf '%s' "$out" | grep -q "Please consult --help."; then + case "$out" in + *"Please consult --help."*) has_hint=1 ;; + *) has_hint=0 ;; + esac + if [ "$rc" -ne 2 ] || [ "$has_hint" -eq 0 ]; then echo "INTERFACE CONTRACT: $entry must reject unknown flags with exit 2 + 'Please consult --help.' (rc=$rc)" >&2 probe_failures=1 fi From e5f3aab02e1c8bdb3d3e04f6b52d78d741e1241f Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 28 Aug 2026 19:03:07 +0200 Subject: [PATCH 10/31] feat(pipeline): index-time importance scoring (weighted degree) Score every Function/Method/Class node during indexing and persist the result as a numeric "importance" key inside the node's existing properties_json: importance = sqrt(num_refs) * priv * generic * distinct * test_penalty num_refs is the incoming CALLS + USAGE degree. The multipliers demote private (leading-underscore) and generic names (defined in >= 5 distinct files), promote distinctive identifiers (snake_case or camelCase, len >= 8), and demote test scaffolding -- symbols in a test file per the graph's own cbm_is_test_path() classifier, or targets of an incoming TESTS edge. Weighted degree only; a transitive (PageRank) refinement is deliberately not built here until it can be shown to beat this on a fixed judgment set. No schema change and no CBM_INDEX_FORMAT_VERSION bump: the score lives in the existing properties_json TEXT column, so no index is forced to rebuild and indexes written by older builds simply lack the key. Three failure modes this shape invites, all of them silent, are addressed directly: * Cost. The generic-name multiplier needs |{files a name is defined in}|. Computing that per NODE with a pairwise file-path scan is O(k^3) in a same-name group of size k -- ruinous on real corpora, where a single Java name group reaches k > 4000. Here the count is computed once per distinct NAME and memoized, and file paths are deduplicated through a hash set, so total distinct-file work is linear in the graph. Measured cold full index of a 666 MB Java corpus, same -O2 binary: 85 s with the pass disabled, 80 s with this implementation, 481 s with the per-node shape. * Registration. PREDUMP_PASS_COUNT is now derived from the pass table with sizeof instead of being hand-written. A hand-written count that lags an appended entry silently skips the last-registered pass with every test still green. * Incremental. The incremental post-pass sequence runs the pass too, over a graph rehydrated from the store whose nodes already carry the key. The write-back overwrites an existing key in place instead of appending a second one, which would otherwise corrupt properties_json into {"importance":1.0,...,"importance":2.0}. tests/test_importance.c covers the formula and binds all three failure modes; the complexity suite gains a GATED counter asserting the pass's same-name-group work stays linear (~2x, not ~4x) under a corpus doubling. Signed-off-by: Martin Vogel Co-authored-by: petercoxphoto <20230727+petercoxphoto@users.noreply.github.com> --- Makefile.cbm | 3 +- src/pipeline/pass_importance.c | 396 +++++++++++++++++ src/pipeline/pipeline.c | 22 +- src/pipeline/pipeline_incremental.c | 14 + src/pipeline/pipeline_internal.h | 25 ++ tests/test_complexity.c | 44 ++ tests/test_importance.c | 641 ++++++++++++++++++++++++++++ tests/test_main.c | 2 + 8 files changed, 1142 insertions(+), 5 deletions(-) create mode 100644 src/pipeline/pass_importance.c create mode 100644 tests/test_importance.c diff --git a/Makefile.cbm b/Makefile.cbm index b71fa6ac9..dc92e1703 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -354,6 +354,7 @@ PIPELINE_SRCS = \ src/pipeline/pass_similarity.c \ src/pipeline/pass_semantic_edges.c \ src/pipeline/pass_complexity.c \ + src/pipeline/pass_importance.c \ src/pipeline/pass_cross_repo.c \ src/pipeline/artifact.c \ src/pipeline/pass_ensemble_routing.c \ @@ -560,7 +561,7 @@ TEST_DISCOVER_SRCS = \ TEST_GRAPH_BUFFER_SRCS = tests/test_graph_buffer.c -TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c +TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c TEST_WATCHER_SRCS = tests/test_watcher.c diff --git a/src/pipeline/pass_importance.c b/src/pipeline/pass_importance.c new file mode 100644 index 000000000..a727ddee0 --- /dev/null +++ b/src/pipeline/pass_importance.c @@ -0,0 +1,396 @@ +/* + * pass_importance.c — Index-time per-symbol importance score (weighted degree). + * + * Predump pass. Computes an importance score for every Function/Method/Class + * node and stores it as a numeric "importance" key inside the node's EXISTING + * properties_json blob — no new column, no schema change, and therefore no + * CBM_INDEX_FORMAT_VERSION bump. Indexes written by older builds simply lack + * the key; every consumer must tolerate its absence. + * + * Model (weighted degree, Aider-repomap shaped): + * + * importance = sqrt(num_refs) * priv * generic * distinct * test_penalty + * + * num_refs incoming CALLS + USAGE edges. 0 -> sqrt(0) = 0. + * priv 0.1 name is private (leading underscore) + * generic 0.1 the name is defined in >= 5 DISTINCT files + * distinct 10 name is snake_case or camelCase AND len >= 8 + * test_penalty + * 0.1 the symbol is test scaffolding — either the target of an + * incoming TESTS edge (a production helper exercised by a + * test), or it lives in a test file per the graph's own + * canonical cbm_is_test_path() classifier (the same one + * pass_tests uses). + * + * Transitive importance (PageRank) is deliberately NOT built here: weighted + * degree is what this pass claims, and a transitive refinement has to earn + * itself against a fixed judgment set before it ships. + * + * ── Cost ───────────────────────────────────────────────────────────── + * The generic-name multiplier needs |{ file a name is defined in }|. The + * obvious shape — for each node, walk its whole same-name group and + * pairwise-compare file paths — is O(k^2) per node and therefore O(k^3) per + * same-name group of size k. That is not a theoretical concern: on a large + * Java corpus a single name group (`toString`, k≈4.6k) costs ~2 minutes, and + * a handful of such names alone multiplies total index time several-fold. + * + * This implementation is linear in the graph instead: + * - the distinct-file count is computed ONCE PER DISTINCT NAME and memoized + * (g_imp_name_counts), so a group of size k is scanned once, not k times; + * - within a group, file paths are deduplicated through a hash set rather + * than pairwise strcmp. + * Total distinct-file work is therefore Σ over distinct names of k = O(N). + * Both gbuf lookups it rests on (find_by_name, find_edges_by_target_type) are + * hash-indexed, so the per-node loop is O(1) amortised per node. + * + * g_importance_name_visits counts same-name-group member visits and is GATED + * in the complexity suite: it must stay linear (ratio ~2) under a corpus + * doubling. A regression to the per-node scan lands at ~4 and fails the gate. + * + * ── Ordering (load-bearing) ────────────────────────────────────────── + * Registered LAST among the predump passes, and last in the incremental + * post-pass sequence, so every edge type the score reads (CALLS, USAGE from + * extraction; TESTS from pass_tests) already exists. A pass ordered earlier + * would silently score everything at num_refs = 0 with no test penalty. + */ +#include "foundation/constants.h" +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" +#include "graph_buffer/graph_buffer.h" +#include "foundation/hash_table.h" +#include "foundation/log.h" +#include "foundation/compat.h" +#include "cbm.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +/* Work counters — deltas are read by the complexity suite's linearity gate. + * Never reset here: callers snapshot before/after so nested runs compose. */ +_Atomic uint64_t g_importance_nodes = 0; +_Atomic uint64_t g_importance_name_visits = 0; + +enum { + CBM_IMPORTANCE_GENERIC_MIN_FILES = 5, /* name defined in >= N files -> generic */ + CBM_IMPORTANCE_DISTINCT_MIN_LEN = 8, /* distinctive-identifier length floor */ +}; +static const double CBM_IMPORTANCE_PRIV_MUL = 0.1; +static const double CBM_IMPORTANCE_GENERIC_MUL = 0.1; +static const double CBM_IMPORTANCE_DISTINCT_MUL = 10.0; +static const double CBM_IMPORTANCE_TEST_MUL = 0.1; + +/* The node labels that carry an importance score. File/Module and other + * non-symbol nodes are deliberately excluded. */ +static const char *const CBM_IMPORTANCE_LABELS[] = {"Function", "Method", "Class"}; +enum { CBM_IMPORTANCE_LABEL_COUNT = 3 }; + +/* Int → string for structured logging (thread-safe ring buffer). */ +static const char *itoa_imp(uint64_t val) { + enum { RING = 2, MASK = 1 }; + static CBM_TLS char bufs[RING][CBM_SZ_32]; + static CBM_TLS int idx = 0; + int i = idx; + idx = (idx + 1) & MASK; + snprintf(bufs[i], sizeof(bufs[i]), "%llu", (unsigned long long)val); + return bufs[i]; +} + +static bool name_is_private(const char *name) { + return name != NULL && name[0] == '_'; +} + +/* Distinctive = (snake_case OR camelCase) AND length >= 8. snake_case is an + * embedded '_'; camelCase is a lower->upper "hump". A plain len>=8 lowercase + * word (no '_', no hump) is NOT distinctive. */ +static bool name_is_distinctive(const char *name) { + if (!name) { + return false; + } + size_t len = strlen(name); + if (len < (size_t)CBM_IMPORTANCE_DISTINCT_MIN_LEN) { + return false; + } + if (strchr(name, '_') != NULL) { + return true; /* snake_case */ + } + for (size_t i = 1; i < len; i++) { + if (islower((unsigned char)name[i - 1]) && isupper((unsigned char)name[i])) { + return true; /* camelCase hump */ + } + } + return false; +} + +/* ── JSON property write-back ───────────────────────────────────────── + * + * properties_json is a flat object here (same assumption the sibling passes' + * json_get_int/json_get_bool helpers make), but the value scanner below is + * written to survive nested values so a future richer blob cannot be + * truncated by an in-place overwrite. */ + +/* Return a pointer just past the JSON value beginning at p (first non-space + * char of the value). NULL if the value is unterminated/malformed. */ +static const char *imp_value_end(const char *p) { + if (*p == '"') { + p++; + while (*p) { + if (*p == '\\' && p[1]) { + p += 2; + continue; + } + if (*p == '"') { + return p + 1; + } + p++; + } + return NULL; + } + if (*p == '{' || *p == '[') { + int depth = 0; + bool in_str = false; + while (*p) { + if (in_str) { + if (*p == '\\' && p[1]) { + p += 2; + continue; + } + if (*p == '"') { + in_str = false; + } + } else if (*p == '"') { + in_str = true; + } else if (*p == '{' || *p == '[') { + depth++; + } else if (*p == '}' || *p == ']') { + depth--; + if (depth == 0) { + return p + 1; + } + } + p++; + } + return NULL; + } + while (*p && *p != ',' && *p != '}' && *p != ']') { + p++; + } + return p; +} + +/* Find `"key"` where it is genuinely in KEY position: preceded (modulo + * whitespace) by '{' or ',' and followed (modulo whitespace) by ':'. A bare + * strstr would also match the text inside a string VALUE. Returns a pointer + * to the opening quote, or NULL. */ +static const char *imp_find_key(const char *json, const char *key) { + char pat[CBM_SZ_64]; + int n = snprintf(pat, sizeof(pat), "\"%s\"", key); + if (n < 0 || (size_t)n >= sizeof(pat)) { + return NULL; + } + const char *p = json; + while ((p = strstr(p, pat)) != NULL) { + const char *before = p; + while (before > json && isspace((unsigned char)before[-1])) { + before--; + } + char prev = (before > json) ? before[-1] : '\0'; + const char *after = p + n; + while (isspace((unsigned char)*after)) { + after++; + } + if ((prev == '{' || prev == ',') && *after == ':') { + return p; + } + p += n; + } + return NULL; +} + +/* Write the numeric "importance" key into a node's properties JSON object. + * + * IDEMPOTENT BY CONTRACT. The incremental path rehydrates nodes straight out + * of the store (cbm_gbuf_load_from_db), and those nodes ALREADY carry an + * "importance" key from the previous run. A pure append would emit + * {"importance":1.0,...,"importance":2.0} — silent property corruption that + * no build or schema check would catch. So: overwrite an existing key in + * place, append only when the key is genuinely absent. + * + * A properties_json that is not a JSON object is left untouched. */ +void cbm_pipeline_importance_append_prop(cbm_gbuf_node_t *node, double score) { + if (!node) { + return; + } + const char *old = node->properties_json ? node->properties_json : "{}"; + size_t olen = strlen(old); + if (olen < 2 || old[olen - 1] != '}') { + return; /* not a JSON object — leave untouched */ + } + + char val[CBM_SZ_32]; + int vn = snprintf(val, sizeof(val), "%.6f", score); + if (vn < 0 || (size_t)vn >= sizeof(val)) { + return; + } + + const char *k = imp_find_key(old, "importance"); + if (k) { + const char *colon = strchr(k, ':'); + if (!colon) { + return; /* imp_find_key guarantees one, but never trust a raw scan */ + } + const char *v = colon + 1; + while (isspace((unsigned char)*v)) { + v++; + } + const char *vend = imp_value_end(v); + if (!vend) { + return; + } + size_t head = (size_t)(v - old); + size_t tail = strlen(vend); + char *neu = malloc(head + (size_t)vn + tail + 1); + if (!neu) { + return; + } + memcpy(neu, old, head); + memcpy(neu + head, val, (size_t)vn); + memcpy(neu + head + (size_t)vn, vend, tail + 1); + free(node->properties_json); + node->properties_json = neu; + return; + } + + char frag[CBM_SZ_64]; + int fn = snprintf(frag, sizeof(frag), "%s\"importance\":%s}", (olen == 2) ? "" : ",", val); + if (fn < 0 || (size_t)fn >= sizeof(frag)) { + return; + } + char *neu = malloc(olen - 1 + (size_t)fn + 1); + if (!neu) { + return; + } + memcpy(neu, old, olen - 1); /* copy without the trailing '}' */ + memcpy(neu + olen - 1, frag, (size_t)fn + 1); + free(node->properties_json); + node->properties_json = neu; +} + +/* ── Distinct-file counting (the linear core) ─────────────────────────── */ + +typedef struct { + CBMHashTable *counts; /* name -> (void *)(intptr_t)(distinct_files + 1) */ + CBMHashTable *paths; /* scratch file-path set, cleared between names */ +} imp_name_index_t; + +/* Distinct files a name is DEFINED in. Counts distinct file paths, not + * distinct nodes — two defs of one name in the same file count once. + * Computed once per distinct name; every later node in the group is a hash + * hit. Name keys are borrowed from the gbuf nodes, which outlive the pass. */ +static int imp_distinct_file_count(const cbm_gbuf_t *gb, const char *name, imp_name_index_t *ix) { + if (!name || !*name || !ix || !ix->counts) { + return 0; + } + void *cached = cbm_ht_get(ix->counts, name); + if (cached) { + return (int)((intptr_t)cached - 1); + } + + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + int distinct = 0; + if (cbm_gbuf_find_by_name(gb, name, &nodes, &count) == 0 && count > 0) { + atomic_fetch_add_explicit(&g_importance_name_visits, (uint64_t)count, memory_order_relaxed); + if (ix->paths) { + cbm_ht_clear(ix->paths); + for (int i = 0; i < count; i++) { + const char *fp = nodes[i]->file_path; + if (!fp || cbm_ht_has(ix->paths, fp)) { + continue; + } + cbm_ht_set(ix->paths, fp, (void *)(intptr_t)1); + distinct++; + } + } else { + /* Scratch set unavailable (allocation failure): the generic-name + * multiplier degrades to "never generic" rather than falling back + * to a pairwise scan — a silent quadratic is worse than a missing + * 0.1 multiplier. */ + distinct = 0; + } + } + cbm_ht_set(ix->counts, name, (void *)(intptr_t)(distinct + 1)); + return distinct; +} + +static int incoming_edge_count(const cbm_gbuf_t *gb, int64_t id, const char *type) { + const cbm_gbuf_edge_t **edges = NULL; + int ne = 0; + if (cbm_gbuf_find_edges_by_target_type(gb, id, type, &edges, &ne) != 0) { + return 0; + } + return ne; +} + +void cbm_pipeline_pass_importance(cbm_pipeline_ctx_t *ctx) { + if (!ctx || !ctx->gbuf) { + return; + } + cbm_gbuf_t *gb = ctx->gbuf; + + /* Modest reservations, deliberately not sized from node_count: the memo + * holds one entry per DISTINCT name, far fewer than nodes, and reserving + * buckets for every node would cost hundreds of MB of transient peak on a + * multi-million-node graph to save a handful of amortised rehashes. */ + imp_name_index_t ix = { + .counts = cbm_ht_create(CBM_SZ_4K), + .paths = cbm_ht_create(CBM_SZ_256), + }; + if (!ix.counts) { + cbm_ht_free(ix.paths); + cbm_log_error("pass.importance", "msg", "name_index_alloc_failed"); + return; + } + + uint64_t updated = 0; + for (int li = 0; li < CBM_IMPORTANCE_LABEL_COUNT; li++) { + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_label(gb, CBM_IMPORTANCE_LABELS[li], &nodes, &count) != 0) { + continue; + } + for (int i = 0; i < count; i++) { + cbm_gbuf_node_t *n = (cbm_gbuf_node_t *)nodes[i]; + + int num_refs = + incoming_edge_count(gb, n->id, "CALLS") + incoming_edge_count(gb, n->id, "USAGE"); + double score = sqrt((double)num_refs); + + if (name_is_private(n->name)) { + score *= CBM_IMPORTANCE_PRIV_MUL; + } + if (imp_distinct_file_count(gb, n->name, &ix) >= CBM_IMPORTANCE_GENERIC_MIN_FILES) { + score *= CBM_IMPORTANCE_GENERIC_MUL; + } + if (name_is_distinctive(n->name)) { + score *= CBM_IMPORTANCE_DISTINCT_MUL; + } + if (cbm_is_test_path(n->file_path) || incoming_edge_count(gb, n->id, "TESTS") > 0) { + score *= CBM_IMPORTANCE_TEST_MUL; + } + + cbm_pipeline_importance_append_prop(n, score); + updated++; + } + } + + cbm_ht_free(ix.paths); + cbm_ht_free(ix.counts); + atomic_fetch_add_explicit(&g_importance_nodes, updated, memory_order_relaxed); + cbm_log_info("pass.importance", "symbols", itoa_imp(updated)); +} diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 0c1d5eee4..091d8c665 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -946,6 +946,9 @@ static void predump_complexity(cbm_pipeline_ctx_t *ctx) { static void predump_ensemble(cbm_pipeline_ctx_t *ctx) { cbm_pipeline_pass_ensemble_routing(ctx); } +static void predump_importance(cbm_pipeline_ctx_t *ctx) { + cbm_pipeline_pass_importance(ctx); +} static void run_predump_passes(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) { static const struct { @@ -953,12 +956,23 @@ static void run_predump_passes(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) { const char *name; bool moderate_only; /* true = skip in fast mode */ } passes[] = { - {predump_deco, "decorator_tags", false}, {predump_cfg, "configlink", false}, - {predump_route, "route_match", false}, {predump_ensemble, "ensemble_routing", false}, - {predump_sim, "similarity", true}, {predump_sem, "semantic_edges", true}, + {predump_deco, "decorator_tags", false}, + {predump_cfg, "configlink", false}, + {predump_route, "route_match", false}, + {predump_ensemble, "ensemble_routing", false}, + {predump_sim, "similarity", true}, + {predump_sem, "semantic_edges", true}, {predump_complexity, "complexity", false}, + /* Importance runs LAST: it reads CALLS/USAGE (extraction) and TESTS + * (pass_tests, which run_post_extraction runs before this loop), so + * every edge type its score depends on already exists here. */ + {predump_importance, "importance", false}, }; - enum { PREDUMP_PASS_COUNT = 7 }; + /* Derived from the table, never hand-written. A hand-written count that + * lags a newly appended entry silently skips the LAST pass while every + * test stays green — exactly the failure this expression makes + * impossible. */ + enum { PREDUMP_PASS_COUNT = (int)(sizeof(passes) / sizeof(passes[0])) }; struct timespec t; for (int i = 0; i < PREDUMP_PASS_COUNT && !check_cancel(p); i++) { /* "moderate_only" passes (similarity/semantic edges) run in FULL, diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index fca1c8f06..5de6bfd6a 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1456,6 +1456,20 @@ static int run_postpasses(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_file return rc < 0 ? rc : CBM_NOT_FOUND; } } + + /* Importance last, in EVERY mode — same ordering rule as the full path: + * it reads CALLS/USAGE (re-extraction, above) and TESTS (pass_tests, first + * in this function). The graph here is the rehydrated whole-project buffer, + * so nodes outside the changed set already carry an "importance" key from + * the previous run; cbm_pipeline_importance_append_prop overwrites it in + * place rather than appending a duplicate. */ + cbm_clock_gettime(CLOCK_MONOTONIC, &t); + cbm_pipeline_pass_importance(ctx); + cbm_log_info("pass.timing", "pass", "incr_importance", "elapsed_ms", + itoa_buf((int)elapsed_ms(t))); + if (cbm_pipeline_check_cancel(ctx)) { + return CBM_NOT_FOUND; + } return 0; } /* Publish the test-only legacy partial result through the same atomic diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 754fa4ffa..a25573733 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -630,6 +630,31 @@ int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx); * cycles (recursive). Runs on the graph buffer before the dump. */ void cbm_pipeline_pass_complexity(cbm_pipeline_ctx_t *ctx); +/* Pre-dump pass: per-symbol importance score (weighted degree). + * importance = sqrt(num_refs) * priv * generic * distinct * test_penalty + * Stored as a numeric "importance" key inside the node's EXISTING + * properties_json — no schema change and no index-format bump; indexes written + * by older builds simply lack the key and consumers must tolerate its absence. + * MUST run after pass_tests and after CALLS/USAGE extraction — a pass ordered + * earlier would see zero TESTS edges and num_refs = 0 everywhere. It is + * therefore registered last in run_predump_passes and last in the incremental + * post-pass sequence. */ +void cbm_pipeline_pass_importance(cbm_pipeline_ctx_t *ctx); + +/* Write the numeric "importance" key into a node's properties JSON object. + * IDEMPOTENT: an existing key is overwritten in place, never appended twice — + * the incremental path re-scores nodes rehydrated from the store that already + * carry the key. Non-static so the idempotence contract is directly testable. */ +void cbm_pipeline_importance_append_prop(cbm_gbuf_node_t *node, double score); + +/* Work counters for the importance pass (pass_importance.c). Deltas are read + * by the complexity suite's linearity gate; never reset by the pass itself, so + * nested/repeated runs compose. g_importance_name_visits counts same-name-group + * member visits during distinct-file counting — the quantity that goes + * superlinear if the per-distinct-name memoization is ever lost. */ +extern _Atomic uint64_t g_importance_nodes; +extern _Atomic uint64_t g_importance_name_visits; + /* ── Env URL scanner (pass_envscan.c) ────────────────────────────── */ typedef struct { diff --git a/tests/test_complexity.c b/tests/test_complexity.c index 1fe68a71e..d9c651ff9 100644 --- a/tests/test_complexity.c +++ b/tests/test_complexity.c @@ -353,6 +353,8 @@ typedef struct { uint64_t tail_lookups; uint64_t tail_candidates; uint64_t fallback_rows; + uint64_t imp_nodes; /* symbols scored by the importance pass */ + uint64_t imp_name_visits; /* same-name-group members visited by that pass */ double wall_s; } CxMetrics; @@ -373,6 +375,8 @@ static int cx_run(const char *root, const char *db_path, CxMetrics *out) { uint64_t tl0 = atomic_load_explicit(&g_lsp_tail_lookups, memory_order_relaxed); uint64_t tc0 = atomic_load_explicit(&g_lsp_tail_candidates, memory_order_relaxed); uint64_t fb0 = cbm_pp_lsp_linear_fallback_rows(); + uint64_t in0 = atomic_load_explicit(&g_importance_nodes, memory_order_relaxed); + uint64_t iv0 = atomic_load_explicit(&g_importance_name_visits, memory_order_relaxed); atomic_store_explicit(&g_cx_pass_count, 0, memory_order_relaxed); cbm_log_set_sink_ex(cx_pass_sink, CBM_LOG_SINK_TEE); @@ -407,6 +411,9 @@ static int cx_run(const char *root, const char *db_path, CxMetrics *out) { out->tail_lookups = atomic_load_explicit(&g_lsp_tail_lookups, memory_order_relaxed) - tl0; out->tail_candidates = atomic_load_explicit(&g_lsp_tail_candidates, memory_order_relaxed) - tc0; out->fallback_rows = cbm_pp_lsp_linear_fallback_rows() - fb0; + out->imp_nodes = atomic_load_explicit(&g_importance_nodes, memory_order_relaxed) - in0; + out->imp_name_visits = + atomic_load_explicit(&g_importance_name_visits, memory_order_relaxed) - iv0; cbm_store_t *s = cbm_store_open_path(db_path); if (!s) { @@ -717,9 +724,46 @@ TEST(complexity_throughput_report_written) { PASS(); } +/* Importance scoring must stay linear in the graph. Its generic-name + * multiplier needs |{files a name is defined in}|; computing that per NODE + * instead of once per distinct NAME makes a same-name group of size k cost + * O(k^3) — measured on a large Java corpus as a multi-minute cost for a + * handful of names. Under replication the same-name groups grow with the copy + * count, so the per-node shape lands at ~4x per doubling here while the + * memoized shape stays at ~2x. Gated on the counter ratio, never on wall time + * (O9). This counter is GATED, not merely recorded: the whole point of the + * work is that this quantity must not go superlinear. */ +TEST(complexity_importance_scoring_is_linear) { + if (cx_measure_pair() != 0) { + FAIL("failed to build/run the complexity corpus pair"); + } + const CxMetrics *a = &g_cx_base; + const CxMetrics *b = &g_cx_doubled; + + printf(" imp_nodes %llu -> %llu imp_name_visits %llu -> %llu\n", + (unsigned long long)a->imp_nodes, (unsigned long long)b->imp_nodes, + (unsigned long long)a->imp_name_visits, (unsigned long long)b->imp_name_visits); + + /* Non-vacuous, twice over: the pass must have run at all (a miscounted + * PREDUMP_PASS_COUNT would silently skip it and zero both counters), and + * the base leg must have produced enough work for a ratio to mean + * anything. A zero here is a wiring defect to fix, never a pass. */ + ASSERT_GT((long long)a->imp_nodes, 0); + ASSERT_GT((long long)a->imp_name_visits, (long long)CX_MIN_BASE_WORK); + + double node_r = cx_ratio((double)b->imp_nodes, (double)a->imp_nodes); + double visit_r = cx_ratio((double)b->imp_name_visits, (double)a->imp_name_visits); + printf(" imp_nodes ratio %.2f imp_name_visits ratio %.2f (linear ~2, per-node ~4)\n", + node_r, visit_r); + ASSERT_TRUE(node_r >= CX_RATIO_LO && node_r <= CX_RATIO_HI); + ASSERT_TRUE(visit_r <= CX_RATIO_HI); + PASS(); +} + SUITE(complexity) { RUN_TEST(complexity_replicated_modules_scale_linearly); RUN_TEST(complexity_perfile_registry_work_is_linear); + RUN_TEST(complexity_importance_scoring_is_linear); RUN_TEST(complexity_shared_package_growth_stays_linear); RUN_TEST(complexity_throughput_report_written); } diff --git a/tests/test_importance.c b/tests/test_importance.c new file mode 100644 index 000000000..9452a49e9 --- /dev/null +++ b/tests/test_importance.c @@ -0,0 +1,641 @@ +/* + * test_importance.c — index-time per-symbol importance score (pass_importance.c). + * + * Formula under test: + * importance = sqrt(num_refs) * priv * generic * distinct * test_penalty + * + * Three of these tests exist to catch failure modes that are SILENT — they + * produce a wrong index with a fully green build: + * + * - importance_pass_actually_runs_in_full_pipeline + * binds the predump registration. A hand-written PREDUMP_PASS_COUNT that + * lags the pass table skips the LAST-registered pass and nothing else + * notices. This test fails loudly if the score never lands. + * - importance_rehydrated_node_keeps_exactly_one_key + * - importance_second_index_run_keeps_exactly_one_key + * bind the idempotent write-back. The incremental path rehydrates nodes + * that already carry "importance"; a plain append yields + * {"importance":1.0,...,"importance":2.0} — property corruption, not a + * build failure. + * - importance_distinct_file_count_is_linear_in_group_size + * binds the cost shape: the distinct-file count must be computed once + * per distinct NAME, not once per node. The per-node shape is O(k^3) in + * a same-name group and is measurably disqualifying on real corpora. + */ +#include "test_framework.h" +#include "test_helpers.h" + +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" +#include "graph_buffer/graph_buffer.h" +#include "store/store.h" +#include "foundation/compat.h" +#include "foundation/log.h" + +#include +#include +#include +#include +#include +#include + +/* ── Shared helpers ───────────────────────────────────────────────────── */ + +/* Parse the numeric value of "importance" out of a properties_json blob. + * Returns a sentinel (-999.0) when the key is absent — callers that care about + * presence assert on imp_key_count() instead. */ +static double imp_value(const char *json) { + if (!json) { + return -999.0; + } + const char *p = strstr(json, "\"importance\":"); + if (!p) { + return -999.0; + } + return strtod(p + strlen("\"importance\":"), NULL); +} + +/* How many times the key appears. The duplicate-key corruption this suite + * guards against shows up here as 2. */ +static int imp_key_count(const char *json) { + if (!json) { + return 0; + } + int n = 0; + const char *p = json; + while ((p = strstr(p, "\"importance\":")) != NULL) { + n++; + p += strlen("\"importance\":"); + } + return n; +} + +static void imp_run_pass(cbm_gbuf_t *gb) { + atomic_int cancelled = 0; + cbm_pipeline_ctx_t ctx = { + .project_name = "test-proj", + .repo_path = "/tmp/test", + .gbuf = gb, + .cancelled = &cancelled, + }; + cbm_pipeline_pass_importance(&ctx); +} + +/* Every stored node of the given label must carry exactly `expect` copies of + * the key. Returns the number of nodes checked, or -1 on a store error. */ +static int imp_check_label(cbm_store_t *s, const char *project, const char *label, int expect) { + cbm_node_t *nodes = NULL; + int count = 0; + if (cbm_store_find_nodes_by_label(s, project, label, &nodes, &count) != CBM_STORE_OK) { + return -1; + } + int bad = 0; + for (int i = 0; i < count; i++) { + if (imp_key_count(nodes[i].properties_json) != expect) { + bad++; + printf(" %s node '%s' props=%s\n", label, nodes[i].name ? nodes[i].name : "?", + nodes[i].properties_json ? nodes[i].properties_json : "(null)"); + } + } + cbm_store_free_nodes(nodes, count); + return bad == 0 ? count : -1; +} + +/* ── Registration: the pass must actually execute ─────────────────────── */ + +/* A miscounted PREDUMP_PASS_COUNT silently drops the last-registered pass. + * The score is the only observable, so assert on the score. */ +TEST(importance_pass_actually_runs_in_full_pipeline) { + char *tmp = th_mktempdir("cbm_imp_run"); + if (!tmp) { + FAIL("tmpdir"); + } + char root[512]; + snprintf(root, sizeof(root), "%s", tmp); /* th_mktempdir returns a static buffer */ + + ASSERT_EQ(th_write_file(TH_PATH(root, "main.py"), + "def helper():\n return 1\n\n" + "def caller():\n return helper() + helper()\n\n" + "class Widget:\n def render(self):\n return helper()\n"), + 0); + + char db_path[600]; + snprintf(db_path, sizeof(db_path), "%s/graph.db", root); + cbm_pipeline_t *p = cbm_pipeline_new(root, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + char project[256]; + snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(p)); + cbm_pipeline_free(p); + + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + + /* Non-vacuous: the corpus really produced symbols of each kind. */ + int fc = imp_check_label(s, project, "Function", 1); + int mc = imp_check_label(s, project, "Method", 1); + int cc = imp_check_label(s, project, "Class", 1); + printf(" scored Function=%d Method=%d Class=%d\n", fc, mc, cc); + ASSERT_GT(fc, 0); + ASSERT_GT(mc, 0); + ASSERT_GT(cc, 0); + + /* File nodes are deliberately excluded from scoring. */ + cbm_node_t *files = NULL; + int filec = 0; + ASSERT_EQ(cbm_store_find_nodes_by_label(s, project, "File", &files, &filec), CBM_STORE_OK); + for (int i = 0; i < filec; i++) { + ASSERT_EQ(imp_key_count(files[i].properties_json), 0); + } + cbm_store_free_nodes(files, filec); + + cbm_store_close(s); + th_rmtree(root); + PASS(); +} + +/* ── Formula ──────────────────────────────────────────────────────────── */ + +TEST(importance_base_is_sqrt_of_incoming_refs) { + cbm_gbuf_t *gb = cbm_gbuf_new("test-proj", "/tmp/test"); + ASSERT_NOT_NULL(gb); + + int64_t target = + cbm_gbuf_upsert_node(gb, "Function", "target", "pkg.target", "pkg/main.go", 1, 1, "{}"); + int64_t c1 = cbm_gbuf_upsert_node(gb, "Function", "c1", "pkg.c1", "pkg/main.go", 2, 2, "{}"); + int64_t c2 = cbm_gbuf_upsert_node(gb, "Function", "c2", "pkg.c2", "pkg/main.go", 3, 3, "{}"); + int64_t c3 = cbm_gbuf_upsert_node(gb, "Function", "c3", "pkg.c3", "pkg/main.go", 4, 4, "{}"); + int64_t lonely = + cbm_gbuf_upsert_node(gb, "Function", "lonely", "pkg.lonely", "pkg/main.go", 5, 5, "{}"); + ASSERT_GT(target, 0); + ASSERT_GT(lonely, 0); + + cbm_gbuf_insert_edge(gb, c1, target, "CALLS", "{}"); + cbm_gbuf_insert_edge(gb, c2, target, "CALLS", "{}"); + cbm_gbuf_insert_edge(gb, c3, target, "USAGE", "{}"); /* spans BOTH edge types */ + + imp_run_pass(gb); + + const cbm_gbuf_node_t *tn = cbm_gbuf_find_by_id(gb, target); + ASSERT_NOT_NULL(tn); + ASSERT_EQ(imp_key_count(tn->properties_json), 1); + ASSERT_FLOAT_EQ(imp_value(tn->properties_json), sqrt(3.0), 1e-6); + + /* num_refs == 0 -> sqrt(0) == 0, still written. */ + const cbm_gbuf_node_t *ln = cbm_gbuf_find_by_id(gb, lonely); + ASSERT_NOT_NULL(ln); + ASSERT_EQ(imp_key_count(ln->properties_json), 1); + ASSERT_FLOAT_EQ(imp_value(ln->properties_json), 0.0, 1e-9); + + cbm_gbuf_free(gb); + PASS(); +} + +TEST(importance_private_names_are_demoted) { + cbm_gbuf_t *gb = cbm_gbuf_new("test-proj", "/tmp/test"); + ASSERT_NOT_NULL(gb); + + int64_t priv = + cbm_gbuf_upsert_node(gb, "Function", "_run", "pkg._run", "pkg/main.go", 1, 1, "{}"); + int64_t pub = cbm_gbuf_upsert_node(gb, "Function", "run", "pkg.run", "pkg/main.go", 2, 2, "{}"); + for (int i = 0; i < 4; i++) { + char name[CBM_SZ_32]; + char qn[CBM_SZ_64]; + snprintf(name, sizeof(name), "p%d", i); + snprintf(qn, sizeof(qn), "pkg.p%d", i); + int64_t c = + cbm_gbuf_upsert_node(gb, "Function", name, qn, "pkg/main.go", 10 + i, 10 + i, "{}"); + cbm_gbuf_insert_edge(gb, c, priv, "CALLS", "{}"); + snprintf(name, sizeof(name), "q%d", i); + snprintf(qn, sizeof(qn), "pkg.q%d", i); + c = cbm_gbuf_upsert_node(gb, "Function", name, qn, "pkg/main.go", 20 + i, 20 + i, "{}"); + cbm_gbuf_insert_edge(gb, c, pub, "CALLS", "{}"); + } + + imp_run_pass(gb); + + ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, priv)->properties_json), sqrt(4.0) * 0.1, + 1e-6); + ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, pub)->properties_json), sqrt(4.0), 1e-6); + + cbm_gbuf_free(gb); + PASS(); +} + +/* Generic = defined in >= 5 DISTINCT FILES. Two defs of one name in the same + * file count once, so the boundary is tested on files, not on node count. */ +TEST(importance_generic_names_are_demoted_on_distinct_files) { + cbm_gbuf_t *gb = cbm_gbuf_new("test-proj", "/tmp/test"); + ASSERT_NOT_NULL(gb); + + /* "generic": 5 distinct files -> demoted. */ + int64_t generic = 0; + for (int i = 0; i < 5; i++) { + char qn[CBM_SZ_64]; + char file[CBM_SZ_64]; + snprintf(qn, sizeof(qn), "pkg%d.generic", i); + snprintf(file, sizeof(file), "pkg%d/main.go", i); + int64_t id = cbm_gbuf_upsert_node(gb, "Function", "generic", qn, file, 1, 1, "{}"); + if (i == 0) { + generic = id; + } + } + /* "narrow": 4 distinct files, but 6 nodes — two duplicates share a file, + * so the DISTINCT count is 4 and it stays undemoted. */ + int64_t narrow = 0; + for (int i = 0; i < 4; i++) { + char qn[CBM_SZ_64]; + char file[CBM_SZ_64]; + snprintf(qn, sizeof(qn), "n%d.narrow", i); + snprintf(file, sizeof(file), "n%d/main.go", i); + int64_t id = cbm_gbuf_upsert_node(gb, "Function", "narrow", qn, file, 1, 1, "{}"); + if (i == 0) { + narrow = id; + } + } + cbm_gbuf_upsert_node(gb, "Function", "narrow", "n0.narrow_b", "n0/main.go", 9, 9, "{}"); + cbm_gbuf_upsert_node(gb, "Function", "narrow", "n1.narrow_b", "n1/main.go", 9, 9, "{}"); + + for (int i = 0; i < 9; i++) { + char name[CBM_SZ_32]; + char qn[CBM_SZ_64]; + snprintf(name, sizeof(name), "z%d", i); + snprintf(qn, sizeof(qn), "z.z%d", i); + int64_t c = cbm_gbuf_upsert_node(gb, "Function", name, qn, "z/main.go", i, i, "{}"); + cbm_gbuf_insert_edge(gb, c, i < 4 ? generic : narrow, "CALLS", "{}"); + } + + imp_run_pass(gb); + + /* generic: sqrt(4) * 0.1; narrow: sqrt(5), undemoted. */ + ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, generic)->properties_json), sqrt(4.0) * 0.1, + 1e-6); + ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, narrow)->properties_json), sqrt(5.0), 1e-6); + + cbm_gbuf_free(gb); + PASS(); +} + +TEST(importance_distinctive_names_are_promoted) { + cbm_gbuf_t *gb = cbm_gbuf_new("test-proj", "/tmp/test"); + ASSERT_NOT_NULL(gb); + + /* snake_case >= 8, camelCase >= 8, and a plain long lowercase word that is + * NOT distinctive (no '_', no hump). */ + int64_t snake = cbm_gbuf_upsert_node(gb, "Function", "load_user_profile", + "pkg.load_user_profile", "pkg/a.go", 1, 1, "{}"); + int64_t camel = cbm_gbuf_upsert_node(gb, "Function", "loadUserProfile", "pkg.loadUserProfile", + "pkg/b.go", 1, 1, "{}"); + int64_t plain = + cbm_gbuf_upsert_node(gb, "Function", "aggregate", "pkg.aggregate", "pkg/c.go", 1, 1, "{}"); + int64_t shortnm = + cbm_gbuf_upsert_node(gb, "Function", "get_x", "pkg.get_x", "pkg/d.go", 1, 1, "{}"); + + const int64_t targets[] = {snake, camel, plain, shortnm}; + for (int t = 0; t < 4; t++) { + char name[CBM_SZ_32]; + char qn[CBM_SZ_64]; + snprintf(name, sizeof(name), "cc%d", t); + snprintf(qn, sizeof(qn), "z.cc%d", t); + int64_t c = cbm_gbuf_upsert_node(gb, "Function", name, qn, "z/main.go", t, t, "{}"); + cbm_gbuf_insert_edge(gb, c, targets[t], "CALLS", "{}"); + } + + imp_run_pass(gb); + + ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, snake)->properties_json), 1.0 * 10.0, 1e-6); + ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, camel)->properties_json), 1.0 * 10.0, 1e-6); + ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, plain)->properties_json), 1.0, 1e-6); + ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, shortnm)->properties_json), 1.0, 1e-6); + + cbm_gbuf_free(gb); + PASS(); +} + +/* Test scaffolding is demoted two ways: living in a test file (the graph's own + * cbm_is_test_path classifier) or being the target of a TESTS edge. */ +TEST(importance_test_scaffolding_is_demoted) { + cbm_gbuf_t *gb = cbm_gbuf_new("test-proj", "/tmp/test"); + ASSERT_NOT_NULL(gb); + + int64_t in_test_file = cbm_gbuf_upsert_node(gb, "Function", "fixture", "t.fixture", + "pkg/thing_test.go", 1, 1, "{}"); + int64_t tests_target = + cbm_gbuf_upsert_node(gb, "Function", "helper", "pkg.helper", "pkg/util.go", 1, 1, "{}"); + int64_t plain = + cbm_gbuf_upsert_node(gb, "Function", "worker", "pkg.worker", "pkg/util.go", 2, 2, "{}"); + /* A production file whose NAME merely contains "test" must NOT be demoted. */ + int64_t testutil = cbm_gbuf_upsert_node(gb, "Function", "spawn", "pkg.spawn", + "pkg/testutil_helpers.go", 1, 1, "{}"); + + const int64_t targets[] = {in_test_file, tests_target, plain, testutil}; + for (int t = 0; t < 4; t++) { + char name[CBM_SZ_32]; + char qn[CBM_SZ_64]; + snprintf(name, sizeof(name), "d%d", t); + snprintf(qn, sizeof(qn), "z.d%d", t); + int64_t c = cbm_gbuf_upsert_node(gb, "Function", name, qn, "z/main.go", t, t, "{}"); + cbm_gbuf_insert_edge(gb, c, targets[t], "CALLS", "{}"); + } + int64_t tfn = cbm_gbuf_upsert_node(gb, "Function", "TestHelper", "t.TestHelper", + "pkg/util_test.go", 5, 5, "{}"); + cbm_gbuf_insert_edge(gb, tfn, tests_target, "TESTS", "{}"); + + imp_run_pass(gb); + + ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, in_test_file)->properties_json), 1.0 * 0.1, + 1e-6); + ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, tests_target)->properties_json), 1.0 * 0.1, + 1e-6); + ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, plain)->properties_json), 1.0, 1e-6); + ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, testutil)->properties_json), 1.0, 1e-6); + + cbm_gbuf_free(gb); + PASS(); +} + +/* ── Idempotent write-back ────────────────────────────────────────────── */ + +TEST(importance_append_prop_overwrites_instead_of_duplicating) { + cbm_gbuf_t *gb = cbm_gbuf_new("test-proj", "/tmp/test"); + ASSERT_NOT_NULL(gb); + int64_t id = cbm_gbuf_upsert_node(gb, "Function", "f", "p.f", "p/a.go", 1, 1, + "{\"loop_depth\":2,\"recursive\":false}"); + cbm_gbuf_node_t *n = (cbm_gbuf_node_t *)cbm_gbuf_find_by_id(gb, id); + ASSERT_NOT_NULL(n); + + cbm_pipeline_importance_append_prop(n, 1.5); + ASSERT_EQ(imp_key_count(n->properties_json), 1); + ASSERT_FLOAT_EQ(imp_value(n->properties_json), 1.5, 1e-6); + + cbm_pipeline_importance_append_prop(n, 42.25); + printf(" after second write: %s\n", n->properties_json); + ASSERT_EQ(imp_key_count(n->properties_json), 1); + ASSERT_FLOAT_EQ(imp_value(n->properties_json), 42.25, 1e-6); + + /* Sibling keys survive the in-place overwrite, on both sides of the key. */ + ASSERT_TRUE(strstr(n->properties_json, "\"loop_depth\":2") != NULL); + ASSERT_TRUE(strstr(n->properties_json, "\"recursive\":false") != NULL); + + /* A key that is NOT last: overwrite must not truncate the tail. */ + free(n->properties_json); + n->properties_json = strdup("{\"importance\":0.000000,\"tail\":7}"); + ASSERT_NOT_NULL(n->properties_json); + cbm_pipeline_importance_append_prop(n, 3.0); + ASSERT_EQ(imp_key_count(n->properties_json), 1); + ASSERT_FLOAT_EQ(imp_value(n->properties_json), 3.0, 1e-6); + ASSERT_TRUE(strstr(n->properties_json, "\"tail\":7") != NULL); + + /* A string VALUE that merely contains the key text is not a key. */ + free(n->properties_json); + n->properties_json = strdup("{\"doc\":\"see \\\"importance\\\": below\"}"); + ASSERT_NOT_NULL(n->properties_json); + cbm_pipeline_importance_append_prop(n, 2.0); + ASSERT_FLOAT_EQ(imp_value(n->properties_json), 2.0, 1e-6); + ASSERT_TRUE(strstr(n->properties_json, "\"doc\":") != NULL); + + /* Empty object and a non-object blob. */ + free(n->properties_json); + n->properties_json = strdup("{}"); + ASSERT_NOT_NULL(n->properties_json); + cbm_pipeline_importance_append_prop(n, 0.5); + ASSERT_STR_EQ(n->properties_json, "{\"importance\":0.500000}"); + + free(n->properties_json); + n->properties_json = strdup("not json"); + ASSERT_NOT_NULL(n->properties_json); + cbm_pipeline_importance_append_prop(n, 9.0); + ASSERT_STR_EQ(n->properties_json, "not json"); + + cbm_gbuf_free(gb); + PASS(); +} + +/* Running the pass twice over the same graph — the shape the incremental path + * produces when it re-scores an already-scored rehydrated buffer. */ +TEST(importance_pass_is_idempotent_across_reruns) { + cbm_gbuf_t *gb = cbm_gbuf_new("test-proj", "/tmp/test"); + ASSERT_NOT_NULL(gb); + int64_t t = + cbm_gbuf_upsert_node(gb, "Function", "target", "pkg.target", "pkg/main.go", 1, 1, "{}"); + int64_t c = + cbm_gbuf_upsert_node(gb, "Function", "caller", "pkg.caller", "pkg/main.go", 2, 2, "{}"); + cbm_gbuf_insert_edge(gb, c, t, "CALLS", "{}"); + + imp_run_pass(gb); + char first[CBM_SZ_512]; + snprintf(first, sizeof(first), "%s", cbm_gbuf_find_by_id(gb, t)->properties_json); + imp_run_pass(gb); + const char *second = cbm_gbuf_find_by_id(gb, t)->properties_json; + + printf(" run1=%s run2=%s\n", first, second); + ASSERT_EQ(imp_key_count(second), 1); + ASSERT_STR_EQ(first, second); /* re-scoring is a fixed point */ + + cbm_gbuf_free(gb); + PASS(); +} + +/* The literal incremental shape: index, rehydrate the stored graph exactly as + * pipeline_incremental.c does, re-run the pass. Nodes arrive already carrying + * the key — an append-only write-back yields two. */ +TEST(importance_rehydrated_node_keeps_exactly_one_key) { + char *tmp = th_mktempdir("cbm_imp_rehy"); + if (!tmp) { + FAIL("tmpdir"); + } + char root[512]; + snprintf(root, sizeof(root), "%s", tmp); /* th_mktempdir returns a static buffer */ + + ASSERT_EQ(th_write_file(TH_PATH(root, "main.py"), + "def helper():\n return 1\n\n" + "def caller():\n return helper() + helper()\n"), + 0); + char db_path[600]; + snprintf(db_path, sizeof(db_path), "%s/graph.db", root); + cbm_pipeline_t *p = cbm_pipeline_new(root, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + char project[256]; + snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(p)); + cbm_pipeline_free(p); + + cbm_gbuf_t *gb = cbm_gbuf_new(project, root); + ASSERT_NOT_NULL(gb); + ASSERT_EQ(cbm_gbuf_load_from_db(gb, db_path, project), 0); + + /* Precondition: the rehydrated nodes already carry the key. Without this + * the test could pass while proving nothing. */ + const cbm_gbuf_node_t **fns = NULL; + int fc = 0; + ASSERT_EQ(cbm_gbuf_find_by_label(gb, "Function", &fns, &fc), 0); + ASSERT_GT(fc, 0); + for (int i = 0; i < fc; i++) { + ASSERT_EQ(imp_key_count(fns[i]->properties_json), 1); + } + + imp_run_pass(gb); + + ASSERT_EQ(cbm_gbuf_find_by_label(gb, "Function", &fns, &fc), 0); + ASSERT_GT(fc, 0); + for (int i = 0; i < fc; i++) { + if (imp_key_count(fns[i]->properties_json) != 1) { + printf(" duplicate key on '%s': %s\n", fns[i]->name, fns[i]->properties_json); + } + ASSERT_EQ(imp_key_count(fns[i]->properties_json), 1); + } + + cbm_gbuf_free(gb); + th_rmtree(root); + PASS(); +} + +/* Pass-timing observer: which importance-pass call sites fired. */ +static atomic_int g_saw_incr_importance = 0; +static atomic_int g_saw_full_importance = 0; +static void imp_pass_sink(const char *line) { + if (!line || !strstr(line, "pass.timing")) { + return; + } + if (strstr(line, "pass=incr_importance")) { + atomic_fetch_add_explicit(&g_saw_incr_importance, 1, memory_order_relaxed); + } else if (strstr(line, "pass=importance")) { + atomic_fetch_add_explicit(&g_saw_full_importance, 1, memory_order_relaxed); + } +} + +/* End-to-end, on the incremental route that actually rehydrates the stored + * graph (LEGACY_PARTIAL — production's other incremental route, closure + * repair, works on a delta buffer and never sees pre-scored nodes). Index, + * edit a file, index again: the run must be observed on that route, the pass + * must have run on it, and every stored symbol must carry exactly one key. An + * append-only write-back produces two here. */ +TEST(importance_second_index_run_keeps_exactly_one_key) { + char *tmp = th_mktempdir("cbm_imp_incr"); + if (!tmp) { + FAIL("tmpdir"); + } + char root[512]; + snprintf(root, sizeof(root), "%s", tmp); /* th_mktempdir returns a static buffer */ + + ASSERT_EQ(th_write_file(TH_PATH(root, "a.py"), "def helper():\n return 1\n"), 0); + ASSERT_EQ( + th_write_file(TH_PATH(root, "b.py"), "import a\n\ndef caller():\n return a.helper()\n"), + 0); + + char db_path[600]; + snprintf(db_path, sizeof(db_path), "%s/graph.db", root); + + atomic_store(&g_saw_incr_importance, 0); + atomic_store(&g_saw_full_importance, 0); + cbm_log_set_sink_ex(imp_pass_sink, CBM_LOG_SINK_TEE); + + cbm_pipeline_t *p1 = cbm_pipeline_new(root, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p1); + ASSERT_EQ(cbm_pipeline_run(p1), 0); + char project[256]; + snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(p1)); + cbm_pipeline_free(p1); + + /* Edit one file so the second run has real work but stays incremental. */ + ASSERT_EQ(th_write_file(TH_PATH(root, "b.py"), + "import a\n\ndef caller():\n return a.helper() + a.helper()\n"), + 0); + + cbm_pipeline_incremental_test_force_legacy_partial_once(); + cbm_pipeline_t *p2 = cbm_pipeline_new(root, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p2); + ASSERT_EQ(cbm_pipeline_run(p2), 0); + cbm_incremental_route_t route = cbm_pipeline_incremental_test_last_route(); + cbm_pipeline_free(p2); + cbm_log_set_sink(NULL); + + /* Route proof: without it a silent fall-back to a full reindex would make + * the duplicate-key assertion below vacuous. */ + ASSERT_EQ((int)route, (int)CBM_INCREMENTAL_ROUTE_LEGACY_PARTIAL); + + int incr = atomic_load(&g_saw_incr_importance); + int full = atomic_load(&g_saw_full_importance); + printf(" importance pass runs: full=%d incremental=%d\n", full, incr); + ASSERT_GT(full, 0); /* first run scored via the predump table */ + /* The second run must have re-scored too — through whichever path it took. + * If it was NOT incremental the duplicate-key trap is untested, so say so + * rather than passing quietly. */ + if (incr == 0) { + th_rmtree(root); + FAIL("second run did not re-score on the incremental path — trap untested"); + } + + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + int fc = imp_check_label(s, project, "Function", 1); + printf(" functions with exactly one importance key: %d\n", fc); + ASSERT_GT(fc, 0); + cbm_store_close(s); + + th_rmtree(root); + PASS(); +} + +/* ── Cost shape ───────────────────────────────────────────────────────── */ + +/* The distinct-file count must be computed once per distinct NAME. Computing + * it per NODE makes a same-name group of size k cost O(k^3) — measurably + * disqualifying on real corpora. Assert on the pass's own work counter, not + * on wall time: doubling a same-name group must roughly double the work. */ +TEST(importance_distinct_file_count_is_linear_in_group_size) { + enum { K = 200 }; + + uint64_t visits[2]; + for (int leg = 0; leg < 2; leg++) { + int k = (leg == 0) ? K : 2 * K; + cbm_gbuf_t *gb = cbm_gbuf_new("test-proj", "/tmp/test"); + ASSERT_NOT_NULL(gb); + for (int i = 0; i < k; i++) { + char qn[CBM_SZ_64]; + char file[CBM_SZ_64]; + snprintf(qn, sizeof(qn), "pkg%d.toString", i); + snprintf(file, sizeof(file), "pkg%d/a.java", i); + ASSERT_GT(cbm_gbuf_upsert_node(gb, "Method", "toString", qn, file, 1, 1, "{}"), 0); + } + uint64_t v0 = atomic_load_explicit(&g_importance_name_visits, memory_order_relaxed); + imp_run_pass(gb); + visits[leg] = atomic_load_explicit(&g_importance_name_visits, memory_order_relaxed) - v0; + cbm_gbuf_free(gb); + } + + double ratio = (visits[0] > 0) ? (double)visits[1] / (double)visits[0] : 0.0; + printf(" name visits k=%d -> %llu, k=%d -> %llu (ratio %.2f; linear ~2, per-node ~4)\n", K, + (unsigned long long)visits[0], 2 * K, (unsigned long long)visits[1], ratio); + + /* Non-vacuous: the group really was scanned. */ + ASSERT_GT((long long)visits[0], 0); + /* Memoized: one scan of the whole group, so visits == k exactly. */ + ASSERT_EQ((long long)visits[0], (long long)K); + ASSERT_EQ((long long)visits[1], (long long)(2 * K)); + ASSERT_TRUE(ratio <= 2.75); + PASS(); +} + +/* ── Suite ────────────────────────────────────────────────────────────── */ + +SUITE(importance) { + /* Registration — catches a miscounted PREDUMP_PASS_COUNT. */ + RUN_TEST(importance_pass_actually_runs_in_full_pipeline); + + /* Formula. */ + RUN_TEST(importance_base_is_sqrt_of_incoming_refs); + RUN_TEST(importance_private_names_are_demoted); + RUN_TEST(importance_generic_names_are_demoted_on_distinct_files); + RUN_TEST(importance_distinctive_names_are_promoted); + RUN_TEST(importance_test_scaffolding_is_demoted); + + /* Idempotent write-back — the incremental duplicate-key trap. */ + RUN_TEST(importance_append_prop_overwrites_instead_of_duplicating); + RUN_TEST(importance_pass_is_idempotent_across_reruns); + RUN_TEST(importance_rehydrated_node_keeps_exactly_one_key); + RUN_TEST(importance_second_index_run_keeps_exactly_one_key); + + /* Cost shape. */ + RUN_TEST(importance_distinct_file_count_is_linear_in_group_size); +} diff --git a/tests/test_main.c b/tests/test_main.c index 0e478ccad..2d57f5468 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -768,6 +768,7 @@ extern void suite_discover(void); extern void suite_graph_buffer(void); extern void suite_registry(void); extern void suite_pipeline(void); +extern void suite_importance(void); extern void suite_pipeline_semantic_manifest_repro(void); extern void suite_cross_repo(void); extern void suite_index_resilience(void); @@ -1041,6 +1042,7 @@ int main(int argc, char **argv) { /* Pipeline (M8) */ RUN_SELECTED_SUITE(registry); RUN_SELECTED_SUITE(pipeline); + RUN_SELECTED_SUITE(importance); RUN_SELECTED_SUITE(index_format); RUN_SELECTED_SUITE(pipeline_semantic_manifest_repro); RUN_SELECTED_SUITE(call_reference_contract); From 396ac348b5abe0e1a7c6cc7bb9d604e8ee76cefc Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 28 Aug 2026 20:56:33 +0200 Subject: [PATCH 11/31] fix(pipeline): score importance over the staging store on the delta route The closure-delta incremental route scored importance off a graph that does not contain the project's edges, and persisted the result. cbm_delta_preseed fills that route's gbuf with PROXY nodes -- id, label, name, qualified name, file path, and nothing else. Only the re-extracted files carry real edges; every inbound edge from an unchanged file is snapshotted and re-linked later, inside cbm_delta_patch. Scoring in run_postpasses therefore read an in-degree of zero for symbols with hundreds of real callers, and because the repaired files' rows are purged and re-inserted, cbm_delta_patch wrote those zeros to disk. Measured on a fixture whose `helper` has 25 callers: full index 5.000000, after one closure-delta re-index 0.000000. closure_try_plan runs first, so this is the preferred warm path, not an edge case. The comment at the call site claimed a rehydrated whole-project buffer; that is true only of the legacy-partial route, and it is now stated correctly. The fix rescores in SQL over the staging store, immediately after cbm_delta_patch has inserted the new nodes and re-linked the inbound edges -- the first moment on this route where the complete graph is queryable. Loading the full graph into RAM instead would have given back the point of the delta route. Rescoring is project-wide, not limited to the changed files. Distinct-files- per-name and in-degree are both GLOBAL inputs: a body-only edit in one file can change the score of a symbol defined in a file that did not change. A scoped rescore would leave those stale. TWO ROUTES, ONE RULE. The scoring rule and the JSON write-back are not reimplemented in SQL. cbm_pipeline_importance_score() and cbm_pipeline_importance_set_prop() are now single definitions that both routes call -- the SQL side reaches them through two SQLite user-defined functions, so the same C code runs either way and the rule cannot drift. SQL contributes only the three aggregates, each an indexed GROUP BY. What remains free is the input GATHERING, and tests bind that directly: a full index and a closure-delta incremental index converging on the same tree must produce identical scores for every symbol. Both new tests assert the route was actually taken, so neither can pass by silently running some other path. Cost, measured on a 666 MB Java corpus: cold full index unchanged at 83 s (the recompute does not run on that path); warm closure-delta re-index 20 s without the recompute, 26 s with it. The delta route keeps its advantage over a full reindex (26 s vs 83 s). Signed-off-by: Martin Vogel --- src/pipeline/pass_importance.c | 242 ++++++++++++++++++++----- src/pipeline/pipeline_incremental.c | 57 ++++-- src/pipeline/pipeline_internal.h | 38 +++- tests/test_importance.c | 270 ++++++++++++++++++++++++++++ 4 files changed, 548 insertions(+), 59 deletions(-) diff --git a/src/pipeline/pass_importance.c b/src/pipeline/pass_importance.c index a727ddee0..c544d0a10 100644 --- a/src/pipeline/pass_importance.c +++ b/src/pipeline/pass_importance.c @@ -60,7 +60,9 @@ #include "foundation/hash_table.h" #include "foundation/log.h" #include "foundation/compat.h" +#include "store/store.h" #include "cbm.h" +#include "sqlite3.h" #include #include @@ -75,6 +77,7 @@ * Never reset here: callers snapshot before/after so nested runs compose. */ _Atomic uint64_t g_importance_nodes = 0; _Atomic uint64_t g_importance_name_visits = 0; +_Atomic uint64_t g_importance_store_rows = 0; enum { CBM_IMPORTANCE_GENERIC_MIN_FILES = 5, /* name defined in >= N files -> generic */ @@ -212,37 +215,39 @@ static const char *imp_find_key(const char *json, const char *key) { return NULL; } -/* Write the numeric "importance" key into a node's properties JSON object. +/* Produce a copy of `json` carrying the numeric "importance" key set to + * `score`, or NULL if `json` is not a JSON object (caller leaves it alone) or + * on allocation failure. Caller owns the result. * - * IDEMPOTENT BY CONTRACT. The incremental path rehydrates nodes straight out - * of the store (cbm_gbuf_load_from_db), and those nodes ALREADY carry an - * "importance" key from the previous run. A pure append would emit - * {"importance":1.0,...,"importance":2.0} — silent property corruption that - * no build or schema check would catch. So: overwrite an existing key in - * place, append only when the key is genuinely absent. + * IDEMPOTENT BY CONTRACT. Nodes reach this function already carrying an + * "importance" key on every re-scoring route: the legacy-partial incremental + * path rehydrates them via cbm_gbuf_load_from_db, and the store-level + * recompute reads them straight out of SQLite. A pure append would emit + * {"importance":1.0,...,"importance":2.0} — silent property corruption that no + * build or schema check would catch. So: overwrite an existing key in place, + * append only when the key is genuinely absent. * - * A properties_json that is not a JSON object is left untouched. */ -void cbm_pipeline_importance_append_prop(cbm_gbuf_node_t *node, double score) { - if (!node) { - return; - } - const char *old = node->properties_json ? node->properties_json : "{}"; + * SINGLE DEFINITION. Both scoring routes (the in-memory gbuf pass and the + * SQL-level store recompute) write through this one function, so the JSON + * shape cannot drift between them. */ +char *cbm_pipeline_importance_set_prop(const char *json, double score) { + const char *old = json ? json : "{}"; size_t olen = strlen(old); if (olen < 2 || old[olen - 1] != '}') { - return; /* not a JSON object — leave untouched */ + return NULL; /* not a JSON object — leave untouched */ } char val[CBM_SZ_32]; int vn = snprintf(val, sizeof(val), "%.6f", score); if (vn < 0 || (size_t)vn >= sizeof(val)) { - return; + return NULL; } const char *k = imp_find_key(old, "importance"); if (k) { const char *colon = strchr(k, ':'); if (!colon) { - return; /* imp_find_key guarantees one, but never trust a raw scan */ + return NULL; /* imp_find_key guarantees one, but never trust a raw scan */ } const char *v = colon + 1; while (isspace((unsigned char)*v)) { @@ -250,37 +255,77 @@ void cbm_pipeline_importance_append_prop(cbm_gbuf_node_t *node, double score) { } const char *vend = imp_value_end(v); if (!vend) { - return; + return NULL; } size_t head = (size_t)(v - old); size_t tail = strlen(vend); char *neu = malloc(head + (size_t)vn + tail + 1); if (!neu) { - return; + return NULL; } memcpy(neu, old, head); memcpy(neu + head, val, (size_t)vn); memcpy(neu + head + (size_t)vn, vend, tail + 1); - free(node->properties_json); - node->properties_json = neu; - return; + return neu; } char frag[CBM_SZ_64]; int fn = snprintf(frag, sizeof(frag), "%s\"importance\":%s}", (olen == 2) ? "" : ",", val); if (fn < 0 || (size_t)fn >= sizeof(frag)) { - return; + return NULL; } char *neu = malloc(olen - 1 + (size_t)fn + 1); if (!neu) { - return; + return NULL; } memcpy(neu, old, olen - 1); /* copy without the trailing '}' */ memcpy(neu + olen - 1, frag, (size_t)fn + 1); + return neu; +} + +/* gbuf-node convenience wrapper around cbm_pipeline_importance_set_prop. */ +void cbm_pipeline_importance_append_prop(cbm_gbuf_node_t *node, double score) { + if (!node) { + return; + } + char *neu = cbm_pipeline_importance_set_prop(node->properties_json, score); + if (!neu) { + return; + } free(node->properties_json); node->properties_json = neu; } +/* ── The scoring rule ───────────────────────────────────────────────── + * + * SINGLE DEFINITION, deliberately. There are two routes that score nodes — + * the in-memory gbuf pass (full index, legacy-partial incremental) and the + * SQL-level recompute over the staging store (closure-delta incremental) — + * and they gather their inputs very differently. Only the GATHERING differs; + * the rule itself lives here alone, so the two routes cannot drift apart in + * what a score means. The equivalence test in tests/test_importance.c guards + * the remaining freedom: that both gatherings produce the same inputs for the + * same graph. */ +double cbm_pipeline_importance_score(const cbm_importance_inputs_t *in) { + if (!in) { + return 0.0; + } + double score = sqrt((double)(in->num_refs > 0 ? in->num_refs : 0)); + if (name_is_private(in->name)) { + score *= CBM_IMPORTANCE_PRIV_MUL; + } + if (in->name_distinct_files >= CBM_IMPORTANCE_GENERIC_MIN_FILES) { + score *= CBM_IMPORTANCE_GENERIC_MUL; + } + if (name_is_distinctive(in->name)) { + score *= CBM_IMPORTANCE_DISTINCT_MUL; + } + if (cbm_is_test_path(in->file_path) || in->tests_target) { + score *= CBM_IMPORTANCE_TEST_MUL; + } + return score; +} + /* ── Distinct-file counting (the linear core) ─────────────────────────── */ typedef struct { @@ -367,24 +412,16 @@ void cbm_pipeline_pass_importance(cbm_pipeline_ctx_t *ctx) { for (int i = 0; i < count; i++) { cbm_gbuf_node_t *n = (cbm_gbuf_node_t *)nodes[i]; - int num_refs = - incoming_edge_count(gb, n->id, "CALLS") + incoming_edge_count(gb, n->id, "USAGE"); - double score = sqrt((double)num_refs); - - if (name_is_private(n->name)) { - score *= CBM_IMPORTANCE_PRIV_MUL; - } - if (imp_distinct_file_count(gb, n->name, &ix) >= CBM_IMPORTANCE_GENERIC_MIN_FILES) { - score *= CBM_IMPORTANCE_GENERIC_MUL; - } - if (name_is_distinctive(n->name)) { - score *= CBM_IMPORTANCE_DISTINCT_MUL; - } - if (cbm_is_test_path(n->file_path) || incoming_edge_count(gb, n->id, "TESTS") > 0) { - score *= CBM_IMPORTANCE_TEST_MUL; - } - - cbm_pipeline_importance_append_prop(n, score); + /* Gather only — the rule itself lives in one place. */ + cbm_importance_inputs_t in = { + .name = n->name, + .file_path = n->file_path, + .num_refs = incoming_edge_count(gb, n->id, "CALLS") + + incoming_edge_count(gb, n->id, "USAGE"), + .name_distinct_files = imp_distinct_file_count(gb, n->name, &ix), + .tests_target = incoming_edge_count(gb, n->id, "TESTS") > 0, + }; + cbm_pipeline_importance_append_prop(n, cbm_pipeline_importance_score(&in)); updated++; } } @@ -394,3 +431,126 @@ void cbm_pipeline_pass_importance(cbm_pipeline_ctx_t *ctx) { atomic_fetch_add_explicit(&g_importance_nodes, updated, memory_order_relaxed); cbm_log_info("pass.importance", "symbols", itoa_imp(updated)); } + +/* ── Store-level recompute (closure-delta incremental) ──────────────── + * + * WHY THIS EXISTS. The closure-delta route never materialises the project + * graph in RAM: cbm_delta_preseed fills the delta gbuf with *proxy* nodes + * (id, label, name, qn, file_path — no edges, empty properties), and only the + * re-extracted files' symbols carry real edges. Scoring there reads a + * near-zero in-degree for a symbol referenced hundreds of times project-wide, + * and cbm_delta_patch then persists exactly those fresh rows. Loading the full + * graph to fix that would give back the point of the delta route, so the + * recompute runs where the complete graph already exists: in SQL, over the + * staging store, AFTER cbm_delta_patch has inserted the new nodes and + * re-linked the snapshotted inbound edges. + * + * WHY IT CANNOT DRIFT FROM THE IN-MEMORY PASS. The scoring rule and the JSON + * write-back are not reimplemented here. They are registered as SQLite + * user-defined functions that call cbm_pipeline_importance_score() and + * cbm_pipeline_importance_set_prop() — the same C code the gbuf pass runs. + * SQL supplies only the three aggregates (in-degree, TESTS presence, distinct + * files per name), each one indexed GROUP BY. What remains free is the + * GATHERING, and the full-vs-incremental equivalence test guards that. + * + * The whole project is rescored, not just the changed files: distinct-files- + * per-name is a global input, so editing one file can legitimately move a name + * across the generic-name threshold for every other definition of that name. A + * scoped recompute would leave those stale — and would fail the equivalence + * test, correctly. */ + +/* UDF: cbm_imp_score(name, file_path, num_refs, name_files, tests_target). */ +static void sql_importance_score(sqlite3_context *c, int argc, sqlite3_value **argv) { + if (argc != 5) { + sqlite3_result_error(c, "cbm_imp_score arity", -1); + return; + } + cbm_importance_inputs_t in = { + .name = (const char *)sqlite3_value_text(argv[0]), + .file_path = (const char *)sqlite3_value_text(argv[1]), + .num_refs = sqlite3_value_int(argv[2]), + .name_distinct_files = sqlite3_value_int(argv[3]), + .tests_target = sqlite3_value_int(argv[4]) != 0, + }; + sqlite3_result_double(c, cbm_pipeline_importance_score(&in)); +} + +/* UDF: cbm_imp_set(properties, score) → properties with "importance" set. + * A non-object blob yields NULL; the UPDATE's COALESCE then leaves that row + * untouched — the same tolerance the in-memory writer has. */ +static void sql_importance_set(sqlite3_context *c, int argc, sqlite3_value **argv) { + if (argc != 2) { + sqlite3_result_error(c, "cbm_imp_set arity", -1); + return; + } + const char *props = (const char *)sqlite3_value_text(argv[0]); + char *neu = cbm_pipeline_importance_set_prop(props, sqlite3_value_double(argv[1])); + if (!neu) { + sqlite3_result_null(c); + return; + } + sqlite3_result_text(c, neu, -1, SQLITE_TRANSIENT); + free(neu); +} + +/* The aggregate CTE is MATERIALIZED on purpose: it reads `nodes` while the + * UPDATE writes `nodes`, and materialising settles that ordering explicitly + * rather than relying on the planner to choose it. */ +static const char *const kImportanceRecomputeSQL = + "WITH agg AS MATERIALIZED (" + " SELECT n.id AS id," + " cbm_imp_score(n.name, n.file_path, COALESCE(d.refs, 0)," + " COALESCE(f.files, 0), COALESCE(d.tests, 0)) AS score" + " FROM nodes n" + " LEFT JOIN (SELECT target_id AS id," + " SUM(CASE WHEN type IN ('CALLS','USAGE') THEN 1 ELSE 0 END) AS refs," + " SUM(CASE WHEN type = 'TESTS' THEN 1 ELSE 0 END) AS tests" + " FROM edges WHERE project = ?1 GROUP BY target_id) d ON d.id = n.id" + " LEFT JOIN (SELECT name, COUNT(DISTINCT file_path) AS files" + " FROM nodes WHERE project = ?1 GROUP BY name) f ON f.name = n.name" + " WHERE n.project = ?1 AND n.label IN ('Function','Method','Class')" + ")" + "UPDATE nodes SET properties = COALESCE(cbm_imp_set(properties, agg.score), properties)" + " FROM agg WHERE nodes.id = agg.id"; + +int cbm_pipeline_importance_recompute_store(cbm_store_t *store, const char *project) { + if (!store || !project) { + return -1; + } + sqlite3 *db = cbm_store_get_db(store); + if (!db) { + return -1; + } + if (sqlite3_create_function(db, "cbm_imp_score", 5, SQLITE_UTF8 | SQLITE_DETERMINISTIC, NULL, + sql_importance_score, NULL, NULL) != SQLITE_OK || + sqlite3_create_function(db, "cbm_imp_set", 2, SQLITE_UTF8 | SQLITE_DETERMINISTIC, NULL, + sql_importance_set, NULL, NULL) != SQLITE_OK) { + cbm_log_error("importance.store_udf_failed", "err", sqlite3_errmsg(db)); + return -1; + } + if (cbm_store_begin(store) != CBM_STORE_OK) { + return -1; + } + sqlite3_stmt *st = NULL; + if (sqlite3_prepare_v2(db, kImportanceRecomputeSQL, -1, &st, NULL) != SQLITE_OK) { + cbm_log_error("importance.store_prepare_failed", "err", sqlite3_errmsg(db)); + cbm_store_rollback(store); + return -1; + } + sqlite3_bind_text(st, 1, project, -1, SQLITE_TRANSIENT); + int step_rc = sqlite3_step(st); + sqlite3_finalize(st); + if (step_rc != SQLITE_DONE) { + cbm_log_error("importance.store_step_failed", "err", sqlite3_errmsg(db)); + cbm_store_rollback(store); + return -1; + } + int changed = sqlite3_changes(db); + if (cbm_store_commit(store) != CBM_STORE_OK) { + return -1; + } + uint64_t rows = (uint64_t)(changed > 0 ? changed : 0); + atomic_fetch_add_explicit(&g_importance_store_rows, rows, memory_order_relaxed); + cbm_log_info("pass.importance_store", "symbols", itoa_imp(rows)); + return 0; +} diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 5de6bfd6a..4ff89aa42 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1410,9 +1410,15 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed } } -/* Run post-extraction passes (tests, decorator tags, configlink). */ +/* Run post-extraction passes (tests, decorator tags, configlink). + * + * score_importance: false on the closure-delta route, whose ctx->gbuf is a + * PROXY buffer — cbm_delta_preseed fills it with id/label/name/qn/file_path + * and no project-wide edges, so an in-memory score there would read a + * near-zero in-degree and cbm_delta_patch would persist it. That route scores + * in SQL instead, after the patch; see cbm_pipeline_importance_recompute_store. */ static int run_postpasses(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_files, int ci, - const char *project) { + const char *project, bool score_importance) { struct timespec t; cbm_clock_gettime(CLOCK_MONOTONIC, &t); @@ -1457,16 +1463,22 @@ static int run_postpasses(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_file } } - /* Importance last, in EVERY mode — same ordering rule as the full path: - * it reads CALLS/USAGE (re-extraction, above) and TESTS (pass_tests, first - * in this function). The graph here is the rehydrated whole-project buffer, - * so nodes outside the changed set already carry an "importance" key from - * the previous run; cbm_pipeline_importance_append_prop overwrites it in - * place rather than appending a duplicate. */ - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - cbm_pipeline_pass_importance(ctx); - cbm_log_info("pass.timing", "pass", "incr_importance", "elapsed_ms", - itoa_buf((int)elapsed_ms(t))); + /* Importance last, in EVERY mode — same ordering rule as the full path: it + * reads CALLS/USAGE (re-extraction, above) and TESTS (pass_tests, first in + * this function). + * + * ONLY on the legacy-partial route, where ctx->gbuf really is the whole + * project rehydrated by cbm_gbuf_load_from_db — those nodes already carry + * an "importance" key from the previous run, and the write-back overwrites + * it in place rather than appending a duplicate. The closure-delta route + * passes score_importance = false and rescores in SQL after its patch, + * because its buffer holds proxies without project-wide edges. */ + if (score_importance) { + cbm_clock_gettime(CLOCK_MONOTONIC, &t); + cbm_pipeline_pass_importance(ctx); + cbm_log_info("pass.timing", "pass", "incr_importance", "elapsed_ms", + itoa_buf((int)elapsed_ms(t))); + } if (cbm_pipeline_check_cancel(ctx)) { return CBM_NOT_FOUND; } @@ -2158,7 +2170,7 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char phase_rc = cbm_pipeline_check_cancel(&ctx); } if (phase_rc == 0) { - phase_rc = run_postpasses(&ctx, changed_files, ci, project); + phase_rc = run_postpasses(&ctx, changed_files, ci, project, false); } if (ctx.return_type_table) { for (int i = 0; i < ctx.return_type_table->count; i++) { @@ -2183,6 +2195,23 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char } cbm_log_info("delta.patch_done", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); + /* Importance, in SQL, over the now-complete staging graph. This is the + * FIRST point on this route where the whole project is queryable: the + * patch has just inserted the repaired files' nodes AND re-linked the + * snapshotted inbound edges, so in-degree is finally correct. Scoring + * earlier — in run_postpasses, off the proxy buffer — would persist a + * near-zero score for any heavily-referenced symbol in a changed file. + * A failure here is a delta-route failure: `result` still holds + * FORCE_FULL_REINDEX, so the run degrades to a correct full rebuild + * rather than publishing wrong scores. */ + cbm_clock_gettime(CLOCK_MONOTONIC, &t); + if (cbm_pipeline_importance_recompute_store(staging, project) != 0) { + cbm_log_error("delta.err", "phase", "importance_recompute"); + goto out; + } + cbm_log_info("pass.timing", "pass", "delta_importance_store", "elapsed_ms", + itoa_buf((int)elapsed_ms(t))); + /* Coverage: previous failure rows for files not re-extracted + this * run's fresh entries — same merge the legacy tail performs. */ cbm_file_error_t *run_errs = NULL; @@ -2713,7 +2742,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil phase_rc = cbm_pipeline_check_cancel(&ctx); } if (phase_rc == 0) { - phase_rc = run_postpasses(&ctx, changed_files, ci, project); + phase_rc = run_postpasses(&ctx, changed_files, ci, project, true); } /* Free ObjectScript tables built by pass_calls during run_extract_resolve. */ diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index a25573733..e686ac3b7 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -641,12 +641,39 @@ void cbm_pipeline_pass_complexity(cbm_pipeline_ctx_t *ctx); * post-pass sequence. */ void cbm_pipeline_pass_importance(cbm_pipeline_ctx_t *ctx); -/* Write the numeric "importance" key into a node's properties JSON object. - * IDEMPOTENT: an existing key is overwritten in place, never appended twice — - * the incremental path re-scores nodes rehydrated from the store that already - * carry the key. Non-static so the idempotence contract is directly testable. */ +/* Gathered inputs for one symbol. Each scoring route fills this its own way + * (gbuf lookups, or SQL aggregates) and then calls the ONE rule below. */ +typedef struct { + const char *name; + const char *file_path; + int num_refs; /* incoming CALLS + USAGE */ + int name_distinct_files; /* distinct files the NAME is defined in */ + bool tests_target; /* has an incoming TESTS edge */ +} cbm_importance_inputs_t; + +/* THE scoring rule — single definition, shared by every route, so the two + * gathering strategies cannot drift apart in what a score means. */ +double cbm_pipeline_importance_score(const cbm_importance_inputs_t *in); + +/* Produce a copy of `json` with the numeric "importance" key set to `score`, + * or NULL when `json` is not a JSON object or allocation fails. Caller owns + * the result. IDEMPOTENT: an existing key is overwritten in place, never + * appended twice — every re-scoring route sees nodes that already carry it. + * Single definition, shared by the in-memory and SQL writers alike. */ +char *cbm_pipeline_importance_set_prop(const char *json, double score); + +/* gbuf-node convenience wrapper around cbm_pipeline_importance_set_prop. */ void cbm_pipeline_importance_append_prop(cbm_gbuf_node_t *node, double score); +/* Recompute importance for an ENTIRE project directly in a store, in SQL. + * Used by the closure-delta incremental route, whose in-RAM graph is a proxy + * buffer with no project-wide edges — scoring there would persist a near-zero + * in-degree for heavily-referenced symbols in the changed files. Must be + * called on the STAGING store AFTER cbm_delta_patch has merged nodes and + * re-linked inbound edges. Returns 0 on success; the caller treats failure as + * a delta-route failure and falls back to a full rebuild. */ +int cbm_pipeline_importance_recompute_store(cbm_store_t *store, const char *project); + /* Work counters for the importance pass (pass_importance.c). Deltas are read * by the complexity suite's linearity gate; never reset by the pass itself, so * nested/repeated runs compose. g_importance_name_visits counts same-name-group @@ -654,6 +681,9 @@ void cbm_pipeline_importance_append_prop(cbm_gbuf_node_t *node, double score); * superlinear if the per-distinct-name memoization is ever lost. */ extern _Atomic uint64_t g_importance_nodes; extern _Atomic uint64_t g_importance_name_visits; +/* Rows rescored by the SQL-level store recompute. Lets a test prove the + * closure-delta route actually took that path instead of passing vacuously. */ +extern _Atomic uint64_t g_importance_store_rows; /* ── Env URL scanner (pass_envscan.c) ────────────────────────────── */ diff --git a/tests/test_importance.c b/tests/test_importance.c index 9452a49e9..afaf41a3c 100644 --- a/tests/test_importance.c +++ b/tests/test_importance.c @@ -101,6 +101,97 @@ static int imp_check_label(cbm_store_t *s, const char *project, const char *labe return bad == 0 ? count : -1; } +/* ── Store-level score readout (for the closure-delta tests) ──────────── */ + +typedef struct { + char **qns; + double *scores; + int count; +} ImpScoreSet; + +static void imp_scores_free(ImpScoreSet *set) { + for (int i = 0; i < set->count; i++) { + free(set->qns[i]); + } + free(set->qns); + free(set->scores); + set->qns = NULL; + set->scores = NULL; + set->count = 0; +} + +/* Collect (qualified_name -> importance) for every scored label in a store. */ +static int imp_collect_scores(const char *db_path, const char *project, ImpScoreSet *out) { + static const char *const kLabels[] = {"Function", "Method", "Class"}; + memset(out, 0, sizeof(*out)); + cbm_store_t *s = cbm_store_open_path(db_path); + if (!s) { + return -1; + } + int rc = 0; + for (int li = 0; li < 3; li++) { + cbm_node_t *nodes = NULL; + int n = 0; + if (cbm_store_find_nodes_by_label(s, project, kLabels[li], &nodes, &n) != CBM_STORE_OK) { + rc = -1; + break; + } + char **qns = realloc(out->qns, (size_t)(out->count + n) * sizeof(*qns)); + double *sc = realloc(out->scores, (size_t)(out->count + n) * sizeof(*sc)); + if (n > 0 && (!qns || !sc)) { + cbm_store_free_nodes(nodes, n); + rc = -1; + break; + } + if (qns) { + out->qns = qns; + } + if (sc) { + out->scores = sc; + } + for (int i = 0; i < n; i++) { + out->qns[out->count] = strdup(nodes[i].qualified_name ? nodes[i].qualified_name : ""); + out->scores[out->count] = imp_value(nodes[i].properties_json); + out->count++; + } + cbm_store_free_nodes(nodes, n); + } + cbm_store_close(s); + return rc; +} + +static double imp_score_of(const ImpScoreSet *set, const char *qn_suffix) { + for (int i = 0; i < set->count; i++) { + size_t l = strlen(set->qns[i]); + size_t k = strlen(qn_suffix); + if (l >= k && strcmp(set->qns[i] + (l - k), qn_suffix) == 0) { + return set->scores[i]; + } + } + return -999.0; +} + +/* Write a fixture whose `helper` in hot.py is called from `callers` other + * files — the inbound edges that only exist project-wide, never in a delta + * buffer. `tag` lets the caller perturb hot.py to force a re-parse. */ +static int imp_write_hot_fixture(const char *root, int callers, const char *tag) { + char body[256]; + snprintf(body, sizeof(body), "%sdef helper():\n return 1\n", tag); + if (th_write_file(TH_PATH(root, "hot.py"), body) != 0) { + return -1; + } + for (int i = 0; i < callers; i++) { + char rel[64]; + char src[192]; + snprintf(rel, sizeof(rel), "c%d.py", i); + snprintf(src, sizeof(src), "import hot\n\ndef use%d():\n return hot.helper()\n", i); + if (th_write_file(TH_PATH(root, rel), src) != 0) { + return -1; + } + } + return 0; +} + /* ── Registration: the pass must actually execute ─────────────────────── */ /* A miscounted PREDUMP_PASS_COUNT silently drops the last-registered pass. @@ -577,6 +668,181 @@ TEST(importance_second_index_run_keeps_exactly_one_key) { PASS(); } +/* ── Closure-delta route: the graph is only complete in the store ─────── */ + +/* REGRESSION. `helper` lives in hot.py and is called from 25 other files. Edit + * hot.py and re-index: the closure-delta route purges hot.py's rows, rebuilds + * them from a PROXY buffer that holds no project-wide edges, and persists what + * it computed. Scoring off that buffer sees in-degree 0 and stores 0.0 for a + * symbol with 25 real callers. The fix rescores in SQL after cbm_delta_patch + * has re-linked the snapshotted inbound edges. */ +TEST(importance_closure_delta_keeps_referenced_symbol_sane) { + enum { CALLERS = 25 }; + char *tmp = th_mktempdir("cbm_imp_delta"); + if (!tmp) { + FAIL("tmpdir"); + } + char root[512]; + snprintf(root, sizeof(root), "%s", tmp); + char *tmp2 = th_mktempdir("cbm_imp_deltadb"); + if (!tmp2) { + th_rmtree(root); + FAIL("tmpdir2"); + } + char dbdir[512]; + snprintf(dbdir, sizeof(dbdir), "%s", tmp2); + + ASSERT_EQ(imp_write_hot_fixture(root, CALLERS, ""), 0); + + char db_path[600]; + snprintf(db_path, sizeof(db_path), "%s/graph.db", dbdir); + cbm_pipeline_t *p1 = cbm_pipeline_new(root, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p1); + ASSERT_EQ(cbm_pipeline_run(p1), 0); + char project[256]; + snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(p1)); + cbm_pipeline_free(p1); + + ImpScoreSet before = {0}; + ASSERT_EQ(imp_collect_scores(db_path, project, &before), 0); + double full_score = imp_score_of(&before, "helper"); + printf(" full-index helper score %.6f (expect sqrt(%d) = %.6f)\n", full_score, CALLERS, + sqrt((double)CALLERS)); + /* Non-vacuous: the callers really produced edges. If this is 0 the fixture + * stopped exercising the bug and the test below would prove nothing. */ + ASSERT_TRUE(full_score > 1.0); + imp_scores_free(&before); + + /* Touch hot.py so the delta route has exactly one file to repair. */ + ASSERT_EQ(imp_write_hot_fixture(root, CALLERS, "# touched\n"), 0); + + uint64_t sql0 = atomic_load_explicit(&g_importance_store_rows, memory_order_relaxed); + cbm_pipeline_t *p2 = cbm_pipeline_new(root, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p2); + ASSERT_EQ(cbm_pipeline_run(p2), 0); + cbm_incremental_route_t route = cbm_pipeline_incremental_test_last_route(); + cbm_pipeline_free(p2); + uint64_t sql_rows = atomic_load_explicit(&g_importance_store_rows, memory_order_relaxed) - sql0; + + /* Route proof — without it a fall-back to a full reindex would repair the + * score for the wrong reason and the test would pass vacuously. */ + printf(" route=%d (closure_repair=%d) sql_rescored_rows=%llu\n", (int)route, + (int)CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR, (unsigned long long)sql_rows); + ASSERT_EQ((int)route, (int)CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR); + + ImpScoreSet after = {0}; + ASSERT_EQ(imp_collect_scores(db_path, project, &after), 0); + double delta_score = imp_score_of(&after, "helper"); + printf(" after closure-delta helper score %.6f\n", delta_score); + imp_scores_free(&after); + + th_rmtree(root); + th_rmtree(dbdir); + /* THE SYMPTOM, asserted before any mechanism guard: the graph is unchanged, + * so the score must be unchanged. Scoring off the proxy buffer yields 0.0. */ + ASSERT_FLOAT_EQ(delta_score, full_score, 1e-6); + /* Mechanism guard second, so a regression fails on the wrong score rather + * than on bookkeeping. */ + ASSERT_GT((long long)sql_rows, 0); + PASS(); +} + +/* EQUIVALENCE. A full index and an incremental index that converge on the same + * tree must produce the same scores — for every symbol, not just the hot one. + * Same directory and therefore the same project name and qualified names, so + * the two score maps are directly comparable. */ +TEST(importance_full_and_incremental_converge_on_same_scores) { + enum { CALLERS = 12 }; + char *tmp = th_mktempdir("cbm_imp_equiv"); + if (!tmp) { + FAIL("tmpdir"); + } + char root[512]; + snprintf(root, sizeof(root), "%s", tmp); + char *tmp2 = th_mktempdir("cbm_imp_equivdb"); + if (!tmp2) { + th_rmtree(root); + FAIL("tmpdir2"); + } + char dbdir[512]; + snprintf(dbdir, sizeof(dbdir), "%s", tmp2); + + /* v1: every caller but c0 calls helper. c0 imports hot but does not. */ + ASSERT_EQ(imp_write_hot_fixture(root, CALLERS, ""), 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "c0.py"), "import hot\n\ndef use0():\n return 1\n"), + 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "extra.py"), + "import hot\n\ndef load_user_profile():\n return hot.helper()\n"), + 0); + + char db_incr[600]; + snprintf(db_incr, sizeof(db_incr), "%s/incr.db", dbdir); + cbm_pipeline_t *p1 = cbm_pipeline_new(root, db_incr, CBM_MODE_FULL); + ASSERT_NOT_NULL(p1); + ASSERT_EQ(cbm_pipeline_run(p1), 0); + char project[256]; + snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(p1)); + cbm_pipeline_free(p1); + + /* v2: c0's BODY starts calling helper. Body-only, so c0's surface is + * unchanged and the closure planner accepts the delta — but the in-degree + * of `helper`, which lives in the UNCHANGED hot.py, goes up by one. That + * is the case a changed-files-only rescore gets wrong: the symbol whose + * score moved is not in the changed set at all. */ + ASSERT_EQ(th_write_file(TH_PATH(root, "c0.py"), + "import hot\n\ndef use0():\n return hot.helper()\n"), + 0); + + uint64_t sql0 = atomic_load_explicit(&g_importance_store_rows, memory_order_relaxed); + cbm_pipeline_t *p2 = cbm_pipeline_new(root, db_incr, CBM_MODE_FULL); + ASSERT_NOT_NULL(p2); + ASSERT_EQ(cbm_pipeline_run(p2), 0); + cbm_incremental_route_t route = cbm_pipeline_incremental_test_last_route(); + cbm_pipeline_free(p2); + uint64_t sql_rows = atomic_load_explicit(&g_importance_store_rows, memory_order_relaxed) - sql0; + printf(" route=%d (closure_repair=%d) sql_rescored_rows=%llu\n", (int)route, + (int)CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR, (unsigned long long)sql_rows); + ASSERT_EQ((int)route, (int)CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR); + + /* Same tree, indexed cold into a separate database. */ + char db_full[600]; + snprintf(db_full, sizeof(db_full), "%s/full.db", dbdir); + cbm_pipeline_t *p3 = cbm_pipeline_new(root, db_full, CBM_MODE_FULL); + ASSERT_NOT_NULL(p3); + ASSERT_EQ(cbm_pipeline_run(p3), 0); + cbm_pipeline_free(p3); + + ImpScoreSet incr = {0}; + ImpScoreSet full = {0}; + ASSERT_EQ(imp_collect_scores(db_incr, project, &incr), 0); + ASSERT_EQ(imp_collect_scores(db_full, project, &full), 0); + printf(" scored symbols: incremental=%d full=%d\n", incr.count, full.count); + + int mismatches = 0; + for (int i = 0; i < full.count; i++) { + double a = full.scores[i]; + double b = imp_score_of(&incr, full.qns[i]); + if (!(fabs(a - b) < 1e-6)) { + mismatches++; + if (mismatches <= 5) { + printf(" MISMATCH %s: full=%.6f incremental=%.6f\n", full.qns[i], a, b); + } + } + } + int count_full = full.count; + int count_incr = incr.count; + imp_scores_free(&incr); + imp_scores_free(&full); + th_rmtree(root); + th_rmtree(dbdir); + + ASSERT_GT(count_full, 5); /* non-vacuous: there really are symbols to compare */ + ASSERT_EQ(count_incr, count_full); + ASSERT_EQ(mismatches, 0); /* the symptom */ + ASSERT_GT((long long)sql_rows, 0); /* mechanism guard, asserted second */ + PASS(); +} + /* ── Cost shape ───────────────────────────────────────────────────────── */ /* The distinct-file count must be computed once per distinct NAME. Computing @@ -636,6 +902,10 @@ SUITE(importance) { RUN_TEST(importance_rehydrated_node_keeps_exactly_one_key); RUN_TEST(importance_second_index_run_keeps_exactly_one_key); + /* Closure-delta route — the store-level recompute. */ + RUN_TEST(importance_closure_delta_keeps_referenced_symbol_sane); + RUN_TEST(importance_full_and_incremental_converge_on_same_scores); + /* Cost shape. */ RUN_TEST(importance_distinct_file_count_is_linear_in_group_size); } From cf6e620431954034523a2705c35f9d4f553c4688 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 28 Aug 2026 18:10:00 +0200 Subject: [PATCH 12/31] feat(mcp): add bounded graph comparison tool Signed-off-by: Martin Vogel --- src/mcp/mcp.c | 431 ++++++++++++++++++++++++++++++++++++++- src/store/store.c | 384 ++++++++++++++++++++++++++++++++++ src/store/store.h | 53 +++++ tests/test_mcp.c | 510 +++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 1371 insertions(+), 7 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 0ce6ae7eb..616bbf6a1 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -1,5 +1,5 @@ /* - * mcp.c — MCP server: JSON-RPC 2.0 over stdio with 14 graph tools. + * mcp.c — MCP server: JSON-RPC 2.0 over stdio with graph tools. * * Uses yyjson for fast JSON parsing/building. * Single-threaded event loop: read line → parse → dispatch → respond. @@ -36,6 +36,11 @@ enum { MCP_TOOLS_PAGE_SIZE = 8, MCP_HELP_TOOLS_WRAP_COL = 74, /* --help tool list stays readable on 80-col terminals */ MCP_MAX_CROSS_REPO_TARGETS = 4096, + MCP_COMPARE_DEFAULT_LIMIT = 200, + MCP_COMPARE_MAX_LIMIT = 1000, + MCP_COMPARE_DEFAULT_SCAN_LIMIT = 2000000, + MCP_COMPARE_MAX_SCAN_LIMIT = 10000000, + MCP_COMPARE_SET_BYTE_BUDGET = 512 * 1024, }; #define MCP_MS_TO_US 1000LL #define MCP_S_TO_US 1000000LL @@ -556,6 +561,19 @@ static const tool_def_t TOOLS[] = { "{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"}},\"required\":[" "\"project\"]}"}, + {"compare_graphs", "Compare graphs", + "Compare two indexed project snapshots. Returns deterministic target-only additions and " + "base-only removals for stable node and edge identities using a bounded streaming merge. " + "Each result set is independently capped by limit and a fixed 512 KiB encoded-byte budget; " + "exact totals and truncation reasons are always reported.", + "{\"type\":\"object\",\"properties\":{" + "\"base_project\":{\"type\":\"string\",\"minLength\":1}," + "\"target_project\":{\"type\":\"string\",\"minLength\":1}," + "\"limit\":{\"type\":\"integer\",\"minimum\":1,\"maximum\":1000,\"default\":200}," + "\"scan_limit\":{\"type\":\"integer\",\"minimum\":1,\"maximum\":10000000," + "\"default\":2000000}},\"required\":[\"base_project\",\"target_project\"]," + "\"additionalProperties\":false}"}, + {"get_architecture", "Get architecture", "Get high-level architecture overview. DEFAULT (no aspects) is a compact summary — " "overview counts, languages, packages, entry_points; request more via aspects:[...] " @@ -719,6 +737,7 @@ static const tool_annotation_def_t TOOL_ANNOTATIONS[] = { {"trace_path", false, true, true, false}, {"get_code_snippet", false, true, true, false}, {"get_graph_schema", false, true, true, false}, + {"compare_graphs", true, false, true, false}, {"get_architecture", false, true, true, false}, {"search_code", false, true, true, false}, {"list_projects", true, false, true, false}, @@ -780,9 +799,9 @@ static void mcp_add_tool_def(yyjson_mut_doc *doc, yyjson_mut_val *tools, int i) static bool mcp_tool_allowed(cbm_mcp_tool_profile_t profile, const char *name) { static const char *const analysis_tools[] = { - "search_graph", "query_graph", "trace_path", "get_code_snippet", - "get_graph_schema", "get_architecture", "search_code", "list_projects", - "index_status", "check_index_coverage", "detect_changes", + "search_graph", "query_graph", "trace_path", "get_code_snippet", + "get_graph_schema", "compare_graphs", "get_architecture", "search_code", + "list_projects", "index_status", "check_index_coverage", "detect_changes", }; static const char *const scout_tools[] = { "search_graph", "trace_path", "get_code_snippet", "get_architecture", @@ -2751,6 +2770,407 @@ static char *verify_project_indexed(cbm_store_t *store, const char *project) { return NULL; } +/* compare_graphs deliberately bypasses resolve_store(): it needs two + * independently-owned request-scoped read handles, while resolve_store caches + * one handle on the server. Direct-name lookup stays the fast path; the + * existing internal-name fallback preserves legacy renamed databases. */ +static cbm_store_t *compare_open_project_store(const char *project) { + char path[CBM_SZ_1K]; + project_db_path(project, path, sizeof(path)); + cbm_store_t *store = path[0] ? cbm_store_open_path_query(path) : NULL; + if (store) { + cbm_project_t row = {0}; + if (cbm_store_get_project(store, project, &row) == CBM_STORE_OK) { + cbm_project_free_fields(&row); + return store; + } + cbm_store_close(store); + } + return resolve_store_fallback_scan(project); +} + +typedef struct { + yyjson_mut_val *items; + size_t returned; + size_t encoded_bytes; + bool budget_exhausted; +} compare_result_set_t; + +typedef struct { + cbm_mcp_server_t *server; + yyjson_mut_doc *doc; + size_t limit; + compare_result_set_t nodes_added; + compare_result_set_t nodes_removed; + compare_result_set_t edges_added; + compare_result_set_t edges_removed; +} compare_response_t; + +static char *compare_graphs_error(const char *code, const char *message) { + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = doc ? yyjson_mut_obj(doc) : NULL; + if (!doc || !root) { + yyjson_mut_doc_free(doc); + return cbm_mcp_text_result("compare_graphs failed: out of memory", true); + } + yyjson_mut_doc_set_root(doc, root); + if (!yyjson_mut_obj_add_strcpy(doc, root, "error", message) || + !yyjson_mut_obj_add_strcpy(doc, root, "code", code)) { + yyjson_mut_doc_free(doc); + return cbm_mcp_text_result("compare_graphs failed: out of memory", true); + } + char *json = yy_doc_to_str(doc); + yyjson_mut_doc_free(doc); + if (!json) { + return cbm_mcp_text_result("compare_graphs failed: out of memory", true); + } + char *result = cbm_mcp_text_result(json, true); + free(json); + return result; +} + +static bool compare_arg_name_allowed(const char *name) { + return strcmp(name, "base_project") == 0 || strcmp(name, "target_project") == 0 || + strcmp(name, "limit") == 0 || strcmp(name, "scan_limit") == 0; +} + +static bool compare_parse_bounded_integer(yyjson_val *root, const char *key, int64_t default_value, + int64_t maximum, uint64_t *out, + const char **error_message) { + yyjson_val *value = yyjson_obj_get(root, key); + int64_t parsed = default_value; + if (value) { + if (!yyjson_is_int(value)) { + *error_message = "limit values must be integers"; + return false; + } + parsed = yyjson_get_int(value); + } + if (parsed < 1 || parsed > maximum) { + *error_message = strcmp(key, "limit") == 0 ? "limit must be between 1 and 1000" + : "scan_limit must be between 1 and 10000000"; + return false; + } + *out = (uint64_t)parsed; + return true; +} + +static bool compare_parse_arguments(const char *args, char **base_project, char **target_project, + uint64_t *limit, uint64_t *scan_limit, + const char **error_message) { + *base_project = NULL; + *target_project = NULL; + yyjson_doc *doc = yyjson_read(args ? args : "{}", args ? strlen(args) : SLEN("{}"), 0); + if (!doc) { + *error_message = "arguments must be valid JSON"; + return false; + } + yyjson_val *root = yyjson_doc_get_root(doc); + if (!yyjson_is_obj(root)) { + *error_message = "arguments must be an object"; + yyjson_doc_free(doc); + return false; + } + + yyjson_obj_iter iterator = yyjson_obj_iter_with(root); + yyjson_val *key = NULL; + while ((key = yyjson_obj_iter_next(&iterator)) != NULL) { + const char *name = yyjson_get_str(key); + if (!name || !compare_arg_name_allowed(name)) { + *error_message = "unknown argument"; + yyjson_doc_free(doc); + return false; + } + } + + yyjson_val *base = yyjson_obj_get(root, "base_project"); + yyjson_val *target = yyjson_obj_get(root, "target_project"); + if (!base || !yyjson_is_str(base) || yyjson_get_len(base) == 0 || !target || + !yyjson_is_str(target) || yyjson_get_len(target) == 0) { + *error_message = "base_project and target_project are required non-empty strings"; + yyjson_doc_free(doc); + return false; + } + if (strcmp(yyjson_get_str(base), yyjson_get_str(target)) == 0) { + *error_message = "base_project and target_project must be distinct"; + yyjson_doc_free(doc); + return false; + } + if (!compare_parse_bounded_integer(root, "limit", MCP_COMPARE_DEFAULT_LIMIT, + MCP_COMPARE_MAX_LIMIT, limit, error_message) || + !compare_parse_bounded_integer(root, "scan_limit", MCP_COMPARE_DEFAULT_SCAN_LIMIT, + MCP_COMPARE_MAX_SCAN_LIMIT, scan_limit, error_message)) { + yyjson_doc_free(doc); + return false; + } + + *base_project = heap_strdup(yyjson_get_str(base)); + *target_project = heap_strdup(yyjson_get_str(target)); + yyjson_doc_free(doc); + if (!*base_project || !*target_project) { + free(*base_project); + free(*target_project); + *base_project = NULL; + *target_project = NULL; + *error_message = "out of memory while validating arguments"; + return false; + } + return true; +} + +static yyjson_mut_val *compare_node_json(yyjson_mut_doc *doc, + const cbm_graph_node_identity_t *node) { + yyjson_mut_val *object = yyjson_mut_obj(doc); + if (!object || + !yyjson_mut_obj_add_strcpy(doc, object, "qualified_name", node->qualified_name) || + !yyjson_mut_obj_add_strcpy(doc, object, "label", node->label) || + !yyjson_mut_obj_add_strcpy(doc, object, "file_path", node->file_path)) { + return NULL; + } + return object; +} + +static bool compare_append_item(compare_response_t *response, compare_result_set_t *set, + yyjson_mut_doc *item_doc, yyjson_mut_val *item) { + if (!item_doc || !item) { + yyjson_mut_doc_free(item_doc); + return false; + } + char *encoded = yy_doc_to_str(item_doc); + if (!encoded) { + yyjson_mut_doc_free(item_doc); + return false; + } + size_t encoded_len = strlen(encoded); + size_t separator = set->returned > 0 ? 1U : 0U; + free(encoded); + + if (set->encoded_bytes > MCP_COMPARE_SET_BYTE_BUDGET || + separator > MCP_COMPARE_SET_BYTE_BUDGET - set->encoded_bytes || + encoded_len > MCP_COMPARE_SET_BYTE_BUDGET - set->encoded_bytes - separator) { + set->budget_exhausted = true; + yyjson_mut_doc_free(item_doc); + return true; + } + + yyjson_mut_val *copy = yyjson_mut_val_mut_copy(response->doc, item); + bool ok = copy && yyjson_mut_arr_add_val(set->items, copy); + yyjson_mut_doc_free(item_doc); + if (!ok) { + return false; + } + set->encoded_bytes += separator + encoded_len; + set->returned++; + return true; +} + +static bool compare_node_callback(void *context, bool added, + const cbm_graph_node_identity_t *node) { + compare_response_t *response = (compare_response_t *)context; + compare_result_set_t *set = added ? &response->nodes_added : &response->nodes_removed; + if (set->returned >= response->limit || set->budget_exhausted) { + return true; + } + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *item = doc ? compare_node_json(doc, node) : NULL; + if (doc && item) { + yyjson_mut_doc_set_root(doc, item); + } + return compare_append_item(response, set, doc, item); +} + +static bool compare_edge_callback(void *context, bool added, + const cbm_graph_edge_identity_t *edge) { + compare_response_t *response = (compare_response_t *)context; + compare_result_set_t *set = added ? &response->edges_added : &response->edges_removed; + if (set->returned >= response->limit || set->budget_exhausted) { + return true; + } + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *item = doc ? yyjson_mut_obj(doc) : NULL; + yyjson_mut_val *source = doc ? compare_node_json(doc, &edge->source) : NULL; + yyjson_mut_val *target = doc ? compare_node_json(doc, &edge->target) : NULL; + bool ok = item && source && target && yyjson_mut_obj_add_val(doc, item, "source", source) && + yyjson_mut_obj_add_val(doc, item, "target", target) && + yyjson_mut_obj_add_strcpy(doc, item, "type", edge->type) && + yyjson_mut_obj_add_strcpy(doc, item, "local_name_gen", edge->local_name_gen); + if (ok) { + yyjson_mut_doc_set_root(doc, item); + } + return compare_append_item(response, set, doc, ok ? item : NULL); +} + +static bool compare_cancel_callback(void *context) { + compare_response_t *response = (compare_response_t *)context; + return mcp_request_cancelled(response->server); +} + +static yyjson_mut_val *compare_project_json(yyjson_mut_doc *doc, const char *project, + const cbm_graph_compare_project_t *metadata) { + yyjson_mut_val *object = yyjson_mut_obj(doc); + if (!object || !yyjson_mut_obj_add_strcpy(doc, object, "project", project) || + !yyjson_mut_obj_add_strcpy(doc, object, "generation", metadata->generation) || + !yyjson_mut_obj_add_strcpy(doc, object, "index_mode", metadata->index_mode) || + !yyjson_mut_obj_add_sint(doc, object, "node_count", metadata->node_count) || + !yyjson_mut_obj_add_sint(doc, object, "edge_count", metadata->edge_count)) { + return NULL; + } + return object; +} + +static yyjson_mut_val *compare_set_json(yyjson_mut_doc *doc, compare_result_set_t *set, + uint64_t total, size_t limit) { + yyjson_mut_val *object = yyjson_mut_obj(doc); + yyjson_mut_val *reasons = yyjson_mut_arr(doc); + bool truncated = total > (uint64_t)set->returned; + if (!object || !reasons || !yyjson_mut_obj_add_val(doc, object, "items", set->items) || + !yyjson_mut_obj_add_uint(doc, object, "returned", set->returned) || + !yyjson_mut_obj_add_uint(doc, object, "total", total) || + !yyjson_mut_obj_add_bool(doc, object, "truncated", truncated)) { + return NULL; + } + if (truncated && set->returned >= limit && !yyjson_mut_arr_add_strcpy(doc, reasons, "limit")) { + return NULL; + } + if (truncated && set->budget_exhausted && + !yyjson_mut_arr_add_strcpy(doc, reasons, "encoded_byte_budget")) { + return NULL; + } + if (!yyjson_mut_obj_add_val(doc, object, "truncation_reasons", reasons)) { + return NULL; + } + return object; +} + +static char *handle_compare_graphs(cbm_mcp_server_t *server, const char *args) { + char *base_project = NULL; + char *target_project = NULL; + uint64_t limit = 0; + uint64_t scan_limit = 0; + const char *argument_error = NULL; + if (!compare_parse_arguments(args, &base_project, &target_project, &limit, &scan_limit, + &argument_error)) { + return compare_graphs_error("invalid_arguments", argument_error); + } + if (mcp_request_cancelled(server)) { + free(base_project); + free(target_project); + return compare_graphs_error("cancelled", "compare_graphs cancelled for this request"); + } + + cbm_store_t *base_store = compare_open_project_store(base_project); + if (!base_store) { + free(base_project); + free(target_project); + return compare_graphs_error("project_not_indexed", "base project is not indexed"); + } + cbm_store_t *target_store = compare_open_project_store(target_project); + if (!target_store) { + cbm_store_close(base_store); + free(base_project); + free(target_project); + return compare_graphs_error("project_not_indexed", "target project is not indexed"); + } + + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = doc ? yyjson_mut_obj(doc) : NULL; + compare_response_t response = { + .server = server, + .doc = doc, + .limit = (size_t)limit, + .nodes_added = {.items = doc ? yyjson_mut_arr(doc) : NULL, .encoded_bytes = 2U}, + .nodes_removed = {.items = doc ? yyjson_mut_arr(doc) : NULL, .encoded_bytes = 2U}, + .edges_added = {.items = doc ? yyjson_mut_arr(doc) : NULL, .encoded_bytes = 2U}, + .edges_removed = {.items = doc ? yyjson_mut_arr(doc) : NULL, .encoded_bytes = 2U}, + }; + if (!doc || !root || !response.nodes_added.items || !response.nodes_removed.items || + !response.edges_added.items || !response.edges_removed.items) { + cbm_store_close(target_store); + cbm_store_close(base_store); + yyjson_mut_doc_free(doc); + free(base_project); + free(target_project); + return compare_graphs_error("allocation_failed", "could not allocate comparison result"); + } + yyjson_mut_doc_set_root(doc, root); + + cbm_graph_compare_result_t comparison = {0}; + int compare_rc = cbm_store_compare_graphs( + base_store, base_project, target_store, target_project, scan_limit, compare_cancel_callback, + compare_node_callback, compare_edge_callback, &response, &comparison); + cbm_store_close(target_store); + cbm_store_close(base_store); + + if (compare_rc != CBM_STORE_OK) { + yyjson_mut_doc_free(doc); + free(base_project); + free(target_project); + if (compare_rc == CBM_STORE_CANCELLED) { + return compare_graphs_error("cancelled", "compare_graphs cancelled for this request"); + } + if (compare_rc == CBM_STORE_NOT_FOUND) { + return compare_graphs_error("project_not_indexed", "project is not indexed"); + } + if (compare_rc == CBM_STORE_SCAN_LIMIT) { + return compare_graphs_error("scan_limit_exceeded", + "combined graph rows exceed scan_limit"); + } + if (compare_rc == CBM_STORE_CALLBACK_ERR) { + return compare_graphs_error("allocation_failed", + "could not allocate comparison result"); + } + return compare_graphs_error("query_failed", "graph comparison query failed"); + } + + yyjson_mut_val *base = compare_project_json(doc, base_project, &comparison.base); + yyjson_mut_val *target = compare_project_json(doc, target_project, &comparison.target); + yyjson_mut_val *nodes = yyjson_mut_obj(doc); + yyjson_mut_val *edges = yyjson_mut_obj(doc); + yyjson_mut_val *limits = yyjson_mut_obj(doc); + yyjson_mut_val *nodes_added = + compare_set_json(doc, &response.nodes_added, comparison.nodes_added_total, response.limit); + yyjson_mut_val *nodes_removed = compare_set_json( + doc, &response.nodes_removed, comparison.nodes_removed_total, response.limit); + yyjson_mut_val *edges_added = + compare_set_json(doc, &response.edges_added, comparison.edges_added_total, response.limit); + yyjson_mut_val *edges_removed = compare_set_json( + doc, &response.edges_removed, comparison.edges_removed_total, response.limit); + bool built = + base && target && nodes && edges && limits && nodes_added && nodes_removed && edges_added && + edges_removed && yyjson_mut_obj_add_int(doc, root, "schema_version", 1) && + yyjson_mut_obj_add_val(doc, root, "base", base) && + yyjson_mut_obj_add_val(doc, root, "target", target) && + yyjson_mut_obj_add_val(doc, nodes, "added", nodes_added) && + yyjson_mut_obj_add_val(doc, nodes, "removed", nodes_removed) && + yyjson_mut_obj_add_val(doc, root, "nodes", nodes) && + yyjson_mut_obj_add_val(doc, edges, "added", edges_added) && + yyjson_mut_obj_add_val(doc, edges, "removed", edges_removed) && + yyjson_mut_obj_add_val(doc, root, "edges", edges) && + yyjson_mut_obj_add_uint(doc, limits, "limit", limit) && + yyjson_mut_obj_add_uint(doc, limits, "scan_limit", scan_limit) && + yyjson_mut_obj_add_uint(doc, limits, "encoded_byte_budget", MCP_COMPARE_SET_BYTE_BUDGET) && + yyjson_mut_obj_add_val(doc, root, "limits", limits); + free(base_project); + free(target_project); + if (!built) { + yyjson_mut_doc_free(doc); + return compare_graphs_error("allocation_failed", "could not allocate comparison result"); + } + if (mcp_request_cancelled(server)) { + yyjson_mut_doc_free(doc); + return compare_graphs_error("cancelled", "compare_graphs cancelled for this request"); + } + + char *json = yy_doc_to_str(doc); + yyjson_mut_doc_free(doc); + if (!json) { + return compare_graphs_error("allocation_failed", "could not serialize comparison result"); + } + char *result = cbm_mcp_text_result(json, false); + free(json); + return result; +} + static bool sg_field_blocked(const char *f); /* internal-only fields, defined with search_graph */ static char *handle_get_graph_schema(cbm_mcp_server_t *srv, const char *args) { @@ -11413,6 +11833,9 @@ static char *dispatch_tool(cbm_mcp_server_t *srv, const char *tool_name, const c if (strcmp(tool_name, "get_graph_schema") == 0) { return handle_get_graph_schema(srv, args_json); } + if (strcmp(tool_name, "compare_graphs") == 0) { + return handle_compare_graphs(srv, args_json); + } if (strcmp(tool_name, "search_graph") == 0) { return handle_search_graph(srv, args_json); } diff --git a/src/store/store.c b/src/store/store.c index ad0b7967b..0f2a5ead6 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -1208,6 +1208,390 @@ int cbm_store_rollback(cbm_store_t *s) { return exec_sql(s, "ROLLBACK;"); } +/* ── Graph comparison ────────────────────────────────────────────── */ + +typedef struct { + cbm_graph_compare_cancel_fn cancel; + void *context; +} graph_compare_progress_t; + +typedef struct { + sqlite3_stmt *stmt; + cbm_graph_node_identity_t identity; + bool has_row; +} graph_node_cursor_t; + +typedef struct { + sqlite3_stmt *stmt; + cbm_graph_edge_identity_t identity; + bool has_row; +} graph_edge_cursor_t; + +static bool graph_compare_is_cancelled(const graph_compare_progress_t *progress) { + return progress && progress->cancel && progress->cancel(progress->context); +} + +static int graph_compare_progress(void *context) { + return graph_compare_is_cancelled((const graph_compare_progress_t *)context) ? 1 : 0; +} + +/* SQLite's BINARY collation compares the unsigned bytes of the UTF-8 text. + * Keep the merge comparator byte-for-byte equivalent so neither cursor can + * be advanced past a row that SQLite considers distinct. */ +static int graph_binary_compare(const char *left, const char *right) { + const unsigned char *a = (const unsigned char *)safe_str(left); + const unsigned char *b = (const unsigned char *)safe_str(right); + while (*a && *a == *b) { + a++; + b++; + } + return (*a > *b) - (*a < *b); +} + +static int graph_node_compare(const cbm_graph_node_identity_t *left, + const cbm_graph_node_identity_t *right) { + int cmp = graph_binary_compare(left->qualified_name, right->qualified_name); + if (cmp == 0) { + cmp = graph_binary_compare(left->label, right->label); + } + if (cmp == 0) { + cmp = graph_binary_compare(left->file_path, right->file_path); + } + return cmp; +} + +static int graph_edge_compare(const cbm_graph_edge_identity_t *left, + const cbm_graph_edge_identity_t *right) { + int cmp = graph_node_compare(&left->source, &right->source); + if (cmp == 0) { + cmp = graph_node_compare(&left->target, &right->target); + } + if (cmp == 0) { + cmp = graph_binary_compare(left->type, right->type); + } + if (cmp == 0) { + cmp = graph_binary_compare(left->local_name_gen, right->local_name_gen); + } + return cmp; +} + +static int graph_count_rows(cbm_store_t *store, const char *sql, const char *project, + int64_t *out) { + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(store->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { + store_set_error_sqlite(store, "compare count prepare"); + return CBM_STORE_ERR; + } + bind_text(stmt, SKIP_ONE, project); + int step_rc = sqlite3_step(stmt); + if (step_rc != SQLITE_ROW) { + store_set_error_sqlite(store, "compare count"); + sqlite3_finalize(stmt); + return step_rc == SQLITE_INTERRUPT ? CBM_STORE_CANCELLED : CBM_STORE_ERR; + } + *out = sqlite3_column_int64(stmt, 0); + step_rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + if (step_rc != SQLITE_DONE) { + store_set_error_sqlite(store, "compare count finish"); + return step_rc == SQLITE_INTERRUPT ? CBM_STORE_CANCELLED : CBM_STORE_ERR; + } + return CBM_STORE_OK; +} + +static int graph_capture_project(cbm_store_t *store, const char *project, + cbm_graph_compare_project_t *out) { + cbm_project_t row = {0}; + int rc = cbm_store_get_project(store, project, &row); + if (rc != CBM_STORE_OK) { + return rc; + } + if (!row.name || !row.indexed_at || !row.root_path) { + cbm_project_free_fields(&row); + store_set_error(store, "compare project allocation failed"); + return CBM_STORE_ERR; + } + + snprintf(out->generation, sizeof(out->generation), "%s", safe_str(row.indexed_at)); + snprintf(out->index_mode, sizeof(out->index_mode), "unknown"); + + cbm_coverage_meta_t meta = {0}; + rc = cbm_store_coverage_meta_get(store, project, &meta); + if (rc == CBM_STORE_OK) { + snprintf(out->generation, sizeof(out->generation), "%s", safe_str(meta.generation)); + snprintf(out->index_mode, sizeof(out->index_mode), "%s", safe_str(meta.index_mode)); + cbm_store_coverage_meta_clear(&meta); + } else if (rc != CBM_STORE_NOT_FOUND) { + cbm_project_free_fields(&row); + return rc; + } + cbm_project_free_fields(&row); + + rc = graph_count_rows(store, "SELECT count(*) FROM nodes WHERE project=?1;", project, + &out->node_count); + if (rc != CBM_STORE_OK) { + return rc; + } + return graph_count_rows(store, "SELECT count(*) FROM edges WHERE project=?1;", project, + &out->edge_count); +} + +static int graph_node_cursor_step(cbm_store_t *store, graph_node_cursor_t *cursor) { + int rc = sqlite3_step(cursor->stmt); + if (rc == SQLITE_DONE) { + cursor->has_row = false; + return CBM_STORE_OK; + } + if (rc != SQLITE_ROW) { + store_set_error_sqlite(store, "compare node scan"); + cursor->has_row = false; + return rc == SQLITE_INTERRUPT ? CBM_STORE_CANCELLED : CBM_STORE_ERR; + } + cursor->identity.qualified_name = safe_str((const char *)sqlite3_column_text(cursor->stmt, 0)); + cursor->identity.label = safe_str((const char *)sqlite3_column_text(cursor->stmt, ST_COL_1)); + cursor->identity.file_path = + safe_str((const char *)sqlite3_column_text(cursor->stmt, ST_COL_2)); + cursor->has_row = true; + return CBM_STORE_OK; +} + +static int graph_edge_cursor_step(cbm_store_t *store, graph_edge_cursor_t *cursor) { + int rc = sqlite3_step(cursor->stmt); + if (rc == SQLITE_DONE) { + cursor->has_row = false; + return CBM_STORE_OK; + } + if (rc != SQLITE_ROW) { + store_set_error_sqlite(store, "compare edge scan"); + cursor->has_row = false; + return rc == SQLITE_INTERRUPT ? CBM_STORE_CANCELLED : CBM_STORE_ERR; + } + cursor->identity.source.qualified_name = + safe_str((const char *)sqlite3_column_text(cursor->stmt, 0)); + cursor->identity.source.label = + safe_str((const char *)sqlite3_column_text(cursor->stmt, ST_COL_1)); + cursor->identity.source.file_path = + safe_str((const char *)sqlite3_column_text(cursor->stmt, ST_COL_2)); + cursor->identity.target.qualified_name = + safe_str((const char *)sqlite3_column_text(cursor->stmt, ST_COL_3)); + cursor->identity.target.label = + safe_str((const char *)sqlite3_column_text(cursor->stmt, ST_COL_4)); + cursor->identity.target.file_path = + safe_str((const char *)sqlite3_column_text(cursor->stmt, ST_COL_5)); + cursor->identity.type = safe_str((const char *)sqlite3_column_text(cursor->stmt, ST_COL_6)); + cursor->identity.local_name_gen = + safe_str((const char *)sqlite3_column_text(cursor->stmt, ST_COL_7)); + cursor->has_row = true; + return CBM_STORE_OK; +} + +static int graph_prepare_node_cursor(cbm_store_t *store, const char *project, + graph_node_cursor_t *cursor) { + static const char sql[] = + "SELECT qualified_name,label,coalesce(file_path,'') FROM nodes WHERE project=?1 " + "ORDER BY qualified_name COLLATE BINARY,label COLLATE BINARY," + "coalesce(file_path,'') COLLATE BINARY;"; + if (sqlite3_prepare_v2(store->db, sql, CBM_NOT_FOUND, &cursor->stmt, NULL) != SQLITE_OK) { + store_set_error_sqlite(store, "compare node prepare"); + return CBM_STORE_ERR; + } + bind_text(cursor->stmt, SKIP_ONE, project); + return graph_node_cursor_step(store, cursor); +} + +static int graph_prepare_edge_cursor(cbm_store_t *store, const char *project, + graph_edge_cursor_t *cursor) { + static const char sql[] = + "SELECT sn.qualified_name,sn.label,coalesce(sn.file_path,'')," + "tn.qualified_name,tn.label,coalesce(tn.file_path,''),e.type," + "coalesce(e.local_name_gen,'') FROM edges e " + "JOIN nodes sn ON sn.id=e.source_id JOIN nodes tn ON tn.id=e.target_id " + "WHERE e.project=?1 ORDER BY " + "sn.qualified_name COLLATE BINARY,sn.label COLLATE BINARY," + "coalesce(sn.file_path,'') COLLATE BINARY," + "tn.qualified_name COLLATE BINARY,tn.label COLLATE BINARY," + "coalesce(tn.file_path,'') COLLATE BINARY," + "e.type COLLATE BINARY,coalesce(e.local_name_gen,'') COLLATE BINARY;"; + if (sqlite3_prepare_v2(store->db, sql, CBM_NOT_FOUND, &cursor->stmt, NULL) != SQLITE_OK) { + store_set_error_sqlite(store, "compare edge prepare"); + return CBM_STORE_ERR; + } + bind_text(cursor->stmt, SKIP_ONE, project); + return graph_edge_cursor_step(store, cursor); +} + +static int graph_compare_nodes(cbm_store_t *base_store, const char *base_project, + cbm_store_t *target_store, const char *target_project, + graph_compare_progress_t *progress, + cbm_graph_compare_node_fn callback, + cbm_graph_compare_result_t *out) { + graph_node_cursor_t base = {0}; + graph_node_cursor_t target = {0}; + int rc = graph_prepare_node_cursor(base_store, base_project, &base); + if (rc == CBM_STORE_OK) { + rc = graph_prepare_node_cursor(target_store, target_project, &target); + } + uint64_t merged = 0; + while (rc == CBM_STORE_OK && (base.has_row || target.has_row)) { + if ((merged++ & UINT64_C(255)) == 0 && graph_compare_is_cancelled(progress)) { + rc = CBM_STORE_CANCELLED; + break; + } + int cmp = + !base.has_row + ? 1 + : (!target.has_row ? -1 : graph_node_compare(&base.identity, &target.identity)); + if (cmp < 0) { + out->nodes_removed_total++; + if (callback && !callback(progress->context, false, &base.identity)) { + rc = CBM_STORE_CALLBACK_ERR; + break; + } + rc = graph_node_cursor_step(base_store, &base); + } else if (cmp > 0) { + out->nodes_added_total++; + if (callback && !callback(progress->context, true, &target.identity)) { + rc = CBM_STORE_CALLBACK_ERR; + break; + } + rc = graph_node_cursor_step(target_store, &target); + } else { + rc = graph_node_cursor_step(base_store, &base); + if (rc == CBM_STORE_OK) { + rc = graph_node_cursor_step(target_store, &target); + } + } + } + sqlite3_finalize(base.stmt); + sqlite3_finalize(target.stmt); + return rc; +} + +static int graph_compare_edges(cbm_store_t *base_store, const char *base_project, + cbm_store_t *target_store, const char *target_project, + graph_compare_progress_t *progress, + cbm_graph_compare_edge_fn callback, + cbm_graph_compare_result_t *out) { + graph_edge_cursor_t base = {0}; + graph_edge_cursor_t target = {0}; + int rc = graph_prepare_edge_cursor(base_store, base_project, &base); + if (rc == CBM_STORE_OK) { + rc = graph_prepare_edge_cursor(target_store, target_project, &target); + } + uint64_t merged = 0; + while (rc == CBM_STORE_OK && (base.has_row || target.has_row)) { + if ((merged++ & UINT64_C(255)) == 0 && graph_compare_is_cancelled(progress)) { + rc = CBM_STORE_CANCELLED; + break; + } + int cmp = + !base.has_row + ? 1 + : (!target.has_row ? -1 : graph_edge_compare(&base.identity, &target.identity)); + if (cmp < 0) { + out->edges_removed_total++; + if (callback && !callback(progress->context, false, &base.identity)) { + rc = CBM_STORE_CALLBACK_ERR; + break; + } + rc = graph_edge_cursor_step(base_store, &base); + } else if (cmp > 0) { + out->edges_added_total++; + if (callback && !callback(progress->context, true, &target.identity)) { + rc = CBM_STORE_CALLBACK_ERR; + break; + } + rc = graph_edge_cursor_step(target_store, &target); + } else { + rc = graph_edge_cursor_step(base_store, &base); + if (rc == CBM_STORE_OK) { + rc = graph_edge_cursor_step(target_store, &target); + } + } + } + sqlite3_finalize(base.stmt); + sqlite3_finalize(target.stmt); + return rc; +} + +static bool graph_scan_limit_exceeded(int64_t base_count, int64_t target_count, + uint64_t scan_limit) { + if (base_count < 0 || target_count < 0 || (uint64_t)base_count > scan_limit || + (uint64_t)target_count > scan_limit) { + return true; + } + return (uint64_t)base_count > scan_limit - (uint64_t)target_count; +} + +int cbm_store_compare_graphs(cbm_store_t *base_store, const char *base_project, + cbm_store_t *target_store, const char *target_project, + uint64_t scan_limit, cbm_graph_compare_cancel_fn cancel, + cbm_graph_compare_node_fn on_node, cbm_graph_compare_edge_fn on_edge, + void *context, cbm_graph_compare_result_t *out) { + if (!base_store || !target_store || base_store == target_store || !base_project || + !base_project[0] || !target_project || !target_project[0] || scan_limit == 0 || !out) { + return CBM_STORE_ERR; + } + memset(out, 0, sizeof(*out)); + graph_compare_progress_t progress = {.cancel = cancel, .context = context}; + if (graph_compare_is_cancelled(&progress)) { + return CBM_STORE_CANCELLED; + } + + int rc = CBM_STORE_OK; + bool base_transaction = false; + bool target_transaction = false; + sqlite3_progress_handler(base_store->db, 1000, graph_compare_progress, &progress); + sqlite3_progress_handler(target_store->db, 1000, graph_compare_progress, &progress); + + rc = exec_sql(base_store, "BEGIN;"); + if (rc == CBM_STORE_OK) { + base_transaction = true; + rc = exec_sql(target_store, "BEGIN;"); + } + if (rc == CBM_STORE_OK) { + target_transaction = true; + rc = graph_capture_project(base_store, base_project, &out->base); + } + if (rc == CBM_STORE_OK) { + rc = graph_capture_project(target_store, target_project, &out->target); + } + if (rc == CBM_STORE_OK && + (graph_scan_limit_exceeded(out->base.node_count, out->target.node_count, scan_limit) || + graph_scan_limit_exceeded(out->base.edge_count, out->target.edge_count, scan_limit))) { + rc = CBM_STORE_SCAN_LIMIT; + } + if (rc == CBM_STORE_OK) { + rc = graph_compare_nodes(base_store, base_project, target_store, target_project, &progress, + on_node, out); + } + if (rc == CBM_STORE_OK) { + rc = graph_compare_edges(base_store, base_project, target_store, target_project, &progress, + on_edge, out); + } + if (rc == CBM_STORE_OK && graph_compare_is_cancelled(&progress)) { + rc = CBM_STORE_CANCELLED; + } + if (rc != CBM_STORE_OK && graph_compare_is_cancelled(&progress)) { + rc = CBM_STORE_CANCELLED; + } + + sqlite3_progress_handler(base_store->db, 0, NULL, NULL); + sqlite3_progress_handler(target_store->db, 0, NULL, NULL); + if (target_transaction && exec_sql(target_store, "ROLLBACK;") != CBM_STORE_OK && + rc == CBM_STORE_OK) { + rc = CBM_STORE_ERR; + } + if (base_transaction && exec_sql(base_store, "ROLLBACK;") != CBM_STORE_OK && + rc == CBM_STORE_OK) { + rc = CBM_STORE_ERR; + } + if (rc != CBM_STORE_OK) { + memset(out, 0, sizeof(*out)); + } + return rc; +} + /* ── Bulk write ─────────────────────────────────────────────────── */ int cbm_store_begin_bulk(cbm_store_t *s) { diff --git a/src/store/store.h b/src/store/store.h index 45a7bb439..43c189792 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -24,6 +24,9 @@ typedef struct cbm_store cbm_store_t; #define CBM_STORE_ERR (-1) #define CBM_STORE_NOT_FOUND (-2) #define CBM_INDEX_FORMAT_VERSION 1 +#define CBM_STORE_CANCELLED (-3) +#define CBM_STORE_SCAN_LIMIT (-4) +#define CBM_STORE_CALLBACK_ERR (-5) /* ── Data structures ────────────────────────────────────────────── */ @@ -255,6 +258,56 @@ typedef struct { int sample_qn_count; } cbm_schema_info_t; +/* ── Graph comparison ──────────────────────────────────────────── */ + +/* Stable graph identities deliberately exclude the project name. Strings are + * borrowed from the active SQLite row and remain valid only for the duration + * of the callback. */ +typedef struct { + const char *qualified_name; + const char *label; + const char *file_path; +} cbm_graph_node_identity_t; + +typedef struct { + cbm_graph_node_identity_t source; + cbm_graph_node_identity_t target; + const char *type; + const char *local_name_gen; +} cbm_graph_edge_identity_t; + +typedef bool (*cbm_graph_compare_cancel_fn)(void *context); +typedef bool (*cbm_graph_compare_node_fn)(void *context, bool added, + const cbm_graph_node_identity_t *node); +typedef bool (*cbm_graph_compare_edge_fn)(void *context, bool added, + const cbm_graph_edge_identity_t *edge); + +typedef struct { + char generation[128]; + char index_mode[32]; + int64_t node_count; + int64_t edge_count; +} cbm_graph_compare_project_t; + +typedef struct { + cbm_graph_compare_project_t base; + cbm_graph_compare_project_t target; + uint64_t nodes_added_total; + uint64_t nodes_removed_total; + uint64_t edges_added_total; + uint64_t edges_removed_total; +} cbm_graph_compare_result_t; + +/* Compare two independently-owned read-only stores using ordered streaming + * cursors. Both read transactions and both SQLite progress handlers are owned + * by this call and released on every exit. scan_limit applies independently to + * the combined node rows and combined edge rows across the two projects. */ +int cbm_store_compare_graphs(cbm_store_t *base_store, const char *base_project, + cbm_store_t *target_store, const char *target_project, + uint64_t scan_limit, cbm_graph_compare_cancel_fn cancel, + cbm_graph_compare_node_fn on_node, cbm_graph_compare_edge_fn on_edge, + void *context, cbm_graph_compare_result_t *out); + /* ── Lifecycle ──────────────────────────────────────────────────── */ /* Open an in-memory database (for testing). */ diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 6b7af64ed..fbf59823e 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -914,6 +914,7 @@ TEST(mcp_tools_have_behavior_annotations) { {"trace_path", false, true, true, false}, {"get_code_snippet", false, true, true, false}, {"get_graph_schema", false, true, true, false}, + {"compare_graphs", true, false, true, false}, {"get_architecture", false, true, true, false}, {"search_code", false, true, true, false}, {"list_projects", true, false, true, false}, @@ -1359,9 +1360,9 @@ TEST(server_handle_analysis_profile_filters_and_rejects_mutators) { resp = cbm_mcp_server_handle(srv, "{\"jsonrpc\":\"2.0\",\"id\":220,\"method\":\"tools/list\"}"); ASSERT_NOT_NULL(resp); static const char *const analysis_tools[] = { - "search_graph", "query_graph", "trace_path", "get_code_snippet", - "get_graph_schema", "get_architecture", "search_code", "list_projects", - "index_status", "check_index_coverage", "detect_changes", + "search_graph", "query_graph", "trace_path", "get_code_snippet", + "get_graph_schema", "compare_graphs", "get_architecture", "search_code", + "list_projects", "index_status", "check_index_coverage", "detect_changes", }; ASSERT_EQ(mcp_response_tool_count(resp), sizeof(analysis_tools) / sizeof(analysis_tools[0])); for (size_t i = 0U; i < sizeof(analysis_tools) / sizeof(analysis_tools[0]); i++) { @@ -1411,6 +1412,7 @@ TEST(server_handle_scout_profile_exposes_only_the_fast_tier) { ASSERT_FALSE(mcp_response_has_exact_tool(resp, "query_graph")); ASSERT_FALSE(mcp_response_has_exact_tool(resp, "search_code")); ASSERT_FALSE(mcp_response_has_exact_tool(resp, "get_graph_schema")); + ASSERT_FALSE(mcp_response_has_exact_tool(resp, "compare_graphs")); ASSERT_FALSE(mcp_response_has_exact_tool(resp, "detect_changes")); ASSERT_FALSE(mcp_response_has_exact_tool(resp, "index_repository")); free(resp); @@ -1670,6 +1672,24 @@ TEST(tool_unknown_tool) { PASS(); } +/* Issue #525: compare_graphs is a first-class analysis tool. The frozen base + * routes this call to the unknown-tool fallback; production registration is + * therefore required before any compare semantics can accidentally look green. */ +TEST(tool_compare_graphs_registered_issue525) { + cbm_mcp_server_t *srv = setup_mcp_with_data(); + + char *resp = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":525,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"compare_graphs\",\"arguments\":{" + "\"base_project\":\"base525\",\"target_project\":\"target525\"}}}"); + ASSERT_NOT_NULL(resp); + ASSERT_NULL(strstr(resp, "unknown tool: compare_graphs")); + free(resp); + + cbm_mcp_server_free(srv); + PASS(); +} + TEST(tool_search_graph_basic) { cbm_mcp_server_t *srv = setup_mcp_with_data(); @@ -7923,6 +7943,485 @@ static char *extract_text_content(const char *mcp_result) { return result; } +typedef struct { + char cache[512]; + char base_path[768]; + char target_path[768]; + char *saved_cache; +} compare_graphs_fixture_t; + +typedef struct { + const char *qualified_name; + const char *label; + const char *file_path; +} compare_node_spec_t; + +static bool compare_write_coverage(cbm_store_t *store, const char *project, + const char *generation) { + cbm_coverage_meta_t meta = { + .generation = generation, + .index_mode = "full", + .recorded_at = "2026-08-28T00:00:00Z", + .recording_status = "complete", + .ignored_files_stored = 0, + .ignored_files_total = 0, + .coverage_version = 1, + .hash_records_complete = true, + }; + return cbm_store_coverage_replace_ex(store, project, NULL, 0, &meta) == CBM_STORE_OK; +} + +static bool compare_write_fixture_store(const char *path, const char *project, bool target) { + static const compare_node_spec_t base_nodes[] = { + {"pkg.common", "Function", "src/common.c"}, {"pkg.base", "Function", "src/base.c"}, + {"pkg.changed", "Function", "src/old.c"}, {"pkg.source", "Module", "src/source.c"}, + {"pkg.sink", "Module", "src/sink.c"}, + }; + static const compare_node_spec_t target_nodes[] = { + {"pkg.common", "Function", "src/common.c"}, {"pkg.target", "Function", "src/target.c"}, + {"pkg.changed", "Method", "src/new.c"}, {"pkg.source", "Module", "src/source.c"}, + {"pkg.sink", "Module", "src/sink.c"}, + }; + static const int base_order[] = {3, 2, 0, 4, 1}; + static const int target_order[] = {1, 4, 0, 2, 3}; + const compare_node_spec_t *nodes = target ? target_nodes : base_nodes; + const int *order = target ? target_order : base_order; + + cbm_store_t *store = cbm_store_open_path(path); + if (!store || + cbm_store_upsert_project(store, project, "/tmp/compare-graphs-525") != CBM_STORE_OK) { + cbm_store_close(store); + return false; + } + int64_t ids[5] = {0}; + bool ok = true; + for (size_t position = 0; position < 5; position++) { + int index = order[position]; + cbm_node_t node = { + .project = project, + .label = nodes[index].label, + .name = nodes[index].qualified_name, + .qualified_name = nodes[index].qualified_name, + .file_path = nodes[index].file_path, + .properties_json = "{}", + }; + ids[index] = cbm_store_upsert_node(store, &node); + ok = ok && ids[index] > 0; + } + + cbm_edge_t common_call = { + .project = project, + .source_id = ids[3], + .target_id = ids[4], + .type = "CALLS", + .properties_json = "{}", + }; + cbm_edge_t common_import = { + .project = project, + .source_id = ids[3], + .target_id = ids[4], + .type = "IMPORTS", + .properties_json = "{\"local_name\":\"Alpha\"}", + }; + cbm_edge_t changed_import = { + .project = project, + .source_id = ids[3], + .target_id = ids[4], + .type = "IMPORTS", + .properties_json = target ? "{\"local_name\":\"Gamma\"}" : "{\"local_name\":\"Beta\"}", + }; + cbm_edge_t side_edge = { + .project = project, + .source_id = ids[1], + .target_id = ids[0], + .type = "USES", + .properties_json = "{}", + }; + if (target) { + ok = ok && cbm_store_insert_edge(store, &side_edge) > 0 && + cbm_store_insert_edge(store, &changed_import) > 0 && + cbm_store_insert_edge(store, &common_call) > 0 && + cbm_store_insert_edge(store, &common_import) > 0; + } else { + ok = ok && cbm_store_insert_edge(store, &common_import) > 0 && + cbm_store_insert_edge(store, &common_call) > 0 && + cbm_store_insert_edge(store, &side_edge) > 0 && + cbm_store_insert_edge(store, &changed_import) > 0; + } + ok = ok && + compare_write_coverage(store, project, + target ? "target-generation-525" : "base-generation-525") && + cbm_store_prepare_for_publish(store) == CBM_STORE_OK; + cbm_store_close(store); + return ok; +} + +static bool compare_graphs_fixture_open(compare_graphs_fixture_t *fixture) { + memset(fixture, 0, sizeof(*fixture)); + snprintf(fixture->cache, sizeof(fixture->cache), "%s/cbm-compare-525-XXXXXX", cbm_tmpdir()); + if (!cbm_mkdtemp(fixture->cache)) { + return false; + } + const char *saved = getenv("CBM_CACHE_DIR"); + fixture->saved_cache = saved ? strdup(saved) : NULL; + if ((saved && !fixture->saved_cache) || cbm_setenv("CBM_CACHE_DIR", fixture->cache, 1) != 0) { + free(fixture->saved_cache); + fixture->saved_cache = NULL; + (void)th_rmtree(fixture->cache); + return false; + } + snprintf(fixture->base_path, sizeof(fixture->base_path), "%s/base525.db", fixture->cache); + snprintf(fixture->target_path, sizeof(fixture->target_path), "%s/target525.db", fixture->cache); + if (!compare_write_fixture_store(fixture->base_path, "base525", false) || + !compare_write_fixture_store(fixture->target_path, "target525", true)) { + restore_cache_dir(fixture->saved_cache); + free(fixture->saved_cache); + fixture->saved_cache = NULL; + (void)th_rmtree(fixture->cache); + return false; + } + return true; +} + +static bool compare_graphs_fixture_close(compare_graphs_fixture_t *fixture) { + restore_cache_dir(fixture->saved_cache); + free(fixture->saved_cache); + fixture->saved_cache = NULL; + return th_rmtree(fixture->cache) == 0; +} + +static char *compare_graphs_call(cbm_mcp_server_t *server, const char *extra_arguments) { + char arguments[1024]; + snprintf(arguments, sizeof(arguments), + "{\"base_project\":\"base525\",\"target_project\":\"target525\"%s}", + extra_arguments ? extra_arguments : ""); + return cbm_mcp_handle_tool(server, "compare_graphs", arguments); +} + +static bool compare_set_has(const yyjson_val *set, uint64_t total, uint64_t returned, + bool truncated, const char *reason) { + if (!set || !yyjson_is_obj(set)) { + return false; + } + yyjson_val *items = yyjson_obj_get(set, "items"); + yyjson_val *reasons = yyjson_obj_get(set, "truncation_reasons"); + return items && yyjson_is_arr(items) && yyjson_arr_size(items) == returned && reasons && + yyjson_is_arr(reasons) && yyjson_get_uint(yyjson_obj_get(set, "total")) == total && + yyjson_get_uint(yyjson_obj_get(set, "returned")) == returned && + yyjson_get_bool(yyjson_obj_get(set, "truncated")) == truncated && + ((!reason && yyjson_arr_size(reasons) == 0) || + (reason && yyjson_arr_size(reasons) == 1 && + strcmp(yyjson_get_str(yyjson_arr_get(reasons, 0)), reason) == 0)); +} + +TEST(tool_compare_graphs_streams_stable_deltas_issue525) { + compare_graphs_fixture_t fixture; + ASSERT_TRUE(compare_graphs_fixture_open(&fixture)); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(server); + cbm_mcp_server_set_tool_profile(server, CBM_MCP_TOOL_PROFILE_ANALYSIS); + + char *first = compare_graphs_call(server, NULL); + char *second = compare_graphs_call(server, NULL); + ASSERT_NOT_NULL(first); + ASSERT_NOT_NULL(second); + ASSERT_STR_EQ(first, second); + ASSERT_NOT_NULL(strstr(first, "\"isError\":false")); + char *inner = extract_text_content(first); + ASSERT_NOT_NULL(inner); + yyjson_doc *doc = yyjson_read(inner, strlen(inner), 0); + ASSERT_NOT_NULL(doc); + yyjson_val *root = yyjson_doc_get_root(doc); + ASSERT_EQ(yyjson_get_int(yyjson_obj_get(root, "schema_version")), 1); + + yyjson_val *base = yyjson_obj_get(root, "base"); + yyjson_val *target = yyjson_obj_get(root, "target"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(base, "project")), "base525"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(base, "generation")), "base-generation-525"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(base, "index_mode")), "full"); + ASSERT_EQ(yyjson_get_int(yyjson_obj_get(base, "node_count")), 5); + ASSERT_EQ(yyjson_get_int(yyjson_obj_get(base, "edge_count")), 4); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(target, "project")), "target525"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(target, "generation")), "target-generation-525"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(target, "index_mode")), "full"); + ASSERT_EQ(yyjson_get_int(yyjson_obj_get(target, "node_count")), 5); + ASSERT_EQ(yyjson_get_int(yyjson_obj_get(target, "edge_count")), 4); + + yyjson_val *nodes = yyjson_obj_get(root, "nodes"); + yyjson_val *nodes_added = yyjson_obj_get(nodes, "added"); + yyjson_val *nodes_removed = yyjson_obj_get(nodes, "removed"); + ASSERT_TRUE(compare_set_has(nodes_added, 2, 2, false, NULL)); + ASSERT_TRUE(compare_set_has(nodes_removed, 2, 2, false, NULL)); + yyjson_val *added_items = yyjson_obj_get(nodes_added, "items"); + yyjson_val *removed_items = yyjson_obj_get(nodes_removed, "items"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(added_items, 0), "qualified_name")), + "pkg.changed"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(added_items, 0), "label")), + "Method"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(added_items, 0), "file_path")), + "src/new.c"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(added_items, 1), "qualified_name")), + "pkg.target"); + ASSERT_STR_EQ( + yyjson_get_str(yyjson_obj_get(yyjson_arr_get(removed_items, 0), "qualified_name")), + "pkg.base"); + ASSERT_STR_EQ( + yyjson_get_str(yyjson_obj_get(yyjson_arr_get(removed_items, 1), "qualified_name")), + "pkg.changed"); + + yyjson_val *edges = yyjson_obj_get(root, "edges"); + yyjson_val *edges_added = yyjson_obj_get(edges, "added"); + yyjson_val *edges_removed = yyjson_obj_get(edges, "removed"); + ASSERT_TRUE(compare_set_has(edges_added, 2, 2, false, NULL)); + ASSERT_TRUE(compare_set_has(edges_removed, 2, 2, false, NULL)); + yyjson_val *edge_add_items = yyjson_obj_get(edges_added, "items"); + yyjson_val *edge_remove_items = yyjson_obj_get(edges_removed, "items"); + ASSERT_STR_EQ( + yyjson_get_str(yyjson_obj_get(yyjson_arr_get(edge_add_items, 0), "local_name_gen")), + "Gamma"); + ASSERT_STR_EQ( + yyjson_get_str(yyjson_obj_get(yyjson_arr_get(edge_remove_items, 1), "local_name_gen")), + "Beta"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(edge_add_items, 1), "type")), + "USES"); + ASSERT_STR_EQ( + yyjson_get_str(yyjson_obj_get(yyjson_arr_get(edge_add_items, 1), "local_name_gen")), ""); + yyjson_val *edge_source = yyjson_obj_get(yyjson_arr_get(edge_add_items, 0), "source"); + yyjson_val *edge_target = yyjson_obj_get(yyjson_arr_get(edge_add_items, 0), "target"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(edge_source, "qualified_name")), "pkg.source"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(edge_target, "qualified_name")), "pkg.sink"); + + yyjson_val *limits = yyjson_obj_get(root, "limits"); + ASSERT_EQ(yyjson_get_uint(yyjson_obj_get(limits, "limit")), 200); + ASSERT_EQ(yyjson_get_uint(yyjson_obj_get(limits, "scan_limit")), 2000000); + ASSERT_EQ(yyjson_get_uint(yyjson_obj_get(limits, "encoded_byte_budget")), 512U * 1024U); + yyjson_doc_free(doc); + free(inner); + free(second); + free(first); + + char *limited = compare_graphs_call(server, ",\"limit\":1"); + ASSERT_NOT_NULL(limited); + inner = extract_text_content(limited); + doc = yyjson_read(inner, strlen(inner), 0); + ASSERT_NOT_NULL(doc); + root = yyjson_doc_get_root(doc); + nodes = yyjson_obj_get(root, "nodes"); + edges = yyjson_obj_get(root, "edges"); + ASSERT_TRUE(compare_set_has(yyjson_obj_get(nodes, "added"), 2, 1, true, "limit")); + ASSERT_TRUE(compare_set_has(yyjson_obj_get(nodes, "removed"), 2, 1, true, "limit")); + ASSERT_TRUE(compare_set_has(yyjson_obj_get(edges, "added"), 2, 1, true, "limit")); + ASSERT_TRUE(compare_set_has(yyjson_obj_get(edges, "removed"), 2, 1, true, "limit")); + yyjson_doc_free(doc); + free(inner); + free(limited); + + cbm_mcp_server_free(server); + ASSERT_TRUE(compare_graphs_fixture_close(&fixture)); + PASS(); +} + +static bool compare_write_single_node_store(const char *path, const char *project, + const char *generation, const char *qualified_name) { + cbm_store_t *store = cbm_store_open_path(path); + if (!store || + cbm_store_upsert_project(store, project, "/tmp/compare-budget-525") != CBM_STORE_OK) { + cbm_store_close(store); + return false; + } + bool ok = true; + if (qualified_name) { + cbm_node_t node = { + .project = project, + .label = "Function", + .name = "large", + .qualified_name = qualified_name, + .file_path = "src/large.c", + .properties_json = "{}", + }; + ok = cbm_store_upsert_node(store, &node) > 0; + } + ok = ok && compare_write_coverage(store, project, generation) && + cbm_store_prepare_for_publish(store) == CBM_STORE_OK; + cbm_store_close(store); + return ok; +} + +TEST(tool_compare_graphs_enforces_encoded_budget_issue525) { + compare_graphs_fixture_t fixture; + ASSERT_TRUE(compare_graphs_fixture_open(&fixture)); + char base_path[768]; + char target_path[768]; + snprintf(base_path, sizeof(base_path), "%s/budgetbase525.db", fixture.cache); + snprintf(target_path, sizeof(target_path), "%s/budgettarget525.db", fixture.cache); + size_t name_length = 530000U; + char *large_name = malloc(name_length + 1U); + ASSERT_NOT_NULL(large_name); + memset(large_name, 'x', name_length); + large_name[0] = 'z'; + large_name[name_length] = '\0'; + ASSERT_TRUE(compare_write_single_node_store(base_path, "budgetbase525", "budget-base", NULL)); + ASSERT_TRUE(compare_write_single_node_store(target_path, "budgettarget525", "budget-target", + large_name)); + free(large_name); + + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(server); + char *response = cbm_mcp_handle_tool( + server, "compare_graphs", + "{\"base_project\":\"budgetbase525\",\"target_project\":\"budgettarget525\"," + "\"limit\":1000}"); + ASSERT_NOT_NULL(response); + ASSERT_LT(strlen(response), 10000U); + char *inner = extract_text_content(response); + yyjson_doc *doc = inner ? yyjson_read(inner, strlen(inner), 0) : NULL; + ASSERT_NOT_NULL(doc); + yyjson_val *nodes = yyjson_obj_get(yyjson_doc_get_root(doc), "nodes"); + ASSERT_TRUE(compare_set_has(yyjson_obj_get(nodes, "added"), 1, 0, true, "encoded_byte_budget")); + yyjson_doc_free(doc); + free(inner); + free(response); + cbm_mcp_server_free(server); + ASSERT_TRUE(compare_graphs_fixture_close(&fixture)); + PASS(); +} + +TEST(tool_compare_graphs_validation_and_scan_cap_are_atomic_issue525) { + compare_graphs_fixture_t fixture; + ASSERT_TRUE(compare_graphs_fixture_open(&fixture)); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(server); + + char equal_path[768]; + char equal_wal[800]; + char equal_shm[800]; + snprintf(equal_path, sizeof(equal_path), "%s/same525.db", fixture.cache); + snprintf(equal_wal, sizeof(equal_wal), "%s-wal", equal_path); + snprintf(equal_shm, sizeof(equal_shm), "%s-shm", equal_path); + ASSERT_FALSE(cbm_file_exists(equal_path)); + + static const char *const invalid_arguments[] = { + "{}", + "{\"base_project\":\"same525\",\"target_project\":\"same525\"}", + "{\"base_project\":\"base525\",\"target_project\":\"target525\",\"limit\":0}", + "{\"base_project\":\"base525\",\"target_project\":\"target525\",\"limit\":1001}", + "{\"base_project\":\"base525\",\"target_project\":\"target525\",\"scan_limit\":0}", + ("{\"base_project\":\"base525\",\"target_project\":\"target525\"," + "\"scan_limit\":10000001}"), + "{\"base_project\":\"base525\",\"target_project\":\"target525\",\"extra\":1}", + "{", + }; + for (size_t index = 0; index < sizeof(invalid_arguments) / sizeof(invalid_arguments[0]); + index++) { + char *response = cbm_mcp_handle_tool(server, "compare_graphs", invalid_arguments[index]); + ASSERT_NOT_NULL(response); + ASSERT_NOT_NULL(strstr(response, "\"isError\":true")); + char *inner = extract_text_content(response); + ASSERT_NOT_NULL(inner); + ASSERT_NULL(strstr(inner, "\"nodes\"")); + free(inner); + free(response); + } + ASSERT_FALSE(cbm_file_exists(equal_path)); + ASSERT_FALSE(cbm_file_exists(equal_wal)); + ASSERT_FALSE(cbm_file_exists(equal_shm)); + + char ghost_path[768]; + char ghost_wal[800]; + char ghost_shm[800]; + snprintf(ghost_path, sizeof(ghost_path), "%s/ghost525.db", fixture.cache); + snprintf(ghost_wal, sizeof(ghost_wal), "%s-wal", ghost_path); + snprintf(ghost_shm, sizeof(ghost_shm), "%s-shm", ghost_path); + ASSERT_FALSE(cbm_file_exists(ghost_path)); + char *missing = cbm_mcp_handle_tool( + server, "compare_graphs", "{\"base_project\":\"base525\",\"target_project\":\"ghost525\"}"); + ASSERT_NOT_NULL(missing); + ASSERT_NOT_NULL(strstr(missing, "project_not_indexed")); + ASSERT_FALSE(cbm_file_exists(ghost_path)); + ASSERT_FALSE(cbm_file_exists(ghost_wal)); + ASSERT_FALSE(cbm_file_exists(ghost_shm)); + free(missing); + + char *scan_limited = compare_graphs_call(server, ",\"scan_limit\":1"); + ASSERT_NOT_NULL(scan_limited); + ASSERT_NOT_NULL(strstr(scan_limited, "scan_limit_exceeded")); + char *scan_inner = extract_text_content(scan_limited); + ASSERT_NOT_NULL(scan_inner); + ASSERT_NULL(strstr(scan_inner, "\"nodes\"")); + free(scan_inner); + free(scan_limited); + + char *malformed_rpc = cbm_mcp_server_handle(server, "{"); + ASSERT_NOT_NULL(malformed_rpc); + ASSERT_NOT_NULL(strstr(malformed_rpc, "-32700")); + free(malformed_rpc); + cbm_mcp_server_free(server); + ASSERT_TRUE(compare_graphs_fixture_close(&fixture)); + PASS(); +} + +TEST(tool_compare_graphs_cancel_and_readonly_handles_release_issue525) { + compare_graphs_fixture_t fixture; + ASSERT_TRUE(compare_graphs_fixture_open(&fixture)); + long base_length = 0; + long target_length = 0; + unsigned char *base_before = mcp_read_file_bytes(fixture.base_path, &base_length); + unsigned char *target_before = mcp_read_file_bytes(fixture.target_path, &target_length); + ASSERT_NOT_NULL(base_before); + ASSERT_NOT_NULL(target_before); + char base_wal[800]; + char base_shm[800]; + char target_wal[800]; + char target_shm[800]; + snprintf(base_wal, sizeof(base_wal), "%s-wal", fixture.base_path); + snprintf(base_shm, sizeof(base_shm), "%s-shm", fixture.base_path); + snprintf(target_wal, sizeof(target_wal), "%s-wal", fixture.target_path); + snprintf(target_shm, sizeof(target_shm), "%s-shm", fixture.target_path); + ASSERT_FALSE(cbm_file_exists(base_wal)); + ASSERT_FALSE(cbm_file_exists(base_shm)); + ASSERT_FALSE(cbm_file_exists(target_wal)); + ASSERT_FALSE(cbm_file_exists(target_shm)); + + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(server); + ASSERT_TRUE(cbm_mcp_server_request_scope_begin(server)); + ASSERT_TRUE(cbm_mcp_server_cancel_active(server)); + char *cancelled = compare_graphs_call(server, NULL); + ASSERT_NOT_NULL(cancelled); + ASSERT_NOT_NULL(strstr(cancelled, "\"isError\":true")); + ASSERT_NOT_NULL(strstr(cancelled, "cancelled")); + char *cancelled_inner = extract_text_content(cancelled); + ASSERT_NOT_NULL(cancelled_inner); + ASSERT_NULL(strstr(cancelled_inner, "\"nodes\"")); + free(cancelled_inner); + free(cancelled); + cbm_mcp_server_request_scope_end(server); + + char moved_path[800]; + snprintf(moved_path, sizeof(moved_path), "%s/base525.moved", fixture.cache); + ASSERT_EQ(rename(fixture.base_path, moved_path), 0); + ASSERT_EQ(rename(moved_path, fixture.base_path), 0); + + char *success = compare_graphs_call(server, NULL); + ASSERT_NOT_NULL(success); + ASSERT_NOT_NULL(strstr(success, "\"isError\":false")); + free(success); + ASSERT_EQ(rename(fixture.base_path, moved_path), 0); + ASSERT_EQ(rename(moved_path, fixture.base_path), 0); + ASSERT_TRUE(mcp_file_matches_snapshot(fixture.base_path, base_before, base_length)); + ASSERT_TRUE(mcp_file_matches_snapshot(fixture.target_path, target_before, target_length)); + ASSERT_FALSE(cbm_file_exists(base_wal)); + ASSERT_FALSE(cbm_file_exists(base_shm)); + ASSERT_FALSE(cbm_file_exists(target_wal)); + ASSERT_FALSE(cbm_file_exists(target_shm)); + + free(base_before); + free(target_before); + cbm_mcp_server_free(server); + ASSERT_TRUE(compare_graphs_fixture_close(&fixture)); + PASS(); +} + /* Call get_code_snippet and extract inner text content. * Caller must free returned string. */ static char *call_snippet(cbm_mcp_server_t *srv, const char *args_json) { @@ -11551,6 +12050,11 @@ SUITE(mcp) { RUN_TEST(tool_list_projects_empty); RUN_TEST(tool_get_graph_schema_empty); RUN_TEST(tool_unknown_tool); + RUN_TEST(tool_compare_graphs_registered_issue525); + RUN_TEST(tool_compare_graphs_streams_stable_deltas_issue525); + RUN_TEST(tool_compare_graphs_enforces_encoded_budget_issue525); + RUN_TEST(tool_compare_graphs_validation_and_scan_cap_are_atomic_issue525); + RUN_TEST(tool_compare_graphs_cancel_and_readonly_handles_release_issue525); RUN_TEST(tool_search_graph_basic); RUN_TEST(tool_search_graph_semantic_only_skips_structural_results_issue1295); RUN_TEST(tool_trace_totals_respect_test_filter); From 079d8dcb891ee2a0d45345124e1207a0c4b1993d Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 28 Aug 2026 19:21:05 +0200 Subject: [PATCH 13/31] fix: harden graph comparison identities Signed-off-by: Martin Vogel --- src/mcp/mcp.c | 23 ++++- src/store/store.c | 71 ++++++++++++-- src/store/store.h | 15 ++- tests/test_mcp.c | 243 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 336 insertions(+), 16 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 616bbf6a1..e7077c12f 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -2918,13 +2918,26 @@ static bool compare_parse_arguments(const char *args, char **base_project, char return true; } +static char *sanitize_utf8_lossy(const char *s); + +static bool compare_add_identity_string(yyjson_mut_doc *doc, yyjson_mut_val *object, + const char *key, const char *value) { + char *sanitized = sanitize_utf8_lossy(value); + if (!sanitized) { + return false; + } + bool ok = yyjson_mut_obj_add_strcpy(doc, object, key, sanitized); + free(sanitized); + return ok; +} + static yyjson_mut_val *compare_node_json(yyjson_mut_doc *doc, const cbm_graph_node_identity_t *node) { yyjson_mut_val *object = yyjson_mut_obj(doc); if (!object || - !yyjson_mut_obj_add_strcpy(doc, object, "qualified_name", node->qualified_name) || - !yyjson_mut_obj_add_strcpy(doc, object, "label", node->label) || - !yyjson_mut_obj_add_strcpy(doc, object, "file_path", node->file_path)) { + !compare_add_identity_string(doc, object, "qualified_name", node->qualified_name) || + !compare_add_identity_string(doc, object, "label", node->label) || + !compare_add_identity_string(doc, object, "file_path", node->file_path)) { return NULL; } return object; @@ -2992,8 +3005,8 @@ static bool compare_edge_callback(void *context, bool added, yyjson_mut_val *target = doc ? compare_node_json(doc, &edge->target) : NULL; bool ok = item && source && target && yyjson_mut_obj_add_val(doc, item, "source", source) && yyjson_mut_obj_add_val(doc, item, "target", target) && - yyjson_mut_obj_add_strcpy(doc, item, "type", edge->type) && - yyjson_mut_obj_add_strcpy(doc, item, "local_name_gen", edge->local_name_gen); + compare_add_identity_string(doc, item, "type", edge->type) && + compare_add_identity_string(doc, item, "local_name_gen", edge->local_name_gen); if (ok) { yyjson_mut_doc_set_root(doc, item); } diff --git a/src/store/store.c b/src/store/store.c index 0f2a5ead6..b991bb58e 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -1227,7 +1227,36 @@ typedef struct { bool has_row; } graph_edge_cursor_t; +#ifdef CBM_ENABLE_TEST_SEAMS +static atomic_int graph_compare_test_bind_countdown = ATOMIC_VAR_INIT(CBM_NOT_FOUND); +static atomic_int graph_compare_test_cancel_countdown = ATOMIC_VAR_INIT(CBM_NOT_FOUND); + +void cbm_store_compare_test_fail_bind_after(int successful_binds) { + atomic_store(&graph_compare_test_bind_countdown, successful_binds); +} + +void cbm_store_compare_test_cancel_after(int successful_checks) { + atomic_store(&graph_compare_test_cancel_countdown, successful_checks); +} + +static bool graph_compare_test_countdown_fires(atomic_int *countdown) { + int remaining = atomic_load(countdown); + while (remaining >= 0) { + int next = remaining == 0 ? CBM_NOT_FOUND : remaining - 1; + if (atomic_compare_exchange_weak(countdown, &remaining, next)) { + return remaining == 0; + } + } + return false; +} +#endif + static bool graph_compare_is_cancelled(const graph_compare_progress_t *progress) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (graph_compare_test_countdown_fires(&graph_compare_test_cancel_countdown)) { + return true; + } +#endif return progress && progress->cancel && progress->cancel(progress->context); } @@ -1235,6 +1264,15 @@ static int graph_compare_progress(void *context) { return graph_compare_is_cancelled((const graph_compare_progress_t *)context) ? 1 : 0; } +static int graph_compare_bind_text(sqlite3_stmt *stmt, int column, const char *value) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (graph_compare_test_countdown_fires(&graph_compare_test_bind_countdown)) { + return SQLITE_NOMEM; + } +#endif + return bind_text(stmt, column, value); +} + /* SQLite's BINARY collation compares the unsigned bytes of the UTF-8 text. * Keep the merge comparator byte-for-byte equivalent so neither cursor can * be advanced past a row that SQLite considers distinct. */ @@ -1282,7 +1320,11 @@ static int graph_count_rows(cbm_store_t *store, const char *sql, const char *pro store_set_error_sqlite(store, "compare count prepare"); return CBM_STORE_ERR; } - bind_text(stmt, SKIP_ONE, project); + if (graph_compare_bind_text(stmt, SKIP_ONE, project) != SQLITE_OK) { + store_set_error(store, "compare count bind failed"); + sqlite3_finalize(stmt); + return CBM_STORE_ERR; + } int step_rc = sqlite3_step(stmt); if (step_rc != SQLITE_ROW) { store_set_error_sqlite(store, "compare count"); @@ -1388,35 +1430,46 @@ static int graph_edge_cursor_step(cbm_store_t *store, graph_edge_cursor_t *curso static int graph_prepare_node_cursor(cbm_store_t *store, const char *project, graph_node_cursor_t *cursor) { static const char sql[] = - "SELECT qualified_name,label,coalesce(file_path,'') FROM nodes WHERE project=?1 " + "SELECT qualified_name,label,replace(coalesce(file_path,''),'\\','/') " + "FROM nodes WHERE project=?1 " "ORDER BY qualified_name COLLATE BINARY,label COLLATE BINARY," - "coalesce(file_path,'') COLLATE BINARY;"; + "replace(coalesce(file_path,''),'\\','/') COLLATE BINARY;"; if (sqlite3_prepare_v2(store->db, sql, CBM_NOT_FOUND, &cursor->stmt, NULL) != SQLITE_OK) { store_set_error_sqlite(store, "compare node prepare"); return CBM_STORE_ERR; } - bind_text(cursor->stmt, SKIP_ONE, project); + if (graph_compare_bind_text(cursor->stmt, SKIP_ONE, project) != SQLITE_OK) { + store_set_error(store, "compare node bind failed"); + sqlite3_finalize(cursor->stmt); + cursor->stmt = NULL; + return CBM_STORE_ERR; + } return graph_node_cursor_step(store, cursor); } static int graph_prepare_edge_cursor(cbm_store_t *store, const char *project, graph_edge_cursor_t *cursor) { static const char sql[] = - "SELECT sn.qualified_name,sn.label,coalesce(sn.file_path,'')," - "tn.qualified_name,tn.label,coalesce(tn.file_path,''),e.type," + "SELECT sn.qualified_name,sn.label,replace(coalesce(sn.file_path,''),'\\','/')," + "tn.qualified_name,tn.label,replace(coalesce(tn.file_path,''),'\\','/'),e.type," "coalesce(e.local_name_gen,'') FROM edges e " "JOIN nodes sn ON sn.id=e.source_id JOIN nodes tn ON tn.id=e.target_id " "WHERE e.project=?1 ORDER BY " "sn.qualified_name COLLATE BINARY,sn.label COLLATE BINARY," - "coalesce(sn.file_path,'') COLLATE BINARY," + "replace(coalesce(sn.file_path,''),'\\','/') COLLATE BINARY," "tn.qualified_name COLLATE BINARY,tn.label COLLATE BINARY," - "coalesce(tn.file_path,'') COLLATE BINARY," + "replace(coalesce(tn.file_path,''),'\\','/') COLLATE BINARY," "e.type COLLATE BINARY,coalesce(e.local_name_gen,'') COLLATE BINARY;"; if (sqlite3_prepare_v2(store->db, sql, CBM_NOT_FOUND, &cursor->stmt, NULL) != SQLITE_OK) { store_set_error_sqlite(store, "compare edge prepare"); return CBM_STORE_ERR; } - bind_text(cursor->stmt, SKIP_ONE, project); + if (graph_compare_bind_text(cursor->stmt, SKIP_ONE, project) != SQLITE_OK) { + store_set_error(store, "compare edge bind failed"); + sqlite3_finalize(cursor->stmt); + cursor->stmt = NULL; + return CBM_STORE_ERR; + } return graph_edge_cursor_step(store, cursor); } diff --git a/src/store/store.h b/src/store/store.h index 43c189792..d9047f28f 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -282,9 +282,12 @@ typedef bool (*cbm_graph_compare_node_fn)(void *context, bool added, typedef bool (*cbm_graph_compare_edge_fn)(void *context, bool added, const cbm_graph_edge_identity_t *edge); +#define CBM_GRAPH_COMPARE_GENERATION_SIZE 128 +#define CBM_GRAPH_COMPARE_INDEX_MODE_SIZE 32 + typedef struct { - char generation[128]; - char index_mode[32]; + char generation[CBM_GRAPH_COMPARE_GENERATION_SIZE]; + char index_mode[CBM_GRAPH_COMPARE_INDEX_MODE_SIZE]; int64_t node_count; int64_t edge_count; } cbm_graph_compare_project_t; @@ -308,6 +311,14 @@ int cbm_store_compare_graphs(cbm_store_t *base_store, const char *base_project, cbm_graph_compare_node_fn on_node, cbm_graph_compare_edge_fn on_edge, void *context, cbm_graph_compare_result_t *out); +#ifdef CBM_ENABLE_TEST_SEAMS +/* Deterministic one-shot comparison fault seams. Values count successful + * comparison binds/cancellation checks before the injected event; -1 disables + * the seam. */ +void cbm_store_compare_test_fail_bind_after(int successful_binds); +void cbm_store_compare_test_cancel_after(int successful_checks); +#endif + /* ── Lifecycle ──────────────────────────────────────────────────── */ /* Open an in-memory database (for testing). */ diff --git a/tests/test_mcp.c b/tests/test_mcp.c index fbf59823e..e867ac0e5 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -8247,6 +8247,245 @@ static bool compare_write_single_node_store(const char *path, const char *projec return ok; } +static bool compare_write_identity_store(const char *path, const char *project, + const char *generation, const char *source_qn, + const char *source_label, const char *source_file, + const char *target_qn, const char *target_label, + const char *target_file, const char *edge_type) { + cbm_store_t *store = cbm_store_open_path(path); + if (!store || + cbm_store_upsert_project(store, project, "/tmp/compare-identity-525") != CBM_STORE_OK) { + cbm_store_close(store); + return false; + } + + bool ok = true; + int64_t source_id = 0; + int64_t target_id = 0; + if (source_qn) { + cbm_node_t source = { + .project = project, + .label = source_label, + .name = source_qn, + .qualified_name = source_qn, + .file_path = source_file, + .properties_json = "{}", + }; + source_id = cbm_store_upsert_node(store, &source); + ok = source_id > 0; + } + if (ok && target_qn) { + cbm_node_t target = { + .project = project, + .label = target_label, + .name = target_qn, + .qualified_name = target_qn, + .file_path = target_file, + .properties_json = "{}", + }; + target_id = cbm_store_upsert_node(store, &target); + ok = target_id > 0; + } + if (ok && edge_type) { + cbm_edge_t edge = { + .project = project, + .source_id = source_id, + .target_id = target_id, + .type = edge_type, + .properties_json = "{}", + }; + ok = cbm_store_insert_edge(store, &edge) > 0; + } + ok = ok && compare_write_coverage(store, project, generation) && + cbm_store_prepare_for_publish(store) == CBM_STORE_OK; + cbm_store_close(store); + return ok; +} + +TEST(tool_compare_graphs_normalizes_legacy_path_separators_issue525) { + compare_graphs_fixture_t fixture; + ASSERT_TRUE(compare_graphs_fixture_open(&fixture)); + char base_path[768]; + char target_path[768]; + snprintf(base_path, sizeof(base_path), "%s/sepbase525.db", fixture.cache); + snprintf(target_path, sizeof(target_path), "%s/septarget525.db", fixture.cache); + ASSERT_TRUE(compare_write_identity_store(base_path, "sepbase525", "sep-base", "pkg.source", + "Function", "src\\same.c", "pkg.target", "Function", + "src\\target.c", "CALLS")); + ASSERT_TRUE(compare_write_identity_store(target_path, "septarget525", "sep-target", + "pkg.source", "Function", "src/same.c", "pkg.target", + "Function", "src/target.c", "CALLS")); + + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(server); + char *response = cbm_mcp_handle_tool( + server, "compare_graphs", + "{\"base_project\":\"sepbase525\",\"target_project\":\"septarget525\"}"); + ASSERT_NOT_NULL(response); + char *inner = extract_text_content(response); + yyjson_doc *doc = inner ? yyjson_read(inner, strlen(inner), 0) : NULL; + ASSERT_NOT_NULL(doc); + yyjson_val *root = yyjson_doc_get_root(doc); + yyjson_val *nodes = yyjson_obj_get(root, "nodes"); + yyjson_val *edges = yyjson_obj_get(root, "edges"); + ASSERT_TRUE(compare_set_has(yyjson_obj_get(nodes, "added"), 0, 0, false, NULL)); + ASSERT_TRUE(compare_set_has(yyjson_obj_get(nodes, "removed"), 0, 0, false, NULL)); + ASSERT_TRUE(compare_set_has(yyjson_obj_get(edges, "added"), 0, 0, false, NULL)); + ASSERT_TRUE(compare_set_has(yyjson_obj_get(edges, "removed"), 0, 0, false, NULL)); + yyjson_doc_free(doc); + free(inner); + free(response); + cbm_mcp_server_free(server); + ASSERT_TRUE(compare_graphs_fixture_close(&fixture)); + PASS(); +} + +TEST(tool_compare_graphs_sanitizes_legacy_invalid_utf8_issue525) { + static const char invalid_qn[] = "pkg.\xFF" + "node"; + static const char invalid_label[] = "\xFE" + "Function"; + static const char invalid_file[] = "src/\x80" + "bad.c"; + static const char invalid_type[] = "\xFF" + "EDGE"; + static const char safe_qn[] = "pkg.sink"; + static const char replacement_qn[] = "pkg.\xEF\xBF\xBD" + "node"; + static const char replacement_label[] = "\xEF\xBF\xBD" + "Function"; + static const char replacement_file[] = "src/\xEF\xBF\xBD" + "bad.c"; + static const char replacement_type[] = "\xEF\xBF\xBD" + "EDGE"; + + compare_graphs_fixture_t fixture; + ASSERT_TRUE(compare_graphs_fixture_open(&fixture)); + char base_path[768]; + char target_path[768]; + snprintf(base_path, sizeof(base_path), "%s/utfbase525.db", fixture.cache); + snprintf(target_path, sizeof(target_path), "%s/utftarget525.db", fixture.cache); + ASSERT_TRUE(compare_write_identity_store(base_path, "utfbase525", "utf-base", NULL, NULL, NULL, + NULL, NULL, NULL, NULL)); + ASSERT_TRUE(compare_write_identity_store(target_path, "utftarget525", "utf-target", invalid_qn, + invalid_label, invalid_file, safe_qn, "Function", + "src/sink.c", invalid_type)); + + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(server); + char *response = cbm_mcp_handle_tool( + server, "compare_graphs", + "{\"base_project\":\"utfbase525\",\"target_project\":\"utftarget525\"}"); + ASSERT_NOT_NULL(response); + yyjson_doc *outer = yyjson_read(response, strlen(response), 0); + ASSERT_NOT_NULL(outer); + yyjson_doc_free(outer); + char *inner = extract_text_content(response); + yyjson_doc *doc = inner ? yyjson_read(inner, strlen(inner), 0) : NULL; + ASSERT_NOT_NULL(doc); + yyjson_val *root = yyjson_doc_get_root(doc); + yyjson_val *items = + yyjson_obj_get(yyjson_obj_get(yyjson_obj_get(root, "nodes"), "added"), "items"); + bool found_node = false; + size_t index, maximum; + yyjson_val *item; + yyjson_arr_foreach(items, index, maximum, item) { + const char *qualified_name = yyjson_get_str(yyjson_obj_get(item, "qualified_name")); + if (qualified_name && strcmp(qualified_name, replacement_qn) == 0) { + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(item, "label")), replacement_label); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(item, "file_path")), replacement_file); + found_node = true; + } + } + ASSERT_TRUE(found_node); + yyjson_val *edge_items = + yyjson_obj_get(yyjson_obj_get(yyjson_obj_get(root, "edges"), "added"), "items"); + ASSERT_EQ(yyjson_arr_size(edge_items), 1); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(edge_items, 0), "type")), + replacement_type); + yyjson_doc_free(doc); + free(inner); + free(response); + cbm_mcp_server_free(server); + ASSERT_TRUE(compare_graphs_fixture_close(&fixture)); + PASS(); +} + +TEST(tool_compare_graphs_bind_failures_are_atomic_issue525) { + compare_graphs_fixture_t fixture; + ASSERT_TRUE(compare_graphs_fixture_open(&fixture)); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(server); + static const int fail_after[] = {0, 4, 6}; + for (size_t index = 0; index < sizeof(fail_after) / sizeof(fail_after[0]); index++) { + cbm_store_compare_test_fail_bind_after(fail_after[index]); + char *response = compare_graphs_call(server, NULL); + ASSERT_NOT_NULL(response); + ASSERT_NOT_NULL(strstr(response, "\"isError\":true")); + ASSERT_NOT_NULL(strstr(response, "query_failed")); + char *inner = extract_text_content(response); + ASSERT_NOT_NULL(inner); + ASSERT_NULL(strstr(inner, "\"nodes\"")); + free(inner); + free(response); + } + cbm_store_compare_test_fail_bind_after(CBM_NOT_FOUND); + cbm_mcp_server_free(server); + ASSERT_TRUE(compare_graphs_fixture_close(&fixture)); + PASS(); +} + +TEST(tool_compare_graphs_midscan_cancel_restores_store_state_issue525) { + compare_graphs_fixture_t fixture; + ASSERT_TRUE(compare_graphs_fixture_open(&fixture)); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(server); + + cbm_store_compare_test_cancel_after(1); + char *cancelled = compare_graphs_call(server, NULL); + ASSERT_NOT_NULL(cancelled); + ASSERT_NOT_NULL(strstr(cancelled, "\"isError\":true")); + ASSERT_NOT_NULL(strstr(cancelled, "cancelled")); + char *inner = extract_text_content(cancelled); + ASSERT_NOT_NULL(inner); + ASSERT_NULL(strstr(inner, "\"nodes\"")); + free(inner); + free(cancelled); + cbm_store_compare_test_cancel_after(CBM_NOT_FOUND); + + cbm_store_t *base_store = cbm_store_open_path_query(fixture.base_path); + cbm_store_t *target_store = cbm_store_open_path_query(fixture.target_path); + ASSERT_NOT_NULL(base_store); + ASSERT_NOT_NULL(target_store); + cbm_graph_compare_result_t result = {0}; + cbm_store_compare_test_cancel_after(1); + ASSERT_EQ(cbm_store_compare_graphs(base_store, "base525", target_store, "target525", 100, NULL, + NULL, NULL, NULL, &result), + CBM_STORE_CANCELLED); + ASSERT_EQ(result.nodes_added_total, 0); + ASSERT_EQ(result.nodes_removed_total, 0); + ASSERT_EQ(result.edges_added_total, 0); + ASSERT_EQ(result.edges_removed_total, 0); + cbm_store_compare_test_cancel_after(CBM_NOT_FOUND); + ASSERT_EQ(cbm_store_compare_graphs(base_store, "base525", target_store, "target525", 100, NULL, + NULL, NULL, NULL, &result), + CBM_STORE_OK); + ASSERT_EQ(result.nodes_added_total, 2); + ASSERT_EQ(result.nodes_removed_total, 2); + ASSERT_EQ(result.edges_added_total, 2); + ASSERT_EQ(result.edges_removed_total, 2); + cbm_store_close(target_store); + cbm_store_close(base_store); + + char *success = compare_graphs_call(server, NULL); + ASSERT_NOT_NULL(success); + ASSERT_NOT_NULL(strstr(success, "\"isError\":false")); + free(success); + cbm_mcp_server_free(server); + ASSERT_TRUE(compare_graphs_fixture_close(&fixture)); + PASS(); +} + TEST(tool_compare_graphs_enforces_encoded_budget_issue525) { compare_graphs_fixture_t fixture; ASSERT_TRUE(compare_graphs_fixture_open(&fixture)); @@ -12052,6 +12291,10 @@ SUITE(mcp) { RUN_TEST(tool_unknown_tool); RUN_TEST(tool_compare_graphs_registered_issue525); RUN_TEST(tool_compare_graphs_streams_stable_deltas_issue525); + RUN_TEST(tool_compare_graphs_normalizes_legacy_path_separators_issue525); + RUN_TEST(tool_compare_graphs_sanitizes_legacy_invalid_utf8_issue525); + RUN_TEST(tool_compare_graphs_bind_failures_are_atomic_issue525); + RUN_TEST(tool_compare_graphs_midscan_cancel_restores_store_state_issue525); RUN_TEST(tool_compare_graphs_enforces_encoded_budget_issue525); RUN_TEST(tool_compare_graphs_validation_and_scan_cap_are_atomic_issue525); RUN_TEST(tool_compare_graphs_cancel_and_readonly_handles_release_issue525); From 7ba26f5216b466c6e939e1d2ac58313cd65ed9a3 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 28 Aug 2026 20:16:22 +0200 Subject: [PATCH 14/31] fix: sanitize graph comparison metadata Signed-off-by: Martin Vogel --- src/mcp/mcp.c | 4 +- src/store/store.c | 40 ++++++++++++++++--- src/store/store.h | 1 + tests/test_mcp.c | 98 +++++++++++++++++++++++++++++++++++++++-------- 4 files changed, 119 insertions(+), 24 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index e7077c12f..ada9e646d 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -3022,8 +3022,8 @@ static yyjson_mut_val *compare_project_json(yyjson_mut_doc *doc, const char *pro const cbm_graph_compare_project_t *metadata) { yyjson_mut_val *object = yyjson_mut_obj(doc); if (!object || !yyjson_mut_obj_add_strcpy(doc, object, "project", project) || - !yyjson_mut_obj_add_strcpy(doc, object, "generation", metadata->generation) || - !yyjson_mut_obj_add_strcpy(doc, object, "index_mode", metadata->index_mode) || + !compare_add_identity_string(doc, object, "generation", metadata->generation) || + !compare_add_identity_string(doc, object, "index_mode", metadata->index_mode) || !yyjson_mut_obj_add_sint(doc, object, "node_count", metadata->node_count) || !yyjson_mut_obj_add_sint(doc, object, "edge_count", metadata->edge_count)) { return NULL; diff --git a/src/store/store.c b/src/store/store.c index b991bb58e..ffb0047d1 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -1213,8 +1213,11 @@ int cbm_store_rollback(cbm_store_t *s) { typedef struct { cbm_graph_compare_cancel_fn cancel; void *context; + bool progress_cancelled; } graph_compare_progress_t; +enum { GRAPH_COMPARE_PROGRESS_INTERVAL = 1000 }; + typedef struct { sqlite3_stmt *stmt; cbm_graph_node_identity_t identity; @@ -1230,6 +1233,7 @@ typedef struct { #ifdef CBM_ENABLE_TEST_SEAMS static atomic_int graph_compare_test_bind_countdown = ATOMIC_VAR_INIT(CBM_NOT_FOUND); static atomic_int graph_compare_test_cancel_countdown = ATOMIC_VAR_INIT(CBM_NOT_FOUND); +static atomic_bool graph_compare_test_progress_cancel = ATOMIC_VAR_INIT(false); void cbm_store_compare_test_fail_bind_after(int successful_binds) { atomic_store(&graph_compare_test_bind_countdown, successful_binds); @@ -1239,6 +1243,10 @@ void cbm_store_compare_test_cancel_after(int successful_checks) { atomic_store(&graph_compare_test_cancel_countdown, successful_checks); } +void cbm_store_compare_test_cancel_from_progress(bool enabled) { + atomic_store(&graph_compare_test_progress_cancel, enabled); +} + static bool graph_compare_test_countdown_fires(atomic_int *countdown) { int remaining = atomic_load(countdown); while (remaining >= 0) { @@ -1261,7 +1269,25 @@ static bool graph_compare_is_cancelled(const graph_compare_progress_t *progress) } static int graph_compare_progress(void *context) { - return graph_compare_is_cancelled((const graph_compare_progress_t *)context) ? 1 : 0; + graph_compare_progress_t *progress = (graph_compare_progress_t *)context; +#ifdef CBM_ENABLE_TEST_SEAMS + if (atomic_load(&graph_compare_test_progress_cancel)) { + progress->progress_cancelled = true; + return true; + } +#endif + bool cancelled = graph_compare_is_cancelled(progress); + progress->progress_cancelled = cancelled; + return cancelled; +} + +static int graph_compare_progress_interval(void) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (atomic_load(&graph_compare_test_progress_cancel)) { + return 1; + } +#endif + return GRAPH_COMPARE_PROGRESS_INTERVAL; } static int graph_compare_bind_text(sqlite3_stmt *stmt, int column, const char *value) { @@ -1594,8 +1620,10 @@ int cbm_store_compare_graphs(cbm_store_t *base_store, const char *base_project, int rc = CBM_STORE_OK; bool base_transaction = false; bool target_transaction = false; - sqlite3_progress_handler(base_store->db, 1000, graph_compare_progress, &progress); - sqlite3_progress_handler(target_store->db, 1000, graph_compare_progress, &progress); + int progress_interval = graph_compare_progress_interval(); + sqlite3_progress_handler(base_store->db, progress_interval, graph_compare_progress, &progress); + sqlite3_progress_handler(target_store->db, progress_interval, graph_compare_progress, + &progress); rc = exec_sql(base_store, "BEGIN;"); if (rc == CBM_STORE_OK) { @@ -1622,10 +1650,12 @@ int cbm_store_compare_graphs(cbm_store_t *base_store, const char *base_project, rc = graph_compare_edges(base_store, base_project, target_store, target_project, &progress, on_edge, out); } - if (rc == CBM_STORE_OK && graph_compare_is_cancelled(&progress)) { + if (rc == CBM_STORE_OK && + (progress.progress_cancelled || graph_compare_is_cancelled(&progress))) { rc = CBM_STORE_CANCELLED; } - if (rc != CBM_STORE_OK && graph_compare_is_cancelled(&progress)) { + if (rc != CBM_STORE_OK && + (progress.progress_cancelled || graph_compare_is_cancelled(&progress))) { rc = CBM_STORE_CANCELLED; } diff --git a/src/store/store.h b/src/store/store.h index d9047f28f..0409071f1 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -317,6 +317,7 @@ int cbm_store_compare_graphs(cbm_store_t *base_store, const char *base_project, * the seam. */ void cbm_store_compare_test_fail_bind_after(int successful_binds); void cbm_store_compare_test_cancel_after(int successful_checks); +void cbm_store_compare_test_cancel_from_progress(bool enabled); #endif /* ── Lifecycle ──────────────────────────────────────────────────── */ diff --git a/tests/test_mcp.c b/tests/test_mcp.c index e867ac0e5..6099c1db2 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -7956,11 +7956,11 @@ typedef struct { const char *file_path; } compare_node_spec_t; -static bool compare_write_coverage(cbm_store_t *store, const char *project, - const char *generation) { +static bool compare_write_coverage_mode(cbm_store_t *store, const char *project, + const char *generation, const char *index_mode) { cbm_coverage_meta_t meta = { .generation = generation, - .index_mode = "full", + .index_mode = index_mode, .recorded_at = "2026-08-28T00:00:00Z", .recording_status = "complete", .ignored_files_stored = 0, @@ -7971,6 +7971,11 @@ static bool compare_write_coverage(cbm_store_t *store, const char *project, return cbm_store_coverage_replace_ex(store, project, NULL, 0, &meta) == CBM_STORE_OK; } +static bool compare_write_coverage(cbm_store_t *store, const char *project, + const char *generation) { + return compare_write_coverage_mode(store, project, generation, "full"); +} + static bool compare_write_fixture_store(const char *path, const char *project, bool target) { static const compare_node_spec_t base_nodes[] = { {"pkg.common", "Function", "src/common.c"}, {"pkg.base", "Function", "src/base.c"}, @@ -8248,10 +8253,11 @@ static bool compare_write_single_node_store(const char *path, const char *projec } static bool compare_write_identity_store(const char *path, const char *project, - const char *generation, const char *source_qn, - const char *source_label, const char *source_file, - const char *target_qn, const char *target_label, - const char *target_file, const char *edge_type) { + const char *generation, const char *index_mode, + const char *source_qn, const char *source_label, + const char *source_file, const char *target_qn, + const char *target_label, const char *target_file, + const char *edge_type) { cbm_store_t *store = cbm_store_open_path(path); if (!store || cbm_store_upsert_project(store, project, "/tmp/compare-identity-525") != CBM_STORE_OK) { @@ -8296,7 +8302,7 @@ static bool compare_write_identity_store(const char *path, const char *project, }; ok = cbm_store_insert_edge(store, &edge) > 0; } - ok = ok && compare_write_coverage(store, project, generation) && + ok = ok && compare_write_coverage_mode(store, project, generation, index_mode) && cbm_store_prepare_for_publish(store) == CBM_STORE_OK; cbm_store_close(store); return ok; @@ -8309,10 +8315,10 @@ TEST(tool_compare_graphs_normalizes_legacy_path_separators_issue525) { char target_path[768]; snprintf(base_path, sizeof(base_path), "%s/sepbase525.db", fixture.cache); snprintf(target_path, sizeof(target_path), "%s/septarget525.db", fixture.cache); - ASSERT_TRUE(compare_write_identity_store(base_path, "sepbase525", "sep-base", "pkg.source", - "Function", "src\\same.c", "pkg.target", "Function", - "src\\target.c", "CALLS")); - ASSERT_TRUE(compare_write_identity_store(target_path, "septarget525", "sep-target", + ASSERT_TRUE(compare_write_identity_store(base_path, "sepbase525", "sep-base", "full", + "pkg.source", "Function", "src\\same.c", "pkg.target", + "Function", "src\\target.c", "CALLS")); + ASSERT_TRUE(compare_write_identity_store(target_path, "septarget525", "sep-target", "full", "pkg.source", "Function", "src/same.c", "pkg.target", "Function", "src/target.c", "CALLS")); @@ -8358,6 +8364,14 @@ TEST(tool_compare_graphs_sanitizes_legacy_invalid_utf8_issue525) { "bad.c"; static const char replacement_type[] = "\xEF\xBF\xBD" "EDGE"; + static const char invalid_generation[] = "utf-\xFF" + "generation"; + static const char invalid_index_mode[] = "\xFE" + "full"; + static const char replacement_generation[] = "utf-\xEF\xBF\xBD" + "generation"; + static const char replacement_index_mode[] = "\xEF\xBF\xBD" + "full"; compare_graphs_fixture_t fixture; ASSERT_TRUE(compare_graphs_fixture_open(&fixture)); @@ -8365,11 +8379,11 @@ TEST(tool_compare_graphs_sanitizes_legacy_invalid_utf8_issue525) { char target_path[768]; snprintf(base_path, sizeof(base_path), "%s/utfbase525.db", fixture.cache); snprintf(target_path, sizeof(target_path), "%s/utftarget525.db", fixture.cache); - ASSERT_TRUE(compare_write_identity_store(base_path, "utfbase525", "utf-base", NULL, NULL, NULL, - NULL, NULL, NULL, NULL)); - ASSERT_TRUE(compare_write_identity_store(target_path, "utftarget525", "utf-target", invalid_qn, - invalid_label, invalid_file, safe_qn, "Function", - "src/sink.c", invalid_type)); + ASSERT_TRUE(compare_write_identity_store(base_path, "utfbase525", "utf-base", "full", NULL, + NULL, NULL, NULL, NULL, NULL, NULL)); + ASSERT_TRUE(compare_write_identity_store( + target_path, "utftarget525", invalid_generation, invalid_index_mode, invalid_qn, + invalid_label, invalid_file, safe_qn, "Function", "src/sink.c", invalid_type)); cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); ASSERT_NOT_NULL(server); @@ -8384,6 +8398,9 @@ TEST(tool_compare_graphs_sanitizes_legacy_invalid_utf8_issue525) { yyjson_doc *doc = inner ? yyjson_read(inner, strlen(inner), 0) : NULL; ASSERT_NOT_NULL(doc); yyjson_val *root = yyjson_doc_get_root(doc); + yyjson_val *target = yyjson_obj_get(root, "target"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(target, "generation")), replacement_generation); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(target, "index_mode")), replacement_index_mode); yyjson_val *items = yyjson_obj_get(yyjson_obj_get(yyjson_obj_get(root, "nodes"), "added"), "items"); bool found_node = false; @@ -8486,6 +8503,52 @@ TEST(tool_compare_graphs_midscan_cancel_restores_store_state_issue525) { PASS(); } +TEST(tool_compare_graphs_progress_cancel_clears_handler_issue525) { + compare_graphs_fixture_t fixture; + ASSERT_TRUE(compare_graphs_fixture_open(&fixture)); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(server); + + cbm_store_compare_test_cancel_from_progress(true); + char *cancelled = compare_graphs_call(server, NULL); + ASSERT_NOT_NULL(cancelled); + ASSERT_NOT_NULL(strstr(cancelled, "\"isError\":true")); + ASSERT_NOT_NULL(strstr(cancelled, "cancelled")); + char *inner = extract_text_content(cancelled); + ASSERT_NOT_NULL(inner); + ASSERT_NULL(strstr(inner, "\"nodes\"")); + free(inner); + free(cancelled); + + cbm_store_t *base_store = cbm_store_open_path_query(fixture.base_path); + cbm_store_t *target_store = cbm_store_open_path_query(fixture.target_path); + ASSERT_NOT_NULL(base_store); + ASSERT_NOT_NULL(target_store); + cbm_graph_compare_result_t result = {0}; + ASSERT_EQ(cbm_store_compare_graphs(base_store, "base525", target_store, "target525", 100, NULL, + NULL, NULL, NULL, &result), + CBM_STORE_CANCELLED); + ASSERT_EQ(result.nodes_added_total, 0); + ASSERT_EQ(result.nodes_removed_total, 0); + ASSERT_EQ(result.edges_added_total, 0); + ASSERT_EQ(result.edges_removed_total, 0); + + static const char expensive_query[] = + "WITH RECURSIVE sequence(value) AS (VALUES(0) UNION ALL " + "SELECT value+1 FROM sequence WHERE value<5000) SELECT sum(value) FROM sequence;"; + int base_query_rc = cbm_store_exec(base_store, expensive_query); + int target_query_rc = cbm_store_exec(target_store, expensive_query); + cbm_store_compare_test_cancel_from_progress(false); + cbm_store_close(target_store); + cbm_store_close(base_store); + ASSERT_EQ(base_query_rc, CBM_STORE_OK); + ASSERT_EQ(target_query_rc, CBM_STORE_OK); + + cbm_mcp_server_free(server); + ASSERT_TRUE(compare_graphs_fixture_close(&fixture)); + PASS(); +} + TEST(tool_compare_graphs_enforces_encoded_budget_issue525) { compare_graphs_fixture_t fixture; ASSERT_TRUE(compare_graphs_fixture_open(&fixture)); @@ -12295,6 +12358,7 @@ SUITE(mcp) { RUN_TEST(tool_compare_graphs_sanitizes_legacy_invalid_utf8_issue525); RUN_TEST(tool_compare_graphs_bind_failures_are_atomic_issue525); RUN_TEST(tool_compare_graphs_midscan_cancel_restores_store_state_issue525); + RUN_TEST(tool_compare_graphs_progress_cancel_clears_handler_issue525); RUN_TEST(tool_compare_graphs_enforces_encoded_budget_issue525); RUN_TEST(tool_compare_graphs_validation_and_scan_cap_are_atomic_issue525); RUN_TEST(tool_compare_graphs_cancel_and_readonly_handles_release_issue525); From 706bb2ce81da399f88b6d010d6c28272434aa9f5 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 28 Aug 2026 21:30:06 +0200 Subject: [PATCH 15/31] feat(lang): add Chialisp support with a first-party tree-sitter grammar Chialisp (.clsp/.clib/.clinc) is the Chia smart-coin language. The only public grammar, Quexington/tree-sitter-chialisp, cannot parse it: on the five reference files from chia-blockchain@main it produced 3-34 ERROR nodes each. Three defects account for all of them, and each desynchronised the rest of the file rather than losing one node: * comments were `/;.*\r\n/` -- CRLF-only, while real files are LF; * `(include foo.clib)` was rejected because of the dot in the filename; * `(defconstant NAME )` accepted only a primitive after the name. So the grammar is written here instead, in tools/tree-sitter-chialisp/, and files as first-party: its vendored LICENSE is the project's own, byte-identical to the repository root, carrying no third-party copyright. It is deliberately GENERIC -- source_file / list / symbol / string / number / hex / dot / comment, modelling the clvm_tools reader rather than the Chialisp form vocabulary. `mod`, `defun`, `defconstant` and `include` are ordinary head symbols; which lists are definitions is decided in extract_defs.c. A dialect that adds a form (cl-21 -> cl-23 -> cl-26 each added several) therefore needs no regenerated parser. Acceptance: all five reference files parse with zero ERROR and zero MISSING nodes; the three defects above are pinned as tests. Extraction, and the four decisions in it: * `Constant` is admitted to the cross-file name registry (cbm_label_is_registry_symbol). Chialisp's condition codes live as `(defconstant ...)` in a shared .clib and are referenced by name from every puzzle; labelled but unregistered they are seeded nowhere, and roughly half of a Chialisp graph would be present as nodes yet never the target of an edge. All three registry-seeding sites route through that one predicate, so they cannot diverge. No other language emits `Constant`, so the blast radius is exactly Chialisp. Pinned by a two-file graph test that goes red when the label is removed from the predicate. * `export` / `namespace` are NOT definition heads. `(export foo)` names a function already defined in the same file, so treating it as a def head mints a second node for one symbol. They stay in the not-a-call filter. * A parameter list is a `list` too, but it binds rather than invokes. Without that gate `(defun check_conditions (HEIGHTLOCK conditions) ...)` mints a phantom call to HEIGHTLOCK, and real Chialisp is mostly such definitions. * `(include *standard-cl-26*)` is a dialect directive, not a file dependency; recording it would invent a dependency on a file that does not exist. Three helpers that the defs, calls and unified walks all need -- the quote-data gate, comment-skipping named-child access, and the Chialisp def-head set -- live in helpers.c rather than being copied per translation unit. They previously existed as byte-identical duplicates that had already drifted: extract_defs.c skipped comments when reading a def head while its mirror in extract_unified.c did not, so a comment between a head and its name desynchronised defs from call scope. Both now use the same accessor. Ledger: 161 -> 162 grammars across the seven published surfaces, ABI-14 86 -> 87, first-party 13 -> 14, CBM_LANG_COUNT 164 -> 165. Distilled from PR #919 by darkverbito, whose extraction logic -- the CLVM not-a-call vocabulary, the quoted-data gate, the filename-derived `mod` name and the include/embed import handling -- is kept and is the substance of this change. The grammar that PR vendored is not used. Co-authored-by: darkverbito <8049985+darkverbito@users.noreply.github.com> Signed-off-by: Martin Vogel --- .gitattributes | 4 + README.md | 12 +- THIRD_PARTY.md | 12 +- docs/index.html | 20 +- docs/llms.txt | 2 +- internal/cbm/cbm.h | 5 +- internal/cbm/extract_calls.c | 105 ++- internal/cbm/extract_defs.c | 92 ++- internal/cbm/extract_imports.c | 41 + internal/cbm/extract_unified.c | 42 +- internal/cbm/extract_usages.c | 26 +- internal/cbm/grammar_chialisp.c | 3 + internal/cbm/helpers.c | 73 +- internal/cbm/helpers.h | 30 + internal/cbm/lang_specs.c | 14 + internal/cbm/vendored/grammars/MANIFEST.md | 8 +- .../cbm/vendored/grammars/chialisp/LICENSE | 22 + .../cbm/vendored/grammars/chialisp/parser.c | 485 ++++++++++++ .../grammars/chialisp/tree_sitter/alloc.h | 54 ++ .../grammars/chialisp/tree_sitter/array.h | 291 +++++++ .../grammars/chialisp/tree_sitter/parser.h | 286 +++++++ pkg/chocolatey/codebase-memory-mcp.nuspec | 2 +- pkg/npm/README.md | 4 +- scripts/audit-license-provenance.py | 6 +- scripts/package-release.sh | 2 +- scripts/vendored-checksums.txt | 7 +- server.json | 2 +- src/discover/language.c | 6 + src/discover/userconfig.c | 1 + tests/repro/repro_call_argument_matrix_b.c | 23 +- tests/repro/repro_call_node_manifest.c | 9 +- tests/repro/repro_invariant_breadth.c | 710 +++++++++--------- tests/repro/repro_language_registry.c | 7 +- tests/test_extraction.c | 228 +++++- tests/test_grammar_labels.c | 1 + tests/test_grammar_regression.c | 18 +- tests/test_lang_contract.c | 79 +- tests/test_language.c | 10 + tests/test_language_count_contract.sh | 2 +- tools/tree-sitter-chialisp/grammar.js | 77 ++ tools/tree-sitter-chialisp/package.json | 21 + tools/tree-sitter-chialisp/src/grammar.json | 185 +++++ .../tree-sitter-chialisp/src/node-types.json | 106 +++ tools/tree-sitter-chialisp/src/parser.c | 485 ++++++++++++ .../src/tree_sitter/alloc.h | 54 ++ .../src/tree_sitter/array.h | 291 +++++++ .../src/tree_sitter/parser.h | 286 +++++++ .../test/corpus/atoms.txt | 119 +++ .../test/corpus/comments.txt | 83 ++ .../test/corpus/forms.txt | 136 ++++ tools/tree-sitter-chialisp/tree-sitter.json | 16 + 51 files changed, 4114 insertions(+), 489 deletions(-) create mode 100644 internal/cbm/grammar_chialisp.c create mode 100644 internal/cbm/vendored/grammars/chialisp/LICENSE create mode 100644 internal/cbm/vendored/grammars/chialisp/parser.c create mode 100644 internal/cbm/vendored/grammars/chialisp/tree_sitter/alloc.h create mode 100644 internal/cbm/vendored/grammars/chialisp/tree_sitter/array.h create mode 100644 internal/cbm/vendored/grammars/chialisp/tree_sitter/parser.h create mode 100644 tools/tree-sitter-chialisp/grammar.js create mode 100644 tools/tree-sitter-chialisp/package.json create mode 100644 tools/tree-sitter-chialisp/src/grammar.json create mode 100644 tools/tree-sitter-chialisp/src/node-types.json create mode 100644 tools/tree-sitter-chialisp/src/parser.c create mode 100644 tools/tree-sitter-chialisp/src/tree_sitter/alloc.h create mode 100644 tools/tree-sitter-chialisp/src/tree_sitter/array.h create mode 100644 tools/tree-sitter-chialisp/src/tree_sitter/parser.h create mode 100644 tools/tree-sitter-chialisp/test/corpus/atoms.txt create mode 100644 tools/tree-sitter-chialisp/test/corpus/comments.txt create mode 100644 tools/tree-sitter-chialisp/test/corpus/forms.txt create mode 100644 tools/tree-sitter-chialisp/tree-sitter.json diff --git a/.gitattributes b/.gitattributes index a76ea0bd6..1b2d4fb75 100644 --- a/.gitattributes +++ b/.gitattributes @@ -11,3 +11,7 @@ internal/cbm/vendored/grammars/**/scanner.c linguist-generated=true diff=false # the git hooks are extensionless and need explicit entries. (#1272) scripts/git-hooks/commit-msg text eol=lf scripts/hooks/pre-commit text eol=lf + +# Chialisp grammar corpus: one case asserts CRLF comment termination, so this +# file must keep its exact bytes on every platform (no eol conversion). (#919) +tools/tree-sitter-chialisp/test/corpus/comments.txt -text diff --git a/README.md b/README.md index 4744d135c..bc1d0b0d2 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![License](https://img.shields.io/badge/license-MIT-green)](LICENSE) [![CI](https://img.shields.io/github/actions/workflow/status/DeusData/codebase-memory-mcp/dry-run.yml?label=CI)](https://github.com/DeusData/codebase-memory-mcp/actions/workflows/dry-run.yml) [![Tests](https://img.shields.io/badge/tests-6768_passing-brightgreen)](https://github.com/DeusData/codebase-memory-mcp) -[![Languages](https://img.shields.io/badge/languages-161-orange)](https://github.com/DeusData/codebase-memory-mcp) +[![Languages](https://img.shields.io/badge/languages-162-orange)](https://github.com/DeusData/codebase-memory-mcp) [![Hybrid LSP](https://img.shields.io/badge/Hybrid_LSP-10_languages-blue)](#hybrid-lsp) [![Agents](https://img.shields.io/badge/agent_surfaces-45-purple)](https://github.com/DeusData/codebase-memory-mcp) [![Pure C](https://img.shields.io/badge/pure_C-no_language_runtime-blue)](https://github.com/DeusData/codebase-memory-mcp) @@ -16,7 +16,7 @@ **The fastest and most efficient code intelligence engine for AI coding agents.** Full-indexes an average repository in milliseconds, the Linux kernel (28M LOC, 75K files) in 3 minutes. Answers structural queries in under 1ms. Ships as a native executable with a small verified runtime-asset set for macOS, Linux, and Windows — download, run `install`, done. -High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across all 161 languages, enhanced with [**Hybrid LSP** semantic type resolution](#hybrid-lsp) for Python, TypeScript / JavaScript / JSX / TSX, PHP, C#, Go, C, C++, Java, Kotlin, Rust, and Perl — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 15 MCP tools. No language runtime, hosted service, or API key. Plug and play across 45 supported automatic/conditional client surfaces. +High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across all 162 languages, enhanced with [**Hybrid LSP** semantic type resolution](#hybrid-lsp) for Python, TypeScript / JavaScript / JSX / TSX, PHP, C#, Go, C, C++, Java, Kotlin, Rust, and Perl — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 15 MCP tools. No language runtime, hosted service, or API key. Plug and play across 45 supported automatic/conditional client surfaces. > **Research** — The design and benchmarks behind this project are described in the preprint [*Codebase-Memory: Tree-Sitter-Based Knowledge Graphs for LLM Code Exploration via MCP*](https://arxiv.org/abs/2603.27277) (arXiv:2603.27277). Evaluated across 31 real-world repositories: 83% answer quality, 10× fewer tokens, 2.1× fewer tool calls vs. file-by-file exploration. @@ -32,7 +32,7 @@ High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-si - **Extreme indexing speed** — Linux kernel (28M LOC, 75K files) in 3 minutes. RAM-first pipeline: LZ4 compression, in-memory SQLite, fused Aho-Corasick pattern matching. Memory released after indexing. - **Plug and play** — native executable plus authenticated release-owned assets for macOS (arm64/amd64), Linux (arm64/amd64), and Windows (amd64). The native install needs no Docker, language runtime, or API keys. Download → `install` → restart agent → done. -- **161 languages** — vendored tree-sitter grammars compiled into the binary. Nothing to install, nothing that breaks. +- **162 languages** — vendored tree-sitter grammars compiled into the binary. Nothing to install, nothing that breaks. - **120x fewer tokens** — 5 structural queries: ~3,400 tokens vs ~412,000 via file-by-file search. One graph query replaces dozens of grep/read cycles. - **45 supported automatic/conditional client surfaces** — `install` configures detected clients and safely activates conditional clients only when their documented platform, marker, or explicit existing config path is present. See [Multi-Agent Support](#multi-agent-support) for the complete matrix and manual/UI-only boundaries. - **Built-in graph visualization** — 3D interactive UI at `localhost:9749`, served from the binary itself. @@ -796,14 +796,14 @@ codebase-memory-mcp ships a **lightweight C implementation of language type-reso **Two-layer architecture:** -1. **Tree-sitter pass** — fast, syntactic, runs for every one of the 161 languages. Extracts definitions, calls, imports. +1. **Tree-sitter pass** — fast, syntactic, runs for every one of the 162 languages. Extracts definitions, calls, imports. 2. **Hybrid LSP pass** — type-aware, runs above the tree-sitter pass per-language. Refines call edges using the import graph plus a per-file or pre-built cross-file definition registry. Languages without a Hybrid LSP pass yet fall back to textual resolution, so you always get *some* answer. The result is a knowledge graph accurate enough to drive `trace_path` across packages, inheritance hierarchies, and stdlib calls — without paying for a language server process per project. ## Language Support -161 languages, all parsed via vendored tree-sitter grammars compiled into the binary. Benchmarked against 64 real open-source repositories (78 to 49K nodes): +162 languages, all parsed via vendored tree-sitter grammars compiled into the binary. Benchmarked against 64 real open-source repositories (78 to 49K nodes): | Tier | Score | Languages | |------|-------|-----------| @@ -829,7 +829,7 @@ src/ traces/ Runtime trace ingestion ui/ Local HTTP server + verified external 3D-UI asset pack foundation/ Platform abstractions (threads, filesystem, logging, memory) -internal/cbm/ Vendored tree-sitter grammars (161 languages) + AST extraction engine +internal/cbm/ Vendored tree-sitter grammars (162 languages) + AST extraction engine ``` ## Security diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md index d148f9f65..fdcb0d5e4 100644 --- a/THIRD_PARTY.md +++ b/THIRD_PARTY.md @@ -52,9 +52,15 @@ License summary: - `clojure` ([sogaiu/tree-sitter-clojure](https://github.com/sogaiu/tree-sitter-clojure)) is **CC0-1.0**; `fennel` is **CC0-1.0**; `jinja2` and `just` are **Apache-2.0**; `pine` is **ISC** (declared by its upstream). -- The grammars authored in-house for this project (`cobol`, `form`, `janet`, - `magma`, `protobuf`, `wolfram`) are **MIT** under the project's own license, - (c) DeusData. Seven further grammars (`arkts`, `assembly`, `cfml`, +- The grammars authored in-house for this project (`chialisp`, `cobol`, `form`, + `janet`, `magma`, `protobuf`, `wolfram`) are **MIT** under the project's own + license, (c) DeusData. Each ships the repository's own LICENSE, byte-identical + to the root copy; they carry no third-party copyright because there is no + third party. `chialisp` is a generic s-expression grammar for the Chia + smart-coin language, written for this project because no usable public + grammar exists; its source and corpus tests live in + `tools/tree-sitter-chialisp/`. +- Seven further grammars (`arkts`, `assembly`, `cfml`, `cfscript`, `dotenv`, `pine`, `qml`) are self-maintained forks that retain their original upstream authors' licenses — see the manifest for per-grammar provenance. `arkts` is a first-party derivative of diff --git a/docs/index.html b/docs/index.html index ac5ecb81f..56496d5d8 100644 --- a/docs/index.html +++ b/docs/index.html @@ -4,7 +4,7 @@ codebase-memory-mcp — Code Intelligence Knowledge Graph for AI Coding Agents - + @@ -12,7 +12,7 @@ - + @@ -24,7 +24,7 @@ - + @@ -41,7 +41,7 @@ "applicationSubCategory": "Model Context Protocol (MCP) server", "operatingSystem": "macOS, Linux, Windows", "softwareVersion": "0.8.1", - "description": "An open-source MCP server that indexes a codebase into a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links, so AI coding agents answer structural questions with roughly 120x fewer tokens than file-by-file search. Parses 161 languages via tree-sitter with Hybrid LSP semantic type resolution for 9 language families. Ships as a native executable with authenticated release-owned assets; the native install needs no language runtime.", + "description": "An open-source MCP server that indexes a codebase into a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links, so AI coding agents answer structural questions with roughly 120x fewer tokens than file-by-file search. Parses 162 languages via tree-sitter with Hybrid LSP semantic type resolution for 9 language families. Ships as a native executable with authenticated release-owned assets; the native install needs no language runtime.", "url": "https://deusdata.github.io/codebase-memory-mcp/", "downloadUrl": "https://github.com/DeusData/codebase-memory-mcp/releases/latest", "codeRepository": "https://github.com/DeusData/codebase-memory-mcp", @@ -125,7 +125,7 @@ "name": "What is codebase-memory-mcp?", "acceptedAnswer": { "@type": "Answer", - "text": "codebase-memory-mcp is an open-source Model Context Protocol (MCP) server that indexes a codebase into a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. AI coding agents query that graph instead of reading files one by one, answering structural questions with roughly 120x fewer tokens. It parses 161 languages and ships as a native executable with authenticated release-owned assets; the native install needs no language runtime." + "text": "codebase-memory-mcp is an open-source Model Context Protocol (MCP) server that indexes a codebase into a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. AI coding agents query that graph instead of reading files one by one, answering structural questions with roughly 120x fewer tokens. It parses 162 languages and ships as a native executable with authenticated release-owned assets; the native install needs no language runtime." } }, { @@ -141,7 +141,7 @@ "name": "Which programming languages does codebase-memory-mcp support?", "acceptedAnswer": { "@type": "Answer", - "text": "It supports 161 languages through vendored tree-sitter grammars compiled into the binary, including Python, Go, JavaScript, TypeScript, Rust, Java, C, C++, C#, PHP, Ruby, Kotlin, Swift, and many more. Nine language families — Python, TypeScript/JavaScript, PHP, C#, Go, C/C++, Java, Kotlin, and Rust — additionally get Hybrid LSP semantic type resolution." + "text": "It supports 162 languages through vendored tree-sitter grammars compiled into the binary, including Python, Go, JavaScript, TypeScript, Rust, Java, C, C++, C#, PHP, Ruby, Kotlin, Swift, and many more. Nine language families — Python, TypeScript/JavaScript, PHP, C#, Go, C/C++, Java, Kotlin, and Rust — additionally get Hybrid LSP semantic type resolution." } }, { @@ -444,7 +444,7 @@ The fastest, most efficient code intelligence engine for AI coding agents. It indexes any repository into a persistent knowledge graph — full-indexing an average repo in seconds and the Linux kernel in 3 minutes — so your agent answers structural questions with ~120x fewer tokens. - Tree-sitter parsing across 161 languages, Hybrid LSP type resolution, native executable with verified assets. + Tree-sitter parsing across 162 languages, Hybrid LSP type resolution, native executable with verified assets.

~120x
fewer tokens
@@ -483,7 +483,7 @@ Protocol (MCP) server that indexes a codebase into a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. Instead of reading files one at a time, an AI coding agent queries the graph — answering structural questions with roughly 120x fewer - tokens. It parses 161 languages and ships as a native executable with authenticated release-owned assets; + tokens. It parses 162 languages and ships as a native executable with authenticated release-owned assets; the native install needs no language runtime.

@@ -606,7 +606,7 @@

- The two-layer pipeline runs a fast syntactic tree-sitter pass for every one of the 161 languages, + The two-layer pipeline runs a fast syntactic tree-sitter pass for every one of the 162 languages, then a type-aware Hybrid LSP pass on top for the families above. Languages without a Hybrid LSP pass yet fall back to textual resolution, so you always get an answer.

@@ -699,7 +699,7 @@

Features

-

161 languages

+

162 languages

Python, Go, JS, TS, TSX, Rust, Java, C++, C#, C, PHP, Ruby, Kotlin, Scala, Zig, Elixir, Haskell, OCaml, Swift, Dart, Lean 4, and many more via vendored tree-sitter grammars compiled into the binary.

diff --git a/docs/llms.txt b/docs/llms.txt index 270d4875a..f3c23970b 100644 --- a/docs/llms.txt +++ b/docs/llms.txt @@ -1,6 +1,6 @@ # codebase-memory-mcp -> An open-source Model Context Protocol (MCP) server that indexes a codebase into a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. AI coding agents query the graph instead of reading files one by one, answering structural questions with roughly 120x fewer tokens (~3,400 vs ~412,000 across five structural queries). Parses 161 languages via vendored tree-sitter grammars, with Hybrid LSP semantic type resolution for Python, TypeScript/JavaScript, PHP, C#, Go, C/C++, Java, Kotlin, and Rust. It also generates semantic graph edges (SEMANTICALLY_RELATED for vocabulary-mismatch matches, SIMILAR_TO for near-clone detection) and supports semantic vector search via bundled nomic-embed-code embeddings compiled into the native executable — no API key, no Ollama, no Docker. Ships a native executable with an authenticated integration asset; the UI variant adds one content-addressed frontend pack. All processing is local — the embeddings run on-device; there is no embedded LLM and no API key. +> An open-source Model Context Protocol (MCP) server that indexes a codebase into a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. AI coding agents query the graph instead of reading files one by one, answering structural questions with roughly 120x fewer tokens (~3,400 vs ~412,000 across five structural queries). Parses 162 languages via vendored tree-sitter grammars, with Hybrid LSP semantic type resolution for Python, TypeScript/JavaScript, PHP, C#, Go, C/C++, Java, Kotlin, and Rust. It also generates semantic graph edges (SEMANTICALLY_RELATED for vocabulary-mismatch matches, SIMILAR_TO for near-clone detection) and supports semantic vector search via bundled nomic-embed-code embeddings compiled into the native executable — no API key, no Ollama, no Docker. Ships a native executable with an authenticated integration asset; the UI variant adds one content-addressed frontend pack. All processing is local — the embeddings run on-device; there is no embedded LLM and no API key. ## Key facts diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index ac187406e..7076015ac 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -174,8 +174,9 @@ typedef enum { CBM_LANG_OBJECTSCRIPT_UDL, // InterSystems ObjectScript UDL (.cls class files) CBM_LANG_OBJECTSCRIPT_ROUTINE, // InterSystems ObjectScript routine (.mac/.int/.rtn/.inc) CBM_LANG_OBJECTSCRIPT_EXPORT, // InterSystems Studio Export XML () - CBM_LANG_ARKTS, // ArkTS (HarmonyOS/OpenHarmony .ets — TypeScript superset + ArkUI) - CBM_LANG_PLSQL, // Oracle PL/SQL + CBM_LANG_ARKTS, // ArkTS (HarmonyOS/OpenHarmony .ets — TypeScript superset + ArkUI) + CBM_LANG_PLSQL, // Oracle PL/SQL + CBM_LANG_CHIALISP, // Chialisp (.clsp/.clib/.clinc — Chia smart-coin s-expression language) CBM_LANG_COUNT } CBMLanguage; diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index abfe21140..58c0c27d6 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -735,7 +735,94 @@ static bool call_node_is_definition_container(CBMLanguage lang, TSNode node, con // Lisp dialects: a call is a list (`list` / `list_lit`) whose head (first named // child) is the function symbol (`symbol` / `sym_lit`). Generic field/first-child // extraction misses it because the head is not an `identifier` node. -static char *extract_lisp_callee(CBMArena *a, TSNode node, const char *source, const char *nk) { +/* Chialisp: a head atom that is a CLVM primitive/opcode, or a Chialisp + * syntax/binding/def keyword, is NOT a call — emit no CALLS edge. Sourced from + * the clvm_tools_rs keyword set and the clvm_rs operator set. The def and + * include heads are here too so `(defun ...)`/`(include ...)` never mint a + * phantom call to their own keyword. `export` and `namespace` are in THIS set + * even though they are deliberately NOT definition heads: `(export foo)` names + * a function already defined in the same file, so it is neither a call nor a + * second definition of it. Real helpers that merely look primitive — + * sha256tree, the curry helpers — are deliberately absent, so they pass through + * and resolve normally. */ +static bool chialisp_head_is_not_call(const char *t) { + if (!t) { + return true; + } + static const char *filtered[] = { + /* --- CLVM primitives (VM ops) --- */ + "q", "a", "i", "c", "f", "r", "l", "x", "=", ">s", "sha256", "substr", "strlen", "concat", + "+", "-", "*", "/", "divmod", ">", "ash", "lsh", "logand", "logior", "logxor", "lognot", + "point_add", "pubkey_for_exp", "not", "any", "all", "softfork", "coinid", "g1_subtract", + "g1_multiply", "g1_negate", "g2_add", "g2_subtract", "g2_multiply", "g2_negate", "g1_map", + "g2_map", "bls_pairing_identity", "bls_verify", "modpow", "%", "secp256k1_verify", + "secp256r1_verify", "keccak256", + /* --- Chialisp syntax / binding / intrinsics (not calls or defs) --- */ + "quote", "qq", "unquote", "&rest", "let", "let*", "assign", "assign-inline", + "assign-lambda", "lambda", "mod", "if", "list", "com", "opt", "@", "@*env*", "print", + /* --- def / export / include heads (never a call) --- */ + "defun", "defun-inline", "defmacro", "defmac", "defconstant", "defconst", "namespace", + "export", "embed-file", "compile-file", "include", NULL}; + for (int i = 0; filtered[i]; i++) { + if (strcmp(t, filtered[i]) == 0) { + return true; + } + } + return false; +} + +/* True when `node` (a `list`) sits in a BINDER position rather than an + * application position: a `(defun NAME (params) ...)` parameter list, a + * `(mod (ARGS) ...)` curried-argument list, a `(lambda (x) ...)` parameter + * list, or a `let` binding container / one of its binding pairs. + * + * A binder NAMES things; it is not an invocation. Without this every function's + * parameter list would mint a CALLS edge to its own first parameter — on + * `(defun check_conditions (HEIGHTLOCK conditions) ...)` that is a phantom call + * to HEIGHTLOCK, and real Chialisp is mostly such definitions. + * + * Cost note: this compares `node` against the parent's first three forms rather + * than computing `node`'s own index. Searching for the index would scan all of + * the parent's children, and since this runs once per child, a single list of N + * forms would cost O(N^2) — `condition_codes.clib` is one list of forty, and a + * generated table is one list of thousands. Reading forms 1 and 2 is O(1). */ +static bool chialisp_node_is_binder_list(CBMArena *a, TSNode node, const char *source) { + /* At most two levels: the binder itself, and one binding pair inside a + * `let` binding container. Bounded on purpose — never an ancestor walk. */ + for (int level = 0; level < 2; level++) { + TSNode parent = ts_node_parent(node); + if (ts_node_is_null(parent) || strcmp(ts_node_type(parent), "list") != 0) { + return false; + } + TSNode head = cbm_lisp_named_child_skip_comments(parent, 0); + if (!ts_node_is_null(head) && strcmp(ts_node_type(head), "symbol") == 0) { + char *ht = cbm_node_text(a, head, source); + if (!ht) { + return false; + } + /* `(defun NAME (params) ...)` — the params are the third form. */ + if (strcmp(ht, "defun") == 0 || strcmp(ht, "defun-inline") == 0 || + strcmp(ht, "defmacro") == 0 || strcmp(ht, "defmac") == 0) { + TSNode params = cbm_lisp_named_child_skip_comments(parent, 2); + return !ts_node_is_null(params) && ts_node_eq(params, node); + } + /* `(mod (ARGS) ...)`, `(lambda (x) ...)`, `(let ((a 1)) ...)`. */ + if (strcmp(ht, "mod") == 0 || strcmp(ht, "lambda") == 0 || strcmp(ht, "let") == 0 || + strcmp(ht, "let*") == 0) { + TSNode binder = cbm_lisp_named_child_skip_comments(parent, 1); + return !ts_node_is_null(binder) && ts_node_eq(binder, node); + } + return false; + } + /* The head is not a symbol, so `parent` may itself be a let-binding + * container and `node` one of its pairs — retry one level up. */ + node = parent; + } + return false; +} + +static char *extract_lisp_callee(CBMArena *a, TSNode node, const char *source, const char *nk, + CBMLanguage lang) { if (strcmp(nk, "list") != 0 && strcmp(nk, "list_lit") != 0) { return NULL; } @@ -744,7 +831,16 @@ static char *extract_lisp_callee(CBMArena *a, TSNode node, const char *source, c const char *hk = ts_node_type(head); if (strcmp(hk, "symbol") == 0 || strcmp(hk, "sym_lit") == 0 || strcmp(hk, "identifier") == 0) { - return cbm_node_text(a, head, source); + char *ht = cbm_node_text(a, head, source); + /* Drop CLVM ops / syntax / def heads, binder positions, and + * anything inside quoted data — a quoted form is a value, not an + * invocation. */ + if (lang == CBM_LANG_CHIALISP && + (chialisp_head_is_not_call(ht) || chialisp_node_is_binder_list(a, node, source) || + cbm_lisp_node_in_quote(a, node, source))) { + return NULL; + } + return ht; } } return NULL; @@ -1383,8 +1479,9 @@ static char *extract_callee_lang_specific(CBMArena *a, TSNode node, const char * } if (lang == CBM_LANG_CLOJURE || lang == CBM_LANG_COMMONLISP || lang == CBM_LANG_SCHEME || - lang == CBM_LANG_FENNEL || lang == CBM_LANG_RACKET || lang == CBM_LANG_EMACSLISP) { - return extract_lisp_callee(a, node, source, nk); + lang == CBM_LANG_FENNEL || lang == CBM_LANG_RACKET || lang == CBM_LANG_EMACSLISP || + lang == CBM_LANG_CHIALISP) { + return extract_lisp_callee(a, node, source, nk, lang); } if (lang == CBM_LANG_FSHARP) { return extract_fsharp_callee(a, node, source, nk); diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 6982d6c13..a91a164e8 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -7254,36 +7254,85 @@ static bool lisp_is_def_head(const char *t) { return false; } +/* Basename stem (no directory, no extension) of a path, into an arena string. + * A Chialisp `(mod ...)` has no name of its own — the file IS the puzzle — so + * the module entry point is named after the file that holds it. */ +static char *lisp_path_stem(CBMArena *a, const char *path) { + if (!path) { + return NULL; + } + const char *slash = strrchr(path, '/'); + const char *base = slash ? slash + SKIP_CHAR : path; + const char *dot = strrchr(base, '.'); + size_t len = (dot && dot != base) ? (size_t)(dot - base) : strlen(base); + return cbm_arena_strndup(a, base, len); +} + static void extract_lisp_def(CBMExtractCtx *ctx, TSNode node) { CBMArena *a = ctx->arena; + bool chialisp = (ctx->language == CBM_LANG_CHIALISP); if (ts_node_named_child_count(node) < 2) { return; } - char *head = cbm_node_text(a, ts_node_named_child(node, 0), ctx->source); - if (!lisp_is_def_head(head)) { + TSNode head_node = + chialisp ? cbm_lisp_named_child_skip_comments(node, 0) : ts_node_named_child(node, 0); + if (ts_node_is_null(head_node)) { return; } - TSNode target = ts_node_named_child(node, 1); - const char *tk = ts_node_type(target); - TSNode name_node = target; - // (define (foo args) ...) — the name is the head symbol of the nested list. - if ((strcmp(tk, "list") == 0 || strcmp(tk, "list_lit") == 0) && - ts_node_named_child_count(target) > 0) { - name_node = ts_node_named_child(target, 0); - } - if (ts_node_is_null(name_node)) { + char *head = cbm_node_text(a, head_node, ctx->source); + if (!(chialisp ? cbm_chialisp_is_def_head(head) : lisp_is_def_head(head))) { return; } - char *name = cbm_node_text(a, name_node, ctx->source); + /* A def head inside `(q ...)`/`(qq ...)` is quoted DATA — Chialisp macros + * embed puzzle-shaped literals there — so it must not mint a node. */ + if (chialisp && cbm_lisp_node_in_quote(a, node, ctx->source)) { + return; + } + char *name = NULL; + if (chialisp && strcmp(head, "mod") == 0) { + /* A top-level `(mod (ARGS) ...)` is the puzzle entry point; its second + * form is the curried-argument LIST, so the generic nested-name path + * below would name the module after its first curried argument. */ + name = lisp_path_stem(a, ctx->rel_path); + } else { + TSNode target = + chialisp ? cbm_lisp_named_child_skip_comments(node, 1) : ts_node_named_child(node, 1); + if (ts_node_is_null(target)) { + return; + } + const char *tk = ts_node_type(target); + TSNode name_node = target; + // (define (foo args) ...) — the name is the head symbol of the nested list. + if ((strcmp(tk, "list") == 0 || strcmp(tk, "list_lit") == 0) && + ts_node_named_child_count(target) > 0) { + name_node = ts_node_named_child(target, 0); + } + if (ts_node_is_null(name_node)) { + return; + } + name = cbm_node_text(a, name_node, ctx->source); + } if (!name || !name[0]) { return; } /* struct/record/type defining forms produce a type node, not a callable * (Racket `(struct point ...)`, Clojure `(defrecord ...)`, etc.). */ const char *lisp_label = "Function"; - if (strcmp(head, "struct") == 0 || strcmp(head, "define-struct") == 0 || - strcmp(head, "define-record-type") == 0 || strcmp(head, "defrecord") == 0 || - strcmp(head, "deftype") == 0) { + if (chialisp) { + /* Chialisp label map. `Constant` is admitted to the cross-file registry + * by cbm_label_is_registry_symbol, so a constant defined in an included + * .clib resolves from every puzzle that includes it. */ + if (strcmp(head, "mod") == 0) { + lisp_label = "Module"; + } else if (strcmp(head, "defconstant") == 0 || strcmp(head, "defconst") == 0 || + strcmp(head, "embed-file") == 0 || strcmp(head, "compile-file") == 0) { + lisp_label = "Constant"; + } else if (strcmp(head, "defmacro") == 0 || strcmp(head, "defmac") == 0) { + lisp_label = "Macro"; + } + } else if (strcmp(head, "struct") == 0 || strcmp(head, "define-struct") == 0 || + strcmp(head, "define-record-type") == 0 || strcmp(head, "defrecord") == 0 || + strcmp(head, "deftype") == 0) { lisp_label = "Struct"; } else if (strcmp(head, "definterface") == 0 || strcmp(head, "defprotocol") == 0) { lisp_label = "Interface"; @@ -7466,9 +7515,20 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec, } if ((ctx->language == CBM_LANG_CLOJURE || ctx->language == CBM_LANG_RACKET || - ctx->language == CBM_LANG_SCHEME) && + ctx->language == CBM_LANG_SCHEME || ctx->language == CBM_LANG_CHIALISP) && (strcmp(kind, "list") == 0 || strcmp(kind, "list_lit") == 0)) { extract_lisp_def(ctx, node); + if (ctx->language == CBM_LANG_CHIALISP) { + /* Chialisp nests every helper `(defun ...)` inside the top-level + * `(mod ...)`, and a .clib wraps its defuns in one enclosing + * list. `list` is also chialisp_func_types, so the generic + * function_node_types match below would fire on this same node + * and `continue` WITHOUT descending — losing every nested def. + * Push the children here and skip that match. Gated to Chialisp; + * the other lisps keep their existing fall-through. */ + wd_push_children_reverse(&s, node, frame.enclosing_class_qn); + continue; + } // fall through: descend into children so nested defs are captured too } diff --git a/internal/cbm/extract_imports.c b/internal/cbm/extract_imports.c index 2d8ed1e4e..9de2fd007 100644 --- a/internal/cbm/extract_imports.c +++ b/internal/cbm/extract_imports.c @@ -1931,6 +1931,46 @@ static void lisp_process_list(CBMExtractCtx *ctx, TSNode node) { return; } + /* Chialisp include/embed forms. + * + * `(include *standard-cl-26*)` selects a dialect; every `*...*` sigil is a + * compiler directive, not a file dependency. The Chialisp compiler filters + * these with its own `!name.starts_with("*")` check and so do we — recording + * one as an import would invent a dependency on a file that does not exist. + * + * `(embed-file NAME kind "file")` / `(compile-file NAME "file")` DO create a + * real dependency on the embedded/compiled artifact; the path is the string + * argument. There is no dedicated EMBEDS edge type, so it is recorded as an + * import — which is what it is for reachability. */ + if (ctx->language == CBM_LANG_CHIALISP) { + if (strcmp(hn, "include") == 0) { + for (uint32_t j = 1; j < nc; j++) { + TSNode mod_node = ts_node_named_child(node, j); + const char *mk = ts_node_type(mod_node); + if (strcmp(mk, "symbol") == 0 || strcmp(mk, "sym_lit") == 0) { + char *sig = cbm_node_text(ctx->arena, mod_node, ctx->source); + if (sig && sig[0] == '*') { + continue; /* dialect sigil, not a file */ + } + } + lisp_push_module(ctx, mod_node); + } + return; + } + if (strcmp(hn, "embed-file") == 0 || strcmp(hn, "compile-file") == 0) { + /* the embedded/compiled path is the last string argument. */ + for (uint32_t j = nc; j-- > 1;) { + TSNode arg = ts_node_named_child(node, j); + const char *ak = ts_node_type(arg); + if (strcmp(ak, "string") == 0 || strcmp(ak, "str_lit") == 0) { + lisp_push_module(ctx, arg); + break; + } + } + return; + } + } + /* Plain import head: `(require :util)`, `(import ...)`, `(use ...)`. */ if (strcmp(hn, "import") == 0 || strcmp(hn, "require") == 0 || strcmp(hn, "load") == 0 || strcmp(hn, "use") == 0 || strcmp(hn, "include") == 0) { @@ -2996,6 +3036,7 @@ void cbm_extract_imports(CBMExtractCtx *ctx) { case CBM_LANG_FENNEL: case CBM_LANG_COMMONLISP: case CBM_LANG_CLOJURE: + case CBM_LANG_CHIALISP: parse_lisp_imports(ctx); break; case CBM_LANG_STARLARK: diff --git a/internal/cbm/extract_unified.c b/internal/cbm/extract_unified.c index f106f5635..a5e314272 100644 --- a/internal/cbm/extract_unified.c +++ b/internal/cbm/extract_unified.c @@ -329,14 +329,48 @@ static bool lisp_head_is_def(const char *t) { * forms, ...), so push_boundary_scopes pushes no scope for them. Mirrors * extract_lisp_def() in extract_defs.c. */ static const char *compute_lisp_func_qn(CBMExtractCtx *ctx, TSNode node) { + bool chialisp = (ctx->language == CBM_LANG_CHIALISP); if (ts_node_named_child_count(node) < 2) { return NULL; } - char *head = cbm_node_text(ctx->arena, ts_node_named_child(node, 0), ctx->source); - if (!lisp_head_is_def(head)) { + /* Chialisp reads its head and name through the comment-skipping accessor — + * the SAME one extract_lisp_def() uses. When these two disagree (one + * skipping comments, the other not), a doc comment between a def head and + * its name shifts the named-child indices for only one of them, and the + * call scope silently stops matching the def it belongs to. */ + TSNode head_node = + chialisp ? cbm_lisp_named_child_skip_comments(node, 0) : ts_node_named_child(node, 0); + if (ts_node_is_null(head_node)) { + return NULL; + } + char *head = cbm_node_text(ctx->arena, head_node, ctx->source); + if (!(chialisp ? cbm_chialisp_is_def_head(head) : lisp_head_is_def(head))) { + return NULL; + } + if (chialisp && cbm_lisp_node_in_quote(ctx->arena, node, ctx->source)) { + return NULL; + } + if (chialisp && head && strcmp(head, "mod") == 0) { + /* `mod`'s second form is the curried-arg list, not a name; the puzzle + * entry point is named by its file, so in-body top-level calls + * attribute to the entry rather than to a curried argument. Mirrors the + * lisp_path_stem() naming in extract_defs.c. */ + const char *path = ctx->rel_path; + const char *slash = path ? strrchr(path, '/') : NULL; + const char *base = slash ? slash + 1 : path; + if (!base || !base[0]) { + return NULL; + } + const char *dot = strrchr(base, '.'); + size_t len = (dot && dot != base) ? (size_t)(dot - base) : strlen(base); + char *stem = cbm_arena_strndup(ctx->arena, base, len); + return cbm_fqn_compute(ctx->arena, ctx->project, ctx->rel_path, stem); + } + TSNode target = + chialisp ? cbm_lisp_named_child_skip_comments(node, 1) : ts_node_named_child(node, 1); + if (ts_node_is_null(target)) { return NULL; } - TSNode target = ts_node_named_child(node, 1); const char *tk = ts_node_type(target); TSNode name_node = target; /* (define (foo args) ...) — the name is the head symbol of the nested list. */ @@ -746,7 +780,7 @@ static const char *compute_func_qn(CBMExtractCtx *ctx, TSNode node, const CBMLan * lives here (we have ctx->source). Non-def lists return NULL → no scope * pushed → the in-body call sources to the enclosing def, not the Module. */ if (ctx->language == CBM_LANG_CLOJURE || ctx->language == CBM_LANG_SCHEME || - ctx->language == CBM_LANG_RACKET) { + ctx->language == CBM_LANG_RACKET || ctx->language == CBM_LANG_CHIALISP) { return compute_lisp_func_qn(ctx, node); } diff --git a/internal/cbm/extract_usages.c b/internal/cbm/extract_usages.c index 1f2e4098e..c47c8434b 100644 --- a/internal/cbm/extract_usages.c +++ b/internal/cbm/extract_usages.c @@ -162,6 +162,7 @@ static bool is_reference_node(TSNode node, CBMLanguage lang) { case CBM_LANG_SCHEME: case CBM_LANG_FENNEL: case CBM_LANG_RACKET: + case CBM_LANG_CHIALISP: case CBM_LANG_LINKERSCRIPT: return strcmp(kind, "symbol") == 0; case CBM_LANG_MAKEFILE: @@ -464,6 +465,7 @@ static const CBMOccurrenceSpec occurrence_specs[CBM_LANG_COUNT] = { [CBM_LANG_CLOJURE] = {NULL, NULL, CBM_OCCURRENCE_LISP_DEF, false}, [CBM_LANG_SCHEME] = {NULL, NULL, CBM_OCCURRENCE_LISP_DEF, false}, [CBM_LANG_RACKET] = {NULL, NULL, CBM_OCCURRENCE_LISP_DEF, false}, + [CBM_LANG_CHIALISP] = {NULL, NULL, CBM_OCCURRENCE_LISP_DEF, false}, [CBM_LANG_COMMONLISP] = {NULL, NULL, CBM_OCCURRENCE_COMMONLISP_DEFUN, false}, [CBM_LANG_FENNEL] = {NULL, NULL, CBM_OCCURRENCE_FENNEL_FN, false}, [CBM_LANG_ELIXIR] = {NULL, NULL, CBM_OCCURRENCE_ELIXIR_DEF, false}, @@ -537,23 +539,39 @@ static bool lisp_def_head(const char *text) { return kind_in_exact_set(text, heads); } +/* Chialisp heads whose THIRD form is a parameter list, so the symbols in it + * bind rather than refer: `(defun NAME (params) body)`. Deliberately not + * `defconstant` — its third form is the VALUE expression, whose symbols are + * genuine usages — and not `mod`, whose binder is the second form and is + * already covered by the shared named_child(1) rule below. */ +static bool chialisp_head_binds_params_at_2(const char *head) { + return head && (strcmp(head, "defun") == 0 || strcmp(head, "defun-inline") == 0 || + strcmp(head, "defmacro") == 0 || strcmp(head, "defmac") == 0); +} + static bool is_lisp_def_binding(CBMExtractCtx *ctx, TSNode node) { + bool chialisp = (ctx->language == CBM_LANG_CHIALISP); for (TSNode form = ts_node_parent(node); !ts_node_is_null(form); form = ts_node_parent(form)) { const char *kind = ts_node_type(form); if ((strcmp(kind, "list") != 0 && strcmp(kind, "list_lit") != 0) || ts_node_named_child_count(form) < 2) { continue; } - TSNode head_node = ts_node_named_child(form, 0); + TSNode head_node = + chialisp ? cbm_lisp_named_child_skip_comments(form, 0) : ts_node_named_child(form, 0); + if (ts_node_is_null(head_node)) { + continue; + } char *head = cbm_node_text(ctx->arena, head_node, ctx->source); - if (!lisp_def_head(head)) { + if (!(chialisp ? cbm_chialisp_is_def_head(head) : lisp_def_head(head))) { continue; } if (node_contains(head_node, node) || named_child_contains(form, 1, node)) { return true; } - if (ctx->language == CBM_LANG_CLOJURE && ts_node_named_child_count(form) > 2 && - named_child_contains(form, 2, node)) { + if ((ctx->language == CBM_LANG_CLOJURE || + (chialisp && chialisp_head_binds_params_at_2(head))) && + ts_node_named_child_count(form) > 2 && named_child_contains(form, 2, node)) { return true; } return false; diff --git a/internal/cbm/grammar_chialisp.c b/internal/cbm/grammar_chialisp.c new file mode 100644 index 000000000..cf5ff3641 --- /dev/null +++ b/internal/cbm/grammar_chialisp.c @@ -0,0 +1,3 @@ +// Vendored tree-sitter grammar: chialisp +// Each grammar compiled as separate unit (conflicting static symbols). +#include "vendored/grammars/chialisp/parser.c" diff --git a/internal/cbm/helpers.c b/internal/cbm/helpers.c index 46d3fda6a..5ea8c6cef 100644 --- a/internal/cbm/helpers.c +++ b/internal/cbm/helpers.c @@ -17,6 +17,10 @@ enum { INIT_FILE_LEN = 8, /* strlen("__init__") */ INDEX_FILE_LEN = 5, /* strlen("index") */ NOT_FOUND = -1, + /* Ancestor-walk bound for cbm_lisp_node_in_quote. A quote nest deeper than + * this is pathological input, not Chialisp; bounding it keeps the walk O(1) + * per node rather than O(depth) on adversarially nested data. */ + CBM_LISP_QUOTE_ANCESTOR_MAX = 256, }; /* Prefix length helper for strncmp with string literals. */ @@ -190,9 +194,76 @@ bool cbm_label_is_registry_symbol(const char *label) { if (!label) { return false; } + /* "Constant" is a named, file-scope-transcending symbol exactly like + * Variable: Chialisp's `(defconstant CREATE_COIN 51)` lives in an included + * .clib and is referenced by name from every puzzle that includes it. Left + * out of the registry it can never be the target of a cross-file resolve, + * and the ~half of a Chialisp graph that is constants would be unreachable. + * It is deliberately NOT type-like (cbm_label_is_type_like): a constant + * must never satisfy an inheritance, impl-receiver or semantic-type + * lookup. */ return strcmp(label, "Function") == 0 || strcmp(label, "Method") == 0 || cbm_label_is_type_like(label) || strcmp(label, "Variable") == 0 || - strcmp(label, "Field") == 0 || cbm_label_is_relation(label); + strcmp(label, "Constant") == 0 || strcmp(label, "Field") == 0 || + cbm_label_is_relation(label); +} + +bool cbm_lisp_node_in_quote(CBMArena *a, TSNode node, const char *source) { + TSNode cur = ts_node_parent(node); + for (int guard = 0; guard < CBM_LISP_QUOTE_ANCESTOR_MAX && !ts_node_is_null(cur); guard++) { + const char *ck = ts_node_type(cur); + if ((strcmp(ck, "list") == 0 || strcmp(ck, "list_lit") == 0) && + ts_node_named_child_count(cur) > 0) { + TSNode h = ts_node_named_child(cur, 0); + const char *hk = ts_node_type(h); + if (strcmp(hk, "symbol") == 0 || strcmp(hk, "sym_lit") == 0) { + char *ht = cbm_node_text(a, h, source); + if (ht && + (strcmp(ht, "q") == 0 || strcmp(ht, "quote") == 0 || strcmp(ht, "qq") == 0)) { + return true; + } + } + } + cur = ts_node_parent(cur); + } + return false; +} + +TSNode cbm_lisp_named_child_skip_comments(TSNode node, uint32_t want) { + uint32_t nc = ts_node_named_child_count(node); + uint32_t seen = 0; + for (uint32_t i = 0; i < nc; i++) { + TSNode c = ts_node_named_child(node, i); + if (strcmp(ts_node_type(c), "comment") == 0) { + continue; + } + if (seen == want) { + return c; + } + seen++; + } + TSNode null_node = {0}; + return null_node; +} + +bool cbm_chialisp_is_def_head(const char *t) { + if (!t) { + return false; + } + /* `export` and `namespace` are absent ON PURPOSE. `(export foo)` re-exports + * a function `(defun foo ...)` already defined in the same file, so + * admitting it here mints a SECOND node for the same symbol. They stay in + * the not-a-call filter (extract_calls.c) because they are still not calls. + */ + static const char *heads[] = {"mod", "defun", "defun-inline", "defmacro", + "defmac", "defconstant", "defconst", "embed-file", + "compile-file", NULL}; + for (int i = 0; heads[i]; i++) { + if (strcmp(t, heads[i]) == 0) { + return true; + } + } + return false; } bool cbm_is_keyword(const char *name, CBMLanguage lang) { diff --git a/internal/cbm/helpers.h b/internal/cbm/helpers.h index b03ef3743..4e8150b6f 100644 --- a/internal/cbm/helpers.h +++ b/internal/cbm/helpers.h @@ -122,6 +122,36 @@ char *cbm_cpp_out_of_line_parent_class(CBMArena *a, TSNode node, const char *sou // Find a child node by kind string. TSNode cbm_find_child_by_kind(TSNode parent, const char *kind); +/* --- Lisp-family shared gates --------------------------------------------- + * The defs, calls and unified extractors each walk the same generic `list` + * node and must agree on what it means. The predicates below therefore live + * here rather than being copied per translation unit: a private copy in each + * drifts, and the drift is silent — defs and call-scope simply stop describing + * the same tree. + */ + +/* True when any ancestor list of `node` is headed by a quote symbol + * (`q` / `quote` / `qq`) — its contents are DATA, not code, so no def and no + * call may be minted from them. Bounded ancestor walk; the arena is used only + * for the head-text reads. */ +bool cbm_lisp_node_in_quote(CBMArena *a, TSNode node, const char *source); + +/* The `want`-th named child of `node`, skipping `comment` nodes. Comments are + * named in the s-expression grammars and so occupy named-child indices: a + * comment between a def head and its name shifts every later index by one. + * Definition extraction and call-scope attribution MUST use this same skipping + * rule or they desynchronise on exactly the files that carry doc comments. */ +TSNode cbm_lisp_named_child_skip_comments(TSNode node, uint32_t want); + +/* True for a Chialisp definition-form head. Deliberately separate from the + * Clojure/Scheme def-head set: Chialisp shares the generic `list` kind, and + * treating `mod`/`defconstant` as defs must not leak into the other lisps (in + * Scheme `(mod x y)` is a call). Excludes the expression-local binding forms + * (`let`, `assign`, `lambda`), which would fragment call attribution, and + * excludes `export`/`namespace`, which NAME an already-defined function rather + * than defining one. */ +bool cbm_chialisp_is_def_head(const char *t); + // Check if node kind matches a set of types (NULL-terminated array of strings). bool cbm_kind_in_set(TSNode node, const char **types); diff --git a/internal/cbm/lang_specs.c b/internal/cbm/lang_specs.c index 040700702..51ca4ba60 100644 --- a/internal/cbm/lang_specs.c +++ b/internal/cbm/lang_specs.c @@ -80,6 +80,7 @@ extern const TSLanguage *tree_sitter_powershell(void); extern const TSLanguage *tree_sitter_pascal(void); extern const TSLanguage *tree_sitter_d(void); extern const TSLanguage *tree_sitter_scheme(void); +extern const TSLanguage *tree_sitter_chialisp(void); extern const TSLanguage *tree_sitter_fennel(void); extern const TSLanguage *tree_sitter_fish(void); extern const TSLanguage *tree_sitter_awk(void); @@ -1107,6 +1108,13 @@ static const char *scheme_func_types[] = {"list", NULL}; static const char *scheme_call_types[] = {"list", NULL}; static const char *scheme_var_types[] = {"symbol", NULL}; static const char *scheme_module_types[] = {"program", NULL}; +// Chialisp: a deliberately generic s-expression grammar (tools/tree-sitter-chialisp). +// Every parenthesized form is a `list` and every atom a `symbol`; which lists are +// definitions is decided in extract_defs.c, not by the parser. Root is `source_file`. +static const char *chialisp_func_types[] = {"list", NULL}; +static const char *chialisp_call_types[] = {"list", NULL}; +static const char *chialisp_var_types[] = {"symbol", NULL}; +static const char *chialisp_module_types[] = {"source_file", NULL}; static const char *fennel_func_types[] = {"fn", "lambda", "hashfn", NULL}; static const char *fennel_call_types[] = {"list", NULL}; static const char *fennel_branch_types[] = {"each", "for", "match", NULL}; @@ -2161,6 +2169,12 @@ static const CBMLangSpec lang_specs[CBM_LANG_COUNT] = { empty_types, scheme_var_types, empty_types, empty_types, NULL, empty_types, NULL, NULL, tree_sitter_scheme, NULL}, + // CBM_LANG_CHIALISP — lisp-family shape (generic list/symbol nodes) + [CBM_LANG_CHIALISP] = {CBM_LANG_CHIALISP, chialisp_func_types, empty_types, empty_types, + chialisp_module_types, chialisp_call_types, empty_types, empty_types, + empty_types, chialisp_var_types, empty_types, empty_types, NULL, + empty_types, NULL, NULL, tree_sitter_chialisp, NULL}, + // CBM_LANG_FENNEL [CBM_LANG_FENNEL] = {CBM_LANG_FENNEL, fennel_func_types, empty_types, empty_types, fennel_module_types, fennel_call_types, empty_types, empty_types, diff --git a/internal/cbm/vendored/grammars/MANIFEST.md b/internal/cbm/vendored/grammars/MANIFEST.md index ea80f8c74..9b5b0127b 100644 --- a/internal/cbm/vendored/grammars/MANIFEST.md +++ b/internal/cbm/vendored/grammars/MANIFEST.md @@ -9,8 +9,8 @@ The grammars were originally vendored as bare `parser.c`+`scanner.c` with **no r ## Summary -- Grammars: **161** — vendored-from-upstream: **143**, first-party/self-maintained: **13**, registry-disagreement: **5** (nim removed 2026-06-12; objectscript_udl + objectscript_routine added 2026-06-24; mojo added 2026-07-01; arkts added 2026-08-26; plsql added 2026-08-27 — see notes below) -- ABI distribution: **7×** ABI-13 **86×** ABI-14 **65×** ABI-15 (runtime ceiling is ABI 15; never vendor ABI 16 without a runtime upgrade) +- Grammars: **162** — vendored-from-upstream: **143**, first-party/self-maintained: **14**, registry-disagreement: **5** (nim removed 2026-06-12; objectscript_udl + objectscript_routine added 2026-06-24; mojo added 2026-07-01; arkts added 2026-08-26; plsql added 2026-08-27; chialisp added 2026-08-28 — see notes below) +- ABI distribution: **7×** ABI-13 **87×** ABI-14 **65×** ABI-15 (runtime ceiling is ABI 15; never vendor ABI 16 without a runtime upgrade) - Vendored copies missing LICENSE: **0** — all upstream LICENSE files restored 2026-06-11 (first-party grammars carry the project MIT license; `move` uses the Helix-listed upstream tzakian/tree-sitter-move MIT text, `zsh` uses georgeharker/tree-sitter-zsh MIT) - `verdict`: VERIFIED-BOTH = our source matches *both* registries; VERIFIED-NVIM/HELIX = matches one; registry-disagreement = registries name a different repo (listed separately); `vendor-maintained` = the language vendor's own grammar, not in nvim/Helix; `community-niche` = an individual/community-maintained grammar for a niche language, not in nvim/Helix — provenance hand-verified against the upstream repository instead of a registry. - **objectscript_udl / objectscript_routine** (added 2026-06-24): vendored from [intersystems/tree-sitter-objectscript](https://github.com/intersystems/tree-sitter-objectscript) @ `a7ffcdf` — MIT, the InterSystems-official grammars (a niche vendor language, hence `vendor-maintained`, not in nvim-treesitter/Helix). **Re-vendor note:** each `scanner.c`'s upstream `#include "../../common/scanner.h"` is repointed to a per-directory `objectscript_common.h` (a verbatim copy of upstream `common/scanner.h`), because this repo's shared `vendored/common/scanner.h` belongs to the cfml/fsharp grammars and differs. The generated `parser.c`/`scanner.c` are otherwise byte-for-byte upstream — on re-vendor, re-apply only that single include rename. **Local modification (2026-07-16):** in `objectscript_common.h`, two loop counters `uint8_t i` were widened to `int i` (the `reverse_marker` scan and the `html_marker_buffer` reversal) to clear CodeQL `cpp/comparison-with-wider-type` — a false positive in practice (both lengths are hard-bounded by `MARKER_BUFFER_MAX_LEN = 30`, so `uint8_t` could never wrap), fixed for cleanliness. On re-vendor, re-apply this widening too (or upstream it at intersystems/tree-sitter-objectscript). @@ -18,6 +18,8 @@ The grammars were originally vendored as bare `parser.c`+`scanner.c` with **no r - **arkts** (added 2026-08-26): **first-party derivative** — a fork of [tree-sitter/tree-sitter-typescript](https://github.com/tree-sitter/tree-sitter-typescript)'s `typescript` dialect pinned @ `75b3874edb2d` (v0.23.2, the same commit our vendored typescript/tsx come from), on the tree-sitter-javascript base @ `3a837b6f3658` (v0.23.1, the version upstream's own package-lock pins), extended with ArkTS/ArkUI syntax (`@Component struct` declarations, UI-DSL trailing-closure calls with post-block attribute chains, decorated function declarations, `import lazy`, `@Extend`/`@Styles` leading-dot attribute chains, anonymous `stateStyles` style blocks). Grammar source + corpus tests live in `tools/tree-sitter-arkts/`; regenerate with `npx tree-sitter-cli@0.25.10 generate` (ABI 15). **External scanner:** `scanner.c` is a symbol-rename trampoline; `_common_scanner.h` is byte-identical (same SHA-256) to the reviewed `typescript/_common_scanner.h` already shipped. **LICENSE:** tree-sitter-typescript's MIT text **verbatim and byte-identical** (© 2017 Max Brunsfeld) — which is what MIT requires of a derivative work, and what lets the provenance audit byte-verify it against upstream rather than take a note on trust. The fork is registered in the audit's `FORKS` map. Our own copyright for the ArkTS additions, and the tree-sitter-javascript attribution (© 2014 Max Brunsfeld), live with the SOURCE in `tools/tree-sitter-arkts/grammar.js` and in `THIRD_PARTY.md`, which is what ships in the release archives. - **plsql** (added 2026-08-27): vendored from [AndreasMaierDe/tree-sitter-plsql](https://github.com/AndreasMaierDe/tree-sitter-plsql) @ `28aebef209be` (full: `28aebef209be57169600e1aa41ca1431cc6c916f`, upstream tip; repo dormant since 2023-02) — MIT, ABI 14, **no external scanner** (`EXTERNAL_TOKEN_COUNT 0`). Not listed in nvim-treesitter/Helix (`community-niche`, an individual-maintained grammar); provenance verified directly against upstream (pin = `git ls-remote` HEAD; vendored `parser.c` byte-identical to the pinned clone apart from the include-quote local patch below; 0 non-ASCII bytes; 0 dangerous calls). Security review covered only the vendored C surface (`parser.c`, `tree_sitter/parser.h`) plus upstream license/provenance metadata; no package manager hooks, workflow files, prompt/agent instruction files, or generated lockfiles were vendored. Known upstream limitation: `CREATE TYPE ... AS OBJECT` currently yields ERROR nodes (pinned by `plsql_create_type_as_object_limitation` in `tests/test_extraction.c` + `tests/fixtures/plsql/create_type_as_object_limitation.tps`). Originally contributed as PR #1033 by Oğuz (@ouzsrcm); re-vendored from upstream per vendoring policy with the PR's language wiring distilled on top. +- **chialisp** (added 2026-08-28): **first-party** — authored in this repository, not vendored from anywhere. Grammar source + corpus tests live in `tools/tree-sitter-chialisp/`; regenerate with `npx tree-sitter-cli@0.25.10 generate --abi 14` and copy `src/parser.c` + `src/tree_sitter/*.h` here. ABI 14, **no external scanner** (`EXTERNAL_TOKEN_COUNT 0`), 0 non-ASCII bytes. It is a deliberately GENERIC s-expression grammar (`source_file`/`list`/`symbol`/`string`/`number`/`hex`/`dot`/`comment`) modelling the clvm_tools reader rather than the Chialisp form vocabulary: `mod`/`defun`/`defconstant`/`include` are ordinary head symbols, and which lists are definitions is decided in `internal/cbm/extract_defs.c`, so a dialect that adds a form does not need a regenerated parser. Written because the only public grammar (`Quexington/tree-sitter-chialisp`) cannot parse the language: it required CRLF to terminate a comment (`/;.*\r\n/`) while real files are LF, rejected the `.` in `(include foo.clib)`, and accepted only a primitive after `(defconstant NAME …)` — each of which desynchronised the rest of the file. Acceptance gate: all five `chia-blockchain@main` reference files parse with **zero ERROR and zero MISSING nodes**. **LICENSE:** the project's own LICENSE, byte-identical to the repository root — no third-party copyright is carried, because there is no third party. + > ⚠️ **Pinned commit = the revision nvim-treesitter/Helix vendor** (battle-tested, canonical source), not bleeding-edge HEAD. When re-vendoring, update the pinned commit here. ## Custom extraction handling (definition extraction) @@ -38,6 +40,7 @@ Guarded by the `contract_all_grammars_in_graph` graph-breadth test in |---|---| | ada | `resolve_func_name`: `subprogram_body`/`subprogram_declaration` → `procedure_specification`/`function_specification` child's `name` field | | cairo | `resolve_func_name`: `function_definition`/`function_signature` → `identifier` child | +| chialisp | `extract_lisp_def`: `(mod/defun/defun-inline/defmacro/defmac/defconstant/defconst …)` head-symbol forms in `list`; `mod` named by filename; `defconstant` → `Constant`; CLVM operators, quoted data and dialect sigils filtered out of calls/imports | | clojure | `extract_lisp_def`: `(defn …)` / `(def …)` head-symbol forms in `list_lit` | | d | `resolve_func_name`: `function_declaration` → `identifier` child | | fortran | `resolve_func_name`: `subroutine`/`function` → inner `*_statement`'s `name` field | @@ -234,6 +237,7 @@ upstream author (correctly retained). The table now records the true origin. | magma | 15 | ✅ project MIT | | protobuf | 13 | ✅ project MIT | | wolfram | 13 | ✅ project MIT | +| chialisp | 14 | ✅ project MIT | ### Self-maintained forks (upstream license retained, byte-verified 2026-06-12) diff --git a/internal/cbm/vendored/grammars/chialisp/LICENSE b/internal/cbm/vendored/grammars/chialisp/LICENSE new file mode 100644 index 000000000..0b9535274 --- /dev/null +++ b/internal/cbm/vendored/grammars/chialisp/LICENSE @@ -0,0 +1,22 @@ +MIT License + + Copyright (c) 2025 DeusData + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + \ No newline at end of file diff --git a/internal/cbm/vendored/grammars/chialisp/parser.c b/internal/cbm/vendored/grammars/chialisp/parser.c new file mode 100644 index 000000000..d929151ea --- /dev/null +++ b/internal/cbm/vendored/grammars/chialisp/parser.c @@ -0,0 +1,485 @@ +/* Automatically @generated by tree-sitter v0.25.10 */ + +#include "tree_sitter/parser.h" + +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic ignored "-Wmissing-field-initializers" +#endif + +#define LANGUAGE_VERSION 14 +#define STATE_COUNT 9 +#define LARGE_STATE_COUNT 8 +#define SYMBOL_COUNT 14 +#define ALIAS_COUNT 0 +#define TOKEN_COUNT 9 +#define EXTERNAL_TOKEN_COUNT 0 +#define FIELD_COUNT 0 +#define MAX_ALIAS_SEQUENCE_LENGTH 3 +#define MAX_RESERVED_WORD_SET_SIZE 0 +#define PRODUCTION_ID_COUNT 1 +#define SUPERTYPE_COUNT 0 + +enum ts_symbol_identifiers { + anon_sym_LPAREN = 1, + anon_sym_RPAREN = 2, + sym_comment = 3, + sym_hex = 4, + sym_number = 5, + sym_string = 6, + sym_dot = 7, + sym_symbol = 8, + sym_source_file = 9, + sym__form = 10, + sym__sexp = 11, + sym_list = 12, + aux_sym_source_file_repeat1 = 13, +}; + +static const char * const ts_symbol_names[] = { + [ts_builtin_sym_end] = "end", + [anon_sym_LPAREN] = "(", + [anon_sym_RPAREN] = ")", + [sym_comment] = "comment", + [sym_hex] = "hex", + [sym_number] = "number", + [sym_string] = "string", + [sym_dot] = "dot", + [sym_symbol] = "symbol", + [sym_source_file] = "source_file", + [sym__form] = "_form", + [sym__sexp] = "_sexp", + [sym_list] = "list", + [aux_sym_source_file_repeat1] = "source_file_repeat1", +}; + +static const TSSymbol ts_symbol_map[] = { + [ts_builtin_sym_end] = ts_builtin_sym_end, + [anon_sym_LPAREN] = anon_sym_LPAREN, + [anon_sym_RPAREN] = anon_sym_RPAREN, + [sym_comment] = sym_comment, + [sym_hex] = sym_hex, + [sym_number] = sym_number, + [sym_string] = sym_string, + [sym_dot] = sym_dot, + [sym_symbol] = sym_symbol, + [sym_source_file] = sym_source_file, + [sym__form] = sym__form, + [sym__sexp] = sym__sexp, + [sym_list] = sym_list, + [aux_sym_source_file_repeat1] = aux_sym_source_file_repeat1, +}; + +static const TSSymbolMetadata ts_symbol_metadata[] = { + [ts_builtin_sym_end] = { + .visible = false, + .named = true, + }, + [anon_sym_LPAREN] = { + .visible = true, + .named = false, + }, + [anon_sym_RPAREN] = { + .visible = true, + .named = false, + }, + [sym_comment] = { + .visible = true, + .named = true, + }, + [sym_hex] = { + .visible = true, + .named = true, + }, + [sym_number] = { + .visible = true, + .named = true, + }, + [sym_string] = { + .visible = true, + .named = true, + }, + [sym_dot] = { + .visible = true, + .named = true, + }, + [sym_symbol] = { + .visible = true, + .named = true, + }, + [sym_source_file] = { + .visible = true, + .named = true, + }, + [sym__form] = { + .visible = false, + .named = true, + }, + [sym__sexp] = { + .visible = false, + .named = true, + }, + [sym_list] = { + .visible = true, + .named = true, + }, + [aux_sym_source_file_repeat1] = { + .visible = false, + .named = false, + }, +}; + +static const TSSymbol ts_alias_sequences[PRODUCTION_ID_COUNT][MAX_ALIAS_SEQUENCE_LENGTH] = { + [0] = {0}, +}; + +static const uint16_t ts_non_terminal_alias_map[] = { + 0, +}; + +static const TSStateId ts_primary_state_ids[STATE_COUNT] = { + [0] = 0, + [1] = 1, + [2] = 2, + [3] = 3, + [4] = 4, + [5] = 5, + [6] = 6, + [7] = 7, + [8] = 8, +}; + +static bool ts_lex(TSLexer *lexer, TSStateId state) { + START_LEXER(); + eof = lexer->eof(lexer); + switch (state) { + case 0: + if (eof) ADVANCE(5); + ADVANCE_MAP( + '"', 1, + '\'', 2, + '(', 6, + ')', 7, + '-', 14, + '.', 13, + '0', 10, + ';', 8, + ); + if (('\t' <= lookahead && lookahead <= '\r') || + lookahead == ' ') SKIP(0); + if (('1' <= lookahead && lookahead <= '9')) ADVANCE(11); + if (lookahead != 0) ADVANCE(16); + END_STATE(); + case 1: + if (lookahead == '"') ADVANCE(12); + if (lookahead == '\\') ADVANCE(3); + if (lookahead != 0) ADVANCE(1); + END_STATE(); + case 2: + if (lookahead == '\'') ADVANCE(12); + if (lookahead == '\\') ADVANCE(4); + if (lookahead != 0) ADVANCE(2); + END_STATE(); + case 3: + if (lookahead != 0 && + lookahead != '\n') ADVANCE(1); + END_STATE(); + case 4: + if (lookahead != 0 && + lookahead != '\n') ADVANCE(2); + END_STATE(); + case 5: + ACCEPT_TOKEN(ts_builtin_sym_end); + END_STATE(); + case 6: + ACCEPT_TOKEN(anon_sym_LPAREN); + END_STATE(); + case 7: + ACCEPT_TOKEN(anon_sym_RPAREN); + END_STATE(); + case 8: + ACCEPT_TOKEN(sym_comment); + if (lookahead != 0 && + lookahead != '\n' && + lookahead != '\r') ADVANCE(8); + END_STATE(); + case 9: + ACCEPT_TOKEN(sym_hex); + if (('0' <= lookahead && lookahead <= '9') || + ('A' <= lookahead && lookahead <= 'F') || + ('a' <= lookahead && lookahead <= 'f')) ADVANCE(9); + if (lookahead != 0 && + (lookahead < '\t' || '\r' < lookahead) && + lookahead != ' ' && + lookahead != '"' && + (lookahead < '\'' || ')' < lookahead) && + lookahead != ';') ADVANCE(16); + END_STATE(); + case 10: + ACCEPT_TOKEN(sym_number); + if (lookahead == 'X' || + lookahead == 'x') ADVANCE(15); + if (('0' <= lookahead && lookahead <= '9')) ADVANCE(11); + if (lookahead != 0 && + (lookahead < '\t' || '\r' < lookahead) && + lookahead != ' ' && + lookahead != '"' && + (lookahead < '\'' || ')' < lookahead) && + lookahead != ';') ADVANCE(16); + END_STATE(); + case 11: + ACCEPT_TOKEN(sym_number); + if (('0' <= lookahead && lookahead <= '9')) ADVANCE(11); + if (lookahead != 0 && + (lookahead < '\t' || '\r' < lookahead) && + lookahead != ' ' && + lookahead != '"' && + (lookahead < '\'' || ')' < lookahead) && + lookahead != ';') ADVANCE(16); + END_STATE(); + case 12: + ACCEPT_TOKEN(sym_string); + END_STATE(); + case 13: + ACCEPT_TOKEN(sym_dot); + if (lookahead != 0 && + (lookahead < '\t' || '\r' < lookahead) && + lookahead != ' ' && + lookahead != '"' && + (lookahead < '\'' || ')' < lookahead) && + lookahead != ';') ADVANCE(16); + END_STATE(); + case 14: + ACCEPT_TOKEN(sym_symbol); + if (('0' <= lookahead && lookahead <= '9')) ADVANCE(11); + if (lookahead != 0 && + (lookahead < '\t' || '\r' < lookahead) && + lookahead != ' ' && + lookahead != '"' && + (lookahead < '\'' || ')' < lookahead) && + lookahead != ';') ADVANCE(16); + END_STATE(); + case 15: + ACCEPT_TOKEN(sym_symbol); + if (('0' <= lookahead && lookahead <= '9') || + ('A' <= lookahead && lookahead <= 'F') || + ('a' <= lookahead && lookahead <= 'f')) ADVANCE(9); + if (lookahead != 0 && + (lookahead < '\t' || '\r' < lookahead) && + lookahead != ' ' && + lookahead != '"' && + (lookahead < '\'' || ')' < lookahead) && + lookahead != ';') ADVANCE(16); + END_STATE(); + case 16: + ACCEPT_TOKEN(sym_symbol); + if (lookahead != 0 && + (lookahead < '\t' || '\r' < lookahead) && + lookahead != ' ' && + lookahead != '"' && + (lookahead < '\'' || ')' < lookahead) && + lookahead != ';') ADVANCE(16); + END_STATE(); + default: + return false; + } +} + +static const TSLexMode ts_lex_modes[STATE_COUNT] = { + [0] = {.lex_state = 0}, + [1] = {.lex_state = 0}, + [2] = {.lex_state = 0}, + [3] = {.lex_state = 0}, + [4] = {.lex_state = 0}, + [5] = {.lex_state = 0}, + [6] = {.lex_state = 0}, + [7] = {.lex_state = 0}, + [8] = {.lex_state = 0}, +}; + +static const uint16_t ts_parse_table[LARGE_STATE_COUNT][SYMBOL_COUNT] = { + [STATE(0)] = { + [ts_builtin_sym_end] = ACTIONS(1), + [anon_sym_LPAREN] = ACTIONS(1), + [anon_sym_RPAREN] = ACTIONS(1), + [sym_comment] = ACTIONS(3), + [sym_hex] = ACTIONS(1), + [sym_number] = ACTIONS(1), + [sym_string] = ACTIONS(1), + [sym_dot] = ACTIONS(1), + [sym_symbol] = ACTIONS(1), + }, + [STATE(1)] = { + [sym_source_file] = STATE(8), + [sym__form] = STATE(4), + [sym__sexp] = STATE(4), + [sym_list] = STATE(4), + [aux_sym_source_file_repeat1] = STATE(4), + [ts_builtin_sym_end] = ACTIONS(5), + [anon_sym_LPAREN] = ACTIONS(7), + [sym_comment] = ACTIONS(3), + [sym_hex] = ACTIONS(9), + [sym_number] = ACTIONS(9), + [sym_string] = ACTIONS(11), + [sym_dot] = ACTIONS(9), + [sym_symbol] = ACTIONS(9), + }, + [STATE(2)] = { + [sym__form] = STATE(2), + [sym__sexp] = STATE(2), + [sym_list] = STATE(2), + [aux_sym_source_file_repeat1] = STATE(2), + [ts_builtin_sym_end] = ACTIONS(13), + [anon_sym_LPAREN] = ACTIONS(15), + [anon_sym_RPAREN] = ACTIONS(13), + [sym_comment] = ACTIONS(3), + [sym_hex] = ACTIONS(18), + [sym_number] = ACTIONS(18), + [sym_string] = ACTIONS(21), + [sym_dot] = ACTIONS(18), + [sym_symbol] = ACTIONS(18), + }, + [STATE(3)] = { + [sym__form] = STATE(5), + [sym__sexp] = STATE(5), + [sym_list] = STATE(5), + [aux_sym_source_file_repeat1] = STATE(5), + [anon_sym_LPAREN] = ACTIONS(7), + [anon_sym_RPAREN] = ACTIONS(24), + [sym_comment] = ACTIONS(3), + [sym_hex] = ACTIONS(26), + [sym_number] = ACTIONS(26), + [sym_string] = ACTIONS(28), + [sym_dot] = ACTIONS(26), + [sym_symbol] = ACTIONS(26), + }, + [STATE(4)] = { + [sym__form] = STATE(2), + [sym__sexp] = STATE(2), + [sym_list] = STATE(2), + [aux_sym_source_file_repeat1] = STATE(2), + [ts_builtin_sym_end] = ACTIONS(30), + [anon_sym_LPAREN] = ACTIONS(7), + [sym_comment] = ACTIONS(3), + [sym_hex] = ACTIONS(32), + [sym_number] = ACTIONS(32), + [sym_string] = ACTIONS(34), + [sym_dot] = ACTIONS(32), + [sym_symbol] = ACTIONS(32), + }, + [STATE(5)] = { + [sym__form] = STATE(2), + [sym__sexp] = STATE(2), + [sym_list] = STATE(2), + [aux_sym_source_file_repeat1] = STATE(2), + [anon_sym_LPAREN] = ACTIONS(7), + [anon_sym_RPAREN] = ACTIONS(36), + [sym_comment] = ACTIONS(3), + [sym_hex] = ACTIONS(32), + [sym_number] = ACTIONS(32), + [sym_string] = ACTIONS(34), + [sym_dot] = ACTIONS(32), + [sym_symbol] = ACTIONS(32), + }, + [STATE(6)] = { + [ts_builtin_sym_end] = ACTIONS(38), + [anon_sym_LPAREN] = ACTIONS(38), + [anon_sym_RPAREN] = ACTIONS(38), + [sym_comment] = ACTIONS(3), + [sym_hex] = ACTIONS(40), + [sym_number] = ACTIONS(40), + [sym_string] = ACTIONS(38), + [sym_dot] = ACTIONS(40), + [sym_symbol] = ACTIONS(40), + }, + [STATE(7)] = { + [ts_builtin_sym_end] = ACTIONS(42), + [anon_sym_LPAREN] = ACTIONS(42), + [anon_sym_RPAREN] = ACTIONS(42), + [sym_comment] = ACTIONS(3), + [sym_hex] = ACTIONS(44), + [sym_number] = ACTIONS(44), + [sym_string] = ACTIONS(42), + [sym_dot] = ACTIONS(44), + [sym_symbol] = ACTIONS(44), + }, +}; + +static const uint16_t ts_small_parse_table[] = { + [0] = 2, + ACTIONS(3), 1, + sym_comment, + ACTIONS(46), 1, + ts_builtin_sym_end, +}; + +static const uint32_t ts_small_parse_table_map[] = { + [SMALL_STATE(8)] = 0, +}; + +static const TSParseActionEntry ts_parse_actions[] = { + [0] = {.entry = {.count = 0, .reusable = false}}, + [1] = {.entry = {.count = 1, .reusable = false}}, RECOVER(), + [3] = {.entry = {.count = 1, .reusable = true}}, SHIFT_EXTRA(), + [5] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_source_file, 0, 0, 0), + [7] = {.entry = {.count = 1, .reusable = true}}, SHIFT(3), + [9] = {.entry = {.count = 1, .reusable = false}}, SHIFT(4), + [11] = {.entry = {.count = 1, .reusable = true}}, SHIFT(4), + [13] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_source_file_repeat1, 2, 0, 0), + [15] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_source_file_repeat1, 2, 0, 0), SHIFT_REPEAT(3), + [18] = {.entry = {.count = 2, .reusable = false}}, REDUCE(aux_sym_source_file_repeat1, 2, 0, 0), SHIFT_REPEAT(2), + [21] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_source_file_repeat1, 2, 0, 0), SHIFT_REPEAT(2), + [24] = {.entry = {.count = 1, .reusable = true}}, SHIFT(6), + [26] = {.entry = {.count = 1, .reusable = false}}, SHIFT(5), + [28] = {.entry = {.count = 1, .reusable = true}}, SHIFT(5), + [30] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_source_file, 1, 0, 0), + [32] = {.entry = {.count = 1, .reusable = false}}, SHIFT(2), + [34] = {.entry = {.count = 1, .reusable = true}}, SHIFT(2), + [36] = {.entry = {.count = 1, .reusable = true}}, SHIFT(7), + [38] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_list, 2, 0, 0), + [40] = {.entry = {.count = 1, .reusable = false}}, REDUCE(sym_list, 2, 0, 0), + [42] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_list, 3, 0, 0), + [44] = {.entry = {.count = 1, .reusable = false}}, REDUCE(sym_list, 3, 0, 0), + [46] = {.entry = {.count = 1, .reusable = true}}, ACCEPT_INPUT(), +}; + +#ifdef __cplusplus +extern "C" { +#endif +#ifdef TREE_SITTER_HIDE_SYMBOLS +#define TS_PUBLIC +#elif defined(_WIN32) +#define TS_PUBLIC __declspec(dllexport) +#else +#define TS_PUBLIC __attribute__((visibility("default"))) +#endif + +TS_PUBLIC const TSLanguage *tree_sitter_chialisp(void) { + static const TSLanguage language = { + .abi_version = LANGUAGE_VERSION, + .symbol_count = SYMBOL_COUNT, + .alias_count = ALIAS_COUNT, + .token_count = TOKEN_COUNT, + .external_token_count = EXTERNAL_TOKEN_COUNT, + .state_count = STATE_COUNT, + .large_state_count = LARGE_STATE_COUNT, + .production_id_count = PRODUCTION_ID_COUNT, + .field_count = FIELD_COUNT, + .max_alias_sequence_length = MAX_ALIAS_SEQUENCE_LENGTH, + .parse_table = &ts_parse_table[0][0], + .small_parse_table = ts_small_parse_table, + .small_parse_table_map = ts_small_parse_table_map, + .parse_actions = ts_parse_actions, + .symbol_names = ts_symbol_names, + .symbol_metadata = ts_symbol_metadata, + .public_symbol_map = ts_symbol_map, + .alias_map = ts_non_terminal_alias_map, + .alias_sequences = &ts_alias_sequences[0][0], + .lex_modes = (const void*)ts_lex_modes, + .lex_fn = ts_lex, + .primary_state_ids = ts_primary_state_ids, + }; + return &language; +} +#ifdef __cplusplus +} +#endif diff --git a/internal/cbm/vendored/grammars/chialisp/tree_sitter/alloc.h b/internal/cbm/vendored/grammars/chialisp/tree_sitter/alloc.h new file mode 100644 index 000000000..1abdd1201 --- /dev/null +++ b/internal/cbm/vendored/grammars/chialisp/tree_sitter/alloc.h @@ -0,0 +1,54 @@ +#ifndef TREE_SITTER_ALLOC_H_ +#define TREE_SITTER_ALLOC_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +// Allow clients to override allocation functions +#ifdef TREE_SITTER_REUSE_ALLOCATOR + +extern void *(*ts_current_malloc)(size_t size); +extern void *(*ts_current_calloc)(size_t count, size_t size); +extern void *(*ts_current_realloc)(void *ptr, size_t size); +extern void (*ts_current_free)(void *ptr); + +#ifndef ts_malloc +#define ts_malloc ts_current_malloc +#endif +#ifndef ts_calloc +#define ts_calloc ts_current_calloc +#endif +#ifndef ts_realloc +#define ts_realloc ts_current_realloc +#endif +#ifndef ts_free +#define ts_free ts_current_free +#endif + +#else + +#ifndef ts_malloc +#define ts_malloc malloc +#endif +#ifndef ts_calloc +#define ts_calloc calloc +#endif +#ifndef ts_realloc +#define ts_realloc realloc +#endif +#ifndef ts_free +#define ts_free free +#endif + +#endif + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_ALLOC_H_ diff --git a/internal/cbm/vendored/grammars/chialisp/tree_sitter/array.h b/internal/cbm/vendored/grammars/chialisp/tree_sitter/array.h new file mode 100644 index 000000000..a17a574f0 --- /dev/null +++ b/internal/cbm/vendored/grammars/chialisp/tree_sitter/array.h @@ -0,0 +1,291 @@ +#ifndef TREE_SITTER_ARRAY_H_ +#define TREE_SITTER_ARRAY_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "./alloc.h" + +#include +#include +#include +#include +#include + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4101) +#elif defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" +#endif + +#define Array(T) \ + struct { \ + T *contents; \ + uint32_t size; \ + uint32_t capacity; \ + } + +/// Initialize an array. +#define array_init(self) \ + ((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL) + +/// Create an empty array. +#define array_new() \ + { NULL, 0, 0 } + +/// Get a pointer to the element at a given `index` in the array. +#define array_get(self, _index) \ + (assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index]) + +/// Get a pointer to the first element in the array. +#define array_front(self) array_get(self, 0) + +/// Get a pointer to the last element in the array. +#define array_back(self) array_get(self, (self)->size - 1) + +/// Clear the array, setting its size to zero. Note that this does not free any +/// memory allocated for the array's contents. +#define array_clear(self) ((self)->size = 0) + +/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is +/// less than the array's current capacity, this function has no effect. +#define array_reserve(self, new_capacity) \ + _array__reserve((Array *)(self), array_elem_size(self), new_capacity) + +/// Free any memory allocated for this array. Note that this does not free any +/// memory allocated for the array's contents. +#define array_delete(self) _array__delete((Array *)(self)) + +/// Push a new `element` onto the end of the array. +#define array_push(self, element) \ + (_array__grow((Array *)(self), 1, array_elem_size(self)), \ + (self)->contents[(self)->size++] = (element)) + +/// Increase the array's size by `count` elements. +/// New elements are zero-initialized. +#define array_grow_by(self, count) \ + do { \ + if ((count) == 0) break; \ + _array__grow((Array *)(self), count, array_elem_size(self)); \ + memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \ + (self)->size += (count); \ + } while (0) + +/// Append all elements from one array to the end of another. +#define array_push_all(self, other) \ + array_extend((self), (other)->size, (other)->contents) + +/// Append `count` elements to the end of the array, reading their values from the +/// `contents` pointer. +#define array_extend(self, count, contents) \ + _array__splice( \ + (Array *)(self), array_elem_size(self), (self)->size, \ + 0, count, contents \ + ) + +/// Remove `old_count` elements from the array starting at the given `index`. At +/// the same index, insert `new_count` new elements, reading their values from the +/// `new_contents` pointer. +#define array_splice(self, _index, old_count, new_count, new_contents) \ + _array__splice( \ + (Array *)(self), array_elem_size(self), _index, \ + old_count, new_count, new_contents \ + ) + +/// Insert one `element` into the array at the given `index`. +#define array_insert(self, _index, element) \ + _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element)) + +/// Remove one element from the array at the given `index`. +#define array_erase(self, _index) \ + _array__erase((Array *)(self), array_elem_size(self), _index) + +/// Pop the last element off the array, returning the element by value. +#define array_pop(self) ((self)->contents[--(self)->size]) + +/// Assign the contents of one array to another, reallocating if necessary. +#define array_assign(self, other) \ + _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self)) + +/// Swap one array with another +#define array_swap(self, other) \ + _array__swap((Array *)(self), (Array *)(other)) + +/// Get the size of the array contents +#define array_elem_size(self) (sizeof *(self)->contents) + +/// Search a sorted array for a given `needle` value, using the given `compare` +/// callback to determine the order. +/// +/// If an existing element is found to be equal to `needle`, then the `index` +/// out-parameter is set to the existing value's index, and the `exists` +/// out-parameter is set to true. Otherwise, `index` is set to an index where +/// `needle` should be inserted in order to preserve the sorting, and `exists` +/// is set to false. +#define array_search_sorted_with(self, compare, needle, _index, _exists) \ + _array__search_sorted(self, 0, compare, , needle, _index, _exists) + +/// Search a sorted array for a given `needle` value, using integer comparisons +/// of a given struct field (specified with a leading dot) to determine the order. +/// +/// See also `array_search_sorted_with`. +#define array_search_sorted_by(self, field, needle, _index, _exists) \ + _array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists) + +/// Insert a given `value` into a sorted array, using the given `compare` +/// callback to determine the order. +#define array_insert_sorted_with(self, compare, value) \ + do { \ + unsigned _index, _exists; \ + array_search_sorted_with(self, compare, &(value), &_index, &_exists); \ + if (!_exists) array_insert(self, _index, value); \ + } while (0) + +/// Insert a given `value` into a sorted array, using integer comparisons of +/// a given struct field (specified with a leading dot) to determine the order. +/// +/// See also `array_search_sorted_by`. +#define array_insert_sorted_by(self, field, value) \ + do { \ + unsigned _index, _exists; \ + array_search_sorted_by(self, field, (value) field, &_index, &_exists); \ + if (!_exists) array_insert(self, _index, value); \ + } while (0) + +// Private + +typedef Array(void) Array; + +/// This is not what you're looking for, see `array_delete`. +static inline void _array__delete(Array *self) { + if (self->contents) { + ts_free(self->contents); + self->contents = NULL; + self->size = 0; + self->capacity = 0; + } +} + +/// This is not what you're looking for, see `array_erase`. +static inline void _array__erase(Array *self, size_t element_size, + uint32_t index) { + assert(index < self->size); + char *contents = (char *)self->contents; + memmove(contents + index * element_size, contents + (index + 1) * element_size, + (self->size - index - 1) * element_size); + self->size--; +} + +/// This is not what you're looking for, see `array_reserve`. +static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) { + if (new_capacity > self->capacity) { + if (self->contents) { + self->contents = ts_realloc(self->contents, new_capacity * element_size); + } else { + self->contents = ts_malloc(new_capacity * element_size); + } + self->capacity = new_capacity; + } +} + +/// This is not what you're looking for, see `array_assign`. +static inline void _array__assign(Array *self, const Array *other, size_t element_size) { + _array__reserve(self, element_size, other->size); + self->size = other->size; + memcpy(self->contents, other->contents, self->size * element_size); +} + +/// This is not what you're looking for, see `array_swap`. +static inline void _array__swap(Array *self, Array *other) { + Array swap = *other; + *other = *self; + *self = swap; +} + +/// This is not what you're looking for, see `array_push` or `array_grow_by`. +static inline void _array__grow(Array *self, uint32_t count, size_t element_size) { + uint32_t new_size = self->size + count; + if (new_size > self->capacity) { + uint32_t new_capacity = self->capacity * 2; + if (new_capacity < 8) new_capacity = 8; + if (new_capacity < new_size) new_capacity = new_size; + _array__reserve(self, element_size, new_capacity); + } +} + +/// This is not what you're looking for, see `array_splice`. +static inline void _array__splice(Array *self, size_t element_size, + uint32_t index, uint32_t old_count, + uint32_t new_count, const void *elements) { + uint32_t new_size = self->size + new_count - old_count; + uint32_t old_end = index + old_count; + uint32_t new_end = index + new_count; + assert(old_end <= self->size); + + _array__reserve(self, element_size, new_size); + + char *contents = (char *)self->contents; + if (self->size > old_end) { + memmove( + contents + new_end * element_size, + contents + old_end * element_size, + (self->size - old_end) * element_size + ); + } + if (new_count > 0) { + if (elements) { + memcpy( + (contents + index * element_size), + elements, + new_count * element_size + ); + } else { + memset( + (contents + index * element_size), + 0, + new_count * element_size + ); + } + } + self->size += new_count - old_count; +} + +/// A binary search routine, based on Rust's `std::slice::binary_search_by`. +/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`. +#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \ + do { \ + *(_index) = start; \ + *(_exists) = false; \ + uint32_t size = (self)->size - *(_index); \ + if (size == 0) break; \ + int comparison; \ + while (size > 1) { \ + uint32_t half_size = size / 2; \ + uint32_t mid_index = *(_index) + half_size; \ + comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \ + if (comparison <= 0) *(_index) = mid_index; \ + size -= half_size; \ + } \ + comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \ + if (comparison == 0) *(_exists) = true; \ + else if (comparison < 0) *(_index) += 1; \ + } while (0) + +/// Helper macro for the `_sorted_by` routines below. This takes the left (existing) +/// parameter by reference in order to work with the generic sorting function above. +#define _compare_int(a, b) ((int)*(a) - (int)(b)) + +#ifdef _MSC_VER +#pragma warning(pop) +#elif defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_ARRAY_H_ diff --git a/internal/cbm/vendored/grammars/chialisp/tree_sitter/parser.h b/internal/cbm/vendored/grammars/chialisp/tree_sitter/parser.h new file mode 100644 index 000000000..858107deb --- /dev/null +++ b/internal/cbm/vendored/grammars/chialisp/tree_sitter/parser.h @@ -0,0 +1,286 @@ +#ifndef TREE_SITTER_PARSER_H_ +#define TREE_SITTER_PARSER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#define ts_builtin_sym_error ((TSSymbol)-1) +#define ts_builtin_sym_end 0 +#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 + +#ifndef TREE_SITTER_API_H_ +typedef uint16_t TSStateId; +typedef uint16_t TSSymbol; +typedef uint16_t TSFieldId; +typedef struct TSLanguage TSLanguage; +typedef struct TSLanguageMetadata { + uint8_t major_version; + uint8_t minor_version; + uint8_t patch_version; +} TSLanguageMetadata; +#endif + +typedef struct { + TSFieldId field_id; + uint8_t child_index; + bool inherited; +} TSFieldMapEntry; + +// Used to index the field and supertype maps. +typedef struct { + uint16_t index; + uint16_t length; +} TSMapSlice; + +typedef struct { + bool visible; + bool named; + bool supertype; +} TSSymbolMetadata; + +typedef struct TSLexer TSLexer; + +struct TSLexer { + int32_t lookahead; + TSSymbol result_symbol; + void (*advance)(TSLexer *, bool); + void (*mark_end)(TSLexer *); + uint32_t (*get_column)(TSLexer *); + bool (*is_at_included_range_start)(const TSLexer *); + bool (*eof)(const TSLexer *); + void (*log)(const TSLexer *, const char *, ...); +}; + +typedef enum { + TSParseActionTypeShift, + TSParseActionTypeReduce, + TSParseActionTypeAccept, + TSParseActionTypeRecover, +} TSParseActionType; + +typedef union { + struct { + uint8_t type; + TSStateId state; + bool extra; + bool repetition; + } shift; + struct { + uint8_t type; + uint8_t child_count; + TSSymbol symbol; + int16_t dynamic_precedence; + uint16_t production_id; + } reduce; + uint8_t type; +} TSParseAction; + +typedef struct { + uint16_t lex_state; + uint16_t external_lex_state; +} TSLexMode; + +typedef struct { + uint16_t lex_state; + uint16_t external_lex_state; + uint16_t reserved_word_set_id; +} TSLexerMode; + +typedef union { + TSParseAction action; + struct { + uint8_t count; + bool reusable; + } entry; +} TSParseActionEntry; + +typedef struct { + int32_t start; + int32_t end; +} TSCharacterRange; + +struct TSLanguage { + uint32_t abi_version; + uint32_t symbol_count; + uint32_t alias_count; + uint32_t token_count; + uint32_t external_token_count; + uint32_t state_count; + uint32_t large_state_count; + uint32_t production_id_count; + uint32_t field_count; + uint16_t max_alias_sequence_length; + const uint16_t *parse_table; + const uint16_t *small_parse_table; + const uint32_t *small_parse_table_map; + const TSParseActionEntry *parse_actions; + const char * const *symbol_names; + const char * const *field_names; + const TSMapSlice *field_map_slices; + const TSFieldMapEntry *field_map_entries; + const TSSymbolMetadata *symbol_metadata; + const TSSymbol *public_symbol_map; + const uint16_t *alias_map; + const TSSymbol *alias_sequences; + const TSLexerMode *lex_modes; + bool (*lex_fn)(TSLexer *, TSStateId); + bool (*keyword_lex_fn)(TSLexer *, TSStateId); + TSSymbol keyword_capture_token; + struct { + const bool *states; + const TSSymbol *symbol_map; + void *(*create)(void); + void (*destroy)(void *); + bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist); + unsigned (*serialize)(void *, char *); + void (*deserialize)(void *, const char *, unsigned); + } external_scanner; + const TSStateId *primary_state_ids; + const char *name; + const TSSymbol *reserved_words; + uint16_t max_reserved_word_set_size; + uint32_t supertype_count; + const TSSymbol *supertype_symbols; + const TSMapSlice *supertype_map_slices; + const TSSymbol *supertype_map_entries; + TSLanguageMetadata metadata; +}; + +static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, int32_t lookahead) { + uint32_t index = 0; + uint32_t size = len - index; + while (size > 1) { + uint32_t half_size = size / 2; + uint32_t mid_index = index + half_size; + const TSCharacterRange *range = &ranges[mid_index]; + if (lookahead >= range->start && lookahead <= range->end) { + return true; + } else if (lookahead > range->end) { + index = mid_index; + } + size -= half_size; + } + const TSCharacterRange *range = &ranges[index]; + return (lookahead >= range->start && lookahead <= range->end); +} + +/* + * Lexer Macros + */ + +#ifdef _MSC_VER +#define UNUSED __pragma(warning(suppress : 4101)) +#else +#define UNUSED __attribute__((unused)) +#endif + +#define START_LEXER() \ + bool result = false; \ + bool skip = false; \ + UNUSED \ + bool eof = false; \ + int32_t lookahead; \ + goto start; \ + next_state: \ + lexer->advance(lexer, skip); \ + start: \ + skip = false; \ + lookahead = lexer->lookahead; + +#define ADVANCE(state_value) \ + { \ + state = state_value; \ + goto next_state; \ + } + +#define ADVANCE_MAP(...) \ + { \ + static const uint16_t map[] = { __VA_ARGS__ }; \ + for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \ + if (map[i] == lookahead) { \ + state = map[i + 1]; \ + goto next_state; \ + } \ + } \ + } + +#define SKIP(state_value) \ + { \ + skip = true; \ + state = state_value; \ + goto next_state; \ + } + +#define ACCEPT_TOKEN(symbol_value) \ + result = true; \ + lexer->result_symbol = symbol_value; \ + lexer->mark_end(lexer); + +#define END_STATE() return result; + +/* + * Parse Table Macros + */ + +#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT) + +#define STATE(id) id + +#define ACTIONS(id) id + +#define SHIFT(state_value) \ + {{ \ + .shift = { \ + .type = TSParseActionTypeShift, \ + .state = (state_value) \ + } \ + }} + +#define SHIFT_REPEAT(state_value) \ + {{ \ + .shift = { \ + .type = TSParseActionTypeShift, \ + .state = (state_value), \ + .repetition = true \ + } \ + }} + +#define SHIFT_EXTRA() \ + {{ \ + .shift = { \ + .type = TSParseActionTypeShift, \ + .extra = true \ + } \ + }} + +#define REDUCE(symbol_name, children, precedence, prod_id) \ + {{ \ + .reduce = { \ + .type = TSParseActionTypeReduce, \ + .symbol = symbol_name, \ + .child_count = children, \ + .dynamic_precedence = precedence, \ + .production_id = prod_id \ + }, \ + }} + +#define RECOVER() \ + {{ \ + .type = TSParseActionTypeRecover \ + }} + +#define ACCEPT_INPUT() \ + {{ \ + .type = TSParseActionTypeAccept \ + }} + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_PARSER_H_ diff --git a/pkg/chocolatey/codebase-memory-mcp.nuspec b/pkg/chocolatey/codebase-memory-mcp.nuspec index d0509ecf1..3d2395fd2 100644 --- a/pkg/chocolatey/codebase-memory-mcp.nuspec +++ b/pkg/chocolatey/codebase-memory-mcp.nuspec @@ -18,7 +18,7 @@ release-owned runtime assets. Chocolatey retains the complete extracted runtime set and exposes its executable through a PATH shim. Full indexes an average repository in milliseconds, the Linux kernel (28M LOC) in -3 minutes. Answers structural queries in under 1ms. Supports 161 languages via +3 minutes. Answers structural queries in under 1ms. Supports 162 languages via vendored tree-sitter grammars. No hosted service or API key. Plug and play across 11 coding agents including Claude Code, Codex CLI, Gemini CLI, and Zed. diff --git a/pkg/npm/README.md b/pkg/npm/README.md index 48ea2e64d..614b81b1c 100644 --- a/pkg/npm/README.md +++ b/pkg/npm/README.md @@ -7,7 +7,7 @@ **The fastest and most efficient code intelligence engine for AI coding agents.** Full-indexes an average repository in milliseconds, the Linux kernel (28M LOC, 75K files) in 3 minutes. Answers structural queries in under 1ms. This npm wrapper downloads, verifies, and caches the selected native runtime set: the executable, its authenticated integration asset, and—when requested—the content-addressed UI pack. -High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across 161 languages — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 15 MCP tools. No hosted service or API key; this package requires Node.js to manage and launch the cached native runtime set. Plug and play across 45 automatic/conditional client surfaces. +High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across 162 languages — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 15 MCP tools. No hosted service or API key; this package requires Node.js to manage and launch the cached native runtime set. Plug and play across 45 automatic/conditional client surfaces. ## Installation @@ -29,7 +29,7 @@ Restart your agent. Say **"Index this project"** — done. - **Extreme indexing speed** — Linux kernel (28M LOC, 75K files) in 3 minutes. RAM-first pipeline with LZ4 compression and in-memory SQLite. - **Plug and play** — verified native runtime sets for macOS (arm64/amd64), Linux (arm64/amd64), and Windows (arm64/amd64). No Docker or API keys; Node.js owns package download, cache repair, and launch. -- **161 languages** — vendored tree-sitter grammars compiled into the binary. Nothing to install, nothing that breaks. +- **162 languages** — vendored tree-sitter grammars compiled into the binary. Nothing to install, nothing that breaks. - **120x fewer tokens** — 5 structural queries: ~3,400 tokens vs ~412,000 via file-by-file search. - **45 supported automatic/conditional client surfaces** — `install` configures the appropriate MCP, durable-context, and documented hook surfaces without widening client permissions. - **Detected automatically (39)** — Claude Code, Codex CLI, Gemini CLI, Zed, OpenCode, Antigravity, Aider, KiloCode, VS Code, Cursor, Windsurf, Augment / Auggie, OpenClaw, Kiro, Junie, Hermes, OpenHands, Cline, Warp, Qwen Code, GitHub Copilot CLI, Factory Droid, Crush, Goose, Mistral Vibe, Grok Build, Qoder CLI, Kimi Code CLI, GitLab Duo CLI, Rovo Dev CLI, Amp, Devin CLI / Local, Tabnine, Amazon Q Developer IDE, CodeBuddy Code CLI, IBM Bob Shell, Pochi, Pi, and Oh My Pi (omp). diff --git a/scripts/audit-license-provenance.py b/scripts/audit-license-provenance.py index edbdbfdb9..d22d4d15b 100755 --- a/scripts/audit-license-provenance.py +++ b/scripts/audit-license-provenance.py @@ -20,7 +20,11 @@ import sys ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) GRAMMARS = os.path.join(ROOT, "internal/cbm/vendored/grammars") -FIRST_PARTY = {"cobol", "form", "janet", "magma", "protobuf", "wolfram"} +# Grammars authored in this repository. This is a REGISTRATION, not an +# exemption: membership only routes the check -- the audit still requires the +# vendored LICENSE to be byte-equal to the project root LICENSE, and reports +# FIRST-PARTY-VAR (a failing verdict) when it is not. +FIRST_PARTY = {"chialisp", "cobol", "form", "janet", "magma", "protobuf", "wolfram"} FORKS = { # self-maintained forks: vendored LICENSE must match the original upstream # arkts: our own ArkTS fork of tree-sitter-typescript. The vendored LICENSE is # upstream's MIT verbatim, which is what MIT requires of a derivative; our own diff --git a/scripts/package-release.sh b/scripts/package-release.sh index 1ee35c539..ff399c76d 100755 --- a/scripts/package-release.sh +++ b/scripts/package-release.sh @@ -293,7 +293,7 @@ build_mcpb_bundle() { "name": "codebase-memory-mcp", "display_name": "Codebase Memory", "version": "$mcpb_version", - "description": "Codebase knowledge graph for AI agents — 161 languages, sub-ms queries, 99% fewer tokens.", + "description": "Codebase knowledge graph for AI agents — 162 languages, sub-ms queries, 99% fewer tokens.", "author": { "name": "DeusData", "url": "https://github.com/DeusData" }, "repository": { "type": "git", "url": "https://github.com/DeusData/codebase-memory-mcp" }, "homepage": "https://deusdata.github.io/codebase-memory-mcp/", diff --git a/scripts/vendored-checksums.txt b/scripts/vendored-checksums.txt index 905af7d48..992216603 100644 --- a/scripts/vendored-checksums.txt +++ b/scripts/vendored-checksums.txt @@ -5,7 +5,7 @@ c5cfb43042b6b72045f4ba997834d0a7786d2793d91680868b5815b39f14fc78 internal/cbm/v b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea internal/cbm/vendored/common/tree_sitter/alloc.h 31e60a1bff6f715afacce03b5b70efe42b58371b4f9595dd4af52a577ff9608c internal/cbm/vendored/common/tree_sitter/array.h 180b893c8734778fd32f372dfbc27bd6ad1cd2221f26150b31256ff6716320d2 internal/cbm/vendored/common/tree_sitter/parser.h -4db1d964d82c2f1c50ff8fc0de7a85708fa2f1884d0afbc587c59dd103163967 internal/cbm/vendored/grammars/MANIFEST.md +1fec60a59cb72bd61b4f9e771a77cc184597f49caf5d33f2e2a40f5739518571 internal/cbm/vendored/grammars/MANIFEST.md ad8425038de519f8c4e3e9339feebf99dfad8a6002dcf79227d91402780a32cc internal/cbm/vendored/grammars/ada/LICENSE 02805ec13939b749c891567be36bf024b09034e04c80683a1ae667272458d549 internal/cbm/vendored/grammars/ada/parser.c 115a75d000bef9c70c4de7dfbb7f2a80b90cbb14ba265056dc7abb1ef3b9b6db internal/cbm/vendored/grammars/ada/tree_sitter/alloc.h @@ -118,6 +118,11 @@ eeadc5447401a424faf2a9214727debb25a2c3f2b8315e43c86b6ed48bbb1290 internal/cbm/v b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea internal/cbm/vendored/grammars/cfscript/tree_sitter/alloc.h 31e60a1bff6f715afacce03b5b70efe42b58371b4f9595dd4af52a577ff9608c internal/cbm/vendored/grammars/cfscript/tree_sitter/array.h 180b893c8734778fd32f372dfbc27bd6ad1cd2221f26150b31256ff6716320d2 internal/cbm/vendored/grammars/cfscript/tree_sitter/parser.h +1f58f9911dc5e3bcb96de28bb28e7b6bb7eb323952d29569c5d7214a152146bb internal/cbm/vendored/grammars/chialisp/LICENSE +c55f7903853b47242a8f3e4d3a945bfe97817deb9673561f1a806b89d734d2f7 internal/cbm/vendored/grammars/chialisp/parser.c +b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea internal/cbm/vendored/grammars/chialisp/tree_sitter/alloc.h +5bdf6ed1a78e3409fd443e085ca967a64c188a5d082aaf7f819bccd53a471c94 internal/cbm/vendored/grammars/chialisp/tree_sitter/array.h +180b893c8734778fd32f372dfbc27bd6ad1cd2221f26150b31256ff6716320d2 internal/cbm/vendored/grammars/chialisp/tree_sitter/parser.h a2010f343487d3f7618affe54f789f5487602331c0a8d03f49e9a7c547cf0499 internal/cbm/vendored/grammars/clojure/LICENSE 93e7bc5496b929e8627c95ec39fa78f267b084868f7ecc2ef34df00919eb9b06 internal/cbm/vendored/grammars/clojure/parser.c 435ac887a97b4199fcf97707649aa987a94524f0e023f4e8287eae5dae49453f internal/cbm/vendored/grammars/clojure/tree_sitter/parser.h diff --git a/server.json b/server.json index 93bcccc3b..0241f5527 100644 --- a/server.json +++ b/server.json @@ -2,7 +2,7 @@ "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json", "name": "io.github.DeusData/codebase-memory-mcp", "title": "Codebase Memory", - "description": "Codebase knowledge graph for AI agents \u2014 161 languages, sub-ms queries, 99% fewer tokens.", + "description": "Codebase knowledge graph for AI agents \u2014 162 languages, sub-ms queries, 99% fewer tokens.", "repository": { "url": "https://github.com/DeusData/codebase-memory-mcp", "source": "github" diff --git a/src/discover/language.c b/src/discover/language.c index 3cb0bf6f8..666db536f 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -574,6 +574,11 @@ static const ext_entry_t EXT_TABLE[] = { /* Scheme */ {".scm", CBM_LANG_SCHEME}, + /* Chialisp — .clsp puzzles, .clib/.clinc includable libraries */ + {".clsp", CBM_LANG_CHIALISP}, + {".clib", CBM_LANG_CHIALISP}, + {".clinc", CBM_LANG_CHIALISP}, + /* Slang */ {".slang", CBM_LANG_SLANG}, @@ -783,6 +788,7 @@ static const char *LANG_NAMES[CBM_LANG_COUNT] = { [CBM_LANG_DLANG] = "D", [CBM_LANG_NIM] = "Nim", [CBM_LANG_SCHEME] = "Scheme", + [CBM_LANG_CHIALISP] = "Chialisp", [CBM_LANG_FENNEL] = "Fennel", [CBM_LANG_FISH] = "Fish", [CBM_LANG_AWK] = "AWK", diff --git a/src/discover/userconfig.c b/src/discover/userconfig.c index c97fd48c9..8ba63d5bd 100644 --- a/src/discover/userconfig.c +++ b/src/discover/userconfig.c @@ -142,6 +142,7 @@ static const lang_name_entry_t LANG_NAME_TABLE[] = { {"matlab", CBM_LANG_MATLAB}, {"mojo", CBM_LANG_MOJO}, {"plsql", CBM_LANG_PLSQL}, + {"chialisp", CBM_LANG_CHIALISP}, {"lean", CBM_LANG_LEAN}, {"form", CBM_LANG_FORM}, {"magma", CBM_LANG_MAGMA}, diff --git a/tests/repro/repro_call_argument_matrix_b.c b/tests/repro/repro_call_argument_matrix_b.c index 733781344..e67bd19f4 100644 --- a/tests/repro/repro_call_argument_matrix_b.c +++ b/tests/repro/repro_call_argument_matrix_b.c @@ -844,6 +844,17 @@ static const char OBJECTSCRIPT_ROUTINE_BARE[] = "SAMPLE\n" "Run(watched)\n" " Quit watched\n"; +/* Chialisp: a .clib wraps its definitions in one enclosing list; a call is a + * `list` whose head symbol is the callee, and a parameter list is a `list` + * too — so `total_calls` here also pins that a binder is not an invocation. */ +static const char CHIALISP_INSIDE[] = "(\n" + " (defun accept (value) value)\n" + " (defun run (watched) (accept watched))\n" + ")\n"; +static const char CHIALISP_BARE[] = "(\n" + " (defun run (watched) watched)\n" + ")\n"; + static const char PLSQL_INSIDE[] = "CREATE OR REPLACE PACKAGE BODY sample_pkg AS\n" " FUNCTION accept(value NUMBER) RETURN NUMBER IS\n" " BEGIN\n" @@ -1007,6 +1018,10 @@ static const RoutineArgumentCase PLSQL_CASE = ROUTINE_ARGUMENT_CASE( "PLSQL", CBM_LANG_PLSQL, "sample_pkg.pkb", PLSQL_INSIDE, PLSQL_BARE, "ref_call", "run", "accept", "watched", 1, 1, 0, "native PL/SQL package-body routine application"); +static const RoutineArgumentCase CHIALISP_CASE = ROUTINE_ARGUMENT_CASE( + "CHIALISP", CBM_LANG_CHIALISP, "sample.clib", CHIALISP_INSIDE, CHIALISP_BARE, "list", "run", + "accept", "watched", 1, 1, 0, "Chialisp list application and symbol-reference vocabulary"); + static const ModuleArgumentCase JUST_CASE = MODULE_ARGUMENT_CASE( "JUST", CBM_LANG_JUST, "justfile", JUST_INSIDE, JUST_BARE, "function_call", "uppercase", "watched", 1, 0, "Just expression call; recipe dependency is checked separately"); @@ -1197,6 +1212,7 @@ DEFINE_ROUTINE_ARGUMENT_TEST(mojo, MOJO_CASE) DEFINE_ROUTINE_ARGUMENT_TEST(objectscript_udl, OBJECTSCRIPT_UDL_CASE) DEFINE_ROUTINE_ARGUMENT_TEST(objectscript_routine, OBJECTSCRIPT_ROUTINE_CASE) DEFINE_ROUTINE_ARGUMENT_TEST(plsql, PLSQL_CASE) +DEFINE_ROUTINE_ARGUMENT_TEST(chialisp, CHIALISP_CASE) #undef DEFINE_ROUTINE_ARGUMENT_TEST @@ -1266,15 +1282,15 @@ TEST(repro_call_argument_matrix_b_domain_bitbake) { } enum { - ROUTINE_ARGUMENT_LANGUAGE_COUNT = 37, + ROUTINE_ARGUMENT_LANGUAGE_COUNT = 38, MODULE_ARGUMENT_LANGUAGE_COUNT = 4, DOMAIN_CONTROL_LANGUAGE_COUNT = 6, MATRIX_LANGUAGE_COUNT = ROUTINE_ARGUMENT_LANGUAGE_COUNT + MODULE_ARGUMENT_LANGUAGE_COUNT + DOMAIN_CONTROL_LANGUAGE_COUNT, }; -_Static_assert(MATRIX_LANGUAGE_COUNT == 47, - "RACKET..PLSQL call-capable matrix must contain exactly 47 " +_Static_assert(MATRIX_LANGUAGE_COUNT == 48, + "RACKET..CHIALISP call-capable matrix must contain exactly 48 " "language rows"); #define MATRIX_B_LANGUAGE_ROWS(X) \ @@ -1317,6 +1333,7 @@ _Static_assert(MATRIX_LANGUAGE_COUNT == 47, X(repro_call_argument_matrix_b_routine_objectscript_routine, \ OBJECTSCRIPT_ROUTINE_CASE.identity.language) \ X(repro_call_argument_matrix_b_routine_plsql, PLSQL_CASE.identity.language) \ + X(repro_call_argument_matrix_b_routine_chialisp, CHIALISP_CASE.identity.language) \ X(repro_call_argument_matrix_b_module_just, JUST_CASE.identity.language) \ X(repro_call_argument_matrix_b_module_gotemplate, GOTEMPLATE_CASE.identity.language) \ X(repro_call_argument_matrix_b_module_linkerscript, LINKERSCRIPT_CASE.identity.language) \ diff --git a/tests/repro/repro_call_node_manifest.c b/tests/repro/repro_call_node_manifest.c index 97042df51..4405590b3 100644 --- a/tests/repro/repro_call_node_manifest.c +++ b/tests/repro/repro_call_node_manifest.c @@ -54,9 +54,9 @@ typedef struct { } CallNodeManifestEntry; enum { - EXPECTED_HISTORICAL_CALL_NODE_TOTAL = 222, - EXPECTED_ACTIVE_PRIMARY_TOTAL = 192, - EXPECTED_DIRECT_CALLS = 157, + EXPECTED_HISTORICAL_CALL_NODE_TOTAL = 223, + EXPECTED_ACTIVE_PRIMARY_TOTAL = 193, + EXPECTED_DIRECT_CALLS = 158, EXPECTED_CONSTRUCTOR_CALLS = 21, EXPECTED_OPERATOR_CALLS = 12, EXPECTED_IMPLICIT_CALLS = 2, @@ -65,7 +65,7 @@ enum { EXPECTED_CALLEE_WRAPPERS = 8, EXPECTED_ARGUMENT_WRAPPERS = 1, EXPECTED_CONTROL_OR_DOCUMENT_NONCALLS = 6, - EXPECTED_PRIMARY_OWNERS = 192, + EXPECTED_PRIMARY_OWNERS = 193, EXPECTED_SYNTHETIC_OWNERS = 10, EXPECTED_NO_CALL_EDGE_OWNERS = 20, EXPECTED_PRIMARY_OPERATOR_CALLS = 4, @@ -220,6 +220,7 @@ static const CallNodeManifestEntry CALL_NODE_MANIFEST[] = { ENTRY(CBM_LANG_DLANG, "function_call_expression", DIRECT_CALL, NO_CALL_EDGE, true, false), ENTRY(CBM_LANG_DLANG, "new_expression", CONSTRUCTOR_CALL, PRIMARY_EXTRACTOR, true, true), ENTRY(CBM_LANG_SCHEME, "list", DIRECT_CALL, PRIMARY_EXTRACTOR, true, true), + ENTRY(CBM_LANG_CHIALISP, "list", DIRECT_CALL, PRIMARY_EXTRACTOR, true, true), ENTRY(CBM_LANG_FENNEL, "list", DIRECT_CALL, PRIMARY_EXTRACTOR, true, true), ENTRY(CBM_LANG_FISH, "command", DSL_INVOCATION, PRIMARY_EXTRACTOR, true, true), ENTRY(CBM_LANG_AWK, "func_call", DIRECT_CALL, PRIMARY_EXTRACTOR, true, true), diff --git a/tests/repro/repro_invariant_breadth.c b/tests/repro/repro_invariant_breadth.c index b4becd790..6aa8924d3 100644 --- a/tests/repro/repro_invariant_breadth.c +++ b/tests/repro/repro_invariant_breadth.c @@ -10,7 +10,7 @@ * * QUALITY_ANALYSIS.md gap #6 reports 27 languages failing this. This file * is the "large breadth table" — one per-language case, table-driven, asserting - * the invariant across 26 languages. + * the invariant across 27 languages. * * Fixture design rule: * Each fixture defines exactly TWO functions: a callee (helper) and a caller @@ -27,6 +27,9 @@ * r, julia, dart, groovy, commonlisp, powershell, ada, clojure, * fsharp, racket, rescript, scheme (12 cases) * + * Added after the original snapshot: + * chialisp (1 case, expected GREEN — lisp-family scope attribution) + * * Note: the "suspicious" group (r, julia, ...) from QUALITY_ANALYSIS may be * GREEN because the calls-breadth table (test_lang_contract.c) already shows * expect_calls=true for most. The module-sourcing assertion is STRICTER: a @@ -51,19 +54,17 @@ /* ---- helper: count CALLS edges by source-node label --------------------- */ -static int ib_calls_from_label(cbm_store_t *store, const char *project, - const char *label) { +static int ib_calls_from_label(cbm_store_t *store, const char *project, const char *label) { cbm_edge_t *edges = NULL; int edge_count = 0; - if (cbm_store_find_edges_by_type(store, project, "CALLS", - &edges, &edge_count) != CBM_STORE_OK) { + if (cbm_store_find_edges_by_type(store, project, "CALLS", &edges, &edge_count) != + CBM_STORE_OK) { return -1; } int total = 0; for (int i = 0; i < edge_count; i++) { cbm_node_t src = {0}; - if (cbm_store_find_node_by_id(store, edges[i].source_id, - &src) != CBM_STORE_OK) { + if (cbm_store_find_node_by_id(store, edges[i].source_id, &src) != CBM_STORE_OK) { continue; } if (src.label && strcmp(src.label, label) == 0) { @@ -135,42 +136,40 @@ static const IBCase IB_CASES[] = { /* CALLS edge is produced -- but it may still be Module-sourced. */ /* ------------------------------------------------------------------ */ - { - "r", "a.R", - "helper <- function(x) {\n" - " x * 2\n" - "}\n" - "\n" - "run <- function() {\n" - " helper(21)\n" - "}\n", - /* - * R: extract_calls.c has an R branch that reads the callee from the - * call node's first child. However, enclosing-function detection - * for R may fall back to Module if func_kinds_for_lang does not - * include R's "function_definition" node type. RED when the CALLS - * edge is sourced at Module instead of the "run" Function node. - */ - 0, "R enclosing-function detection likely missing from func_kinds_for_lang; " - "call may be sourced at Module" - }, + {"r", "a.R", + "helper <- function(x) {\n" + " x * 2\n" + "}\n" + "\n" + "run <- function() {\n" + " helper(21)\n" + "}\n", + /* + * R: extract_calls.c has an R branch that reads the callee from the + * call node's first child. However, enclosing-function detection + * for R may fall back to Module if func_kinds_for_lang does not + * include R's "function_definition" node type. RED when the CALLS + * edge is sourced at Module instead of the "run" Function node. + */ + 0, + "R enclosing-function detection likely missing from func_kinds_for_lang; " + "call may be sourced at Module"}, - { - "julia", "a.jl", - "function helper(x)\n" - " return x + 1\n" - "end\n" - "\n" - "function run(n)\n" - " return helper(n)\n" - "end\n", - /* - * Julia: same issue -- function body extraction may not detect the - * enclosing Julia function node correctly, sourcing the call at Module. - */ - 0, "Julia enclosing-function detection may not map function_definition to " - "a callable QN; call sourced at Module" - }, + {"julia", "a.jl", + "function helper(x)\n" + " return x + 1\n" + "end\n" + "\n" + "function run(n)\n" + " return helper(n)\n" + "end\n", + /* + * Julia: same issue -- function body extraction may not detect the + * enclosing Julia function node correctly, sourcing the call at Module. + */ + 0, + "Julia enclosing-function detection may not map function_definition to " + "a callable QN; call sourced at Module"}, /* ------------------------------------------------------------------ */ /* EXPECTED-GREEN GROUP (regression guards) */ @@ -179,191 +178,163 @@ static const IBCase IB_CASES[] = { /* them will turn the corresponding case RED. */ /* ------------------------------------------------------------------ */ - { - "elixir", "a.ex", - "defmodule Sample do\n" - " def helper(x) do\n" - " x + 1\n" - " end\n" - "\n" - " def run do\n" - " helper(41)\n" - " end\n" - "end\n", - 1, NULL - }, + {"elixir", "a.ex", + "defmodule Sample do\n" + " def helper(x) do\n" + " x + 1\n" + " end\n" + "\n" + " def run do\n" + " helper(41)\n" + " end\n" + "end\n", + 1, NULL}, - { - "ocaml", "a.ml", - "let helper x = x + 1\n" - "\n" - "let run () =\n" - " let result = helper 41 in\n" - " print_int result\n", - 1, NULL - }, + {"ocaml", "a.ml", + "let helper x = x + 1\n" + "\n" + "let run () =\n" + " let result = helper 41 in\n" + " print_int result\n", + 1, NULL}, - { - "fortran", "a.f90", - "function helper(x) result(y)\n" - " integer, intent(in) :: x\n" - " integer :: y\n" - " y = x + 1\n" - "end function helper\n" - "\n" - "function run(n) result(total)\n" - " integer, intent(in) :: n\n" - " integer :: total\n" - " total = helper(n) + helper(n + 1)\n" - "end function run\n", - 1, NULL - }, + {"fortran", "a.f90", + "function helper(x) result(y)\n" + " integer, intent(in) :: x\n" + " integer :: y\n" + " y = x + 1\n" + "end function helper\n" + "\n" + "function run(n) result(total)\n" + " integer, intent(in) :: n\n" + " integer :: total\n" + " total = helper(n) + helper(n + 1)\n" + "end function run\n", + 1, NULL}, - { - "pascal", "a.pas", - "procedure Helper(x: Integer);\n" - "begin\n" - " WriteLn(x);\n" - "end;\n" - "\n" - "procedure Run;\n" - "begin\n" - " Helper(1);\n" - "end;\n", - 1, NULL - }, + {"pascal", "a.pas", + "procedure Helper(x: Integer);\n" + "begin\n" + " WriteLn(x);\n" + "end;\n" + "\n" + "procedure Run;\n" + "begin\n" + " Helper(1);\n" + "end;\n", + 1, NULL}, - { - "cuda", "a.cu", - "__device__ int helper(int x) {\n" - " return x * 2;\n" - "}\n" - "\n" - "__global__ void run(int *out) {\n" - " out[0] = helper(21);\n" - "}\n", - 1, NULL - }, + {"cuda", "a.cu", + "__device__ int helper(int x) {\n" + " return x * 2;\n" + "}\n" + "\n" + "__global__ void run(int *out) {\n" + " out[0] = helper(21);\n" + "}\n", + 1, NULL}, - { - "d", "a.d", - "int helper(int x)\n" - "{\n" - " return x + 1;\n" - "}\n" - "\n" - "void run()\n" - "{\n" - " int y = helper(41);\n" - "}\n", - 1, NULL - }, + {"d", "a.d", + "int helper(int x)\n" + "{\n" + " return x + 1;\n" + "}\n" + "\n" + "void run()\n" + "{\n" + " int y = helper(41);\n" + "}\n", + 1, NULL}, - { - "glsl", "a.glsl", - "float helper(float x) {\n" - " return x * 2.0;\n" - "}\n" - "\n" - "void run() {\n" - " float y = helper(3.0);\n" - "}\n", - 1, NULL - }, + {"glsl", "a.glsl", + "float helper(float x) {\n" + " return x * 2.0;\n" + "}\n" + "\n" + "void run() {\n" + " float y = helper(3.0);\n" + "}\n", + 1, NULL}, - { - "hlsl", "a.hlsl", - "float helper(float x)\n" - "{\n" - " return x * 2.0;\n" - "}\n" - "\n" - "float run(float v)\n" - "{\n" - " return helper(v) + 1.0;\n" - "}\n", - 1, NULL - }, + {"hlsl", "a.hlsl", + "float helper(float x)\n" + "{\n" + " return x * 2.0;\n" + "}\n" + "\n" + "float run(float v)\n" + "{\n" + " return helper(v) + 1.0;\n" + "}\n", + 1, NULL}, - { - "ispc", "a.ispc", - "static inline uniform float helper(uniform float x) {\n" - " return x * 2.0f;\n" - "}\n" - "\n" - "export void run(uniform float in[], uniform float out[],\n" - " uniform int n) {\n" - " foreach (i = 0 ... n) {\n" - " out[i] = helper(in[i]);\n" - " }\n" - "}\n", - 1, NULL - }, + {"ispc", "a.ispc", + "static inline uniform float helper(uniform float x) {\n" + " return x * 2.0f;\n" + "}\n" + "\n" + "export void run(uniform float in[], uniform float out[],\n" + " uniform int n) {\n" + " foreach (i = 0 ... n) {\n" + " out[i] = helper(in[i]);\n" + " }\n" + "}\n", + 1, NULL}, - { - "odin", "a.odin", - "package fixture\n" - "\n" - "helper :: proc() -> int {\n" - "\treturn 42\n" - "}\n" - "\n" - "run :: proc() {\n" - "\tx := helper()\n" - "\t_ = x\n" - "}\n", - 1, NULL - }, + {"odin", "a.odin", + "package fixture\n" + "\n" + "helper :: proc() -> int {\n" + "\treturn 42\n" + "}\n" + "\n" + "run :: proc() {\n" + "\tx := helper()\n" + "\t_ = x\n" + "}\n", + 1, NULL}, - { - "slang", "a.slang", - "void helper()\n" - "{\n" - " int x = 1;\n" - "}\n" - "\n" - "void run()\n" - "{\n" - " helper();\n" - "}\n", - 1, NULL - }, + {"slang", "a.slang", + "void helper()\n" + "{\n" + " int x = 1;\n" + "}\n" + "\n" + "void run()\n" + "{\n" + " helper();\n" + "}\n", + 1, NULL}, - { - "squirrel", "a.nut", - "function helper(x) {\n" - " return x + 1;\n" - "}\n" - "\n" - "function run() {\n" - " return helper(41);\n" - "}\n", - 1, NULL - }, + {"squirrel", "a.nut", + "function helper(x) {\n" + " return x + 1;\n" + "}\n" + "\n" + "function run() {\n" + " return helper(41);\n" + "}\n", + 1, NULL}, - { - "vimscript", "a.vim", - "function! Helper() abort\n" - " return 1\n" - "endfunction\n" - "\n" - "function! Run() abort\n" - " call Helper()\n" - "endfunction\n", - 1, NULL - }, + {"vimscript", "a.vim", + "function! Helper() abort\n" + " return 1\n" + "endfunction\n" + "\n" + "function! Run() abort\n" + " call Helper()\n" + "endfunction\n", + 1, NULL}, - { - "cairo", "a.cairo", - "fn helper(x: felt252) -> felt252 {\n" - " x + 1\n" - "}\n" - "\n" - "fn run() -> felt252 {\n" - " helper(41)\n" - "}\n", - 1, NULL - }, + {"cairo", "a.cairo", + "fn helper(x: felt252) -> felt252 {\n" + " x + 1\n" + "}\n" + "\n" + "fn run() -> felt252 {\n" + " helper(41)\n" + "}\n", + 1, NULL}, /* ------------------------------------------------------------------ */ /* KNOWN-GAP GROUP */ @@ -374,163 +345,169 @@ static const IBCase IB_CASES[] = { /* calls >= 1 AND module_calls == 0. */ /* ------------------------------------------------------------------ */ - { - "dart", "a.dart", - "void helper() {\n" - " print('helper');\n" - "}\n" - "\n" - "void run() {\n" - " helper();\n" - "}\n", - /* - * Dart: selector call node carries no callee field and the first child - * is not an identifier; no dart branch in extract_calls.c. No CALLS - * edge is produced at all, so callable-sourcing cannot be tested - * independently. Both gaps (no CALLS + callable-sourcing) are RED. - */ - 0, "selector call node: no callee field, first child not identifier; " - "no dart branch in extract_calls.c" - }, + {"dart", "a.dart", + "void helper() {\n" + " print('helper');\n" + "}\n" + "\n" + "void run() {\n" + " helper();\n" + "}\n", + /* + * Dart: selector call node carries no callee field and the first child + * is not an identifier; no dart branch in extract_calls.c. No CALLS + * edge is produced at all, so callable-sourcing cannot be tested + * independently. Both gaps (no CALLS + callable-sourcing) are RED. + */ + 0, + "selector call node: no callee field, first child not identifier; " + "no dart branch in extract_calls.c"}, - { - "groovy", "a.groovy", - "def helper() {\n" - " println 'helping'\n" - "}\n" - "\n" - "def run() {\n" - " helper()\n" - "}\n", - /* - * Groovy: function_call callee not on a function/name field and first - * child is not 'identifier'; no groovy branch in extract_calls.c. - */ - 0, "function_call callee not on function/name field; " - "first child is not identifier; no groovy branch in extract_calls.c" - }, + {"groovy", "a.groovy", + "def helper() {\n" + " println 'helping'\n" + "}\n" + "\n" + "def run() {\n" + " helper()\n" + "}\n", + /* + * Groovy: function_call callee not on a function/name field and first + * child is not 'identifier'; no groovy branch in extract_calls.c. + */ + 0, + "function_call callee not on function/name field; " + "first child is not identifier; no groovy branch in extract_calls.c"}, - { - "commonlisp", "a.lisp", - "(defun helper (x)\n" - " (* x 2))\n" - "\n" - "(defun run ()\n" - " (helper 21))\n", - /* - * Common Lisp: list_lit call head is sym_lit not identifier; - * no commonlisp branch in extract_callee_name. - */ - 0, "list_lit call head is sym_lit not identifier; " - "no commonlisp branch in extract_callee_name" - }, + {"commonlisp", "a.lisp", + "(defun helper (x)\n" + " (* x 2))\n" + "\n" + "(defun run ()\n" + " (helper 21))\n", + /* + * Common Lisp: list_lit call head is sym_lit not identifier; + * no commonlisp branch in extract_callee_name. + */ + 0, + "list_lit call head is sym_lit not identifier; " + "no commonlisp branch in extract_callee_name"}, - { - "powershell", "a.ps1", - "function helper {\n" - " Write-Output 'hi'\n" - "}\n" - "\n" - "function run {\n" - " helper\n" - "}\n", - /* - * PowerShell: command node child is command_name not identifier; - * extract_scripting_callee handles MATLAB not PowerShell. - */ - 0, "command node child is command_name not identifier; " - "extract_scripting_callee handles MATLAB not PowerShell" - }, + {"powershell", "a.ps1", + "function helper {\n" + " Write-Output 'hi'\n" + "}\n" + "\n" + "function run {\n" + " helper\n" + "}\n", + /* + * PowerShell: command node child is command_name not identifier; + * extract_scripting_callee handles MATLAB not PowerShell. + */ + 0, + "command node child is command_name not identifier; " + "extract_scripting_callee handles MATLAB not PowerShell"}, - { - "ada", "a.adb", - "procedure Run is\n" - " procedure Helper is\n" - " begin\n" - " null;\n" - " end Helper;\n" - "begin\n" - " Helper;\n" - "end Run;\n", - /* - * Ada: procedure_call_statement callee did not resolve to a CALLS edge; - * no Ada branch in extract_calls.c. - */ - 0, "procedure_call_statement callee not resolved; " - "no Ada branch in extract_calls.c" - }, + {"ada", "a.adb", + "procedure Run is\n" + " procedure Helper is\n" + " begin\n" + " null;\n" + " end Helper;\n" + "begin\n" + " Helper;\n" + "end Run;\n", + /* + * Ada: procedure_call_statement callee did not resolve to a CALLS edge; + * no Ada branch in extract_calls.c. + */ + 0, + "procedure_call_statement callee not resolved; " + "no Ada branch in extract_calls.c"}, - { - "clojure", "a.clj", - "(defn helper [] 42)\n" - "\n" - "(defn run [] (helper))\n", - /* - * Clojure: lisp call is a list_lit whose head is a sym_lit (not a - * field, not a first-child 'identifier'); no lisp branch in - * extract_callee_name. - */ - 0, "list_lit head is sym_lit not identifier; " - "no lisp/clojure branch in extract_callee_name" - }, + {"clojure", "a.clj", + "(defn helper [] 42)\n" + "\n" + "(defn run [] (helper))\n", + /* + * Clojure: lisp call is a list_lit whose head is a sym_lit (not a + * field, not a first-child 'identifier'); no lisp branch in + * extract_callee_name. + */ + 0, + "list_lit head is sym_lit not identifier; " + "no lisp/clojure branch in extract_callee_name"}, - { - "fsharp", "a.fs", - "let helper x = x + 1\n" - "\n" - "let run () = helper 41\n", - /* - * F#: application_expression callee head is a long_identifier_or_op - * wrapper, not a bare identifier/field; no fsharp callee branch. - */ - 0, "application_expression callee head is long_identifier_or_op wrapper; " - "no fsharp callee branch in extract_callee_name" - }, + {"fsharp", "a.fs", + "let helper x = x + 1\n" + "\n" + "let run () = helper 41\n", + /* + * F#: application_expression callee head is a long_identifier_or_op + * wrapper, not a bare identifier/field; no fsharp callee branch. + */ + 0, + "application_expression callee head is long_identifier_or_op wrapper; " + "no fsharp callee branch in extract_callee_name"}, - { - "racket", "a.rkt", - "#lang racket\n" - "\n" - "(define (helper x)\n" - " (+ x 1))\n" - "\n" - "(define (run)\n" - " (helper 41))\n", - /* - * Racket: lisp call is a 'list' whose head is a 'symbol' (grammar has - * no 'identifier' node); no racket branch in extract_callee_name. - */ - 0, "list head is symbol not identifier; " - "no racket branch in extract_callee_name" - }, + {"racket", "a.rkt", + "#lang racket\n" + "\n" + "(define (helper x)\n" + " (+ x 1))\n" + "\n" + "(define (run)\n" + " (helper 41))\n", + /* + * Racket: lisp call is a 'list' whose head is a 'symbol' (grammar has + * no 'identifier' node); no racket branch in extract_callee_name. + */ + 0, + "list head is symbol not identifier; " + "no racket branch in extract_callee_name"}, - { - "rescript", "a.res", - "let helper = (x) => x + 1\n" - "\n" - "let run = () => helper(41)\n", - /* - * ReScript: call_expression 'function' field is a 'value_identifier' - * (not in extract_callee_from_fields' accepted type list). - */ - 0, "call_expression function field is value_identifier; " - "not in extract_callee_from_fields accepted type list" - }, + {"rescript", "a.res", + "let helper = (x) => x + 1\n" + "\n" + "let run = () => helper(41)\n", + /* + * ReScript: call_expression 'function' field is a 'value_identifier' + * (not in extract_callee_from_fields' accepted type list). + */ + 0, + "call_expression function field is value_identifier; " + "not in extract_callee_from_fields accepted type list"}, - { - "scheme", "a.scm", - "(define (helper x)\n" - " (* x 2))\n" - "\n" - "(define (run)\n" - " (helper 21))\n", - /* - * Scheme: lisp call is a 'list' whose head is a 'symbol'; - * no scheme branch in extract_callee_name. - */ - 0, "list head is symbol not identifier; " - "no scheme branch in extract_callee_name" - }, + {"scheme", "a.scm", + "(define (helper x)\n" + " (* x 2))\n" + "\n" + "(define (run)\n" + " (helper 21))\n", + /* + * Scheme: lisp call is a 'list' whose head is a 'symbol'; + * no scheme branch in extract_callee_name. + */ + 0, + "list head is symbol not identifier; " + "no scheme branch in extract_callee_name"}, + + {"chialisp", "a.clsp", + "(mod ()\n" + " (defun helper (x)\n" + " (* x 2))\n" + " (defun run ()\n" + " (helper 21))\n" + ")\n", + /* + * Chialisp: EXPECTED-GREEN. Every def is nested inside the top-level + * `(mod ...)`, so the in-body call can only be Function-sourced if + * compute_lisp_func_qn pushes a scope for the inner `(defun run ...)` + * rather than stopping at the module. That is exactly what would + * regress if the Chialisp def-head set drifted from the defs walk. + */ + 1, NULL}, }; enum { IB_CASES_COUNT = (int)(sizeof(IB_CASES) / sizeof(IB_CASES[0])) }; @@ -564,17 +541,14 @@ TEST(repro_invariant_breadth_callable_sourcing) { const IBCase *c = &IB_CASES[i]; IBMetrics m = ib_metrics(c->filename, c->src); - int pass = (m.ok && m.calls >= 1 && m.callable_calls >= 1 && - m.module_calls == 0); + int pass = (m.ok && m.calls >= 1 && m.callable_calls >= 1 && m.module_calls == 0); if (!pass) { fprintf(stderr, " [INV-BREADTH] FAIL %-12s ok=%d calls=%d " "callable=%d module=%d%s%s\n", - c->lang, m.ok, m.calls, m.callable_calls, - m.module_calls, - c->gap_note ? " -- " : "", - c->gap_note ? c->gap_note : ""); + c->lang, m.ok, m.calls, m.callable_calls, m.module_calls, + c->gap_note ? " -- " : "", c->gap_note ? c->gap_note : ""); failures++; } else { fprintf(stderr, diff --git a/tests/repro/repro_language_registry.c b/tests/repro/repro_language_registry.c index afa1ee596..31a55b97e 100644 --- a/tests/repro/repro_language_registry.c +++ b/tests/repro/repro_language_registry.c @@ -212,6 +212,7 @@ static const LanguageCapabilityEntry LANGUAGE_CAPABILITIES[CBM_LANG_COUNT] = { TRANSFORM_ONLY(OBJECTSCRIPT_EXPORT), CALL_WITH_REFERENCE_VOCAB(ARKTS), CALL_WITHOUT_REFERENCE_VOCAB(PLSQL), + CALL_WITHOUT_REFERENCE_VOCAB(CHIALISP), }; #undef CALL_WITH_REFERENCE_VOCAB @@ -301,7 +302,7 @@ TEST(repro_language_capability_ledger_covers_every_enum) { } if (counts[CAP_CALL_WITH_REFERENCE_VOCAB] != 87 || - counts[CAP_CALL_WITHOUT_REFERENCE_VOCAB] != 26 || counts[CAP_NO_CALL] != 50 || + counts[CAP_CALL_WITHOUT_REFERENCE_VOCAB] != 27 || counts[CAP_NO_CALL] != 50 || counts[CAP_TRANSFORM_ONLY] != 1 || counts[CAP_UNSUPPORTED] != 1) { fprintf(stderr, " [language-registry] invariant=capability_partition call_ref_vocab=%d ref_gap=%d " @@ -318,8 +319,8 @@ TEST(repro_language_capability_ledger_covers_every_enum) { TEST(repro_call_argument_matrices_equal_call_capability_ledger) { enum { EXPECTED_MATRIX_A_ROWS = 68, - EXPECTED_MATRIX_B_ROWS = 47, - EXPECTED_CALL_CAPABLE_LANGUAGES = 113, + EXPECTED_MATRIX_B_ROWS = 48, + EXPECTED_CALL_CAPABLE_LANGUAGES = 114, EXPECTED_NON_CALL_LANGUAGES = 52, EXPECTED_NON_CALL_DOMAIN_CONTROLS = 2, }; diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 935eda07c..ac75b16a6 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -1225,19 +1225,18 @@ TEST(form_procedure) { /* --- Oracle PL/SQL --- */ TEST(plsql_package_and_call) { - const char *src = - "CREATE OR REPLACE PACKAGE BODY emp_pkg AS\n" - " FUNCTION hire(p_name VARCHAR2) RETURN NUMBER IS\n" - " v_sal NUMBER;\n" - " BEGIN\n" - " v_sal := util_pkg.calc_salary(p_name);\n" - " IF v_sal > 0 THEN\n" - " RETURN v_sal;\n" - " END IF;\n" - " RAISE no_data_found;\n" - " END;\n" - "END emp_pkg;\n" - "/\n"; + const char *src = "CREATE OR REPLACE PACKAGE BODY emp_pkg AS\n" + " FUNCTION hire(p_name VARCHAR2) RETURN NUMBER IS\n" + " v_sal NUMBER;\n" + " BEGIN\n" + " v_sal := util_pkg.calc_salary(p_name);\n" + " IF v_sal > 0 THEN\n" + " RETURN v_sal;\n" + " END IF;\n" + " RAISE no_data_found;\n" + " END;\n" + "END emp_pkg;\n" + "/\n"; CBMFileResult *r = extract(src, CBM_LANG_PLSQL, "t", "emp_pkg.pkb"); ASSERT_NOT_NULL(r); ASSERT_FALSE(r->has_error); @@ -1286,6 +1285,156 @@ TEST(plsql_create_type_as_object_limitation) { PASS(); } +/* --- Chialisp --- + * + * Three defects in the only public Chialisp grammar + * (Quexington/tree-sitter-chialisp) motivated writing our own, and each is + * pinned below as a parse-level assertion rather than a note: comments that + * required CRLF to terminate, `(include foo.clib)` rejected because of the dot + * in the filename, and `(defconstant NAME )` accepting only a primitive. + * All three desynchronised the rest of the file, so a regression here shows up + * as `has_error` plus missing defs, not as one lost node. */ +TEST(chialisp_puzzle_defs_and_labels) { + const char *src = "; a Chialisp puzzle\n" + "(mod (ARG)\n" + " (include condition_codes.clib)\n" + " (defconstant TWO 2)\n" + " (defconstant HASH (sha256 1))\n" + " (defun-inline square (x) (* x x))\n" + " (defun apply_twice (v) (square (square v)))\n" + " (defmacro assert items (f items))\n" + " (apply_twice ARG)\n" + ")\n"; + CBMFileResult *r = extract(src, CBM_LANG_CHIALISP, "t", "puzzles/my_puzzle.clsp"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + /* `mod` has no name of its own — the puzzle is named by its file. */ + ASSERT(has_def(r, "Module", "my_puzzle")); + /* Defs are NESTED inside `(mod ...)`; a walk that stops at the module + * loses every one of them. */ + ASSERT(has_def(r, "Function", "square")); + ASSERT(has_def(r, "Function", "apply_twice")); + ASSERT(has_def(r, "Macro", "assert")); + /* `defconstant` binds an arbitrary EXPRESSION, not just a primitive. */ + ASSERT(has_def(r, "Constant", "TWO")); + ASSERT(has_def(r, "Constant", "HASH")); + /* A dotted include filename resolves as one symbol. */ + ASSERT(has_import(r, "condition_codes.clib")); + /* Real calls survive; CLVM primitives and def heads do not become calls. */ + ASSERT(has_call(r, "square")); + ASSERT(has_call(r, "apply_twice")); + ASSERT(!has_call(r, "sha256")); + ASSERT(!has_call(r, "defun")); + ASSERT(!has_call(r, "mod")); + ASSERT(!has_call(r, "include")); + /* A parameter list is a `list` too — but it binds, it does not invoke. */ + ASSERT(!has_call(r, "x")); + ASSERT(!has_call(r, "v")); + ASSERT(!has_call(r, "items")); + cbm_free_result(r); + PASS(); +} + +TEST(chialisp_comment_line_endings) { + /* LF, CRLF, and a comment closed by EOF with no trailing newline. The + * public grammar's comment rule was `/;.*\r\n/`, so an LF file lost every + * form after the first comment. */ + const char *lf = "(mod (A)\n; note\n(defun f (x) x)\n)\n"; + const char *crlf = "(mod (A)\r\n; note\r\n(defun f (x) x)\r\n)\r\n"; + const char *eof = "(mod (A)\n(defun f (x) x)\n)\n; trailing comment, no newline"; + const char *srcs[] = {lf, crlf, eof}; + for (int i = 0; i < 3; i++) { + CBMFileResult *r = extract(srcs[i], CBM_LANG_CHIALISP, "t", "c.clsp"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT_FALSE(r->parse_incomplete); + ASSERT(has_def(r, "Function", "f")); + cbm_free_result(r); + } + PASS(); +} + +TEST(chialisp_library_defs_and_quoted_data) { + /* A .clib wraps its definitions in one enclosing list, and its macros embed + * puzzle-shaped literals under `(q ...)`. Those literals are DATA: a def + * head or call symbol inside one must mint nothing. */ + const char *src = "(\n" + " (defconstant TWO 2)\n" + " (defmacro emit () (q . (defun ghost (x) (real_helper x))))\n" + " (defun real_helper (x) (+ x TWO))\n" + ")\n"; + CBMFileResult *r = extract(src, CBM_LANG_CHIALISP, "t", "curry.clib"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT(has_def(r, "Constant", "TWO")); + ASSERT(has_def(r, "Macro", "emit")); + ASSERT(has_def(r, "Function", "real_helper")); + ASSERT(!has_def_any(r, "ghost")); + ASSERT(!has_call(r, "real_helper")); + cbm_free_result(r); + PASS(); +} + +TEST(chialisp_export_names_do_not_duplicate_defs) { + /* `(export foo)` names a function already defined in the same file. As a + * def head it would mint a SECOND node for `foo`; as a call head it would + * mint a phantom call. It is neither. */ + const char *src = "(\n" + " (defun foo (x) x)\n" + " (export foo)\n" + ")\n"; + CBMFileResult *r = extract(src, CBM_LANG_CHIALISP, "t", "e.clsp"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + int foo_defs = 0; + for (int i = 0; i < r->defs.count; i++) { + if (r->defs.items[i].name && strcmp(r->defs.items[i].name, "foo") == 0) { + foo_defs++; + } + } + ASSERT_EQ(foo_defs, 1); + ASSERT(!has_call(r, "export")); + ASSERT(!has_call(r, "foo")); + cbm_free_result(r); + PASS(); +} + +TEST(chialisp_comment_before_def_head_keeps_the_name) { + /* Comments are NAMED nodes and so occupy named-child indices. A comment + * between the head and the name shifts them by one; defs and call-scope + * must skip comments the same way or they stop describing the same tree. */ + const char *src = "(mod ()\n" + " (defun ; why this exists\n" + " documented (x) (* x 2))\n" + " (defun caller () (documented 21))\n" + ")\n"; + CBMFileResult *r = extract(src, CBM_LANG_CHIALISP, "t", "d.clsp"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT(has_def(r, "Function", "documented")); + ASSERT(has_def(r, "Function", "caller")); + ASSERT(has_call(r, "documented")); + cbm_free_result(r); + PASS(); +} + +TEST(chialisp_dialect_sigil_is_not_a_file_import) { + /* `(include *standard-cl-26*)` selects a dialect; recording it as an import + * invents a dependency on a file that does not exist. The compiler filters + * `*...*` names and so must we. */ + const char *src = "(mod ()\n" + " (include *standard-cl-26*)\n" + " (include curry.clib)\n" + ")\n"; + CBMFileResult *r = extract(src, CBM_LANG_CHIALISP, "t", "s.clsp"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT(has_import(r, "curry.clib")); + ASSERT(!has_import(r, "standard-cl-26")); + cbm_free_result(r); + PASS(); +} + /* --- Wolfram --- */ TEST(wolfram_function) { CBMFileResult *r = @@ -2052,9 +2201,10 @@ TEST(dbt_source_and_two_arg_ref) { /* Both dbt builtins name the relation in their LAST string argument: * source('group','table') -> table, and the two-argument * ref('package','model') form -> model. */ - CBMFileResult *r = extract("SELECT * FROM {{ source('raw', 'customers') }}\n" - "UNION ALL SELECT * FROM {{ ref('analytics', 'legacy_customers') }}\n", - CBM_LANG_SQL, "t", "models/stg_customers.sql"); + CBMFileResult *r = + extract("SELECT * FROM {{ source('raw', 'customers') }}\n" + "UNION ALL SELECT * FROM {{ ref('analytics', 'legacy_customers') }}\n", + CBM_LANG_SQL, "t", "models/stg_customers.sql"); ASSERT_NOT_NULL(r); ASSERT(has_def(r, "Model", "stg_customers")); ASSERT(has_usage(r, "customers")); @@ -3073,8 +3223,8 @@ TEST(vue_embedded_structure_negative_controls_issue1410) { } TEST(vue_embedded_structure_host_controls_issue1410) { - CBMFileResult *plain = extract("function plainTs(): void { target(); }\n", CBM_LANG_TYPESCRIPT, - "t", "plain.ts"); + CBMFileResult *plain = + extract("function plainTs(): void { target(); }\n", CBM_LANG_TYPESCRIPT, "t", "plain.ts"); ASSERT_NOT_NULL(plain); ASSERT_FALSE(plain->has_error); ASSERT_EQ(count_defs_named(plain, "Function", "plainTs"), 1); @@ -3532,11 +3682,11 @@ TEST(extract_java_method_annotations_issue382) { /* ── ArkTS (HarmonyOS .ets) ─────────────────────────────────────── */ TEST(arkts_component_struct) { - CBMFileResult *r = extract( - "@Entry\n@Component\nstruct Index {\n @State message: string = 'Hello'\n\n" - " build() {\n Column() {\n Text(this.message).fontSize(20)\n }\n" - " .width('100%')\n }\n}\n", - CBM_LANG_ARKTS, "t", "Index.ets"); + CBMFileResult *r = + extract("@Entry\n@Component\nstruct Index {\n @State message: string = 'Hello'\n\n" + " build() {\n Column() {\n Text(this.message).fontSize(20)\n }\n" + " .width('100%')\n }\n}\n", + CBM_LANG_ARKTS, "t", "Index.ets"); ASSERT_NOT_NULL(r); ASSERT_FALSE(r->has_error); ASSERT(has_def(r, "Struct", "Index")); @@ -3567,12 +3717,12 @@ TEST(arkts_exported_struct_decorators) { } TEST(arkts_member_decorators) { - CBMFileResult *r = extract( - "@Component\nstruct S {\n @State a: number = 0\n @Prop b: string\n" - " @Link c: boolean\n @Provide('k') d: string = ''\n @Consume('k') e: string\n" - " @StorageLink('s') f: number = 1\n @State @Watch('onW') g: boolean = false\n\n" - " build() {\n }\n}\n", - CBM_LANG_ARKTS, "t", "S.ets"); + CBMFileResult *r = + extract("@Component\nstruct S {\n @State a: number = 0\n @Prop b: string\n" + " @Link c: boolean\n @Provide('k') d: string = ''\n @Consume('k') e: string\n" + " @StorageLink('s') f: number = 1\n @State @Watch('onW') g: boolean = false\n\n" + " build() {\n }\n}\n", + CBM_LANG_ARKTS, "t", "S.ets"); ASSERT_NOT_NULL(r); ASSERT_FALSE(r->has_error); ASSERT(decorators_contain(find_def_by_name(r, "a"), "State")); @@ -3612,11 +3762,11 @@ TEST(arkts_no_phantom_builtin_defs) { } TEST(arkts_builder_extend_styles) { - CBMFileResult *r = extract( - "@Builder\nfunction card(t: string) {\n Column() {\n Text(t)\n }\n}\n\n" - "@Extend(Text)\nfunction fancy(size: number) {\n .fontSize(size)\n}\n\n" - "@Styles\nfunction pressed() {\n .backgroundColor('#eee')\n}\n", - CBM_LANG_ARKTS, "t", "b.ets"); + CBMFileResult *r = + extract("@Builder\nfunction card(t: string) {\n Column() {\n Text(t)\n }\n}\n\n" + "@Extend(Text)\nfunction fancy(size: number) {\n .fontSize(size)\n}\n\n" + "@Styles\nfunction pressed() {\n .backgroundColor('#eee')\n}\n", + CBM_LANG_ARKTS, "t", "b.ets"); ASSERT_NOT_NULL(r); ASSERT_FALSE(r->has_error); ASSERT(has_def(r, "Function", "card")); @@ -4960,8 +5110,8 @@ TEST(extract_c_test_dir_marks_is_test_issue1294) { * not be (#1294). */ TEST(extract_python_method_test_dir_marks_is_test_issue1294) { const char *src = "class Foo:\n" - " def helper(self):\n" - " pass\n"; + " def helper(self):\n" + " pass\n"; /* Python's LSP layer injects synthetic builtin stub Methods (str.upper, * dict.get, ...) into defs.items alongside real ones (py_builtins.c), so @@ -6150,6 +6300,12 @@ SUITE(extraction) { RUN_TEST(plsql_package_and_call); RUN_TEST(plsql_standalone_function); RUN_TEST(plsql_create_type_as_object_limitation); + RUN_TEST(chialisp_puzzle_defs_and_labels); + RUN_TEST(chialisp_comment_line_endings); + RUN_TEST(chialisp_library_defs_and_quoted_data); + RUN_TEST(chialisp_export_names_do_not_duplicate_defs); + RUN_TEST(chialisp_comment_before_def_head_keeps_the_name); + RUN_TEST(chialisp_dialect_sigil_is_not_a_file_import); RUN_TEST(wolfram_function); RUN_TEST(magma_function); diff --git a/tests/test_grammar_labels.c b/tests/test_grammar_labels.c index 6f7a8780e..4008b6d2c 100644 --- a/tests/test_grammar_labels.c +++ b/tests/test_grammar_labels.c @@ -141,6 +141,7 @@ static const LabelGolden LABEL_GOLDENS[] = { {"racket", "Function:2,Module:1"}, {"rescript", "Function:2,Module:1"}, {"scheme", "Function:2,Module:1"}, + {"chialisp", "Constant:1,Function:1,Macro:1,Module:2"}, {"slang", "Function:2,Module:1"}, {"squirrel", "Function:2,Module:1"}, {"starlark", "Function:2,Module:1"}, diff --git a/tests/test_grammar_regression.c b/tests/test_grammar_regression.c index cdf598ab3..219c68edb 100644 --- a/tests/test_grammar_regression.c +++ b/tests/test_grammar_regression.c @@ -171,6 +171,21 @@ const GrammarCase CBM_GRAMMAR_CASES[] = { {"racket", CBM_LANG_RACKET, "a.rkt", "(define (foo) 1)\n(define (bar) 2)\n", 1, {NULL}}, {"rescript", CBM_LANG_RESCRIPT, "a.res", "let foo = () => 1\nlet bar = () => 2\n", 1, {NULL}}, {"scheme", CBM_LANG_SCHEME, "a.scm", "(define (foo) 1)\n(define (bar) 2)\n", 1, {NULL}}, + /* Chialisp: every def is a head symbol of a generic `list`, and they are + * NESTED inside the top-level `(mod ...)` — the fixture must exercise that + * nesting, or the walk could stop at `mod` and still look green. */ + {"chialisp", + CBM_LANG_CHIALISP, + "a.clsp", + "(mod (ARG)\n" + " (include condition_codes.clib)\n" + " (defconstant TWO 2)\n" + " (defun-inline square (x) (* x x))\n" + " (defmacro twice (v) (qq (+ (unquote v) (unquote v))))\n" + " (square ARG)\n" + ")\n", + 3, + {"square", "TWO", "twice", NULL}}, {"slang", CBM_LANG_SLANG, "a.slang", "void foo() {}\nvoid bar() {}\n", 1, {NULL}}, {"squirrel", CBM_LANG_SQUIRREL, "a.nut", "function foo() {}\nfunction bar() {}\n", 1, {NULL}}, {"starlark", @@ -199,7 +214,8 @@ const GrammarCase CBM_GRAMMAR_CASES[] = { {"mojo", CBM_LANG_MOJO, "a.mojo", - "fn foo() -> Int:\n return 1\n\nstruct A:\n fn bar(self) -> Int:\n return foo()\n", + "fn foo() -> Int:\n return 1\n\nstruct A:\n fn bar(self) -> Int:\n return " + "foo()\n", 2, {"foo", "A", NULL}}, {"arkts", diff --git a/tests/test_lang_contract.c b/tests/test_lang_contract.c index 586452e2b..3ec63d80e 100644 --- a/tests/test_lang_contract.c +++ b/tests/test_lang_contract.c @@ -463,11 +463,10 @@ TEST(contract_python_relative_import) { /* Python: relative aliased from-import must CALLS the real def (execute). */ TEST(contract_python_aliased_from_import_calls) { - static const LangFile f[] = { - {"gate.py", "def execute(x):\n return x\n"}, - {"router.py", - "from .gate import execute as bridge_execute\n\n\n" - "def submit_task(y):\n return bridge_execute(y)\n"}}; + static const LangFile f[] = {{"gate.py", "def execute(x):\n return x\n"}, + {"router.py", + "from .gate import execute as bridge_execute\n\n\n" + "def submit_task(y):\n return bridge_execute(y)\n"}}; LangProj lp; cbm_store_t *store = lang_index_files(&lp, f, 2); ASSERT_TRUE(store != NULL); @@ -560,10 +559,8 @@ TEST(contract_python_absolute_aliased_from_import_calls) { /* Python: unresolvable module + alias must emit no CALLS (no invented edge). */ TEST(contract_python_ghost_module_aliased_from_import_no_calls) { - static const LangFile f[] = { - {"router.py", - "from ghost_module import nope as alias\n\n\n" - "def submit_task(y):\n return alias(y)\n"}}; + static const LangFile f[] = {{"router.py", "from ghost_module import nope as alias\n\n\n" + "def submit_task(y):\n return alias(y)\n"}}; LangProj lp; cbm_store_t *store = lang_index_files(&lp, f, 1); ASSERT_TRUE(store != NULL); @@ -694,6 +691,57 @@ static void breadth_diag(cbm_store_t *store, const char *project, const char *re } } +/* Chialisp: a constant defined in an included .clib must resolve from the + * puzzle that includes it. + * + * This is the whole reason `Constant` is admitted to the cross-file name + * registry (cbm_label_is_registry_symbol). Chialisp's condition codes, + * CREATE_COIN and friends, are `(defconstant ...)` forms in a shared .clib and + * are referenced by name from every puzzle. Labelled `Constant` but left OUT of + * the registry, they are seeded nowhere, cbm_registry_resolve returns nothing, + * and roughly half of a Chialisp graph is unreachable — present as nodes, but + * never the target of an edge. This test fails (zero cross-file USAGE edges) + * the moment "Constant" is dropped from that predicate. */ +TEST(contract_chialisp_constant_resolves_across_files) { + static const LangFile f[] = {{"condition_codes.clib", "(\n (defconstant CREATE_COIN 51)\n)\n"}, + {"puzzle.clsp", "(mod (ARG)\n" + " (include condition_codes.clib)\n" + " (defun run (amount)\n" + " (list CREATE_COIN amount))\n" + ")\n"}}; + LangProj lp; + cbm_store_t *store = lang_index_files(&lp, f, 2); + ASSERT_TRUE(store != NULL); + + cbm_node_t *consts = NULL; + int nconst = 0; + ASSERT_EQ(cbm_store_find_nodes_by_name(store, lp.project, "CREATE_COIN", &consts, &nconst), + CBM_STORE_OK); + ASSERT_TRUE(nconst >= 1); + ASSERT_TRUE(consts[0].label != NULL && strcmp(consts[0].label, "Constant") == 0); + ASSERT_TRUE(consts[0].file_path != NULL && strstr(consts[0].file_path, "condition_codes.clib")); + + cbm_edge_t *edges = NULL; + int nedges = 0; + ASSERT_EQ(cbm_store_find_edges_by_target_type(store, consts[0].id, "USAGE", &edges, &nedges), + CBM_STORE_OK); + int cross_file = 0; + for (int i = 0; i < nedges; i++) { + cbm_node_t src; + if (cbm_store_find_node_by_id(store, edges[i].source_id, &src) != CBM_STORE_OK) { + continue; + } + if (src.file_path && strstr(src.file_path, "puzzle.clsp")) { + cross_file++; + } + } + cbm_store_free_edges(edges, nedges); + cbm_store_free_nodes(consts, nconst); + lang_cleanup(&lp, store); + ASSERT_TRUE(cross_file >= 1); + PASS(); +} + TEST(contract_all_grammars_in_graph) { int n = (int)CBM_GRAMMAR_CASES_COUNT; if (n > GRAMMAR_BREADTH_MAX) { @@ -859,6 +907,9 @@ static const CallCase CALL_CASES[] = { "fn helper(x: felt252) -> felt252 {\n x + 1\n}\n\nfn run() -> felt252 {\n " "helper(41)\n}\n", true, NULL}, + {"chialisp", "a.clsp", + "(mod ()\n (defun helper (x)\n (* x 2))\n (defun run ()\n (helper 21))\n)\n", true, + NULL}, {"clojure", "a.clj", "(defn helper [] 42)\n\n(defn run [] (helper))\n", false, "lisp: call is a list_lit whose head is a sym_lit (not a field, not a first-child " "'identifier'); no lisp branch in extract_callee_name"}, @@ -1704,10 +1755,11 @@ TEST(contract_edge_parallel_service_edges) { if (store) { sqlite3_stmt *stmt = NULL; sqlite3 *db = cbm_store_get_db(store); - if (db && sqlite3_prepare_v2(db, - "SELECT count(*) FROM edges WHERE properties IS NOT NULL " - "AND properties != '' AND json_valid(properties)=0;", - -1, &stmt, NULL) == SQLITE_OK && + if (db && + sqlite3_prepare_v2(db, + "SELECT count(*) FROM edges WHERE properties IS NOT NULL " + "AND properties != '' AND json_valid(properties)=0;", + -1, &stmt, NULL) == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW) { invalid_props = sqlite3_column_int(stmt, 0); } @@ -1809,6 +1861,7 @@ SUITE(lang_contract) { RUN_TEST(contract_typescript_relative_import); /* Graph-level breadth across all grammars (P4). */ + RUN_TEST(contract_chialisp_constant_resolves_across_files); RUN_TEST(contract_all_grammars_in_graph); /* CALLS-edge breadth across non-LSP languages (P5). */ diff --git a/tests/test_language.c b/tests/test_language.c index 6914e64f4..2c8d6576b 100644 --- a/tests/test_language.c +++ b/tests/test_language.c @@ -748,6 +748,15 @@ TEST(lang_ext_scheme) { PASS(); } +TEST(lang_ext_chialisp) { + ASSERT_EQ(cbm_language_for_extension(".clsp"), CBM_LANG_CHIALISP); + ASSERT_EQ(cbm_language_for_extension(".clib"), CBM_LANG_CHIALISP); + ASSERT_EQ(cbm_language_for_extension(".clinc"), CBM_LANG_CHIALISP); + /* .clj stays Clojure — the Chialisp extensions must not widen it. */ + ASSERT_EQ(cbm_language_for_extension(".clj"), CBM_LANG_CLOJURE); + PASS(); +} + TEST(lang_ext_fennel) { ASSERT_EQ(cbm_language_for_extension(".fnl"), CBM_LANG_FENNEL); PASS(); @@ -1338,6 +1347,7 @@ SUITE(language) { RUN_TEST(lang_ext_d); RUN_TEST(lang_ext_nim); RUN_TEST(lang_ext_scheme); + RUN_TEST(lang_ext_chialisp); RUN_TEST(lang_ext_fennel); RUN_TEST(lang_ext_fish); RUN_TEST(lang_ext_awk); diff --git a/tests/test_language_count_contract.sh b/tests/test_language_count_contract.sh index bf910c771..57d26186c 100644 --- a/tests/test_language_count_contract.sh +++ b/tests/test_language_count_contract.sh @@ -63,7 +63,7 @@ EXEMPT=( 'docs/EVALUATION_PLAN.md' # historical plan, pinned to its authoring date 'docs/BENCHMARK.md' # benchmark corpus subsets (63/17 scored) 'scripts/clone-bench-repos.sh' # bench repo tiers (44 + 22) - 'tests/repro/repro_invariant_breadth.c' # breadth of one invariant (26/27) + 'tests/repro/repro_invariant_breadth.c' # breadth of one invariant (27/27) 'tests/repro/repro_grammar_scripting.c' # scripting-grammar probe scope (12) 'tests/test_grammar_probe_b.c' # grammar probe scope (12) 'tests/test_language_count_contract.sh' # this file, which names the numbers above diff --git a/tools/tree-sitter-chialisp/grammar.js b/tools/tree-sitter-chialisp/grammar.js new file mode 100644 index 000000000..f736411ff --- /dev/null +++ b/tools/tree-sitter-chialisp/grammar.js @@ -0,0 +1,77 @@ +/// +// tree-sitter grammar for Chialisp (Chia smart-coin s-expression language). +// +// Copyright (c) DeusData — MIT (the project's own license). +// +// Deliberately GENERIC. Chialisp has no reserved words at the reader level: +// `mod`, `defun`, `defconstant`, `include` and friends are ordinary head +// symbols of ordinary lists, resolved by the compiler, not by the tokenizer. +// Encoding them as grammar rules is how a Chialisp grammar breaks the moment +// the dialect gains a form (cl-21 -> cl-23 -> cl-26 each added several). So we +// model exactly what the clvm_tools reader models: nested s-expressions over +// five atom kinds. Definition/call/import shape is then decided in the +// extractor (internal/cbm/extract_defs.c and friends), where it can be revised +// without regenerating a parser. +// +// Reader model taken from clvm_tools' IR reader: +// - a token ends at whitespace, `(`, `)`, `;` or a quote character; +// - `;` starts a comment that runs to end of line (LF, CRLF or EOF); +// - both `"` and `'` delimit strings; +// - `.` standing alone is the dotted-pair marker, but a `.` INSIDE a token is +// an ordinary character — which is why `(include foo.clib)` must parse. +// +// Atom kinds are ordered hex -> number -> symbol so that the lexer's +// equal-length tiebreak (match specificity, then rule order) prefers the +// specific reading: `0x17` is hex, `-5` is a number, and anything longer than +// either match — `0xdeadZZ`, `123abc` — falls through to `symbol` by longest +// match. Do not add token(prec(...)) here: lexical precedence outranks match +// length in tree-sitter, which would split `123abc` into `123` + `abc`. +module.exports = grammar({ + name: 'chialisp', + + extras: $ => [ + /\s/, + $.comment, + ], + + rules: { + source_file: $ => repeat($._form), + + // `dot` is admitted anywhere a form is, rather than only in the one + // grammatical dotted-pair slot. clvm_tools' reader is equally permissive, + // and a stricter rule would turn a hand-written constant table into an + // ERROR node — a mis-parse we would then have to explain rather than index. + _form: $ => choice($._sexp, $.dot), + + _sexp: $ => choice( + $.list, + $.hex, + $.number, + $.string, + $.symbol, + ), + + list: $ => seq('(', repeat($._form), ')'), + + // `[^\r\n]*` (not `[^\n]*`) so a CRLF file does not carry the CR inside the + // comment node; the CR is whitespace and is consumed by `extras`. Runs to + // EOF when the last line has no terminator. + comment: _ => token(seq(';', /[^\r\n]*/)), + + hex: _ => /0[xX][0-9a-fA-F]+/, + + // `-?` and not a separate sign token: a bare `-` is the subtraction + // operator and must stay a `symbol`, which it does because this pattern + // requires at least one digit. + number: _ => /-?[0-9]+/, + + string: _ => token(choice( + seq('"', repeat(choice(/[^"\\]/, /\\./)), '"'), + seq("'", repeat(choice(/[^'\\]/, /\\./)), "'"), + )), + + dot: _ => '.', + + symbol: _ => /[^\s()";']+/, + }, +}); diff --git a/tools/tree-sitter-chialisp/package.json b/tools/tree-sitter-chialisp/package.json new file mode 100644 index 000000000..4ac1a8fae --- /dev/null +++ b/tools/tree-sitter-chialisp/package.json @@ -0,0 +1,21 @@ +{ + "name": "tree-sitter-chialisp", + "version": "0.1.0", + "description": "Chialisp (Chia smart-coin s-expression language) grammar for tree-sitter", + "main": "bindings/node", + "keywords": ["tree-sitter", "chialisp", "chia", "clvm", "lisp"], + "license": "MIT", + "dependencies": { + "nan": "^2.18.0" + }, + "devDependencies": { + "tree-sitter-cli": "^0.25.0" + }, + "tree-sitter": [ + { + "scope": "source.chialisp", + "file-types": ["clsp", "clib", "clinc"], + "highlights": "queries/highlights.scm" + } + ] +} diff --git a/tools/tree-sitter-chialisp/src/grammar.json b/tools/tree-sitter-chialisp/src/grammar.json new file mode 100644 index 000000000..0b1a7f3a5 --- /dev/null +++ b/tools/tree-sitter-chialisp/src/grammar.json @@ -0,0 +1,185 @@ +{ + "$schema": "https://tree-sitter.github.io/tree-sitter/assets/schemas/grammar.schema.json", + "name": "chialisp", + "rules": { + "source_file": { + "type": "REPEAT", + "content": { + "type": "SYMBOL", + "name": "_form" + } + }, + "_form": { + "type": "CHOICE", + "members": [ + { + "type": "SYMBOL", + "name": "_sexp" + }, + { + "type": "SYMBOL", + "name": "dot" + } + ] + }, + "_sexp": { + "type": "CHOICE", + "members": [ + { + "type": "SYMBOL", + "name": "list" + }, + { + "type": "SYMBOL", + "name": "hex" + }, + { + "type": "SYMBOL", + "name": "number" + }, + { + "type": "SYMBOL", + "name": "string" + }, + { + "type": "SYMBOL", + "name": "symbol" + } + ] + }, + "list": { + "type": "SEQ", + "members": [ + { + "type": "STRING", + "value": "(" + }, + { + "type": "REPEAT", + "content": { + "type": "SYMBOL", + "name": "_form" + } + }, + { + "type": "STRING", + "value": ")" + } + ] + }, + "comment": { + "type": "TOKEN", + "content": { + "type": "SEQ", + "members": [ + { + "type": "STRING", + "value": ";" + }, + { + "type": "PATTERN", + "value": "[^\\r\\n]*" + } + ] + } + }, + "hex": { + "type": "PATTERN", + "value": "0[xX][0-9a-fA-F]+" + }, + "number": { + "type": "PATTERN", + "value": "-?[0-9]+" + }, + "string": { + "type": "TOKEN", + "content": { + "type": "CHOICE", + "members": [ + { + "type": "SEQ", + "members": [ + { + "type": "STRING", + "value": "\"" + }, + { + "type": "REPEAT", + "content": { + "type": "CHOICE", + "members": [ + { + "type": "PATTERN", + "value": "[^\"\\\\]" + }, + { + "type": "PATTERN", + "value": "\\\\." + } + ] + } + }, + { + "type": "STRING", + "value": "\"" + } + ] + }, + { + "type": "SEQ", + "members": [ + { + "type": "STRING", + "value": "'" + }, + { + "type": "REPEAT", + "content": { + "type": "CHOICE", + "members": [ + { + "type": "PATTERN", + "value": "[^'\\\\]" + }, + { + "type": "PATTERN", + "value": "\\\\." + } + ] + } + }, + { + "type": "STRING", + "value": "'" + } + ] + } + ] + } + }, + "dot": { + "type": "STRING", + "value": "." + }, + "symbol": { + "type": "PATTERN", + "value": "[^\\s()\";']+" + } + }, + "extras": [ + { + "type": "PATTERN", + "value": "\\s" + }, + { + "type": "SYMBOL", + "name": "comment" + } + ], + "conflicts": [], + "precedences": [], + "externals": [], + "inline": [], + "supertypes": [], + "reserved": {} +} \ No newline at end of file diff --git a/tools/tree-sitter-chialisp/src/node-types.json b/tools/tree-sitter-chialisp/src/node-types.json new file mode 100644 index 000000000..c63cb5367 --- /dev/null +++ b/tools/tree-sitter-chialisp/src/node-types.json @@ -0,0 +1,106 @@ +[ + { + "type": "list", + "named": true, + "fields": {}, + "children": { + "multiple": true, + "required": false, + "types": [ + { + "type": "dot", + "named": true + }, + { + "type": "hex", + "named": true + }, + { + "type": "list", + "named": true + }, + { + "type": "number", + "named": true + }, + { + "type": "string", + "named": true + }, + { + "type": "symbol", + "named": true + } + ] + } + }, + { + "type": "source_file", + "named": true, + "root": true, + "fields": {}, + "children": { + "multiple": true, + "required": false, + "types": [ + { + "type": "dot", + "named": true + }, + { + "type": "hex", + "named": true + }, + { + "type": "list", + "named": true + }, + { + "type": "number", + "named": true + }, + { + "type": "string", + "named": true + }, + { + "type": "symbol", + "named": true + } + ] + } + }, + { + "type": "(", + "named": false + }, + { + "type": ")", + "named": false + }, + { + "type": "comment", + "named": true, + "extra": true + }, + { + "type": "dot", + "named": true + }, + { + "type": "hex", + "named": true + }, + { + "type": "number", + "named": true + }, + { + "type": "string", + "named": true + }, + { + "type": "symbol", + "named": true + } +] \ No newline at end of file diff --git a/tools/tree-sitter-chialisp/src/parser.c b/tools/tree-sitter-chialisp/src/parser.c new file mode 100644 index 000000000..d929151ea --- /dev/null +++ b/tools/tree-sitter-chialisp/src/parser.c @@ -0,0 +1,485 @@ +/* Automatically @generated by tree-sitter v0.25.10 */ + +#include "tree_sitter/parser.h" + +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic ignored "-Wmissing-field-initializers" +#endif + +#define LANGUAGE_VERSION 14 +#define STATE_COUNT 9 +#define LARGE_STATE_COUNT 8 +#define SYMBOL_COUNT 14 +#define ALIAS_COUNT 0 +#define TOKEN_COUNT 9 +#define EXTERNAL_TOKEN_COUNT 0 +#define FIELD_COUNT 0 +#define MAX_ALIAS_SEQUENCE_LENGTH 3 +#define MAX_RESERVED_WORD_SET_SIZE 0 +#define PRODUCTION_ID_COUNT 1 +#define SUPERTYPE_COUNT 0 + +enum ts_symbol_identifiers { + anon_sym_LPAREN = 1, + anon_sym_RPAREN = 2, + sym_comment = 3, + sym_hex = 4, + sym_number = 5, + sym_string = 6, + sym_dot = 7, + sym_symbol = 8, + sym_source_file = 9, + sym__form = 10, + sym__sexp = 11, + sym_list = 12, + aux_sym_source_file_repeat1 = 13, +}; + +static const char * const ts_symbol_names[] = { + [ts_builtin_sym_end] = "end", + [anon_sym_LPAREN] = "(", + [anon_sym_RPAREN] = ")", + [sym_comment] = "comment", + [sym_hex] = "hex", + [sym_number] = "number", + [sym_string] = "string", + [sym_dot] = "dot", + [sym_symbol] = "symbol", + [sym_source_file] = "source_file", + [sym__form] = "_form", + [sym__sexp] = "_sexp", + [sym_list] = "list", + [aux_sym_source_file_repeat1] = "source_file_repeat1", +}; + +static const TSSymbol ts_symbol_map[] = { + [ts_builtin_sym_end] = ts_builtin_sym_end, + [anon_sym_LPAREN] = anon_sym_LPAREN, + [anon_sym_RPAREN] = anon_sym_RPAREN, + [sym_comment] = sym_comment, + [sym_hex] = sym_hex, + [sym_number] = sym_number, + [sym_string] = sym_string, + [sym_dot] = sym_dot, + [sym_symbol] = sym_symbol, + [sym_source_file] = sym_source_file, + [sym__form] = sym__form, + [sym__sexp] = sym__sexp, + [sym_list] = sym_list, + [aux_sym_source_file_repeat1] = aux_sym_source_file_repeat1, +}; + +static const TSSymbolMetadata ts_symbol_metadata[] = { + [ts_builtin_sym_end] = { + .visible = false, + .named = true, + }, + [anon_sym_LPAREN] = { + .visible = true, + .named = false, + }, + [anon_sym_RPAREN] = { + .visible = true, + .named = false, + }, + [sym_comment] = { + .visible = true, + .named = true, + }, + [sym_hex] = { + .visible = true, + .named = true, + }, + [sym_number] = { + .visible = true, + .named = true, + }, + [sym_string] = { + .visible = true, + .named = true, + }, + [sym_dot] = { + .visible = true, + .named = true, + }, + [sym_symbol] = { + .visible = true, + .named = true, + }, + [sym_source_file] = { + .visible = true, + .named = true, + }, + [sym__form] = { + .visible = false, + .named = true, + }, + [sym__sexp] = { + .visible = false, + .named = true, + }, + [sym_list] = { + .visible = true, + .named = true, + }, + [aux_sym_source_file_repeat1] = { + .visible = false, + .named = false, + }, +}; + +static const TSSymbol ts_alias_sequences[PRODUCTION_ID_COUNT][MAX_ALIAS_SEQUENCE_LENGTH] = { + [0] = {0}, +}; + +static const uint16_t ts_non_terminal_alias_map[] = { + 0, +}; + +static const TSStateId ts_primary_state_ids[STATE_COUNT] = { + [0] = 0, + [1] = 1, + [2] = 2, + [3] = 3, + [4] = 4, + [5] = 5, + [6] = 6, + [7] = 7, + [8] = 8, +}; + +static bool ts_lex(TSLexer *lexer, TSStateId state) { + START_LEXER(); + eof = lexer->eof(lexer); + switch (state) { + case 0: + if (eof) ADVANCE(5); + ADVANCE_MAP( + '"', 1, + '\'', 2, + '(', 6, + ')', 7, + '-', 14, + '.', 13, + '0', 10, + ';', 8, + ); + if (('\t' <= lookahead && lookahead <= '\r') || + lookahead == ' ') SKIP(0); + if (('1' <= lookahead && lookahead <= '9')) ADVANCE(11); + if (lookahead != 0) ADVANCE(16); + END_STATE(); + case 1: + if (lookahead == '"') ADVANCE(12); + if (lookahead == '\\') ADVANCE(3); + if (lookahead != 0) ADVANCE(1); + END_STATE(); + case 2: + if (lookahead == '\'') ADVANCE(12); + if (lookahead == '\\') ADVANCE(4); + if (lookahead != 0) ADVANCE(2); + END_STATE(); + case 3: + if (lookahead != 0 && + lookahead != '\n') ADVANCE(1); + END_STATE(); + case 4: + if (lookahead != 0 && + lookahead != '\n') ADVANCE(2); + END_STATE(); + case 5: + ACCEPT_TOKEN(ts_builtin_sym_end); + END_STATE(); + case 6: + ACCEPT_TOKEN(anon_sym_LPAREN); + END_STATE(); + case 7: + ACCEPT_TOKEN(anon_sym_RPAREN); + END_STATE(); + case 8: + ACCEPT_TOKEN(sym_comment); + if (lookahead != 0 && + lookahead != '\n' && + lookahead != '\r') ADVANCE(8); + END_STATE(); + case 9: + ACCEPT_TOKEN(sym_hex); + if (('0' <= lookahead && lookahead <= '9') || + ('A' <= lookahead && lookahead <= 'F') || + ('a' <= lookahead && lookahead <= 'f')) ADVANCE(9); + if (lookahead != 0 && + (lookahead < '\t' || '\r' < lookahead) && + lookahead != ' ' && + lookahead != '"' && + (lookahead < '\'' || ')' < lookahead) && + lookahead != ';') ADVANCE(16); + END_STATE(); + case 10: + ACCEPT_TOKEN(sym_number); + if (lookahead == 'X' || + lookahead == 'x') ADVANCE(15); + if (('0' <= lookahead && lookahead <= '9')) ADVANCE(11); + if (lookahead != 0 && + (lookahead < '\t' || '\r' < lookahead) && + lookahead != ' ' && + lookahead != '"' && + (lookahead < '\'' || ')' < lookahead) && + lookahead != ';') ADVANCE(16); + END_STATE(); + case 11: + ACCEPT_TOKEN(sym_number); + if (('0' <= lookahead && lookahead <= '9')) ADVANCE(11); + if (lookahead != 0 && + (lookahead < '\t' || '\r' < lookahead) && + lookahead != ' ' && + lookahead != '"' && + (lookahead < '\'' || ')' < lookahead) && + lookahead != ';') ADVANCE(16); + END_STATE(); + case 12: + ACCEPT_TOKEN(sym_string); + END_STATE(); + case 13: + ACCEPT_TOKEN(sym_dot); + if (lookahead != 0 && + (lookahead < '\t' || '\r' < lookahead) && + lookahead != ' ' && + lookahead != '"' && + (lookahead < '\'' || ')' < lookahead) && + lookahead != ';') ADVANCE(16); + END_STATE(); + case 14: + ACCEPT_TOKEN(sym_symbol); + if (('0' <= lookahead && lookahead <= '9')) ADVANCE(11); + if (lookahead != 0 && + (lookahead < '\t' || '\r' < lookahead) && + lookahead != ' ' && + lookahead != '"' && + (lookahead < '\'' || ')' < lookahead) && + lookahead != ';') ADVANCE(16); + END_STATE(); + case 15: + ACCEPT_TOKEN(sym_symbol); + if (('0' <= lookahead && lookahead <= '9') || + ('A' <= lookahead && lookahead <= 'F') || + ('a' <= lookahead && lookahead <= 'f')) ADVANCE(9); + if (lookahead != 0 && + (lookahead < '\t' || '\r' < lookahead) && + lookahead != ' ' && + lookahead != '"' && + (lookahead < '\'' || ')' < lookahead) && + lookahead != ';') ADVANCE(16); + END_STATE(); + case 16: + ACCEPT_TOKEN(sym_symbol); + if (lookahead != 0 && + (lookahead < '\t' || '\r' < lookahead) && + lookahead != ' ' && + lookahead != '"' && + (lookahead < '\'' || ')' < lookahead) && + lookahead != ';') ADVANCE(16); + END_STATE(); + default: + return false; + } +} + +static const TSLexMode ts_lex_modes[STATE_COUNT] = { + [0] = {.lex_state = 0}, + [1] = {.lex_state = 0}, + [2] = {.lex_state = 0}, + [3] = {.lex_state = 0}, + [4] = {.lex_state = 0}, + [5] = {.lex_state = 0}, + [6] = {.lex_state = 0}, + [7] = {.lex_state = 0}, + [8] = {.lex_state = 0}, +}; + +static const uint16_t ts_parse_table[LARGE_STATE_COUNT][SYMBOL_COUNT] = { + [STATE(0)] = { + [ts_builtin_sym_end] = ACTIONS(1), + [anon_sym_LPAREN] = ACTIONS(1), + [anon_sym_RPAREN] = ACTIONS(1), + [sym_comment] = ACTIONS(3), + [sym_hex] = ACTIONS(1), + [sym_number] = ACTIONS(1), + [sym_string] = ACTIONS(1), + [sym_dot] = ACTIONS(1), + [sym_symbol] = ACTIONS(1), + }, + [STATE(1)] = { + [sym_source_file] = STATE(8), + [sym__form] = STATE(4), + [sym__sexp] = STATE(4), + [sym_list] = STATE(4), + [aux_sym_source_file_repeat1] = STATE(4), + [ts_builtin_sym_end] = ACTIONS(5), + [anon_sym_LPAREN] = ACTIONS(7), + [sym_comment] = ACTIONS(3), + [sym_hex] = ACTIONS(9), + [sym_number] = ACTIONS(9), + [sym_string] = ACTIONS(11), + [sym_dot] = ACTIONS(9), + [sym_symbol] = ACTIONS(9), + }, + [STATE(2)] = { + [sym__form] = STATE(2), + [sym__sexp] = STATE(2), + [sym_list] = STATE(2), + [aux_sym_source_file_repeat1] = STATE(2), + [ts_builtin_sym_end] = ACTIONS(13), + [anon_sym_LPAREN] = ACTIONS(15), + [anon_sym_RPAREN] = ACTIONS(13), + [sym_comment] = ACTIONS(3), + [sym_hex] = ACTIONS(18), + [sym_number] = ACTIONS(18), + [sym_string] = ACTIONS(21), + [sym_dot] = ACTIONS(18), + [sym_symbol] = ACTIONS(18), + }, + [STATE(3)] = { + [sym__form] = STATE(5), + [sym__sexp] = STATE(5), + [sym_list] = STATE(5), + [aux_sym_source_file_repeat1] = STATE(5), + [anon_sym_LPAREN] = ACTIONS(7), + [anon_sym_RPAREN] = ACTIONS(24), + [sym_comment] = ACTIONS(3), + [sym_hex] = ACTIONS(26), + [sym_number] = ACTIONS(26), + [sym_string] = ACTIONS(28), + [sym_dot] = ACTIONS(26), + [sym_symbol] = ACTIONS(26), + }, + [STATE(4)] = { + [sym__form] = STATE(2), + [sym__sexp] = STATE(2), + [sym_list] = STATE(2), + [aux_sym_source_file_repeat1] = STATE(2), + [ts_builtin_sym_end] = ACTIONS(30), + [anon_sym_LPAREN] = ACTIONS(7), + [sym_comment] = ACTIONS(3), + [sym_hex] = ACTIONS(32), + [sym_number] = ACTIONS(32), + [sym_string] = ACTIONS(34), + [sym_dot] = ACTIONS(32), + [sym_symbol] = ACTIONS(32), + }, + [STATE(5)] = { + [sym__form] = STATE(2), + [sym__sexp] = STATE(2), + [sym_list] = STATE(2), + [aux_sym_source_file_repeat1] = STATE(2), + [anon_sym_LPAREN] = ACTIONS(7), + [anon_sym_RPAREN] = ACTIONS(36), + [sym_comment] = ACTIONS(3), + [sym_hex] = ACTIONS(32), + [sym_number] = ACTIONS(32), + [sym_string] = ACTIONS(34), + [sym_dot] = ACTIONS(32), + [sym_symbol] = ACTIONS(32), + }, + [STATE(6)] = { + [ts_builtin_sym_end] = ACTIONS(38), + [anon_sym_LPAREN] = ACTIONS(38), + [anon_sym_RPAREN] = ACTIONS(38), + [sym_comment] = ACTIONS(3), + [sym_hex] = ACTIONS(40), + [sym_number] = ACTIONS(40), + [sym_string] = ACTIONS(38), + [sym_dot] = ACTIONS(40), + [sym_symbol] = ACTIONS(40), + }, + [STATE(7)] = { + [ts_builtin_sym_end] = ACTIONS(42), + [anon_sym_LPAREN] = ACTIONS(42), + [anon_sym_RPAREN] = ACTIONS(42), + [sym_comment] = ACTIONS(3), + [sym_hex] = ACTIONS(44), + [sym_number] = ACTIONS(44), + [sym_string] = ACTIONS(42), + [sym_dot] = ACTIONS(44), + [sym_symbol] = ACTIONS(44), + }, +}; + +static const uint16_t ts_small_parse_table[] = { + [0] = 2, + ACTIONS(3), 1, + sym_comment, + ACTIONS(46), 1, + ts_builtin_sym_end, +}; + +static const uint32_t ts_small_parse_table_map[] = { + [SMALL_STATE(8)] = 0, +}; + +static const TSParseActionEntry ts_parse_actions[] = { + [0] = {.entry = {.count = 0, .reusable = false}}, + [1] = {.entry = {.count = 1, .reusable = false}}, RECOVER(), + [3] = {.entry = {.count = 1, .reusable = true}}, SHIFT_EXTRA(), + [5] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_source_file, 0, 0, 0), + [7] = {.entry = {.count = 1, .reusable = true}}, SHIFT(3), + [9] = {.entry = {.count = 1, .reusable = false}}, SHIFT(4), + [11] = {.entry = {.count = 1, .reusable = true}}, SHIFT(4), + [13] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_source_file_repeat1, 2, 0, 0), + [15] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_source_file_repeat1, 2, 0, 0), SHIFT_REPEAT(3), + [18] = {.entry = {.count = 2, .reusable = false}}, REDUCE(aux_sym_source_file_repeat1, 2, 0, 0), SHIFT_REPEAT(2), + [21] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_source_file_repeat1, 2, 0, 0), SHIFT_REPEAT(2), + [24] = {.entry = {.count = 1, .reusable = true}}, SHIFT(6), + [26] = {.entry = {.count = 1, .reusable = false}}, SHIFT(5), + [28] = {.entry = {.count = 1, .reusable = true}}, SHIFT(5), + [30] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_source_file, 1, 0, 0), + [32] = {.entry = {.count = 1, .reusable = false}}, SHIFT(2), + [34] = {.entry = {.count = 1, .reusable = true}}, SHIFT(2), + [36] = {.entry = {.count = 1, .reusable = true}}, SHIFT(7), + [38] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_list, 2, 0, 0), + [40] = {.entry = {.count = 1, .reusable = false}}, REDUCE(sym_list, 2, 0, 0), + [42] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_list, 3, 0, 0), + [44] = {.entry = {.count = 1, .reusable = false}}, REDUCE(sym_list, 3, 0, 0), + [46] = {.entry = {.count = 1, .reusable = true}}, ACCEPT_INPUT(), +}; + +#ifdef __cplusplus +extern "C" { +#endif +#ifdef TREE_SITTER_HIDE_SYMBOLS +#define TS_PUBLIC +#elif defined(_WIN32) +#define TS_PUBLIC __declspec(dllexport) +#else +#define TS_PUBLIC __attribute__((visibility("default"))) +#endif + +TS_PUBLIC const TSLanguage *tree_sitter_chialisp(void) { + static const TSLanguage language = { + .abi_version = LANGUAGE_VERSION, + .symbol_count = SYMBOL_COUNT, + .alias_count = ALIAS_COUNT, + .token_count = TOKEN_COUNT, + .external_token_count = EXTERNAL_TOKEN_COUNT, + .state_count = STATE_COUNT, + .large_state_count = LARGE_STATE_COUNT, + .production_id_count = PRODUCTION_ID_COUNT, + .field_count = FIELD_COUNT, + .max_alias_sequence_length = MAX_ALIAS_SEQUENCE_LENGTH, + .parse_table = &ts_parse_table[0][0], + .small_parse_table = ts_small_parse_table, + .small_parse_table_map = ts_small_parse_table_map, + .parse_actions = ts_parse_actions, + .symbol_names = ts_symbol_names, + .symbol_metadata = ts_symbol_metadata, + .public_symbol_map = ts_symbol_map, + .alias_map = ts_non_terminal_alias_map, + .alias_sequences = &ts_alias_sequences[0][0], + .lex_modes = (const void*)ts_lex_modes, + .lex_fn = ts_lex, + .primary_state_ids = ts_primary_state_ids, + }; + return &language; +} +#ifdef __cplusplus +} +#endif diff --git a/tools/tree-sitter-chialisp/src/tree_sitter/alloc.h b/tools/tree-sitter-chialisp/src/tree_sitter/alloc.h new file mode 100644 index 000000000..1abdd1201 --- /dev/null +++ b/tools/tree-sitter-chialisp/src/tree_sitter/alloc.h @@ -0,0 +1,54 @@ +#ifndef TREE_SITTER_ALLOC_H_ +#define TREE_SITTER_ALLOC_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +// Allow clients to override allocation functions +#ifdef TREE_SITTER_REUSE_ALLOCATOR + +extern void *(*ts_current_malloc)(size_t size); +extern void *(*ts_current_calloc)(size_t count, size_t size); +extern void *(*ts_current_realloc)(void *ptr, size_t size); +extern void (*ts_current_free)(void *ptr); + +#ifndef ts_malloc +#define ts_malloc ts_current_malloc +#endif +#ifndef ts_calloc +#define ts_calloc ts_current_calloc +#endif +#ifndef ts_realloc +#define ts_realloc ts_current_realloc +#endif +#ifndef ts_free +#define ts_free ts_current_free +#endif + +#else + +#ifndef ts_malloc +#define ts_malloc malloc +#endif +#ifndef ts_calloc +#define ts_calloc calloc +#endif +#ifndef ts_realloc +#define ts_realloc realloc +#endif +#ifndef ts_free +#define ts_free free +#endif + +#endif + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_ALLOC_H_ diff --git a/tools/tree-sitter-chialisp/src/tree_sitter/array.h b/tools/tree-sitter-chialisp/src/tree_sitter/array.h new file mode 100644 index 000000000..a17a574f0 --- /dev/null +++ b/tools/tree-sitter-chialisp/src/tree_sitter/array.h @@ -0,0 +1,291 @@ +#ifndef TREE_SITTER_ARRAY_H_ +#define TREE_SITTER_ARRAY_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "./alloc.h" + +#include +#include +#include +#include +#include + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4101) +#elif defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" +#endif + +#define Array(T) \ + struct { \ + T *contents; \ + uint32_t size; \ + uint32_t capacity; \ + } + +/// Initialize an array. +#define array_init(self) \ + ((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL) + +/// Create an empty array. +#define array_new() \ + { NULL, 0, 0 } + +/// Get a pointer to the element at a given `index` in the array. +#define array_get(self, _index) \ + (assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index]) + +/// Get a pointer to the first element in the array. +#define array_front(self) array_get(self, 0) + +/// Get a pointer to the last element in the array. +#define array_back(self) array_get(self, (self)->size - 1) + +/// Clear the array, setting its size to zero. Note that this does not free any +/// memory allocated for the array's contents. +#define array_clear(self) ((self)->size = 0) + +/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is +/// less than the array's current capacity, this function has no effect. +#define array_reserve(self, new_capacity) \ + _array__reserve((Array *)(self), array_elem_size(self), new_capacity) + +/// Free any memory allocated for this array. Note that this does not free any +/// memory allocated for the array's contents. +#define array_delete(self) _array__delete((Array *)(self)) + +/// Push a new `element` onto the end of the array. +#define array_push(self, element) \ + (_array__grow((Array *)(self), 1, array_elem_size(self)), \ + (self)->contents[(self)->size++] = (element)) + +/// Increase the array's size by `count` elements. +/// New elements are zero-initialized. +#define array_grow_by(self, count) \ + do { \ + if ((count) == 0) break; \ + _array__grow((Array *)(self), count, array_elem_size(self)); \ + memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \ + (self)->size += (count); \ + } while (0) + +/// Append all elements from one array to the end of another. +#define array_push_all(self, other) \ + array_extend((self), (other)->size, (other)->contents) + +/// Append `count` elements to the end of the array, reading their values from the +/// `contents` pointer. +#define array_extend(self, count, contents) \ + _array__splice( \ + (Array *)(self), array_elem_size(self), (self)->size, \ + 0, count, contents \ + ) + +/// Remove `old_count` elements from the array starting at the given `index`. At +/// the same index, insert `new_count` new elements, reading their values from the +/// `new_contents` pointer. +#define array_splice(self, _index, old_count, new_count, new_contents) \ + _array__splice( \ + (Array *)(self), array_elem_size(self), _index, \ + old_count, new_count, new_contents \ + ) + +/// Insert one `element` into the array at the given `index`. +#define array_insert(self, _index, element) \ + _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element)) + +/// Remove one element from the array at the given `index`. +#define array_erase(self, _index) \ + _array__erase((Array *)(self), array_elem_size(self), _index) + +/// Pop the last element off the array, returning the element by value. +#define array_pop(self) ((self)->contents[--(self)->size]) + +/// Assign the contents of one array to another, reallocating if necessary. +#define array_assign(self, other) \ + _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self)) + +/// Swap one array with another +#define array_swap(self, other) \ + _array__swap((Array *)(self), (Array *)(other)) + +/// Get the size of the array contents +#define array_elem_size(self) (sizeof *(self)->contents) + +/// Search a sorted array for a given `needle` value, using the given `compare` +/// callback to determine the order. +/// +/// If an existing element is found to be equal to `needle`, then the `index` +/// out-parameter is set to the existing value's index, and the `exists` +/// out-parameter is set to true. Otherwise, `index` is set to an index where +/// `needle` should be inserted in order to preserve the sorting, and `exists` +/// is set to false. +#define array_search_sorted_with(self, compare, needle, _index, _exists) \ + _array__search_sorted(self, 0, compare, , needle, _index, _exists) + +/// Search a sorted array for a given `needle` value, using integer comparisons +/// of a given struct field (specified with a leading dot) to determine the order. +/// +/// See also `array_search_sorted_with`. +#define array_search_sorted_by(self, field, needle, _index, _exists) \ + _array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists) + +/// Insert a given `value` into a sorted array, using the given `compare` +/// callback to determine the order. +#define array_insert_sorted_with(self, compare, value) \ + do { \ + unsigned _index, _exists; \ + array_search_sorted_with(self, compare, &(value), &_index, &_exists); \ + if (!_exists) array_insert(self, _index, value); \ + } while (0) + +/// Insert a given `value` into a sorted array, using integer comparisons of +/// a given struct field (specified with a leading dot) to determine the order. +/// +/// See also `array_search_sorted_by`. +#define array_insert_sorted_by(self, field, value) \ + do { \ + unsigned _index, _exists; \ + array_search_sorted_by(self, field, (value) field, &_index, &_exists); \ + if (!_exists) array_insert(self, _index, value); \ + } while (0) + +// Private + +typedef Array(void) Array; + +/// This is not what you're looking for, see `array_delete`. +static inline void _array__delete(Array *self) { + if (self->contents) { + ts_free(self->contents); + self->contents = NULL; + self->size = 0; + self->capacity = 0; + } +} + +/// This is not what you're looking for, see `array_erase`. +static inline void _array__erase(Array *self, size_t element_size, + uint32_t index) { + assert(index < self->size); + char *contents = (char *)self->contents; + memmove(contents + index * element_size, contents + (index + 1) * element_size, + (self->size - index - 1) * element_size); + self->size--; +} + +/// This is not what you're looking for, see `array_reserve`. +static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) { + if (new_capacity > self->capacity) { + if (self->contents) { + self->contents = ts_realloc(self->contents, new_capacity * element_size); + } else { + self->contents = ts_malloc(new_capacity * element_size); + } + self->capacity = new_capacity; + } +} + +/// This is not what you're looking for, see `array_assign`. +static inline void _array__assign(Array *self, const Array *other, size_t element_size) { + _array__reserve(self, element_size, other->size); + self->size = other->size; + memcpy(self->contents, other->contents, self->size * element_size); +} + +/// This is not what you're looking for, see `array_swap`. +static inline void _array__swap(Array *self, Array *other) { + Array swap = *other; + *other = *self; + *self = swap; +} + +/// This is not what you're looking for, see `array_push` or `array_grow_by`. +static inline void _array__grow(Array *self, uint32_t count, size_t element_size) { + uint32_t new_size = self->size + count; + if (new_size > self->capacity) { + uint32_t new_capacity = self->capacity * 2; + if (new_capacity < 8) new_capacity = 8; + if (new_capacity < new_size) new_capacity = new_size; + _array__reserve(self, element_size, new_capacity); + } +} + +/// This is not what you're looking for, see `array_splice`. +static inline void _array__splice(Array *self, size_t element_size, + uint32_t index, uint32_t old_count, + uint32_t new_count, const void *elements) { + uint32_t new_size = self->size + new_count - old_count; + uint32_t old_end = index + old_count; + uint32_t new_end = index + new_count; + assert(old_end <= self->size); + + _array__reserve(self, element_size, new_size); + + char *contents = (char *)self->contents; + if (self->size > old_end) { + memmove( + contents + new_end * element_size, + contents + old_end * element_size, + (self->size - old_end) * element_size + ); + } + if (new_count > 0) { + if (elements) { + memcpy( + (contents + index * element_size), + elements, + new_count * element_size + ); + } else { + memset( + (contents + index * element_size), + 0, + new_count * element_size + ); + } + } + self->size += new_count - old_count; +} + +/// A binary search routine, based on Rust's `std::slice::binary_search_by`. +/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`. +#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \ + do { \ + *(_index) = start; \ + *(_exists) = false; \ + uint32_t size = (self)->size - *(_index); \ + if (size == 0) break; \ + int comparison; \ + while (size > 1) { \ + uint32_t half_size = size / 2; \ + uint32_t mid_index = *(_index) + half_size; \ + comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \ + if (comparison <= 0) *(_index) = mid_index; \ + size -= half_size; \ + } \ + comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \ + if (comparison == 0) *(_exists) = true; \ + else if (comparison < 0) *(_index) += 1; \ + } while (0) + +/// Helper macro for the `_sorted_by` routines below. This takes the left (existing) +/// parameter by reference in order to work with the generic sorting function above. +#define _compare_int(a, b) ((int)*(a) - (int)(b)) + +#ifdef _MSC_VER +#pragma warning(pop) +#elif defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_ARRAY_H_ diff --git a/tools/tree-sitter-chialisp/src/tree_sitter/parser.h b/tools/tree-sitter-chialisp/src/tree_sitter/parser.h new file mode 100644 index 000000000..858107deb --- /dev/null +++ b/tools/tree-sitter-chialisp/src/tree_sitter/parser.h @@ -0,0 +1,286 @@ +#ifndef TREE_SITTER_PARSER_H_ +#define TREE_SITTER_PARSER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#define ts_builtin_sym_error ((TSSymbol)-1) +#define ts_builtin_sym_end 0 +#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 + +#ifndef TREE_SITTER_API_H_ +typedef uint16_t TSStateId; +typedef uint16_t TSSymbol; +typedef uint16_t TSFieldId; +typedef struct TSLanguage TSLanguage; +typedef struct TSLanguageMetadata { + uint8_t major_version; + uint8_t minor_version; + uint8_t patch_version; +} TSLanguageMetadata; +#endif + +typedef struct { + TSFieldId field_id; + uint8_t child_index; + bool inherited; +} TSFieldMapEntry; + +// Used to index the field and supertype maps. +typedef struct { + uint16_t index; + uint16_t length; +} TSMapSlice; + +typedef struct { + bool visible; + bool named; + bool supertype; +} TSSymbolMetadata; + +typedef struct TSLexer TSLexer; + +struct TSLexer { + int32_t lookahead; + TSSymbol result_symbol; + void (*advance)(TSLexer *, bool); + void (*mark_end)(TSLexer *); + uint32_t (*get_column)(TSLexer *); + bool (*is_at_included_range_start)(const TSLexer *); + bool (*eof)(const TSLexer *); + void (*log)(const TSLexer *, const char *, ...); +}; + +typedef enum { + TSParseActionTypeShift, + TSParseActionTypeReduce, + TSParseActionTypeAccept, + TSParseActionTypeRecover, +} TSParseActionType; + +typedef union { + struct { + uint8_t type; + TSStateId state; + bool extra; + bool repetition; + } shift; + struct { + uint8_t type; + uint8_t child_count; + TSSymbol symbol; + int16_t dynamic_precedence; + uint16_t production_id; + } reduce; + uint8_t type; +} TSParseAction; + +typedef struct { + uint16_t lex_state; + uint16_t external_lex_state; +} TSLexMode; + +typedef struct { + uint16_t lex_state; + uint16_t external_lex_state; + uint16_t reserved_word_set_id; +} TSLexerMode; + +typedef union { + TSParseAction action; + struct { + uint8_t count; + bool reusable; + } entry; +} TSParseActionEntry; + +typedef struct { + int32_t start; + int32_t end; +} TSCharacterRange; + +struct TSLanguage { + uint32_t abi_version; + uint32_t symbol_count; + uint32_t alias_count; + uint32_t token_count; + uint32_t external_token_count; + uint32_t state_count; + uint32_t large_state_count; + uint32_t production_id_count; + uint32_t field_count; + uint16_t max_alias_sequence_length; + const uint16_t *parse_table; + const uint16_t *small_parse_table; + const uint32_t *small_parse_table_map; + const TSParseActionEntry *parse_actions; + const char * const *symbol_names; + const char * const *field_names; + const TSMapSlice *field_map_slices; + const TSFieldMapEntry *field_map_entries; + const TSSymbolMetadata *symbol_metadata; + const TSSymbol *public_symbol_map; + const uint16_t *alias_map; + const TSSymbol *alias_sequences; + const TSLexerMode *lex_modes; + bool (*lex_fn)(TSLexer *, TSStateId); + bool (*keyword_lex_fn)(TSLexer *, TSStateId); + TSSymbol keyword_capture_token; + struct { + const bool *states; + const TSSymbol *symbol_map; + void *(*create)(void); + void (*destroy)(void *); + bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist); + unsigned (*serialize)(void *, char *); + void (*deserialize)(void *, const char *, unsigned); + } external_scanner; + const TSStateId *primary_state_ids; + const char *name; + const TSSymbol *reserved_words; + uint16_t max_reserved_word_set_size; + uint32_t supertype_count; + const TSSymbol *supertype_symbols; + const TSMapSlice *supertype_map_slices; + const TSSymbol *supertype_map_entries; + TSLanguageMetadata metadata; +}; + +static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, int32_t lookahead) { + uint32_t index = 0; + uint32_t size = len - index; + while (size > 1) { + uint32_t half_size = size / 2; + uint32_t mid_index = index + half_size; + const TSCharacterRange *range = &ranges[mid_index]; + if (lookahead >= range->start && lookahead <= range->end) { + return true; + } else if (lookahead > range->end) { + index = mid_index; + } + size -= half_size; + } + const TSCharacterRange *range = &ranges[index]; + return (lookahead >= range->start && lookahead <= range->end); +} + +/* + * Lexer Macros + */ + +#ifdef _MSC_VER +#define UNUSED __pragma(warning(suppress : 4101)) +#else +#define UNUSED __attribute__((unused)) +#endif + +#define START_LEXER() \ + bool result = false; \ + bool skip = false; \ + UNUSED \ + bool eof = false; \ + int32_t lookahead; \ + goto start; \ + next_state: \ + lexer->advance(lexer, skip); \ + start: \ + skip = false; \ + lookahead = lexer->lookahead; + +#define ADVANCE(state_value) \ + { \ + state = state_value; \ + goto next_state; \ + } + +#define ADVANCE_MAP(...) \ + { \ + static const uint16_t map[] = { __VA_ARGS__ }; \ + for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \ + if (map[i] == lookahead) { \ + state = map[i + 1]; \ + goto next_state; \ + } \ + } \ + } + +#define SKIP(state_value) \ + { \ + skip = true; \ + state = state_value; \ + goto next_state; \ + } + +#define ACCEPT_TOKEN(symbol_value) \ + result = true; \ + lexer->result_symbol = symbol_value; \ + lexer->mark_end(lexer); + +#define END_STATE() return result; + +/* + * Parse Table Macros + */ + +#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT) + +#define STATE(id) id + +#define ACTIONS(id) id + +#define SHIFT(state_value) \ + {{ \ + .shift = { \ + .type = TSParseActionTypeShift, \ + .state = (state_value) \ + } \ + }} + +#define SHIFT_REPEAT(state_value) \ + {{ \ + .shift = { \ + .type = TSParseActionTypeShift, \ + .state = (state_value), \ + .repetition = true \ + } \ + }} + +#define SHIFT_EXTRA() \ + {{ \ + .shift = { \ + .type = TSParseActionTypeShift, \ + .extra = true \ + } \ + }} + +#define REDUCE(symbol_name, children, precedence, prod_id) \ + {{ \ + .reduce = { \ + .type = TSParseActionTypeReduce, \ + .symbol = symbol_name, \ + .child_count = children, \ + .dynamic_precedence = precedence, \ + .production_id = prod_id \ + }, \ + }} + +#define RECOVER() \ + {{ \ + .type = TSParseActionTypeRecover \ + }} + +#define ACCEPT_INPUT() \ + {{ \ + .type = TSParseActionTypeAccept \ + }} + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_PARSER_H_ diff --git a/tools/tree-sitter-chialisp/test/corpus/atoms.txt b/tools/tree-sitter-chialisp/test/corpus/atoms.txt new file mode 100644 index 000000000..06ee243b6 --- /dev/null +++ b/tools/tree-sitter-chialisp/test/corpus/atoms.txt @@ -0,0 +1,119 @@ +================================================================================ +Hex literals, integers and negative numbers +================================================================================ + +(0x17 0x0a1B2c 12345 -42 0) + +-------------------------------------------------------------------------------- + +(source_file + (list + (hex) + (hex) + (number) + (number) + (number))) + +================================================================================ +Strings with escapes, both quote characters +================================================================================ + +("plain" "a\"b" 'single' "back\\slash") + +-------------------------------------------------------------------------------- + +(source_file + (list + (string) + (string) + (string) + (string))) + +================================================================================ +Dot in dotted-pair position +================================================================================ + +(defun curry_hashes (mod_hash . curry_parameter_hashes) + (q . (x)) +) + +-------------------------------------------------------------------------------- + +(source_file + (list + (symbol) + (symbol) + (list + (symbol) + (dot) + (symbol)) + (list + (symbol) + (dot) + (list + (symbol))))) + +================================================================================ +Bare minus is a symbol, not a number +================================================================================ + +(- (lsh 1 128) 1) + +-------------------------------------------------------------------------------- + +(source_file + (list + (symbol) + (list + (symbol) + (number) + (number)) + (number))) + +================================================================================ +Longest match wins over the specific atom kinds +================================================================================ + +(0xdeadZZ 123abc foo.clib @ #q .5) + +-------------------------------------------------------------------------------- + +(source_file + (list + (symbol) + (symbol) + (symbol) + (symbol) + (symbol) + (symbol))) + +================================================================================ +Constant table of hex pairs with interleaved comments +================================================================================ + +(defconstant constant_tree ( + (0x4b . ; = `(sha256 1)` + 0x9d) . ; = `(sha256 1 1)` + (0x02 . + 0xa8) + ) +) + +-------------------------------------------------------------------------------- + +(source_file + (list + (symbol) + (symbol) + (list + (list + (hex) + (dot) + (comment) + (hex)) + (dot) + (comment) + (list + (hex) + (dot) + (hex))))) diff --git a/tools/tree-sitter-chialisp/test/corpus/comments.txt b/tools/tree-sitter-chialisp/test/corpus/comments.txt new file mode 100644 index 000000000..0c0a173ed --- /dev/null +++ b/tools/tree-sitter-chialisp/test/corpus/comments.txt @@ -0,0 +1,83 @@ +================================================================================ +Comment with LF line endings +================================================================================ + +(mod (A) + ; a comment + (f A) +) + +-------------------------------------------------------------------------------- + +(source_file + (list + (symbol) + (list + (symbol)) + (comment) + (list + (symbol) + (symbol)))) + +================================================================================ +Comment with CRLF line endings +================================================================================ + +(mod (A) + ; a comment + (f A) +) + +-------------------------------------------------------------------------------- + +(source_file + (list + (symbol) + (list + (symbol)) + (comment) + (list + (symbol) + (symbol)))) + +================================================================================ +Comment terminated by EOF with no trailing newline +================================================================================ + +(mod (A) (f A)) +; trailing comment at EOF +-------------------------------------------------------------------------------- + +(source_file + (list + (symbol) + (list + (symbol)) + (list + (symbol) + (symbol))) + (comment)) + +================================================================================ +Double-semicolon doc comments between definitions +================================================================================ + +( + ;; the tree hash of a curried function + (defun-inline sha256_one _noargs (f (f constant_tree))) +) + +-------------------------------------------------------------------------------- + +(source_file + (list + (comment) + (list + (symbol) + (symbol) + (symbol) + (list + (symbol) + (list + (symbol) + (symbol)))))) diff --git a/tools/tree-sitter-chialisp/test/corpus/forms.txt b/tools/tree-sitter-chialisp/test/corpus/forms.txt new file mode 100644 index 000000000..d79a92e5d --- /dev/null +++ b/tools/tree-sitter-chialisp/test/corpus/forms.txt @@ -0,0 +1,136 @@ +================================================================================ +Include with a dotted filename +================================================================================ + +(include condition_codes.clib) +(include curry-and-treehash.clinc) +(include *standard-cl-26*) + +-------------------------------------------------------------------------------- + +(source_file + (list + (symbol) + (symbol)) + (list + (symbol) + (symbol)) + (list + (symbol) + (symbol))) + +================================================================================ +defconstant bound to an arbitrary expression +================================================================================ + +(defconstant HASH (sha256 1)) +(defconstant TWO 2) +(defconst MAX 1000) + +-------------------------------------------------------------------------------- + +(source_file + (list + (symbol) + (symbol) + (list + (symbol) + (number))) + (list + (symbol) + (symbol) + (number)) + (list + (symbol) + (symbol) + (number))) + +================================================================================ +Nested mod / defun / defun-inline / defmacro / defmac / let / assign +================================================================================ + +(mod (ARG) + (defun-inline sq (x) (* x x)) + (defmacro assert items (f items)) + (defmac and ARGS 1) + (defun outer (y) (let ((a 1)) (+ a y))) + (defun outer2 (y) (assign z (* y 2) (+ z 1))) + (sq ARG) +) + +-------------------------------------------------------------------------------- + +(source_file + (list + (symbol) + (list + (symbol)) + (list + (symbol) + (symbol) + (list + (symbol)) + (list + (symbol) + (symbol) + (symbol))) + (list + (symbol) + (symbol) + (symbol) + (list + (symbol) + (symbol))) + (list + (symbol) + (symbol) + (symbol) + (number)) + (list + (symbol) + (symbol) + (list + (symbol)) + (list + (symbol) + (list + (list + (symbol) + (number))) + (list + (symbol) + (symbol) + (symbol)))) + (list + (symbol) + (symbol) + (list + (symbol)) + (list + (symbol) + (symbol) + (list + (symbol) + (symbol) + (number)) + (list + (symbol) + (symbol) + (number)))) + (list + (symbol) + (symbol)))) + +================================================================================ +Empty list and empty file body +================================================================================ + +(mod () ()) + +-------------------------------------------------------------------------------- + +(source_file + (list + (symbol) + (list) + (list))) diff --git a/tools/tree-sitter-chialisp/tree-sitter.json b/tools/tree-sitter-chialisp/tree-sitter.json new file mode 100644 index 000000000..e6841d702 --- /dev/null +++ b/tools/tree-sitter-chialisp/tree-sitter.json @@ -0,0 +1,16 @@ +{ + "grammars": [ + { + "name": "chialisp", + "camelcase": "Chialisp", + "scope": "source.chialisp", + "path": ".", + "file-types": ["clsp", "clib", "clinc"] + } + ], + "metadata": { + "version": "0.1.0", + "license": "MIT", + "description": "Chialisp (Chia smart-coin s-expression language) grammar for tree-sitter" + } +} From e6398fd9579f428a9ae418c75ec95dfd134c27dd Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 28 Aug 2026 21:43:50 +0200 Subject: [PATCH 16/31] style(tests): keep the Chialisp test diffs to the lines that changed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Chialisp commit was formatted with the pinned Homebrew clang-format, which is right for the production sources but wrong for the three test files it also touched: tests/ is not in LINT_SRCS, so those files have never been under the format gate and were already format-dirty on main. Running the formatter over them rewrote code that had nothing to do with Chialisp — 478 whitespace-only lines in repro_invariant_breadth.c alone, plus 50 in test_extraction.c and 14 in test_lang_contract.c — which is a review burden and hides the actual change. Restore those three files to main's bytes and re-apply only the Chialisp additions, in each file's own house style. The result is token-identical to the version that was tested, apart from one em-dash written as `--` in a comment. Re-verified after the rewrite: the six chialisp extraction tests, the cross-file Constant graph test, the matrix-B row, the breadth case (`[INV-BREADTH] PASS chialisp calls=1 callable=1 module=0`), and `make lint-ci`. The production sources keep their formatting: they ARE in LINT_SRCS, and the only whitespace they carry is the trailing-comment realignment in cbm.h that adding a longer enumerator forces, plus the re-indent in extract_defs.c from moving the name-resolution block into an else branch. Signed-off-by: Martin Vogel --- tests/repro/repro_invariant_breadth.c | 723 ++++++++++++++------------ tests/test_extraction.c | 72 +-- tests/test_lang_contract.c | 24 +- 3 files changed, 434 insertions(+), 385 deletions(-) diff --git a/tests/repro/repro_invariant_breadth.c b/tests/repro/repro_invariant_breadth.c index 6aa8924d3..5c7b94f1b 100644 --- a/tests/repro/repro_invariant_breadth.c +++ b/tests/repro/repro_invariant_breadth.c @@ -28,7 +28,7 @@ * fsharp, racket, rescript, scheme (12 cases) * * Added after the original snapshot: - * chialisp (1 case, expected GREEN — lisp-family scope attribution) + * chialisp (1 case, expected GREEN -- lisp-family scope attribution) * * Note: the "suspicious" group (r, julia, ...) from QUALITY_ANALYSIS may be * GREEN because the calls-breadth table (test_lang_contract.c) already shows @@ -54,17 +54,19 @@ /* ---- helper: count CALLS edges by source-node label --------------------- */ -static int ib_calls_from_label(cbm_store_t *store, const char *project, const char *label) { +static int ib_calls_from_label(cbm_store_t *store, const char *project, + const char *label) { cbm_edge_t *edges = NULL; int edge_count = 0; - if (cbm_store_find_edges_by_type(store, project, "CALLS", &edges, &edge_count) != - CBM_STORE_OK) { + if (cbm_store_find_edges_by_type(store, project, "CALLS", + &edges, &edge_count) != CBM_STORE_OK) { return -1; } int total = 0; for (int i = 0; i < edge_count; i++) { cbm_node_t src = {0}; - if (cbm_store_find_node_by_id(store, edges[i].source_id, &src) != CBM_STORE_OK) { + if (cbm_store_find_node_by_id(store, edges[i].source_id, + &src) != CBM_STORE_OK) { continue; } if (src.label && strcmp(src.label, label) == 0) { @@ -136,40 +138,42 @@ static const IBCase IB_CASES[] = { /* CALLS edge is produced -- but it may still be Module-sourced. */ /* ------------------------------------------------------------------ */ - {"r", "a.R", - "helper <- function(x) {\n" - " x * 2\n" - "}\n" - "\n" - "run <- function() {\n" - " helper(21)\n" - "}\n", - /* - * R: extract_calls.c has an R branch that reads the callee from the - * call node's first child. However, enclosing-function detection - * for R may fall back to Module if func_kinds_for_lang does not - * include R's "function_definition" node type. RED when the CALLS - * edge is sourced at Module instead of the "run" Function node. - */ - 0, - "R enclosing-function detection likely missing from func_kinds_for_lang; " - "call may be sourced at Module"}, + { + "r", "a.R", + "helper <- function(x) {\n" + " x * 2\n" + "}\n" + "\n" + "run <- function() {\n" + " helper(21)\n" + "}\n", + /* + * R: extract_calls.c has an R branch that reads the callee from the + * call node's first child. However, enclosing-function detection + * for R may fall back to Module if func_kinds_for_lang does not + * include R's "function_definition" node type. RED when the CALLS + * edge is sourced at Module instead of the "run" Function node. + */ + 0, "R enclosing-function detection likely missing from func_kinds_for_lang; " + "call may be sourced at Module" + }, - {"julia", "a.jl", - "function helper(x)\n" - " return x + 1\n" - "end\n" - "\n" - "function run(n)\n" - " return helper(n)\n" - "end\n", - /* - * Julia: same issue -- function body extraction may not detect the - * enclosing Julia function node correctly, sourcing the call at Module. - */ - 0, - "Julia enclosing-function detection may not map function_definition to " - "a callable QN; call sourced at Module"}, + { + "julia", "a.jl", + "function helper(x)\n" + " return x + 1\n" + "end\n" + "\n" + "function run(n)\n" + " return helper(n)\n" + "end\n", + /* + * Julia: same issue -- function body extraction may not detect the + * enclosing Julia function node correctly, sourcing the call at Module. + */ + 0, "Julia enclosing-function detection may not map function_definition to " + "a callable QN; call sourced at Module" + }, /* ------------------------------------------------------------------ */ /* EXPECTED-GREEN GROUP (regression guards) */ @@ -178,163 +182,191 @@ static const IBCase IB_CASES[] = { /* them will turn the corresponding case RED. */ /* ------------------------------------------------------------------ */ - {"elixir", "a.ex", - "defmodule Sample do\n" - " def helper(x) do\n" - " x + 1\n" - " end\n" - "\n" - " def run do\n" - " helper(41)\n" - " end\n" - "end\n", - 1, NULL}, + { + "elixir", "a.ex", + "defmodule Sample do\n" + " def helper(x) do\n" + " x + 1\n" + " end\n" + "\n" + " def run do\n" + " helper(41)\n" + " end\n" + "end\n", + 1, NULL + }, - {"ocaml", "a.ml", - "let helper x = x + 1\n" - "\n" - "let run () =\n" - " let result = helper 41 in\n" - " print_int result\n", - 1, NULL}, + { + "ocaml", "a.ml", + "let helper x = x + 1\n" + "\n" + "let run () =\n" + " let result = helper 41 in\n" + " print_int result\n", + 1, NULL + }, - {"fortran", "a.f90", - "function helper(x) result(y)\n" - " integer, intent(in) :: x\n" - " integer :: y\n" - " y = x + 1\n" - "end function helper\n" - "\n" - "function run(n) result(total)\n" - " integer, intent(in) :: n\n" - " integer :: total\n" - " total = helper(n) + helper(n + 1)\n" - "end function run\n", - 1, NULL}, + { + "fortran", "a.f90", + "function helper(x) result(y)\n" + " integer, intent(in) :: x\n" + " integer :: y\n" + " y = x + 1\n" + "end function helper\n" + "\n" + "function run(n) result(total)\n" + " integer, intent(in) :: n\n" + " integer :: total\n" + " total = helper(n) + helper(n + 1)\n" + "end function run\n", + 1, NULL + }, - {"pascal", "a.pas", - "procedure Helper(x: Integer);\n" - "begin\n" - " WriteLn(x);\n" - "end;\n" - "\n" - "procedure Run;\n" - "begin\n" - " Helper(1);\n" - "end;\n", - 1, NULL}, + { + "pascal", "a.pas", + "procedure Helper(x: Integer);\n" + "begin\n" + " WriteLn(x);\n" + "end;\n" + "\n" + "procedure Run;\n" + "begin\n" + " Helper(1);\n" + "end;\n", + 1, NULL + }, - {"cuda", "a.cu", - "__device__ int helper(int x) {\n" - " return x * 2;\n" - "}\n" - "\n" - "__global__ void run(int *out) {\n" - " out[0] = helper(21);\n" - "}\n", - 1, NULL}, + { + "cuda", "a.cu", + "__device__ int helper(int x) {\n" + " return x * 2;\n" + "}\n" + "\n" + "__global__ void run(int *out) {\n" + " out[0] = helper(21);\n" + "}\n", + 1, NULL + }, - {"d", "a.d", - "int helper(int x)\n" - "{\n" - " return x + 1;\n" - "}\n" - "\n" - "void run()\n" - "{\n" - " int y = helper(41);\n" - "}\n", - 1, NULL}, + { + "d", "a.d", + "int helper(int x)\n" + "{\n" + " return x + 1;\n" + "}\n" + "\n" + "void run()\n" + "{\n" + " int y = helper(41);\n" + "}\n", + 1, NULL + }, - {"glsl", "a.glsl", - "float helper(float x) {\n" - " return x * 2.0;\n" - "}\n" - "\n" - "void run() {\n" - " float y = helper(3.0);\n" - "}\n", - 1, NULL}, + { + "glsl", "a.glsl", + "float helper(float x) {\n" + " return x * 2.0;\n" + "}\n" + "\n" + "void run() {\n" + " float y = helper(3.0);\n" + "}\n", + 1, NULL + }, - {"hlsl", "a.hlsl", - "float helper(float x)\n" - "{\n" - " return x * 2.0;\n" - "}\n" - "\n" - "float run(float v)\n" - "{\n" - " return helper(v) + 1.0;\n" - "}\n", - 1, NULL}, + { + "hlsl", "a.hlsl", + "float helper(float x)\n" + "{\n" + " return x * 2.0;\n" + "}\n" + "\n" + "float run(float v)\n" + "{\n" + " return helper(v) + 1.0;\n" + "}\n", + 1, NULL + }, - {"ispc", "a.ispc", - "static inline uniform float helper(uniform float x) {\n" - " return x * 2.0f;\n" - "}\n" - "\n" - "export void run(uniform float in[], uniform float out[],\n" - " uniform int n) {\n" - " foreach (i = 0 ... n) {\n" - " out[i] = helper(in[i]);\n" - " }\n" - "}\n", - 1, NULL}, + { + "ispc", "a.ispc", + "static inline uniform float helper(uniform float x) {\n" + " return x * 2.0f;\n" + "}\n" + "\n" + "export void run(uniform float in[], uniform float out[],\n" + " uniform int n) {\n" + " foreach (i = 0 ... n) {\n" + " out[i] = helper(in[i]);\n" + " }\n" + "}\n", + 1, NULL + }, - {"odin", "a.odin", - "package fixture\n" - "\n" - "helper :: proc() -> int {\n" - "\treturn 42\n" - "}\n" - "\n" - "run :: proc() {\n" - "\tx := helper()\n" - "\t_ = x\n" - "}\n", - 1, NULL}, + { + "odin", "a.odin", + "package fixture\n" + "\n" + "helper :: proc() -> int {\n" + "\treturn 42\n" + "}\n" + "\n" + "run :: proc() {\n" + "\tx := helper()\n" + "\t_ = x\n" + "}\n", + 1, NULL + }, - {"slang", "a.slang", - "void helper()\n" - "{\n" - " int x = 1;\n" - "}\n" - "\n" - "void run()\n" - "{\n" - " helper();\n" - "}\n", - 1, NULL}, + { + "slang", "a.slang", + "void helper()\n" + "{\n" + " int x = 1;\n" + "}\n" + "\n" + "void run()\n" + "{\n" + " helper();\n" + "}\n", + 1, NULL + }, - {"squirrel", "a.nut", - "function helper(x) {\n" - " return x + 1;\n" - "}\n" - "\n" - "function run() {\n" - " return helper(41);\n" - "}\n", - 1, NULL}, + { + "squirrel", "a.nut", + "function helper(x) {\n" + " return x + 1;\n" + "}\n" + "\n" + "function run() {\n" + " return helper(41);\n" + "}\n", + 1, NULL + }, - {"vimscript", "a.vim", - "function! Helper() abort\n" - " return 1\n" - "endfunction\n" - "\n" - "function! Run() abort\n" - " call Helper()\n" - "endfunction\n", - 1, NULL}, + { + "vimscript", "a.vim", + "function! Helper() abort\n" + " return 1\n" + "endfunction\n" + "\n" + "function! Run() abort\n" + " call Helper()\n" + "endfunction\n", + 1, NULL + }, - {"cairo", "a.cairo", - "fn helper(x: felt252) -> felt252 {\n" - " x + 1\n" - "}\n" - "\n" - "fn run() -> felt252 {\n" - " helper(41)\n" - "}\n", - 1, NULL}, + { + "cairo", "a.cairo", + "fn helper(x: felt252) -> felt252 {\n" + " x + 1\n" + "}\n" + "\n" + "fn run() -> felt252 {\n" + " helper(41)\n" + "}\n", + 1, NULL + }, /* ------------------------------------------------------------------ */ /* KNOWN-GAP GROUP */ @@ -345,169 +377,181 @@ static const IBCase IB_CASES[] = { /* calls >= 1 AND module_calls == 0. */ /* ------------------------------------------------------------------ */ - {"dart", "a.dart", - "void helper() {\n" - " print('helper');\n" - "}\n" - "\n" - "void run() {\n" - " helper();\n" - "}\n", - /* - * Dart: selector call node carries no callee field and the first child - * is not an identifier; no dart branch in extract_calls.c. No CALLS - * edge is produced at all, so callable-sourcing cannot be tested - * independently. Both gaps (no CALLS + callable-sourcing) are RED. - */ - 0, - "selector call node: no callee field, first child not identifier; " - "no dart branch in extract_calls.c"}, + { + "dart", "a.dart", + "void helper() {\n" + " print('helper');\n" + "}\n" + "\n" + "void run() {\n" + " helper();\n" + "}\n", + /* + * Dart: selector call node carries no callee field and the first child + * is not an identifier; no dart branch in extract_calls.c. No CALLS + * edge is produced at all, so callable-sourcing cannot be tested + * independently. Both gaps (no CALLS + callable-sourcing) are RED. + */ + 0, "selector call node: no callee field, first child not identifier; " + "no dart branch in extract_calls.c" + }, - {"groovy", "a.groovy", - "def helper() {\n" - " println 'helping'\n" - "}\n" - "\n" - "def run() {\n" - " helper()\n" - "}\n", - /* - * Groovy: function_call callee not on a function/name field and first - * child is not 'identifier'; no groovy branch in extract_calls.c. - */ - 0, - "function_call callee not on function/name field; " - "first child is not identifier; no groovy branch in extract_calls.c"}, + { + "groovy", "a.groovy", + "def helper() {\n" + " println 'helping'\n" + "}\n" + "\n" + "def run() {\n" + " helper()\n" + "}\n", + /* + * Groovy: function_call callee not on a function/name field and first + * child is not 'identifier'; no groovy branch in extract_calls.c. + */ + 0, "function_call callee not on function/name field; " + "first child is not identifier; no groovy branch in extract_calls.c" + }, - {"commonlisp", "a.lisp", - "(defun helper (x)\n" - " (* x 2))\n" - "\n" - "(defun run ()\n" - " (helper 21))\n", - /* - * Common Lisp: list_lit call head is sym_lit not identifier; - * no commonlisp branch in extract_callee_name. - */ - 0, - "list_lit call head is sym_lit not identifier; " - "no commonlisp branch in extract_callee_name"}, + { + "commonlisp", "a.lisp", + "(defun helper (x)\n" + " (* x 2))\n" + "\n" + "(defun run ()\n" + " (helper 21))\n", + /* + * Common Lisp: list_lit call head is sym_lit not identifier; + * no commonlisp branch in extract_callee_name. + */ + 0, "list_lit call head is sym_lit not identifier; " + "no commonlisp branch in extract_callee_name" + }, - {"powershell", "a.ps1", - "function helper {\n" - " Write-Output 'hi'\n" - "}\n" - "\n" - "function run {\n" - " helper\n" - "}\n", - /* - * PowerShell: command node child is command_name not identifier; - * extract_scripting_callee handles MATLAB not PowerShell. - */ - 0, - "command node child is command_name not identifier; " - "extract_scripting_callee handles MATLAB not PowerShell"}, + { + "powershell", "a.ps1", + "function helper {\n" + " Write-Output 'hi'\n" + "}\n" + "\n" + "function run {\n" + " helper\n" + "}\n", + /* + * PowerShell: command node child is command_name not identifier; + * extract_scripting_callee handles MATLAB not PowerShell. + */ + 0, "command node child is command_name not identifier; " + "extract_scripting_callee handles MATLAB not PowerShell" + }, - {"ada", "a.adb", - "procedure Run is\n" - " procedure Helper is\n" - " begin\n" - " null;\n" - " end Helper;\n" - "begin\n" - " Helper;\n" - "end Run;\n", - /* - * Ada: procedure_call_statement callee did not resolve to a CALLS edge; - * no Ada branch in extract_calls.c. - */ - 0, - "procedure_call_statement callee not resolved; " - "no Ada branch in extract_calls.c"}, + { + "ada", "a.adb", + "procedure Run is\n" + " procedure Helper is\n" + " begin\n" + " null;\n" + " end Helper;\n" + "begin\n" + " Helper;\n" + "end Run;\n", + /* + * Ada: procedure_call_statement callee did not resolve to a CALLS edge; + * no Ada branch in extract_calls.c. + */ + 0, "procedure_call_statement callee not resolved; " + "no Ada branch in extract_calls.c" + }, - {"clojure", "a.clj", - "(defn helper [] 42)\n" - "\n" - "(defn run [] (helper))\n", - /* - * Clojure: lisp call is a list_lit whose head is a sym_lit (not a - * field, not a first-child 'identifier'); no lisp branch in - * extract_callee_name. - */ - 0, - "list_lit head is sym_lit not identifier; " - "no lisp/clojure branch in extract_callee_name"}, + { + "clojure", "a.clj", + "(defn helper [] 42)\n" + "\n" + "(defn run [] (helper))\n", + /* + * Clojure: lisp call is a list_lit whose head is a sym_lit (not a + * field, not a first-child 'identifier'); no lisp branch in + * extract_callee_name. + */ + 0, "list_lit head is sym_lit not identifier; " + "no lisp/clojure branch in extract_callee_name" + }, - {"fsharp", "a.fs", - "let helper x = x + 1\n" - "\n" - "let run () = helper 41\n", - /* - * F#: application_expression callee head is a long_identifier_or_op - * wrapper, not a bare identifier/field; no fsharp callee branch. - */ - 0, - "application_expression callee head is long_identifier_or_op wrapper; " - "no fsharp callee branch in extract_callee_name"}, + { + "fsharp", "a.fs", + "let helper x = x + 1\n" + "\n" + "let run () = helper 41\n", + /* + * F#: application_expression callee head is a long_identifier_or_op + * wrapper, not a bare identifier/field; no fsharp callee branch. + */ + 0, "application_expression callee head is long_identifier_or_op wrapper; " + "no fsharp callee branch in extract_callee_name" + }, - {"racket", "a.rkt", - "#lang racket\n" - "\n" - "(define (helper x)\n" - " (+ x 1))\n" - "\n" - "(define (run)\n" - " (helper 41))\n", - /* - * Racket: lisp call is a 'list' whose head is a 'symbol' (grammar has - * no 'identifier' node); no racket branch in extract_callee_name. - */ - 0, - "list head is symbol not identifier; " - "no racket branch in extract_callee_name"}, + { + "racket", "a.rkt", + "#lang racket\n" + "\n" + "(define (helper x)\n" + " (+ x 1))\n" + "\n" + "(define (run)\n" + " (helper 41))\n", + /* + * Racket: lisp call is a 'list' whose head is a 'symbol' (grammar has + * no 'identifier' node); no racket branch in extract_callee_name. + */ + 0, "list head is symbol not identifier; " + "no racket branch in extract_callee_name" + }, - {"rescript", "a.res", - "let helper = (x) => x + 1\n" - "\n" - "let run = () => helper(41)\n", - /* - * ReScript: call_expression 'function' field is a 'value_identifier' - * (not in extract_callee_from_fields' accepted type list). - */ - 0, - "call_expression function field is value_identifier; " - "not in extract_callee_from_fields accepted type list"}, + { + "rescript", "a.res", + "let helper = (x) => x + 1\n" + "\n" + "let run = () => helper(41)\n", + /* + * ReScript: call_expression 'function' field is a 'value_identifier' + * (not in extract_callee_from_fields' accepted type list). + */ + 0, "call_expression function field is value_identifier; " + "not in extract_callee_from_fields accepted type list" + }, - {"scheme", "a.scm", - "(define (helper x)\n" - " (* x 2))\n" - "\n" - "(define (run)\n" - " (helper 21))\n", - /* - * Scheme: lisp call is a 'list' whose head is a 'symbol'; - * no scheme branch in extract_callee_name. - */ - 0, - "list head is symbol not identifier; " - "no scheme branch in extract_callee_name"}, + { + "scheme", "a.scm", + "(define (helper x)\n" + " (* x 2))\n" + "\n" + "(define (run)\n" + " (helper 21))\n", + /* + * Scheme: lisp call is a 'list' whose head is a 'symbol'; + * no scheme branch in extract_callee_name. + */ + 0, "list head is symbol not identifier; " + "no scheme branch in extract_callee_name" + }, - {"chialisp", "a.clsp", - "(mod ()\n" - " (defun helper (x)\n" - " (* x 2))\n" - " (defun run ()\n" - " (helper 21))\n" - ")\n", - /* - * Chialisp: EXPECTED-GREEN. Every def is nested inside the top-level - * `(mod ...)`, so the in-body call can only be Function-sourced if - * compute_lisp_func_qn pushes a scope for the inner `(defun run ...)` - * rather than stopping at the module. That is exactly what would - * regress if the Chialisp def-head set drifted from the defs walk. - */ - 1, NULL}, + { + "chialisp", "a.clsp", + "(mod ()\n" + " (defun helper (x)\n" + " (* x 2))\n" + " (defun run ()\n" + " (helper 21))\n" + ")\n", + /* + * Chialisp: EXPECTED-GREEN. Every def is nested inside the top-level + * `(mod ...)`, so the in-body call can only be Function-sourced if + * compute_lisp_func_qn pushes a scope for the inner `(defun run ...)` + * rather than stopping at the module. That is exactly what would + * regress if the Chialisp def-head set drifted from the defs walk. + */ + 1, NULL + }, }; enum { IB_CASES_COUNT = (int)(sizeof(IB_CASES) / sizeof(IB_CASES[0])) }; @@ -541,14 +585,17 @@ TEST(repro_invariant_breadth_callable_sourcing) { const IBCase *c = &IB_CASES[i]; IBMetrics m = ib_metrics(c->filename, c->src); - int pass = (m.ok && m.calls >= 1 && m.callable_calls >= 1 && m.module_calls == 0); + int pass = (m.ok && m.calls >= 1 && m.callable_calls >= 1 && + m.module_calls == 0); if (!pass) { fprintf(stderr, " [INV-BREADTH] FAIL %-12s ok=%d calls=%d " "callable=%d module=%d%s%s\n", - c->lang, m.ok, m.calls, m.callable_calls, m.module_calls, - c->gap_note ? " -- " : "", c->gap_note ? c->gap_note : ""); + c->lang, m.ok, m.calls, m.callable_calls, + m.module_calls, + c->gap_note ? " -- " : "", + c->gap_note ? c->gap_note : ""); failures++; } else { fprintf(stderr, diff --git a/tests/test_extraction.c b/tests/test_extraction.c index ac75b16a6..ea733b0c3 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -1225,18 +1225,19 @@ TEST(form_procedure) { /* --- Oracle PL/SQL --- */ TEST(plsql_package_and_call) { - const char *src = "CREATE OR REPLACE PACKAGE BODY emp_pkg AS\n" - " FUNCTION hire(p_name VARCHAR2) RETURN NUMBER IS\n" - " v_sal NUMBER;\n" - " BEGIN\n" - " v_sal := util_pkg.calc_salary(p_name);\n" - " IF v_sal > 0 THEN\n" - " RETURN v_sal;\n" - " END IF;\n" - " RAISE no_data_found;\n" - " END;\n" - "END emp_pkg;\n" - "/\n"; + const char *src = + "CREATE OR REPLACE PACKAGE BODY emp_pkg AS\n" + " FUNCTION hire(p_name VARCHAR2) RETURN NUMBER IS\n" + " v_sal NUMBER;\n" + " BEGIN\n" + " v_sal := util_pkg.calc_salary(p_name);\n" + " IF v_sal > 0 THEN\n" + " RETURN v_sal;\n" + " END IF;\n" + " RAISE no_data_found;\n" + " END;\n" + "END emp_pkg;\n" + "/\n"; CBMFileResult *r = extract(src, CBM_LANG_PLSQL, "t", "emp_pkg.pkb"); ASSERT_NOT_NULL(r); ASSERT_FALSE(r->has_error); @@ -2201,10 +2202,9 @@ TEST(dbt_source_and_two_arg_ref) { /* Both dbt builtins name the relation in their LAST string argument: * source('group','table') -> table, and the two-argument * ref('package','model') form -> model. */ - CBMFileResult *r = - extract("SELECT * FROM {{ source('raw', 'customers') }}\n" - "UNION ALL SELECT * FROM {{ ref('analytics', 'legacy_customers') }}\n", - CBM_LANG_SQL, "t", "models/stg_customers.sql"); + CBMFileResult *r = extract("SELECT * FROM {{ source('raw', 'customers') }}\n" + "UNION ALL SELECT * FROM {{ ref('analytics', 'legacy_customers') }}\n", + CBM_LANG_SQL, "t", "models/stg_customers.sql"); ASSERT_NOT_NULL(r); ASSERT(has_def(r, "Model", "stg_customers")); ASSERT(has_usage(r, "customers")); @@ -3223,8 +3223,8 @@ TEST(vue_embedded_structure_negative_controls_issue1410) { } TEST(vue_embedded_structure_host_controls_issue1410) { - CBMFileResult *plain = - extract("function plainTs(): void { target(); }\n", CBM_LANG_TYPESCRIPT, "t", "plain.ts"); + CBMFileResult *plain = extract("function plainTs(): void { target(); }\n", CBM_LANG_TYPESCRIPT, + "t", "plain.ts"); ASSERT_NOT_NULL(plain); ASSERT_FALSE(plain->has_error); ASSERT_EQ(count_defs_named(plain, "Function", "plainTs"), 1); @@ -3682,11 +3682,11 @@ TEST(extract_java_method_annotations_issue382) { /* ── ArkTS (HarmonyOS .ets) ─────────────────────────────────────── */ TEST(arkts_component_struct) { - CBMFileResult *r = - extract("@Entry\n@Component\nstruct Index {\n @State message: string = 'Hello'\n\n" - " build() {\n Column() {\n Text(this.message).fontSize(20)\n }\n" - " .width('100%')\n }\n}\n", - CBM_LANG_ARKTS, "t", "Index.ets"); + CBMFileResult *r = extract( + "@Entry\n@Component\nstruct Index {\n @State message: string = 'Hello'\n\n" + " build() {\n Column() {\n Text(this.message).fontSize(20)\n }\n" + " .width('100%')\n }\n}\n", + CBM_LANG_ARKTS, "t", "Index.ets"); ASSERT_NOT_NULL(r); ASSERT_FALSE(r->has_error); ASSERT(has_def(r, "Struct", "Index")); @@ -3717,12 +3717,12 @@ TEST(arkts_exported_struct_decorators) { } TEST(arkts_member_decorators) { - CBMFileResult *r = - extract("@Component\nstruct S {\n @State a: number = 0\n @Prop b: string\n" - " @Link c: boolean\n @Provide('k') d: string = ''\n @Consume('k') e: string\n" - " @StorageLink('s') f: number = 1\n @State @Watch('onW') g: boolean = false\n\n" - " build() {\n }\n}\n", - CBM_LANG_ARKTS, "t", "S.ets"); + CBMFileResult *r = extract( + "@Component\nstruct S {\n @State a: number = 0\n @Prop b: string\n" + " @Link c: boolean\n @Provide('k') d: string = ''\n @Consume('k') e: string\n" + " @StorageLink('s') f: number = 1\n @State @Watch('onW') g: boolean = false\n\n" + " build() {\n }\n}\n", + CBM_LANG_ARKTS, "t", "S.ets"); ASSERT_NOT_NULL(r); ASSERT_FALSE(r->has_error); ASSERT(decorators_contain(find_def_by_name(r, "a"), "State")); @@ -3762,11 +3762,11 @@ TEST(arkts_no_phantom_builtin_defs) { } TEST(arkts_builder_extend_styles) { - CBMFileResult *r = - extract("@Builder\nfunction card(t: string) {\n Column() {\n Text(t)\n }\n}\n\n" - "@Extend(Text)\nfunction fancy(size: number) {\n .fontSize(size)\n}\n\n" - "@Styles\nfunction pressed() {\n .backgroundColor('#eee')\n}\n", - CBM_LANG_ARKTS, "t", "b.ets"); + CBMFileResult *r = extract( + "@Builder\nfunction card(t: string) {\n Column() {\n Text(t)\n }\n}\n\n" + "@Extend(Text)\nfunction fancy(size: number) {\n .fontSize(size)\n}\n\n" + "@Styles\nfunction pressed() {\n .backgroundColor('#eee')\n}\n", + CBM_LANG_ARKTS, "t", "b.ets"); ASSERT_NOT_NULL(r); ASSERT_FALSE(r->has_error); ASSERT(has_def(r, "Function", "card")); @@ -5110,8 +5110,8 @@ TEST(extract_c_test_dir_marks_is_test_issue1294) { * not be (#1294). */ TEST(extract_python_method_test_dir_marks_is_test_issue1294) { const char *src = "class Foo:\n" - " def helper(self):\n" - " pass\n"; + " def helper(self):\n" + " pass\n"; /* Python's LSP layer injects synthetic builtin stub Methods (str.upper, * dict.get, ...) into defs.items alongside real ones (py_builtins.c), so diff --git a/tests/test_lang_contract.c b/tests/test_lang_contract.c index 3ec63d80e..2c2441767 100644 --- a/tests/test_lang_contract.c +++ b/tests/test_lang_contract.c @@ -463,10 +463,11 @@ TEST(contract_python_relative_import) { /* Python: relative aliased from-import must CALLS the real def (execute). */ TEST(contract_python_aliased_from_import_calls) { - static const LangFile f[] = {{"gate.py", "def execute(x):\n return x\n"}, - {"router.py", - "from .gate import execute as bridge_execute\n\n\n" - "def submit_task(y):\n return bridge_execute(y)\n"}}; + static const LangFile f[] = { + {"gate.py", "def execute(x):\n return x\n"}, + {"router.py", + "from .gate import execute as bridge_execute\n\n\n" + "def submit_task(y):\n return bridge_execute(y)\n"}}; LangProj lp; cbm_store_t *store = lang_index_files(&lp, f, 2); ASSERT_TRUE(store != NULL); @@ -559,8 +560,10 @@ TEST(contract_python_absolute_aliased_from_import_calls) { /* Python: unresolvable module + alias must emit no CALLS (no invented edge). */ TEST(contract_python_ghost_module_aliased_from_import_no_calls) { - static const LangFile f[] = {{"router.py", "from ghost_module import nope as alias\n\n\n" - "def submit_task(y):\n return alias(y)\n"}}; + static const LangFile f[] = { + {"router.py", + "from ghost_module import nope as alias\n\n\n" + "def submit_task(y):\n return alias(y)\n"}}; LangProj lp; cbm_store_t *store = lang_index_files(&lp, f, 1); ASSERT_TRUE(store != NULL); @@ -1755,11 +1758,10 @@ TEST(contract_edge_parallel_service_edges) { if (store) { sqlite3_stmt *stmt = NULL; sqlite3 *db = cbm_store_get_db(store); - if (db && - sqlite3_prepare_v2(db, - "SELECT count(*) FROM edges WHERE properties IS NOT NULL " - "AND properties != '' AND json_valid(properties)=0;", - -1, &stmt, NULL) == SQLITE_OK && + if (db && sqlite3_prepare_v2(db, + "SELECT count(*) FROM edges WHERE properties IS NOT NULL " + "AND properties != '' AND json_valid(properties)=0;", + -1, &stmt, NULL) == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW) { invalid_props = sqlite3_column_int(stmt, 0); } From ee0a2e6dc6e72e574e84c59133941e23e7d51a0c Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 28 Aug 2026 21:47:39 +0200 Subject: [PATCH 17/31] docs(grammars): recount the manifest's ABI distribution from the tree The line read `7x ABI-13, 86x ABI-14, 65x ABI-15`, which sums to 158 against 161 vendored grammars. Counting `#define LANGUAGE_VERSION` in each parser.c gives 9 / 79 / 73 = 161, so the line was already wrong before Chialisp; bumping the ABI-14 bucket by one, as the Chialisp change did, would have carried a four-off error forward under a fresh timestamp. Recount with Chialisp included: 9 / 80 / 73 = 162, which matches both the grammar-directory count and the language-count contract. The regeneration command is now recorded next to the numbers so the next person does not have to rediscover it: grep -h '#define LANGUAGE_VERSION' internal/cbm/vendored/grammars/*/parser.c \ | sort | uniq -c Vendored-integrity gate and the language-count contract both re-run green. Signed-off-by: Martin Vogel --- internal/cbm/vendored/grammars/MANIFEST.md | 4 +++- scripts/vendored-checksums.txt | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/cbm/vendored/grammars/MANIFEST.md b/internal/cbm/vendored/grammars/MANIFEST.md index 9b5b0127b..57cd072ad 100644 --- a/internal/cbm/vendored/grammars/MANIFEST.md +++ b/internal/cbm/vendored/grammars/MANIFEST.md @@ -10,7 +10,9 @@ The grammars were originally vendored as bare `parser.c`+`scanner.c` with **no r ## Summary - Grammars: **162** — vendored-from-upstream: **143**, first-party/self-maintained: **14**, registry-disagreement: **5** (nim removed 2026-06-12; objectscript_udl + objectscript_routine added 2026-06-24; mojo added 2026-07-01; arkts added 2026-08-26; plsql added 2026-08-27; chialisp added 2026-08-28 — see notes below) -- ABI distribution: **7×** ABI-13 **87×** ABI-14 **65×** ABI-15 (runtime ceiling is ABI 15; never vendor ABI 16 without a runtime upgrade) +- ABI distribution: **9×** ABI-13 **80×** ABI-14 **73×** ABI-15 (runtime ceiling is ABI 15; never vendor ABI 16 without a runtime upgrade) + — recounted from the tree 2026-08-28. This line had drifted: it read `7×/86×/65×`, which sums to 158 against 161 vendored grammars, so it was wrong before Chialisp was added and incrementing it would have carried the error forward. Regenerate with: + `grep -h '#define LANGUAGE_VERSION' internal/cbm/vendored/grammars/*/parser.c | sort | uniq -c` - Vendored copies missing LICENSE: **0** — all upstream LICENSE files restored 2026-06-11 (first-party grammars carry the project MIT license; `move` uses the Helix-listed upstream tzakian/tree-sitter-move MIT text, `zsh` uses georgeharker/tree-sitter-zsh MIT) - `verdict`: VERIFIED-BOTH = our source matches *both* registries; VERIFIED-NVIM/HELIX = matches one; registry-disagreement = registries name a different repo (listed separately); `vendor-maintained` = the language vendor's own grammar, not in nvim/Helix; `community-niche` = an individual/community-maintained grammar for a niche language, not in nvim/Helix — provenance hand-verified against the upstream repository instead of a registry. - **objectscript_udl / objectscript_routine** (added 2026-06-24): vendored from [intersystems/tree-sitter-objectscript](https://github.com/intersystems/tree-sitter-objectscript) @ `a7ffcdf` — MIT, the InterSystems-official grammars (a niche vendor language, hence `vendor-maintained`, not in nvim-treesitter/Helix). **Re-vendor note:** each `scanner.c`'s upstream `#include "../../common/scanner.h"` is repointed to a per-directory `objectscript_common.h` (a verbatim copy of upstream `common/scanner.h`), because this repo's shared `vendored/common/scanner.h` belongs to the cfml/fsharp grammars and differs. The generated `parser.c`/`scanner.c` are otherwise byte-for-byte upstream — on re-vendor, re-apply only that single include rename. **Local modification (2026-07-16):** in `objectscript_common.h`, two loop counters `uint8_t i` were widened to `int i` (the `reverse_marker` scan and the `html_marker_buffer` reversal) to clear CodeQL `cpp/comparison-with-wider-type` — a false positive in practice (both lengths are hard-bounded by `MARKER_BUFFER_MAX_LEN = 30`, so `uint8_t` could never wrap), fixed for cleanliness. On re-vendor, re-apply this widening too (or upstream it at intersystems/tree-sitter-objectscript). diff --git a/scripts/vendored-checksums.txt b/scripts/vendored-checksums.txt index 992216603..991a684aa 100644 --- a/scripts/vendored-checksums.txt +++ b/scripts/vendored-checksums.txt @@ -5,7 +5,7 @@ c5cfb43042b6b72045f4ba997834d0a7786d2793d91680868b5815b39f14fc78 internal/cbm/v b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea internal/cbm/vendored/common/tree_sitter/alloc.h 31e60a1bff6f715afacce03b5b70efe42b58371b4f9595dd4af52a577ff9608c internal/cbm/vendored/common/tree_sitter/array.h 180b893c8734778fd32f372dfbc27bd6ad1cd2221f26150b31256ff6716320d2 internal/cbm/vendored/common/tree_sitter/parser.h -1fec60a59cb72bd61b4f9e771a77cc184597f49caf5d33f2e2a40f5739518571 internal/cbm/vendored/grammars/MANIFEST.md +73be629708008f0cb31e0f8d41dc71ae901626baba191c24efd41bf52220c913 internal/cbm/vendored/grammars/MANIFEST.md ad8425038de519f8c4e3e9339feebf99dfad8a6002dcf79227d91402780a32cc internal/cbm/vendored/grammars/ada/LICENSE 02805ec13939b749c891567be36bf024b09034e04c80683a1ae667272458d549 internal/cbm/vendored/grammars/ada/parser.c 115a75d000bef9c70c4de7dfbb7f2a80b90cbb14ba265056dc7abb1ef3b9b6db internal/cbm/vendored/grammars/ada/tree_sitter/alloc.h From 01c7e549ef633d147191e2b06097b39227c05f7c Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 28 Aug 2026 18:05:30 +0200 Subject: [PATCH 18/31] feat(search): index prose in BM25 via a nodes_fts body column Closes #518 (Section nodes don't index body text) and #519 (META.yaml / frontmatter description values not indexed). A question asked in words could not reach the nodes that hold the words: Section carried only its heading, a config Module only its path, and both labels were filtered out of BM25 results outright. nodes_fts gains a fifth column, `body`, backfilled from each node's EXISTING properties.docstring. No new column on `nodes`, no new table, and CBM_INDEX_FORMAT_VERSION deliberately does NOT move -- nobody is forced to reindex. A legacy four-column table survives untouched under CREATE VIRTUAL TABLE IF NOT EXISTS: the backfill probes for the column before naming it, and bm25()'s surplus fifth weight is inert on a table that has no fifth column (FTS5 reads a weight only for a column an instance landed in). Such a database keeps opening and searching, just without prose. Two node kinds gain the docstring they lacked: - Markdown section bodies (#518) -- the blocks following a heading, up to the next heading or subsection, collapsed to single spaces and capped at MAX_COMMENT_LEN (500 B) with a UTF-8 backoff that never splits a multi-byte sequence. - Top-level YAML/JSON description / summary / purpose (#519), promoted onto the file's Module node. Section and Module are no longer excluded from BM25 results. Both the ranked query and the mirrored count query were changed together; removing the filter from only one desynchronises results from counts. Column weights are bm25(nodes_fts, 1.0, 1.0, 1.0, 1.0, 0.3): the four identifier columns keep parity with each other and prose sits well below them, so a prose-only hit surfaces but never outranks an identifier match. The FTS backfill is consolidated into one writer, cbm_store_fts_rebuild(), used by the wholesale rebuild AND the delta merge. That second site matters: it had its own hand-written four-column INSERT, which with a fifth column would leave body NULL for every node arriving by delta -- silently, on the warm path most users hit, while a full reindex looked perfect. There is now exactly one place the column list is decided, and a test that fails if the delta path stops writing prose. The backfill guards json_extract with json_valid: malformed properties JSON exists in pre-fix databases and would otherwise abort the whole rebuild. Tests: markdown section bodies (own body, stops at the next heading, UTF-8 safe cap, empty body stays NULL), YAML/JSON description promotion (block scalars, synonym fallback, top-level only, config-only), FTS body indexing and its column isolation, malformed-JSON tolerance, legacy four-column degrade, incremental watermark, BM25 finding Section and Module by prose, results/total consistency, identifier-outranks-prose, and the delta-merge regression. Signed-off-by: Martin Vogel Co-authored-by: ShauryaaSharma <196896785+ShauryaaSharma@users.noreply.github.com> Signed-off-by: Martin Vogel --- internal/cbm/extract_defs.c | 268 +++++++++++++++++++++++++++++++++- src/mcp/mcp.c | 54 +++++-- src/pipeline/pipeline.c | 21 +-- src/pipeline/pipeline_delta.c | 122 +++++++++------- src/store/store.c | 109 +++++++++++++- src/store/store.h | 16 ++ tests/test_extraction.c | 214 +++++++++++++++++++++++++++ tests/test_mcp.c | 242 ++++++++++++++++++++++++++++++ tests/test_pipeline.c | 72 +++++++++ tests/test_store_search.c | 172 ++++++++++++++++++++++ 10 files changed, 1200 insertions(+), 90 deletions(-) diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 6982d6c13..dfef6c3f9 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -3861,7 +3861,132 @@ static const char *qn_safe_segment(CBMArena *a, const char *name) { return out; } -static void push_simple_class_def(CBMExtractCtx *ctx, TSNode node, char *name, const char *label) { +/* UTF-8 sequence classification: a lead byte's high bits name the sequence + * length, and every continuation byte matches 10xxxxxx. */ +enum { + UTF8_CONT_MASK = 0xC0, /* isolate the two high bits ... */ + UTF8_CONT_MARK = 0x80, /* ... which are 10 on a continuation byte */ + UTF8_LEAD2_MASK = 0xE0, /* 110xxxxx → 2-byte sequence */ + UTF8_LEAD2_MARK = 0xC0, + UTF8_LEAD3_MASK = 0xF0, /* 1110xxxx → 3-byte sequence */ + UTF8_LEAD3_MARK = 0xE0, + UTF8_LEAD4_MASK = 0xF8, /* 11110xxx → 4-byte sequence */ + UTF8_LEAD4_MARK = 0xF0, + UTF8_LEN_1 = 1, + UTF8_LEN_2 = 2, + UTF8_LEN_3 = 3, + UTF8_LEN_4 = 4, +}; + +/* Bytes the sequence starting with `lead` occupies (1 for ASCII or a byte that + * is not a valid lead). */ +static size_t utf8_sequence_len(unsigned char lead) { + if ((lead & UTF8_LEAD2_MASK) == UTF8_LEAD2_MARK) { + return UTF8_LEN_2; + } + if ((lead & UTF8_LEAD3_MASK) == UTF8_LEAD3_MARK) { + return UTF8_LEN_3; + } + if ((lead & UTF8_LEAD4_MASK) == UTF8_LEAD4_MARK) { + return UTF8_LEN_4; + } + return UTF8_LEN_1; +} + +/* Drop a trailing PARTIAL UTF-8 sequence left behind by a byte-length cut, so + * a capped prose value never ends mid-codepoint (#1017's rule, applied to the + * prose bodies below rather than to comments). */ +static void utf8_trim_partial_tail(char *text) { + size_t n = strlen(text); + if (n == 0) { + return; + } + size_t i = n; + while (i > 0 && ((unsigned char)text[i - UTF8_LEN_1] & UTF8_CONT_MASK) == UTF8_CONT_MARK) { + i--; + } + if (i == 0) { + text[0] = '\0'; /* continuation bytes only — not decodable */ + return; + } + size_t need = utf8_sequence_len((unsigned char)text[i - UTF8_LEN_1]); + if (i - UTF8_LEN_1 + need > n) { + text[i - UTF8_LEN_1] = '\0'; + } +} + +/* Collapse `len` bytes of raw prose into a single-spaced value capped at + * MAX_COMMENT_LEN (the same 500-byte ceiling docstrings already use, which + * leaves room inside the 2 KB properties buffer that carries it). + * + * The output buffer is FIXED at that cap, so a section body of any size costs a + * bounded copy rather than a copy of the whole section. Returns NULL when + * nothing but whitespace was there. */ +static char *collapse_prose(CBMArena *a, const char *src, size_t len) { + if (!src || len == 0) { + return NULL; + } + char *out = (char *)cbm_arena_alloc(a, MAX_COMMENT_LEN + NULL_TERM); + if (!out) { + return NULL; + } + size_t w = 0; + bool in_ws = true; /* start true so leading whitespace is swallowed */ + for (size_t i = 0; i < len && w < MAX_COMMENT_LEN; i++) { + unsigned char c = (unsigned char)src[i]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + in_ws = true; + continue; + } + if (in_ws && w > 0) { + out[w++] = ' '; + if (w >= MAX_COMMENT_LEN) { + break; + } + } + in_ws = false; + out[w++] = (char)c; + } + out[w] = '\0'; + utf8_trim_partial_tail(out); + return out[0] ? out : NULL; +} + +static bool is_markdown_heading_kind(const char *kind) { + return strcmp(kind, "atx_heading") == 0 || strcmp(kind, "setext_heading") == 0; +} + +/* #518: a Markdown heading node is only the title line — the section's prose + * lives in the blocks that FOLLOW it. Collect that prose as the Section's + * docstring so the node carries the text a reader would search for; nodes_fts + * indexes it from there (`body`), which is the whole point of the issue. + * + * tree-sitter-markdown wraps a heading and its content in a `section` node, + * with nested subsections as further `section` children that own their own + * text. Stopping at either a `section` or another heading therefore gives each + * Section exactly its OWN body under both that shape and a flat one. The bytes + * between are contiguous in the source, so one slice beats concatenation. */ +static const char *extract_markdown_section_body(CBMArena *a, TSNode heading, const char *source) { + uint32_t start = ts_node_end_byte(heading); + uint32_t end = start; + for (TSNode sib = ts_node_next_sibling(heading); !ts_node_is_null(sib); + sib = ts_node_next_sibling(sib)) { + const char *sk = ts_node_type(sib); + if (strcmp(sk, "section") == 0 || is_markdown_heading_kind(sk)) { + break; + } + end = ts_node_end_byte(sib); + } + if (end <= start) { + return NULL; + } + return collapse_prose(a, source + start, (size_t)(end - start)); +} + +/* Push a config-language definition that carries prose. push_simple_class_def + * delegates here with no docstring. */ +static void push_simple_class_def_doc(CBMExtractCtx *ctx, TSNode node, char *name, + const char *label, const char *docstring) { CBMArena *a = ctx->arena; CBMDefinition def; memset(&def, 0, sizeof(def)); @@ -3872,9 +3997,14 @@ static void push_simple_class_def(CBMExtractCtx *ctx, TSNode node, char *name, c def.start_line = ts_node_start_point(node).row + TS_LINE_OFFSET; def.end_line = ts_node_end_point(node).row + TS_LINE_OFFSET; def.is_exported = true; + def.docstring = docstring; cbm_defs_push(&ctx->result->defs, a, def); } +static void push_simple_class_def(CBMExtractCtx *ctx, TSNode node, char *name, const char *label) { + push_simple_class_def_doc(ctx, node, name, label, NULL); +} + // Find TOML table key name from children. static char *find_toml_key_name(CBMArena *a, TSNode node, const char *source) { uint32_t nc = ts_node_child_count(node); @@ -4033,6 +4163,7 @@ static bool extract_config_class_def(CBMExtractCtx *ctx, TSNode node, const char CBMArena *a = ctx->arena; char *name = NULL; const char *label = "Class"; + const char *docstring = NULL; if (ctx->language == CBM_LANG_TOML && (strcmp(kind, "table") == 0 || strcmp(kind, "table_array_element") == 0)) { @@ -4048,6 +4179,8 @@ static bool extract_config_class_def(CBMExtractCtx *ctx, TSNode node, const char // label rather than degrade it to match a test. The markdown repro asserts // "Class"; that assertion is the inaccurate side and is flagged for review. label = "Section"; + // #518: index what the section SAYS, not just its title. + docstring = extract_markdown_section_body(a, node, ctx->source); } else if (ctx->language == CBM_LANG_HCL && strcmp(kind, "block") == 0) { name = find_hcl_block_name(a, node, ctx->source); } else { @@ -4055,7 +4188,7 @@ static bool extract_config_class_def(CBMExtractCtx *ctx, TSNode node, const char } if (name && name[0]) { - push_simple_class_def(ctx, node, name, label); + push_simple_class_def_doc(ctx, node, name, label, docstring); } return true; } @@ -6175,12 +6308,9 @@ static bool is_helm_values_file(const char *rel) { return strcmp(b, "values.yaml") == 0 || strcmp(b, "values.yml") == 0; } -// Extract ONLY top-level keys of a YAML document (no leaf explosion). Used for -// Helm values.yaml so each chart's tunables surface as a handful of structured -// Variables instead of one node per nested leaf (#338). -static void extract_yaml_toplevel_keys(CBMExtractCtx *ctx, TSNode root) { - CBMArena *a = ctx->arena; - // Descend stream -> document -> block_node down to the first block_mapping. +// Descend stream -> document -> block_node to a YAML document's top-level +// block_mapping. Returns a null node when the document has none. +static TSNode find_yaml_toplevel_mapping(TSNode root) { TSNode bm = {0}; TSNode cur = root; for (int depth = 0; depth < 6 && ts_node_is_null(bm); depth++) { @@ -6204,6 +6334,126 @@ static void extract_yaml_toplevel_keys(CBMExtractCtx *ctx, TSNode root) { } cur = next; } + return bm; +} + +// Descend to a JSON document's top-level object. Returns a null node if absent. +static TSNode find_json_toplevel_object(TSNode root) { + if (strcmp(ts_node_type(root), "object") == 0) { + return root; + } + TSNode obj = cbm_find_child_by_kind(root, "object"); + if (!ts_node_is_null(obj)) { + return obj; + } + uint32_t n = ts_node_named_child_count(root); + for (uint32_t i = 0; i < n; i++) { + TSNode inner = cbm_find_child_by_kind(ts_node_named_child(root, i), "object"); + if (!ts_node_is_null(inner)) { + return inner; + } + } + TSNode null_node = {0}; + return null_node; +} + +/* #519: a config file's own prose sits in a top-level `description` (or one of + * its usual synonyms) — META.yaml, action.yml, an OpenAPI document, + * package.json. Nothing indexed it: the VALUE is not a definition, so no node + * carried it and BM25 could not see it at all. Checked in priority order; the + * first key present wins. */ +static const char *const config_desc_keys[] = {"description", "summary", "purpose", NULL}; + +// Strip one layer of matching surrounding quotes. Operates on arena text. +static char *strip_surrounding_quotes(char *t) { + if (!t) { + return t; + } + size_t n = strlen(t); + if (n >= PAIR_CHARS && (t[0] == '"' || t[0] == '\'') && t[n - SKIP_CHAR] == t[0]) { + t[n - SKIP_CHAR] = '\0'; + return t + SKIP_CHAR; + } + return t; +} + +// Normalise a config scalar into an indexable value: drop a YAML block-scalar +// header (`|`/`>` plus its chomping and indent modifiers), collapse whitespace +// to the shared MAX_COMMENT_LEN cap, then unquote. +static const char *config_scalar_value(CBMArena *a, const char *raw) { + if (!raw) { + return NULL; + } + while (*raw == ' ' || *raw == '\t' || *raw == '\n' || *raw == '\r') { + raw++; + } + if (*raw == '|' || *raw == '>') { + while (*raw && *raw != '\n') { + raw++; + } + } + char *v = collapse_prose(a, raw, strlen(raw)); + return v ? strip_surrounding_quotes(v) : NULL; +} + +// Value of `pair` when its key is `want`, else NULL. A YAML block_mapping_pair +// and a JSON pair both expose key/value fields, so one reader serves both. +static const char *config_pair_value_if_key(CBMExtractCtx *ctx, TSNode pair, const char *want) { + TSNode key = ts_node_child_by_field_name(pair, TS_FIELD("key")); + TSNode val = ts_node_child_by_field_name(pair, TS_FIELD("value")); + if (ts_node_is_null(key) || ts_node_is_null(val)) { + return NULL; + } + char *kt = cbm_node_text(ctx->arena, key, ctx->source); + if (!kt || strcmp(strip_surrounding_quotes(kt), want) != 0) { + return NULL; + } + return config_scalar_value(ctx->arena, cbm_node_text(ctx->arena, val, ctx->source)); +} + +// First non-empty description value among `container`'s direct pairs, scanned +// in config_desc_keys priority order. +static const char *config_container_description(CBMExtractCtx *ctx, TSNode container, + const char *pair_kind) { + uint32_t n = ts_node_named_child_count(container); + for (const char *const *k = config_desc_keys; *k; k++) { + for (uint32_t i = 0; i < n; i++) { + TSNode pair = ts_node_named_child(container, i); + if (strcmp(ts_node_type(pair), pair_kind) != 0) { + continue; + } + const char *v = config_pair_value_if_key(ctx, pair, *k); + if (v && v[0]) { + return v; + } + } + } + return NULL; +} + +/* #519 entry point: the file-level description a config document declares about + * itself, promoted onto the Module node by cbm_extract_definitions so + * nodes_fts.body indexes it and the file is findable by what it SAYS it does, + * not only by its path. NULL for every other language. */ +static const char *extract_config_module_description(CBMExtractCtx *ctx) { + if (ctx->language == CBM_LANG_YAML) { + TSNode bm = find_yaml_toplevel_mapping(ctx->root); + return ts_node_is_null(bm) ? NULL + : config_container_description(ctx, bm, "block_mapping_pair"); + } + if (ctx->language == CBM_LANG_JSON) { + TSNode obj = find_json_toplevel_object(ctx->root); + return ts_node_is_null(obj) ? NULL : config_container_description(ctx, obj, "pair"); + } + return NULL; +} + +// Extract ONLY top-level keys of a YAML document (no leaf explosion). Used for +// Helm values.yaml so each chart's tunables surface as a handful of structured +// Variables instead of one node per nested leaf (#338). +static void extract_yaml_toplevel_keys(CBMExtractCtx *ctx, TSNode root) { + CBMArena *a = ctx->arena; + TSNode bm = find_yaml_toplevel_mapping(root); if (ts_node_is_null(bm)) { return; } @@ -7587,6 +7837,8 @@ void cbm_extract_definitions(CBMExtractCtx *ctx) { mod.end_line = ts_node_end_point(ctx->root).row + TS_LINE_OFFSET; mod.is_exported = true; mod.is_test = ctx->result->is_test_file; + // #519: index what a config file declares itself to be, not only its path. + mod.docstring = extract_config_module_description(ctx); cbm_defs_push(&ctx->result->defs, a, mod); cbm_extract_definitions_without_module(ctx); diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 0ce6ae7eb..96a2e71ae 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -2936,6 +2936,25 @@ enum { BM25_INNER_LIMIT = 2000, }; +/* Column weights for nodes_fts (name, qualified_name, label, file_path, body). + * The four identifier columns stay at parity; prose sits well below them. + * FTS5 applies these to per-column term frequency BEFORE the tf-saturation + * term, which is what makes the weighting BM25F-correct rather than a post-hoc + * rescale. 0.3 is the findability-favouring end of the field weighting the IR + * literature settles on for body text (typical title:body ratios run 3:1 to + * 10:1): a prose-only hit still surfaces, but never outranks a node whose + * IDENTIFIER matches. + * + * Defined once and used by BOTH the ranked query and the count query — they + * share an inner candidate window, so different weights would silently + * desynchronise the reported total from the rows returned. + * + * Safe against a legacy four-column nodes_fts: FTS5's bm25() reads a weight + * only when an instance actually lands in that column (`nVal > ic`), so the + * fifth weight is simply never consulted on a table that has no fifth + * column. */ +#define BM25_WEIGHTS "bm25(nodes_fts, 1.0, 1.0, 1.0, 1.0, 0.3)" + /* Module-local SQLITE_TRANSIENT wrapper to dodge performance-no-int-to-ptr. * See the matching helper in src/store/store.c for the same pattern. */ static sqlite3_destructor_type mcp_sqlite_transient(void) { @@ -3031,7 +3050,7 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu * matches, this causes multi-minute queries. * * The fix: let FTS5 drive the inner subquery alone. SQLite CAN early-terminate - * SELECT rowid, bm25(nodes_fts) FROM nodes_fts WHERE MATCH ? ORDER BY bm25() LIMIT N + * SELECT rowid, bm25(nodes_fts,...) FROM nodes_fts WHERE MATCH ? ORDER BY bm25() LIMIT N * because no outer predicate blocks it. We fetch BM25_INNER_LIMIT top candidates * from the FTS5 index, then join/filter/boost only those rows. bm25() returns a * NEGATIVE score (lower = more relevant). */ @@ -3046,13 +3065,18 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu " WHEN n.label IN (" CBM_SQL_RELATION_LABELS ") THEN 5.0 " " ELSE 0.0 END) AS rank " "FROM (" - " SELECT rowid, bm25(nodes_fts) AS base_rank" + " SELECT rowid, " BM25_WEIGHTS " AS base_rank" " FROM nodes_fts WHERE nodes_fts MATCH ?1" " ORDER BY base_rank LIMIT ?5" ") fts " "JOIN nodes n ON n.id = fts.rowid " "WHERE n.project = ?2 " - " AND n.label NOT IN ('File','Folder','Module','Section','Variable','Project') " + /* Section and Module are NO LONGER excluded (#518/#519): they are the + * labels that carry prose — a Markdown section's body, a config file's + * description — so excluding them made the body column unreachable. + * This exclusion list is MIRRORED in the count query below; the two + * must be changed together or results desynchronise from counts. */ + " AND n.label NOT IN ('File','Folder','Variable','Project') " " AND (?6 IS NULL OR n.file_path LIKE ?6) " /* rank ties are common (boosted floats) — the id tie-break makes * offset pages contractually stable across calls. */ @@ -3079,17 +3103,19 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu * Uses the identical subquery structure so the FTS5 early-exit applies here too. */ int total = 0; { - const char *count_sql = - "SELECT COUNT(*) FROM (" - " SELECT fts.rowid FROM (" - " SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1" - " ORDER BY bm25(nodes_fts) LIMIT ?3" - " ) fts " - " JOIN nodes n ON n.id = fts.rowid " - " WHERE n.project = ?2 " - " AND n.label NOT IN ('File','Folder','Module','Section','Variable','Project')" - " AND (?6 IS NULL OR n.file_path LIKE ?6)" - ")"; + const char *count_sql = "SELECT COUNT(*) FROM (" + " SELECT fts.rowid FROM (" + " SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1" + " ORDER BY " BM25_WEIGHTS " LIMIT ?3" + " ) fts " + " JOIN nodes n ON n.id = fts.rowid " + " WHERE n.project = ?2 " + /* MIRRORS the ranked query's filter verbatim — same weights, same + * label exclusions. Changing one alone reports a total that does + * not describe the rows returned. */ + " AND n.label NOT IN ('File','Folder','Variable','Project')" + " AND (?6 IS NULL OR n.file_path LIKE ?6)" + ")"; sqlite3_stmt *cs = NULL; if (sqlite3_prepare_v2(db, count_sql, BM25_SQL_AUTO_LEN, &cs, NULL) == SQLITE_OK) { sqlite3_bind_text(cs, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN, diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 0c1d5eee4..9ba818224 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -1524,22 +1524,6 @@ static void discard_generation_stage(const char *stage_path) { cbm_remove_db_sidecars(stage_path); } -static int generation_rebuild_fts(cbm_store_t *store) { - if (cbm_store_exec(store, "INSERT INTO nodes_fts(nodes_fts) VALUES('delete-all');") != - CBM_STORE_OK) { - return CBM_STORE_ERR; - } - if (cbm_store_exec(store, - "INSERT INTO nodes_fts(rowid, name, qualified_name, label, file_path) " - "SELECT id, cbm_camel_split(name), qualified_name, label, file_path " - "FROM nodes;") == CBM_STORE_OK) { - return CBM_STORE_OK; - } - return cbm_store_exec(store, - "INSERT INTO nodes_fts(rowid, name, qualified_name, label, file_path) " - "SELECT id, name, qualified_name, label, file_path FROM nodes;"); -} - typedef struct { bool quarantined; char backup_path[CBM_SZ_4K]; @@ -1809,7 +1793,10 @@ int cbm_pipeline_publish_staged(char *stage_path, const cbm_pipeline_generation_ cbm_log_info("publish.timing", "block", "coverage_replace", "elapsed_ms", itoa_buf((int)elapsed_ms(t_pub))); cbm_clock_gettime(CLOCK_MONOTONIC, &t_pub); - if (fts_wholesale && generation_rebuild_fts(store) != CBM_STORE_OK) { + /* The column list lives in cbm_store_fts_rebuild() alone — see the delta + * merge, which must index the SAME columns or prose goes missing on the + * warm path while a full reindex looks perfect. */ + if (fts_wholesale && cbm_store_fts_rebuild(store, NULL, 0) != CBM_STORE_OK) { ok = false; } cbm_log_info("publish.timing", "block", "fts", "elapsed_ms", itoa_buf((int)elapsed_ms(t_pub))); diff --git a/src/pipeline/pipeline_delta.c b/src/pipeline/pipeline_delta.c index 50fc094c2..0596eb238 100644 --- a/src/pipeline/pipeline_delta.c +++ b/src/pipeline/pipeline_delta.c @@ -21,12 +21,17 @@ * deleted individually on existing databases; their rowids can never alias * a live node again (AUTOINCREMENT), so dead entries simply drop out of the * rowid join at query time. The patch inserts rows for exactly the new - * nodes, via the same cbm_camel_split SQL function the wholesale rebuild - * uses. + * nodes through cbm_store_fts_rebuild() — the SAME writer the wholesale + * rebuild uses, so the two paths cannot index different column sets. This + * path is the one most users actually hit: a hand-written INSERT here that + * named only the identifier columns would leave nodes_fts.body NULL for + * every node arriving by delta merge, invisibly, while a full reindex + * looked perfect (#518/#519). */ #include "foundation/constants.h" #include "pipeline/pipeline_internal.h" +#include /* intptr_t — SQLITE_TRANSIENT wrapper */ #include #include #include @@ -38,6 +43,25 @@ enum { DELTA_IN_CHUNK = 200 }; +/* Positional bind indices for the snapshot re-link statement. */ +enum { + DELTA_BIND_1 = 1, + DELTA_BIND_2 = 2, + DELTA_BIND_3 = 3, + DELTA_BIND_4 = 4, + DELTA_BIND_5 = 5, +}; + +/* Module-local SQLITE_TRANSIENT wrapper to dodge performance-no-int-to-ptr. + * Same pattern as store.c's BIND_TRANSIENT and mcp.c's MCP_SQLITE_TRANSIENT. */ +static sqlite3_destructor_type delta_sqlite_transient(void) { + static const volatile intptr_t raw = -1; + sqlite3_destructor_type dtor = NULL; + memcpy(&dtor, (const void *)&raw, sizeof(dtor)); + return dtor; +} +#define DELTA_SQLITE_TRANSIENT (delta_sqlite_transient()) + /* Assemble "?,?,...,?" for a chunked IN list. buf must hold 2*count. */ static void delta_placeholders(char *buf, int count) { int pos = 0; @@ -448,6 +472,42 @@ static void delta_patch_edge(const cbm_gbuf_edge_t *edge, void *userdata) { ctx->edges++; } +/* Re-link the snapshotted inbound edges by qualified name. A target whose QN + * no longer exists simply matches no row — full-reindex semantics for deleted + * symbols, dedup by the UNIQUE edge constraint. Returns false on failure. + * + * Extracted from cbm_delta_patch to keep that function inside the + * cognitive-complexity budget; behaviour is unchanged. */ +static bool delta_relink_snapshot(sqlite3 *db, const char *project, + const cbm_delta_saved_edge_t *snapshot, int snapshot_count) { + sqlite3_stmt *relink = NULL; + if (sqlite3_prepare_v2(db, + "INSERT OR IGNORE INTO edges (project, source_id, target_id," + " type, properties)" + " SELECT ?1, s.id, t.id, ?2, ?3 FROM nodes s, nodes t" + " WHERE s.project = ?1 AND s.qualified_name = ?4" + " AND t.project = ?1 AND t.qualified_name = ?5", + CBM_NOT_FOUND, &relink, NULL) != SQLITE_OK) { + return false; + } + bool ok = true; + for (int i = 0; i < snapshot_count && ok; i++) { + sqlite3_reset(relink); + sqlite3_bind_text(relink, DELTA_BIND_1, project, CBM_NOT_FOUND, DELTA_SQLITE_TRANSIENT); + sqlite3_bind_text(relink, DELTA_BIND_2, snapshot[i].type, CBM_NOT_FOUND, + DELTA_SQLITE_TRANSIENT); + sqlite3_bind_text(relink, DELTA_BIND_3, snapshot[i].props, CBM_NOT_FOUND, + DELTA_SQLITE_TRANSIENT); + sqlite3_bind_text(relink, DELTA_BIND_4, snapshot[i].source_qn, CBM_NOT_FOUND, + DELTA_SQLITE_TRANSIENT); + sqlite3_bind_text(relink, DELTA_BIND_5, snapshot[i].target_qn, CBM_NOT_FOUND, + DELTA_SQLITE_TRANSIENT); + ok = sqlite3_step(relink) == SQLITE_DONE; + } + sqlite3_finalize(relink); + return ok; +} + int cbm_delta_patch(cbm_store_t *store, const char *project, cbm_gbuf_t *gbuf, int64_t max_db_id, const cbm_delta_saved_edge_t *snapshot, int snapshot_count) { sqlite3 *db = cbm_store_get_db(store); @@ -489,57 +549,21 @@ int cbm_delta_patch(cbm_store_t *store, const char *project, cbm_gbuf_t *gbuf, i sqlite3_finalize(ctx.edge_stmt); sqlite3_finalize(ctx.qn_lookup); - /* Re-link the snapshotted inbound edges by qualified name. A target - * whose QN no longer exists simply matches no row — full-reindex - * semantics for deleted symbols, dedup by the UNIQUE edge constraint. */ - if (!ctx.failed && snapshot_count > 0) { - sqlite3_stmt *relink = NULL; - if (sqlite3_prepare_v2(db, - "INSERT OR IGNORE INTO edges (project, source_id, target_id," - " type, properties)" - " SELECT ?1, s.id, t.id, ?2, ?3 FROM nodes s, nodes t" - " WHERE s.project = ?1 AND s.qualified_name = ?4" - " AND t.project = ?1 AND t.qualified_name = ?5", - CBM_NOT_FOUND, &relink, NULL) != SQLITE_OK) { - ctx.failed = true; - } else { - for (int i = 0; i < snapshot_count && !ctx.failed; i++) { - sqlite3_reset(relink); - sqlite3_bind_text(relink, 1, project, CBM_NOT_FOUND, SQLITE_TRANSIENT); - sqlite3_bind_text(relink, 2, snapshot[i].type, CBM_NOT_FOUND, SQLITE_TRANSIENT); - sqlite3_bind_text(relink, 3, snapshot[i].props, CBM_NOT_FOUND, SQLITE_TRANSIENT); - sqlite3_bind_text(relink, 4, snapshot[i].source_qn, CBM_NOT_FOUND, - SQLITE_TRANSIENT); - sqlite3_bind_text(relink, 5, snapshot[i].target_qn, CBM_NOT_FOUND, - SQLITE_TRANSIENT); - if (sqlite3_step(relink) != SQLITE_DONE) { - ctx.failed = true; - } - } - sqlite3_finalize(relink); - } + if (!ctx.failed && snapshot_count > 0 && + !delta_relink_snapshot(db, project, snapshot, snapshot_count)) { + ctx.failed = true; } - /* Row-level FTS for exactly the new nodes, through the same tokenizer - * function the wholesale rebuild uses. */ + /* Row-level FTS for exactly the new nodes, through the shared writer — + * same columns, same tokenizer, same prose backfill as the wholesale + * rebuild. NOT_FOUND means the index could not be written at all (FTS5 + * compiled out); search then runs without it, matching the dump path. */ if (!ctx.failed) { - sqlite3_stmt *fts = NULL; - if (sqlite3_prepare_v2(db, - "INSERT INTO nodes_fts (rowid, name, qualified_name, label," - " file_path)" - " SELECT id, cbm_camel_split(name), qualified_name, label," - " file_path FROM nodes WHERE project = ?1 AND id > ?2", - CBM_NOT_FOUND, &fts, NULL) == SQLITE_OK) { - sqlite3_bind_text(fts, 1, project, CBM_NOT_FOUND, SQLITE_TRANSIENT); - sqlite3_bind_int64(fts, 2, max_db_id); - if (sqlite3_step(fts) != SQLITE_DONE) { - ctx.failed = true; - } - sqlite3_finalize(fts); - } else { - /* FTS5 may be compiled out; the table then never existed and - * search runs without it — matching the dump path's behavior. */ + int fts_rc = cbm_store_fts_rebuild(store, project, max_db_id); + if (fts_rc == CBM_STORE_NOT_FOUND) { cbm_log_warn("delta.fts_insert_unavailable", "project", project); + } else if (fts_rc != CBM_STORE_OK) { + ctx.failed = true; } } diff --git a/src/store/store.c b/src/store/store.c index ad0b7967b..8e44583b3 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -356,12 +356,21 @@ static int init_schema(cbm_store_t *s) { * not a copy of the source text — required for camelCase tokenization * because we feed it `cbm_camel_split(name)` at insert time but want * queries to match against the split tokens, not the original. - * Fails silently if FTS5 is not compiled in (SQLITE_ENABLE_FTS5). */ + * Fails silently if FTS5 is not compiled in (SQLITE_ENABLE_FTS5). + * + * `body` (#518/#519) carries each node's prose — its docstring — so a + * question phrased in words rather than identifiers still finds the node. + * It is deliberately NOT an index-format change: IF NOT EXISTS leaves a + * legacy four-column table exactly as it is, cbm_store_fts_rebuild() + * probes for the column before naming it, and bm25()'s surplus weight is + * inert on a table that has no fifth column. A database written by an + * older build therefore keeps opening and searching — without prose — + * instead of forcing every user to reindex. */ { char *fts_err = NULL; int fts_rc = sqlite3_exec(s->db, "CREATE VIRTUAL TABLE IF NOT EXISTS nodes_fts USING fts5(" - " name, qualified_name, label, file_path," + " name, qualified_name, label, file_path, body," " content=''," " tokenize='unicode61 remove_diacritics 2'" ");", @@ -373,6 +382,102 @@ static int init_schema(cbm_store_t *s) { return CBM_STORE_OK; } +/* ── FTS backfill ───────────────────────────────────────────────── */ + +/* Prose source for nodes_fts.body: the docstring each node already carries in + * its properties JSON. Deriving it costs no new column on `nodes` and no new + * table, which is exactly why CBM_INDEX_FORMAT_VERSION does not move and no + * user is forced to reindex. + * - the LIKE prefilter keeps the JSON parse off the large majority of nodes + * that have no docstring at all; + * - json_valid() guards the parse because json_extract() RAISES on malformed + * properties, and pre-fix databases contain such rows (see the reverted + * is_entry_point expression index in create_user_indexes) — unguarded, one + * bad row would abort the entire backfill. */ +#define FTS_BODY_EXPR \ + "CASE WHEN properties LIKE '%\"docstring\"%' AND json_valid(properties) " \ + "THEN json_extract(properties, '$.docstring') END" + +enum { FTS_SQL_BUF = 512 }; + +/* Does nodes_fts carry the `body` column? A database created by an older + * build has only the four identifier columns, and CREATE VIRTUAL TABLE IF NOT + * EXISTS does not widen it. Probe rather than assume: naming a column that + * is not there would fail the backfill outright, and the contract for a legacy + * database is "still searchable, just without prose". */ +static bool fts_has_body_column(cbm_store_t *s) { + sqlite3_stmt *probe = NULL; + if (sqlite3_prepare_v2(s->db, "SELECT body FROM nodes_fts LIMIT 0;", CBM_NOT_FOUND, &probe, + NULL) != SQLITE_OK) { + return false; + } + sqlite3_finalize(probe); + return true; +} + +/* One backfill attempt. CBM_STORE_NOT_FOUND means the statement would not + * even prepare (missing table, column or SQL function) so the caller may retry + * with a narrower shape; CBM_STORE_ERR means the write itself failed. */ +static int fts_backfill_try(cbm_store_t *s, const char *project, int64_t after_id, bool with_body, + bool camel) { + char sql[FTS_SQL_BUF]; + int n = snprintf(sql, sizeof(sql), + "INSERT INTO nodes_fts (rowid, name, qualified_name, label, file_path%s)" + " SELECT id, %s, qualified_name, label, file_path%s FROM nodes%s;", + with_body ? ", body" : "", camel ? "cbm_camel_split(name)" : "name", + with_body ? ", " FTS_BODY_EXPR : "", + project ? " WHERE project = ?1 AND id > ?2" : ""); + if (n <= 0 || (size_t)n >= sizeof(sql)) { + return CBM_STORE_ERR; + } + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { + return CBM_STORE_NOT_FOUND; + } + if (project) { + sqlite3_bind_text(stmt, ST_COL_1, project, CBM_NOT_FOUND, BIND_TRANSIENT); + sqlite3_bind_int64(stmt, ST_COL_2, after_id); + } + int rc = sqlite3_step(stmt) == SQLITE_DONE ? CBM_STORE_OK : CBM_STORE_ERR; + if (rc != CBM_STORE_OK) { + store_set_error_sqlite(s, "fts_backfill"); + } + sqlite3_finalize(stmt); + return rc; +} + +int cbm_store_fts_rebuild(cbm_store_t *s, const char *project, int64_t after_id) { + if (!s || !s->db) { + return CBM_STORE_ERR; + } + /* Wholesale only: the incremental caller adds rows to a live index. */ + if (!project && + exec_sql(s, "INSERT INTO nodes_fts(nodes_fts) VALUES('delete-all');") != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + /* Degrade one capability at a time, widest first. Dropping `body` covers a + * legacy four-column table or a build without JSON1; dropping + * cbm_camel_split covers a connection where the function is not registered + * (the pre-existing fallback). Each attempt is a single statement, so a + * failed one rolls itself back before the next runs. */ + const bool with_body = fts_has_body_column(s); + const struct { + bool body; + bool camel; + } ladder[] = {{true, true}, {true, false}, {false, true}, {false, false}}; + int rc = CBM_STORE_NOT_FOUND; + for (size_t i = 0; i < sizeof(ladder) / sizeof(ladder[0]); i++) { + if (ladder[i].body && !with_body) { + continue; + } + rc = fts_backfill_try(s, project, after_id, ladder[i].body, ladder[i].camel); + if (rc == CBM_STORE_OK) { + return rc; + } + } + return rc; +} + static int create_user_indexes(cbm_store_t *s) { const char *sql = "CREATE INDEX IF NOT EXISTS idx_nodes_label ON nodes(project, label);" diff --git a/src/store/store.h b/src/store/store.h index 45a7bb439..71fbc911d 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -912,4 +912,20 @@ int cbm_store_count_vectors(cbm_store_t *s, const char *project); * Returns CBM_STORE_OK on success. */ int cbm_store_exec(cbm_store_t *s, const char *sql); +/* Populate nodes_fts from the `nodes` table — the single writer for the BM25 + * index. Every backfill site routes through it so the column list is decided + * in exactly ONE place: a hand-written INSERT that names only the identifier + * columns silently leaves `body` NULL for every node it writes, which looks + * perfect after a full reindex and de-indexes prose on the warm path. + * + * project == NULL → wholesale rebuild: clears the index, then reindexes + * every node in the database. + * project != NULL → incremental: indexes only that project's nodes with + * id > after_id and leaves existing rows untouched. + * + * Returns CBM_STORE_OK on success; CBM_STORE_NOT_FOUND when nodes_fts cannot + * be written at all (FTS5 compiled out — the caller decides whether that is + * fatal); CBM_STORE_ERR on a genuine write failure. */ +int cbm_store_fts_rebuild(cbm_store_t *s, const char *project, int64_t after_id); + #endif /* CBM_STORE_H */ diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 935eda07c..b34b14024 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -6013,6 +6013,208 @@ TEST(iris_export_xml_multi_class) { PASS(); } +/* ── #518 / #519: prose that BM25 can index ──────────────────────── + * + * A Section carried only its heading and a config Module only its path, so a + * question asked in words could not reach either. Both now carry the prose in + * `docstring`, which is what nodes_fts indexes into its `body` column. */ + +/* First Module-labelled definition, or NULL. */ +static const CBMDefinition *find_module_def(CBMFileResult *r) { + for (int i = 0; i < r->defs.count; i++) { + if (strcmp(r->defs.items[i].label, "Module") == 0) { + return &r->defs.items[i]; + } + } + return NULL; +} + +TEST(markdown_section_body_becomes_docstring_issue518) { + CBMFileResult *r = extract("# Installation\n" + "Run the bootstrap script to provision a workstation.\n" + "It installs the toolchain and seeds the cache.\n", + CBM_LANG_MARKDOWN, "t", "README.md"); + ASSERT_NOT_NULL(r); + const CBMDefinition *d = find_def_by_name(r, "Installation"); + ASSERT_NOT_NULL(d); + ASSERT_STR_EQ(d->label, "Section"); + ASSERT_NOT_NULL(d->docstring); + /* The prose — not the heading — is what makes the section findable. */ + ASSERT_NOT_NULL(strstr(d->docstring, "bootstrap script")); + ASSERT_NOT_NULL(strstr(d->docstring, "seeds the cache")); + /* Newlines collapse to single spaces so the 500-byte cap buys real words. */ + ASSERT_NULL(strchr(d->docstring, '\n')); + cbm_free_result(r); + PASS(); +} + +TEST(markdown_section_body_stops_at_next_heading_issue518) { + CBMFileResult *r = extract("# Alpha\n" + "alphatext belongs to the first section.\n" + "\n" + "# Beta\n" + "betatext belongs to the second section.\n", + CBM_LANG_MARKDOWN, "t", "README.md"); + ASSERT_NOT_NULL(r); + const CBMDefinition *alpha = find_def_by_name(r, "Alpha"); + const CBMDefinition *beta = find_def_by_name(r, "Beta"); + ASSERT_NOT_NULL(alpha); + ASSERT_NOT_NULL(beta); + ASSERT_NOT_NULL(alpha->docstring); + ASSERT_NOT_NULL(beta->docstring); + /* Each section owns ITS body: bleeding across the boundary would make every + * heading match every word in the file. */ + ASSERT_NOT_NULL(strstr(alpha->docstring, "alphatext")); + ASSERT_NULL(strstr(alpha->docstring, "betatext")); + ASSERT_NOT_NULL(strstr(beta->docstring, "betatext")); + ASSERT_NULL(strstr(beta->docstring, "alphatext")); + cbm_free_result(r); + PASS(); +} + +TEST(markdown_section_body_heading_only_has_no_docstring_issue518) { + CBMFileResult *r = extract("# Lonely\n", CBM_LANG_MARKDOWN, "t", "README.md"); + ASSERT_NOT_NULL(r); + const CBMDefinition *d = find_def_by_name(r, "Lonely"); + ASSERT_NOT_NULL(d); + /* An empty body stays NULL rather than "": append_json_string drops empty + * values, so an empty string would be a difference with no observable + * meaning — and a docstring key that promises prose it does not have. */ + ASSERT_NULL(d->docstring); + cbm_free_result(r); + PASS(); +} + +TEST(markdown_section_body_capped_utf8_safe_issue518) { + /* 400 three-byte codepoints (1200 bytes) guarantees the 500-byte cut lands + * mid-sequence unless the backoff works. */ + char src[4096]; + int pos = snprintf(src, sizeof(src), "# Unicode\n"); + for (int i = 0; i < 400; i++) { + pos += snprintf(src + pos, sizeof(src) - (size_t)pos, "\xe2\x9c\x93"); + } + snprintf(src + pos, sizeof(src) - (size_t)pos, "\n"); + + CBMFileResult *r = extract(src, CBM_LANG_MARKDOWN, "t", "README.md"); + ASSERT_NOT_NULL(r); + const CBMDefinition *d = find_def_by_name(r, "Unicode"); + ASSERT_NOT_NULL(d); + ASSERT_NOT_NULL(d->docstring); + size_t n = strlen(d->docstring); + ASSERT_LTE((int)n, 500); /* MAX_COMMENT_LEN — fits the 2 KB properties buffer */ + ASSERT_GT((int)n, 0); + /* Every byte must belong to a COMPLETE sequence: walk the string and check + * each lead byte is followed by its full continuation run. */ + for (size_t i = 0; i < n;) { + unsigned char c = (unsigned char)d->docstring[i]; + size_t need; + if ((c & 0x80) == 0) { + need = 1; + } else if ((c & 0xE0) == 0xC0) { + need = 2; + } else if ((c & 0xF0) == 0xE0) { + need = 3; + } else if ((c & 0xF8) == 0xF0) { + need = 4; + } else { + FAIL("stray UTF-8 continuation byte at a sequence start"); + } + ASSERT_LTE((int)(i + need), (int)n); /* no truncated tail sequence */ + i += need; + } + cbm_free_result(r); + PASS(); +} + +TEST(yaml_toplevel_description_promoted_to_module_issue519) { + CBMFileResult *r = extract("name: my-action\n" + "description: Provisions an ephemeral build runner.\n" + "runs:\n" + " using: node20\n", + CBM_LANG_YAML, "t", "META.yaml"); + ASSERT_NOT_NULL(r); + const CBMDefinition *mod = find_module_def(r); + ASSERT_NOT_NULL(mod); + ASSERT_NOT_NULL(mod->docstring); + ASSERT_NOT_NULL(strstr(mod->docstring, "ephemeral build runner")); + cbm_free_result(r); + PASS(); +} + +TEST(yaml_block_scalar_description_promoted_issue519) { + CBMFileResult *r = extract("name: my-action\n" + "description: |\n" + " Provisions an ephemeral build runner\n" + " and tears it down afterwards.\n", + CBM_LANG_YAML, "t", "META.yaml"); + ASSERT_NOT_NULL(r); + const CBMDefinition *mod = find_module_def(r); + ASSERT_NOT_NULL(mod); + ASSERT_NOT_NULL(mod->docstring); + /* The `|` indicator itself must not survive into the indexed text. */ + ASSERT_NOT_NULL(strstr(mod->docstring, "tears it down")); + ASSERT_NULL(strchr(mod->docstring, '|')); + cbm_free_result(r); + PASS(); +} + +TEST(yaml_summary_promoted_when_no_description_issue519) { + CBMFileResult *r = extract("name: thing\n" + "summary: Aggregates telemetry from every shard.\n", + CBM_LANG_YAML, "t", "META.yaml"); + ASSERT_NOT_NULL(r); + const CBMDefinition *mod = find_module_def(r); + ASSERT_NOT_NULL(mod); + ASSERT_NOT_NULL(mod->docstring); + ASSERT_NOT_NULL(strstr(mod->docstring, "every shard")); + cbm_free_result(r); + PASS(); +} + +TEST(json_toplevel_description_promoted_to_module_issue519) { + CBMFileResult *r = extract("{\n" + " \"name\": \"widget\",\n" + " \"description\": \"Renders dashboards from graph queries.\"\n" + "}\n", + CBM_LANG_JSON, "t", "package.json"); + ASSERT_NOT_NULL(r); + const CBMDefinition *mod = find_module_def(r); + ASSERT_NOT_NULL(mod); + ASSERT_NOT_NULL(mod->docstring); + ASSERT_NOT_NULL(strstr(mod->docstring, "Renders dashboards")); + /* The JSON string quotes are stripped — they are not part of the value. */ + ASSERT_NULL(strchr(mod->docstring, '"')); + cbm_free_result(r); + PASS(); +} + +TEST(config_description_only_at_top_level_issue519) { + /* A nested `description` describes the nested thing, not the file. */ + CBMFileResult *r = extract("name: chart\n" + "values:\n" + " description: nestedonly\n", + CBM_LANG_YAML, "t", "META.yaml"); + ASSERT_NOT_NULL(r); + const CBMDefinition *mod = find_module_def(r); + ASSERT_NOT_NULL(mod); + ASSERT(mod->docstring == NULL || strstr(mod->docstring, "nestedonly") == NULL); + cbm_free_result(r); + PASS(); +} + +TEST(non_config_language_module_has_no_promoted_description_issue519) { + /* The promotion is config-only: a Python file's Module node must not pick + * up a variable that merely happens to be called `description`. */ + CBMFileResult *r = + extract("description = 'not a config file'\n", CBM_LANG_PYTHON, "t", "conf.py"); + ASSERT_NOT_NULL(r); + const CBMDefinition *mod = find_module_def(r); + ASSERT_NOT_NULL(mod); + ASSERT_NULL(mod->docstring); + cbm_free_result(r); + PASS(); +} + SUITE(extraction) { /* Initialize extraction library */ cbm_init(); @@ -6366,5 +6568,17 @@ SUITE(extraction) { RUN_TEST(docstring_utf8_truncation_boundary_issue1017); RUN_TEST(extract_ts_decorators_survive_interleaved_comment); + /* #518/#519 — prose carried into docstring so nodes_fts can index it */ + RUN_TEST(markdown_section_body_becomes_docstring_issue518); + RUN_TEST(markdown_section_body_stops_at_next_heading_issue518); + RUN_TEST(markdown_section_body_heading_only_has_no_docstring_issue518); + RUN_TEST(markdown_section_body_capped_utf8_safe_issue518); + RUN_TEST(yaml_toplevel_description_promoted_to_module_issue519); + RUN_TEST(yaml_block_scalar_description_promoted_issue519); + RUN_TEST(yaml_summary_promoted_when_no_description_issue519); + RUN_TEST(json_toplevel_description_promoted_to_module_issue519); + RUN_TEST(config_description_only_at_top_level_issue519); + RUN_TEST(non_config_language_module_has_no_promoted_description_issue519); + cbm_shutdown(); } diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 6b7af64ed..18bfbd3b1 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -11460,7 +11460,249 @@ TEST(index_repository_supervisor_uses_canonical_session_path) { * SUITE * ══════════════════════════════════════════════════════════════════ */ +/* ── BM25 prose search (#518 / #519) ─────────────────────────────── + * + * Section and Module used to be filtered out of BM25 results outright, which + * made the prose they carry unreachable no matter how it was indexed. These + * cover the query side: prose is findable, the excluded labels come back, and + * the ranked query and the count query still agree. */ + +static cbm_mcp_server_t *setup_prose_search_server(const char *proj) { + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + if (!srv) { + return NULL; + } + cbm_store_t *st = cbm_mcp_server_store(srv); + cbm_mcp_server_set_project(srv, proj); + cbm_store_upsert_project(st, proj, "/tmp/prose"); + + cbm_node_t section = {.project = proj, + .label = "Section", + .name = "Installation", + .qualified_name = "prose.README.Installation", + .file_path = "README.md", + .start_line = 1, + .end_line = 20, + .properties_json = "{\"docstring\":\"provisions an ephemeral " + "workstation runner and seeds the cache\"}"}; + cbm_store_upsert_node(st, §ion); + + cbm_node_t module = {.project = proj, + .label = "Module", + .name = "action.yml", + .qualified_name = "prose.action_yml", + .file_path = "action.yml", + .start_line = 1, + .end_line = 40, + .properties_json = + "{\"docstring\":\"aggregates telemetry from every shard\"}"}; + cbm_store_upsert_node(st, &module); + + cbm_node_t fn = {.project = proj, + .label = "Function", + .name = "telemetryCollector", + .qualified_name = "prose.src.telemetryCollector", + .file_path = "src/collect.c", + .start_line = 3, + .end_line = 9}; + cbm_store_upsert_node(st, &fn); + + cbm_store_fts_rebuild(st, NULL, 0); + return srv; +} + +static char *prose_search(cbm_mcp_server_t *srv, const char *proj, const char *query) { + char req[512]; + snprintf(req, sizeof(req), + "{\"jsonrpc\":\"2.0\",\"id\":518,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"search_graph\"," + "\"arguments\":{\"project\":\"%s\",\"query\":\"%s\",\"limit\":10}}}", + proj, query); + char *resp = cbm_mcp_server_handle(srv, req); + if (!resp) { + return NULL; + } + char *inner = extract_text_content(resp); + free(resp); + return inner; +} + +TEST(bm25_finds_section_by_its_prose_issue518) { + cbm_mcp_server_t *srv = setup_prose_search_server("prose518"); + ASSERT_NOT_NULL(srv); + + /* "ephemeral" appears NOWHERE in any identifier — only in the section's + * body. Before the body column it was unfindable. */ + char *inner = prose_search(srv, "prose518", "ephemeral"); + ASSERT_NOT_NULL(inner); + ASSERT_NOT_NULL(strstr(inner, "search_mode: bm25")); + ASSERT_NOT_NULL(strstr(inner, "prose.README.Installation")); + ASSERT_NOT_NULL(strstr(inner, "Section")); + free(inner); + + cbm_mcp_server_free(srv); + PASS(); +} + +TEST(bm25_finds_module_by_promoted_description_issue519) { + cbm_mcp_server_t *srv = setup_prose_search_server("prose519"); + ASSERT_NOT_NULL(srv); + + /* A config file's own description, promoted onto its Module node. Module + * was one of the labels the BM25 filter used to drop unconditionally. */ + char *inner = prose_search(srv, "prose519", "shard"); + ASSERT_NOT_NULL(inner); + ASSERT_NOT_NULL(strstr(inner, "prose.action_yml")); + ASSERT_NOT_NULL(strstr(inner, "Module")); + free(inner); + + cbm_mcp_server_free(srv); + PASS(); +} + +TEST(bm25_results_and_total_stay_consistent_issue518) { + /* The ranked query and the count query share an inner candidate window and + * MIRROR each other's filter. Changing the label exclusion (or the weights) + * in only one silently reports a total that does not describe the rows. */ + cbm_mcp_server_t *srv = setup_prose_search_server("prosecount"); + ASSERT_NOT_NULL(srv); + + char *inner = prose_search(srv, "prosecount", "telemetry"); + ASSERT_NOT_NULL(inner); + /* Both the Module (body: "aggregates telemetry...") and the Function + * (name: telemetryCollector) match, and both are now eligible. */ + const char *total = strstr(inner, "total: "); + const char *results = strstr(inner, "results: "); + ASSERT_NOT_NULL(total); + ASSERT_NOT_NULL(results); + int total_n = atoi(total + strlen("total: ")); + int results_n = atoi(results + strlen("results: ")); + ASSERT_EQ(total_n, 2); + ASSERT_EQ(results_n, total_n); + ASSERT_NOT_NULL(strstr(inner, "prose.action_yml")); + ASSERT_NOT_NULL(strstr(inner, "prose.src.telemetryCollector")); + free(inner); + + cbm_mcp_server_free(srv); + PASS(); +} + +TEST(bm25_identifier_match_outranks_prose_only_match_issue518) { + /* The 0.3 body weight is what keeps prose from drowning identifiers. Both + * candidates carry the same label boost, so the ordering here is decided by + * the column weights alone. */ + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + cbm_store_t *st = cbm_mcp_server_store(srv); + const char *proj = "prose-rank"; + cbm_mcp_server_set_project(srv, proj); + cbm_store_upsert_project(st, proj, "/tmp/prose-rank"); + + cbm_node_t by_name = {.project = proj, + .label = "Function", + .name = "reconcile", + .qualified_name = "pr.a.reconcile", + .file_path = "a.c"}; + ASSERT_TRUE(cbm_store_upsert_node(st, &by_name) > 0); + + cbm_node_t by_body = {.project = proj, + .label = "Function", + .name = "zzz", + .qualified_name = "pr.b.zzz", + .file_path = "b.c", + .properties_json = "{\"docstring\":\"reconcile the ledger\"}"}; + ASSERT_TRUE(cbm_store_upsert_node(st, &by_body) > 0); + ASSERT_EQ(cbm_store_fts_rebuild(st, NULL, 0), CBM_STORE_OK); + + char *inner = prose_search(srv, proj, "reconcile"); + ASSERT_NOT_NULL(inner); + const char *name_hit = strstr(inner, "pr.a.reconcile"); + const char *body_hit = strstr(inner, "pr.b.zzz"); + ASSERT_NOT_NULL(name_hit); /* the identifier match */ + ASSERT_NOT_NULL(body_hit); /* the prose-only match still SURFACES ... */ + ASSERT_TRUE(name_hit < body_hit); /* ... but never above the identifier */ + free(inner); + + cbm_mcp_server_free(srv); + PASS(); +} + +TEST(bm25_searches_legacy_four_column_fts_without_error_issue518) { + /* No index-format bump means a database whose nodes_fts predates the body + * column is opened by the current binary. bm25()'s fifth weight must be + * inert there, not an error: FTS5 reads a weight only for a column an + * instance actually landed in. */ + char *td = th_mktempdir("cbm_mcp_legacy_fts"); + ASSERT_NOT_NULL(td); + /* cbm_mcp_server_new(project) opens /.db, so seeding + * the legacy table THERE is what puts the server on a pre-body database — + * the same way a real upgrade finds one. */ + char saved_cache[512] = {0}; + const char *prev = getenv("CBM_CACHE_DIR"); + if (prev) { + snprintf(saved_cache, sizeof(saved_cache), "%s", prev); + } + cbm_setenv("CBM_CACHE_DIR", td, 1); + + const char *proj = "legacyfts"; + char dbpath[600]; + snprintf(dbpath, sizeof(dbpath), "%s/%s.db", td, proj); + + sqlite3 *raw = NULL; + ASSERT_EQ(sqlite3_open(dbpath, &raw), SQLITE_OK); + ASSERT_EQ(sqlite3_exec(raw, + "CREATE VIRTUAL TABLE nodes_fts USING fts5(" + " name, qualified_name, label, file_path," + " content='', tokenize='unicode61 remove_diacritics 2');", + NULL, NULL, NULL), + SQLITE_OK); + sqlite3_close(raw); + + cbm_mcp_server_t *srv = cbm_mcp_server_new(proj); + ASSERT_NOT_NULL(srv); + cbm_store_t *st = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(st); + cbm_mcp_server_set_project(srv, proj); + cbm_store_upsert_project(st, proj, td); + + cbm_node_t fn = {.project = proj, + .label = "Function", + .name = "reconcile", + .qualified_name = "legacy.a.reconcile", + .file_path = "a.c", + .properties_json = "{\"docstring\":\"prose that cannot be indexed here\"}"}; + ASSERT_TRUE(cbm_store_upsert_node(st, &fn) > 0); + ASSERT_EQ(cbm_store_fts_rebuild(st, NULL, 0), CBM_STORE_OK); + + char *inner = prose_search(srv, proj, "reconcile"); + ASSERT_NOT_NULL(inner); + ASSERT_NOT_NULL(strstr(inner, "search_mode: bm25")); + ASSERT_NOT_NULL(strstr(inner, "legacy.a.reconcile")); + free(inner); + + /* Prose is absent rather than broken — a degrade, not a failure. */ + inner = prose_search(srv, proj, "indexed"); + ASSERT_NOT_NULL(inner); + ASSERT_NULL(strstr(inner, "legacy.a.reconcile")); + free(inner); + + cbm_mcp_server_free(srv); + if (saved_cache[0]) { + cbm_setenv("CBM_CACHE_DIR", saved_cache, 1); + } else { + cbm_unsetenv("CBM_CACHE_DIR"); + } + th_rmtree(td); + PASS(); +} + SUITE(mcp) { + /* #518/#519 — BM25 prose search */ + RUN_TEST(bm25_finds_section_by_its_prose_issue518); + RUN_TEST(bm25_finds_module_by_promoted_description_issue519); + RUN_TEST(bm25_results_and_total_stay_consistent_issue518); + RUN_TEST(bm25_identifier_match_outranks_prose_only_match_issue518); + RUN_TEST(bm25_searches_legacy_four_column_fts_without_error_issue518); RUN_TEST(mcp_path_within_root_rejects_escape); RUN_TEST(detect_changes_rejects_option_like_base_branch); RUN_TEST(detect_changes_rejects_windows_cmd_metacharacters_in_base_branch); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 564792d2b..3f874bd19 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -12457,6 +12457,77 @@ TEST(pipeline_ensemble_routing_method_scoping) { PASS(); } +/* #518/#519 item-7 regression: the DELTA merge is the warm path most users + * hit. It used to write nodes_fts with a hand-rolled four-column INSERT of + * its own; with a fifth `body` column that literal leaves prose NULL for every + * node arriving by delta, invisibly — a full reindex still looks perfect. + * Routing the write through cbm_store_fts_rebuild() is what this test binds: + * revert pipeline_delta.c to a four-column INSERT and it fails. */ +TEST(pipeline_delta_patch_indexes_docstring_into_fts_body) { + char *td = th_mktempdir("cbm_delta_fts"); + ASSERT_NOT_NULL(td); + char dbpath[512]; + snprintf(dbpath, sizeof(dbpath), "%s/delta.db", td); + + cbm_store_t *store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + const char *proj = "deltafts"; + ASSERT_EQ(cbm_store_upsert_project(store, proj, td), CBM_STORE_OK); + + /* A previous generation already on disk, so the patch runs against a real + * id watermark rather than an empty database. */ + cbm_node_t existing = {.project = proj, + .label = "Function", + .name = "alreadyThere", + .qualified_name = "deltafts.main.alreadyThere", + .file_path = "main.c"}; + ASSERT_TRUE(cbm_store_upsert_node(store, &existing) > 0); + ASSERT_EQ(cbm_store_fts_rebuild(store, NULL, 0), CBM_STORE_OK); + + /* The delta patch: preseed proxies the resident rows and sets the + * watermark, then the NEW node is minted above it. */ + cbm_gbuf_t *gb = cbm_gbuf_new(proj, td); + ASSERT_NOT_NULL(gb); + int64_t max_db_id = cbm_delta_preseed(store, proj, gb); + ASSERT_TRUE(max_db_id > 0); + int64_t new_id = cbm_gbuf_upsert_node( + gb, "Section", "Upgrading", "deltafts.README.Upgrading", "README.md", 1, 9, + "{\"docstring\":\"migrates the retention ledger before the cutover\"}"); + ASSERT_TRUE(new_id > max_db_id); + + ASSERT_EQ(cbm_delta_patch(store, proj, gb, max_db_id, NULL, 0), 0); + cbm_gbuf_free(gb); + + /* nodes_fts is contentless, so a column-filtered MATCH is the only way to + * observe WHICH column a token landed in — and the only assertion that + * distinguishes "indexed" from "indexed without its prose". */ + sqlite3_stmt *st = NULL; + ASSERT_EQ(sqlite3_prepare_v2(cbm_store_get_db(store), + "SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1", -1, &st, + NULL), + SQLITE_OK); + sqlite3_bind_text(st, 1, "body:retention", -1, SQLITE_TRANSIENT); + ASSERT_EQ(sqlite3_step(st), SQLITE_ROW); + ASSERT_EQ((long long)sqlite3_column_int64(st, 0), (long long)new_id); + ASSERT_EQ(sqlite3_step(st), SQLITE_DONE); + sqlite3_finalize(st); + + /* The identifier columns are still written on the same path. */ + st = NULL; + ASSERT_EQ(sqlite3_prepare_v2(cbm_store_get_db(store), + "SELECT COUNT(*) FROM nodes_fts WHERE nodes_fts MATCH ?1", -1, &st, + NULL), + SQLITE_OK); + sqlite3_bind_text(st, 1, "name:Upgrading", -1, SQLITE_TRANSIENT); + ASSERT_EQ(sqlite3_step(st), SQLITE_ROW); + ASSERT_EQ(sqlite3_column_int(st, 0), 1); + sqlite3_finalize(st); + + cbm_store_close(store); + th_rmtree(td); + PASS(); +} + SUITE(pipeline) { RUN_TEST(pipeline_lsp_surface_persisted_and_body_edit_invariant); /* Index lock */ @@ -12774,6 +12845,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_ensemble_routing_attr_does_not_leak_across_items); RUN_TEST(pipeline_ensemble_routing_settings_targets); RUN_TEST(pipeline_ensemble_routing_unterminated_item_is_safe); + RUN_TEST(pipeline_delta_patch_indexes_docstring_into_fts_body); } /* Focused semantic-manifest and publication contracts. Kept separate from the diff --git a/tests/test_store_search.c b/tests/test_store_search.c index a82b15240..814cb5fef 100644 --- a/tests/test_store_search.c +++ b/tests/test_store_search.c @@ -8,6 +8,7 @@ #include "test_framework.h" #include "test_helpers.h" #include +#include "sqlite3.h" /* vendored/sqlite3 — raw nodes_fts MATCH probes */ #include #include #include @@ -1629,6 +1630,172 @@ TEST(store_find_nodes_rejects_null_store_without_ub) { PASS(); } +/* ── nodes_fts prose column (#518 / #519) ────────────────────────── + * + * nodes_fts is CONTENTLESS, so a column's value cannot be selected back — a + * MATCH with a column filter is the only way to prove a token landed in `body` + * rather than in one of the identifier columns. That distinction IS the test: + * a four-column INSERT still "works", it just silently indexes no prose. */ + +static int fts_match_count(cbm_store_t *s, const char *match) { + sqlite3 *db = cbm_store_get_db(s); + sqlite3_stmt *st = NULL; + if (sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM nodes_fts WHERE nodes_fts MATCH ?1", -1, &st, + NULL) != SQLITE_OK) { + return -1; + } + sqlite3_bind_text(st, 1, match, -1, SQLITE_TRANSIENT); + int n = sqlite3_step(st) == SQLITE_ROW ? sqlite3_column_int(st, 0) : -1; + sqlite3_finalize(st); + return n; +} + +/* One Section carrying prose, one Function carrying none. */ +static void seed_prose_nodes(cbm_store_t *s) { + cbm_store_upsert_project(s, "p", "/tmp/p"); + cbm_node_t sec = {.project = "p", + .label = "Section", + .name = "Installation", + .qualified_name = "p.README.Installation", + .file_path = "README.md", + .properties_json = + "{\"docstring\":\"provisions an ephemeral workstation runner\"}"}; + cbm_store_upsert_node(s, &sec); + cbm_node_t fn = {.project = "p", + .label = "Function", + .name = "plainFunction", + .qualified_name = "p.main.plainFunction", + .file_path = "main.c"}; + cbm_store_upsert_node(s, &fn); +} + +static cbm_store_t *setup_prose_store(void) { + cbm_store_t *s = cbm_store_open_memory(); + seed_prose_nodes(s); + return s; +} + +TEST(store_fts_rebuild_indexes_docstring_as_body_issue518) { + cbm_store_t *s = setup_prose_store(); + ASSERT_EQ(cbm_store_fts_rebuild(s, NULL, 0), CBM_STORE_OK); + + /* The prose is in `body` — the column BM25 weights at 0.3. */ + ASSERT_EQ(fts_match_count(s, "body:ephemeral"), 1); + ASSERT_EQ(fts_match_count(s, "body:workstation"), 1); + /* ...and specifically NOT smeared into an identifier column. */ + ASSERT_EQ(fts_match_count(s, "name:ephemeral"), 0); + /* A node with no docstring contributes no body tokens. */ + ASSERT_EQ(fts_match_count(s, "body:plainFunction"), 0); + /* Identifier indexing is untouched. */ + ASSERT_EQ(fts_match_count(s, "name:plainFunction"), 1); + + cbm_store_close(s); + PASS(); +} + +TEST(store_fts_rebuild_survives_malformed_properties_json) { + /* Pre-fix databases contain rows whose properties JSON does not parse. + * json_extract() RAISES on those, so an unguarded backfill would abort + * outright and leave the whole index empty — the same trap that reverted + * the is_entry_point expression index. */ + cbm_store_t *s = setup_prose_store(); + cbm_node_t broken = {.project = "p", + .label = "Function", + .name = "brokenProps", + .qualified_name = "p.main.brokenProps", + .file_path = "main.c", + .properties_json = "{\"docstring\":\"unterminated"}; + ASSERT_TRUE(cbm_store_upsert_node(s, &broken) > 0); + + ASSERT_EQ(cbm_store_fts_rebuild(s, NULL, 0), CBM_STORE_OK); + ASSERT_EQ(fts_match_count(s, "name:brokenProps"), 1); + ASSERT_EQ(fts_match_count(s, "body:ephemeral"), 1); + + cbm_store_close(s); + PASS(); +} + +TEST(store_fts_rebuild_tolerates_legacy_four_column_table) { + /* CBM_INDEX_FORMAT_VERSION deliberately does NOT move for the body column, + * so real users open databases whose nodes_fts predates it. Reproduce that + * exactly: lay down the four-column table first, then let the current + * store open the file — CREATE VIRTUAL TABLE IF NOT EXISTS leaves it be. */ + char *td = th_mktempdir("cbm_fts_legacy"); + ASSERT_NOT_NULL(td); + char path[512]; + snprintf(path, sizeof(path), "%s/legacy.db", td); + + sqlite3 *raw = NULL; + ASSERT_EQ(sqlite3_open(path, &raw), SQLITE_OK); + ASSERT_EQ(sqlite3_exec(raw, + "CREATE VIRTUAL TABLE nodes_fts USING fts5(" + " name, qualified_name, label, file_path," + " content='', tokenize='unicode61 remove_diacritics 2');", + NULL, NULL, NULL), + SQLITE_OK); + sqlite3_close(raw); + + cbm_store_t *s = cbm_store_open_path(path); + ASSERT_NOT_NULL(s); /* a legacy database still OPENS */ + seed_prose_nodes(s); + + /* ...and still backfills, degrading to the four-column write. */ + ASSERT_EQ(cbm_store_fts_rebuild(s, NULL, 0), CBM_STORE_OK); + ASSERT_EQ(fts_match_count(s, "name:plainFunction"), 1); + ASSERT_EQ(fts_match_count(s, "qualified_name:Installation"), 1); + /* No prose, exactly as promised — the words are simply not in the index. */ + ASSERT_EQ(fts_match_count(s, "ephemeral"), 0); + + /* The ranked query passes FIVE column weights. On a four-column table the + * fifth is never consulted (FTS5 reads a weight only for a column an + * instance landed in), so the SAME expression the search uses must run + * here without error rather than needing a second, forked query. */ + sqlite3_stmt *ranked = NULL; + ASSERT_EQ(sqlite3_prepare_v2(cbm_store_get_db(s), + "SELECT rowid, bm25(nodes_fts, 1.0, 1.0, 1.0, 1.0, 0.3) AS r" + " FROM nodes_fts WHERE nodes_fts MATCH ?1 ORDER BY r LIMIT 10", + -1, &ranked, NULL), + SQLITE_OK); + sqlite3_bind_text(ranked, 1, "plainFunction", -1, SQLITE_TRANSIENT); + ASSERT_EQ(sqlite3_step(ranked), SQLITE_ROW); + ASSERT_EQ(sqlite3_step(ranked), SQLITE_DONE); /* stepped to completion: no error */ + sqlite3_finalize(ranked); + + cbm_store_close(s); + th_rmtree(td); + PASS(); +} + +TEST(store_fts_rebuild_incremental_adds_only_nodes_above_watermark) { + cbm_store_t *s = setup_prose_store(); + ASSERT_EQ(cbm_store_fts_rebuild(s, NULL, 0), CBM_STORE_OK); + + sqlite3_stmt *st = NULL; + ASSERT_EQ(sqlite3_prepare_v2(cbm_store_get_db(s), "SELECT COALESCE(MAX(id),0) FROM nodes", -1, + &st, NULL), + SQLITE_OK); + ASSERT_EQ(sqlite3_step(st), SQLITE_ROW); + int64_t watermark = sqlite3_column_int64(st, 0); + sqlite3_finalize(st); + + cbm_node_t added = {.project = "p", + .label = "Section", + .name = "Upgrading", + .qualified_name = "p.README.Upgrading", + .file_path = "README.md", + .properties_json = "{\"docstring\":\"migrates the retention ledger\"}"}; + ASSERT_TRUE(cbm_store_upsert_node(s, &added) > 0); + + ASSERT_EQ(cbm_store_fts_rebuild(s, "p", watermark), CBM_STORE_OK); + ASSERT_EQ(fts_match_count(s, "body:retention"), 1); + /* Pre-existing rows are not duplicated by the incremental pass. */ + ASSERT_EQ(fts_match_count(s, "body:ephemeral"), 1); + ASSERT_EQ(fts_match_count(s, "name:plainFunction"), 1); + + cbm_store_close(s); + PASS(); +} + SUITE(store_search) { RUN_TEST(store_search_by_label); RUN_TEST(store_search_by_name_pattern); @@ -1700,4 +1867,9 @@ SUITE(store_search) { RUN_TEST(store_risk_label_all_levels); RUN_TEST(store_impact_summary_empty); RUN_TEST(store_find_nodes_rejects_null_store_without_ub); + /* #518/#519 — nodes_fts prose column */ + RUN_TEST(store_fts_rebuild_indexes_docstring_as_body_issue518); + RUN_TEST(store_fts_rebuild_survives_malformed_properties_json); + RUN_TEST(store_fts_rebuild_tolerates_legacy_four_column_table); + RUN_TEST(store_fts_rebuild_incremental_adds_only_nodes_above_watermark); } From b493b644cf4e77f4ff9bd5b30862b01ba7ed3198 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 28 Aug 2026 18:33:48 +0200 Subject: [PATCH 19/31] test(search): end-to-end guard from source prose to a findable body match The #518/#519 tests so far each cover one layer: extraction produces the docstring, the store indexes it into nodes_fts.body, the delta merge writes the same columns, and the query returns Section and Module. Nothing yet proved the layers connect. This indexes a real temporary repository -- a README.md whose section body runs well past the 500-byte cap, plus a META.yaml with a top-level description -- through the full pipeline, then asserts both prose words are matchable in the `body` column. It is also the guard on the size budget. build_def_props drops an oversized field ATOMICALLY, so a section body that did not fit the 2 KB properties buffer would vanish without a trace and every narrower test would still pass. Signed-off-by: Martin Vogel --- tests/test_pipeline.c | 66 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 3f874bd19..1eb10290d 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -12528,6 +12528,71 @@ TEST(pipeline_delta_patch_indexes_docstring_into_fts_body) { PASS(); } + +/* End-to-end for #518/#519: source → docstring → properties JSON → nodes_fts + * `body` → findable. Each layer has its own test; this one proves they connect. + * It is also the guard on the size budget: build_def_props drops an oversized + * field ATOMICALLY, so a 500-byte section body that did not fit the 2 KB + * properties buffer would vanish silently and every narrower test would still + * pass. */ +TEST(pipeline_markdown_and_config_prose_reaches_fts_body) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_prose_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char path[512]; + char dbpath[512]; + snprintf(dbpath, sizeof(dbpath), "%s/prose.db", tmp); + + /* A section body well past the 500-byte cap, so the truncating path — the + * one that produces the longest properties JSON — is what gets indexed. */ + snprintf(path, sizeof(path), "%s/README.md", tmp); + FILE *f = fopen(path, "w"); + ASSERT_NOT_NULL(f); + fprintf(f, "# Installation\n\nThe phlogiston bootstrap provisions a workstation.\n"); + for (int i = 0; i < 40; i++) { + fprintf(f, "Filler prose line %d that pads the section well past the cap.\n", i); + } + fclose(f); + + snprintf(path, sizeof(path), "%s/META.yaml", tmp); + f = fopen(path, "w"); + ASSERT_NOT_NULL(f); + fprintf(f, "name: widget\ndescription: Aggregates quicksilver telemetry per shard.\n"); + fclose(f); + + /* One code file so the run is a normal index rather than a docs-only edge. */ + snprintf(path, sizeof(path), "%s/main.go", tmp); + f = fopen(path, "w"); + ASSERT_NOT_NULL(f); + fprintf(f, "package main\n\nfunc main() {}\n"); + fclose(f); + + cbm_pipeline_t *p = cbm_pipeline_new(tmp, dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_pipeline_free(p); + + cbm_store_t *s = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(s); + sqlite3_stmt *st = NULL; + ASSERT_EQ(sqlite3_prepare_v2(cbm_store_get_db(s), + "SELECT COUNT(*) FROM nodes_fts WHERE nodes_fts MATCH ?1", -1, &st, + NULL), + SQLITE_OK); + const char *cases[] = {"body:phlogiston", "body:quicksilver"}; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + sqlite3_reset(st); + sqlite3_bind_text(st, 1, cases[i], -1, SQLITE_TRANSIENT); + ASSERT_EQ(sqlite3_step(st), SQLITE_ROW); + ASSERT_GT(sqlite3_column_int(st, 0), 0); + } + sqlite3_finalize(st); + cbm_store_close(s); + + th_rmtree(tmp); + PASS(); +} + SUITE(pipeline) { RUN_TEST(pipeline_lsp_surface_persisted_and_body_edit_invariant); /* Index lock */ @@ -12846,6 +12911,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_ensemble_routing_settings_targets); RUN_TEST(pipeline_ensemble_routing_unterminated_item_is_safe); RUN_TEST(pipeline_delta_patch_indexes_docstring_into_fts_body); + RUN_TEST(pipeline_markdown_and_config_prose_reaches_fts_body); } /* Focused semantic-manifest and publication contracts. Kept separate from the From dc23628c6f00a818466fe09f0f180413333aee80 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 28 Aug 2026 18:47:05 +0200 Subject: [PATCH 20/31] test(search): pin that nodes_fts.body is indexed raw, not camelCase-split Only `name` goes through cbm_camel_split; qualified_name, label, file_path and now `body` are indexed verbatim. `body` is natural-language prose, so splitting it would mangle the very text the column exists to make findable, and the unicode61 tokenizer already handles prose correctly. That invariant lived only in the shape of one format string, where the splitter is substituted into the name position alone. Nothing failed if a later edit wrapped the body expression too. cbm_camel_split("getUserById") yields "getUserById get User By Id", so a split body would additionally match the fragment "User" -- seeding a camelCase identifier into the prose and asserting the fragment does NOT match turns the invariant into a test. The mirror assertion, that `name` DOES match its fragment, pins the other half so the guard cannot be satisfied by simply indexing nothing. Verified binding: wrapping the body expression in cbm_camel_split makes body:User match 1 instead of 0 and the test fails. Signed-off-by: Martin Vogel --- tests/test_store_search.c | 45 ++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/tests/test_store_search.c b/tests/test_store_search.c index 814cb5fef..9110c420a 100644 --- a/tests/test_store_search.c +++ b/tests/test_store_search.c @@ -1008,10 +1008,8 @@ TEST(store_bfs_reachability_is_not_trail_capped) { cbm_store_t *s = cbm_store_open_memory(); cbm_store_upsert_project(s, "test", "/tmp/test"); - cbm_node_t root = {.project = "test", - .label = "Function", - .name = "Root", - .qualified_name = "test.Root"}; + cbm_node_t root = { + .project = "test", .label = "Function", .name = "Root", .qualified_name = "test.Root"}; int64_t root_id = cbm_store_upsert_node(s, &root); enum { NODE_COUNT = 4200 }; @@ -1083,18 +1081,12 @@ TEST(store_bfs_trail_preserves_deeper_match_under_hub_budget) { cbm_store_t *s = cbm_store_open_memory(); cbm_store_upsert_project(s, "test", "/tmp/test"); - cbm_node_t root = {.project = "test", - .label = "Function", - .name = "root", - .qualified_name = "test.root"}; - cbm_node_t branch = {.project = "test", - .label = "Function", - .name = "branch", - .qualified_name = "test.branch"}; - cbm_node_t target = {.project = "test", - .label = "Function", - .name = "target", - .qualified_name = "test.target"}; + cbm_node_t root = { + .project = "test", .label = "Function", .name = "root", .qualified_name = "test.root"}; + cbm_node_t branch = { + .project = "test", .label = "Function", .name = "branch", .qualified_name = "test.branch"}; + cbm_node_t target = { + .project = "test", .label = "Function", .name = "target", .qualified_name = "test.target"}; int64_t root_id = cbm_store_upsert_node(s, &root); int64_t branch_id = cbm_store_upsert_node(s, &branch); int64_t target_id = cbm_store_upsert_node(s, &target); @@ -1112,10 +1104,8 @@ TEST(store_bfs_trail_preserves_deeper_match_under_hub_budget) { for (int i = 0; i < LEAF_COUNT; i++) { char name[32]; snprintf(name, sizeof(name), "leaf-%d", i); - cbm_node_t leaf = {.project = "test", - .label = "Function", - .name = name, - .qualified_name = name}; + cbm_node_t leaf = { + .project = "test", .label = "Function", .name = name, .qualified_name = name}; int64_t leaf_id = cbm_store_upsert_node(s, &leaf); edge.source_id = root_id; edge.target_id = leaf_id; @@ -1658,8 +1648,8 @@ static void seed_prose_nodes(cbm_store_t *s) { .name = "Installation", .qualified_name = "p.README.Installation", .file_path = "README.md", - .properties_json = - "{\"docstring\":\"provisions an ephemeral workstation runner\"}"}; + .properties_json = "{\"docstring\":\"provisions an ephemeral " + "workstation runner via getUserById\"}"}; cbm_store_upsert_node(s, &sec); cbm_node_t fn = {.project = "p", .label = "Function", @@ -1689,6 +1679,17 @@ TEST(store_fts_rebuild_indexes_docstring_as_body_issue518) { /* Identifier indexing is untouched. */ ASSERT_EQ(fts_match_count(s, "name:plainFunction"), 1); + /* `body` is PROSE and must be indexed RAW — only `name` gets the camelCase + * splitter. cbm_camel_split("getUserById") yields "getUserById get User By + * Id", so a split body would additionally match the fragment "User". + * Asserting the fragment does NOT match is what makes this test fail if + * anyone ever wraps the body expression in cbm_camel_split(). */ + ASSERT_EQ(fts_match_count(s, "body:getUserById"), 1); + ASSERT_EQ(fts_match_count(s, "body:User"), 0); + /* ...and the mirror image: `name` IS split, so its fragment DOES match. + * Together these pin both halves of the invariant. */ + ASSERT_EQ(fts_match_count(s, "name:plain"), 1); + cbm_store_close(s); PASS(); } From 31bdd4d682eb9812b1130f4a1b55b3725feed834 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 28 Aug 2026 05:59:36 +0200 Subject: [PATCH 21/31] fix(mcp): bound search code scans Signed-off-by: Martin Vogel --- src/mcp/mcp.c | 346 +++++++++++++++++++++++++------- src/mcp/mcp_internal.h | 3 + tests/test_mcp.c | 426 ++++++++++++++++++++++++++++++++++++++-- tests/test_subprocess.c | 140 +++++++++++++ 4 files changed, 824 insertions(+), 91 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index a6dd1f868..4707f61ca 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -124,6 +124,7 @@ enum { #define MCP_MAX_MESSAGE_SIZE ((size_t)10U * 1024U * 1024U) #define MCP_MAX_HEADER_SIZE ((size_t)8U * 1024U) #define MCP_SEARCH_OUTPUT_MAX ((size_t)64U * 1024U * 1024U) +#define MCP_SEARCH_SCAN_TIMEOUT_MS ((uint64_t)30000U) /* ── Helpers ────────────────────────────────────────────────────── */ @@ -1605,6 +1606,9 @@ struct cbm_mcp_server { void *auto_index_count_test_context; #endif size_t search_output_limit_override; + const char *search_scan_command_override; + uint64_t search_scan_timeout_override_ms; + bool search_scan_timeout_override_set; cbm_thread_t autoindex_tid; bool autoindex_active; /* true if auto-index thread was started */ @@ -1920,6 +1924,20 @@ void cbm_mcp_server_set_search_output_limit_for_test(cbm_mcp_server_t *srv, size } } +void cbm_mcp_server_set_search_scan_command_for_test(cbm_mcp_server_t *srv, const char *command) { + if (srv) { + srv->search_scan_command_override = command; + } +} + +void cbm_mcp_server_set_search_scan_timeout_for_test(cbm_mcp_server_t *srv, uint64_t timeout_ms, + bool override_set) { + if (srv) { + srv->search_scan_timeout_override_ms = timeout_ms; + srv->search_scan_timeout_override_set = override_set; + } +} + /* ── Cache dir + project DB path helpers ───────────────────────── */ /* Returns the cache directory. Writes to buf, returns buf for convenience. */ @@ -9054,11 +9072,16 @@ void cbm_search_code_build_grep_cmd(char *cmd, size_t cmd_sz, bool use_regex, bo /* -0: read NUL-separated paths from the filelist so paths containing * spaces stay one argument (issue #687). Pairs with the NUL separator * written by write_scoped_filelist. */ - snprintf(cmd, cmd_sz, "xargs -0 grep -Hn %s --include='%s' -f '%s' < '%s' 2>/dev/null", + snprintf(cmd, cmd_sz, + "xargs -0 sh -c 'grep -Hn -d skip %s --include=\"%s\" -f \"%s\" \"$@\"; " + "status=$?; [ \"$status\" -eq 0 ] || [ \"$status\" -eq 1 ]' sh < '%s' " + "2>/dev/null", flag, file_pattern, tmpfile, filelist); } else { - snprintf(cmd, cmd_sz, "xargs -0 grep -Hn %s -f '%s' < '%s' 2>/dev/null", flag, tmpfile, - filelist); + snprintf(cmd, cmd_sz, + "xargs -0 sh -c 'grep -Hn -d skip %s -f \"%s\" \"$@\"; status=$?; " + "[ \"$status\" -eq 0 ] || [ \"$status\" -eq 1 ]' sh < '%s' 2>/dev/null", + flag, tmpfile, filelist); } } else { if (file_pattern) { @@ -9659,13 +9682,18 @@ static bool write_scoped_filelist(cbm_mcp_server_t *srv, const char *project, co } char **indexed_files = NULL; int indexed_count = 0; - if (cbm_store_list_files(pre_store, project, &indexed_files, &indexed_count) != CBM_STORE_OK || - indexed_count == 0) { + int list_rc = cbm_store_list_files(pre_store, project, &indexed_files, &indexed_count); + if (list_rc != CBM_STORE_OK || indexed_count == 0) { + for (int fi = 0; fi < indexed_count; fi++) { + free(indexed_files[fi]); + } + free(indexed_files); return false; } bool ok = false; int written = 0; if (fl) { + ok = true; for (int fi = 0; fi < indexed_count; fi++) { /* A source path never legitimately contains a newline or carriage * return. Those bytes are exactly the record separator on the @@ -9683,6 +9711,31 @@ static bool write_scoped_filelist(cbm_mcp_server_t *srv, const char *project, co continue; } } + size_t root_len = strlen(root_path); + size_t file_len = strlen(indexed_files[fi]); + if (root_len > SIZE_MAX - file_len - 2) { + continue; + } + size_t scan_path_len = root_len + 1 + file_len; + char *scan_path = malloc(scan_path_len + 1); + if (!scan_path) { + ok = false; + break; + } + memcpy(scan_path, root_path, root_len); + scan_path[root_len] = '/'; + memcpy(scan_path + root_len + 1, indexed_files[fi], file_len + 1); + + /* Incremental stores can retain structural directory nodes and + * briefly stale deleted-file paths. Neither is a content-scan + * operand. Filter them before spawning so an expected stale entry + * cannot turn otherwise valid matches into grep status 2. This + * deliberately does not follow symlinks/reparse points. */ + cbm_path_info_t path_info; + if (cbm_path_info_utf8(scan_path, &path_info) != 0 || !path_info.is_regular) { + free(scan_path); + continue; + } /* Write "/" piece-by-piece (no fixed-size buffer, so an * arbitrarily long absolute path cannot overflow). Forward slash join * so xargs doesn't treat Windows backslashes as escapes; binary mode @@ -9691,9 +9744,8 @@ static bool write_scoped_filelist(cbm_mcp_server_t *srv, const char *project, co * newline separator would split plain xargs on the space). * - Windows: newline, consumed by PowerShell `Get-Content | * Select-String -LiteralPath` (NUL bytes break Get-Content). */ - (void)fwrite(root_path, 1, strlen(root_path), fl); - (void)fputc('/', fl); - (void)fwrite(indexed_files[fi], 1, strlen(indexed_files[fi]), fl); + (void)fwrite(scan_path, 1, scan_path_len, fl); + free(scan_path); #ifdef _WIN32 (void)fputc('\n', fl); #else @@ -9703,7 +9755,6 @@ static bool write_scoped_filelist(cbm_mcp_server_t *srv, const char *project, co } /* The stream stays open — the caller owns it and closes it (flushing * these records to disk) before the grep subprocess reads the list. */ - ok = true; } for (int fi = 0; fi < indexed_count; fi++) { free(indexed_files[fi]); @@ -9807,6 +9858,16 @@ typedef struct { FILE *filelist; /* held open for write_scoped_filelist; closed by the caller */ } search_scratch_t; +typedef enum { + MCP_SCAN_SUCCESS = 0, + MCP_SCAN_COMMAND_FAILURE, + MCP_SCAN_CONTAINED_COMMAND_FAILURE, + MCP_SCAN_OUTPUT_LIMIT, + MCP_SCAN_DEADLINE, + MCP_SCAN_CANCELLED, + MCP_SCAN_SUPERVISION_FAILURE, +} mcp_scan_cause_t; + /* Create /-XXXXXX exclusively and return a stream on the * descriptor. On failure `path_out` is emptied so cleanup skips it. */ static FILE *search_scratch_file(const char *dir, const char *basename, char *path_out, @@ -9903,17 +9964,44 @@ static bool compile_path_filter(const char *filter, cbm_regex_t *re) { return cbm_regcomp(re, filter, CBM_REG_EXTENDED | CBM_REG_NOSUB) == CBM_REG_OK; } -static int mcp_run_shell_command_cancellable_bounded(cbm_mcp_server_t *srv, const char *command, - char output_path[CBM_SZ_2K], - size_t output_limit, - bool *output_limit_exceeded, - cbm_proc_result_t *result_out); +static mcp_scan_cause_t mcp_run_shell_command_cancellable_bounded( + cbm_mcp_server_t *srv, const char *command, char output_path[CBM_SZ_2K], size_t output_limit, + uint64_t deadline_ms, bool deadline_enabled, bool deadline_latched, bool exit_one_is_no_match, + cbm_proc_result_t *result_out); + +static char *search_code_timeout_result(void) { + static const char message[] = "search_code scan exceeded its execution deadline"; + static const char fallback[] = + "{\"content\":[{\"type\":\"text\",\"text\":\"search_code scan exceeded its execution " + "deadline\"}],\"structuredContent\":{\"code\":\"request_timeout\",\"message\":\"search_" + "code " + "scan exceeded its execution deadline\"},\"isError\":true}"; + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + if (!doc) { + return heap_strdup(fallback); + } + yyjson_mut_val *root = yyjson_mut_obj(doc); + yyjson_mut_doc_set_root(doc, root); + yyjson_mut_val *content = yyjson_mut_arr(doc); + yyjson_mut_val *item = yyjson_mut_obj(doc); + yyjson_mut_obj_add_str(doc, item, "type", "text"); + yyjson_mut_obj_add_str(doc, item, "text", message); + yyjson_mut_arr_add_val(content, item); + yyjson_mut_obj_add_val(doc, root, "content", content); + yyjson_mut_val *structured = yyjson_mut_obj(doc); + yyjson_mut_obj_add_str(doc, structured, "code", "request_timeout"); + yyjson_mut_obj_add_str(doc, structured, "message", message); + yyjson_mut_obj_add_val(doc, root, "structuredContent", structured); + yyjson_mut_obj_add_bool(doc, root, "isError", true); + char *result = yy_doc_to_str(doc); + yyjson_mut_doc_free(doc); + return result ? result : heap_strdup(fallback); +} -#ifdef _WIN32 static char *search_code_scan_error(search_scratch_t *scratch, const char *output_path, bool has_path_filter, cbm_regex_t *path_regex, char *root_path, char *pattern, char *project, char *file_pattern, - const char *message) { + mcp_scan_cause_t cause, const char *message) { if (output_path && output_path[0]) { (void)cbm_unlink(output_path); } @@ -9925,9 +10013,11 @@ static char *search_code_scan_error(search_scratch_t *scratch, const char *outpu free(pattern); free(project); free(file_pattern); + if (cause == MCP_SCAN_DEADLINE) { + return search_code_timeout_result(); + } return cbm_mcp_text_result(message, true); } -#endif static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) { char *pattern = cbm_mcp_get_string_arg(args, "pattern"); @@ -10057,8 +10147,18 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) { } /* ── Phase 1: Grep scan ──────────────────────────────────── */ + uint64_t scan_budget_ms = srv->search_scan_timeout_override_set + ? srv->search_scan_timeout_override_ms + : MCP_SEARCH_SCAN_TIMEOUT_MS; + uint64_t scan_started_ms = cbm_now_ms(); + uint64_t scan_deadline_ms = UINT64_MAX - scan_started_ms < scan_budget_ms + ? UINT64_MAX + : scan_started_ms + scan_budget_ms; + bool scan_deadline_latched = false; search_scratch_t scratch; if (!search_scratch_open(&scratch, pattern)) { + bool scan_cancelled = mcp_request_cancelled(srv); + bool scan_timed_out = cbm_now_ms() >= scan_deadline_ms; char errmsg[CBM_SZ_256]; snprintf(errmsg, sizeof(errmsg), "search failed: cannot create temp file (%s)", strerror(errno)); @@ -10066,8 +10166,16 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) { free(pattern); free(project); free(file_pattern); + if (scan_cancelled) { + return cbm_mcp_text_result("search_code cancelled for this request", true); + } + if (scan_timed_out) { + return search_code_timeout_result(); + } return cbm_mcp_text_result(errmsg, true); } + scan_deadline_latched = cbm_now_ms() >= scan_deadline_ms; + bool scan_cancellation_latched = mcp_request_cancelled(srv); const char *tmpfile = scratch.pattern_path; const char *filelist = scratch.filelist_path; @@ -10085,13 +10193,17 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) { int scoped_written = 0; uint64_t scope_t0 = metrics.include_phase_timings ? cbm_now_ms() : 0; - scoped = write_scoped_filelist(srv, project, root_path, scratch.filelist, has_path_filter, - has_path_filter ? &path_regex : NULL, &scoped_written); + if (!scan_cancellation_latched && !scan_deadline_latched) { + scoped = write_scoped_filelist(srv, project, root_path, scratch.filelist, has_path_filter, + has_path_filter ? &path_regex : NULL, &scoped_written); + } /* Close before grep runs: this is what flushes the records the helper wrote * through the descriptor. Clearing the field hands ownership to * search_scratch_close, which still unlinks the file itself. */ (void)fclose(scratch.filelist); scratch.filelist = NULL; + scan_cancellation_latched = scan_cancellation_latched || mcp_request_cancelled(srv); + scan_deadline_latched = scan_deadline_latched || cbm_now_ms() >= scan_deadline_ms; if (metrics.include_phase_timings) { metrics.scope_ms = cbm_now_ms() - scope_t0; } @@ -10100,7 +10212,7 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) { int gm_count = 0; grep_match_t *gm = NULL; uint64_t scan_t0 = metrics.include_phase_timings ? cbm_now_ms() : 0; - if (scoped && scoped_written == 0) { + if (scoped && scoped_written == 0 && !scan_cancellation_latched && !scan_deadline_latched) { /* The path_filter excluded every indexed file — nothing to scan. * Skip the grep subprocess: xargs on an empty filelist is * platform-dependent (GNU execs grep once with no operands, BSD @@ -10112,61 +10224,57 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) { cbm_search_code_build_grep_cmd(cmd, sizeof(cmd), use_regex, scoped, file_pattern, tmpfile, filelist, root_path); -#ifdef _WIN32 char output_path[CBM_SZ_2K] = {0}; cbm_proc_result_t scan_result = {0}; - bool scan_output_exceeded = false; size_t scan_output_limit = srv->search_output_limit_override ? srv->search_output_limit_override : MCP_SEARCH_OUTPUT_MAX; - int scan_run = mcp_run_shell_command_cancellable_bounded( - srv, cmd, output_path, scan_output_limit, &scan_output_exceeded, &scan_result); - if (scan_output_exceeded) { + const char *scan_command = + srv->search_scan_command_override ? srv->search_scan_command_override : cmd; + mcp_scan_cause_t scan_cause = mcp_run_shell_command_cancellable_bounded( + srv, scan_command, output_path, scan_output_limit, scan_deadline_ms, true, + scan_deadline_latched, !scoped, &scan_result); + if (scan_cause == MCP_SCAN_SUPERVISION_FAILURE) { + return search_code_scan_error(&scratch, output_path, has_path_filter, &path_regex, + root_path, pattern, project, file_pattern, scan_cause, + "search failed: process supervision could not quiesce"); + } + if (scan_cause == MCP_SCAN_CANCELLED) { + return search_code_scan_error(&scratch, output_path, has_path_filter, &path_regex, + root_path, pattern, project, file_pattern, scan_cause, + "search_code cancelled for this request"); + } + if (scan_cause == MCP_SCAN_DEADLINE) { + return search_code_scan_error(&scratch, output_path, has_path_filter, &path_regex, + root_path, pattern, project, file_pattern, scan_cause, + NULL); + } + if (scan_cause == MCP_SCAN_OUTPUT_LIMIT) { char message[CBM_SZ_128]; snprintf(message, sizeof(message), "search failed: output exceeded the %zu-byte safety limit", scan_output_limit); return search_code_scan_error(&scratch, output_path, has_path_filter, &path_regex, - root_path, pattern, project, file_pattern, message); + root_path, pattern, project, file_pattern, scan_cause, + message); } - bool scan_cancelled = scan_result.cancellation_requested || mcp_request_cancelled(srv); - if (scan_cancelled) { - return search_code_scan_error(&scratch, output_path, has_path_filter, &path_regex, - root_path, pattern, project, file_pattern, - "search_code cancelled for this request"); - } - if (scan_run != 0) { + if (scan_cause == MCP_SCAN_COMMAND_FAILURE || + scan_cause == MCP_SCAN_CONTAINED_COMMAND_FAILURE) { return search_code_scan_error( &scratch, output_path, has_path_filter, &path_regex, root_path, pattern, project, - file_pattern, "search failed: the contained command could not complete"); + file_pattern, scan_cause, + "search failed: the contained command could not complete"); } FILE *fp = cbm_fopen(output_path, "rb"); if (!fp) { return search_code_scan_error(&scratch, output_path, has_path_filter, &path_regex, root_path, pattern, project, file_pattern, + MCP_SCAN_COMMAND_FAILURE, "search failed: contained output could not be read"); } gm = collect_grep_matches(fp, root_path, strlen(root_path), has_path_filter, &path_regex, grep_limit, &gm_count); (void)fclose(fp); (void)cbm_unlink(output_path); -#else - FILE *fp = cbm_popen(cmd, "r"); - if (!fp) { - search_scratch_close(&scratch); - if (has_path_filter) { - cbm_regfree(&path_regex); - } - free(root_path); - free(pattern); - free(project); - free(file_pattern); - return cbm_mcp_text_result("search failed", true); - } - - gm = collect_grep_matches(fp, root_path, strlen(root_path), has_path_filter, &path_regex, - grep_limit, &gm_count); - cbm_pclose(fp); -#endif /* Both scratch files and the private directory go here — unlike the old * code, the file list is removed even when the scan was not scoped. */ search_scratch_close(&scratch); @@ -10327,29 +10435,74 @@ static bool mcp_resolve_windows_cmd(char out[CBM_SZ_4K]) { } #endif -static int mcp_run_shell_command_cancellable_bounded(cbm_mcp_server_t *srv, const char *command, - char output_path[CBM_SZ_2K], - size_t output_limit, - bool *output_limit_exceeded, - cbm_proc_result_t *result_out) { - if (output_limit_exceeded) { - *output_limit_exceeded = false; +static mcp_scan_cause_t mcp_scan_pre_spawn_cause(cbm_mcp_server_t *srv, const char *output_path, + size_t output_limit, uint64_t deadline_ms, + bool deadline_enabled, bool *cancellation_latched, + bool *deadline_latched, + bool *output_limit_latched) { + *cancellation_latched = *cancellation_latched || mcp_request_cancelled(srv); + *deadline_latched = *deadline_latched || (deadline_enabled && cbm_now_ms() >= deadline_ms); + if (!*output_limit_latched && output_limit > 0 && output_path && output_path[0]) { + int64_t output_size = cbm_file_size(output_path); + *output_limit_latched = output_size > 0 && (uint64_t)output_size > output_limit; } - if (!srv || !command || !output_path || !result_out || - (output_limit > 0 && !output_limit_exceeded) || !mcp_command_output_path(output_path)) { - return -1; + if (*cancellation_latched) { + return MCP_SCAN_CANCELLED; + } + if (*deadline_latched) { + return MCP_SCAN_DEADLINE; + } + if (*output_limit_latched) { + return MCP_SCAN_OUTPUT_LIMIT; + } + return MCP_SCAN_SUCCESS; +} + +static mcp_scan_cause_t mcp_run_shell_command_cancellable_bounded( + cbm_mcp_server_t *srv, const char *command, char output_path[CBM_SZ_2K], size_t output_limit, + uint64_t deadline_ms, bool deadline_enabled, bool deadline_latched, bool exit_one_is_no_match, + cbm_proc_result_t *result_out) { + if (!srv || !command || !output_path || !result_out) { + return MCP_SCAN_COMMAND_FAILURE; + } + memset(result_out, 0, sizeof(*result_out)); + bool cancellation_latched = false; + bool output_limit_latched = false; + mcp_scan_cause_t pre_spawn_cause = + mcp_scan_pre_spawn_cause(srv, NULL, output_limit, deadline_ms, deadline_enabled, + &cancellation_latched, &deadline_latched, &output_limit_latched); + if (pre_spawn_cause != MCP_SCAN_SUCCESS) { + result_out->tree_quiesced = true; /* no child was spawned */ + return pre_spawn_cause; + } + if (!mcp_command_output_path(output_path)) { + pre_spawn_cause = mcp_scan_pre_spawn_cause(srv, NULL, output_limit, deadline_ms, + deadline_enabled, &cancellation_latched, + &deadline_latched, &output_limit_latched); + result_out->tree_quiesced = true; + return pre_spawn_cause != MCP_SCAN_SUCCESS ? pre_spawn_cause : MCP_SCAN_COMMAND_FAILURE; } /* Internal test seam: rejecting after output allocation exercises the same * cleanup contract as a contained process-tree failure. */ - if (srv->command_test_hook && !srv->command_test_hook(srv->command_test_context, command)) { - return -1; + bool command_rejected = + srv->command_test_hook && !srv->command_test_hook(srv->command_test_context, command); + pre_spawn_cause = + mcp_scan_pre_spawn_cause(srv, output_path, output_limit, deadline_ms, deadline_enabled, + &cancellation_latched, &deadline_latched, &output_limit_latched); + if (pre_spawn_cause != MCP_SCAN_SUCCESS || command_rejected) { + result_out->tree_quiesced = true; /* no child was spawned */ + return pre_spawn_cause != MCP_SCAN_SUCCESS ? pre_spawn_cause : MCP_SCAN_COMMAND_FAILURE; } #ifdef _WIN32 char shell[CBM_SZ_4K]; if (!mcp_resolve_windows_cmd(shell)) { + pre_spawn_cause = mcp_scan_pre_spawn_cause(srv, output_path, output_limit, deadline_ms, + deadline_enabled, &cancellation_latched, + &deadline_latched, &output_limit_latched); (void)cbm_unlink(output_path); output_path[0] = '\0'; - return -1; + result_out->tree_quiesced = true; + return pre_spawn_cause != MCP_SCAN_SUCCESS ? pre_spawn_cause : MCP_SCAN_COMMAND_FAILURE; } #else const char *shell = "/bin/sh"; @@ -10367,27 +10520,48 @@ static int mcp_run_shell_command_cancellable_bounded(cbm_mcp_server_t *srv, cons .cancel_grace_ms = CBM_SUBPROCESS_DEFAULT_CANCEL_GRACE_MS, .delete_log_on_exit = false, }; + pre_spawn_cause = + mcp_scan_pre_spawn_cause(srv, output_path, output_limit, deadline_ms, deadline_enabled, + &cancellation_latched, &deadline_latched, &output_limit_latched); + if (pre_spawn_cause != MCP_SCAN_SUCCESS) { + result_out->tree_quiesced = true; + return pre_spawn_cause; + } cbm_subprocess_t *process = NULL; if (cbm_subprocess_spawn(&options, &process) != 0) { + pre_spawn_cause = mcp_scan_pre_spawn_cause(srv, output_path, output_limit, deadline_ms, + deadline_enabled, &cancellation_latched, + &deadline_latched, &output_limit_latched); (void)cbm_unlink(output_path); output_path[0] = '\0'; - return -1; + result_out->tree_quiesced = true; + return pre_spawn_cause != MCP_SCAN_SUCCESS ? pre_spawn_cause : MCP_SCAN_COMMAND_FAILURE; } cbm_proc_poll_t state; - bool limit_exceeded = false; for (;;) { if (mcp_request_cancelled(srv)) { + cancellation_latched = true; (void)cbm_subprocess_request_cancel(process); } - if (!limit_exceeded && output_limit > 0) { + if (deadline_enabled && cbm_now_ms() >= deadline_ms) { + deadline_latched = true; + (void)cbm_subprocess_request_cancel(process); + } + if (!output_limit_latched && output_limit > 0) { int64_t output_size = cbm_file_size(output_path); if (output_size > 0 && (uint64_t)output_size > output_limit) { - limit_exceeded = true; + output_limit_latched = true; (void)cbm_subprocess_request_cancel(process); } } state = cbm_subprocess_poll(process, result_out); + cancellation_latched = cancellation_latched || mcp_request_cancelled(srv); + deadline_latched = deadline_latched || (deadline_enabled && cbm_now_ms() >= deadline_ms); + if (!output_limit_latched && output_limit > 0) { + int64_t output_size = cbm_file_size(output_path); + output_limit_latched = output_size > 0 && (uint64_t)output_size > output_limit; + } if (state != CBM_PROC_POLL_RUNNING) { break; } @@ -10396,21 +10570,41 @@ static int mcp_run_shell_command_cancellable_bounded(cbm_mcp_server_t *srv, cons bool contained = state == CBM_PROC_POLL_TERMINAL && result_out->tree_quiesced && !result_out->supervision_failed; cbm_subprocess_destroy(process); - if (!limit_exceeded && output_limit > 0) { + if (!output_limit_latched && output_limit > 0) { int64_t final_size = cbm_file_size(output_path); - limit_exceeded = final_size > 0 && (uint64_t)final_size > output_limit; + output_limit_latched = final_size > 0 && (uint64_t)final_size > output_limit; } - if (output_limit_exceeded) { - *output_limit_exceeded = limit_exceeded; + if (!contained) { + return MCP_SCAN_SUPERVISION_FAILURE; } - return contained ? 0 : -1; + if (cancellation_latched) { + return MCP_SCAN_CANCELLED; + } + if (deadline_latched) { + return MCP_SCAN_DEADLINE; + } + if (output_limit_latched) { + return MCP_SCAN_OUTPUT_LIMIT; + } +#ifndef _WIN32 + if (exit_one_is_no_match && result_out->outcome == CBM_PROC_EXIT_NONZERO && + result_out->exit_code == 1) { + return MCP_SCAN_SUCCESS; /* ordinary direct grep no-match */ + } +#endif + return result_out->outcome == CBM_PROC_CLEAN ? MCP_SCAN_SUCCESS + : MCP_SCAN_CONTAINED_COMMAND_FAILURE; } static int mcp_run_shell_command_cancellable(cbm_mcp_server_t *srv, const char *command, char output_path[CBM_SZ_2K], cbm_proc_result_t *result_out) { - return mcp_run_shell_command_cancellable_bounded(srv, command, output_path, 0, NULL, - result_out); + mcp_scan_cause_t cause = mcp_run_shell_command_cancellable_bounded( + srv, command, output_path, 0, 0, false, false, false, result_out); + /* Legacy callers inspect result_out for cancellation/exit status. Preserve + * their original contract: any contained terminal tree is transport + * success; only spawn/rejection or failed supervision is a wrapper error. */ + return cause == MCP_SCAN_COMMAND_FAILURE || cause == MCP_SCAN_SUPERVISION_FAILURE ? -1 : 0; } /* Does `node`'s line range overlap any recorded hunk for `file`? Used to scope diff --git a/src/mcp/mcp_internal.h b/src/mcp/mcp_internal.h index 61b9383c9..8355bb390 100644 --- a/src/mcp/mcp_internal.h +++ b/src/mcp/mcp_internal.h @@ -23,6 +23,9 @@ void cbm_mcp_server_set_auto_index_count_test_hook(cbm_mcp_server_t *srv, cbm_mcp_auto_index_count_test_hook_fn hook, void *context); #endif +void cbm_mcp_server_set_search_scan_command_for_test(cbm_mcp_server_t *srv, const char *command); +void cbm_mcp_server_set_search_scan_timeout_for_test(cbm_mcp_server_t *srv, uint64_t timeout_ms, + bool override_set); /* Release only the constructor-created pristine in-memory store. Public * cbm_mcp_server_new(NULL) semantics remain unchanged; daemon sessions use diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 9a4e9a6f1..45104aecb 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -163,11 +163,14 @@ typedef struct { int merge_base_calls; } mcp_command_hook_probe_t; -#ifdef _WIN32 typedef struct { cbm_mcp_server_t *server; bool cancel_on_call; bool cancel_accepted; + bool reject; + const char *fill_output_directory; + uint64_t delay_ms; + bool output_filled; int calls; char command[CBM_SZ_4K]; } mcp_search_command_probe_t; @@ -176,7 +179,6 @@ typedef struct { char path[512]; char *saved_cache; } mcp_search_cache_t; -#endif static bool mcp_quarantine_hook_probe(void *context, const char *step) { mcp_quarantine_hook_probe_t *probe = context; @@ -203,7 +205,6 @@ static bool mcp_command_hook_probe(void *context, const char *command) { return true; } -#ifdef _WIN32 static bool mcp_search_command_hook_probe(void *context, const char *command) { mcp_search_command_probe_t *probe = context; if (!probe || !command) { @@ -211,10 +212,35 @@ static bool mcp_search_command_hook_probe(void *context, const char *command) { } probe->calls++; snprintf(probe->command, sizeof(probe->command), "%s", command); + if (probe->delay_ms > 0) { + cbm_usleep(probe->delay_ms * 1000U); + } if (probe->cancel_on_call && probe->server) { probe->cancel_accepted = cbm_mcp_server_cancel_active(probe->server); } - return true; + if (probe->fill_output_directory) { + static const char prefix[] = ".mcp-command-"; + cbm_dir_t *directory = cbm_opendir(probe->fill_output_directory); + cbm_dirent_t *entry; + while (directory && (entry = cbm_readdir(directory)) != NULL) { + if (strncmp(entry->name, prefix, sizeof(prefix) - SKIP_ONE) != 0) { + continue; + } + char path[CBM_SZ_2K]; + snprintf(path, sizeof(path), "%s/%s", probe->fill_output_directory, entry->name); + FILE *output = cbm_fopen(path, "ab"); + if (output) { + bool wrote = fputs("bounded-output-sentinel\n", output) >= 0; + bool closed = fclose(output) == 0; + probe->output_filled = wrote && closed; + } + break; + } + if (directory) { + cbm_closedir(directory); + } + } + return !probe->reject; } static bool mcp_search_cache_open(mcp_search_cache_t *cache, const char *prefix) { @@ -240,7 +266,6 @@ static bool mcp_search_cache_close(mcp_search_cache_t *cache) { cache->saved_cache = NULL; return th_rmtree(cache->path) == 0; } -#endif typedef struct { const char *name; @@ -5050,8 +5075,7 @@ TEST(search_code_windows_prefilter_precedes_content_scan) { #endif } -TEST(search_code_windows_cancel_cleans_supervised_scan) { -#ifdef _WIN32 +TEST(search_code_cancel_cleans_supervised_scan) { mcp_search_cache_t cache; ASSERT_TRUE(mcp_search_cache_open(&cache, "cbm-search-cancel")); @@ -5083,13 +5107,9 @@ TEST(search_code_windows_cancel_cleans_supervised_scan) { cleanup_prefilter_dir(tmp, src_path, vendor_path); ASSERT_TRUE(mcp_search_cache_close(&cache)); PASS(); -#else - SKIP_PLATFORM("supervised Select-String cancellation runs on Windows"); -#endif } -TEST(search_code_windows_output_limit_fails_closed_and_cleans_scan) { -#ifdef _WIN32 +TEST(search_code_output_limit_fails_closed_and_cleans_scan) { mcp_search_cache_t cache; ASSERT_TRUE(mcp_search_cache_open(&cache, "cbm-search-limit")); @@ -5123,9 +5143,375 @@ TEST(search_code_windows_output_limit_fails_closed_and_cleans_scan) { cleanup_prefilter_dir(tmp, src_path, vendor_path); ASSERT_TRUE(mcp_search_cache_close(&cache)); PASS(); +} + +TEST(search_code_scan_deadline_fails_closed_and_resets) { + mcp_search_cache_t cache; + ASSERT_TRUE(mcp_search_cache_open(&cache, "cbm-search-deadline")); + int scratch_before = mcp_count_directory_entries_with_prefix(cbm_tmpdir(), "cbm-search-"); + ASSERT_TRUE(scratch_before >= 0); + + char tmp[512], src_path[768], vendor_path[768]; + cbm_mcp_server_t *srv = setup_prefilter_server(tmp, sizeof(tmp), src_path, sizeof(src_path), + vendor_path, sizeof(vendor_path)); + ASSERT_NOT_NULL(srv); + cbm_mcp_server_set_search_scan_timeout_for_test(srv, 0, true); + + char *response = + cbm_mcp_handle_tool(srv, "search_code", + "{\"pattern\":\"HandleRequest\",\"project\":\"prefilter-search\"," + "\"file_pattern\":\"*.go\"}"); + ASSERT_NOT_NULL(response); + ASSERT_NOT_NULL(strstr(response, "\"isError\":true")); + ASSERT_NOT_NULL(strstr(response, "request_timeout")); + ASSERT_NOT_NULL(strstr(response, "execution deadline")); + ASSERT_NOT_NULL( + strstr(response, "\"text\":\"search_code scan exceeded its execution deadline\"")); + ASSERT_NOT_NULL(strstr( + response, "\"structuredContent\":{\"code\":\"request_timeout\",\"message\":\"search_code " + "scan exceeded its execution deadline\"}")); + ASSERT_NULL(strstr(response, "src/handler.go")); + ASSERT_NULL(strstr(response, "return nil")); + + char logs[640]; + snprintf(logs, sizeof(logs), "%s/logs", cache.path); + /* An immediate deadline can return before the log directory is created; + * both a missing directory (-1) and an empty one (0) prove no artifact. */ + ASSERT_TRUE(mcp_count_directory_entries_with_prefix(logs, ".mcp-command-") <= 0); + ASSERT_EQ(mcp_count_directory_entries_with_prefix(cbm_tmpdir(), "cbm-search-"), scratch_before); + free(response); + + cbm_mcp_server_set_search_scan_timeout_for_test(srv, 0, false); + response = cbm_mcp_handle_tool(srv, "search_code", + "{\"pattern\":\"HandleRequest\",\"project\":\"prefilter-" + "search\",\"file_pattern\":\"*.go\"}"); + ASSERT_NOT_NULL(response); + ASSERT_NULL(strstr(response, "\"isError\":true")); + ASSERT_NOT_NULL(strstr(response, "src/handler.go")); + free(response); + + cbm_mcp_server_free(srv); + cleanup_prefilter_dir(tmp, src_path, vendor_path); + ASSERT_TRUE(mcp_search_cache_close(&cache)); + PASS(); +} + +TEST(search_code_scan_deadline_override_is_per_server) { + char tmp_a[512], src_a[768], vendor_a[768]; + char tmp_b[512], src_b[768], vendor_b[768]; + cbm_mcp_server_t *server_a = setup_prefilter_server(tmp_a, sizeof(tmp_a), src_a, sizeof(src_a), + vendor_a, sizeof(vendor_a)); + cbm_mcp_server_t *server_b = setup_prefilter_server(tmp_b, sizeof(tmp_b), src_b, sizeof(src_b), + vendor_b, sizeof(vendor_b)); + ASSERT_NOT_NULL(server_a); + ASSERT_NOT_NULL(server_b); + cbm_mcp_server_set_search_scan_timeout_for_test(server_a, 0, true); + + char *timed_out = cbm_mcp_handle_tool(server_a, "search_code", + "{\"pattern\":\"HandleRequest\",\"project\":\"prefilter-" + "search\",\"file_pattern\":\"*.go\"}"); + char *normal = cbm_mcp_handle_tool(server_b, "search_code", + "{\"pattern\":\"HandleRequest\",\"project\":\"prefilter-" + "search\",\"file_pattern\":\"*.go\"}"); + ASSERT_NOT_NULL(timed_out); + ASSERT_NOT_NULL(normal); + ASSERT_NOT_NULL(strstr(timed_out, "request_timeout")); + ASSERT_NULL(strstr(normal, "\"isError\":true")); + ASSERT_NOT_NULL(strstr(normal, "src/handler.go")); + + free(timed_out); + free(normal); + cbm_mcp_server_free(server_a); + cbm_mcp_server_free(server_b); + cleanup_prefilter_dir(tmp_a, src_a, vendor_a); + cleanup_prefilter_dir(tmp_b, src_b, vendor_b); + PASS(); +} + +TEST(search_code_scan_setup_failures_respect_cause_precedence) { + mcp_search_cache_t cache; + ASSERT_TRUE(mcp_search_cache_open(&cache, "cbm-search-setup-precedence")); + + char cache_blocker[640]; + snprintf(cache_blocker, sizeof(cache_blocker), "%s/not-a-directory", cache.path); + FILE *blocker = cbm_fopen(cache_blocker, "wb"); + ASSERT_NOT_NULL(blocker); + ASSERT_EQ(fclose(blocker), 0); + + char tmp[512], src_path[768], vendor_path[768]; + cbm_mcp_server_t *srv = setup_prefilter_server(tmp, sizeof(tmp), src_path, sizeof(src_path), + vendor_path, sizeof(vendor_path)); + ASSERT_NOT_NULL(srv); + ASSERT_EQ(cbm_setenv("CBM_CACHE_DIR", cache_blocker, 1), 0); + cbm_mcp_server_set_search_scan_timeout_for_test(srv, 0, true); + + /* Keep an outer request scope active so cancellation is latched before the + * nested tool call starts. It must beat both the zero deadline and the + * deliberately broken command-output directory. */ + ASSERT_TRUE(cbm_mcp_server_request_scope_begin(srv)); + ASSERT_TRUE(cbm_mcp_server_cancel_active(srv)); + char *response = cbm_mcp_handle_tool( + srv, "search_code", "{\"pattern\":\"HandleRequest\",\"project\":\"prefilter-search\"}"); + bool cancellation_won = response && strstr(response, "cancelled") != NULL && + strstr(response, "request_timeout") == NULL && + strstr(response, "contained command") == NULL; + free(response); + cbm_mcp_server_request_scope_end(srv); + + /* With cancellation cleared, the same broken setup must not hide the + * already-latched deadline. */ + response = cbm_mcp_handle_tool( + srv, "search_code", "{\"pattern\":\"HandleRequest\",\"project\":\"prefilter-search\"}"); + bool deadline_won = response && strstr(response, "request_timeout") != NULL && + strstr(response, "contained command") == NULL; + free(response); + + ASSERT_EQ(cbm_setenv("CBM_CACHE_DIR", cache.path, 1), 0); + ASSERT_EQ(cbm_unlink(cache_blocker), 0); + cbm_mcp_server_free(srv); + cleanup_prefilter_dir(tmp, src_path, vendor_path); + ASSERT_TRUE(mcp_search_cache_close(&cache)); + ASSERT_TRUE(cancellation_won); + ASSERT_TRUE(deadline_won); + PASS(); +} + +TEST(search_code_scan_live_child_deadline_is_bounded_and_fails_closed) { + mcp_search_cache_t cache; + ASSERT_TRUE(mcp_search_cache_open(&cache, "cbm-search-live-deadline")); + int scratch_before = mcp_count_directory_entries_with_prefix(cbm_tmpdir(), "cbm-search-"); + ASSERT_TRUE(scratch_before >= 0); + + char tmp[512], src_path[768], vendor_path[768]; + cbm_mcp_server_t *srv = setup_prefilter_server(tmp, sizeof(tmp), src_path, sizeof(src_path), + vendor_path, sizeof(vendor_path)); + ASSERT_NOT_NULL(srv); +#ifdef _WIN32 + const char *slow_command = + "echo deadline-partial-output & powershell.exe -NoProfile -Command \"Start-Sleep -Seconds " + "6\""; #else - SKIP_PLATFORM("supervised Select-String output limit runs on Windows"); + const char *slow_command = "printf 'deadline-partial-output\\n'; trap '' TERM; sleep 6"; #endif + cbm_mcp_server_set_search_scan_command_for_test(srv, slow_command); + cbm_mcp_server_set_search_scan_timeout_for_test(srv, 100, true); + + uint64_t started = cbm_now_ms(); + char *response = cbm_mcp_handle_tool( + srv, "search_code", "{\"pattern\":\"HandleRequest\",\"project\":\"prefilter-search\"}"); + uint64_t elapsed = cbm_now_ms() - started; + bool bounded = elapsed < 3000U; + bool timed_out = response && strstr(response, "request_timeout") != NULL && + strstr(response, "\"isError\":true") != NULL; + bool partial_hidden = !response || strstr(response, "deadline-partial-output") == NULL; + + char logs[640]; + snprintf(logs, sizeof(logs), "%s/logs", cache.path); + int command_artifacts = mcp_count_directory_entries_with_prefix(logs, ".mcp-command-"); + int scratch_after = mcp_count_directory_entries_with_prefix(cbm_tmpdir(), "cbm-search-"); + + free(response); + cbm_mcp_server_set_search_scan_command_for_test(srv, NULL); + cbm_mcp_server_set_search_scan_timeout_for_test(srv, 0, false); + cbm_mcp_server_free(srv); + cleanup_prefilter_dir(tmp, src_path, vendor_path); + ASSERT_TRUE(mcp_search_cache_close(&cache)); + ASSERT_TRUE(bounded); + ASSERT_TRUE(timed_out); + ASSERT_TRUE(partial_hidden); + ASSERT_EQ(command_artifacts, 0); + ASSERT_EQ(scratch_after, scratch_before); + PASS(); +} + +TEST(search_code_scan_cancellation_precedes_zero_deadline) { + char tmp[512], src_path[768], vendor_path[768]; + cbm_mcp_server_t *srv = setup_prefilter_server(tmp, sizeof(tmp), src_path, sizeof(src_path), + vendor_path, sizeof(vendor_path)); + ASSERT_NOT_NULL(srv); + cbm_mcp_server_set_search_scan_timeout_for_test(srv, 0, true); + ASSERT_TRUE(cbm_mcp_server_request_scope_begin(srv)); + ASSERT_TRUE(cbm_mcp_server_cancel_active(srv)); + + char *response = cbm_mcp_handle_tool(srv, "search_code", + "{\"pattern\":\"HandleRequest\",\"project\":\"prefilter-" + "search\",\"file_pattern\":\"*.go\"}"); + ASSERT_NOT_NULL(response); + ASSERT_NOT_NULL(strstr(response, "cancelled")); + ASSERT_NULL(strstr(response, "request_timeout")); + ASSERT_NOT_NULL(strstr(response, "\"isError\":true")); + + free(response); + cbm_mcp_server_request_scope_end(srv); + cbm_mcp_server_free(srv); + cleanup_prefilter_dir(tmp, src_path, vendor_path); + PASS(); +} + +TEST(search_code_scan_deadline_precedes_output_limit) { + mcp_search_cache_t cache; + ASSERT_TRUE(mcp_search_cache_open(&cache, "cbm-search-precedence")); + char logs[640]; + snprintf(logs, sizeof(logs), "%s/logs", cache.path); + + char tmp[512], src_path[768], vendor_path[768]; + cbm_mcp_server_t *srv = setup_prefilter_server(tmp, sizeof(tmp), src_path, sizeof(src_path), + vendor_path, sizeof(vendor_path)); + ASSERT_NOT_NULL(srv); + mcp_search_command_probe_t probe = {.fill_output_directory = logs, .delay_ms = 1100}; + cbm_mcp_server_set_command_test_hook(srv, mcp_search_command_hook_probe, &probe); + cbm_mcp_server_set_search_scan_timeout_for_test(srv, 1000, true); + cbm_mcp_server_set_search_output_limit_for_test(srv, 1); + + char *response = cbm_mcp_handle_tool(srv, "search_code", + "{\"pattern\":\"HandleRequest\",\"project\":\"prefilter-" + "search\",\"file_pattern\":\"*.go\"}"); + ASSERT_NOT_NULL(response); + ASSERT_TRUE(probe.output_filled); + ASSERT_NOT_NULL(strstr(response, "request_timeout")); + ASSERT_NULL(strstr(response, "output exceeded")); + ASSERT_NOT_NULL(strstr(response, "\"isError\":true")); + ASSERT_EQ(mcp_count_directory_entries_with_prefix(logs, ".mcp-command-"), 0); + + free(response); + cbm_mcp_server_free(srv); + cleanup_prefilter_dir(tmp, src_path, vendor_path); + ASSERT_TRUE(mcp_search_cache_close(&cache)); + PASS(); +} + +TEST(search_code_scan_hook_rejection_is_contained_and_cleans_up) { + mcp_search_cache_t cache; + ASSERT_TRUE(mcp_search_cache_open(&cache, "cbm-search-reject")); + int scratch_before = mcp_count_directory_entries_with_prefix(cbm_tmpdir(), "cbm-search-"); + ASSERT_TRUE(scratch_before >= 0); + + char tmp[512], src_path[768], vendor_path[768]; + cbm_mcp_server_t *srv = setup_prefilter_server(tmp, sizeof(tmp), src_path, sizeof(src_path), + vendor_path, sizeof(vendor_path)); + ASSERT_NOT_NULL(srv); + mcp_search_command_probe_t probe = {.reject = true}; + cbm_mcp_server_set_command_test_hook(srv, mcp_search_command_hook_probe, &probe); + + char *response = cbm_mcp_handle_tool(srv, "search_code", + "{\"pattern\":\"HandleRequest\",\"project\":\"prefilter-" + "search\",\"file_pattern\":\"*.go\"}"); + ASSERT_NOT_NULL(response); + ASSERT_EQ(probe.calls, 1); + ASSERT_NOT_NULL(strstr(response, "contained command could not complete")); + ASSERT_NOT_NULL(strstr(response, "\"isError\":true")); + ASSERT_NULL(strstr(response, "src/handler.go")); + + char logs[640]; + snprintf(logs, sizeof(logs), "%s/logs", cache.path); + ASSERT_EQ(mcp_count_directory_entries_with_prefix(logs, ".mcp-command-"), 0); + ASSERT_EQ(mcp_count_directory_entries_with_prefix(cbm_tmpdir(), "cbm-search-"), scratch_before); + free(response); + cbm_mcp_server_free(srv); + cleanup_prefilter_dir(tmp, src_path, vendor_path); + ASSERT_TRUE(mcp_search_cache_close(&cache)); + PASS(); +} + +TEST(search_code_scoped_exit_one_is_not_no_match) { + char tmp[512], src_path[768], vendor_path[768]; + cbm_mcp_server_t *srv = setup_prefilter_server(tmp, sizeof(tmp), src_path, sizeof(src_path), + vendor_path, sizeof(vendor_path)); + ASSERT_NOT_NULL(srv); +#ifdef _WIN32 + cbm_mcp_server_set_search_scan_command_for_test(srv, "exit /b 1"); +#else + cbm_mcp_server_set_search_scan_command_for_test(srv, "exit 1"); +#endif + + char *response = cbm_mcp_handle_tool( + srv, "search_code", "{\"pattern\":\"HandleRequest\",\"project\":\"prefilter-search\"}"); + bool failed_closed = response && strstr(response, "contained command could not complete") && + strstr(response, "\"isError\":true") && + !strstr(response, "total_grep_matches: 0"); + + free(response); + cbm_mcp_server_free(srv); + cleanup_prefilter_dir(tmp, src_path, vendor_path); + ASSERT_TRUE(failed_closed); + PASS(); +} + +TEST(search_code_no_match_is_empty_for_direct_and_scoped_routes) { + char tmp[512], src_path[768], vendor_path[768]; + cbm_mcp_server_t *scoped = setup_prefilter_server(tmp, sizeof(tmp), src_path, sizeof(src_path), + vendor_path, sizeof(vendor_path)); + ASSERT_NOT_NULL(scoped); + char *scoped_response = cbm_mcp_handle_tool( + scoped, "search_code", + "{\"pattern\":\"DefinitelyAbsentSymbol\",\"project\":\"prefilter-search\"}"); + ASSERT_NOT_NULL(scoped_response); + ASSERT_NULL(strstr(scoped_response, "\"isError\":true")); + ASSERT_NOT_NULL(strstr(scoped_response, "total_grep_matches: 0")); + + cbm_mcp_server_t *direct = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(direct); + cbm_store_t *store = cbm_mcp_server_store(direct); + ASSERT_NOT_NULL(store); + cbm_mcp_server_set_project(direct, "direct-search"); + ASSERT_EQ(cbm_store_upsert_project(store, "direct-search", tmp), CBM_STORE_OK); + char *direct_response = cbm_mcp_handle_tool( + direct, "search_code", + "{\"pattern\":\"DefinitelyAbsentSymbol\",\"project\":\"direct-search\"}"); + ASSERT_NOT_NULL(direct_response); + ASSERT_NULL(strstr(direct_response, "\"isError\":true")); + ASSERT_NOT_NULL(strstr(direct_response, "total_grep_matches: 0")); + + free(scoped_response); + free(direct_response); + cbm_mcp_server_free(scoped); + cbm_mcp_server_free(direct); + cleanup_prefilter_dir(tmp, src_path, vendor_path); + PASS(); +} + +/* A store may contain an indexed path that is non-regular or no longer exists. + * Neither is a content-scan operand. Scoped search must skip both while + * preserving matches from regular files; actual command failures remain + * contained. */ +TEST(search_code_scoped_scan_skips_non_regular_indexed_paths) { + char tmp[512], src_path[768], vendor_path[768]; + cbm_mcp_server_t *srv = setup_prefilter_server(tmp, sizeof(tmp), src_path, sizeof(src_path), + vendor_path, sizeof(vendor_path)); + ASSERT_NOT_NULL(srv); + + char indexed_dir[768]; + snprintf(indexed_dir, sizeof(indexed_dir), "%s/indexed-dir", tmp); + ASSERT_EQ(cbm_mkdir(indexed_dir), 0); + cbm_store_t *store = cbm_mcp_server_store(srv); + cbm_node_t directory_node = {.project = "prefilter-search", + .label = "File", + .name = "indexed-dir", + .qualified_name = "prefilter-search.indexed-dir", + .file_path = "indexed-dir", + .start_line = 1, + .end_line = 1}; + ASSERT_GT(cbm_store_upsert_node(store, &directory_node), 0); + cbm_node_t missing_node = {.project = "prefilter-search", + .label = "File", + .name = "missing.go", + .qualified_name = "prefilter-search.missing.go", + .file_path = "missing.go", + .start_line = 1, + .end_line = 1}; + ASSERT_GT(cbm_store_upsert_node(store, &missing_node), 0); + + char *response = cbm_mcp_handle_tool( + srv, "search_code", "{\"pattern\":\"HandleRequest\",\"project\":\"prefilter-search\"}"); + ASSERT_NOT_NULL(response); + ASSERT_NULL(strstr(response, "\"isError\":true")); + ASSERT_NOT_NULL(strstr(response, "src/handler.go")); + + free(response); + cbm_mcp_server_free(srv); + ASSERT_EQ(cbm_rmdir(indexed_dir), 0); + cleanup_prefilter_dir(tmp, src_path, vendor_path); + PASS(); } /* Windows raw scans must pin the PowerShell pipe to UTF-8: PS 5.1 otherwise @@ -11702,8 +12088,18 @@ SUITE(mcp) { RUN_TEST(search_code_path_filter_matches_nothing); RUN_TEST(search_code_file_pattern_prefilter_boundaries); RUN_TEST(search_code_windows_prefilter_precedes_content_scan); - RUN_TEST(search_code_windows_cancel_cleans_supervised_scan); - RUN_TEST(search_code_windows_output_limit_fails_closed_and_cleans_scan); + RUN_TEST(search_code_cancel_cleans_supervised_scan); + RUN_TEST(search_code_output_limit_fails_closed_and_cleans_scan); + RUN_TEST(search_code_scan_deadline_fails_closed_and_resets); + RUN_TEST(search_code_scan_deadline_override_is_per_server); + RUN_TEST(search_code_scan_setup_failures_respect_cause_precedence); + RUN_TEST(search_code_scan_live_child_deadline_is_bounded_and_fails_closed); + RUN_TEST(search_code_scan_cancellation_precedes_zero_deadline); + RUN_TEST(search_code_scan_deadline_precedes_output_limit); + RUN_TEST(search_code_scan_hook_rejection_is_contained_and_cleans_up); + RUN_TEST(search_code_scoped_exit_one_is_not_no_match); + RUN_TEST(search_code_no_match_is_empty_for_direct_and_scoped_routes); + RUN_TEST(search_code_scoped_scan_skips_non_regular_indexed_paths); RUN_TEST(search_code_windows_scan_pins_utf8_output); RUN_TEST(search_code_invalid_regex_errors_issue283); RUN_TEST(search_code_literal_pipe_warns_issue282); diff --git a/tests/test_subprocess.c b/tests/test_subprocess.c index 30fd1537d..c9932bf2d 100644 --- a/tests/test_subprocess.c +++ b/tests/test_subprocess.c @@ -22,6 +22,8 @@ #include #include #include +#else +#include #endif /* ── Layer 1: pure classifier (all platforms) ─────────────────────────────── */ @@ -558,6 +560,143 @@ TEST(subprocess_quiet_timeout_kills_ignoring_tree) { #endif } +TEST(subprocess_windows_job_object_cancellation_quiesces_descendant_tree) { +#ifndef _WIN32 + SKIP_PLATFORM("native Windows Job Object descendant-tree probe"); +#else + char temp_dir[MAX_PATH]; + DWORD temp_length = GetTempPathA((DWORD)sizeof(temp_dir), temp_dir); + ASSERT_TRUE(temp_length > 0 && temp_length < sizeof(temp_dir)); + char pid_path[MAX_PATH]; + ASSERT_TRUE(GetTempFileNameA(temp_dir, "cbm", 0, pid_path) != 0); + ASSERT_TRUE(DeleteFileA(pid_path)); + + char system_directory[MAX_PATH]; + UINT system_length = GetSystemDirectoryA(system_directory, (UINT)sizeof(system_directory)); + ASSERT_TRUE(system_length > 0 && system_length < sizeof(system_directory)); + char powershell_path[MAX_PATH]; + ASSERT_TRUE(snprintf(powershell_path, sizeof(powershell_path), + "%s\\WindowsPowerShell\\v1.0\\powershell.exe", system_directory) > 0); + + char script[4096]; + int script_length = snprintf( + script, sizeof(script), + "$child=Start-Process powershell.exe " + "-ArgumentList '-NoProfile','-Command','while ($true) { Start-Sleep -Milliseconds 100 }' " + "-WindowStyle Hidden -PassThru; Set-Content -Encoding ASCII -LiteralPath '%s' " + "-Value ($PID.ToString() + ' ' + $child.Id.ToString()); " + "while ($true) { Start-Sleep -Milliseconds 100 }", + pid_path); + ASSERT_TRUE(script_length > 0 && (size_t)script_length < sizeof(script)); + const char *argv[] = {powershell_path, "-NoProfile", "-Command", script, NULL}; + + cbm_proc_opts_t opts = {0}; + opts.bin = powershell_path; + opts.argv = argv; + opts.cancel_grace_ms = 100; + cbm_subprocess_t *process = NULL; + ASSERT_EQ(cbm_subprocess_spawn(&opts, &process), 0); + ASSERT_NOT_NULL(process); + + DWORD root_pid = 0; + DWORD grandchild_pid = 0; + uint64_t ready_deadline = cbm_now_ms() + 3000U; + bool ready = false; + while (cbm_now_ms() < ready_deadline) { + FILE *pid_file = fopen(pid_path, "r"); + if (pid_file) { + unsigned long root_value = 0; + unsigned long grandchild_value = 0; + int fields = fscanf(pid_file, "%lu %lu", &root_value, &grandchild_value); + (void)fclose(pid_file); + if (fields == 2 && root_value > 0 && grandchild_value > 0) { + root_pid = (DWORD)root_value; + grandchild_pid = (DWORD)grandchild_value; + ready = true; + break; + } + } + cbm_proc_result_t ignored; + if (cbm_subprocess_poll(process, &ignored) != CBM_PROC_POLL_RUNNING) { + break; + } + Sleep(10); + } + + HANDLE root_handle = + ready ? OpenProcess(SYNCHRONIZE | PROCESS_TERMINATE, FALSE, root_pid) : NULL; + HANDLE grandchild_handle = + ready ? OpenProcess(SYNCHRONIZE | PROCESS_TERMINATE, FALSE, grandchild_pid) : NULL; + /* Request cancellation even if the PID probe failed so a failing test never + * leaves its supervised tree running in the shared Windows VM. */ + bool cancelled = cbm_subprocess_request_cancel(process); + cbm_proc_result_t result = {0}; + bool terminal = false; + uint64_t terminal_deadline = cbm_now_ms() + 4000U; + while (cbm_now_ms() < terminal_deadline) { + cbm_proc_poll_t state = cbm_subprocess_poll(process, &result); + if (state == CBM_PROC_POLL_TERMINAL) { + terminal = true; + break; + } + if (state == CBM_PROC_POLL_ERROR) { + break; + } + Sleep(10); + } + /* If the behavior under test regresses, terminate both known Job members + * directly and give the supervisor one bounded drain interval. The test + * still fails on the original `terminal` verdict, but it cannot strand its + * infinite root/grandchild or retain a terminal subprocess/Job handle. */ + bool cleanup_terminal = terminal; + if (!cleanup_terminal) { + if (grandchild_handle) { + (void)TerminateProcess(grandchild_handle, 1); + } + if (root_handle) { + (void)TerminateProcess(root_handle, 1); + } + (void)cbm_subprocess_request_cancel(process); + uint64_t cleanup_deadline = cbm_now_ms() + 2000U; + while (cbm_now_ms() < cleanup_deadline) { + cbm_proc_result_t cleanup_result; + cbm_proc_poll_t state = cbm_subprocess_poll(process, &cleanup_result); + if (state == CBM_PROC_POLL_TERMINAL) { + cleanup_terminal = true; + break; + } + if (state == CBM_PROC_POLL_ERROR) { + break; + } + Sleep(10); + } + } + bool root_gone = root_handle && WaitForSingleObject(root_handle, 2000) == WAIT_OBJECT_0; + bool grandchild_gone = + grandchild_handle && WaitForSingleObject(grandchild_handle, 2000) == WAIT_OBJECT_0; + if (root_handle) { + CloseHandle(root_handle); + } + if (grandchild_handle) { + CloseHandle(grandchild_handle); + } + if (cleanup_terminal) { + cbm_subprocess_destroy(process); + } + (void)DeleteFileA(pid_path); + + ASSERT_TRUE(ready); + ASSERT_TRUE(cancelled); + ASSERT_TRUE(terminal); + ASSERT_TRUE(result.cancellation_requested); + ASSERT_TRUE(result.tree_quiesced); + ASSERT_FALSE(result.supervision_failed); + ASSERT_TRUE(root_gone); + ASSERT_TRUE(grandchild_gone); + PASS(); +#endif +} + TEST(subprocess_cancel_grace_is_hard_capped) { #ifdef _WIN32 SKIP_PLATFORM("POSIX process-group grace cap probe; native Windows coverage pending"); @@ -1040,6 +1179,7 @@ SUITE(subprocess) { RUN_TEST(subprocess_natural_completion_is_cached_across_polls); RUN_TEST(subprocess_cancel_is_idempotent_and_kills_ignoring_tree); RUN_TEST(subprocess_quiet_timeout_kills_ignoring_tree); + RUN_TEST(subprocess_windows_job_object_cancellation_quiesces_descendant_tree); RUN_TEST(subprocess_cancel_grace_is_hard_capped); RUN_TEST(subprocess_poll_log_delivery_is_bounded_and_terminal_is_lossless); RUN_TEST(subprocess_final_log_drain_error_is_terminal_and_preserves_classification); From b10c2744d6c1f180dd4b417d35f3c28dc9ba86fc Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 29 Aug 2026 08:34:17 +0200 Subject: [PATCH 22/31] docs: define runtime trace observation model Signed-off-by: Martin Vogel --- docs/RUNTIME_TRACE_MODEL.md | 541 ++++++++++++++++++++++++++++++++++++ 1 file changed, 541 insertions(+) create mode 100644 docs/RUNTIME_TRACE_MODEL.md diff --git a/docs/RUNTIME_TRACE_MODEL.md b/docs/RUNTIME_TRACE_MODEL.md new file mode 100644 index 000000000..5bc3cb6bd --- /dev/null +++ b/docs/RUNTIME_TRACE_MODEL.md @@ -0,0 +1,541 @@ +# Runtime Trace Observation Model + +## Status and recommendation + +This document is an architecture contract, not an implementation. It recommends a +durable, versioned runtime-observation sidecar whose publications can be read alongside +a pinned static index generation. Runtime observations MUST NOT patch static `CALLS` +edges, fabricate static nodes, or appear as persistent edges in the default graph. + +The model intentionally separates facts observed at runtime from facts derived from +source. A caller may explicitly request a `RUNTIME_CALL` overlay; existing static +search, architecture, and default trace behavior remain unchanged. + +This document-only change does not alter the runtime wire version, runtime schema +version, runtime semantic version, runtime artifact version, static semantic index +version, or artifact schema version. In particular, the static semantic index version +remains 3. + +## Scope + +The model defines: + +- stable identities for spans, producer contributions, observations, endpoints, and + composite publications; +- deterministic ingestion, aggregation, publication, and replay semantics; +- generation-scoped resolution from runtime endpoints to static symbols; +- an opt-in runtime call overlay without changing the stored static graph; +- concurrency, retry, conflict, recovery, privacy, and cardinality boundaries; and +- six bounded implementation packages that can be reviewed and shipped separately. + +## Non-goals + +This design does not: + +- infer new static `CALLS` edges or change confidence on existing ones; +- create placeholder nodes in the static graph for runtime-only identities; +- persist runtime edges in the generic graph edge table; +- make runtime data visible to existing graph tools unless an overlay is requested; +- define distributed tracing transport, sampling policy, or collector deployment; +- promise complete traces, causal proof from timing alone, or source-line attribution; +- store arbitrary span payloads, headers, request bodies, query strings, or secrets; +- average percentile summaries or treat a producer's p99 as a mergeable percentile; +- provide a compatibility promise for a future mutation API before its versioned + contract exists; or +- change any runtime or static version merely by adding this document. + +## Current surface + +The current `ingest_traces` surface requires `project` and `traces[]`. Each trace item +permits `caller`, `callee`, and `count`, but those item fields are not currently +individually required. Its handler counts the supplied records and returns +`status="accepted"`, `traces_received`, and an unimplemented note; this is not an +accepted durable mutation, and the handler does not open or mutate a store. + +The existing trace helper surface can extract `service.name`, HTTP method, HTTP path, +HTTP status, span kind, duration, URL path, and p99. Verified span-kind values are: + +- `1`: internal; +- `2`: server; and +- `3`: client. + +Generic graph edges are unique by source, target, type, and the local-name +discriminator. Generic edge upsert merges properties with `json_patch`. Those rules are +appropriate for the static graph but are not the contribution or aggregation rules in +this document. The store already exposes `BEGIN IMMEDIATE`, `COMMIT`, and `ROLLBACK`. +Static publication already uses opaque generations and request-scoped reads. + +## Normative terms + +`MUST`, `MUST NOT`, `SHOULD`, and `MAY` are normative. "Canonical JSON" means UTF-8 +JSON with a fixed schema, lexicographically sorted object keys, arrays in their +specified order, no insignificant whitespace, and integers rather than locale- or +platform-dependent numeric strings. Hashes are computed over those canonical bytes. + +## Identity layers + +The following identities are different and MUST NOT be substituted for one another. + +### Trace and span identity + +A span is identified within one producer namespace by: + +```text +(project, producer_id, producer_epoch, trace_id, span_id) +``` + +`trace_id` and `span_id` are opaque byte strings rendered in one canonical lowercase +hex form at the boundary. `producer_epoch` is part of span identity so a producer can +reuse trace or span identifiers in a later epoch without collision. The tuple identifies +an input span for diagnostics and deduplication; it does not identify a static node or +an aggregate. + +### Producer contribution identity + +Every accepted batch has a producer-stable identifier: + +```text +contribution_id = H(canonical JSON { + producer_id, + producer_epoch, + source_batch_id +}) +``` + +`producer_id` names the authenticated logical producer, not a transient process. +`producer_epoch` changes only when that producer intentionally starts a new identity +domain. `source_batch_id` is stable across delivery retries. The resulting +`contribution_id` is scoped by project. + +An accepted contribution is immutable. Retrying the same `(project, producer_id, +contribution_id)` with the same canonical payload hash is an idempotent success and +returns the original disposition. Retrying it with different canonical bytes is a +conflict and MUST be rejected without mutation. A producer MUST NOT reuse another +producer's namespace, and the service MUST NOT silently mint a replacement ID to turn +a conflict into a second contribution. + +The payload hash covers the canonical normalized allowlisted contribution map together +with its pinned wire and runtime semantic version fields. It does not cover original +envelope ordering, unsupported fields, discarded fields, or raw payload bytes. + +### Endpoint identity + +Every endpoint has an explicit tag and a canonical key. + +1. `symbol`: a source-derived symbol reference `(project, qualified_name)`. This stable + endpoint identity deliberately excludes a static generation. A separate resolution + record maps it within one named static generation and resolver version. A later + generation MUST resolve again and MUST NOT inherit a node identifier from an older + generation. +2. `runtime`: a runtime-only endpoint derived from an allowlisted protocol identity, + such as canonical service name plus normalized HTTP route and method. It is stored + only in the sidecar and never creates a static node. +3. `unknown`: an opaque, scoped endpoint used when the allowlisted evidence cannot + identify either a symbol or a runtime service endpoint. Unknown keys MUST be stable + only inside their documented scope (for example, producer plus trace) and MUST NOT + collapse unrelated missing values into one global endpoint. + +Endpoint canonicalization is versioned by `runtime_semantic_version`. All strings are +Unicode-normalized, methods and protocol tokens use their specified canonical case, +and absent values are distinct from empty values. Raw URL queries, fragments, and +credentials are excluded. + +### Observation identity + +An observation describes one directed runtime perspective, not one delivery. Its +canonical key is: + +```text +observation_key = H(canonical JSON { + runtime_semantic_version, + project, + protocol, + perspective, + source_endpoint_key, + target_endpoint_key, + operation_key +}) +``` + +The key excludes counters, timestamps, duration summaries, status summaries, +contribution IDs, and static database row IDs. `operation_key` contains only the +versioned allowlisted operation identity, such as canonical HTTP method and normalized +route template. Consequently, identical observations from multiple producers converge +while semantically different directions or operations remain distinct. + +### Composite publication identity + +A reader-visible snapshot is identified by the atomic pair: + +```text +{ static_generation, runtime_generation } +``` + +The pair is the head identity, but its halves are independent immutable publications. +A runtime generation identifies runtime contributions and aggregates without embedding +a static generation. Generation-scoped symbol resolutions and overlay projections are +keyed by the pair plus resolver version. Neither half may be silently advanced while a +request is using the pair. + +## Protocol and perspective mapping + +Ingestion parses and normalizes only versioned, allowlisted span fields into the +contribution, then derives a perspective from that evidence. The raw span payload is +not durably preserved. The HTTP mapper uses the existing service-name, HTTP method, +HTTP path or URL path, status, span-kind, and duration helpers. The route field is +normalized as a route template when the source provides one; a raw high-cardinality URL +path MUST NOT be promoted to a template by guesswork. + +For the verified span kinds: + +- `client` (`3`) represents an outbound attempt. The source is the resolved local + symbol when available, otherwise a tagged local runtime or scoped unknown endpoint. + The target is the allowlisted remote service/HTTP endpoint or a scoped unknown + endpoint. The perspective token is `outbound`. +- `server` (`2`) represents an inbound handling attempt. The source is an allowlisted + peer/runtime endpoint or a scoped unknown endpoint. The target is the resolved local + handler when available, otherwise the local service/HTTP runtime endpoint. The + perspective token is `inbound`. +- `internal` (`1`) represents local work. A directed call projection is produced only + when explicit parent/child evidence supplies two endpoints. Otherwise the + observation remains queryable as internal activity but does not become a + `RUNTIME_CALL`. The perspective token is `internal`. + +Unsupported or absent span kinds are retained only as normalized, allowlisted non-call +observation metadata if policy allows; unsupported fields and raw payload bytes are +discarded. They MUST NOT be guessed into one of the three perspectives. Status and +duration contribute to summaries but never to observation identity. + +Symbol resolution is best-effort and generation-scoped. Ambiguous matches remain +tagged runtime or unknown endpoints. Resolution MUST fail closed: it must not choose a +symbol by filename substring, short-name coincidence, timing, or insertion order. + +## Sidecar model + +The durable sidecar is conceptually separate from the static graph store even if an +implementation initially shares the same database file. It contains these logical +relations: + +### `runtime_endpoints` + +Stores the endpoint tag, canonical endpoint key, and canonical allowlisted identity. +For a `symbol` endpoint, the durable identity contains project plus qualified name but +no static generation. Generation and static database row IDs belong only to resolution +records or caches, not durable endpoint identity. + +### `runtime_contributions` + +Stores project, producer ID, contribution ID, canonical payload hash, ingestion state, +and immutable canonical contribution metadata. States include at least `staged` and +`finalized`; an aborted transaction leaves neither state visible. + +### `runtime_observation_contributions` + +Maps `(contribution_id, observation_key)` to that contribution's canonical summary. +The map entry is immutable after finalization. Replacing it requires a new contribution +identity. + +### `runtime_aggregates` + +Stores the deterministic fold of finalized contribution-map entries for each +observation key and runtime generation. It is derived state and can be rebuilt from +finalized contributions. + +### `runtime_resolutions` + +Maps a stable runtime or unknown endpoint key to an optional stable SymbolEndpoint +`(project, qualified_name)` plus its current static node/cache result, unresolved +result, or ambiguity result. Each record is keyed by project, runtime generation, +static generation, stable endpoint key, and resolver version. Resolution records are +derived caches, not endpoint or observation identity, and a record from one static +generation MUST NOT be reused for another. + +### `runtime_publications` + +Stores immutable runtime generations independently of static generations. A separate +atomic per-project composite head pairs one current runtime generation with one current +static generation. Publication metadata includes the relevant runtime versions; the +composite head records the static index/artifact versions used for compatibility +checks. + +No table above is a static graph node or edge table. The implementation MUST NOT reuse +generic `json_patch` edge upsert for contribution merging. + +## Deterministic aggregation + +Before immutable summaries are stored, one contribution is normalized deterministically: + +1. canonicalize only allowlisted fields and compute the exact span identity + `(project, producer_id, producer_epoch, trace_id, span_id)`; +2. deduplicate records with the same span identity and identical normalized bytes; +3. reject the entire contribution if one span identity has different normalized bytes; +4. group the remaining unique normalized records by `observation_key`; +5. after keyed exact-span deduplication/set union supplies idempotence, fold integer + counts and duration sums, fixed status buckets, and fixed-schema histogram bins over + the unique normalized span map using associative and commutative exact addition; and +6. sort keys and canonical-serialize the contribution map before hashing or storage. + +The normalized contribution map and its immutable summaries MUST be byte-identical for +every input ordering and worker partitioning of the same unique normalized span set. +Discarded or unsupported fields cannot change the payload hash. + +For each observation, aggregation is a map union keyed by contribution identity: + +```text +aggregate(observation) = fold(sorted union { + contribution_id -> canonical contribution summary +}) +``` + +Union of identical key/value entries is idempotent. A differing value under the same +key is a conflict, not a winner selection. Map union is associative and commutative; +sorting by contribution ID before serialization makes the resulting bytes independent +of producer, retry, transaction, or worker order. + +Canonical contribution summaries MAY contain integer count, integer duration sum in a +fixed unit, fixed-schema status buckets, and a mergeable duration distribution whose +encoding is pinned by `runtime_semantic_version`. Aggregate totals are exact integer +sums over the contribution map. Derived display values are computed after the fold. + +Percentiles are not additive. A batch p99 MUST NOT be averaged, weighted-averaged, or +selected as the aggregate p99. The existing p99 helper may summarize a single raw batch +for diagnostics, but a published cross-contribution p99 requires a versioned mergeable +distribution (for example fixed histogram boundaries). Without such a distribution, +the aggregate p99 is unavailable. + +Canonical aggregate JSON MUST be byte-identical for every permutation and +parenthesization of the same finalized contribution set. + +## Ingestion and publication transaction + +Full wire parsing, canonicalization, payload hashing, and every validation that does +not depend on mutable project state happen before acquiring the publication lock. +Publication is then serialized by one shared per-project publication lock used by both +static and runtime publishers. An implementation MUST NOT introduce an independent +runtime lock that permits lost updates to either half of the composite head. + +Within that lock, a runtime publisher: + +1. begins a write transaction with `BEGIN IMMEDIATE`; +2. reads and pins the current composite head and validates compatibility versions; +3. performs state-dependent authorization, quota, producer identity, deduplication, + contribution conflict, and expected-head checks; +4. stages all endpoints, immutable contribution-map entries, and rebuilt aggregates; +5. allocates an opaque runtime generation independent of the pinned static generation; +6. durably finalizes every contribution in the publication; +7. writes the immutable runtime publication and atomically changes only the runtime + half of the composite head, preserving its pinned static generation; +8. commits all changes with `COMMIT`; and +9. releases the per-project publication lock. + +Every validation or write failure before commit uses `ROLLBACK`. There is no visible +partial contribution, aggregate, generation, or head. Finalization and head movement +are in the same transaction, so a success response is sent only after durable commit. + +Static publication follows the same lock discipline and atomically changes only the +static half of the composite head, preserving its pinned runtime generation. It does +not create a new runtime generation merely because source was reindexed. Resolution +records for the new `{static_generation, runtime_generation}` pair are recomputed or +materialized under their pair-and-resolver-version key; records from the old static +generation are never treated as resolutions for the new one. + +## Reader contract + +A request that opts into runtime data pins one composite head at request start. Every +page, endpoint resolution, aggregate, and overlay edge in that request is read from +that pair. Default requests pin only their existing static request scope and do not pay +for or expose the overlay. + +Pagination cursors encode the project, composite head, query shape, overlay mode, and +runtime semantic version. If the current head differs, a subsequent request with that +cursor returns a stale-cursor error; it never resumes against mixed generations. +Readers MAY explicitly request a still-retained historical composite head. Missing or +garbage-collected heads fail explicitly. + +## `RUNTIME_CALL` overlay + +Runtime call projection is an explicit read option. It presents directed overlay edges +derived from one pinned runtime generation. These edges: + +- use the distinct type `RUNTIME_CALL`; +- carry endpoint tags so callers can distinguish symbol, runtime, and unknown ends; +- include deterministic aggregate summaries and the composite head identity; +- never mutate, replace, increase confidence on, or suppress a static `CALLS` edge; +- are omitted from existing search, architecture, and default trace responses unless + the caller requests the overlay; and +- are regenerated when either half of the composite publication changes. + +If a projected runtime call has symbol endpoints, it references their qualified-name +identity within the pinned static generation. Runtime-only and unknown endpoints remain +sidecar objects and are not fabricated as graph nodes. + +## Version boundaries + +Compatibility is divided deliberately: + +- `wire_version`: request and response envelope, field encodings, and retry result; +- `runtime_schema_version`: sidecar tables, indexes, and migration compatibility; +- `runtime_semantic_version`: canonical endpoint/observation keys, mappings, and + aggregation meaning; +- `runtime_artifact_version`: exported/imported sidecar artifact layout; +- `static_semantic_index_version`: source-derived graph semantics; and +- `static_artifact_schema_version`: static artifact representation. + +A change bumps only the boundary it changes. A runtime semantic change can invalidate +runtime generations without changing static semantic index version. A storage-only +migration can change runtime schema version without changing observation identity. +Import rejects incompatible artifact versions before mutation. This document changes +none of these versions. + +## Security, privacy, and cardinality + +Ingestion is project-authorized and producer-authenticated. Producer IDs, contribution +IDs, and project scope are server-validated; caller-provided project or producer fields +do not grant access. Reads apply the same project authorization to runtime and static +data. + +Only allowlisted attributes enter canonical identities or summaries. Credentials, +authorization data, cookies, request/response bodies, query strings, fragments, +arbitrary baggage, and raw exception text are rejected or discarded before durable +storage. Logs and errors use opaque IDs and counts, not rejected payloads. + +The service enforces bounded request bytes, records, endpoints, observations, attribute +lengths, route-template segments, status buckets, and contribution-map growth. Raw URL +paths are not accepted as unbounded route labels. Unknown endpoints are scoped to avoid +both accidental global collapse and attacker-controlled global cardinality. Quotas and +retention apply per project and producer. Limit failure is all-or-nothing and does not +publish a truncated contribution unless a future wire version explicitly defines that +behavior. + +## Failure and recovery matrix + +| Condition | Required disposition | Visible state | +| --- | --- | --- | +| Same contribution ID, same payload hash | Idempotent success; return prior result | Unchanged | +| Same contribution ID, different payload hash | Conflict; reject entire request | Unchanged | +| Unauthorized project or producer | Reject before staging | Unchanged | +| Invalid/unsupported wire or semantic version | Reject before staging | Unchanged | +| Invalid endpoint, attribute, or cardinality limit | Reject entire contribution | Unchanged | +| Ambiguous symbol resolution | Keep runtime/unknown tag; do not guess | Valid sidecar observation only | +| Client-supplied expected composite head differs from the head read under lock | Reject as stale; retry from the newly read head | Unchanged | +| Failure while staging or aggregating | `ROLLBACK` | No partial state | +| Crash before commit | Database recovery rolls back the transaction | Prior head remains | +| Commit succeeds but response is lost | Producer retry resolves idempotently | One finalized contribution | +| Derived aggregate missing or corrupt | Rebuild from finalized contribution map before publish | Prior valid head remains | +| Cursor references a different or removed head | Return stale/missing cursor error | No mixed-generation read | +| Unsupported span kind | Retain only normalized, allowlisted non-call observation metadata; discard unsupported fields and raw payload bytes | No guessed call overlay | +| No mergeable duration distribution | Publish counts/sums; report p99 unavailable | No averaged p99 | + +## Alternatives considered + +### Patch static `CALLS` edges + +Benefit: existing graph queries would see runtime evidence with no overlay option. + +Trade-offs: it conflates observed and source-derived semantics, lets sampled or hostile +telemetry alter the static graph, cannot represent runtime-only endpoints honestly, +couples runtime retention to static indexing, and makes rollback and provenance +ambiguous. It is rejected. + +### Fabricate static nodes for runtime services and unknown peers + +Benefit: all endpoints would fit the existing graph node/edge shape. + +Trade-offs: fabricated nodes look source-derived, create unstable identities and high +cardinality, collide with future symbol resolution, and pollute architecture/search +results. It is rejected. + +### Store persistent runtime edges in the default graph + +Benefit: generic edge uniqueness and `json_patch` upsert are already available. + +Trade-offs: generic uniqueness has no immutable producer-contribution dimension; +`json_patch` is neither the required commutative map union nor a conflict detector; +default readers would unexpectedly mix generations and data classes. It is rejected. + +### Keep runtime observations entirely ephemeral + +Benefit: minimal schema, retention, and migration burden; no durable sensitive data. + +Trade-offs: retries cannot be made idempotent across restarts, results are not +reproducible or pageable, multi-producer aggregation is unstable, and a reader cannot +pin a coherent composite publication. It is rejected as the primary model, though +ephemeral pre-validation remains useful. + +### Durable versioned sidecar with an opt-in overlay (selected) + +Benefits: preserves static semantics, gives runtime-only endpoints an honest identity, +supports deterministic replay and aggregation, isolates retention and authorization, +and provides atomic cross-generation reads. + +Trade-offs: adds schema and version boundaries, a shared publication lock, endpoint +resolution, aggregate rebuilds, overlay-aware APIs, and explicit operational limits. +These costs are accepted because they make provenance, concurrency, failure recovery, +and compatibility testable rather than implicit. + +## Bounded implementation packages + +Implementation remains deferred. The following packages are intentionally ordered and +bounded; each requires its own reproduce-first tests and review. + +1. **Versioned wire validation and canonicalization.** Define authenticated producer + identity and epoch, contribution ID, exact span-identity deduplication, conflicting + duplicate rejection, canonical contribution maps and payload hashes, allowlisted + HTTP/span mapping, limits, and dry-run validation. No store mutation. +2. **Sidecar schema and immutable contributions.** Add runtime schema migrations, + endpoints, contribution records, observation-contribution maps, idempotent retry, + conflict rejection, and transactional rollback. No publication head or overlay. +3. **Deterministic aggregation.** Add canonical intra-contribution grouping, keyed + exact-span idempotent set union/deduplication, associative and commutative exact + integer/status/histogram addition, cross-contribution map-union folds, fixed + mergeable duration distributions, input/worker/merge-order permutation tests, + rebuild verification, and explicit unavailable p99 behavior. +4. **Generation-scoped resolution and publication.** Add the shared per-project lock, + independent runtime generation allocation, durable finalization, pair-and-version + resolution records, atomic composite head updates that preserve the other half, and + crash/retry recovery tests. +5. **Pinned read and `RUNTIME_CALL` overlay.** Add opt-in queries, tagged endpoints, + request-scoped composite pins, stale cursors, historical-head handling, and proofs + that default static tools remain byte-for-byte unchanged. +6. **Artifact, retention, and operational hardening.** Add versioned runtime artifact + import/export, quota/retention policy, authorization audits, corruption rebuild, + migration/rollback tests, metrics, and multi-producer concurrency tests. + +No package may silently absorb a later package's public surface or persistence model. + +## Acceptance invariants + +An implementation of this contract is acceptable only if all of these hold: + +1. Static nodes and `CALLS` edges are byte-identical with runtime ingestion enabled or + disabled for the same source input. +2. Existing search, architecture, and default trace responses exclude runtime data. +3. Runtime data appears only through an explicit, versioned overlay request. +4. Same producer contribution plus same payload is idempotent across retries and + restarts; a different payload under that identity is a no-mutation conflict. +5. Normalized contribution and aggregate bytes are identical for every input, delivery, + worker, and merge order of the same unique normalized span set, including identical + duplicate replay; a conflicting duplicate span identity rejects the whole + contribution without mutation. +6. Aggregate p99 is derived only from a pinned mergeable distribution and is otherwise + unavailable; producer p99 values are never averaged. +7. Symbol resolution is qualified-name based, generation-scoped, and fail-closed on + ambiguity; runtime and unknown endpoints never fabricate static nodes. +8. Runtime generation state contains all of its finalized contributions, endpoints, + and aggregates, and reader visibility requires the separate atomic composite-head + update; the transaction exposes both the complete generation and head update, or + neither. +9. Every overlay read and page uses one immutable + `{static_generation, runtime_generation}` pair; stale cursors fail explicitly. +10. Static and runtime publication serialize through the same per-project lock; each + publisher changes only its own half of the composite head and preserves the other, + and symbol-resolution records are never reused across static generations or + resolver versions. +11. Unsupported span kinds, missing identities, and limit violations never become + guessed calls or partial publications. +12. Authorization, privacy filtering, quotas, and retention are enforced before data + becomes durable or reader-visible. +13. Wire, runtime schema, runtime semantic, runtime artifact, static semantic, and + static artifact versions can change independently according to their boundaries. +14. Documentation-only adoption leaves all six version values and all runtime behavior + unchanged. From 5821078ee503a7a4e0735b0255c647e5d9f57339 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 29 Aug 2026 11:02:18 +0200 Subject: [PATCH 23/31] fix(pkg): correct stale packaging versions to v0.10.8 + gate the drift MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seven packaging surfaces declared 0.8.1 while v0.10.8 was the shipped release — nine releases of drift in metadata published to package registries, and the third version-metadata drift this month. pkg/go was not cosmetic: it hardcodes its version into the release URL it downloads from, so `go install` fetched v0.8.1 binaries. Chocolatey had to move as a unit. The nuspec is version-only but chocolateyInstall.ps1 pins the asset sha256, so bumping the nuspec alone would ship a package declaring 0.10.8 while installing 0.8.1. The new checksum is the v0.10.8 windows-amd64.zip digest, verified by downloading the published asset and hashing it, not merely copied from checksums.txt. The gate (tests/test_version_metadata_contract.sh, scripts/test.sh Step 0x) checks against the newest stable v-prefixed tag, because nothing in the tree declares the version: CBM_VERSION is injected at build time and the release is dispatched with a version input. That absence is exactly why these files drifted. Mutual consistency across the surfaces needs no git and always runs, so a shallow clone degrades the gate loudly instead of passing silently. Homebrew (0.10.3), Scoop and AUR (0.8.1) are left pinned: each embeds per-release asset checksums, so bumping them is a distribution chore rather than a metadata edit. They are still gated — internally consistent, and never naming a release that does not exist — and each carries a recorded reason for being frozen. Signed-off-by: Martin Vogel --- pkg/chocolatey/codebase-memory-mcp.nuspec | 4 +- pkg/chocolatey/tools/chocolateyInstall.ps1 | 4 +- pkg/go/cmd/codebase-memory-mcp/main.go | 2 +- pkg/npm/package.json | 2 +- pkg/pypi/pyproject.toml | 2 +- pkg/pypi/src/codebase_memory_mcp/_cli.py | 2 +- scripts/test.sh | 3 + server.json | 6 +- tests/test_version_metadata_contract.sh | 307 +++++++++++++++++++++ 9 files changed, 321 insertions(+), 11 deletions(-) create mode 100644 tests/test_version_metadata_contract.sh diff --git a/pkg/chocolatey/codebase-memory-mcp.nuspec b/pkg/chocolatey/codebase-memory-mcp.nuspec index d0509ecf1..0c1705cf6 100644 --- a/pkg/chocolatey/codebase-memory-mcp.nuspec +++ b/pkg/chocolatey/codebase-memory-mcp.nuspec @@ -2,7 +2,7 @@ codebase-memory-mcp - 0.8.1 + 0.10.8 codebase-memory-mcp DeusData https://github.com/DeusData/codebase-memory-mcp @@ -28,7 +28,7 @@ After installation, run: to configure your coding agents automatically. - https://github.com/DeusData/codebase-memory-mcp/releases/tag/v0.8.1 + https://github.com/DeusData/codebase-memory-mcp/releases/tag/v0.10.8 diff --git a/pkg/chocolatey/tools/chocolateyInstall.ps1 b/pkg/chocolatey/tools/chocolateyInstall.ps1 index 3c83ee8c1..132b86d90 100644 --- a/pkg/chocolatey/tools/chocolateyInstall.ps1 +++ b/pkg/chocolatey/tools/chocolateyInstall.ps1 @@ -1,9 +1,9 @@ $ErrorActionPreference = 'Stop' $packageName = 'codebase-memory-mcp' -$version = '0.8.1' +$version = '0.10.8' $url64 = "https://github.com/DeusData/codebase-memory-mcp/releases/download/v${version}/codebase-memory-mcp-windows-amd64.zip" -$checksum64 = 'a602ad090ed3f49d86c55472f73f27ad7055222806a82358f2e08513e027f00f' +$checksum64 = 'b43ad982994c4d829670749e08d3b622a74bb20041fc0a7d02bef6113f81c34d' $installDir = Join-Path $env:ChocolateyBinRoot $packageName Install-ChocolateyZipPackage ` diff --git a/pkg/go/cmd/codebase-memory-mcp/main.go b/pkg/go/cmd/codebase-memory-mcp/main.go index 93b626a83..91f1adc33 100644 --- a/pkg/go/cmd/codebase-memory-mcp/main.go +++ b/pkg/go/cmd/codebase-memory-mcp/main.go @@ -36,7 +36,7 @@ import ( const ( repo = "DeusData/codebase-memory-mcp" - version = "0.8.1" + version = "0.10.8" windowsBinaryName = "codebase-memory-mcp.exe" maxRedirects = 5 diff --git a/pkg/npm/package.json b/pkg/npm/package.json index d82a14bd1..f2b122d84 100644 --- a/pkg/npm/package.json +++ b/pkg/npm/package.json @@ -1,6 +1,6 @@ { "name": "codebase-memory-mcp", - "version": "0.8.1", + "version": "0.10.8", "description": "Fast code intelligence for AI coding agents — installs a verified native runtime set", "mcpName": "io.github.DeusData/codebase-memory-mcp", "license": "MIT", diff --git a/pkg/pypi/pyproject.toml b/pkg/pypi/pyproject.toml index 38ad6ec66..5821d2a4b 100644 --- a/pkg/pypi/pyproject.toml +++ b/pkg/pypi/pyproject.toml @@ -8,7 +8,7 @@ build-backend = "hatchling.build" [project] name = "codebase-memory-mcp" -version = "0.8.1" +version = "0.10.8" description = "Fast code intelligence for AI coding agents — installs a verified native runtime set" readme = "README.md" license = { text = "MIT" } diff --git a/pkg/pypi/src/codebase_memory_mcp/_cli.py b/pkg/pypi/src/codebase_memory_mcp/_cli.py index 2750248a5..8a4aa112c 100644 --- a/pkg/pypi/src/codebase_memory_mcp/_cli.py +++ b/pkg/pypi/src/codebase_memory_mcp/_cli.py @@ -338,7 +338,7 @@ def _version() -> str: from importlib.metadata import version return version("codebase-memory-mcp") except Exception: - return "0.8.1" + return "0.10.8" def _os_name() -> str: diff --git a/scripts/test.sh b/scripts/test.sh index b6c9d042b..0aafe6c20 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -269,6 +269,9 @@ bash "$ROOT/tests/test_nomic_blob_generator_contract.sh" echo "=== Step 0w: published language-count contract ===" bash "$ROOT/tests/test_language_count_contract.sh" +echo "=== Step 0x: packaging version-metadata contract ===" +bash "$ROOT/tests/test_version_metadata_contract.sh" + # Verify compiler supports target arch verify_compiler "$CC" diff --git a/server.json b/server.json index 93bcccc3b..a30de71f2 100644 --- a/server.json +++ b/server.json @@ -8,12 +8,12 @@ "source": "github" }, "websiteUrl": "https://deusdata.github.io/codebase-memory-mcp/", - "version": "0.8.1", + "version": "0.10.8", "packages": [ { "registryType": "npm", "identifier": "codebase-memory-mcp", - "version": "0.8.1", + "version": "0.10.8", "runtimeHint": "npx", "transport": { "type": "stdio" @@ -22,7 +22,7 @@ { "registryType": "pypi", "identifier": "codebase-memory-mcp", - "version": "0.8.1", + "version": "0.10.8", "runtimeHint": "uvx", "transport": { "type": "stdio" diff --git a/tests/test_version_metadata_contract.sh b/tests/test_version_metadata_contract.sh new file mode 100644 index 000000000..a3878764f --- /dev/null +++ b/tests/test_version_metadata_contract.sh @@ -0,0 +1,307 @@ +#!/usr/bin/env bash +# Every packaging surface that states a product version must state a version we +# actually released, and the registry-facing ones must state the newest one. +# +# Why this exists: server.json, pkg/npm/package.json and the Chocolatey nuspec +# sat at 0.8.1 while v0.10.8 was the shipped release — nine releases of drift in +# metadata that is published to package registries. It was the third +# version-metadata drift in one month. pkg/go additionally hardcodes its version +# into the release URL it downloads from, so the stale value was not cosmetic: +# `go install` fetched v0.8.1 binaries. +# +# Why there is no single canonical version file to check against: CBM_VERSION is +# injected at BUILD time (-DCBM_VERSION, see Makefile.cbm) and the release is +# dispatched with a version input, so nothing in the tree declares the truth. +# That absence is precisely why these files drifted — there was nothing to +# check them against. The release workflow then institutionalised the drift for +# three of them: publish-registries rewrites pkg/npm/package.json and +# pkg/pypi/pyproject.toml from the dispatch input, and publish-mcp-registry +# rewrites server.json, each with a comment noting the repo copy "can lag". The +# published artifact was therefore correct while the repo told everyone reading +# it a false version, and no surface outside that sync path was corrected at +# all. +# +# So the truth used here is the newest STABLE v-prefixed git tag: the only +# in-repo statement of what actually shipped. Between releases the repo +# legitimately sits at the last released version — no surface is bumped ahead of +# a release — so "every surface equals the newest tag" is stable, not a +# perpetual red. +# +# Two independent assertions, so the gate cannot go vacuous when tags are absent +# (shallow clone, `git archive` tarball, a container that cloned with +# --no-tags). Mutual consistency needs no git at all and therefore ALWAYS runs; +# the tag comparison runs when tags exist and announces itself loudly when they +# do not. A missing truth source degrades this gate, and it says so — it never +# reports a silent pass. +# +# Usage: tests/test_version_metadata_contract.sh +# Exit 0 = every assertion passed. Exit 1 = at least one violation. A run that +# could extract no version from a registered surface is a FAILURE, not a pass. + +set -euo pipefail + +case "${1:-}" in +-h | --help) + sed -n '2,44p' "$0" | sed 's/^# \{0,1\}//' + exit 0 + ;; +esac + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +failures=0 + +fail() { + echo "FAIL: $*" >&2 + failures=$((failures + 1)) +} + +# ── 1. The truth: the newest STABLE released tag ──────────────────── +# Prereleases are excluded (v0.9.1-rc.1): an RC is not what the packaging +# surfaces track. Bare tags are excluded too — the v0.10.7 release incident left +# a tag named "0.10.7" with no v prefix, and the installers resolve +# releases/download/v/..., so only v-prefixed tags describe a release +# a user can actually fetch. +newest_tag="" +if git rev-parse --git-dir >/dev/null 2>&1; then + newest_tag=$( + git tag -l 'v*' 2>/dev/null | + grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | + sed 's/^v//' | + sort -t. -k1,1n -k2,2n -k3,3n | + tail -1 + ) +fi + +if [[ -n "$newest_tag" ]]; then + echo "Truth: newest stable release tag is v$newest_tag" +else + # Not a skip of the contract — only of the half that needs git history. + echo "NOTE: no stable v* tags reachable (shallow clone, tarball, or --no-tags)." >&2 + echo "NOTE: the tag comparison is UNAVAILABLE; mutual consistency is still enforced." >&2 +fi + +# ── 2. Registered surfaces ────────────────────────────────────────── +# "release" = must equal the newest stable tag. Every one of these is +# version-only or carries a checksum we re-pin with it, so bumping is a +# mechanical edit with no external lookup beyond the release's own checksums.txt. +# +# "pin:" = frozen at a version we have NOT re-pinned. These embed a +# per-release sha256 of a published asset, so bumping them means re-pinning +# checksums — a distribution chore, not a metadata edit. They are still gated: +# the version must be internally consistent within the file and must be a +# version that really shipped. Every pin needs a reason below (PIN_REASONS), so +# a frozen surface can never become an unexplained exemption. +SURFACES=( + "server.json|release" + "pkg/npm/package.json|release" + "pkg/pypi/pyproject.toml|release" + "pkg/pypi/src/codebase_memory_mcp/_cli.py|release" + "pkg/go/cmd/codebase-memory-mcp/main.go|release" + "pkg/chocolatey/codebase-memory-mcp.nuspec|release" + "pkg/chocolatey/tools/chocolateyInstall.ps1|release" + "pkg/homebrew/Formula/codebase-memory-mcp.rb|pin:0.10.3" + "pkg/scoop/codebase-memory-mcp.json|pin:0.8.1" + "pkg/aur/PKGBUILD|pin:0.8.1" + "pkg/aur/.SRCINFO|pin:0.8.1" +) + +# Why each pinned surface is frozen, and what clearing it costs. To bring one +# up to date: download the release's checksums.txt +# (gh release download v --pattern checksums.txt), replace the version +# AND every sha256 it pins, then move the entry to "release" above. +PIN_REASONS=( + "pkg/homebrew/Formula/codebase-memory-mcp.rb|pins 4 per-asset sha256 (darwin/linux x arm/intel); last re-pinned for v0.10.3" + "pkg/scoop/codebase-memory-mcp.json|pins the windows-amd64.zip sha256; last re-pinned for v0.8.1" + "pkg/aur/PKGBUILD|pins sha256sums_x86_64 + sha256sums_aarch64; last re-pinned for v0.8.1" + "pkg/aur/.SRCINFO|generated from PKGBUILD, so it must move with it, not before it" +) + +# Paths that legitimately contain a semver but make no claim about the +# product's current version. Each entry states what its number really is, so an +# unexplained exemption cannot hide here. +EXEMPT=( + 'pkg/winget/manifests/' # published version-pinned manifests: the directory + # name IS the version, and rewriting a shipped + # manifest would falsify what 0.8.1 actually was + 'pkg/pypi/tests/' # unit-test fixtures; the literal is arbitrary + # cache-path input, not a published claim + 'pkg/npm/test/' # same: publication tests assert shape, not value + 'pkg/pypi/requirements-publish.txt' # publish-toolchain pins (twine/hatchling) + 'tests/test_version_metadata_contract.sh' # this file, which names the numbers above +) + +# ── 3. Extraction ─────────────────────────────────────────────────── +# Two independent claim forms, because a surface can lie in either one and the +# language-count contract learned the hard way that a gate sharing its regex +# with the enumeration can only confirm what the enumeration already saw: +# declaration version "X.Y.Z" / "version": "X.Y.Z" / pkgver=X.Y.Z / +# release URL releases/download/vX.Y.Z/... , releases/tag/vX.Y.Z , -X.Y.Z-linux +# The Homebrew formula interpolates v#{version} into its URLs and so has no URL +# form at all; that is fine, the anti-vacuity check below is per FILE, not per +# form. +versions_in() { + local path="$1" + { + # shellcheck disable=SC2016 # the literal text $version is what the + # PowerShell surface writes, so this regex must NOT expand it + grep -oiE '(pkgver|packageversion|\$version|"version"||^[[:space:]]*version)[^0-9]{0,16}[0-9]+\.[0-9]+\.[0-9]+' "$path" | + grep -oE '[0-9]+\.[0-9]+\.[0-9]+$' + grep -oE 'releases/(download|tag)/v[0-9]+\.[0-9]+\.[0-9]+' "$path" | + grep -oE '[0-9]+\.[0-9]+\.[0-9]+$' + grep -oE -- '-[0-9]+\.[0-9]+\.[0-9]+-(linux|darwin|windows)' "$path" | + grep -oE '[0-9]+\.[0-9]+\.[0-9]+' + # Bare literal form: pkg/pypi returns a hardcoded fallback from + # _version() when importlib.metadata cannot resolve the installed + # distribution, and that string is what `--version` then prints. The + # anti-vacuity check below caught this one: the declaration form above + # cannot see it, because the version keyword is in the function NAME. + grep -oE 'return "[0-9]+\.[0-9]+\.[0-9]+"' "$path" | + grep -oE '[0-9]+\.[0-9]+\.[0-9]+' + } | sort -u +} + +pin_reason_for() { + local want="$1" entry + for entry in "${PIN_REASONS[@]}"; do + if [[ "${entry%%|*}" == "$want" ]]; then + printf '%s' "${entry#*|}" + return 0 + fi + done + printf '' +} + +# ── 4. Every registered surface ───────────────────────────────────── +release_surface_count=0 +checked_values=0 + +for entry in "${SURFACES[@]}"; do + path="${entry%%|*}" + rule="${entry#*|}" + + if [[ ! -f "$path" ]]; then + fail "registered surface $path does not exist — update SURFACES" + continue + fi + + hits=$(versions_in "$path" || true) + + # Anti-vacuity. A surface that states no version is either no longer a + # surface or has changed wording in a way this contract can no longer read. + # Either way the silence is a defect, not a pass — the whole point of this + # gate is that nothing states a version unchecked. + if [[ -z "$hits" ]]; then + fail "$path is registered as a version surface but states no version —" \ + "either it lost the claim (drop it from SURFACES) or its wording" \ + "changed and this check went vacuous" + continue + fi + + if [[ "$rule" == "release" ]]; then + release_surface_count=$((release_surface_count + 1)) + else + pinned="${rule#pin:}" + reason="$(pin_reason_for "$path")" + if [[ -z "$reason" ]]; then + fail "$path is pinned at $pinned with no entry in PIN_REASONS —" \ + "a frozen surface must say WHY it is frozen" + fi + if [[ -n "$newest_tag" ]] && [[ "$pinned" == "$newest_tag" ]]; then + fail "$path is pinned at $pinned, which is now the newest release —" \ + "move it to the \"release\" rule and drop its PIN_REASONS entry" + fi + fi + + while IFS= read -r found; do + [[ -z "$found" ]] && continue + checked_values=$((checked_values + 1)) + + if [[ "$rule" == "release" ]]; then + # Mutual consistency runs with or without git; the tag comparison + # only when a tag exists. + if [[ -n "$newest_tag" ]] && [[ "$found" != "$newest_tag" ]]; then + fail "$path declares $found, the newest release is $newest_tag" + fi + else + pinned="${rule#pin:}" + if [[ "$found" != "$pinned" ]]; then + fail "$path declares $found but is pinned at $pinned —" \ + "it embeds a per-release sha256, so the version and the" \ + "checksums must move together (update PIN_REASONS or re-pin both)" + fi + if [[ -n "$newest_tag" ]] && + [[ "$(printf '%s\n%s\n' "$found" "$newest_tag" | + sort -t. -k1,1n -k2,2n -k3,3n | tail -1)" != "$newest_tag" ]]; then + fail "$path declares $found, which is NEWER than the newest" \ + "release tag v$newest_tag — it names a release that does not exist" + fi + fi + done <<<"$hits" +done + +# ── 5. Mutual consistency, independent of git ─────────────────────── +# This is the half that still works in a shallow clone. If the tag comparison +# above was unavailable, this is the only thing standing between the repo and +# silent drift, so it must never be conditional on git. +release_values="" +for entry in "${SURFACES[@]}"; do + path="${entry%%|*}" + [[ "${entry#*|}" == "release" ]] || continue + [[ -f "$path" ]] || continue + # `|| true` is load-bearing: versions_in ends in a pipeline and runs under + # pipefail, so a claim form absent from THIS file (server.json has no bare + # return form) makes it exit non-zero and set -e would kill the contract + # mid-run — exiting 1 with no message, which reads exactly like a failure + # that forgot to explain itself. + release_values="$release_values$(versions_in "$path" || true) +" +done +distinct=$(printf '%s' "$release_values" | grep -vE '^$' | sort -u || true) +distinct_count=$(printf '%s\n' "$distinct" | grep -cE '^[0-9]' || true) + +if ((distinct_count > 1)); then + fail "the release-tracking surfaces disagree with each other: $( + printf '%s' "$distinct" | tr '\n' ' ' + )" +fi +if ((distinct_count == 0)); then + fail "no version could be read from ANY release-tracking surface —" \ + "this contract cannot vouch for anything and must not report success" +fi + +# ── 6. No UNREGISTERED packaging surface may state a version ──────── +# Without this, adding an eighth packaging target reintroduces exactly the drift +# this contract exists to prevent, and every check above would still pass. The +# sweep pattern is deliberately BROADER than the extractors in section 3: if the +# two shared a regex, a claim form the extractors cannot see would be invisible +# to the sweep as well. +while IFS= read -r path; do + [[ -z "$path" ]] && continue + for entry in "${SURFACES[@]}"; do + [[ "$path" == "${entry%%|*}" ]] && continue 2 + done + for skip in "${EXEMPT[@]}"; do + [[ "$path" == "$skip"* ]] && continue 2 + done + fail "$path states a product version but is not registered. Add it to" \ + "SURFACES so it stays in step with the release, or to EXEMPT with a" \ + "note saying what its number actually is." +done < <( + git grep -IlE '(version|pkgver|releases/(download|tag)/v)[^0-9]{0,16}[0-9]+\.[0-9]+\.[0-9]+|return "[0-9]+\.[0-9]+\.[0-9]+"' \ + -- server.json pkg tests/test_version_metadata_contract.sh 2>/dev/null || true +) + +if ((failures > 0)); then + echo "FAIL: $failures version-metadata contract violation(s)" >&2 + exit 1 +fi + +if [[ -n "$newest_tag" ]]; then + echo "Version-metadata contract passed (${#SURFACES[@]} surfaces," \ + "$release_surface_count tracking v$newest_tag, $checked_values value(s) checked)" +else + echo "Version-metadata contract passed WITHOUT a tag truth source" \ + "(${#SURFACES[@]} surfaces, $checked_values value(s) checked, mutual consistency only)" +fi From ae8fd2c903b08010107b688017e5ee423191e73d Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 29 Aug 2026 11:03:25 +0200 Subject: [PATCH 24/31] fix(test): keep the version contract alive under pipefail with no tags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tag lookup and one mutual-consistency call ended in pipelines whose grep legitimately matches nothing — no v* tags in a --no-tags CI checkout, no bare-return form in server.json. Under `set -euo pipefail` that exit status killed the contract mid-run: it exited 1 having printed nothing, which reads like a failure that forgot to explain itself, and in the tagless case it died before it could report that its truth source was unavailable. Found by running the contract in a real `git clone --no-tags --depth 1`, not by inspection. With the guards, that clone now passes while announcing the degradation, and still fails on a drifted surface through mutual consistency alone. Signed-off-by: Martin Vogel --- tests/test_version_metadata_contract.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/test_version_metadata_contract.sh b/tests/test_version_metadata_contract.sh index a3878764f..b191597bf 100644 --- a/tests/test_version_metadata_contract.sh +++ b/tests/test_version_metadata_contract.sh @@ -65,12 +65,18 @@ fail() { # a user can actually fetch. newest_tag="" if git rev-parse --git-dir >/dev/null 2>&1; then + # `|| true` is load-bearing: under pipefail a tagless clone makes grep exit + # 1, and set -e would kill the contract HERE — exiting non-zero with no + # output at all, in exactly the venue (a --no-tags CI checkout) where this + # gate most needs to say what it could and could not verify. newest_tag=$( - git tag -l 'v*' 2>/dev/null | - grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | - sed 's/^v//' | - sort -t. -k1,1n -k2,2n -k3,3n | - tail -1 + { + git tag -l 'v*' 2>/dev/null | + grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | + sed 's/^v//' | + sort -t. -k1,1n -k2,2n -k3,3n | + tail -1 + } || true ) fi From 82c4067f79593279f7b6ae5f0e62b5117631ce4a Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 29 Aug 2026 11:26:00 +0200 Subject: [PATCH 25/31] fix(test): release the node retrieved per USAGE edge in the Chialisp contract The cross-file Constant resolution test walks USAGE edges and calls cbm_store_find_node_by_id() for each source, which populates the caller's cbm_node_t with heap_strdup'd fields (store.c:195 via scan_node). It freed the arrays it owned -- consts and edges -- but never released the per-edge node, leaking one set of fields per iteration. LeakSanitizer caught it in every sanitizer-enabled lane while the plain test shards and both Windows legs stayed green, which is why the local test-runner could not see it: test-lsan-macos 417 bytes in 6 allocations test-unix (ubuntu-latest, 2/3) 401 bytes in 6 allocations test-unix (ubuntu-24.04-arm, 2/3) same signature test-diag same signature Use cbm_node_free_fields(), not cbm_store_free_nodes(). The latter frees the array pointer itself after releasing each element's fields, so calling it on a stack-allocated node is a bad free -- ASan aborts with "bad-free test_lang_contract.c" if you try. cbm_node_free_fields() releases the strdup'd members and leaves the struct alone, which is the idiom the neighbouring probes already use (test_edge_types_probe.c:245). The three other find_node_by_id call sites in this file predate this branch and already handle their own cleanup; only the new Chialisp contract leaked. Signed-off-by: Martin Vogel Co-authored-by: darkverbito <8049985+darkverbito@users.noreply.github.com> Signed-off-by: Martin Vogel --- tests/test_lang_contract.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_lang_contract.c b/tests/test_lang_contract.c index 2c2441767..5a32bcda0 100644 --- a/tests/test_lang_contract.c +++ b/tests/test_lang_contract.c @@ -737,6 +737,7 @@ TEST(contract_chialisp_constant_resolves_across_files) { if (src.file_path && strstr(src.file_path, "puzzle.clsp")) { cross_file++; } + cbm_node_free_fields(&src); } cbm_store_free_edges(edges, nedges); cbm_store_free_nodes(consts, nconst); From b6a22843207af16aa533952dce4a697d2e26a241 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 29 Aug 2026 11:44:03 +0200 Subject: [PATCH 26/31] fix(pipeline): suppress weak Python member calls (distilled from #1324) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python attribute calls were not classified as member calls, so weak short-name strategies (suffix_match / unique_name / field_type_hint) could bind `obj.method()` to an arbitrary same-named project symbol — `accelerator.print()` resolving to MockAccelerator.print. Flag a Python attribute call is_method when its receiver is unresolved, and extend the existing TS/JS weak-member guard to Python. Receivers that ARE known stay exempt: self/cls, super(), and any identifier bound by one of the file's imports (including the root of an attribute chain), so Python's canonical `module.function()` cross-file call still resolves. The guard helper is renamed cbm_tsjs_suppress_weak_method_match -> cbm_suppress_weak_member_match. Its drop-list is unchanged, and the per-language gate stays at the two call sites, which enumerate the same language set in pass_calls.c and pass_parallel.c so the sequential and parallel resolvers cannot diverge. ArkTS remains in that set. python/S6/inherited_method in the resolution probe now asserts calls == 0, recording the same trade TypeScript took when the receiver-aware guard landed: the edge it used to produce came from a unique_name registry fallback (measured on main: strategy=unique_name, cands=1, conf=0.7500) where a weak short-name guess happened to be right in a 2-file fixture. It flips back to >= 1 once py_lsp_cross resolves inheritance, matching the flip-back note already carried on the TypeScript S6 case. Fixes #1276 Signed-off-by: Martin Vogel Co-authored-by: Enferlain <15861396+Enferlain@users.noreply.github.com> --- internal/cbm/cbm.h | 8 +- internal/cbm/extract_calls.c | 74 ++++++++++++++ src/pipeline/pass_calls.c | 39 +++++--- src/pipeline/pass_parallel.c | 35 ++++--- src/pipeline/pipeline.h | 13 ++- src/pipeline/registry.c | 34 ++++--- tests/test_extraction.c | 70 +++++++++++++ tests/test_lsp_resolution_probe.c | 29 ++++-- tests/test_pipeline.c | 160 ++++++++++++++++++++++++++++++ tests/test_registry.c | 48 ++++----- 10 files changed, 430 insertions(+), 80 deletions(-) diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index ac187406e..3ea5c48f5 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -259,9 +259,13 @@ typedef struct { uint32_t site_start_byte; // exact AST occurrence span; end > start when present uint32_t site_end_byte; // exclusive byte offset in the source file CBMSourceOrigin source_origin; // raw source or C-family preprocessed buffer - bool is_method; // method/member call with a non-self receiver. Perl: + bool is_method; // method/member call with an UNRESOLVED receiver. Perl: // arrow/method call ($obj->m). TS/JS/TSX: member call - // x.foo() whose receiver is not this/super. Default false. + // x.foo() whose receiver is not this/super. Python: + // x.foo() where x is not self/cls/super() and is not + // rooted in an imported name. Read by the weak-member + // guard and by the pxc synthetic-carrier dedup key in + // pass_lsp_cross.c. Default false. bool requires_lsp_resolution; // synthetic semantic candidate (for example an implicit // C++ operator). Never fall back to textual resolution. } CBMCall; diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index abfe21140..7e5dcecae 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -2740,6 +2740,66 @@ static char *resolve_objectscript_instance_call(CBMArena *a, TSNode node, const return NULL; } +/* True when a Python attribute-call receiver is EXEMPT from the weak-member + * guard (#1276). Three exemptions, each because the receiver is in fact known: + * - `self.x()` / `cls.x()` — a direct self/cls receiver keeps class-local + * semantics; the enclosing class is the right namespace for a weak match. + * - `super().x()` — same, via the base class. + * - `helper.compute()` — an identifier bound by one of THIS file's + * imports, including the root of an attribute chain (`pkg.sub.fn()`). + * module.function() is Python's canonical cross-file call shape and the + * import map resolves it; flagging it would kill the true edge. + * Everything else — a parameter, a local, an attribute of self — has no + * statically-known type here, so the call must not bind by short name alone. + * Note `self.client.send()` is NOT exempt: the receiver is `self.client`, an + * attribute of unknown type, not `self` itself. */ +static bool python_receiver_is_exempt(CBMExtractCtx *ctx, TSNode receiver) { + if (ts_node_is_null(receiver)) { + return false; + } + + /* super().m() — the receiver is a call node whose function is `super`. */ + if (strcmp(ts_node_type(receiver), "call") == 0) { + TSNode fn = ts_node_child_by_field_name(receiver, TS_FIELD("function")); + if (!ts_node_is_null(fn) && strcmp(ts_node_type(fn), "identifier") == 0) { + char *name = cbm_node_text(ctx->arena, fn, ctx->source); + return name && strcmp(name, "super") == 0; + } + return false; + } + + /* Walk an attribute chain down to its root identifier: for `pkg.sub.fn()` + * the receiver is `pkg.sub`, whose root is `pkg` — the name an import binds. */ + bool direct_identifier = strcmp(ts_node_type(receiver), "identifier") == 0; + TSNode root = receiver; + while (!ts_node_is_null(root) && strcmp(ts_node_type(root), "attribute") == 0) { + root = ts_node_child_by_field_name(root, TS_FIELD("object")); + } + if (ts_node_is_null(root) || strcmp(ts_node_type(root), "identifier") != 0) { + return false; + } + + char *name = cbm_node_text(ctx->arena, root, ctx->source); + if (!name) { + return false; + } + /* self/cls only as a DIRECT receiver: `self.m()` is class-local, but + * `self.client.m()` has receiver `self.client` of unknown type. */ + if (direct_identifier && (strcmp(name, "self") == 0 || strcmp(name, "cls") == 0)) { + return true; + } + /* Import-bound root, incl. aliases (`import tools as toolkit` binds + * local_name "toolkit"). Per-file scan: imports.count is a file-local + * number (tens), never the corpus, so this stays O(file), not O(corpus). */ + for (int i = 0; i < ctx->result->imports.count; i++) { + const char *local_name = ctx->result->imports.items[i].local_name; + if (local_name && strcmp(local_name, name) == 0) { + return true; + } + } + return false; +} + static bool is_objectscript_language(CBMLanguage language) { return language == CBM_LANG_OBJECTSCRIPT_UDL || language == CBM_LANG_OBJECTSCRIPT_ROUTINE; } @@ -3357,6 +3417,20 @@ CBMInvocationDescriptor handle_calls(CBMExtractCtx *ctx, TSNode node, const CBML strcmp(ts_node_type(node), "method_call_expression") == 0) { call.is_method = true; } + // Python receiver-aware guard (#1276; same intent as the Perl and + // TS/JS flags). Flag an attribute call x.foo() whose receiver is not + // self/cls/super() and is not rooted in an imported name, so the + // call-resolution pass can suppress weak short-name matches for it + // (`accelerator.print()` must not bind MockAccelerator.print). + // Imported receivers stay unflagged: module.function() is Python's + // canonical cross-file call and the import map resolves it. + if (ctx->language == CBM_LANG_PYTHON && strcmp(ts_node_type(node), "call") == 0) { + TSNode fn = ts_node_child_by_field_name(node, TS_FIELD("function")); + if (!ts_node_is_null(fn) && strcmp(ts_node_type(fn), "attribute") == 0) { + TSNode obj = ts_node_child_by_field_name(fn, TS_FIELD("object")); + call.is_method = !python_receiver_is_exempt(ctx, obj); + } + } // TS/JS/TSX receiver-aware guard (#592/#606 direction; same intent // as the Perl flag above). Flag a member call x.foo() whose receiver // is NOT `this`/`super`. When the TS-LSP cannot resolve the receiver diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index 64255e27f..b25e9f592 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -606,20 +606,29 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, return 0; } - /* TS/JS/TSX weak-method suppression (#592/#606). A member call x.foo() only - * reaches the registry when the TS-LSP could not resolve the receiver type - * (the LSP block above already returned for type-resolved calls, including - * the "resolved but target out of gbuf" fall-through). Binding such a call - * by a weak short-name strategy fabricates an edge (`re.test()` -> a project - * `test`). Rather than drop it here — which would also skip the service - * bypasses below and emit_classified_edge's route/HTTP/CONFIG branches — - * defer to emit_classified_edge and suppress ONLY the plain-CALLS - * fall-through, so every service edge stays main-identical. res.strategy may - * be lsp_* here; the helper's explicit drop-list leaves lsp_* untouched. */ - bool is_tsjs = lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || - lang == CBM_LANG_TSX || lang == CBM_LANG_ARKTS; - bool tsjs_drop_plain_call = - cbm_tsjs_suppress_weak_method_match(is_tsjs, call->is_method, res.strategy); + /* Dynamic-language weak-member suppression (#592/#606/#1276). A member call + * x.foo() only reaches the registry when the language's LSP could not + * resolve the receiver type (the LSP block above already returned for + * type-resolved calls, including the "resolved but target out of gbuf" + * fall-through). Binding such a call by a weak short-name strategy + * fabricates an edge (`re.test()` -> a project `test`, + * `accelerator.print()` -> MockAccelerator.print). Rather than drop it here + * — which would also skip the service bypasses below and + * emit_classified_edge's route/HTTP/CONFIG branches — defer to + * emit_classified_edge and suppress ONLY the plain-CALLS fall-through, so + * every service edge stays main-identical. res.strategy may be lsp_* here; + * the helper's explicit drop-list leaves lsp_* untouched. + * + * This language set MUST match the one in pass_parallel.c exactly — a + * language gated on only one resolver produces an edge on the sequential + * path and not the parallel one (or vice versa), breaking MT determinism. + * ArkTS belongs to the JS/TS family here (#1842); dropping it would + * reintroduce the #592/#606 false-edge class for .ets files. */ + bool suppress_weak_member = lang == CBM_LANG_PYTHON || lang == CBM_LANG_JAVASCRIPT || + lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX || + lang == CBM_LANG_ARKTS; + bool drop_plain_call = + cbm_suppress_weak_member_match(suppress_weak_member, call->is_method, res.strategy); /* Service-pattern HTTP/ASYNC calls to an EXTERNAL client library (e.g. * `requests.get("/api/orders/{id}")`) resolve to a QN containing the library @@ -652,7 +661,7 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, return 0; } emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys, imp_vals, - imp_count, tsjs_drop_plain_call); + imp_count, drop_plain_call); return SKIP_ONE; } diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index fbc4c6160..1eeb55f83 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -2465,20 +2465,25 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB continue; } - /* TS/JS/TSX weak-method suppression (#592/#606). The receiver-aware guard - * must NOT drop this call here: doing so would also skip the #523 - * callee-name service bypass below, emit_service_edge's route/gRPC/config - * branches, and its unconditional detect_url_in_args (which classifies - * verb-suffix HTTP clients like api.patch('/x')). Instead, defer to the - * emit path and suppress ONLY the plain-CALLS fall-through - * (emit_normal_calls_edge), so every service edge stays main-identical by - * construction. res.strategy may carry an lsp_* value here (LSP-resolved - * calls keep res through this point); the helper's EXPLICIT drop-list - * leaves lsp_ts_method / lsp_cross untouched. See #606 direction. */ - bool is_tsjs = lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || - lang == CBM_LANG_TSX || lang == CBM_LANG_ARKTS; - bool tsjs_drop_plain_call = - cbm_tsjs_suppress_weak_method_match(is_tsjs, call->is_method, res.strategy); + /* Dynamic-language weak-member suppression (#592/#606/#1276). The + * receiver-aware guard must NOT drop this call here: doing so would also + * skip the #523 callee-name service bypass below, emit_service_edge's + * route/gRPC/config branches, and its unconditional detect_url_in_args + * (which classifies verb-suffix HTTP clients like api.patch('/x')). + * Instead, defer to the emit path and suppress ONLY the plain-CALLS + * fall-through (emit_normal_calls_edge), so every service edge stays + * main-identical by construction. res.strategy may carry an lsp_* value + * here (LSP-resolved calls keep res through this point); the helper's + * EXPLICIT drop-list leaves lsp_ts_method / lsp_cross untouched. See + * #606 direction. + * + * This language set MUST match the one in pass_calls.c exactly — see the + * note there. ArkTS belongs to the JS/TS family (#1842). */ + bool suppress_weak_member = lang == CBM_LANG_PYTHON || lang == CBM_LANG_JAVASCRIPT || + lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX || + lang == CBM_LANG_ARKTS; + bool drop_plain_call = + cbm_suppress_weak_member_match(suppress_weak_member, call->is_method, res.strategy); /* Service-pattern HTTP/ASYNC client call (`requests.get(url)`): the * service signal lives in the callee_name. The registry can mis-resolve @@ -2573,7 +2578,7 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB _rc_t0 = extract_now_ns(); emit_service_edge(ws->local_edge_buf, source_node, target_node, call, &res, module_qn, rc->registry, rc->main_gbuf, imp_keys, imp_vals, imp_count, - tsjs_drop_plain_call); + drop_plain_call); atomic_fetch_add_explicit(&rc->time_ns_rc_emit, extract_now_ns() - _rc_t0, memory_order_relaxed); ws->calls_resolved++; diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index e54732df5..4b1d15563 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -269,13 +269,16 @@ bool cbm_perl_is_builtin(const char *name); bool cbm_perl_suppress_generic_match(bool is_perl, bool is_method, const char *callee_name, const char *strategy); -/* Decide whether a resolved TS/JS/TSX member-call edge is weak-strategy noise to - * drop (#592/#606): true only for TS/JS, only for a member call with a - * non-this/super receiver (is_method), and only when the match used a weak - * short-name strategy (suffix_match / unique_name / field_type_hint / fuzzy). +/* Decide whether a resolved member-call edge is weak-strategy noise to drop + * (#592/#606/#1276): true only when the CALLER's per-language gate says the + * guard applies (`enabled`), only for a member call with an unresolved receiver + * (is_method), and only when the match used a weak short-name strategy + * (suffix_match / unique_name / field_type_hint / fuzzy). * Explicit drop-list keeps every lsp_* / import / same-module / qualified match. + * The language set lives at the call sites (pass_calls.c / pass_parallel.c) and + * must be identical in both, or the sequential and parallel resolvers diverge. * Pure; unit-tested in test_registry.c. */ -bool cbm_tsjs_suppress_weak_method_match(bool is_tsjs, bool is_method, const char *strategy); +bool cbm_suppress_weak_member_match(bool enabled, bool is_method, const char *strategy); /* #725: drop a suffix_match CALLS edge when the caller language and the * target file's language disagree. unique_name (candidates == 1) is #1572 diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index 9f2f430b5..5126bcbfe 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -429,19 +429,27 @@ bool cbm_perl_suppress_generic_match(bool is_perl, bool is_method, const char *c return true; /* weak short-name match (suffix_match / unique_name / …) → drop */ } -/* TS/JS analogue of the Perl guard above (#592/#606 direction; precedent #477). - * A member call `x.foo()` reaches the weak textual cascade ONLY when the TS-LSP - * could not resolve the receiver type — type-resolved calls win via lsp_* - * strategies before the registry runs. Binding such a call to a project symbol - * by a weak short-name strategy fabricates a CALLS edge (`re.test()` -> - * SalesforceRestClient.test, `date.toISOString()` -> any project toISOString). - * Drop ONLY the weak strategies; keep import/same-module/qualified-tail matches - * and every lsp_* strategy. Uses an EXPLICIT drop-list (not keep-list + - * default-drop) because the parallel resolver runs lsp_* strategies through the - * same guard variable — a default-drop would silently kill lsp_ts_method. Pure - * + side-effect-free so the contract is unit-testable without a full pipeline. */ -bool cbm_tsjs_suppress_weak_method_match(bool is_tsjs, bool is_method, const char *strategy) { - if (!is_tsjs || !is_method || !strategy || !strategy[0]) { +/* Dynamic-language analogue of the Perl guard above (#592/#606/#1276 + * direction; precedent #477). A member call `x.foo()` reaches the weak textual + * cascade ONLY when the language's LSP could not resolve the receiver type — + * type-resolved calls win via lsp_* strategies before the registry runs. + * Binding such a call to a project symbol by a weak short-name strategy + * fabricates a CALLS edge (`re.test()` -> SalesforceRestClient.test, + * `accelerator.print()` -> MockAccelerator.print). Drop ONLY the weak + * strategies; keep import/same-module/qualified-tail matches and every lsp_* + * strategy. Uses an EXPLICIT drop-list (not keep-list + default-drop) because + * the parallel resolver runs lsp_* strategies through the same guard variable — + * a default-drop would silently kill lsp_ts_method. Pure + side-effect-free so + * the contract is unit-testable without a full pipeline. + * + * `enabled` is the CALLER's per-language gate, deliberately kept OUT of this + * helper: the guard applies only to the language set each call site enumerates + * (today Python plus the JS/TS family including ArkTS). Widening it is a + * per-language decision made at the call sites in pass_calls.c and + * pass_parallel.c, which MUST stay in lockstep — a gate added to only one of + * them diverges the sequential and parallel resolvers. */ +bool cbm_suppress_weak_member_match(bool enabled, bool is_method, const char *strategy) { + if (!enabled || !is_method || !strategy || !strategy[0]) { return false; } /* Weak short-name strategies that actually reach the call-resolution guards: diff --git a/tests/test_extraction.c b/tests/test_extraction.c index b34b14024..73219f4da 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -4552,6 +4552,75 @@ TEST(extract_flag_exempt_method_call_not_flagged_is_method) { PASS(); } +/* Python receiver-aware flag (#1276; same intent as the Perl and TS/JS flags). + * Pins BOTH directions: an unknown receiver (a parameter, or an attribute of + * self) IS flagged so the resolver can suppress a weak short-name match, while + * self/cls/super() and import-bound receivers — Python's canonical cross-file + * call shape — are NOT, so their true edges survive. */ +TEST(extract_python_member_call_flags_is_method) { + CBMFileResult *r = extract("from pkg import helper\n" + "import tools as toolkit\n" + "\n" + "class C(Base):\n" + " def run(self, external):\n" + " external.commit()\n" + " self.client.send()\n" + " self.helper()\n" + " super ( ).render()\n" + " helper.compute()\n" + " toolkit.format()\n" + " helper()\n", + CBM_LANG_PYTHON, "t", "x.py"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + int external = 0; + int nested = 0; + int self_call = 0; + int super_call = 0; + int imported_module = 0; + int imported_alias = 0; + int bare = 0; + for (int i = 0; i < r->calls.count; i++) { + const char *cn = r->calls.items[i].callee_name; + if (strcmp(cn, "external.commit") == 0) { + /* parameter receiver — unknown type */ + external++; + ASSERT_TRUE(r->calls.items[i].is_method); + } else if (strcmp(cn, "self.client.send") == 0) { + /* receiver is `self.client`, an attribute of unknown type — NOT self */ + nested++; + ASSERT_TRUE(r->calls.items[i].is_method); + } else if (strcmp(cn, "self.helper") == 0) { + self_call++; + ASSERT_FALSE(r->calls.items[i].is_method); + } else if (strstr(cn, "render") != NULL) { + super_call++; + ASSERT_FALSE(r->calls.items[i].is_method); + } else if (strcmp(cn, "helper.compute") == 0) { + imported_module++; + ASSERT_FALSE(r->calls.items[i].is_method); + } else if (strcmp(cn, "toolkit.format") == 0) { + /* aliased import: local_name is "toolkit" */ + imported_alias++; + ASSERT_FALSE(r->calls.items[i].is_method); + } else if (strcmp(cn, "helper") == 0) { + bare++; + ASSERT_FALSE(r->calls.items[i].is_method); + } + } + /* Each shape must appear exactly once, so a missed extraction cannot make + * the loop above pass vacuously. */ + ASSERT_EQ(external, 1); + ASSERT_EQ(nested, 1); + ASSERT_EQ(self_call, 1); + ASSERT_EQ(super_call, 1); + ASSERT_EQ(imported_module, 1); + ASSERT_EQ(imported_alias, 1); + ASSERT_EQ(bare, 1); + cbm_free_result(r); + PASS(); +} + /* TS/JS/TSX receiver-aware flag (#592/#606; same intent as the Perl flag above). * A member call x.foo() with a non-this/super receiver is flagged is_method so * the resolver can suppress a weak short-name match (`re.test()` must not bind a @@ -6230,6 +6299,7 @@ SUITE(extraction) { RUN_TEST(extract_perl_builtin_call_is_function_not_method); RUN_TEST(extract_perl_method_call_flags_is_method); RUN_TEST(extract_flag_exempt_method_call_not_flagged_is_method); + RUN_TEST(extract_python_member_call_flags_is_method); RUN_TEST(extract_ts_member_call_flags_is_method); RUN_TEST(extract_ts_this_super_receiver_not_flagged); RUN_TEST(extract_js_member_call_flags_is_method); diff --git a/tests/test_lsp_resolution_probe.c b/tests/test_lsp_resolution_probe.c index e45043905..7386710f9 100644 --- a/tests/test_lsp_resolution_probe.c +++ b/tests/test_lsp_resolution_probe.c @@ -952,11 +952,27 @@ TEST(lrp_python_s6_inherited_method) { {"child.py", "from .base import Base\n\n\nclass Child(Base):\n" " def extra(self):\n return 'extra'\n\n\n" "def run(c):\n return c.describe()\n"}}; - /* Uncertain: c.describe() on a Child — py_lsp_cross must see Child inherits Base - * (requires INHERITS edge resolution). Given the Python extraction bug for - * base_classes, this may be RED end-to-end even if py_lsp_cross is correct. - * Assert the correct outcome; RED if extraction bug blocks resolution. */ - ASSERT_TRUE(lrp_assert_calls(f, 2, 1, "python/S6/inherited_method", 0)); + /* RED: c.describe() needs py_lsp_cross to walk `class Child(Base)`. Exactly + * the TS/S6 situation above, and resolved the same way (#1276). + * Until the receiver-aware guard landed, this scenario passed via a + * unique_name registry fallback — MEASURED on main before the guard: + * strategy=unique_name, cands=1, conf=0.7500. "describe" is the only symbol + * of that name in this 2-file fixture, so a weak short-name guess happened + * to be right. In a real repo the same guess binds an arbitrary same-named + * method (the false edges #1276 targets: accelerator.print() -> + * MockAccelerator.print), so the guard now suppresses weak member-call + * matches whose receiver is unresolved — here `c`, a bare parameter. + * c.describe() is the ONLY call in the fixture, so a correctly-suppressed + * run yields exactly zero CALLS. + * Tripwire: assert the store opened AND calls == 0 exactly, so an infra/DB + * failure cannot pass vacuously; flip to ASSERT calls >= 1 once py_lsp_cross + * resolves inheritance (lsp_py_*, a strategy the guard keeps). */ + LRP_Proj lp; + cbm_store_t *store = lrp_index(&lp, f, 2); + ASSERT_NOT_NULL(store); + int calls = cbm_store_count_edges_by_type(store, lp.project, "CALLS"); + lrp_cleanup(&lp, store); + ASSERT_EQ(calls, 0); PASS(); } @@ -1776,7 +1792,8 @@ SUITE(lsp_resolution_probe) { RUN_TEST(lrp_rust_s8_field_call); RUN_TEST(lrp_rust_crossfile_macro_hidden_call_carrier); - /* ── Python (lsp_cross WIRED) — S1–S5,S7,S8 GREEN; S6 uncertain (extraction bug) ── */ + /* ── Python (lsp_cross WIRED) — S1–S5,S7,S8 GREEN; S6 suppressed by the + * weak-member guard (#1276), asserted == 0 until py_lsp_cross inherits ── */ RUN_TEST(lrp_python_s1_crossfile_call); RUN_TEST(lrp_python_s2_method_dispatch); RUN_TEST(lrp_python_s3_constructor); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 1eb10290d..a45541e57 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -4627,6 +4627,67 @@ TEST(pipeline_tsjs_receiver_suppresses_weak_method_edge) { PASS(); } +/* Python counterpart to the TS/JS receiver guard (#1276), sequential path. + * Pins BOTH directions. NEGATIVE: an attribute call on an unknown receiver + * (a parameter) must not bind the lone same-named project method through + * unique_name. POSITIVE: an import-bound receiver (helper.compute) and a bare + * local call must still produce their CALLS edges. + * Fewer than 50 files exercises pass_calls.c. */ +TEST(pipeline_python_receiver_suppresses_weak_method_edge) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_python_recv_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + + write_temp_file(tmp, "worker.py", + "class Worker:\n" + " def backward(self):\n" + " return 1\n"); + write_temp_file(tmp, "pkg/__init__.py", "from .helper import compute\n"); + write_temp_file(tmp, "pkg/helper.py", + "def compute(value):\n" + " return value * 2\n"); + write_temp_file(tmp, "caller.py", + "from pkg import helper\n" + "\n" + "def external_call(accelerator):\n" + " return accelerator.backward()\n" + "\n" + "def imported_call():\n" + " return helper.compute(42)\n" + "\n" + "def local_helper():\n" + " return 1\n" + "\n" + "def bare_call():\n" + " return local_helper()\n"); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/python_recv.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + const char *project = cbm_pipeline_project_name(p); + + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + + /* NEGATIVE: `accelerator` is a parameter — Worker.backward must not bind. */ + ASSERT_FALSE(cross_file_call_exists(s, project, "external_call", "backward")); + /* POSITIVE: import-bound receiver and bare local call both survive. */ + ASSERT_TRUE(cross_file_call_exists(s, project, "imported_call", "compute")); + ASSERT_TRUE(cross_file_call_exists(s, project, "bare_call", "local_helper")); + /* Tripwire: a run that emitted no edges at all would satisfy the negative + * assertion vacuously. */ + ASSERT_GTE(cbm_store_count_edges_by_type(s, project, "CALLS"), 1); + + cbm_store_close(s); + cbm_pipeline_free(p); + th_rmtree(tmp); + PASS(); +} + /* Count nodes with the given exact name in the project (e.g. a Route path). */ static int count_nodes_named(cbm_store_t *s, const char *project, const char *name) { cbm_node_t *ns = NULL; @@ -4775,6 +4836,103 @@ TEST(pipeline_tsjs_receiver_parallel_keeps_service_edges) { PASS(); } +/* Parallel Python regression for #1276. The field-type heuristic capitalizes + * the receiver token and previously promoted accelerator.print() to + * MockAccelerator.print at 0.85; ordinary suffix matching also selected one + * arbitrary backward()/step() implementation. All are unresolved-receiver + * calls and must remain absent, while the import-bound call and the bare local + * call remain — the same both-directions pin as the sequential test above. + * >= 50 files forces pass_parallel.c and try_field_type_hint(), so this is also + * the guard against a sequential/parallel divergence. */ +TEST(pipeline_python_receiver_parallel_suppresses_weak_method_edges) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_python_par_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + + write_temp_file(tmp, "targets.py", + "class MockAccelerator:\n" + " def print(self, value):\n" + " return value\n" + "\n" + "class OtherPrinter:\n" + " def print(self, value):\n" + " return value\n" + "\n" + "class FlashAttentionFunction:\n" + " def backward(self, loss):\n" + " return loss\n" + "\n" + "class OtherBackward:\n" + " def backward(self, loss):\n" + " return loss\n" + "\n" + "class EulerScheduler:\n" + " def step(self):\n" + " return 1\n" + "\n" + "class OtherScheduler:\n" + " def step(self):\n" + " return 1\n"); + write_temp_file(tmp, "pkg/__init__.py", "from .helper import compute\n"); + write_temp_file(tmp, "pkg/helper.py", + "def compute(value):\n" + " return value * 2\n"); + write_temp_file(tmp, "caller.py", + "from pkg import helper\n" + "\n" + "def local_helper():\n" + " return 1\n" + "\n" + "def train(accelerator, trainer):\n" + " accelerator.print('hello')\n" + " accelerator.backward(1)\n" + " trainer.lr_scheduler.step()\n" + " helper.compute(42)\n" + " return local_helper()\n"); + for (int i = 0; i < 52; i++) { + char name[64]; + char body[128]; + snprintf(name, sizeof(name), "filler%d.py", i); + snprintf(body, sizeof(body), "def filler%d():\n return %d\n", i, i); + write_temp_file(tmp, name, body); + } + + char *old_workers = getenv("CBM_WORKERS"); + char *saved = old_workers ? strdup(old_workers) : NULL; + cbm_setenv("CBM_WORKERS", "4", 1); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/python_par.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + const char *project = cbm_pipeline_project_name(p); + + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + + /* NEGATIVE: every receiver here is a parameter or an attribute of one. */ + ASSERT_FALSE(cross_file_call_exists(s, project, "train", "print")); + ASSERT_FALSE(cross_file_call_exists(s, project, "train", "backward")); + ASSERT_FALSE(cross_file_call_exists(s, project, "train", "step")); + /* POSITIVE: import-bound and bare local calls survive the parallel path too. */ + ASSERT_TRUE(cross_file_call_exists(s, project, "train", "compute")); + ASSERT_TRUE(cross_file_call_exists(s, project, "train", "local_helper")); + + cbm_store_close(s); + cbm_pipeline_free(p); + if (saved) { + cbm_setenv("CBM_WORKERS", saved, 1); + free(saved); + } else { + cbm_unsetenv("CBM_WORKERS"); + } + th_rmtree(tmp); + PASS(); +} + /* Reproduce-first: pass_parallel's fused cross-LSP eligibility currently counts * only parser-backed calls/call references. A Python binary operator has no * parser CBMCall; its __add__ semantic record and carrier are created together @@ -12646,7 +12804,9 @@ SUITE(pipeline) { RUN_TEST(pipeline_incremental_parallel_result_cache_alloc_failure_preserves_db_and_retries); #endif RUN_TEST(pipeline_tsjs_receiver_suppresses_weak_method_edge); + RUN_TEST(pipeline_python_receiver_suppresses_weak_method_edge); RUN_TEST(pipeline_tsjs_receiver_parallel_keeps_service_edges); + RUN_TEST(pipeline_python_receiver_parallel_suppresses_weak_method_edges); RUN_TEST(pipeline_parallel_python_cross_only_dunder_gets_synthetic_carrier); RUN_TEST(pipeline_parallel_rust_cross_only_macro_hidden_gets_synthetic_carrier); RUN_TEST(pipeline_native_fetch_classified_as_http_calls); diff --git a/tests/test_registry.c b/tests/test_registry.c index 75cf4f630..ff81a50ee 100644 --- a/tests/test_registry.c +++ b/tests/test_registry.c @@ -811,21 +811,21 @@ TEST(cross_language_suffix_match_drops_py_vs_js) { PASS(); } -TEST(tsjs_suppress_drops_weak_method_matches) { - /* #592/#606: a TS/JS member call whose receiver the LSP could not type, that +TEST(dynamic_suppress_drops_weak_method_matches) { + /* #592/#606/#1276: a member call whose receiver the LSP could not type, that * landed via a WEAK short-name strategy, is generic-resolver noise → drop. * The strategies that actually reach the guards are the registry's * suffix_match / unique_name and the parallel field_type_hint; "fuzzy" is * covered defensively (cbm_registry_fuzzy_resolve is not wired into the * resolvers today) so a future wiring cannot silently reintroduce it. */ - ASSERT_TRUE(cbm_tsjs_suppress_weak_method_match(true, true, "suffix_match")); - ASSERT_TRUE(cbm_tsjs_suppress_weak_method_match(true, true, "unique_name")); - ASSERT_TRUE(cbm_tsjs_suppress_weak_method_match(true, true, "field_type_hint")); - ASSERT_TRUE(cbm_tsjs_suppress_weak_method_match(true, true, "fuzzy")); + ASSERT_TRUE(cbm_suppress_weak_member_match(true, true, "suffix_match")); + ASSERT_TRUE(cbm_suppress_weak_member_match(true, true, "unique_name")); + ASSERT_TRUE(cbm_suppress_weak_member_match(true, true, "field_type_hint")); + ASSERT_TRUE(cbm_suppress_weak_member_match(true, true, "fuzzy")); PASS(); } -TEST(tsjs_suppress_keeps_high_confidence_and_non_methods) { +TEST(dynamic_suppress_keeps_high_confidence_and_non_methods) { /* Keep every receiver-/import-aware strategy. Because the PARALLEL resolver * runs lsp_* strategies through this same guard variable, an explicit * drop-list must never touch them — asserting the lsp_* keeps here is the @@ -833,23 +833,23 @@ TEST(tsjs_suppress_keeps_high_confidence_and_non_methods) { * enumerates the resolver's non-weak strategies: registry * {import_map, import_map_suffix, same_module, qualified_suffix}, parallel * {callee_suffix, service_pattern}, and lsp_*. */ - ASSERT_FALSE(cbm_tsjs_suppress_weak_method_match(true, true, "same_module")); - ASSERT_FALSE(cbm_tsjs_suppress_weak_method_match(true, true, "import_map")); - ASSERT_FALSE(cbm_tsjs_suppress_weak_method_match(true, true, "import_map_suffix")); - ASSERT_FALSE(cbm_tsjs_suppress_weak_method_match(true, true, "qualified_suffix")); - ASSERT_FALSE(cbm_tsjs_suppress_weak_method_match(true, true, "callee_suffix")); - ASSERT_FALSE(cbm_tsjs_suppress_weak_method_match(true, true, "service_pattern")); - ASSERT_FALSE(cbm_tsjs_suppress_weak_method_match(true, true, "lsp_ts_method")); - ASSERT_FALSE(cbm_tsjs_suppress_weak_method_match(true, true, "lsp_cross")); - ASSERT_FALSE(cbm_tsjs_suppress_weak_method_match(true, true, "lsp_ts_local")); + ASSERT_FALSE(cbm_suppress_weak_member_match(true, true, "same_module")); + ASSERT_FALSE(cbm_suppress_weak_member_match(true, true, "import_map")); + ASSERT_FALSE(cbm_suppress_weak_member_match(true, true, "import_map_suffix")); + ASSERT_FALSE(cbm_suppress_weak_member_match(true, true, "qualified_suffix")); + ASSERT_FALSE(cbm_suppress_weak_member_match(true, true, "callee_suffix")); + ASSERT_FALSE(cbm_suppress_weak_member_match(true, true, "service_pattern")); + ASSERT_FALSE(cbm_suppress_weak_member_match(true, true, "lsp_ts_method")); + ASSERT_FALSE(cbm_suppress_weak_member_match(true, true, "lsp_cross")); + ASSERT_FALSE(cbm_suppress_weak_member_match(true, true, "lsp_ts_local")); /* A bare call (is_method=false) is a free-function call → never suppressed. */ - ASSERT_FALSE(cbm_tsjs_suppress_weak_method_match(true, false, "unique_name")); - ASSERT_FALSE(cbm_tsjs_suppress_weak_method_match(true, false, "suffix_match")); - /* Non-TS/JS languages are never affected. */ - ASSERT_FALSE(cbm_tsjs_suppress_weak_method_match(false, true, "suffix_match")); + ASSERT_FALSE(cbm_suppress_weak_member_match(true, false, "unique_name")); + ASSERT_FALSE(cbm_suppress_weak_member_match(true, false, "suffix_match")); + /* Languages outside the caller's guard set are never affected. */ + ASSERT_FALSE(cbm_suppress_weak_member_match(false, true, "suffix_match")); /* No match (NULL/empty strategy) → nothing to suppress. */ - ASSERT_FALSE(cbm_tsjs_suppress_weak_method_match(true, true, NULL)); - ASSERT_FALSE(cbm_tsjs_suppress_weak_method_match(true, true, "")); + ASSERT_FALSE(cbm_suppress_weak_member_match(true, true, NULL)); + ASSERT_FALSE(cbm_suppress_weak_member_match(true, true, "")); PASS(); } @@ -945,6 +945,6 @@ SUITE(registry) { RUN_TEST(perl_suppress_drops_weak_builtin_and_method_matches); RUN_TEST(perl_suppress_keeps_high_confidence_and_genuine_calls); RUN_TEST(cross_language_suffix_match_drops_py_vs_js); - RUN_TEST(tsjs_suppress_drops_weak_method_matches); - RUN_TEST(tsjs_suppress_keeps_high_confidence_and_non_methods); + RUN_TEST(dynamic_suppress_drops_weak_method_matches); + RUN_TEST(dynamic_suppress_keeps_high_confidence_and_non_methods); } From 7bef7f4d663eddab37f3470e8dbcf7d4a5545327 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 29 Aug 2026 11:27:38 +0200 Subject: [PATCH 27/31] feat(mcp): add manage_adr mode='set_sections' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit manage_adr could only replace: mode='update' overwrites the stored document in full, so adding one entry costs a re-send of the whole ADR. That is a data-integrity problem before it is a cost one — the caller has to reproduce every byte it did not intend to change, so the unchanged prefix survives only as well as the round-trip that carried it. mode='set_sections' rewrites only the sections named in `section_updates` and leaves the rest of the stored document untouched, so the stored copy stays the authority for everything the caller did not name. Unlike a whole-document append it is idempotent: a client that loses a response and retries re-sets the same section to the same body and the document is byte-identical, where an append would silently duplicate the chunk. Only the six canonical section names are writable. That is a correctness constraint rather than a style rule: adr_try_section_header() parses ONLY canonical headers, so a non-canonical '## FOO' written here would be read back as body text of the section above it, and a second identical write would append a duplicate — destroying the idempotence the mode exists for. Three things the new mode needed that were not there: - cbm_store_adr_update_sections() now wraps its read-modify-write in BEGIN IMMEDIATE. Three writers replace this row wholesale — the indexing pipeline, the UI POST /api/adr handler, and mode='update' — so the unguarded get/merge/store lost whichever of them committed between the read and the UPSERT. mode='update' is a single atomic UPSERT and never had that window; a section merge introduces it. - set_sections joins the write_request classification. A mode missing from it takes no per-project mutation lease, resolves the store query-only, and never reaches open_adr_store_for_write — its write would be attempted through a read-only handle while an index runs. - The write path reads the legacy /.codebase-memory/adr.md itself. The existing migration runs on the read path only because it must not block on the lease; without this a section write would merge onto an empty document and discard an ADR still present on disk. CBM_ADR_MAX_LENGTH now applies to an MCP write path: it is enforced inside cbm_store_adr_update_sections, which mode='update' does not go through. An empty section body, an unknown section name and a missing section_updates are all rejected before any store is opened, so a caller that meant to write never receives a success-shaped read. Section-level update was chosen over the whole-document append proposed in PR #1243; the analysis that established the problem is from that PR. Co-authored-by: andis777 <24672074+andis777@users.noreply.github.com> Signed-off-by: Martin Vogel --- README.md | 2 +- src/mcp/mcp.c | 238 +++++++++++++++++++++++++++++++++++++- src/store/store.c | 38 +++++- tests/test_mcp.c | 286 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 555 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index df5c33441..3c99104e5 100644 --- a/README.md +++ b/README.md @@ -666,7 +666,7 @@ JSON arguments can also be piped on stdin, for tools that take arguments. A tool | `get_code_snippet` | Read source code for a function by qualified name. | | `get_architecture` | Codebase overview: languages, packages, routes, hotspots, clusters, ADR. | | `search_code` | Grep-like text search within indexed project files. | -| `manage_adr` | CRUD for Architecture Decision Records. Query modes do not wait behind a same-project reindex; writes remain serialized. | +| `manage_adr` | CRUD for Architecture Decision Records (`get` reads, `update` replaces the whole document, `set_sections` rewrites only the named sections, `sections` lists headings). Query modes do not wait behind a same-project reindex; writes remain serialized. | | `ingest_traces` | Ingest runtime traces to validate HTTP_CALLS edges. | `manage_adr` query modes (`get` and `sections`) use the server's cached query store so they can proceed while a same-project reindex is running. If another process publishes a replacement store during reindexing, they can return the pre-publication ADR until idle eviction refreshes that cache. Updates remain serialized through the project mutation guard. diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 5e64fabdf..f07ab1f33 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -706,10 +706,19 @@ static const tool_def_t TOOLS[] = { {"manage_adr", "Manage ADR", "Create or update Architecture Decision Records", "{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"},\"mode\":{\"type\":" - "\"string\",\"enum\":[\"get\",\"update\",\"sections\"],\"description\":\"update replaces " - "the entire ADR document; sections only lists existing " - "headings\"},\"content\":{\"type\":\"string\",\"description\":\"Complete replacement document " - "required by update\"}},\"additionalProperties\":false," + "\"string\",\"enum\":[\"get\",\"update\",\"set_sections\",\"sections\"],\"description\":" + "\"update REPLACES the entire ADR document; set_sections rewrites only the named sections " + "and leaves every other byte of the stored document untouched, so adding one entry does not " + "mean re-sending the whole ADR (setting the same section to the same body twice leaves the " + "document byte-identical, so retrying after a lost response is safe); sections only lists " + "existing headings\"},\"content\":{\"type\":\"string\",\"description\":\"Complete replacement " + "document required by update\"},\"section_updates\":{\"type\":\"object\",\"description\":" + "\"Required by set_sections: section name -> new body for that section. Only the six " + "canonical sections exist; any other name is rejected.\",\"properties\":{" + "\"PURPOSE\":{\"type\":\"string\"},\"STACK\":{\"type\":\"string\"}," + "\"ARCHITECTURE\":{\"type\":\"string\"},\"PATTERNS\":{\"type\":\"string\"}," + "\"TRADEOFFS\":{\"type\":\"string\"},\"PHILOSOPHY\":{\"type\":\"string\"}}," + "\"additionalProperties\":false}},\"additionalProperties\":false," "\"required\":[\"project\"]}"}, {"ingest_traces", "Ingest traces", "Ingest runtime traces to enhance the knowledge graph", @@ -11848,6 +11857,123 @@ static cbm_store_t *open_adr_store_for_write(cbm_mcp_server_t *srv, cbm_store_t return *owned_rw; } +/* Parsed `section_updates` for mode='set_sections'. + * + * mode='update' replaces the whole document, so adding one entry costs a full + * re-send and the stored ADR is only ever as good as that round-trip. Writing + * named sections instead leaves the rest of the document as the authority for + * itself — and, unlike a whole-document append, applying the same request + * twice yields the same document, so a client that retries after a lost + * response cannot silently duplicate content. */ +typedef struct { + char *keys[PROPS_MAX]; + char *values[PROPS_MAX]; + int count; + /* Rejection reason, or NULL when the request parsed cleanly. Set means no + * store was opened and nothing was written. */ + const char *status; + const char *error; +} adr_section_updates_t; + +static void adr_section_updates_free(adr_section_updates_t *u) { + for (int i = 0; i < u->count; i++) { + free(u->keys[i]); + free(u->values[i]); + } + u->count = 0; +} + +static bool adr_collect_section_update(adr_section_updates_t *u, yyjson_val *key, yyjson_val *val) { + const char *name = yyjson_get_str(key); + if (!name || !name[0]) { + u->status = "invalid_section_updates"; + u->error = "'section_updates' keys must be non-empty section names. " + "No ADR write was performed."; + return false; + } + if (!yyjson_is_str(val)) { + u->status = "invalid_section_updates"; + u->error = "'section_updates' values must be strings (the new body for that section). " + "No ADR write was performed."; + return false; + } + const char *body = yyjson_get_str(val); + /* An empty body would render a heading with nothing under it — a silent + * content deletion wearing the response shape of an update. Clearing a + * section is whole-document surgery; that is what mode='update' is for. */ + if (!body || !body[0]) { + u->status = "empty_section_content"; + u->error = "'section_updates' values must be non-empty; use mode='update' to remove a " + "section. No ADR write was performed."; + return false; + } + if (u->count >= PROPS_MAX) { + u->status = "too_many_sections"; + u->error = "'section_updates' carries more entries than an ADR can hold. " + "No ADR write was performed."; + return false; + } + u->keys[u->count] = heap_strdup(name); + u->values[u->count] = heap_strdup(body); + if (!u->keys[u->count] || !u->values[u->count]) { + free(u->keys[u->count]); + free(u->values[u->count]); + u->status = "write_error"; + u->error = "out of memory parsing 'section_updates'. No ADR write was performed."; + return false; + } + u->count++; + return true; +} + +static adr_section_updates_t adr_parse_section_updates(const char *args) { + adr_section_updates_t u; + memset(&u, 0, sizeof(u)); + + yyjson_doc *doc = yyjson_read(args, strlen(args), 0); + yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; + yyjson_val *updates = + (root && yyjson_is_obj(root)) ? yyjson_obj_get(root, "section_updates") : NULL; + + if (!updates) { + /* Never fall through to 'get': a caller that meant to write must not + * receive a success-shaped read. */ + u.status = "missing_section_updates"; + u.error = "mode='set_sections' requires 'section_updates', an object mapping section " + "name to its new body. No ADR write was performed."; + } else if (!yyjson_is_obj(updates) || yyjson_obj_size(updates) == 0) { + u.status = "invalid_section_updates"; + u.error = "'section_updates' must be a non-empty object mapping section name to its new " + "body. No ADR write was performed."; + } else { + size_t idx = 0; + size_t max = 0; + yyjson_val *key = NULL; + yyjson_val *val = NULL; + yyjson_obj_foreach(updates, idx, max, key, val) { + if (!adr_collect_section_update(&u, key, val)) { + adr_section_updates_free(&u); + break; + } + } + } + yyjson_doc_free(doc); + return u; +} + +/* Build the rejection payload for a set_sections request that never reached a + * store. Caller frees. */ +static char *adr_section_updates_error(const adr_section_updates_t *u) { + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root_obj = yyjson_mut_obj(doc); + yyjson_mut_doc_set_root(doc, root_obj); + yyjson_mut_obj_add_str(doc, root_obj, "status", u->status); + yyjson_mut_obj_add_strcpy(doc, root_obj, "error", u->error); + char *json = yy_doc_to_str(doc); + yyjson_mut_doc_free(doc); + return json; +} + static char *handle_manage_adr(cbm_mcp_server_t *srv, const char *args) { char *project = get_project_arg(args); char *mode_str = cbm_mcp_get_string_arg(args, "mode"); @@ -11879,12 +12005,51 @@ static char *handle_manage_adr(cbm_mcp_server_t *srv, const char *args) { true); } + bool set_sections_mode = (strcmp(mode_str, "set_sections") == 0); + adr_section_updates_t updates; + memset(&updates, 0, sizeof(updates)); + char section_key_err[CBM_SZ_256] = ""; + if (set_sections_mode) { + updates = adr_parse_section_updates(args); + /* Only the six canonical names are writable, and that is a correctness + * constraint rather than a house style: adr_try_section_header() parses + * ONLY canonical headers, so a '## FOO' written here would be read back + * as body text of the section above it and a second identical write + * would append a duplicate — destroying exactly the idempotence this + * mode exists to provide. */ + if (!updates.status && cbm_adr_validate_section_keys( + (const char **)updates.keys, updates.count, section_key_err, + (int)sizeof(section_key_err)) != CBM_STORE_OK) { + adr_section_updates_free(&updates); + updates.status = "invalid_section_name"; + updates.error = section_key_err; + } + if (updates.status) { + /* Reject before taking the project lease or opening a store: a + * malformed write must not block an index, and must not read. */ + char *err = adr_section_updates_error(&updates); + adr_section_updates_free(&updates); + free(project); + free(mode_str); + free(content); + char *res = cbm_mcp_text_result(err, true); + free(err); + return res; + } + } + + /* This classification is load-bearing. A mode missing from it takes no + * per-project mutation lease, resolves the store query-only, and never + * reaches open_adr_store_for_write — so its write would be attempted + * through a read-only handle, concurrently with an active index. */ bool write_request = - content && (strcmp(mode_str, "update") == 0 || strcmp(mode_str, "store") == 0); + (content && (strcmp(mode_str, "update") == 0 || strcmp(mode_str, "store") == 0)) || + set_sections_mode; bool mutation_held = false; if (write_request && project) { mutation_held = mcp_project_mutation_begin(srv, project); if (!mutation_held) { + adr_section_updates_free(&updates); free(project); free(mode_str); free(content); @@ -11893,6 +12058,7 @@ static char *handle_manage_adr(cbm_mcp_server_t *srv, const char *args) { } if (mcp_request_cancelled(srv)) { mcp_project_mutation_end(srv, project); + adr_section_updates_free(&updates); free(project); free(mode_str); free(content); @@ -11921,6 +12087,7 @@ static char *handle_manage_adr(cbm_mcp_server_t *srv, const char *args) { if (mutation_held) { mcp_project_mutation_end(srv, project); } + adr_section_updates_free(&updates); free(project); free(mode_str); free(content); @@ -11935,6 +12102,7 @@ static char *handle_manage_adr(cbm_mcp_server_t *srv, const char *args) { if (mutation_held) { mcp_project_mutation_end(srv, project); } + adr_section_updates_free(&updates); free(project); free(mode_str); free(content); @@ -11981,13 +12149,69 @@ static char *handle_manage_adr(cbm_mcp_server_t *srv, const char *args) { } } + /* A set_sections write must see a legacy file-backed ADR too. The + * migration above deliberately runs on the read path only — it must never + * block on the lease — so the write path reads the file here, where the + * exclusive project lease and a writable store are already held. Merging + * onto an empty document instead would silently discard an ADR the user + * still has on disk. */ + char *legacy_seed = NULL; + if (set_sections_mode && !have_adr) { + char *root_path = project_root_from_store(store, project); + legacy_seed = adr_read_legacy_file(root_path); + free(root_path); + } + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); yyjson_mut_val *root_obj = yyjson_mut_obj(doc); yyjson_mut_doc_set_root(doc, root_obj); bool is_error = false; const char *adr_content = have_adr ? adr.content : legacy; - if (write_request) { + if (set_sections_mode) { + /* cbm_store_adr_update_sections requires an existing row — its + * contract, pinned by TEST(adr_update_no_existing). Seed one when the + * project has none, so the mode degrades to a plain create: the legacy + * document when there is one, an empty document otherwise. */ + bool base_present = have_adr; + bool seeded_empty = false; + if (!base_present) { + const char *seed = legacy_seed ? legacy_seed : ""; + if (cbm_store_adr_store(store, project, seed) == CBM_STORE_OK) { + base_present = true; + seeded_empty = (legacy_seed == NULL); + } + } + cbm_adr_t updated; + memset(&updated, 0, sizeof(updated)); + int section_rc = base_present ? cbm_store_adr_update_sections( + store, project, (const char **)updates.keys, + (const char **)updates.values, updates.count, &updated) + : CBM_STORE_ERR; + if (section_rc == CBM_STORE_OK) { + yyjson_mut_obj_add_str(doc, root_obj, "status", "sections_updated"); + yyjson_mut_obj_add_str(doc, root_obj, "semantics", + "named_sections_replaced_rest_preserved"); + yyjson_mut_obj_add_uint(doc, root_obj, "sections_written", (uint64_t)updates.count); + /* Callers confirm the write landed without re-fetching the ADR. */ + yyjson_mut_obj_add_uint(doc, root_obj, "content_length", + (uint64_t)strlen(updated.content)); + cbm_store_adr_free(&updated); + } else { + /* Undo an empty seed. A rejected write must not leave the project + * holding a blank ADR where `get` used to answer no_adr. A legacy + * seed is a real migration and is kept. */ + if (seeded_empty) { + (void)cbm_store_adr_delete(store, project); + } + yyjson_mut_obj_add_str(doc, root_obj, "status", "write_error"); + const char *store_err = cbm_store_error(store); + if (store_err && store_err[0]) { + yyjson_mut_obj_add_strcpy(doc, root_obj, "error", store_err); + } + is_error = true; + } + } else if (write_request) { if (cbm_store_adr_store(store, project, content) == CBM_STORE_OK) { yyjson_mut_obj_add_str(doc, root_obj, "status", "updated"); yyjson_mut_obj_add_str(doc, root_obj, "semantics", "whole_document_replaced"); @@ -12018,6 +12242,8 @@ static char *handle_manage_adr(cbm_mcp_server_t *srv, const char *args) { if (mutation_held) { mcp_project_mutation_end(srv, project); } + adr_section_updates_free(&updates); + free(legacy_seed); free(legacy); free(project); free(mode_str); diff --git a/src/store/store.c b/src/store/store.c index c8a3528d7..e8bc8e5f4 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -8883,10 +8883,25 @@ int cbm_store_adr_delete(cbm_store_t *s, const char *project) { int cbm_store_adr_update_sections(cbm_store_t *s, const char *project, const char **keys, const char **values, int count, cbm_adr_t *out) { + if (!s || !s->db || !project || !keys || !values || !out) { + return CBM_STORE_ERR; + } + + /* The read-modify-write below must be ONE transaction. Three writers + * replace this row wholesale — the indexing pipeline, the UI POST + * /api/adr handler, and manage_adr mode='update' — so an unguarded + * get/merge/store silently loses whichever of them commits between the + * read and the UPSERT. BEGIN IMMEDIATE takes the write lock up front, so a + * competing writer waits rather than being overwritten. */ + if (cbm_store_begin(s) != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + /* Get existing ADR */ cbm_adr_t existing; int rc = cbm_store_adr_get(s, project, &existing); if (rc != CBM_STORE_OK) { + (void)cbm_store_rollback(s); store_set_error(s, "no existing ADR to update"); return rc; } @@ -8917,13 +8932,20 @@ int cbm_store_adr_update_sections(cbm_store_t *s, const char *project, const cha char *merged = cbm_adr_render(§ions); cbm_adr_sections_free(§ions); + if (!merged) { + (void)cbm_store_rollback(s); + store_set_error(s, "failed to render merged ADR"); + return CBM_STORE_ERR; + } + /* Check length */ if ((int)strlen(merged) > CBM_ADR_MAX_LENGTH) { char msg[CBM_SZ_128]; snprintf(msg, sizeof(msg), "merged ADR exceeds %d chars (%d chars)", CBM_ADR_MAX_LENGTH, (int)strlen(merged)); - store_set_error(s, msg); free(merged); + (void)cbm_store_rollback(s); + store_set_error(s, msg); return CBM_STORE_ERR; } @@ -8931,10 +8953,22 @@ int cbm_store_adr_update_sections(cbm_store_t *s, const char *project, const cha rc = cbm_store_adr_store(s, project, merged); free(merged); if (rc != CBM_STORE_OK) { + (void)cbm_store_rollback(s); return rc; } - return cbm_store_adr_get(s, project, out); + /* Read back INSIDE the transaction so `out` is exactly what commits. */ + rc = cbm_store_adr_get(s, project, out); + if (rc != CBM_STORE_OK) { + (void)cbm_store_rollback(s); + return rc; + } + if (cbm_store_commit(s) != CBM_STORE_OK) { + cbm_store_adr_free(out); + (void)cbm_store_rollback(s); + return CBM_STORE_ERR; + } + return CBM_STORE_OK; } void cbm_store_adr_free(cbm_adr_t *adr) { diff --git a/tests/test_mcp.c b/tests/test_mcp.c index de36cb6bd..21b5826b3 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -5847,6 +5847,285 @@ TEST(tool_manage_adr_rejects_removed_sections_argument) { PASS(); } +/* mode='set_sections' rewrites only the named sections. mode='update' replaces + * the whole document, so adding one entry costs a full re-send and every byte + * the caller did not mean to touch survives only as well as that round-trip. */ +TEST(tool_manage_adr_set_sections_replaces_only_named) { + const char *project = "adr-sec-named"; + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + cbm_store_t *st = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(st); + ASSERT_EQ(cbm_store_upsert_project(st, project, "/tmp/adr-sec-named"), CBM_STORE_OK); + cbm_mcp_server_set_project(srv, project); + ASSERT_EQ(cbm_store_adr_store(st, project, "## PURPOSE\nOriginal purpose.\n\n## STACK\nC."), + CBM_STORE_OK); + + /* A section write is a mutation: it must take the per-project lease, or it + * runs concurrently with an index through a query-only store handle. */ + mcp_mutation_guard_probe_t probe = {0}; + cbm_mcp_server_set_project_mutation_guard(srv, mcp_mutation_guard_probe_begin, + mcp_mutation_guard_probe_end, &probe); + + char *resp = cbm_mcp_handle_tool(srv, "manage_adr", + "{\"project\":\"adr-sec-named\",\"mode\":\"set_sections\"," + "\"section_updates\":{\"PATTERNS\":\"- Pipeline stages.\"}}"); + ASSERT_NOT_NULL(resp); + ASSERT_NOT_NULL(strstr(resp, "sections_updated")); + ASSERT_NULL(strstr(resp, "\"isError\":true")); + free(resp); + ASSERT_EQ(probe.begin_count, 1); + ASSERT_EQ(probe.end_count, 1); + ASSERT_STR_EQ(probe.begin_projects[0], project); + + /* The sections nobody named survive verbatim, and the named one landed. */ + cbm_adr_t adr; + memset(&adr, 0, sizeof(adr)); + ASSERT_EQ(cbm_store_adr_get(st, project, &adr), CBM_STORE_OK); + ASSERT_NOT_NULL(adr.content); + ASSERT_NOT_NULL(strstr(adr.content, "## PURPOSE\nOriginal purpose.")); + ASSERT_NOT_NULL(strstr(adr.content, "## STACK\nC.")); + ASSERT_NOT_NULL(strstr(adr.content, "## PATTERNS\n- Pipeline stages.")); + cbm_store_adr_free(&adr); + + cbm_mcp_server_free(srv); + PASS(); +} + +/* THE reason this shape was chosen over a whole-document append: applying the + * same request twice must leave the document byte-identical. An MCP client that + * loses a response and retries would silently duplicate an appended chunk. */ +TEST(tool_manage_adr_set_sections_is_idempotent) { + const char *project = "adr-sec-idem"; + const char *request = "{\"project\":\"adr-sec-idem\",\"mode\":\"set_sections\"," + "\"section_updates\":{\"PATTERNS\":\"- Pipeline stages.\"}}"; + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + cbm_store_t *st = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(st); + ASSERT_EQ(cbm_store_upsert_project(st, project, "/tmp/adr-sec-idem"), CBM_STORE_OK); + cbm_mcp_server_set_project(srv, project); + ASSERT_EQ(cbm_store_adr_store(st, project, "## PURPOSE\nOriginal purpose.\n\n## STACK\nC."), + CBM_STORE_OK); + + char *resp = cbm_mcp_handle_tool(srv, "manage_adr", request); + ASSERT_NOT_NULL(resp); + ASSERT_NOT_NULL(strstr(resp, "sections_updated")); + free(resp); + + cbm_adr_t first; + memset(&first, 0, sizeof(first)); + ASSERT_EQ(cbm_store_adr_get(st, project, &first), CBM_STORE_OK); + ASSERT_NOT_NULL(first.content); + char *after_first = strdup(first.content); + ASSERT_NOT_NULL(after_first); + cbm_store_adr_free(&first); + + /* Replay the identical request — the lost-response retry. */ + resp = cbm_mcp_handle_tool(srv, "manage_adr", request); + ASSERT_NOT_NULL(resp); + ASSERT_NOT_NULL(strstr(resp, "sections_updated")); + ASSERT_NULL(strstr(resp, "\"isError\":true")); + free(resp); + + cbm_adr_t second; + memset(&second, 0, sizeof(second)); + ASSERT_EQ(cbm_store_adr_get(st, project, &second), CBM_STORE_OK); + ASSERT_NOT_NULL(second.content); + ASSERT_STR_EQ(second.content, after_first); + /* And the body is present exactly once, not appended twice. */ + const char *hit = strstr(second.content, "- Pipeline stages."); + ASSERT_NOT_NULL(hit); + ASSERT_NULL(strstr(hit + 1, "- Pipeline stages.")); + cbm_store_adr_free(&second); + free(after_first); + + cbm_mcp_server_free(srv); + PASS(); +} + +/* A project with no ADR yet degrades to a plain create rather than erroring: + * the store primitive requires an existing row, so the handler seeds one. */ +TEST(tool_manage_adr_set_sections_creates_when_absent) { + const char *project = "adr-sec-new"; + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + cbm_store_t *st = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(st); + ASSERT_EQ(cbm_store_upsert_project(st, project, "/tmp/adr-sec-new"), CBM_STORE_OK); + cbm_mcp_server_set_project(srv, project); + + char *resp = cbm_mcp_handle_tool(srv, "manage_adr", + "{\"project\":\"adr-sec-new\",\"mode\":\"set_sections\"," + "\"section_updates\":{\"PURPOSE\":\"Only entry.\"}}"); + ASSERT_NOT_NULL(resp); + ASSERT_NOT_NULL(strstr(resp, "sections_updated")); + ASSERT_NULL(strstr(resp, "\"isError\":true")); + free(resp); + + cbm_adr_t adr; + memset(&adr, 0, sizeof(adr)); + ASSERT_EQ(cbm_store_adr_get(st, project, &adr), CBM_STORE_OK); + /* Exact match: a create must not leave a leading separator behind. */ + ASSERT_STR_EQ(adr.content, "## PURPOSE\nOnly entry."); + cbm_store_adr_free(&adr); + + cbm_mcp_server_free(srv); + PASS(); +} + +/* set_sections without section_updates must fail loudly. Falling through to + * 'get' would hand a caller that meant to write a success-shaped read — and it + * must not take the mutation lease on the way to being rejected. */ +TEST(tool_manage_adr_set_sections_without_updates_errors) { + const char *project = "adr-sec-missing"; + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + cbm_store_t *st = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(st); + ASSERT_EQ(cbm_store_upsert_project(st, project, "/tmp/adr-sec-missing"), CBM_STORE_OK); + cbm_mcp_server_set_project(srv, project); + ASSERT_EQ(cbm_store_adr_store(st, project, "## PURPOSE\nUntouched.\n"), CBM_STORE_OK); + + mcp_mutation_guard_probe_t probe = {0}; + cbm_mcp_server_set_project_mutation_guard(srv, mcp_mutation_guard_probe_begin, + mcp_mutation_guard_probe_end, &probe); + + char *resp = cbm_mcp_handle_tool(srv, "manage_adr", + "{\"project\":\"adr-sec-missing\",\"mode\":\"set_sections\"}"); + ASSERT_NOT_NULL(resp); + ASSERT_NOT_NULL(strstr(resp, "missing_section_updates")); + ASSERT_NOT_NULL(strstr(resp, "No ADR write was performed")); + ASSERT_NOT_NULL(strstr(resp, "\"isError\":true")); + ASSERT_NULL(strstr(resp, "adr_hint")); + free(resp); + ASSERT_EQ(probe.begin_count, 0); + ASSERT_EQ(probe.end_count, 0); + + cbm_adr_t adr; + memset(&adr, 0, sizeof(adr)); + ASSERT_EQ(cbm_store_adr_get(st, project, &adr), CBM_STORE_OK); + ASSERT_STR_EQ(adr.content, "## PURPOSE\nUntouched.\n"); + cbm_store_adr_free(&adr); + + cbm_mcp_server_free(srv); + PASS(); +} + +/* An empty body would render a heading with nothing under it — a content + * deletion wearing the response shape of an update. A non-canonical name would + * be parsed back as body text of the section above it, so writing it twice + * would duplicate it: both are rejected before any store is opened. */ +TEST(tool_manage_adr_set_sections_rejects_empty_and_unknown_sections) { + const char *project = "adr-sec-guards"; + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + cbm_store_t *st = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(st); + ASSERT_EQ(cbm_store_upsert_project(st, project, "/tmp/adr-sec-guards"), CBM_STORE_OK); + cbm_mcp_server_set_project(srv, project); + ASSERT_EQ(cbm_store_adr_store(st, project, "## PURPOSE\nUntouched.\n"), CBM_STORE_OK); + + char *resp = cbm_mcp_handle_tool(srv, "manage_adr", + "{\"project\":\"adr-sec-guards\",\"mode\":\"set_sections\"," + "\"section_updates\":{\"PURPOSE\":\"\"}}"); + ASSERT_NOT_NULL(resp); + ASSERT_NOT_NULL(strstr(resp, "empty_section_content")); + ASSERT_NOT_NULL(strstr(resp, "\"isError\":true")); + free(resp); + + resp = cbm_mcp_handle_tool(srv, "manage_adr", + "{\"project\":\"adr-sec-guards\",\"mode\":\"set_sections\"," + "\"section_updates\":{\"DECISIONS\":\"- Chose SQLite.\"}}"); + ASSERT_NOT_NULL(resp); + ASSERT_NOT_NULL(strstr(resp, "invalid_section_name")); + ASSERT_NOT_NULL(strstr(resp, "DECISIONS")); + ASSERT_NOT_NULL(strstr(resp, "\"isError\":true")); + free(resp); + + resp = cbm_mcp_handle_tool(srv, "manage_adr", + "{\"project\":\"adr-sec-guards\",\"mode\":\"set_sections\"," + "\"section_updates\":{}}"); + ASSERT_NOT_NULL(resp); + ASSERT_NOT_NULL(strstr(resp, "invalid_section_updates")); + ASSERT_NOT_NULL(strstr(resp, "\"isError\":true")); + free(resp); + + /* Every rejection left the stored ADR byte-identical. */ + cbm_adr_t adr; + memset(&adr, 0, sizeof(adr)); + ASSERT_EQ(cbm_store_adr_get(st, project, &adr), CBM_STORE_OK); + ASSERT_STR_EQ(adr.content, "## PURPOSE\nUntouched.\n"); + cbm_store_adr_free(&adr); + + cbm_mcp_server_free(srv); + PASS(); +} + +/* CBM_ADR_MAX_LENGTH is enforced on this path — mode='update' bypasses it, so + * exposing an incremental writer without the cap would make unbounded growth + * cheap. The rejected merge must also roll back to the byte-identical prior + * document rather than leaving a half-applied write. */ +TEST(tool_manage_adr_set_sections_rejects_oversize) { + const char *project = "adr-sec-cap"; + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + cbm_store_t *st = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(st); + ASSERT_EQ(cbm_store_upsert_project(st, project, "/tmp/adr-sec-cap"), CBM_STORE_OK); + cbm_mcp_server_set_project(srv, project); + ASSERT_EQ(cbm_store_adr_store(st, project, "## PURPOSE\nSmall.\n"), CBM_STORE_OK); + + size_t huge_len = (size_t)CBM_ADR_MAX_LENGTH + 100; + char *huge = malloc(huge_len + 1); + ASSERT_NOT_NULL(huge); + memset(huge, 'x', huge_len); + huge[huge_len] = '\0'; + + size_t args_len = huge_len + 256; + char *args = malloc(args_len); + ASSERT_NOT_NULL(args); + snprintf(args, args_len, + "{\"project\":\"adr-sec-cap\",\"mode\":\"set_sections\"," + "\"section_updates\":{\"STACK\":\"%s\"}}", + huge); + + char *resp = cbm_mcp_handle_tool(srv, "manage_adr", args); + ASSERT_NOT_NULL(resp); + ASSERT_NOT_NULL(strstr(resp, "write_error")); + ASSERT_NOT_NULL(strstr(resp, "exceeds")); + ASSERT_NOT_NULL(strstr(resp, "\"isError\":true")); + free(resp); + free(args); + free(huge); + + cbm_adr_t adr; + memset(&adr, 0, sizeof(adr)); + ASSERT_EQ(cbm_store_adr_get(st, project, &adr), CBM_STORE_OK); + ASSERT_STR_EQ(adr.content, "## PURPOSE\nSmall.\n"); + cbm_store_adr_free(&adr); + + cbm_mcp_server_free(srv); + PASS(); +} + +/* The mode must be advertised, or callers never learn it exists and keep + * paying for whole-document rewrites. */ +TEST(tool_manage_adr_set_sections_is_advertised) { + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + char *resp = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/list\",\"params\":{}}"); + ASSERT_NOT_NULL(resp); + const char *adr_tool = strstr(resp, "manage_adr"); + ASSERT_NOT_NULL(adr_tool); + ASSERT_NOT_NULL(strstr(adr_tool, "set_sections")); + ASSERT_NOT_NULL(strstr(adr_tool, "section_updates")); + free(resp); + cbm_mcp_server_free(srv); + PASS(); +} + TEST(tool_manage_adr_mutation_guard_balances_success) { const char *project = "guard-adr-success"; cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); @@ -13163,6 +13442,13 @@ SUITE(mcp) { RUN_TEST(tool_manage_adr_get_with_existing_adr); RUN_TEST(tool_manage_adr_unified_backend_issue256); RUN_TEST(tool_manage_adr_rejects_removed_sections_argument); + RUN_TEST(tool_manage_adr_set_sections_replaces_only_named); + RUN_TEST(tool_manage_adr_set_sections_is_idempotent); + RUN_TEST(tool_manage_adr_set_sections_creates_when_absent); + RUN_TEST(tool_manage_adr_set_sections_without_updates_errors); + RUN_TEST(tool_manage_adr_set_sections_rejects_empty_and_unknown_sections); + RUN_TEST(tool_manage_adr_set_sections_rejects_oversize); + RUN_TEST(tool_manage_adr_set_sections_is_advertised); RUN_TEST(tool_index_repository_reports_store_backed_adr); RUN_TEST(tool_index_repository_resolves_root_path_from_project_name_issue1211); RUN_TEST(tool_index_repository_unknown_project_name_still_requires_repo_path); From d013555d28f618948e0b927ab612d5737488894a Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 29 Aug 2026 12:42:02 +0200 Subject: [PATCH 28/31] feat(mcp): splice ADR sections instead of rebuilding the document MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit manage_adr mode='set_sections' rebuilt the ADR from cbm_adr_parse_sections() and cbm_adr_render(). That model is lossy, so the rebuild silently rewrote text nobody asked to change. Measured on the previous commit: - Everything before the first recognised heading was dropped. - A mis-cased '## Purpose' was not a heading, so when it came first its whole block was dropped with the preamble — an ordinary typo cost the author a section. - Sections stored out of canonical order were reordered. mode='update' never had this exposure because it writes the caller's bytes verbatim; the section merge introduced it. The merge now locates the target heading's byte span and replaces only that span, appending a block when the heading is absent. Bytes outside the span are never reconstructed, so a preamble, a code fence, an unrecognised heading and the author's section order all survive byte-for-byte. That property is the acceptance test: every document in adr_splice_preserves_untouched_bytes is a case the rebuild corrupted. Arbitrary headings now work, which is what an "add an entry" mode needs. The canonical six become a convention rather than a privilege: they are still what ADR_EMPTY_HINT suggests, but they get no ordering priority and none is required. Under splicing, ordering is whatever the document says, and any privileged reordering would reintroduce the data loss above. Names match exactly, including case, so '## Purpose' and '## PURPOSE' are different sections. Folding them would silently merge two blocks the author chose to keep apart — the same class of loss this commit removes. cbm_adr_validate_section_keys() no longer enforces the canonical set. It now rejects only names that could not round-trip through a '## NAME' line (empty, '#'-leading, newline-bearing, edge-whitespace, over-long), because such a name scans back as a different heading or as none, and a second identical write would append a duplicate instead of being a no-op. A heading inside a fenced code block is a code sample, not a section. An unterminated fence hides every heading after it, so a write would append a duplicate rather than replace: that is refused explicitly, and mode='sections' reports the same ambiguity instead of answering with a partial list. manage_adr mode='sections' now uses cbm_adr_scan_headings(), the same classifier the write path splices with. It used to list any '#'-prefixed line, so a '## Foo' in prose or inside a fence was reported as a section no write could target. Two components disagreeing about what a section is was how a section write came to be able to destroy one. cbm_adr_parse_sections() is unchanged, so documents parse exactly as before. TEST(adr_parse_sections_non_canonical) still passes untouched. Co-authored-by: andis777 <24672074+andis777@users.noreply.github.com> Signed-off-by: Martin Vogel --- README.md | 4 +- src/mcp/mcp.c | 69 +++---- src/store/store.c | 386 ++++++++++++++++++++++++++++++++++------ src/store/store.h | 41 +++++ tests/test_mcp.c | 224 ++++++++++++++++++++++- tests/test_store_arch.c | 188 ++++++++++++++++++- 6 files changed, 807 insertions(+), 105 deletions(-) diff --git a/README.md b/README.md index 3c99104e5..271773f45 100644 --- a/README.md +++ b/README.md @@ -666,9 +666,11 @@ JSON arguments can also be piped on stdin, for tools that take arguments. A tool | `get_code_snippet` | Read source code for a function by qualified name. | | `get_architecture` | Codebase overview: languages, packages, routes, hotspots, clusters, ADR. | | `search_code` | Grep-like text search within indexed project files. | -| `manage_adr` | CRUD for Architecture Decision Records (`get` reads, `update` replaces the whole document, `set_sections` rewrites only the named sections, `sections` lists headings). Query modes do not wait behind a same-project reindex; writes remain serialized. | +| `manage_adr` | CRUD for Architecture Decision Records (`get` reads, `update` replaces the whole document, `set_sections` rewrites only the named sections and leaves every other byte untouched, `sections` lists headings). Query modes do not wait behind a same-project reindex; writes remain serialized. | | `ingest_traces` | Ingest runtime traces to validate HTTP_CALLS edges. | +`manage_adr(mode='set_sections')` writes one or more sections by name and splices them into the stored document, so text outside the named sections — including a preamble, code fences and section ordering — is preserved byte-for-byte. Any `## Heading` works, not just the conventional PURPOSE / STACK / ARCHITECTURE / PATTERNS / TRADEOFFS / PHILOSOPHY set; names match exactly, including case. Writing the same section twice is a no-op, so a retry after a lost response cannot duplicate content. + `manage_adr` query modes (`get` and `sections`) use the server's cached query store so they can proceed while a same-project reindex is running. If another process publishes a replacement store during reindexing, they can return the pre-publication ADR until idle eviction refreshes that cache. Updates remain serialized through the project mutation guard. ## Graph Data Model diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index f07ab1f33..67df96831 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -713,12 +713,11 @@ static const tool_def_t TOOLS[] = { "document byte-identical, so retrying after a lost response is safe); sections only lists " "existing headings\"},\"content\":{\"type\":\"string\",\"description\":\"Complete replacement " "document required by update\"},\"section_updates\":{\"type\":\"object\",\"description\":" - "\"Required by set_sections: section name -> new body for that section. Only the six " - "canonical sections exist; any other name is rejected.\",\"properties\":{" - "\"PURPOSE\":{\"type\":\"string\"},\"STACK\":{\"type\":\"string\"}," - "\"ARCHITECTURE\":{\"type\":\"string\"},\"PATTERNS\":{\"type\":\"string\"}," - "\"TRADEOFFS\":{\"type\":\"string\"},\"PHILOSOPHY\":{\"type\":\"string\"}}," - "\"additionalProperties\":false}},\"additionalProperties\":false," + "\"Required by set_sections: section name -> new body for that section. Any heading name " + "works, so a new entry can be added under its own heading; PURPOSE, STACK, ARCHITECTURE, " + "PATTERNS, TRADEOFFS and PHILOSOPHY are the conventional ones. Names match exactly, " + "including case, so 'Purpose' and 'PURPOSE' are different sections.\"," + "\"additionalProperties\":{\"type\":\"string\"}}},\"additionalProperties\":false," "\"required\":[\"project\"]}"}, {"ingest_traces", "Ingest traces", "Ingest runtime traces to enhance the knowledge graph", @@ -11763,31 +11762,33 @@ static char *handle_detect_changes(cbm_mcp_server_t *srv, const char *args) { /* ── manage_adr ───────────────────────────────────────────────── */ -/* ADR "sections" mode: list markdown headers ('#'-prefixed lines) from the - * ADR content string. */ +typedef struct { + yyjson_mut_doc *doc; + yyjson_mut_val *arr; +} adr_sections_ctx_t; + +static void adr_sections_cb(void *ctx, const cbm_adr_heading_t *h) { + adr_sections_ctx_t *c = (adr_sections_ctx_t *)ctx; + char hdr[CBM_SZ_1K]; + snprintf(hdr, sizeof(hdr), "## %.*s", h->name_len, h->name); + yyjson_mut_arr_add_strcpy(c->doc, c->arr, hdr); +} + +/* ADR "sections" mode: list the section headings of the ADR. + * + * This uses cbm_adr_scan_headings(), the SAME classifier the section-write + * path splices with. It used to list any '#'-prefixed line, so a '## Foo' in + * prose — or inside a fenced code block — was reported as a section that no + * write could target. Two components disagreeing about what a section is was + * how a section write came to be able to destroy one. */ static void adr_list_sections_from_content(yyjson_mut_doc *doc, yyjson_mut_val *root_obj, const char *content) { yyjson_mut_val *sections = yyjson_mut_arr(doc); - const char *p = content; - while (p && *p) { - const char *eol = strchr(p, '\n'); - size_t linelen = eol ? (size_t)(eol - p) : strlen(p); - while (linelen > 0 && p[linelen - SKIP_ONE] == '\r') { - linelen--; - } - if (linelen > 0 && p[0] == '#') { - char hdr[CBM_SZ_1K]; - if (linelen >= sizeof(hdr)) { - linelen = sizeof(hdr) - SKIP_ONE; - } - memcpy(hdr, p, linelen); - hdr[linelen] = '\0'; - yyjson_mut_arr_add_strcpy(doc, sections, hdr); - } - if (!eol) { - break; - } - p = eol + SKIP_ONE; + adr_sections_ctx_t ctx = {doc, sections}; + if (content && cbm_adr_scan_headings(content, adr_sections_cb, &ctx) != CBM_STORE_OK) { + /* The ambiguity that refuses a section write is reported here too, + * rather than answering with a heading list that is quietly partial. */ + yyjson_mut_obj_add_str(doc, root_obj, "sections_status", "unterminated_code_fence"); } yyjson_mut_obj_add_val(doc, root_obj, "sections", sections); } @@ -12011,12 +12012,12 @@ static char *handle_manage_adr(cbm_mcp_server_t *srv, const char *args) { char section_key_err[CBM_SZ_256] = ""; if (set_sections_mode) { updates = adr_parse_section_updates(args); - /* Only the six canonical names are writable, and that is a correctness - * constraint rather than a house style: adr_try_section_header() parses - * ONLY canonical headers, so a '## FOO' written here would be read back - * as body text of the section above it and a second identical write - * would append a duplicate — destroying exactly the idempotence this - * mode exists to provide. */ + /* Any heading name is writable — the canonical six are a convention, + * not a privilege. What is still refused is a name that could not + * round-trip through a "## NAME" line (empty, '#'-leading, newline- or + * edge-whitespace-bearing, over-long): such a name would scan back as + * a different heading or as none, so a second identical write would + * append a duplicate instead of being a no-op. */ if (!updates.status && cbm_adr_validate_section_keys( (const char **)updates.keys, updates.count, section_key_err, (int)sizeof(section_key_err)) != CBM_STORE_OK) { diff --git a/src/store/store.c b/src/store/store.c index e8bc8e5f4..dd97ef2d8 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -8708,47 +8708,324 @@ int cbm_adr_validate_content(const char *content, char *errbuf, int errbuf_size) return CBM_STORE_OK; } -int cbm_adr_validate_section_keys(const char **keys, int count, char *errbuf, int errbuf_size) { - char invalid[CBM_SZ_256] = ""; - int ilen = 0; - int ninvalid = 0; +/* ── ADR heading scanning and splicing ──────────────────────────── */ - /* Collect and sort invalid keys */ - const char *inv_keys[ST_MAX_SECTIONS]; - int inv_n = 0; - for (int i = 0; i < count; i++) { - if (!is_canonical_section(keys[i])) { - if (inv_n < ST_BUF_16) { - inv_keys[inv_n++] = keys[i]; +/* Length of a code-fence run (``` or ~~~, after at most three spaces of + * indent), or 0 when the line does not open or close a fence. */ +static int adr_fence_run(const char *line, int line_len, char *ch_out) { + int i = 0; + while (i < line_len && i < ST_HEADER_PREFIX && line[i] == ' ') { + i++; + } + if (i >= line_len || (line[i] != '`' && line[i] != '~')) { + return 0; + } + char c = line[i]; + int n = 0; + while (i + n < line_len && line[i + n] == c) { + n++; + } + if (n < ST_HEADER_PREFIX) { + return 0; + } + *ch_out = c; + return n; +} + +/* True when the line is a closing fence for an open run of `open_n` `open_ch`: + * same character, at least as long, and nothing but whitespace after it. */ +static bool adr_fence_closes(const char *line, int line_len, char open_ch, int open_n) { + char ch = 0; + int n = adr_fence_run(line, line_len, &ch); + if (n == 0 || ch != open_ch || n < open_n) { + return false; + } + int i = 0; + while (i < line_len && line[i] != open_ch) { + i++; + } + i += n; + while (i < line_len) { + if (line[i] != ' ' && line[i] != '\t' && line[i] != '\r') { + return false; + } + i++; + } + return true; +} + +/* Name length of a "## NAME" heading line, or 0 when it is not a heading. + * "###" fails on the third '#', so deeper markdown headings are body text. */ +static int adr_heading_name_len(const char *line, int line_len) { + if (line_len <= ST_HEADER_PREFIX || line[0] != '#' || line[SKIP_ONE] != '#' || + line[PAIR_LEN] != ' ') { + return 0; + } + int len = line_len - ST_HEADER_PREFIX; + const char *name = line + ST_HEADER_PREFIX; + while (len > 0 && (name[len - SKIP_ONE] == ' ' || name[len - SKIP_ONE] == '\t' || + name[len - SKIP_ONE] == '\r')) { + len--; + } + /* A leading space would not survive a render/scan round-trip. */ + if (len == 0 || name[0] == ' ' || name[0] == '\t') { + return 0; + } + return len; +} + +/* True when the document leaves a code fence open. Such a document cannot be + * spliced: every heading after the opener is hidden, so an update would append + * a duplicate heading instead of replacing the existing one. */ +static bool adr_has_unterminated_fence(const char *content) { + const char *p = content; + bool in_fence = false; + char fence_ch = 0; + int fence_n = 0; + while (p && *p) { + const char *eol = strchr(p, '\n'); + int line_len = eol ? (int)(eol - p) : (int)strlen(p); + if (in_fence) { + if (adr_fence_closes(p, line_len, fence_ch, fence_n)) { + in_fence = false; + } + } else { + char ch = 0; + int n = adr_fence_run(p, line_len, &ch); + if (n > 0) { + in_fence = true; + fence_ch = ch; + fence_n = n; } } - } - /* Sort alphabetically */ - for (int i = SKIP_ONE; i < inv_n; i++) { - int j = i; - while (j > 0 && strcmp(inv_keys[j], inv_keys[j - SKIP_ONE]) < 0) { - const char *tmp = inv_keys[j]; - inv_keys[j] = inv_keys[j - SKIP_ONE]; - inv_keys[j - SKIP_ONE] = tmp; - j--; + if (!eol) { + break; } + p = eol + SKIP_ONE; } + return in_fence; +} - for (int i = 0; i < inv_n; i++) { - if (ilen > 0) { - ilen += snprintf(invalid + ilen, sizeof(invalid) - ilen, ", "); - } - ilen += snprintf(invalid + ilen, sizeof(invalid) - ilen, "%s", inv_keys[i]); - ninvalid++; +int cbm_adr_scan_headings(const char *content, void (*cb)(void *ctx, const cbm_adr_heading_t *h), + void *ctx) { + if (!content || !cb) { + return content ? CBM_STORE_ERR : CBM_STORE_OK; } - - if (ninvalid > 0) { - snprintf(errbuf, errbuf_size, - "invalid section names: %s. Valid sections: PURPOSE, STACK, ARCHITECTURE, " - "PATTERNS, TRADEOFFS, PHILOSOPHY", - invalid); + if (adr_has_unterminated_fence(content)) { return CBM_STORE_ERR; } + + cbm_adr_heading_t pending; + memset(&pending, 0, sizeof(pending)); + bool have_pending = false; + bool in_fence = false; + char fence_ch = 0; + int fence_n = 0; + + const char *p = content; + while (*p) { + const char *eol = strchr(p, '\n'); + int line_len = eol ? (int)(eol - p) : (int)strlen(p); + size_t line_off = (size_t)(p - content); + + if (in_fence) { + if (adr_fence_closes(p, line_len, fence_ch, fence_n)) { + in_fence = false; + } + } else { + char ch = 0; + int fn = adr_fence_run(p, line_len, &ch); + if (fn > 0) { + in_fence = true; + fence_ch = ch; + fence_n = fn; + } else { + int name_len = adr_heading_name_len(p, line_len); + if (name_len > 0) { + /* The previous heading's body ends where this one starts. */ + if (have_pending) { + pending.body_end = line_off; + cb(ctx, &pending); + } + pending.name = p + ST_HEADER_PREFIX; + pending.name_len = name_len; + pending.heading_start = line_off; + pending.body_start = + eol ? line_off + (size_t)line_len + SKIP_ONE : line_off + (size_t)line_len; + have_pending = true; + } + } + } + if (!eol) { + break; + } + p = eol + SKIP_ONE; + } + if (have_pending) { + pending.body_end = strlen(content); + cb(ctx, &pending); + } + return CBM_STORE_OK; +} + +int cbm_adr_check_structure(const char *content, char *errbuf, int errbuf_size) { + if (content && adr_has_unterminated_fence(content)) { + snprintf(errbuf, (size_t)errbuf_size, + "the stored ADR leaves a code fence open, so its section boundaries are " + "ambiguous and a section write could splice into a code sample; repair it " + "with mode='update'"); + return CBM_STORE_ERR; + } + return CBM_STORE_OK; +} + +typedef struct { + const char *name; + int name_len; + bool found; + size_t body_start; + size_t body_end; +} adr_find_ctx_t; + +static void adr_find_cb(void *ctx, const cbm_adr_heading_t *h) { + adr_find_ctx_t *f = (adr_find_ctx_t *)ctx; + /* First occurrence wins, so a document that repeats a heading updates + * deterministically rather than depending on scan order. */ + if (f->found || h->name_len != f->name_len || + memcmp(h->name, f->name, (size_t)f->name_len) != 0) { + return; + } + f->found = true; + f->body_start = h->body_start; + f->body_end = h->body_end; +} + +char *cbm_adr_splice_section(const char *content, const char *name, const char *body) { + if (!name || !body) { + return NULL; + } + const char *doc = content ? content : ""; + size_t doc_len = strlen(doc); + + /* Trailing newlines are stripped from the incoming body so that splicing + * the same body twice is byte-identical: without this the separator run + * would grow by one blank line on every repeat. */ + size_t body_len = strlen(body); + while (body_len > 0 && + (body[body_len - SKIP_ONE] == '\n' || body[body_len - SKIP_ONE] == '\r')) { + body_len--; + } + + adr_find_ctx_t find; + memset(&find, 0, sizeof(find)); + find.name = name; + find.name_len = (int)strlen(name); + if (cbm_adr_scan_headings(doc, adr_find_cb, &find) != CBM_STORE_OK) { + return NULL; + } + + char *out = NULL; + if (find.found) { + /* Keep the replaced region's own trailing newline run, so the blank + * line that separates this section from the next survives untouched. */ + size_t ws = find.body_end; + while (ws > find.body_start && (doc[ws - SKIP_ONE] == '\n' || doc[ws - SKIP_ONE] == '\r')) { + ws--; + } + size_t ws_len = find.body_end - ws; + size_t total = find.body_start + body_len + ws_len + (doc_len - find.body_end); + out = malloc(total + SKIP_ONE); + if (!out) { + return NULL; + } + size_t at = 0; + memcpy(out, doc, find.body_start); + at += find.body_start; + memcpy(out + at, body, body_len); + at += body_len; + memcpy(out + at, doc + ws, ws_len); + at += ws_len; + memcpy(out + at, doc + find.body_end, doc_len - find.body_end); + at += doc_len - find.body_end; + out[at] = '\0'; + return out; + } + + /* Absent: append. Enough newlines are added to leave exactly one blank + * line before the new heading — never fewer, and never rewriting the + * newlines the document already ends with. */ + size_t trailing = 0; + while (trailing < doc_len && doc[doc_len - SKIP_ONE - trailing] == '\n') { + trailing++; + } + size_t sep = 0; + if (doc_len > 0 && trailing < PAIR_LEN) { + sep = PAIR_LEN - trailing; + } + size_t name_len = strlen(name); + size_t total = doc_len + sep + ST_HEADER_PREFIX + name_len + SKIP_ONE + body_len; + out = malloc(total + SKIP_ONE); + if (!out) { + return NULL; + } + size_t at = 0; + memcpy(out, doc, doc_len); + at += doc_len; + for (size_t i = 0; i < sep; i++) { + out[at++] = '\n'; + } + memcpy(out + at, "## ", ST_HEADER_PREFIX); + at += ST_HEADER_PREFIX; + memcpy(out + at, name, name_len); + at += name_len; + out[at++] = '\n'; + memcpy(out + at, body, body_len); + at += body_len; + out[at] = '\0'; + return out; +} + +int cbm_adr_validate_section_name(const char *name, char *errbuf, int errbuf_size) { + if (!name || !name[0]) { + snprintf(errbuf, (size_t)errbuf_size, "section name must not be empty"); + return CBM_STORE_ERR; + } + size_t len = strlen(name); + if (len >= CBM_SZ_64) { + snprintf(errbuf, (size_t)errbuf_size, "section name must be shorter than %d characters", + CBM_SZ_64); + return CBM_STORE_ERR; + } + /* Everything below would make the written "## NAME" line scan back as a + * different heading, or as no heading at all. */ + if (name[0] == '#') { + snprintf(errbuf, (size_t)errbuf_size, "section name must not start with '#': %s", name); + return CBM_STORE_ERR; + } + for (size_t i = 0; i < len; i++) { + if (name[i] == '\n' || name[i] == '\r') { + snprintf(errbuf, (size_t)errbuf_size, "section name must not contain a line break"); + return CBM_STORE_ERR; + } + } + if (name[0] == ' ' || name[0] == '\t' || name[len - SKIP_ONE] == ' ' || + name[len - SKIP_ONE] == '\t') { + snprintf(errbuf, (size_t)errbuf_size, + "section name must not start or end with whitespace: '%s'", name); + return CBM_STORE_ERR; + } + return CBM_STORE_OK; +} + +int cbm_adr_validate_section_keys(const char **keys, int count, char *errbuf, int errbuf_size) { + /* Section names used to be restricted to the six canonical headings. They + * are now a convention rather than a privilege, so the only names refused + * are the ones that could not round-trip through a "## NAME" line. */ + for (int i = 0; i < count; i++) { + if (cbm_adr_validate_section_name(keys[i], errbuf, errbuf_size) != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + } return CBM_STORE_OK; } @@ -8906,35 +9183,32 @@ int cbm_store_adr_update_sections(cbm_store_t *s, const char *project, const cha return rc; } - /* Parse existing sections */ - cbm_adr_sections_t sections = cbm_adr_parse_sections(existing.content); - cbm_store_adr_free(&existing); - - /* Merge new sections */ - for (int i = 0; i < count; i++) { - bool found = false; - for (int j = 0; j < sections.count; j++) { - if (strcmp(sections.keys[j], keys[i]) == 0) { - free(sections.values[j]); - sections.values[j] = heap_strdup(values[i]); - found = true; - break; - } - } - if (!found && sections.count < ST_BUF_16) { - sections.keys[sections.count] = heap_strdup(keys[i]); - sections.values[sections.count] = heap_strdup(values[i]); - sections.count++; - } + /* Splice each section in place. The document is NOT rebuilt from parsed + * sections: that model drops the preamble, drops headings it does not + * recognise (including a mis-cased '## Purpose', whose whole block then + * disappears) and reorders the rest, so rendering from it would silently + * rewrite text nobody asked to change. Replacing byte spans leaves every + * untouched byte exactly as it was. */ + char structure_err[CBM_SZ_256] = ""; + if (cbm_adr_check_structure(existing.content, structure_err, (int)sizeof(structure_err)) != + CBM_STORE_OK) { + cbm_store_adr_free(&existing); + (void)cbm_store_rollback(s); + store_set_error(s, structure_err); + return CBM_STORE_ERR; } - /* Render merged */ - char *merged = cbm_adr_render(§ions); - cbm_adr_sections_free(§ions); + char *merged = heap_strdup(existing.content ? existing.content : ""); + cbm_store_adr_free(&existing); + for (int i = 0; merged && i < count; i++) { + char *next = cbm_adr_splice_section(merged, keys[i], values[i]); + free(merged); + merged = next; + } if (!merged) { (void)cbm_store_rollback(s); - store_set_error(s, "failed to render merged ADR"); + store_set_error(s, "failed to splice ADR section"); return CBM_STORE_ERR; } diff --git a/src/store/store.h b/src/store/store.h index 2e952d064..e6538c35d 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -877,6 +877,47 @@ int cbm_adr_validate_content(const char *content, char *errbuf, int errbuf_size) int cbm_adr_validate_section_keys(const char **keys, int count, char *errbuf, int errbuf_size); void cbm_adr_sections_free(cbm_adr_sections_t *s); +/* ── ADR section headings (the splice model) ──────────────────── + * + * A section write must not rebuild the document. cbm_adr_parse_sections() + + * cbm_adr_render() is a lossy model — it drops everything before the first + * heading, drops headings it does not recognise, and reorders what is left — + * so rebuilding from it silently rewrites text nobody asked to change. These + * functions locate a heading's byte span instead, so a write replaces that + * span and leaves every other byte of the document exactly as it was. + * + * A heading is a line of the form "## NAME" that is NOT inside a fenced code + * block. NAME is matched EXACTLY, including case: "## Purpose" and + * "## PURPOSE" are different sections, because folding them would silently + * merge two blocks the author chose to keep apart. The six canonical names + * are a convention (see ADR_EMPTY_HINT), not a privilege: any name that + * round-trips is a section, and none of them gets ordering priority. */ +typedef struct { + const char *name; /* into the source buffer; NOT NUL-terminated */ + int name_len; + size_t heading_start; /* offset of the first '#' of the heading line */ + size_t body_start; /* offset just past the heading line's newline */ + size_t body_end; /* offset of the next heading, or end of document */ +} cbm_adr_heading_t; + +/* Reports every heading in document order. Returns CBM_STORE_ERR without + * calling `cb` when the document has an unterminated code fence: its structure + * is ambiguous, and guessing could splice into a code sample. */ +int cbm_adr_scan_headings(const char *content, void (*cb)(void *ctx, const cbm_adr_heading_t *h), + void *ctx); + +/* CBM_STORE_ERR + message when the document cannot be spliced safely. */ +int cbm_adr_check_structure(const char *content, char *errbuf, int errbuf_size); + +/* Replaces the body of `name`, or appends the section when it is absent. + * Returns a new document (caller frees), or NULL on bad input, an unterminated + * fence, or OOM. Bytes outside the replaced span are preserved exactly, and + * splicing the same name and body twice is byte-identical to doing it once. */ +char *cbm_adr_splice_section(const char *content, const char *name, const char *body); + +/* Rejects names that could not round-trip through a "## NAME" heading. */ +int cbm_adr_validate_section_name(const char *name, char *errbuf, int errbuf_size); + /* ── Search helpers (exposed for testing) ───────────────────────── */ /* Convert a glob pattern to SQL LIKE pattern. Caller must free result. */ diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 21b5826b3..cacd4d88d 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -6012,11 +6012,11 @@ TEST(tool_manage_adr_set_sections_without_updates_errors) { PASS(); } -/* An empty body would render a heading with nothing under it — a content - * deletion wearing the response shape of an update. A non-canonical name would - * be parsed back as body text of the section above it, so writing it twice - * would duplicate it: both are rejected before any store is opened. */ -TEST(tool_manage_adr_set_sections_rejects_empty_and_unknown_sections) { +/* An empty body would leave a heading with nothing under it — a content + * deletion wearing the response shape of an update. A name that cannot survive + * a "## NAME" round-trip would scan back as a different heading or as none, so + * writing it twice would duplicate it. Both are refused before a store opens. */ +TEST(tool_manage_adr_set_sections_rejects_unwritable_sections) { const char *project = "adr-sec-guards"; cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); ASSERT_NOT_NULL(srv); @@ -6034,12 +6034,21 @@ TEST(tool_manage_adr_set_sections_rejects_empty_and_unknown_sections) { ASSERT_NOT_NULL(strstr(resp, "\"isError\":true")); free(resp); + /* A '#'-leading name would render "## # PURPOSE" and scan back different. */ resp = cbm_mcp_handle_tool(srv, "manage_adr", "{\"project\":\"adr-sec-guards\",\"mode\":\"set_sections\"," - "\"section_updates\":{\"DECISIONS\":\"- Chose SQLite.\"}}"); + "\"section_updates\":{\"# PURPOSE\":\"x\"}}"); + ASSERT_NOT_NULL(resp); + ASSERT_NOT_NULL(strstr(resp, "invalid_section_name")); + ASSERT_NOT_NULL(strstr(resp, "\"isError\":true")); + free(resp); + + /* A newline in the name would forge a second heading line. */ + resp = cbm_mcp_handle_tool(srv, "manage_adr", + "{\"project\":\"adr-sec-guards\",\"mode\":\"set_sections\"," + "\"section_updates\":{\"A\\nB\":\"x\"}}"); ASSERT_NOT_NULL(resp); ASSERT_NOT_NULL(strstr(resp, "invalid_section_name")); - ASSERT_NOT_NULL(strstr(resp, "DECISIONS")); ASSERT_NOT_NULL(strstr(resp, "\"isError\":true")); free(resp); @@ -6062,6 +6071,201 @@ TEST(tool_manage_adr_set_sections_rejects_empty_and_unknown_sections) { PASS(); } +/* The use case the whole change exists for: add an entry under its own + * heading, and be able to retry it. Under the old canonical-only rules this + * was the exact request that would have corrupted an ADR. */ +TEST(tool_manage_adr_set_sections_adds_custom_heading) { + const char *project = "adr-sec-custom"; + const char *request = "{\"project\":\"adr-sec-custom\",\"mode\":\"set_sections\"," + "\"section_updates\":{\"DECISIONS\":\"- Chose SQLite.\"}}"; + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + cbm_store_t *st = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(st); + ASSERT_EQ(cbm_store_upsert_project(st, project, "/tmp/adr-sec-custom"), CBM_STORE_OK); + cbm_mcp_server_set_project(srv, project); + ASSERT_EQ(cbm_store_adr_store(st, project, "## PURPOSE\nFoo"), CBM_STORE_OK); + + char *resp = cbm_mcp_handle_tool(srv, "manage_adr", request); + ASSERT_NOT_NULL(resp); + ASSERT_NOT_NULL(strstr(resp, "sections_updated")); + ASSERT_NULL(strstr(resp, "\"isError\":true")); + free(resp); + + cbm_adr_t adr; + memset(&adr, 0, sizeof(adr)); + ASSERT_EQ(cbm_store_adr_get(st, project, &adr), CBM_STORE_OK); + ASSERT_STR_EQ(adr.content, "## PURPOSE\nFoo\n\n## DECISIONS\n- Chose SQLite."); + cbm_store_adr_free(&adr); + + /* Retry the identical request: byte-identical, not duplicated. */ + resp = cbm_mcp_handle_tool(srv, "manage_adr", request); + ASSERT_NOT_NULL(resp); + ASSERT_NOT_NULL(strstr(resp, "sections_updated")); + free(resp); + + memset(&adr, 0, sizeof(adr)); + ASSERT_EQ(cbm_store_adr_get(st, project, &adr), CBM_STORE_OK); + ASSERT_STR_EQ(adr.content, "## PURPOSE\nFoo\n\n## DECISIONS\n- Chose SQLite."); + cbm_store_adr_free(&adr); + + /* And it is now a real section the write path can target again. */ + resp = cbm_mcp_handle_tool(srv, "manage_adr", + "{\"project\":\"adr-sec-custom\",\"mode\":\"set_sections\"," + "\"section_updates\":{\"DECISIONS\":\"- Chose DuckDB.\"}}"); + ASSERT_NOT_NULL(resp); + ASSERT_NOT_NULL(strstr(resp, "sections_updated")); + free(resp); + + memset(&adr, 0, sizeof(adr)); + ASSERT_EQ(cbm_store_adr_get(st, project, &adr), CBM_STORE_OK); + ASSERT_STR_EQ(adr.content, "## PURPOSE\nFoo\n\n## DECISIONS\n- Chose DuckDB."); + cbm_store_adr_free(&adr); + + cbm_mcp_server_free(srv); + PASS(); +} + +/* Regression for real data loss: rebuilding the document from parsed sections + * dropped the preamble, dropped a mis-cased heading together with its whole + * block, and reordered what survived. Splicing leaves all of it alone. */ +TEST(tool_manage_adr_set_sections_preserves_preamble_and_order) { + const char *project = "adr-sec-preserve"; + /* The fenced block sits inside the mis-cased section, NOT the one being + * written: a section's body runs to the next heading, so rewriting STACK + * would legitimately replace a fence that belonged to STACK. */ + const char *stored = "Notes before any heading.\n\n" + "## Purpose\nMis-cased but real.\n\n" + "```md\n## Example\nfenced sample\n```\n\n" + "## STACK\nC and SQLite.\n\n" + "## PURPOSE\nCanonical one."; + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + cbm_store_t *st = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(st); + ASSERT_EQ(cbm_store_upsert_project(st, project, "/tmp/adr-sec-preserve"), CBM_STORE_OK); + cbm_mcp_server_set_project(srv, project); + ASSERT_EQ(cbm_store_adr_store(st, project, stored), CBM_STORE_OK); + + char *resp = cbm_mcp_handle_tool(srv, "manage_adr", + "{\"project\":\"adr-sec-preserve\",\"mode\":\"set_sections\"," + "\"section_updates\":{\"STACK\":\"C only.\"}}"); + ASSERT_NOT_NULL(resp); + ASSERT_NOT_NULL(strstr(resp, "sections_updated")); + free(resp); + + cbm_adr_t adr; + memset(&adr, 0, sizeof(adr)); + ASSERT_EQ(cbm_store_adr_get(st, project, &adr), CBM_STORE_OK); + ASSERT_STR_EQ(adr.content, "Notes before any heading.\n\n" + "## Purpose\nMis-cased but real.\n\n" + "```md\n## Example\nfenced sample\n```\n\n" + "## STACK\nC only.\n\n" + "## PURPOSE\nCanonical one."); + cbm_store_adr_free(&adr); + + cbm_mcp_server_free(srv); + PASS(); +} + +/* An unterminated fence hides every heading after it, so a write would append + * a duplicate heading rather than replace the real one. Refuse, explicitly, + * and leave the document alone. */ +TEST(tool_manage_adr_set_sections_refuses_unterminated_fence) { + const char *project = "adr-sec-fence"; + const char *stored = "## PURPOSE\nFoo\n\n```\nunclosed sample\n\n## STACK\nBar"; + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + cbm_store_t *st = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(st); + ASSERT_EQ(cbm_store_upsert_project(st, project, "/tmp/adr-sec-fence"), CBM_STORE_OK); + cbm_mcp_server_set_project(srv, project); + ASSERT_EQ(cbm_store_adr_store(st, project, stored), CBM_STORE_OK); + + char *resp = cbm_mcp_handle_tool(srv, "manage_adr", + "{\"project\":\"adr-sec-fence\",\"mode\":\"set_sections\"," + "\"section_updates\":{\"STACK\":\"New bar.\"}}"); + ASSERT_NOT_NULL(resp); + ASSERT_NOT_NULL(strstr(resp, "write_error")); + ASSERT_NOT_NULL(strstr(resp, "code fence")); + ASSERT_NOT_NULL(strstr(resp, "\"isError\":true")); + free(resp); + + cbm_adr_t adr; + memset(&adr, 0, sizeof(adr)); + ASSERT_EQ(cbm_store_adr_get(st, project, &adr), CBM_STORE_OK); + ASSERT_STR_EQ(adr.content, stored); + cbm_store_adr_free(&adr); + + /* mode='sections' reports the same ambiguity rather than a partial list. */ + resp = cbm_mcp_handle_tool(srv, "manage_adr", + "{\"project\":\"adr-sec-fence\",\"mode\":\"sections\"}"); + ASSERT_NOT_NULL(resp); + ASSERT_NOT_NULL(strstr(resp, "unterminated_code_fence")); + free(resp); + + cbm_mcp_server_free(srv); + PASS(); +} + +/* mode='sections' and the section write path must agree on what a heading is. + * They used to disagree — 'sections' listed every '#'-prefixed line, including + * ones inside code fences that no write could ever target — and two components + * disagreeing about what a section is is how a section write came to be able + * to destroy one. */ +TEST(tool_manage_adr_sections_agrees_with_write_path) { + const char *project = "adr-sec-agree"; + const char *stored = "Preamble.\n\n" + "# Title\n\n" + "## PURPOSE\nFoo\n\n" + "### Sub\n\n" + "```md\n## Fenced\n```\n\n" + "## DECISIONS\nBar"; + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + cbm_store_t *st = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(st); + ASSERT_EQ(cbm_store_upsert_project(st, project, "/tmp/adr-sec-agree"), CBM_STORE_OK); + cbm_mcp_server_set_project(srv, project); + ASSERT_EQ(cbm_store_adr_store(st, project, stored), CBM_STORE_OK); + + char *resp = cbm_mcp_handle_tool(srv, "manage_adr", + "{\"project\":\"adr-sec-agree\",\"mode\":\"sections\"}"); + ASSERT_NOT_NULL(resp); + /* Listed: exactly the two headings the write path can target. */ + ASSERT_NOT_NULL(strstr(resp, "## PURPOSE")); + ASSERT_NOT_NULL(strstr(resp, "## DECISIONS")); + /* Not listed: a fenced '##', a '#' title, a '###' subheading. */ + ASSERT_NULL(strstr(resp, "## Fenced")); + ASSERT_NULL(strstr(resp, "# Title")); + ASSERT_NULL(strstr(resp, "### Sub")); + free(resp); + + /* Every listed heading is writable, and the unlisted ones stay as text. */ + resp = cbm_mcp_handle_tool(srv, "manage_adr", + "{\"project\":\"adr-sec-agree\",\"mode\":\"set_sections\"," + "\"section_updates\":{\"DECISIONS\":\"New bar\"}}"); + ASSERT_NOT_NULL(resp); + ASSERT_NOT_NULL(strstr(resp, "sections_updated")); + free(resp); + + /* The '#', '###' and fenced lines are body text of PURPOSE, and writing a + * different section leaves every byte of them alone. */ + cbm_adr_t adr; + memset(&adr, 0, sizeof(adr)); + ASSERT_EQ(cbm_store_adr_get(st, project, &adr), CBM_STORE_OK); + ASSERT_STR_EQ(adr.content, "Preamble.\n\n" + "# Title\n\n" + "## PURPOSE\nFoo\n\n" + "### Sub\n\n" + "```md\n## Fenced\n```\n\n" + "## DECISIONS\nNew bar"); + cbm_store_adr_free(&adr); + + cbm_mcp_server_free(srv); + PASS(); +} + /* CBM_ADR_MAX_LENGTH is enforced on this path — mode='update' bypasses it, so * exposing an incremental writer without the cap would make unbounded growth * cheap. The rejected merge must also roll back to the byte-identical prior @@ -13446,7 +13650,11 @@ SUITE(mcp) { RUN_TEST(tool_manage_adr_set_sections_is_idempotent); RUN_TEST(tool_manage_adr_set_sections_creates_when_absent); RUN_TEST(tool_manage_adr_set_sections_without_updates_errors); - RUN_TEST(tool_manage_adr_set_sections_rejects_empty_and_unknown_sections); + RUN_TEST(tool_manage_adr_set_sections_rejects_unwritable_sections); + RUN_TEST(tool_manage_adr_set_sections_adds_custom_heading); + RUN_TEST(tool_manage_adr_set_sections_preserves_preamble_and_order); + RUN_TEST(tool_manage_adr_set_sections_refuses_unterminated_fence); + RUN_TEST(tool_manage_adr_sections_agrees_with_write_path); RUN_TEST(tool_manage_adr_set_sections_rejects_oversize); RUN_TEST(tool_manage_adr_set_sections_is_advertised); RUN_TEST(tool_index_repository_reports_store_backed_adr); diff --git a/tests/test_store_arch.c b/tests/test_store_arch.c index 159c07f95..3a93c4722 100644 --- a/tests/test_store_arch.c +++ b/tests/test_store_arch.c @@ -908,12 +908,181 @@ TEST(adr_validate_keys_valid) { PASS(); } -TEST(adr_validate_keys_invalid) { - const char *keys[] = {"PURPOSE", "STACKS", "CUSTOM"}; +/* Section names are no longer restricted to the canonical six — those are a + * convention now, so "STACKS" and "CUSTOM" are ordinary sections. What is + * still refused is a name that could not round-trip through a "## NAME" line: + * such a name scans back as a different heading or as none, so a second + * identical write would append a duplicate instead of being a no-op. */ +TEST(adr_validate_keys_accepts_arbitrary_names) { + const char *keys[] = {"PURPOSE", "STACKS", "CUSTOM", "Decisions (2026)"}; char errbuf[256]; - ASSERT_TRUE(cbm_adr_validate_section_keys(keys, 3, errbuf, sizeof(errbuf)) != CBM_STORE_OK); - ASSERT_TRUE(strstr(errbuf, "STACKS") != NULL); - ASSERT_TRUE(strstr(errbuf, "CUSTOM") != NULL); + ASSERT_EQ(cbm_adr_validate_section_keys(keys, 4, errbuf, sizeof(errbuf)), CBM_STORE_OK); + PASS(); +} + +TEST(adr_validate_keys_rejects_unroundtrippable) { + char errbuf[256]; + const char *empty[] = {""}; + ASSERT_TRUE(cbm_adr_validate_section_keys(empty, 1, errbuf, sizeof(errbuf)) != CBM_STORE_OK); + + const char *hashed[] = {"# PURPOSE"}; + ASSERT_TRUE(cbm_adr_validate_section_keys(hashed, 1, errbuf, sizeof(errbuf)) != CBM_STORE_OK); + + const char *newline[] = {"PUR\nPOSE"}; + ASSERT_TRUE(cbm_adr_validate_section_keys(newline, 1, errbuf, sizeof(errbuf)) != CBM_STORE_OK); + + const char *padded[] = {" PURPOSE "}; + ASSERT_TRUE(cbm_adr_validate_section_keys(padded, 1, errbuf, sizeof(errbuf)) != CBM_STORE_OK); + + char toolong[128]; + memset(toolong, 'X', sizeof(toolong) - 1); + toolong[sizeof(toolong) - 1] = '\0'; + const char *big[] = {toolong}; + ASSERT_TRUE(cbm_adr_validate_section_keys(big, 1, errbuf, sizeof(errbuf)) != CBM_STORE_OK); + PASS(); +} + +/* ── Splice: bytes outside the target span never change ─────────── */ + +typedef struct { + char buf[512]; + int n; +} adr_name_collect_t; + +static void adr_collect_names(void *ctx, const cbm_adr_heading_t *h) { + adr_name_collect_t *c = (adr_name_collect_t *)ctx; + c->n += snprintf(c->buf + c->n, sizeof(c->buf) - (size_t)c->n, "[%.*s]", h->name_len, h->name); +} + +static int adr_check_splice(const char *in, const char *name, const char *body, + const char *expect) { + char *out = cbm_adr_splice_section(in, name, body); + ASSERT_NOT_NULL(out); + if (strcmp(out, expect) != 0) { + printf(" %sFAIL%s %s: splice(\"%s\")\n in >>>%s<<<\n got >>>%s<<<\n" + " expect >>>%s<<<\n", + tf_red(), tf_reset(), __FILE__, name, in, out, expect); + free(out); + return 1; + } + free(out); + return 0; +} + +#define CHECK_SPLICE(in, name, body, expect) \ + do { \ + if (adr_check_splice((in), (name), (body), (expect)) != 0) { \ + return 1; \ + } \ + } while (0) + +/* THE acceptance property. Every document below is a case where rebuilding + * from cbm_adr_parse_sections() would have rewritten or destroyed text: a + * preamble (dropped), a mis-cased heading (dropped with its whole block), an + * unrecognised heading in prose (absorbed), a fenced '##' (absorbed), and + * out-of-canonical-order sections (reordered). Splicing touches only the + * target span, so each survives byte-for-byte. */ +TEST(adr_splice_preserves_untouched_bytes) { + /* Preamble before the first heading. */ + CHECK_SPLICE("Some preamble.\n\n## PURPOSE\nFoo", "PURPOSE", "New foo", + "Some preamble.\n\n## PURPOSE\nNew foo"); + + /* Mis-cased heading: a real section now, and untouched when not targeted. */ + CHECK_SPLICE("## Purpose\nFoo\n\n## STACK\nBar", "STACK", "New bar", + "## Purpose\nFoo\n\n## STACK\nNew bar"); + + /* A heading in prose that the old parser absorbed into the section above. */ + CHECK_SPLICE("## PURPOSE\nFoo\n## CUSTOM\nStill here\n\n## STACK\nBar", "STACK", "New bar", + "## PURPOSE\nFoo\n## CUSTOM\nStill here\n\n## STACK\nNew bar"); + + /* Fenced code containing a '##' line. */ + CHECK_SPLICE("## PURPOSE\nFoo\n\n```md\n## Example\n```\n\n## STACK\nBar", "STACK", "New bar", + "## PURPOSE\nFoo\n\n```md\n## Example\n```\n\n## STACK\nNew bar"); + + /* Sections stored out of canonical order keep the author's order. */ + CHECK_SPLICE("## STACK\nBar\n\n## PURPOSE\nFoo", "PURPOSE", "New foo", + "## STACK\nBar\n\n## PURPOSE\nNew foo"); + + /* The separator run before the next heading is preserved exactly. */ + CHECK_SPLICE("## A\nx\n\n\n## B\ny", "A", "z", "## A\nz\n\n\n## B\ny"); + + /* A trailing newline on the document is preserved. */ + CHECK_SPLICE("## A\nx\n", "A", "z", "## A\nz\n"); + PASS(); +} + +/* The original use case: add an entry under its own heading. */ +TEST(adr_splice_appends_arbitrary_heading) { + CHECK_SPLICE("## PURPOSE\nFoo", "DECISIONS", "- Chose SQLite.", + "## PURPOSE\nFoo\n\n## DECISIONS\n- Chose SQLite."); + /* A document already ending in a blank line does not gain another. */ + CHECK_SPLICE("## PURPOSE\nFoo\n\n", "DECISIONS", "- Chose SQLite.", + "## PURPOSE\nFoo\n\n## DECISIONS\n- Chose SQLite."); + /* An empty document becomes a plain create with no leading separator. */ + CHECK_SPLICE("", "PURPOSE", "Only entry.", "## PURPOSE\nOnly entry."); + PASS(); +} + +/* Splicing twice must be byte-identical to splicing once — including for a + * non-canonical heading, the case that would have corrupted an ADR under the + * old rules. Trailing newlines on the body are normalised so the separator run + * cannot grow by a blank line on every repeat. */ +TEST(adr_splice_is_idempotent) { + const char *doc = "## PURPOSE\nFoo\n\n## STACK\nBar"; + char *once = cbm_adr_splice_section(doc, "DECISIONS", "- Chose SQLite.\n"); + ASSERT_NOT_NULL(once); + char *twice = cbm_adr_splice_section(once, "DECISIONS", "- Chose SQLite.\n"); + ASSERT_NOT_NULL(twice); + ASSERT_STR_EQ(twice, once); + char *thrice = cbm_adr_splice_section(twice, "DECISIONS", "- Chose SQLite.\n"); + ASSERT_NOT_NULL(thrice); + ASSERT_STR_EQ(thrice, once); + free(once); + free(twice); + free(thrice); + PASS(); +} + +/* Names match exactly, including case: folding them would silently merge two + * blocks the author chose to keep apart. */ +TEST(adr_splice_matches_case_exactly) { + CHECK_SPLICE("## Purpose\nLower\n\n## PURPOSE\nUpper", "PURPOSE", "New", + "## Purpose\nLower\n\n## PURPOSE\nNew"); + CHECK_SPLICE("## Purpose\nLower\n\n## PURPOSE\nUpper", "Purpose", "New", + "## Purpose\nNew\n\n## PURPOSE\nUpper"); + PASS(); +} + +/* A '##' line inside a fenced code block is a code sample, not a heading — + * and '#' / '###' are not section headings at any position. */ +TEST(adr_splice_ignores_heading_inside_fence) { + adr_name_collect_t c; + memset(&c, 0, sizeof(c)); + ASSERT_EQ(cbm_adr_scan_headings( + "## PURPOSE\nFoo\n\n```md\n## Example\n```\n\n### Sub\n# Title\n\n## STACK\nBar", + adr_collect_names, &c), + CBM_STORE_OK); + ASSERT_STR_EQ(c.buf, "[PURPOSE][STACK]"); + PASS(); +} + +/* An unterminated fence hides every heading after it, so an update would + * append a duplicate rather than replace. Refuse instead of guessing. */ +TEST(adr_splice_refuses_unterminated_fence) { + const char *doc = "## PURPOSE\nFoo\n\n```\nunclosed sample\n\n## STACK\nBar"; + char errbuf[256]; + ASSERT_TRUE(cbm_adr_check_structure(doc, errbuf, sizeof(errbuf)) != CBM_STORE_OK); + ASSERT_TRUE(strstr(errbuf, "code fence") != NULL); + + adr_name_collect_t c; + memset(&c, 0, sizeof(c)); + ASSERT_TRUE(cbm_adr_scan_headings(doc, adr_collect_names, &c) != CBM_STORE_OK); + ASSERT_EQ(c.n, 0); + + ASSERT_NULL(cbm_adr_splice_section(doc, "STACK", "New bar")); + + /* A closed fence is fine. */ + ASSERT_EQ(cbm_adr_check_structure("## A\n```\nx\n```\n", errbuf, sizeof(errbuf)), CBM_STORE_OK); PASS(); } @@ -1448,7 +1617,14 @@ SUITE(store_arch) { RUN_TEST(adr_validate_missing_sections); RUN_TEST(adr_validate_empty); RUN_TEST(adr_validate_keys_valid); - RUN_TEST(adr_validate_keys_invalid); + RUN_TEST(adr_validate_keys_accepts_arbitrary_names); + RUN_TEST(adr_validate_keys_rejects_unroundtrippable); + RUN_TEST(adr_splice_preserves_untouched_bytes); + RUN_TEST(adr_splice_appends_arbitrary_heading); + RUN_TEST(adr_splice_is_idempotent); + RUN_TEST(adr_splice_matches_case_exactly); + RUN_TEST(adr_splice_ignores_heading_inside_fence); + RUN_TEST(adr_splice_refuses_unterminated_fence); RUN_TEST(adr_validate_keys_empty); /* Louvain */ From 5eae752f489654a7e1b04450a0f77256d0b4b05f Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 29 Aug 2026 14:02:27 +0200 Subject: [PATCH 29/31] test(parallel): record the weak-member trade for the LSP-ambiguity probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit parallel_lsp_index_exact_ambiguity_does_not_fall_through_to_legacy asserted alpha_edges == 1, which the Python weak-member guard suppresses. That assertion is a side effect, not the test's subject. Its subject — that an ambiguous exact LSP match must not fall through to the lower-ranked legacy semantic row — is unchanged: beta_edges == 0, legacy_edges == 0, and all four probe assertions (ambiguity.injected, legacy_injected, carrier_allows_fallback, shared_matcher_failed_closed) still hold. Verified before changing the number, not after. The harness deliberately injects an ambiguous exact match and forces the shared matcher to fail closed, so value.render() drops past the LSP into the registry with three same-named candidates (Alpha/Beta/Legacy.render) and bound by suffix_match -- measured: strategy=suffix_match, cands=3, conf=0.5500. That is a 1-in-3 guess that happened to land on Alpha: exactly the weak member-call class this change suppresses, and the same accepted trade recorded on python/S6 in test_lsp_resolution_probe.c. Not over-suppression: sibling tests in this suite index the same value.render() source and resolve it via lsp_method (cands=1, conf=0.9000), a strategy the guard keeps -- they still assert 1. Only the sabotaged-LSP path degrades to a weak textual guess. Flips back to 1 once py_lsp_cross resolves the receiver on this path. Refs #1276 Signed-off-by: Martin Vogel --- tests/test_parallel.c | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/tests/test_parallel.c b/tests/test_parallel.c index 1a1c42ce3..2b94130e5 100644 --- a/tests/test_parallel.c +++ b/tests/test_parallel.c @@ -1308,7 +1308,7 @@ TEST(parallel_lsp_index_exact_ambiguity_does_not_fall_through_to_legacy) { const cbm_gbuf_edge_t *legacy_edge = find_calls_edge_by_tails(gbuf, "Caller.run", "Legacy.render"); if (!probe.ambiguity.injected || !probe.legacy_injected || !probe.carrier_allows_fallback || - !probe.shared_matcher_failed_closed || alpha_edges != 1 || beta_edges != 0 || + !probe.shared_matcher_failed_closed || alpha_edges != 0 || beta_edges != 0 || legacy_edges != 0) { printf(" exact ambiguity + legacy diagnostic: exact_ambiguity=%d legacy=%d " "fallback_allowed=%d shared_failed_closed=%d alpha=%d beta=%d legacy_edge=%d " @@ -1325,9 +1325,26 @@ TEST(parallel_lsp_index_exact_ambiguity_does_not_fall_through_to_legacy) { ASSERT_TRUE(probe.legacy_injected); ASSERT_TRUE(probe.carrier_allows_fallback); ASSERT_TRUE(probe.shared_matcher_failed_closed); - /* The ordinary registry/type fallback may still recover the source-proven - * Alpha receiver. Only the lower-ranked legacy semantic row is forbidden. */ - ASSERT_EQ(alpha_edges, 1); + /* alpha_edges is a SIDE EFFECT of this test, not its subject. The subject is + * beta_edges == 0 and legacy_edges == 0 below: an ambiguous exact LSP match + * must not fall through to the lower-ranked legacy semantic row. Both are + * unchanged, as are all four probe assertions above. + * + * It used to be 1 because this harness DELIBERATELY injects an ambiguous + * exact match and forces the shared matcher to fail closed, so + * `value.render()` drops past the LSP into the registry with THREE + * same-named candidates (Alpha/Beta/Legacy.render) and bound by + * suffix_match — MEASURED: strategy=suffix_match, cands=3, conf=0.5500. + * That is a 1-in-3 guess that happened to land on Alpha: exactly the weak + * member-call class the Python guard now suppresses (#1276), and the same + * accepted trade recorded on python/S6 in test_lsp_resolution_probe.c. + * + * NOT over-suppression: sibling tests in this suite index the same + * `value.render()` source and resolve it via lsp_method (cands=1, + * conf=0.9000), a strategy the guard keeps — they still assert 1. Only the + * sabotaged-LSP path degrades to a weak textual guess. + * Flips back to 1 once py_lsp_cross resolves the receiver on this path. */ + ASSERT_EQ(alpha_edges, 0); ASSERT_EQ(beta_edges, 0); ASSERT_EQ(legacy_edges, 0); PASS(); From 2910e2842f912be2dbb3a8c7fbcd939881cc6616 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 29 Aug 2026 12:39:53 +0200 Subject: [PATCH 30/31] feat(lsp): resolve cross-file base classes for the Python/TS cross-LSP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `CBMDefinition.base_classes` carries the SOURCE SPELLING of each base ("Base", "django.db.Model"): extraction strips keywords and generic arguments, but it cannot know where the name is declared. The Python and TS cross-file registrars, however, consume `CBMLSPDef.embedded_types` as fully-qualified names — py_lookup_attribute and ts_lookup_member feed each entry straight into cbm_registry_lookup_type. An unqualified spelling therefore matched nothing declared in ANOTHER file. `class Child(Base)` in child.py never saw Base in base.py, so a call to an inherited method through a typed receiver found no member and fell through to the weak textual cascade, where the receiver-aware guard (#592/#606) correctly kills it. The member-lookup walk over embedded_types was already there in both languages; only the names it walked were unusable across files. cbm_pxc_collect_all_defs now resolves each base spelling to a project QN using exactly the inputs pass_semantic uses to draw its INHERITS edge: the project registry, the declaring module, and the file's import map. Two properties follow. The LSP's inheritance view is the same relation the graph records, so the two cannot diverge. And the binding is import- or same-module-backed rather than a short-name guess, so the CALLS edge it enables is a supported fact that the weak-member guard keeps. Weak registry strategies (suffix_match / unique_name / field_type_hint / fuzzy) are rejected for bases via an explicit drop-list: a base bound because some project type happens to share its name is precisely the fabricated relation #606 removed, and inheritance multiplies it — every inherited member of the wrong base would become a callable target. An unresolved base keeps its raw spelling and the behaviour that predates this change, so stdlib and third-party bases are unaffected. Scoped to the languages whose registrars read embedded_types as QNs (Python, JS/TS/TSX). Go, JVM, C#, C++ and Rust already qualify their own embedded types and keep the raw join. Cost is O(defs) hash lookups plus ONE import map per file: no per-call- site hierarchy walk, no registry scan, no per-file registry rebuild. The complexity guard stays linear (nodes 1.93, edges 2.07, per-file defs 2.00 against a files x corpus coupling of ~4). Tests: - ts/S6 flips to its own documented condition: was `calls == 0` as a tripwire recording the gap, now asserts calls >= 1 AND an lsp_ts_* strategy. Measured: strategy=lsp_ts_method, confidence 0.95. - python/S6b is new and covers what python/S6 structurally cannot: S6's `def run(c)` parameter is un-annotated, so no receiver type exists to inherit through and any binding there is a guess about a name (it resolves via unique_name at 0.75). S6b supplies the receiver three ways — annotated parameter, `self` in the subclass, constructor result — and asserts INHERITS >= 1 plus an lsp_* strategy. Revert-check: all three fall back to unique_name at 0.75 without this change, lsp_method at 0.90 with it. - Both rows assert the STRATEGY, not just the edge count: a short-name guess and a resolved inherited member produce the same count in a two-file fixture, and only the strategy tells them apart. - Corrects python/S6's stale comment citing a base_classes extraction bug; extraction_inheritance is green for Python and TS, and base_classes now holds clean names. macOS full suite 7690 passed / 0 failed / 8 skipped (140 suites); make lint-ci clean. Signed-off-by: Martin Vogel --- src/pipeline/pass_lsp_cross.c | 149 +++++++++++++++++++++++-- src/pipeline/pass_lsp_cross.h | 17 ++- src/pipeline/pipeline.c | 2 +- src/pipeline/pipeline_incremental.c | 6 +- tests/test_lsp_resolution_probe.c | 162 +++++++++++++++++++++++----- tests/test_parallel.c | 4 +- 6 files changed, 295 insertions(+), 45 deletions(-) diff --git a/src/pipeline/pass_lsp_cross.c b/src/pipeline/pass_lsp_cross.c index 684554455..ab92301e8 100644 --- a/src/pipeline/pass_lsp_cross.c +++ b/src/pipeline/pass_lsp_cross.c @@ -150,6 +150,114 @@ static const char *pxc_join_pipe(CBMArena *arena, const char *const *items) { return buf; } +/* ── Cross-file base-class QN resolution ────────────────────────── + * + * CBMDefinition.base_classes carries the SOURCE SPELLING of each base + * ("Base", "django.db.Model", "React.Component"): extraction strips + * keywords and generic arguments, but it cannot know WHERE the name is + * declared. The Python and TS cross-file registrars, however, consume + * CBMLSPDef.embedded_types as fully-qualified names — py_lookup_attribute + * and ts_lookup_member feed each entry straight into + * cbm_registry_lookup_type. An unqualified spelling therefore matched + * nothing declared in ANOTHER file: `class Child(Base)` in child.py never + * saw Base in base.py, so a call to an inherited method through a typed + * receiver had no member to find and fell through to the weak textual + * cascade (where the #592/#606 guard correctly kills it). + * + * Resolve each base name ONCE per definition here, from exactly the + * inputs pass_semantic uses to draw its INHERITS edge: the project + * registry, the declaring module, and the file's import map. Two + * properties follow. The LSP's inheritance view is the same relation the + * graph records, so the two cannot diverge. And the binding is + * import- or same-module-backed, not a short-name guess, so the CALLS + * edge it enables is a supported fact that the weak-member guard keeps. + * + * Cost: O(defs) hash lookups + ONE import map per file. No per-call-site + * hierarchy walk, no registry scan, no per-file registry rebuild. + */ +static bool pxc_lang_resolves_base_qns(CBMLanguage lang) { + switch (lang) { + case CBM_LANG_PYTHON: + case CBM_LANG_JAVASCRIPT: + case CBM_LANG_TYPESCRIPT: + case CBM_LANG_TSX: + return true; + default: + /* Go / JVM / C# / C++ / Rust registrars qualify their own embedded + * types already (struct embedding, parent_class chains, impl-trait + * provenance). Re-resolving here would fight those paths, so they + * keep the raw join. */ + return false; + } +} + +/* True for the registry strategies that are pure short-name guesses. + * EXPLICIT drop-list, mirroring cbm_tsjs_suppress_weak_method_match: a + * base class bound by "some project type happens to share this name" is + * exactly the fabricated relation #606 removed, and inheritance + * multiplies it — every inherited member of the wrong base would become + * a callable target. Import-, module- and suffix-aware strategies are + * kept; anything unresolved simply retains its source spelling and the + * behaviour that predates this resolution. */ +static bool pxc_base_strategy_is_weak(const char *strategy) { + if (!strategy || !strategy[0]) { + return true; + } + return strcmp(strategy, "suffix_match") == 0 || strcmp(strategy, "unique_name") == 0 || + strcmp(strategy, "field_type_hint") == 0 || strcmp(strategy, "fuzzy") == 0; +} + +/* Resolve one base-class spelling to a project QN. Mirrors + * pass_semantic.c::resolve_as_class — same registry, same type-like veto — + * then additionally rejects weak short-name strategies (see above). + * Returns NULL when the base is not a confidently-known project type; + * stdlib and third-party bases land here and keep their raw spelling. */ +static const char *pxc_resolve_base_qn(const cbm_registry_t *reg, const char *raw, + const char *module_qn, const char **imp_keys, + const char **imp_vals, int imp_count) { + if (!reg || !raw || !raw[0]) { + return NULL; + } + cbm_resolution_t res = cbm_registry_resolve(reg, raw, module_qn, imp_keys, imp_vals, imp_count); + if (!res.qualified_name || !res.qualified_name[0]) { + return NULL; + } + if (pxc_base_strategy_is_weak(res.strategy)) { + return NULL; + } + if (!cbm_label_is_type_like(cbm_registry_label_of(reg, res.qualified_name))) { + return NULL; + } + return res.qualified_name; +} + +/* pxc_join_pipe over base_classes, substituting each resolved QN for its + * source spelling. Unresolved entries pass through verbatim so a base the + * registry does not know keeps working exactly as before. */ +static const char *pxc_join_base_qns(CBMArena *arena, const char *const *bases, + const cbm_registry_t *reg, const char *module_qn, + const char **imp_keys, const char **imp_vals, int imp_count) { + if (!bases || !bases[0]) { + return NULL; + } + int count = 0; + while (bases[count]) { + count++; + } + const char **resolved = + (const char **)cbm_arena_alloc(arena, (size_t)(count + 1) * sizeof(const char *)); + if (!resolved) { + return pxc_join_pipe(arena, bases); + } + for (int i = 0; i < count; i++) { + const char *qn = + pxc_resolve_base_qn(reg, bases[i], module_qn, imp_keys, imp_vals, imp_count); + resolved[i] = qn ? qn : bases[i]; + } + resolved[count] = NULL; + return pxc_join_pipe(arena, resolved); +} + static bool pxc_is_jvm_lang(CBMLanguage lang); static const char *pxc_last_component(const char *qn) { @@ -262,7 +370,9 @@ static const char *pxc_qn_leaf(const char *name) { * to skip (unsupported label or missing required field). dst gets borrowed * pointers into src and into `arena` for synthesised composites. */ static int pxc_build_lsp_def(CBMArena *arena, const CBMDefinition *src, const char *module_qn, - const char *namespace_name, CBMLanguage lang, CBMLSPDef *dst) { + const char *namespace_name, CBMLanguage lang, CBMLSPDef *dst, + const cbm_registry_t *reg, const char **imp_keys, + const char **imp_vals, int imp_count) { const char *label = pxc_map_label(src->label); if (!label || !src->qualified_name || !src->name) return -1; @@ -283,7 +393,13 @@ static int pxc_build_lsp_def(CBMArena *arena, const CBMDefinition *src, const ch * for multi-return languages (Go); single-return languages just see one * piece, which is what's already stored. */ dst->return_types = src->return_type; - dst->embedded_types = pxc_join_pipe(arena, src->base_classes); + /* Languages whose cross registrars read embedded_types as QNs get their + * bases resolved against the project registry; everyone else keeps the + * raw source spelling their own registrar already knows how to handle. */ + dst->embedded_types = (reg && pxc_lang_resolves_base_qns(lang)) + ? pxc_join_base_qns(arena, src->base_classes, reg, module_qn, + imp_keys, imp_vals, imp_count) + : pxc_join_pipe(arena, src->base_classes); dst->signature_param_types = src->signature_param_types; dst->signature_param_count = src->signature_param_count; dst->lang = lang; @@ -323,9 +439,10 @@ static int pxc_build_rust_impl_relation(CBMArena *arena, const CBMImplTrait *imp /* Collect a project-wide CBMLSPDef[] from all cached results. Returns a * malloc'd array (caller frees) of length *out_count. String fields are * borrowed from cache[i]->arena and from def_modules[i] (also borrowed). */ -CBMLSPDef *cbm_pxc_collect_all_defs(CBMFileResult **cache, const cbm_file_info_t *files, - int file_count, const char *project_name, char **def_modules, - int *out_count, int *out_def_starts) { +CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMFileResult **cache, + const cbm_file_info_t *files, int file_count, + const char *project_name, char **def_modules, int *out_count, + int *out_def_starts) { int total = 0; for (int i = 0; i < file_count; i++) { if (cache[i]) { @@ -369,12 +486,30 @@ CBMLSPDef *cbm_pxc_collect_all_defs(CBMFileResult **cache, const cbm_file_info_t cache[fi]->namespace_name = namespace_name; } } + /* One import map per FILE (not per def, and not per base name): the + * cross-file base-class resolution below needs the same local-name → + * import-QN view pass_semantic uses. Built only for the languages + * that consume resolved base QNs, and only when a caller supplied the + * pipeline context (the surface-probe path passes NULL and keeps the + * raw spelling). */ + const cbm_registry_t *base_reg = NULL; + const char **imp_keys = NULL; + const char **imp_vals = NULL; + int imp_count = 0; + if (ctx && ctx->registry && pxc_lang_resolves_base_qns(files[fi].language)) { + base_reg = ctx->registry; + cbm_pxc_build_import_map(ctx->gbuf, project_name, files[fi].rel_path, + files[fi].language, cache[fi], &imp_keys, &imp_vals, + &imp_count); + } for (int di = 0; di < cache[fi]->defs.count; di++) { if (pxc_build_lsp_def(&cache[fi]->arena, &cache[fi]->defs.items[di], def_modules[fi], - namespace_name, files[fi].language, &defs[idx]) == 0) { + namespace_name, files[fi].language, &defs[idx], base_reg, + imp_keys, imp_vals, imp_count) == 0) { idx++; } } + cbm_pxc_free_import_map(imp_keys, imp_vals, imp_count); /* NULL-safe */ if (files[fi].language == CBM_LANG_RUST) { for (int ii = 0; ii < cache[fi]->impl_traits.count; ii++) { if (pxc_build_rust_impl_relation( @@ -1284,7 +1419,7 @@ int cbm_pipeline_pass_lsp_cross(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t * int def_count = 0; int *def_starts = (int *)calloc((size_t)file_count + 1, sizeof(int)); - CBMLSPDef *all_defs = cbm_pxc_collect_all_defs(cache, files, file_count, ctx->project_name, + CBMLSPDef *all_defs = cbm_pxc_collect_all_defs(ctx, cache, files, file_count, ctx->project_name, def_modules, &def_count, def_starts); /* Same seam as the parallel driver: serialize per-file surfaces while the * result cache is alive. Failure only degrades to a full rebuild on the diff --git a/src/pipeline/pass_lsp_cross.h b/src/pipeline/pass_lsp_cross.h index 7630d8c72..705b804cc 100644 --- a/src/pipeline/pass_lsp_cross.h +++ b/src/pipeline/pass_lsp_cross.h @@ -54,10 +54,19 @@ bool cbm_pxc_has_cross_lsp(CBMLanguage lang); * receives per-file prefix offsets: file i's defs occupy * [out_def_starts[i], out_def_starts[i+1]) — the LSP-surface serializer * needs the per-file slices, which the flat array does not otherwise - * record. */ -CBMLSPDef *cbm_pxc_collect_all_defs(CBMFileResult **cache, const cbm_file_info_t *files, - int file_count, const char *project_name, char **def_modules, - int *out_count, int *out_def_starts); + * record. + * + * `ctx` (nullable) enables cross-file base-class resolution: for the + * languages whose cross registrars read embedded_types as qualified names + * (Python, JS/TS/TSX), every CBMDefinition.base_classes spelling is resolved + * to a project QN through ctx->registry plus the file's import map — the same + * inputs pass_semantic uses to draw its INHERITS edge, so the LSP's + * inheritance view and the graph's cannot diverge. Pass NULL to keep the raw + * source spelling (surface-probe paths that build no registry). */ +CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMFileResult **cache, + const cbm_file_info_t *files, int file_count, + const char *project_name, char **def_modules, int *out_count, + int *out_def_starts); /* Detect TS dialect flags from a relative path. */ void cbm_pxc_ts_modes(CBMLanguage lang, const char *rel_path, bool *out_js, bool *out_jsx, diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index cc9190f52..938437549 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -1253,7 +1253,7 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, def_modules = (char **)calloc((size_t)file_count, sizeof(char *)); def_starts = (int *)calloc((size_t)file_count + 1, sizeof(int)); all_defs = def_modules - ? cbm_pxc_collect_all_defs(cache, files, file_count, ctx->project_name, + ? cbm_pxc_collect_all_defs(ctx, cache, files, file_count, ctx->project_name, def_modules, &def_count, def_starts) : NULL; } diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 4ff89aa42..bf71a1f84 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1285,7 +1285,7 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed int fresh_count = 0; CBMLSPDef *fresh_defs = def_modules && def_starts - ? cbm_pxc_collect_all_defs(cache, changed_files, ci, ctx->project_name, + ? cbm_pxc_collect_all_defs(ctx, cache, changed_files, ci, ctx->project_name, def_modules, &fresh_count, def_starts) : NULL; if ((fresh_defs || fresh_count == 0) && def_starts && @@ -1588,8 +1588,8 @@ static int closure_probe_surfaces(cbm_pipeline_t *p, const char *project, int def_count = 0; CBMLSPDef *defs = NULL; if (def_modules && def_starts) { - defs = cbm_pxc_collect_all_defs(cache, probe_files, probe_count, project, def_modules, - &def_count, def_starts); + defs = cbm_pxc_collect_all_defs(NULL, cache, probe_files, probe_count, project, + def_modules, &def_count, def_starts); rc = cbm_lsp_surface_build_rows(project, cache, probe_files, probe_count, defs, def_starts, out_rows, out_count); } else { diff --git a/tests/test_lsp_resolution_probe.c b/tests/test_lsp_resolution_probe.c index 7386710f9..91844f67e 100644 --- a/tests/test_lsp_resolution_probe.c +++ b/tests/test_lsp_resolution_probe.c @@ -222,6 +222,36 @@ static void lrp_diag(cbm_store_t *store, const char *project, const char *scenar fprintf(stderr, " [LRP] %s edges=[%s]\n", scenario, line[0] ? line : "(none)"); } +/* Count CALLS edges whose resolution strategy starts with `prefix`. + * + * The S6 inheritance scenarios need more than "an edge exists": a fixture + * small enough to have exactly one same-named method can be satisfied by a + * weak short-name guess that happens to be right, which is precisely the + * class of edge the receiver-aware guard (#592/#606) removes. Asserting an + * `lsp_*` strategy is what distinguishes inheritance RESOLUTION from a lucky + * coincidence. Returns -1 when the edges cannot be read at all. */ +static int lrp_count_calls_with_strategy(cbm_store_t *store, const char *project, + const char *prefix) { + if (!store) { + return -1; + } + cbm_edge_t *edges = NULL; + int n = 0; + if (cbm_store_find_edges_by_type(store, project, "CALLS", &edges, &n) != CBM_STORE_OK) { + return -1; + } + char needle[64]; + snprintf(needle, sizeof(needle), "\"strategy\":\"%s", prefix); + int hits = 0; + for (int i = 0; i < n; i++) { + if (edges[i].properties_json && strstr(edges[i].properties_json, needle)) { + hits++; + } + } + cbm_store_free_edges(edges, n); + return hits; +} + /* Index + assert CALLS >= floor; on failure emit diagnostics. * expect_green=true: ASSERT_TRUE (green guard). * expect_green=false: still asserts CALLS>=1 (the correct outcome), so RED @@ -952,21 +982,28 @@ TEST(lrp_python_s6_inherited_method) { {"child.py", "from .base import Base\n\n\nclass Child(Base):\n" " def extra(self):\n return 'extra'\n\n\n" "def run(c):\n return c.describe()\n"}}; - /* RED: c.describe() needs py_lsp_cross to walk `class Child(Base)`. Exactly - * the TS/S6 situation above, and resolved the same way (#1276). - * Until the receiver-aware guard landed, this scenario passed via a - * unique_name registry fallback — MEASURED on main before the guard: - * strategy=unique_name, cands=1, conf=0.7500. "describe" is the only symbol - * of that name in this 2-file fixture, so a weak short-name guess happened - * to be right. In a real repo the same guess binds an arbitrary same-named - * method (the false edges #1276 targets: accelerator.print() -> - * MockAccelerator.print), so the guard now suppresses weak member-call - * matches whose receiver is unresolved — here `c`, a bare parameter. - * c.describe() is the ONLY call in the fixture, so a correctly-suppressed - * run yields exactly zero CALLS. + /* MEASURED on main before the guard: strategy=unique_name, cands=1, + * conf=0.7500. "describe" is the only symbol of that name in this 2-file + * fixture, so a weak short-name guess happened to be right. In a real repo + * the same guess binds an arbitrary same-named method (the false edges + * #1276 targets: accelerator.print() -> MockAccelerator.print), so the + * Python weak-member guard suppresses member calls whose receiver is + * unresolved -- here `c`, a bare parameter. c.describe() is the ONLY call + * in the fixture, so a correctly-suppressed run yields exactly zero CALLS. + * + * This is NOT an inheritance case, despite the name, and it has no + * flip-back condition: `c` is un-annotated, so no receiver type exists to + * inherit THROUGH. Resolving cross-file base classes cannot recover this + * edge, because there is nothing to say `c` is a Child. Cross-file + * inheritance resolution is covered by S6b below, which supplies the + * receiver type this fixture deliberately withholds. + * + * The base_classes extraction bug this comment used to cite is FIXED -- + * extraction_inheritance is green for Python, and base_classes holds clean + * names with parens, kwargs and generic subscripts already stripped. + * * Tripwire: assert the store opened AND calls == 0 exactly, so an infra/DB - * failure cannot pass vacuously; flip to ASSERT calls >= 1 once py_lsp_cross - * resolves inheritance (lsp_py_*, a strategy the guard keeps). */ + * failure cannot pass vacuously. */ LRP_Proj lp; cbm_store_t *store = lrp_index(&lp, f, 2); ASSERT_NOT_NULL(store); @@ -976,6 +1013,61 @@ TEST(lrp_python_s6_inherited_method) { PASS(); } +/* S6b — Python inherited method call through a receiver whose type IS known. + * + * S6 above is deliberately kept as-is, but it cannot demonstrate inheritance + * resolution: its `def run(c)` parameter carries no annotation, so nothing in + * the program says `c` is a Child. Whatever binds c.describe() there is a + * guess about a name, not a fact about a type — which is why it resolves via + * unique_name and why the Python weak-member guard (#1324) is expected to take + * it to zero. + * + * This row supplies the receiver type three ways — annotated parameter, `self` + * inside the subclass, and a constructor result — so `Base.describe` is + * reachable ONLY by crossing the file boundary that `class Child(Base)` + * declares. Before cbm_pxc_collect_all_defs resolved base spellings to project + * QNs, py_lsp_cross received "Base" and looked it up as a qualified name, found + * nothing (Base lives in base.py), and all three fell through to unique_name at + * confidence 0.75. Asserting the lsp_* strategy is what separates the two + * outcomes: a short-name guess and a resolved inherited member produce the same + * edge COUNT here, and only the strategy tells them apart. */ +TEST(lrp_python_s6b_inherited_method_typed_receiver) { + static const LRP_File annotated[] = { + {"base.py", "class Base:\n def describe(self):\n return 'base'\n"}, + {"child.py", "from .base import Base\n\n\nclass Child(Base):\n" + " def extra(self):\n return 'extra'\n\n\n" + "def run(c: Child):\n return c.describe()\n"}}; + static const LRP_File self_attr[] = { + {"base.py", "class Base:\n def describe(self):\n return 'base'\n"}, + {"child.py", "from .base import Base\n\n\nclass Child(Base):\n" + " def go(self):\n return self.describe()\n"}}; + static const LRP_File constructed[] = { + {"base.py", "class Base:\n def describe(self):\n return 'base'\n"}, + {"child.py", "from .base import Base\n\n\nclass Child(Base):\n pass\n\n\n" + "def run():\n return Child().describe()\n"}}; + const LRP_File *shapes[3] = {annotated, self_attr, constructed}; + const char *names[3] = {"annotated_param", "self_attr", "constructor"}; + + for (int k = 0; k < 3; k++) { + LRP_Proj lp; + cbm_store_t *store = lrp_index(&lp, shapes[k], 2); + ASSERT_NOT_NULL(store); + int inherits = cbm_store_count_edges_by_type(store, lp.project, "INHERITS"); + int lsp_calls = lrp_count_calls_with_strategy(store, lp.project, "lsp_"); + if (inherits < 1 || lsp_calls < 1) { + fprintf(stderr, " [LRP] python/S6b/%s FAIL inherits=%d lsp_calls=%d\n", names[k], + inherits, lsp_calls); + lrp_diag(store, lp.project, names[k]); + } + lrp_cleanup(&lp, store); + /* INHERITS proves the cross-file base really was resolved; the lsp_ + * strategy proves the CALL rode that relation instead of guessing. */ + ASSERT_TRUE(inherits >= 1); + ASSERT_TRUE(lsp_calls >= 1); + } + PASS(); +} + /* S7 — Python type-annotated call (PEP 484 type hint). */ TEST(lrp_python_s7_annotated_call) { static const LRP_File f[] = { @@ -1116,25 +1208,38 @@ TEST(lrp_ts_s6_inherited_method) { "import { Base } from './base';\n\n" "export class Child extends Base {\n extra(): string { return 'child'; }\n}\n\n" "export function run(c: Child): string { return c.describe(); }\n"}}; - /* RED: c.describe() needs ts_lsp_cross to walk `Child extends Base`. The - * INHERITS edge IS extracted (the probe diagnostic shows INHERITS=1); the - * gap is in ts_lsp_cross's cross-file inheritance RESOLUTION, not extraction. - * Until the receiver-aware guard (#592/#606) landed, this scenario passed via - * a unique_name registry fallback — "describe" is unique in this 2-file - * fixture, so a weak short-name guess happened to be right. In a real repo the - * same guess binds an arbitrary same-named method (the false edges #606 - * targets), so the guard now suppresses weak member-call matches with an - * unresolved receiver. c.describe() is the ONLY call in the fixture, so a - * correctly-suppressed run yields exactly zero CALLS. - * Tripwire: assert store opened AND calls == 0 exactly (an infra/DB failure - * must not vacuously pass); flip to ASSERT calls >= 1 once ts_lsp_cross - * resolves inheritance (lsp_ts_*, a strategy the guard keeps). */ + /* GREEN: c.describe() resolves through `Child extends Base` across files. + * + * This scenario was RED for as long as ts_lsp_cross received the base class + * as its SOURCE SPELLING ("Base") while ts_lookup_member fed embedded_types + * straight into cbm_registry_lookup_type as a QN — so a base declared in + * ANOTHER file never matched, the receiver stayed unresolved, and the only + * call in the fixture died in the weak textual cascade. Its predecessor + * "pass" came from a unique_name registry fallback ("describe" is unique + * here, so a weak short-name guess happened to be right); the receiver-aware + * guard (#592/#606) correctly suppressed that, which is why the tripwire + * asserted exactly zero. + * + * cbm_pxc_collect_all_defs now resolves each base spelling to a project QN + * through the import map + project registry — the same inputs pass_semantic + * uses for its INHERITS edge — so `Child extends Base` is a cross-file fact + * the TS LSP can walk. Asserting the lsp_ts_* strategy is the whole point: + * it proves the edge came from inheritance RESOLUTION and not from another + * lucky short-name guess. Keep BOTH assertions — calls>=1 alone would go + * green again for the wrong reason if the guard ever regressed. */ LRP_Proj lp; cbm_store_t *store = lrp_index(&lp, f, 2); ASSERT_NOT_NULL(store); int calls = cbm_store_count_edges_by_type(store, lp.project, "CALLS"); + int lsp_calls = lrp_count_calls_with_strategy(store, lp.project, "lsp_ts"); + if (calls < 1 || lsp_calls < 1) { + fprintf(stderr, " [LRP] ts/S6/inherited_method FAIL calls=%d lsp_ts_calls=%d\n", calls, + lsp_calls); + lrp_diag(store, lp.project, "ts/S6/inherited_method"); + } lrp_cleanup(&lp, store); - ASSERT_EQ(calls, 0); + ASSERT_TRUE(calls >= 1); + ASSERT_TRUE(lsp_calls >= 1); PASS(); } @@ -1800,6 +1905,7 @@ SUITE(lsp_resolution_probe) { RUN_TEST(lrp_python_s4_class_method); RUN_TEST(lrp_python_s5_chained); RUN_TEST(lrp_python_s6_inherited_method); + RUN_TEST(lrp_python_s6b_inherited_method_typed_receiver); RUN_TEST(lrp_python_s7_annotated_call); RUN_TEST(lrp_python_s8_field_type_hint); RUN_TEST(lrp_python_crossfile_dunder_carrier); diff --git a/tests/test_parallel.c b/tests/test_parallel.c index 2b94130e5..181c1f7ca 100644 --- a/tests/test_parallel.c +++ b/tests/test_parallel.c @@ -324,8 +324,8 @@ static cbm_gbuf_t *run_parallel_with_extract_opts_and_mutator( char **def_modules = (char **)calloc((size_t)file_count, sizeof(char *)); int def_count = 0; CBMLSPDef *all_defs = - def_modules ? cbm_pxc_collect_all_defs(result_cache, files, file_count, ctx.project_name, - def_modules, &def_count, NULL) + def_modules ? cbm_pxc_collect_all_defs(&ctx, result_cache, files, file_count, + ctx.project_name, def_modules, &def_count, NULL) : NULL; CBMModuleDefIndex *module_def_index = all_defs ? cbm_pxc_build_module_def_index(all_defs, def_count) : NULL; From d90f98f5f4f183eedfe2c10990b307790b944780 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 29 Aug 2026 17:44:43 +0200 Subject: [PATCH 31/31] fix(store): keep ADR line endings intact when splicing a section MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every ADR fixture in the tree used \n, so nothing exercised a CRLF-stored document — and CI could not have found this, because the fixtures are LF whichever platform runs them. A red would only have appeared once a real user's CRLF ADR met this code. The exposure is specific to splicing. The old rebuild normalised everything on the way out, so line-ending confusion was invisible; a byte-span replacement can miscount a separator run, and byte-identity is the property the splice design exists to provide. Two of the three paths were already correct, and now say so in tests: - Replacing a section preserves CRLF exactly. The walk-back over a trailing break run steps across both '\r' and '\n', so it cannot cut mid-pair. - Heading matching is ending-agnostic: "## PURPOSE\r\n" locates the same as "## PURPOSE\n", because the scanner trims '\r' from the name. Trimming and matching were different claims and only trimming was pinned. Appending a new section was wrong. It counted trailing '\n' characters, so a document ending "...Foo\r\n" was read as having one break but then had a BARE '\n' appended, and the "## NAME" line it wrote used '\n' too — leaving a mixed document that neither the author nor the tool asked for. A CRLF file ending in a blank line also gained a spurious extra break. Appends now count a "\r\n" pair as one break, and write the separator and heading line using the document's own ending. The contract, now that a document can be mixed: - Bytes that already exist are never rewritten, so each line keeps whatever ending it had. This follows from splicing and is what the acceptance property already guaranteed. - Text this code writes itself — the separator and the "## NAME" line — follows the document's majority ending, LF on a tie. - A caller's body is inserted verbatim. Rewriting bytes a caller supplied would be the same silent modification this whole change set removes. Co-authored-by: andis777 <24672074+andis777@users.noreply.github.com> Signed-off-by: Martin Vogel --- src/store/store.c | 81 ++++++++++++++++++++++++-------- tests/test_store_arch.c | 100 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 20 deletions(-) diff --git a/src/store/store.c b/src/store/store.c index dd97ef2d8..ee58d10f6 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -8900,6 +8900,44 @@ static void adr_find_cb(void *ctx, const cbm_adr_heading_t *h) { f->body_end = h->body_end; } +/* The line ending this code should WRITE into `doc`: whichever of "\r\n" and + * "\n" the document already uses more often, LF on a tie or an empty document. + * Lines that already exist are never rewritten — this decides only the + * separator and the "## NAME" line an append adds, so a CRLF document does not + * acquire a stray bare newline nobody asked for. */ +static const char *adr_dominant_eol(const char *doc, size_t doc_len) { + size_t crlf = 0; + size_t lf = 0; + for (size_t i = 0; i < doc_len; i++) { + if (doc[i] != '\n') { + continue; + } + if (i > 0 && doc[i - SKIP_ONE] == '\r') { + crlf++; + } else { + lf++; + } + } + return crlf > lf ? "\r\n" : "\n"; +} + +/* Trailing line breaks, counting a "\r\n" PAIR as ONE break. Counting raw '\n' + * characters gets the count right but loses the shape, and the separator that + * is then appended has to match the ending it is continuing. */ +static size_t adr_trailing_breaks(const char *doc, size_t doc_len) { + size_t idx = doc_len; + size_t breaks = 0; + while (idx > 0 && (doc[idx - SKIP_ONE] == '\n' || doc[idx - SKIP_ONE] == '\r')) { + bool was_lf = doc[idx - SKIP_ONE] == '\n'; + idx--; + if (was_lf && idx > 0 && doc[idx - SKIP_ONE] == '\r') { + idx--; + } + breaks++; + } + return breaks; +} + char *cbm_adr_splice_section(const char *content, const char *name, const char *body) { if (!name || !body) { return NULL; @@ -8951,37 +8989,40 @@ char *cbm_adr_splice_section(const char *content, const char *name, const char * return out; } - /* Absent: append. Enough newlines are added to leave exactly one blank + /* Absent: append. Enough line breaks are added to leave exactly one blank * line before the new heading — never fewer, and never rewriting the - * newlines the document already ends with. */ - size_t trailing = 0; - while (trailing < doc_len && doc[doc_len - SKIP_ONE - trailing] == '\n') { - trailing++; - } + * breaks the document already ends with. Both the separator and the + * heading line use the document's OWN ending: counting raw '\n' here would + * drop a bare newline into a CRLF document, leaving it mixed. */ + const char *eol = adr_dominant_eol(doc, doc_len); + size_t eol_len = strlen(eol); + size_t breaks = adr_trailing_breaks(doc, doc_len); size_t sep = 0; - if (doc_len > 0 && trailing < PAIR_LEN) { - sep = PAIR_LEN - trailing; + if (doc_len > 0 && breaks < PAIR_LEN) { + sep = PAIR_LEN - breaks; } size_t name_len = strlen(name); - size_t total = doc_len + sep + ST_HEADER_PREFIX + name_len + SKIP_ONE + body_len; + size_t total = doc_len + (sep * eol_len) + ST_HEADER_PREFIX + name_len + eol_len + body_len; out = malloc(total + SKIP_ONE); if (!out) { return NULL; } - size_t at = 0; + size_t cursor = 0; memcpy(out, doc, doc_len); - at += doc_len; + cursor += doc_len; for (size_t i = 0; i < sep; i++) { - out[at++] = '\n'; + memcpy(out + cursor, eol, eol_len); + cursor += eol_len; } - memcpy(out + at, "## ", ST_HEADER_PREFIX); - at += ST_HEADER_PREFIX; - memcpy(out + at, name, name_len); - at += name_len; - out[at++] = '\n'; - memcpy(out + at, body, body_len); - at += body_len; - out[at] = '\0'; + memcpy(out + cursor, "## ", ST_HEADER_PREFIX); + cursor += ST_HEADER_PREFIX; + memcpy(out + cursor, name, name_len); + cursor += name_len; + memcpy(out + cursor, eol, eol_len); + cursor += eol_len; + memcpy(out + cursor, body, body_len); + cursor += body_len; + out[cursor] = '\0'; return out; } diff --git a/tests/test_store_arch.c b/tests/test_store_arch.c index 3a93c4722..4e9f59068 100644 --- a/tests/test_store_arch.c +++ b/tests/test_store_arch.c @@ -1011,6 +1011,103 @@ TEST(adr_splice_preserves_untouched_bytes) { PASS(); } +/* Line endings. Every other ADR fixture in this file uses \n, so nothing + * exercised a CRLF-stored document — and CI could not have caught that, + * because the fixtures are LF whichever platform runs them. + * + * The exposure is specific to splicing: the old rebuild normalised everything + * on the way out, so ending confusion was invisible. A byte-span replacement + * can cut mid-"\r\n" or miscount a separator run, and byte-identity is the + * property this design exists to provide. + * + * Contract: bytes that already exist are never rewritten, so each line keeps + * whatever ending it had. Text this code writes itself — the separator before + * an appended section and the "## NAME" line — follows the document's majority + * ending, LF on a tie. A caller's body is inserted verbatim; rewriting the + * bytes a caller supplied would be the same silent modification this whole + * change removes. */ +TEST(adr_splice_preserves_line_endings) { + /* CRLF throughout: untouched bytes keep their \r\n. */ + CHECK_SPLICE("## PURPOSE\r\nFoo\r\n\r\n## STACK\r\nBar", "STACK", "New bar", + "## PURPOSE\r\nFoo\r\n\r\n## STACK\r\nNew bar"); + + /* The separator run before the next heading survives as CRLF, not as the + * two bare newlines a naive walk-back would leave. */ + CHECK_SPLICE("## A\r\nx\r\n\r\n## B\r\ny", "A", "z", "## A\r\nz\r\n\r\n## B\r\ny"); + + /* A trailing CRLF on the document is preserved as CRLF. */ + CHECK_SPLICE("## A\r\nx\r\n", "A", "z", "## A\r\nz\r\n"); + + /* Appending to a CRLF document writes CRLF — a bare \n here would leave a + * mixed document that neither the author nor the tool asked for. */ + CHECK_SPLICE("## PURPOSE\r\nFoo", "DECISIONS", "- New.", + "## PURPOSE\r\nFoo\r\n\r\n## DECISIONS\r\n- New."); + + /* One existing trailing break means one more is added, counting \r\n as a + * single break rather than as two characters. */ + CHECK_SPLICE("## PURPOSE\r\nFoo\r\n", "DECISIONS", "- New.", + "## PURPOSE\r\nFoo\r\n\r\n## DECISIONS\r\n- New."); + + /* A document already ending in a CRLF blank line gains no extra break. */ + CHECK_SPLICE("## PURPOSE\r\nFoo\r\n\r\n", "DECISIONS", "- New.", + "## PURPOSE\r\nFoo\r\n\r\n## DECISIONS\r\n- New."); + + /* LF documents are unaffected by any of the above. */ + CHECK_SPLICE("## PURPOSE\nFoo\n", "DECISIONS", "- New.", + "## PURPOSE\nFoo\n\n## DECISIONS\n- New."); + PASS(); +} + +/* Mixed endings are what real files become through editors and merges. Lines + * that already exist keep their own endings; the majority decides only what + * this code writes itself. */ +TEST(adr_splice_mixed_line_endings) { + /* Majority CRLF (2 of 3) -> the appended block is CRLF, and the one bare + * LF line in the middle is left exactly as it was. */ + CHECK_SPLICE("## PURPOSE\r\nFoo\nBar\r\n", "DECISIONS", "- New.", + "## PURPOSE\r\nFoo\nBar\r\n\r\n## DECISIONS\r\n- New."); + + /* Majority LF (2 of 3) -> the appended block is LF, and the lone CRLF line + * survives untouched. */ + CHECK_SPLICE("## PURPOSE\nFoo\r\nBar\n", "DECISIONS", "- New.", + "## PURPOSE\nFoo\r\nBar\n\n## DECISIONS\n- New."); + + /* A tie falls to LF. */ + CHECK_SPLICE("## PURPOSE\r\nFoo\n", "DECISIONS", "- New.", + "## PURPOSE\r\nFoo\n\n## DECISIONS\n- New."); + + /* Replacing a section in a mixed document rewrites nothing around it. */ + CHECK_SPLICE("## PURPOSE\r\nFoo\n\n## STACK\nBar\r\n", "STACK", "New", + "## PURPOSE\r\nFoo\n\n## STACK\nNew\r\n"); + PASS(); +} + +/* "## PURPOSE\r\n" must locate exactly as "## PURPOSE\n" does. The scanner + * trims \r from heading names, but trimming and matching are different claims + * and only one of them was pinned. */ +TEST(adr_splice_matches_headings_across_line_endings) { + adr_name_collect_t c; + memset(&c, 0, sizeof(c)); + ASSERT_EQ(cbm_adr_scan_headings("## PURPOSE\r\nFoo\r\n\r\n## STACK\r\nBar", + adr_collect_names, &c), + CBM_STORE_OK); + ASSERT_STR_EQ(c.buf, "[PURPOSE][STACK]"); + + /* A fenced block with CRLF still hides its heading, and still closes. */ + memset(&c, 0, sizeof(c)); + ASSERT_EQ(cbm_adr_scan_headings("## PURPOSE\r\nFoo\r\n\r\n```md\r\n## Example\r\n```" + "\r\n\r\n## STACK\r\nBar", + adr_collect_names, &c), + CBM_STORE_OK); + ASSERT_STR_EQ(c.buf, "[PURPOSE][STACK]"); + + /* And an unterminated CRLF fence is still refused. */ + char errbuf[256]; + ASSERT_TRUE(cbm_adr_check_structure("## PURPOSE\r\nFoo\r\n\r\n```\r\nopen\r\n", errbuf, + sizeof(errbuf)) != CBM_STORE_OK); + PASS(); +} + /* The original use case: add an entry under its own heading. */ TEST(adr_splice_appends_arbitrary_heading) { CHECK_SPLICE("## PURPOSE\nFoo", "DECISIONS", "- Chose SQLite.", @@ -1620,6 +1717,9 @@ SUITE(store_arch) { RUN_TEST(adr_validate_keys_accepts_arbitrary_names); RUN_TEST(adr_validate_keys_rejects_unroundtrippable); RUN_TEST(adr_splice_preserves_untouched_bytes); + RUN_TEST(adr_splice_preserves_line_endings); + RUN_TEST(adr_splice_mixed_line_endings); + RUN_TEST(adr_splice_matches_headings_across_line_endings); RUN_TEST(adr_splice_appends_arbitrary_heading); RUN_TEST(adr_splice_is_idempotent); RUN_TEST(adr_splice_matches_case_exactly);