From 2bff501a62a3bba1aeb39f49f7c785d541ca209e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Cochard-Labb=C3=A9?= Date: Fri, 21 Aug 2026 08:14:09 +0200 Subject: [PATCH 01/22] fix(freebsd): treat ${LOCALBASE}/bin as pkg-managed; resolve self via sysctl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two FreeBSD gaps remain after the runtime rework of install/update (#1566) and the KERN_PROC_PATHNAME self-path fix landed on main. 1. cli_external_manager_name() matches only positive evidence (mise, Homebrew, nix, asdf, cargo). ${LOCALBASE}/bin was not among them, so a ports/pkg install of cbm was still copied into ~/.local/bin and a shell rc was edited -- exactly what externally-managed detection is meant to prevent. Recognise ${LOCALBASE}/bin and /sbin on FreeBSD so install skips the copy and update defers to `pkg upgrade`. LOCALBASE is configurable, so the FreeBSD port passes its real PREFIX via -DCBM_PKG_PREFIX and we honour it, falling back to the documented /usr/local default for a plain upstream build. --force-binary remains the escape hatch for anyone self-managing that prefix. 2. resolve_self_executable() in the HTTP server still read /proc/self/exe, which FreeBSD does not mount by default. Add a KERN_PROC_PATHNAME sysctl branch, mirroring cbm_detect_self_path(). Test updated: on FreeBSD ${LOCALBASE}/bin/cbm is now externally managed; a non-LOCALBASE prefix (/opt/local) is not. Signed-off-by: Olivier Cochard-Labbé --- src/cli/cli.c | 23 +++++++++++++++++++++++ src/ui/http_server.c | 13 +++++++++++++ tests/test_cli.c | 10 +++++++++- 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/src/cli/cli.c b/src/cli/cli.c index d981af266..e90d8b885 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -9699,6 +9699,27 @@ static const char *cli_external_manager_name(const char *self_path) { if (strstr(self_path, "/.cargo/bin/")) { return "cargo"; } +#ifdef __FreeBSD__ + /* FreeBSD ports/pkg install under ${LOCALBASE} (default /usr/local). pkg owns + * that file, so install must not drop a second copy in ~/.local/bin or edit a + * shell rc, and update must defer to pkg(8). Anchor at the start so a manual + * `install --dir=/opt/...` elsewhere is still treated as ours; --force-binary + * is the escape hatch for anyone who really does self-manage that prefix. + * + * LOCALBASE is configurable, so the port passes its real PREFIX via + * -DCBM_PKG_PREFIX; when it is absent we fall back to the documented default. */ +#ifdef CBM_PKG_PREFIX + if (strncmp(self_path, CBM_PKG_PREFIX "/bin/", sizeof(CBM_PKG_PREFIX "/bin/") - 1) == 0 || + strncmp(self_path, CBM_PKG_PREFIX "/sbin/", sizeof(CBM_PKG_PREFIX "/sbin/") - 1) == 0) { + return "FreeBSD pkg"; + } +#else + if (strncmp(self_path, "/usr/local/bin/", 15) == 0 || + strncmp(self_path, "/usr/local/sbin/", 16) == 0) { + return "FreeBSD pkg"; + } +#endif +#endif return NULL; } @@ -12227,6 +12248,8 @@ int cbm_cmd_update(int argc, char **argv) { (void)fprintf(stderr, " update it with: mise upgrade codebase-memory-mcp\n"); } else if (manager && strcmp(manager, "Homebrew") == 0) { (void)fprintf(stderr, " update it with: brew upgrade codebase-memory-mcp\n"); + } else if (manager && strcmp(manager, "FreeBSD pkg") == 0) { + (void)fprintf(stderr, " update it with: pkg upgrade codebase-memory-mcp\n"); } else { (void)fprintf(stderr, " update it through whichever tool installed it.\n"); } diff --git a/src/ui/http_server.c b/src/ui/http_server.c index 18589d8d8..409267358 100644 --- a/src/ui/http_server.c +++ b/src/ui/http_server.c @@ -63,6 +63,10 @@ #ifdef __APPLE__ #include #endif +#ifdef __FreeBSD__ +#include +#include +#endif /* ── Constants ────────────────────────────────────────────────── */ @@ -1003,6 +1007,15 @@ static bool resolve_self_executable(char *out, size_t outsz) { return copy_path(out, outsz, buf); } return false; +#elif defined(__FreeBSD__) + /* No /proc by default on FreeBSD; ask the kernel for our own path. */ + char buf[1024]; + int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1}; + size_t cb = sizeof(buf); + if (sysctl(mib, 4, buf, &cb, NULL, 0) == 0 && cb > 0) { + return copy_path(out, outsz, buf); + } + return false; #else char buf[1024]; ssize_t len = readlink("/proc/self/exe", buf, sizeof(buf) - 1); diff --git a/tests/test_cli.c b/tests/test_cli.c index 95fff9d76..f9f2ed99d 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -12752,8 +12752,16 @@ TEST(cli_external_manager_detection_needs_positive_evidence_issue1566) { ASSERT_NULL( cbm_cli_external_manager_name_for_testing("/Users/x/.local/bin/codebase-memory-mcp")); ASSERT_NULL(cbm_cli_external_manager_name_for_testing("/opt/cbm/codebase-memory-mcp")); - ASSERT_NULL(cbm_cli_external_manager_name_for_testing("/usr/local/bin/codebase-memory-mcp")); ASSERT_NULL(cbm_cli_external_manager_name_for_testing("build/c/test-runner")); +#ifdef __FreeBSD__ + /* On FreeBSD the port/pkg owns ${LOCALBASE}/bin, so a binary there IS + * externally managed; elsewhere the same path is unremarkable. */ + ASSERT_NOT_NULL( + cbm_cli_external_manager_name_for_testing("/usr/local/bin/codebase-memory-mcp")); + ASSERT_NULL(cbm_cli_external_manager_name_for_testing("/opt/local/bin/codebase-memory-mcp")); +#else + ASSERT_NULL(cbm_cli_external_manager_name_for_testing("/usr/local/bin/codebase-memory-mcp")); +#endif ASSERT_NULL(cbm_cli_external_manager_name_for_testing("")); ASSERT_NULL(cbm_cli_external_manager_name_for_testing(NULL)); PASS(); From c537bf0f8b3b6a77e39df49fec78560b5e8e87b7 Mon Sep 17 00:00:00 2001 From: Rares Popa <2606875+rarepops@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:54:12 +0200 Subject: [PATCH 02/22] fix(foundation): resume spawn backoff after EINTR Signed-off-by: Rares Popa <2606875+rarepops@users.noreply.github.com> --- src/foundation/compat.c | 14 +++++++++++ src/foundation/compat.h | 3 +++ src/foundation/subprocess.c | 2 +- tests/test_subprocess.c | 48 +++++++++++++++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 1 deletion(-) diff --git a/src/foundation/compat.c b/src/foundation/compat.c index f0c4d3571..74c25bd8e 100644 --- a/src/foundation/compat.c +++ b/src/foundation/compat.c @@ -17,6 +17,20 @@ #include #endif +int cbm_nanosleep_full(const struct timespec *req) { +#ifdef _WIN32 + return cbm_nanosleep(req, NULL); +#else + struct timespec remaining = *req; + while (nanosleep(&remaining, &remaining) != 0) { + if (errno != EINTR) { + return -1; + } + } + return 0; +#endif +} + /* ── strndup (Windows lacks it) ───────────────────────────────── */ #ifdef _WIN32 diff --git a/src/foundation/compat.h b/src/foundation/compat.h index 055a45687..175d775e4 100644 --- a/src/foundation/compat.h +++ b/src/foundation/compat.h @@ -96,6 +96,9 @@ static inline int cbm_nanosleep(const struct timespec *req, struct timespec *rem #define cbm_nanosleep nanosleep #endif +/* Sleeps for the full requested duration even when POSIX signals interrupt it. */ +int cbm_nanosleep_full(const struct timespec *req); + /* ── gmtime_r (Windows lacks it) ─────────────────────────────── */ #ifdef _WIN32 static inline struct tm *cbm_gmtime_r(const time_t *timep, struct tm *result) { diff --git a/src/foundation/subprocess.c b/src/foundation/subprocess.c index 9174391f4..196165cea 100644 --- a/src/foundation/subprocess.c +++ b/src/foundation/subprocess.c @@ -970,7 +970,7 @@ static void cbm_spawn_backoff(int attempt) { int shift = attempt < CBM_SPAWN_BACKOFF_MAX_SHIFT ? attempt : CBM_SPAWN_BACKOFF_MAX_SHIFT; long ms = (long)CBM_SPAWN_BACKOFF_BASE_MS << shift; struct timespec delay = {ms / 1000L, (ms % 1000L) * 1000L * 1000L}; - (void)cbm_nanosleep(&delay, NULL); + (void)cbm_nanosleep_full(&delay); } /* fork() fails with EAGAIN under the same pressure posix_spawn does, and the diff --git a/tests/test_subprocess.c b/tests/test_subprocess.c index 30fd1537d..c0fb2224c 100644 --- a/tests/test_subprocess.c +++ b/tests/test_subprocess.c @@ -21,6 +21,7 @@ #ifndef _WIN32 #include #include +#include #include #endif @@ -192,6 +193,52 @@ TEST(subprocess_retries_transient_spawn_refusal) { #endif } +#ifndef _WIN32 +static volatile sig_atomic_t g_spawn_backoff_alarm_count = 0; + +static void spawn_backoff_alarm_handler(int signal_number) { + (void)signal_number; + g_spawn_backoff_alarm_count++; +} +#endif + +TEST(subprocess_spawn_backoff_resumes_after_eintr) { +#ifdef _WIN32 + SKIP_PLATFORM("POSIX signal interruption"); +#else + struct sigaction action = {0}; + struct sigaction previous_action = {0}; + action.sa_handler = spawn_backoff_alarm_handler; + (void)sigemptyset(&action.sa_mask); + bool handler_installed = sigaction(SIGALRM, &action, &previous_action) == 0; + + struct itimerval timer = { + .it_interval = {.tv_sec = 0, .tv_usec = 1000}, + .it_value = {.tv_sec = 0, .tv_usec = 1000}, + }; + g_spawn_backoff_alarm_count = 0; + bool timer_started = handler_installed && setitimer(ITIMER_REAL, &timer, NULL) == 0; + + uint64_t started_at = cbm_now_ms(); + cbm_subprocess_force_spawn_eagain_for_testing(3); + cbm_proc_result_t result = run_sh("exit 0", 0); + uint64_t elapsed_ms = cbm_now_ms() - started_at; + + struct itimerval disabled = {0}; + (void)setitimer(ITIMER_REAL, &disabled, NULL); + if (handler_installed) { + (void)sigaction(SIGALRM, &previous_action, NULL); + } + + ASSERT_TRUE(handler_installed); + ASSERT_TRUE(timer_started); + ASSERT_TRUE(g_spawn_backoff_alarm_count > 0); + ASSERT_TRUE(elapsed_ms >= 50); + ASSERT_EQ(result.outcome, CBM_PROC_CLEAN); + PASS(); +#endif +} + TEST(subprocess_gives_up_after_the_retry_budget) { #ifdef _WIN32 SKIP_PLATFORM("POSIX fork/EAGAIN path"); @@ -1033,6 +1080,7 @@ SUITE(subprocess) { RUN_TEST(subprocess_run_crash_is_crash); RUN_TEST(subprocess_run_hang_is_hang); RUN_TEST(subprocess_retries_transient_spawn_refusal); + RUN_TEST(subprocess_spawn_backoff_resumes_after_eintr); RUN_TEST(subprocess_gives_up_after_the_retry_budget); RUN_TEST(subprocess_run_spawn_failure); RUN_TEST(subprocess_run_null_bin_rejected); From 1e0aac527b016d519eb7f8fc7dd610dbeb66e988 Mon Sep 17 00:00:00 2001 From: kilianpaquier Date: Sat, 22 Aug 2026 19:13:39 +0200 Subject: [PATCH 03/22] fix(help): add --clients to CLI help Signed-off-by: kilianpaquier --- src/main.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main.c b/src/main.c index c5c936e49..02e3db3a9 100644 --- a/src/main.c +++ b/src/main.c @@ -902,6 +902,8 @@ static void print_help(void) { printf(" Run one tool locally, then exit\n"); printf(" codebase-memory-mcp install [-y|-n] [--force] [--dry-run] " "[--dir=] [--skip-config]\n"); + printf(" [--clients=] Run " + "'install --clients' to list tokens\n"); printf(" codebase-memory-mcp uninstall [-y|-n] [--dry-run]\n"); printf(" codebase-memory-mcp update [-y|-n]\n"); printf(" codebase-memory-mcp config \n"); From 1e25d962d69999438229b0562d44ea60e97f5456 Mon Sep 17 00:00:00 2001 From: Rares Popa <2606875+rarepops@users.noreply.github.com> Date: Sat, 22 Aug 2026 20:42:04 +0200 Subject: [PATCH 04/22] fix(cli): apply client selector to registry integrations (#1798) Signed-off-by: Rares Popa <2606875+rarepops@users.noreply.github.com> --- src/cli/cli.c | 67 ++++++++++++++++++++++++++++++++++++++---------- src/cli/cli.h | 1 + tests/test_cli.c | 37 +++++++++++++++++++++++++- 3 files changed, 91 insertions(+), 14 deletions(-) diff --git a/src/cli/cli.c b/src/cli/cli.c index d981af266..3d2405610 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -7376,6 +7376,12 @@ static const char *detect_arch(void) { /* ── Agent config install/refresh (shared by install + update) ── */ +/* Set by `install --clients=...` after validation and consumed by both the + * legacy detector and the registry-backed client path (#1558, #1798). */ +static const char *g_client_selection = NULL; +static bool cli_clients_apply_selection(const char *spec, cbm_detected_agents_t *detected); +static bool cli_clients_selects_registry_client(const char *spec, cbm_agent_client_id_t client_id); +static void cli_clients_print_list(FILE *out); static void print_detected_registry_agents(const char *home, bool *any); /* Print detected agent names on a single line. */ @@ -8169,7 +8175,10 @@ static void print_detected_registry_agents(const char *home, bool *any) { cbm_init_agent_registry_context(home, ®istry); for (size_t index = 0U; index < cbm_agent_client_count(); index++) { const cbm_agent_client_profile_t *profile = cbm_agent_client_at(index); - if (profile && cbm_agent_client_detect(profile->id, ®istry.options)) { + if (profile && + (!g_client_selection || + cli_clients_selects_registry_client(g_client_selection, profile->id)) && + cbm_agent_client_detect(profile->id, ®istry.options)) { printf(" %s", profile->display_name); *any = true; } @@ -8525,7 +8534,10 @@ static void install_agent_client_registry(const char *home, const char *binary_p cbm_init_agent_registry_context(home, ®istry); for (size_t index = 0U; index < cbm_agent_client_count(); index++) { const cbm_agent_client_profile_t *profile = cbm_agent_client_at(index); - if (!profile || !cbm_agent_client_detect(profile->id, ®istry.options)) { + if (!profile || + (g_client_selection && + !cli_clients_selects_registry_client(g_client_selection, profile->id)) || + !cbm_agent_client_detect(profile->id, ®istry.options)) { continue; } if (!g_install_plan) { @@ -9365,13 +9377,6 @@ static void install_additional_agent_configs(const cbm_detected_agents_t *agents } } -/* #1558: set by `install --clients=...` after validation, consumed at the one - * place detection happens. Validation runs in cbm_cmd_install so an unknown - * token fails before anything is written, not midway through configuring. */ -static const char *g_client_selection = NULL; -static bool cli_clients_apply_selection(const char *spec, cbm_detected_agents_t *detected); -static void cli_clients_print_list(FILE *out); - int cbm_install_agent_configs(const char *home, const char *binary_path, bool force, bool dry_run) { g_agent_install_errors = 0; cbm_detected_agents_t agents = cbm_detect_agents(home); @@ -9499,9 +9504,9 @@ int cbm_install_handle_existing_indexes(const char *home, bool reset, bool dry_r * Codex had to revert the OpenCode and Cursor integrations by hand, and the * next install silently recreated them. `--clients=claude,codex` restricts it. * - * The table is the flag's vocabulary AND its documentation: 26 clients ship - * here, with tokens nobody would guess (factory-droid, mistral-vibe, - * copilot-cli), so `--clients=help` prints every token. A selector whose + * The tables are the flag's vocabulary AND its documentation, with tokens + * nobody would guess (factory-droid, mistral-vibe, copilot-cli), so + * `--clients=help` prints every token. A selector whose * accepted values can only be learned from the source is not a usable * selector, and an unrecognised token fails loudly with the list rather than * silently configuring nothing. */ @@ -9549,10 +9554,40 @@ static void cli_clients_print_list(FILE *out) { for (size_t i = 0; i < CLI_CLIENT_COUNT; i++) { (void)fprintf(out, " %-16s %s\n", CLI_CLIENTS[i].token, CLI_CLIENTS[i].display); } + for (size_t i = 0; i < cbm_agent_client_count(); i++) { + const cbm_agent_client_profile_t *profile = cbm_agent_client_at(i); + if (profile) { + (void)fprintf(out, " %-16s %s\n", profile->stable_id, profile->display_name); + } + } (void)fprintf(out, "\nExample: --clients=claude,codex\n" "Omit --clients to configure every detected client.\n"); } +static bool cli_clients_spec_contains(const char *spec, const char *wanted_token) { + char buf[CLI_BUF_1K]; + snprintf(buf, sizeof(buf), "%s", spec ? spec : ""); + char *save = NULL; + for (char *tok = strtok_r(buf, ",", &save); tok; tok = strtok_r(NULL, ",", &save)) { + while (*tok == ' ') { + tok++; + } + size_t len = strlen(tok); + while (len > 0 && tok[len - 1] == ' ') { + tok[--len] = '\0'; + } + if (strcmp(tok, wanted_token) == 0) { + return true; + } + } + return false; +} + +static bool cli_clients_selects_registry_client(const char *spec, cbm_agent_client_id_t client_id) { + const cbm_agent_client_profile_t *profile = cbm_agent_client_by_id(client_id); + return profile && cli_clients_spec_contains(spec, profile->stable_id); +} + /* Restrict `detected` to the comma-separated token list. Returns false (after * printing the vocabulary) when a token is unknown, so a typo can never be * mistaken for "that client was not installed". */ @@ -9581,6 +9616,9 @@ static bool cli_clients_apply_selection(const char *spec, cbm_detected_agents_t break; } } + if (!matched) { + matched = cbm_agent_client_by_stable_id(tok) != NULL; + } if (!matched) { (void)fprintf(stderr, "error: unknown client: %s\n\n", tok); cli_clients_print_list(stderr); @@ -9606,6 +9644,9 @@ size_t cbm_cli_clients_count_for_testing(void) { const char *cbm_cli_clients_token_for_testing(size_t index) { return index < CLI_CLIENT_COUNT ? CLI_CLIENTS[index].token : NULL; } +void cbm_cli_set_client_selection_for_testing(const char *spec) { + g_client_selection = spec; +} #endif /* ── Subcommand: install ──────────────────────────────────────── */ @@ -10018,7 +10059,7 @@ int cbm_cmd_install(int argc, char **argv) { } } else if (strcmp(argv[i], "--clients") == 0) { /* Bare `--clients` (and `--clients=help`/`list`) print the - * vocabulary. 26 clients ship with tokens nobody would guess. */ + * vocabulary, including tokens users would not readily guess. */ cli_clients_print_list(stdout); return 0; } else if (strcmp(argv[i], "--skip-binary") == 0) { diff --git a/src/cli/cli.h b/src/cli/cli.h index c2121a940..fe115c5ac 100644 --- a/src/cli/cli.h +++ b/src/cli/cli.h @@ -208,6 +208,7 @@ int cbm_install_agent_configs(const char *home, const char *binary_path, bool fo bool cbm_cli_clients_apply_selection_for_testing(const char *spec, cbm_detected_agents_t *detected); size_t cbm_cli_clients_count_for_testing(void); const char *cbm_cli_clients_token_for_testing(size_t index); +void cbm_cli_set_client_selection_for_testing(const char *spec); #endif #ifdef CBM_CLI_ENABLE_TEST_API diff --git a/tests/test_cli.c b/tests/test_cli.c index 6295479f2..abcbf8b66 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -12794,7 +12794,7 @@ TEST(cli_external_manager_detection_needs_positive_evidence_issue1566) { * Claude and Codex had to revert OpenCode and Cursor by hand — and the next * install silently recreated them. The selector restricts it. * - * The vocabulary is the part that makes it usable: 26 clients ship, with tokens + * The vocabulary is the part that makes it usable: clients ship with tokens * nobody would guess (factory-droid, mistral-vibe, copilot-cli). A selector * whose accepted values can only be learned by reading our source is not a * usable selector, so this pins that every client is listed and that an unknown @@ -12823,6 +12823,12 @@ TEST(cli_clients_selector_vocabulary_is_complete_and_strict_issue1558) { cbm_detected_agents_t typo = all; ASSERT_FALSE(cbm_cli_clients_apply_selection_for_testing("claude,codx", &typo)); + /* Registry-backed clients share the public selector vocabulary. */ + cbm_detected_agents_t registry = all; + ASSERT_TRUE(cbm_cli_clients_apply_selection_for_testing("qoder", ®istry)); + ASSERT_FALSE(registry.claude_code); + ASSERT_FALSE(registry.cursor); + /* Every token in the table must resolve — a client added to detection but * forgotten here is invisible to the selector. */ for (size_t i = 0; i < cbm_cli_clients_count_for_testing(); i++) { @@ -12834,6 +12840,34 @@ TEST(cli_clients_selector_vocabulary_is_complete_and_strict_issue1558) { PASS(); } +TEST(cli_clients_selector_filters_registry_installs_issue1798) { + char *tmpdir = th_mktempdir("cbm_cli_clients_registry"); + ASSERT_NOT_NULL(tmpdir); + + char qoder_dir[512]; + char settings_path[512]; + ASSERT(snprintf(qoder_dir, sizeof(qoder_dir), "%s/.qoder", tmpdir) > 0); + ASSERT(snprintf(settings_path, sizeof(settings_path), "%s/settings.json", qoder_dir) > 0); + ASSERT_EQ(test_mkdirp(qoder_dir), 0); + + cbm_cli_set_client_selection_for_testing("cursor"); + int excluded_rc = cbm_install_agent_configs(tmpdir, "/opt/codebase-memory-mcp", false, false); + struct stat settings_state; + bool excluded = stat(settings_path, &settings_state) != 0; + + cbm_cli_set_client_selection_for_testing("qoder"); + int included_rc = cbm_install_agent_configs(tmpdir, "/opt/codebase-memory-mcp", false, false); + bool included = stat(settings_path, &settings_state) == 0; + cbm_cli_set_client_selection_for_testing(NULL); + + test_rmdir_r(tmpdir); + ASSERT_EQ(excluded_rc, 0); + ASSERT_TRUE(excluded); + ASSERT_EQ(included_rc, 0); + ASSERT_TRUE(included); + PASS(); +} + TEST(cli_update_accepts_retired_variant_flags_issue1544) { char *ui_argv[] = {"--dry-run", "--ui", "--yes"}; char *standard_argv[] = {"--dry-run", "--standard", "--yes"}; @@ -12981,6 +13015,7 @@ SUITE(cli) { RUN_TEST(cli_skill_frontmatter_scalars_with_colons_are_quoted_issue1554); RUN_TEST(cli_external_manager_detection_needs_positive_evidence_issue1566); RUN_TEST(cli_clients_selector_vocabulary_is_complete_and_strict_issue1558); + RUN_TEST(cli_clients_selector_filters_registry_installs_issue1798); RUN_TEST(cli_update_accepts_retired_variant_flags_issue1544); RUN_TEST(cli_update_agent_configs_finish_before_guard_release); RUN_TEST(cli_uninstall_quiesces_active_cohort_before_removing_binary_and_index); From 95689b5c20101db7d92b69767bbea75e5bf0d4fc Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 29 Aug 2026 17:59:58 +0200 Subject: [PATCH 05/22] fix(python): suppress bare calls shadowed by an enclosing parameter A Python `foo()` whose callee identifier is bound as a parameter of an enclosing scope cannot be the module-level `foo` -- the parameter shadows it for the whole body -- so resolving the call to a project Function/Method by a weak short-name strategy fabricates the edge by construction: def _run_with_heavy_slot(run): return run() # bound an unrelated SatoriLive.run The receiver-aware weak-member guard (#1276) cannot see this class at all: a bare call has no receiver, so is_method is false and the guard never fires. This is the bare-call counterpart of python_receiver_is_exempt. Keyed on the SCOPE FACT, not on the callee's spelling. A list of generic-looking names (get / run / execute) asserts that certain spellings are usually noise, which is a claim about corpus fashion rather than about what the resolver knew, and it ages invisibly: nothing fails when the distribution shifts, the graph just quietly loses different edges. A parameter binding is decidable from this file's AST outright. Parameters only, deliberately. A parameter is in scope for the entire body regardless of position and Python forbids `global` on one, so no flow analysis is needed. Local assignments are flow- and binding-form-sensitive (`for`, `with as`, `except as`, `:=`, unpacking, plus global/nonlocal overrides); a partial body scan would suppress the wrong edges invisibly -- the same failure mode that rules out the name list. Enclosing scopes are walked to the file root so a closure over an outer parameter counts. Wired at both pass_calls.c and pass_parallel.c with an identical language gate: a guard on one resolver only diverges the sequential and parallel paths. The weak-strategy drop-list is now a single shared predicate used by both the member guard and this one, so they cannot disagree about what "weak" means; a unit test pins that agreement. Tests pin both directions. The pipeline positive control is a cross-file bare call with NO import, so it resolves by a weak strategy this guard could have killed -- asserting a same_module edge would prove nothing, since no guard touches same_module for any input. Verified by breaking the guard in both directions: under-suppressing fails the negatives and the extraction flag; over-suppressing fails the positives, so they are not vacuous. Disabling only the pass_parallel.c gate fails only the >=50-file test and nothing else. 805 passed / 0 failed across pipeline registry parallel extraction complexity lsp_resolution_probe. The defect class was identified by Joseph-MingEn in #1386, which proposed a name-keyed shape; the diagnosis is theirs. Signed-off-by: Martin Vogel Co-authored-by: Joseph-MingEn <125283161+Joseph-MingEn@users.noreply.github.com> --- internal/cbm/cbm.h | 4 ++ internal/cbm/extract_calls.c | 87 ++++++++++++++++++++++++ src/pipeline/pass_calls.c | 10 ++- src/pipeline/pass_parallel.c | 7 +- src/pipeline/pipeline.h | 12 ++++ src/pipeline/registry.c | 57 +++++++++++++--- tests/test_extraction.c | 80 ++++++++++++++++++++++ tests/test_pipeline.c | 128 +++++++++++++++++++++++++++++++++++ tests/test_registry.c | 63 +++++++++++++++++ 9 files changed, 436 insertions(+), 12 deletions(-) diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 3ea5c48f5..f06104ee0 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -268,6 +268,10 @@ typedef struct { // pass_lsp_cross.c. Default false. bool requires_lsp_resolution; // synthetic semantic candidate (for example an implicit // C++ operator). Never fall back to textual resolution. + bool callee_is_locally_bound; // bare call foo() whose callee identifier is bound as a + // parameter of an enclosing function, so it cannot be the + // module-level foo. Python only today. Read by the + // weak-local-binding guard. Default false. } CBMCall; typedef struct { diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index 7e5dcecae..c65725abb 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -2800,6 +2800,86 @@ static bool python_receiver_is_exempt(CBMExtractCtx *ctx, TSNode receiver) { return false; } +/* Name bound by one Python parameter node, or NULL when the shape binds none. + * Covers every binding form a `parameters` / `lambda_parameters` list produces: + * a bare `identifier`, the `name` field of default/typed parameters, and the + * identifier under a `*args` / `**kwargs` splat. A shape with no identifier (the + * bare `*` keyword separator) yields NULL and simply matches nothing. */ +static const char *python_parameter_name(CBMExtractCtx *ctx, TSNode param) { + if (ts_node_is_null(param)) { + return NULL; + } + if (strcmp(ts_node_type(param), "identifier") == 0) { + return cbm_node_text(ctx->arena, param, ctx->source); + } + TSNode name = ts_node_child_by_field_name(param, TS_FIELD("name")); + if (!ts_node_is_null(name) && strcmp(ts_node_type(name), "identifier") == 0) { + return cbm_node_text(ctx->arena, name, ctx->source); + } + /* `*args` / `**kwargs`, and any typed shape without a `name` field: the + * bound identifier is the first named child. */ + TSNode first = ts_node_named_child(param, 0); + if (!ts_node_is_null(first) && strcmp(ts_node_type(first), "identifier") == 0) { + return cbm_node_text(ctx->arena, first, ctx->source); + } + return NULL; +} + +/* True when the callee of a BARE Python call `foo()` is bound as a parameter of + * an enclosing function or lambda — the bare-call counterpart of + * python_receiver_is_exempt above. + * + * A parameter binding shadows any module-level `foo` for the whole body, so + * resolving such a call to a project Function/Method by short name alone + * fabricates the edge BY CONSTRUCTION: `def _run_with_heavy_slot(run): run()` + * must not bind an unrelated `SatoriLive.run`. Unlike a receiver type this is + * decidable from the AST outright, with no flow analysis and no list of + * "generic-looking" callee names — Python forbids `global` on a parameter, and a + * parameter is in scope for the entire body regardless of position, so there is + * no ordering subtlety to get wrong. + * + * Enclosing scopes are walked to the file root so a closure over an outer + * parameter counts (`def outer(run): def inner(): return run()`). + * + * LOCAL ASSIGNMENTS are deliberately NOT covered. They are flow- and + * binding-form-sensitive (`for`, `with ... as`, `except ... as`, `:=`, + * unpacking, plus `global`/`nonlocal` overrides), so a partial body scan would + * suppress the wrong edges invisibly — the same failure mode that rules out a + * hardcoded name list. Parameters alone already cover the Callable-parameter + * shape that motivated this guard. Cost is O(enclosing depth x params) per bare + * call, never the corpus. + * + * Known and accepted: `def inner(): global run; return run()` nested in a + * function whose parameter is `run` is still flagged. Detecting it needs exactly + * the body scan this helper avoids, and it costs one edge in a shape that + * essentially does not occur. */ +static bool python_callee_is_bound_parameter(CBMExtractCtx *ctx, TSNode call_node, + TSNode callee_ident) { + const char *callee_name = cbm_node_text(ctx->arena, callee_ident, ctx->source); + if (!callee_name || !callee_name[0]) { + return false; + } + for (TSNode scope = ts_node_parent(call_node); !ts_node_is_null(scope); + scope = ts_node_parent(scope)) { + const char *kind = ts_node_type(scope); + if (strcmp(kind, "function_definition") != 0 && strcmp(kind, "lambda") != 0) { + continue; + } + TSNode params = ts_node_child_by_field_name(scope, TS_FIELD("parameters")); + if (ts_node_is_null(params)) { + continue; + } + uint32_t count = ts_node_named_child_count(params); + for (uint32_t i = 0; i < count; i++) { + const char *pname = python_parameter_name(ctx, ts_node_named_child(params, i)); + if (pname && strcmp(pname, callee_name) == 0) { + return true; + } + } + } + return false; +} + static bool is_objectscript_language(CBMLanguage language) { return language == CBM_LANG_OBJECTSCRIPT_UDL || language == CBM_LANG_OBJECTSCRIPT_ROUTINE; } @@ -3424,11 +3504,18 @@ CBMInvocationDescriptor handle_calls(CBMExtractCtx *ctx, TSNode node, const CBML // (`accelerator.print()` must not bind MockAccelerator.print). // Imported receivers stay unflagged: module.function() is Python's // canonical cross-file call and the import map resolves it. + // A BARE Python call foo() whose callee is bound as a parameter of an + // enclosing scope cannot be the module-level foo, so short-name + // resolution would fabricate the edge (`def f(run): run()` must not + // bind SatoriLive.run). Distinct from is_method: there is no receiver + // here, so the weak-member guard cannot see this class at all. if (ctx->language == CBM_LANG_PYTHON && strcmp(ts_node_type(node), "call") == 0) { TSNode fn = ts_node_child_by_field_name(node, TS_FIELD("function")); if (!ts_node_is_null(fn) && strcmp(ts_node_type(fn), "attribute") == 0) { TSNode obj = ts_node_child_by_field_name(fn, TS_FIELD("object")); call.is_method = !python_receiver_is_exempt(ctx, obj); + } else if (!ts_node_is_null(fn) && strcmp(ts_node_type(fn), "identifier") == 0) { + call.callee_is_locally_bound = python_callee_is_bound_parameter(ctx, node, fn); } } // TS/JS/TSX receiver-aware guard (#592/#606 direction; same intent diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index b25e9f592..c9744dda2 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -627,8 +627,16 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, bool suppress_weak_member = lang == CBM_LANG_PYTHON || lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX || lang == CBM_LANG_ARKTS; + /* Bare-call local-binding suppression. A member call has a receiver the + * guard above can reason about; a bare `run()` has none, so that guard + * cannot see this class at all. Python-only today because the extraction + * flag is set only for Python — this gate MUST match pass_parallel.c's + * exactly, for the same divergence reason noted above. */ + bool suppress_weak_local_binding = lang == CBM_LANG_PYTHON; bool drop_plain_call = - cbm_suppress_weak_member_match(suppress_weak_member, call->is_method, res.strategy); + cbm_suppress_weak_member_match(suppress_weak_member, call->is_method, res.strategy) || + cbm_suppress_weak_local_binding_call(suppress_weak_local_binding, + call->callee_is_locally_bound, res.strategy); /* Service-pattern HTTP/ASYNC calls to an EXTERNAL client library (e.g. * `requests.get("/api/orders/{id}")`) resolve to a QN containing the library diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 1eeb55f83..d6cc648af 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -2482,8 +2482,13 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB bool suppress_weak_member = lang == CBM_LANG_PYTHON || lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX || lang == CBM_LANG_ARKTS; + /* Bare-call local-binding suppression — see the note in pass_calls.c. + * This gate MUST stay identical to the one there. */ + bool suppress_weak_local_binding = lang == CBM_LANG_PYTHON; bool drop_plain_call = - cbm_suppress_weak_member_match(suppress_weak_member, call->is_method, res.strategy); + cbm_suppress_weak_member_match(suppress_weak_member, call->is_method, res.strategy) || + cbm_suppress_weak_local_binding_call(suppress_weak_local_binding, + call->callee_is_locally_bound, res.strategy); /* Service-pattern HTTP/ASYNC client call (`requests.get(url)`): the * service signal lives in the callee_name. The registry can mis-resolve diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index 4b1d15563..37f709a68 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -280,6 +280,18 @@ bool cbm_perl_suppress_generic_match(bool is_perl, bool is_method, const char *c * Pure; unit-tested in test_registry.c. */ bool cbm_suppress_weak_member_match(bool enabled, bool is_method, const char *strategy); +/* Bare-call counterpart of the guard above. True when a resolved BARE call edge + * binds a callee that is shadowed by an enclosing parameter, and the match came + * from a weak short-name strategy — so the edge is fabricated by construction + * (`def f(run): run()` must not bind an unrelated `SatoriLive.run`). Shares the + * member guard's drop-list, so lsp_* / import / same-module matches are kept. + * Deliberately keyed on the SCOPE FACT, not on the callee's spelling. The + * language set lives at the call sites (pass_calls.c / pass_parallel.c) and must + * be identical in both, or the sequential and parallel resolvers diverge. + * Pure; unit-tested in test_registry.c. */ +bool cbm_suppress_weak_local_binding_call(bool enabled, bool callee_is_locally_bound, + const char *strategy); + /* #725: drop a suffix_match CALLS edge when the caller language and the * target file's language disagree. unique_name (candidates == 1) is #1572 * and is left alone; same_module / import_map / lsp_* are kept. JS/TS/TSX diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index 5126bcbfe..790e34a60 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -448,22 +448,59 @@ bool cbm_perl_suppress_generic_match(bool is_perl, bool is_method, const char *c * per-language decision made at the call sites in pass_calls.c and * pass_parallel.c, which MUST stay in lockstep — a gate added to only one of * them diverges the sequential and parallel resolvers. */ -bool cbm_suppress_weak_member_match(bool enabled, bool is_method, const char *strategy) { - if (!enabled || !is_method || !strategy || !strategy[0]) { +/* The weak short-name strategies that actually reach the call-resolution + * guards: the registry's suffix_match / unique_name and the parallel + * field_type_hint. "fuzzy" is listed as defensive insurance only — + * cbm_registry_fuzzy_resolve is not wired into the sequential/parallel resolvers + * today, so it never reaches these helpers, but naming it keeps a future wiring + * from silently reintroducing the noise. Everything else — same_module / + * import_map / import_map_suffix / qualified_suffix / callee_suffix / + * service_pattern / lsp_* — is a receiver- or import-aware match and is KEPT. + * + * Shared by BOTH weak-call guards below so the drop-list exists exactly once: a + * list that drifted between the member guard and the local-binding guard would + * make the two disagree about what "weak" means. */ +static bool weak_short_name_strategy(const char *strategy) { + if (!strategy || !strategy[0]) { return false; } - /* Weak short-name strategies that actually reach the call-resolution guards: - * the registry's suffix_match / unique_name and the parallel field_type_hint. - * "fuzzy" is listed as defensive insurance only — cbm_registry_fuzzy_resolve - * is not wired into the sequential/parallel resolvers today, so it never - * reaches this helper, but naming it keeps a future wiring from silently - * reintroducing the noise. Everything else — same_module / import_map / - * import_map_suffix / qualified_suffix / callee_suffix / service_pattern / - * lsp_* — is a receiver- or import-aware match and is KEPT. */ return strcmp(strategy, "suffix_match") == 0 || strcmp(strategy, "unique_name") == 0 || strcmp(strategy, "field_type_hint") == 0 || strcmp(strategy, "fuzzy") == 0; } +bool cbm_suppress_weak_member_match(bool enabled, bool is_method, const char *strategy) { + if (!enabled || !is_method) { + return false; + } + return weak_short_name_strategy(strategy); +} + +/* Bare-call counterpart of the member guard above. A Python call `foo()` whose + * callee identifier is bound as a parameter of an enclosing scope cannot be the + * module-level `foo`: the parameter shadows it for the whole body. Binding such + * a call to a project Function/Method by a weak short-name strategy fabricates + * the edge by construction (`def _run_with_heavy_slot(run): run()` -> + * SatoriLive.run). + * + * This is deliberately NOT keyed on the callee's spelling. A list of + * "generic-looking" names (get / run / execute) asserts that certain spellings + * are usually noise, which is a claim about corpus fashion rather than about + * what the resolver knew — and it ages invisibly, because nothing fails when the + * distribution shifts, the graph just quietly loses different edges. A parameter + * binding is a fact about THIS file's scope, decidable outright. + * + * `enabled` is the caller's per-language gate, kept out of the helper for the + * same reason as the member guard: the two call sites in pass_calls.c and + * pass_parallel.c MUST enumerate the identical language set, or the sequential + * and parallel resolvers diverge. Pure; unit-tested in test_registry.c. */ +bool cbm_suppress_weak_local_binding_call(bool enabled, bool callee_is_locally_bound, + const char *strategy) { + if (!enabled || !callee_is_locally_bound) { + return false; + } + return weak_short_name_strategy(strategy); +} + static bool js_ts_family(CBMLanguage lang) { return lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX || lang == CBM_LANG_ARKTS; diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 73219f4da..7657835eb 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -4621,6 +4621,85 @@ TEST(extract_python_member_call_flags_is_method) { PASS(); } +/* Python bare-call local-binding flag (the bare-call counterpart of the + * receiver flag above). Pins BOTH directions: a callee shadowed by a parameter + * of an enclosing scope IS flagged so the resolver can suppress a weak + * short-name match, while an unshadowed callee — a genuine module-level + * function, an imported name, or a nested `def` — is NOT, so its true edge + * survives. Every parameter binding form the grammar produces is covered, since + * a form the extractor silently missed would leave that shape unguarded. */ +TEST(extract_python_bare_call_flags_locally_bound_callee) { + CBMFileResult *r = extract("from pkg import helper\n" + "\n" + "def outer(run, *rest, timeout=5, label: str = 'x', **opts):\n" + " def inner():\n" + " return run()\n" + " rest()\n" + " timeout()\n" + " label()\n" + " opts()\n" + " module_level()\n" + " helper()\n" + " return inner()\n" + "\n" + "def typed(cb: Callable):\n" + " return cb()\n" + "\n" + "apply_it = lambda fn: fn()\n", + CBM_LANG_PYTHON, "t", "x.py"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + /* callee name -> (expected flag, seen count) */ + struct { + const char *callee; + bool expect_bound; + int seen; + } cases[] = { + {"run", true, 0}, /* closure over an ENCLOSING function's parameter */ + {"rest", true, 0}, /* *args -> list_splat_pattern */ + {"timeout", true, 0}, /* default_parameter */ + {"label", true, 0}, /* typed_default_parameter (keyword-only) */ + {"opts", true, 0}, /* **kwargs -> dictionary_splat_pattern */ + {"cb", true, 0}, /* typed_parameter, no default */ + {"fn", true, 0}, /* lambda parameter */ + {"module_level", false, 0}, /* unbound: the true cross-file edge */ + {"helper", false, 0}, /* imported name, not a parameter */ + {"inner", false, 0}, /* nested def: a real target, keep the edge */ + }; + const int case_count = (int)(sizeof(cases) / sizeof(cases[0])); + + for (int i = 0; i < r->calls.count; i++) { + const char *cn = r->calls.items[i].callee_name; + if (!cn) { + continue; + } + for (int c = 0; c < case_count; c++) { + if (strcmp(cn, cases[c].callee) != 0) { + continue; + } + cases[c].seen++; + if (r->calls.items[i].callee_is_locally_bound != cases[c].expect_bound) { + printf(" bare-call flag mismatch for %s(): got %d, expected %d\n", cases[c].callee, + r->calls.items[i].callee_is_locally_bound ? 1 : 0, + cases[c].expect_bound ? 1 : 0); + } + ASSERT_EQ(r->calls.items[i].callee_is_locally_bound, cases[c].expect_bound); + } + } + /* Each shape must appear exactly once, so a missed extraction cannot let the + * loop above pass vacuously. */ + for (int c = 0; c < case_count; c++) { + if (cases[c].seen != 1) { + printf(" bare call %s() extracted %d times, expected 1\n", cases[c].callee, + cases[c].seen); + } + ASSERT_EQ(cases[c].seen, 1); + } + cbm_free_result(r); + PASS(); +} + /* TS/JS/TSX receiver-aware flag (#592/#606; same intent as the Perl flag above). * A member call x.foo() with a non-this/super receiver is flagged is_method so * the resolver can suppress a weak short-name match (`re.test()` must not bind a @@ -6300,6 +6379,7 @@ SUITE(extraction) { RUN_TEST(extract_perl_method_call_flags_is_method); RUN_TEST(extract_flag_exempt_method_call_not_flagged_is_method); RUN_TEST(extract_python_member_call_flags_is_method); + RUN_TEST(extract_python_bare_call_flags_locally_bound_callee); RUN_TEST(extract_ts_member_call_flags_is_method); RUN_TEST(extract_ts_this_super_receiver_not_flagged); RUN_TEST(extract_js_member_call_flags_is_method); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index a45541e57..716b839a8 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -4836,6 +4836,60 @@ TEST(pipeline_tsjs_receiver_parallel_keeps_service_edges) { PASS(); } +/* Python bare-call local-binding suppression, sequential path. The bare-call + * counterpart of the receiver guard above: `run` is a PARAMETER, so `run()` + * cannot be the module-level `run` and must not bind SatoriLive.run. + * + * The positive control is deliberately a CROSS-FILE bare call with no import, + * so it resolves by a weak short-name strategy — one this guard could have + * killed. Asserting a same-file (same_module) edge instead would prove nothing, + * because no guard in this codebase touches same_module for any input. + * Fewer than 50 files exercises pass_calls.c. */ +TEST(pipeline_python_bare_local_binding_suppresses_weak_edge) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_py_bare_seq_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + + write_temp_file(tmp, "live.py", + "class SatoriLive:\n" + " def run(self):\n" + " return 1\n"); + write_temp_file(tmp, "helpers.py", + "def compute_widget_total():\n" + " return 7\n"); + write_temp_file(tmp, "gate.py", + "def _run_with_heavy_slot(run):\n" + " return run()\n" + "\n" + "def uses_free_function():\n" + " return compute_widget_total()\n"); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/py_bare.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + const char *project = cbm_pipeline_project_name(p); + + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + + /* NEGATIVE: the callee is shadowed by a parameter. */ + ASSERT_FALSE(cross_file_call_exists(s, project, "_run_with_heavy_slot", "run")); + /* POSITIVE: an unshadowed cross-file bare call survives. */ + ASSERT_TRUE(cross_file_call_exists(s, project, "uses_free_function", "compute_widget_total")); + /* Tripwire: a run that emitted no edges at all would satisfy the negative + * assertion vacuously. */ + ASSERT_GTE(cbm_store_count_edges_by_type(s, project, "CALLS"), 1); + + cbm_store_close(s); + cbm_pipeline_free(p); + th_rmtree(tmp); + PASS(); +} + /* Parallel Python regression for #1276. The field-type heuristic capitalizes * the receiver token and previously promoted accelerator.print() to * MockAccelerator.print at 0.85; ordinary suffix matching also selected one @@ -4933,6 +4987,78 @@ TEST(pipeline_python_receiver_parallel_suppresses_weak_method_edges) { PASS(); } +/* Parallel counterpart. >= 50 files forces pass_parallel.c, which is wired with + * the same gate: a guard wired on only one resolver produces an edge on the + * sequential path and not the parallel one, breaking MT determinism. #1386 + * wired both and tested only the sequential path, and the `parallel` suite is + * exactly what catches that. Same both-directions pin as the sequential test. */ +TEST(pipeline_python_bare_local_binding_parallel_suppresses_weak_edge) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_py_bare_par_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + + write_temp_file(tmp, "live.py", + "class SatoriLive:\n" + " def run(self):\n" + " return 1\n" + "\n" + "class BatchJob:\n" + " def execute(self):\n" + " return 2\n"); + write_temp_file(tmp, "helpers.py", + "def compute_widget_total():\n" + " return 7\n"); + write_temp_file(tmp, "gate.py", + "def _run_with_heavy_slot(run, execute):\n" + " run()\n" + " return execute()\n" + "\n" + "def uses_free_function():\n" + " return compute_widget_total()\n"); + for (int i = 0; i < 52; i++) { + char name[64]; + char body[128]; + snprintf(name, sizeof(name), "filler%d.py", i); + snprintf(body, sizeof(body), "def filler%d():\n return %d\n", i, i); + write_temp_file(tmp, name, body); + } + + char *old_workers = getenv("CBM_WORKERS"); + char *saved = old_workers ? strdup(old_workers) : NULL; + cbm_setenv("CBM_WORKERS", "4", 1); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/py_bare_par.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + const char *project = cbm_pipeline_project_name(p); + + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + + /* NEGATIVE: both callees are shadowed by parameters. */ + ASSERT_FALSE(cross_file_call_exists(s, project, "_run_with_heavy_slot", "run")); + ASSERT_FALSE(cross_file_call_exists(s, project, "_run_with_heavy_slot", "execute")); + /* POSITIVE: the unshadowed cross-file bare call survives the parallel path. */ + ASSERT_TRUE(cross_file_call_exists(s, project, "uses_free_function", "compute_widget_total")); + /* Tripwire against a vacuous pass. */ + ASSERT_GTE(cbm_store_count_edges_by_type(s, project, "CALLS"), 1); + + cbm_store_close(s); + cbm_pipeline_free(p); + if (saved) { + cbm_setenv("CBM_WORKERS", saved, 1); + free(saved); + } else { + cbm_unsetenv("CBM_WORKERS"); + } + th_rmtree(tmp); + PASS(); +} + /* Reproduce-first: pass_parallel's fused cross-LSP eligibility currently counts * only parser-backed calls/call references. A Python binary operator has no * parser CBMCall; its __add__ semantic record and carrier are created together @@ -12807,6 +12933,8 @@ SUITE(pipeline) { RUN_TEST(pipeline_python_receiver_suppresses_weak_method_edge); RUN_TEST(pipeline_tsjs_receiver_parallel_keeps_service_edges); RUN_TEST(pipeline_python_receiver_parallel_suppresses_weak_method_edges); + RUN_TEST(pipeline_python_bare_local_binding_suppresses_weak_edge); + RUN_TEST(pipeline_python_bare_local_binding_parallel_suppresses_weak_edge); RUN_TEST(pipeline_parallel_python_cross_only_dunder_gets_synthetic_carrier); RUN_TEST(pipeline_parallel_rust_cross_only_macro_hidden_gets_synthetic_carrier); RUN_TEST(pipeline_native_fetch_classified_as_http_calls); diff --git a/tests/test_registry.c b/tests/test_registry.c index ff81a50ee..3747501ab 100644 --- a/tests/test_registry.c +++ b/tests/test_registry.c @@ -853,6 +853,66 @@ TEST(dynamic_suppress_keeps_high_confidence_and_non_methods) { PASS(); } +TEST(local_binding_suppress_drops_weak_shadowed_bare_calls) { + /* A bare `run()` whose callee is a parameter of an enclosing scope cannot be + * the module-level `run`, so a weak short-name match fabricates the edge. */ + ASSERT_TRUE(cbm_suppress_weak_local_binding_call(true, true, "suffix_match")); + ASSERT_TRUE(cbm_suppress_weak_local_binding_call(true, true, "unique_name")); + ASSERT_TRUE(cbm_suppress_weak_local_binding_call(true, true, "field_type_hint")); + ASSERT_TRUE(cbm_suppress_weak_local_binding_call(true, true, "fuzzy")); + PASS(); +} + +TEST(local_binding_suppress_keeps_unshadowed_and_strong_strategies) { + /* THE RECALL PIN. A bare call to a genuine module-level function is NOT + * locally bound, so it is never suppressed — whatever the callee is spelled. + * This is the assertion a name-keyed guard (get/run/execute) would fail: it + * would drop these purely because of how the callee reads. */ + ASSERT_FALSE(cbm_suppress_weak_local_binding_call(true, false, "suffix_match")); + ASSERT_FALSE(cbm_suppress_weak_local_binding_call(true, false, "unique_name")); + ASSERT_FALSE(cbm_suppress_weak_local_binding_call(true, false, "field_type_hint")); + ASSERT_FALSE(cbm_suppress_weak_local_binding_call(true, false, "fuzzy")); + /* Every receiver-/import-aware strategy is kept even when shadowed. */ + ASSERT_FALSE(cbm_suppress_weak_local_binding_call(true, true, "same_module")); + ASSERT_FALSE(cbm_suppress_weak_local_binding_call(true, true, "import_map")); + ASSERT_FALSE(cbm_suppress_weak_local_binding_call(true, true, "import_map_suffix")); + ASSERT_FALSE(cbm_suppress_weak_local_binding_call(true, true, "qualified_suffix")); + ASSERT_FALSE(cbm_suppress_weak_local_binding_call(true, true, "callee_suffix")); + ASSERT_FALSE(cbm_suppress_weak_local_binding_call(true, true, "service_pattern")); + ASSERT_FALSE(cbm_suppress_weak_local_binding_call(true, true, "lsp_cross")); + ASSERT_FALSE(cbm_suppress_weak_local_binding_call(true, true, "lsp_py_method")); + ASSERT_FALSE(cbm_suppress_weak_local_binding_call(true, true, "lsp_direct")); + /* Languages outside the caller's gate are never affected. */ + ASSERT_FALSE(cbm_suppress_weak_local_binding_call(false, true, "suffix_match")); + /* No match (NULL/empty strategy) → nothing to suppress. */ + ASSERT_FALSE(cbm_suppress_weak_local_binding_call(true, true, NULL)); + ASSERT_FALSE(cbm_suppress_weak_local_binding_call(true, true, "")); + PASS(); +} + +TEST(weak_call_guards_share_one_drop_list) { + /* The member guard and the local-binding guard must agree on what "weak" + * means. They share a single static predicate for exactly this reason; if + * someone re-inlines one of the lists and edits only that copy, the two + * guards start disagreeing and this test catches it at the contract level + * rather than in a corpus months later. */ + static const char *const strategies[] = { + "suffix_match", "unique_name", "field_type_hint", "fuzzy", "same_module", + "import_map", "import_map_suffix", "qualified_suffix", "callee_suffix", + "service_pattern", "lsp_cross", "lsp_ts_method", "lsp_py_method", "lsp_direct", + "", NULL}; + for (int i = 0; strategies[i] != NULL; i++) { + bool member = cbm_suppress_weak_member_match(true, true, strategies[i]); + bool binding = cbm_suppress_weak_local_binding_call(true, true, strategies[i]); + if (member != binding) { + printf(" drop-list divergence on strategy \"%s\": member=%d binding=%d\n", + strategies[i], member, binding); + } + ASSERT_EQ(member, binding); + } + PASS(); +} + /* ── Suite ─────────────────────────────────────────────────────── */ /* Method call THROUGH an imported symbol that is itself an indexed node @@ -947,4 +1007,7 @@ SUITE(registry) { RUN_TEST(cross_language_suffix_match_drops_py_vs_js); RUN_TEST(dynamic_suppress_drops_weak_method_matches); RUN_TEST(dynamic_suppress_keeps_high_confidence_and_non_methods); + RUN_TEST(local_binding_suppress_drops_weak_shadowed_bare_calls); + RUN_TEST(local_binding_suppress_keeps_unshadowed_and_strong_strategies); + RUN_TEST(weak_call_guards_share_one_drop_list); } From 8bcfa2915fa02721c5b6d2a6c7082a26fef79d88 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 29 Aug 2026 18:03:01 +0200 Subject: [PATCH 06/22] style: clang-format the new bare-call tests Changed-range clang-format (Homebrew LLVM 22.1.8, the build CI's lint-ci uses) flagged two alignment violations in the tests added by the previous commit. Whitespace only; 805 passed / 0 failed re-run after the reformat. Signed-off-by: Martin Vogel --- tests/test_extraction.c | 14 +++++++------- tests/test_registry.c | 21 ++++++++++++++++----- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 7657835eb..0bfcaf552 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -4656,13 +4656,13 @@ TEST(extract_python_bare_call_flags_locally_bound_callee) { bool expect_bound; int seen; } cases[] = { - {"run", true, 0}, /* closure over an ENCLOSING function's parameter */ - {"rest", true, 0}, /* *args -> list_splat_pattern */ - {"timeout", true, 0}, /* default_parameter */ - {"label", true, 0}, /* typed_default_parameter (keyword-only) */ - {"opts", true, 0}, /* **kwargs -> dictionary_splat_pattern */ - {"cb", true, 0}, /* typed_parameter, no default */ - {"fn", true, 0}, /* lambda parameter */ + {"run", true, 0}, /* closure over an ENCLOSING function's parameter */ + {"rest", true, 0}, /* *args -> list_splat_pattern */ + {"timeout", true, 0}, /* default_parameter */ + {"label", true, 0}, /* typed_default_parameter (keyword-only) */ + {"opts", true, 0}, /* **kwargs -> dictionary_splat_pattern */ + {"cb", true, 0}, /* typed_parameter, no default */ + {"fn", true, 0}, /* lambda parameter */ {"module_level", false, 0}, /* unbound: the true cross-file edge */ {"helper", false, 0}, /* imported name, not a parameter */ {"inner", false, 0}, /* nested def: a real target, keep the edge */ diff --git a/tests/test_registry.c b/tests/test_registry.c index 3747501ab..2fd966948 100644 --- a/tests/test_registry.c +++ b/tests/test_registry.c @@ -896,11 +896,22 @@ TEST(weak_call_guards_share_one_drop_list) { * someone re-inlines one of the lists and edits only that copy, the two * guards start disagreeing and this test catches it at the contract level * rather than in a corpus months later. */ - static const char *const strategies[] = { - "suffix_match", "unique_name", "field_type_hint", "fuzzy", "same_module", - "import_map", "import_map_suffix", "qualified_suffix", "callee_suffix", - "service_pattern", "lsp_cross", "lsp_ts_method", "lsp_py_method", "lsp_direct", - "", NULL}; + static const char *const strategies[] = {"suffix_match", + "unique_name", + "field_type_hint", + "fuzzy", + "same_module", + "import_map", + "import_map_suffix", + "qualified_suffix", + "callee_suffix", + "service_pattern", + "lsp_cross", + "lsp_ts_method", + "lsp_py_method", + "lsp_direct", + "", + NULL}; for (int i = 0; strategies[i] != NULL; i++) { bool member = cbm_suppress_weak_member_match(true, true, strategies[i]); bool binding = cbm_suppress_weak_local_binding_call(true, true, strategies[i]); From 0b0d143c2d733e3313eef5b9d9745b0fb08cc2c0 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 29 Aug 2026 21:11:45 +0200 Subject: [PATCH 07/22] fix(python): walk bare-call scopes with the walk cursor, bounded #1912 hung stack_overflow_b on all seven CI legs: rc=124 at 900s with zero tests completed, on lsp_python_deep_nesting_no_crash. The bare-call scope walk used ts_node_parent(), which is NOT O(1) -- it restarts at the tree root and descends to find the parent (vendored ts_runtime/src/node.c). Each hop therefore costs O(depth), a parent chain is O(depth^2) per call, and since every level of f(f(f(...))) is itself a bare call, O(depth^3) across the file. The fixture nests 30,000 deep, so this is a hang rather than a slowdown. A hop-count cap alone is NOT sufficient, and this was measured rather than assumed: with a 64-hop cap the suite still timed out, because 64 x O(30,000) x 30,000 calls is still ~5.7e10. When the cost is per hop, bounding the number of hops cannot fix it -- the walk itself has to be cheap. Use the unified walk's own cursor. WalkState.current_cursor is already parked on the current node and ts_tree_cursor_goto_parent() IS O(1), because the cursor carries its path stack, so ascending costs nothing per hop. Same current_cursor idiom, including the ts_node_eq identity guard before trusting the shared cursor, as usage_current_field_name in extract_usages.c. This is the same lesson CBMWalkScope already records: carry walk state, never recompute it per node. The 64-hop cap is retained and now does real work, bounding the remaining O(depth) per call. Precedent for the shape: CBM_LSP_PERL_MAX_WALK_DEPTH and LEAN_MAX_PARENT_DEPTH in this file. Both the cap and a missing or mismatched cursor FAIL OPEN -- they can only ever cost a suppression, never a true edge, which is the safe direction for a guard whose justification is precision. lsp_python_deep_nesting_no_crash: 900s timeout, 0 completions -> PASS. stack_overflow_b overall: 6 passed in 46s. The regression test pins the cap's contract deterministically rather than by wall clock: within the cap a shadowed callee is flagged; past 64 ancestors the guard fails open and leaves it unflagged. Raising the cap flips the deep case and fails the test, verified. 826 passed / 0 failed across stack_overflow_a stack_overflow_b stack_overflow_c pipeline registry parallel extraction complexity lsp_resolution_probe. Signed-off-by: Martin Vogel Co-authored-by: Joseph-MingEn <125283161+Joseph-MingEn@users.noreply.github.com> --- internal/cbm/extract_calls.c | 55 ++++++++++++++++++++++++++++---- tests/test_extraction.c | 62 ++++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 6 deletions(-) diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index c65725abb..94d8b96b9 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -2853,14 +2853,54 @@ static const char *python_parameter_name(CBMExtractCtx *ctx, TSNode param) { * function whose parameter is `run` is still flagged. Detecting it needs exactly * the body scan this helper avoids, and it costs one edge in a shape that * essentially does not occur. */ -static bool python_callee_is_bound_parameter(CBMExtractCtx *ctx, TSNode call_node, +static bool python_callee_is_bound_parameter(CBMExtractCtx *ctx, WalkState *state, TSNode call_node, TSNode callee_ident) { + /* Ancestors are walked with the UNIFIED WALK'S OWN CURSOR, not ts_node_parent(). + * + * ts_node_parent() is not O(1): it restarts at the tree root and descends to + * find the parent (vendored ts_runtime/src/node.c), so it costs O(depth) per + * hop. A parent-chain walk is therefore O(depth^2) per call, and since every + * level of f(f(f(...))) is ITSELF a bare call, O(depth^3) across the file -- + * tests/test_stack_overflow.c nests 30,000 deep, which is a hang, not a + * slowdown. Capping the hop COUNT does not fix that, because the cost is per + * hop; the walk itself has to be cheap. + * + * ts_tree_cursor_goto_parent() IS O(1) -- the cursor carries its path stack -- + * so copying the walk cursor and ascending costs nothing per hop. Same reason + * CBMWalkScope keeps a frame stack instead of recomputing enclosing state, and + * same current_cursor idiom (including the identity guard) as + * usage_current_field_name in extract_usages.c. + * + * The hop cap then bounds the remaining O(depth) per call. 64 rather than + * Lean's 20 above: that walk looks for an IMMEDIATE declaration boundary, + * while this one crosses whatever statement/expression nesting separates a + * call from its enclosing def, so it needs headroom before it can bite on + * ordinary code. + * + * Both the cap and a missing/mismatched cursor FAIL OPEN -- return false, do + * NOT suppress. A guard whose whole justification is precision must never + * destroy an edge it did not actually prove was fabricated, so either can + * only ever cost a suppression, never a true edge. Pinned in + * test_extraction.c. */ + enum { PY_MAX_SCOPE_WALK_DEPTH = 64 }; + const char *callee_name = cbm_node_text(ctx->arena, callee_ident, ctx->source); if (!callee_name || !callee_name[0]) { return false; } - for (TSNode scope = ts_node_parent(call_node); !ts_node_is_null(scope); - scope = ts_node_parent(scope)) { + /* Only trust the shared cursor when it is actually parked on this call. */ + if (!state || !state->current_cursor || + !ts_node_eq(ts_tree_cursor_current_node(state->current_cursor), call_node)) { + return false; + } + + TSTreeCursor up = ts_tree_cursor_copy(state->current_cursor); + bool bound = false; + for (int walked = 0; walked < PY_MAX_SCOPE_WALK_DEPTH && !bound; walked++) { + if (!ts_tree_cursor_goto_parent(&up)) { + break; + } + TSNode scope = ts_tree_cursor_current_node(&up); const char *kind = ts_node_type(scope); if (strcmp(kind, "function_definition") != 0 && strcmp(kind, "lambda") != 0) { continue; @@ -2873,11 +2913,13 @@ static bool python_callee_is_bound_parameter(CBMExtractCtx *ctx, TSNode call_nod for (uint32_t i = 0; i < count; i++) { const char *pname = python_parameter_name(ctx, ts_node_named_child(params, i)); if (pname && strcmp(pname, callee_name) == 0) { - return true; + bound = true; + break; } } } - return false; + ts_tree_cursor_delete(&up); + return bound; } static bool is_objectscript_language(CBMLanguage language) { @@ -3515,7 +3557,8 @@ CBMInvocationDescriptor handle_calls(CBMExtractCtx *ctx, TSNode node, const CBML TSNode obj = ts_node_child_by_field_name(fn, TS_FIELD("object")); call.is_method = !python_receiver_is_exempt(ctx, obj); } else if (!ts_node_is_null(fn) && strcmp(ts_node_type(fn), "identifier") == 0) { - call.callee_is_locally_bound = python_callee_is_bound_parameter(ctx, node, fn); + call.callee_is_locally_bound = + python_callee_is_bound_parameter(ctx, state, node, fn); } } // TS/JS/TSX receiver-aware guard (#592/#606 direction; same intent diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 0bfcaf552..c67e0a7aa 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -4700,6 +4700,67 @@ TEST(extract_python_bare_call_flags_locally_bound_callee) { PASS(); } +/* The scope walk behind the bare-call flag is BOUNDED (64 ancestors). An + * unbounded walk is QUADRATIC in a file's nesting depth -- every level of + * f(f(f(...))) is itself a bare call re-walking its own chain -- which hung + * stack_overflow_b's 30,000-deep fixture rather than merely slowing it. + * + * This pins the cap's CONTRACT deterministically rather than by wall clock (a + * timing assertion would be a lottery, not a gate): within the cap the shadowed + * callee is flagged; past it the guard FAILS OPEN and leaves the call alone, so + * the cap can only ever cost a suppression, never a true edge. Removing the cap + * flips the deep case to flagged and fails this test. */ +TEST(extract_python_bare_call_scope_walk_is_bounded) { + /* Shallow: return_statement / block / function_definition — 3 ancestors. */ + CBMFileResult *shallow = extract("def shallow(handler):\n" + " return handler()\n", + CBM_LANG_PYTHON, "t", "s.py"); + ASSERT_NOT_NULL(shallow); + ASSERT_FALSE(shallow->has_error); + int shallow_seen = 0; + for (int i = 0; i < shallow->calls.count; i++) { + const char *cn = shallow->calls.items[i].callee_name; + if (cn && strcmp(cn, "handler") == 0) { + shallow_seen++; + ASSERT_TRUE(shallow->calls.items[i].callee_is_locally_bound); + } + } + ASSERT_EQ(shallow_seen, 1); + cbm_free_result(shallow); + + /* Deep: 200 parenthesized_expression ancestors put the SAME call well past + * the cap, so the enclosing parameter is never reached and the call stays + * unflagged. */ + const int PARENS = 200; + size_t sz = (size_t)PARENS * 2 + 128; + char *src = malloc(sz); + ASSERT_NOT_NULL(src); + char *w = src; + w += snprintf(w, sz, "def deep(handler):\n return "); + memset(w, '(', (size_t)PARENS); + w += PARENS; + w += snprintf(w, sz - (size_t)(w - src), "handler()"); + memset(w, ')', (size_t)PARENS); + w += PARENS; + snprintf(w, sz - (size_t)(w - src), "\n"); + + CBMFileResult *deep = extract(src, CBM_LANG_PYTHON, "t", "d.py"); + ASSERT_NOT_NULL(deep); + ASSERT_FALSE(deep->has_error); + int deep_seen = 0; + for (int i = 0; i < deep->calls.count; i++) { + const char *cn = deep->calls.items[i].callee_name; + if (cn && strcmp(cn, "handler") == 0) { + deep_seen++; + ASSERT_FALSE(deep->calls.items[i].callee_is_locally_bound); + } + } + ASSERT_EQ(deep_seen, 1); + cbm_free_result(deep); + free(src); + PASS(); +} + /* TS/JS/TSX receiver-aware flag (#592/#606; same intent as the Perl flag above). * A member call x.foo() with a non-this/super receiver is flagged is_method so * the resolver can suppress a weak short-name match (`re.test()` must not bind a @@ -6380,6 +6441,7 @@ SUITE(extraction) { RUN_TEST(extract_flag_exempt_method_call_not_flagged_is_method); RUN_TEST(extract_python_member_call_flags_is_method); RUN_TEST(extract_python_bare_call_flags_locally_bound_callee); + RUN_TEST(extract_python_bare_call_scope_walk_is_bounded); RUN_TEST(extract_ts_member_call_flags_is_method); RUN_TEST(extract_ts_this_super_receiver_not_flagged); RUN_TEST(extract_js_member_call_flags_is_method); From 3f831936b5f3012d13175e7c97326165efecefad Mon Sep 17 00:00:00 2001 From: Joshua Richter Date: Mon, 31 Aug 2026 19:57:11 -0400 Subject: [PATCH 08/22] fix(grammar): stop the Swift scanner shifting past the width of an int The Swift scanner keeps a 64-bit mask of the symbols that suppress a match -- the rule that stops `try!` emitting its `!` as a token of its own. It tests one bit per candidate: uint64_t suppressing_symbols = OP_SYMBOL_SUPPRESSOR[full_match]; for (uint64_t suppressor = 0; suppressor < TOKEN_COUNT; suppressor++) { if (!(suppressing_symbols & 1 << suppressor)) { The mask is uint64_t but the literal `1` is an int, so the shift is an int shift. TOKEN_COUNT is larger than 32, so once suppressor reaches 31 the shift runs past the width of the type. That is undefined behavior, and every bit above 31 is tested against a value the standard does not define. Nothing caught it because nothing in the tree reached the suppressor path. Any Swift force-unwrap does: `cached!` is enough. UBSan reports it as: scanner.c:514:47: runtime error: left shift of 1 by 31 places cannot be represented in type 'int' `1ULL` makes the literal as wide as the mask it is tested against. The new test in tests/test_extraction.c cannot go red on its own. The normal test build prints the UBSan message and carries on, which is why this survived. The Windows CLANGARM64 leg runs UBSan in trap mode, and there the same shift is an illegal-instruction crash -- so the test exists to make sure that leg keeps parsing a force-unwrap at all. scripts/vendored-checksums.txt records the new hash for the one changed file, as scripts/security-vendored.sh --update writes it. Layer 8 of the security gate compares vendored content against that manifest, so the edit and its recorded hash belong in the same commit. Found while adding Swift URL extraction in #1892 / #1976, and split out of that PR so the vendored change can be reviewed on its own. Signed-off-by: Joshua Richter --- .../cbm/vendored/grammars/swift/scanner.c | 2 +- scripts/vendored-checksums.txt | 2 +- tests/test_extraction.c | 19 +++++++++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/internal/cbm/vendored/grammars/swift/scanner.c b/internal/cbm/vendored/grammars/swift/scanner.c index 2615afe43..bb2dcac58 100644 --- a/internal/cbm/vendored/grammars/swift/scanner.c +++ b/internal/cbm/vendored/grammars/swift/scanner.c @@ -511,7 +511,7 @@ static bool eat_operators( uint64_t suppressing_symbols = OP_SYMBOL_SUPPRESSOR[full_match]; if (suppressing_symbols) { for (uint64_t suppressor = 0; suppressor < TOKEN_COUNT; suppressor++) { - if (!(suppressing_symbols & 1 << suppressor)) { + if (!(suppressing_symbols & 1ULL << suppressor)) { continue; } diff --git a/scripts/vendored-checksums.txt b/scripts/vendored-checksums.txt index 4b15db072..5412a8fdb 100644 --- a/scripts/vendored-checksums.txt +++ b/scripts/vendored-checksums.txt @@ -791,7 +791,7 @@ c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4 internal/cbm/v 2db740ea1be6b71014d2be1385491f01bf2d15b607e61c5b30b4128535efe68a internal/cbm/vendored/grammars/sway/tree_sitter/parser.h 3533cec129bb4bba015c0d61d86dd7c3b7e82110e4d2ff7837a01eff5bad5ccc internal/cbm/vendored/grammars/swift/LICENSE 93e6b39fc5b16ef9d5869862ec8d56380a838355c362481942942d59ca666de2 internal/cbm/vendored/grammars/swift/parser.c -b835c1ded068e902944fe82c3770b9dcf85f67dcaab8fbf772f1606007da7373 internal/cbm/vendored/grammars/swift/scanner.c +f3d6271d64f58c39eed544104a70ca2cf9ecbf80c5d900620f1afd38836542cb internal/cbm/vendored/grammars/swift/scanner.c b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea internal/cbm/vendored/grammars/swift/tree_sitter/alloc.h 5bdf6ed1a78e3409fd443e085ca967a64c188a5d082aaf7f819bccd53a471c94 internal/cbm/vendored/grammars/swift/tree_sitter/array.h a1f6ef161fbaf48a0e10fca90ef5290a062462b307b3898aa562993853b9f80a internal/cbm/vendored/grammars/swift/tree_sitter/parser.h diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 5b8d16f61..cb7a1a8c8 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -1810,6 +1810,24 @@ TEST(swift_chained_call) { PASS(); } +/* A Swift force-unwrap is the one thing that reaches the scanner's suppressor + * path -- the rule that stops `try!` emitting its `!` as a token of its own. + * That path shifted an int by up to TOKEN_COUNT bits, which runs past the + * width of the type once the index reaches 31. + * + * This test cannot go red here. The normal test build prints the UBSan + * message and carries on, which is why the bug survived. The Windows + * CLANGARM64 leg runs UBSan in trap mode, where the same shift is an + * illegal-instruction crash, so parsing this file at all is the check. */ +TEST(swift_force_unwrap_scanner_shift) { + CBMFileResult *r = + extract("func load() { let u = cached! }\n", CBM_LANG_SWIFT, "t", "Load.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + cbm_free_result(r); + PASS(); +} + /* --- Objective-C --- */ TEST(objc_interface) { CBMFileResult *r = @@ -6661,6 +6679,7 @@ SUITE(extraction) { RUN_TEST(swift_method_call); RUN_TEST(swift_constructor_call); RUN_TEST(swift_chained_call); + RUN_TEST(swift_force_unwrap_scanner_shift); RUN_TEST(objc_interface); RUN_TEST(objc_implementation); RUN_TEST(dart_top_level_function); From a2ea711c29560511eb8467bea7ad309ebf500d6e Mon Sep 17 00:00:00 2001 From: Joshua Richter Date: Mon, 31 Aug 2026 23:01:44 -0400 Subject: [PATCH 09/22] fix(grammar): stop the Swift try-bang entry shifting past its own type scanner.c line 131 shifts a literal 1UL by 32 to build the entry that suppresses the bang of a try!. unsigned long is 64 bits on Linux and macOS but 32 bits on Windows, so there the shift count equals the width of the type, which is undefined. 1ULL is 64 bits on every target this project builds for. Nothing reports it today. The expression is a compile-time constant, so no sanitizer sees it run, and Makefile.cbm:719 builds vendored grammars with -w, which switches off -Wshift-count-overflow. Upstream tree-sitter-swift made the same change in 6ab8d1d74ebd, after the commit this grammar is pinned to. Refs #1892 Signed-off-by: Joshua Richter --- internal/cbm/vendored/grammars/swift/scanner.c | 2 +- scripts/vendored-checksums.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/cbm/vendored/grammars/swift/scanner.c b/internal/cbm/vendored/grammars/swift/scanner.c index bb2dcac58..357350282 100644 --- a/internal/cbm/vendored/grammars/swift/scanner.c +++ b/internal/cbm/vendored/grammars/swift/scanner.c @@ -128,7 +128,7 @@ const uint64_t OP_SYMBOL_SUPPRESSOR[OPERATOR_COUNT] = { 0, // EQ_EQ, 0, // PLUS_THEN_WS, 0, // MINUS_THEN_WS, - 1UL << FAKE_TRY_BANG, // BANG, + 1ULL << FAKE_TRY_BANG, // BANG, 0, // THROWS_KEYWORD, 0, // RETHROWS_KEYWORD, 0, // DEFAULT_KEYWORD, diff --git a/scripts/vendored-checksums.txt b/scripts/vendored-checksums.txt index 5412a8fdb..a54ab6440 100644 --- a/scripts/vendored-checksums.txt +++ b/scripts/vendored-checksums.txt @@ -791,7 +791,7 @@ c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4 internal/cbm/v 2db740ea1be6b71014d2be1385491f01bf2d15b607e61c5b30b4128535efe68a internal/cbm/vendored/grammars/sway/tree_sitter/parser.h 3533cec129bb4bba015c0d61d86dd7c3b7e82110e4d2ff7837a01eff5bad5ccc internal/cbm/vendored/grammars/swift/LICENSE 93e6b39fc5b16ef9d5869862ec8d56380a838355c362481942942d59ca666de2 internal/cbm/vendored/grammars/swift/parser.c -f3d6271d64f58c39eed544104a70ca2cf9ecbf80c5d900620f1afd38836542cb internal/cbm/vendored/grammars/swift/scanner.c +8a14f26c9011a02298a076674461beb48625a4ca7cea4a0f63947930d93631fd internal/cbm/vendored/grammars/swift/scanner.c b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea internal/cbm/vendored/grammars/swift/tree_sitter/alloc.h 5bdf6ed1a78e3409fd443e085ca967a64c188a5d082aaf7f819bccd53a471c94 internal/cbm/vendored/grammars/swift/tree_sitter/array.h a1f6ef161fbaf48a0e10fca90ef5290a062462b307b3898aa562993853b9f80a internal/cbm/vendored/grammars/swift/tree_sitter/parser.h From 9d4ac2b98b9c19633ce2cbf8d9c6bf9edc57bba6 Mon Sep 17 00:00:00 2001 From: Ertan Date: Fri, 28 Aug 2026 16:23:29 +0300 Subject: [PATCH 10/22] fix(daemon): accept Unicode session roots on Windows Signed-off-by: Ertan --- src/daemon/application.c | 12 +++++---- tests/test_daemon_application.c | 40 ++++++++++++++++++++++++++++ tests/windows/test_non_ascii_path.py | 4 +-- 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/src/daemon/application.c b/src/daemon/application.c index 12543808e..69ff52ec7 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -374,6 +374,11 @@ static bool application_regular_db_exists(const char *project) { return stat(path, &status) == 0 && S_ISREG(status.st_mode); } +static bool application_canonical_directory_exists(const char *path) { + cbm_path_info_t info = {0}; + return cbm_path_info_utf8(path, &info) == 0 && info.is_directory; +} + static cbm_daemon_application_watch_t *application_find_watch_locked( cbm_daemon_application_t *application, const char *project) { for (cbm_daemon_application_watch_t *watch = application->watches; watch; watch = watch->next) { @@ -2354,9 +2359,7 @@ static cbm_daemon_runtime_application_status_t application_set_context( if (canonical && allowed_present) { canonical = cbm_canonical_path(allowed, canonical_allowed, sizeof(canonical_allowed)); } - struct stat root_status; - canonical = - canonical && stat(canonical_root, &root_status) == 0 && S_ISDIR(root_status.st_mode); + canonical = canonical && application_canonical_directory_exists(canonical_root); bool set = canonical && cbm_mcp_server_set_session_context(session->mcp, canonical_root, allowed_present ? canonical_allowed : NULL); @@ -3369,9 +3372,8 @@ static int application_background_index(cbm_daemon_application_t *application, return -1; } char canonical_root[APPLICATION_PATH_CAP]; - struct stat root_status; if (!cbm_canonical_path(root_path, canonical_root, sizeof(canonical_root)) || - stat(canonical_root, &root_status) != 0 || !S_ISDIR(root_status.st_mode)) { + !application_canonical_directory_exists(canonical_root)) { return -1; } yyjson_mut_doc *document = yyjson_mut_doc_new(NULL); diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index e0ef6b459..fceeb7130 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -714,6 +714,45 @@ TEST(daemon_application_requires_immutable_explicit_context) { PASS(); } +TEST(daemon_application_accepts_utf8_session_context_root) { + char root[APP_TEST_PATH_CAP]; + snprintf(root, sizeof(root), "%s/cbm-app-context-caf\xC3\xA9-XXXXXX", cbm_tmpdir()); + bool root_ok = cbm_mkdtemp(root) != NULL; + cbm_daemon_application_t *application = cbm_daemon_application_new(NULL); + cbm_daemon_runtime_application_callbacks_t callbacks = + cbm_daemon_application_runtime_callbacks(application); + cbm_daemon_runtime_application_session_t *session = app_test_open(&callbacks, 311); + uint8_t *context = NULL; + uint32_t context_length = 0; + uint8_t *response = NULL; + uint32_t response_length = 0; + bool context_ok = root_ok && app_test_context_request(root, root, &context, &context_length); + cbm_daemon_runtime_application_status_t status = + context_ok ? app_test_request(&callbacks, session, context, context_length, &response, + &response_length) + : CBM_DAEMON_RUNTIME_APPLICATION_TRANSPORT_ERROR; + + free(context); + free(response); + if (session) { + callbacks.session_close(callbacks.context, session); + } + bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + cbm_daemon_application_free(application); + if (root_ok) { + (void)cbm_rmdir(root); + } + + ASSERT_TRUE(root_ok); + ASSERT_NOT_NULL(application); + ASSERT_NOT_NULL(session); + ASSERT_TRUE(context_ok); + ASSERT_EQ(status, CBM_DAEMON_RUNTIME_APPLICATION_OK); + ASSERT_EQ(response_length, 0); + ASSERT_TRUE(stopped); + PASS(); +} + TEST(daemon_application_mcp_notification_has_no_response) { cbm_daemon_application_t *application = cbm_daemon_application_new(NULL); cbm_daemon_runtime_application_callbacks_t callbacks = @@ -5348,6 +5387,7 @@ SUITE(daemon_application) { RUN_TEST(daemon_application_new_session_does_not_retain_initial_store); RUN_TEST(daemon_application_request_cancel_is_scoped_to_exact_token); RUN_TEST(daemon_application_requires_immutable_explicit_context); + RUN_TEST(daemon_application_accepts_utf8_session_context_root); RUN_TEST(daemon_application_ui_config_updates_are_masked_and_serialized); RUN_TEST(daemon_application_ui_config_rejects_noncanonical_frames); RUN_TEST(daemon_application_ui_readiness_proof_is_generation_bound_before_context); diff --git a/tests/windows/test_non_ascii_path.py b/tests/windows/test_non_ascii_path.py index bd7de22a4..bb181688f 100644 --- a/tests/windows/test_non_ascii_path.py +++ b/tests/windows/test_non_ascii_path.py @@ -358,9 +358,9 @@ def no_project_error(index_txt, repo, cache): def index_and_count(binary, repo, cache): - """Index `repo` into an isolated cache and return label-resolved counts.""" + """Start in and index `repo`, then return label-resolved counts.""" os.makedirs(cache, exist_ok=True) - with McpServer(binary, cache_dir=cache) as s: + with McpServer(binary, cache_dir=cache, cwd=repo) as s: s.initialize() resp = s.call_tool("index_repository", {"repo_path": repo}, timeout=180) index_txt, err = s.tool_text(resp) From 25a7aacc87250bff752ff61cd33e9236be5d1310 Mon Sep 17 00:00:00 2001 From: Joshua Richter Date: Fri, 28 Aug 2026 08:48:57 -0400 Subject: [PATCH 11/22] fix(cypher): refuse a query the parser did not fully read cbm_parse built a query and returned success without checking that it had read every token. The grammar accepts at most one WITH and treats RETURN as optional, so the parser stopped at the first thing it did not understand and reported success anyway. The dropped tail took the filter and the RETURN with it. The engine then answered from the fragment it had parsed, using its default projection. It reported success and returned wrong rows, which is worse than a refusal, because nothing tells the caller to look. Two shapes hit this: MATCH (f:Function) WHERE f.name = 'x' RETURN f.name AS n BANANA SPLIT 99 -> one row, no error, the trailing words silently dropped MATCH (f:Function) OPTIONAL MATCH (a)-[:CALLS]->(f) WITH f, count(a) AS calls OPTIONAL MATCH (b)-[:USAGE]->(f) WITH f, calls, count(b) AS usages WHERE calls = 0 AND usages = 0 RETURN f.name AS n -> every function, unfiltered, under the default column names cbm_parse now checks that the cursor sits on the end-of-input token before it returns success, and names the leftover token when it does not. The message adds a note about the one-WITH limit, but only when a standalone WITH really sits in the part that went unread. Appending it every time pointed the reader at WITH when the problem was a typo -- the BANANA query above holds no WITH anywhere. The test cannot simply look at the token the parse stopped on either: on the second-WITH query above it stops at OPTIONAL rather than at WITH, because parse_post_where consumes the first WITH and then has no way to take another MATCH stage. So it scans the unread tail, and skips a WITH straight after STARTS, which is the STARTS WITH operator rather than a clause. The check exposed a second defect. parse_post_where parses the branch after UNION by calling cbm_parse on a slice of the same tokens, using a separate parser. The outer parser's cursor never moved past the UNION keyword, so a valid UNION query looked unfinished. That cursor was already wrong; nothing caught it, because nothing checked where the cursor ended up. The UNION branch now moves the cursor to the end after the sub-parse succeeds, which is sound because that sub-parse no longer returns success with tokens left over. Three tests cover both directions. Removing only the guard turns the two rejection tests red with rc == 0 -- the parser reporting success on input it never finished reading -- while the acceptance test stays green, so the guard is load-bearing for exactly these two behaviours. The two rejection tests also pin the message. The BANANA case must name BANANA and must not mention WITH; that assertion was seen red first, at tests/test_cypher.c:226. The second-WITH case must still carry the note; that one passed before and after, and is the control that stops the note being suppressed everywhere instead of only where it misleads. Verified on macOS with Apple clang: make -f Makefile.cbm test-focused TEST_SUITES=cypher -> 186 passed guard removed, tests kept -> 184 passed, 2 failed make -f Makefile.cbm cbm -> exit 0, no warnings The full suite reports 7623 passed, 2 failed. Both failures are in tests/test_cli.c (lines 1748 and 6723) and reproduce on a clean tree with this change stashed out. They depend on the coding agents installed on the machine, not on this change. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Joshua Richter --- src/cypher/cypher.c | 37 ++++++++++++++++++++++++ tests/test_cypher.c | 68 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/src/cypher/cypher.c b/src/cypher/cypher.c index 36c07ea4c..17840b04c 100644 --- a/src/cypher/cypher.c +++ b/src/cypher/cypher.c @@ -1992,6 +1992,13 @@ static int parse_post_where(parser_t *p, cbm_query_t *q, // NOLINT(misc-no-recur q->union_next = sub.query; sub.query = NULL; cbm_parse_free(&sub); + /* The branch after UNION was parsed by a SEPARATE parser over a slice + * of these tokens, so this parser's cursor never moved past the UNION + * keyword. That sub-parse now refuses to succeed with anything left + * over, so everything from here to the end is accounted for. Move the + * cursor to the end to say so, or cbm_parse's end-of-input check reads + * a fully parsed UNION query as unfinished. */ + p->pos = p->count; } return 0; } @@ -2055,6 +2062,36 @@ int cbm_parse(const cbm_token_t *tokens, int token_count, // NOLINT(misc-no-recu return CBM_NOT_FOUND; } + /* Every token must be consumed. The grammar accepts at most one WITH and + * treats RETURN as optional, so a query with a second WITH stage — or any + * typo after RETURN — used to stop parsing there and succeed anyway. The + * dropped tail took the filter and the RETURN with it, and the engine + * answered from the fragment it had parsed, using its default projection. + * That reported success and returned wrong rows, which is worse than a + * refusal because nothing tells the caller to look. Refuse instead. */ + if (peek(&p)->type != TOK_EOF) { + /* Only mention the one-WITH limit when a standalone WITH really is + * sitting in the part we could not read. Saying it every time points + * a reader at WITH when the problem is a typo. A WITH straight after + * STARTS is the STARTS WITH operator rather than a clause, the same + * guard parse_post_where uses. */ + const char *hint = ""; + for (int i = p.pos; i < p.count; i++) { + if (p.tokens[i].type == TOK_WITH && + (i == 0 || p.tokens[i - SKIP_ONE].type != TOK_STARTS)) { + hint = " Note that only one WITH clause is supported."; + break; + } + } + snprintf(p.error, sizeof(p.error), + "unexpected input at pos %d ('%s') — the query was not fully " + "parsed.%s", + peek(&p)->pos, peek(&p)->text ? peek(&p)->text : "", hint); + out->error = heap_strdup(p.error); + cbm_query_free(q); + return CBM_NOT_FOUND; + } + out->query = q; return 0; } diff --git a/tests/test_cypher.c b/tests/test_cypher.c index 7882be82f..b674da15d 100644 --- a/tests/test_cypher.c +++ b/tests/test_cypher.c @@ -209,6 +209,71 @@ TEST(cypher_parse_simple_node) { PASS(); } +/* Trailing input must be an error, never a silent drop. The parser used to + * stop at the first thing it did not understand and report success, so the + * engine answered from the fragment it had parsed. */ +TEST(cypher_parse_rejects_trailing_tokens) { + cbm_query_t *q = NULL; + char *err = NULL; + int rc = cbm_cypher_parse("MATCH (f:Function) RETURN f.name AS n BANANA SPLIT 99", &q, &err); + ASSERT_NEQ(rc, 0); + ASSERT_NOT_NULL(err); + ASSERT_NULL(q); + + /* The message must name what actually stopped the parse, and must not + * mention WITH: this query has no WITH in it anywhere. */ + ASSERT(strstr(err, "BANANA") != NULL); + ASSERT(strstr(err, "WITH") == NULL); + + free(err); + PASS(); +} + +/* Only one WITH is supported. A second one used to take the rest of the + * query with it — the filter and the RETURN both vanished, and every row + * came back unfiltered under the default projection. */ +TEST(cypher_parse_rejects_second_with_clause) { + cbm_query_t *q = NULL; + char *err = NULL; + int rc = cbm_cypher_parse("MATCH (f:Function) " + "OPTIONAL MATCH (a)-[:CALLS]->(f) " + "WITH f, count(a) AS calls " + "OPTIONAL MATCH (b)-[:USAGE]->(f) " + "WITH f, calls, count(b) AS usages " + "WHERE calls = 0 AND usages = 0 " + "RETURN f.name AS n", + &q, &err); + ASSERT_NEQ(rc, 0); + ASSERT_NOT_NULL(err); + ASSERT_NULL(q); + + /* Here the note earns its place. The parse stops at OPTIONAL, and the + * reason is the second WITH further along, which the reader cannot see + * from the stopping point alone. */ + ASSERT(strstr(err, "only one WITH clause is supported") != NULL); + + free(err); + PASS(); +} + +/* The guard must not reject a query that is simply finished. One WITH, a + * WHERE after it and a RETURN is the shape the grammar does support. */ +TEST(cypher_parse_accepts_single_with_clause) { + cbm_query_t *q = NULL; + char *err = NULL; + int rc = cbm_cypher_parse("MATCH (f:Function) " + "WITH f, f.name AS n " + "WHERE n = 'buildTree' " + "RETURN n", + &q, &err); + ASSERT_EQ(rc, 0); + ASSERT_NULL(err); + ASSERT_NOT_NULL(q); + + cbm_query_free(q); + PASS(); +} + TEST(cypher_parse_relationship_outbound) { cbm_query_t *q = NULL; char *err = NULL; @@ -4102,6 +4167,9 @@ SUITE(cypher) { RUN_TEST(cypher_lex_full_query); /* Parser */ RUN_TEST(cypher_parse_simple_node); + RUN_TEST(cypher_parse_rejects_trailing_tokens); + RUN_TEST(cypher_parse_rejects_second_with_clause); + RUN_TEST(cypher_parse_accepts_single_with_clause); RUN_TEST(cypher_parse_relationship_outbound); RUN_TEST(cypher_parse_relationship_inbound); RUN_TEST(cypher_parse_relationship_any); From 62b44519c68b403ea44d9a85ba8ee74cddee66e6 Mon Sep 17 00:00:00 2001 From: Joshua Richter Date: Fri, 28 Aug 2026 09:57:59 -0400 Subject: [PATCH 12/22] fix(confidence): stop an unreadable confidence from reading as 0.0 strtod answers 0.0 for text it cannot read, and 0.0 is a real confidence in both places that call it. So a property blob carrying a malformed value, such as "confidence":null, came back as a recorded confidence of zero. Two things went wrong with that. In src/graph_buffer/graph_buffer.c, edge_props_confidence answers CBM_EDGE_CONF_ABSENT (-1) when an edge carries no confidence, so any real confidence outranks it. A malformed value answered 0.0, which beats -1 in the merge comparison, so the malformed blob displaced a clean stored one. The function's own comment already promised that "absent/unparseable reads as -1". Only the absent half was true. In src/mcp/mcp.c, bfs_edge_evidence_for_hop sets the confidence to -1 and the emitter publishes any value of 0 or more as a recorded number. A malformed value printed as 0.00, which reads as "the resolver was certain this is wrong" rather than "nobody wrote a number here". Both sites now pass an end pointer to strtod and keep the absent sentinel when the pointer never moved, which is the shape src/store/store.c:402 already uses. Two tests come with the change, and both were seen failing before the fix and passing after: gbuf_edge_props_unreadable_confidence_does_not_displace_absent tool_trace_path_unreadable_confidence_reports_not_recorded Red: 258 passed, 2 failed. Green: 260 passed, 0 failed. The full suite reports 7633 passed, 2 failed. Both failures are in tests/test_cli.c (lines 1749 and 6725) and reproduce on a clean tree without this change. They depend on the coding agents installed on the machine, not on this change. make -f Makefile.cbm lint-ci passes. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Joshua Richter --- src/graph_buffer/graph_buffer.c | 13 ++++++- src/mcp/mcp.c | 11 +++++- tests/test_graph_buffer.c | 29 +++++++++++++++ tests/test_mcp.c | 64 +++++++++++++++++++++++++++++++++ 4 files changed, 115 insertions(+), 2 deletions(-) diff --git a/src/graph_buffer/graph_buffer.c b/src/graph_buffer/graph_buffer.c index b173267ff..939cc8cdb 100644 --- a/src/graph_buffer/graph_buffer.c +++ b/src/graph_buffer/graph_buffer.c @@ -1045,7 +1045,18 @@ static double edge_props_confidence(const char *props_json) { if (!p) { return CBM_EDGE_CONF_ABSENT; } - return strtod(p + sizeof(conf_key) - SKIP_ONE, NULL); + /* strtod answers 0.0 for text it cannot read, and 0.0 is a real + * confidence that beats CBM_EDGE_CONF_ABSENT. So a blob carrying + * "confidence":null used to outrank a clean blob that carries no + * confidence at all, and displace it. Ask strtod where it stopped: an + * end pointer that never moved means it read nothing. */ + const char *value = p + sizeof(conf_key) - SKIP_ONE; + char *end = NULL; + double conf = strtod(value, &end); + if (end == value) { + return CBM_EDGE_CONF_ABSENT; + } + return conf; } /* Decide whether an incoming property blob replaces the stored one on a diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 73534404a..39607d85c 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -6498,7 +6498,16 @@ static bool bfs_edge_evidence_for_hop(cbm_traverse_result_t *tr, int64_t hop_nod if (conf) { const char *colon = strchr(conf, ':'); if (colon) { - *confidence_out = strtod(colon + 1, NULL); + /* strtod answers 0.0 for text it cannot read, and the caller + * publishes any value >= 0 as a recorded confidence. So a + * malformed value used to print as 0.00 — the one number the + * surrounding code works to keep meaningful. Keep the -1 when + * the end pointer never moved: nothing was read. */ + char *end = NULL; + double parsed = strtod(colon + 1, &end); + if (end != colon + 1) { + *confidence_out = parsed; + } } } return true; diff --git a/tests/test_graph_buffer.c b/tests/test_graph_buffer.c index c4aa5c270..232286045 100644 --- a/tests/test_graph_buffer.c +++ b/tests/test_graph_buffer.c @@ -236,6 +236,34 @@ TEST(gbuf_edge_props_merge_prefers_higher_confidence) { PASS(); } +/* A confidence the code cannot read is not evidence of anything, so it must + * not outrank an edge that simply carries no confidence at all. + * + * edge_props_confidence answers -1 for "absent" so that any real confidence + * beats it. strtod answers 0.0 for text it cannot read, so an unreadable + * value used to come back as a real confidence of zero -- which beats -1 and + * displaced the stored blob. The function's own comment already promised + * that "absent/unparseable reads as -1"; only the absent half was true. */ +TEST(gbuf_edge_props_unreadable_confidence_does_not_displace_absent) { + const char *no_conf = "{\"callee\":\"f\",\"strategy\":\"lsp\"}"; + const char *bad_conf = "{\"callee\":\"f\",\"confidence\":null,\"strategy\":\"registry\"}"; + + cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp"); + int64_t a = cbm_gbuf_upsert_node(gb, "Function", "a", "pkg.a", "f.go", 1, 5, "{}"); + int64_t b = cbm_gbuf_upsert_node(gb, "Function", "b", "pkg.b", "f.go", 6, 10, "{}"); + cbm_gbuf_insert_edge(gb, a, b, "CALLS", no_conf); + cbm_gbuf_insert_edge(gb, a, b, "CALLS", bad_conf); /* unreadable, arrives last */ + + const cbm_gbuf_edge_t **edges = NULL; + int count = 0; + cbm_gbuf_find_edges_by_type(gb, "CALLS", &edges, &count); + ASSERT_EQ(count, 1); + ASSERT_TRUE(strstr(edges[0]->properties_json, "\"strategy\":\"lsp\"") != NULL); + + cbm_gbuf_free(gb); + PASS(); +} + /* An empty incoming blob must never displace real stored properties. */ TEST(gbuf_edge_props_merge_keeps_existing_on_empty) { const char *lsp = "{\"callee\":\"f\",\"confidence\":0.95,\"strategy\":\"lsp\"}"; @@ -1152,6 +1180,7 @@ SUITE(graph_buffer) { /* Edge property merge determinism */ RUN_TEST(gbuf_edge_props_merge_is_order_independent); RUN_TEST(gbuf_edge_props_merge_prefers_higher_confidence); + RUN_TEST(gbuf_edge_props_unreadable_confidence_does_not_displace_absent); RUN_TEST(gbuf_edge_props_merge_keeps_existing_on_empty); /* Shared ID tests */ diff --git a/tests/test_mcp.c b/tests/test_mcp.c index e8722ffc1..64f128820 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -3629,6 +3629,69 @@ TEST(trace_evidence_strategy_class_vocabulary_is_closed) { PASS(); } +/* A confidence the code cannot read must be reported as "not recorded", not + * as a recorded zero. + * + * The emitter reserves ev_conf < 0 for "no confidence on this edge" and + * prints "-" (text) or null (json). The reader set ev_conf with + * strtod(colon + 1, NULL), and strtod answers 0.0 for text it cannot read -- + * so a malformed value passed the ev_conf >= 0.0 test and printed 0.00, the + * one value the surrounding code goes out of its way to keep meaningful. + * A caller then cannot tell "the resolver was certain this is wrong" from + * "nobody wrote a number here". */ +TEST(tool_trace_path_unreadable_confidence_reports_not_recorded) { + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + cbm_store_t *st = cbm_mcp_server_store(srv); + const char *proj = "badconf-proj"; + cbm_mcp_server_set_project(srv, proj); + cbm_store_upsert_project(st, proj, "/tmp/badconf"); + cbm_node_t caller = {.project = proj, + .label = "Function", + .name = "caller", + .qualified_name = "badconf-proj.src.caller", + .file_path = "src/a.c", + .start_line = 1, + .end_line = 5}; + cbm_node_t callee = {.project = proj, + .label = "Function", + .name = "target", + .qualified_name = "badconf-proj.src.target", + .file_path = "src/a.c", + .start_line = 10, + .end_line = 20}; + int64_t id_caller = cbm_store_upsert_node(st, &caller); + int64_t id_callee = cbm_store_upsert_node(st, &callee); + ASSERT_GT(id_caller, 0); + ASSERT_GT(id_callee, 0); + /* The strategy reads fine; only the confidence is malformed. */ + cbm_edge_t e = {.project = proj, + .source_id = id_caller, + .target_id = id_callee, + .type = "CALLS", + .properties_json = "{\"callee\":\"target\",\"confidence\":null," + "\"strategy\":\"lsp_trait_dispatch\",\"candidates\":1}"}; + ASSERT_GT(cbm_store_insert_edge(st, &e), 0); + + char *ev = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":93,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"trace_path\",\"arguments\":{\"function_name\":\"caller\"," + "\"project\":\"badconf-proj\",\"direction\":\"outbound\",\"include_evidence\":true}}}"); + ASSERT_NOT_NULL(ev); + char *ev_txt = extract_text_content(ev); + ASSERT_NOT_NULL(ev_txt); + /* Positive controls: the hop and its readable class still come through, so + * a failure below is about the confidence and not a broken request. */ + ASSERT_NOT_NULL(strstr(ev_txt, "target")); + ASSERT_NOT_NULL(strstr(ev_txt, "lsp")); + /* The claim: an unreadable confidence is never published as 0.00. */ + ASSERT_NULL(strstr(ev_txt, "0.00")); + free(ev_txt); + free(ev); + + cbm_mcp_server_free(srv); + PASS(); +} + /* Distilled from #559 (@vvenegasv). The indexer already records * {strategy, confidence} on every CALLS edge (pass_calls.c:355) and the store * reads it back, but no tool ever surfaced it — an agent could see THAT A->B @@ -13870,6 +13933,7 @@ SUITE(mcp) { RUN_TEST(trace_evidence_strategy_class_vocabulary_is_closed); RUN_TEST(tool_trace_path_evidence_is_opt_in_and_class_mapped); RUN_TEST(tool_trace_path_evidence_columns_match_header_issue1542); + RUN_TEST(tool_trace_path_unreadable_confidence_reports_not_recorded); RUN_TEST(tool_trace_call_path_depth_clamped); RUN_TEST(tool_trace_call_path_distinct_defs_not_over_unioned); RUN_TEST(tool_trace_call_path_dts_stub_unions_with_impl); From 44caa4c386c578e59c5919735e1e4d6094ac1af7 Mon Sep 17 00:00:00 2001 From: The1nk <23087600+The1nk@users.noreply.github.com> Date: Sat, 15 Aug 2026 19:43:35 -0400 Subject: [PATCH 13/22] feat(discover): map .razor to C# so Blazor components are discovered A .razor file had no entry in EXT_TABLE, so discovery skipped it entirely and a Blazor application produced no graph nodes for any of its components. The only way in was an undocumented extra_extensions entry in a per-project .codebase-memory.json. Map .razor to CBM_LANG_CSHARP. This is best-effort by design: the C# grammar recovers the @code block, while the surrounding markup lands in ERROR regions and is reported through parse_partial. That is strictly more than the Module-and-imports the other markup-hosted languages (Vue, Svelte, Astro) extract today, and it needs no new grammar. Covered by tests/test_language.c:lang_ext_razor. Co-Authored-By: Claude Opus 5 Signed-off-by: The1nk <23087600+The1nk@users.noreply.github.com> --- src/discover/language.c | 5 +++++ tests/test_language.c | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/src/discover/language.c b/src/discover/language.c index 666db536f..a9b49b863 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -50,6 +50,11 @@ static const ext_entry_t EXT_TABLE[] = { /* C# */ {".cs", CBM_LANG_CSHARP}, + /* Blazor components. The C# grammar recovers the @code block; the + * surrounding markup parses as ERROR regions and is reported via + * parse_partial, which is why this is a best-effort mapping rather + * than a dedicated grammar. */ + {".razor", CBM_LANG_CSHARP}, /* Clojure */ {".clj", CBM_LANG_CLOJURE}, diff --git a/tests/test_language.c b/tests/test_language.c index 2c8d6576b..1bcd5e040 100644 --- a/tests/test_language.c +++ b/tests/test_language.c @@ -89,6 +89,14 @@ TEST(lang_ext_csharp) { ASSERT_EQ(cbm_language_for_extension(".cs"), CBM_LANG_CSHARP); PASS(); } +/* Blazor components were unmapped, so a .razor file was never discovered at + * all: indexing a Blazor app produced no nodes for any component, and reaching + * them required an undocumented extra_extensions entry in a per-project + * .codebase-memory.json. */ +TEST(lang_ext_razor) { + ASSERT_EQ(cbm_language_for_extension(".razor"), CBM_LANG_CSHARP); + PASS(); +} TEST(lang_ext_php) { ASSERT_EQ(cbm_language_for_extension(".php"), CBM_LANG_PHP); PASS(); @@ -1197,6 +1205,7 @@ SUITE(language) { RUN_TEST(lang_ext_h); RUN_TEST(lang_ext_ixx); RUN_TEST(lang_ext_csharp); + RUN_TEST(lang_ext_razor); RUN_TEST(lang_ext_php); RUN_TEST(lang_ext_lua); RUN_TEST(lang_ext_scala); From e8204092333ee1c38cd681a81abb36f125288894 Mon Sep 17 00:00:00 2001 From: Joshua Richter Date: Tue, 1 Sep 2026 14:06:41 -0400 Subject: [PATCH 14/22] docs(vendored): record the Swift scanner shift patch in MANIFEST.md The vendoring table claims our copy of a grammar matches the pinned upstream commit. A local patch makes that claim false unless the patch is written down, so a future re-vendor would drop the fix without anyone noticing. The row covers both changed lines, because both are the same defect in the same file: OP_SYMBOL_SUPPRESSOR at line 131 and eat_operators at line 514. The row also names the two upstream commits that already carry these changes, and the table intro gains one clause for the case. Upstream fixed eat_operators in fb63a7004f07 on 2026-04-06 (their issue #558) and the OP_SYMBOL_SUPPRESSOR entry in 6ab8d1d74ebd on 2026-08-10. Our pin 8abb3e8b3325 is from 2026-03-20 and predates both, so these are backports rather than local inventions. That distinction changes what a re-vendor should do. The table's instruction is "must re-apply these", which is right for the other four rows -- upstream never took those. For this row it would mean hand-applying a change the newer file already has. The row now says to delete it instead, and the intro allows for that case. MANIFEST.md is itself listed in scripts/vendored-checksums.txt, so its own recorded checksum moves with the edit. Refs #1892 Signed-off-by: Joshua Richter --- internal/cbm/vendored/grammars/MANIFEST.md | 5 ++++- scripts/vendored-checksums.txt | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/cbm/vendored/grammars/MANIFEST.md b/internal/cbm/vendored/grammars/MANIFEST.md index 7dd9688b4..4025b5f50 100644 --- a/internal/cbm/vendored/grammars/MANIFEST.md +++ b/internal/cbm/vendored/grammars/MANIFEST.md @@ -77,7 +77,9 @@ Guarded by the `contract_all_grammars_in_graph` graph-breadth test in The grammars below carry a small local patch to their vendored sources, on top of the pinned upstream commit recorded in the vendoring table below. -Re-vendoring from upstream must re-apply these. +Re-vendoring from upstream must re-apply these, unless the reason column +names an upstream commit that already carries the change — then drop the +row instead. | grammar | location | patch | reason | |---|---|---|---| @@ -85,6 +87,7 @@ Re-vendoring from upstream must re-apply these. | rescript | `rescript/scanner.c`, deserialize | guard `memcpy(state, buffer, n_bytes)` with `if (n_bytes > 0)` | UBSan: zero-length `memcpy` with a NULL `buffer` / `n_bytes == 0` on empty-state deserialize (formal UB, harmless). The sibling serialize copies a fixed `sizeof(ScannerState)` (always > 0, non-NULL src) and needs no guard. | | purescript | `purescript/scanner.c`, serialize | guard `memcpy(buffer, indents->data, to_copy)` with `if (to_copy > 0)` | UBSan: zero-length `memcpy` with a NULL/0-size source when the indent vector is empty (formal UB, harmless) | | plsql | `plsql/parser.c`, include | `#include ` → `#include "tree_sitter/parser.h"` | The older ABI-14 generator emits angle brackets; every other vendored grammar uses the quoted form, which resolves the per-grammar `tree_sitter/` header from the including file's directory | +| swift | `swift/scanner.c`, `OP_SYMBOL_SUPPRESSOR` + `eat_operators` | `1UL <<` / `1 <<` → `1ULL <<` | UBSan: `1 << suppressor` shifts an `int` by up to `TOKEN_COUNT` bits, undefined once the index reaches 31, while the mask it feeds is `uint64_t`. `1UL << FAKE_TRY_BANG` is the same defect on Windows, where `unsigned long` is 32 bits and `FAKE_TRY_BANG` is 32; the CLANGARM64 leg runs UBSan in trap mode, so there it is an illegal instruction rather than a log line. Upstream already carries both changes: `fb63a7004f07` (2026-04-06, upstream #558) for `eat_operators`, `6ab8d1d74ebd` (2026-08-10) for the `OP_SYMBOL_SUPPRESSOR` entry. Our pin `8abb3e8b3325` (2026-03-20) predates both, so this is a backport rather than a local invention — a re-vendor past 2026-08-10 should delete this row, not re-apply it | ## Vendored from verified upstream diff --git a/scripts/vendored-checksums.txt b/scripts/vendored-checksums.txt index a54ab6440..c4b809dc6 100644 --- a/scripts/vendored-checksums.txt +++ b/scripts/vendored-checksums.txt @@ -5,7 +5,7 @@ c5cfb43042b6b72045f4ba997834d0a7786d2793d91680868b5815b39f14fc78 internal/cbm/v b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea internal/cbm/vendored/common/tree_sitter/alloc.h 31e60a1bff6f715afacce03b5b70efe42b58371b4f9595dd4af52a577ff9608c internal/cbm/vendored/common/tree_sitter/array.h 180b893c8734778fd32f372dfbc27bd6ad1cd2221f26150b31256ff6716320d2 internal/cbm/vendored/common/tree_sitter/parser.h -e1233d81f21d868b8f2d9e8fca10e15f2ec3ae4db4a7ba0301161cc011deef65 internal/cbm/vendored/grammars/MANIFEST.md +00a84162eb34fb58bcdd38c08922d4fea4c87b7c2219dd64b773dd15ba18de71 internal/cbm/vendored/grammars/MANIFEST.md ad8425038de519f8c4e3e9339feebf99dfad8a6002dcf79227d91402780a32cc internal/cbm/vendored/grammars/ada/LICENSE 02805ec13939b749c891567be36bf024b09034e04c80683a1ae667272458d549 internal/cbm/vendored/grammars/ada/parser.c 115a75d000bef9c70c4de7dfbb7f2a80b90cbb14ba265056dc7abb1ef3b9b6db internal/cbm/vendored/grammars/ada/tree_sitter/alloc.h From fd73c347fbcfd7f66c210577b4d713fc6ce7780b Mon Sep 17 00:00:00 2001 From: The1nk <23087600+The1nk@users.noreply.github.com> Date: Sat, 15 Aug 2026 19:43:35 -0400 Subject: [PATCH 15/22] feat(extract): emit Route nodes for Blazor @page directives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_architecture's routes aspect returned nothing for a Blazor application: the entry surface of the app was invisible in the graph even though every @page directive names one. @page lives in markup above the @code block, which tree-sitter's C# grammar never parses, so there is no AST node to read it from. Scan the raw source for it instead. The match is deliberately strict — the directive must be the first token on its line, followed by whitespace and a double-quoted path beginning with '/' — so @pageSize and prose mentions cannot match. The route is attached to the file's module def. A .razor component's class is implicit, so there is no class node to carry it, and the module QN already is the component's identity. insert_def_into_gbuf is label-agnostic and creates Route + HANDLES from route_path. pass_route_nodes.c's ensure_decorator_routes gains "Module" for the same reason. Extraction covers the full-index path on its own; this backstop is what runs on an incremental re-index, so without it a component's Route would appear on a full index and vanish the next time that one file changed. Its loop bound now derives from the labels array rather than borrowing the unrelated RN_STRIP_PASSES, so adding a label cannot silently skip it. Verified no double-creation when both paths can fire. Measured on a real Blazor application (36 .razor files): routes went from 0 to 11, with 11 matching HANDLES edges, and no change to the 363 Class / 1775 Method / 3805 CALLS already extracted from its C#. Covered by two new tests in tests/test_extraction.c. Co-Authored-By: Claude Opus 5 Signed-off-by: The1nk <23087600+The1nk@users.noreply.github.com> --- internal/cbm/extract_defs.c | 94 +++++++++++++++++++++++++++++++++ src/pipeline/pass_route_nodes.c | 13 ++++- tests/test_extraction.c | 81 ++++++++++++++++++++++++---- 3 files changed, 176 insertions(+), 12 deletions(-) diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 2e359af67..7ec967dc6 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -7902,6 +7902,89 @@ void cbm_extract_definitions_without_module(CBMExtractCtx *ctx) { extract_variables(ctx, ctx->root, spec); } +/* True when rel_path names a Blazor component file. */ +static bool cbm_path_is_razor(const char *rel_path) { + if (!rel_path) { + return false; + } + size_t len = strlen(rel_path); + static const char suffix[] = ".razor"; + size_t slen = sizeof(suffix) - 1U; + return len > slen && strcmp(rel_path + (len - slen), suffix) == 0; +} + +/* Match `@page "/route"` on ONE line; returns the route text or NULL. + * + * Deliberately strict: the directive must be the first token on the line and be + * followed by whitespace and a double-quoted path beginning with '/', so + * neither `@pageSize` nor a `@page` mentioned in markup prose can match. + * + * A blank line is rejected up front rather than falling through the length + * check, which keeps every later comparison reachable on some path — the + * all-whitespace case would otherwise leave `line_end - p` provably zero. */ +static const char *razor_page_route_on_line(CBMArena *a, const char *line, const char *line_end) { + static const char directive[] = "@page"; + const size_t dlen = sizeof(directive) - 1U; + + const char *p = line; + while (p < line_end && (*p == ' ' || *p == '\t')) { + p++; + } + if (p == line_end) { + return NULL; /* blank line — nothing can follow */ + } + if ((size_t)(line_end - p) <= dlen || strncmp(p, directive, dlen) != 0) { + return NULL; + } + p += dlen; + if (*p != ' ' && *p != '\t') { + return NULL; /* `@pageSize` and friends */ + } + while (p < line_end && (*p == ' ' || *p == '\t')) { + p++; + } + if (p == line_end || *p != '"') { + return NULL; + } + p++; + const char *route = p; + while (p < line_end && *p != '"') { + p++; + } + if (p == line_end || p == route || *route != '/') { + return NULL; /* unterminated, empty, or not a rooted path */ + } + return cbm_arena_strndup(a, route, (size_t)(p - route)); +} + +/* Blazor route directive: `@page "/counter"` lives in MARKUP above the `@code` + * block. Tree-sitter's C# grammar recovers `@code` but never parses the + * directive, so there is no AST node to read it from — this scans the raw + * source instead. That is why routes need no Razor grammar. + * + * A component may declare several routes; the first is taken, because + * CBMDefinition carries a single route_path. */ +static const char *cbm_razor_page_route(CBMArena *a, const char *source, int source_len) { + if (!source || source_len <= 0) { + return NULL; + } + const char *end = source + source_len; + const char *line = source; + + while (line < end) { + const char *nl = memchr(line, '\n', (size_t)(end - line)); + const char *route = razor_page_route_on_line(a, line, nl ? nl : end); + if (route) { + return route; + } + if (!nl) { + break; + } + line = nl + 1; + } + return NULL; +} + void cbm_extract_definitions(CBMExtractCtx *ctx) { const CBMLangSpec *spec = cbm_lang_spec(ctx->language); if (!spec) { @@ -7923,6 +8006,17 @@ void cbm_extract_definitions(CBMExtractCtx *ctx) { mod.is_test = ctx->result->is_test_file; // #519: index what a config file declares itself to be, not only its path. mod.docstring = extract_config_module_description(ctx); + /* A routable Blazor component carries its route on the module def: the + * component's class is implicit in a .razor file, so there is no class node + * to hang it on, and the module QN already is the component's identity. + * insert_def_into_gbuf creates Route+HANDLES for any def with route_path. */ + if (ctx->language == CBM_LANG_CSHARP && cbm_path_is_razor(ctx->rel_path)) { + const char *route = cbm_razor_page_route(a, ctx->source, ctx->source_len); + if (route) { + mod.route_path = route; + mod.route_method = "GET"; /* a routable page is reached by navigation */ + } + } cbm_defs_push(&ctx->result->defs, a, mod); cbm_extract_definitions_without_module(ctx); diff --git a/src/pipeline/pass_route_nodes.c b/src/pipeline/pass_route_nodes.c index 163e5a1fc..b1fac1670 100644 --- a/src/pipeline/pass_route_nodes.c +++ b/src/pipeline/pass_route_nodes.c @@ -480,10 +480,19 @@ static int ensure_one_decorator_route(cbm_gbuf_t *gb, const cbm_gbuf_node_t *fun /* Phase 2a: Ensure all functions with route_path properties have Route+HANDLES edges. */ static void ensure_decorator_routes(cbm_gbuf_t *gb) { - const char *labels[] = {"Function", "Method"}; + /* "Module" is here for Blazor: a .razor component's class is implicit, so + * its @page route is carried by the file's Module def. Extraction's own + * insert_def_into_gbuf is label-agnostic and creates the Route either way — + * this backstop is what runs on an INCREMENTAL re-index, so leaving Module + * out would make a component's Route appear on a full index and vanish the + * next time that one file changed. + * Bound comes from the array, not the unrelated RN_STRIP_PASSES it used to + * borrow, so adding a label cannot silently skip it. */ + const char *labels[] = {"Function", "Method", "Module"}; + const int label_count = (int)(sizeof(labels) / sizeof(labels[0])); int created = 0; - for (int li = 0; li < RN_STRIP_PASSES; li++) { + for (int li = 0; li < label_count; li++) { const cbm_gbuf_node_t **nodes = NULL; int count = 0; if (cbm_gbuf_find_by_label(gb, labels[li], &nodes, &count) != 0) { diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 5b8d16f61..eb38e3421 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -3906,6 +3906,75 @@ TEST(extract_java_jaxrs_path_composition_issue1005) { PASS(); } +/* Return the file's Module definition (extraction pushes it first), or NULL. */ +static const CBMDefinition *find_module_def(CBMFileResult *r) { + for (int i = 0; i < r->defs.count; i++) { + if (r->defs.items[i].label && strcmp(r->defs.items[i].label, "Module") == 0) { + return &r->defs.items[i]; + } + } + return NULL; +} + +/* Blazor: a routable component declares its route with a `@page` directive in + * MARKUP, above the `@code` block. The C# grammar recovers `@code` (that is why + * .razor already yields methods via extra_extensions) but never sees the + * directive, so a routable page contributes no Route node and + * get_architecture(routes) is empty for a whole Blazor app. + * + * The route hangs off the file's Module definition, not off a class: a .razor + * component's class is implicit — it is never written in the source — so there + * is no class node to carry it. The Module's qualified name already IS the + * component's identity (t.Pages.Counter), and insert_def_into_gbuf creates + * Route+HANDLES for any definition carrying route_path, whatever its label. */ +TEST(extract_blazor_page_directive_routes_component) { + CBMFileResult *r = extract("@page \"/counter\"\n" + "@inject NavigationManager Nav\n" + "\n" + "

Counter

\n" + "\n" + "\n" + "@code {\n" + " private int count;\n" + " private void Increment() { count++; }\n" + "}\n", + CBM_LANG_CSHARP, "t", "Pages/Counter.razor"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + /* The markup must not cost us the @code block we already extract today. */ + ASSERT_NOT_NULL(find_def_by_name(r, "Increment")); + const CBMDefinition *mod = find_module_def(r); + ASSERT_NOT_NULL(mod); + ASSERT_NOT_NULL(mod->route_path); + ASSERT_STR_EQ(mod->route_path, "/counter"); + /* A routable Blazor page is reached by navigation, i.e. GET. */ + ASSERT_NOT_NULL(mod->route_method); + ASSERT_STR_EQ(mod->route_method, "GET"); + cbm_free_result(r); + PASS(); +} + +/* The directive scan must not fire on every .razor file. A non-routable + * component (no @page) has to stay route-free, or every shared component in the + * tree becomes a bogus Route node. */ +TEST(extract_blazor_component_without_page_has_no_route) { + CBMFileResult *r = extract("@inject IJSRuntime JS\n" + "\n" + "
@Title
\n" + "\n" + "@code {\n" + " private void Refresh() { }\n" + "}\n", + CBM_LANG_CSHARP, "t", "Shared/Card.razor"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + const CBMDefinition *mod = find_module_def(r); + ASSERT_NOT_NULL(mod); + ASSERT_NULL(mod->route_path); + cbm_free_result(r); + PASS(); +} + /* A comment between decorators must not drop the decorators above it. * Comments are NAMED tree-sitter nodes, so the prev-sibling walk used to stop * at one — a documented route (@Post + @HttpCode above an explanatory comment) @@ -6291,16 +6360,6 @@ TEST(iris_export_xml_multi_class) { * question asked in words could not reach either. Both now carry the prose in * `docstring`, which is what nodes_fts indexes into its `body` column. */ -/* First Module-labelled definition, or NULL. */ -static const CBMDefinition *find_module_def(CBMFileResult *r) { - for (int i = 0; i < r->defs.count; i++) { - if (strcmp(r->defs.items[i].label, "Module") == 0) { - return &r->defs.items[i]; - } - } - return NULL; -} - TEST(markdown_section_body_becomes_docstring_issue518) { CBMFileResult *r = extract("# Installation\n" "Run the bootstrap script to provision a workstation.\n" @@ -6808,6 +6867,8 @@ SUITE(extraction) { RUN_TEST(arkts_lazy_import); RUN_TEST(arkts_ts_compat); RUN_TEST(extract_java_jaxrs_path_composition_issue1005); + RUN_TEST(extract_blazor_page_directive_routes_component); + RUN_TEST(extract_blazor_component_without_page_has_no_route); RUN_TEST(extract_ts_template_string_url_issue1006); RUN_TEST(extract_go_binary_concat_url_issue1249); RUN_TEST(extract_go_binary_concat_url_no_literal_suffix_issue1249); From c36b4fbc446f9085704a6073b81ec743fd4180fc Mon Sep 17 00:00:00 2001 From: Joshua Richter Date: Tue, 1 Sep 2026 22:30:54 -0400 Subject: [PATCH 16/22] fix(routes): find a route path wherever it sits in a decorator's arguments Java and Kotlin put no order on annotation attributes, so `path` can appear after `method`, `produces` and `consumes`: @RequestMapping(method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE, path = "/orders") extract_route_path_from_args stopped after the third argument, so the path was never read and no Route node formed. The endpoint was in the source and not in the graph. Four call sites read this function, so the miss covered plain decorator arguments, Spring mappings, and both JAX-RS paths. The loop now checks every argument. That matches find_drf_kwarg_in_args a few lines below, which walks the same kind of node with no cap. Cost stays small: the loop returns on the first path-shaped string, and each argument's own subtree walk is still bounded by find_route_path_literal. The recursive breadth guard inside find_route_path_literal keeps DECORATOR_SCAN_LIMIT. That loop descends to CBM_DESCENDANT_MAX_DEPTH, so breadth 3 by depth 6 is 729 visits at worst and unbounded breadth there is not bounded at all. The test fails without the change, reporting the class-level "/api" route present and "/api/orders" missing. Signed-off-by: Joshua Richter --- internal/cbm/extract_defs.c | 6 +++++- tests/test_edge_types_probe.c | 25 +++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 2e359af67..2de509833 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -1450,8 +1450,12 @@ static const char *find_route_path_literal(CBMArena *a, TSNode node, const char // Extract route path from decorator arguments (first string that starts with /). static const char *extract_route_path_from_args(CBMArena *a, TSNode args, const char *source) { + /* Every argument is checked. Java and Kotlin put no order on annotation + * attributes, so `path` can sit anywhere in the list. Stopping early left + * a real route unread and formed no Route node. Each argument's own + * subtree walk stays bounded by find_route_path_literal below. */ uint32_t nc = ts_node_named_child_count(args); - for (uint32_t ai = 0; ai < nc && ai < DECORATOR_SCAN_LIMIT; ai++) { + for (uint32_t ai = 0; ai < nc; ai++) { TSNode arg = ts_node_named_child(args, ai); /* Spring/Kotlin frequently uses named or array-valued annotation args: * @RequestMapping(value = ["/internal/v1"]) diff --git a/tests/test_edge_types_probe.c b/tests/test_edge_types_probe.c index f71091ae4..84d7713ee 100644 --- a/tests/test_edge_types_probe.c +++ b/tests/test_edge_types_probe.c @@ -416,6 +416,30 @@ TEST(handles_spring_java) { PASS(); } +/* Spring (Java) — the path attribute may sit anywhere in the annotation. + * Java puts no order on annotation attributes, so `path` after `name`, + * `produces` and `consumes` is ordinary source. The argument scan stopped + * after the third attribute, so the path was never read and no Route node + * formed. A HANDLES count alone cannot catch that, because the class-level + * @RequestMapping still produces one route on its own. */ +TEST(handles_spring_java_path_attribute_fourth) { + static const char *routes[] = {"/api/orders", NULL}; + static const EtFile f[] = { + {"OrderController.java", + "package com.example;\n\n" + "import org.springframework.web.bind.annotation.RequestMapping;\n" + "import org.springframework.web.bind.annotation.GetMapping;\n\n" + "@RequestMapping(\"/api\")\npublic class OrderController {\n" + " @GetMapping(name = \"listOrders\",\n" + " produces = \"application/json\",\n" + " consumes = \"application/json\",\n" + " path = \"/orders\")\n" + " public String listOrders() {\n" + " return \"orders\";\n }\n}\n"}}; + ASSERT_TRUE(et_routes_exact(f, 1, routes)); + PASS(); +} + /* Spring (Kotlin) — same prefix contract, including Kotlin's named array form * for class-level RequestMapping values. */ TEST(handles_spring_kotlin) { @@ -1578,6 +1602,7 @@ SUITE(edge_types_probe) { RUN_TEST(handles_fastify_js); RUN_TEST(handles_gin_go); RUN_TEST(handles_spring_java); + RUN_TEST(handles_spring_java_path_attribute_fourth); RUN_TEST(handles_spring_kotlin); RUN_TEST(handles_jaxrs_java); RUN_TEST(handles_aspnet_csharp); From 592894a4387a50e1d128ff6a2695ff48f8cc32a5 Mon Sep 17 00:00:00 2001 From: Joshua Richter Date: Tue, 1 Sep 2026 22:31:03 -0400 Subject: [PATCH 17/22] fix(routes): take a route's handler from the last argument, not the first Express, Fastify, gin and Laravel all put middleware between the route path and the handler, and the handler comes last: routerGet("/users", requireAuth, rateLimit, listUsers); extract_handler_arg returned the FIRST argument that looked like a function reference, so requireAuth won and the HANDLES edge pointed at the middleware instead of listUsers. That is a wrong edge, not a missing one, and it misleads anyone tracing a request through the graph. The scan bound hid a second fault. An arrow function matches none of the accepted node kinds, so three inline middlewares pushed the real handler past MAX_HANDLER_SCAN and no handler came back at all. The loop now examines every argument and keeps the last eligible one. Nothing that follows a handler matches the accepted kinds either, since an options argument is an object node, so the last match is the handler. HANDLER_START_IDX stays, because argument 0 really is the path. MAX_HANDLER_SCAN had no other use and is gone. A two-argument route has one eligible argument, so first and last agree and the existing Express, Fastify, gin and Laravel route tests are unchanged. Both tests fail without the change, the first reporting "requireAuth" where "listUsers" belongs and the second reporting no handler at all. Signed-off-by: Joshua Richter --- internal/cbm/extract_calls.c | 20 +++++++++++----- tests/test_extraction.c | 45 ++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index 6bb4a32bc..c190e36b7 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -21,8 +21,6 @@ enum { LEAN_MAX_PARENT_DEPTH = 20 }; /* Max positional args to scan for URL/string. */ enum { MAX_POSITIONAL_SCAN = 3 }; -/* Max positional args to scan for handler ref. */ -enum { MAX_HANDLER_SCAN = 4 }; /* Max string arg length before rejection. */ enum { MAX_STRING_ARG_LEN = CBM_SZ_512 }; /* Min printable ASCII (space). */ @@ -2349,8 +2347,17 @@ static const char *normalize_string_handler(CBMArena *a, const char *raw) { } static const char *extract_handler_arg(CBMExtractCtx *ctx, TSNode args) { + /* The LAST eligible argument wins, and every argument is examined. + * Express, Fastify, gin and Laravel all put middleware between the route + * path and the handler, so the first function-shaped argument is usually a + * middleware. Taking the first one pointed the HANDLES edge at the + * middleware; stopping the scan early missed the handler outright when the + * middleware was written inline, because an arrow function matches none of + * the kinds below. Nothing that follows a handler matches them either — an + * options argument is an object node — so the last match is the handler. */ + const char *handler = NULL; uint32_t nc = ts_node_named_child_count(args); - for (uint32_t ai = HANDLER_START_IDX; ai < nc && ai < MAX_HANDLER_SCAN; ai++) { + for (uint32_t ai = HANDLER_START_IDX; ai < nc; ai++) { TSNode arg2 = ts_node_named_child(args, ai); /* PHP wraps each argument in an `argument` node — unwrap to the value. */ if (strcmp(ts_node_type(arg2), "argument") == 0 && ts_node_named_child_count(arg2) > 0) { @@ -2362,17 +2369,18 @@ static const char *extract_handler_arg(CBMExtractCtx *ctx, TSNode args) { if (strcmp(ak2, "identifier") == 0 || strcmp(ak2, "member_expression") == 0 || strcmp(ak2, "selector_expression") == 0 || strcmp(ak2, "attribute") == 0 || strcmp(ak2, "field_expression") == 0 || strcmp(ak2, "name") == 0) { - return cbm_node_text(ctx->arena, arg2, ctx->source); + handler = cbm_node_text(ctx->arena, arg2, ctx->source); + continue; } if (is_string_like(ak2)) { const char *h = normalize_string_handler(ctx->arena, cbm_node_text(ctx->arena, arg2, ctx->source)); if (h && h[0]) { - return h; + handler = h; } } } - return NULL; + return handler; } // Extract JSX component refs (uppercase tags) as CALLS edges. diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 5b8d16f61..e1a1643dd 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -3970,6 +3970,49 @@ TEST(extract_ts_url_builder_issue1009) { PASS(); } +/* A route registration names its middleware before its handler, and every + * framework here puts the handler last. The handler scan took the FIRST + * argument that looked like a function reference, so a named middleware won + * and the HANDLES edge pointed at the middleware instead of the handler. */ +TEST(extract_ts_route_handler_after_named_middleware) { + CBMFileResult *r = extract("function requireAuth(req: any, res: any, next: any) { next(); }\n" + "function rateLimit(req: any, res: any, next: any) { next(); }\n" + "function listUsers(req: any, res: any) { res.json([]); }\n" + "routerGet(\"/users\", requireAuth, rateLimit, listUsers);\n", + CBM_LANG_TYPESCRIPT, "t", "routes.ts"); + ASSERT_NOT_NULL(r); + const CBMCall *c = find_call_by_callee(r, "routerGet"); + ASSERT_NOT_NULL(c); + ASSERT_NOT_NULL(c->first_string_arg); + ASSERT_STR_EQ(c->first_string_arg, "/users"); + ASSERT_NOT_NULL(c->second_arg_name); + ASSERT_STR_EQ(c->second_arg_name, "listUsers"); + cbm_free_result(r); + PASS(); +} + +/* The same route with its middleware written inline. An arrow function is not + * one of the kinds the handler scan accepts, so three of them pushed the real + * handler past the scan bound and no handler came back at all. */ +TEST(extract_ts_route_handler_after_inline_middleware) { + CBMFileResult *r = extract("function listOrders(req: any, res: any) { res.json([]); }\n" + "routerGet(\"/orders\",\n" + " (req: any, res: any, next: any) => { next(); },\n" + " (req: any, res: any, next: any) => { next(); },\n" + " (req: any, res: any, next: any) => { next(); },\n" + " listOrders);\n", + CBM_LANG_TYPESCRIPT, "t", "orders.ts"); + ASSERT_NOT_NULL(r); + const CBMCall *c = find_call_by_callee(r, "routerGet"); + ASSERT_NOT_NULL(c); + ASSERT_NOT_NULL(c->first_string_arg); + ASSERT_STR_EQ(c->first_string_arg, "/orders"); + ASSERT_NOT_NULL(c->second_arg_name); + ASSERT_STR_EQ(c->second_arg_name, "listOrders"); + cbm_free_result(r); + PASS(); +} + /* Issue #1009 (composed builders): a builder whose template inlines an earlier * builder's call plus a query string: `return \`${basePath(id)}?${params}\``. * The known-substitution is inlined and the query string is truncated, so the @@ -6812,6 +6855,8 @@ SUITE(extraction) { RUN_TEST(extract_go_binary_concat_url_issue1249); RUN_TEST(extract_go_binary_concat_url_no_literal_suffix_issue1249); RUN_TEST(extract_ts_url_builder_issue1009); + RUN_TEST(extract_ts_route_handler_after_named_middleware); + RUN_TEST(extract_ts_route_handler_after_inline_middleware); RUN_TEST(extract_ts_url_builder_composed_issue1009); RUN_TEST(extract_c_url_builder_gated_issue1009); RUN_TEST(extract_ts_url_builder_mixed_returns_issue1009); From 2f37f61e8848f31f308bce8623f66dc2fdb50619 Mon Sep 17 00:00:00 2001 From: Ertan Date: Wed, 26 Aug 2026 06:32:49 +0300 Subject: [PATCH 18/22] perf(mcp): filter suffix globs while building search scope Signed-off-by: Ertan --- src/mcp/mcp.c | 79 +++++++++++++++++++++++++++--------------- src/mcp/mcp_internal.h | 1 + tests/test_mcp.c | 13 ++++--- 3 files changed, 60 insertions(+), 33 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 73534404a..33de0e79e 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -9747,6 +9747,36 @@ bool cbm_search_code_file_pattern_can_prefilter(const char *file_pattern) { return true; } +bool cbm_search_code_windows_path_matches_prefilter(const char *path, const char *file_pattern) { + if (!path || !cbm_search_code_file_pattern_can_prefilter(file_pattern)) { + return false; + } + + const char *suffix = file_pattern + 1; + size_t path_len = strlen(path); + size_t suffix_len = strlen(suffix); + if (path_len < suffix_len) { + return false; + } + + const unsigned char *candidate = (const unsigned char *)path + path_len - suffix_len; + const unsigned char *expected = (const unsigned char *)suffix; + for (size_t i = 0; i < suffix_len; i++) { + unsigned char left = candidate[i]; + unsigned char right = expected[i]; + if (left >= 'A' && left <= 'Z') { + left = (unsigned char)(left - 'A' + 'a'); + } + if (right >= 'A' && right <= 'Z') { + right = (unsigned char)(right - 'A' + 'a'); + } + if (left != right) { + return false; + } + } + return true; +} + /* Build the grep/search command string based on scoped vs recursive mode. * On Windows, uses PowerShell Select-String with tab-delimited output. * On POSIX, uses grep with colon-delimited output. */ @@ -9767,30 +9797,16 @@ void cbm_search_code_build_grep_cmd(char *cmd, size_t cmd_sz, bool use_regex, bo const char *sm = use_regex ? "" : " -SimpleMatch"; if (scoped) { if (file_pattern) { - if (cbm_search_code_file_pattern_can_prefilter(file_pattern)) { - snprintf( - cmd, cmd_sz, - "powershell -Command \"" CBM_PS_UTF8_PRELUDE - "$pat = Get-Content -Encoding UTF8 -LiteralPath '%s'; " - "Get-Content -Encoding UTF8 -LiteralPath '%s'" - " | Where-Object { $_ -like '%s' }" - " | ForEach-Object { Select-String -LiteralPath $_ -Pattern $pat%s " - "-ErrorAction SilentlyContinue }" - " | Where-Object { $_.Path -like '*%s' }" - " | ForEach-Object { $_.Path + [char]9 + $_.LineNumber + [char]9 + $_.Line }\"", - tmpfile, filelist, file_pattern, sm, file_pattern); - } else { - snprintf( - cmd, cmd_sz, - "powershell -Command \"" CBM_PS_UTF8_PRELUDE - "$pat = Get-Content -Encoding UTF8 -LiteralPath '%s'; " - "Get-Content -Encoding UTF8 -LiteralPath '%s' | ForEach-Object { Select-String " - "-LiteralPath $_ -Pattern $pat%s " - "-ErrorAction SilentlyContinue }" - " | Where-Object { $_.Path -like '*%s' }" - " | ForEach-Object { $_.Path + [char]9 + $_.LineNumber + [char]9 + $_.Line }\"", - tmpfile, filelist, sm, file_pattern); - } + snprintf( + cmd, cmd_sz, + "powershell -Command \"" CBM_PS_UTF8_PRELUDE + "$pat = Get-Content -Encoding UTF8 -LiteralPath '%s'; " + "Get-Content -Encoding UTF8 -LiteralPath '%s' | ForEach-Object { Select-String " + "-LiteralPath $_ -Pattern $pat%s " + "-ErrorAction SilentlyContinue }" + " | Where-Object { $_.Path -like '*%s' }" + " | ForEach-Object { $_.Path + [char]9 + $_.LineNumber + [char]9 + $_.Line }\"", + tmpfile, filelist, sm, file_pattern); } else { snprintf( cmd, cmd_sz, @@ -10433,8 +10449,8 @@ static void classify_all_grep_hits(grep_match_t *gm, int gm_count, cbm_store_t * * created inside the private scratch directory; this function never opens or * closes it, so the list is never reachable through a predictable pathname. */ static bool write_scoped_filelist(cbm_mcp_server_t *srv, const char *project, const char *root_path, - FILE *fl, bool has_path_filter, cbm_regex_t *path_regex, - int *out_written) { + FILE *fl, const char *file_pattern, bool has_path_filter, + cbm_regex_t *path_regex, int *out_written) { *out_written = 0; cbm_store_t *pre_store = resolve_store(srv, project); if (!pre_store) { @@ -10471,6 +10487,12 @@ static bool write_scoped_filelist(cbm_mcp_server_t *srv, const char *project, co continue; } } +#ifdef _WIN32 + if (cbm_search_code_file_pattern_can_prefilter(file_pattern) && + !cbm_search_code_windows_path_matches_prefilter(indexed_files[fi], file_pattern)) { + continue; + } +#endif size_t root_len = strlen(root_path); size_t file_len = strlen(indexed_files[fi]); if (root_len > SIZE_MAX - file_len - 2) { @@ -10954,8 +10976,9 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) { uint64_t scope_t0 = metrics.include_phase_timings ? cbm_now_ms() : 0; if (!scan_cancellation_latched && !scan_deadline_latched) { - scoped = write_scoped_filelist(srv, project, root_path, scratch.filelist, has_path_filter, - has_path_filter ? &path_regex : NULL, &scoped_written); + scoped = write_scoped_filelist(srv, project, root_path, scratch.filelist, file_pattern, + has_path_filter, has_path_filter ? &path_regex : NULL, + &scoped_written); } /* Close before grep runs: this is what flushes the records the helper wrote * through the descriptor. Clearing the field hands ownership to diff --git a/src/mcp/mcp_internal.h b/src/mcp/mcp_internal.h index 8355bb390..673fd0c5d 100644 --- a/src/mcp/mcp_internal.h +++ b/src/mcp/mcp_internal.h @@ -67,6 +67,7 @@ bool cbm_detect_node_in_hunks(const cbm_node_t *node, const cbm_changed_hunk_t * * PowerShell -like contract. Exposed for * direct boundary tests only. */ bool cbm_search_code_file_pattern_can_prefilter(const char *file_pattern); +bool cbm_search_code_windows_path_matches_prefilter(const char *path, const char *file_pattern); /* Internal command builder exposed so tests can pin the PowerShell pipeline * ordering without diff --git a/tests/test_mcp.c b/tests/test_mcp.c index e8722ffc1..a5c2e5f03 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -5345,22 +5345,25 @@ TEST(search_code_file_pattern_prefilter_boundaries) { ASSERT_FALSE(cbm_search_code_file_pattern_can_prefilter("src\\*.pas")); ASSERT_FALSE(cbm_search_code_file_pattern_can_prefilter("*.c++")); ASSERT_FALSE(cbm_search_code_file_pattern_can_prefilter("*R&D*.go")); + + ASSERT_TRUE(cbm_search_code_windows_path_matches_prefilter("src/UnitMain.PAS", "*.pas")); + ASSERT_TRUE(cbm_search_code_windows_path_matches_prefilter("types/index.D.TS", "*.d.ts")); + ASSERT_FALSE(cbm_search_code_windows_path_matches_prefilter("src/UnitMain.pas.bak", "*.pas")); + ASSERT_FALSE(cbm_search_code_windows_path_matches_prefilter("types/index.ts", "*.d.ts")); PASS(); } -TEST(search_code_windows_prefilter_precedes_content_scan) { +TEST(search_code_windows_scope_prefilter_removes_pipeline_filter) { #ifdef _WIN32 char command[CBM_SZ_4K]; cbm_search_code_build_grep_cmd(command, sizeof(command), false, true, "*.go", "C:/tmp/pattern", "C:/tmp/filelist", "C:/tmp/root"); - const char *prefilter = strstr(command, "Where-Object { $_ -like '*.go' }"); const char *content_scan = strstr(command, "ForEach-Object { Select-String"); const char *postfilter = strstr(command, "Where-Object { $_.Path -like '**.go' }"); - ASSERT_NOT_NULL(prefilter); + ASSERT_NULL(strstr(command, "Where-Object { $_ -like '*.go' }")); ASSERT_NOT_NULL(content_scan); ASSERT_NOT_NULL(postfilter); - ASSERT_TRUE(prefilter < content_scan); ASSERT_TRUE(content_scan < postfilter); cbm_search_code_build_grep_cmd(command, sizeof(command), false, true, "*handler*.go", @@ -13904,7 +13907,7 @@ SUITE(mcp) { RUN_TEST(search_code_path_filter_prefilter_keeps_matches); RUN_TEST(search_code_path_filter_matches_nothing); RUN_TEST(search_code_file_pattern_prefilter_boundaries); - RUN_TEST(search_code_windows_prefilter_precedes_content_scan); + RUN_TEST(search_code_windows_scope_prefilter_removes_pipeline_filter); RUN_TEST(search_code_cancel_cleans_supervised_scan); RUN_TEST(search_code_output_limit_fails_closed_and_cleans_scan); RUN_TEST(search_code_scan_deadline_fails_closed_and_resets); From 349aecba9528427fa635975e246f5b8959b14f30 Mon Sep 17 00:00:00 2001 From: Ilya Brykau Date: Wed, 2 Sep 2026 10:39:03 +0200 Subject: [PATCH 19/22] fix(pipeline): key the Go Field guard on the recorded selector shape, not reference text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cbm_go_suppress_bare_field_ref dropped a Field-targeted reference when strchr(ref_name, '.') == NULL — but the extractor strips the receiver on every path that reaches the resolver (resolve_lhs_write_name records the trailing field name of a selector LHS; is_reference_node records the inner field_identifier), so the dot test could never be false for Go. The guard was a blanket veto: all ~4588 Go Field nodes recovered by one 61de19bb were unreachable by USAGE/READS/WRITES, including genuine selector references (issue #1962, found in maintainer post-merge review of #1944). The selector-vs-bare distinction exists at extraction time and was discarded; record it and let the resolver consume it: - CBMUsage/CBMReadWrite gain is_member_access. The usage recorder sets it when the reference node is a field_identifier (the member half of a selector); resolve_lhs_write_name reports it through an out-param when it takes the field/member LHS branch. - cbm_go_suppress_bare_field_ref(is_go, is_member_access, target_label) refuses a Field bind only for references that were never the member half of a selector. All four resolver sites (sequential and parallel, USAGE and READS/WRITES) pass the recorded flag. Reproduce-first: with the veto restored, the extended #1942 fixtures go RED on both resolver paths at the new asserts (t.err = nil must WRITE the field; t.n must produce USAGE); with the fix both twins are GREEN and the original bare-local negatives still hold. The dead ASSERT_FALSE(..., "t.err", ...) unit case — an input Go's extractor cannot produce — is replaced by flag-based cases. Fixes #1962. Signed-off-by: Ilya Brykau --- internal/cbm/cbm.h | 7 +++++++ internal/cbm/extract_semantic.c | 16 +++++++++++++--- internal/cbm/extract_usages.c | 4 ++++ src/pipeline/pass_parallel.c | 12 +++++++----- src/pipeline/pass_usages.c | 12 +++++++----- src/pipeline/pipeline.h | 2 +- src/pipeline/registry.c | 27 +++++++++++++-------------- tests/test_pipeline.c | 20 ++++++++++++++++++++ tests/test_registry.c | 27 ++++++++++++++------------- 9 files changed, 86 insertions(+), 41 deletions(-) diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 4f06bebb7..e9d307c57 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -292,6 +292,10 @@ typedef struct { uint32_t site_start_byte; // exact reference-token span; end > start when present uint32_t site_end_byte; // exclusive byte offset in the source file CBMSourceOrigin source_origin; // raw source or C-family preprocessed buffer + bool is_member_access; // token is the member half of a selector/attribute + // (Go x.f — field_identifier). The extractor strips + // the receiver, so this is the only surviving record + // of selector shape (#1962). Default false. } CBMUsage; typedef struct { @@ -303,6 +307,9 @@ typedef struct { const char *var_name; // variable name const char *enclosing_func_qn; // QN of enclosing function bool is_write; // true = write, false = read + bool is_member_access; // var_name is the field half of a selector/member LHS + // (`t.err = x` → "err"); the receiver is stripped here, + // so this is the only record of selector shape (#1962) } CBMReadWrite; typedef struct { diff --git a/internal/cbm/extract_semantic.c b/internal/cbm/extract_semantic.c index 9379f7171..36709d2dd 100644 --- a/internal/cbm/extract_semantic.c +++ b/internal/cbm/extract_semantic.c @@ -146,7 +146,10 @@ static void walk_throws(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec // - field/member/selector access (`self.total = ...`, `obj.Field = ...` → // write the trailing field name `total`/`Field`) // Returns NULL if no simple write target can be determined. -static char *resolve_lhs_write_name(CBMExtractCtx *ctx, TSNode left) { +// Sets *is_member_out when the written name is the field half of a +// member/selector LHS — the receiver is stripped from the returned name, so +// this out-param is the only surviving record of the selector shape (#1962). +static char *resolve_lhs_write_name(CBMExtractCtx *ctx, TSNode left, bool *is_member_out) { // Unwrap a single-element expression_list (Go). if (strcmp(ts_node_type(left), "expression_list") == 0) { if (ts_node_named_child_count(left) != 1) { @@ -184,6 +187,9 @@ static char *resolve_lhs_write_name(CBMExtractCtx *ctx, TSNode left) { fld = ts_node_child_by_field_name(left, TS_FIELD("name")); } if (!ts_node_is_null(fld)) { + if (is_member_out) { + *is_member_out = true; + } return cbm_node_text(ctx->arena, fld, ctx->source); } return NULL; @@ -251,12 +257,14 @@ static void try_emit_assignment_write(CBMExtractCtx *ctx, TSNode node, const cha if (ts_node_is_null(left)) { return; } - char *name = resolve_lhs_write_name(ctx, left); + bool is_member = false; + char *name = resolve_lhs_write_name(ctx, left, &is_member); if (name && name[0] && !cbm_is_keyword(name, ctx->language)) { CBMReadWrite rw; rw.var_name = name; rw.is_write = true; rw.enclosing_func_qn = func_qn; + rw.is_member_access = is_member; cbm_rw_push(&ctx->result->rw, ctx->arena, rw); } } @@ -329,12 +337,14 @@ void handle_readwrites(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec, TSNode left = resolve_write_lhs_node(node); if (!ts_node_is_null(left)) { - char *name = resolve_lhs_write_name(ctx, left); + bool is_member = false; + char *name = resolve_lhs_write_name(ctx, left, &is_member); if (name && name[0] && !cbm_is_keyword(name, ctx->language)) { CBMReadWrite rw; rw.var_name = name; rw.is_write = true; rw.enclosing_func_qn = state->enclosing_func_qn; + rw.is_member_access = is_member; cbm_rw_push(&ctx->result->rw, ctx->arena, rw); } } diff --git a/internal/cbm/extract_usages.c b/internal/cbm/extract_usages.c index 4e3218fd1..fd83864ac 100644 --- a/internal/cbm/extract_usages.c +++ b/internal/cbm/extract_usages.c @@ -2673,6 +2673,10 @@ void handle_usages(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec, Wal usage.ref_name = name; usage.enclosing_func_qn = state->enclosing_func_qn; usage.lexical_scope_id = usage_lexical_scope_id_for_node(ctx, state, node); + /* The member half of a selector is its own reference node + * (field_identifier); record that shape — the name alone cannot carry + * it, and the Go Field guard keys on it (#1962). */ + usage.is_member_access = strcmp(ts_node_type(node), "field_identifier") == 0; stamp_usage_site(ctx, &usage, node, name, state); cbm_usages_push(&ctx->result->usages, ctx->arena, usage); } diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 1ce766927..77a64c408 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -2670,9 +2670,10 @@ static void resolve_file_usages(resolve_ctx_t *rc, resolve_worker_state_t *ws, if (tgt && cbm_suppress_cross_language_ref(lang, tgt->file_path)) { continue; } - /* #1942: a bare Go reference can never denote a struct field. */ - if (tgt && - cbm_go_suppress_bare_field_ref(lang == CBM_LANG_GO, usage->ref_name, tgt->label)) { + /* #1942/#1962: a bare Go reference can never denote a struct + * field; the member half of a selector may. */ + if (tgt && cbm_go_suppress_bare_field_ref(lang == CBM_LANG_GO, usage->is_member_access, + tgt->label)) { continue; } if (usage->semantic_reference_blocked && (usage->semantic_reference_local_shadow || @@ -2760,8 +2761,9 @@ static void resolve_file_rw(resolve_ctx_t *rc, resolve_worker_state_t *ws, CBMFi if (cbm_suppress_cross_language_ref(lang, tgt->file_path)) { continue; } - /* #1942: a bare Go reference can never denote a struct field. */ - if (cbm_go_suppress_bare_field_ref(lang == CBM_LANG_GO, rw->var_name, tgt->label)) { + /* #1942/#1962: a bare Go reference can never denote a struct field; + * a selector-LHS write (`t.err = x`) may bind it. */ + if (cbm_go_suppress_bare_field_ref(lang == CBM_LANG_GO, rw->is_member_access, tgt->label)) { continue; } const char *etype = rw->is_write ? "WRITES" : "READS"; diff --git a/src/pipeline/pass_usages.c b/src/pipeline/pass_usages.c index 690b45ad9..9738f7198 100644 --- a/src/pipeline/pass_usages.c +++ b/src/pipeline/pass_usages.c @@ -216,9 +216,10 @@ static int resolve_usage_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *res if (tgt && cbm_suppress_cross_language_ref(lang, tgt->file_path)) { continue; } - /* #1942: a bare Go reference can never denote a struct field. */ - if (tgt && - cbm_go_suppress_bare_field_ref(lang == CBM_LANG_GO, usage->ref_name, tgt->label)) { + /* #1942/#1962: a bare Go reference can never denote a struct + * field; the member half of a selector may. */ + if (tgt && cbm_go_suppress_bare_field_ref(lang == CBM_LANG_GO, usage->is_member_access, + tgt->label)) { continue; } if (usage->semantic_reference_blocked && (usage->semantic_reference_local_shadow || @@ -309,8 +310,9 @@ static int resolve_rw_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *result if (cbm_suppress_cross_language_ref(lang, tgt->file_path)) { continue; } - /* #1942: a bare Go reference can never denote a struct field. */ - if (cbm_go_suppress_bare_field_ref(lang == CBM_LANG_GO, rw->var_name, tgt->label)) { + /* #1942/#1962: a bare Go reference can never denote a struct field; + * a selector-LHS write (`t.err = x`) may bind it. */ + if (cbm_go_suppress_bare_field_ref(lang == CBM_LANG_GO, rw->is_member_access, tgt->label)) { continue; } diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index d174bb6d6..109da9781 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -302,7 +302,7 @@ bool cbm_suppress_cross_language_ref(CBMLanguage caller_lang, const char *target * Field when the reference text carries no '.'. Go only: other OO languages * legitimately reference their own members bare inside method bodies. Pure; * unit-tested in test_registry.c. */ -bool cbm_go_suppress_bare_field_ref(bool is_go, const char *ref_name, const char *target_label); +bool cbm_go_suppress_bare_field_ref(bool is_go, bool is_member_access, const char *target_label); /* Get the label of a qualified name, or NULL if not found. */ const char *cbm_registry_label_of(const cbm_registry_t *r, const char *qn); diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index eefdb7596..b29eba494 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -546,22 +546,21 @@ bool cbm_suppress_cross_language_ref(CBMLanguage caller_lang, const char *target return true; } -bool cbm_go_suppress_bare_field_ref(bool is_go, const char *ref_name, const char *target_label) { - /* #1942: a bare (dot-less) Go reference can never denote a struct field — - * field access is always a selector expression (x.f), and selector - * references resolve through the LSP join, never through the bare-name - * registry fallback. Every Field-targeted reference edge in the field - * census carried dot-less text, so dropping the bind loses nothing real. - * Go-gated: a C#/Java/C++/Python method body legitimately references its - * own members bare (cp_reads_writes_cs_static_field pins that shape as - * required), so a global veto would break those languages. */ - if (!is_go || !ref_name || !ref_name[0] || !target_label) { +bool cbm_go_suppress_bare_field_ref(bool is_go, bool is_member_access, const char *target_label) { + /* #1942/#1962: a bare Go identifier can never denote a struct field — + * field access is always a selector expression (x.f). The extractor + * strips the receiver before the resolver runs (resolve_lhs_write_name + * records the trailing field name; is_reference_node records the inner + * field_identifier), so the reference TEXT is always dot-less and cannot + * carry the distinction — the recorded is_member_access shape can. Only a + * reference that was never the member half of a selector is refused a + * Field bind. Go-gated: a C#/Java/C++/Python method body legitimately + * references its own members bare (cp_reads_writes_cs_static_field pins + * that shape as required), so a global veto would break those languages. */ + if (!is_go || is_member_access || !target_label) { return false; } - if (strcmp(target_label, "Field") != 0) { - return false; - } - return strchr(ref_name, '.') == NULL; + return strcmp(target_label, "Field") == 0; } /* ── Lifecycle ──────────────────────────────────────────────────── */ diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index d62a5572f..2af0366f8 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -4857,6 +4857,17 @@ static void write_go_bare_field_fixture(const char *tmp, int pad_files) { "\terr := errors.New(\"x\")\n" "\treturn err\n" "}\n"); + /* #1962: genuine selector references from a sibling file of the same + * package. `t.err = nil` writes the field through a selector; `t.n` reads + * it. The extractor strips the receiver on both paths, so only the + * is_member_access signal can distinguish these from Run's bare local. */ + write_temp_file(tmp, "state/reset.go", + "package state\n" + "\n" + "func (t *Tracker) Reset() int {\n" + "\tt.err = nil\n" + "\treturn t.n\n" + "}\n"); for (int i = 0; i < pad_files; i++) { char name[64]; char body[128]; @@ -4896,6 +4907,11 @@ TEST(pipeline_go_bare_ref_never_binds_field) { ASSERT_FALSE(cross_file_edge_exists(s, project, "Run", "err", "WRITES")); ASSERT_FALSE(cross_file_edge_exists(s, project, "Run", "err", "READS")); ASSERT_FALSE(cross_file_edge_exists(s, project, "Run", "err", "USAGE")); + /* #1962, reproduce-first: RED while the guard is a blanket veto — genuine + * selector references must reach the field (write via `t.err = nil`, + * value use via `t.n`). */ + ASSERT_TRUE(cross_file_edge_exists(s, project, "Reset", "err", "WRITES")); + ASSERT_TRUE(cross_file_edge_exists(s, project, "Reset", "n", "USAGE")); cbm_store_close(s); cbm_pipeline_free(p); @@ -4927,6 +4943,10 @@ TEST(pipeline_go_bare_ref_never_binds_field_parallel) { ASSERT_FALSE(cross_file_edge_exists(s, project, "Run", "err", "WRITES")); ASSERT_FALSE(cross_file_edge_exists(s, project, "Run", "err", "READS")); ASSERT_FALSE(cross_file_edge_exists(s, project, "Run", "err", "USAGE")); + /* #1962 parallel twin: resolve_file_rw / resolve_file_usages must honour + * the member-access signal exactly like the sequential resolvers. */ + ASSERT_TRUE(cross_file_edge_exists(s, project, "Reset", "err", "WRITES")); + ASSERT_TRUE(cross_file_edge_exists(s, project, "Reset", "n", "USAGE")); cbm_store_close(s); cbm_pipeline_free(p); diff --git a/tests/test_registry.c b/tests/test_registry.c index 3e1d604af..0f3f3861a 100644 --- a/tests/test_registry.c +++ b/tests/test_registry.c @@ -840,22 +840,23 @@ TEST(cross_language_ref_drops_go_vs_c) { } TEST(go_bare_ref_never_binds_field) { - /* #1942: a bare (dot-less) Go reference can never denote a struct field — - * field access is always a selector expression. */ - ASSERT_TRUE(cbm_go_suppress_bare_field_ref(true, "err", "Field")); - ASSERT_TRUE(cbm_go_suppress_bare_field_ref(true, "config", "Field")); - /* A selector-shaped reference may bind a field. */ - ASSERT_FALSE(cbm_go_suppress_bare_field_ref(true, "t.err", "Field")); + /* #1942/#1962: a bare Go identifier can never denote a struct field — + * field access is always a selector expression. The extractor strips the + * receiver before the resolver runs (resolve_lhs_write_name writes the + * trailing name; is_reference_node records the inner field_identifier), + * so the selector-vs-bare distinction arrives as the recorded + * is_member_access signal, never as a dot in the reference text. */ + ASSERT_TRUE(cbm_go_suppress_bare_field_ref(true, false, "Field")); + /* The member half of a selector may bind a field. */ + ASSERT_FALSE(cbm_go_suppress_bare_field_ref(true, true, "Field")); /* Bare references to non-fields are untouched. */ - ASSERT_FALSE(cbm_go_suppress_bare_field_ref(true, "err", "Variable")); - ASSERT_FALSE(cbm_go_suppress_bare_field_ref(true, "err", "Function")); + ASSERT_FALSE(cbm_go_suppress_bare_field_ref(true, false, "Variable")); + ASSERT_FALSE(cbm_go_suppress_bare_field_ref(true, false, "Function")); /* Other languages reference their own members bare inside methods — * never suppressed (cp_reads_writes_cs_static_field pins the C# shape). */ - ASSERT_FALSE(cbm_go_suppress_bare_field_ref(false, "_count", "Field")); - /* Degenerate inputs → nothing to judge. */ - ASSERT_FALSE(cbm_go_suppress_bare_field_ref(true, NULL, "Field")); - ASSERT_FALSE(cbm_go_suppress_bare_field_ref(true, "", "Field")); - ASSERT_FALSE(cbm_go_suppress_bare_field_ref(true, "err", NULL)); + ASSERT_FALSE(cbm_go_suppress_bare_field_ref(false, false, "Field")); + /* Degenerate input → nothing to judge. */ + ASSERT_FALSE(cbm_go_suppress_bare_field_ref(true, false, NULL)); PASS(); } From 97517a4652f376cb6bf096255e235edf456f86b1 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 2 Sep 2026 10:45:37 +0200 Subject: [PATCH 20/22] fix(python): carry bare-call parameter bindings in the walk Replaces the cursor ascent from 0b0d143c. That commit fixed a real hang -- ts_node_parent() is O(depth), so a parent chain is O(depth^2) per call and O(depth^3) across a file where every level of f(f(f(...))) is itself a bare call, which hung stack_overflow_b's 30,000-deep fixture at 900s. But it fixed it by making each hop cheap rather than by removing the per-call walk, so the cost stayed O(depth) per call and needed a 64-ancestor cap to stay bounded. That cap FAILS OPEN. Past 64 ancestors the guard stopped suppressing, so deep-but-ordinary code silently kept the fabricated edges the guard exists to remove -- and nothing failed when it did. Carry the answer instead. The unified walk binds a def's or lambda's parameters when it opens that scope and unwinds them when it closes, so the guard is an O(1) map lookup. This is the lesson CBMWalkScope already records one struct away: carry walk state, never recompute it per node. - CBMParamSlot is a name -> active-count map, not a set. `def outer(run): def inner(run):` binds one name twice and leaving the inner scope must not unbind the outer. - Bindings are pushed at the end of push_boundary_scopes, after every scope push for that node, so they land in the frame just pushed -- covering function_definition (push_function_scope) and lambda (push_lexical_boundary) alike -- and pop_expired_scopes unwinds to the frame's entry height. - Allocation failure sets py_param_tracking_failed and the lookup answers false forever after: a lost suppression, never a lost edge. The repo's other binding table cannot serve this. CBMLexicalBinding is qsort-ed in cbm_finalize_lexical_usages AFTER the walk and its active_end stays 0 until then, so its binary search is invalid from handle_calls, which runs mid-walk and before handle_usages. extract_python_bare_call_scope_walk_is_bounded pinned the cap's fail-open contract, which no longer exists, and is replaced by extract_python_bare_call_flag_is_depth_independent. It keeps the shallow and 200-ancestor cases -- the deep one now asserts the call IS flagged, so reintroducing a cap fails it -- and adds an unwind case: `handler` as both a parameter and a module-level def, asserting the two shadowed calls are flagged and the sibling call after that scope closed is NOT. A binding outliving its frame would destroy that true edge, the one direction this guard must never fail in. It passed on the first run. extraction 327 passed; pipeline registry parallel complexity 407 passed; stack_overflow_b 6 passed in 38s (46s with the cursor ascent, 900s rc=124 before it), lsp_python_deep_nesting_no_crash PASS. Signed-off-by: Martin Vogel --- internal/cbm/extract_calls.c | 122 +++------------------ internal/cbm/extract_unified.c | 195 +++++++++++++++++++++++++++++++++ internal/cbm/extract_unified.h | 46 ++++++++ tests/test_extraction.c | 146 +++++++++++++++--------- 4 files changed, 353 insertions(+), 156 deletions(-) diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index cb81fc543..b481fab6d 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -2970,28 +2970,8 @@ static bool python_receiver_is_exempt(CBMExtractCtx *ctx, TSNode receiver) { * a bare `identifier`, the `name` field of default/typed parameters, and the * identifier under a `*args` / `**kwargs` splat. A shape with no identifier (the * bare `*` keyword separator) yields NULL and simply matches nothing. */ -static const char *python_parameter_name(CBMExtractCtx *ctx, TSNode param) { - if (ts_node_is_null(param)) { - return NULL; - } - if (strcmp(ts_node_type(param), "identifier") == 0) { - return cbm_node_text(ctx->arena, param, ctx->source); - } - TSNode name = ts_node_child_by_field_name(param, TS_FIELD("name")); - if (!ts_node_is_null(name) && strcmp(ts_node_type(name), "identifier") == 0) { - return cbm_node_text(ctx->arena, name, ctx->source); - } - /* `*args` / `**kwargs`, and any typed shape without a `name` field: the - * bound identifier is the first named child. */ - TSNode first = ts_node_named_child(param, 0); - if (!ts_node_is_null(first) && strcmp(ts_node_type(first), "identifier") == 0) { - return cbm_node_text(ctx->arena, first, ctx->source); - } - return NULL; -} - /* True when the callee of a BARE Python call `foo()` is bound as a parameter of - * an enclosing function or lambda — the bare-call counterpart of + * an enclosing function or lambda -- the bare-call counterpart of * python_receiver_is_exempt above. * * A parameter binding shadows any module-level `foo` for the whole body, so @@ -2999,92 +2979,25 @@ static const char *python_parameter_name(CBMExtractCtx *ctx, TSNode param) { * fabricates the edge BY CONSTRUCTION: `def _run_with_heavy_slot(run): run()` * must not bind an unrelated `SatoriLive.run`. Unlike a receiver type this is * decidable from the AST outright, with no flow analysis and no list of - * "generic-looking" callee names — Python forbids `global` on a parameter, and a - * parameter is in scope for the entire body regardless of position, so there is - * no ordering subtlety to get wrong. + * "generic-looking" callee names -- Python forbids `global` on a parameter, and + * a parameter is in scope for the entire body regardless of position. * - * Enclosing scopes are walked to the file root so a closure over an outer - * parameter counts (`def outer(run): def inner(): return run()`). + * The answer is CARRIED BY THE WALK rather than recomputed here. The unified + * walk binds a def's or lambda's parameters when it opens that scope and + * unwinds them when it closes it, so this is an O(1) map lookup. Deciding it + * by ascending the tree instead -- with ts_node_parent() or with a copied walk + * cursor -- costs O(depth) per call, and since every level of f(f(f(...))) is + * itself a bare call, that is quadratic across the file: the 30,000-deep + * fixture in tests/test_stack_overflow.c turned it into a hang rather than a + * slowdown. An O(1) lookup needs no hop cap, so unlike a capped walk this can + * no longer fail open on deep-but-ordinary code. * - * LOCAL ASSIGNMENTS are deliberately NOT covered. They are flow- and - * binding-form-sensitive (`for`, `with ... as`, `except ... as`, `:=`, - * unpacking, plus `global`/`nonlocal` overrides), so a partial body scan would - * suppress the wrong edges invisibly — the same failure mode that rules out a - * hardcoded name list. Parameters alone already cover the Callable-parameter - * shape that motivated this guard. Cost is O(enclosing depth x params) per bare - * call, never the corpus. - * - * Known and accepted: `def inner(): global run; return run()` nested in a - * function whose parameter is `run` is still flagged. Detecting it needs exactly - * the body scan this helper avoids, and it costs one edge in a shape that - * essentially does not occur. */ -static bool python_callee_is_bound_parameter(CBMExtractCtx *ctx, WalkState *state, TSNode call_node, + * It still answers false if the walk's own tracking hit an allocation failure, + * which costs a suppression and never a true edge. */ +static bool python_callee_is_bound_parameter(CBMExtractCtx *ctx, WalkState *state, TSNode callee_ident) { - /* Ancestors are walked with the UNIFIED WALK'S OWN CURSOR, not ts_node_parent(). - * - * ts_node_parent() is not O(1): it restarts at the tree root and descends to - * find the parent (vendored ts_runtime/src/node.c), so it costs O(depth) per - * hop. A parent-chain walk is therefore O(depth^2) per call, and since every - * level of f(f(f(...))) is ITSELF a bare call, O(depth^3) across the file -- - * tests/test_stack_overflow.c nests 30,000 deep, which is a hang, not a - * slowdown. Capping the hop COUNT does not fix that, because the cost is per - * hop; the walk itself has to be cheap. - * - * ts_tree_cursor_goto_parent() IS O(1) -- the cursor carries its path stack -- - * so copying the walk cursor and ascending costs nothing per hop. Same reason - * CBMWalkScope keeps a frame stack instead of recomputing enclosing state, and - * same current_cursor idiom (including the identity guard) as - * usage_current_field_name in extract_usages.c. - * - * The hop cap then bounds the remaining O(depth) per call. 64 rather than - * Lean's 20 above: that walk looks for an IMMEDIATE declaration boundary, - * while this one crosses whatever statement/expression nesting separates a - * call from its enclosing def, so it needs headroom before it can bite on - * ordinary code. - * - * Both the cap and a missing/mismatched cursor FAIL OPEN -- return false, do - * NOT suppress. A guard whose whole justification is precision must never - * destroy an edge it did not actually prove was fabricated, so either can - * only ever cost a suppression, never a true edge. Pinned in - * test_extraction.c. */ - enum { PY_MAX_SCOPE_WALK_DEPTH = 64 }; - const char *callee_name = cbm_node_text(ctx->arena, callee_ident, ctx->source); - if (!callee_name || !callee_name[0]) { - return false; - } - /* Only trust the shared cursor when it is actually parked on this call. */ - if (!state || !state->current_cursor || - !ts_node_eq(ts_tree_cursor_current_node(state->current_cursor), call_node)) { - return false; - } - - TSTreeCursor up = ts_tree_cursor_copy(state->current_cursor); - bool bound = false; - for (int walked = 0; walked < PY_MAX_SCOPE_WALK_DEPTH && !bound; walked++) { - if (!ts_tree_cursor_goto_parent(&up)) { - break; - } - TSNode scope = ts_tree_cursor_current_node(&up); - const char *kind = ts_node_type(scope); - if (strcmp(kind, "function_definition") != 0 && strcmp(kind, "lambda") != 0) { - continue; - } - TSNode params = ts_node_child_by_field_name(scope, TS_FIELD("parameters")); - if (ts_node_is_null(params)) { - continue; - } - uint32_t count = ts_node_named_child_count(params); - for (uint32_t i = 0; i < count; i++) { - const char *pname = python_parameter_name(ctx, ts_node_named_child(params, i)); - if (pname && strcmp(pname, callee_name) == 0) { - bound = true; - break; - } - } - } - ts_tree_cursor_delete(&up); - return bound; + return cbm_walk_python_param_is_bound(state, callee_name); } static bool is_objectscript_language(CBMLanguage language) { @@ -3722,8 +3635,7 @@ CBMInvocationDescriptor handle_calls(CBMExtractCtx *ctx, TSNode node, const CBML TSNode obj = ts_node_child_by_field_name(fn, TS_FIELD("object")); call.is_method = !python_receiver_is_exempt(ctx, obj); } else if (!ts_node_is_null(fn) && strcmp(ts_node_type(fn), "identifier") == 0) { - call.callee_is_locally_bound = - python_callee_is_bound_parameter(ctx, state, node, fn); + call.callee_is_locally_bound = python_callee_is_bound_parameter(ctx, state, fn); } } // TS/JS/TSX receiver-aware guard (#592/#606 direction; same intent diff --git a/internal/cbm/extract_unified.c b/internal/cbm/extract_unified.c index a5e314272..5c2905f9c 100644 --- a/internal/cbm/extract_unified.c +++ b/internal/cbm/extract_unified.c @@ -119,11 +119,196 @@ static uint32_t add_lexical_scope(WalkState *state, TSNode node, CBMLexicalScope return scope->id; } +/* ── #1912: Python parameter bindings held live by the walk ─────────────────── + * + * A bare Python call `run()` cannot honestly resolve to a project function by + * short name when `run` is bound as a parameter of an enclosing def or lambda: + * the parameter shadows any module-level `run` for the whole body, so the match + * is fabricated by construction. + * + * Deciding that per call by ASCENDING the tree is the trap this replaces. Both + * ts_node_parent() (O(depth) per hop) and a walk-cursor ascent (O(1) per hop, + * O(depth) per call) are per-call costs, and every level of f(f(f(...))) is + * itself a bare call, so the file-wide cost is quadratic or worse -- exactly + * what CBMWalkScope's comment records for the state it replaced. Scanning the + * frame stack has the same shape. + * + * So the walk carries the answer instead: a name -> active-count map, pushed + * when a Python function or lambda scope opens and unwound when it closes. + * Lookup is O(1), which is what removes the need for a hop cap. + * + * A COUNT, not a flag: `def outer(run): def inner(run):` binds one name twice, + * and leaving the inner scope must not unbind the outer one. + * + * Every failure path answers "not bound", which can only ever cost a + * suppression -- never a true edge. */ + +static uint32_t py_param_hash(const char *s) { + uint32_t h = 2166136261u; /* FNV-1a */ + for (const unsigned char *p = (const unsigned char *)s; *p; p++) { + h = (h ^ *p) * 16777619u; + } + return h ? h : 1u; /* 0 marks an empty slot */ +} + +static CBMParamSlot *py_param_slot(WalkState *state, const char *name, uint32_t hash) { + int mask = state->py_param_slot_capacity - 1; + int i = (int)(hash & (uint32_t)mask); + for (;;) { + CBMParamSlot *slot = &state->py_param_slots[i]; + if (slot->hash == 0) { + return slot; /* free slot -- caller decides whether to claim it */ + } + if (slot->hash == hash && strcmp(slot->name, name) == 0) { + return slot; + } + i = (i + 1) & mask; + } +} + +static bool py_param_grow(WalkState *state) { + int new_capacity = state->py_param_slot_capacity * PAIR_LEN; + if (new_capacity <= state->py_param_slot_capacity) { + return false; + } + CBMParamSlot *grown = + (CBMParamSlot *)cbm_arena_alloc(state->arena, (size_t)new_capacity * sizeof(*grown)); + if (!grown) { + return false; + } + memset(grown, 0, (size_t)new_capacity * sizeof(*grown)); + CBMParamSlot *old = state->py_param_slots; + int old_capacity = state->py_param_slot_capacity; + state->py_param_slots = grown; + state->py_param_slot_capacity = new_capacity; + for (int i = 0; i < old_capacity; i++) { + if (old[i].hash != 0) { + *py_param_slot(state, old[i].name, old[i].hash) = old[i]; + } + } + return true; +} + +static bool py_param_stack_reserve(WalkState *state) { + if (state->py_param_stack_count < state->py_param_stack_capacity) { + return true; + } + int new_capacity = state->py_param_stack_capacity * PAIR_LEN; + if (new_capacity <= state->py_param_stack_capacity) { + return false; + } + const char **grown = + (const char **)cbm_arena_alloc(state->arena, (size_t)new_capacity * sizeof(*grown)); + if (!grown) { + return false; + } + memcpy(grown, state->py_param_stack, (size_t)state->py_param_stack_count * sizeof(*grown)); + state->py_param_stack = grown; + state->py_param_stack_capacity = new_capacity; + return true; +} + +/* Bind one parameter name for the lifetime of the frame currently on top. */ +static void py_param_bind(WalkState *state, const char *name) { + if (state->py_param_tracking_failed || !name || !name[0]) { + return; + } + /* Grow before inserting: the probe below must always find a free slot, and + * a table above ~70% load degrades toward linear probing. */ + if ((state->py_param_slot_used + 1) * 10 >= state->py_param_slot_capacity * 7) { + if (!py_param_grow(state)) { + state->py_param_tracking_failed = true; + return; + } + } + if (!py_param_stack_reserve(state)) { + state->py_param_tracking_failed = true; + return; + } + uint32_t hash = py_param_hash(name); + CBMParamSlot *slot = py_param_slot(state, name, hash); + if (slot->hash == 0) { + slot->hash = hash; + slot->name = name; + slot->count = 0; + state->py_param_slot_used++; + } + slot->count++; + state->py_param_stack[state->py_param_stack_count++] = name; +} + +/* Unwind to a frame's entry height, undoing exactly what that frame bound. */ +static void py_param_unwind_to(WalkState *state, int base) { + if (state->py_param_tracking_failed) { + return; + } + while (state->py_param_stack_count > base) { + const char *name = state->py_param_stack[--state->py_param_stack_count]; + CBMParamSlot *slot = py_param_slot(state, name, py_param_hash(name)); + if (slot->hash != 0 && slot->count > 0) { + slot->count--; + } + } +} + +bool cbm_walk_python_param_is_bound(const WalkState *state, const char *name) { + if (!state || !name || !name[0] || state->py_param_tracking_failed || + state->py_param_slot_used == 0) { + return false; + } + const CBMParamSlot *slot = py_param_slot((WalkState *)state, name, py_param_hash(name)); + return slot->hash != 0 && slot->count > 0; +} + +/* The identifier a Python parameter node binds. Handles the bare, typed, + * defaulted, keyword-only, *args and **kwargs shapes. */ +static const char *py_parameter_name(CBMExtractCtx *ctx, TSNode param) { + if (ts_node_is_null(param)) { + return NULL; + } + if (strcmp(ts_node_type(param), "identifier") == 0) { + return cbm_node_text(ctx->arena, param, ctx->source); + } + TSNode name = ts_node_child_by_field_name(param, TS_FIELD("name")); + if (!ts_node_is_null(name) && strcmp(ts_node_type(name), "identifier") == 0) { + return cbm_node_text(ctx->arena, name, ctx->source); + } + /* `*args` / `**kwargs`, and any typed shape without a `name` field: the + * bound identifier is the first named child. */ + TSNode first = ts_node_named_child(param, 0); + if (!ts_node_is_null(first) && strcmp(ts_node_type(first), "identifier") == 0) { + return cbm_node_text(ctx->arena, first, ctx->source); + } + return NULL; +} + +/* Bind this node's parameters into the frame just pushed for it. Called after + * every push in push_boundary_scopes, so the top frame is this node's own and + * pop_expired_scopes unwinds them at exactly the right depth. */ +static void py_bind_scope_parameters(CBMExtractCtx *ctx, TSNode node, WalkState *state) { + if (ctx->language != CBM_LANG_PYTHON || state->scope_top == 0) { + return; + } + const char *kind = ts_node_type(node); + if (strcmp(kind, "function_definition") != 0 && strcmp(kind, "lambda") != 0) { + return; + } + TSNode params = ts_node_child_by_field_name(node, TS_FIELD("parameters")); + if (ts_node_is_null(params)) { + return; + } + uint32_t count = ts_node_named_child_count(params); + for (uint32_t i = 0; i < count; i++) { + py_param_bind(state, py_parameter_name(ctx, ts_node_named_child(params, i))); + } +} + static bool push_scope(WalkState *state, uint8_t kind, uint32_t depth, const char *qn) { if (!ensure_scope_capacity(state)) { return false; } CBMWalkScope *f = &state->scopes[state->scope_top]; + f->prev_py_param_stack_count = state->py_param_stack_count; f->kind = kind; f->depth = depth; f->qn = qn; @@ -260,6 +445,7 @@ static void pop_expired_scopes(WalkState *state, uint32_t cur_depth) { state->inside_import = f->prev_inside_import; state->loop_depth = f->prev_loop_depth; state->branch_depth = f->prev_branch_depth; + py_param_unwind_to(state, f->prev_py_param_stack_count); } } @@ -2354,6 +2540,7 @@ static void push_boundary_scopes(CBMExtractCtx *ctx, TSNode node, const CBMLangS } push_lexical_boundary(node, state, depth); + py_bind_scope_parameters(ctx, node, state); push_call_scope(state, depth, invocation); if (is_actual_import_boundary(ctx, node, spec) && !is_export_of_declaration(node)) { push_scope(state, SCOPE_IMPORT, depth, NULL); @@ -2387,6 +2574,14 @@ void cbm_extract_unified(CBMExtractCtx *ctx) { state.scope_capacity = MAX_SCOPES; state.lexical_scopes = state.inline_lexical_scopes; state.lexical_scope_capacity = INLINE_LEXICAL_SCOPES; + state.py_param_slots = state.inline_py_param_slots; + state.py_param_slot_capacity = INLINE_PY_PARAM_SLOTS; + state.py_param_slot_used = 0; + memset(state.inline_py_param_slots, 0, sizeof(state.inline_py_param_slots)); + state.py_param_stack = state.inline_py_param_stack; + state.py_param_stack_capacity = INLINE_PY_PARAM_STACK; + state.py_param_stack_count = 0; + state.py_param_tracking_failed = false; state.usage_start_index = ctx->result->usages.count; state.root_lexical_scope_id = add_lexical_scope(&state, ctx->root, CBM_LEXICAL_SCOPE_MODULE); /* Base walk-state tuple (previously established by the first diff --git a/internal/cbm/extract_unified.h b/internal/cbm/extract_unified.h index 820d2dc0a..f80237229 100644 --- a/internal/cbm/extract_unified.h +++ b/internal/cbm/extract_unified.h @@ -18,6 +18,8 @@ #define INLINE_LEXICAL_SCOPES 64 #define INLINE_LEXICAL_BINDINGS 64 #define INLINE_PYTHON_DIRECTIVES 16 +#define INLINE_PY_PARAM_SLOTS 64 +#define INLINE_PY_PARAM_STACK 64 // ObjectScript type map: variable name → class name (for instance_method_call // resolution). Stack-allocated, per-method scope. Overflow is silent (no crash). @@ -51,6 +53,15 @@ typedef struct { bool raw_call_emitted; } CBMInvocationDescriptor; +/* One name bound as a function/lambda parameter by a scope currently OPEN on + * the walk stack. A count, not a flag: `def outer(run): def inner(run):` binds + * the same name twice and the inner pop must not unbind the outer. */ +typedef struct { + const char *name; + uint32_t hash; + int count; +} CBMParamSlot; + typedef struct { const char *qn; uint32_t depth; @@ -73,6 +84,9 @@ typedef struct { bool prev_inside_import; int prev_loop_depth; int prev_branch_depth; + /* #1912: py_param_stack height on entry. Pop unwinds back to it, so a + * frame unbinds exactly the parameters it bound and nothing else. */ + int prev_py_param_stack_count; } CBMWalkScope; typedef enum { @@ -158,11 +172,43 @@ typedef struct { CBMPythonDirective inline_python_directives[INLINE_PYTHON_DIRECTIVES]; int python_directive_capacity; int python_directive_count; + /* #1912 -- Python bare-call shadowing. A name bound as a parameter by ANY + * enclosing function or lambda shadows every project function, so a bare + * `run()` under `def outer(run)` cannot honestly resolve by short name. + * + * Maintained as a live name->count map pushed and popped BY THE WALK, not + * recomputed per call. Walking ancestors (either ts_node_parent or a + * cursor) and scanning the frame stack are both O(depth) per call, and + * since every level of f(f(f(...))) is itself a bare call that is + * quadratic across the file -- the same trap CBMWalkScope records above. + * Lookup here is O(1), so no hop cap and no fail-open cap are needed. + * + * The other binding table (CBMLexicalBinding) cannot serve this: it is + * qsort-ed in cbm_finalize_lexical_usages AFTER the walk and its + * active_end stays 0 until then, so its binary search is invalid from + * handle_calls, which runs mid-walk and before handle_usages. */ + CBMParamSlot *py_param_slots; + CBMParamSlot inline_py_param_slots[INLINE_PY_PARAM_SLOTS]; + int py_param_slot_capacity; + int py_param_slot_used; + const char **py_param_stack; + const char *inline_py_param_stack[INLINE_PY_PARAM_STACK]; + int py_param_stack_capacity; + int py_param_stack_count; + /* Allocation failure: stop tracking and answer "not bound" forever after, + * which can only cost a suppression, never a true edge. */ + bool py_param_tracking_failed; + CBMLanguage language; os_type_map_t os_type_map; // ObjectScript variable → type mapping } WalkState; +/* #1912: is `name` bound as a parameter by a Python function or lambda scope + * currently open on the walk stack? O(1). Answers false on any failure, so a + * caller can only ever lose a suppression, never a true edge. */ +bool cbm_walk_python_param_is_bound(const WalkState *state, const char *name); + // Per-node handler prototypes. Each is called once per node during the // unified cursor walk, replacing the old recursive walk_* functions. CBMInvocationDescriptor handle_calls(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec, diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 654bfe406..55000a395 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -1225,19 +1225,18 @@ TEST(form_procedure) { /* --- Oracle PL/SQL --- */ TEST(plsql_package_and_call) { - const char *src = - "CREATE OR REPLACE PACKAGE BODY emp_pkg AS\n" - " FUNCTION hire(p_name VARCHAR2) RETURN NUMBER IS\n" - " v_sal NUMBER;\n" - " BEGIN\n" - " v_sal := util_pkg.calc_salary(p_name);\n" - " IF v_sal > 0 THEN\n" - " RETURN v_sal;\n" - " END IF;\n" - " RAISE no_data_found;\n" - " END;\n" - "END emp_pkg;\n" - "/\n"; + const char *src = "CREATE OR REPLACE PACKAGE BODY emp_pkg AS\n" + " FUNCTION hire(p_name VARCHAR2) RETURN NUMBER IS\n" + " v_sal NUMBER;\n" + " BEGIN\n" + " v_sal := util_pkg.calc_salary(p_name);\n" + " IF v_sal > 0 THEN\n" + " RETURN v_sal;\n" + " END IF;\n" + " RAISE no_data_found;\n" + " END;\n" + "END emp_pkg;\n" + "/\n"; CBMFileResult *r = extract(src, CBM_LANG_PLSQL, "t", "emp_pkg.pkb"); ASSERT_NOT_NULL(r); ASSERT_FALSE(r->has_error); @@ -2202,9 +2201,10 @@ TEST(dbt_source_and_two_arg_ref) { /* Both dbt builtins name the relation in their LAST string argument: * source('group','table') -> table, and the two-argument * ref('package','model') form -> model. */ - CBMFileResult *r = extract("SELECT * FROM {{ source('raw', 'customers') }}\n" - "UNION ALL SELECT * FROM {{ ref('analytics', 'legacy_customers') }}\n", - CBM_LANG_SQL, "t", "models/stg_customers.sql"); + CBMFileResult *r = + extract("SELECT * FROM {{ source('raw', 'customers') }}\n" + "UNION ALL SELECT * FROM {{ ref('analytics', 'legacy_customers') }}\n", + CBM_LANG_SQL, "t", "models/stg_customers.sql"); ASSERT_NOT_NULL(r); ASSERT(has_def(r, "Model", "stg_customers")); ASSERT(has_usage(r, "customers")); @@ -3276,8 +3276,8 @@ TEST(vue_embedded_structure_negative_controls_issue1410) { } TEST(vue_embedded_structure_host_controls_issue1410) { - CBMFileResult *plain = extract("function plainTs(): void { target(); }\n", CBM_LANG_TYPESCRIPT, - "t", "plain.ts"); + CBMFileResult *plain = + extract("function plainTs(): void { target(); }\n", CBM_LANG_TYPESCRIPT, "t", "plain.ts"); ASSERT_NOT_NULL(plain); ASSERT_FALSE(plain->has_error); ASSERT_EQ(count_defs_named(plain, "Function", "plainTs"), 1); @@ -3735,11 +3735,11 @@ TEST(extract_java_method_annotations_issue382) { /* ── ArkTS (HarmonyOS .ets) ─────────────────────────────────────── */ TEST(arkts_component_struct) { - CBMFileResult *r = extract( - "@Entry\n@Component\nstruct Index {\n @State message: string = 'Hello'\n\n" - " build() {\n Column() {\n Text(this.message).fontSize(20)\n }\n" - " .width('100%')\n }\n}\n", - CBM_LANG_ARKTS, "t", "Index.ets"); + CBMFileResult *r = + extract("@Entry\n@Component\nstruct Index {\n @State message: string = 'Hello'\n\n" + " build() {\n Column() {\n Text(this.message).fontSize(20)\n }\n" + " .width('100%')\n }\n}\n", + CBM_LANG_ARKTS, "t", "Index.ets"); ASSERT_NOT_NULL(r); ASSERT_FALSE(r->has_error); ASSERT(has_def(r, "Struct", "Index")); @@ -3770,12 +3770,12 @@ TEST(arkts_exported_struct_decorators) { } TEST(arkts_member_decorators) { - CBMFileResult *r = extract( - "@Component\nstruct S {\n @State a: number = 0\n @Prop b: string\n" - " @Link c: boolean\n @Provide('k') d: string = ''\n @Consume('k') e: string\n" - " @StorageLink('s') f: number = 1\n @State @Watch('onW') g: boolean = false\n\n" - " build() {\n }\n}\n", - CBM_LANG_ARKTS, "t", "S.ets"); + CBMFileResult *r = + extract("@Component\nstruct S {\n @State a: number = 0\n @Prop b: string\n" + " @Link c: boolean\n @Provide('k') d: string = ''\n @Consume('k') e: string\n" + " @StorageLink('s') f: number = 1\n @State @Watch('onW') g: boolean = false\n\n" + " build() {\n }\n}\n", + CBM_LANG_ARKTS, "t", "S.ets"); ASSERT_NOT_NULL(r); ASSERT_FALSE(r->has_error); ASSERT(decorators_contain(find_def_by_name(r, "a"), "State")); @@ -3815,11 +3815,11 @@ TEST(arkts_no_phantom_builtin_defs) { } TEST(arkts_builder_extend_styles) { - CBMFileResult *r = extract( - "@Builder\nfunction card(t: string) {\n Column() {\n Text(t)\n }\n}\n\n" - "@Extend(Text)\nfunction fancy(size: number) {\n .fontSize(size)\n}\n\n" - "@Styles\nfunction pressed() {\n .backgroundColor('#eee')\n}\n", - CBM_LANG_ARKTS, "t", "b.ets"); + CBMFileResult *r = + extract("@Builder\nfunction card(t: string) {\n Column() {\n Text(t)\n }\n}\n\n" + "@Extend(Text)\nfunction fancy(size: number) {\n .fontSize(size)\n}\n\n" + "@Styles\nfunction pressed() {\n .backgroundColor('#eee')\n}\n", + CBM_LANG_ARKTS, "t", "b.ets"); ASSERT_NOT_NULL(r); ASSERT_FALSE(r->has_error); ASSERT(has_def(r, "Function", "card")); @@ -4903,17 +4903,22 @@ TEST(extract_python_bare_call_flags_locally_bound_callee) { PASS(); } -/* The scope walk behind the bare-call flag is BOUNDED (64 ancestors). An - * unbounded walk is QUADRATIC in a file's nesting depth -- every level of - * f(f(f(...))) is itself a bare call re-walking its own chain -- which hung - * stack_overflow_b's 30,000-deep fixture rather than merely slowing it. +/* The bare-call flag is DEPTH-INDEPENDENT, and the binding is UNWOUND when its + * scope closes. * - * This pins the cap's CONTRACT deterministically rather than by wall clock (a - * timing assertion would be a lottery, not a gate): within the cap the shadowed - * callee is flagged; past it the guard FAILS OPEN and leaves the call alone, so - * the cap can only ever cost a suppression, never a true edge. Removing the cap - * flips the deep case to flagged and fails this test. */ -TEST(extract_python_bare_call_scope_walk_is_bounded) { + * Both properties come from the same design decision. The answer is carried by + * the unified walk -- parameters are bound when a def or lambda scope opens and + * unwound when it closes -- rather than recomputed per call by ascending the + * tree. An ascending walk is O(depth) per call, and every level of f(f(f(...))) + * is itself a bare call, so it is quadratic in a file's nesting depth; that hung + * stack_overflow_b's 30,000-deep fixture rather than merely slowing it. An + * earlier cut capped the ascent at 64 ancestors and FAILED OPEN past it, which + * silently stopped suppressing on deep-but-ordinary code. + * + * Pinned deterministically rather than by wall clock -- a timing assertion would + * be a lottery, not a gate. The depth case fails if a cap is reintroduced; the + * unwind case fails if a frame's bindings outlive its scope. */ +TEST(extract_python_bare_call_flag_is_depth_independent) { /* Shallow: return_statement / block / function_definition — 3 ancestors. */ CBMFileResult *shallow = extract("def shallow(handler):\n" " return handler()\n", @@ -4931,9 +4936,9 @@ TEST(extract_python_bare_call_scope_walk_is_bounded) { ASSERT_EQ(shallow_seen, 1); cbm_free_result(shallow); - /* Deep: 200 parenthesized_expression ancestors put the SAME call well past - * the cap, so the enclosing parameter is never reached and the call stays - * unflagged. */ + /* Deep: 200 parenthesized_expression ancestors separate the SAME call from + * its enclosing def. The parameter still shadows it, so it stays flagged -- + * depth changes nothing. This is the case a 64-ancestor cap got wrong. */ const int PARENS = 200; size_t sz = (size_t)PARENS * 2 + 128; char *src = malloc(sz); @@ -4955,12 +4960,51 @@ TEST(extract_python_bare_call_scope_walk_is_bounded) { const char *cn = deep->calls.items[i].callee_name; if (cn && strcmp(cn, "handler") == 0) { deep_seen++; - ASSERT_FALSE(deep->calls.items[i].callee_is_locally_bound); + ASSERT_TRUE(deep->calls.items[i].callee_is_locally_bound); } } ASSERT_EQ(deep_seen, 1); cbm_free_result(deep); free(src); + + /* Unwind: `handler` is a parameter of shadowed() and a module-level function + * of the same name. The call INSIDE shadowed() is flagged; the call in + * sibling(), after that scope closed, must NOT be — it really does resolve + * to the module-level def. A binding that outlived its frame would flag it + * and destroy a true edge, which is the one direction this guard must never + * fail in. Nested same-name defs also pin the count: leaving the inner scope + * must not unbind the outer one. */ + CBMFileResult *unwound = extract("def handler():\n" + " return 1\n" + "\n" + "def shadowed(handler):\n" + " def inner(handler):\n" + " return handler()\n" + " return inner(handler) or handler()\n" + "\n" + "def sibling():\n" + " return handler()\n", + CBM_LANG_PYTHON, "t", "u.py"); + ASSERT_NOT_NULL(unwound); + ASSERT_FALSE(unwound->has_error); + int flagged = 0; + int unflagged = 0; + for (int i = 0; i < unwound->calls.count; i++) { + const char *cn = unwound->calls.items[i].callee_name; + if (!cn || strcmp(cn, "handler") != 0) { + continue; + } + if (unwound->calls.items[i].callee_is_locally_bound) { + flagged++; + } else { + unflagged++; + } + } + /* Two shadowed calls (inner body, and shadowed()'s own tail) and exactly one + * unshadowed call in sibling(). */ + ASSERT_EQ(flagged, 2); + ASSERT_EQ(unflagged, 1); + cbm_free_result(unwound); PASS(); } @@ -5372,8 +5416,8 @@ TEST(extract_c_test_dir_marks_is_test_issue1294) { * not be (#1294). */ TEST(extract_python_method_test_dir_marks_is_test_issue1294) { const char *src = "class Foo:\n" - " def helper(self):\n" - " pass\n"; + " def helper(self):\n" + " pass\n"; /* Python's LSP layer injects synthetic builtin stub Methods (str.upper, * dict.get, ...) into defs.items alongside real ones (py_builtins.c), so @@ -6644,7 +6688,7 @@ SUITE(extraction) { RUN_TEST(extract_flag_exempt_method_call_not_flagged_is_method); RUN_TEST(extract_python_member_call_flags_is_method); RUN_TEST(extract_python_bare_call_flags_locally_bound_callee); - RUN_TEST(extract_python_bare_call_scope_walk_is_bounded); + RUN_TEST(extract_python_bare_call_flag_is_depth_independent); RUN_TEST(extract_ts_member_call_flags_is_method); RUN_TEST(extract_ts_this_super_receiver_not_flagged); RUN_TEST(extract_js_member_call_flags_is_method); From 40f2722d6166c1f97fa1667c1e63e8c97e6f05f0 Mon Sep 17 00:00:00 2001 From: Daniel Ochoa Date: Wed, 2 Sep 2026 07:59:32 -0500 Subject: [PATCH 21/22] fix(extract): AST traversal stacks no longer outlive the file they walk Fixes #2010. Refs #1997. Indexing a 14k-file TypeScript repository holds 13.3 GB resident at post_extract on Linux aarch64, and about 3.1 GB of it is AST traversal scratch that no CBMFileResult points at. ts_nstack_init cut its stacks from ctx->arena, which is result->arena. In the parallel path that arena is stored into ec->result_cache[file_idx] (pass_parallel.c:963) and freed only after parallel_resolve and the infra passes (pipeline.c:1324), so a structure written for a one-file lifetime is in fact held for the whole index, once per file, concurrently. The header claimed otherwise and was wrong: "freed when the arena is destroyed at end of file extraction". The largest instance is channels. CHAN_STACK_CAP is 4096 and sizeof(TSNode) is 32, so scan_string_consts_js (extract_channels.c:104) and extract_channels_js (:376) take 128 KB each, per JavaScript or TypeScript file. Across 12,673 such files that is 3,168 MB, and it produced 23 Channel nodes out of 164,684. CBMExtractCtx gains a scratch arena, created and destroyed by the cbm_extract_file_ex call that builds the context. Nothing else changes about where anything lives: ctx->arena still owns every string a CBMFileResult points at, and ctx->scratch owns only traversal stacks, which nothing points at. Nothing a CBMFileResult points at moved. A context built without a scratch, and a file whose scratch arena fails to allocate, both fall back to ctx->arena, which is exactly the behaviour that shipped before this change, so no path can be handed a NULL arena and silently lose nodes. The signature change is what makes the mistake unrepresentable. ts_nstack_init takes the context rather than an arena, so handing it ctx->arena is an incompatible pointer type; push and push_children read the arena the stack recorded, so a stale argument is an arity error. Both fail under -Werror rather than silently retaining memory. The 55 init sites are NOT one uniform sed. 29 pass ctx->arena and 26 pass a local alias, and that alias is shared with result allocations: in parse_zig_imports (extract_imports.c:1093) the same `a` feeds cbm_node_text, strip_quotes, path_last and cbm_imports_push (extract_imports.c:1101, 1106, 1108, 1109). Rebinding it would have moved import names and module paths into memory that dies with the call while the result kept pointing at them. Each site was edited on its own and no alias was rebound. Re-derive the set with: grep -rn 'ts_nstack_init' internal/cbm/extract_*.c grep -rn 'ts_nstack_push' internal/cbm/extract_*.c parse_lisp_imports used its alias only for the stack, so the alias goes too. push_nested_class_nodes and push_class_body_children took an arena purely to reach the stack, so they take the context now; the one call site passes ctx. Two measurements, from two different builds, and they are not the same number. A macOS census build, which instruments every cbm_arena_alloc and sums the blocks behind every live CBMFileResult arena, reports the retained per-file arena set dropping from 11,226 MB mapped to 5,680 MB, and blocks from 48,045 to 34,186. Those are mapped bytes, not resident: that build logs mem.allocator.bound_populations_only, so untouched block tails never commit. The resident figures come from Linux aarch64, gcc 13.3, static build, where the build logs mem.allocator.owned classes=all and mapped equals resident. Five runs per side on the full 13,999-file tree: mem.collect post_extract rss_mb 13293 -> 6752 (-49.2%), mem.phase peak_mb 14840 -> 8565 (-42.3%). nodes 164684, parse_partial 280 and registry entries/defines/imports 787954/804999/100027 identical on all ten runs; edges 835869-835874 against 835867-835881, overlapping ranges, so run to run noise. Wall clock means 35651 ms against 30502 ms, but the ranges overlap and the host had other load, so the only supportable claim is no regression. ObjectScript Studio Export on a small fixture: nodes 18, edges 25, both sides. Scratch block size is 512 KB, chosen from measurement. arena_grow fires on one file in 12,000 at both 512 KB and 1 MB, and constantly at 256 KB, where the two channel stacks alone are exactly 262144 bytes. 512 KB therefore costs half the resident block per worker for the same growth behaviour, and stays under MI_LARGE_MAX_OBJ_SIZE in the vendored mimalloc, which is MI_LARGE_PAGE_SIZE/8, exactly 524288 bytes (types.h:426, with MI_ENABLE_LARGE_PAGES defaulting to 1 at :115 and not overridden by this build), so a 512 KB block is still bin-allocated from a 4 MiB large page. Growth is not free at that size for the same reason: arena_grow doubles to 1 MiB, above that bound and therefore a singleton OS allocation. One file in twelve thousand pays it, which is why it is accepted. The largest single file's scratch high water mark on this corpus is 33.5 MB, now transient rather than retained. Tests. Nothing in the suite asserted on result->arena size, block count or total_alloc after cbm_extract_file, which is why this went unnoticed for the life of extract_node_stack.h. traversal_stack_not_in_result_arena_issue2010 pins the byte budget: extracting "export const x = 1;" as TypeScript charged 365984 bytes to the result arena before and charges 87456 after, a difference of 278528, exactly the two 4096-entry channel walks plus the 512-entry ES import walk at 32 bytes per TSNode. Of the 87456 that remain, 7680 is the defs item array at GROW_ARRAY's starting capacity of 32 times sizeof(CBMDefinition) 240 and the other 79776 is everything else the extraction interns, none of it traversal scratch; the 128 KB bound is therefore above the new figure with room and a factor of four below the old one. extract_traversal_stacks_come_from_ctx_scratch_issue2010 in test_mem.c pins the lifetime, since a byte budget alone would also be satisfied by shrinking CHAN_STACK_CAP: it builds the extraction context directly over two arenas it owns, runs cbm_extract_channels, and asserts the scratch took at least the 262144 bytes of the two walks while the result arena did not. extract_c_macro_hidden_call_survives_preprocessed_pass_issue2010 is not a scratch test. pp_ctx carries ctx->scratch so every context in the file is uniform, but nothing reads it there: pp_ctx reaches only cbm_extract_unified and cbm_run_c_lsp, and neither extract_unified.c nor anything under internal/cbm/lsp/ includes extract_node_stack.h, so no traversal stack is built on that path today. The test guards the macro-expansion path itself, which had no assertion on a call that exists only after expansion. ObjectScript composite extraction is already covered by pipeline_objectscript_export_preserves_calls_sequential_parallel and iris_export_xml_multi_class, so no new test is added there; a per-call arena is correct on that path by construction, since each call owns and destroys its own. Local: scripts/test.sh, 7558 passed, 0 failed, 8 skipped, 139 suites. Alternatives considered Thread-local scratch arena, reset per file, with a depth guard for nesting. Rejected on three counts. This codebase never uses thread-local storage for owned bulk memory; every existing _Thread_local here is a small cache, a guard or a log buffer, and every per-worker resource is a field in a context struct. It would have needed cbm_arena_reset, which has no production caller and a documented subtlety with grown block sizes (arena.c:213-217). And extraction threads are created per cbm_parallel_for call and joined (worker_pool.c:71,88) with no thread-exit hook in this project's own code, so the arena would have leaked once per thread per dispatch and turned the leak lane red. Save and restore mark and release on the arena. Rejected because a per-call arena already makes nesting correct by construction, and a mark type plus two functions would have to be added to both arena headers. Per-walk arenas, one per traversal rather than one per file. Rejected on cost: 55 sites would each need an explicit span, for the same bound. Right-sizing the initial capacities alone. CHAN_STACK_CAP at 256 rather than 4096 would cut those two sites from 3,168 MB to about 198 MB, which is real, but it tunes the constant while the lifetime defect remains at all 55 sites, and growth is not free: 256 doubling to 4096 abandons 122,880 bytes, and one file in this corpus took 33.5 MB of stacks. The cap and the scratch block size are coupled, which is a further reason to change one at a time. Worth doing separately. Routing to a scratch inside extract_node_stack.h while keeping the arena parameters. Rejected because a parameter that is ignored lies about the API, and the alias hazard above would stay a review obligation instead of a compile error. Freeing the per-file result arena earlier. Rejected: the results are read by parallel_resolve and the infra passes. That is the retention half #1925 addresses, and this change is independent of it. Signed-off-by: Daniel Ochoa --- internal/cbm/cbm.c | 52 ++++++- internal/cbm/cbm.h | 7 + internal/cbm/extract_channels.c | 60 ++++---- internal/cbm/extract_defs.c | 26 ++-- internal/cbm/extract_env_accesses.c | 6 +- internal/cbm/extract_imports.c | 217 ++++++++++++++-------------- internal/cbm/extract_node_stack.h | 41 ++++-- internal/cbm/extract_semantic.c | 12 +- internal/cbm/extract_type_assigns.c | 6 +- internal/cbm/extract_type_refs.c | 12 +- tests/test_extraction.c | 54 +++++++ tests/test_mem.c | 52 +++++++ 12 files changed, 359 insertions(+), 186 deletions(-) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index d7468af5a..9b4d16977 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -1,3 +1,7 @@ +/* Full declaration set for the same CBMArena, and it must precede cbm.h: + * internal/cbm/arena.h declares a subset and the two share the CBM_ARENA_H + * guard, so whichever is included first is the one this file sees. */ +#include "foundation/arena.h" // cbm_arena_init_sized #include "cbm.h" #include "arena.h" // CBMArena, cbm_arena_init/alloc/strdup/destroy #include "helpers.h" @@ -1162,11 +1166,27 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage return r; } -CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLanguage language, - const char *project, const char *rel_path, - int64_t timeout_micros, const char **extra_defines, - const char **include_paths, const CBMMacroTable *macro_table, - const CBMReturnTypeTable *return_type_table) { +/* Initial block for the per-file traversal scratch arena, chosen by measuring + * arena_grow on a 14k-file TypeScript tree: it fires on one file in 12,000 at + * both this size and at 1 MB, and on most files at 256 KB, where the two + * channel walks alone are exactly 262144 bytes. 512 KB therefore buys the same + * growth behaviour as 1 MB for half the resident block per worker. It is also + * exactly MI_LARGE_MAX_OBJ_SIZE in the vendored mimalloc + * (vendored/mimalloc/include/mimalloc/types.h:426, MI_LARGE_PAGE_SIZE/8 with + * MI_ENABLE_LARGE_PAGES defaulting to 1 at :115 and not overridden here), so + * the block is still bin-allocated from a large page. Growth is not free at + * this size for the same reason: arena_grow doubles to 1 MiB, which is above + * that bound and so a singleton OS allocation. One file in twelve thousand + * pays it, which is why the cost is accepted. */ +enum { CBM_EXTRACT_SCRATCH_BLOCK = CBM_SZ_512 * CBM_SZ_1K }; + +static CBMFileResult *extract_file_ex_body(const char *source, int source_len, CBMLanguage language, + const char *project, const char *rel_path, + int64_t timeout_micros, const char **extra_defines, + const char **include_paths, + const CBMMacroTable *macro_table, + const CBMReturnTypeTable *return_type_table, + CBMArena *scratch) { // Allocate result on heap (arena inside for all string data) enum { SINGLE = 1 }; CBMFileResult *result = (CBMFileResult *)calloc(SINGLE, sizeof(CBMFileResult)); @@ -1277,6 +1297,7 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua // Build extraction context CBMExtractCtx ctx = { .arena = a, + .scratch = scratch, .result = result, .source = source, .source_len = source_len, @@ -1398,6 +1419,7 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua // Build context for expanded source — extract only calls via unified extractor CBMExtractCtx pp_ctx = { .arena = a, + .scratch = scratch, .result = result, .source = expanded, .source_len = expanded_len, @@ -1634,6 +1656,26 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua return result; } +/* Public entry. Owns the traversal scratch arena for the whole of one file's + * extraction: created here, handed to the body as ctx->scratch, destroyed on + * the way out. The body has seven early returns, so bracketing it in a wrapper + * is what keeps that to one create and one destroy. If the arena cannot be + * created, the body is handed NULL and the traversal stacks fall back to the + * result arena, which is what shipped before #1997. */ +CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLanguage language, + const char *project, const char *rel_path, + int64_t timeout_micros, const char **extra_defines, + const char **include_paths, const CBMMacroTable *macro_table, + const CBMReturnTypeTable *return_type_table) { + CBMArena scratch; + cbm_arena_init_sized(&scratch, CBM_EXTRACT_SCRATCH_BLOCK); + CBMFileResult *result = extract_file_ex_body( + source, source_len, language, project, rel_path, timeout_micros, extra_defines, + include_paths, macro_table, return_type_table, scratch.nblocks > 0 ? &scratch : NULL); + cbm_arena_destroy(&scratch); + return result; +} + void cbm_free_result(CBMFileResult *result) { if (!result) { return; diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 440d80fce..b4f09a90f 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -574,6 +574,13 @@ typedef struct { typedef struct { CBMArena *arena; + /* Scratch for AST traversal, owned by the cbm_extract_file_ex call that + * built this context and destroyed when it returns. Nothing a + * CBMFileResult points at may be allocated here: `arena` is the result's + * own, and it outlives extraction by the whole pipeline (#1997). NULL in a + * context built without one, in which case the stacks fall back to + * `arena`. */ + CBMArena *scratch; CBMFileResult *result; const char *source; int source_len; diff --git a/internal/cbm/extract_channels.c b/internal/cbm/extract_channels.c index c12d4ebeb..493133a13 100644 --- a/internal/cbm/extract_channels.c +++ b/internal/cbm/extract_channels.c @@ -101,8 +101,8 @@ static const char *literal_from_first_child(CBMExtractCtx *ctx, TSNode node) { * const table per file is sufficient for the common Socket.IO pattern). */ static void scan_string_consts_js(CBMExtractCtx *ctx, chan_const_table_t *tbl) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP); - ts_nstack_push(&stack, ctx->arena, ctx->root); + ts_nstack_init(&stack, ctx, CHAN_STACK_CAP); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0 && tbl->count < CHAN_CONST_CAP) { TSNode node = ts_nstack_pop(&stack); @@ -128,15 +128,15 @@ static void scan_string_consts_js(CBMExtractCtx *ctx, chan_const_table_t *tbl) { } } - ts_nstack_push_children(&stack, ctx->arena, node); + ts_nstack_push_children(&stack, node); } } /* Python constant resolution: NAME = "value" (assignment node). */ static void scan_string_consts_python(CBMExtractCtx *ctx, chan_const_table_t *tbl) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP); - ts_nstack_push(&stack, ctx->arena, ctx->root); + ts_nstack_init(&stack, ctx, CHAN_STACK_CAP); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0 && tbl->count < CHAN_CONST_CAP) { TSNode node = ts_nstack_pop(&stack); @@ -163,7 +163,7 @@ static void scan_string_consts_python(CBMExtractCtx *ctx, chan_const_table_t *tb uint32_t count = ts_node_child_count(node); for (int i = (int)count - SKIP_ONE; i >= 0; i--) { - ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i)); + ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i)); } } } @@ -373,15 +373,15 @@ static void extract_channels_js(CBMExtractCtx *ctx) { /* Second pass: walk the tree looking for call_expression nodes. */ TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP); - ts_nstack_push(&stack, ctx->arena, ctx->root); + ts_nstack_init(&stack, ctx, CHAN_STACK_CAP); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (strcmp(ts_node_type(node), "call_expression") == 0) { js_process_call(ctx, node, &consts); } - ts_nstack_push_children(&stack, ctx->arena, node); + ts_nstack_push_children(&stack, node); } } @@ -549,8 +549,8 @@ static void extract_channels_python(CBMExtractCtx *ctx) { scan_string_consts_python(ctx, &consts); TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP); - ts_nstack_push(&stack, ctx->arena, ctx->root); + ts_nstack_init(&stack, ctx, CHAN_STACK_CAP); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); @@ -562,7 +562,7 @@ static void extract_channels_python(CBMExtractCtx *ctx) { } uint32_t count = ts_node_child_count(node); for (int i = (int)count - SKIP_ONE; i >= 0; i--) { - ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i)); + ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i)); } } } @@ -627,8 +627,8 @@ static void go_process_call(CBMExtractCtx *ctx, TSNode call) { static void extract_channels_go(CBMExtractCtx *ctx) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP); - ts_nstack_push(&stack, ctx->arena, ctx->root); + ts_nstack_init(&stack, ctx, CHAN_STACK_CAP); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); @@ -637,7 +637,7 @@ static void extract_channels_go(CBMExtractCtx *ctx) { } uint32_t count = ts_node_child_count(node); for (int i = (int)count - SKIP_ONE; i >= 0; i--) { - ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i)); + ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i)); } } } @@ -724,8 +724,8 @@ static void java_process_annotation(CBMExtractCtx *ctx, TSNode annotation) { static void extract_channels_java(CBMExtractCtx *ctx) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP); - ts_nstack_push(&stack, ctx->arena, ctx->root); + ts_nstack_init(&stack, ctx, CHAN_STACK_CAP); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); @@ -737,7 +737,7 @@ static void extract_channels_java(CBMExtractCtx *ctx) { } uint32_t count = ts_node_child_count(node); for (int i = (int)count - SKIP_ONE; i >= 0; i--) { - ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i)); + ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i)); } } } @@ -793,8 +793,8 @@ static void csharp_process_call(CBMExtractCtx *ctx, TSNode call) { static void extract_channels_csharp(CBMExtractCtx *ctx) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP); - ts_nstack_push(&stack, ctx->arena, ctx->root); + ts_nstack_init(&stack, ctx, CHAN_STACK_CAP); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); @@ -803,7 +803,7 @@ static void extract_channels_csharp(CBMExtractCtx *ctx) { } uint32_t count = ts_node_child_count(node); for (int i = (int)count - SKIP_ONE; i >= 0; i--) { - ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i)); + ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i)); } } } @@ -856,8 +856,8 @@ static void ruby_process_call(CBMExtractCtx *ctx, TSNode call) { static void extract_channels_ruby(CBMExtractCtx *ctx) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP); - ts_nstack_push(&stack, ctx->arena, ctx->root); + ts_nstack_init(&stack, ctx, CHAN_STACK_CAP); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); @@ -866,7 +866,7 @@ static void extract_channels_ruby(CBMExtractCtx *ctx) { } uint32_t count = ts_node_child_count(node); for (int i = (int)count - SKIP_ONE; i >= 0; i--) { - ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i)); + ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i)); } } } @@ -958,8 +958,8 @@ static void elixir_process_function_def(CBMExtractCtx *ctx, TSNode func_def) { static void extract_channels_elixir(CBMExtractCtx *ctx) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP); - ts_nstack_push(&stack, ctx->arena, ctx->root); + ts_nstack_init(&stack, ctx, CHAN_STACK_CAP); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); @@ -971,7 +971,7 @@ static void extract_channels_elixir(CBMExtractCtx *ctx) { } uint32_t count = ts_node_child_count(node); for (int i = (int)count - SKIP_ONE; i >= 0; i--) { - ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i)); + ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i)); } } } @@ -1034,8 +1034,8 @@ static void rust_process_call(CBMExtractCtx *ctx, TSNode call) { static void extract_channels_rust(CBMExtractCtx *ctx) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP); - ts_nstack_push(&stack, ctx->arena, ctx->root); + ts_nstack_init(&stack, ctx, CHAN_STACK_CAP); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); @@ -1044,7 +1044,7 @@ static void extract_channels_rust(CBMExtractCtx *ctx) { } uint32_t count = ts_node_child_count(node); for (int i = (int)count - SKIP_ONE; i >= 0; i--) { - ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i)); + ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i)); } } } diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index f9854b03e..e6656067b 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -5054,8 +5054,8 @@ static TSNode emit_elixir_module_class(CBMExtractCtx *ctx, TSNode cur) { static void extract_elixir_call(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec) { (void)spec; TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CBM_SZ_64); - ts_nstack_push(&stack, ctx->arena, node); + ts_nstack_init(&stack, ctx, CBM_SZ_64); + ts_nstack_push(&stack, node); while (stack.count > 0) { TSNode cur = ts_nstack_pop(&stack); @@ -5083,7 +5083,7 @@ static void extract_elixir_call(CBMExtractCtx *ctx, TSNode node, const CBMLangSp for (int di = (int)dbc - SKIP_CHAR; di >= 0; di--) { TSNode dchild = ts_node_child(do_block, (uint32_t)di); if (!ts_node_is_null(dchild) && strcmp(ts_node_type(dchild), "call") == 0) { - ts_nstack_push(&stack, ctx->arena, dchild); + ts_nstack_push(&stack, dchild); } } } @@ -6021,8 +6021,8 @@ static void extract_var_names(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec // Used by YAML, TOML, INI, JSON. static void walk_variables_iter(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CBM_SZ_256); - ts_nstack_push(&stack, ctx->arena, root); + ts_nstack_init(&stack, ctx, CBM_SZ_256); + ts_nstack_push(&stack, root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); @@ -6046,7 +6046,7 @@ static void walk_variables_iter(CBMExtractCtx *ctx, TSNode root, const CBMLangSp strcmp(ck, "section") == 0 || strcmp(ck, "object") == 0 || strcmp(ck, "array") == 0 || strcmp(ck, "pair") == 0 || strcmp(ck, "element") == 0 || strcmp(ck, "content") == 0) { - ts_nstack_push(&stack, ctx->arena, child); + ts_nstack_push(&stack, child); } } } @@ -6749,10 +6749,10 @@ static void wd_push_children_reverse(wd_stack_t *s, TSNode node, const char *enc // Push nested class nodes from a class body container onto the defs stack. // Iteratively walks into wrapper nodes (field_declaration, template_declaration). static void push_nested_class_nodes(TSNode body, const CBMLangSpec *spec, wd_stack_t *s, - const char *enclosing_qn, CBMArena *arena) { + const char *enclosing_qn, const CBMExtractCtx *ctx) { TSNodeStack nc_stack; - ts_nstack_init(&nc_stack, arena, NESTED_CLASS_STACK_CAP); - ts_nstack_push(&nc_stack, arena, body); + ts_nstack_init(&nc_stack, ctx, NESTED_CLASS_STACK_CAP); + ts_nstack_push(&nc_stack, body); while (nc_stack.count > 0) { TSNode cur = ts_nstack_pop(&nc_stack); @@ -6767,7 +6767,7 @@ static void push_nested_class_nodes(TSNode body, const CBMLangSpec *spec, wd_sta const char *ck = ts_node_type(child); if (strcmp(ck, "field_declaration") == 0 || strcmp(ck, "template_declaration") == 0 || strcmp(ck, "declaration") == 0) { - ts_nstack_push(&nc_stack, arena, child); + ts_nstack_push(&nc_stack, child); } } } @@ -6883,7 +6883,7 @@ static void extract_typescript_namespace_def(CBMExtractCtx *ctx, TSNode node, // Push nested class children from a class body container onto the walk stack. static void push_class_body_children(TSNode node, const CBMLangSpec *spec, wd_stack_t *s, - const char *new_enclosing, CBMArena *arena) { + const char *new_enclosing, const CBMExtractCtx *ctx) { /* Use the same language-aware body selection as method extraction. The old * independent spelling list omitted valid containers such as Scala's * `template_body` and Solidity's contract body. Methods were extracted @@ -6899,7 +6899,7 @@ static void push_class_body_children(TSNode node, const CBMLangSpec *spec, wd_st body = find_class_member_body(node, spec->language); } if (!ts_node_is_null(body)) { - push_nested_class_nodes(body, spec, s, new_enclosing, arena); + push_nested_class_nodes(body, spec, s, new_enclosing, ctx); return; } @@ -7423,7 +7423,7 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec, if (cbm_kind_in_set(node, spec->class_node_types)) { extract_class_def(ctx, node, spec); const char *new_enclosing = compute_class_qn(ctx, node, frame.enclosing_class_qn); - push_class_body_children(node, spec, &s, new_enclosing, ctx->arena); + push_class_body_children(node, spec, &s, new_enclosing, ctx); continue; } diff --git a/internal/cbm/extract_env_accesses.c b/internal/cbm/extract_env_accesses.c index ab0fc8f96..902d13d53 100644 --- a/internal/cbm/extract_env_accesses.c +++ b/internal/cbm/extract_env_accesses.c @@ -140,8 +140,8 @@ static bool is_env_var_name(const char *s) { // Iterative env access walker — explicit stack static void walk_env_accesses(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, 4096); - ts_nstack_push(&stack, ctx->arena, root); + ts_nstack_init(&stack, ctx, 4096); + ts_nstack_push(&stack, root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); @@ -163,7 +163,7 @@ static void walk_env_accesses(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec continue; // don't push children (avoid double-counting) } - ts_nstack_push_children(&stack, ctx->arena, node); + ts_nstack_push_children(&stack, node); } } diff --git a/internal/cbm/extract_imports.c b/internal/cbm/extract_imports.c index 8696e4b30..c31e2bbfd 100644 --- a/internal/cbm/extract_imports.c +++ b/internal/cbm/extract_imports.c @@ -506,8 +506,8 @@ static bool process_commonjs_require(CBMExtractCtx *ctx, TSNode call) { static void walk_es_imports(CBMExtractCtx *ctx, TSNode root) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CBM_SZ_512); - ts_nstack_push(&stack, ctx->arena, root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); @@ -540,7 +540,7 @@ static void walk_es_imports(CBMExtractCtx *ctx, TSNode root) { } if (push_children) { - ts_nstack_push_children(&stack, ctx->arena, node); + ts_nstack_push_children(&stack, node); } } } @@ -1093,8 +1093,8 @@ static void parse_haskell_imports(CBMExtractCtx *ctx) { static void parse_zig_imports(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (strcmp(ts_node_type(node), "builtin_function") == 0) { @@ -1111,7 +1111,7 @@ static void parse_zig_imports(CBMExtractCtx *ctx) { } } } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } @@ -1163,8 +1163,8 @@ static void process_wolfram_needs(CBMExtractCtx *ctx, TSNode node) { static void walk_wolfram_imports(CBMExtractCtx *ctx, TSNode root) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CBM_SZ_512); - ts_nstack_push(&stack, ctx->arena, root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); @@ -1176,7 +1176,7 @@ static void walk_wolfram_imports(CBMExtractCtx *ctx, TSNode root) { process_wolfram_needs(ctx, node); } - ts_nstack_push_children(&stack, ctx->arena, node); + ts_nstack_push_children(&stack, node); } } @@ -1534,8 +1534,8 @@ static void capture_namespace_decl(CBMExtractCtx *ctx) { static void parse_hare_imports(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (strcmp(ts_node_type(node), "use_statement") == 0) { @@ -1554,7 +1554,7 @@ static void parse_hare_imports(CBMExtractCtx *ctx) { } continue; } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } @@ -1566,8 +1566,8 @@ static void parse_hare_imports(CBMExtractCtx *ctx) { static void parse_pascal_imports(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (strcmp(ts_node_type(node), "declUses") == 0) { @@ -1585,7 +1585,7 @@ static void parse_pascal_imports(CBMExtractCtx *ctx) { } continue; } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } @@ -1598,8 +1598,8 @@ static void parse_pascal_imports(CBMExtractCtx *ctx) { static void parse_powershell_imports(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (strcmp(ts_node_type(node), "command") == 0) { @@ -1609,8 +1609,8 @@ static void parse_powershell_imports(CBMExtractCtx *ctx) { /* Find the last generic_token anywhere under the command — that * is the module path / namespace / assembly being imported. */ TSNodeStack inner; - ts_nstack_init(&inner, a, CBM_SZ_512); - ts_nstack_push(&inner, a, node); + ts_nstack_init(&inner, ctx, CBM_SZ_512); + ts_nstack_push(&inner, node); const char *last_tok = NULL; uint32_t last_start = 0; while (inner.count > 0) { @@ -1624,7 +1624,7 @@ static void parse_powershell_imports(CBMExtractCtx *ctx) { last_start = sb; } } - ts_nstack_push_children(&inner, a, c); + ts_nstack_push_children(&inner, c); } if (last_tok && last_tok[0]) { CBMImport imp = {.local_name = path_last(a, last_tok), @@ -1634,7 +1634,7 @@ static void parse_powershell_imports(CBMExtractCtx *ctx) { } continue; /* commands don't nest imports further */ } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } @@ -1821,17 +1821,16 @@ static void lisp_process_list(CBMExtractCtx *ctx, TSNode node) { * just root children. Stack-based (not recursive) to avoid deep-nesting stack * overflow, matching the other walkers in this file. */ static void parse_lisp_imports(CBMExtractCtx *ctx) { - CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); const char *nt = ts_node_type(node); if (strcmp(nt, "list") == 0 || strcmp(nt, "list_lit") == 0) { lisp_process_list(ctx, node); } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } @@ -1841,8 +1840,8 @@ static void parse_lisp_imports(CBMExtractCtx *ctx) { static void parse_starlark_imports(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (strcmp(ts_node_type(node), "call") == 0) { @@ -1870,7 +1869,7 @@ static void parse_starlark_imports(CBMExtractCtx *ctx) { } } } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } @@ -1881,8 +1880,8 @@ static void parse_starlark_imports(CBMExtractCtx *ctx) { static void parse_tcl_imports(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (strcmp(ts_node_type(node), "command") == 0) { @@ -1904,7 +1903,7 @@ static void parse_tcl_imports(CBMExtractCtx *ctx) { } } } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } @@ -1915,8 +1914,8 @@ static void parse_tcl_imports(CBMExtractCtx *ctx) { static void parse_teal_imports(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (strcmp(ts_node_type(node), "function_call") == 0) { @@ -1940,7 +1939,7 @@ static void parse_teal_imports(CBMExtractCtx *ctx) { } } } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } @@ -1950,8 +1949,8 @@ static void parse_teal_imports(CBMExtractCtx *ctx) { static void parse_zsh_imports(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (strcmp(ts_node_type(node), "command") == 0) { @@ -1971,7 +1970,7 @@ static void parse_zsh_imports(CBMExtractCtx *ctx) { } } } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } @@ -2058,15 +2057,15 @@ static void parse_html_imports(CBMExtractCtx *ctx) { parse_embedded_imports(ctx); TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CBM_SZ_512); - ts_nstack_push(&stack, ctx->arena, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (strcmp(ts_node_type(node), "start_tag") == 0 || strcmp(ts_node_type(node), "self_closing_tag") == 0) { html_extract_tag_src(ctx, node); } - ts_nstack_push_children(&stack, ctx->arena, node); + ts_nstack_push_children(&stack, node); } } @@ -2125,8 +2124,8 @@ static void push_string_descendant_import(CBMExtractCtx *ctx, TSNode node) { static void parse_cmake_imports(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (strcmp(ts_node_type(node), "normal_command") == 0) { @@ -2144,15 +2143,15 @@ static void parse_cmake_imports(CBMExtractCtx *ctx) { } } } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } // --- BitBake imports: require/include path --- static void parse_bitbake_imports(CBMExtractCtx *ctx) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CBM_SZ_512); - ts_nstack_push(&stack, ctx->arena, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); const char *k = ts_node_type(node); @@ -2160,7 +2159,7 @@ static void parse_bitbake_imports(CBMExtractCtx *ctx) { strcmp(k, "inherit_directive") == 0) { push_string_descendant_import(ctx, node); } - ts_nstack_push_children(&stack, ctx->arena, node); + ts_nstack_push_children(&stack, node); } } @@ -2172,8 +2171,8 @@ static void parse_bitbake_imports(CBMExtractCtx *ctx) { static void parse_meson_imports(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); const char *k = ts_node_type(node); @@ -2196,35 +2195,35 @@ static void parse_meson_imports(CBMExtractCtx *ctx) { } } } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } // --- Kconfig imports: source "path" --- static void parse_kconfig_imports(CBMExtractCtx *ctx) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CBM_SZ_512); - ts_nstack_push(&stack, ctx->arena, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (ts_node_is_named(node) && strcmp(ts_node_type(node), "source") == 0) { push_string_descendant_import(ctx, node); } - ts_nstack_push_children(&stack, ctx->arena, node); + ts_nstack_push_children(&stack, node); } } // --- GN imports: import("//path") --- static void parse_gn_imports(CBMExtractCtx *ctx) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CBM_SZ_512); - ts_nstack_push(&stack, ctx->arena, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (strcmp(ts_node_type(node), "import_statement") == 0) { push_string_descendant_import(ctx, node); } - ts_nstack_push_children(&stack, ctx->arena, node); + ts_nstack_push_children(&stack, node); } } @@ -2248,8 +2247,8 @@ static void parse_just_imports(CBMExtractCtx *ctx) { static void parse_nix_imports(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (strcmp(ts_node_type(node), "apply_expression") == 0) { @@ -2268,15 +2267,15 @@ static void parse_nix_imports(CBMExtractCtx *ctx) { } } } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } // --- Jsonnet imports: import 'path' / importstr 'path' --- static void parse_jsonnet_imports(CBMExtractCtx *ctx) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CBM_SZ_512); - ts_nstack_push(&stack, ctx->arena, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); const char *k = ts_node_type(node); @@ -2284,15 +2283,15 @@ static void parse_jsonnet_imports(CBMExtractCtx *ctx) { strcmp(k, "importbin") == 0)) { push_string_descendant_import(ctx, node); } - ts_nstack_push_children(&stack, ctx->arena, node); + ts_nstack_push_children(&stack, node); } } // --- Pkl imports: amends/extends/import "path" --- static void parse_pkl_imports(CBMExtractCtx *ctx) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CBM_SZ_512); - ts_nstack_push(&stack, ctx->arena, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); const char *k = ts_node_type(node); @@ -2300,7 +2299,7 @@ static void parse_pkl_imports(CBMExtractCtx *ctx) { strcmp(k, "importExpr") == 0) { push_string_descendant_import(ctx, node); } - ts_nstack_push_children(&stack, ctx->arena, node); + ts_nstack_push_children(&stack, node); } } @@ -2308,8 +2307,8 @@ static void parse_pkl_imports(CBMExtractCtx *ctx) { static void parse_nickel_imports(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); /* The grammar emits an anonymous `import` token followed by a @@ -2330,7 +2329,7 @@ static void parse_nickel_imports(CBMExtractCtx *ctx) { } } } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } @@ -2354,8 +2353,8 @@ static void parse_thrift_imports(CBMExtractCtx *ctx) { // --- Cap'n Proto imports: using X = import "path" --- static void parse_capnp_imports(CBMExtractCtx *ctx) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CBM_SZ_512); - ts_nstack_push(&stack, ctx->arena, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); const char *k = ts_node_type(node); @@ -2364,7 +2363,7 @@ static void parse_capnp_imports(CBMExtractCtx *ctx) { push_string_descendant_import(ctx, node); continue; /* don't double-emit from nested import_path */ } - ts_nstack_push_children(&stack, ctx->arena, node); + ts_nstack_push_children(&stack, node); } } @@ -2372,8 +2371,8 @@ static void parse_capnp_imports(CBMExtractCtx *ctx) { static void parse_dlang_imports(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (strcmp(ts_node_type(node), "import_declaration") == 0) { @@ -2387,15 +2386,15 @@ static void parse_dlang_imports(CBMExtractCtx *ctx) { } continue; } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } // --- TableGen imports: include "path.td" --- static void parse_tablegen_imports(CBMExtractCtx *ctx) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CBM_SZ_512); - ts_nstack_push(&stack, ctx->arena, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); const char *k = ts_node_type(node); @@ -2404,22 +2403,22 @@ static void parse_tablegen_imports(CBMExtractCtx *ctx) { push_string_descendant_import(ctx, node); continue; } - ts_nstack_push_children(&stack, ctx->arena, node); + ts_nstack_push_children(&stack, node); } } // --- Crystal imports: require "./path" --- static void parse_crystal_imports(CBMExtractCtx *ctx) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CBM_SZ_512); - ts_nstack_push(&stack, ctx->arena, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (ts_node_is_named(node) && strcmp(ts_node_type(node), "require") == 0) { push_string_descendant_import(ctx, node); continue; } - ts_nstack_push_children(&stack, ctx->arena, node); + ts_nstack_push_children(&stack, node); } } @@ -2427,8 +2426,8 @@ static void parse_crystal_imports(CBMExtractCtx *ctx) { static void parse_fsharp_imports(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); const char *k = ts_node_type(node); @@ -2444,7 +2443,7 @@ static void parse_fsharp_imports(CBMExtractCtx *ctx) { } continue; } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } @@ -2452,8 +2451,8 @@ static void parse_fsharp_imports(CBMExtractCtx *ctx) { static void parse_ada_imports(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (strcmp(ts_node_type(node), "with_clause") == 0) { @@ -2472,7 +2471,7 @@ static void parse_ada_imports(CBMExtractCtx *ctx) { } continue; } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } @@ -2483,8 +2482,8 @@ static void parse_ada_imports(CBMExtractCtx *ctx) { static void parse_elm_imports(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (strcmp(ts_node_type(node), "import_clause") == 0) { @@ -2506,7 +2505,7 @@ static void parse_elm_imports(CBMExtractCtx *ctx) { } continue; } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } @@ -2517,8 +2516,8 @@ static void parse_elm_imports(CBMExtractCtx *ctx) { static void parse_move_imports(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (strcmp(ts_node_type(node), "use_declaration") == 0) { @@ -2535,7 +2534,7 @@ static void parse_move_imports(CBMExtractCtx *ctx) { } continue; } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } @@ -2569,8 +2568,8 @@ static char *smali_demangle_descriptor(CBMArena *a, const char *desc) { static void parse_smali_imports(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); const char *k = ts_node_type(node); @@ -2585,7 +2584,7 @@ static void parse_smali_imports(CBMExtractCtx *ctx) { } continue; } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } @@ -2595,8 +2594,8 @@ static void parse_smali_imports(CBMExtractCtx *ctx) { static void parse_tlaplus_imports(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); const char *k = ts_node_type(node); @@ -2615,7 +2614,7 @@ static void parse_tlaplus_imports(CBMExtractCtx *ctx) { } continue; } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } @@ -2627,8 +2626,8 @@ static void parse_tlaplus_imports(CBMExtractCtx *ctx) { static void parse_vhdl_imports(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (strcmp(ts_node_type(node), "use_clause") == 0) { @@ -2646,7 +2645,7 @@ static void parse_vhdl_imports(CBMExtractCtx *ctx) { } continue; } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } @@ -2656,8 +2655,8 @@ static void parse_vhdl_imports(CBMExtractCtx *ctx) { static void parse_wit_imports(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (strcmp(ts_node_type(node), "use_item") == 0) { @@ -2675,7 +2674,7 @@ static void parse_wit_imports(CBMExtractCtx *ctx) { } continue; } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } @@ -2686,8 +2685,8 @@ static void parse_wit_imports(CBMExtractCtx *ctx) { static void parse_smithy_imports(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (strcmp(ts_node_type(node), "use_statement") == 0) { @@ -2709,7 +2708,7 @@ static void parse_smithy_imports(CBMExtractCtx *ctx) { } continue; } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } @@ -2719,8 +2718,8 @@ static void parse_smithy_imports(CBMExtractCtx *ctx) { static void parse_hyprlang_imports(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; TSNodeStack stack; - ts_nstack_init(&stack, a, CBM_SZ_512); - ts_nstack_push(&stack, a, ctx->root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (strcmp(ts_node_type(node), "source") == 0) { @@ -2737,7 +2736,7 @@ static void parse_hyprlang_imports(CBMExtractCtx *ctx) { } continue; } - ts_nstack_push_children(&stack, a, node); + ts_nstack_push_children(&stack, node); } } diff --git a/internal/cbm/extract_node_stack.h b/internal/cbm/extract_node_stack.h index d112ccc2f..0013d53b1 100644 --- a/internal/cbm/extract_node_stack.h +++ b/internal/cbm/extract_node_stack.h @@ -4,14 +4,22 @@ * Replaces fixed-size TSNode stack[] arrays that silently drop AST subtrees * when the stack overflows (GitHub issue #199). * - * Uses the arena allocator for zero-fragmentation growth: old blocks are - * abandoned (freed when the arena is destroyed at end of file extraction). - * Initial capacity matches the previous fixed caps so small files allocate - * no extra memory. + * Traversal stacks are scratch: nothing in a CBMFileResult ever points into + * one. They are therefore cut from ctx->scratch, which the enclosing + * cbm_extract_file_ex call owns and destroys on the way out, and never from + * ctx->arena, which the pipeline holds for every file until the whole result + * cache is freed (#1997). + * + * Growth abandons the old buffer in that scratch arena, which is free because + * the arena dies with the file. The initial capacities below are unchanged, but + * they are no longer what a small file costs: cbm_extract_file_ex creates the + * scratch arena's first block for every call, including calls that build no + * stack at all, and reclaims it on the way out. */ #ifndef CBM_EXTRACT_NODE_STACK_H #define CBM_EXTRACT_NODE_STACK_H +#include "cbm.h" /* CBMExtractCtx: a stack draws from ctx->scratch */ #include "arena.h" #include "tree_sitter/api.h" #include /* memcpy */ @@ -20,20 +28,31 @@ typedef struct { TSNode *items; int count; int cap; + /* The arena every allocation for this stack comes from, recorded once by + * ts_nstack_init so push() cannot be handed a different one. It is + * ctx->scratch, or ctx->arena as the fallback when the context has none. */ + CBMArena *scratch; } TSNodeStack; -/* Initialize a stack with the given initial capacity, arena-allocated. */ -static inline void ts_nstack_init(TSNodeStack *s, CBMArena *arena, int initial_cap) { +/* Initialize a stack with the given initial capacity, allocated from the + * context's traversal scratch. Taking the context rather than an arena is + * deliberate: it makes handing over ctx->arena, or a local alias of it, a type + * error rather than a retention bug nobody notices. A context built without a + * scratch falls back to ctx->arena, which is the behaviour that shipped before + * #1997, so no caller ever gets a NULL arena and silently loses nodes. */ +static inline void ts_nstack_init(TSNodeStack *s, const CBMExtractCtx *ctx, int initial_cap) { + CBMArena *arena = ctx->scratch ? ctx->scratch : ctx->arena; + s->scratch = arena; s->items = (TSNode *)cbm_arena_alloc(arena, (size_t)initial_cap * sizeof(TSNode)); s->count = 0; s->cap = s->items ? initial_cap : 0; } /* Push a node onto the stack, growing 2x if needed. */ -static inline void ts_nstack_push(TSNodeStack *s, CBMArena *arena, TSNode node) { +static inline void ts_nstack_push(TSNodeStack *s, TSNode node) { if (s->count >= s->cap) { int new_cap = s->cap ? s->cap * 2 : 512; - TSNode *new_items = (TSNode *)cbm_arena_alloc(arena, (size_t)new_cap * sizeof(TSNode)); + TSNode *new_items = (TSNode *)cbm_arena_alloc(s->scratch, (size_t)new_cap * sizeof(TSNode)); if (!new_items) return; /* OOM: best-effort, stop growing */ if (s->items && s->count > 0) { @@ -54,7 +73,7 @@ static inline TSNode ts_nstack_pop(TSNodeStack *s) { /* * Push all children of `node` so they POP in forward (source) order — a drop-in * replacement for the common idiom: - * for (int i = (int)count - 1; i >= 0; i--) ts_nstack_push(s, a, ts_node_child(node, i)); + * for (int i = (int)count - 1; i >= 0; i--) ts_nstack_push(s, ts_node_child(node, i)); * * That idiom calls ts_node_child(node, i) once per index, and ts_node_child is * O(i) in tree-sitter (it walks the child iterator from the first child each @@ -63,12 +82,12 @@ static inline TSNode ts_nstack_pop(TSNodeStack *s) { * files). This helper enumerates children in a single O(N) cursor pass, then * reverses the just-pushed segment so pop order is identical to the old idiom. */ -static inline void ts_nstack_push_children(TSNodeStack *s, CBMArena *arena, TSNode node) { +static inline void ts_nstack_push_children(TSNodeStack *s, TSNode node) { int base = s->count; TSTreeCursor cursor = ts_tree_cursor_new(node); if (ts_tree_cursor_goto_first_child(&cursor)) { do { - ts_nstack_push(s, arena, ts_tree_cursor_current_node(&cursor)); + ts_nstack_push(s, ts_tree_cursor_current_node(&cursor)); } while (ts_tree_cursor_goto_next_sibling(&cursor)); } ts_tree_cursor_delete(&cursor); diff --git a/internal/cbm/extract_semantic.c b/internal/cbm/extract_semantic.c index 9379f7171..a4ab6f223 100644 --- a/internal/cbm/extract_semantic.c +++ b/internal/cbm/extract_semantic.c @@ -125,14 +125,14 @@ static void process_throw_node(CBMExtractCtx *ctx, TSNode node, const CBMLangSpe // Iterative throw walker static void walk_throws(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CBM_SZ_512); - ts_nstack_push(&stack, ctx->arena, root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); process_throw_node(ctx, node, spec); uint32_t count = ts_node_child_count(node); for (int i = (int)count - LAST_IDX; i >= 0; i--) { - ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i)); + ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i)); } } } @@ -263,8 +263,8 @@ static void try_emit_assignment_write(CBMExtractCtx *ctx, TSNode node, const cha static void walk_readwrites(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, CBM_SZ_512); - ts_nstack_push(&stack, ctx->arena, root); + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); if (cbm_kind_in_set(node, spec->assignment_node_types)) { @@ -272,7 +272,7 @@ static void walk_readwrites(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec * } uint32_t count = ts_node_child_count(node); for (int i = (int)count - LAST_IDX; i >= 0; i--) { - ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i)); + ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i)); } } } diff --git a/internal/cbm/extract_type_assigns.c b/internal/cbm/extract_type_assigns.c index 7830a8b18..7ac30ee1d 100644 --- a/internal/cbm/extract_type_assigns.c +++ b/internal/cbm/extract_type_assigns.c @@ -186,12 +186,12 @@ static void process_type_assign_node(CBMExtractCtx *ctx, TSNode node, const CBML // Walk AST for assignment patterns where RHS is a constructor call. static void walk_type_assigns(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, 4096); - ts_nstack_push(&stack, ctx->arena, root); + ts_nstack_init(&stack, ctx, 4096); + ts_nstack_push(&stack, root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); process_type_assign_node(ctx, node, spec, cbm_enclosing_func_qn_cached(ctx, node)); - ts_nstack_push_children(&stack, ctx->arena, node); + ts_nstack_push_children(&stack, node); } } diff --git a/internal/cbm/extract_type_refs.c b/internal/cbm/extract_type_refs.c index 8f78101f4..9bfdb0e49 100644 --- a/internal/cbm/extract_type_refs.c +++ b/internal/cbm/extract_type_refs.c @@ -245,14 +245,14 @@ static void process_body_type_ref(CBMExtractCtx *ctx, TSNode node, const char *f // Walk function body for type references (casts, type assertions, local var types, generics). static void walk_body_type_refs(CBMExtractCtx *ctx, TSNode root, const char *func_qn) { TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, 4096); - ts_nstack_push(&stack, ctx->arena, root); + ts_nstack_init(&stack, ctx, 4096); + ts_nstack_push(&stack, root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); process_body_type_ref(ctx, node, func_qn); uint32_t count = ts_node_child_count(node); for (int i = (int)count - SKIP_ONE; i >= 0; i--) { - ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i)); + ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i)); } } } @@ -289,8 +289,8 @@ static void walk_type_refs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *s } TSNodeStack stack; - ts_nstack_init(&stack, ctx->arena, 4096); - ts_nstack_push(&stack, ctx->arena, root); + ts_nstack_init(&stack, ctx, 4096); + ts_nstack_push(&stack, root); while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); @@ -301,7 +301,7 @@ static void walk_type_refs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *s } uint32_t count = ts_node_child_count(node); for (int i = (int)count - SKIP_ONE; i >= 0; i--) { - ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i)); + ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i)); } } } diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 447e7c104..28f7feb48 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -7,6 +7,7 @@ */ #include "test_framework.h" #include "cbm.h" +#include "foundation/constants.h" /* CBM_SZ_* */ #include "../src/foundation/compat.h" /* cbm_clock_gettime (wide-flat scaling guard) */ #include "../src/foundation/compat_fs.h" #include @@ -145,6 +146,57 @@ TEST(extract_ts_factory_object_methods_issue341) { PASS(); } +/* #2010, split out of #1997: AST traversal stacks were allocated from + * result->arena, which the parallel pass holds for every file until the whole + * result cache is freed, so a one-file scratch structure was retained for the + * length of the index. cbm_extract_channels runs for every file and dispatches + * TypeScript to extract_channels_js, whose two walks take a 4096-entry TSNode + * stack each, and the ES import walk takes a 512-entry one: + * 2 * 4096 * 32 + 512 * 32 = 278528 bytes charged to the arena of a one-line + * file. + * + * The bound is derived, not tuned. Measured on this source, total_alloc was + * 365984 before the scratch arena and is 87456 after, exactly that difference. + * Of the 87456 that remain, 7680 is the defs item array at GROW_ARRAY's + * starting capacity of 32 times sizeof(CBMDefinition) 240, and the other 79776 + * is everything else this file's extraction interns; none of it is traversal + * scratch. So the bound sits above 87456 with room and a factor of four below + * 365984. + * + * It is a byte budget, not a proof of lifetime; that is + * extract_traversal_stacks_come_from_ctx_scratch_issue2010 in test_mem.c. */ +TEST(traversal_stack_not_in_result_arena_issue2010) { + CBMFileResult *r = extract("export const x = 1;\n", CBM_LANG_TYPESCRIPT, "t", "a.ts"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT_TRUE(has_def_any(r, "x")); + /* Read the field rather than cbm_arena_total(): this file sees + * internal/cbm/arena.h, which declares a subset of the API. test_mem.c + * includes foundation/arena.h ahead of cbm.h and can call the accessor. */ + ASSERT_LT(r->arena.total_alloc, (size_t)CBM_SZ_128 * CBM_SZ_1K); + cbm_free_result(r); + PASS(); +} + +/* Not a scratch test. The C and C++ preprocessed second pass builds its own + * extraction context (pp_ctx in cbm_extract_file_ex), and that context carries + * ctx->scratch so every context in the file is uniform, but nothing reads it + * there: pp_ctx reaches only cbm_extract_unified and cbm_run_c_lsp, and neither + * extract_unified.c nor anything under internal/cbm/lsp/ includes + * extract_node_stack.h, so no traversal stack is built on that path today. + * This guards the macro-expansion path itself, which had no assertion on a call + * that exists only after expansion. */ +TEST(extract_c_macro_hidden_call_survives_preprocessed_pass_issue2010) { + CBMFileResult *r = extract("void target(void) {}\n" + "#define INVOKE() target()\n" + "void caller(void) { INVOKE(); }\n", + CBM_LANG_C, "t", "macro_call.c"); + ASSERT_NOT_NULL(r); + ASSERT_TRUE(has_call(r, "target")); + cbm_free_result(r); + PASS(); +} + /* --- C/C++ preprocessor macros become Macro nodes (#375) --- */ TEST(extract_c_macros_issue375) { CBMFileResult *r = extract("#define SIMPLE_MACRO 1\n" @@ -5371,6 +5423,8 @@ SUITE(extraction) { RUN_TEST(extract_r_box_use_imports_issue218); RUN_TEST(extract_r_dollar_call_issue219); RUN_TEST(extract_ts_factory_object_methods_issue341); + RUN_TEST(traversal_stack_not_in_result_arena_issue2010); + RUN_TEST(extract_c_macro_hidden_call_survives_preprocessed_pass_issue2010); RUN_TEST(extract_c_macros_issue375); RUN_TEST(extract_cpp_macros_issue375); RUN_TEST(extract_cpp_functionlike_macro_type_arg_no_false_parse_partial_issue1071); diff --git a/tests/test_mem.c b/tests/test_mem.c index 31c5a686f..e42eabb3d 100644 --- a/tests/test_mem.c +++ b/tests/test_mem.c @@ -14,6 +14,8 @@ #include "graph_buffer/graph_buffer.h" #include "discover/discover.h" #include "cbm.h" +#include "lang_specs.h" /* cbm_ts_language */ +#include "foundation/constants.h" /* CBM_SZ_* */ #include #include @@ -1277,6 +1279,53 @@ TEST(mem_map_attributes_a_known_allocation) { PASS(); } +/* #2010, the lifetime half. traversal_stack_not_in_result_arena in + * test_extraction.c pins the byte budget of the result arena, but a smaller + * CHAN_STACK_CAP would satisfy that too. This pins where the bytes actually + * went: the scratch arena takes the two 128 KB channel walks and the result + * arena does not. Builds the extraction context directly, since a completed + * cbm_extract_file_ex has already destroyed its scratch. */ +TEST(extract_traversal_stacks_come_from_ctx_scratch_issue2010) { + enum { CHANNEL_WALK_BYTES = 2 * 4096 * (int)sizeof(TSNode) }; + const char *src = "export const x = 1;\n"; + const TSLanguage *ts_lang = cbm_ts_language(CBM_LANG_TYPESCRIPT); + ASSERT_NOT_NULL((void *)ts_lang); + + TSParser *parser = ts_parser_new(); + ASSERT_NOT_NULL(parser); + ts_parser_set_language(parser, ts_lang); + TSTree *tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)strlen(src)); + ASSERT_NOT_NULL(tree); + + CBMFileResult result; + memset(&result, 0, sizeof(result)); + cbm_arena_init(&result.arena); + CBMArena scratch; + cbm_arena_init_sized(&scratch, (size_t)CBM_SZ_512 * CBM_SZ_1K); + + CBMExtractCtx ctx = { + .arena = &result.arena, + .scratch = &scratch, + .result = &result, + .source = src, + .source_len = (int)strlen(src), + .language = CBM_LANG_TYPESCRIPT, + .project = "t", + .rel_path = "a.ts", + .root = ts_tree_root_node(tree), + }; + cbm_extract_channels(&ctx); + + ASSERT_GTE(cbm_arena_total(&scratch), (size_t)CHANNEL_WALK_BYTES); + ASSERT_LT(cbm_arena_total(&result.arena), (size_t)CBM_SZ_64 * CBM_SZ_1K); + + cbm_arena_destroy(&scratch); + cbm_arena_destroy(&result.arena); + ts_tree_delete(tree); + ts_parser_delete(parser); + PASS(); +} + SUITE(mem) { /* mem API */ RUN_TEST(mem_arena_eager_commit_follows_platform_commit_cost); @@ -1340,4 +1389,7 @@ SUITE(mem) { RUN_TEST(parallel_extract_without_source_retention); RUN_TEST(parallel_extract_tiny_source_retention_budget); RUN_TEST(parallel_extract_with_slab); + + /* extraction scratch arena (#2010) */ + RUN_TEST(extract_traversal_stacks_come_from_ctx_scratch_issue2010); } From e0785b353af1af3812d5d14cad55c839efc5749d Mon Sep 17 00:00:00 2001 From: Rares Popa <2606875+rarepops@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:46:29 +0200 Subject: [PATCH 22/22] feat(ui): show serving binary version Expose the compiled server version through the existing UI config endpoint and render it beside the product name. Keep older or unavailable config responses compatible by hiding the label, with focused frontend and native endpoint coverage. Fixes #1820 Signed-off-by: Rares Popa <2606875+rarepops@users.noreply.github.com> --- graph-ui/src/App.test.tsx | 59 +++++++++++++++++++++++++++++++++++++++ graph-ui/src/App.tsx | 24 ++++++++++++++++ src/ui/http_server.c | 10 +++++-- tests/test_httpd.c | 21 ++++++++++++++ 4 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 graph-ui/src/App.test.tsx diff --git a/graph-ui/src/App.test.tsx b/graph-ui/src/App.test.tsx new file mode 100644 index 000000000..f6437c40d --- /dev/null +++ b/graph-ui/src/App.test.tsx @@ -0,0 +1,59 @@ +/* @vitest-environment jsdom */ +import "@testing-library/jest-dom/vitest"; +import { cleanup, render, screen, waitFor } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { App } from "./App"; +import { messages } from "./lib/i18n"; + +vi.mock("./components/GraphTab", () => ({ GraphTab: () => null })); +vi.mock("./components/StatsTab", () => ({ StatsTab: () => null })); +vi.mock("./components/ControlTab", () => ({ ControlTab: () => null })); +vi.mock("./lib/i18n", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, useUiMessages: () => messages.en }; +}); + +describe("App", () => { + afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); + window.history.replaceState(null, "", "/"); + }); + + it("shows the serving binary version", async () => { + vi.stubGlobal("fetch", vi.fn(async () => + new Response(JSON.stringify({ lang: "en", version: "0.10.8" }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }), + )); + + render(); + + expect(await screen.findByText("v0.10.8")).toBeVisible(); + }); + + it("hides the version when the config has no string version", async () => { + const fetchMock = vi.fn(async () => + new Response(JSON.stringify({ lang: "en", version: 108 }), { status: 200 }), + ); + vi.stubGlobal("fetch", fetchMock); + + render(); + + await waitFor(() => expect(fetchMock).toHaveBeenCalledWith("/api/ui-config")); + expect(screen.queryByTitle("Server version")).not.toBeInTheDocument(); + }); + + it("hides the version when the config request fails", async () => { + const fetchMock = vi.fn(async () => { + throw new Error("offline"); + }); + vi.stubGlobal("fetch", fetchMock); + + render(); + + await waitFor(() => expect(fetchMock).toHaveBeenCalledWith("/api/ui-config")); + expect(screen.queryByTitle("Server version")).not.toBeInTheDocument(); + }); +}); \ No newline at end of file diff --git a/graph-ui/src/App.tsx b/graph-ui/src/App.tsx index 18392667c..a0d84b0b6 100644 --- a/graph-ui/src/App.tsx +++ b/graph-ui/src/App.tsx @@ -33,8 +33,24 @@ function routeUrl(tab: TabId, project: string | null): string { export function App() { const t = useUiMessages(); const [route, setRoute] = useState(readRoute); + const [version, setVersion] = useState(null); const { tab: activeTab, project: selectedProject } = route; + useEffect(() => { + let cancelled = false; + void fetch("/api/ui-config") + .then((response) => (response.ok ? response.json() : null)) + .then((config) => { + if (!cancelled && typeof config?.version === "string" && config.version) { + setVersion(config.version); + } + }) + .catch(() => {}); + return () => { + cancelled = true; + }; + }, []); + /* Normalize the URL on first load so it always carries the current route. */ useEffect(() => { const initial = readRoute(); @@ -73,6 +89,14 @@ export function App() { Codebase Memory + {version && ( + + {version.startsWith("v") ? version : `v${version}`} + + )} {/* Tabs inline in header */} diff --git a/src/ui/http_server.c b/src/ui/http_server.c index b187438a5..77a600b11 100644 --- a/src/ui/http_server.c +++ b/src/ui/http_server.c @@ -66,6 +66,10 @@ /* ── Constants ────────────────────────────────────────────────── */ +#ifndef CBM_VERSION +#define CBM_VERSION "dev" +#endif + /* Max JSON-RPC request body size (1 MB) — transport enforces the same cap. */ #define MAX_BODY_SIZE CBM_HTTP_MAX_BODY @@ -143,8 +147,10 @@ static void handle_ui_config(cbm_http_conn_t *c, const cbm_http_req_t *req) { * audit forbids hardcoded external URLs in graph-ui source (external * targets must come from an auditable backend response, same pattern as * the /api/repo-info deep-links). */ - cbm_http_replyf(c, 200, g_cors_json, "{\"lang\":\"%s\",\"upstream_issues_url\":\"%s\"}", - lang_buf, "https://github.com/DeusData/codebase-memory-mcp/issues/new"); + cbm_http_replyf(c, 200, g_cors_json, + "{\"lang\":\"%s\",\"version\":\"%s\",\"upstream_issues_url\":\"%s\"}", + lang_buf, CBM_VERSION, + "https://github.com/DeusData/codebase-memory-mcp/issues/new"); } /* ── Server state ─────────────────────────────────────────────── */ diff --git a/tests/test_httpd.c b/tests/test_httpd.c index eacf92634..3ac680197 100644 --- a/tests/test_httpd.c +++ b/tests/test_httpd.c @@ -33,6 +33,9 @@ #include #include #include +#ifndef CBM_VERSION +#define CBM_VERSION "dev" +#endif #ifndef _WIN32 #include #endif @@ -1573,6 +1576,23 @@ TEST(ui_server_ui_config_detects_zh_accept_language) { PASS(); } +TEST(ui_server_ui_config_includes_serving_version_issue1820) { + th_server_t ts; + ASSERT_EQ(th_server_start(&ts), 0); + + char resp[4096]; + int n = th_http(cbm_http_server_port(ts.srv), "GET /api/ui-config HTTP/1.1\r\n\r\n", resp, + sizeof(resp)); + ASSERT_TRUE(n > 0); + ASSERT_EQ(th_status(resp), 200); + char expected_version[128]; + snprintf(expected_version, sizeof(expected_version), "\"version\":\"%s\"", CBM_VERSION); + ASSERT_NOT_NULL(strstr(resp, expected_version)); + + th_server_stop(&ts); + PASS(); +} + TEST(ui_server_ui_config_prefers_config_lang) { char tmpdir[256]; snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_httpd_cfg_XXXXXX"); @@ -2397,6 +2417,7 @@ SUITE(httpd) { RUN_TEST(ui_server_delete_project_invalid_name_keeps_watch); RUN_TEST(ui_server_delete_project_unlink_failure_keeps_watch); RUN_TEST(ui_server_ui_config_detects_zh_accept_language); + RUN_TEST(ui_server_ui_config_includes_serving_version_issue1820); RUN_TEST(ui_server_ui_config_prefers_config_lang); RUN_TEST(ui_server_slow_request_hits_deadline); RUN_TEST(ui_server_access_log_redacts_query);