diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 082bca1d6..0c63510a6 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -1284,24 +1284,63 @@ static bool cbm_is_eof_terminator_miss(TSNode n, const char *source, int source_ return true; } +/* The two walks below visit, in source order, every ERROR or MISSING node that + * is top-most in its chain: they step into a child that merely contains an + * error and never into an ERROR subtree. That descent follows the nesting of + * the construct that failed — every ancestor of the failing node carries + * ts_node_has_error — so a recursive walk used one stack frame per nesting + * level of the indexed file, and a deep enough nest ran the worker thread out + * of stack. The cursor keeps that path on the heap. The nodes visited and + * their order are those of the child-index loop it replaced: the cursor + * stops on exactly the visible children ts_node_child enumerates. */ +typedef void (*cbm_error_node_fn)(TSNode node, void *arg); + +static void cbm_walk_error_nodes(TSNode root, const char *source, int source_len, + cbm_error_node_fn fn, void *arg) { + /* The thread's reusable cursor: the walk completes before anything else on + * this thread can ask for it, and the callbacks never do. */ + TSTreeCursor *cursor = cbm_thread_cursor(root); + if (!ts_tree_cursor_goto_first_child(cursor)) { + return; + } + int depth = 0; /* 0 while on a direct child of root */ + for (;;) { + TSNode c = ts_tree_cursor_current_node(cursor); + bool descend = false; + if (ts_node_is_missing(c) || strcmp(ts_node_type(c), "ERROR") == 0) { + /* An absent final newline only means nothing was dropped. Anything + * else is the top-most region; do not descend. */ + if (!cbm_is_eof_terminator_miss(c, source, source_len)) { + fn(c, arg); + } + } else if (ts_node_has_error(c)) { + descend = true; + } + if (descend && ts_tree_cursor_goto_first_child(cursor)) { + depth++; + continue; + } + while (!ts_tree_cursor_goto_next_sibling(cursor)) { + if (depth == 0) { + return; + } + (void)ts_tree_cursor_goto_parent(cursor); + depth--; + } + } +} + /* Walks to the end even after the cap is full, so `dropped` is the real number * of ranges lost rather than a lower bound. This costs little: the walk never * descends into an ERROR subtree — it records the top-most node and moves on — * so it only visits the spine of nodes that contain an error, plus one level. */ +static void cbm_collect_error_region(TSNode c, void *arg) { + cbm_error_regions_push((cbm_error_regions_t *)arg, c); +} + static void cbm_collect_error_regions(TSNode n, cbm_error_regions_t *acc, const char *source, int source_len) { - uint32_t k = ts_node_child_count(n); - for (uint32_t i = 0; i < k; i++) { - TSNode c = ts_node_child(n, i); - if (ts_node_is_missing(c) || strcmp(ts_node_type(c), "ERROR") == 0) { - if (cbm_is_eof_terminator_miss(c, source, source_len)) { - continue; /* absent final newline only — nothing was dropped */ - } - cbm_error_regions_push(acc, c); /* top-most region; do not descend */ - } else if (ts_node_has_error(c)) { - cbm_collect_error_regions(c, acc, source, source_len); - } - } + cbm_walk_error_nodes(n, source, source_len, cbm_collect_error_region, acc); } /* ── Phase 2 line map: what the preprocessed parse already explained ─────── @@ -1408,24 +1447,24 @@ static void cbm_mark_no_code_lines(const char *src, int src_len, uint8_t *map, * Step 2 walks the expanded lines and, for each one that is unmarked, belongs * to the file itself (not an included header) and maps back to a real * original line, records that original line as parsed. */ +typedef struct { + uint8_t *rows; + uint32_t row_count; +} cbm_pp_error_rows_t; + +static void cbm_mark_pp_error_row(TSNode c, void *arg) { + cbm_pp_error_rows_t *marks = (cbm_pp_error_rows_t *)arg; + uint32_t s = ts_node_start_point(c).row + 1; + uint32_t e = ts_node_end_point(c).row + 1; + for (uint32_t r = s; r <= e && r <= marks->row_count; r++) { + marks->rows[r] = 1; + } +} + static void cbm_mark_pp_error_rows(TSNode n, uint8_t *rows, uint32_t row_count, const char *src, int src_len) { - uint32_t k = ts_node_child_count(n); - for (uint32_t i = 0; i < k; i++) { - TSNode c = ts_node_child(n, i); - if (ts_node_is_missing(c) || strcmp(ts_node_type(c), "ERROR") == 0) { - if (cbm_is_eof_terminator_miss(c, src, src_len)) { - continue; /* absent final newline only — nothing was dropped */ - } - uint32_t s = ts_node_start_point(c).row + 1; - uint32_t e = ts_node_end_point(c).row + 1; - for (uint32_t r = s; r <= e && r <= row_count; r++) { - rows[r] = 1; - } - } else if (ts_node_has_error(c)) { - cbm_mark_pp_error_rows(c, rows, row_count, src, src_len); - } - } + cbm_pp_error_rows_t marks = {rows, row_count}; + cbm_walk_error_nodes(n, src, src_len, cbm_mark_pp_error_row, &marks); } /* Recovery subtraction (#963): tree-sitter error recovery plus the diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index f14efb766..de2a2fedd 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -3411,43 +3411,48 @@ static TSNode primary_callee_expr(TSNode node) { } static TSNode terminal_callee_leaf(const CBMExtractCtx *ctx, TSNode expr) { - if (ts_node_is_null(expr) || is_dynamic_callee_expr(ctx, expr)) { - return (TSNode){0}; - } - static const char *const terminal_fields[] = {"property", "field", "attribute", "method", "name", NULL}; - TSNode terminal = first_present_field(expr, terminal_fields); - if (!ts_node_is_null(terminal)) { - if (is_dynamic_callee_expr(ctx, expr)) { + // Each step either returns or moves `expr` to the child the search goes + // on in. Recursing for that step cost one stack frame per wrapper level of + // the callee — `((((f))))(...)`, or the head of a curried application — + // so stack use followed the nesting depth of the indexed file and a deep + // enough callee ran the worker thread out of stack. The loop visits the + // same nodes in the same order. + for (;;) { + if (ts_node_is_null(expr) || is_dynamic_callee_expr(ctx, expr)) { return (TSNode){0}; } - return terminal_callee_leaf(ctx, terminal); - } - const char *kind = ts_node_type(expr); - if (strcmp(kind, "identifier") == 0 || strcmp(kind, "simple_identifier") == 0 || - strcmp(kind, "type_identifier") == 0 || strcmp(kind, "field_identifier") == 0 || - strcmp(kind, "property_identifier") == 0 || strcmp(kind, "name") == 0 || - strcmp(kind, "variable_name") == 0 || strcmp(kind, "attribute") == 0 || - strcmp(kind, "symbol") == 0 || strcmp(kind, "sym_lit") == 0 || - strcmp(kind, "user_symbol") == 0) { - return expr; - } + TSNode terminal = first_present_field(expr, terminal_fields); + if (!ts_node_is_null(terminal)) { + expr = terminal; + continue; + } - // Known wrappers keep their static terminal in one of their named children. - // Prefer the last child for qualified/member/navigation forms and the first - // for transparent parentheses and generic wrappers whose name field was not - // exposed by the grammar. - uint32_t count = ts_node_named_child_count(expr); - if (count == 0) { - return (TSNode){0}; + const char *kind = ts_node_type(expr); + if (strcmp(kind, "identifier") == 0 || strcmp(kind, "simple_identifier") == 0 || + strcmp(kind, "type_identifier") == 0 || strcmp(kind, "field_identifier") == 0 || + strcmp(kind, "property_identifier") == 0 || strcmp(kind, "name") == 0 || + strcmp(kind, "variable_name") == 0 || strcmp(kind, "attribute") == 0 || + strcmp(kind, "symbol") == 0 || strcmp(kind, "sym_lit") == 0 || + strcmp(kind, "user_symbol") == 0) { + return expr; + } + + // Known wrappers keep their static terminal in one of their named + // children. Prefer the last child for qualified/member/navigation forms + // and the first for transparent parentheses and generic wrappers whose + // name field was not exposed by the grammar. + uint32_t count = ts_node_named_child_count(expr); + if (count == 0) { + return (TSNode){0}; + } + bool terminal_is_last = strstr(kind, "qualified") || strstr(kind, "scoped") || + strstr(kind, "member") || strstr(kind, "selector") || + strstr(kind, "navigation") || strstr(kind, "field_expression"); + expr = ts_node_named_child(expr, terminal_is_last ? count - 1 : 0); } - bool terminal_is_last = strstr(kind, "qualified") || strstr(kind, "scoped") || - strstr(kind, "member") || strstr(kind, "selector") || - strstr(kind, "navigation") || strstr(kind, "field_expression"); - TSNode child = ts_node_named_child(expr, terminal_is_last ? count - 1 : 0); - return terminal_callee_leaf(ctx, child); } typedef struct { diff --git a/internal/cbm/extract_usages.c b/internal/cbm/extract_usages.c index bdad1c8c7..db08c43d2 100644 --- a/internal/cbm/extract_usages.c +++ b/internal/cbm/extract_usages.c @@ -1346,37 +1346,47 @@ static bool is_call_argument_label_walk(TSNode node, WalkState *state) { } static bool is_direct_argument_value(TSNode node) { - usage_slow_parent_fallback_test_note(); - TSNode parent = ts_node_parent(node); - if (ts_node_is_null(parent)) { - return false; - } - if (is_labeled_argument_kind(ts_node_type(parent))) { - TSNode value = ts_node_child_by_field_name(parent, TS_FIELD("value")); - return !ts_node_is_null(value) && ts_node_eq(value, node) && - is_direct_argument_value(parent); - } - TSNode direct_arguments = ts_node_child_by_field_name(parent, TS_FIELD("arguments")); - if (!ts_node_is_null(direct_arguments) && ts_node_eq(direct_arguments, node)) { - return true; - } - if (is_argument_container_kind(ts_node_type(parent))) { - return true; - } - const char *parent_kind = ts_node_type(parent); - if (strcmp(parent_kind, "list_expression") == 0) { - TSNode call = ts_node_parent(parent); - if (!ts_node_is_null(call)) { - TSNode arguments = ts_node_child_by_field_name(call, TS_FIELD("arguments")); - return !ts_node_is_null(arguments) && ts_node_eq(arguments, parent); + /* A labeled argument (keyword_argument and kin) is itself what the call + * site holds, so when `node` is its value the question moves one level up. + * That step used to recurse; the loop asks it in place and takes the + * slow-parent note once per level, as each recursive entry did. */ + for (;;) { + usage_slow_parent_fallback_test_note(); + TSNode parent = ts_node_parent(node); + if (ts_node_is_null(parent)) { + return false; } - return false; + if (is_labeled_argument_kind(ts_node_type(parent))) { + TSNode value = ts_node_child_by_field_name(parent, TS_FIELD("value")); + if (ts_node_is_null(value) || !ts_node_eq(value, node)) { + return false; + } + node = parent; + continue; + } + TSNode direct_arguments = ts_node_child_by_field_name(parent, TS_FIELD("arguments")); + if (!ts_node_is_null(direct_arguments) && ts_node_eq(direct_arguments, node)) { + return true; + } + if (is_argument_container_kind(ts_node_type(parent))) { + return true; + } + const char *parent_kind = ts_node_type(parent); + if (strcmp(parent_kind, "list_expression") == 0) { + TSNode call = ts_node_parent(parent); + if (!ts_node_is_null(call)) { + TSNode arguments = ts_node_child_by_field_name(call, TS_FIELD("arguments")); + return !ts_node_is_null(arguments) && ts_node_eq(arguments, parent); + } + return false; + } + if (strcmp(parent_kind, "argument") != 0 && strcmp(parent_kind, "value_argument") != 0) { + return false; + } + TSNode grandparent = ts_node_parent(parent); + return !ts_node_is_null(grandparent) && + is_argument_container_kind(ts_node_type(grandparent)); } - if (strcmp(parent_kind, "argument") != 0 && strcmp(parent_kind, "value_argument") != 0) { - return false; - } - TSNode grandparent = ts_node_parent(parent); - return !ts_node_is_null(grandparent) && is_argument_container_kind(ts_node_type(grandparent)); } /* The body of the walk above, entered one level in: for a caller that has diff --git a/internal/cbm/lsp/php_lsp.c b/internal/cbm/lsp/php_lsp.c index 4caf5ba0f..b9dac6014 100644 --- a/internal/cbm/lsp/php_lsp.c +++ b/internal/cbm/lsp/php_lsp.c @@ -1352,39 +1352,45 @@ static bool php_callable_reference_has_dynamic_member_name(PHPLSPContext *ctx, T * USAGE passes attach the same exact token occurrence to a first-class * callable, including qualified free-function references. */ static TSNode php_reference_callee_leaf(PHPLSPContext *ctx, TSNode expr) { - if (ts_node_is_null(expr) || php_dynamic_reference_expr(ctx, expr)) { - return (TSNode){0}; - } - static const char *const terminal_fields[] = {"property", "field", "attribute", "method", "name", NULL}; - for (int i = 0; terminal_fields[i]; i++) { - TSNode terminal = ts_node_child_by_field_name(expr, terminal_fields[i], - (uint32_t)strlen(terminal_fields[i])); - if (!ts_node_is_null(terminal)) { - return php_reference_callee_leaf(ctx, terminal); + /* A loop for the same reason as terminal_callee_leaf: recursing once per + * wrapper level of the callee made stack use follow the nesting depth of + * the file. Same nodes, same order. */ + for (;;) { + if (ts_node_is_null(expr) || php_dynamic_reference_expr(ctx, expr)) { + return (TSNode){0}; } - } - const char *kind = ts_node_type(expr); - if (strcmp(kind, "identifier") == 0 || strcmp(kind, "simple_identifier") == 0 || - strcmp(kind, "type_identifier") == 0 || strcmp(kind, "field_identifier") == 0 || - strcmp(kind, "property_identifier") == 0 || strcmp(kind, "name") == 0 || - strcmp(kind, "variable_name") == 0 || strcmp(kind, "attribute") == 0 || - strcmp(kind, "symbol") == 0 || strcmp(kind, "sym_lit") == 0 || - strcmp(kind, "user_symbol") == 0) { - return expr; - } + TSNode terminal = {0}; + for (int i = 0; terminal_fields[i] && ts_node_is_null(terminal); i++) { + terminal = ts_node_child_by_field_name(expr, terminal_fields[i], + (uint32_t)strlen(terminal_fields[i])); + } + if (!ts_node_is_null(terminal)) { + expr = terminal; + continue; + } - uint32_t count = ts_node_named_child_count(expr); - if (count == 0) { - return (TSNode){0}; + const char *kind = ts_node_type(expr); + if (strcmp(kind, "identifier") == 0 || strcmp(kind, "simple_identifier") == 0 || + strcmp(kind, "type_identifier") == 0 || strcmp(kind, "field_identifier") == 0 || + strcmp(kind, "property_identifier") == 0 || strcmp(kind, "name") == 0 || + strcmp(kind, "variable_name") == 0 || strcmp(kind, "attribute") == 0 || + strcmp(kind, "symbol") == 0 || strcmp(kind, "sym_lit") == 0 || + strcmp(kind, "user_symbol") == 0) { + return expr; + } + + uint32_t count = ts_node_named_child_count(expr); + if (count == 0) { + return (TSNode){0}; + } + bool terminal_is_last = strstr(kind, "qualified") || strstr(kind, "scoped") || + strstr(kind, "member") || strstr(kind, "selector") || + strstr(kind, "navigation") || strstr(kind, "field_expression"); + expr = ts_node_named_child(expr, terminal_is_last ? count - 1 : 0); } - bool terminal_is_last = strstr(kind, "qualified") || strstr(kind, "scoped") || - strstr(kind, "member") || strstr(kind, "selector") || - strstr(kind, "navigation") || strstr(kind, "field_expression"); - return php_reference_callee_leaf(ctx, - ts_node_named_child(expr, terminal_is_last ? count - 1 : 0)); } static TSNode php_callable_reference_leaf(PHPLSPContext *ctx, TSNode call, const char *kind) { diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index f068ad411..0c330d9ad 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -486,8 +486,14 @@ static void calls_append_args(char *props, size_t cap, const CBMCall *call) { /* Add rather than subtract: pos is unsigned, so `cap - pos - PAIR_LEN` * wraps once pos reaches cap - PAIR_LEN and stops bounding the memcpy * below. The closing write at the end of this function already guards - * additively; match it. */ - if (n <= 0 || pos + (size_t)n + PAIR_LEN >= cap) { + * additively; match it. + * + * snprintf reports the untruncated length, so an argument `one` could + * not hold whole is never copied out of it: copying n bytes would read + * past the local and plant its terminator inside the props text. It + * ends the array exactly like an argument the props buffer cannot + * take. */ + if (n <= 0 || (size_t)n >= sizeof(one) || pos + (size_t)n + PAIR_LEN >= cap) { break; /* not enough room — close the array with what fits */ } memcpy(props + pos, one, (size_t)n); @@ -500,6 +506,11 @@ static void calls_append_args(char *props, size_t cap, const CBMCall *call) { } } +/* Test seam for the formatter above. Not part of the public API. */ +void cbm_pipeline_calls_append_args_for_tests(char *props, size_t cap, const CBMCall *call) { + calls_append_args(props, cap, call); +} + static void calls_emit_edge(cbm_gbuf_t *gbuf, int64_t src, int64_t tgt, const char *type, char *props, size_t cap, const CBMCall *call) { if (call && call->start_line > 0 && strcmp(type, "CALLS") == 0) { diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 7f266eae9..1149deeb0 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -927,6 +927,12 @@ void cbm_pipeline_set_rename_hook_for_tests(cbm_pipeline_t *p, int (*hook)(const char *, const char *, void *), void *ctx); +/* Test seam: the sequential CALLS pass's args formatter (pass_calls.c), so a + * test can hand it an argument longer than its formatting buffer and read the + * props it leaves behind. `props` is a NUL-terminated JSON object of `cap` + * bytes; the formatter appends its "args" array in place. */ +void cbm_pipeline_calls_append_args_for_tests(char *props, size_t cap, const CBMCall *call); + /* Synchronous thread-local seam for deterministic cross-repo cancellation * tests. The callback runs immediately after a CROSS_* edge is committed and * is never retained; it must not re-enter cross-repo matching. */ diff --git a/src/pipeline/worker_pool.c b/src/pipeline/worker_pool.c index 2247fccd5..2943fd746 100644 --- a/src/pipeline/worker_pool.c +++ b/src/pipeline/worker_pool.c @@ -20,9 +20,7 @@ enum { WP_TRUE = 1, WP_MIN = 1, WP_STEP = 1 }; #include #include -/* 8 MB stack per worker — matches main thread default. - * Required for deep AST recursion (tree-sitter + walk_defs). */ -#define CBM_WORKER_STACK_SIZE ((size_t)8 * CBM_SZ_1K * CBM_SZ_1K) +/* CBM_WORKER_STACK_SIZE (8 MB per worker) lives in worker_pool.h. */ /* ── Serial fallback ─────────────────────────────────────────────── */ diff --git a/src/pipeline/worker_pool.h b/src/pipeline/worker_pool.h index 590d67b53..91697b961 100644 --- a/src/pipeline/worker_pool.h +++ b/src/pipeline/worker_pool.h @@ -11,6 +11,13 @@ #define CBM_WORKER_POOL_H #include +#include +#include "foundation/constants.h" /* CBM_SZ_1K */ + +/* 8 MB stack per worker — matches main thread default. Required for deep AST + * recursion (tree-sitter + walk_defs). Shared here so a test can run an + * extraction on a thread of exactly the size the pipeline gives it. */ +#define CBM_WORKER_STACK_SIZE ((size_t)8 * CBM_SZ_1K * CBM_SZ_1K) /* Worker callback: called once per iteration with index [0..count-1]. */ typedef void (*cbm_parallel_fn)(int idx, void *ctx); diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 0b6800cd1..33057ea6d 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -19,6 +19,13 @@ #include "helpers.h" /* cbm_count_branching (walker stack cap) */ #include "lang_specs.h" /* cbm_lang_spec, cbm_ts_language */ +/* Deep-nesting probes: extraction on a worker-sized stack, crash-isolated. */ +#include "pipeline/worker_pool.h" /* CBM_WORKER_STACK_SIZE */ +#include "../src/foundation/compat_thread.h" /* cbm_thread_create */ +#if !defined(_WIN32) +#include /* fork/waitpid */ +#endif + /* ── Helpers ───────────────────────────────────────────────────── */ /* Check if any definition with the given label has the given name. */ @@ -8863,6 +8870,315 @@ TEST(extract_walk_truncated_when_a_node_budget_is_set) { PASS(); } +/* ═══════════════════════════════════════════════════════════════════ + * Deep nesting: the syntax-tree walkers must not spend stack per level + * + * Extraction runs on pipeline worker threads whose stack is + * CBM_WORKER_STACK_SIZE (src/pipeline/worker_pool.h), not on the test runner's + * main thread, whose size depends on the platform and its rlimit. Each probe + * below therefore runs cbm_extract_file on a thread of exactly that size, so + * a walker that recurses once per nesting level runs out of stack at the + * depth it would in the pipeline. On POSIX the thread lives in a forked child + * and an overflow surfaces as the child's killing signal instead of taking + * the runner with it (the pattern of tests/test_stack_overflow.c); Windows + * runs it in-process, where a real overflow is a visible runner abort. A + * probe returns 0 when every expectation held and prints what did not. + * + * DEEP_NEST_LEVELS: a walker frame holds at least its saved frame pointer and + * return address plus the 32-byte TSNode it passes down (on arm64 a TSNode is + * passed by reference to a copy in the caller's frame), 48 bytes, so 200,000 + * frames need 9.6 MB — past the 8 MB worker stack even at that floor; + * measured frames are larger and the sanitizer build adds red zones. The + * parser is not the limit: tree-sitter keeps the nesting on its heap parse + * stack and its node and cursor helpers are iterative, so the depth is the + * walker's problem alone. The fixtures nest parentheses, which copy no text + * per level (a nested call copies its argument text at every level, which is + * what makes the stack_overflow_b suite slow). + * ═══════════════════════════════════════════════════════════════════ */ + +enum { DEEP_NEST_LEVELS = 200000 }; + +typedef int (*deep_probe_fn)(void); + +typedef struct { + deep_probe_fn fn; + int rc; +} deep_probe_job_t; + +static void *deep_probe_thread_main(void *arg) { + deep_probe_job_t *job = (deep_probe_job_t *)arg; + job->rc = job->fn(); + return NULL; +} + +/* 0 when the probe passed, its own code otherwise, 2 if no thread came up. */ +static int deep_probe_on_worker_stack(deep_probe_fn fn) { + deep_probe_job_t job = {fn, 3}; + cbm_thread_t thread; + if (cbm_thread_create(&thread, CBM_WORKER_STACK_SIZE, deep_probe_thread_main, &job) != 0) { + return 2; + } + (void)cbm_thread_join(&thread); + return job.rc; +} + +/* 1 when the probe passed; otherwise `why` says what happened. */ +static int deep_probe_passed(deep_probe_fn fn, char *why, size_t why_len) { +#if defined(_WIN32) + int rc = deep_probe_on_worker_stack(fn); + if (rc != 0) { + snprintf(why, why_len, "probe returned %d", rc); + } + return rc == 0; +#else + fflush(NULL); + pid_t pid = fork(); + if (pid < 0) { + snprintf(why, why_len, "fork failed"); + return 0; + } + if (pid == 0) { + int rc = deep_probe_on_worker_stack(fn); + fflush(NULL); + _exit(rc); + } + int status = 0; + (void)waitpid(pid, &status, 0); + if (WIFSIGNALED(status)) { + snprintf(why, why_len, + "extraction killed by signal %d on a worker-sized stack: a walker recursed once " + "per nesting level", + WTERMSIG(status)); + return 0; + } + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + snprintf(why, why_len, "probe returned %d", WIFEXITED(status) ? WEXITSTATUS(status) : -1); + return 0; + } + return 1; +#endif +} + +/* head, `levels` times '(', core, `levels` times ')', tail. *core_at receives + * the byte offset of core. */ +static char *build_nested_source(const char *head, const char *core, const char *tail, int levels, + uint32_t *core_at) { + size_t head_len = strlen(head); + size_t core_len = strlen(core); + size_t tail_len = strlen(tail); + char *src = malloc(head_len + core_len + tail_len + 2 * (size_t)levels + 1); + if (!src) { + return NULL; + } + char *w = src; + memcpy(w, head, head_len); + w += head_len; + memset(w, '(', (size_t)levels); + w += levels; + *core_at = (uint32_t)(w - src); + memcpy(w, core, core_len); + w += core_len; + memset(w, ')', (size_t)levels); + w += levels; + memcpy(w, tail, tail_len + 1); + return src; +} + +/* `$x = ((...((f))...))(...);` — a first-class callable whose callee sits under + * DEEP_NEST_LEVELS parentheses. PHP, because its first-class callable syntax + * is the path that hands a parenthesised callee to terminal_callee_leaf + * (extract_calls.c): the leaf finder steps into one named child per wrapper + * node that has no terminal field, while every ordinary call path first needs + * a textual callee name, which the field resolver only yields for identifier + * and member shapes whose leaf sits one or two levels down (a member chain + * `a.b.c()` stops at its `property`; nested calls `f(f(f()))` are arguments, + * visited by the explicit-stack walker). Expected: the exact occurrence the + * leaf finder names — one CALL_REFERENCE usage of `f` at the byte of the `f` + * token — in a cleanly parsed file. */ +static int deep_callee_chain_probe(void) { + uint32_t f_at = 0; + char *src = build_nested_source("has_error || r->parse_incomplete) { + printf(" deep callee: extraction failed or the parse was not clean\n"); + rc = 1; + } else { + int references = 0; + for (int i = 0; i < r->usages.count; i++) { + const CBMUsage *u = &r->usages.items[i]; + if (u->kind != CBM_USAGE_CALL_REFERENCE || !u->ref_name || + strcmp(u->ref_name, "f") != 0) { + continue; + } + references++; + if (u->site_start_byte != f_at || u->site_end_byte != f_at + 1) { + printf(" deep callee: reference site %u-%u, expected %u-%u\n", + u->site_start_byte, u->site_end_byte, f_at, f_at + 1); + rc = 1; + } + } + if (references != 1) { + printf(" deep callee: %d CALL_REFERENCE usages of f, expected 1\n", references); + rc = 1; + } + } + if (r) { + cbm_free_result(r); + } + free(src); + return rc; +} + +TEST(extract_deep_callee_chain_completes) { + char why[160]; + if (!deep_probe_passed(deep_callee_chain_probe, why, sizeof(why))) { + FAIL(why); + } + PASS(); +} + +/* One invalid token at the bottom of DEEP_NEST_LEVELS parentheses. Every + * enclosing parenthesized_expression then carries ts_node_has_error without + * being an ERROR node, which is the chain cbm_collect_error_regions (cbm.c) + * descends: it steps into a child that contains an error and stops at the + * top-most ERROR or MISSING node. Nested ERROR nodes would not do — the walk + * never enters an ERROR subtree, so they cost it one level. Expected: one + * region, on the line of the nest only, with the neighbouring definition + * intact and the file still usable. */ +static int deep_error_regions_probe(void) { + uint32_t core_at = 0; + char *src = build_nested_source("def ok():\n return 1\nx = ", "1 2", "\ny = ok()\n", + DEEP_NEST_LEVELS, &core_at); + if (!src) { + return 4; + } + CBMFileResult *r = extract(src, CBM_LANG_PYTHON, "t", "deep_error.py"); + int rc = 0; + if (!r) { + printf(" deep error regions: extraction failed\n"); + rc = 1; + } else { + if (!r->parse_incomplete || r->error_region_count != 1 || !r->error_ranges || + strcmp(r->error_ranges, "3-3") != 0) { + printf(" deep error regions: incomplete=%d count=%d ranges=%s, expected one " + "region 3-3\n", + r->parse_incomplete ? 1 : 0, r->error_region_count, + r->error_ranges ? r->error_ranges : "(none)"); + rc = 1; + } + if (r->parse_unusable) { + printf(" deep error regions: one line of five must not make the file unusable\n"); + rc = 1; + } + if (!has_def(r, "Function", "ok")) { + printf(" deep error regions: the definition next to the nest was lost\n"); + rc = 1; + } + cbm_free_result(r); + } + free(src); + return rc; +} + +TEST(extract_deep_error_regions_complete) { + char why[160]; + if (!deep_probe_passed(deep_error_regions_probe, why, sizeof(why))) { + FAIL(why); + } + PASS(); +} + +/* `f(cb=((...(obj.handler)...)))`: a labeled argument whose value is a bound + * method under LABELED_WRAPPERS parentheses. The usage pass classifies + * `handler` by climbing the wrappers (python_direct_callable_attribute_site, a + * loop) and then asking is_direct_argument_value, whose one recursive step + * moved from the value to the labeled argument holding it. The grammar never + * nests a labeled argument inside another's value, so that step is bounded + * and no stack depth is at stake; what the test pins is the answer across the + * conversion of that step to a loop: the usage is a callable-value candidate + * whose site is the outermost wrapper, and the keyword `cb` is no usage. The + * bare form pins the site on the attribute itself. */ +enum { LABELED_WRAPPERS = 2000 }; + +static int handler_candidate_site(const CBMFileResult *r, uint32_t expect_start, + uint32_t expect_end, const char *form) { + int seen = 0; + int rc = 0; + for (int i = 0; i < r->usages.count; i++) { + const CBMUsage *u = &r->usages.items[i]; + if (!u->ref_name) { + continue; + } + if (strcmp(u->ref_name, "cb") == 0) { + printf(" %s argument: the keyword label became a usage\n", form); + rc = 1; + } + if (strcmp(u->ref_name, "handler") != 0) { + continue; + } + seen++; + if (!u->may_be_call_reference || u->site_start_byte != expect_start || + u->site_end_byte != expect_end) { + printf(" %s argument: handler candidate=%d site %u-%u, expected candidate site " + "%u-%u\n", + form, u->may_be_call_reference ? 1 : 0, u->site_start_byte, u->site_end_byte, + expect_start, expect_end); + rc = 1; + } + } + if (seen != 1) { + printf(" %s argument: %d usages of handler, expected 1\n", form, seen); + rc = 1; + } + return rc; +} + +static int deep_argument_nesting_probe(void) { + const uint32_t attribute_len = (uint32_t)strlen("obj.handler"); + uint32_t core_at = 0; + char *src = build_nested_source("f(cb=", "obj.handler", ")\n", LABELED_WRAPPERS, &core_at); + if (!src) { + return 4; + } + int rc = 0; + CBMFileResult *wrapped = extract(src, CBM_LANG_PYTHON, "t", "deep_arg.py"); + if (!wrapped || wrapped->has_error || wrapped->parse_incomplete) { + printf(" wrapped argument: extraction failed or the parse was not clean\n"); + rc = 1; + } else { + rc |= handler_candidate_site(wrapped, core_at - LABELED_WRAPPERS, + core_at + attribute_len + LABELED_WRAPPERS, "wrapped"); + } + if (wrapped) { + cbm_free_result(wrapped); + } + free(src); + + CBMFileResult *bare = extract("f(cb=obj.handler)\n", CBM_LANG_PYTHON, "t", "bare_arg.py"); + if (!bare || bare->has_error || bare->parse_incomplete) { + printf(" bare argument: extraction failed or the parse was not clean\n"); + rc = 1; + } else { + rc |= handler_candidate_site(bare, 5, 5 + attribute_len, "bare"); + } + if (bare) { + cbm_free_result(bare); + } + return rc; +} + +TEST(extract_deep_argument_nesting_completes) { + char why[160]; + if (!deep_probe_passed(deep_argument_nesting_probe, why, sizeof(why))) { + FAIL(why); + } + PASS(); +} + SUITE(extraction) { RUN_TEST(extract_compact_keeps_every_field_and_shrinks_the_arena); RUN_TEST(extract_compact_is_idempotent_and_survives_empty_results); @@ -9305,5 +9621,10 @@ SUITE(extraction) { RUN_TEST(config_description_only_at_top_level_issue519); RUN_TEST(non_config_language_module_has_no_promoted_description_issue519); + /* Syntax-tree walkers spend no stack per nesting level */ + RUN_TEST(extract_deep_callee_chain_completes); + RUN_TEST(extract_deep_error_regions_complete); + RUN_TEST(extract_deep_argument_nesting_completes); + cbm_shutdown(); } diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 56be6b88e..828f2062f 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -16270,6 +16270,85 @@ TEST(pipeline_objectscript_export_range_join_keeps_one_trailing_marker) { } #endif +/* ── calls_append_args: an argument longer than its formatting buffer ────── */ + +#if !defined(_WIN32) +#include /* fork/waitpid: isolate a copy that runs past a local buffer */ +#endif + +/* The sequential CALLS pass formats each argument into a 512-byte local before + * copying it into the props buffer (calls_append_args, pass_calls.c). snprintf + * reports the untruncated length, so an argument the local cannot hold must end + * the array exactly like one the props buffer cannot take; copying that length + * out of the local reads past it and plants the local's terminator inside the + * props text. Two 250-byte strings make the formatted argument about 520 + * bytes, and the props buffer is large enough to take it, so only the local's + * own bound can stop the copy. 0 when the props came out right, 1 otherwise. */ +static int calls_args_overlong_probe(void) { + enum { CAP = 4 * CBM_SZ_1K, LONG_LEN = 250 }; + static char expr[LONG_LEN + 1]; + static char value[LONG_LEN + 1]; + static char props[CAP]; + memset(expr, 'a', LONG_LEN); + expr[LONG_LEN] = '\0'; + memset(value, 'b', LONG_LEN); + value[LONG_LEN] = '\0'; + CBMCallArg args[2] = {{"x", NULL, NULL, 0}, {expr, value, NULL, 1}}; + CBMCall call = {0}; + call.callee_name = "sink"; + call.args = args; + call.arg_count = 2; + + const char *base = "{\"line\":7}"; + memset(props, 0x7f, CAP); /* a sentinel the formatter never writes */ + memcpy(props, base, strlen(base) + 1); + cbm_pipeline_calls_append_args_for_tests(props, CAP, &call); + + /* The short argument is kept; the overlong one ends the array cleanly. */ + const char *expected = "{\"line\":7,\"args\":[{\"i\":0,\"e\":\"x\"}]}"; + size_t expected_len = strlen(expected); + if (strncmp(props, expected, expected_len + 1) != 0) { + printf(" props: %.80s\n expected: %s\n", props, expected); + return 1; + } + for (size_t i = expected_len + 1; i < CAP; i++) { + if ((unsigned char)props[i] != 0x7f) { + printf(" byte %zu past the terminator was written\n", i); + return 1; + } + } + return 0; +} + +TEST(pipeline_calls_args_overlong_argument_stops_cleanly) { +#if defined(_WIN32) + ASSERT_EQ(calls_args_overlong_probe(), 0); +#else + /* A forked child, so a copy that runs past the local surfaces as the + * child's killing signal (the sanitizer build aborts on it) rather than + * taking the runner down. */ + fflush(NULL); + pid_t pid = fork(); + ASSERT_TRUE(pid >= 0); + if (pid == 0) { + int rc = calls_args_overlong_probe(); + fflush(NULL); + _exit(rc); + } + int status = 0; + (void)waitpid(pid, &status, 0); + if (WIFSIGNALED(status)) { + char m[96]; + snprintf(m, sizeof(m), "formatter killed by signal %d: a copy ran past its local buffer", + WTERMSIG(status)); + FAIL(m); + } + ASSERT_TRUE(WIFEXITED(status)); + ASSERT_EQ(WEXITSTATUS(status), 0); +#endif + PASS(); +} + SUITE(pipeline) { RUN_TEST(pipeline_nested_fixture_files_are_written); RUN_TEST(pipeline_fixture_file_parent_is_preserved); @@ -16636,6 +16715,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_markdown_and_config_prose_reaches_fts_body); RUN_TEST(pipeline_semantic_edges_no_functions); RUN_TEST(pipeline_semantic_batched_matches_unbatched); + RUN_TEST(pipeline_calls_args_overlong_argument_stops_cleanly); } /* Focused semantic-manifest and publication contracts. Kept separate from the