mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-10-16 05:10:08 +08:00
fix(extract): walk nested syntax iteratively, bound the args formatter
Four syntax-tree walkers recursed once per nesting level of the indexed file, so their stack use followed the depth of the parse tree rather than anything bounded. They walk iteratively now, with identical results. - terminal_callee_leaf (extract_calls.c) and its PHP LSP mirror php_reference_callee_leaf descended one wrapper node per frame on a callee such as ((((f))))(...). Both are loops now: each step either returns or moves to the child the search continues in, so the nodes visited and the leaf chosen are unchanged. - cbm_collect_error_regions and cbm_mark_pp_error_rows (cbm.c) recursed into every child that contains an error. One cursor walk now drives both. It stops on exactly the visible children ts_node_child enumerated, in the same order, and never enters an ERROR subtree. - is_direct_argument_value (extract_usages.c) moved from a labeled argument's value to the argument itself by recursing. The grammar bounds that step, but it is a loop now, taking the slow-parent test note once per level as before. calls_append_args (pass_calls.c) formatted each argument into a 512-byte local and assumed the result fit. An argument that does not fit now ends the array like one the props buffer cannot take. The parallel twin formats in place and already stops on truncation. Indexing this repository's src/ and internal/cbm/ with and without the change gives byte-identical sorted dumps of nodes, edges per type, CALLS edges and parse-coverage rows. CBM_WORKER_STACK_SIZE moves to worker_pool.h so the new extraction tests run on a thread of exactly the pipeline's size, in a forked child on POSIX so an overflow is a signal instead of a dead runner. They nest 200,000 parentheses around a first-class callable and around an invalid token, pin the labeled-argument classification across the loop conversion, and hand the formatter an argument longer than its local buffer. Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
+67
-28
@@ -1284,24 +1284,63 @@ static bool cbm_is_eof_terminator_miss(TSNode n, const char *source, int source_
|
||||
return true;
|
||||
}
|
||||
|
||||
/* The two walks below visit, in source order, every ERROR or MISSING node that
|
||||
* is top-most in its chain: they step into a child that merely contains an
|
||||
* error and never into an ERROR subtree. That descent follows the nesting of
|
||||
* the construct that failed — every ancestor of the failing node carries
|
||||
* ts_node_has_error — so a recursive walk used one stack frame per nesting
|
||||
* level of the indexed file, and a deep enough nest ran the worker thread out
|
||||
* of stack. The cursor keeps that path on the heap. The nodes visited and
|
||||
* their order are those of the child-index loop it replaced: the cursor
|
||||
* stops on exactly the visible children ts_node_child enumerates. */
|
||||
typedef void (*cbm_error_node_fn)(TSNode node, void *arg);
|
||||
|
||||
static void cbm_walk_error_nodes(TSNode root, const char *source, int source_len,
|
||||
cbm_error_node_fn fn, void *arg) {
|
||||
/* The thread's reusable cursor: the walk completes before anything else on
|
||||
* this thread can ask for it, and the callbacks never do. */
|
||||
TSTreeCursor *cursor = cbm_thread_cursor(root);
|
||||
if (!ts_tree_cursor_goto_first_child(cursor)) {
|
||||
return;
|
||||
}
|
||||
int depth = 0; /* 0 while on a direct child of root */
|
||||
for (;;) {
|
||||
TSNode c = ts_tree_cursor_current_node(cursor);
|
||||
bool descend = false;
|
||||
if (ts_node_is_missing(c) || strcmp(ts_node_type(c), "ERROR") == 0) {
|
||||
/* An absent final newline only means nothing was dropped. Anything
|
||||
* else is the top-most region; do not descend. */
|
||||
if (!cbm_is_eof_terminator_miss(c, source, source_len)) {
|
||||
fn(c, arg);
|
||||
}
|
||||
} else if (ts_node_has_error(c)) {
|
||||
descend = true;
|
||||
}
|
||||
if (descend && ts_tree_cursor_goto_first_child(cursor)) {
|
||||
depth++;
|
||||
continue;
|
||||
}
|
||||
while (!ts_tree_cursor_goto_next_sibling(cursor)) {
|
||||
if (depth == 0) {
|
||||
return;
|
||||
}
|
||||
(void)ts_tree_cursor_goto_parent(cursor);
|
||||
depth--;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Walks to the end even after the cap is full, so `dropped` is the real number
|
||||
* of ranges lost rather than a lower bound. This costs little: the walk never
|
||||
* descends into an ERROR subtree — it records the top-most node and moves on —
|
||||
* so it only visits the spine of nodes that contain an error, plus one level. */
|
||||
static void cbm_collect_error_region(TSNode c, void *arg) {
|
||||
cbm_error_regions_push((cbm_error_regions_t *)arg, c);
|
||||
}
|
||||
|
||||
static void cbm_collect_error_regions(TSNode n, cbm_error_regions_t *acc, const char *source,
|
||||
int source_len) {
|
||||
uint32_t k = ts_node_child_count(n);
|
||||
for (uint32_t i = 0; i < k; i++) {
|
||||
TSNode c = ts_node_child(n, i);
|
||||
if (ts_node_is_missing(c) || strcmp(ts_node_type(c), "ERROR") == 0) {
|
||||
if (cbm_is_eof_terminator_miss(c, source, source_len)) {
|
||||
continue; /* absent final newline only — nothing was dropped */
|
||||
}
|
||||
cbm_error_regions_push(acc, c); /* top-most region; do not descend */
|
||||
} else if (ts_node_has_error(c)) {
|
||||
cbm_collect_error_regions(c, acc, source, source_len);
|
||||
}
|
||||
}
|
||||
cbm_walk_error_nodes(n, source, source_len, cbm_collect_error_region, acc);
|
||||
}
|
||||
|
||||
/* ── Phase 2 line map: what the preprocessed parse already explained ───────
|
||||
@@ -1408,24 +1447,24 @@ static void cbm_mark_no_code_lines(const char *src, int src_len, uint8_t *map,
|
||||
* Step 2 walks the expanded lines and, for each one that is unmarked, belongs
|
||||
* to the file itself (not an included header) and maps back to a real
|
||||
* original line, records that original line as parsed. */
|
||||
typedef struct {
|
||||
uint8_t *rows;
|
||||
uint32_t row_count;
|
||||
} cbm_pp_error_rows_t;
|
||||
|
||||
static void cbm_mark_pp_error_row(TSNode c, void *arg) {
|
||||
cbm_pp_error_rows_t *marks = (cbm_pp_error_rows_t *)arg;
|
||||
uint32_t s = ts_node_start_point(c).row + 1;
|
||||
uint32_t e = ts_node_end_point(c).row + 1;
|
||||
for (uint32_t r = s; r <= e && r <= marks->row_count; r++) {
|
||||
marks->rows[r] = 1;
|
||||
}
|
||||
}
|
||||
|
||||
static void cbm_mark_pp_error_rows(TSNode n, uint8_t *rows, uint32_t row_count, const char *src,
|
||||
int src_len) {
|
||||
uint32_t k = ts_node_child_count(n);
|
||||
for (uint32_t i = 0; i < k; i++) {
|
||||
TSNode c = ts_node_child(n, i);
|
||||
if (ts_node_is_missing(c) || strcmp(ts_node_type(c), "ERROR") == 0) {
|
||||
if (cbm_is_eof_terminator_miss(c, src, src_len)) {
|
||||
continue; /* absent final newline only — nothing was dropped */
|
||||
}
|
||||
uint32_t s = ts_node_start_point(c).row + 1;
|
||||
uint32_t e = ts_node_end_point(c).row + 1;
|
||||
for (uint32_t r = s; r <= e && r <= row_count; r++) {
|
||||
rows[r] = 1;
|
||||
}
|
||||
} else if (ts_node_has_error(c)) {
|
||||
cbm_mark_pp_error_rows(c, rows, row_count, src, src_len);
|
||||
}
|
||||
}
|
||||
cbm_pp_error_rows_t marks = {rows, row_count};
|
||||
cbm_walk_error_nodes(n, src, src_len, cbm_mark_pp_error_row, &marks);
|
||||
}
|
||||
|
||||
/* Recovery subtraction (#963): tree-sitter error recovery plus the
|
||||
|
||||
@@ -3411,43 +3411,48 @@ static TSNode primary_callee_expr(TSNode node) {
|
||||
}
|
||||
|
||||
static TSNode terminal_callee_leaf(const CBMExtractCtx *ctx, TSNode expr) {
|
||||
if (ts_node_is_null(expr) || is_dynamic_callee_expr(ctx, expr)) {
|
||||
return (TSNode){0};
|
||||
}
|
||||
|
||||
static const char *const terminal_fields[] = {"property", "field", "attribute",
|
||||
"method", "name", NULL};
|
||||
TSNode terminal = first_present_field(expr, terminal_fields);
|
||||
if (!ts_node_is_null(terminal)) {
|
||||
if (is_dynamic_callee_expr(ctx, expr)) {
|
||||
// Each step either returns or moves `expr` to the child the search goes
|
||||
// on in. Recursing for that step cost one stack frame per wrapper level of
|
||||
// the callee — `((((f))))(...)`, or the head of a curried application —
|
||||
// so stack use followed the nesting depth of the indexed file and a deep
|
||||
// enough callee ran the worker thread out of stack. The loop visits the
|
||||
// same nodes in the same order.
|
||||
for (;;) {
|
||||
if (ts_node_is_null(expr) || is_dynamic_callee_expr(ctx, expr)) {
|
||||
return (TSNode){0};
|
||||
}
|
||||
return terminal_callee_leaf(ctx, terminal);
|
||||
}
|
||||
|
||||
const char *kind = ts_node_type(expr);
|
||||
if (strcmp(kind, "identifier") == 0 || strcmp(kind, "simple_identifier") == 0 ||
|
||||
strcmp(kind, "type_identifier") == 0 || strcmp(kind, "field_identifier") == 0 ||
|
||||
strcmp(kind, "property_identifier") == 0 || strcmp(kind, "name") == 0 ||
|
||||
strcmp(kind, "variable_name") == 0 || strcmp(kind, "attribute") == 0 ||
|
||||
strcmp(kind, "symbol") == 0 || strcmp(kind, "sym_lit") == 0 ||
|
||||
strcmp(kind, "user_symbol") == 0) {
|
||||
return expr;
|
||||
}
|
||||
TSNode terminal = first_present_field(expr, terminal_fields);
|
||||
if (!ts_node_is_null(terminal)) {
|
||||
expr = terminal;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Known wrappers keep their static terminal in one of their named children.
|
||||
// Prefer the last child for qualified/member/navigation forms and the first
|
||||
// for transparent parentheses and generic wrappers whose name field was not
|
||||
// exposed by the grammar.
|
||||
uint32_t count = ts_node_named_child_count(expr);
|
||||
if (count == 0) {
|
||||
return (TSNode){0};
|
||||
const char *kind = ts_node_type(expr);
|
||||
if (strcmp(kind, "identifier") == 0 || strcmp(kind, "simple_identifier") == 0 ||
|
||||
strcmp(kind, "type_identifier") == 0 || strcmp(kind, "field_identifier") == 0 ||
|
||||
strcmp(kind, "property_identifier") == 0 || strcmp(kind, "name") == 0 ||
|
||||
strcmp(kind, "variable_name") == 0 || strcmp(kind, "attribute") == 0 ||
|
||||
strcmp(kind, "symbol") == 0 || strcmp(kind, "sym_lit") == 0 ||
|
||||
strcmp(kind, "user_symbol") == 0) {
|
||||
return expr;
|
||||
}
|
||||
|
||||
// Known wrappers keep their static terminal in one of their named
|
||||
// children. Prefer the last child for qualified/member/navigation forms
|
||||
// and the first for transparent parentheses and generic wrappers whose
|
||||
// name field was not exposed by the grammar.
|
||||
uint32_t count = ts_node_named_child_count(expr);
|
||||
if (count == 0) {
|
||||
return (TSNode){0};
|
||||
}
|
||||
bool terminal_is_last = strstr(kind, "qualified") || strstr(kind, "scoped") ||
|
||||
strstr(kind, "member") || strstr(kind, "selector") ||
|
||||
strstr(kind, "navigation") || strstr(kind, "field_expression");
|
||||
expr = ts_node_named_child(expr, terminal_is_last ? count - 1 : 0);
|
||||
}
|
||||
bool terminal_is_last = strstr(kind, "qualified") || strstr(kind, "scoped") ||
|
||||
strstr(kind, "member") || strstr(kind, "selector") ||
|
||||
strstr(kind, "navigation") || strstr(kind, "field_expression");
|
||||
TSNode child = ts_node_named_child(expr, terminal_is_last ? count - 1 : 0);
|
||||
return terminal_callee_leaf(ctx, child);
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
|
||||
@@ -1346,37 +1346,47 @@ static bool is_call_argument_label_walk(TSNode node, WalkState *state) {
|
||||
}
|
||||
|
||||
static bool is_direct_argument_value(TSNode node) {
|
||||
usage_slow_parent_fallback_test_note();
|
||||
TSNode parent = ts_node_parent(node);
|
||||
if (ts_node_is_null(parent)) {
|
||||
return false;
|
||||
}
|
||||
if (is_labeled_argument_kind(ts_node_type(parent))) {
|
||||
TSNode value = ts_node_child_by_field_name(parent, TS_FIELD("value"));
|
||||
return !ts_node_is_null(value) && ts_node_eq(value, node) &&
|
||||
is_direct_argument_value(parent);
|
||||
}
|
||||
TSNode direct_arguments = ts_node_child_by_field_name(parent, TS_FIELD("arguments"));
|
||||
if (!ts_node_is_null(direct_arguments) && ts_node_eq(direct_arguments, node)) {
|
||||
return true;
|
||||
}
|
||||
if (is_argument_container_kind(ts_node_type(parent))) {
|
||||
return true;
|
||||
}
|
||||
const char *parent_kind = ts_node_type(parent);
|
||||
if (strcmp(parent_kind, "list_expression") == 0) {
|
||||
TSNode call = ts_node_parent(parent);
|
||||
if (!ts_node_is_null(call)) {
|
||||
TSNode arguments = ts_node_child_by_field_name(call, TS_FIELD("arguments"));
|
||||
return !ts_node_is_null(arguments) && ts_node_eq(arguments, parent);
|
||||
/* A labeled argument (keyword_argument and kin) is itself what the call
|
||||
* site holds, so when `node` is its value the question moves one level up.
|
||||
* That step used to recurse; the loop asks it in place and takes the
|
||||
* slow-parent note once per level, as each recursive entry did. */
|
||||
for (;;) {
|
||||
usage_slow_parent_fallback_test_note();
|
||||
TSNode parent = ts_node_parent(node);
|
||||
if (ts_node_is_null(parent)) {
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
if (is_labeled_argument_kind(ts_node_type(parent))) {
|
||||
TSNode value = ts_node_child_by_field_name(parent, TS_FIELD("value"));
|
||||
if (ts_node_is_null(value) || !ts_node_eq(value, node)) {
|
||||
return false;
|
||||
}
|
||||
node = parent;
|
||||
continue;
|
||||
}
|
||||
TSNode direct_arguments = ts_node_child_by_field_name(parent, TS_FIELD("arguments"));
|
||||
if (!ts_node_is_null(direct_arguments) && ts_node_eq(direct_arguments, node)) {
|
||||
return true;
|
||||
}
|
||||
if (is_argument_container_kind(ts_node_type(parent))) {
|
||||
return true;
|
||||
}
|
||||
const char *parent_kind = ts_node_type(parent);
|
||||
if (strcmp(parent_kind, "list_expression") == 0) {
|
||||
TSNode call = ts_node_parent(parent);
|
||||
if (!ts_node_is_null(call)) {
|
||||
TSNode arguments = ts_node_child_by_field_name(call, TS_FIELD("arguments"));
|
||||
return !ts_node_is_null(arguments) && ts_node_eq(arguments, parent);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
if (strcmp(parent_kind, "argument") != 0 && strcmp(parent_kind, "value_argument") != 0) {
|
||||
return false;
|
||||
}
|
||||
TSNode grandparent = ts_node_parent(parent);
|
||||
return !ts_node_is_null(grandparent) &&
|
||||
is_argument_container_kind(ts_node_type(grandparent));
|
||||
}
|
||||
if (strcmp(parent_kind, "argument") != 0 && strcmp(parent_kind, "value_argument") != 0) {
|
||||
return false;
|
||||
}
|
||||
TSNode grandparent = ts_node_parent(parent);
|
||||
return !ts_node_is_null(grandparent) && is_argument_container_kind(ts_node_type(grandparent));
|
||||
}
|
||||
|
||||
/* The body of the walk above, entered one level in: for a caller that has
|
||||
|
||||
+33
-27
@@ -1352,39 +1352,45 @@ static bool php_callable_reference_has_dynamic_member_name(PHPLSPContext *ctx, T
|
||||
* USAGE passes attach the same exact token occurrence to a first-class
|
||||
* callable, including qualified free-function references. */
|
||||
static TSNode php_reference_callee_leaf(PHPLSPContext *ctx, TSNode expr) {
|
||||
if (ts_node_is_null(expr) || php_dynamic_reference_expr(ctx, expr)) {
|
||||
return (TSNode){0};
|
||||
}
|
||||
|
||||
static const char *const terminal_fields[] = {"property", "field", "attribute",
|
||||
"method", "name", NULL};
|
||||
for (int i = 0; terminal_fields[i]; i++) {
|
||||
TSNode terminal = ts_node_child_by_field_name(expr, terminal_fields[i],
|
||||
(uint32_t)strlen(terminal_fields[i]));
|
||||
if (!ts_node_is_null(terminal)) {
|
||||
return php_reference_callee_leaf(ctx, terminal);
|
||||
/* A loop for the same reason as terminal_callee_leaf: recursing once per
|
||||
* wrapper level of the callee made stack use follow the nesting depth of
|
||||
* the file. Same nodes, same order. */
|
||||
for (;;) {
|
||||
if (ts_node_is_null(expr) || php_dynamic_reference_expr(ctx, expr)) {
|
||||
return (TSNode){0};
|
||||
}
|
||||
}
|
||||
|
||||
const char *kind = ts_node_type(expr);
|
||||
if (strcmp(kind, "identifier") == 0 || strcmp(kind, "simple_identifier") == 0 ||
|
||||
strcmp(kind, "type_identifier") == 0 || strcmp(kind, "field_identifier") == 0 ||
|
||||
strcmp(kind, "property_identifier") == 0 || strcmp(kind, "name") == 0 ||
|
||||
strcmp(kind, "variable_name") == 0 || strcmp(kind, "attribute") == 0 ||
|
||||
strcmp(kind, "symbol") == 0 || strcmp(kind, "sym_lit") == 0 ||
|
||||
strcmp(kind, "user_symbol") == 0) {
|
||||
return expr;
|
||||
}
|
||||
TSNode terminal = {0};
|
||||
for (int i = 0; terminal_fields[i] && ts_node_is_null(terminal); i++) {
|
||||
terminal = ts_node_child_by_field_name(expr, terminal_fields[i],
|
||||
(uint32_t)strlen(terminal_fields[i]));
|
||||
}
|
||||
if (!ts_node_is_null(terminal)) {
|
||||
expr = terminal;
|
||||
continue;
|
||||
}
|
||||
|
||||
uint32_t count = ts_node_named_child_count(expr);
|
||||
if (count == 0) {
|
||||
return (TSNode){0};
|
||||
const char *kind = ts_node_type(expr);
|
||||
if (strcmp(kind, "identifier") == 0 || strcmp(kind, "simple_identifier") == 0 ||
|
||||
strcmp(kind, "type_identifier") == 0 || strcmp(kind, "field_identifier") == 0 ||
|
||||
strcmp(kind, "property_identifier") == 0 || strcmp(kind, "name") == 0 ||
|
||||
strcmp(kind, "variable_name") == 0 || strcmp(kind, "attribute") == 0 ||
|
||||
strcmp(kind, "symbol") == 0 || strcmp(kind, "sym_lit") == 0 ||
|
||||
strcmp(kind, "user_symbol") == 0) {
|
||||
return expr;
|
||||
}
|
||||
|
||||
uint32_t count = ts_node_named_child_count(expr);
|
||||
if (count == 0) {
|
||||
return (TSNode){0};
|
||||
}
|
||||
bool terminal_is_last = strstr(kind, "qualified") || strstr(kind, "scoped") ||
|
||||
strstr(kind, "member") || strstr(kind, "selector") ||
|
||||
strstr(kind, "navigation") || strstr(kind, "field_expression");
|
||||
expr = ts_node_named_child(expr, terminal_is_last ? count - 1 : 0);
|
||||
}
|
||||
bool terminal_is_last = strstr(kind, "qualified") || strstr(kind, "scoped") ||
|
||||
strstr(kind, "member") || strstr(kind, "selector") ||
|
||||
strstr(kind, "navigation") || strstr(kind, "field_expression");
|
||||
return php_reference_callee_leaf(ctx,
|
||||
ts_node_named_child(expr, terminal_is_last ? count - 1 : 0));
|
||||
}
|
||||
|
||||
static TSNode php_callable_reference_leaf(PHPLSPContext *ctx, TSNode call, const char *kind) {
|
||||
|
||||
@@ -486,8 +486,14 @@ static void calls_append_args(char *props, size_t cap, const CBMCall *call) {
|
||||
/* Add rather than subtract: pos is unsigned, so `cap - pos - PAIR_LEN`
|
||||
* wraps once pos reaches cap - PAIR_LEN and stops bounding the memcpy
|
||||
* below. The closing write at the end of this function already guards
|
||||
* additively; match it. */
|
||||
if (n <= 0 || pos + (size_t)n + PAIR_LEN >= cap) {
|
||||
* additively; match it.
|
||||
*
|
||||
* snprintf reports the untruncated length, so an argument `one` could
|
||||
* not hold whole is never copied out of it: copying n bytes would read
|
||||
* past the local and plant its terminator inside the props text. It
|
||||
* ends the array exactly like an argument the props buffer cannot
|
||||
* take. */
|
||||
if (n <= 0 || (size_t)n >= sizeof(one) || pos + (size_t)n + PAIR_LEN >= cap) {
|
||||
break; /* not enough room — close the array with what fits */
|
||||
}
|
||||
memcpy(props + pos, one, (size_t)n);
|
||||
@@ -500,6 +506,11 @@ static void calls_append_args(char *props, size_t cap, const CBMCall *call) {
|
||||
}
|
||||
}
|
||||
|
||||
/* Test seam for the formatter above. Not part of the public API. */
|
||||
void cbm_pipeline_calls_append_args_for_tests(char *props, size_t cap, const CBMCall *call) {
|
||||
calls_append_args(props, cap, call);
|
||||
}
|
||||
|
||||
static void calls_emit_edge(cbm_gbuf_t *gbuf, int64_t src, int64_t tgt, const char *type,
|
||||
char *props, size_t cap, const CBMCall *call) {
|
||||
if (call && call->start_line > 0 && strcmp(type, "CALLS") == 0) {
|
||||
|
||||
@@ -927,6 +927,12 @@ void cbm_pipeline_set_rename_hook_for_tests(cbm_pipeline_t *p,
|
||||
int (*hook)(const char *, const char *, void *),
|
||||
void *ctx);
|
||||
|
||||
/* Test seam: the sequential CALLS pass's args formatter (pass_calls.c), so a
|
||||
* test can hand it an argument longer than its formatting buffer and read the
|
||||
* props it leaves behind. `props` is a NUL-terminated JSON object of `cap`
|
||||
* bytes; the formatter appends its "args" array in place. */
|
||||
void cbm_pipeline_calls_append_args_for_tests(char *props, size_t cap, const CBMCall *call);
|
||||
|
||||
/* Synchronous thread-local seam for deterministic cross-repo cancellation
|
||||
* tests. The callback runs immediately after a CROSS_* edge is committed and
|
||||
* is never retained; it must not re-enter cross-repo matching. */
|
||||
|
||||
@@ -20,9 +20,7 @@ enum { WP_TRUE = 1, WP_MIN = 1, WP_STEP = 1 };
|
||||
#include <stdatomic.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
/* 8 MB stack per worker — matches main thread default.
|
||||
* Required for deep AST recursion (tree-sitter + walk_defs). */
|
||||
#define CBM_WORKER_STACK_SIZE ((size_t)8 * CBM_SZ_1K * CBM_SZ_1K)
|
||||
/* CBM_WORKER_STACK_SIZE (8 MB per worker) lives in worker_pool.h. */
|
||||
|
||||
/* ── Serial fallback ─────────────────────────────────────────────── */
|
||||
|
||||
|
||||
@@ -11,6 +11,13 @@
|
||||
#define CBM_WORKER_POOL_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include "foundation/constants.h" /* CBM_SZ_1K */
|
||||
|
||||
/* 8 MB stack per worker — matches main thread default. Required for deep AST
|
||||
* recursion (tree-sitter + walk_defs). Shared here so a test can run an
|
||||
* extraction on a thread of exactly the size the pipeline gives it. */
|
||||
#define CBM_WORKER_STACK_SIZE ((size_t)8 * CBM_SZ_1K * CBM_SZ_1K)
|
||||
|
||||
/* Worker callback: called once per iteration with index [0..count-1]. */
|
||||
typedef void (*cbm_parallel_fn)(int idx, void *ctx);
|
||||
|
||||
@@ -19,6 +19,13 @@
|
||||
#include "helpers.h" /* cbm_count_branching (walker stack cap) */
|
||||
#include "lang_specs.h" /* cbm_lang_spec, cbm_ts_language */
|
||||
|
||||
/* Deep-nesting probes: extraction on a worker-sized stack, crash-isolated. */
|
||||
#include "pipeline/worker_pool.h" /* CBM_WORKER_STACK_SIZE */
|
||||
#include "../src/foundation/compat_thread.h" /* cbm_thread_create */
|
||||
#if !defined(_WIN32)
|
||||
#include <sys/wait.h> /* fork/waitpid */
|
||||
#endif
|
||||
|
||||
/* ── Helpers ───────────────────────────────────────────────────── */
|
||||
|
||||
/* Check if any definition with the given label has the given name. */
|
||||
@@ -8863,6 +8870,315 @@ TEST(extract_walk_truncated_when_a_node_budget_is_set) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════
|
||||
* Deep nesting: the syntax-tree walkers must not spend stack per level
|
||||
*
|
||||
* Extraction runs on pipeline worker threads whose stack is
|
||||
* CBM_WORKER_STACK_SIZE (src/pipeline/worker_pool.h), not on the test runner's
|
||||
* main thread, whose size depends on the platform and its rlimit. Each probe
|
||||
* below therefore runs cbm_extract_file on a thread of exactly that size, so
|
||||
* a walker that recurses once per nesting level runs out of stack at the
|
||||
* depth it would in the pipeline. On POSIX the thread lives in a forked child
|
||||
* and an overflow surfaces as the child's killing signal instead of taking
|
||||
* the runner with it (the pattern of tests/test_stack_overflow.c); Windows
|
||||
* runs it in-process, where a real overflow is a visible runner abort. A
|
||||
* probe returns 0 when every expectation held and prints what did not.
|
||||
*
|
||||
* DEEP_NEST_LEVELS: a walker frame holds at least its saved frame pointer and
|
||||
* return address plus the 32-byte TSNode it passes down (on arm64 a TSNode is
|
||||
* passed by reference to a copy in the caller's frame), 48 bytes, so 200,000
|
||||
* frames need 9.6 MB — past the 8 MB worker stack even at that floor;
|
||||
* measured frames are larger and the sanitizer build adds red zones. The
|
||||
* parser is not the limit: tree-sitter keeps the nesting on its heap parse
|
||||
* stack and its node and cursor helpers are iterative, so the depth is the
|
||||
* walker's problem alone. The fixtures nest parentheses, which copy no text
|
||||
* per level (a nested call copies its argument text at every level, which is
|
||||
* what makes the stack_overflow_b suite slow).
|
||||
* ═══════════════════════════════════════════════════════════════════ */
|
||||
|
||||
enum { DEEP_NEST_LEVELS = 200000 };
|
||||
|
||||
typedef int (*deep_probe_fn)(void);
|
||||
|
||||
typedef struct {
|
||||
deep_probe_fn fn;
|
||||
int rc;
|
||||
} deep_probe_job_t;
|
||||
|
||||
static void *deep_probe_thread_main(void *arg) {
|
||||
deep_probe_job_t *job = (deep_probe_job_t *)arg;
|
||||
job->rc = job->fn();
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* 0 when the probe passed, its own code otherwise, 2 if no thread came up. */
|
||||
static int deep_probe_on_worker_stack(deep_probe_fn fn) {
|
||||
deep_probe_job_t job = {fn, 3};
|
||||
cbm_thread_t thread;
|
||||
if (cbm_thread_create(&thread, CBM_WORKER_STACK_SIZE, deep_probe_thread_main, &job) != 0) {
|
||||
return 2;
|
||||
}
|
||||
(void)cbm_thread_join(&thread);
|
||||
return job.rc;
|
||||
}
|
||||
|
||||
/* 1 when the probe passed; otherwise `why` says what happened. */
|
||||
static int deep_probe_passed(deep_probe_fn fn, char *why, size_t why_len) {
|
||||
#if defined(_WIN32)
|
||||
int rc = deep_probe_on_worker_stack(fn);
|
||||
if (rc != 0) {
|
||||
snprintf(why, why_len, "probe returned %d", rc);
|
||||
}
|
||||
return rc == 0;
|
||||
#else
|
||||
fflush(NULL);
|
||||
pid_t pid = fork();
|
||||
if (pid < 0) {
|
||||
snprintf(why, why_len, "fork failed");
|
||||
return 0;
|
||||
}
|
||||
if (pid == 0) {
|
||||
int rc = deep_probe_on_worker_stack(fn);
|
||||
fflush(NULL);
|
||||
_exit(rc);
|
||||
}
|
||||
int status = 0;
|
||||
(void)waitpid(pid, &status, 0);
|
||||
if (WIFSIGNALED(status)) {
|
||||
snprintf(why, why_len,
|
||||
"extraction killed by signal %d on a worker-sized stack: a walker recursed once "
|
||||
"per nesting level",
|
||||
WTERMSIG(status));
|
||||
return 0;
|
||||
}
|
||||
if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
|
||||
snprintf(why, why_len, "probe returned %d", WIFEXITED(status) ? WEXITSTATUS(status) : -1);
|
||||
return 0;
|
||||
}
|
||||
return 1;
|
||||
#endif
|
||||
}
|
||||
|
||||
/* head, `levels` times '(', core, `levels` times ')', tail. *core_at receives
|
||||
* the byte offset of core. */
|
||||
static char *build_nested_source(const char *head, const char *core, const char *tail, int levels,
|
||||
uint32_t *core_at) {
|
||||
size_t head_len = strlen(head);
|
||||
size_t core_len = strlen(core);
|
||||
size_t tail_len = strlen(tail);
|
||||
char *src = malloc(head_len + core_len + tail_len + 2 * (size_t)levels + 1);
|
||||
if (!src) {
|
||||
return NULL;
|
||||
}
|
||||
char *w = src;
|
||||
memcpy(w, head, head_len);
|
||||
w += head_len;
|
||||
memset(w, '(', (size_t)levels);
|
||||
w += levels;
|
||||
*core_at = (uint32_t)(w - src);
|
||||
memcpy(w, core, core_len);
|
||||
w += core_len;
|
||||
memset(w, ')', (size_t)levels);
|
||||
w += levels;
|
||||
memcpy(w, tail, tail_len + 1);
|
||||
return src;
|
||||
}
|
||||
|
||||
/* `$x = ((...((f))...))(...);` — a first-class callable whose callee sits under
|
||||
* DEEP_NEST_LEVELS parentheses. PHP, because its first-class callable syntax
|
||||
* is the path that hands a parenthesised callee to terminal_callee_leaf
|
||||
* (extract_calls.c): the leaf finder steps into one named child per wrapper
|
||||
* node that has no terminal field, while every ordinary call path first needs
|
||||
* a textual callee name, which the field resolver only yields for identifier
|
||||
* and member shapes whose leaf sits one or two levels down (a member chain
|
||||
* `a.b.c()` stops at its `property`; nested calls `f(f(f()))` are arguments,
|
||||
* visited by the explicit-stack walker). Expected: the exact occurrence the
|
||||
* leaf finder names — one CALL_REFERENCE usage of `f` at the byte of the `f`
|
||||
* token — in a cleanly parsed file. */
|
||||
static int deep_callee_chain_probe(void) {
|
||||
uint32_t f_at = 0;
|
||||
char *src = build_nested_source("<?php\n$x = ", "f", "(...);\n", DEEP_NEST_LEVELS, &f_at);
|
||||
if (!src) {
|
||||
return 4;
|
||||
}
|
||||
CBMFileResult *r = extract(src, CBM_LANG_PHP, "t", "deep_callee.php");
|
||||
int rc = 0;
|
||||
if (!r || r->has_error || r->parse_incomplete) {
|
||||
printf(" deep callee: extraction failed or the parse was not clean\n");
|
||||
rc = 1;
|
||||
} else {
|
||||
int references = 0;
|
||||
for (int i = 0; i < r->usages.count; i++) {
|
||||
const CBMUsage *u = &r->usages.items[i];
|
||||
if (u->kind != CBM_USAGE_CALL_REFERENCE || !u->ref_name ||
|
||||
strcmp(u->ref_name, "f") != 0) {
|
||||
continue;
|
||||
}
|
||||
references++;
|
||||
if (u->site_start_byte != f_at || u->site_end_byte != f_at + 1) {
|
||||
printf(" deep callee: reference site %u-%u, expected %u-%u\n",
|
||||
u->site_start_byte, u->site_end_byte, f_at, f_at + 1);
|
||||
rc = 1;
|
||||
}
|
||||
}
|
||||
if (references != 1) {
|
||||
printf(" deep callee: %d CALL_REFERENCE usages of f, expected 1\n", references);
|
||||
rc = 1;
|
||||
}
|
||||
}
|
||||
if (r) {
|
||||
cbm_free_result(r);
|
||||
}
|
||||
free(src);
|
||||
return rc;
|
||||
}
|
||||
|
||||
TEST(extract_deep_callee_chain_completes) {
|
||||
char why[160];
|
||||
if (!deep_probe_passed(deep_callee_chain_probe, why, sizeof(why))) {
|
||||
FAIL(why);
|
||||
}
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* One invalid token at the bottom of DEEP_NEST_LEVELS parentheses. Every
|
||||
* enclosing parenthesized_expression then carries ts_node_has_error without
|
||||
* being an ERROR node, which is the chain cbm_collect_error_regions (cbm.c)
|
||||
* descends: it steps into a child that contains an error and stops at the
|
||||
* top-most ERROR or MISSING node. Nested ERROR nodes would not do — the walk
|
||||
* never enters an ERROR subtree, so they cost it one level. Expected: one
|
||||
* region, on the line of the nest only, with the neighbouring definition
|
||||
* intact and the file still usable. */
|
||||
static int deep_error_regions_probe(void) {
|
||||
uint32_t core_at = 0;
|
||||
char *src = build_nested_source("def ok():\n return 1\nx = ", "1 2", "\ny = ok()\n",
|
||||
DEEP_NEST_LEVELS, &core_at);
|
||||
if (!src) {
|
||||
return 4;
|
||||
}
|
||||
CBMFileResult *r = extract(src, CBM_LANG_PYTHON, "t", "deep_error.py");
|
||||
int rc = 0;
|
||||
if (!r) {
|
||||
printf(" deep error regions: extraction failed\n");
|
||||
rc = 1;
|
||||
} else {
|
||||
if (!r->parse_incomplete || r->error_region_count != 1 || !r->error_ranges ||
|
||||
strcmp(r->error_ranges, "3-3") != 0) {
|
||||
printf(" deep error regions: incomplete=%d count=%d ranges=%s, expected one "
|
||||
"region 3-3\n",
|
||||
r->parse_incomplete ? 1 : 0, r->error_region_count,
|
||||
r->error_ranges ? r->error_ranges : "(none)");
|
||||
rc = 1;
|
||||
}
|
||||
if (r->parse_unusable) {
|
||||
printf(" deep error regions: one line of five must not make the file unusable\n");
|
||||
rc = 1;
|
||||
}
|
||||
if (!has_def(r, "Function", "ok")) {
|
||||
printf(" deep error regions: the definition next to the nest was lost\n");
|
||||
rc = 1;
|
||||
}
|
||||
cbm_free_result(r);
|
||||
}
|
||||
free(src);
|
||||
return rc;
|
||||
}
|
||||
|
||||
TEST(extract_deep_error_regions_complete) {
|
||||
char why[160];
|
||||
if (!deep_probe_passed(deep_error_regions_probe, why, sizeof(why))) {
|
||||
FAIL(why);
|
||||
}
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* `f(cb=((...(obj.handler)...)))`: a labeled argument whose value is a bound
|
||||
* method under LABELED_WRAPPERS parentheses. The usage pass classifies
|
||||
* `handler` by climbing the wrappers (python_direct_callable_attribute_site, a
|
||||
* loop) and then asking is_direct_argument_value, whose one recursive step
|
||||
* moved from the value to the labeled argument holding it. The grammar never
|
||||
* nests a labeled argument inside another's value, so that step is bounded
|
||||
* and no stack depth is at stake; what the test pins is the answer across the
|
||||
* conversion of that step to a loop: the usage is a callable-value candidate
|
||||
* whose site is the outermost wrapper, and the keyword `cb` is no usage. The
|
||||
* bare form pins the site on the attribute itself. */
|
||||
enum { LABELED_WRAPPERS = 2000 };
|
||||
|
||||
static int handler_candidate_site(const CBMFileResult *r, uint32_t expect_start,
|
||||
uint32_t expect_end, const char *form) {
|
||||
int seen = 0;
|
||||
int rc = 0;
|
||||
for (int i = 0; i < r->usages.count; i++) {
|
||||
const CBMUsage *u = &r->usages.items[i];
|
||||
if (!u->ref_name) {
|
||||
continue;
|
||||
}
|
||||
if (strcmp(u->ref_name, "cb") == 0) {
|
||||
printf(" %s argument: the keyword label became a usage\n", form);
|
||||
rc = 1;
|
||||
}
|
||||
if (strcmp(u->ref_name, "handler") != 0) {
|
||||
continue;
|
||||
}
|
||||
seen++;
|
||||
if (!u->may_be_call_reference || u->site_start_byte != expect_start ||
|
||||
u->site_end_byte != expect_end) {
|
||||
printf(" %s argument: handler candidate=%d site %u-%u, expected candidate site "
|
||||
"%u-%u\n",
|
||||
form, u->may_be_call_reference ? 1 : 0, u->site_start_byte, u->site_end_byte,
|
||||
expect_start, expect_end);
|
||||
rc = 1;
|
||||
}
|
||||
}
|
||||
if (seen != 1) {
|
||||
printf(" %s argument: %d usages of handler, expected 1\n", form, seen);
|
||||
rc = 1;
|
||||
}
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int deep_argument_nesting_probe(void) {
|
||||
const uint32_t attribute_len = (uint32_t)strlen("obj.handler");
|
||||
uint32_t core_at = 0;
|
||||
char *src = build_nested_source("f(cb=", "obj.handler", ")\n", LABELED_WRAPPERS, &core_at);
|
||||
if (!src) {
|
||||
return 4;
|
||||
}
|
||||
int rc = 0;
|
||||
CBMFileResult *wrapped = extract(src, CBM_LANG_PYTHON, "t", "deep_arg.py");
|
||||
if (!wrapped || wrapped->has_error || wrapped->parse_incomplete) {
|
||||
printf(" wrapped argument: extraction failed or the parse was not clean\n");
|
||||
rc = 1;
|
||||
} else {
|
||||
rc |= handler_candidate_site(wrapped, core_at - LABELED_WRAPPERS,
|
||||
core_at + attribute_len + LABELED_WRAPPERS, "wrapped");
|
||||
}
|
||||
if (wrapped) {
|
||||
cbm_free_result(wrapped);
|
||||
}
|
||||
free(src);
|
||||
|
||||
CBMFileResult *bare = extract("f(cb=obj.handler)\n", CBM_LANG_PYTHON, "t", "bare_arg.py");
|
||||
if (!bare || bare->has_error || bare->parse_incomplete) {
|
||||
printf(" bare argument: extraction failed or the parse was not clean\n");
|
||||
rc = 1;
|
||||
} else {
|
||||
rc |= handler_candidate_site(bare, 5, 5 + attribute_len, "bare");
|
||||
}
|
||||
if (bare) {
|
||||
cbm_free_result(bare);
|
||||
}
|
||||
return rc;
|
||||
}
|
||||
|
||||
TEST(extract_deep_argument_nesting_completes) {
|
||||
char why[160];
|
||||
if (!deep_probe_passed(deep_argument_nesting_probe, why, sizeof(why))) {
|
||||
FAIL(why);
|
||||
}
|
||||
PASS();
|
||||
}
|
||||
|
||||
SUITE(extraction) {
|
||||
RUN_TEST(extract_compact_keeps_every_field_and_shrinks_the_arena);
|
||||
RUN_TEST(extract_compact_is_idempotent_and_survives_empty_results);
|
||||
@@ -9305,5 +9621,10 @@ SUITE(extraction) {
|
||||
RUN_TEST(config_description_only_at_top_level_issue519);
|
||||
RUN_TEST(non_config_language_module_has_no_promoted_description_issue519);
|
||||
|
||||
/* Syntax-tree walkers spend no stack per nesting level */
|
||||
RUN_TEST(extract_deep_callee_chain_completes);
|
||||
RUN_TEST(extract_deep_error_regions_complete);
|
||||
RUN_TEST(extract_deep_argument_nesting_completes);
|
||||
|
||||
cbm_shutdown();
|
||||
}
|
||||
|
||||
@@ -16270,6 +16270,85 @@ TEST(pipeline_objectscript_export_range_join_keeps_one_trailing_marker) {
|
||||
}
|
||||
#endif
|
||||
|
||||
/* ── calls_append_args: an argument longer than its formatting buffer ────── */
|
||||
|
||||
#if !defined(_WIN32)
|
||||
#include <sys/wait.h> /* fork/waitpid: isolate a copy that runs past a local buffer */
|
||||
#endif
|
||||
|
||||
/* The sequential CALLS pass formats each argument into a 512-byte local before
|
||||
* copying it into the props buffer (calls_append_args, pass_calls.c). snprintf
|
||||
* reports the untruncated length, so an argument the local cannot hold must end
|
||||
* the array exactly like one the props buffer cannot take; copying that length
|
||||
* out of the local reads past it and plants the local's terminator inside the
|
||||
* props text. Two 250-byte strings make the formatted argument about 520
|
||||
* bytes, and the props buffer is large enough to take it, so only the local's
|
||||
* own bound can stop the copy. 0 when the props came out right, 1 otherwise. */
|
||||
static int calls_args_overlong_probe(void) {
|
||||
enum { CAP = 4 * CBM_SZ_1K, LONG_LEN = 250 };
|
||||
static char expr[LONG_LEN + 1];
|
||||
static char value[LONG_LEN + 1];
|
||||
static char props[CAP];
|
||||
memset(expr, 'a', LONG_LEN);
|
||||
expr[LONG_LEN] = '\0';
|
||||
memset(value, 'b', LONG_LEN);
|
||||
value[LONG_LEN] = '\0';
|
||||
CBMCallArg args[2] = {{"x", NULL, NULL, 0}, {expr, value, NULL, 1}};
|
||||
CBMCall call = {0};
|
||||
call.callee_name = "sink";
|
||||
call.args = args;
|
||||
call.arg_count = 2;
|
||||
|
||||
const char *base = "{\"line\":7}";
|
||||
memset(props, 0x7f, CAP); /* a sentinel the formatter never writes */
|
||||
memcpy(props, base, strlen(base) + 1);
|
||||
cbm_pipeline_calls_append_args_for_tests(props, CAP, &call);
|
||||
|
||||
/* The short argument is kept; the overlong one ends the array cleanly. */
|
||||
const char *expected = "{\"line\":7,\"args\":[{\"i\":0,\"e\":\"x\"}]}";
|
||||
size_t expected_len = strlen(expected);
|
||||
if (strncmp(props, expected, expected_len + 1) != 0) {
|
||||
printf(" props: %.80s\n expected: %s\n", props, expected);
|
||||
return 1;
|
||||
}
|
||||
for (size_t i = expected_len + 1; i < CAP; i++) {
|
||||
if ((unsigned char)props[i] != 0x7f) {
|
||||
printf(" byte %zu past the terminator was written\n", i);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
TEST(pipeline_calls_args_overlong_argument_stops_cleanly) {
|
||||
#if defined(_WIN32)
|
||||
ASSERT_EQ(calls_args_overlong_probe(), 0);
|
||||
#else
|
||||
/* A forked child, so a copy that runs past the local surfaces as the
|
||||
* child's killing signal (the sanitizer build aborts on it) rather than
|
||||
* taking the runner down. */
|
||||
fflush(NULL);
|
||||
pid_t pid = fork();
|
||||
ASSERT_TRUE(pid >= 0);
|
||||
if (pid == 0) {
|
||||
int rc = calls_args_overlong_probe();
|
||||
fflush(NULL);
|
||||
_exit(rc);
|
||||
}
|
||||
int status = 0;
|
||||
(void)waitpid(pid, &status, 0);
|
||||
if (WIFSIGNALED(status)) {
|
||||
char m[96];
|
||||
snprintf(m, sizeof(m), "formatter killed by signal %d: a copy ran past its local buffer",
|
||||
WTERMSIG(status));
|
||||
FAIL(m);
|
||||
}
|
||||
ASSERT_TRUE(WIFEXITED(status));
|
||||
ASSERT_EQ(WEXITSTATUS(status), 0);
|
||||
#endif
|
||||
PASS();
|
||||
}
|
||||
|
||||
SUITE(pipeline) {
|
||||
RUN_TEST(pipeline_nested_fixture_files_are_written);
|
||||
RUN_TEST(pipeline_fixture_file_parent_is_preserved);
|
||||
@@ -16636,6 +16715,7 @@ SUITE(pipeline) {
|
||||
RUN_TEST(pipeline_markdown_and_config_prose_reaches_fts_body);
|
||||
RUN_TEST(pipeline_semantic_edges_no_functions);
|
||||
RUN_TEST(pipeline_semantic_batched_matches_unbatched);
|
||||
RUN_TEST(pipeline_calls_args_overlong_argument_stops_cleanly);
|
||||
}
|
||||
|
||||
/* Focused semantic-manifest and publication contracts. Kept separate from the
|
||||
|
||||
Reference in New Issue
Block a user