Merge branch 'main' into fix/issue-1682-dotted-ts-imports

This commit is contained in:
Pcristin
2026-09-01 21:13:59 +03:00
committed by GitHub
48 changed files with 765249 additions and 470344 deletions
+10 -4
View File
@@ -42,7 +42,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 240
steps:
- name: Wait for CodeQL on current commit (max 45 min)
- name: Wait for CodeQL on current commit (max 150 min)
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
@@ -50,11 +50,17 @@ jobs:
# CodeQL runs are recorded against the PR head SHA.
CURRENT_SHA="${{ github.event.pull_request.head.sha || github.sha }}"
echo "Waiting for CodeQL to complete on $CURRENT_SHA..."
for attempt in $(seq 1 90); do
# 300 attempts x 30s = 150 min. The previous budget was 90 x 30s = 45 min,
# which is shorter than CodeQL actually takes on this repository: a measured
# run on PR #1426 (head 7b72652a) completed with conclusion=success after
# 124 min, 7 minutes AFTER this gate had already given up. That marked at
# least five contributor PRs red for a scan that passed. The job's own
# timeout-minutes is 240, so 150 still leaves headroom.
for attempt in $(seq 1 300); do
LATEST=$(gh api "repos/${{ github.repository }}/actions/workflows/codeql.yml/runs?head_sha=$CURRENT_SHA&per_page=1" \
--jq '.workflow_runs[] | "\(.conclusion) \(.status)"' 2>/dev/null | head -1 || echo "")
if [ -z "$LATEST" ]; then
echo " $attempt/90: no run yet..."; sleep 30; continue
echo " $attempt/300: no run yet..."; sleep 30; continue
fi
CONCLUSION=$(echo "$LATEST" | cut -d' ' -f1)
STATUS=$(echo "$LATEST" | cut -d' ' -f2)
@@ -63,7 +69,7 @@ jobs:
elif [ "$STATUS" = "completed" ]; then
echo "BLOCKED: CodeQL $CONCLUSION"; exit 1
fi
echo " $attempt/90: $STATUS..."; sleep 30
echo " $attempt/300: $STATUS..."; sleep 30
done
echo "BLOCKED: CodeQL timeout"; exit 1
+68
View File
@@ -1161,6 +1161,65 @@ static char *extract_nickel_callee(CBMArena *a, TSNode node, const char *source,
return NULL;
}
// Pkl: `unqualifiedAccessExpr` / `qualifiedAccessExpr` are the same node whether
// they are a call (`helper(a)`) or a bare property read (`host`) — the only
// discriminator is an `argumentList` child, so both are gated on it. For a
// qualified call the method name is the `identifier` child that is not the
// `receiver`; the receiver is prefixed only when it is itself a plain name
// (`utils.fallback(a)` -> "utils.fallback", module-qualified, which cbm.c
// shortens to the last dotted segment when resolving). A receiver that is itself
// a call must NOT be prefixed: `s.trim().toLowerCase()` -> "toLowerCase", since
// the receiver's text carries parens and would never resolve.
// `newExpr` resolves to its `declaredType` so `new Server {}` links to the class.
static char *extract_pkl_callee(CBMArena *a, TSNode node, const char *source, const char *nk) {
if (strcmp(nk, "newExpr") == 0) {
// `new { ... }` with an inferred type has no declaredType child.
TSNode dt = cbm_find_child_by_kind(node, "declaredType");
return ts_node_is_null(dt) ? NULL : cbm_node_text(a, dt, source);
}
bool qualified = strcmp(nk, "qualifiedAccessExpr") == 0;
if (!qualified && strcmp(nk, "unqualifiedAccessExpr") != 0) {
return NULL;
}
// No argument list -> property read, not a call.
if (ts_node_is_null(cbm_find_child_by_kind(node, "argumentList"))) {
return NULL;
}
TSNode recv = ts_node_child_by_field_name(node, TS_FIELD("receiver"));
TSNode name = (TSNode){0};
uint32_t nc = ts_node_named_child_count(node);
for (uint32_t i = 0; i < nc; i++) {
TSNode child = ts_node_named_child(node, i);
if (!ts_node_is_null(recv) && ts_node_eq(child, recv)) {
continue;
}
if (strcmp(ts_node_type(child), "identifier") == 0) {
name = child;
break;
}
}
if (ts_node_is_null(name)) {
return NULL;
}
char *mn = cbm_node_text(a, name, source);
if (!mn || !mn[0]) {
return NULL;
}
if (!qualified || ts_node_is_null(recv)) {
return mn;
}
if (strcmp(ts_node_type(recv), "unqualifiedAccessExpr") == 0 &&
ts_node_is_null(cbm_find_child_by_kind(recv, "argumentList"))) {
char *rt = cbm_node_text(a, recv, source);
if (rt && rt[0]) {
return cbm_arena_sprintf(a, "%s.%s", rt, mn);
}
}
return mn;
}
// Typst: a `call` node's callee is its `item` field (an ident), matching the
// def-side resolution of `#let greet(name) = ...`.
static char *extract_typst_callee(CBMArena *a, TSNode node, const char *source, const char *nk) {
@@ -1789,6 +1848,15 @@ static char *extract_callee_name(CBMArena *a, TSNode node, const char *source, C
}
}
/* Pkl: resolve here and return unconditionally — the access-expr call node
* types double as plain property reads, so falling through to field-based or
* generic first-identifier resolution would mint a CALLS edge for every
* property read (a bare `host` has an `identifier` first child, which the
* generic fallback would happily emit). NULL here means "not a call". */
if (lang == CBM_LANG_PKL) {
return extract_pkl_callee(a, node, source, ts_node_type(node));
}
// Helm / Go templates: resolve `include "x"` / `template "x"` to the
// referenced named template so it links to the define'd Function (#338).
if (lang == CBM_LANG_GOTEMPLATE) {
+28
View File
@@ -378,6 +378,7 @@ typedef enum {
CBM_OCCURRENCE_VHDL_INTERFACE,
CBM_OCCURRENCE_PINE_FUNCTION,
CBM_OCCURRENCE_LLVM_FUNCTION,
CBM_OCCURRENCE_PKL_DECLARATION,
} CBMOccurrencePolicy;
typedef struct {
@@ -497,6 +498,7 @@ static const CBMOccurrenceSpec occurrence_specs[CBM_LANG_COUNT] = {
[CBM_LANG_PINE] = {NULL, NULL, CBM_OCCURRENCE_PINE_FUNCTION, false},
[CBM_LANG_PUPPET] = {NULL, NULL, CBM_OCCURRENCE_STANDARD, true},
[CBM_LANG_LLVM_IR] = {llvm_binding_nodes, NULL, CBM_OCCURRENCE_LLVM_FUNCTION, false},
[CBM_LANG_PKL] = {NULL, NULL, CBM_OCCURRENCE_PKL_DECLARATION, false},
[CBM_LANG_MESON] = {NULL, meson_write_nodes, CBM_OCCURRENCE_STANDARD, true},
[CBM_LANG_GN] = {NULL, gn_write_nodes, CBM_OCCURRENCE_STANDARD, true},
[CBM_LANG_LINKERSCRIPT] = {NULL, linkerscript_write_nodes, CBM_OCCURRENCE_STANDARD, true},
@@ -1006,6 +1008,30 @@ static bool is_pine_function_binding(TSNode node) {
return ancestor_field_binds(node, "function_declaration_statement", fields);
}
/* Pkl declares names positionally: no production labels the declared identifier
* with a `name` field, so the generic declared-container rule never binds them
* and every method name, parameter name, and property name would be re-emitted
* as an ordinary read of itself. Each container below holds its declared name
* as named child 0; annotations, defaults, and bodies follow it and stay reads.
* Resolve against the NEAREST container so a nested declaration's own name is
* the only occurrence its parent can bind. */
static bool is_pkl_declaration_binding(TSNode node) {
static const char *const declaration_kinds[] = {"methodHeader",
"typedIdentifier",
"classProperty",
"objectProperty",
"clazz",
"typeAlias",
NULL};
for (TSNode parent = ts_node_parent(node); !ts_node_is_null(parent);
parent = ts_node_parent(parent)) {
if (kind_in_exact_set(ts_node_type(parent), declaration_kinds)) {
return named_child_contains(parent, 0, node);
}
}
return false;
}
static bool is_policy_binding(CBMExtractCtx *ctx, TSNode node,
const CBMOccurrenceSpec *occurrence) {
switch (occurrence->policy) {
@@ -1062,6 +1088,8 @@ static bool is_policy_binding(CBMExtractCtx *ctx, TSNode node,
return is_vhdl_interface_binding(node);
case CBM_OCCURRENCE_PINE_FUNCTION:
return is_pine_function_binding(node);
case CBM_OCCURRENCE_PKL_DECLARATION:
return is_pkl_declaration_binding(node);
case CBM_OCCURRENCE_LLVM_FUNCTION:
for (TSNode parent = ts_node_parent(node); !ts_node_is_null(parent);
parent = ts_node_parent(parent)) {
+8 -23
View File
@@ -651,29 +651,14 @@ int cbm_count_branching(TSNode node, const char **branching_types) {
// Loop node-type names across tree-sitter grammars, for loop-nesting depth.
bool cbm_is_loop_node_type(const char *kind) {
static const char *const loops[] = {"for_statement",
"while_statement",
"do_statement",
"do_while_statement",
"for_in_statement",
"for_of_statement",
"for_each_statement",
"foreach_statement",
"enhanced_for_statement",
"for_range_loop",
"c_style_for_statement",
"for_expression",
"while_expression",
"loop_expression",
"while_let_expression",
"repeat_statement",
"repeat_while_statement",
"until",
"while_modifier",
"until_modifier",
"for",
"while",
NULL};
static const char *const loops[] = {
"for_statement", "while_statement", "do_statement", "do_while_statement",
"for_in_statement", "for_of_statement", "for_each_statement", "foreach_statement",
"enhanced_for_statement", "for_range_loop", "c_style_for_statement", "for_expression",
"while_expression", "loop_expression", "while_let_expression", "repeat_statement",
"repeat_while_statement",
// Pkl: `for (x in xs) { ... }` inside an object body.
"forGenerator", "until", "while_modifier", "until_modifier", "for", "while", NULL};
for (const char *const *l = loops; *l; l++) {
if (strcmp(kind, *l) == 0) {
return true;
+16 -6
View File
@@ -1594,11 +1594,21 @@ static const char *tlaplus_branch_types[] = {"if_then_else", "case", NULL};
static const char *tlaplus_var_types[] = {"variable_declaration", NULL};
static const char *tlaplus_module_types[] = {"source_file", NULL};
static const char *pkl_func_types[] = {"classMethod", "objectMethod", NULL};
static const char *pkl_class_types[] = {"clazz", NULL};
static const char *pkl_import_types[] = {"importClause", "extendsOrAmendsClause", "extends",
"import", NULL};
static const char *pkl_class_types[] = {"clazz", "typeAlias", NULL};
static const char *pkl_import_types[] = {
"importClause", "importGlobClause", "importExpr", "extendsOrAmendsClause",
"extends", "import", NULL};
static const char *pkl_var_types[] = {"classProperty", "objectProperty", NULL};
static const char *pkl_module_types[] = {"module", NULL};
/* Both access exprs double as plain property reads; extract_pkl_callee keeps
* only the ones carrying an argumentList. `newExpr` resolves to its type. */
static const char *pkl_call_types[] = {"unqualifiedAccessExpr", "qualifiedAccessExpr", "newExpr",
NULL};
/* Control-flow only, matching every other spec (short-circuit operators are
* deliberately excluded). `forGenerator` is also a loop — see helpers.c. */
static const char *pkl_branch_types[] = {"ifExpr", "whenGenerator", "forGenerator", NULL};
static const char *pkl_throw_types[] = {"throwExpr", NULL};
static const char *pkl_decorator_types[] = {"annotation", NULL};
static const char *gomod_var_types[] = {"require_directive", "replace_directive", NULL};
static const char *gomod_import_types[] = {"require", NULL};
static const char *gomod_module_types[] = {"source_file", NULL};
@@ -2630,9 +2640,9 @@ static const CBMLangSpec lang_specs[CBM_LANG_COUNT] = {
// CBM_LANG_PKL
[CBM_LANG_PKL] = {CBM_LANG_PKL, pkl_func_types, pkl_class_types, empty_types, pkl_module_types,
empty_types, pkl_import_types, empty_types, empty_types, pkl_var_types,
empty_types, empty_types, NULL, empty_types, NULL, NULL, tree_sitter_pkl,
NULL},
pkl_call_types, pkl_import_types, empty_types, pkl_branch_types,
pkl_var_types, empty_types, pkl_throw_types, NULL, pkl_decorator_types, NULL,
NULL, tree_sitter_pkl, NULL},
// CBM_LANG_GOMOD
[CBM_LANG_GOMOD] = {CBM_LANG_GOMOD, empty_types, empty_types, empty_types, gomod_module_types,
+6 -1
View File
@@ -2818,7 +2818,10 @@ static const CBMRegisteredFunc *c_lookup_member_depth(CLSPContext *ctx, const ch
const char *shortn = dot ? dot + 1 : type_qn;
size_t slen = strlen(shortn);
const char *best_qn = NULL;
for (int i = 0; i < ctx->registry->type_count; i++) {
CBMTypeNameIter it;
cbm_registry_types_by_short_name(ctx->registry, shortn, &it);
int i;
while ((i = cbm_type_name_iter_next(&it)) >= 0) {
const char *q = ctx->registry->types[i].qualified_name;
if (!q) {
continue;
@@ -5972,6 +5975,7 @@ CBMTypeRegistry *cbm_c_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, in
c_register_lsp_defs(arena, reg, "", d, 1);
}
cbm_registry_finalize(reg);
cbm_registry_build_type_short_index(reg);
reg->read_only = true; /* seal: shared Tier-2 registry is read-only during resolve */
return reg;
}
@@ -6055,6 +6059,7 @@ void cbm_run_c_lsp_cross(CBMArena *arena, const char *source, int source_len, co
// Finalize registry — O(1) lookups. See go_lsp.c "3c. Finalize"
// comment for the rationale (linear-scan fallback otherwise).
cbm_registry_finalize(&reg);
cbm_registry_build_type_short_index(&reg);
// Initialize context and run
CLSPContext ctx;
+85 -40
View File
@@ -866,11 +866,53 @@ const CBMType* go_eval_builtin_call(GoLSPContext* ctx, const char* name, TSNode
// --- go_lookup_field: struct field lookup with embedding recursion ---
// --- Import-alias re-qualification ------------------------------------
//
// parse_field_defs_into_type qualifies struct field type texts as
// "<def_module>.<text>". When the author wrote the text through an import
// alias ("Svc:svc.Svc" in module test.main), that yields a QN
// ("test.main.svc.Svc") that exists nowhere in the project-wide registry —
// the real QN is "<import_qn>.Svc" and only the calling file's import map
// can say so. On an exact-QN miss this rewrites the alias segment through
// the file's imports; returns NULL when no alias segment is involved.
static const char *go_requalify_via_imports(GoLSPContext *ctx, const char *type_qn) {
if (!ctx || !type_qn || !type_qn[0] || ctx->import_count <= 0) return NULL;
for (int j = 0; j < ctx->import_count; j++) {
const char *alias = ctx->import_local_names[j];
const char *alias_qn = ctx->import_package_qns[j];
if (!alias || !alias[0] || !alias_qn || strchr(alias, '.')) continue;
size_t alias_len = strlen(alias);
/* Last occurrence of a "<dot>alias<dot>" segment in type_qn. */
const char *hit = NULL;
for (const char *p = type_qn;;) {
p = strstr(p, ".");
if (!p) break;
p++;
if (strncmp(p, alias, alias_len) == 0 && p[alias_len] == '.') {
hit = p;
p += alias_len;
}
}
if (hit) {
const char *rest = hit + alias_len + 1; /* past "<alias>." */
return cbm_arena_sprintf(ctx->arena, "%s.%s", alias_qn, rest);
}
}
return NULL;
}
static const CBMType* go_lookup_field(GoLSPContext* ctx,
const char* type_qn, const char* field_name, int depth) {
if (!type_qn || !field_name || depth > 5) return NULL;
const CBMRegisteredType* rt = cbm_registry_lookup_type(ctx->registry, type_qn);
if (!rt && depth == 0) {
/* Import-alias re-qualification: field texts from cross-package defs
* may embed an alias segment only this file's import map resolves. */
const char* alt_qn = go_requalify_via_imports(ctx, type_qn);
if (alt_qn) rt = cbm_registry_lookup_type(ctx->registry, alt_qn);
}
if (!rt) return NULL;
// Follow alias chain
@@ -907,6 +949,18 @@ static const CBMRegisteredFunc* go_lookup_field_or_method_depth(GoLSPContext* ct
const CBMRegisteredFunc* f = cbm_registry_lookup_method(ctx->registry, type_qn, member_name);
if (f) return f;
/* Import-alias re-qualification fallback: NAMED receivers built from
* cross-package field type texts can carry a "<module>.svc.Svc" QN;
* retry the method set on the import-resolved QN (see
* go_requalify_via_imports). */
if (depth == 0) {
const char* alt_qn = go_requalify_via_imports(ctx, type_qn);
if (alt_qn) {
f = go_lookup_field_or_method_depth(ctx, alt_qn, member_name, depth + 1);
if (f) return f;
}
}
const CBMRegisteredType* rt = cbm_registry_lookup_type(ctx->registry, type_qn);
if (rt) {
// Follow type alias chain
@@ -1430,20 +1484,32 @@ static void resolve_calls_in_node_inner(GoLSPContext* ctx, TSNode node) {
if (base && base->kind == CBM_TYPE_POINTER) base = cbm_type_deref(base);
if (base && base->kind == CBM_TYPE_NAMED) {
const char *recv_qn = base->data.named.qualified_name;
const CBMRegisteredType *receiver_type = cbm_registry_lookup_type(
ctx->registry, base->data.named.qualified_name);
ctx->registry, recv_qn);
const char *alt_qn = NULL;
/* Re-qualify NAMED receivers that embed an import
* alias segment (cross-package field type texts) —
* the real type only exists under the import QN. */
if (!receiver_type) {
alt_qn = go_requalify_via_imports(ctx, recv_qn);
if (alt_qn) {
receiver_type = cbm_registry_lookup_type(ctx->registry, alt_qn);
if (receiver_type) recv_qn = alt_qn;
}
}
/* Registered interface receivers must reach the
* interface-resolution branch below. Their semantic
* method registrations are signatures, not concrete
* dispatch targets. */
if (!receiver_type || !receiver_type->is_interface) {
const CBMRegisteredFunc *method = go_lookup_field_or_method(
ctx, base->data.named.qualified_name, field_name);
ctx, recv_qn, field_name);
if (method) {
const char *strategy = "lsp_type_dispatch";
if (method->receiver_type &&
strcmp(method->receiver_type,
base->data.named.qualified_name) != 0) {
recv_qn) != 0) {
strategy = "lsp_embed_dispatch";
}
emit_resolved_call(ctx, method->qualified_name, strategy, 0.95f,
@@ -1460,9 +1526,15 @@ static void resolve_calls_in_node_inner(GoLSPContext* ctx, TSNode node) {
if (!is_iface && base->kind == CBM_TYPE_NAMED) {
const CBMRegisteredType* rt = cbm_registry_lookup_type(ctx->registry,
base->data.named.qualified_name);
if (!rt) {
const char* alt_qn = go_requalify_via_imports(
ctx, base->data.named.qualified_name);
if (alt_qn)
rt = cbm_registry_lookup_type(ctx->registry, alt_qn);
}
if (rt && rt->is_interface) {
is_iface = true;
iface_qn = base->data.named.qualified_name;
iface_qn = rt->qualified_name;
}
}
if (is_iface) {
@@ -1787,42 +1859,15 @@ static void process_function(GoLSPContext* ctx, TSNode func_node) {
char* func_name = lsp_node_text(ctx, name_node);
if (!func_name || !func_name[0]) return;
// For methods, the enclosing-function QN must include the receiver type
// (package.Type.Method), matching how the textual extractor and the
// registry qualify the method. Building it as package.Method (no receiver)
// here made the LSP-resolved call's caller_qn disagree with the textual
// call's enclosing_func_qn, so cbm_pipeline_find_lsp_resolution never
// joined them — every call inside a method body silently lost its
// type-aware LSP strategy. Derive the bare receiver type name the same way
// the receiver binding below does.
char* recv_type_name = NULL;
{
TSNode recv0 = ts_node_child_by_field_name(func_node, "receiver", 8);
if (!ts_node_is_null(recv0)) {
uint32_t rnc0 = ts_node_child_count(recv0);
for (uint32_t i = 0; i < rnc0 && !recv_type_name; i++) {
TSNode rp = ts_node_child(recv0, i);
if (ts_node_is_null(rp) || !ts_node_is_named(rp)) continue;
if (strcmp(ts_node_type(rp), "parameter_declaration") != 0) continue;
TSNode rtype = ts_node_child_by_field_name(rp, "type", 4);
if (ts_node_is_null(rtype)) continue;
// Unwrap a pointer receiver (*Type) to the bare type identifier.
const char* rtk = ts_node_type(rtype);
if (strcmp(rtk, "pointer_type") == 0 && ts_node_named_child_count(rtype) > 0) {
rtype = ts_node_named_child(rtype, 0);
}
char* tn = lsp_node_text(ctx, rtype);
if (tn && tn[0]) recv_type_name = tn;
}
}
}
if (recv_type_name) {
ctx->enclosing_func_qn =
cbm_arena_sprintf(ctx->arena, "%s.%s.%s", ctx->package_qn, recv_type_name, func_name);
} else {
ctx->enclosing_func_qn = cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->package_qn, func_name);
}
// Enclosing-function QN must be the BARE package.Func form (no receiver
// type segment). The textual call events (extract_unified.c) source calls
// as package_qn.func_name — methods included — and the defs pass creates
// the graph Method node under the same QN, so any other form breaks the
// caller-QN join in cbm_pipeline_find_lsp_resolution and the LSP-resolved
// call silently falls back to the registry short-name resolver. The
// receiver type still reaches the registry via the def's parent_class /
// method->receiver_type; it just does not appear in the caller QN.
ctx->enclosing_func_qn = cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->package_qn, func_name);
// Push function scope
CBMScope* saved_scope = ctx->current_scope;
+87
View File
@@ -167,6 +167,56 @@ static void build_type_embed_index(CBMTypeRegistry *reg, CBMArena *idx_arena) {
reg->type_embed_entry_count = idx;
}
/* Optional index: final qualified-name segment -> chain of TYPE indices.
* Descending iteration plus prepend preserves ascending registry scan order.
* Consumers opt in after the QN index captures the finalized tail boundary. */
void cbm_registry_build_type_short_index(CBMTypeRegistry *reg) {
if (!reg)
return;
reg->type_short_buckets = NULL;
reg->type_short_entries = NULL;
reg->type_short_bucket_count = 0;
if (!reg->arena || !reg->type_qn_buckets || reg->type_qn_bucket_count <= 0)
return;
CBMArena *idx_arena = reg->arena;
int count = 0;
for (int i = 0; i < reg->type_count; i++) {
if (reg->types[i].qualified_name)
count++;
}
if (count == 0)
return;
int bucket_count = next_pow2(count * 2);
if (bucket_count < 16)
bucket_count = 16;
int *buckets = (int *)cbm_arena_alloc(idx_arena, (size_t)bucket_count * sizeof(int));
CBMRegistryHashEntry *entries = (CBMRegistryHashEntry *)cbm_arena_alloc(
idx_arena, (size_t)count * sizeof(CBMRegistryHashEntry));
if (!buckets || !entries)
return;
for (int i = 0; i < bucket_count; i++)
buckets[i] = -1;
int idx = 0;
for (int i = reg->type_count - 1; i >= 0; i--) {
const char *qn = reg->types[i].qualified_name;
if (!qn)
continue;
const char *dot = strrchr(qn, '.');
const char *short_name = dot ? dot + 1 : qn;
uint64_t h = fnv1a(short_name);
int slot = (int)(h & (uint64_t)(bucket_count - 1));
entries[idx].hash = h;
entries[idx].payload_index = i;
entries[idx].next_index = buckets[slot];
entries[idx].slot = slot;
buckets[slot] = idx;
idx++;
}
reg->type_short_buckets = buckets;
reg->type_short_entries = entries;
reg->type_short_bucket_count = bucket_count;
}
/* Index: short_name -> chain of FREE-function (receiver_type==NULL) indices.
* Descending-iterate + prepend for ascending chain order (as above). */
static void build_type_short_index(CBMTypeRegistry *reg, CBMArena *idx_arena) {
@@ -248,6 +298,43 @@ static void build_ffunc_short_index(CBMTypeRegistry *reg, CBMArena *idx_arena) {
reg->ffunc_short_entry_count = idx;
}
void cbm_registry_types_by_short_name(const CBMTypeRegistry *reg, const char *short_name,
CBMTypeNameIter *out) {
out->reg = reg;
out->hash = fnv1a(short_name);
if (reg->type_qn_buckets && reg->type_qn_bucket_count > 0) {
if (reg->type_short_buckets && reg->type_short_bucket_count > 0) {
int slot = (int)(out->hash & (uint64_t)(reg->type_short_bucket_count - 1));
out->chain_idx = reg->type_short_buckets[slot];
out->tail_i = reg->type_qn_entry_count;
} else {
out->chain_idx = -1;
out->tail_i = 0; /* auxiliary-index allocation failed: preserve full-scan semantics */
}
out->tail_end = reg->type_count;
} else {
out->chain_idx = -1;
out->tail_i = 0;
out->tail_end = reg->type_count;
}
}
int cbm_type_name_iter_next(CBMTypeNameIter *it) {
const CBMTypeRegistry *reg = it->reg;
while (it->chain_idx >= 0) {
const CBMRegistryHashEntry *e = &reg->type_short_entries[it->chain_idx];
int p = e->payload_index;
uint64_t h = e->hash;
it->chain_idx = e->next_index;
if (h != it->hash)
continue;
return p;
}
if (it->tail_i < it->tail_end)
return it->tail_i++;
return -1;
}
void cbm_registry_types_by_embedded_bare(const CBMTypeRegistry *reg, const char *bare,
CBMTypeEmbedIter *out) {
out->reg = reg;
+30 -4
View File
@@ -111,9 +111,14 @@ typedef struct CBMTypeRegistry {
int method_bucket_count;
int method_entry_count;
// Auxiliary short-name / embedded-type indexes (built by finalize alongside the
// QN buckets). Turn the Rust trait- and free-function fallback scans from
// O(type_count)/O(func_count) into O(chain). Read-only after finalize.
// Auxiliary short-name / embedded-type indexes. The type short-name index is
// opt-in after finalize; the others are built by finalize. Current C/C++ cross
// registries opt in, while other languages incur no construction cost.
// Type short-name index: fnv1a(last-'.'-segment of qualified_name) -> chain
// of TYPE indices. payload_index = type index.
int *type_short_buckets;
CBMRegistryHashEntry *type_short_entries;
int type_short_bucket_count;
// Embedded-type index: fnv1a(bare last-'.'-segment of each embedded_type) -> chain
// of TYPE indices declaring it. payload_index = type index (a type may appear once
// per embedded entry; consumers dedup adjacent same-type via the iterator).
@@ -160,6 +165,11 @@ void cbm_registry_finalize(CBMTypeRegistry *reg);
// there add GBs across a large repo (FastAPI incremental test: +1.1 GB RSS).
void cbm_registry_finalize_into(CBMTypeRegistry *reg, CBMArena *idx_arena);
// Build the optional final-QN-segment type index. Call immediately after
// finalization. Consumers that do not opt in avoid its type_count-sized allocation
// and scan. Allocation failure preserves the iterator's linear fallback.
void cbm_registry_build_type_short_index(CBMTypeRegistry *reg);
// Register a function/method.
void cbm_registry_add_func(CBMTypeRegistry *reg, CBMRegisteredFunc func);
@@ -219,8 +229,24 @@ const CBMRegisteredFunc *cbm_registry_lookup_symbol_by_types(const CBMTypeRegist
const CBMType **arg_types,
int arg_count);
// --- Auxiliary index iterators (Rust trait / free-function fallback fast paths) ---
// --- Auxiliary index iterators (language fallback fast paths) ---
//
// Iterate registry TYPE indices whose qualified-name final segment may equal
// `short_name`. When the optional index is present this walks its hash chain plus
// any post-finalize tail; otherwise it degrades to the original full scan. Results
// preserve ascending registry order. The index is a hash prefilter; callers must
// re-check their exact predicate, including tail entries.
typedef struct {
const CBMTypeRegistry *reg;
uint64_t hash;
int chain_idx;
int tail_i;
int tail_end;
} CBMTypeNameIter;
void cbm_registry_types_by_short_name(const CBMTypeRegistry *reg, const char *short_name,
CBMTypeNameIter *out);
int cbm_type_name_iter_next(CBMTypeNameIter *it);
// Iterate registry TYPE indices whose embedded_types contain an entry whose BARE
// name (last '.'-segment) equals `bare`. On a finalized registry this walks the
// embedded-type index plus any post-finalize tail; on an unfinalized registry it
+5 -1
View File
@@ -678,7 +678,11 @@ static bool callee_is_delimiter_or_filesystem_builder(const char *callee_name) {
method = last_colon + 2;
}
if (strcmp(method, "split") == 0 || strcmp(method, "rsplit") == 0 ||
strcmp(method, "partition") == 0 || strcmp(method, "join") == 0) {
strcmp(method, "partition") == 0 || strcmp(method, "join") == 0 ||
strcmp(method, "replace") == 0 || strcmp(method, "replaceAll") == 0 ||
strcmp(method, "match") == 0 || strcmp(method, "matchAll") == 0 ||
strcmp(method, "search") == 0 || strcmp(method, "test") == 0 ||
strcmp(method, "exec") == 0) {
return true;
}
return strstr(callee_name, "os.path.join") != NULL || strstr(callee_name, "path.join") != NULL;
+4
View File
@@ -44,6 +44,10 @@ cbm_svc_kind_t cbm_service_pattern_match(const char *resolved_qn);
* classified via its real resolved QN instead and never reaches this check. */
bool cbm_service_pattern_is_global_fetch(const char *callee_name);
/* True when a string literal is a plausible HTTP route for the given callee.
* Rejects filesystem paths and non-HTTP string consumers. */
bool cbm_service_pattern_is_http_route_literal(const char *literal, const char *callee_name);
/* Per-worker TLS cache for cbm_service_pattern_match results. The
* pattern matcher runs once per resolved CALL edge in emit_service_
* edge — that's 6 pattern lists × ~30 patterns × strstr per call ≈
+16 -2
View File
@@ -10,11 +10,25 @@ The grammars were originally vendored as bare `parser.c`+`scanner.c` with **no r
## Summary
- Grammars: **162** — vendored-from-upstream: **143**, first-party/self-maintained: **14**, registry-disagreement: **5** (nim removed 2026-06-12; objectscript_udl + objectscript_routine added 2026-06-24; mojo added 2026-07-01; arkts added 2026-08-26; plsql added 2026-08-27; chialisp added 2026-08-28 — see notes below)
- ABI distribution: **9×** ABI-13 **80×** ABI-14 **73×** ABI-15 (runtime ceiling is ABI 15; never vendor ABI 16 without a runtime upgrade)
- ABI distribution: **9×** ABI-13 **79×** ABI-14 **74×** ABI-15 (runtime ceiling is ABI 15; never vendor ABI 16 without a runtime upgrade)
— recounted from the tree 2026-08-30 after the perl v1.2.1 refresh moved `perl` from ABI 14 to ABI 15 (was `9×/80×/73×`). Neither side of the rebase had this right: main's line was correct for main, and this branch still carried the pre-2026-08-28 `7×/84×/65×`. Regenerate, never increment.
— recounted from the tree 2026-08-28. This line had drifted: it read `7×/86×/65×`, which sums to 158 against 161 vendored grammars, so it was wrong before Chialisp was added and incrementing it would have carried the error forward. Regenerate with:
`grep -h '#define LANGUAGE_VERSION' internal/cbm/vendored/grammars/*/parser.c | sort | uniq -c`
- Vendored copies missing LICENSE: **0** — all upstream LICENSE files restored 2026-06-11 (first-party grammars carry the project MIT license; `move` uses the Helix-listed upstream tzakian/tree-sitter-move MIT text, `zsh` uses georgeharker/tree-sitter-zsh MIT)
- `verdict`: VERIFIED-BOTH = our source matches *both* registries; VERIFIED-NVIM/HELIX = matches one; registry-disagreement = registries name a different repo (listed separately); `vendor-maintained` = the language vendor's own grammar, not in nvim/Helix; `community-niche` = an individual/community-maintained grammar for a niche language, not in nvim/Helix — provenance hand-verified against the upstream repository instead of a registry.
- **perl** (refreshed 2026-08-27): byte-for-byte generated bundle from
[tree-sitter-perl/tree-sitter-perl](https://github.com/tree-sitter-perl/tree-sitter-perl)
tag `v1.2.1`, full commit `c3e17b31179bf8f658c9f37c7a3ea6a202212d5a` (ABI 15).
The upstream GitHub source asset SHA-256 is
`95c5fa0966dd431eb2f96b941c37b413ae7e9083729433f4d3d41fbc2a4f14a6`; the
screened ordered source-manifest SHA-256 is
`4e7bf02e8bd14b410309ce12bab80edb730826db79e1139434c1459a3117b900`.
The existing MIT `LICENSE` was preserved byte-for-byte at SHA-256
`68a9a526ae357ed5a2f8ca5dabb14131b283554b69639bb04816088d4b1f2fa0`.
Exact per-file SHA-256 values are pinned in `scripts/vendored-checksums.txt`.
The table uses `UPSTREAM-RELEASE` because this refresh verified the tagged
upstream release itself but did not independently verify a current registry
pin; it intentionally does not claim `VERIFIED-NVIM` or `VERIFIED-BOTH`.
- **objectscript_udl / objectscript_routine** (added 2026-06-24): vendored from [intersystems/tree-sitter-objectscript](https://github.com/intersystems/tree-sitter-objectscript) @ `a7ffcdf` — MIT, the InterSystems-official grammars (a niche vendor language, hence `vendor-maintained`, not in nvim-treesitter/Helix). **Re-vendor note:** each `scanner.c`'s upstream `#include "../../common/scanner.h"` is repointed to a per-directory `objectscript_common.h` (a verbatim copy of upstream `common/scanner.h`), because this repo's shared `vendored/common/scanner.h` belongs to the cfml/fsharp grammars and differs. The generated `parser.c`/`scanner.c` are otherwise byte-for-byte upstream — on re-vendor, re-apply only that single include rename. **Local modification (2026-07-16):** in `objectscript_common.h`, two loop counters `uint8_t i` were widened to `int i` (the `reverse_marker` scan and the `html_marker_buffer` reversal) to clear CodeQL `cpp/comparison-with-wider-type` — a false positive in practice (both lengths are hard-bounded by `MARKER_BUFFER_MAX_LEN = 30`, so `uint8_t` could never wrap), fixed for cleanliness. On re-vendor, re-apply this widening too (or upstream it at intersystems/tree-sitter-objectscript).
- **mojo** (added 2026-07-01): vendored from [lsh/tree-sitter-mojo](https://github.com/lsh/tree-sitter-mojo) @ `33193a99afe6` — MIT, ABI 15. Helix tracks `lsh/tree-sitter-mojo` as its Mojo grammar source, but the Helix-pinned commit (`3d7c53b8038f`) no longer resolves in the upstream repository after a force-push, so this vendor uses current upstream `main` rather than the stale registry SHA. Security review covered only the vendored C surface (`parser.c`, `scanner.c`, `tree_sitter/*.h`) plus upstream license/provenance metadata; no package manager hooks, workflow files, prompt/agent instruction files, or generated lockfiles were vendored.
- **arkts** (added 2026-08-26): **first-party derivative** — a fork of [tree-sitter/tree-sitter-typescript](https://github.com/tree-sitter/tree-sitter-typescript)'s `typescript` dialect pinned @ `75b3874edb2d` (v0.23.2, the same commit our vendored typescript/tsx come from), on the tree-sitter-javascript base @ `3a837b6f3658` (v0.23.1, the version upstream's own package-lock pins), extended with ArkTS/ArkUI syntax (`@Component struct` declarations, UI-DSL trailing-closure calls with post-block attribute chains, decorated function declarations, `import lazy`, `@Extend`/`@Styles` leading-dot attribute chains, anonymous `stateStyles` style blocks). Grammar source + corpus tests live in `tools/tree-sitter-arkts/`; regenerate with `npx tree-sitter-cli@0.25.10 generate` (ABI 15). **External scanner:** `scanner.c` is a symbol-rename trampoline; `_common_scanner.h` is byte-identical (same SHA-256) to the reviewed `typescript/_common_scanner.h` already shipped. **LICENSE:** tree-sitter-typescript's MIT text **verbatim and byte-identical** (© 2017 Max Brunsfeld) — which is what MIT requires of a derivative work, and what lets the provenance audit byte-verify it against upstream rather than take a note on trust. The fork is registered in the audit's `FORKS` map. Our own copyright for the ArkTS additions, and the tree-sitter-javascript attribution (© 2014 Max Brunsfeld), live with the SOURCE in `tools/tree-sitter-arkts/grammar.js` and in `THIRD_PARTY.md`, which is what ships in the release archives.
@@ -163,7 +177,7 @@ Re-vendoring from upstream must re-apply these.
| ocaml | 14 | tree-sitter/tree-sitter-ocaml | `5a979b3ec7f1` | VERIFIED-BOTH | ✅ |
| odin | 14 | tree-sitter-grammars/tree-sitter-odin | `d2ca8efb4487` | VERIFIED-BOTH | ✅ |
| pascal | 14 | Isopod/tree-sitter-pascal | `042119eca2e1` | VERIFIED-BOTH | ✅ |
| perl | 14 | tree-sitter-perl/tree-sitter-perl | `ea9667dc65a8` | VERIFIED-BOTH | ✅ |
| perl | 15 | tree-sitter-perl/tree-sitter-perl | `c3e17b31179b` | UPSTREAM-RELEASE | ✅ |
| php | 15 | tree-sitter/tree-sitter-php | `3f2465c217d0` | VERIFIED-BOTH | ✅ |
| pkl | 15 | apple/tree-sitter-pkl | `f5beed1da8e5` | VERIFIED-BOTH | ✅ |
| plsql | 14 | AndreasMaierDe/tree-sitter-plsql | `28aebef209be` | community-niche | ✅ |
@@ -69,7 +69,7 @@ is respectively less than, matching, or greater than the array member.
* have to make lim 3, then halve, obtaining 1, so that we will only
* look at item 3.
*/
void *bsearch(const void *key, const void *base0, size_t nmemb, size_t size,
static void *tsp_bsearch(const void *key, const void *base0, size_t nmemb, size_t size,
int (*compar)(const void *, const void *)) {
const char *base = (const char *)base0;
int lim, cmp;
+760516 -469989
View File
File diff suppressed because it is too large Load Diff
+687 -55
View File
@@ -1,6 +1,9 @@
#include "tree_sitter/array.h"
#include "tree_sitter/parser.h"
#include "tsp_unicode.h"
#include "tsp_keywords.h"
#include "tsp_intuit_more.h"
#include "tsp_intuit_readline.h"
// grumble grumble no stdlib
static char *tsp_strchr(register const char *s, int c) {
@@ -49,6 +52,8 @@ enum TokenType {
TOKEN_ESCAPE_SEQUENCE,
TOKEN_ESCAPED_DELIMITER,
TOKEN_DOLLAR_IN_REGEXP,
TOKEN_REGEXP_OPEN_BRACKET,
TOKEN_REGEXP_OPEN_BRACE,
TOKEN_POD,
TOKEN_GOBBLED_CONTENT,
TOKEN_ATTRIBUTE_VALUE_BEGIN,
@@ -61,6 +66,7 @@ enum TokenType {
TOKEN_HEREDOC_MIDDLE,
TOKEN_HEREDOC_END,
TOKEN_FAT_COMMA_AUTOQUOTED,
TOKEN_FAT_COMMA_AUTOQUOTED_AHEAD,
TOKEN_FILETEST,
TOKEN_BRACE_AUTOQUOTED,
/* zero-width lookahead tokens */
@@ -69,6 +75,24 @@ enum TokenType {
TOKEN_NO_INTERP_WHITESPACE_ZW,
/* zero-width high priority token */
TOKEN_NONASSOC,
/* synthetic tokens for error recovery */
TOKEN_RECOVER_PAREN_CLOSE,
TOKEN_RECOVER_BRACKET_CLOSE,
TOKEN_RECOVER_BRACE_CLOSE,
TOKEN_RECOVER_ARROW,
TOKEN_RECOVER_BLOCK_CLOSE,
/* opaque body of a `format NAME = ... .` declaration: everything from the
* line after `=` up to and including the lone-`.` terminator line */
TOKEN_FORMAT_CONTENT,
/* `x` repetition operator glued to its count (`"ab"x3`) */
TOKEN_X_OP,
/* `class`/`role`/`method` emitted only in declaration position */
TOKEN_KW_CLASS,
TOKEN_KW_ROLE,
TOKEN_KW_METHOD,
/* `async`/`try` contextual keywords, emitted only in keyword position */
TOKEN_KW_ASYNC,
TOKEN_KW_TRY,
/* error condition is always last */
TOKEN_ERROR
};
@@ -115,18 +139,47 @@ static int32_t close_for_open(int32_t c) {
typedef struct {
int32_t open, close, count;
/* Recognises a pattern-LEADING '['/'{' -- the quote's own opening delimiter
* appearing (after optional whitespace) as the literal first element of the
* body (m{{...}}, qr{ {...} }, s{{...}}, m[[...]]) -- where the S_intuit_more
* subscript heuristic (tsp_intuit_more) has no preceding variable to weigh and
* so must not be consulted.
*
* Decided ONCE by a bounded lookahead at the opening delimiter (see the open
* handler), where the lexer sits exactly at the body start: if the first
* non-space body char is the delimiter again, the body leads with a literal
* group. Looking at the actual first char sidesteps both problems of the
* earlier column+flag scheme -- columns repeat across lines, and the scanner
* can't see grammar-lexed interpolation -- because we never have to ask "has
* content been seen?": we just check what the first char *is*.
*
* Set at the opener, consumed (cleared) by that first bracket. */
bool body_leads_with_delim;
} TSPQuote;
static TSPQuote tspquote_new() { return (TSPQuote){0, 0, 0}; }
static TSPQuote tspquote_new() { return (TSPQuote){0, 0, 0, false}; }
enum HeredocState { HEREDOC_NONE, HEREDOC_START, HEREDOC_UNKNOWN, HEREDOC_CONTINUE, HEREDOC_END };
/* Perl allows multiple heredocs queued on a single line, consumed FIFO after
* the newline. We hold up to HEREDOC_QUEUE_MAX pending heredocs; each one
* carries its own delimiter and interpolate/indent flags. The consume-FSM
* state (heredoc_state) applies to the FRONT of the queue (the one currently
* being consumed); when it finishes we dequeue and advance to the next. */
#define HEREDOC_QUEUE_MAX 8
typedef struct {
TSPString delim;
bool interpolates, indents;
} HeredocEntry;
typedef struct {
Array(TSPQuote) quotes;
/* heredoc - we need to track if we should start the heredoc, if it's
* interpolating, how many chars the delimiter is and what the delimiter is */
bool heredoc_interpolates, heredoc_indents;
/* FIFO queue of pending heredocs (front = index 0). heredoc_count is how
* many entries are live; heredoc_state is the consume state of the front. */
HeredocEntry heredoc_queue[HEREDOC_QUEUE_MAX];
uint8_t heredoc_count;
enum HeredocState heredoc_state;
TSPString heredoc_delim;
bool recovery_emitted;
} LexerState;
static void lexerstate_push_quote(LexerState *state, int32_t opener) {
@@ -192,6 +245,20 @@ static bool lexerstate_is_paired_delimiter(LexerState *state) {
return !!q->open;
}
/* Is `c` the innermost quote's pattern-leading bracket -- the delimiter-matching
* '['/'{' the open-handler lookahead flagged as the body's first literal element?
* One-shot: clears the flag so a later same-delimiter bracket is treated as
* ordinary (subscript/class) content. */
static bool lexerstate_take_body_lead(LexerState *state, int32_t c) {
if (!state->quotes.size) return false;
TSPQuote *q = array_back(&state->quotes);
if (q->body_leads_with_delim && c == q->open) {
q->body_leads_with_delim = false;
return true;
}
return false;
}
// in order to emulate a sublex, we basically need to have a new escape type character
// that can pop up anywhere so long as there's an active string, to escape the string
// char
@@ -210,16 +277,48 @@ static bool lexerstate_is_paired_delimiter(LexerState *state) {
// we can match perl's behavior if we are intentionally destructive here and find our match
static void lexerstate_add_heredoc(LexerState *state, TSPString *delim, bool interp, bool indent) {
state->heredoc_delim = *delim;
state->heredoc_interpolates = interp;
state->heredoc_indents = indent;
state->heredoc_state = HEREDOC_START;
/* The front of the queue is the heredoc currently being consumed. */
static HeredocEntry *lexerstate_front_heredoc(LexerState *state) {
return &state->heredoc_queue[0];
}
/* ENQUEUE a pending heredoc to the back of the FIFO. When this is the first
* pending heredoc, prime the consume FSM at HEREDOC_START.
*
* Overflow handling: if the queue is already full we OVERWRITE THE LAST SLOT
* with the new entry rather than dropping it. The last slot then always
* tracks the FINAL heredoc's terminator, so the scanner stays in heredoc-mode
* and greedily consumes all overflow content up to that final terminator —
* absorbing the excess into the last body instead of leaking it into the code
* stream. For >MAX heredocs on a line this means the first MAX-1 bodies parse
* correctly, the last body greedily swallows the remaining overflow (wrong but
* BOUNDED), and code after the heredoc block is unaffected (no desync). This
* is graceful degradation for a pathological input. */
static void lexerstate_add_heredoc(LexerState *state, TSPString *delim, bool interp, bool indent) {
HeredocEntry *e;
if (state->heredoc_count >= HEREDOC_QUEUE_MAX)
e = &state->heredoc_queue[HEREDOC_QUEUE_MAX - 1];
else
e = &state->heredoc_queue[state->heredoc_count++];
e->delim = *delim;
e->interpolates = interp;
e->indents = indent;
if (state->heredoc_count == 1) state->heredoc_state = HEREDOC_START;
}
/* DEQUEUE the finished front heredoc and advance to the next pending one.
* Each heredoc body is wrapped in its own `heredoc_content` grammar node which
* begins with a fresh `_heredoc_start` zero-width token; that token only fires
* in HEREDOC_START. So when another heredoc remains queued we re-arm the FSM
* to HEREDOC_START, exactly as if its delimiter had just been declared. */
static void lexerstate_finish_heredoc(LexerState *state) {
state->heredoc_delim.length = 0;
state->heredoc_state = HEREDOC_NONE;
if (state->heredoc_count > 0) {
state->heredoc_count--;
for (int i = 0; i < state->heredoc_count; i++)
state->heredoc_queue[i] = state->heredoc_queue[i + 1];
}
state->heredoc_queue[state->heredoc_count] = (HeredocEntry){0};
state->heredoc_state = state->heredoc_count > 0 ? HEREDOC_START : HEREDOC_NONE;
}
#define ADVANCE_C \
@@ -247,6 +346,16 @@ static void lexerstate_finish_heredoc(LexerState *state) {
DEBUG("marking end of token\n", 0); \
} while (0)
// Try each recovery token in priority order. Uses a macro because
// TOKEN() contains `return true`. brace_ok: skip brace recovery at '}'
// (that's the real closer); ';' is safe (only valid in subscript context).
#define EMIT_RECOVERY_TOKENS(brace_ok) do { \
if (valid_symbols[TOKEN_RECOVER_ARROW]) { state->recovery_emitted = true; TOKEN(TOKEN_RECOVER_ARROW); } \
if (valid_symbols[TOKEN_RECOVER_PAREN_CLOSE]) { state->recovery_emitted = true; TOKEN(TOKEN_RECOVER_PAREN_CLOSE); } \
if (valid_symbols[TOKEN_RECOVER_BRACKET_CLOSE]) { state->recovery_emitted = true; TOKEN(TOKEN_RECOVER_BRACKET_CLOSE); } \
if ((brace_ok) && valid_symbols[TOKEN_RECOVER_BRACE_CLOSE]) { state->recovery_emitted = true; TOKEN(TOKEN_RECOVER_BRACE_CLOSE); } \
} while(0)
static void skip_whitespace(TSLexer *lexer) {
while (1) {
int32_t c = lexer->lookahead;
@@ -258,19 +367,130 @@ static void skip_whitespace(TSLexer *lexer) {
}
}
static void skip_ws_to_eol(TSLexer *lexer) {
static bool skip_ws_to_eol(TSLexer *lexer) {
while (1) {
int32_t c = lexer->lookahead;
if (!c) return;
if (!c) return false;
if (is_tsp_whitespace(c)) {
lexer->advance(lexer, true);
// return after eating the newline
if (c == '\n') return;
if (c == '\n') return true;
} else
return;
return false;
}
}
// Forward declarations (defined after _skip_chars)
static bool isidfirst(int32_t c);
static bool isidcont(int32_t c);
enum PeekResult {
PEEK_NO_MATCH, // first char not a keyword starter — lexer untouched
PEEK_KEYWORD, // statement keyword (not followed by =>)
PEEK_FAT_COMMA, // keyword followed by => — caller MARK_ENDs then goto fat_comma_check
PEEK_AUTOQUOTE_KEY, // non-keyword word followed by => — peek already marked the
// word boundary; caller goes straight to fat_comma_check
PEEK_NOT_KEYWORD, // word read but not a keyword — caller MUST return false
};
// How a word read by the keyword peek classifies (set by KEYWORD_MATCH).
enum KwKind {
KW_NONE, // not a statement keyword
KW_ALWAYS, // always a statement keyword (package/use/no/class/role)
KW_NEEDS_NAME, // declaration only if followed by a name (sub/method)
};
static enum PeekResult peek_is_statement_keyword(TSLexer *lexer, bool *is_structural) {
int32_t la = lexer->lookahead;
*is_structural = false;
if (KEYWORD_FIRST_CHAR_FILTER(la))
return PEEK_NO_MATCH;
// Read the word (character set derived from keyword list)
char word[16];
int len = 0;
while (KEYWORD_WORD_CHAR(la)) {
if (len < 15) word[len++] = (char)la;
lexer->advance(lexer, false);
la = lexer->lookahead;
}
word[len] = '\0';
// Classify *before* consuming any identifier tail: a statement keyword must
// end exactly at the keyword charset boundary. A longer identifier (e.g.
// "service", "naming") read only partway by the loop above is never a
// keyword — but it can still be a fat-comma autoquote key, so we don't bail.
enum KwKind kind = KW_NONE;
if (!isidcont(la))
KEYWORD_MATCH(word, kind);
if (kind == KW_NONE) {
// Not a statement keyword. It may still be a fat-comma autoquote key
// (`name => 1` on a continuation line inside an open list, where the
// recovery gate fires and would otherwise lex it as a plain bareword).
// Consume the rest of the identifier, mark the token end at the word
// boundary, then look past whitespace for '='. The whole word must be
// covered, so skip with advance(false): advance(true) would reset the
// token start past the word and emit a zero-width key. mark_end fixes the
// end at the word, so trailing whitespace stays out of the token anyway.
while (isidcont(la)) {
lexer->advance(lexer, false);
la = lexer->lookahead;
}
lexer->mark_end(lexer);
while (is_tsp_whitespace(la)) {
lexer->advance(lexer, false);
la = lexer->lookahead;
}
if (la == '=') return PEEK_AUTOQUOTE_KEY;
return PEEK_NOT_KEYWORD;
}
// A statement keyword. Structural OO keywords (method/class/role) never
// legitimately nest inside a sub/method body, so they alone trigger body
// block-close recovery. use/no/sub/package commonly DO appear in a body
// (e.g. `no strict 'refs';`) and must not trigger it.
//
// `sub` is the tempting one: a bare nested `sub foo {...}` is almost always a
// "won't stay shared" bug, not intent — so closing the enclosing body at it
// would often be the *helpful* read. We deliberately don't. It is valid Perl
// and it is genuinely out there (~140 nested bare subs across a 92k-file
// corpus: perl's own regen/Porting codegen scripts, some CPAN, real app code),
// so the parser tree-s it faithfully; diagnosing the footgun belongs to the
// layer above (linter/LSP), not the grammar. Lexical `my sub` (which *does*
// truly nest, and is legit) is a separate story and is excluded for free
// anyway: it starts with `my`, takes the KW_NONE path above, and never reaches
// here. (`kind` was already classified above; don't re-run KEYWORD_MATCH.)
*is_structural = (strcmp(word, "method") == 0 ||
strcmp(word, "class") == 0 ||
strcmp(word, "role") == 0);
// The caller already marked a zero-width position at the keyword start for
// recovery-token emission; leave that mark alone (don't mark_end here) and
// skip trailing whitespace with advance(true) — the recovery-safe behavior.
// A word-end mark here would make the recovery token swallow the keyword
// (and breaks the keyword-boundary tests).
while (is_tsp_whitespace(la)) {
lexer->advance(lexer, true);
la = lexer->lookahead;
}
// Fat comma => means even a keyword is a hash key, not a statement
// (`package => 1`). We can't mark the word boundary here without clobbering
// the recovery mark, so the caller emits a zero-width _fat_comma_autoquoted
// _ahead marker and tree-sitter re-lexes the word through the normal
// autoquote path with a correct span. (Plain words and non-keyword builtins
// — sort/map/ref/state/… — take the KW_NONE path above and autoquote in a
// single pass.)
if (la == '=') return PEEK_FAT_COMMA;
// For sub/method: only a declaration if followed by an identifier (name)
if (kind == KW_NEEDS_NAME && !isidfirst(la))
return PEEK_NOT_KEYWORD; // anonymous sub/method
return PEEK_KEYWORD;
}
static void _skip_chars(TSLexer *lexer, int maxlen, const char *allow) {
int32_t c = lexer->lookahead;
@@ -306,14 +526,37 @@ static bool isidcont(int32_t c) { return c == '_' || is_tsp_id_continue(c); }
// there's a matching rule in the grammar to catch when it doesn't match a rule
static bool is_interpolation_escape(int32_t c) { return c < 256 && tsp_strchr("$@-[{\\", c); }
/* The runtime hands serialize() a FIXED buffer of TREE_SITTER_SERIALIZATION_BUFFER_SIZE
* (1024) bytes — `self->lexer.debug_buffer`, a member embedded in the Lexer/TSParser
* struct. Writing past it is intra-object memory corruption (ASAN doesn't flag it),
* and returning a length > the buffer trips the runtime's `assert(length <= 1024)`
* (or, in NDEBUG builds, silently clobbers adjacent parser fields). So we must bound
* the serialized size to the buffer, NOT to UINT8_MAX.
*
* Budget: everything except the quotes is fixed-size — the two count bytes, the
* recovery flag, and the whole heredoc FIFO at its worst case (HEREDOC_QUEUE_MAX
* full entries). Whatever room is left holds quotes. A deeply nested (or, in a
* mid-edit buffer, deeply *unclosed*) interpolation like `qq{ ${\ qq{ ... }} }`
* stacks one quote per level and can blow past this; we cap it. Truncating the
* stack degrades parsing of pathologically-nested input (>~59 levels) but keeps it
* BOUNDED — the same graceful-degradation contract the heredoc overflow already uses.
*/
#define SER_FIXED_OVERHEAD \
(1 /* quote_count */ + \
1 /* heredoc_state */ + 1 /* heredoc_count */ + \
HEREDOC_QUEUE_MAX * (1 /* interpolates */ + 1 /* indents */ + (int)sizeof(TSPString)) + \
1 /* recovery_emitted */)
#define MAX_SERIALIZED_QUOTES \
((TREE_SITTER_SERIALIZATION_BUFFER_SIZE - SER_FIXED_OVERHEAD) / (int)sizeof(TSPQuote))
unsigned int tree_sitter_perl_external_scanner_serialize(void *payload, char *buffer) {
LexerState *state = payload;
size_t size = 0;
// Serialize the quotes array
// Serialize the quotes array, capped to what fits in the fixed buffer (see above).
size_t quote_count = state->quotes.size;
if (quote_count > UINT8_MAX) {
quote_count = UINT8_MAX;
if (quote_count > MAX_SERIALIZED_QUOTES) {
quote_count = MAX_SERIALIZED_QUOTES;
}
buffer[size++] = (char)quote_count;
@@ -322,12 +565,18 @@ unsigned int tree_sitter_perl_external_scanner_serialize(void *payload, char *bu
}
size += quote_count * sizeof(TSPQuote);
// Serialize the heredoc state and delimiter
buffer[size++] = (char)state->heredoc_interpolates;
buffer[size++] = (char)state->heredoc_indents;
// Serialize the heredoc consume state, then the whole pending FIFO queue:
// a count byte followed by each entry's flags + delimiter.
buffer[size++] = (char)state->heredoc_state;
memcpy(&buffer[size], &state->heredoc_delim, sizeof(TSPString));
size += sizeof(TSPString);
buffer[size++] = (char)state->heredoc_count;
for (uint8_t i = 0; i < state->heredoc_count; i++) {
HeredocEntry *e = &state->heredoc_queue[i];
buffer[size++] = (char)e->interpolates;
buffer[size++] = (char)e->indents;
memcpy(&buffer[size], &e->delim, sizeof(TSPString));
size += sizeof(TSPString);
}
buffer[size++] = (char)state->recovery_emitted;
return size;
}
@@ -340,6 +589,9 @@ void tree_sitter_perl_external_scanner_deserialize(void *payload, const char *bu
if (length > 0) {
// Deserialize the quotes array
size_t quote_count = (uint8_t)buffer[size++];
// Defensive: a well-formed buffer never exceeds this (serialize caps it), but
// clamp anyway so a malformed/old buffer can't drive an oversized memcpy.
if (quote_count > MAX_SERIALIZED_QUOTES) quote_count = MAX_SERIALIZED_QUOTES;
if (quote_count > 0) {
array_reserve(&state->quotes, quote_count);
state->quotes.size = quote_count;
@@ -347,13 +599,21 @@ void tree_sitter_perl_external_scanner_deserialize(void *payload, const char *bu
size += quote_count * sizeof(TSPQuote);
}
// Deserialize the heredoc state and delimiter
state->heredoc_interpolates = (bool)buffer[size++];
state->heredoc_indents = (bool)buffer[size++];
// Deserialize the heredoc consume state and the pending FIFO queue.
state->heredoc_state = (enum HeredocState)buffer[size++];
memcpy(&state->heredoc_delim, &buffer[size], sizeof(TSPString));
size += sizeof(TSPString);
state->heredoc_count = (uint8_t)buffer[size++];
if (state->heredoc_count > HEREDOC_QUEUE_MAX) state->heredoc_count = HEREDOC_QUEUE_MAX;
for (uint8_t i = 0; i < state->heredoc_count; i++) {
HeredocEntry *e = &state->heredoc_queue[i];
e->interpolates = (bool)buffer[size++];
e->indents = (bool)buffer[size++];
memcpy(&e->delim, &buffer[size], sizeof(TSPString));
size += sizeof(TSPString);
}
state->recovery_emitted = (bool)buffer[size++];
} else {
state->heredoc_count = 0;
state->heredoc_state = HEREDOC_NONE;
}
}
@@ -363,6 +623,9 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
bool is_ERROR = valid_symbols[TOKEN_ERROR];
bool skipped_whitespace = false;
bool crossed_newline = false;
bool recovery_emitted = state->recovery_emitted;
state->recovery_emitted = false;
int32_t c = lexer->lookahead;
@@ -372,16 +635,60 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
TOKEN(TOKEN_GOBBLED_CONTENT);
}
/* `format NAME = <newline> BODY .` — the body (line after `=` up to and
* including a lone `.` line) is opaque content the LR grammar can't parse, read
* as one TOKEN_FORMAT_CONTENT. Lives before whitespace handling so the body's
* leading newline isn't skipped separately. */
if (!is_ERROR && valid_symbols[TOKEN_FORMAT_CONTENT]) {
/* swallow the remainder of the `format ... =` line, then its newline */
while (c != '\n' && !lexer->eof(lexer)) ADVANCE_C;
if (c == '\n') ADVANCE_C;
/* now read body lines; stop after a line consisting solely of `.`
* (perl: a lone `.`, trailing whitespace tolerated) */
while (!lexer->eof(lexer)) {
/* peek whether this whole line is just `.` */
bool only_dot = false;
if (c == '.') {
ADVANCE_C;
only_dot = true;
while (c == ' ' || c == '\t' || c == '\r') ADVANCE_C;
if (c != '\n' && !lexer->eof(lexer)) only_dot = false;
}
if (only_dot) {
if (c == '\n') ADVANCE_C; /* include the terminator's newline */
break;
}
/* not the terminator: consume to end of this line */
while (c != '\n' && !lexer->eof(lexer)) ADVANCE_C;
if (c == '\n') ADVANCE_C;
}
MARK_END;
TOKEN(TOKEN_FORMAT_CONTENT);
}
/* we use this to force tree-sitter to stay on the error branch of a nonassoc
* operator */
if (!is_ERROR && valid_symbols[TOKEN_NONASSOC]) TOKEN(TOKEN_NONASSOC);
/* The `x` repetition operator glued to its count (`"ab"x3`, `("")x4`). The
* internal lexer would greedily take `x3` as one identifier, so when the
* grammar offers the operator here (an operator is expected — perl's
* XOPERATOR state, which `valid_symbols` encodes) and `x` is glued to a digit,
* we emit it explicitly, consuming just the `x`. Anything else (`xor`, an
* identifier, a space-delimited `x`) falls through to the normal lexer. */
if (!is_ERROR && valid_symbols[TOKEN_X_OP] && c == 'x') {
ADVANCE_C;
if (c >= '0' && c <= '9') { MARK_END; TOKEN(TOKEN_X_OP); }
return false;
}
// this is whitespace sensitive, so it must go before any whitespace is
// skipped
if (valid_symbols[TOKEN_HEREDOC_MIDDLE] && !is_ERROR) {
if (valid_symbols[TOKEN_HEREDOC_MIDDLE] && !is_ERROR && state->heredoc_count > 0) {
HeredocEntry *front = lexerstate_front_heredoc(state);
DEBUG("Beginning heredoc contents\n", 0);
if (state->heredoc_state != HEREDOC_CONTINUE) {
TSPString line;
TSPString line = {0};
// read as many lines as we can
while (!lexer->eof(lexer)) {
tspstring_reset(&line);
@@ -391,7 +698,7 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
state->heredoc_state == HEREDOC_END || lexer->get_column(lexer) == 0;
bool saw_escape = false;
DEBUG("Starting loop at col %d\n", lexer->get_column(lexer));
if (is_valid_start_pos && state->heredoc_indents) {
if (is_valid_start_pos && front->indents) {
DEBUG("Skipping initial whitespace in heredoc\n", 0);
skip_whitespace(lexer);
c = lexer->lookahead;
@@ -411,8 +718,8 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
if (c == '$' || c == '@' || c == '\\') saw_escape = true;
ADVANCE_C;
}
DEBUG("got length %d, want length %d\n", line.length, state->heredoc_delim.length);
if (is_valid_start_pos && tspstring_eq(&line, &state->heredoc_delim)) {
DEBUG("got length %d, want length %d\n", line.length, front->delim.length);
if (is_valid_start_pos && tspstring_eq(&line, &front->delim)) {
// if we've read already, we return everything up until now
if (state->heredoc_state != HEREDOC_END) {
state->heredoc_state = HEREDOC_END;
@@ -422,7 +729,7 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
lexerstate_finish_heredoc(state);
TOKEN(TOKEN_HEREDOC_END);
}
if (saw_escape && state->heredoc_interpolates) {
if (saw_escape && front->interpolates) {
// we'll repeat this line in continue mode where we'll pause midline
state->heredoc_state = HEREDOC_CONTINUE;
TOKEN(TOKEN_HEREDOC_MIDDLE);
@@ -452,10 +759,10 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
}
}
if (iswspace(c) && valid_symbols[TOKEN_NO_INTERP_WHITESPACE_ZW]) {
if (!is_ERROR && iswspace(c) && valid_symbols[TOKEN_NO_INTERP_WHITESPACE_ZW]) {
TOKEN(TOKEN_NO_INTERP_WHITESPACE_ZW);
}
skip_ws_to_eol(lexer);
crossed_newline = skip_ws_to_eol(lexer);
/* heredocs override everything, so they must be here before */
if (valid_symbols[TOKEN_HEREDOC_START]) {
if (state->heredoc_state == HEREDOC_START && lexer->get_column(lexer) == 0) {
@@ -506,39 +813,256 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
// CTRL-Z must be here, b/c it cares about whitespace
if (c == 26 && valid_symbols[TOKEN_CTRL_Z]) TOKEN(TOKEN_CTRL_Z);
bool any_recovery_valid =
valid_symbols[TOKEN_RECOVER_ARROW] ||
valid_symbols[TOKEN_RECOVER_PAREN_CLOSE] ||
valid_symbols[TOKEN_RECOVER_BRACKET_CLOSE] ||
valid_symbols[TOKEN_RECOVER_BRACE_CLOSE] ||
valid_symbols[TOKEN_RECOVER_BLOCK_CLOSE] ||
valid_symbols[PERLY_SEMICOLON];
// Contextual keywords: class/role/method are declarations only when a name (or
// a block, for method) follows; otherwise they're ordinary barewords
// (`class($cv)`, `role { … }`, `-role`). Decide here, before emitting, so
// there's no keyword-vs-call ambiguity. The OO keywords are deferred while
// recovery is pending because they trigger the structural body block-close,
// which must fire first; async/try don't, so they're classified unconditionally.
bool oo_kw_valid = valid_symbols[TOKEN_KW_CLASS] || valid_symbols[TOKEN_KW_ROLE] || valid_symbols[TOKEN_KW_METHOD];
bool asynctry_valid = valid_symbols[TOKEN_KW_ASYNC] || valid_symbols[TOKEN_KW_TRY];
bool recovery_pending = any_recovery_valid && (crossed_newline || recovery_emitted);
bool enter_oo = oo_kw_valid && !recovery_pending && (c == 'c' || c == 'r' || c == 'm');
bool enter_asynctry = asynctry_valid && (c == 'a' || c == 't');
if (!is_ERROR && (enter_oo || enter_asynctry)) {
char word[8];
int wlen = 0;
while (isidcont(c)) { // consume the whole identifier (store first 7 chars)
if (wlen < 7) word[wlen++] = (char)c;
lexer->advance(lexer, false);
c = lexer->lookahead;
}
word[wlen] = '\0';
MARK_END; // the token covers exactly the word (lookahead past here is free)
while (is_tsp_whitespace(c)) { // peek past whitespace at the next token
lexer->advance(lexer, false);
c = lexer->lookahead;
}
bool is_class = !recovery_pending && valid_symbols[TOKEN_KW_CLASS] && strcmp(word, "class") == 0;
bool is_role = !recovery_pending && valid_symbols[TOKEN_KW_ROLE] && strcmp(word, "role") == 0;
bool is_method = !recovery_pending && valid_symbols[TOKEN_KW_METHOD] && strcmp(word, "method") == 0;
bool is_async = valid_symbols[TOKEN_KW_ASYNC] && strcmp(word, "async") == 0;
bool is_try = valid_symbols[TOKEN_KW_TRY] && strcmp(word, "try") == 0;
// `try` is the try/catch keyword only before a block `{`; `try(...)`/`try =>`
// /`try;` are bareword calls. (`catch` is never over-reserved.)
if (is_try) {
if (c == '{') TOKEN(TOKEN_KW_TRY);
}
// `async` is a keyword before a block (`async { … }` — threads::async, a bare
// block run as an anon sub; F::AA itself has only `async sub`) or before a
// sub/method/extended declaration. Else it's a bareword/call.
if (is_async) {
if (c == '{') TOKEN(TOKEN_KW_ASYNC);
if (c == 's' || c == 'm' || c == 'e') { // peek for sub/method/extended
char nword[9];
int nlen = 0;
while (isidcont(c)) {
if (nlen < 8) nword[nlen++] = (char)c;
lexer->advance(lexer, false);
c = lexer->lookahead;
}
nword[nlen] = '\0';
if (strcmp(nword, "sub") == 0 || strcmp(nword, "method") == 0 ||
strcmp(nword, "extended") == 0)
TOKEN(TOKEN_KW_ASYNC);
// Not a decl: `async` is a bareword/call. Must `return false` (re-lex),
// NOT `goto kw_autoquote` like the path below — we consumed past `async`
// into the next word, so the handler would read *that* word's `=>` as
// async's own fat comma and wrongly autoquote `async` (`async send=>1`).
return false;
}
}
// `class NAME` / `role NAME` — a name following => declaration; else bareword.
if ((is_class || is_role) && isidfirst(c))
TOKEN(is_class ? TOKEN_KW_CLASS : TOKEN_KW_ROLE);
// `method NAME { … }` / `method { … }` are declarations (named/anonymous);
// `method NAME => sub { … }` (MooseX) is a call — peek past the name for `=>`.
if (is_method) {
if (c == '{' || c == '(') TOKEN(TOKEN_KW_METHOD); // anonymous method
if (isidfirst(c)) {
while (isidcont(c)) { lexer->advance(lexer, false); c = lexer->lookahead; } // skip the name
while (is_tsp_whitespace(c)) { lexer->advance(lexer, false); c = lexer->lookahead; }
if (c != '=') TOKEN(TOKEN_KW_METHOD); // `{`/`(`/`:`/`;` => decl; `=>` => call
return false; // method NAME => … : re-lex as a call (the name autoquotes)
}
}
// Bareword/call, or an autoquoted key (`class => 1`, `$h{m}`). Can't just
// `return false` — a quote-op word (`m`/`s`/`y`) would re-lex as a match — so
// fall through to the canonical autoquote handler; MARK_END already fixed the
// token at the word, and it emits the autoquote or returns false on no match.
goto kw_autoquote;
}
// === Error recovery: close unclosed delimiters and insert semicolons ===
//
// When the scanner sees '}', ';', EOF, or a statement keyword on a new
// line, it emits recovery tokens to unwind open delimiters. Each call
// peels one layer — the parser keeps calling until everything is closed.
//
// Syntactic boundary: '}', ';', or EOF
if (!is_ERROR) {
if (c == '}' || c == ';' || lexer->eof(lexer)) {
EMIT_RECOVERY_TOKENS(c != '}');
}
}
if (valid_symbols[PERLY_SEMICOLON]) {
if (c == '}' || lexer->eof(lexer)) {
// do a PERLY_SEMICOLON unless we're in brace autoquoting
if (is_ERROR || !valid_symbols[TOKEN_BRACE_END_ZW]) {
DEBUG("Fake PERLY_SEMICOLON at end-of-scope\n", 0);
// no advance
TOKEN(PERLY_SEMICOLON);
}
}
}
if (lexer->eof(lexer)) return false;
// Statement keyword boundary: a statement keyword on a new line means
// the previous expression is done. Also fires when a recovery token
// was just emitted (recovery_emitted flag) since the newline was already
// consumed by the previous call.
// Gate on the first-char filter so we don't call MARK_END (and freeze
// the token end at the keyword-start position) when the lookahead can't
// possibly start a keyword. Without this, a later TOKEN() emission
// (e.g. TOKEN_APOSTROPHE/TOKEN_DOUBLE_QUOTE which do ADVANCE_C+TOKEN
// and never re-mark) inherits the stale mark and reports a zero-width
// open-quote, which derails the string body.
if ((crossed_newline || recovery_emitted) && !is_ERROR && any_recovery_valid &&
!KEYWORD_FIRST_CHAR_FILTER(c)) {
MARK_END; // zero-width position for recovery tokens
bool is_structural = false;
enum PeekResult peek = peek_is_statement_keyword(lexer, &is_structural);
if (peek == PEEK_KEYWORD) {
DEBUG("keyword boundary\n", 0);
EMIT_RECOVERY_TOKENS(true);
// Body block-close: only a structural OO keyword (method/class/role)
// closes a half-typed sub/method body. Fires after any inner paren/
// bracket closers above; the enclosing class block can't accept this
// token, so the cascade stops at one body (no over-closing).
if (is_structural && valid_symbols[TOKEN_RECOVER_BLOCK_CLOSE]) {
DEBUG("body block recovery at structural keyword\n", 0);
// Re-arm the cascade so the next scan re-enters and can close the NEXT
// enclosing recover-aware block (e.g. an unclosed `if`/loop body inside
// the method: close the if-block, then the method body).
state->recovery_emitted = true;
TOKEN(TOKEN_RECOVER_BLOCK_CLOSE);
}
// A statement terminator may need to fire BEFORE the body block-close:
// after an inner closer (e.g. `my @x = [` -> `]`) completes an expression,
// the statement still needs `;` before the enclosing body can reduce and
// close. Re-arm the recovery cascade (recovery_emitted) so the next scan
// re-enters here and can emit the block-close. Harmless when nothing
// follows: the re-entry finds no further valid recovery token.
if (valid_symbols[PERLY_SEMICOLON]) {
state->recovery_emitted = true;
TOKEN(PERLY_SEMICOLON);
}
}
if (peek == PEEK_FAT_COMMA) {
// A statement keyword used as a fat-comma key (`package => 1`). We can't
// mark the word boundary here (peek preserved the caller's zero-width
// recovery mark for the keyword path, and re-marking would consume the
// keyword). Instead emit a zero-width "autoquote ahead" marker at the
// word start: tree-sitter re-lexes the word via the normal autoquote
// path, which marks the word boundary correctly. The recovery gate
// won't re-fire on re-entry (no newline precedes the word that scan), so
// there's no loop. Falls back to the old (zero-width-span) emission if
// the marker isn't accepted in this state.
if (valid_symbols[TOKEN_FAT_COMMA_AUTOQUOTED_AHEAD])
TOKEN(TOKEN_FAT_COMMA_AUTOQUOTED_AHEAD);
MARK_END;
c = lexer->lookahead;
goto fat_comma_check;
}
if (peek == PEEK_AUTOQUOTE_KEY) {
// A non-keyword bareword used as a fat-comma key (`name => 1`) on a
// continuation line where the recovery gate fired. Peek already marked
// the token end at the word boundary (covering exactly the word), so do
// NOT MARK_END again — that would re-mark at '=' and emit a zero-width
// key. Resume at the autoquote handler's => check.
c = lexer->lookahead;
goto fat_comma_check;
}
if (peek == PEEK_NOT_KEYWORD)
return false;
}
if (valid_symbols[TOKEN_OPEN_FILEGLOB_BRACKET] || valid_symbols[TOKEN_OPEN_READLINE_BRACKET] || valid_symbols[PERLY_HEREDOC]) {
if (c == '<') {
ADVANCE_C;
MARK_END;
// ah, we have a heredoc; let's just go down to that section then
if (c == '<') goto heredoc_token_handling;
if (c == '$') ADVANCE_C;
// Gather the `<...>` body as we go, starting right after `<`, so the
// fileglob content heuristic below sees the WHOLE body -- including
// the `$ident` prefix the readline probe consumes here (otherwise a
// `<$sner >` glob would reach the heuristic with content " ").
char content[256];
size_t clen = 0;
if (c == '$') { content[clen++] = '$'; ADVANCE_C; }
// we now zoooom as many ident chars as we can
while (isidcont(c)) ADVANCE_C;
while (isidcont(c)) {
if (clen < sizeof(content)) content[clen++] = (c < 0x80) ? (char)c : (char)0x7f;
ADVANCE_C;
}
// if ident chars took us until the closing `>` then we're readline FILEHANDLE
if (c == '>') TOKEN(TOKEN_OPEN_READLINE_BRACKET);
// otherwise we're a fileglob operator, and we set up our string parse + be excellent
lexerstate_push_quote(state, '<');
TOKEN(TOKEN_OPEN_FILEGLOB_BRACKET);
// Otherwise `<...>` is either a fileglob (`<*.c>`) or the relational
// `<` operator (`CONST < 0`); both are live after an ambiguous
// bareword. Gather the body and the bytes after the `>` (pure
// lookahead past MARK_END, so the token stays the one-char `<`;
// non-ASCII -> 0x7f), then let tsp_is_fileglob() decide -- see
// tsp_intuit_readline.h.
if (valid_symbols[TOKEN_OPEN_FILEGLOB_BRACKET]) {
// `content`/`clen` already hold the `$ident` prefix from above.
// Stop at a statement boundary so we don't chase a `>` onto the
// next line.
while (c != '>' && c != '<' && c != ';' && c != '\n' &&
!lexer->eof(lexer)) {
if (clen < sizeof(content))
content[clen++] = (c < 0x80) ? (char)c : (char)0x7f;
ADVANCE_C;
}
if (c == '>') {
ADVANCE_C; // step past the closing `>` (still pure lookahead)
char after[256];
size_t alen = 0;
while (alen < sizeof(after) && c != '\n' && !lexer->eof(lexer)) {
after[alen++] = (c < 0x80) ? (char)c : (char)0x7f;
ADVANCE_C;
}
if (tsp_is_fileglob(content, clen, after, alen)) {
lexerstate_push_quote(state, '<');
TOKEN(TOKEN_OPEN_FILEGLOB_BRACKET);
}
}
}
// not a readline, not a fileglob — let `<` fall through to the
// grammar as the relational operator.
return false;
}
}
if (valid_symbols[TOKEN_DOLLAR_IDENT_ZW]) {
// false on word chars, another dollar or {
if (!isidcont(c) && !tsp_strchr("${", c)) {
if (!is_ERROR && valid_symbols[TOKEN_DOLLAR_IDENT_ZW]) {
// false on word chars, another dollar or { -- but if whitespace intervened
// the `$` can't begin a glued deref ($$foo needs the name glued on), so a
// following word char is a separate token: `$$ eq …`, `$$ and …` are the
// PID var `$$`, not `${$eq}`. Treat skipped whitespace as a hard boundary.
if (!tsp_strchr("${", c) && (skipped_whitespace || !isidcont(c))) {
if (c == ':') {
// NOTE - it's a syntax error to do $$:, so that's why we return
// dollar_ident_zw in that case
@@ -601,6 +1125,64 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
return false;
}
/* A '[' or '{' in a pattern is ambiguous: it can begin a subscript on the
* preceding variable (/$foo[0]/, /$foo{k}/) or a character class /
* quantifier (/[abc]/, /a{2,3}/). We replicate perl's S_intuit_more
* heuristic (tsp_intuit_more) to decide. When it's a class/quantifier we
* emit a literal opener (aliased back to '[' / '{' in the grammar); when
* it's a subscript we decline so the grammar's token.immediate('[' / '{')
* takes over. */
if ((valid_symbols[TOKEN_REGEXP_OPEN_BRACKET] && c == '[') ||
(valid_symbols[TOKEN_REGEXP_OPEN_BRACE] && c == '{')) {
/* A pattern-LEADING '['/'{' that is the quote's own opening delimiter is a
* balanced nested delimiter (m{{...}}, qr{ {...} }, s{{...}}, m[[...]]),
* NOT a subscript: there is no preceding variable for intuit_more to weigh,
* and its subscript verdict here would be meaningless. Fall through to the
* Q/QQ content scanner, which consumes the opener while tracking the nesting
* count so the matching inner close doesn't terminate the quote early.
* Anything beyond the leading position is genuinely ambiguous again, so the
* heuristic below applies as before. */
bool leading = lexerstate_take_body_lead(state, c);
if (leading) {
/* leave the opener for the content scanner below */
} else {
int32_t open = c;
int32_t close = (open == '[') ? ']' : '}';
char buf[256];
int n = 0;
buf[n++] = (char)open;
ADVANCE_C; /* consume the opener... */
MARK_END; /* ...the emitted token is exactly that one char */
/* Read ahead to gather the construct for the heuristic, stopping once we
* have buffered the matching close (so intuit_more can find it) or we run
* out of room / input. These advances past MARK_END are pure lookahead;
* the token stays one char wide. Non-ASCII collapses to a placeholder
* since the heuristic is ASCII-only by design (a best-effort gap). */
while (n < (int)sizeof(buf) && c != 0 && !lexer->eof(lexer)) {
buf[n++] = (c < 0x80) ? (char)c : (char)0x7f;
if (c == close) break;
ADVANCE_C;
}
if (!tsp_intuit_more(buf, n)) {
/* If this class/quantifier opener is also the quote's own delimiter
* (`m{ \d{2} }`, `m[ [abc] ]`), it opens a nested level — bump the count
* so the matching inner close doesn't terminate the quote early. The
* content scanner does this when it consumes `{`/`[` inline; we must do
* the same here, since we only reach this branch when the opener leads a
* scan (e.g. right after an escape) and the content scanner is bypassed. */
int32_t qi = lexerstate_is_quote_opener(state, open);
if (qi) lexerstate_saw_opener(state, qi);
TOKEN(open == '[' ? TOKEN_REGEXP_OPEN_BRACKET : TOKEN_REGEXP_OPEN_BRACE);
}
/* subscript: let the grammar's immediate '[' / '{' win */
return false;
}
}
if (valid_symbols[TOKEN_POD]) {
int column = lexer->get_column(lexer);
if (column == 0 && c == '=') {
@@ -648,7 +1230,7 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
bool should_indent = false;
bool should_interpolate = true;
TSPString delim;
TSPString delim = {0};
tspstring_reset(&delim);
if (!skipped_whitespace) {
if (c == '~') {
@@ -694,7 +1276,13 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
ADVANCE_C;
}
}
if (delim.length > 0) {
// We stop the loop above either at the closing quote or at EOF. Only
// commit a heredoc if we actually found the closer; an EMPTY delimiter
// (`<<''` / `<<""`) is legal — perl terminates its body at the next blank
// line, which the body matcher already handles (an empty delim compares
// equal to an empty line). Keying off the closer rather than a non-empty
// delim is also more correct: it won't mis-fire on EOF-without-closer.
if (c == delim_open) {
// gotta eat that delimiter
ADVANCE_C;
// gotta null terminate up in here
@@ -724,10 +1312,28 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
}
MARK_END;
// In a paired multi-part quote (s{}{}, tr[][], ...), the replacement pair
// opens a fresh quote whose delimiter may differ from the pattern's. The
// pattern's quote is still on top of the stack (middle_close consumes its
// closer but doesn't pop it), so its closer would wrongly stay active while
// we scan the replacement. Now that the second pair is opening, retire it.
// MIDDLE_SKIP being a valid symbol here means we're at the middle choice
// point (between pattern and replacement) rather than an initial begin.
if (valid_symbols[TOKEN_QUOTELIKE_MIDDLE_SKIP] && state->quotes.size) {
lexerstate_pop_quote(state, state->quotes.size);
}
lexerstate_push_quote(state, delim);
// TODO - fill in this debug print here?
// DEBUG("Generic QSTRING open='%c' close='%c'\n", state->delim_open,
// state->delim_close);
/* Pattern-leading bracket lookahead (see TSPQuote.body_leads_with_delim):
* for a paired delimiter, peek past any leading whitespace -- if the body's
* first real char is the delimiter again (m{{...}}, qr{ {...} }, m[[...]]),
* it's a literal leading group, not a subscript. This is pure lookahead:
* MARK_END already covers just the opener, so these advances don't extend
* the emitted token and the body is re-scanned from the opener's end. */
if (close_for_open(delim)) {
while (is_tsp_whitespace(c)) ADVANCE_C;
if (c == delim) array_back(&state->quotes)->body_leads_with_delim = true;
}
TOKEN(TOKEN_QUOTELIKE_BEGIN);
}
@@ -885,6 +1491,23 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
}
return false;
}
// A lone non-identifier punctuation variable inside ${...} — ${@}, ${!},
// ${%}, ${$}, ${"} … — names the punctuation variable $@/$!/…, NOT a
// sigil/operator. The main lexer would otherwise pick the sigil/operator token
// (and for @ / % that token swallows the closing brace, decapitating the rest
// of the file). Recognize it here with the same '}'-lookahead the bareword
// path uses: one var char immediately followed (modulo whitespace) by '}' is
// the autoquoted name. Identifiers, digits and ^carets are handled by the
// grammar's other varname alternatives; '{' would begin a block, '#' a comment.
if (valid_symbols[TOKEN_BRACE_AUTOQUOTED] && !isidfirst(c) &&
c > ' ' && c != '}' && c != '{' && c != '^' && c != '#' &&
!(c >= '0' && c <= '9')) {
ADVANCE_C;
MARK_END;
while (is_tsp_whitespace(c)) ADVANCE_C;
if (c == '}') TOKEN(TOKEN_BRACE_AUTOQUOTED);
return false;
}
if (isidfirst(c) &&
(valid_symbols[TOKEN_FAT_COMMA_AUTOQUOTED] || valid_symbols[TOKEN_BRACE_AUTOQUOTED])) {
// we zip until the end of the identifier; then we do a lookeahed to see if it's autoquoted
@@ -898,17 +1521,26 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
// NOTE - TS is annoying about skipping chars after you've hit done
// mark_end, so we have to do the regular advance so our token actually shows up
// The contextual-keyword block jumps here (after reading its word + MARK_END)
// to reuse this comment-skip + fat_comma_check instead of duplicating them.
kw_autoquote:
while (is_tsp_whitespace(c) || c == '#') {
while (is_tsp_whitespace(c)) ADVANCE_C;
// now we need to skip comments - we get in a funny way if we have a quotelike
// operator followed by a comment as the quote char
if (c == '#') {
ADVANCE_C;
while (lexer->get_column(lexer)) ADVANCE_C;
while (lexer->get_column(lexer) && !lexer->eof(lexer)) ADVANCE_C;
}
if (lexer->eof(lexer)) return false;
// TODO - in theory there could be POD here that we needa skip over (EYES ROLL)
}
// The keyword peek's fat comma detection gotos here when it finds a
// statement keyword followed by '=' on a new line. The peek has already
// consumed the word + whitespace; lexer is at '='. MARK_END was set by
// the peek's caller (covers the word). Same goto pattern as heredoc vs
// diamond operator disambiguation.
fat_comma_check:
c1 = lexer->lookahead;
ADVANCE_C;
if (valid_symbols[TOKEN_FAT_COMMA_AUTOQUOTED]) {
@@ -52,67 +52,96 @@ extern "C" {
/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
/// less than the array's current capacity, this function has no effect.
#define array_reserve(self, new_capacity) \
_array__reserve((Array *)(self), array_elem_size(self), new_capacity)
#define array_reserve(self, new_capacity) \
((self)->contents = _array__reserve( \
(void *)(self)->contents, &(self)->capacity, \
array_elem_size(self), new_capacity) \
)
/// Free any memory allocated for this array. Note that this does not free any
/// memory allocated for the array's contents.
#define array_delete(self) _array__delete((Array *)(self))
#define array_delete(self) \
do { \
if ((self)->contents) ts_free((self)->contents); \
(self)->contents = NULL; \
(self)->size = 0; \
(self)->capacity = 0; \
} while (0)
/// Push a new `element` onto the end of the array.
#define array_push(self, element) \
(_array__grow((Array *)(self), 1, array_elem_size(self)), \
(self)->contents[(self)->size++] = (element))
#define array_push(self, element) \
do { \
(self)->contents = _array__grow( \
(void *)(self)->contents, (self)->size, &(self)->capacity, \
1, array_elem_size(self) \
); \
(self)->contents[(self)->size++] = (element); \
} while(0)
/// Increase the array's size by `count` elements.
/// New elements are zero-initialized.
#define array_grow_by(self, count) \
do { \
if ((count) == 0) break; \
_array__grow((Array *)(self), count, array_elem_size(self)); \
#define array_grow_by(self, count) \
do { \
if ((count) == 0) break; \
(self)->contents = _array__grow( \
(self)->contents, (self)->size, &(self)->capacity, \
count, array_elem_size(self) \
); \
memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
(self)->size += (count); \
(self)->size += (count); \
} while (0)
/// Append all elements from one array to the end of another.
#define array_push_all(self, other) \
#define array_push_all(self, other) \
array_extend((self), (other)->size, (other)->contents)
/// Append `count` elements to the end of the array, reading their values from the
/// `contents` pointer.
#define array_extend(self, count, contents) \
_array__splice( \
(Array *)(self), array_elem_size(self), (self)->size, \
0, count, contents \
#define array_extend(self, count, other_contents) \
(self)->contents = _array__splice( \
(void*)(self)->contents, &(self)->size, &(self)->capacity, \
array_elem_size(self), (self)->size, 0, count, other_contents \
)
/// Remove `old_count` elements from the array starting at the given `index`. At
/// the same index, insert `new_count` new elements, reading their values from the
/// `new_contents` pointer.
#define array_splice(self, _index, old_count, new_count, new_contents) \
_array__splice( \
(Array *)(self), array_elem_size(self), _index, \
old_count, new_count, new_contents \
#define array_splice(self, _index, old_count, new_count, new_contents) \
(self)->contents = _array__splice( \
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
array_elem_size(self), _index, old_count, new_count, new_contents \
)
/// Insert one `element` into the array at the given `index`.
#define array_insert(self, _index, element) \
_array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
#define array_insert(self, _index, element) \
(self)->contents = _array__splice( \
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
array_elem_size(self), _index, 0, 1, &(element) \
)
/// Remove one element from the array at the given `index`.
#define array_erase(self, _index) \
_array__erase((Array *)(self), array_elem_size(self), _index)
_array__erase((void *)(self)->contents, &(self)->size, array_elem_size(self), _index)
/// Pop the last element off the array, returning the element by value.
#define array_pop(self) ((self)->contents[--(self)->size])
/// Assign the contents of one array to another, reallocating if necessary.
#define array_assign(self, other) \
_array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
#define array_assign(self, other) \
(self)->contents = _array__assign( \
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
(const void *)(other)->contents, (other)->size, array_elem_size(self) \
)
/// Swap one array with another
#define array_swap(self, other) \
_array__swap((Array *)(self), (Array *)(other))
#define array_swap(self, other) \
do { \
void *_array_swap_tmp = (void *)(self)->contents; \
(self)->contents = (other)->contents; \
(other)->contents = _array_swap_tmp; \
_array__swap(&(self)->size, &(self)->capacity, \
&(other)->size, &(other)->capacity); \
} while (0)
/// Get the size of the array contents
#define array_elem_size(self) (sizeof *(self)->contents)
@@ -157,82 +186,90 @@ extern "C" {
// Private
typedef Array(void) Array;
/// This is not what you're looking for, see `array_delete`.
static inline void _array__delete(Array *self) {
if (self->contents) {
ts_free(self->contents);
self->contents = NULL;
self->size = 0;
self->capacity = 0;
}
}
// Pointers to individual `Array` fields (rather than the entire `Array` itself)
// are passed to the various `_array__*` functions below to address strict aliasing
// violations that arises when the _entire_ `Array` struct is passed as `Array(void)*`.
//
// The `Array` type itself was not altered as a solution in order to avoid breakage
// with existing consumers (in particular, parsers with external scanners).
/// This is not what you're looking for, see `array_erase`.
static inline void _array__erase(Array *self, size_t element_size,
uint32_t index) {
assert(index < self->size);
char *contents = (char *)self->contents;
static inline void _array__erase(void* self_contents, uint32_t *size,
size_t element_size, uint32_t index) {
assert(index < *size);
char *contents = (char *)self_contents;
memmove(contents + index * element_size, contents + (index + 1) * element_size,
(self->size - index - 1) * element_size);
self->size--;
(*size - index - 1) * element_size);
(*size)--;
}
/// This is not what you're looking for, see `array_reserve`.
static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
if (new_capacity > self->capacity) {
if (self->contents) {
self->contents = ts_realloc(self->contents, new_capacity * element_size);
static inline void *_array__reserve(void *contents, uint32_t *capacity,
size_t element_size, uint32_t new_capacity) {
void *new_contents = contents;
if (new_capacity > *capacity) {
if (contents) {
new_contents = ts_realloc(contents, new_capacity * element_size);
} else {
self->contents = ts_malloc(new_capacity * element_size);
new_contents = ts_malloc(new_capacity * element_size);
}
self->capacity = new_capacity;
*capacity = new_capacity;
}
return new_contents;
}
/// This is not what you're looking for, see `array_assign`.
static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
_array__reserve(self, element_size, other->size);
self->size = other->size;
memcpy(self->contents, other->contents, self->size * element_size);
static inline void *_array__assign(void* self_contents, uint32_t *self_size, uint32_t *self_capacity,
const void *other_contents, uint32_t other_size, size_t element_size) {
void *new_contents = _array__reserve(self_contents, self_capacity, element_size, other_size);
*self_size = other_size;
memcpy(new_contents, other_contents, *self_size * element_size);
return new_contents;
}
/// This is not what you're looking for, see `array_swap`.
static inline void _array__swap(Array *self, Array *other) {
Array swap = *other;
*other = *self;
*self = swap;
static inline void _array__swap(uint32_t *self_size, uint32_t *self_capacity,
uint32_t *other_size, uint32_t *other_capacity) {
uint32_t tmp_size = *self_size;
uint32_t tmp_capacity = *self_capacity;
*self_size = *other_size;
*self_capacity = *other_capacity;
*other_size = tmp_size;
*other_capacity = tmp_capacity;
}
/// This is not what you're looking for, see `array_push` or `array_grow_by`.
static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
uint32_t new_size = self->size + count;
if (new_size > self->capacity) {
uint32_t new_capacity = self->capacity * 2;
static inline void *_array__grow(void *contents, uint32_t size, uint32_t *capacity,
uint32_t count, size_t element_size) {
void *new_contents = contents;
uint32_t new_size = size + count;
if (new_size > *capacity) {
uint32_t new_capacity = *capacity * 2;
if (new_capacity < 8) new_capacity = 8;
if (new_capacity < new_size) new_capacity = new_size;
_array__reserve(self, element_size, new_capacity);
new_contents = _array__reserve(contents, capacity, element_size, new_capacity);
}
return new_contents;
}
/// This is not what you're looking for, see `array_splice`.
static inline void _array__splice(Array *self, size_t element_size,
static inline void *_array__splice(void *self_contents, uint32_t *size, uint32_t *capacity,
size_t element_size,
uint32_t index, uint32_t old_count,
uint32_t new_count, const void *elements) {
uint32_t new_size = self->size + new_count - old_count;
uint32_t new_size = *size + new_count - old_count;
uint32_t old_end = index + old_count;
uint32_t new_end = index + new_count;
assert(old_end <= self->size);
assert(old_end <= *size);
_array__reserve(self, element_size, new_size);
void *new_contents = _array__reserve(self_contents, capacity, element_size, new_size);
char *contents = (char *)self->contents;
if (self->size > old_end) {
char *contents = (char *)new_contents;
if (*size > old_end) {
memmove(
contents + new_end * element_size,
contents + old_end * element_size,
(self->size - old_end) * element_size
(*size - old_end) * element_size
);
}
if (new_count > 0) {
@@ -250,7 +287,9 @@ static inline void _array__splice(Array *self, size_t element_size,
);
}
}
self->size += new_count - old_count;
*size += new_count - old_count;
return new_contents;
}
/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
@@ -18,6 +18,11 @@ typedef uint16_t TSStateId;
typedef uint16_t TSSymbol;
typedef uint16_t TSFieldId;
typedef struct TSLanguage TSLanguage;
typedef struct TSLanguageMetadata {
uint8_t major_version;
uint8_t minor_version;
uint8_t patch_version;
} TSLanguageMetadata;
#endif
typedef struct {
@@ -26,10 +31,11 @@ typedef struct {
bool inherited;
} TSFieldMapEntry;
// Used to index the field and supertype maps.
typedef struct {
uint16_t index;
uint16_t length;
} TSFieldMapSlice;
} TSMapSlice;
typedef struct {
bool visible;
@@ -79,6 +85,12 @@ typedef struct {
uint16_t external_lex_state;
} TSLexMode;
typedef struct {
uint16_t lex_state;
uint16_t external_lex_state;
uint16_t reserved_word_set_id;
} TSLexerMode;
typedef union {
TSParseAction action;
struct {
@@ -93,7 +105,7 @@ typedef struct {
} TSCharacterRange;
struct TSLanguage {
uint32_t version;
uint32_t abi_version;
uint32_t symbol_count;
uint32_t alias_count;
uint32_t token_count;
@@ -109,13 +121,13 @@ struct TSLanguage {
const TSParseActionEntry *parse_actions;
const char * const *symbol_names;
const char * const *field_names;
const TSFieldMapSlice *field_map_slices;
const TSMapSlice *field_map_slices;
const TSFieldMapEntry *field_map_entries;
const TSSymbolMetadata *symbol_metadata;
const TSSymbol *public_symbol_map;
const uint16_t *alias_map;
const TSSymbol *alias_sequences;
const TSLexMode *lex_modes;
const TSLexerMode *lex_modes;
bool (*lex_fn)(TSLexer *, TSStateId);
bool (*keyword_lex_fn)(TSLexer *, TSStateId);
TSSymbol keyword_capture_token;
@@ -129,15 +141,23 @@ struct TSLanguage {
void (*deserialize)(void *, const char *, unsigned);
} external_scanner;
const TSStateId *primary_state_ids;
const char *name;
const TSSymbol *reserved_words;
uint16_t max_reserved_word_set_size;
uint32_t supertype_count;
const TSSymbol *supertype_symbols;
const TSMapSlice *supertype_map_slices;
const TSSymbol *supertype_map_entries;
TSLanguageMetadata metadata;
};
static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
uint32_t index = 0;
uint32_t size = len - index;
while (size > 1) {
uint32_t half_size = size / 2;
uint32_t mid_index = index + half_size;
TSCharacterRange *range = &ranges[mid_index];
const TSCharacterRange *range = &ranges[mid_index];
if (lookahead >= range->start && lookahead <= range->end) {
return true;
} else if (lookahead > range->end) {
@@ -145,7 +165,7 @@ static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t
}
size -= half_size;
}
TSCharacterRange *range = &ranges[index];
const TSCharacterRange *range = &ranges[index];
return (lookahead >= range->start && lookahead <= range->end);
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,210 @@
/* tsp_intuit_more.h
*
* A faithful, stateless port of perl's S_intuit_more() (toke.c) and
* S_regcurly() (regcomp.c) -- "the one truly awful dwimmer necessary to
* conflate C and sed".
*
* Inside a pattern, a `[` or `{` following a variable is ambiguous:
* /$foo[...]/ could be an array subscript OR a character class
* /$foo{...}/ could be a hash subscript OR a {n,m} quantifier
* Perl resolves this with a weighting heuristic. We reproduce it so the
* external scanner can decide whether to emit a literal regex `[` / `{`.
*
* tsp_intuit_more(s, len) returns TRUE => "there is more to the expression",
* i.e. it's a SUBSCRIPT; FALSE => it's a character class / quantifier. This
* matches perl's return-value convention exactly.
*
* The caller is the scanner; it has already established that we are inside a
* pattern and that s[0] is '[' or '{'. `s` points at that bracket and spans
* `len` buffered bytes, ideally through the matching close bracket (when one
* exists in range). The function never reads past s[len-1].
*
* Three branches of perl's heuristic depend on state a stateless lexer can't
* see; these are intentional best-effort gaps an LSP can refine:
* - PL_lex_brackets (parser-stack recursion) -> handled by the scanner
* - gv_fetchpvn_flags (runtime symbol table, -100) -> treated as the -10 case
* - keyword() (-150) -> backed by our own
* regenerated keyword trie (tsp_intuit_keywords.h)
*/
#ifndef TSP_INTUIT_MORE_H
#define TSP_INTUIT_MORE_H
#include <stdbool.h>
#include <stddef.h>
#include "tsp_intuit_keywords.h"
/* --- ASCII classification (perl's is* macros, ASCII subset) ----------------
* UTF-8 word characters beyond ASCII are a best-effort gap. */
static inline bool tsp_im_isdigit(int c) { return c >= '0' && c <= '9'; }
static inline bool tsp_im_isalpha(int c) {
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
}
static inline bool tsp_im_isword(int c) {
return tsp_im_isalpha(c) || tsp_im_isdigit(c) || c == '_';
}
static inline bool tsp_im_isblank(int c) { return c == ' ' || c == '\t'; }
/* strchr over a literal set, NUL-safe (a NUL `c` never matches). */
static inline bool tsp_im_in(const char *set, int c) {
if (c == 0) return false;
for (; *set; set++)
if ((unsigned char)*set == (unsigned char)c) return true;
return false;
}
/* --- S_regcurly: does s[0..len) begin a {m,n} quantifier? ------------------
* Mirrors regcomp.c. We only need the syntactic yes/no, so the result[]
* out-param is dropped. Accepts {m}, {m,}, {m,n}, and {,n}; rejects {} . */
static bool tsp_regcurly(const char *s, int len) {
const char *e = s + len;
bool min_present = false; /* perl's min_start: digit(s) before the comma */
bool max_present = false; /* perl's max_end: digit(s) after the comma */
if (s >= e || *s++ != '{') return false;
while (s < e && tsp_im_isblank(*s)) s++;
if (s < e && tsp_im_isdigit(*s)) {
min_present = true;
do { s++; } while (s < e && tsp_im_isdigit(*s));
}
while (s < e && tsp_im_isblank(*s)) s++;
if (s < e && *s == ',') {
s++;
while (s < e && tsp_im_isblank(*s)) s++;
if (s < e && tsp_im_isdigit(*s)) {
max_present = true;
do { s++; } while (s < e && tsp_im_isdigit(*s));
}
while (s < e && tsp_im_isblank(*s)) s++;
}
/* Mirrors regcomp.c: must close with '}' and have at least one number,
* before or after the comma ({3}, {3,}, {3,5}, {,5}); '{}' and '{,}' fail. */
if (s >= e || *s != '}' || (!min_present && !max_present)) return false;
return true;
}
/* --- S_intuit_more: subscript (TRUE) vs char-class/quantifier (FALSE) ------ */
static bool tsp_intuit_more(const char *s, int len) {
const char *e = s + len;
if (s >= e) return true; /* nothing buffered; be safe */
/* In a pattern, a leading '{' might be a {n,m} quantifier. */
if (s[0] == '{') {
return !tsp_regcurly(s, len);
}
if (s[0] != '[') return true; /* caller only sends '[' or '{' */
s++; /* step past '[' */
/* '^' implies a char class; empty '[]' isn't legal but means "no more". */
if (s >= e) return true; /* '[' with nothing after -> expr */
if (s[0] == ']' || s[0] == '^') return false;
/* Find the matching ']'. No ']' in range => has to be an expression. */
const char *send = NULL;
for (const char *p = s; p < e; p++) {
if (*p == ']') { send = p; break; }
}
if (!send) return true;
/* One or two digits only => subscript, e.g. [12]. */
if (tsp_im_isdigit(s[0]) && send - s <= 2 &&
(send - s == 1 || tsp_im_isdigit(s[1]))) {
return true;
}
/* The weighting machine. weight >= 0 => char class, < 0 => subscript. */
int weight = (s[0] == '$') ? -1 : 2; /* '$' leans slightly to subscript */
unsigned char seen[256] = {0};
unsigned char un_char = 0;
bool first_time = true;
for (; s < send; s++, first_time = false) {
unsigned char prev_un_char = un_char;
un_char = (unsigned char)s[0];
switch (s[0]) {
case '@':
case '&':
case '$':
weight -= seen[un_char] * 10;
/* A word char following one of these: perl would consult the symbol
* table for a -100; we lack it, so always take the -10 path (gap). */
if (s + 1 < e && tsp_im_isword((unsigned char)s[1])) {
weight -= 10;
}
else if (s[0] == '$' && s + 1 < e && s[1] &&
tsp_im_in("[#!%*<>()-=", (unsigned char)s[1])) {
/* Possible punctuation variable. */
char s2 = (s + 2 < e) ? s[2] : 0;
if (tsp_im_in("])} =", (unsigned char)s2)) weight -= 10;
else weight -= 1;
}
break;
case '\\':
if (s + 1 < e && s[1]) {
if (tsp_im_in("wds]", (unsigned char)s[1])) weight += 100;
else if (seen[(unsigned char)'\''] || seen[(unsigned char)'"']) weight += 1;
else if (tsp_im_in("abcfnrtvx", (unsigned char)s[1])) weight += 40;
else if (tsp_im_isdigit((unsigned char)s[1])) {
weight += 40;
while (s + 1 < send && tsp_im_isdigit((unsigned char)s[1])) s++;
}
}
else weight += 100; /* '\' at end strongly char class */
break;
case '-':
if (s + 1 < e && s[1] == '\\') weight += 50;
if (!first_time && tsp_im_in("aA01! ", prev_un_char)) weight += 30;
if (s + 1 < e && tsp_im_in("zZ79~", (unsigned char)s[1])) weight += 30;
if (first_time && s + 1 < e &&
(tsp_im_isdigit((unsigned char)s[1]) || s[1] == '$')) {
weight -= 5; /* negative subscript */
}
break;
default:
if ((first_time || (!tsp_im_isword(prev_un_char)
&& prev_un_char != '$'
&& prev_un_char != '@'
&& prev_un_char != '&'))
&& tsp_im_isalpha((unsigned char)s[0])
&& s + 1 < send && tsp_im_isalpha((unsigned char)s[1])) {
/* A run of >= 2 alphas at a boundary: if it spells a keyword it's
* almost certainly not a character class. (Mirrors perl's pointer
* walk: s lands on the first non-alpha, which the loop's s++ then
* skips -- intentional, matching toke.c.) */
const char *d = s;
while (s < send && tsp_im_isalpha((unsigned char)s[0])) s++;
if (tsp_is_perl_keyword(d, (int)(s - d))) weight -= 150;
}
/* Ascending runs like ...12... or ...ab... lean char class. */
if (!first_time && un_char == (unsigned char)(prev_un_char + 1)) {
weight += 5;
}
/* Repeats lean subscript (nobody writes [aba] as a class). */
weight -= seen[un_char];
break;
}
seen[un_char]++;
}
if (weight >= 0) return false; /* probably a character class */
return true; /* subscript */
}
#endif /* TSP_INTUIT_MORE_H */
@@ -0,0 +1,207 @@
/* tsp_intuit_readline.h
*
* Disambiguates `<...>` after an ambiguous bareword: fileglob/readline
* (`print <*.c>`) vs the relational `<` operator (`CONST < 7 > $x`). Real
* perl decides this by PL_expect (was a value or a list-op in front of `<`?),
* which the scanner can't see, so we hybridise two heuristics whose blind
* spots are disjoint:
*
* (a) glob-shape: a real target looks like a glob pattern (metachar `*?[]{}~`,
* path `/`, filename-dot `foo.c`) or a pure filehandle name; content that
* is neither (a number, a spaced scalar, an operand) is relational. This
* catches `CONST < $a > foo`, which (b) alone misreads.
* (b) orphan lookahead: a glob is a complete term, so a bare term right after
* its `>` would be orphaned -> `<` was the operator. This catches
* `CONST < $a/$b > $c` (glob-shaped content, but `$c` orphans), which (a)
* alone misreads.
*
* tsp_is_fileglob(content, clen, after, alen): TRUE => emit fileglob; FALSE =>
* bail to the `<` operator. See the decision body below. Best-effort and
* ASCII-only (an LSP with parser state can refine it); never reads past either
* buffer. `after` is a capped window; if it holds only whitespace we treat it
* as EOF and allow the glob.
*/
#ifndef TSP_INTUIT_READLINE_H
#define TSP_INTUIT_READLINE_H
#include <stdbool.h>
#include <stddef.h>
#include <string.h>
/* --- ASCII classification (mirrors the subset used elsewhere) -------------- */
static inline bool tsp_rl_isdigit(int c) { return c >= '0' && c <= '9'; }
static inline bool tsp_rl_isalpha(int c) {
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
}
static inline bool tsp_rl_isword(int c) {
return tsp_rl_isalpha(c) || tsp_rl_isdigit(c) || c == '_';
}
static inline bool tsp_rl_isspace(int c) {
return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' ||
c == '\v';
}
/* The low-precedence operator / statement-modifier keywords that may legally
* FOLLOW a complete `bareword <glob>` term. Seeing one of these after the `>`
* means the glob is complete, not orphaning a term. */
static inline bool tsp_rl_is_trailing_keyword(const char *w, size_t n) {
static const char *const kw[] = {
"if", "unless", "while", "until", "for", "foreach", "and", "or", "xor",
"not", "cmp", "eq", "ne", "lt", "gt", "le", "ge", "x"};
for (size_t i = 0; i < sizeof(kw) / sizeof(kw[0]); i++) {
if (strlen(kw[i]) == n && memcmp(w, kw[i], n) == 0) return true;
}
return false;
}
/* The infix word-operators that, when appearing INSIDE the `<...>` content as a
* whitespace-delimited word, prove it is a relational expression rather than a
* glob pattern (a glob never contains a bare perl operator word). */
static inline bool tsp_rl_is_infix_word(const char *w, size_t n) {
static const char *const kw[] = {"and", "or", "xor", "not", "cmp", "eq",
"ne", "lt", "gt", "le", "ge", "x"};
for (size_t i = 0; i < sizeof(kw) / sizeof(kw[0]); i++) {
if (strlen(kw[i]) == n && memcmp(w, kw[i], n) == 0) return true;
}
return false;
}
/* Scan the `<...>` content for a whitespace-delimited infix operator word.
* Only alpha runs that start at a whitespace boundary (and are not glued to
* sigils/digits/punctuation) count; `$x` or `*.c` never qualify. */
static inline bool tsp_rl_content_has_infix_op(const char *s, size_t len) {
size_t i = 0;
while (i < len) {
/* skip leading whitespace */
while (i < len && tsp_rl_isspace((unsigned char)s[i])) i++;
size_t start = i;
bool clean = true; /* a clean bare word: only alphas */
while (i < len && !tsp_rl_isspace((unsigned char)s[i])) {
if (!tsp_rl_isalpha((unsigned char)s[i])) clean = false;
i++;
}
if (clean && i > start && tsp_rl_is_infix_word(s + start, i - start))
return true;
}
return false;
}
/* The glob metacharacters whose presence in the content is decisive evidence
* of a glob pattern (as opposed to a relational operand). */
static inline bool tsp_rl_is_glob_meta(int c) {
return c == '*' || c == '?' || c == '[' || c == ']' || c == '{' || c == '}' ||
c == '~';
}
/* Does the content POSITIVELY look like a glob pattern? True if it contains a
* glob metacharacter, a path separator `/`, or a filename-looking `.` (a dot
* flanked by word characters, e.g. `foo.c`). A bare number (`7`), a lone
* scalar (`$a`), or an operator expression has none of these. */
static inline bool tsp_rl_content_is_globshaped(const char *s, size_t len) {
for (size_t i = 0; i < len; i++) {
unsigned char c = (unsigned char)s[i];
if (tsp_rl_is_glob_meta(c) || c == '/') return true;
if (c == '.') {
bool prev_word = (i > 0) && tsp_rl_isword((unsigned char)s[i - 1]);
bool next_word = (i + 1 < len) && tsp_rl_isword((unsigned char)s[i + 1]);
if (prev_word && next_word) return true;
}
}
return false;
}
/* Is the content a pure filehandle / scalar NAME (no leading whitespace): an
* optional leading `$`, then word characters with `::` or `'` package
* separators (e.g. `$fh`, `STDIN`, `Foo::BAR`), tolerating only trailing
* whitespace? Such content is a readline target, not a relational operand.
* The leading-vs-trailing whitespace asymmetry is load-bearing: a glob target
* is written tight against `<` (`<$sner >`), whereas a relational `< EXPR`
* carries a separating space (`< $a > $b`). */
static inline bool tsp_rl_is_filehandle(const char *buf, size_t len) {
size_t i = 0;
if (i >= len || tsp_rl_isspace((unsigned char)buf[i])) return false;
if (buf[i] == '$') i++;
if (i >= len) return false; /* bare `$`: not a name */
bool last_was_word = false;
while (i < len) {
if (tsp_rl_isword((unsigned char)buf[i])) {
last_was_word = true;
i++;
} else if (buf[i] == ':') {
if (!last_was_word) return false;
if (i + 1 >= len || buf[i + 1] != ':') return false;
last_was_word = false;
i += 2;
} else if (buf[i] == '\'') {
if (!last_was_word) return false;
last_was_word = false;
i++;
} else if (tsp_rl_isspace((unsigned char)buf[i])) {
if (!last_was_word) return false;
for (i++; i < len; i++)
if (!tsp_rl_isspace((unsigned char)buf[i])) return false;
return true;
} else {
return false;
}
}
return last_was_word;
}
/* --- the decision ---------------------------------------------------------- */
static inline bool tsp_is_fileglob(const char *content, size_t clen,
const char *after, size_t alen) {
/* The empty diamond `<>` (rare here; usually handled before us). */
if (clen == 0) return true;
/* (1) operator-word inside the content => relational expression, not glob. */
if (tsp_rl_content_has_infix_op(content, clen)) return false;
/* (2) GLOB-SHAPE GATE. A real glob/readline target positively looks like a
* glob pattern (metachar / path / filename-dot) or a pure filehandle name.
* Content that is neither -- a bare number (`< 7 >`), a lone scalar with a
* separating space (`< $a > $b`), an arithmetic sub-expression -- is a
* relational operand, so bail. This is what keeps `CONST < $a > foo` and
* `CONST < 7 > $x` relational without any lookahead. */
if (!tsp_rl_content_is_globshaped(content, clen) &&
!tsp_rl_is_filehandle(content, clen))
return false;
/* (3) post-`>` orphan lookahead. The content looks glob-shaped, but a real
* glob is a COMPLETE term: if a bare term follows the `>` with no connecting
* operator it would be orphaned, which means `<` was the relational operator
* after all (e.g. `CONST < $a/$b > $c` -- glob-shaped content `$a/$b`, yet
* the trailing `$c` orphans). */
size_t i = 0;
while (i < alen && tsp_rl_isspace((unsigned char)after[i])) i++;
/* Window exhausted before any non-space byte: treat as EOF-like -> glob ok. */
if (i >= alen) return true;
unsigned char nc = (unsigned char)after[i];
/* An alphabetic word: keyword => complete term (glob ok); otherwise an
* ambiguous bareword, which we lean toward NOT treating as an orphan. */
if (tsp_rl_isalpha(nc)) {
size_t start = i;
while (i < alen && tsp_rl_isword((unsigned char)after[i])) i++;
if (tsp_rl_is_trailing_keyword(after + start, i - start)) return true;
/* Unknown bareword after the glob: lean toward glob (do not orphan). */
return true;
}
/* Bare-term openers with no connecting operator => orphan => bail. */
if (nc == '$' || nc == '@' || nc == '%' || nc == '"' || nc == '\'' ||
tsp_rl_isdigit(nc) || nc == '(' || nc == '[' || nc == '{') {
return false;
}
/* Everything else — operators (`+ - * / . < > ! ~ ? : | & ^`), `=`, `=>`,
* comma, `;`, closing brackets `) ] }` — terminates the term cleanly, so the
* glob is complete. */
return true;
}
#endif /* TSP_INTUIT_READLINE_H */
@@ -0,0 +1,33 @@
/* THIS FILE IS GENERATED BY generate_keywords.pl
*
* Statement keywords recognized by peek_is_statement_keyword().
* To add a keyword, edit the @keywords list in generate_keywords.pl
* and re-run: perl generate_keywords.pl
*
* Current keywords: class, method, no, package, role, sub, use
*/
// First-char filter: returns PEEK_NO_MATCH immediately if the
// lookahead can't start any statement keyword.
#define KEYWORD_FIRST_CHAR_FILTER(la) \
(!(la == 'c' || la == 'm' || la == 'n' || la == 'p' || la == 'r' || la == 's' || la == 'u'))
// Word-reading loop: characters that appear in any keyword.
#define KEYWORD_WORD_CHAR(la) \
(la == 'a' || la == 'b' || la == 'c' || la == 'd' || la == 'e' || la == 'g' || la == 'h' || la == 'k' || la == 'l' || la == 'm' || la == 'n' || la == 'o' || la == 'p' || la == 'r' || la == 's' || la == 't' || la == 'u')
// Keyword classification: sets `kind` to one of the KwKind values.
// KW_NONE — not a statement keyword (a plain bareword / autoquote key)
// KW_ALWAYS — always a statement keyword (package, use, no, class, role)
// KW_NEEDS_NAME — only a declaration if followed by a name (sub, method)
// Unlike a bare match this never returns: the caller decides what to do with a
// non-keyword, which may still be a fat-comma autoquote key.
#define KEYWORD_MATCH(word, kind) do { \
if (strcmp(word, "package") == 0 || strcmp(word, "use") == 0 || strcmp(word, "no") == 0 || strcmp(word, "class") == 0 || strcmp(word, "role") == 0) { \
(kind) = KW_ALWAYS; \
} else if (strcmp(word, "sub") == 0 || strcmp(word, "method") == 0) { \
(kind) = KW_NEEDS_NAME; \
} else { \
(kind) = KW_NONE; \
} \
} while(0)
@@ -541,8 +541,8 @@ static const struct TSPRange tsp_id_start[] = {
{ 201552, 205744 }
};
bool is_tsp_id_start (int32_t codepoint) {
return bsearch(&codepoint, tsp_id_start, sizeof(tsp_id_start) / sizeof(tsp_id_start[0]), sizeof(tsp_id_start[0]), tsprange_contains);
static bool is_tsp_id_start (int32_t codepoint) {
return tsp_bsearch(&codepoint, tsp_id_start, sizeof(tsp_id_start) / sizeof(tsp_id_start[0]), sizeof(tsp_id_start[0]), tsprange_contains);
}
static const struct TSPRange tsp_id_continue[] = {
@@ -1205,8 +1205,8 @@ static const struct TSPRange tsp_id_continue[] = {
{ 917760, 918000 }
};
bool is_tsp_id_continue (int32_t codepoint) {
return bsearch(&codepoint, tsp_id_continue, sizeof(tsp_id_continue) / sizeof(tsp_id_continue[0]), sizeof(tsp_id_continue[0]), tsprange_contains);
static bool is_tsp_id_continue (int32_t codepoint) {
return tsp_bsearch(&codepoint, tsp_id_continue, sizeof(tsp_id_continue) / sizeof(tsp_id_continue[0]), sizeof(tsp_id_continue[0]), tsprange_contains);
}
static const struct TSPRange tsp_whitespace[] = {
@@ -1221,6 +1221,6 @@ static const struct TSPRange tsp_whitespace[] = {
{ 8287, 8288 },
{ 12288, 12289 }
};
bool is_tsp_whitespace (int32_t codepoint) {
return bsearch(&codepoint, tsp_whitespace, sizeof(tsp_whitespace) / sizeof(tsp_whitespace[0]), sizeof(tsp_whitespace[0]), tsprange_contains);
static bool is_tsp_whitespace (int32_t codepoint) {
return tsp_bsearch(&codepoint, tsp_whitespace, sizeof(tsp_whitespace) / sizeof(tsp_whitespace[0]), sizeof(tsp_whitespace[0]), tsprange_contains);
}
+11 -7
View File
@@ -5,7 +5,7 @@ c5cfb43042b6b72045f4ba997834d0a7786d2793d91680868b5815b39f14fc78 internal/cbm/v
b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea internal/cbm/vendored/common/tree_sitter/alloc.h
31e60a1bff6f715afacce03b5b70efe42b58371b4f9595dd4af52a577ff9608c internal/cbm/vendored/common/tree_sitter/array.h
180b893c8734778fd32f372dfbc27bd6ad1cd2221f26150b31256ff6716320d2 internal/cbm/vendored/common/tree_sitter/parser.h
73be629708008f0cb31e0f8d41dc71ae901626baba191c24efd41bf52220c913 internal/cbm/vendored/grammars/MANIFEST.md
e1233d81f21d868b8f2d9e8fca10e15f2ec3ae4db4a7ba0301161cc011deef65 internal/cbm/vendored/grammars/MANIFEST.md
ad8425038de519f8c4e3e9339feebf99dfad8a6002dcf79227d91402780a32cc internal/cbm/vendored/grammars/ada/LICENSE
02805ec13939b749c891567be36bf024b09034e04c80683a1ae667272458d549 internal/cbm/vendored/grammars/ada/parser.c
115a75d000bef9c70c4de7dfbb7f2a80b90cbb14ba265056dc7abb1ef3b9b6db internal/cbm/vendored/grammars/ada/tree_sitter/alloc.h
@@ -553,13 +553,17 @@ fafd2e286865aacc132e938c1640f51619ec9e7866cd2d8debff39540336c14a internal/cbm/v
6db3411bfeff9fea8e64a1436f13b4c3558c52df0c19ea038a6a22fecae1b6eb internal/cbm/vendored/grammars/pascal/parser.c
2db740ea1be6b71014d2be1385491f01bf2d15b607e61c5b30b4128535efe68a internal/cbm/vendored/grammars/pascal/tree_sitter/parser.h
68a9a526ae357ed5a2f8ca5dabb14131b283554b69639bb04816088d4b1f2fa0 internal/cbm/vendored/grammars/perl/LICENSE
cb08206e89750c1fab700b89fc9876afb5cc689827e514ef49a5569c54635b61 internal/cbm/vendored/grammars/perl/bsearch.h
25c014ab8ce027992a6cfd76ef7b88bf3a41f63fd6ad43f38a553e218a7a9ea8 internal/cbm/vendored/grammars/perl/parser.c
fe4dc0394501b3d6211dcc1e31e9731befae844fcd6db0130e3c16f992d85860 internal/cbm/vendored/grammars/perl/scanner.c
6f953a91955c72b1b6b063c34fa877c8b36f5b43c320083ad7361256e67b9484 internal/cbm/vendored/grammars/perl/bsearch.h
ab3f9bf80dda8a4b919ffc8b556bd6e6e44995974e71acf4f4c0d6ebf5653625 internal/cbm/vendored/grammars/perl/parser.c
117d53b68474bf11e0b207d3e63e359b05be6f98d91d2c31b6abcec3a960eb1b internal/cbm/vendored/grammars/perl/scanner.c
b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea internal/cbm/vendored/grammars/perl/tree_sitter/alloc.h
5bdf6ed1a78e3409fd443e085ca967a64c188a5d082aaf7f819bccd53a471c94 internal/cbm/vendored/grammars/perl/tree_sitter/array.h
a1f6ef161fbaf48a0e10fca90ef5290a062462b307b3898aa562993853b9f80a internal/cbm/vendored/grammars/perl/tree_sitter/parser.h
9cbc0731f8c9bd52bd3de9644fd887f20cecea2d17634b20769db4940eadb566 internal/cbm/vendored/grammars/perl/tsp_unicode.h
31e60a1bff6f715afacce03b5b70efe42b58371b4f9595dd4af52a577ff9608c internal/cbm/vendored/grammars/perl/tree_sitter/array.h
180b893c8734778fd32f372dfbc27bd6ad1cd2221f26150b31256ff6716320d2 internal/cbm/vendored/grammars/perl/tree_sitter/parser.h
e65b7139547077b59ad62cb109efcf73d420311706c9dc7b6f9831cb246f2c61 internal/cbm/vendored/grammars/perl/tsp_intuit_keywords.h
45ada959185bb4b15f4fe702d52f0aeb5cd992b64cb129248aa843644e6026a2 internal/cbm/vendored/grammars/perl/tsp_intuit_more.h
9504b2ef7cce98bd3b22e2226ca05c1b63fd17b56ed3375bf72c4e429af42180 internal/cbm/vendored/grammars/perl/tsp_intuit_readline.h
2c86ede8eedbaf7854c2f9eef4b835ae55e563d7a85c05e47d8bfee502e5a78f internal/cbm/vendored/grammars/perl/tsp_keywords.h
1c81d6e72511578b6ebcafc3e9002427207f5167a7d916b92e34b5f1c36fa01c internal/cbm/vendored/grammars/perl/tsp_unicode.h
664764fabf40c0b318e06324a5a12379092a2df25b8cbf2415f71bc0bb352346 internal/cbm/vendored/grammars/php/LICENSE
de8eb36bc8f517ab9f3eaf82e3825d7b3e11b62e9471545f4c906100cfce0e07 internal/cbm/vendored/grammars/php/_common_scanner.h
bc796948b625f74e390cfd1254392136c86fafbd2dc0c9d93360ec20bc6b0dfb internal/cbm/vendored/grammars/php/parser.c
+11 -1
View File
@@ -248,9 +248,19 @@ cbm_mem_budget_t cbm_mem_resolve_budget(size_t total_ram, double ram_fraction,
cbm_mem_budget_t cbm_mem_resolve_budget_capped(size_t total_ram, double ram_fraction,
const char *budget_mb, size_t hard_cap_bytes) {
cbm_mem_budget_t result = cbm_mem_resolve_budget(total_ram, ram_fraction, budget_mb);
/* The parent already divided the aggregate budget (env override or
* ram_fraction) across job slots. That per-slot share is the hard cap:
* N workers × a per-worker absolute override would oversubscribe the host
* (#1654). A lower explicit value still wins. Keep CBM_MEM_BUDGET_MB as
* the source when the env discriminator fired so the ceiling is visible
* as the user's aggregate, not as a silent daemon_worker_cap rewrite. */
if (hard_cap_bytes > 0 && (result.budget == 0 || result.budget > hard_cap_bytes)) {
bool explicit_override =
result.source != NULL && strcmp(result.source, "CBM_MEM_BUDGET_MB") == 0;
result.budget = hard_cap_bytes;
result.source = "daemon_worker_cap";
if (!explicit_override) {
result.source = "daemon_worker_cap";
}
result.hard_capped = true;
}
return result;
+1 -1
View File
@@ -31,7 +31,7 @@ void cbm_mem_init_with_cap(double ram_fraction, size_t hard_cap_bytes);
* the caller never re-parses the env string. */
typedef struct {
size_t budget; /* resolved budget in bytes */
const char *source; /* log token: "ram_fraction" | "CBM_MEM_BUDGET_MB" */
const char *source; /* "ram_fraction" | "CBM_MEM_BUDGET_MB" | "daemon_worker_cap" */
bool clamped; /* override was valid but exceeded total_ram → clamped down */
bool invalid; /* override was present but unparseable / out-of-range / ≤0 */
bool hard_capped; /* internal worker hard cap reduced the resolved budget */
+250 -5
View File
@@ -130,6 +130,7 @@ enum {
#define MCP_MAX_HEADER_SIZE ((size_t)8U * 1024U)
#define MCP_SEARCH_OUTPUT_MAX ((size_t)64U * 1024U * 1024U)
#define MCP_SEARCH_SCAN_TIMEOUT_MS ((uint64_t)30000U)
#define MCP_FILE_OUTLINE_OUTPUT_MAX ((size_t)2U * 1024U * 1024U)
/* ── Helpers ────────────────────────────────────────────────────── */
@@ -557,6 +558,21 @@ static const tool_def_t TOOLS[] = {
"\"type\":\"string\"},\"include_neighbors\":{"
"\"type\":\"boolean\",\"default\":false}},\"required\":[\"qualified_name\",\"project\"]}"},
{"get_file_outline", "Get file outline",
"Return a compact declaration outline for one exact repository-relative file. Results are "
"filtered by optional exact labels, ordered deterministically by source position, and "
"bounded with exact total/offset/limit pagination metadata. File/folder/container nodes "
"are excluded. The query observes request cancellation and fails without partial output.",
"{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"},"
"\"file_path\":{\"type\":\"string\",\"description\":\"Exact repository-relative "
"file path\"},\"labels\":{\"type\":\"array\",\"items\":{\"type\":\"string\"},"
"\"maxItems\":16,\"description\":\"Optional exact node-label filter\"},"
"\"limit\":{\"type\":\"integer\",\"minimum\":1,\"maximum\":200,\"default\":100},"
"\"offset\":{\"type\":\"integer\",\"minimum\":0,\"default\":0},"
"\"format\":{\"type\":\"string\",\"enum\":[\"tree\",\"json\"],"
"\"default\":\"tree\"}},\"additionalProperties\":false,"
"\"required\":[\"project\",\"file_path\"]}"},
{"get_graph_schema", "Get graph schema",
"Get the schema of the knowledge graph (node labels, edge types)",
"{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"}},\"required\":["
@@ -745,6 +761,7 @@ static const tool_annotation_def_t TOOL_ANNOTATIONS[] = {
{"query_graph", false, true, true, false},
{"trace_path", false, true, true, false},
{"get_code_snippet", false, true, true, false},
{"get_file_outline", false, true, true, false},
{"get_graph_schema", false, true, true, false},
{"compare_graphs", true, false, true, false},
{"get_architecture", false, true, true, false},
@@ -808,13 +825,14 @@ static void mcp_add_tool_def(yyjson_mut_doc *doc, yyjson_mut_val *tools, int i)
static bool mcp_tool_allowed(cbm_mcp_tool_profile_t profile, const char *name) {
static const char *const analysis_tools[] = {
"search_graph", "query_graph", "trace_path", "get_code_snippet",
"get_graph_schema", "compare_graphs", "get_architecture", "search_code",
"list_projects", "index_status", "check_index_coverage", "detect_changes",
"search_graph", "query_graph", "trace_path", "get_code_snippet",
"get_file_outline", "get_graph_schema", "compare_graphs", "get_architecture",
"search_code", "list_projects", "index_status", "check_index_coverage",
"detect_changes",
};
static const char *const scout_tools[] = {
"search_graph", "trace_path", "get_code_snippet", "get_architecture",
"list_projects", "index_status", "check_index_coverage",
"search_graph", "trace_path", "get_code_snippet", "get_file_outline",
"get_architecture", "list_projects", "index_status", "check_index_coverage",
};
if (!name) {
return false;
@@ -9303,6 +9321,230 @@ static char *build_snippet_response(cbm_mcp_server_t *srv, cbm_node_t *node,
return result;
}
static bool file_outline_request_cancelled(void *context) {
return mcp_request_cancelled((const cbm_mcp_server_t *)context);
}
static char *handle_get_file_outline(cbm_mcp_server_t *srv, const char *args) {
const char *args_text = args ? args : "{}";
yyjson_doc *args_doc = yyjson_read(args_text, strlen(args_text), 0);
yyjson_val *root = args_doc ? yyjson_doc_get_root(args_doc) : NULL;
if (!root || !yyjson_is_obj(root)) {
yyjson_doc_free(args_doc);
return cbm_mcp_text_result("get_file_outline arguments must be a JSON object", true);
}
char *project = get_project_arg(args_text);
yyjson_val *file_value = yyjson_obj_get(root, "file_path");
const char *file_path =
file_value && yyjson_is_str(file_value) ? yyjson_get_str(file_value) : NULL;
if (!project) {
yyjson_doc_free(args_doc);
return cbm_mcp_text_result("project is required", true);
}
if (!file_path || !file_path[0]) {
free(project);
yyjson_doc_free(args_doc);
return cbm_mcp_text_result("file_path is required", true);
}
char normalized_path[CBM_SZ_4K];
if (coverage_normalize_rel(file_path, false, normalized_path, sizeof(normalized_path)) !=
COVERAGE_PATH_OK) {
free(project);
yyjson_doc_free(args_doc);
return cbm_mcp_text_result("file_path must be a repository-relative path without '..'",
true);
}
int limit = 100;
yyjson_val *limit_value = yyjson_obj_get(root, "limit");
if (limit_value) {
if (!yyjson_is_int(limit_value)) {
free(project);
yyjson_doc_free(args_doc);
return cbm_mcp_text_result("limit must be an integer", true);
}
int64_t parsed = yyjson_get_sint(limit_value);
if (parsed < 1 || parsed > CBM_STORE_FILE_OUTLINE_MAX_LIMIT) {
free(project);
yyjson_doc_free(args_doc);
return cbm_mcp_text_result("limit must be between 1 and 200", true);
}
limit = (int)parsed;
}
int offset = 0;
yyjson_val *offset_value = yyjson_obj_get(root, "offset");
if (offset_value) {
if (!yyjson_is_int(offset_value)) {
free(project);
yyjson_doc_free(args_doc);
return cbm_mcp_text_result("offset must be a non-negative integer", true);
}
int64_t parsed = yyjson_get_sint(offset_value);
if (parsed < 0 || parsed > INT_MAX) {
free(project);
yyjson_doc_free(args_doc);
return cbm_mcp_text_result("offset must be a non-negative integer", true);
}
offset = (int)parsed;
}
bool json_format = false;
yyjson_val *format_value = yyjson_obj_get(root, "format");
if (format_value) {
const char *format = yyjson_is_str(format_value) ? yyjson_get_str(format_value) : NULL;
if (!format || (strcmp(format, "tree") != 0 && strcmp(format, "json") != 0)) {
free(project);
yyjson_doc_free(args_doc);
return cbm_mcp_text_result("format must be either 'tree' or 'json'", true);
}
json_format = strcmp(format, "json") == 0;
}
const char *labels[CBM_STORE_FILE_OUTLINE_MAX_LABELS];
int label_count = 0;
yyjson_val *labels_value = yyjson_obj_get(root, "labels");
if (labels_value) {
if (!yyjson_is_arr(labels_value) ||
yyjson_arr_size(labels_value) > CBM_STORE_FILE_OUTLINE_MAX_LABELS) {
free(project);
yyjson_doc_free(args_doc);
return cbm_mcp_text_result("labels must be an array of at most 16 strings", true);
}
size_t index = 0;
size_t max = 0;
yyjson_val *item;
yyjson_arr_foreach(labels_value, index, max, item) {
const char *label = yyjson_is_str(item) ? yyjson_get_str(item) : NULL;
if (!label || !label[0] || strlen(label) >= CBM_SZ_128) {
free(project);
yyjson_doc_free(args_doc);
return cbm_mcp_text_result("labels must contain only non-empty bounded strings",
true);
}
labels[label_count++] = label;
}
}
cbm_store_t *store = resolve_store(srv, project);
if (!store) {
char *error = build_project_list_error("project not found or not indexed");
char *result = cbm_mcp_text_result(error, true);
free(error);
free(project);
yyjson_doc_free(args_doc);
return result;
}
char *not_indexed = verify_project_indexed(store, project);
if (not_indexed) {
free(project);
yyjson_doc_free(args_doc);
return not_indexed;
}
cbm_file_outline_row_t *rows = NULL;
int row_count = 0;
int total = 0;
int rc = cbm_store_get_file_outline(store, project, normalized_path, labels, label_count, limit,
offset, file_outline_request_cancelled, srv, &rows,
&row_count, &total);
if (rc != CBM_STORE_OK) {
free(project);
yyjson_doc_free(args_doc);
if (rc == CBM_STORE_CANCELLED) {
return cbm_mcp_text_result("get_file_outline cancelled for this request", true);
}
if (rc == CBM_STORE_SCAN_LIMIT) {
return cbm_mcp_text_result("get_file_outline exceeded its fixed output safety limit",
true);
}
return cbm_mcp_text_result("get_file_outline query failed", true);
}
char *payload = NULL;
if (!json_format) {
cbm_sb_t sb;
cbm_sb_init(&sb);
cbm_tree_scalar_str(&sb, "file_path", normalized_path);
static const char *const columns[] = {"name", "label", "lines", "qn"};
cbm_tree_table_header(&sb, "results", row_count, columns, 4);
for (int i = 0; i < row_count; i++) {
char lines[CBM_SZ_32];
if (rows[i].start_line > 0) {
snprintf(lines, sizeof(lines), "%d-%d", rows[i].start_line,
rows[i].end_line > rows[i].start_line ? rows[i].end_line
: rows[i].start_line);
} else {
lines[0] = '\0';
}
cbm_tree_row_begin(&sb);
cbm_tree_cell_str(&sb, rows[i].name, true);
cbm_tree_cell_str(&sb, rows[i].label, false);
cbm_tree_cell_str(&sb, lines, false);
cbm_tree_cell_str(&sb, rows[i].qualified_name, false);
cbm_tree_row_end(&sb);
}
cbm_tree_scalar_int(&sb, "total", total);
cbm_tree_scalar_int(&sb, "offset", offset);
cbm_tree_scalar_int(&sb, "limit", limit);
cbm_tree_scalar_int(&sb, "returned", row_count);
cbm_tree_scalar_bool(&sb, "has_more", (int64_t)offset + row_count < total);
payload = cbm_sb_finish(&sb);
} else {
yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
yyjson_mut_val *object = yyjson_mut_obj(doc);
yyjson_mut_doc_set_root(doc, object);
yyjson_mut_obj_add_str(doc, object, "file_path", normalized_path);
yyjson_mut_val *columns = yyjson_mut_arr(doc);
static const char *const column_names[] = {"name", "label", "lines", "qn"};
for (size_t i = 0; i < sizeof(column_names) / sizeof(column_names[0]); i++) {
yyjson_mut_arr_add_str(doc, columns, column_names[i]);
}
yyjson_mut_obj_add_val(doc, object, "cols", columns);
yyjson_mut_val *json_rows = yyjson_mut_arr(doc);
for (int i = 0; i < row_count; i++) {
char lines[CBM_SZ_32];
if (rows[i].start_line > 0) {
snprintf(lines, sizeof(lines), "%d-%d", rows[i].start_line,
rows[i].end_line > rows[i].start_line ? rows[i].end_line
: rows[i].start_line);
} else {
lines[0] = '\0';
}
yyjson_mut_val *row = yyjson_mut_arr(doc);
yyjson_mut_arr_add_strcpy(doc, row, rows[i].name);
yyjson_mut_arr_add_strcpy(doc, row, rows[i].label);
yyjson_mut_arr_add_strcpy(doc, row, lines);
yyjson_mut_arr_add_strcpy(doc, row, rows[i].qualified_name);
yyjson_mut_arr_add_val(json_rows, row);
}
yyjson_mut_obj_add_val(doc, object, "rows", json_rows);
yyjson_mut_obj_add_int(doc, object, "total", total);
yyjson_mut_obj_add_int(doc, object, "offset", offset);
yyjson_mut_obj_add_int(doc, object, "limit", limit);
yyjson_mut_obj_add_int(doc, object, "returned", row_count);
yyjson_mut_obj_add_bool(doc, object, "has_more", (int64_t)offset + row_count < total);
payload = yy_doc_to_str(doc);
yyjson_mut_doc_free(doc);
}
cbm_store_free_file_outline(rows, row_count);
free(project);
yyjson_doc_free(args_doc);
if (!payload) {
return cbm_mcp_text_result("get_file_outline output allocation failed", true);
}
if (strlen(payload) > MCP_FILE_OUTLINE_OUTPUT_MAX) {
free(payload);
return cbm_mcp_text_result("get_file_outline exceeded its fixed output safety limit", true);
}
char *result = cbm_mcp_text_result(payload, false);
free(payload);
return result;
}
static char *handle_get_code_snippet(cbm_mcp_server_t *srv, const char *args) {
char *qn = cbm_mcp_get_string_arg(args, "qualified_name");
char *project = get_project_arg(args);
@@ -12361,6 +12603,9 @@ static char *dispatch_tool(cbm_mcp_server_t *srv, const char *tool_name, const c
if (strcmp(tool_name, "get_code_snippet") == 0) {
return handle_get_code_snippet(srv, args_json);
}
if (strcmp(tool_name, "get_file_outline") == 0) {
return handle_get_file_outline(srv, args_json);
}
if (strcmp(tool_name, "search_code") == 0) {
return handle_search_code(srv, args_json);
}
+80 -1
View File
@@ -413,6 +413,73 @@ static int pxc_build_lsp_def(CBMArena *arena, const CBMDefinition *src, const ch
return 0;
}
/* Go: fold per-field "Field" definitions into their owning struct's
* field_defs. extract_defs.c emits one flat CBMDefinition per struct field
* (label "Field", parent_class = owning struct QN, name = field name,
* return_type = raw type text). Those rows are dropped by pxc_build_lsp_def
* (pxc_map_label excludes "Field"), so without this fold every Go struct
* registers with zero fields and field-chain calls (h.svc.Handle) can
* never resolve. Fields are always declared in the same file as their struct,
* so scanning the file's own defs covers every case. Runs inside
* cbm_pxc_collect_all_defs — one site covers both the prebuilt-registry path
* and the per-file fallback, since both consume all_defs. */
static void pxc_fold_go_struct_fields(CBMArena *arena, const CBMFileResult *result, CBMLSPDef *defs,
int start, int end) {
if (!arena || !result || !defs || start >= end) {
return;
}
for (int si = start; si < end; si++) {
CBMLSPDef *dst = &defs[si];
if (!dst->label || strcmp(dst->label, "Struct") != 0 || !dst->qualified_name) {
continue;
}
int count = 0;
size_t total = 0; /* "name:type" bytes; separators and NUL added below */
for (int di = 0; di < result->defs.count; di++) {
const CBMDefinition *fd = &result->defs.items[di];
if (!fd->label || !fd->parent_class || !fd->name || !fd->name[0] || !fd->return_type ||
!fd->return_type[0] || strcmp(fd->label, "Field") != 0 ||
strcmp(fd->parent_class, dst->qualified_name) != 0) {
continue;
}
total += strlen(fd->name) + 1 + strlen(fd->return_type);
count++;
}
if (count == 0) {
continue;
}
/* count - 1 separators + NUL. */
size_t bufsz = total + (size_t)(count - 1) + 1;
char *buf = (char *)cbm_arena_alloc(arena, bufsz);
if (!buf) {
continue;
}
char *p = buf;
int written = 0;
for (int di = 0; di < result->defs.count; di++) {
const CBMDefinition *fd = &result->defs.items[di];
if (!fd->label || !fd->parent_class || !fd->name || !fd->name[0] || !fd->return_type ||
!fd->return_type[0] || strcmp(fd->label, "Field") != 0 ||
strcmp(fd->parent_class, dst->qualified_name) != 0) {
continue;
}
size_t n = strlen(fd->name);
memcpy(p, fd->name, n);
p += n;
*p++ = ':';
n = strlen(fd->return_type);
memcpy(p, fd->return_type, n);
p += n;
if (written + 1 < count) {
*p++ = '|';
}
written++;
}
*p = '\0';
dst->field_defs = buf;
}
}
/* Carry one Rust type-level impl independently of any method definition.
* `impl Trait for Type {}` is semantically meaningful even when the block is
* empty (the trait may provide defaults), so attaching the relation only to
@@ -472,6 +539,7 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMFileResult
if (out_def_starts) {
out_def_starts[fi] = idx;
}
const int file_start = idx;
if (!cache[fi])
continue;
if (!def_modules[fi]) {
@@ -510,6 +578,9 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMFileResult
}
}
cbm_pxc_free_import_map(imp_keys, imp_vals, imp_count); /* NULL-safe */
if (files[fi].language == CBM_LANG_GO) {
pxc_fold_go_struct_fields(&cache[fi]->arena, cache[fi], defs, file_start, idx);
}
if (files[fi].language == CBM_LANG_RUST) {
for (int ii = 0; ii < cache[fi]->impl_traits.count; ii++) {
if (pxc_build_rust_impl_relation(
@@ -1218,8 +1289,16 @@ void cbm_pxc_dispatch_file(CBMLanguage lang, CBMFileResult *result, const char *
switch (lang) {
case CBM_LANG_GO:
/* Tier 3 (metadata-driven): pure lookup over the Tier-1
* lsp_unresolved entries — no parse, no AST walk. */
* lsp_unresolved entries — no parse, no AST walk. Then the
* AST walk on the shared Tier-2 registry (mirroring every
* other language) so NAMED receivers evaluated against
* project-wide defs also resolve. The walk variant below is
* read-only — the sealed registry is safe for parallel
* workers. */
cbm_go_fast_resolve_qualified_calls(result, prebuilt, imp_keys, imp_vals, imp_count);
cbm_run_go_lsp_cross_with_registry(&result->arena, source, source_len, def_module,
prebuilt, imp_keys, imp_vals, imp_count,
result->cached_tree, &result->resolved_calls);
used_prebuilt = true;
break;
case CBM_LANG_PYTHON: {
+10
View File
@@ -1792,6 +1792,13 @@ static void detect_url_in_args(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source,
const CBMCall *call) {
for (int ai = 0; ai < call->arg_count; ai++) {
const CBMCallArg *ca = &call->args[ai];
/* A slash-prefixed raw expression is not a URL string. In JS/TS this
* is notably a regex literal (`/<table/i`); genuine string literals
* and propagated constants are carried in `value`, while template
* literals keep their leading backtick in `expr`. */
if (!ca->value && ca->expr && ca->expr[0] == '/') {
continue;
}
const char *url = ca->value ? ca->value : ca->expr;
if (!url || (url[0] != '/' && url[0] != '`')) {
continue;
@@ -1800,6 +1807,9 @@ static void detect_url_in_args(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source,
if (!normalize_url_arg(url, norm, (int)sizeof(norm))) {
continue;
}
if (!cbm_service_pattern_is_http_route_literal(norm, call->callee_name)) {
continue;
}
char route_qn[CBM_ROUTE_QN_SIZE];
char cpath[CBM_SZ_256];
snprintf(route_qn, sizeof(route_qn), "__route__ANY__%s",
+1 -2
View File
@@ -33,12 +33,11 @@ enum {
#include <stdint.h>
#include "graph_buffer/graph_buffer.h"
#include "foundation/log.h"
#include "service_patterns.h" /* cbm_service_pattern_is_http_route_literal */
#include <stdio.h>
#include <string.h>
bool cbm_service_pattern_is_http_route_literal(const char *literal, const char *callee_name);
/* True for characters that may appear in a ":name" route parameter. */
static inline bool is_route_ident_char(char c) {
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_';
+227
View File
@@ -2640,6 +2640,233 @@ int cbm_store_find_nodes_by_file(cbm_store_t *s, const char *project, const char
project, file_path, out, count);
}
typedef struct {
cbm_store_cancel_fn cancel;
void *context;
} file_outline_cancel_guard_t;
static bool file_outline_cancelled(const file_outline_cancel_guard_t *guard) {
return guard && guard->cancel && guard->cancel(guard->context);
}
static int file_outline_progress_cancel(void *context) {
return file_outline_cancelled((const file_outline_cancel_guard_t *)context) ? 1 : 0;
}
static bool file_outline_build_sql(char *sql, size_t sql_size, bool count_only, int label_count) {
int written =
snprintf(sql, sql_size,
count_only ? "SELECT COUNT(*) FROM nodes WHERE project = ?1 AND file_path = ?2 "
"AND label NOT IN ('Module','Package','File','Folder','Project')"
: "SELECT name, label, qualified_name, start_line, end_line FROM nodes "
"WHERE project = ?1 AND file_path = ?2 "
"AND label NOT IN ('Module','Package','File','Folder','Project')");
if (written < 0 || (size_t)written >= sql_size) {
return false;
}
size_t used = (size_t)written;
if (label_count > 0) {
written = snprintf(sql + used, sql_size - used, " AND label IN (");
if (written < 0 || (size_t)written >= sql_size - used) {
return false;
}
used += (size_t)written;
for (int i = 0; i < label_count; i++) {
written = snprintf(sql + used, sql_size - used, "%s?%d", i > 0 ? "," : "", i + 3);
if (written < 0 || (size_t)written >= sql_size - used) {
return false;
}
used += (size_t)written;
}
written = snprintf(sql + used, sql_size - used, ")");
if (written < 0 || (size_t)written >= sql_size - used) {
return false;
}
used += (size_t)written;
}
if (!count_only) {
int limit_bind = label_count + 3;
int offset_bind = limit_bind + 1;
written = snprintf(sql + used, sql_size - used,
" ORDER BY start_line, end_line, label COLLATE BINARY, "
"name COLLATE BINARY, qualified_name COLLATE BINARY, id "
"LIMIT ?%d OFFSET ?%d",
limit_bind, offset_bind);
if (written < 0 || (size_t)written >= sql_size - used) {
return false;
}
}
return true;
}
static void file_outline_bind_common(sqlite3_stmt *stmt, const char *project, const char *file_path,
const char *const *labels, int label_count) {
bind_text(stmt, ST_COL_1, project);
bind_text(stmt, ST_COL_2, file_path);
for (int i = 0; i < label_count; i++) {
bind_text(stmt, i + 3, labels[i]);
}
}
void cbm_store_free_file_outline(cbm_file_outline_row_t *rows, int count) {
if (!rows) {
return;
}
for (int i = 0; i < count; i++) {
free((char *)rows[i].name);
free((char *)rows[i].label);
free((char *)rows[i].qualified_name);
}
free(rows);
}
int cbm_store_get_file_outline(cbm_store_t *s, const char *project, const char *file_path,
const char *const *labels, int label_count, int limit, int offset,
cbm_store_cancel_fn cancel, void *cancel_context,
cbm_file_outline_row_t **out, int *count, int *total) {
if (out) {
*out = NULL;
}
if (count) {
*count = 0;
}
if (total) {
*total = 0;
}
if (!s || !s->db || !project || !project[0] || !file_path || !file_path[0] || !out || !count ||
!total || label_count < 0 || label_count > CBM_STORE_FILE_OUTLINE_MAX_LABELS ||
(label_count > 0 && !labels) || limit < 1 || limit > CBM_STORE_FILE_OUTLINE_MAX_LIMIT ||
offset < 0) {
return CBM_STORE_ERR;
}
for (int i = 0; i < label_count; i++) {
if (!labels[i] || !labels[i][0]) {
return CBM_STORE_ERR;
}
}
file_outline_cancel_guard_t guard = {.cancel = cancel, .context = cancel_context};
if (file_outline_cancelled(&guard)) {
return CBM_STORE_CANCELLED;
}
if (cancel) {
sqlite3_progress_handler(s->db, 1000, file_outline_progress_cancel, &guard);
}
char sql[ST_SQL_BUF];
if (!file_outline_build_sql(sql, sizeof(sql), true, label_count)) {
sqlite3_progress_handler(s->db, 0, NULL, NULL);
return CBM_STORE_ERR;
}
sqlite3_stmt *stmt = NULL;
int rc = sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL);
if (rc != SQLITE_OK) {
store_set_error_sqlite(s, "file outline count prepare");
sqlite3_progress_handler(s->db, 0, NULL, NULL);
return CBM_STORE_ERR;
}
file_outline_bind_common(stmt, project, file_path, labels, label_count);
rc = sqlite3_step(stmt);
if (rc == SQLITE_ROW) {
*total = sqlite3_column_int(stmt, 0);
}
sqlite3_finalize(stmt);
if (rc == SQLITE_INTERRUPT || file_outline_cancelled(&guard)) {
sqlite3_progress_handler(s->db, 0, NULL, NULL);
*total = 0;
return CBM_STORE_CANCELLED;
}
if (rc != SQLITE_ROW) {
store_set_error_sqlite(s, "file outline count");
sqlite3_progress_handler(s->db, 0, NULL, NULL);
*total = 0;
return CBM_STORE_ERR;
}
if (!file_outline_build_sql(sql, sizeof(sql), false, label_count)) {
sqlite3_progress_handler(s->db, 0, NULL, NULL);
*total = 0;
return CBM_STORE_ERR;
}
stmt = NULL;
rc = sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL);
if (rc != SQLITE_OK) {
store_set_error_sqlite(s, "file outline prepare");
sqlite3_progress_handler(s->db, 0, NULL, NULL);
*total = 0;
return CBM_STORE_ERR;
}
file_outline_bind_common(stmt, project, file_path, labels, label_count);
sqlite3_bind_int(stmt, label_count + 3, limit);
sqlite3_bind_int(stmt, label_count + 4, offset);
cbm_file_outline_row_t *rows = calloc((size_t)limit, sizeof(*rows));
if (!rows) {
sqlite3_finalize(stmt);
sqlite3_progress_handler(s->db, 0, NULL, NULL);
*total = 0;
return CBM_STORE_ERR;
}
size_t text_bytes = 0U;
int n = 0;
bool was_cancelled = false;
for (;;) {
if (file_outline_cancelled(&guard)) {
was_cancelled = true;
break;
}
rc = sqlite3_step(stmt);
if (rc != SQLITE_ROW) {
break;
}
const char *name = (const char *)sqlite3_column_text(stmt, 0);
const char *label = (const char *)sqlite3_column_text(stmt, 1);
const char *qn = (const char *)sqlite3_column_text(stmt, 2);
size_t row_bytes = (size_t)sqlite3_column_bytes(stmt, 0) +
(size_t)sqlite3_column_bytes(stmt, 1) +
(size_t)sqlite3_column_bytes(stmt, 2) + 3U;
if (row_bytes > CBM_STORE_FILE_OUTLINE_MAX_TEXT_BYTES - text_bytes) {
sqlite3_finalize(stmt);
sqlite3_progress_handler(s->db, 0, NULL, NULL);
cbm_store_free_file_outline(rows, n);
*total = 0;
return CBM_STORE_SCAN_LIMIT;
}
rows[n].name = heap_strdup(name ? name : "");
rows[n].label = heap_strdup(label ? label : "");
rows[n].qualified_name = heap_strdup(qn ? qn : "");
rows[n].start_line = sqlite3_column_int(stmt, 3);
rows[n].end_line = sqlite3_column_int(stmt, 4);
if (!rows[n].name || !rows[n].label || !rows[n].qualified_name) {
sqlite3_finalize(stmt);
sqlite3_progress_handler(s->db, 0, NULL, NULL);
cbm_store_free_file_outline(rows, n + 1);
*total = 0;
return CBM_STORE_ERR;
}
text_bytes += row_bytes;
n++;
}
was_cancelled = was_cancelled || rc == SQLITE_INTERRUPT || file_outline_cancelled(&guard);
sqlite3_finalize(stmt);
sqlite3_progress_handler(s->db, 0, NULL, NULL);
if (was_cancelled) {
cbm_store_free_file_outline(rows, n);
*total = 0;
return CBM_STORE_CANCELLED;
}
if (rc != SQLITE_DONE) {
store_set_error_sqlite(s, "file outline rows");
cbm_store_free_file_outline(rows, n);
*total = 0;
return CBM_STORE_ERR;
}
*out = rows;
*count = n;
return CBM_STORE_OK;
}
int cbm_store_count_nodes(cbm_store_t *s, const char *project) {
if (!s || !s->db) {
return 0;
+28
View File
@@ -28,6 +28,10 @@ typedef struct cbm_store cbm_store_t;
#define CBM_STORE_SCAN_LIMIT (-4)
#define CBM_STORE_CALLBACK_ERR (-5)
#define CBM_STORE_FILE_OUTLINE_MAX_LIMIT 200
#define CBM_STORE_FILE_OUTLINE_MAX_LABELS 16
#define CBM_STORE_FILE_OUTLINE_MAX_TEXT_BYTES (256U * 1024U)
/* ── Data structures ────────────────────────────────────────────── */
typedef struct {
@@ -42,6 +46,18 @@ typedef struct {
const char *properties_json; /* JSON string, NULL → "{}" */
} cbm_node_t;
/* Compact declaration row returned by the bounded file-outline query. */
typedef struct {
const char *name;
const char *label;
const char *qualified_name;
int start_line;
int end_line;
} cbm_file_outline_row_t;
/* Optional cancellation callback for bounded store queries. */
typedef bool (*cbm_store_cancel_fn)(void *context);
typedef struct {
int64_t id;
const char *project;
@@ -497,6 +513,18 @@ int cbm_store_find_nodes_by_label(cbm_store_t *s, const char *project, const cha
int cbm_store_find_nodes_by_file(cbm_store_t *s, const char *project, const char *file_path,
cbm_node_t **out, int *count);
/* Return a stable, paginated outline for one exact repository-relative file.
* File/folder/container nodes are excluded. labels may be NULL when
* label_count is zero; otherwise labels are exact-match filters. The query is
* capped by CBM_STORE_FILE_OUTLINE_MAX_LIMIT and a fixed aggregate text-byte
* budget, and fails without partial rows when cancelled or over budget.
* total is the exact filtered count before pagination. */
int cbm_store_get_file_outline(cbm_store_t *s, const char *project, const char *file_path,
const char *const *labels, int label_count, int limit, int offset,
cbm_store_cancel_fn cancel, void *cancel_context,
cbm_file_outline_row_t **out, int *count, int *total);
void cbm_store_free_file_outline(cbm_file_outline_row_t *rows, int count);
/* Batch lookup: map qualified names → node IDs.
* qns[i] is resolved; out_ids[i] receives the ID or 0 if not found.
* Returns number of QNs actually found, or CBM_STORE_ERR. */
+14 -3
View File
@@ -752,6 +752,12 @@ static const char TLAPLUS_BOUNDED_QUANTIFICATION[] =
"Guard(values) == \\A item \\in values : item = item\n"
"====\n";
/* Pkl access expressions double as plain property reads, so the bare fixture
* also proves the un-applied reference is not promoted to a call. */
static const char PKL_INSIDE[] = "function accept(value: Int): Int = value\n"
"function run(watched: Int): Int = accept(watched)\n";
static const char PKL_BARE[] = "function run(watched: Int): Int = watched\n";
static const char APEX_INSIDE[] = "public class Sample {\n"
" private static Integer accept(Integer value) {\n"
" return value;\n"
@@ -988,6 +994,9 @@ static const RoutineArgumentCase LLVM_IR_CASE = ROUTINE_ARGUMENT_CASE(
static const RoutineArgumentCase TLAPLUS_CASE = ROUTINE_ARGUMENT_CASE(
"TLAPLUS", CBM_LANG_TLAPLUS, "Sample.tla", TLAPLUS_INSIDE, TLAPLUS_BARE, "bound_op", "Guard",
"Accept", "values", 1, 1, 0, "TLA+ operator application with a value argument");
static const RoutineArgumentCase PKL_CASE = ROUTINE_ARGUMENT_CASE(
"PKL", CBM_LANG_PKL, "sample.pkl", PKL_INSIDE, PKL_BARE, "unqualifiedAccessExpr", "run",
"accept", "watched", 1, 1, 0, "native Pkl method application and property-read vocabulary");
static const RoutineArgumentCase APEX_CASE = ROUTINE_ARGUMENT_CASE(
"APEX", CBM_LANG_APEX, "Sample.cls", APEX_INSIDE, APEX_BARE, "method_invocation", "run",
"accept", "watched", 1, 1, 0, "native method application");
@@ -1203,6 +1212,7 @@ DEFINE_ROUTINE_ARGUMENT_TEST(func, FUNC_CASE)
DEFINE_ROUTINE_ARGUMENT_TEST(puppet, PUPPET_CASE)
DEFINE_ROUTINE_ARGUMENT_TEST(slang, SLANG_CASE)
DEFINE_ROUTINE_ARGUMENT_TEST(llvm_ir, LLVM_IR_CASE)
DEFINE_ROUTINE_ARGUMENT_TEST(pkl, PKL_CASE)
DEFINE_ROUTINE_ARGUMENT_TEST(apex, APEX_CASE)
DEFINE_ROUTINE_ARGUMENT_TEST(pine, PINE_CASE)
DEFINE_ROUTINE_ARGUMENT_TEST(qml, QML_CASE)
@@ -1282,15 +1292,15 @@ TEST(repro_call_argument_matrix_b_domain_bitbake) {
}
enum {
ROUTINE_ARGUMENT_LANGUAGE_COUNT = 38,
ROUTINE_ARGUMENT_LANGUAGE_COUNT = 39,
MODULE_ARGUMENT_LANGUAGE_COUNT = 4,
DOMAIN_CONTROL_LANGUAGE_COUNT = 6,
MATRIX_LANGUAGE_COUNT = ROUTINE_ARGUMENT_LANGUAGE_COUNT + MODULE_ARGUMENT_LANGUAGE_COUNT +
DOMAIN_CONTROL_LANGUAGE_COUNT,
};
_Static_assert(MATRIX_LANGUAGE_COUNT == 48,
"RACKET..CHIALISP call-capable matrix must contain exactly 48 "
_Static_assert(MATRIX_LANGUAGE_COUNT == 49,
"RACKET..CHIALISP call-capable matrix must contain exactly 49 "
"language rows");
#define MATRIX_B_LANGUAGE_ROWS(X) \
@@ -1322,6 +1332,7 @@ _Static_assert(MATRIX_LANGUAGE_COUNT == 48,
X(repro_call_argument_matrix_b_routine_slang, SLANG_CASE.identity.language) \
X(repro_call_argument_matrix_b_routine_llvm_ir, LLVM_IR_CASE.identity.language) \
X(repro_call_argument_matrix_b_routine_tlaplus, TLAPLUS_CASE.identity.language) \
X(repro_call_argument_matrix_b_routine_pkl, PKL_CASE.identity.language) \
X(repro_call_argument_matrix_b_routine_apex, APEX_CASE.identity.language) \
X(repro_call_argument_matrix_b_routine_pine, PINE_CASE.identity.language) \
X(repro_call_argument_matrix_b_routine_qml, QML_CASE.identity.language) \
+13 -5
View File
@@ -16,6 +16,11 @@
* TLA+ bound_op is historical direct-call metadata. Bounded quantification is
* not a historical ledger row; repro_call_argument_matrix_b is its separate
* negative behavior guard.
*
* The historical-snapshot column is the audited ledger, not an archaeology
* record: every registered call node owns a row, so a newly registered kind
* (Pkl's access/new expressions) joins the snapshot with the change that
* registers it.
*/
#include "test_framework.h"
#include "lang_specs.h"
@@ -54,10 +59,10 @@ typedef struct {
} CallNodeManifestEntry;
enum {
EXPECTED_HISTORICAL_CALL_NODE_TOTAL = 223,
EXPECTED_ACTIVE_PRIMARY_TOTAL = 193,
EXPECTED_DIRECT_CALLS = 158,
EXPECTED_CONSTRUCTOR_CALLS = 21,
EXPECTED_HISTORICAL_CALL_NODE_TOTAL = 226,
EXPECTED_ACTIVE_PRIMARY_TOTAL = 196,
EXPECTED_DIRECT_CALLS = 160,
EXPECTED_CONSTRUCTOR_CALLS = 22,
EXPECTED_OPERATOR_CALLS = 12,
EXPECTED_IMPLICIT_CALLS = 2,
EXPECTED_DSL_INVOCATIONS = 14,
@@ -65,7 +70,7 @@ enum {
EXPECTED_CALLEE_WRAPPERS = 8,
EXPECTED_ARGUMENT_WRAPPERS = 1,
EXPECTED_CONTROL_OR_DOCUMENT_NONCALLS = 6,
EXPECTED_PRIMARY_OWNERS = 193,
EXPECTED_PRIMARY_OWNERS = 196,
EXPECTED_SYNTHETIC_OWNERS = 10,
EXPECTED_NO_CALL_EDGE_OWNERS = 20,
EXPECTED_PRIMARY_OPERATOR_CALLS = 4,
@@ -297,6 +302,9 @@ static const CallNodeManifestEntry CALL_NODE_MANIFEST[] = {
ENTRY(CBM_LANG_TLAPLUS, "function_evaluation", DIRECT_CALL, PRIMARY_EXTRACTOR, true, true),
ENTRY(CBM_LANG_TLAPLUS, "call", DIRECT_CALL, PRIMARY_EXTRACTOR, true, true),
ENTRY(CBM_LANG_TLAPLUS, "bound_op", DIRECT_CALL, PRIMARY_EXTRACTOR, true, true),
ENTRY(CBM_LANG_PKL, "unqualifiedAccessExpr", DIRECT_CALL, PRIMARY_EXTRACTOR, true, true),
ENTRY(CBM_LANG_PKL, "qualifiedAccessExpr", DIRECT_CALL, PRIMARY_EXTRACTOR, true, true),
ENTRY(CBM_LANG_PKL, "newExpr", CONSTRUCTOR_CALL, PRIMARY_EXTRACTOR, true, true),
ENTRY(CBM_LANG_APEX, "method_invocation", DIRECT_CALL, PRIMARY_EXTRACTOR, true, true),
ENTRY(CBM_LANG_PINE, "call", DIRECT_CALL, PRIMARY_EXTRACTOR, true, true),
ENTRY(CBM_LANG_MOJO, "call", DIRECT_CALL, PRIMARY_EXTRACTOR, true, true),
+58 -20
View File
@@ -45,7 +45,8 @@
* 6. calls-extracted : inv_has_call(r, callee) == 1.
* Only asserted for languages that have non-empty
* call_types: HCL (function_call), NICKEL (infix_expr),
* JSONNET (functioncall), STARLARK (call).
* JSONNET (functioncall), STARLARK (call),
* PKL (unqualified/qualifiedAccessExpr, newExpr).
*
* FULL-PIPELINE (rh_index_files -> cbm_store_t*, via inv_count_* store helpers):
* 7. callable-sourcing : inv_count_calls_by_source(store,project,&mod,&call).
@@ -85,11 +86,15 @@
* Dims 1-5 ("Class"). No calls.
* XML -- class_types = element -> "Class". Dims 1-5 ("Class"). No calls.
* PROPERTIES -- var_types = property -> "Variable". Dims 1-5 ("Variable"). No calls.
* PKL -- func_types = classMethod/objectMethod -> "Function";
* class_types = clazz -> "Class"; var_types = classProperty/objectProperty.
* call_types = empty_types. Dims 1-5 ("Function", "Class"). No call dim.
*
* LANGUAGES WITH CALLABLES (dims 1-6 + R, and pipeline dims 7-8 where applicable):
* PKL -- func_types = classMethod/objectMethod -> "Function";
* class_types = clazz/typeAlias -> "Class";
* var_types = classProperty/objectProperty;
* call_types = unqualifiedAccessExpr/qualifiedAccessExpr/newExpr.
* Dims 1-8. The access-expr call types double as property reads,
* so extract_pkl_callee gates them on an `argumentList` child;
* the test adds an inline negative assertion for that gate.
* HCL -- class_types = block -> "Class"; var_types = attribute;
* call_types = function_call. Dims 1-6. No func_types so no pipeline
* dim 7 (calls would be module-sourced with no Function anchor).
@@ -766,39 +771,72 @@ TEST(repro_grammar_config_ron) {
/* ── PKL ──────────────────────────────────────────────────────────────────────
* Idiomatic PKL (Apple Pkl) module with a class definition
* (pkl_class_types = {"clazz"} -> "Class"), a method inside it
* (pkl_class_types = {"clazz", "typeAlias"} -> "Class"), methods inside it
* (pkl_func_types = {"classMethod", "objectMethod"} -> "Function"), and
* class properties (pkl_var_types = {"classProperty", "objectProperty"}).
* pkl_call_types = empty_types so no call extraction occurs.
* pkl_call_types = {"unqualifiedAccessExpr", "qualifiedAccessExpr", "newExpr"}.
*
* Dims asserted: 1-5 + R ("Class" for the class def, "Function" for the method).
* Dims 6-8 SKIPPED: call_types = empty_types in spec.
* Expected GREEN: dims 1-5. Dim 5 RED would indicate clazz->Class or
* classMethod->Function mapping is broken in the PKL grammar walker.
* Dims asserted: 1-8 (full battery) + R.
* Dim 6 GREEN: `makeUrl(host, port)` inside url() extracts callee "makeUrl".
* Dim 7 GREEN: every call site in the fixture is inside a classMethod body, so
* no CALLS edge is Module-sourced. (Real-world Pkl does call at module level;
* the fixture deliberately avoids it because dim 7 treats Module-sourced
* in-body calls as the enclosing-func gap.)
* Dim 8 GREEN: makeUrl and Server are both defined in-file, so neither the
* unqualified call nor the newExpr constructor edge dangles.
*
* PKL-SPECIFIC REGRESSION (asserted inline below): `unqualifiedAccessExpr` and
* `qualifiedAccessExpr` are the same node for a call and for a bare property
* read, so the interpolated `host` / `port` reads inside makeUrl must NOT be
* emitted as CALLS. extract_pkl_callee gates on an `argumentList` child; without
* that gate every property read in every Pkl file becomes a call edge.
*/
TEST(repro_grammar_config_pkl) {
static const char src[] =
"module cbm.Config\n"
"\n"
"function makeUrl(host: String, port: Int): String = \"http://\\(host):\\(port)\"\n"
"typealias Port = Int\n"
"\n"
"function makeUrl(host: String, port: Port): String = \"http://\\(host):\\(port)\"\n"
"\n"
"class Server {\n"
" host: String = \"localhost\"\n"
" port: Int = 8080\n"
" port: Port = 8080\n"
" tls: Boolean = false\n"
"\n"
" function url(): String = \"http://\\(host):\\(port)\"\n"
"}\n"
" function url(): String = makeUrl(host, port)\n"
"\n"
"server = new Server {\n"
" host = \"0.0.0.0\"\n"
" port = 9000\n"
" function clone(): Server = new Server { host = host }\n"
"}\n";
static const char bad[] = "module cbm.Config\nclass Server {\n host:";
if (config_struct_battery("PKL", src, CBM_LANG_PKL, "config.pkl",
"Class", "Function") != 0)
if (config_callable_battery("PKL", src, CBM_LANG_PKL, "config.pkl",
"Function", "makeUrl") != 0)
return 1;
return config_robustness("PKL", bad, CBM_LANG_PKL, "config.pkl");
/* Bare property reads must not be calls (see PKL-SPECIFIC REGRESSION above). */
CBMFileResult *pr = inv_rx(src, CBM_LANG_PKL, "config.pkl");
if (!pr) {
printf(" %sFAIL%s [PKL] inv_rx returned NULL\n", tf_red(), tf_reset());
return 1;
}
int bogus = 0;
for (int i = 0; i < pr->calls.count; i++) {
const char *cn = pr->calls.items[i].callee_name;
if (cn && (strcmp(cn, "host") == 0 || strcmp(cn, "port") == 0 ||
strcmp(cn, "tls") == 0)) {
bogus++;
}
}
cbm_free_result(pr);
if (bogus != 0) {
printf(" %sFAIL%s [PKL] property-read-not-call: %d bare property read(s) "
"emitted as a CALLS edge\n", tf_red(), tf_reset(), bogus);
return 1;
}
if (config_robustness("PKL", bad, CBM_LANG_PKL, "config.pkl") != 0)
return 1;
return config_pipeline_battery("PKL", "config.pkl", src);
}
/* ── NICKEL ───────────────────────────────────────────────────────────────────
+6 -6
View File
@@ -195,7 +195,7 @@ static const LanguageCapabilityEntry LANGUAGE_CAPABILITIES[CBM_LANG_COUNT] = {
NO_CALL(SMITHY),
NO_CALL(WIT),
CALL_WITH_REFERENCE_VOCAB(TLAPLUS),
NO_CALL(PKL),
CALL_WITH_REFERENCE_VOCAB(PKL),
NO_CALL(GOMOD),
CALL_WITH_REFERENCE_VOCAB(APEX),
NO_CALL(SOQL),
@@ -301,8 +301,8 @@ TEST(repro_language_capability_ledger_covers_every_enum) {
}
}
if (counts[CAP_CALL_WITH_REFERENCE_VOCAB] != 87 ||
counts[CAP_CALL_WITHOUT_REFERENCE_VOCAB] != 27 || counts[CAP_NO_CALL] != 50 ||
if (counts[CAP_CALL_WITH_REFERENCE_VOCAB] != 88 ||
counts[CAP_CALL_WITHOUT_REFERENCE_VOCAB] != 27 || counts[CAP_NO_CALL] != 49 ||
counts[CAP_TRANSFORM_ONLY] != 1 || counts[CAP_UNSUPPORTED] != 1) {
fprintf(stderr,
" [language-registry] invariant=capability_partition call_ref_vocab=%d ref_gap=%d "
@@ -319,9 +319,9 @@ TEST(repro_language_capability_ledger_covers_every_enum) {
TEST(repro_call_argument_matrices_equal_call_capability_ledger) {
enum {
EXPECTED_MATRIX_A_ROWS = 68,
EXPECTED_MATRIX_B_ROWS = 48,
EXPECTED_CALL_CAPABLE_LANGUAGES = 114,
EXPECTED_NON_CALL_LANGUAGES = 52,
EXPECTED_MATRIX_B_ROWS = 49,
EXPECTED_CALL_CAPABLE_LANGUAGES = 115,
EXPECTED_NON_CALL_LANGUAGES = 51,
EXPECTED_NON_CALL_DOMAIN_CONTROLS = 2,
};
CBMLanguage matrix_a_ids[CBM_LANG_COUNT];
+157 -6
View File
@@ -15250,12 +15250,9 @@ TEST(clsp_tier2_shared_registry_readonly_c) {
PASS();
}
/* Direct guard for the finalize-time short-name / embedded-type indexes and their
* iterators (type_registry.c), which the Rust trait/free-func fast paths rely on.
* Verifies: embed index yields exactly the types whose embedded_types carry a
* matching BARE name, in ascending registry order, deduped when a type lists the
* same bare twice; free-func index yields only free funcs (receiver_type==NULL) with
* the given short_name, not methods. */
/* Direct guard for the type-name / embedded-type / free-function registry
* indexes and their iterators (type_registry.c). Verifies that every iterator
* preserves ascending registry order, which is part of resolver tie-breaking. */
TEST(registry_short_name_indexes) {
CBMArena arena;
cbm_arena_init(&arena);
@@ -15288,6 +15285,25 @@ TEST(registry_short_name_indexes) {
t.short_name = "C";
t.embedded_types = c_emb;
cbm_registry_add_type(&reg, t);
memset(&t, 0, sizeof(t));
t.qualified_name = "other.Trait";
t.short_name = "Trait";
cbm_registry_add_type(&reg, t);
memset(&t, 0, sizeof(t));
t.qualified_name = "Trait";
t.short_name = "Trait";
cbm_registry_add_type(&reg, t);
/* Populate enough distinct names to force many hash-bucket collisions, with
* repeated short names spread across the registry. The differential check
* below compares the index against the exact former linear-scan order. */
for (int i = 0; i < 512; i++) {
memset(&t, 0, sizeof(t));
t.qualified_name = cbm_arena_sprintf(&arena, "bulk.mod%d.Name%d", i, i % 23);
ASSERT_NOT_NULL(t.qualified_name);
t.short_name = strrchr(t.qualified_name, '.') + 1;
cbm_registry_add_type(&reg, t);
}
/* free func "helper" (x2 — different QNs), method "M.helper", free func "other". */
CBMRegisteredFunc f;
@@ -15310,6 +15326,77 @@ TEST(registry_short_name_indexes) {
cbm_registry_add_func(&reg, f);
cbm_registry_finalize(&reg);
cbm_registry_build_type_short_index(&reg);
/* Qualified-name bare segment "Trait": types 0, 4, then the bare-QN type
* 5. The iterator is a hash prefilter, so apply the exact predicate before
* asserting its order just like production consumers do. */
CBMTypeNameIter nit;
cbm_registry_types_by_short_name(&reg, "Trait", &nit);
{
int expected[] = {0, 4, 5};
int exact_count = 0;
int candidate;
while ((candidate = cbm_type_name_iter_next(&nit)) >= 0) {
const char *qn = reg.types[candidate].qualified_name;
const char *last_dot = qn ? strrchr(qn, '.') : NULL;
const char *candidate_short = last_dot ? last_dot + 1 : qn;
if (!candidate_short || strcmp(candidate_short, "Trait") != 0)
continue;
ASSERT_TRUE(exact_count < 3);
ASSERT_EQ(candidate, expected[exact_count++]);
}
ASSERT_EQ(exact_count, 3);
}
cbm_registry_types_by_short_name(&reg, "Missing", &nit);
{
int exact_count = 0;
int candidate;
while ((candidate = cbm_type_name_iter_next(&nit)) >= 0) {
const char *qn = reg.types[candidate].qualified_name;
const char *last_dot = qn ? strrchr(qn, '.') : NULL;
const char *candidate_short = last_dot ? last_dot + 1 : qn;
if (candidate_short && strcmp(candidate_short, "Missing") == 0)
exact_count++;
}
ASSERT_EQ(exact_count, 0);
}
for (int name_i = 0; name_i < 23; name_i++) {
char short_name[32];
snprintf(short_name, sizeof(short_name), "Name%d", name_i);
cbm_registry_types_by_short_name(&reg, short_name, &nit);
int linear_i = 0;
for (;;) {
int expected = -1;
while (linear_i < reg.type_count) {
int candidate = linear_i++;
const char *qn = reg.types[candidate].qualified_name;
const char *last_dot = qn ? strrchr(qn, '.') : NULL;
const char *candidate_short = last_dot ? last_dot + 1 : qn;
if (candidate_short && strcmp(candidate_short, short_name) == 0) {
expected = candidate;
break;
}
}
int actual;
for (;;) {
actual = cbm_type_name_iter_next(&nit);
if (actual < 0)
break;
const char *qn = reg.types[actual].qualified_name;
const char *last_dot = qn ? strrchr(qn, '.') : NULL;
const char *candidate_short = last_dot ? last_dot + 1 : qn;
if (candidate_short && strcmp(candidate_short, short_name) == 0)
break;
}
ASSERT_EQ(actual, expected);
if (expected < 0)
break;
}
}
/* Embed index for bare "Trait": types 1 (A) then 2 (B), ascending, B once. */
CBMTypeEmbedIter it;
@@ -15339,6 +15426,70 @@ TEST(registry_short_name_indexes) {
ASSERT_EQ(cbm_free_func_iter_next(&fit), 3);
ASSERT_EQ(cbm_free_func_iter_next(&fit), -1);
/* A type appended after finalize is covered by the iterator tail. If the
* auxiliary allocation is unavailable, the iterator falls back to the same
* complete linear scan rather than silently dropping candidates. Keep this
* after the other iterator assertions because their tails intentionally
* expose post-finalize candidates for caller-side exact filtering too. */
int tail_type_i = reg.type_count;
memset(&t, 0, sizeof(t));
t.qualified_name = "tail.Trait";
t.short_name = "Trait";
cbm_registry_add_type(&reg, t);
cbm_registry_types_by_short_name(&reg, "Trait", &nit);
ASSERT_EQ(cbm_type_name_iter_next(&nit), 0);
ASSERT_EQ(cbm_type_name_iter_next(&nit), 4);
ASSERT_EQ(cbm_type_name_iter_next(&nit), 5);
ASSERT_EQ(cbm_type_name_iter_next(&nit), tail_type_i);
ASSERT_EQ(cbm_type_name_iter_next(&nit), -1);
int *saved_type_short_buckets = reg.type_short_buckets;
reg.type_short_buckets = NULL;
cbm_registry_types_by_short_name(&reg, "Trait", &nit);
int fallback_expected[] = {0, 4, 5, tail_type_i};
int fallback_count = 0;
int candidate;
while ((candidate = cbm_type_name_iter_next(&nit)) >= 0) {
const char *qn = reg.types[candidate].qualified_name;
const char *last_dot = qn ? strrchr(qn, '.') : NULL;
const char *candidate_short = last_dot ? last_dot + 1 : qn;
if (!candidate_short || strcmp(candidate_short, "Trait") != 0)
continue;
ASSERT_TRUE(fallback_count < 4);
ASSERT_EQ(candidate, fallback_expected[fallback_count++]);
}
ASSERT_EQ(fallback_count, 4);
reg.type_short_buckets = saved_type_short_buckets;
/* A failed rebuild must discard the previous auxiliary index. Re-finalize
* after adding the tail type so the QN boundary advances, then force arena
* exhaustion for the optional rebuild. The iterator must fall back to the
* complete linear scan instead of retaining the stale pre-tail index. */
cbm_registry_finalize(&reg);
int saved_nblocks = arena.nblocks;
size_t saved_used = arena.used;
arena.nblocks = CBM_ARENA_MAX_BLOCKS;
arena.used = arena.block_size;
cbm_registry_build_type_short_index(&reg);
arena.nblocks = saved_nblocks;
arena.used = saved_used;
ASSERT_NULL(reg.type_short_buckets);
ASSERT_NULL(reg.type_short_entries);
ASSERT_EQ(reg.type_short_bucket_count, 0);
cbm_registry_types_by_short_name(&reg, "Trait", &nit);
fallback_count = 0;
while ((candidate = cbm_type_name_iter_next(&nit)) >= 0) {
const char *qn = reg.types[candidate].qualified_name;
const char *last_dot = qn ? strrchr(qn, '.') : NULL;
const char *candidate_short = last_dot ? last_dot + 1 : qn;
if (!candidate_short || strcmp(candidate_short, "Trait") != 0)
continue;
ASSERT_TRUE(fallback_count < 4);
ASSERT_EQ(candidate, fallback_expected[fallback_count++]);
}
ASSERT_EQ(fallback_count, 4);
cbm_arena_destroy(&arena);
PASS();
}
+94
View File
@@ -1285,6 +1285,99 @@ TEST(golsp_crossfile_stdlib_interface) {
PASS();
}
/* Cross-package receiver-method resolution when struct field type texts are
* written through import aliases. parse_field_defs_into_type qualifies the
* text with the defining module ("Svc:svc.Svc" in module test.main becomes
* "test.main.svc.Svc") — a QN that exists nowhere in the project-wide
* registry, so the dispatch must re-qualify the alias segment through the
* calling file's import map and land on the real receiver type. Mirrors a
* common cross-module shape: a service struct field typed from an sdk module
* and an interface-typed client field from an api module. */
TEST(golsp_crossfile_aliased_field_requal) {
const char *source = "package main\n\n"
"func callSvc(h *Handler) error {\n"
"\th.Svc.Ping()\n\treturn nil\n}\n\n"
"func callPb(h *Holder) error {\n"
"\th.C.Ping()\n\treturn nil\n}\n";
CBMLSPDef defs[] = {
/* myapp/svc — concrete service struct */
{.qualified_name = "myapp/svc.Svc",
.short_name = "Svc",
.label = "Struct",
.def_module_qn = "myapp/svc"},
/* test.main — receiver structs; field texts use import aliases, the
* trigger for the wrong qualification */
{.qualified_name = "test.main.Handler",
.short_name = "Handler",
.label = "Struct",
.def_module_qn = "test.main",
.field_defs = "Svc:svc.Svc"},
{.qualified_name = "test.main.Holder",
.short_name = "Holder",
.label = "Struct",
.def_module_qn = "test.main",
.field_defs = "C:pb.Client"},
/* myapp/pb — client interface */
{.qualified_name = "myapp/pb.Client",
.short_name = "Client",
.label = "Interface",
.def_module_qn = "myapp/pb",
.is_interface = true,
.method_names_str = "Ping"},
/* methods, after their receiver types (extraction order) */
{.qualified_name = "myapp/svc.Svc.Ping",
.short_name = "Ping",
.label = "Method",
.def_module_qn = "myapp/svc",
.receiver_type = "myapp/svc.Svc",
.return_types = "error"},
{.qualified_name = "myapp/pb.Client.Ping",
.short_name = "Ping",
.label = "Method",
.def_module_qn = "myapp/pb",
.receiver_type = "myapp/pb.Client",
.return_types = "error"},
};
const char *imp_names[] = {"svc", "pb"};
const char *imp_qns[] = {"myapp/svc", "myapp/pb"};
CBMArena arena;
cbm_arena_init(&arena);
CBMResolvedCallArray out = {0};
CBMTypeRegistry *reg = cbm_go_build_cross_registry(&arena, defs, 6);
ASSERT_NOT_NULL(reg);
cbm_run_go_lsp_cross_with_registry(&arena, source, (int)strlen(source), "test.main", reg,
imp_names, imp_qns, 2, NULL, &out);
int svc_idx = find_resolved_arr_confident(&out, "callSvc", "Svc.Ping");
if (svc_idx < 0) {
printf(" cross-registry diagnostics (%d records):\n", out.count);
for (int i = 0; i < out.count; i++) {
const CBMResolvedCall *rc = &out.items[i];
printf(" %s -> %s [%s %.2f]\n", rc->caller_qn ? rc->caller_qn : "(null)",
rc->callee_qn ? rc->callee_qn : "(null)",
rc->strategy ? rc->strategy : "(null)", rc->confidence);
}
}
ASSERT_GTE(svc_idx, 0);
ASSERT_STR_EQ(out.items[svc_idx].callee_qn, "myapp/svc.Svc.Ping");
ASSERT_STR_EQ(out.items[svc_idx].strategy, "lsp_type_dispatch");
ASSERT_TRUE(out.items[svc_idx].confidence >= 0.9f);
int pb_idx = find_resolved_arr_confident(&out, "callPb", "Client.Ping");
ASSERT_GTE(pb_idx, 0);
ASSERT_STR_EQ(out.items[pb_idx].callee_qn, "myapp/pb.Client.Ping");
ASSERT_TRUE(strcmp(out.items[pb_idx].strategy, "lsp_interface_dispatch") == 0 ||
strcmp(out.items[pb_idx].strategy, "lsp_type_dispatch") == 0);
ASSERT_TRUE(out.items[pb_idx].confidence >= 0.8f);
cbm_arena_destroy(&arena);
PASS();
}
TEST(golsp_crossfile_local_interface_single_impl) {
const char *source =
"package main\n\n"
@@ -1448,6 +1541,7 @@ SUITE(go_lsp) {
RUN_TEST(golsp_crossfile_return_type_chain);
RUN_TEST(golsp_crossfile_interface_dispatch);
RUN_TEST(golsp_crossfile_interface_field_chain);
RUN_TEST(golsp_crossfile_aliased_field_requal);
RUN_TEST(golsp_crossfile_map_index);
RUN_TEST(golsp_crossfile_stdlib_interface);
RUN_TEST(golsp_crossfile_local_interface_single_impl);
+1
View File
@@ -26,6 +26,7 @@ TEST(infrascan_http_route_literal_guard_rejects_filesystem_paths) {
ASSERT_FALSE(cbm_service_pattern_is_http_route_literal("/var/run/app.json", "requests.get"));
ASSERT_FALSE(cbm_service_pattern_is_http_route_literal("/locations/", "str.split"));
ASSERT_FALSE(cbm_service_pattern_is_http_route_literal("/api", "os.path.join"));
ASSERT_FALSE(cbm_service_pattern_is_http_route_literal("/html/g", "template.replace"));
ASSERT_FALSE(cbm_service_pattern_is_http_route_literal(NULL, "requests.get"));
ASSERT_FALSE(cbm_service_pattern_is_http_route_literal("", "requests.get"));
ASSERT_TRUE(cbm_service_pattern_is_http_route_literal("/api/orders", "requests.get"));
+182 -14
View File
@@ -851,6 +851,7 @@ TEST(mcp_tools_list) {
ASSERT_NOT_NULL(strstr(json, "query_graph"));
ASSERT_NOT_NULL(strstr(json, "trace_path"));
ASSERT_NOT_NULL(strstr(json, "get_code_snippet"));
ASSERT_NOT_NULL(strstr(json, "get_file_outline"));
ASSERT_NOT_NULL(strstr(json, "get_graph_schema"));
ASSERT_NOT_NULL(strstr(json, "get_architecture"));
ASSERT_NOT_NULL(strstr(json, "search_code"));
@@ -905,6 +906,7 @@ TEST(mcp_tools_list_latest_metadata) {
ASSERT_NOT_NULL(strstr(json, "\"title\":\"Search graph\""));
ASSERT_NOT_NULL(strstr(json, "\"title\":\"Index repository\""));
ASSERT_NOT_NULL(strstr(json, "\"title\":\"Check index coverage\""));
ASSERT_NOT_NULL(strstr(json, "\"title\":\"Get file outline\""));
/* No tool may declare an outputSchema. The blanket permissive schema
* ({"type":"object","additionalProperties":true}) carried zero information
* for clients, but its presence made spec-compliant clients read
@@ -939,6 +941,7 @@ TEST(mcp_tools_have_behavior_annotations) {
{"query_graph", false, true, true, false},
{"trace_path", false, true, true, false},
{"get_code_snippet", false, true, true, false},
{"get_file_outline", false, true, true, false},
{"get_graph_schema", false, true, true, false},
{"compare_graphs", true, false, true, false},
{"get_architecture", false, true, true, false},
@@ -1336,9 +1339,9 @@ static int issue403_initialize_count_calls(const char *session_root, bool approv
cbm_setenv("CBM_CACHE_DIR", cache, 1);
char err[1024];
bool approved = !approve_sensitive ||
cbm_workspace_grant_add(cache, cbm_workspace_home_dir(), session_root, true,
err, sizeof(err));
bool approved =
!approve_sensitive || cbm_workspace_grant_add(cache, cbm_workspace_home_dir(), session_root,
true, err, sizeof(err));
cbm_config_t *cfg = approved ? cbm_config_open(cache) : NULL;
cbm_mcp_server_t *srv = cfg ? cbm_mcp_server_new(NULL) : NULL;
int calls = -2;
@@ -1368,8 +1371,8 @@ static int issue403_initialize_count_calls(const char *session_root, bool approv
}
TEST(mcp_issue403_sensitive_root_stops_before_discovery_count) {
int sensitive = issue403_initialize_count_calls(
"C:/Users/dev/AppData/Local/Programs/Antigravity", false);
int sensitive =
issue403_initialize_count_calls("C:/Users/dev/AppData/Local/Programs/Antigravity", false);
int ordinary = issue403_initialize_count_calls("C:/Users/dev/projects/app", false);
ASSERT_EQ(sensitive, 0);
ASSERT_EQ(ordinary, 1);
@@ -1434,13 +1437,37 @@ TEST(server_handle_tools_list_defaults_to_all_tools_and_accepts_cursor) {
ASSERT_NOT_NULL(strstr(resp, "ingest_traces"));
free(resp);
resp = cbm_mcp_server_handle(
srv,
"{\"jsonrpc\":\"2.0\",\"id\":201,\"method\":\"tools/list\",\"params\":{\"cursor\":\"8\"}}");
/* A cursored page advertises nextCursor exactly while tools remain after
* it. This used to pass the literal cursor "8", which silently encoded
* "there are at most MCP_TOOLS_PAGE_SIZE * 2 tools" -- so registering a
* 17th tool broke it for a reason that had nothing to do with pagination.
* Derive the offset from the live count instead: the final page, wherever
* it falls, is the one that must not advertise more. */
resp = cbm_mcp_server_handle(srv, "{\"jsonrpc\":\"2.0\",\"id\":203,\"method\":\"tools/list\"}");
ASSERT_NOT_NULL(resp);
size_t total_tools = mcp_response_tool_count(resp);
free(resp);
ASSERT_TRUE(total_tools > 1U);
char last_page_req[160];
snprintf(last_page_req, sizeof(last_page_req),
"{\"jsonrpc\":\"2.0\",\"id\":201,\"method\":\"tools/list\","
"\"params\":{\"cursor\":\"%zu\"}}",
total_tools - 1U);
resp = cbm_mcp_server_handle(srv, last_page_req);
ASSERT_NOT_NULL(resp);
ASSERT_NOT_NULL(strstr(resp, "\"id\":201"));
ASSERT_NULL(strstr(resp, "\"nextCursor\""));
ASSERT_NOT_NULL(strstr(resp, "manage_adr"));
ASSERT_EQ(mcp_response_tool_count(resp), 1U);
free(resp);
/* ...and a page that does have tools after it MUST advertise the cursor,
* so the assertion above cannot pass merely because paging never emits. */
resp = cbm_mcp_server_handle(
srv,
"{\"jsonrpc\":\"2.0\",\"id\":204,\"method\":\"tools/list\",\"params\":{\"cursor\":\"0\"}}");
ASSERT_NOT_NULL(resp);
ASSERT_NOT_NULL(strstr(resp, "\"nextCursor\""));
free(resp);
cbm_mcp_server_free(srv);
@@ -1463,9 +1490,10 @@ TEST(server_handle_analysis_profile_filters_and_rejects_mutators) {
resp = cbm_mcp_server_handle(srv, "{\"jsonrpc\":\"2.0\",\"id\":220,\"method\":\"tools/list\"}");
ASSERT_NOT_NULL(resp);
static const char *const analysis_tools[] = {
"search_graph", "query_graph", "trace_path", "get_code_snippet",
"get_graph_schema", "compare_graphs", "get_architecture", "search_code",
"list_projects", "index_status", "check_index_coverage", "detect_changes",
"search_graph", "query_graph", "trace_path", "get_code_snippet",
"get_file_outline", "get_graph_schema", "compare_graphs", "get_architecture",
"search_code", "list_projects", "index_status", "check_index_coverage",
"detect_changes",
};
ASSERT_EQ(mcp_response_tool_count(resp), sizeof(analysis_tools) / sizeof(analysis_tools[0]));
for (size_t i = 0U; i < sizeof(analysis_tools) / sizeof(analysis_tools[0]); i++) {
@@ -1504,10 +1532,11 @@ TEST(server_handle_scout_profile_exposes_only_the_fast_tier) {
resp = cbm_mcp_server_handle(srv, "{\"jsonrpc\":\"2.0\",\"id\":223,\"method\":\"tools/list\"}");
ASSERT_NOT_NULL(resp);
ASSERT_EQ(mcp_response_tool_count(resp), 7U);
ASSERT_EQ(mcp_response_tool_count(resp), 8U);
ASSERT_TRUE(mcp_response_has_exact_tool(resp, "search_graph"));
ASSERT_TRUE(mcp_response_has_exact_tool(resp, "trace_path"));
ASSERT_TRUE(mcp_response_has_exact_tool(resp, "get_code_snippet"));
ASSERT_TRUE(mcp_response_has_exact_tool(resp, "get_file_outline"));
ASSERT_TRUE(mcp_response_has_exact_tool(resp, "get_architecture"));
ASSERT_TRUE(mcp_response_has_exact_tool(resp, "list_projects"));
ASSERT_TRUE(mcp_response_has_exact_tool(resp, "index_status"));
@@ -1730,6 +1759,8 @@ static cbm_mcp_server_t *setup_mcp_with_data(void) {
return srv;
}
static char *extract_text_content(const char *mcp_result);
TEST(tool_list_projects_empty) {
cbm_mcp_server_t *srv = setup_mcp_with_data();
@@ -1788,6 +1819,142 @@ TEST(tool_compare_graphs_registered_issue525) {
ASSERT_NOT_NULL(resp);
ASSERT_NULL(strstr(resp, "unknown tool: compare_graphs"));
free(resp);
cbm_mcp_server_free(srv);
PASS();
}
TEST(tool_get_file_outline_returns_bounded_filtered_columnar_rows_issue469) {
cbm_mcp_server_t *srv = setup_mcp_with_data();
cbm_store_t *store = cbm_mcp_server_store(srv);
ASSERT_NOT_NULL(store);
ASSERT_EQ(cbm_store_upsert_project(store, "outline-project", "/tmp/outline-project"),
CBM_STORE_OK);
cbm_mcp_server_set_project(srv, "outline-project");
cbm_node_t nodes[] = {
{.project = "outline-project",
.label = "Module",
.name = "main",
.qualified_name = "outline-project.main",
.file_path = "src/main.c",
.start_line = 1,
.end_line = 80},
{.project = "outline-project",
.label = "Function",
.name = "alpha",
.qualified_name = "outline-project.src.main.alpha",
.file_path = "src/main.c",
.start_line = 10,
.end_line = 14},
{.project = "outline-project",
.label = "Class",
.name = "IgnoredClass",
.qualified_name = "outline-project.src.main.IgnoredClass",
.file_path = "src/main.c",
.start_line = 15,
.end_line = 40},
{.project = "outline-project",
.label = "Method",
.name = "omega",
.qualified_name = "outline-project.src.main.omega",
.file_path = "src/main.c",
.start_line = 30,
.end_line = 33},
{.project = "outline-project",
.label = "Function",
.name = "other",
.qualified_name = "outline-project.src.other.other",
.file_path = "src/other.c",
.start_line = 1,
.end_line = 2},
};
for (size_t i = 0; i < sizeof(nodes) / sizeof(nodes[0]); i++) {
ASSERT_GT(cbm_store_upsert_node(store, &nodes[i]), 0);
}
char *response =
cbm_mcp_handle_tool(srv, "get_file_outline",
"{\"project\":\"outline-project\",\"file_path\":\"src/main.c\","
"\"labels\":[\"Function\",\"Method\"],\"limit\":1}");
ASSERT_NOT_NULL(response);
ASSERT_NOT_NULL(strstr(response, "cols"));
ASSERT_NOT_NULL(strstr(response, "(cols: name label lines qn)"));
ASSERT_NOT_NULL(strstr(response, "alpha"));
ASSERT_NULL(strstr(response, "omega"));
ASSERT_NULL(strstr(response, "IgnoredClass"));
ASSERT_NOT_NULL(strstr(response, "total: 2"));
ASSERT_NOT_NULL(strstr(response, "has_more: true"));
ASSERT_NULL(strstr(response, "unknown tool"));
free(response);
cbm_mcp_server_free(srv);
PASS();
}
TEST(tool_get_file_outline_validates_json_path_limit_and_cancel_issue469) {
cbm_mcp_server_t *srv = setup_mcp_with_data();
cbm_store_t *store = cbm_mcp_server_store(srv);
ASSERT_NOT_NULL(store);
ASSERT_EQ(cbm_store_upsert_project(store, "outline-controls", "/tmp/outline-controls"),
CBM_STORE_OK);
cbm_mcp_server_set_project(srv, "outline-controls");
cbm_node_t node = {.project = "outline-controls",
.label = "Function",
.name = "bounded",
.qualified_name = "outline-controls.src.main.bounded",
.file_path = "src/main.c",
.start_line = 7,
.end_line = 9};
ASSERT_GT(cbm_store_upsert_node(store, &node), 0);
char *response =
cbm_mcp_handle_tool(srv, "get_file_outline",
"{\"project\":\"outline-controls\",\"file_path\":\"../outside.c\"}");
ASSERT_NOT_NULL(response);
ASSERT_NOT_NULL(strstr(response, "repository-relative"));
ASSERT_NOT_NULL(strstr(response, "isError"));
free(response);
response = cbm_mcp_handle_tool(
srv, "get_file_outline",
"{\"project\":\"outline-controls\",\"file_path\":\"src/main.c\",\"limit\":201}");
ASSERT_NOT_NULL(response);
ASSERT_NOT_NULL(strstr(response, "between 1 and 200"));
ASSERT_NOT_NULL(strstr(response, "isError"));
free(response);
response = cbm_mcp_handle_tool(srv, "get_file_outline",
"{\"project\":\"outline-controls\",\"file_path\":\"src/main.c\","
"\"format\":\"json\"}");
ASSERT_NOT_NULL(response);
char *inner = extract_text_content(response);
ASSERT_NOT_NULL(inner);
yyjson_doc *doc = yyjson_read(inner, strlen(inner), 0);
ASSERT_NOT_NULL(doc);
yyjson_val *root = yyjson_doc_get_root(doc);
ASSERT_TRUE(yyjson_is_arr(yyjson_obj_get(root, "cols")));
ASSERT_TRUE(yyjson_is_arr(yyjson_obj_get(root, "rows")));
ASSERT_EQ(yyjson_get_int(yyjson_obj_get(root, "total")), 1);
yyjson_doc_free(doc);
free(inner);
free(response);
ASSERT_TRUE(cbm_mcp_server_request_scope_begin(srv));
ASSERT_TRUE(cbm_mcp_server_cancel_active(srv));
response = cbm_mcp_handle_tool(
srv, "get_file_outline", "{\"project\":\"outline-controls\",\"file_path\":\"src/main.c\"}");
ASSERT_NOT_NULL(response);
ASSERT_NOT_NULL(strstr(response, "cancelled for this request"));
ASSERT_NOT_NULL(strstr(response, "isError"));
free(response);
cbm_mcp_server_request_scope_end(srv);
response = cbm_mcp_handle_tool(
srv, "get_file_outline", "{\"project\":\"outline-controls\",\"file_path\":\"src/main.c\"}");
ASSERT_NOT_NULL(response);
ASSERT_NOT_NULL(strstr(response, "bounded"));
ASSERT_NULL(strstr(response, "cancelled"));
free(response);
cbm_mcp_server_free(srv);
PASS();
@@ -1812,7 +1979,6 @@ TEST(tool_search_graph_basic) {
/* Forward declarations for helpers defined later in this file */
static cbm_mcp_server_t *setup_snippet_server(char *tmp_dir, size_t tmp_sz);
static void cleanup_snippet_dir(const char *tmp_dir);
static char *extract_text_content(const char *mcp_result);
TEST(tool_search_graph_semantic_only_skips_structural_results_issue1295) {
cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL);
@@ -13555,6 +13721,8 @@ SUITE(mcp) {
RUN_TEST(tool_compare_graphs_enforces_encoded_budget_issue525);
RUN_TEST(tool_compare_graphs_validation_and_scan_cap_are_atomic_issue525);
RUN_TEST(tool_compare_graphs_cancel_and_readonly_handles_release_issue525);
RUN_TEST(tool_get_file_outline_returns_bounded_filtered_columnar_rows_issue469);
RUN_TEST(tool_get_file_outline_validates_json_path_limit_and_cancel_issue469);
RUN_TEST(tool_search_graph_basic);
RUN_TEST(tool_search_graph_semantic_only_skips_structural_results_issue1295);
RUN_TEST(tool_trace_totals_respect_test_filter);
+14 -4
View File
@@ -516,6 +516,11 @@ TEST(resolve_budget_override_when_total_unknown) {
PASS();
}
/* CBM_MEM_BUDGET_MB is an aggregate ceiling the parent divides across job
* slots. A lower explicit value still wins; a raise is clipped to the per-slot
* share so N workers cannot oversubscribe the host (#1654). The source stays
* CBM_MEM_BUDGET_MB so the clip is the user's aggregate, not a silent
* daemon_worker_cap rewrite of a fraction-derived default. */
TEST(resolve_budget_worker_cap_preserves_lower_user_override) {
size_t total = 8192 * CBM_TEST_MB;
size_t worker_cap = 16 * CBM_TEST_MB;
@@ -524,10 +529,15 @@ TEST(resolve_budget_worker_cap_preserves_lower_user_override) {
ASSERT_STR_EQ(lower.source, "CBM_MEM_BUDGET_MB");
ASSERT_FALSE(lower.hard_capped);
cbm_mem_budget_t capped = cbm_mem_resolve_budget_capped(total, 0.5, "64", worker_cap);
ASSERT_EQ(capped.budget, worker_cap);
ASSERT_STR_EQ(capped.source, "daemon_worker_cap");
ASSERT_TRUE(capped.hard_capped);
cbm_mem_budget_t raised = cbm_mem_resolve_budget_capped(total, 0.5, "64", worker_cap);
ASSERT_EQ(raised.budget, worker_cap);
ASSERT_STR_EQ(raised.source, "CBM_MEM_BUDGET_MB");
ASSERT_TRUE(raised.hard_capped);
cbm_mem_budget_t fraction = cbm_mem_resolve_budget_capped(total, 0.5, NULL, worker_cap);
ASSERT_EQ(fraction.budget, worker_cap);
ASSERT_STR_EQ(fraction.source, "daemon_worker_cap");
ASSERT_TRUE(fraction.hard_capped);
PASS();
}
+228
View File
@@ -3203,6 +3203,233 @@ TEST(parallel_python_lsp_override_emits_lsp_strategy_edges) {
PASS();
}
/* ── Go cross-package field-chain fixture (field_defs fold) ─────── */
/* Production's sequential driver seeds Folder nodes via pass_structure; the
* compact harness (run_sequential_with_lsp_cross_*) does not. Go imports
* resolve to Folder nodes, so a Go import-map fixture must seed File and
* Folder nodes itself — replicating the production shape. */
static cbm_gbuf_t *run_go_field_chain_sequential(const char *project, const char *repo_path,
cbm_file_info_t *files, int file_count) {
cbm_gbuf_t *gbuf = cbm_gbuf_new(project, repo_path);
cbm_registry_t *reg = cbm_registry_new();
CBMFileResult **cache = (CBMFileResult **)calloc((size_t)file_count, sizeof(CBMFileResult *));
if (!gbuf || !reg || !cache) {
cbm_gbuf_free(gbuf);
cbm_registry_free(reg);
free(cache);
return NULL;
}
atomic_int cancelled;
atomic_init(&cancelled, 0);
cbm_pipeline_ctx_t ctx = {
.project_name = project,
.repo_path = repo_path,
.gbuf = gbuf,
.registry = reg,
.cancelled = &cancelled,
.result_cache = cache,
};
seed_test_file_nodes(gbuf, project, files, file_count);
char *svc_dir_qn = cbm_pipeline_fqn_folder(project, "svc");
if (svc_dir_qn) {
cbm_gbuf_upsert_node(gbuf, "Folder", "svc", svc_dir_qn, "svc", 0, 0, "{}");
free(svc_dir_qn);
}
cbm_init();
cbm_pipeline_pass_definitions(&ctx, files, file_count);
cbm_pipeline_pass_lsp_cross(&ctx, files, file_count, cache);
cbm_pipeline_pass_calls(&ctx, files, file_count);
cbm_pipeline_pass_usages(&ctx, files, file_count);
cbm_pipeline_pass_semantic(&ctx, files, file_count);
/* CBM_GO_FIELD_DIAG dump. NOTE: resolved-call records may borrow QN
* strings across file results (cross-file resolution in pass_lsp_cross),
* so every result must stay alive while ANY is inspected. Print all
* first, free all second. */
if (getenv("CBM_GO_FIELD_DIAG")) {
for (int i = 0; i < file_count; i++) {
if (!cache[i]) {
continue;
}
const CBMFileResult *r = cache[i];
printf(" [diag] file %s defs=%d imports=%d calls=%d resolved=%d\n", files[i].rel_path,
r->defs.count, r->imports.count, r->calls.count, r->resolved_calls.count);
for (int j = 0; j < r->resolved_calls.count; j++) {
const CBMResolvedCall *rc = &r->resolved_calls.items[j];
printf(" [diag] rc caller=%s callee=%s strategy=%s conf=%.2f kind=%d span=[%u,%u)\n",
rc->caller_qn ? rc->caller_qn : "?", rc->callee_qn ? rc->callee_qn : "?",
rc->strategy ? rc->strategy : "?", rc->confidence, (int)rc->kind,
(unsigned)rc->site_start_byte, (unsigned)rc->site_end_byte);
}
for (int j = 0; j < r->calls.count; j++) {
const CBMCall *c = &r->calls.items[j];
printf(" [diag] call callee=%s enclosing=%s span=[%u,%u) req=%d\n",
c->callee_name ? c->callee_name : "?", c->enclosing_func_qn ? c->enclosing_func_qn : "?",
(unsigned)c->site_start_byte, (unsigned)c->site_end_byte, (int)c->requires_lsp_resolution);
}
for (int j = 0; j < r->defs.count; j++) {
const CBMDefinition *d = &r->defs.items[j];
printf(" [diag] def label=%s qn=%s parent=%s ret=%s\n", d->label ? d->label : "?",
d->qualified_name ? d->qualified_name : "?", d->parent_class ? d->parent_class : "?",
d->return_type ? d->return_type : "?");
}
}
}
for (int i = 0; i < file_count; i++) {
cbm_free_result(cache[i]);
}
free(cache);
harness_ctx_free_tables(&ctx);
cbm_registry_free(reg);
if (ctx.seq_cross_arena_live) {
cbm_arena_destroy(&ctx.seq_cross_arena);
ctx.seq_cross_arena_live = false;
}
if (ctx.seq_cross_def_modules) {
for (int i = 0; i < ctx.seq_cross_def_module_count; i++) {
free(ctx.seq_cross_def_modules[i]);
}
free(ctx.seq_cross_def_modules);
ctx.seq_cross_def_modules = NULL;
}
return gbuf;
}
/* Cross-package field chains (h.S.Ping()) resolve end to end through the
* shared prebuilt Go registry. Regression for the field_defs fold: Go struct
* fields are extracted as flat "Field" definitions that pxc_map_label drops,
* so structs registered with zero fields and field chains never resolved.
* Mirrors the real-world handler shape — an app package holds an aliased
* cross-package field ("S *s.Svc") and app.Call calls through it. */
TEST(parallel_go_cross_package_field_chain_resolves) {
char tmpdir[256];
snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_par_gofold_XXXXXX");
if (!cbm_mkdtemp(tmpdir)) {
FAIL("mkdtemp failed");
}
char svc_path[512];
char app_path[512];
snprintf(svc_path, sizeof(svc_path), "%s/acme-order/internal/service/order_service.go", tmpdir);
snprintf(app_path, sizeof(app_path), "%s/acme-order/internal/handler/order_handler.go", tmpdir);
if (th_write_file(svc_path, "package service\n"
"\n"
"import (\n"
" \"context\"\n"
" pb \"acme-sdk/apis/order\"\n"
")\n"
"\n"
"type OrderService struct{}\n"
"\n"
"func (s *OrderService) PlaceOrder(ctx context.Context, req *pb.PlaceOrderReq) (*pb.PlaceOrderRsp, error) {\n"
" return nil, nil\n"
"}\n"
"\n"
"func (s *OrderService) ListOrders(ctx context.Context, req *pb.ListOrdersReq) (*pb.ListOrdersRsp, error) {\n"
" return nil, nil\n"
"}\n") != 0 ||
th_write_file(app_path, "package handler\n"
"\n"
"import (\n"
" \"context\"\n"
"\n"
" pb \"acme-sdk/apis/order\"\n"
"\n"
" \"acme-order/internal/service\"\n"
")\n"
"\n"
"type OrderHandler struct {\n"
" pb.UnimplementedOrderServer\n"
" cartSvc *service.CartService\n"
" userSvc *service.UserService\n"
" orderSvc *service.OrderService\n"
" inventorySvc *service.InventoryService\n"
" paymentSvc *service.PaymentService\n"
" shipmentSvc *service.ShipmentService\n"
" couponSvc *service.CouponService\n"
" searchSvc *service.SearchService\n"
"}\n"
"\n"
"func (h *OrderHandler) ListOrders(ctx context.Context, req *pb.ListOrdersReq) (*pb.ListOrdersRsp, error) {\n"
" return h.orderSvc.ListOrders(ctx, req)\n"
"}\n"
"\n"
"func (h *OrderHandler) PlaceOrder(ctx context.Context, req *pb.PlaceOrderReq) (*pb.PlaceOrderRsp, error) {\n"
" return h.orderSvc.PlaceOrder(ctx, req)\n"
"}\n"
"\n"
"func (h *OrderHandler) UpdateCart(ctx context.Context, req *pb.UpdateCartReq) (*pb.UpdateCartRsp, error) {\n"
" return h.cartSvc.UpdateCart(ctx, req)\n"
"}\n") != 0) {
th_rmtree(tmpdir);
FAIL("write fixture failed");
}
cbm_file_info_t files[2] = {0};
files[0].path = svc_path;
files[0].rel_path = (char *)"acme-order/internal/service/order_service.go";
files[0].language = CBM_LANG_GO;
files[1].path = app_path;
files[1].rel_path = (char *)"acme-order/internal/handler/order_handler.go";
files[1].language = CBM_LANG_GO;
cbm_gbuf_t *gbuf = run_go_field_chain_sequential("go_field_fold", tmpdir, files, 2);
ASSERT_NOT_NULL(gbuf);
const cbm_gbuf_edge_t *edge = find_call_edge_to_target_fragment(gbuf, "handler.PlaceOrder", ".service.PlaceOrder");
const bool found = edge != NULL;
const bool dispatch =
edge && edge->properties_json &&
strstr(edge->properties_json, "\"strategy\":\"lsp_type_dispatch\"");
if (!found || !dispatch) {
printf(" go field chain diagnostic: found=%d dispatch=%d\n", found, dispatch);
if (edge && edge->properties_json) {
printf(" go field chain props: %s\n", edge->properties_json);
}
/* Dump all callable nodes and CALLS edges for cross-referencing */
{
const cbm_gbuf_node_t **nodes = NULL;
int ncount = 0;
if (cbm_gbuf_find_by_label(gbuf, "Function", &nodes, &ncount) == 0) {
for (int i = 0; i < ncount; i++) {
printf(" node Function: %s\n", nodes[i]->qualified_name);
}
}
if (cbm_gbuf_find_by_label(gbuf, "Method", &nodes, &ncount) == 0) {
for (int i = 0; i < ncount; i++) {
printf(" node Method: %s\n", nodes[i]->qualified_name);
}
}
cbm_gbuf_edge_visitor_fn edge_dump = NULL;
(void)edge_dump;
/* print every CALLS edge with endpoints */
const cbm_gbuf_edge_t **all = NULL;
int ecount = 0;
if (cbm_gbuf_find_edges_by_type(gbuf, "CALLS", &all, &ecount) == 0) {
for (int i = 0; i < ecount; i++) {
const cbm_gbuf_node_t *src =
all[i] ? cbm_gbuf_find_by_id(gbuf, all[i]->source_id) : NULL;
const cbm_gbuf_node_t *dst =
all[i] ? cbm_gbuf_find_by_id(gbuf, all[i]->target_id) : NULL;
printf(" CALLS edge: %s -> %s props=%s\n",
src ? src->qualified_name : "?",
dst ? dst->qualified_name : "?",
all[i]->properties_json ? all[i]->properties_json : "{}");
}
}
}
}
cbm_gbuf_free(gbuf);
th_rmtree(tmpdir);
ASSERT_TRUE(found);
ASSERT_TRUE(dispatch);
PASS();
}
/* Cross-file regression for the QN-mismatch bug: py_lsp's per-file mode
* emits resolved_calls.callee_qn as the raw import-module path (e.g.
* `greeter.Greeter` from `from greeter import Greeter`) rather than the
@@ -4108,6 +4335,7 @@ SUITE(parallel) {
RUN_TEST(parallel_cpp_preprocessed_coordinate_collision_preserves_hidden_target);
RUN_TEST(parallel_cuda_preprocessed_coordinate_collision_preserves_hidden_target);
RUN_TEST(parallel_python_lsp_override_cross_file_emits_lsp_strategy_edges);
RUN_TEST(parallel_go_cross_package_field_chain_resolves);
RUN_TEST(parallel_cross_file_reread_preserves_unretained_edges);
RUN_TEST(parallel_java_kotlin_lsp_override_cross_file_emits_lsp_strategy_edges);
RUN_TEST(parallel_lsp_tail_match_fallbacks_gated_to_jvm);
+54
View File
@@ -104,6 +104,22 @@ static const char *PY_CLEAN = "def ok():\n"
"def ok2():\n"
" return 2\n";
/* Perl formats have a line-oriented body terminated by a lone dot. The
* following named sub pins the important recovery boundary: a grammar must
* both accept the format and resume normal declaration parsing afterwards. */
static const char *PERL_FORMAT_WITH_FOLLOWING_SUB = "package Report;\n"
"format REPORT =\n"
"@<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n"
"$headline\n"
".\n"
"sub after_format { return 1; }\n";
/* A grammar refresh must not hide real Perl syntax loss. */
static const char *PERL_MALFORMED = "package Broken;\n"
"sub before_error { return 1; }\n"
"} ] } ]\n"
"sub after_error { return 2; }\n";
/* ── Tests ────────────────────────────────────────────────────────────────── */
TEST(c_ifdef_split_brace_sets_parse_incomplete) {
@@ -418,6 +434,42 @@ TEST(width_bearing_error_at_eof_still_flagged_issue1610) {
PASS();
}
/* #1838: tree-sitter-perl v1.0.0 rejects a valid line-oriented format and
* reports the file as partial. The supported upstream v1.2.1 grammar accepts
* the format and preserves declaration extraction beyond its dot terminator. */
TEST(perl_format_followed_by_named_sub_is_complete_issue1838) {
CBMFileResult *r = do_extract(PERL_FORMAT_WITH_FOLLOWING_SUB, CBM_LANG_PERL, "report.pl");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
bool partial = r->parse_incomplete;
int error_regions = r->error_region_count;
bool has_error_ranges = r->error_ranges != NULL;
bool has_following_sub = has_def(r, "after_format");
if (partial || error_regions != 0 || has_error_ranges || !has_following_sub) {
fprintf(stderr,
" Perl format result: partial=%d regions=%d ranges=%s following_sub=%d\n",
partial,
error_regions,
r->error_ranges ? r->error_ranges : "(none)",
has_following_sub);
cbm_free_result(r);
FAIL("a valid Perl format and its following named sub must parse completely");
}
cbm_free_result(r);
PASS();
}
TEST(perl_malformed_source_remains_partial_issue1838) {
CBMFileResult *r = do_extract(PERL_MALFORMED, CBM_LANG_PERL, "broken.pl");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT_TRUE(r->parse_incomplete);
ASSERT_GTE(r->error_region_count, 1);
ASSERT_NOT_NULL(r->error_ranges);
cbm_free_result(r);
PASS();
}
SUITE(parse_coverage) {
RUN_TEST(c_ifdef_split_brace_sets_parse_incomplete);
RUN_TEST(c_ifdef_split_brace_neighbors_still_extracted);
@@ -437,4 +489,6 @@ SUITE(parse_coverage) {
RUN_TEST(missing_final_newline_not_flagged_across_grammars_issue1610);
RUN_TEST(real_error_before_eof_still_flagged_without_final_newline_issue1610);
RUN_TEST(width_bearing_error_at_eof_still_flagged_issue1610);
RUN_TEST(perl_format_followed_by_named_sub_is_complete_issue1838);
RUN_TEST(perl_malformed_source_remains_partial_issue1838);
}
+65
View File
@@ -5352,6 +5352,70 @@ TEST(pipeline_parallel_rust_cross_only_macro_hidden_gets_synthetic_carrier) {
PASS();
}
/* Slash-prefixed call arguments are not necessarily HTTP routes. Keep the
* parallel arg-url heuristic from minting Route nodes for filesystem paths or
* regex-replacement operands, while preserving a genuine API path. */
TEST(pipeline_arg_url_rejects_non_http_slash_arguments) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_arg_url_guard_XXXXXX");
if (!cbm_mkdtemp(tmp)) {
FAIL("tmpdir");
}
write_temp_file(tmp, "src/args.py",
"import requests\n"
"TMP = '/tmp/pgv_fuzz.bin'\n"
"def run_copy(path):\n"
" return path\n"
"def write_fixture():\n"
" return run_copy(TMP)\n"
"def load_api():\n"
" return requests.get('/api/data')\n");
write_temp_file(tmp, "src/regex.js",
"function sink(value) { return value; }\n"
"export function sanitize(template) {\n"
" sink(/<table/i);\n"
" return template.replace('/html/g', '');\n"
"}\n");
for (int i = 0; i < 52; i++) {
char name[64];
char body[128];
snprintf(name, sizeof(name), "src/filler%d.ts", i);
snprintf(body, sizeof(body), "export function filler%d(): number { return %d; }\n", i, i);
write_temp_file(tmp, name, body);
}
char *old_workers = getenv("CBM_WORKERS");
char *saved = old_workers ? strdup(old_workers) : NULL;
cbm_setenv("CBM_WORKERS", "4", 1);
char db_path[512];
snprintf(db_path, sizeof(db_path), "%s/arg_url_guard.db", tmp);
cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL);
ASSERT_NOT_NULL(p);
ASSERT_EQ(cbm_pipeline_run(p), 0);
const char *project = cbm_pipeline_project_name(p);
cbm_store_t *s = cbm_store_open_path(db_path);
ASSERT_NOT_NULL(s);
ASSERT_EQ(count_nodes_named(s, project, "/html/g"), 0);
ASSERT_EQ(count_nodes_named(s, project, "/<table/i"), 0);
ASSERT_EQ(count_nodes_named(s, project, "/tmp/pgv_fuzz.bin"), 0);
ASSERT_GTE(count_nodes_named(s, project, "/api/data"), 1);
cbm_store_close(s);
cbm_pipeline_free(p);
if (saved) {
cbm_setenv("CBM_WORKERS", saved, 1);
free(saved);
} else {
cbm_unsetenv("CBM_WORKERS");
}
th_rmtree(tmp);
PASS();
}
/* Native `fetch()` (#856), sequential path (< 50 files → pass_calls.c). A bare
* unqualified call to the global fetch API has no import and no local
* definition anywhere in this project, so registry resolution comes back
@@ -13059,6 +13123,7 @@ SUITE(pipeline) {
RUN_TEST(pipeline_python_receiver_parallel_suppresses_weak_method_edges);
RUN_TEST(pipeline_parallel_python_cross_only_dunder_gets_synthetic_carrier);
RUN_TEST(pipeline_parallel_rust_cross_only_macro_hidden_gets_synthetic_carrier);
RUN_TEST(pipeline_arg_url_rejects_non_http_slash_arguments);
RUN_TEST(pipeline_native_fetch_classified_as_http_calls);
RUN_TEST(pipeline_native_fetch_parallel_classified_as_http_calls);
RUN_TEST(pipeline_local_fetch_shadow_not_classified_as_http);
+163
View File
@@ -13,6 +13,17 @@
#include <stdlib.h>
#include <stdio.h>
typedef struct {
int calls;
int cancel_on_call;
} file_outline_cancel_probe_t;
static bool file_outline_cancel_probe(void *context) {
file_outline_cancel_probe_t *probe = context;
probe->calls++;
return probe->calls >= probe->cancel_on_call;
}
/* ── Label allowlist / SQL drift guard ──────────────────────────── */
/* CONTRACT PIN. `cbm_label_is_type_like()` is documented in cbm.h as the single
@@ -330,6 +341,156 @@ TEST(store_node_find_by_file) {
PASS();
}
TEST(store_file_outline_is_filtered_stable_bounded_and_cancellable_issue469) {
cbm_store_t *s = cbm_store_open_memory();
ASSERT_NOT_NULL(s);
ASSERT_EQ(cbm_store_upsert_project(s, "outline", "/tmp/outline"), CBM_STORE_OK);
cbm_node_t nodes[] = {
{.project = "outline",
.label = "Method",
.name = "omega",
.qualified_name = "outline.main.omega",
.file_path = "main.c",
.start_line = 30,
.end_line = 33},
{.project = "outline",
.label = "Module",
.name = "main",
.qualified_name = "outline.main",
.file_path = "main.c",
.start_line = 1,
.end_line = 80},
{.project = "outline",
.label = "Function",
.name = "zeta",
.qualified_name = "outline.main.zeta",
.file_path = "main.c",
.start_line = 10,
.end_line = 20},
{.project = "outline",
.label = "Function",
.name = "alpha",
.qualified_name = "outline.main.alpha",
.file_path = "main.c",
.start_line = 10,
.end_line = 15},
{.project = "outline",
.label = "Class",
.name = "VisibleWithoutFilter",
.qualified_name = "outline.main.VisibleWithoutFilter",
.file_path = "main.c",
.start_line = 5,
.end_line = 40},
{.project = "outline",
.label = "Function",
.name = "other",
.qualified_name = "outline.other.other",
.file_path = "other.c",
.start_line = 1,
.end_line = 2},
};
for (size_t i = 0; i < sizeof(nodes) / sizeof(nodes[0]); i++) {
ASSERT_GT(cbm_store_upsert_node(s, &nodes[i]), 0);
}
static const char *const labels[] = {"Function", "Method"};
cbm_file_outline_row_t *rows = NULL;
int count = 0;
int total = 0;
ASSERT_EQ(cbm_store_get_file_outline(s, "outline", "main.c", labels, 2, 2, 0, NULL, NULL, &rows,
&count, &total),
CBM_STORE_OK);
ASSERT_EQ(total, 3);
ASSERT_EQ(count, 2);
ASSERT_STR_EQ(rows[0].name, "alpha");
ASSERT_STR_EQ(rows[1].name, "zeta");
cbm_store_free_file_outline(rows, count);
rows = NULL;
count = 0;
total = 0;
ASSERT_EQ(cbm_store_get_file_outline(s, "outline", "main.c", labels, 2, 2, 2, NULL, NULL, &rows,
&count, &total),
CBM_STORE_OK);
ASSERT_EQ(total, 3);
ASSERT_EQ(count, 1);
ASSERT_STR_EQ(rows[0].name, "omega");
cbm_store_free_file_outline(rows, count);
rows = NULL;
count = 0;
total = 0;
ASSERT_EQ(cbm_store_get_file_outline(s, "outline", "main.c", NULL, 0, 10, 0, NULL, NULL, &rows,
&count, &total),
CBM_STORE_OK);
ASSERT_EQ(total, 4);
ASSERT_EQ(count, 4);
ASSERT_STR_EQ(rows[0].name, "VisibleWithoutFilter");
ASSERT_STR_EQ(rows[1].name, "alpha");
ASSERT_STR_EQ(rows[2].name, "zeta");
ASSERT_STR_EQ(rows[3].name, "omega");
cbm_store_free_file_outline(rows, count);
file_outline_cancel_probe_t probe = {.cancel_on_call = 3};
rows = (cbm_file_outline_row_t *)(uintptr_t)1;
count = 99;
total = 99;
ASSERT_EQ(cbm_store_get_file_outline(s, "outline", "main.c", labels, 2, 2, 0,
file_outline_cancel_probe, &probe, &rows, &count, &total),
CBM_STORE_CANCELLED);
ASSERT_NULL(rows);
ASSERT_EQ(count, 0);
ASSERT_EQ(total, 0);
ASSERT_TRUE(probe.calls >= 3);
const char *empty_label[] = {""};
ASSERT_EQ(cbm_store_get_file_outline(s, "outline", "main.c", empty_label, 1, 1, 0, NULL, NULL,
&rows, &count, &total),
CBM_STORE_ERR);
ASSERT_EQ(cbm_store_get_file_outline(s, "outline", "main.c", NULL, 0,
CBM_STORE_FILE_OUTLINE_MAX_LIMIT + 1, 0, NULL, NULL, &rows,
&count, &total),
CBM_STORE_ERR);
cbm_store_close(s);
PASS();
}
TEST(store_file_outline_fails_closed_on_text_budget_issue469) {
cbm_store_t *s = cbm_store_open_memory();
ASSERT_NOT_NULL(s);
ASSERT_EQ(cbm_store_upsert_project(s, "outline-budget", "/tmp/outline-budget"), CBM_STORE_OK);
size_t qn_size = CBM_STORE_FILE_OUTLINE_MAX_TEXT_BYTES + 64U;
char *large_qn = malloc(qn_size + 1U);
ASSERT_NOT_NULL(large_qn);
memset(large_qn, 'q', qn_size);
large_qn[qn_size] = '\0';
cbm_node_t node = {.project = "outline-budget",
.label = "Function",
.name = "oversized",
.qualified_name = large_qn,
.file_path = "large.c",
.start_line = 1,
.end_line = 2};
ASSERT_GT(cbm_store_upsert_node(s, &node), 0);
free(large_qn);
cbm_file_outline_row_t *rows = (cbm_file_outline_row_t *)(uintptr_t)1;
int count = 99;
int total = 99;
ASSERT_EQ(cbm_store_get_file_outline(s, "outline-budget", "large.c", NULL, 0, 1, 0, NULL, NULL,
&rows, &count, &total),
CBM_STORE_SCAN_LIMIT);
ASSERT_NULL(rows);
ASSERT_EQ(count, 0);
ASSERT_EQ(total, 0);
cbm_store_close(s);
PASS();
}
TEST(store_node_find_not_found) {
cbm_store_t *s = cbm_store_open_memory();
cbm_store_upsert_project(s, "test", "/tmp/test");
@@ -2227,6 +2388,8 @@ SUITE(store_nodes) {
RUN_TEST(store_node_dedup);
RUN_TEST(store_node_find_by_label);
RUN_TEST(store_node_find_by_file);
RUN_TEST(store_file_outline_is_filtered_stable_bounded_and_cancellable_issue469);
RUN_TEST(store_file_outline_fails_closed_on_text_budget_issue469);
RUN_TEST(store_node_find_not_found);
RUN_TEST(store_node_count_empty);
RUN_TEST(store_node_delete_by_file);
+31
View File
@@ -148,3 +148,34 @@ class McpServer:
self.proc.kill()
except Exception:
pass
def wait_projects_with_stats(server, timeout=90.0, poll=1.0):
"""Poll list_projects until a project row carries node/edge counts.
index_repository returns when the pipeline finishes, but the project
stats are published asynchronously; on slow CI runners the first
list_projects can observe the registration row before its counts land
(`nodes` absent/None). That window is environmental, not a product
regression (#1952), so wait it out instead of failing setup on it.
Returns (projects, last_text): the parsed project list (possibly [])
and the last raw list_projects payload for diagnostics.
"""
deadline = time.time() + timeout
last_txt = ""
while True:
resp = server.call_tool("list_projects", {}, timeout=60)
txt, err = server.tool_text(resp)
projects = []
if not err and txt:
last_txt = txt
try:
projects = json.loads(txt).get("projects") or []
except ValueError:
projects = []
if projects and projects[0].get("nodes") is not None:
return projects, last_txt
if time.time() >= deadline:
return projects, last_txt
time.sleep(poll)
+19 -7
View File
@@ -31,6 +31,7 @@ import shutil
import subprocess
import sys
import tempfile
import time
MATH_TS = (
"export function add(a: number, b: number): number { return a + b; }\n"
@@ -67,13 +68,24 @@ def main():
# CLI path itself works and isolating the failure to argv encoding.
ascii_repo = os.path.join(work, "ascii_repo")
make_fixture(ascii_repo)
env = dict(os.environ)
env["CBM_CACHE_DIR"] = os.path.join(work, "cache_ascii")
ctrl = subprocess.run(
[binary, "cli", "index_repository",
json.dumps({"repo_path": ascii_repo})],
capture_output=True, timeout=120, env=env)
ctrl_out = (ctrl.stdout or b"").decode("utf-8", "replace")
# The control is setup, not the surface under test: a cold runner can
# lose the first one-shot to coordination-daemon startup latency
# (#1952). One retry against a fresh cache separates that from a real
# CLI indexing failure.
ctrl_out = ""
for attempt in ("cache_ascii", "cache_ascii_retry"):
env = dict(os.environ)
env["CBM_CACHE_DIR"] = os.path.join(work, attempt)
ctrl = subprocess.run(
[binary, "cli", "index_repository",
json.dumps({"repo_path": ascii_repo})],
capture_output=True, timeout=120, env=env)
ctrl_out = (ctrl.stdout or b"").decode("utf-8", "replace")
if '"nodes"' in ctrl_out:
break
print("SETUP: ASCII control attempt %r did not index via CLI:\n%s"
% (attempt, ctrl_out[:300]))
time.sleep(2)
if '"nodes"' not in ctrl_out:
print("SETUP FAIL: ASCII control did not index via CLI:\n%s" %
ctrl_out[:300])
+25 -7
View File
@@ -31,6 +31,7 @@ import shutil
import subprocess
import sys
import tempfile
import time
SYMBOL = "someIndexedSymbol"
SRC = "export function %s(a: number): number { return a + 1; }\n" % SYMBOL
@@ -63,9 +64,19 @@ def main():
# repo_path / cwd in the forward-slash drive form Claude Code passes.
repo_fwd = repo.replace("\\", "/")
idx = run_cli(binary, cache, ["cli", "index_repository",
json.dumps({"repo_path": repo_fwd})])
idx_out = (idx.stdout or b"").decode("utf-8", "replace")
# Setup, not the surface under test: retry once so a cold runner's
# coordination-daemon startup latency (#1952) is not misread as a
# broken CLI index. Reindexing the same cache is idempotent.
idx_out = ""
for attempt in (1, 2):
idx = run_cli(binary, cache, ["cli", "index_repository",
json.dumps({"repo_path": repo_fwd})])
idx_out = (idx.stdout or b"").decode("utf-8", "replace")
if '"nodes"' in idx_out:
break
print("SETUP: index attempt %d did not run:\n%s"
% (attempt, idx_out[:300]))
time.sleep(2)
if '"nodes"' not in idx_out:
print("SETUP FAIL: index did not run:\n%s" % idx_out[:300])
return 2
@@ -88,10 +99,17 @@ def main():
# first — the supported way to keep hooks armed outside MCP sessions —
# and retire it afterwards (with a kill-by-pid backstop so a stuck stop
# can never hang CI).
start = run_cli(binary, cache, ["daemon", "start"], timeout=60)
start_out = (start.stdout or b"").decode("utf-8", "replace")
print("daemon start rc=%d %r" % (start.returncode, start_out[:120]))
if start.returncode != 0:
start_out = ""
rc = 1
for attempt in (1, 2):
start = run_cli(binary, cache, ["daemon", "start"], timeout=60)
start_out = (start.stdout or b"").decode("utf-8", "replace")
rc = start.returncode
print("daemon start attempt %d rc=%d %r" % (attempt, rc, start_out[:120]))
if rc == 0:
break
time.sleep(2)
if rc != 0:
print("SETUP FAIL: permanent daemon did not start")
return 2
pid_match = re.search(r"pid (\d+)", start_out)
+181 -45
View File
@@ -2,8 +2,8 @@
Guards the fix for issue #636 / #357 (landed on main via #700) at the product
surface (real codebase-memory-mcp process, real SQLite DB, real stdio). Two
byte-identical TypeScript fixtures are indexed: one under an ASCII parent path,
one under a non-ASCII parent path. The invariant under test:
byte-identical polyglot fixtures (TypeScript + Go, #1959) are indexed: one under
an ASCII parent path, one under a non-ASCII parent path. The invariant under test:
A byte-identical fixture must produce equivalent graph counts regardless of
whether its absolute path contains non-ASCII characters.
@@ -34,7 +34,7 @@ import sys
import tempfile
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from mcp_stdio import McpServer # noqa: E402
from mcp_stdio import McpServer, wait_projects_with_stats # noqa: E402
MATH_TS = (
"export function add(a: number, b: number): number { return a + b; }\n"
@@ -54,6 +54,81 @@ MAIN_TS = (
"run();\n"
)
# Dummy Go package covering the shapes the Go/cgo extraction work (#1932
# family) showed were previously invisible to CI: struct fields and a method,
# a build-tag twin pair sharing one function name, a cgo file (C preamble,
# `C.` call, `//export` directive), and a channel producer/consumer pair
# (#1959). Go qualified names embed the containing directory, so these also
# put non-ASCII repo paths through the Go pipeline passes.
GO_MOD = "module example.com/fixture\n\ngo 1.22\n"
GO_STORE = """package gopkg
type Store struct {
Total int
Name string
}
func (s *Store) Push(x int) {
s.Total = s.Total + x
}
func NewStore(name string) *Store {
return &Store{Name: name}
}
"""
GO_FLUSH_LINUX = """//go:build linux
package gopkg
func FlushDisk() int { return 1 }
"""
GO_FLUSH_WINDOWS = """//go:build windows
package gopkg
func FlushDisk() int { return 2 }
"""
GO_BRIDGE = """package gopkg
/*
#include <stdio.h>
static void announce(void) { puts("hi"); }
*/
import "C"
//export FixtureReady
func FixtureReady() int { return 3 }
func Announce() {
C.announce()
}
"""
GO_EVENTS = """package gopkg
func Produce(events chan string) {
events <- "tick"
}
func Drain(events chan string) string {
v := <-events
return v
}
"""
GO_FILES = (
("go.mod", GO_MOD),
("gopkg/store.go", GO_STORE),
("gopkg/flush_linux.go", GO_FLUSH_LINUX),
("gopkg/flush_windows.go", GO_FLUSH_WINDOWS),
("gopkg/bridge.go", GO_BRIDGE),
("gopkg/events.go", GO_EVENTS),
)
# Distinct non-ASCII scripts — each must behave like the ASCII baseline.
NON_ASCII_SEGMENTS = {
"latin1_accents": "café_repo",
@@ -244,6 +319,42 @@ def make_fixture(root):
for name, text in (("math.ts", MATH_TS), ("main.ts", MAIN_TS)):
with open(os.path.join(src, name), "wb") as f:
f.write(text.encode("utf-8")) # exact bytes, identical across copies
for rel, text in GO_FILES:
path = os.path.join(root, rel.replace("/", os.sep))
os.makedirs(os.path.dirname(path) or root, exist_ok=True)
with open(path, "wb") as f:
f.write(text.encode("utf-8"))
def no_project_error(index_txt, repo, cache):
"""Assemble the venue diagnostics for an index that left no project row.
The count summary cannot explain a venue-specific empty listing; carry the
index response, the cache contents and the supervisor's worker logs (the
only record of the pipeline's own error) into the CI log.
"""
try:
cache_entries = sorted(os.listdir(cache))
except OSError as exc:
cache_entries = ["<listdir failed: %s>" % exc]
log_tails = []
logs_dir = os.path.join(cache, "logs")
if os.path.isdir(logs_dir):
for log_name in sorted(os.listdir(logs_dir)):
try:
with open(os.path.join(logs_dir, log_name), "rb") as lf:
tail = lf.read()[-800:].decode("utf-8", "replace")
log_tails.append("%s: %s" % (log_name, tail))
except OSError as exc:
log_tails.append("%s: <unreadable: %s>" % (log_name, exc))
try:
repo_entries = sorted(os.listdir(repo))
except OSError as exc:
repo_entries = ["<listdir failed: %s>" % exc]
return {"error": "no project listed after index; index said %r; "
"cache holds %r; repo holds %r; worker logs: %s"
% (index_txt[:400], cache_entries, repo_entries,
" | ".join(log_tails) or "<none>")}
def index_and_count(binary, repo, cache):
@@ -255,42 +366,29 @@ def index_and_count(binary, repo, cache):
index_txt, err = s.tool_text(resp)
if err:
return {"error": "index tools/call error: %r" % err}
lp = s.call_tool("list_projects", {}, timeout=60)
lp_txt, _ = s.tool_text(lp)
projects = json.loads(lp_txt).get("projects") or []
if not projects:
# The count summary cannot explain a venue-specific empty listing;
# carry the index response and the cache contents into the log.
try:
cache_entries = sorted(os.listdir(cache))
except OSError as exc:
cache_entries = ["<listdir failed: %s>" % exc]
# The supervisor's worker logs carry the actual pipeline error.
log_tails = []
logs_dir = os.path.join(cache, "logs")
if os.path.isdir(logs_dir):
for log_name in sorted(os.listdir(logs_dir)):
try:
with open(os.path.join(logs_dir, log_name), "rb") as lf:
tail = lf.read()[-800:].decode("utf-8", "replace")
log_tails.append("%s: %s" % (log_name, tail))
except OSError as exc:
log_tails.append("%s: <unreadable: %s>" % (log_name, exc))
try:
repo_entries = sorted(os.listdir(repo))
except OSError as exc:
repo_entries = ["<listdir failed: %s>" % exc]
return {"error": "no project listed after index; index said %r; "
"cache holds %r; repo holds %r; worker logs: %s"
% (index_txt[:400], cache_entries, repo_entries,
" | ".join(log_tails) or "<none>")}
p = projects[0]
out = {"name": p.get("name"), "nodes": p.get("nodes"),
"edges": p.get("edges")}
# The index response itself carries the synchronous, authoritative
# counts ("nodes"/"edges"). list_projects publishes its stats columns
# asynchronously — on some venues never within a one-shot session — so
# gating on it misreads a healthy index as a setup failure (#1952).
try:
summary = json.loads(index_txt)
except ValueError:
summary = {}
out = {"name": summary.get("project"), "nodes": summary.get("nodes"),
"edges": summary.get("edges")}
if out["nodes"] is None:
# Payload without counts: fall back to list_projects, polled
# because its stats row can trail the index on slow runners.
projects, _ = wait_projects_with_stats(s)
if not projects:
return no_project_error(index_txt, repo, cache)
p = projects[0]
out = {"name": p.get("name"), "nodes": p.get("nodes"),
"edges": p.get("edges")}
# Definition-level counts prove the parser ran (not just discovery).
# query_graph defaults to TOON text; this scripted consumer requests
# format="json" ({"columns":[...],"rows":[["<n>"]],...}) explicitly.
name = p.get("name")
name = out["name"]
defs = 0
for label in ("Function", "Class", "Method"):
q = "MATCH (n:%s) RETURN count(n)" % label
@@ -305,6 +403,25 @@ def index_and_count(binary, repo, cache):
except Exception:
pass
out["definition_nodes"] = defs
# Go-file definitions separately: the Go passes derive qualified names
# from the containing directory, so they meet non-ASCII paths on a
# different route than the TS passes; a Go-specific count catches a
# regression that total counts could mask (#1959).
go_defs = 0
for label in ("Function", "Method"):
q = ("MATCH (n:%s) WHERE n.file_path CONTAINS '.go' "
"RETURN count(n)" % label)
r = s.call_tool("query_graph",
{"query": q, "project": name, "format": "json"},
timeout=60)
t, _ = s.tool_text(r)
try:
rows = json.loads(t).get("rows") or []
if rows and rows[0]:
go_defs += int(rows[0][0])
except Exception:
pass
out["go_definition_nodes"] = go_defs
return out
@@ -331,15 +448,30 @@ def main():
ascii_repo = os.path.join(work, "ascii_repo")
make_fixture(ascii_repo)
base = index_and_count(binary, ascii_repo, os.path.join(work, "c_ascii"))
# The baseline is setup, not the surface under test: a cold runner can
# lose the first index to daemon startup latency (#1952). One retry
# against a fresh cache separates that environmental window from a
# real indexing failure before the guard declares a precondition skip.
base = {}
for attempt in ("c_ascii", "c_ascii_retry"):
base = index_and_count(binary, ascii_repo, os.path.join(work, attempt))
if not base.get("error") and base.get("nodes"):
break
print("SETUP: ASCII baseline attempt %r did not index: %r"
% (attempt, base))
if base.get("error") or not base.get("nodes"):
print("SETUP FAIL: ASCII baseline did not index: %r" % base)
return 2
print("baseline (ASCII): nodes=%s edges=%s definitions=%s" %
(base["nodes"], base["edges"], base["definition_nodes"]))
print("baseline (ASCII): nodes=%s edges=%s definitions=%s go=%s" %
(base["nodes"], base["edges"], base["definition_nodes"],
base["go_definition_nodes"]))
if base["definition_nodes"] < 1:
print("SETUP FAIL: ASCII baseline produced no definitions: %r" % base)
return 2
if base["go_definition_nodes"] < 1:
print("SETUP FAIL: ASCII baseline extracted no Go definitions: %r"
% base)
return 2
for key, seg in NON_ASCII_SEGMENTS.items():
repo = os.path.join(work, seg)
@@ -348,13 +480,15 @@ def main():
ok = (not got.get("error")
and got.get("nodes") == base["nodes"]
and got.get("edges") == base["edges"]
and got.get("definition_nodes") == base["definition_nodes"])
and got.get("definition_nodes") == base["definition_nodes"]
and got.get("go_definition_nodes") == base["go_definition_nodes"])
status = "PASS" if ok else "FAIL"
print("[%s] non-ascii/%-14s nodes=%s edges=%s definitions=%s "
"(baseline %s/%s/%s) name=%r" %
print("[%s] non-ascii/%-14s nodes=%s edges=%s definitions=%s go=%s "
"(baseline %s/%s/%s/%s) name=%r" %
(status, key, got.get("nodes"), got.get("edges"),
got.get("definition_nodes"), base["nodes"], base["edges"],
base["definition_nodes"], got.get("name")))
got.get("definition_nodes"), got.get("go_definition_nodes"),
base["nodes"], base["edges"], base["definition_nodes"],
base["go_definition_nodes"], got.get("name")))
if not ok:
# The counts alone cannot explain a venue-specific failure;
# surface the captured error verbatim so CI logs carry the
@@ -372,7 +506,9 @@ def main():
if (not retry.get("error")
and retry.get("nodes") == base["nodes"]
and retry.get("edges") == base["edges"]
and retry.get("definition_nodes") == base["definition_nodes"]):
and retry.get("definition_nodes") == base["definition_nodes"]
and retry.get("go_definition_nodes")
== base["go_definition_nodes"]):
print(" %s retry matched baseline -- order-dependent, "
"not path-dependent" % key)
else: