mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-11-09 09:11:00 +08:00
Merge branch 'main' into fix/issue-1682-dotted-ts-imports
This commit is contained in:
@@ -42,7 +42,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 240
|
||||
steps:
|
||||
- name: Wait for CodeQL on current commit (max 45 min)
|
||||
- name: Wait for CodeQL on current commit (max 150 min)
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
@@ -50,11 +50,17 @@ jobs:
|
||||
# CodeQL runs are recorded against the PR head SHA.
|
||||
CURRENT_SHA="${{ github.event.pull_request.head.sha || github.sha }}"
|
||||
echo "Waiting for CodeQL to complete on $CURRENT_SHA..."
|
||||
for attempt in $(seq 1 90); do
|
||||
# 300 attempts x 30s = 150 min. The previous budget was 90 x 30s = 45 min,
|
||||
# which is shorter than CodeQL actually takes on this repository: a measured
|
||||
# run on PR #1426 (head 7b72652a) completed with conclusion=success after
|
||||
# 124 min, 7 minutes AFTER this gate had already given up. That marked at
|
||||
# least five contributor PRs red for a scan that passed. The job's own
|
||||
# timeout-minutes is 240, so 150 still leaves headroom.
|
||||
for attempt in $(seq 1 300); do
|
||||
LATEST=$(gh api "repos/${{ github.repository }}/actions/workflows/codeql.yml/runs?head_sha=$CURRENT_SHA&per_page=1" \
|
||||
--jq '.workflow_runs[] | "\(.conclusion) \(.status)"' 2>/dev/null | head -1 || echo "")
|
||||
if [ -z "$LATEST" ]; then
|
||||
echo " $attempt/90: no run yet..."; sleep 30; continue
|
||||
echo " $attempt/300: no run yet..."; sleep 30; continue
|
||||
fi
|
||||
CONCLUSION=$(echo "$LATEST" | cut -d' ' -f1)
|
||||
STATUS=$(echo "$LATEST" | cut -d' ' -f2)
|
||||
@@ -63,7 +69,7 @@ jobs:
|
||||
elif [ "$STATUS" = "completed" ]; then
|
||||
echo "BLOCKED: CodeQL $CONCLUSION"; exit 1
|
||||
fi
|
||||
echo " $attempt/90: $STATUS..."; sleep 30
|
||||
echo " $attempt/300: $STATUS..."; sleep 30
|
||||
done
|
||||
echo "BLOCKED: CodeQL timeout"; exit 1
|
||||
|
||||
|
||||
@@ -1161,6 +1161,65 @@ static char *extract_nickel_callee(CBMArena *a, TSNode node, const char *source,
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// Pkl: `unqualifiedAccessExpr` / `qualifiedAccessExpr` are the same node whether
|
||||
// they are a call (`helper(a)`) or a bare property read (`host`) — the only
|
||||
// discriminator is an `argumentList` child, so both are gated on it. For a
|
||||
// qualified call the method name is the `identifier` child that is not the
|
||||
// `receiver`; the receiver is prefixed only when it is itself a plain name
|
||||
// (`utils.fallback(a)` -> "utils.fallback", module-qualified, which cbm.c
|
||||
// shortens to the last dotted segment when resolving). A receiver that is itself
|
||||
// a call must NOT be prefixed: `s.trim().toLowerCase()` -> "toLowerCase", since
|
||||
// the receiver's text carries parens and would never resolve.
|
||||
// `newExpr` resolves to its `declaredType` so `new Server {}` links to the class.
|
||||
static char *extract_pkl_callee(CBMArena *a, TSNode node, const char *source, const char *nk) {
|
||||
if (strcmp(nk, "newExpr") == 0) {
|
||||
// `new { ... }` with an inferred type has no declaredType child.
|
||||
TSNode dt = cbm_find_child_by_kind(node, "declaredType");
|
||||
return ts_node_is_null(dt) ? NULL : cbm_node_text(a, dt, source);
|
||||
}
|
||||
|
||||
bool qualified = strcmp(nk, "qualifiedAccessExpr") == 0;
|
||||
if (!qualified && strcmp(nk, "unqualifiedAccessExpr") != 0) {
|
||||
return NULL;
|
||||
}
|
||||
// No argument list -> property read, not a call.
|
||||
if (ts_node_is_null(cbm_find_child_by_kind(node, "argumentList"))) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
TSNode recv = ts_node_child_by_field_name(node, TS_FIELD("receiver"));
|
||||
TSNode name = (TSNode){0};
|
||||
uint32_t nc = ts_node_named_child_count(node);
|
||||
for (uint32_t i = 0; i < nc; i++) {
|
||||
TSNode child = ts_node_named_child(node, i);
|
||||
if (!ts_node_is_null(recv) && ts_node_eq(child, recv)) {
|
||||
continue;
|
||||
}
|
||||
if (strcmp(ts_node_type(child), "identifier") == 0) {
|
||||
name = child;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (ts_node_is_null(name)) {
|
||||
return NULL;
|
||||
}
|
||||
char *mn = cbm_node_text(a, name, source);
|
||||
if (!mn || !mn[0]) {
|
||||
return NULL;
|
||||
}
|
||||
if (!qualified || ts_node_is_null(recv)) {
|
||||
return mn;
|
||||
}
|
||||
if (strcmp(ts_node_type(recv), "unqualifiedAccessExpr") == 0 &&
|
||||
ts_node_is_null(cbm_find_child_by_kind(recv, "argumentList"))) {
|
||||
char *rt = cbm_node_text(a, recv, source);
|
||||
if (rt && rt[0]) {
|
||||
return cbm_arena_sprintf(a, "%s.%s", rt, mn);
|
||||
}
|
||||
}
|
||||
return mn;
|
||||
}
|
||||
|
||||
// Typst: a `call` node's callee is its `item` field (an ident), matching the
|
||||
// def-side resolution of `#let greet(name) = ...`.
|
||||
static char *extract_typst_callee(CBMArena *a, TSNode node, const char *source, const char *nk) {
|
||||
@@ -1789,6 +1848,15 @@ static char *extract_callee_name(CBMArena *a, TSNode node, const char *source, C
|
||||
}
|
||||
}
|
||||
|
||||
/* Pkl: resolve here and return unconditionally — the access-expr call node
|
||||
* types double as plain property reads, so falling through to field-based or
|
||||
* generic first-identifier resolution would mint a CALLS edge for every
|
||||
* property read (a bare `host` has an `identifier` first child, which the
|
||||
* generic fallback would happily emit). NULL here means "not a call". */
|
||||
if (lang == CBM_LANG_PKL) {
|
||||
return extract_pkl_callee(a, node, source, ts_node_type(node));
|
||||
}
|
||||
|
||||
// Helm / Go templates: resolve `include "x"` / `template "x"` to the
|
||||
// referenced named template so it links to the define'd Function (#338).
|
||||
if (lang == CBM_LANG_GOTEMPLATE) {
|
||||
|
||||
@@ -378,6 +378,7 @@ typedef enum {
|
||||
CBM_OCCURRENCE_VHDL_INTERFACE,
|
||||
CBM_OCCURRENCE_PINE_FUNCTION,
|
||||
CBM_OCCURRENCE_LLVM_FUNCTION,
|
||||
CBM_OCCURRENCE_PKL_DECLARATION,
|
||||
} CBMOccurrencePolicy;
|
||||
|
||||
typedef struct {
|
||||
@@ -497,6 +498,7 @@ static const CBMOccurrenceSpec occurrence_specs[CBM_LANG_COUNT] = {
|
||||
[CBM_LANG_PINE] = {NULL, NULL, CBM_OCCURRENCE_PINE_FUNCTION, false},
|
||||
[CBM_LANG_PUPPET] = {NULL, NULL, CBM_OCCURRENCE_STANDARD, true},
|
||||
[CBM_LANG_LLVM_IR] = {llvm_binding_nodes, NULL, CBM_OCCURRENCE_LLVM_FUNCTION, false},
|
||||
[CBM_LANG_PKL] = {NULL, NULL, CBM_OCCURRENCE_PKL_DECLARATION, false},
|
||||
[CBM_LANG_MESON] = {NULL, meson_write_nodes, CBM_OCCURRENCE_STANDARD, true},
|
||||
[CBM_LANG_GN] = {NULL, gn_write_nodes, CBM_OCCURRENCE_STANDARD, true},
|
||||
[CBM_LANG_LINKERSCRIPT] = {NULL, linkerscript_write_nodes, CBM_OCCURRENCE_STANDARD, true},
|
||||
@@ -1006,6 +1008,30 @@ static bool is_pine_function_binding(TSNode node) {
|
||||
return ancestor_field_binds(node, "function_declaration_statement", fields);
|
||||
}
|
||||
|
||||
/* Pkl declares names positionally: no production labels the declared identifier
|
||||
* with a `name` field, so the generic declared-container rule never binds them
|
||||
* and every method name, parameter name, and property name would be re-emitted
|
||||
* as an ordinary read of itself. Each container below holds its declared name
|
||||
* as named child 0; annotations, defaults, and bodies follow it and stay reads.
|
||||
* Resolve against the NEAREST container so a nested declaration's own name is
|
||||
* the only occurrence its parent can bind. */
|
||||
static bool is_pkl_declaration_binding(TSNode node) {
|
||||
static const char *const declaration_kinds[] = {"methodHeader",
|
||||
"typedIdentifier",
|
||||
"classProperty",
|
||||
"objectProperty",
|
||||
"clazz",
|
||||
"typeAlias",
|
||||
NULL};
|
||||
for (TSNode parent = ts_node_parent(node); !ts_node_is_null(parent);
|
||||
parent = ts_node_parent(parent)) {
|
||||
if (kind_in_exact_set(ts_node_type(parent), declaration_kinds)) {
|
||||
return named_child_contains(parent, 0, node);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool is_policy_binding(CBMExtractCtx *ctx, TSNode node,
|
||||
const CBMOccurrenceSpec *occurrence) {
|
||||
switch (occurrence->policy) {
|
||||
@@ -1062,6 +1088,8 @@ static bool is_policy_binding(CBMExtractCtx *ctx, TSNode node,
|
||||
return is_vhdl_interface_binding(node);
|
||||
case CBM_OCCURRENCE_PINE_FUNCTION:
|
||||
return is_pine_function_binding(node);
|
||||
case CBM_OCCURRENCE_PKL_DECLARATION:
|
||||
return is_pkl_declaration_binding(node);
|
||||
case CBM_OCCURRENCE_LLVM_FUNCTION:
|
||||
for (TSNode parent = ts_node_parent(node); !ts_node_is_null(parent);
|
||||
parent = ts_node_parent(parent)) {
|
||||
|
||||
+8
-23
@@ -651,29 +651,14 @@ int cbm_count_branching(TSNode node, const char **branching_types) {
|
||||
|
||||
// Loop node-type names across tree-sitter grammars, for loop-nesting depth.
|
||||
bool cbm_is_loop_node_type(const char *kind) {
|
||||
static const char *const loops[] = {"for_statement",
|
||||
"while_statement",
|
||||
"do_statement",
|
||||
"do_while_statement",
|
||||
"for_in_statement",
|
||||
"for_of_statement",
|
||||
"for_each_statement",
|
||||
"foreach_statement",
|
||||
"enhanced_for_statement",
|
||||
"for_range_loop",
|
||||
"c_style_for_statement",
|
||||
"for_expression",
|
||||
"while_expression",
|
||||
"loop_expression",
|
||||
"while_let_expression",
|
||||
"repeat_statement",
|
||||
"repeat_while_statement",
|
||||
"until",
|
||||
"while_modifier",
|
||||
"until_modifier",
|
||||
"for",
|
||||
"while",
|
||||
NULL};
|
||||
static const char *const loops[] = {
|
||||
"for_statement", "while_statement", "do_statement", "do_while_statement",
|
||||
"for_in_statement", "for_of_statement", "for_each_statement", "foreach_statement",
|
||||
"enhanced_for_statement", "for_range_loop", "c_style_for_statement", "for_expression",
|
||||
"while_expression", "loop_expression", "while_let_expression", "repeat_statement",
|
||||
"repeat_while_statement",
|
||||
// Pkl: `for (x in xs) { ... }` inside an object body.
|
||||
"forGenerator", "until", "while_modifier", "until_modifier", "for", "while", NULL};
|
||||
for (const char *const *l = loops; *l; l++) {
|
||||
if (strcmp(kind, *l) == 0) {
|
||||
return true;
|
||||
|
||||
@@ -1594,11 +1594,21 @@ static const char *tlaplus_branch_types[] = {"if_then_else", "case", NULL};
|
||||
static const char *tlaplus_var_types[] = {"variable_declaration", NULL};
|
||||
static const char *tlaplus_module_types[] = {"source_file", NULL};
|
||||
static const char *pkl_func_types[] = {"classMethod", "objectMethod", NULL};
|
||||
static const char *pkl_class_types[] = {"clazz", NULL};
|
||||
static const char *pkl_import_types[] = {"importClause", "extendsOrAmendsClause", "extends",
|
||||
"import", NULL};
|
||||
static const char *pkl_class_types[] = {"clazz", "typeAlias", NULL};
|
||||
static const char *pkl_import_types[] = {
|
||||
"importClause", "importGlobClause", "importExpr", "extendsOrAmendsClause",
|
||||
"extends", "import", NULL};
|
||||
static const char *pkl_var_types[] = {"classProperty", "objectProperty", NULL};
|
||||
static const char *pkl_module_types[] = {"module", NULL};
|
||||
/* Both access exprs double as plain property reads; extract_pkl_callee keeps
|
||||
* only the ones carrying an argumentList. `newExpr` resolves to its type. */
|
||||
static const char *pkl_call_types[] = {"unqualifiedAccessExpr", "qualifiedAccessExpr", "newExpr",
|
||||
NULL};
|
||||
/* Control-flow only, matching every other spec (short-circuit operators are
|
||||
* deliberately excluded). `forGenerator` is also a loop — see helpers.c. */
|
||||
static const char *pkl_branch_types[] = {"ifExpr", "whenGenerator", "forGenerator", NULL};
|
||||
static const char *pkl_throw_types[] = {"throwExpr", NULL};
|
||||
static const char *pkl_decorator_types[] = {"annotation", NULL};
|
||||
static const char *gomod_var_types[] = {"require_directive", "replace_directive", NULL};
|
||||
static const char *gomod_import_types[] = {"require", NULL};
|
||||
static const char *gomod_module_types[] = {"source_file", NULL};
|
||||
@@ -2630,9 +2640,9 @@ static const CBMLangSpec lang_specs[CBM_LANG_COUNT] = {
|
||||
|
||||
// CBM_LANG_PKL
|
||||
[CBM_LANG_PKL] = {CBM_LANG_PKL, pkl_func_types, pkl_class_types, empty_types, pkl_module_types,
|
||||
empty_types, pkl_import_types, empty_types, empty_types, pkl_var_types,
|
||||
empty_types, empty_types, NULL, empty_types, NULL, NULL, tree_sitter_pkl,
|
||||
NULL},
|
||||
pkl_call_types, pkl_import_types, empty_types, pkl_branch_types,
|
||||
pkl_var_types, empty_types, pkl_throw_types, NULL, pkl_decorator_types, NULL,
|
||||
NULL, tree_sitter_pkl, NULL},
|
||||
|
||||
// CBM_LANG_GOMOD
|
||||
[CBM_LANG_GOMOD] = {CBM_LANG_GOMOD, empty_types, empty_types, empty_types, gomod_module_types,
|
||||
|
||||
@@ -2818,7 +2818,10 @@ static const CBMRegisteredFunc *c_lookup_member_depth(CLSPContext *ctx, const ch
|
||||
const char *shortn = dot ? dot + 1 : type_qn;
|
||||
size_t slen = strlen(shortn);
|
||||
const char *best_qn = NULL;
|
||||
for (int i = 0; i < ctx->registry->type_count; i++) {
|
||||
CBMTypeNameIter it;
|
||||
cbm_registry_types_by_short_name(ctx->registry, shortn, &it);
|
||||
int i;
|
||||
while ((i = cbm_type_name_iter_next(&it)) >= 0) {
|
||||
const char *q = ctx->registry->types[i].qualified_name;
|
||||
if (!q) {
|
||||
continue;
|
||||
@@ -5972,6 +5975,7 @@ CBMTypeRegistry *cbm_c_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, in
|
||||
c_register_lsp_defs(arena, reg, "", d, 1);
|
||||
}
|
||||
cbm_registry_finalize(reg);
|
||||
cbm_registry_build_type_short_index(reg);
|
||||
reg->read_only = true; /* seal: shared Tier-2 registry is read-only during resolve */
|
||||
return reg;
|
||||
}
|
||||
@@ -6055,6 +6059,7 @@ void cbm_run_c_lsp_cross(CBMArena *arena, const char *source, int source_len, co
|
||||
// Finalize registry — O(1) lookups. See go_lsp.c "3c. Finalize"
|
||||
// comment for the rationale (linear-scan fallback otherwise).
|
||||
cbm_registry_finalize(®);
|
||||
cbm_registry_build_type_short_index(®);
|
||||
|
||||
// Initialize context and run
|
||||
CLSPContext ctx;
|
||||
|
||||
+85
-40
@@ -866,11 +866,53 @@ const CBMType* go_eval_builtin_call(GoLSPContext* ctx, const char* name, TSNode
|
||||
|
||||
// --- go_lookup_field: struct field lookup with embedding recursion ---
|
||||
|
||||
// --- Import-alias re-qualification ------------------------------------
|
||||
//
|
||||
// parse_field_defs_into_type qualifies struct field type texts as
|
||||
// "<def_module>.<text>". When the author wrote the text through an import
|
||||
// alias ("Svc:svc.Svc" in module test.main), that yields a QN
|
||||
// ("test.main.svc.Svc") that exists nowhere in the project-wide registry —
|
||||
// the real QN is "<import_qn>.Svc" and only the calling file's import map
|
||||
// can say so. On an exact-QN miss this rewrites the alias segment through
|
||||
// the file's imports; returns NULL when no alias segment is involved.
|
||||
|
||||
static const char *go_requalify_via_imports(GoLSPContext *ctx, const char *type_qn) {
|
||||
if (!ctx || !type_qn || !type_qn[0] || ctx->import_count <= 0) return NULL;
|
||||
for (int j = 0; j < ctx->import_count; j++) {
|
||||
const char *alias = ctx->import_local_names[j];
|
||||
const char *alias_qn = ctx->import_package_qns[j];
|
||||
if (!alias || !alias[0] || !alias_qn || strchr(alias, '.')) continue;
|
||||
size_t alias_len = strlen(alias);
|
||||
/* Last occurrence of a "<dot>alias<dot>" segment in type_qn. */
|
||||
const char *hit = NULL;
|
||||
for (const char *p = type_qn;;) {
|
||||
p = strstr(p, ".");
|
||||
if (!p) break;
|
||||
p++;
|
||||
if (strncmp(p, alias, alias_len) == 0 && p[alias_len] == '.') {
|
||||
hit = p;
|
||||
p += alias_len;
|
||||
}
|
||||
}
|
||||
if (hit) {
|
||||
const char *rest = hit + alias_len + 1; /* past "<alias>." */
|
||||
return cbm_arena_sprintf(ctx->arena, "%s.%s", alias_qn, rest);
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static const CBMType* go_lookup_field(GoLSPContext* ctx,
|
||||
const char* type_qn, const char* field_name, int depth) {
|
||||
if (!type_qn || !field_name || depth > 5) return NULL;
|
||||
|
||||
const CBMRegisteredType* rt = cbm_registry_lookup_type(ctx->registry, type_qn);
|
||||
if (!rt && depth == 0) {
|
||||
/* Import-alias re-qualification: field texts from cross-package defs
|
||||
* may embed an alias segment only this file's import map resolves. */
|
||||
const char* alt_qn = go_requalify_via_imports(ctx, type_qn);
|
||||
if (alt_qn) rt = cbm_registry_lookup_type(ctx->registry, alt_qn);
|
||||
}
|
||||
if (!rt) return NULL;
|
||||
|
||||
// Follow alias chain
|
||||
@@ -907,6 +949,18 @@ static const CBMRegisteredFunc* go_lookup_field_or_method_depth(GoLSPContext* ct
|
||||
const CBMRegisteredFunc* f = cbm_registry_lookup_method(ctx->registry, type_qn, member_name);
|
||||
if (f) return f;
|
||||
|
||||
/* Import-alias re-qualification fallback: NAMED receivers built from
|
||||
* cross-package field type texts can carry a "<module>.svc.Svc" QN;
|
||||
* retry the method set on the import-resolved QN (see
|
||||
* go_requalify_via_imports). */
|
||||
if (depth == 0) {
|
||||
const char* alt_qn = go_requalify_via_imports(ctx, type_qn);
|
||||
if (alt_qn) {
|
||||
f = go_lookup_field_or_method_depth(ctx, alt_qn, member_name, depth + 1);
|
||||
if (f) return f;
|
||||
}
|
||||
}
|
||||
|
||||
const CBMRegisteredType* rt = cbm_registry_lookup_type(ctx->registry, type_qn);
|
||||
if (rt) {
|
||||
// Follow type alias chain
|
||||
@@ -1430,20 +1484,32 @@ static void resolve_calls_in_node_inner(GoLSPContext* ctx, TSNode node) {
|
||||
if (base && base->kind == CBM_TYPE_POINTER) base = cbm_type_deref(base);
|
||||
|
||||
if (base && base->kind == CBM_TYPE_NAMED) {
|
||||
const char *recv_qn = base->data.named.qualified_name;
|
||||
const CBMRegisteredType *receiver_type = cbm_registry_lookup_type(
|
||||
ctx->registry, base->data.named.qualified_name);
|
||||
ctx->registry, recv_qn);
|
||||
const char *alt_qn = NULL;
|
||||
/* Re-qualify NAMED receivers that embed an import
|
||||
* alias segment (cross-package field type texts) —
|
||||
* the real type only exists under the import QN. */
|
||||
if (!receiver_type) {
|
||||
alt_qn = go_requalify_via_imports(ctx, recv_qn);
|
||||
if (alt_qn) {
|
||||
receiver_type = cbm_registry_lookup_type(ctx->registry, alt_qn);
|
||||
if (receiver_type) recv_qn = alt_qn;
|
||||
}
|
||||
}
|
||||
/* Registered interface receivers must reach the
|
||||
* interface-resolution branch below. Their semantic
|
||||
* method registrations are signatures, not concrete
|
||||
* dispatch targets. */
|
||||
if (!receiver_type || !receiver_type->is_interface) {
|
||||
const CBMRegisteredFunc *method = go_lookup_field_or_method(
|
||||
ctx, base->data.named.qualified_name, field_name);
|
||||
ctx, recv_qn, field_name);
|
||||
if (method) {
|
||||
const char *strategy = "lsp_type_dispatch";
|
||||
if (method->receiver_type &&
|
||||
strcmp(method->receiver_type,
|
||||
base->data.named.qualified_name) != 0) {
|
||||
recv_qn) != 0) {
|
||||
strategy = "lsp_embed_dispatch";
|
||||
}
|
||||
emit_resolved_call(ctx, method->qualified_name, strategy, 0.95f,
|
||||
@@ -1460,9 +1526,15 @@ static void resolve_calls_in_node_inner(GoLSPContext* ctx, TSNode node) {
|
||||
if (!is_iface && base->kind == CBM_TYPE_NAMED) {
|
||||
const CBMRegisteredType* rt = cbm_registry_lookup_type(ctx->registry,
|
||||
base->data.named.qualified_name);
|
||||
if (!rt) {
|
||||
const char* alt_qn = go_requalify_via_imports(
|
||||
ctx, base->data.named.qualified_name);
|
||||
if (alt_qn)
|
||||
rt = cbm_registry_lookup_type(ctx->registry, alt_qn);
|
||||
}
|
||||
if (rt && rt->is_interface) {
|
||||
is_iface = true;
|
||||
iface_qn = base->data.named.qualified_name;
|
||||
iface_qn = rt->qualified_name;
|
||||
}
|
||||
}
|
||||
if (is_iface) {
|
||||
@@ -1787,42 +1859,15 @@ static void process_function(GoLSPContext* ctx, TSNode func_node) {
|
||||
char* func_name = lsp_node_text(ctx, name_node);
|
||||
if (!func_name || !func_name[0]) return;
|
||||
|
||||
// For methods, the enclosing-function QN must include the receiver type
|
||||
// (package.Type.Method), matching how the textual extractor and the
|
||||
// registry qualify the method. Building it as package.Method (no receiver)
|
||||
// here made the LSP-resolved call's caller_qn disagree with the textual
|
||||
// call's enclosing_func_qn, so cbm_pipeline_find_lsp_resolution never
|
||||
// joined them — every call inside a method body silently lost its
|
||||
// type-aware LSP strategy. Derive the bare receiver type name the same way
|
||||
// the receiver binding below does.
|
||||
char* recv_type_name = NULL;
|
||||
{
|
||||
TSNode recv0 = ts_node_child_by_field_name(func_node, "receiver", 8);
|
||||
if (!ts_node_is_null(recv0)) {
|
||||
uint32_t rnc0 = ts_node_child_count(recv0);
|
||||
for (uint32_t i = 0; i < rnc0 && !recv_type_name; i++) {
|
||||
TSNode rp = ts_node_child(recv0, i);
|
||||
if (ts_node_is_null(rp) || !ts_node_is_named(rp)) continue;
|
||||
if (strcmp(ts_node_type(rp), "parameter_declaration") != 0) continue;
|
||||
TSNode rtype = ts_node_child_by_field_name(rp, "type", 4);
|
||||
if (ts_node_is_null(rtype)) continue;
|
||||
// Unwrap a pointer receiver (*Type) to the bare type identifier.
|
||||
const char* rtk = ts_node_type(rtype);
|
||||
if (strcmp(rtk, "pointer_type") == 0 && ts_node_named_child_count(rtype) > 0) {
|
||||
rtype = ts_node_named_child(rtype, 0);
|
||||
}
|
||||
char* tn = lsp_node_text(ctx, rtype);
|
||||
if (tn && tn[0]) recv_type_name = tn;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (recv_type_name) {
|
||||
ctx->enclosing_func_qn =
|
||||
cbm_arena_sprintf(ctx->arena, "%s.%s.%s", ctx->package_qn, recv_type_name, func_name);
|
||||
} else {
|
||||
ctx->enclosing_func_qn = cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->package_qn, func_name);
|
||||
}
|
||||
// Enclosing-function QN must be the BARE package.Func form (no receiver
|
||||
// type segment). The textual call events (extract_unified.c) source calls
|
||||
// as package_qn.func_name — methods included — and the defs pass creates
|
||||
// the graph Method node under the same QN, so any other form breaks the
|
||||
// caller-QN join in cbm_pipeline_find_lsp_resolution and the LSP-resolved
|
||||
// call silently falls back to the registry short-name resolver. The
|
||||
// receiver type still reaches the registry via the def's parent_class /
|
||||
// method->receiver_type; it just does not appear in the caller QN.
|
||||
ctx->enclosing_func_qn = cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->package_qn, func_name);
|
||||
|
||||
// Push function scope
|
||||
CBMScope* saved_scope = ctx->current_scope;
|
||||
|
||||
@@ -167,6 +167,56 @@ static void build_type_embed_index(CBMTypeRegistry *reg, CBMArena *idx_arena) {
|
||||
reg->type_embed_entry_count = idx;
|
||||
}
|
||||
|
||||
/* Optional index: final qualified-name segment -> chain of TYPE indices.
|
||||
* Descending iteration plus prepend preserves ascending registry scan order.
|
||||
* Consumers opt in after the QN index captures the finalized tail boundary. */
|
||||
void cbm_registry_build_type_short_index(CBMTypeRegistry *reg) {
|
||||
if (!reg)
|
||||
return;
|
||||
reg->type_short_buckets = NULL;
|
||||
reg->type_short_entries = NULL;
|
||||
reg->type_short_bucket_count = 0;
|
||||
if (!reg->arena || !reg->type_qn_buckets || reg->type_qn_bucket_count <= 0)
|
||||
return;
|
||||
CBMArena *idx_arena = reg->arena;
|
||||
int count = 0;
|
||||
for (int i = 0; i < reg->type_count; i++) {
|
||||
if (reg->types[i].qualified_name)
|
||||
count++;
|
||||
}
|
||||
if (count == 0)
|
||||
return;
|
||||
int bucket_count = next_pow2(count * 2);
|
||||
if (bucket_count < 16)
|
||||
bucket_count = 16;
|
||||
int *buckets = (int *)cbm_arena_alloc(idx_arena, (size_t)bucket_count * sizeof(int));
|
||||
CBMRegistryHashEntry *entries = (CBMRegistryHashEntry *)cbm_arena_alloc(
|
||||
idx_arena, (size_t)count * sizeof(CBMRegistryHashEntry));
|
||||
if (!buckets || !entries)
|
||||
return;
|
||||
for (int i = 0; i < bucket_count; i++)
|
||||
buckets[i] = -1;
|
||||
int idx = 0;
|
||||
for (int i = reg->type_count - 1; i >= 0; i--) {
|
||||
const char *qn = reg->types[i].qualified_name;
|
||||
if (!qn)
|
||||
continue;
|
||||
const char *dot = strrchr(qn, '.');
|
||||
const char *short_name = dot ? dot + 1 : qn;
|
||||
uint64_t h = fnv1a(short_name);
|
||||
int slot = (int)(h & (uint64_t)(bucket_count - 1));
|
||||
entries[idx].hash = h;
|
||||
entries[idx].payload_index = i;
|
||||
entries[idx].next_index = buckets[slot];
|
||||
entries[idx].slot = slot;
|
||||
buckets[slot] = idx;
|
||||
idx++;
|
||||
}
|
||||
reg->type_short_buckets = buckets;
|
||||
reg->type_short_entries = entries;
|
||||
reg->type_short_bucket_count = bucket_count;
|
||||
}
|
||||
|
||||
/* Index: short_name -> chain of FREE-function (receiver_type==NULL) indices.
|
||||
* Descending-iterate + prepend for ascending chain order (as above). */
|
||||
static void build_type_short_index(CBMTypeRegistry *reg, CBMArena *idx_arena) {
|
||||
@@ -248,6 +298,43 @@ static void build_ffunc_short_index(CBMTypeRegistry *reg, CBMArena *idx_arena) {
|
||||
reg->ffunc_short_entry_count = idx;
|
||||
}
|
||||
|
||||
void cbm_registry_types_by_short_name(const CBMTypeRegistry *reg, const char *short_name,
|
||||
CBMTypeNameIter *out) {
|
||||
out->reg = reg;
|
||||
out->hash = fnv1a(short_name);
|
||||
if (reg->type_qn_buckets && reg->type_qn_bucket_count > 0) {
|
||||
if (reg->type_short_buckets && reg->type_short_bucket_count > 0) {
|
||||
int slot = (int)(out->hash & (uint64_t)(reg->type_short_bucket_count - 1));
|
||||
out->chain_idx = reg->type_short_buckets[slot];
|
||||
out->tail_i = reg->type_qn_entry_count;
|
||||
} else {
|
||||
out->chain_idx = -1;
|
||||
out->tail_i = 0; /* auxiliary-index allocation failed: preserve full-scan semantics */
|
||||
}
|
||||
out->tail_end = reg->type_count;
|
||||
} else {
|
||||
out->chain_idx = -1;
|
||||
out->tail_i = 0;
|
||||
out->tail_end = reg->type_count;
|
||||
}
|
||||
}
|
||||
|
||||
int cbm_type_name_iter_next(CBMTypeNameIter *it) {
|
||||
const CBMTypeRegistry *reg = it->reg;
|
||||
while (it->chain_idx >= 0) {
|
||||
const CBMRegistryHashEntry *e = ®->type_short_entries[it->chain_idx];
|
||||
int p = e->payload_index;
|
||||
uint64_t h = e->hash;
|
||||
it->chain_idx = e->next_index;
|
||||
if (h != it->hash)
|
||||
continue;
|
||||
return p;
|
||||
}
|
||||
if (it->tail_i < it->tail_end)
|
||||
return it->tail_i++;
|
||||
return -1;
|
||||
}
|
||||
|
||||
void cbm_registry_types_by_embedded_bare(const CBMTypeRegistry *reg, const char *bare,
|
||||
CBMTypeEmbedIter *out) {
|
||||
out->reg = reg;
|
||||
|
||||
@@ -111,9 +111,14 @@ typedef struct CBMTypeRegistry {
|
||||
int method_bucket_count;
|
||||
int method_entry_count;
|
||||
|
||||
// Auxiliary short-name / embedded-type indexes (built by finalize alongside the
|
||||
// QN buckets). Turn the Rust trait- and free-function fallback scans from
|
||||
// O(type_count)/O(func_count) into O(chain). Read-only after finalize.
|
||||
// Auxiliary short-name / embedded-type indexes. The type short-name index is
|
||||
// opt-in after finalize; the others are built by finalize. Current C/C++ cross
|
||||
// registries opt in, while other languages incur no construction cost.
|
||||
// Type short-name index: fnv1a(last-'.'-segment of qualified_name) -> chain
|
||||
// of TYPE indices. payload_index = type index.
|
||||
int *type_short_buckets;
|
||||
CBMRegistryHashEntry *type_short_entries;
|
||||
int type_short_bucket_count;
|
||||
// Embedded-type index: fnv1a(bare last-'.'-segment of each embedded_type) -> chain
|
||||
// of TYPE indices declaring it. payload_index = type index (a type may appear once
|
||||
// per embedded entry; consumers dedup adjacent same-type via the iterator).
|
||||
@@ -160,6 +165,11 @@ void cbm_registry_finalize(CBMTypeRegistry *reg);
|
||||
// there add GBs across a large repo (FastAPI incremental test: +1.1 GB RSS).
|
||||
void cbm_registry_finalize_into(CBMTypeRegistry *reg, CBMArena *idx_arena);
|
||||
|
||||
// Build the optional final-QN-segment type index. Call immediately after
|
||||
// finalization. Consumers that do not opt in avoid its type_count-sized allocation
|
||||
// and scan. Allocation failure preserves the iterator's linear fallback.
|
||||
void cbm_registry_build_type_short_index(CBMTypeRegistry *reg);
|
||||
|
||||
// Register a function/method.
|
||||
void cbm_registry_add_func(CBMTypeRegistry *reg, CBMRegisteredFunc func);
|
||||
|
||||
@@ -219,8 +229,24 @@ const CBMRegisteredFunc *cbm_registry_lookup_symbol_by_types(const CBMTypeRegist
|
||||
const CBMType **arg_types,
|
||||
int arg_count);
|
||||
|
||||
// --- Auxiliary index iterators (Rust trait / free-function fallback fast paths) ---
|
||||
// --- Auxiliary index iterators (language fallback fast paths) ---
|
||||
//
|
||||
// Iterate registry TYPE indices whose qualified-name final segment may equal
|
||||
// `short_name`. When the optional index is present this walks its hash chain plus
|
||||
// any post-finalize tail; otherwise it degrades to the original full scan. Results
|
||||
// preserve ascending registry order. The index is a hash prefilter; callers must
|
||||
// re-check their exact predicate, including tail entries.
|
||||
typedef struct {
|
||||
const CBMTypeRegistry *reg;
|
||||
uint64_t hash;
|
||||
int chain_idx;
|
||||
int tail_i;
|
||||
int tail_end;
|
||||
} CBMTypeNameIter;
|
||||
void cbm_registry_types_by_short_name(const CBMTypeRegistry *reg, const char *short_name,
|
||||
CBMTypeNameIter *out);
|
||||
int cbm_type_name_iter_next(CBMTypeNameIter *it);
|
||||
|
||||
// Iterate registry TYPE indices whose embedded_types contain an entry whose BARE
|
||||
// name (last '.'-segment) equals `bare`. On a finalized registry this walks the
|
||||
// embedded-type index plus any post-finalize tail; on an unfinalized registry it
|
||||
|
||||
@@ -678,7 +678,11 @@ static bool callee_is_delimiter_or_filesystem_builder(const char *callee_name) {
|
||||
method = last_colon + 2;
|
||||
}
|
||||
if (strcmp(method, "split") == 0 || strcmp(method, "rsplit") == 0 ||
|
||||
strcmp(method, "partition") == 0 || strcmp(method, "join") == 0) {
|
||||
strcmp(method, "partition") == 0 || strcmp(method, "join") == 0 ||
|
||||
strcmp(method, "replace") == 0 || strcmp(method, "replaceAll") == 0 ||
|
||||
strcmp(method, "match") == 0 || strcmp(method, "matchAll") == 0 ||
|
||||
strcmp(method, "search") == 0 || strcmp(method, "test") == 0 ||
|
||||
strcmp(method, "exec") == 0) {
|
||||
return true;
|
||||
}
|
||||
return strstr(callee_name, "os.path.join") != NULL || strstr(callee_name, "path.join") != NULL;
|
||||
|
||||
@@ -44,6 +44,10 @@ cbm_svc_kind_t cbm_service_pattern_match(const char *resolved_qn);
|
||||
* classified via its real resolved QN instead and never reaches this check. */
|
||||
bool cbm_service_pattern_is_global_fetch(const char *callee_name);
|
||||
|
||||
/* True when a string literal is a plausible HTTP route for the given callee.
|
||||
* Rejects filesystem paths and non-HTTP string consumers. */
|
||||
bool cbm_service_pattern_is_http_route_literal(const char *literal, const char *callee_name);
|
||||
|
||||
/* Per-worker TLS cache for cbm_service_pattern_match results. The
|
||||
* pattern matcher runs once per resolved CALL edge in emit_service_
|
||||
* edge — that's 6 pattern lists × ~30 patterns × strstr per call ≈
|
||||
|
||||
@@ -10,11 +10,25 @@ The grammars were originally vendored as bare `parser.c`+`scanner.c` with **no r
|
||||
## Summary
|
||||
|
||||
- Grammars: **162** — vendored-from-upstream: **143**, first-party/self-maintained: **14**, registry-disagreement: **5** (nim removed 2026-06-12; objectscript_udl + objectscript_routine added 2026-06-24; mojo added 2026-07-01; arkts added 2026-08-26; plsql added 2026-08-27; chialisp added 2026-08-28 — see notes below)
|
||||
- ABI distribution: **9×** ABI-13 **80×** ABI-14 **73×** ABI-15 (runtime ceiling is ABI 15; never vendor ABI 16 without a runtime upgrade)
|
||||
- ABI distribution: **9×** ABI-13 **79×** ABI-14 **74×** ABI-15 (runtime ceiling is ABI 15; never vendor ABI 16 without a runtime upgrade)
|
||||
— recounted from the tree 2026-08-30 after the perl v1.2.1 refresh moved `perl` from ABI 14 to ABI 15 (was `9×/80×/73×`). Neither side of the rebase had this right: main's line was correct for main, and this branch still carried the pre-2026-08-28 `7×/84×/65×`. Regenerate, never increment.
|
||||
— recounted from the tree 2026-08-28. This line had drifted: it read `7×/86×/65×`, which sums to 158 against 161 vendored grammars, so it was wrong before Chialisp was added and incrementing it would have carried the error forward. Regenerate with:
|
||||
`grep -h '#define LANGUAGE_VERSION' internal/cbm/vendored/grammars/*/parser.c | sort | uniq -c`
|
||||
- Vendored copies missing LICENSE: **0** — all upstream LICENSE files restored 2026-06-11 (first-party grammars carry the project MIT license; `move` uses the Helix-listed upstream tzakian/tree-sitter-move MIT text, `zsh` uses georgeharker/tree-sitter-zsh MIT)
|
||||
- `verdict`: VERIFIED-BOTH = our source matches *both* registries; VERIFIED-NVIM/HELIX = matches one; registry-disagreement = registries name a different repo (listed separately); `vendor-maintained` = the language vendor's own grammar, not in nvim/Helix; `community-niche` = an individual/community-maintained grammar for a niche language, not in nvim/Helix — provenance hand-verified against the upstream repository instead of a registry.
|
||||
- **perl** (refreshed 2026-08-27): byte-for-byte generated bundle from
|
||||
[tree-sitter-perl/tree-sitter-perl](https://github.com/tree-sitter-perl/tree-sitter-perl)
|
||||
tag `v1.2.1`, full commit `c3e17b31179bf8f658c9f37c7a3ea6a202212d5a` (ABI 15).
|
||||
The upstream GitHub source asset SHA-256 is
|
||||
`95c5fa0966dd431eb2f96b941c37b413ae7e9083729433f4d3d41fbc2a4f14a6`; the
|
||||
screened ordered source-manifest SHA-256 is
|
||||
`4e7bf02e8bd14b410309ce12bab80edb730826db79e1139434c1459a3117b900`.
|
||||
The existing MIT `LICENSE` was preserved byte-for-byte at SHA-256
|
||||
`68a9a526ae357ed5a2f8ca5dabb14131b283554b69639bb04816088d4b1f2fa0`.
|
||||
Exact per-file SHA-256 values are pinned in `scripts/vendored-checksums.txt`.
|
||||
The table uses `UPSTREAM-RELEASE` because this refresh verified the tagged
|
||||
upstream release itself but did not independently verify a current registry
|
||||
pin; it intentionally does not claim `VERIFIED-NVIM` or `VERIFIED-BOTH`.
|
||||
- **objectscript_udl / objectscript_routine** (added 2026-06-24): vendored from [intersystems/tree-sitter-objectscript](https://github.com/intersystems/tree-sitter-objectscript) @ `a7ffcdf` — MIT, the InterSystems-official grammars (a niche vendor language, hence `vendor-maintained`, not in nvim-treesitter/Helix). **Re-vendor note:** each `scanner.c`'s upstream `#include "../../common/scanner.h"` is repointed to a per-directory `objectscript_common.h` (a verbatim copy of upstream `common/scanner.h`), because this repo's shared `vendored/common/scanner.h` belongs to the cfml/fsharp grammars and differs. The generated `parser.c`/`scanner.c` are otherwise byte-for-byte upstream — on re-vendor, re-apply only that single include rename. **Local modification (2026-07-16):** in `objectscript_common.h`, two loop counters `uint8_t i` were widened to `int i` (the `reverse_marker` scan and the `html_marker_buffer` reversal) to clear CodeQL `cpp/comparison-with-wider-type` — a false positive in practice (both lengths are hard-bounded by `MARKER_BUFFER_MAX_LEN = 30`, so `uint8_t` could never wrap), fixed for cleanliness. On re-vendor, re-apply this widening too (or upstream it at intersystems/tree-sitter-objectscript).
|
||||
- **mojo** (added 2026-07-01): vendored from [lsh/tree-sitter-mojo](https://github.com/lsh/tree-sitter-mojo) @ `33193a99afe6` — MIT, ABI 15. Helix tracks `lsh/tree-sitter-mojo` as its Mojo grammar source, but the Helix-pinned commit (`3d7c53b8038f`) no longer resolves in the upstream repository after a force-push, so this vendor uses current upstream `main` rather than the stale registry SHA. Security review covered only the vendored C surface (`parser.c`, `scanner.c`, `tree_sitter/*.h`) plus upstream license/provenance metadata; no package manager hooks, workflow files, prompt/agent instruction files, or generated lockfiles were vendored.
|
||||
- **arkts** (added 2026-08-26): **first-party derivative** — a fork of [tree-sitter/tree-sitter-typescript](https://github.com/tree-sitter/tree-sitter-typescript)'s `typescript` dialect pinned @ `75b3874edb2d` (v0.23.2, the same commit our vendored typescript/tsx come from), on the tree-sitter-javascript base @ `3a837b6f3658` (v0.23.1, the version upstream's own package-lock pins), extended with ArkTS/ArkUI syntax (`@Component struct` declarations, UI-DSL trailing-closure calls with post-block attribute chains, decorated function declarations, `import lazy`, `@Extend`/`@Styles` leading-dot attribute chains, anonymous `stateStyles` style blocks). Grammar source + corpus tests live in `tools/tree-sitter-arkts/`; regenerate with `npx tree-sitter-cli@0.25.10 generate` (ABI 15). **External scanner:** `scanner.c` is a symbol-rename trampoline; `_common_scanner.h` is byte-identical (same SHA-256) to the reviewed `typescript/_common_scanner.h` already shipped. **LICENSE:** tree-sitter-typescript's MIT text **verbatim and byte-identical** (© 2017 Max Brunsfeld) — which is what MIT requires of a derivative work, and what lets the provenance audit byte-verify it against upstream rather than take a note on trust. The fork is registered in the audit's `FORKS` map. Our own copyright for the ArkTS additions, and the tree-sitter-javascript attribution (© 2014 Max Brunsfeld), live with the SOURCE in `tools/tree-sitter-arkts/grammar.js` and in `THIRD_PARTY.md`, which is what ships in the release archives.
|
||||
@@ -163,7 +177,7 @@ Re-vendoring from upstream must re-apply these.
|
||||
| ocaml | 14 | tree-sitter/tree-sitter-ocaml | `5a979b3ec7f1` | VERIFIED-BOTH | ✅ |
|
||||
| odin | 14 | tree-sitter-grammars/tree-sitter-odin | `d2ca8efb4487` | VERIFIED-BOTH | ✅ |
|
||||
| pascal | 14 | Isopod/tree-sitter-pascal | `042119eca2e1` | VERIFIED-BOTH | ✅ |
|
||||
| perl | 14 | tree-sitter-perl/tree-sitter-perl | `ea9667dc65a8` | VERIFIED-BOTH | ✅ |
|
||||
| perl | 15 | tree-sitter-perl/tree-sitter-perl | `c3e17b31179b` | UPSTREAM-RELEASE | ✅ |
|
||||
| php | 15 | tree-sitter/tree-sitter-php | `3f2465c217d0` | VERIFIED-BOTH | ✅ |
|
||||
| pkl | 15 | apple/tree-sitter-pkl | `f5beed1da8e5` | VERIFIED-BOTH | ✅ |
|
||||
| plsql | 14 | AndreasMaierDe/tree-sitter-plsql | `28aebef209be` | community-niche | ✅ |
|
||||
|
||||
@@ -69,7 +69,7 @@ is respectively less than, matching, or greater than the array member.
|
||||
* have to make lim 3, then halve, obtaining 1, so that we will only
|
||||
* look at item 3.
|
||||
*/
|
||||
void *bsearch(const void *key, const void *base0, size_t nmemb, size_t size,
|
||||
static void *tsp_bsearch(const void *key, const void *base0, size_t nmemb, size_t size,
|
||||
int (*compar)(const void *, const void *)) {
|
||||
const char *base = (const char *)base0;
|
||||
int lim, cmp;
|
||||
|
||||
+760516
-469989
File diff suppressed because it is too large
Load Diff
+687
-55
@@ -1,6 +1,9 @@
|
||||
#include "tree_sitter/array.h"
|
||||
#include "tree_sitter/parser.h"
|
||||
#include "tsp_unicode.h"
|
||||
#include "tsp_keywords.h"
|
||||
#include "tsp_intuit_more.h"
|
||||
#include "tsp_intuit_readline.h"
|
||||
|
||||
// grumble grumble no stdlib
|
||||
static char *tsp_strchr(register const char *s, int c) {
|
||||
@@ -49,6 +52,8 @@ enum TokenType {
|
||||
TOKEN_ESCAPE_SEQUENCE,
|
||||
TOKEN_ESCAPED_DELIMITER,
|
||||
TOKEN_DOLLAR_IN_REGEXP,
|
||||
TOKEN_REGEXP_OPEN_BRACKET,
|
||||
TOKEN_REGEXP_OPEN_BRACE,
|
||||
TOKEN_POD,
|
||||
TOKEN_GOBBLED_CONTENT,
|
||||
TOKEN_ATTRIBUTE_VALUE_BEGIN,
|
||||
@@ -61,6 +66,7 @@ enum TokenType {
|
||||
TOKEN_HEREDOC_MIDDLE,
|
||||
TOKEN_HEREDOC_END,
|
||||
TOKEN_FAT_COMMA_AUTOQUOTED,
|
||||
TOKEN_FAT_COMMA_AUTOQUOTED_AHEAD,
|
||||
TOKEN_FILETEST,
|
||||
TOKEN_BRACE_AUTOQUOTED,
|
||||
/* zero-width lookahead tokens */
|
||||
@@ -69,6 +75,24 @@ enum TokenType {
|
||||
TOKEN_NO_INTERP_WHITESPACE_ZW,
|
||||
/* zero-width high priority token */
|
||||
TOKEN_NONASSOC,
|
||||
/* synthetic tokens for error recovery */
|
||||
TOKEN_RECOVER_PAREN_CLOSE,
|
||||
TOKEN_RECOVER_BRACKET_CLOSE,
|
||||
TOKEN_RECOVER_BRACE_CLOSE,
|
||||
TOKEN_RECOVER_ARROW,
|
||||
TOKEN_RECOVER_BLOCK_CLOSE,
|
||||
/* opaque body of a `format NAME = ... .` declaration: everything from the
|
||||
* line after `=` up to and including the lone-`.` terminator line */
|
||||
TOKEN_FORMAT_CONTENT,
|
||||
/* `x` repetition operator glued to its count (`"ab"x3`) */
|
||||
TOKEN_X_OP,
|
||||
/* `class`/`role`/`method` emitted only in declaration position */
|
||||
TOKEN_KW_CLASS,
|
||||
TOKEN_KW_ROLE,
|
||||
TOKEN_KW_METHOD,
|
||||
/* `async`/`try` contextual keywords, emitted only in keyword position */
|
||||
TOKEN_KW_ASYNC,
|
||||
TOKEN_KW_TRY,
|
||||
/* error condition is always last */
|
||||
TOKEN_ERROR
|
||||
};
|
||||
@@ -115,18 +139,47 @@ static int32_t close_for_open(int32_t c) {
|
||||
|
||||
typedef struct {
|
||||
int32_t open, close, count;
|
||||
/* Recognises a pattern-LEADING '['/'{' -- the quote's own opening delimiter
|
||||
* appearing (after optional whitespace) as the literal first element of the
|
||||
* body (m{{...}}, qr{ {...} }, s{{...}}, m[[...]]) -- where the S_intuit_more
|
||||
* subscript heuristic (tsp_intuit_more) has no preceding variable to weigh and
|
||||
* so must not be consulted.
|
||||
*
|
||||
* Decided ONCE by a bounded lookahead at the opening delimiter (see the open
|
||||
* handler), where the lexer sits exactly at the body start: if the first
|
||||
* non-space body char is the delimiter again, the body leads with a literal
|
||||
* group. Looking at the actual first char sidesteps both problems of the
|
||||
* earlier column+flag scheme -- columns repeat across lines, and the scanner
|
||||
* can't see grammar-lexed interpolation -- because we never have to ask "has
|
||||
* content been seen?": we just check what the first char *is*.
|
||||
*
|
||||
* Set at the opener, consumed (cleared) by that first bracket. */
|
||||
bool body_leads_with_delim;
|
||||
} TSPQuote;
|
||||
|
||||
static TSPQuote tspquote_new() { return (TSPQuote){0, 0, 0}; }
|
||||
static TSPQuote tspquote_new() { return (TSPQuote){0, 0, 0, false}; }
|
||||
|
||||
enum HeredocState { HEREDOC_NONE, HEREDOC_START, HEREDOC_UNKNOWN, HEREDOC_CONTINUE, HEREDOC_END };
|
||||
|
||||
/* Perl allows multiple heredocs queued on a single line, consumed FIFO after
|
||||
* the newline. We hold up to HEREDOC_QUEUE_MAX pending heredocs; each one
|
||||
* carries its own delimiter and interpolate/indent flags. The consume-FSM
|
||||
* state (heredoc_state) applies to the FRONT of the queue (the one currently
|
||||
* being consumed); when it finishes we dequeue and advance to the next. */
|
||||
#define HEREDOC_QUEUE_MAX 8
|
||||
typedef struct {
|
||||
TSPString delim;
|
||||
bool interpolates, indents;
|
||||
} HeredocEntry;
|
||||
|
||||
typedef struct {
|
||||
Array(TSPQuote) quotes;
|
||||
/* heredoc - we need to track if we should start the heredoc, if it's
|
||||
* interpolating, how many chars the delimiter is and what the delimiter is */
|
||||
bool heredoc_interpolates, heredoc_indents;
|
||||
/* FIFO queue of pending heredocs (front = index 0). heredoc_count is how
|
||||
* many entries are live; heredoc_state is the consume state of the front. */
|
||||
HeredocEntry heredoc_queue[HEREDOC_QUEUE_MAX];
|
||||
uint8_t heredoc_count;
|
||||
enum HeredocState heredoc_state;
|
||||
TSPString heredoc_delim;
|
||||
bool recovery_emitted;
|
||||
} LexerState;
|
||||
|
||||
static void lexerstate_push_quote(LexerState *state, int32_t opener) {
|
||||
@@ -192,6 +245,20 @@ static bool lexerstate_is_paired_delimiter(LexerState *state) {
|
||||
return !!q->open;
|
||||
}
|
||||
|
||||
/* Is `c` the innermost quote's pattern-leading bracket -- the delimiter-matching
|
||||
* '['/'{' the open-handler lookahead flagged as the body's first literal element?
|
||||
* One-shot: clears the flag so a later same-delimiter bracket is treated as
|
||||
* ordinary (subscript/class) content. */
|
||||
static bool lexerstate_take_body_lead(LexerState *state, int32_t c) {
|
||||
if (!state->quotes.size) return false;
|
||||
TSPQuote *q = array_back(&state->quotes);
|
||||
if (q->body_leads_with_delim && c == q->open) {
|
||||
q->body_leads_with_delim = false;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// in order to emulate a sublex, we basically need to have a new escape type character
|
||||
// that can pop up anywhere so long as there's an active string, to escape the string
|
||||
// char
|
||||
@@ -210,16 +277,48 @@ static bool lexerstate_is_paired_delimiter(LexerState *state) {
|
||||
|
||||
// we can match perl's behavior if we are intentionally destructive here and find our match
|
||||
|
||||
static void lexerstate_add_heredoc(LexerState *state, TSPString *delim, bool interp, bool indent) {
|
||||
state->heredoc_delim = *delim;
|
||||
state->heredoc_interpolates = interp;
|
||||
state->heredoc_indents = indent;
|
||||
state->heredoc_state = HEREDOC_START;
|
||||
/* The front of the queue is the heredoc currently being consumed. */
|
||||
static HeredocEntry *lexerstate_front_heredoc(LexerState *state) {
|
||||
return &state->heredoc_queue[0];
|
||||
}
|
||||
|
||||
/* ENQUEUE a pending heredoc to the back of the FIFO. When this is the first
|
||||
* pending heredoc, prime the consume FSM at HEREDOC_START.
|
||||
*
|
||||
* Overflow handling: if the queue is already full we OVERWRITE THE LAST SLOT
|
||||
* with the new entry rather than dropping it. The last slot then always
|
||||
* tracks the FINAL heredoc's terminator, so the scanner stays in heredoc-mode
|
||||
* and greedily consumes all overflow content up to that final terminator —
|
||||
* absorbing the excess into the last body instead of leaking it into the code
|
||||
* stream. For >MAX heredocs on a line this means the first MAX-1 bodies parse
|
||||
* correctly, the last body greedily swallows the remaining overflow (wrong but
|
||||
* BOUNDED), and code after the heredoc block is unaffected (no desync). This
|
||||
* is graceful degradation for a pathological input. */
|
||||
static void lexerstate_add_heredoc(LexerState *state, TSPString *delim, bool interp, bool indent) {
|
||||
HeredocEntry *e;
|
||||
if (state->heredoc_count >= HEREDOC_QUEUE_MAX)
|
||||
e = &state->heredoc_queue[HEREDOC_QUEUE_MAX - 1];
|
||||
else
|
||||
e = &state->heredoc_queue[state->heredoc_count++];
|
||||
e->delim = *delim;
|
||||
e->interpolates = interp;
|
||||
e->indents = indent;
|
||||
if (state->heredoc_count == 1) state->heredoc_state = HEREDOC_START;
|
||||
}
|
||||
|
||||
/* DEQUEUE the finished front heredoc and advance to the next pending one.
|
||||
* Each heredoc body is wrapped in its own `heredoc_content` grammar node which
|
||||
* begins with a fresh `_heredoc_start` zero-width token; that token only fires
|
||||
* in HEREDOC_START. So when another heredoc remains queued we re-arm the FSM
|
||||
* to HEREDOC_START, exactly as if its delimiter had just been declared. */
|
||||
static void lexerstate_finish_heredoc(LexerState *state) {
|
||||
state->heredoc_delim.length = 0;
|
||||
state->heredoc_state = HEREDOC_NONE;
|
||||
if (state->heredoc_count > 0) {
|
||||
state->heredoc_count--;
|
||||
for (int i = 0; i < state->heredoc_count; i++)
|
||||
state->heredoc_queue[i] = state->heredoc_queue[i + 1];
|
||||
}
|
||||
state->heredoc_queue[state->heredoc_count] = (HeredocEntry){0};
|
||||
state->heredoc_state = state->heredoc_count > 0 ? HEREDOC_START : HEREDOC_NONE;
|
||||
}
|
||||
|
||||
#define ADVANCE_C \
|
||||
@@ -247,6 +346,16 @@ static void lexerstate_finish_heredoc(LexerState *state) {
|
||||
DEBUG("marking end of token\n", 0); \
|
||||
} while (0)
|
||||
|
||||
// Try each recovery token in priority order. Uses a macro because
|
||||
// TOKEN() contains `return true`. brace_ok: skip brace recovery at '}'
|
||||
// (that's the real closer); ';' is safe (only valid in subscript context).
|
||||
#define EMIT_RECOVERY_TOKENS(brace_ok) do { \
|
||||
if (valid_symbols[TOKEN_RECOVER_ARROW]) { state->recovery_emitted = true; TOKEN(TOKEN_RECOVER_ARROW); } \
|
||||
if (valid_symbols[TOKEN_RECOVER_PAREN_CLOSE]) { state->recovery_emitted = true; TOKEN(TOKEN_RECOVER_PAREN_CLOSE); } \
|
||||
if (valid_symbols[TOKEN_RECOVER_BRACKET_CLOSE]) { state->recovery_emitted = true; TOKEN(TOKEN_RECOVER_BRACKET_CLOSE); } \
|
||||
if ((brace_ok) && valid_symbols[TOKEN_RECOVER_BRACE_CLOSE]) { state->recovery_emitted = true; TOKEN(TOKEN_RECOVER_BRACE_CLOSE); } \
|
||||
} while(0)
|
||||
|
||||
static void skip_whitespace(TSLexer *lexer) {
|
||||
while (1) {
|
||||
int32_t c = lexer->lookahead;
|
||||
@@ -258,19 +367,130 @@ static void skip_whitespace(TSLexer *lexer) {
|
||||
}
|
||||
}
|
||||
|
||||
static void skip_ws_to_eol(TSLexer *lexer) {
|
||||
static bool skip_ws_to_eol(TSLexer *lexer) {
|
||||
while (1) {
|
||||
int32_t c = lexer->lookahead;
|
||||
if (!c) return;
|
||||
if (!c) return false;
|
||||
if (is_tsp_whitespace(c)) {
|
||||
lexer->advance(lexer, true);
|
||||
// return after eating the newline
|
||||
if (c == '\n') return;
|
||||
if (c == '\n') return true;
|
||||
} else
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Forward declarations (defined after _skip_chars)
|
||||
static bool isidfirst(int32_t c);
|
||||
static bool isidcont(int32_t c);
|
||||
|
||||
enum PeekResult {
|
||||
PEEK_NO_MATCH, // first char not a keyword starter — lexer untouched
|
||||
PEEK_KEYWORD, // statement keyword (not followed by =>)
|
||||
PEEK_FAT_COMMA, // keyword followed by => — caller MARK_ENDs then goto fat_comma_check
|
||||
PEEK_AUTOQUOTE_KEY, // non-keyword word followed by => — peek already marked the
|
||||
// word boundary; caller goes straight to fat_comma_check
|
||||
PEEK_NOT_KEYWORD, // word read but not a keyword — caller MUST return false
|
||||
};
|
||||
|
||||
// How a word read by the keyword peek classifies (set by KEYWORD_MATCH).
|
||||
enum KwKind {
|
||||
KW_NONE, // not a statement keyword
|
||||
KW_ALWAYS, // always a statement keyword (package/use/no/class/role)
|
||||
KW_NEEDS_NAME, // declaration only if followed by a name (sub/method)
|
||||
};
|
||||
|
||||
static enum PeekResult peek_is_statement_keyword(TSLexer *lexer, bool *is_structural) {
|
||||
int32_t la = lexer->lookahead;
|
||||
*is_structural = false;
|
||||
|
||||
if (KEYWORD_FIRST_CHAR_FILTER(la))
|
||||
return PEEK_NO_MATCH;
|
||||
|
||||
// Read the word (character set derived from keyword list)
|
||||
char word[16];
|
||||
int len = 0;
|
||||
while (KEYWORD_WORD_CHAR(la)) {
|
||||
if (len < 15) word[len++] = (char)la;
|
||||
lexer->advance(lexer, false);
|
||||
la = lexer->lookahead;
|
||||
}
|
||||
word[len] = '\0';
|
||||
|
||||
// Classify *before* consuming any identifier tail: a statement keyword must
|
||||
// end exactly at the keyword charset boundary. A longer identifier (e.g.
|
||||
// "service", "naming") read only partway by the loop above is never a
|
||||
// keyword — but it can still be a fat-comma autoquote key, so we don't bail.
|
||||
enum KwKind kind = KW_NONE;
|
||||
if (!isidcont(la))
|
||||
KEYWORD_MATCH(word, kind);
|
||||
|
||||
if (kind == KW_NONE) {
|
||||
// Not a statement keyword. It may still be a fat-comma autoquote key
|
||||
// (`name => 1` on a continuation line inside an open list, where the
|
||||
// recovery gate fires and would otherwise lex it as a plain bareword).
|
||||
// Consume the rest of the identifier, mark the token end at the word
|
||||
// boundary, then look past whitespace for '='. The whole word must be
|
||||
// covered, so skip with advance(false): advance(true) would reset the
|
||||
// token start past the word and emit a zero-width key. mark_end fixes the
|
||||
// end at the word, so trailing whitespace stays out of the token anyway.
|
||||
while (isidcont(la)) {
|
||||
lexer->advance(lexer, false);
|
||||
la = lexer->lookahead;
|
||||
}
|
||||
lexer->mark_end(lexer);
|
||||
while (is_tsp_whitespace(la)) {
|
||||
lexer->advance(lexer, false);
|
||||
la = lexer->lookahead;
|
||||
}
|
||||
if (la == '=') return PEEK_AUTOQUOTE_KEY;
|
||||
return PEEK_NOT_KEYWORD;
|
||||
}
|
||||
|
||||
// A statement keyword. Structural OO keywords (method/class/role) never
|
||||
// legitimately nest inside a sub/method body, so they alone trigger body
|
||||
// block-close recovery. use/no/sub/package commonly DO appear in a body
|
||||
// (e.g. `no strict 'refs';`) and must not trigger it.
|
||||
//
|
||||
// `sub` is the tempting one: a bare nested `sub foo {...}` is almost always a
|
||||
// "won't stay shared" bug, not intent — so closing the enclosing body at it
|
||||
// would often be the *helpful* read. We deliberately don't. It is valid Perl
|
||||
// and it is genuinely out there (~140 nested bare subs across a 92k-file
|
||||
// corpus: perl's own regen/Porting codegen scripts, some CPAN, real app code),
|
||||
// so the parser tree-s it faithfully; diagnosing the footgun belongs to the
|
||||
// layer above (linter/LSP), not the grammar. Lexical `my sub` (which *does*
|
||||
// truly nest, and is legit) is a separate story and is excluded for free
|
||||
// anyway: it starts with `my`, takes the KW_NONE path above, and never reaches
|
||||
// here. (`kind` was already classified above; don't re-run KEYWORD_MATCH.)
|
||||
*is_structural = (strcmp(word, "method") == 0 ||
|
||||
strcmp(word, "class") == 0 ||
|
||||
strcmp(word, "role") == 0);
|
||||
|
||||
// The caller already marked a zero-width position at the keyword start for
|
||||
// recovery-token emission; leave that mark alone (don't mark_end here) and
|
||||
// skip trailing whitespace with advance(true) — the recovery-safe behavior.
|
||||
// A word-end mark here would make the recovery token swallow the keyword
|
||||
// (and breaks the keyword-boundary tests).
|
||||
while (is_tsp_whitespace(la)) {
|
||||
lexer->advance(lexer, true);
|
||||
la = lexer->lookahead;
|
||||
}
|
||||
|
||||
// Fat comma => means even a keyword is a hash key, not a statement
|
||||
// (`package => 1`). We can't mark the word boundary here without clobbering
|
||||
// the recovery mark, so the caller emits a zero-width _fat_comma_autoquoted
|
||||
// _ahead marker and tree-sitter re-lexes the word through the normal
|
||||
// autoquote path with a correct span. (Plain words and non-keyword builtins
|
||||
// — sort/map/ref/state/… — take the KW_NONE path above and autoquote in a
|
||||
// single pass.)
|
||||
if (la == '=') return PEEK_FAT_COMMA;
|
||||
|
||||
// For sub/method: only a declaration if followed by an identifier (name)
|
||||
if (kind == KW_NEEDS_NAME && !isidfirst(la))
|
||||
return PEEK_NOT_KEYWORD; // anonymous sub/method
|
||||
|
||||
return PEEK_KEYWORD;
|
||||
}
|
||||
|
||||
static void _skip_chars(TSLexer *lexer, int maxlen, const char *allow) {
|
||||
int32_t c = lexer->lookahead;
|
||||
|
||||
@@ -306,14 +526,37 @@ static bool isidcont(int32_t c) { return c == '_' || is_tsp_id_continue(c); }
|
||||
// there's a matching rule in the grammar to catch when it doesn't match a rule
|
||||
static bool is_interpolation_escape(int32_t c) { return c < 256 && tsp_strchr("$@-[{\\", c); }
|
||||
|
||||
/* The runtime hands serialize() a FIXED buffer of TREE_SITTER_SERIALIZATION_BUFFER_SIZE
|
||||
* (1024) bytes — `self->lexer.debug_buffer`, a member embedded in the Lexer/TSParser
|
||||
* struct. Writing past it is intra-object memory corruption (ASAN doesn't flag it),
|
||||
* and returning a length > the buffer trips the runtime's `assert(length <= 1024)`
|
||||
* (or, in NDEBUG builds, silently clobbers adjacent parser fields). So we must bound
|
||||
* the serialized size to the buffer, NOT to UINT8_MAX.
|
||||
*
|
||||
* Budget: everything except the quotes is fixed-size — the two count bytes, the
|
||||
* recovery flag, and the whole heredoc FIFO at its worst case (HEREDOC_QUEUE_MAX
|
||||
* full entries). Whatever room is left holds quotes. A deeply nested (or, in a
|
||||
* mid-edit buffer, deeply *unclosed*) interpolation like `qq{ ${\ qq{ ... }} }`
|
||||
* stacks one quote per level and can blow past this; we cap it. Truncating the
|
||||
* stack degrades parsing of pathologically-nested input (>~59 levels) but keeps it
|
||||
* BOUNDED — the same graceful-degradation contract the heredoc overflow already uses.
|
||||
*/
|
||||
#define SER_FIXED_OVERHEAD \
|
||||
(1 /* quote_count */ + \
|
||||
1 /* heredoc_state */ + 1 /* heredoc_count */ + \
|
||||
HEREDOC_QUEUE_MAX * (1 /* interpolates */ + 1 /* indents */ + (int)sizeof(TSPString)) + \
|
||||
1 /* recovery_emitted */)
|
||||
#define MAX_SERIALIZED_QUOTES \
|
||||
((TREE_SITTER_SERIALIZATION_BUFFER_SIZE - SER_FIXED_OVERHEAD) / (int)sizeof(TSPQuote))
|
||||
|
||||
unsigned int tree_sitter_perl_external_scanner_serialize(void *payload, char *buffer) {
|
||||
LexerState *state = payload;
|
||||
size_t size = 0;
|
||||
|
||||
// Serialize the quotes array
|
||||
// Serialize the quotes array, capped to what fits in the fixed buffer (see above).
|
||||
size_t quote_count = state->quotes.size;
|
||||
if (quote_count > UINT8_MAX) {
|
||||
quote_count = UINT8_MAX;
|
||||
if (quote_count > MAX_SERIALIZED_QUOTES) {
|
||||
quote_count = MAX_SERIALIZED_QUOTES;
|
||||
}
|
||||
buffer[size++] = (char)quote_count;
|
||||
|
||||
@@ -322,12 +565,18 @@ unsigned int tree_sitter_perl_external_scanner_serialize(void *payload, char *bu
|
||||
}
|
||||
size += quote_count * sizeof(TSPQuote);
|
||||
|
||||
// Serialize the heredoc state and delimiter
|
||||
buffer[size++] = (char)state->heredoc_interpolates;
|
||||
buffer[size++] = (char)state->heredoc_indents;
|
||||
// Serialize the heredoc consume state, then the whole pending FIFO queue:
|
||||
// a count byte followed by each entry's flags + delimiter.
|
||||
buffer[size++] = (char)state->heredoc_state;
|
||||
memcpy(&buffer[size], &state->heredoc_delim, sizeof(TSPString));
|
||||
size += sizeof(TSPString);
|
||||
buffer[size++] = (char)state->heredoc_count;
|
||||
for (uint8_t i = 0; i < state->heredoc_count; i++) {
|
||||
HeredocEntry *e = &state->heredoc_queue[i];
|
||||
buffer[size++] = (char)e->interpolates;
|
||||
buffer[size++] = (char)e->indents;
|
||||
memcpy(&buffer[size], &e->delim, sizeof(TSPString));
|
||||
size += sizeof(TSPString);
|
||||
}
|
||||
buffer[size++] = (char)state->recovery_emitted;
|
||||
|
||||
return size;
|
||||
}
|
||||
@@ -340,6 +589,9 @@ void tree_sitter_perl_external_scanner_deserialize(void *payload, const char *bu
|
||||
if (length > 0) {
|
||||
// Deserialize the quotes array
|
||||
size_t quote_count = (uint8_t)buffer[size++];
|
||||
// Defensive: a well-formed buffer never exceeds this (serialize caps it), but
|
||||
// clamp anyway so a malformed/old buffer can't drive an oversized memcpy.
|
||||
if (quote_count > MAX_SERIALIZED_QUOTES) quote_count = MAX_SERIALIZED_QUOTES;
|
||||
if (quote_count > 0) {
|
||||
array_reserve(&state->quotes, quote_count);
|
||||
state->quotes.size = quote_count;
|
||||
@@ -347,13 +599,21 @@ void tree_sitter_perl_external_scanner_deserialize(void *payload, const char *bu
|
||||
size += quote_count * sizeof(TSPQuote);
|
||||
}
|
||||
|
||||
// Deserialize the heredoc state and delimiter
|
||||
state->heredoc_interpolates = (bool)buffer[size++];
|
||||
state->heredoc_indents = (bool)buffer[size++];
|
||||
// Deserialize the heredoc consume state and the pending FIFO queue.
|
||||
state->heredoc_state = (enum HeredocState)buffer[size++];
|
||||
|
||||
memcpy(&state->heredoc_delim, &buffer[size], sizeof(TSPString));
|
||||
size += sizeof(TSPString);
|
||||
state->heredoc_count = (uint8_t)buffer[size++];
|
||||
if (state->heredoc_count > HEREDOC_QUEUE_MAX) state->heredoc_count = HEREDOC_QUEUE_MAX;
|
||||
for (uint8_t i = 0; i < state->heredoc_count; i++) {
|
||||
HeredocEntry *e = &state->heredoc_queue[i];
|
||||
e->interpolates = (bool)buffer[size++];
|
||||
e->indents = (bool)buffer[size++];
|
||||
memcpy(&e->delim, &buffer[size], sizeof(TSPString));
|
||||
size += sizeof(TSPString);
|
||||
}
|
||||
state->recovery_emitted = (bool)buffer[size++];
|
||||
} else {
|
||||
state->heredoc_count = 0;
|
||||
state->heredoc_state = HEREDOC_NONE;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -363,6 +623,9 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
|
||||
|
||||
bool is_ERROR = valid_symbols[TOKEN_ERROR];
|
||||
bool skipped_whitespace = false;
|
||||
bool crossed_newline = false;
|
||||
bool recovery_emitted = state->recovery_emitted;
|
||||
state->recovery_emitted = false;
|
||||
|
||||
int32_t c = lexer->lookahead;
|
||||
|
||||
@@ -372,16 +635,60 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
|
||||
TOKEN(TOKEN_GOBBLED_CONTENT);
|
||||
}
|
||||
|
||||
/* `format NAME = <newline> BODY .` — the body (line after `=` up to and
|
||||
* including a lone `.` line) is opaque content the LR grammar can't parse, read
|
||||
* as one TOKEN_FORMAT_CONTENT. Lives before whitespace handling so the body's
|
||||
* leading newline isn't skipped separately. */
|
||||
if (!is_ERROR && valid_symbols[TOKEN_FORMAT_CONTENT]) {
|
||||
/* swallow the remainder of the `format ... =` line, then its newline */
|
||||
while (c != '\n' && !lexer->eof(lexer)) ADVANCE_C;
|
||||
if (c == '\n') ADVANCE_C;
|
||||
/* now read body lines; stop after a line consisting solely of `.`
|
||||
* (perl: a lone `.`, trailing whitespace tolerated) */
|
||||
while (!lexer->eof(lexer)) {
|
||||
/* peek whether this whole line is just `.` */
|
||||
bool only_dot = false;
|
||||
if (c == '.') {
|
||||
ADVANCE_C;
|
||||
only_dot = true;
|
||||
while (c == ' ' || c == '\t' || c == '\r') ADVANCE_C;
|
||||
if (c != '\n' && !lexer->eof(lexer)) only_dot = false;
|
||||
}
|
||||
if (only_dot) {
|
||||
if (c == '\n') ADVANCE_C; /* include the terminator's newline */
|
||||
break;
|
||||
}
|
||||
/* not the terminator: consume to end of this line */
|
||||
while (c != '\n' && !lexer->eof(lexer)) ADVANCE_C;
|
||||
if (c == '\n') ADVANCE_C;
|
||||
}
|
||||
MARK_END;
|
||||
TOKEN(TOKEN_FORMAT_CONTENT);
|
||||
}
|
||||
|
||||
/* we use this to force tree-sitter to stay on the error branch of a nonassoc
|
||||
* operator */
|
||||
if (!is_ERROR && valid_symbols[TOKEN_NONASSOC]) TOKEN(TOKEN_NONASSOC);
|
||||
|
||||
/* The `x` repetition operator glued to its count (`"ab"x3`, `("")x4`). The
|
||||
* internal lexer would greedily take `x3` as one identifier, so when the
|
||||
* grammar offers the operator here (an operator is expected — perl's
|
||||
* XOPERATOR state, which `valid_symbols` encodes) and `x` is glued to a digit,
|
||||
* we emit it explicitly, consuming just the `x`. Anything else (`xor`, an
|
||||
* identifier, a space-delimited `x`) falls through to the normal lexer. */
|
||||
if (!is_ERROR && valid_symbols[TOKEN_X_OP] && c == 'x') {
|
||||
ADVANCE_C;
|
||||
if (c >= '0' && c <= '9') { MARK_END; TOKEN(TOKEN_X_OP); }
|
||||
return false;
|
||||
}
|
||||
|
||||
// this is whitespace sensitive, so it must go before any whitespace is
|
||||
// skipped
|
||||
if (valid_symbols[TOKEN_HEREDOC_MIDDLE] && !is_ERROR) {
|
||||
if (valid_symbols[TOKEN_HEREDOC_MIDDLE] && !is_ERROR && state->heredoc_count > 0) {
|
||||
HeredocEntry *front = lexerstate_front_heredoc(state);
|
||||
DEBUG("Beginning heredoc contents\n", 0);
|
||||
if (state->heredoc_state != HEREDOC_CONTINUE) {
|
||||
TSPString line;
|
||||
TSPString line = {0};
|
||||
// read as many lines as we can
|
||||
while (!lexer->eof(lexer)) {
|
||||
tspstring_reset(&line);
|
||||
@@ -391,7 +698,7 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
|
||||
state->heredoc_state == HEREDOC_END || lexer->get_column(lexer) == 0;
|
||||
bool saw_escape = false;
|
||||
DEBUG("Starting loop at col %d\n", lexer->get_column(lexer));
|
||||
if (is_valid_start_pos && state->heredoc_indents) {
|
||||
if (is_valid_start_pos && front->indents) {
|
||||
DEBUG("Skipping initial whitespace in heredoc\n", 0);
|
||||
skip_whitespace(lexer);
|
||||
c = lexer->lookahead;
|
||||
@@ -411,8 +718,8 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
|
||||
if (c == '$' || c == '@' || c == '\\') saw_escape = true;
|
||||
ADVANCE_C;
|
||||
}
|
||||
DEBUG("got length %d, want length %d\n", line.length, state->heredoc_delim.length);
|
||||
if (is_valid_start_pos && tspstring_eq(&line, &state->heredoc_delim)) {
|
||||
DEBUG("got length %d, want length %d\n", line.length, front->delim.length);
|
||||
if (is_valid_start_pos && tspstring_eq(&line, &front->delim)) {
|
||||
// if we've read already, we return everything up until now
|
||||
if (state->heredoc_state != HEREDOC_END) {
|
||||
state->heredoc_state = HEREDOC_END;
|
||||
@@ -422,7 +729,7 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
|
||||
lexerstate_finish_heredoc(state);
|
||||
TOKEN(TOKEN_HEREDOC_END);
|
||||
}
|
||||
if (saw_escape && state->heredoc_interpolates) {
|
||||
if (saw_escape && front->interpolates) {
|
||||
// we'll repeat this line in continue mode where we'll pause midline
|
||||
state->heredoc_state = HEREDOC_CONTINUE;
|
||||
TOKEN(TOKEN_HEREDOC_MIDDLE);
|
||||
@@ -452,10 +759,10 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
|
||||
}
|
||||
}
|
||||
|
||||
if (iswspace(c) && valid_symbols[TOKEN_NO_INTERP_WHITESPACE_ZW]) {
|
||||
if (!is_ERROR && iswspace(c) && valid_symbols[TOKEN_NO_INTERP_WHITESPACE_ZW]) {
|
||||
TOKEN(TOKEN_NO_INTERP_WHITESPACE_ZW);
|
||||
}
|
||||
skip_ws_to_eol(lexer);
|
||||
crossed_newline = skip_ws_to_eol(lexer);
|
||||
/* heredocs override everything, so they must be here before */
|
||||
if (valid_symbols[TOKEN_HEREDOC_START]) {
|
||||
if (state->heredoc_state == HEREDOC_START && lexer->get_column(lexer) == 0) {
|
||||
@@ -506,39 +813,256 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
|
||||
// CTRL-Z must be here, b/c it cares about whitespace
|
||||
if (c == 26 && valid_symbols[TOKEN_CTRL_Z]) TOKEN(TOKEN_CTRL_Z);
|
||||
|
||||
bool any_recovery_valid =
|
||||
valid_symbols[TOKEN_RECOVER_ARROW] ||
|
||||
valid_symbols[TOKEN_RECOVER_PAREN_CLOSE] ||
|
||||
valid_symbols[TOKEN_RECOVER_BRACKET_CLOSE] ||
|
||||
valid_symbols[TOKEN_RECOVER_BRACE_CLOSE] ||
|
||||
valid_symbols[TOKEN_RECOVER_BLOCK_CLOSE] ||
|
||||
valid_symbols[PERLY_SEMICOLON];
|
||||
|
||||
// Contextual keywords: class/role/method are declarations only when a name (or
|
||||
// a block, for method) follows; otherwise they're ordinary barewords
|
||||
// (`class($cv)`, `role { … }`, `-role`). Decide here, before emitting, so
|
||||
// there's no keyword-vs-call ambiguity. The OO keywords are deferred while
|
||||
// recovery is pending because they trigger the structural body block-close,
|
||||
// which must fire first; async/try don't, so they're classified unconditionally.
|
||||
bool oo_kw_valid = valid_symbols[TOKEN_KW_CLASS] || valid_symbols[TOKEN_KW_ROLE] || valid_symbols[TOKEN_KW_METHOD];
|
||||
bool asynctry_valid = valid_symbols[TOKEN_KW_ASYNC] || valid_symbols[TOKEN_KW_TRY];
|
||||
bool recovery_pending = any_recovery_valid && (crossed_newline || recovery_emitted);
|
||||
bool enter_oo = oo_kw_valid && !recovery_pending && (c == 'c' || c == 'r' || c == 'm');
|
||||
bool enter_asynctry = asynctry_valid && (c == 'a' || c == 't');
|
||||
if (!is_ERROR && (enter_oo || enter_asynctry)) {
|
||||
char word[8];
|
||||
int wlen = 0;
|
||||
while (isidcont(c)) { // consume the whole identifier (store first 7 chars)
|
||||
if (wlen < 7) word[wlen++] = (char)c;
|
||||
lexer->advance(lexer, false);
|
||||
c = lexer->lookahead;
|
||||
}
|
||||
word[wlen] = '\0';
|
||||
MARK_END; // the token covers exactly the word (lookahead past here is free)
|
||||
while (is_tsp_whitespace(c)) { // peek past whitespace at the next token
|
||||
lexer->advance(lexer, false);
|
||||
c = lexer->lookahead;
|
||||
}
|
||||
bool is_class = !recovery_pending && valid_symbols[TOKEN_KW_CLASS] && strcmp(word, "class") == 0;
|
||||
bool is_role = !recovery_pending && valid_symbols[TOKEN_KW_ROLE] && strcmp(word, "role") == 0;
|
||||
bool is_method = !recovery_pending && valid_symbols[TOKEN_KW_METHOD] && strcmp(word, "method") == 0;
|
||||
bool is_async = valid_symbols[TOKEN_KW_ASYNC] && strcmp(word, "async") == 0;
|
||||
bool is_try = valid_symbols[TOKEN_KW_TRY] && strcmp(word, "try") == 0;
|
||||
|
||||
// `try` is the try/catch keyword only before a block `{`; `try(...)`/`try =>`
|
||||
// /`try;` are bareword calls. (`catch` is never over-reserved.)
|
||||
if (is_try) {
|
||||
if (c == '{') TOKEN(TOKEN_KW_TRY);
|
||||
}
|
||||
|
||||
// `async` is a keyword before a block (`async { … }` — threads::async, a bare
|
||||
// block run as an anon sub; F::AA itself has only `async sub`) or before a
|
||||
// sub/method/extended declaration. Else it's a bareword/call.
|
||||
if (is_async) {
|
||||
if (c == '{') TOKEN(TOKEN_KW_ASYNC);
|
||||
if (c == 's' || c == 'm' || c == 'e') { // peek for sub/method/extended
|
||||
char nword[9];
|
||||
int nlen = 0;
|
||||
while (isidcont(c)) {
|
||||
if (nlen < 8) nword[nlen++] = (char)c;
|
||||
lexer->advance(lexer, false);
|
||||
c = lexer->lookahead;
|
||||
}
|
||||
nword[nlen] = '\0';
|
||||
if (strcmp(nword, "sub") == 0 || strcmp(nword, "method") == 0 ||
|
||||
strcmp(nword, "extended") == 0)
|
||||
TOKEN(TOKEN_KW_ASYNC);
|
||||
// Not a decl: `async` is a bareword/call. Must `return false` (re-lex),
|
||||
// NOT `goto kw_autoquote` like the path below — we consumed past `async`
|
||||
// into the next word, so the handler would read *that* word's `=>` as
|
||||
// async's own fat comma and wrongly autoquote `async` (`async send=>1`).
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// `class NAME` / `role NAME` — a name following => declaration; else bareword.
|
||||
if ((is_class || is_role) && isidfirst(c))
|
||||
TOKEN(is_class ? TOKEN_KW_CLASS : TOKEN_KW_ROLE);
|
||||
|
||||
// `method NAME { … }` / `method { … }` are declarations (named/anonymous);
|
||||
// `method NAME => sub { … }` (MooseX) is a call — peek past the name for `=>`.
|
||||
if (is_method) {
|
||||
if (c == '{' || c == '(') TOKEN(TOKEN_KW_METHOD); // anonymous method
|
||||
if (isidfirst(c)) {
|
||||
while (isidcont(c)) { lexer->advance(lexer, false); c = lexer->lookahead; } // skip the name
|
||||
while (is_tsp_whitespace(c)) { lexer->advance(lexer, false); c = lexer->lookahead; }
|
||||
if (c != '=') TOKEN(TOKEN_KW_METHOD); // `{`/`(`/`:`/`;` => decl; `=>` => call
|
||||
return false; // method NAME => … : re-lex as a call (the name autoquotes)
|
||||
}
|
||||
}
|
||||
|
||||
// Bareword/call, or an autoquoted key (`class => 1`, `$h{m}`). Can't just
|
||||
// `return false` — a quote-op word (`m`/`s`/`y`) would re-lex as a match — so
|
||||
// fall through to the canonical autoquote handler; MARK_END already fixed the
|
||||
// token at the word, and it emits the autoquote or returns false on no match.
|
||||
goto kw_autoquote;
|
||||
}
|
||||
|
||||
// === Error recovery: close unclosed delimiters and insert semicolons ===
|
||||
//
|
||||
// When the scanner sees '}', ';', EOF, or a statement keyword on a new
|
||||
// line, it emits recovery tokens to unwind open delimiters. Each call
|
||||
// peels one layer — the parser keeps calling until everything is closed.
|
||||
//
|
||||
// Syntactic boundary: '}', ';', or EOF
|
||||
if (!is_ERROR) {
|
||||
if (c == '}' || c == ';' || lexer->eof(lexer)) {
|
||||
EMIT_RECOVERY_TOKENS(c != '}');
|
||||
}
|
||||
}
|
||||
|
||||
if (valid_symbols[PERLY_SEMICOLON]) {
|
||||
if (c == '}' || lexer->eof(lexer)) {
|
||||
// do a PERLY_SEMICOLON unless we're in brace autoquoting
|
||||
if (is_ERROR || !valid_symbols[TOKEN_BRACE_END_ZW]) {
|
||||
DEBUG("Fake PERLY_SEMICOLON at end-of-scope\n", 0);
|
||||
// no advance
|
||||
TOKEN(PERLY_SEMICOLON);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (lexer->eof(lexer)) return false;
|
||||
|
||||
// Statement keyword boundary: a statement keyword on a new line means
|
||||
// the previous expression is done. Also fires when a recovery token
|
||||
// was just emitted (recovery_emitted flag) since the newline was already
|
||||
// consumed by the previous call.
|
||||
// Gate on the first-char filter so we don't call MARK_END (and freeze
|
||||
// the token end at the keyword-start position) when the lookahead can't
|
||||
// possibly start a keyword. Without this, a later TOKEN() emission
|
||||
// (e.g. TOKEN_APOSTROPHE/TOKEN_DOUBLE_QUOTE which do ADVANCE_C+TOKEN
|
||||
// and never re-mark) inherits the stale mark and reports a zero-width
|
||||
// open-quote, which derails the string body.
|
||||
if ((crossed_newline || recovery_emitted) && !is_ERROR && any_recovery_valid &&
|
||||
!KEYWORD_FIRST_CHAR_FILTER(c)) {
|
||||
MARK_END; // zero-width position for recovery tokens
|
||||
bool is_structural = false;
|
||||
enum PeekResult peek = peek_is_statement_keyword(lexer, &is_structural);
|
||||
if (peek == PEEK_KEYWORD) {
|
||||
DEBUG("keyword boundary\n", 0);
|
||||
EMIT_RECOVERY_TOKENS(true);
|
||||
// Body block-close: only a structural OO keyword (method/class/role)
|
||||
// closes a half-typed sub/method body. Fires after any inner paren/
|
||||
// bracket closers above; the enclosing class block can't accept this
|
||||
// token, so the cascade stops at one body (no over-closing).
|
||||
if (is_structural && valid_symbols[TOKEN_RECOVER_BLOCK_CLOSE]) {
|
||||
DEBUG("body block recovery at structural keyword\n", 0);
|
||||
// Re-arm the cascade so the next scan re-enters and can close the NEXT
|
||||
// enclosing recover-aware block (e.g. an unclosed `if`/loop body inside
|
||||
// the method: close the if-block, then the method body).
|
||||
state->recovery_emitted = true;
|
||||
TOKEN(TOKEN_RECOVER_BLOCK_CLOSE);
|
||||
}
|
||||
// A statement terminator may need to fire BEFORE the body block-close:
|
||||
// after an inner closer (e.g. `my @x = [` -> `]`) completes an expression,
|
||||
// the statement still needs `;` before the enclosing body can reduce and
|
||||
// close. Re-arm the recovery cascade (recovery_emitted) so the next scan
|
||||
// re-enters here and can emit the block-close. Harmless when nothing
|
||||
// follows: the re-entry finds no further valid recovery token.
|
||||
if (valid_symbols[PERLY_SEMICOLON]) {
|
||||
state->recovery_emitted = true;
|
||||
TOKEN(PERLY_SEMICOLON);
|
||||
}
|
||||
}
|
||||
if (peek == PEEK_FAT_COMMA) {
|
||||
// A statement keyword used as a fat-comma key (`package => 1`). We can't
|
||||
// mark the word boundary here (peek preserved the caller's zero-width
|
||||
// recovery mark for the keyword path, and re-marking would consume the
|
||||
// keyword). Instead emit a zero-width "autoquote ahead" marker at the
|
||||
// word start: tree-sitter re-lexes the word via the normal autoquote
|
||||
// path, which marks the word boundary correctly. The recovery gate
|
||||
// won't re-fire on re-entry (no newline precedes the word that scan), so
|
||||
// there's no loop. Falls back to the old (zero-width-span) emission if
|
||||
// the marker isn't accepted in this state.
|
||||
if (valid_symbols[TOKEN_FAT_COMMA_AUTOQUOTED_AHEAD])
|
||||
TOKEN(TOKEN_FAT_COMMA_AUTOQUOTED_AHEAD);
|
||||
MARK_END;
|
||||
c = lexer->lookahead;
|
||||
goto fat_comma_check;
|
||||
}
|
||||
if (peek == PEEK_AUTOQUOTE_KEY) {
|
||||
// A non-keyword bareword used as a fat-comma key (`name => 1`) on a
|
||||
// continuation line where the recovery gate fired. Peek already marked
|
||||
// the token end at the word boundary (covering exactly the word), so do
|
||||
// NOT MARK_END again — that would re-mark at '=' and emit a zero-width
|
||||
// key. Resume at the autoquote handler's => check.
|
||||
c = lexer->lookahead;
|
||||
goto fat_comma_check;
|
||||
}
|
||||
if (peek == PEEK_NOT_KEYWORD)
|
||||
return false;
|
||||
}
|
||||
|
||||
if (valid_symbols[TOKEN_OPEN_FILEGLOB_BRACKET] || valid_symbols[TOKEN_OPEN_READLINE_BRACKET] || valid_symbols[PERLY_HEREDOC]) {
|
||||
if (c == '<') {
|
||||
ADVANCE_C;
|
||||
MARK_END;
|
||||
// ah, we have a heredoc; let's just go down to that section then
|
||||
if (c == '<') goto heredoc_token_handling;
|
||||
if (c == '$') ADVANCE_C;
|
||||
// Gather the `<...>` body as we go, starting right after `<`, so the
|
||||
// fileglob content heuristic below sees the WHOLE body -- including
|
||||
// the `$ident` prefix the readline probe consumes here (otherwise a
|
||||
// `<$sner >` glob would reach the heuristic with content " ").
|
||||
char content[256];
|
||||
size_t clen = 0;
|
||||
if (c == '$') { content[clen++] = '$'; ADVANCE_C; }
|
||||
// we now zoooom as many ident chars as we can
|
||||
while (isidcont(c)) ADVANCE_C;
|
||||
while (isidcont(c)) {
|
||||
if (clen < sizeof(content)) content[clen++] = (c < 0x80) ? (char)c : (char)0x7f;
|
||||
ADVANCE_C;
|
||||
}
|
||||
// if ident chars took us until the closing `>` then we're readline FILEHANDLE
|
||||
if (c == '>') TOKEN(TOKEN_OPEN_READLINE_BRACKET);
|
||||
// otherwise we're a fileglob operator, and we set up our string parse + be excellent
|
||||
lexerstate_push_quote(state, '<');
|
||||
TOKEN(TOKEN_OPEN_FILEGLOB_BRACKET);
|
||||
// Otherwise `<...>` is either a fileglob (`<*.c>`) or the relational
|
||||
// `<` operator (`CONST < 0`); both are live after an ambiguous
|
||||
// bareword. Gather the body and the bytes after the `>` (pure
|
||||
// lookahead past MARK_END, so the token stays the one-char `<`;
|
||||
// non-ASCII -> 0x7f), then let tsp_is_fileglob() decide -- see
|
||||
// tsp_intuit_readline.h.
|
||||
if (valid_symbols[TOKEN_OPEN_FILEGLOB_BRACKET]) {
|
||||
// `content`/`clen` already hold the `$ident` prefix from above.
|
||||
// Stop at a statement boundary so we don't chase a `>` onto the
|
||||
// next line.
|
||||
while (c != '>' && c != '<' && c != ';' && c != '\n' &&
|
||||
!lexer->eof(lexer)) {
|
||||
if (clen < sizeof(content))
|
||||
content[clen++] = (c < 0x80) ? (char)c : (char)0x7f;
|
||||
ADVANCE_C;
|
||||
}
|
||||
|
||||
if (c == '>') {
|
||||
ADVANCE_C; // step past the closing `>` (still pure lookahead)
|
||||
char after[256];
|
||||
size_t alen = 0;
|
||||
while (alen < sizeof(after) && c != '\n' && !lexer->eof(lexer)) {
|
||||
after[alen++] = (c < 0x80) ? (char)c : (char)0x7f;
|
||||
ADVANCE_C;
|
||||
}
|
||||
if (tsp_is_fileglob(content, clen, after, alen)) {
|
||||
lexerstate_push_quote(state, '<');
|
||||
TOKEN(TOKEN_OPEN_FILEGLOB_BRACKET);
|
||||
}
|
||||
}
|
||||
}
|
||||
// not a readline, not a fileglob — let `<` fall through to the
|
||||
// grammar as the relational operator.
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (valid_symbols[TOKEN_DOLLAR_IDENT_ZW]) {
|
||||
// false on word chars, another dollar or {
|
||||
if (!isidcont(c) && !tsp_strchr("${", c)) {
|
||||
if (!is_ERROR && valid_symbols[TOKEN_DOLLAR_IDENT_ZW]) {
|
||||
// false on word chars, another dollar or { -- but if whitespace intervened
|
||||
// the `$` can't begin a glued deref ($$foo needs the name glued on), so a
|
||||
// following word char is a separate token: `$$ eq …`, `$$ and …` are the
|
||||
// PID var `$$`, not `${$eq}`. Treat skipped whitespace as a hard boundary.
|
||||
if (!tsp_strchr("${", c) && (skipped_whitespace || !isidcont(c))) {
|
||||
if (c == ':') {
|
||||
// NOTE - it's a syntax error to do $$:, so that's why we return
|
||||
// dollar_ident_zw in that case
|
||||
@@ -601,6 +1125,64 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
|
||||
return false;
|
||||
}
|
||||
|
||||
/* A '[' or '{' in a pattern is ambiguous: it can begin a subscript on the
|
||||
* preceding variable (/$foo[0]/, /$foo{k}/) or a character class /
|
||||
* quantifier (/[abc]/, /a{2,3}/). We replicate perl's S_intuit_more
|
||||
* heuristic (tsp_intuit_more) to decide. When it's a class/quantifier we
|
||||
* emit a literal opener (aliased back to '[' / '{' in the grammar); when
|
||||
* it's a subscript we decline so the grammar's token.immediate('[' / '{')
|
||||
* takes over. */
|
||||
if ((valid_symbols[TOKEN_REGEXP_OPEN_BRACKET] && c == '[') ||
|
||||
(valid_symbols[TOKEN_REGEXP_OPEN_BRACE] && c == '{')) {
|
||||
/* A pattern-LEADING '['/'{' that is the quote's own opening delimiter is a
|
||||
* balanced nested delimiter (m{{...}}, qr{ {...} }, s{{...}}, m[[...]]),
|
||||
* NOT a subscript: there is no preceding variable for intuit_more to weigh,
|
||||
* and its subscript verdict here would be meaningless. Fall through to the
|
||||
* Q/QQ content scanner, which consumes the opener while tracking the nesting
|
||||
* count so the matching inner close doesn't terminate the quote early.
|
||||
* Anything beyond the leading position is genuinely ambiguous again, so the
|
||||
* heuristic below applies as before. */
|
||||
bool leading = lexerstate_take_body_lead(state, c);
|
||||
if (leading) {
|
||||
/* leave the opener for the content scanner below */
|
||||
} else {
|
||||
int32_t open = c;
|
||||
int32_t close = (open == '[') ? ']' : '}';
|
||||
|
||||
char buf[256];
|
||||
int n = 0;
|
||||
buf[n++] = (char)open;
|
||||
|
||||
ADVANCE_C; /* consume the opener... */
|
||||
MARK_END; /* ...the emitted token is exactly that one char */
|
||||
|
||||
/* Read ahead to gather the construct for the heuristic, stopping once we
|
||||
* have buffered the matching close (so intuit_more can find it) or we run
|
||||
* out of room / input. These advances past MARK_END are pure lookahead;
|
||||
* the token stays one char wide. Non-ASCII collapses to a placeholder
|
||||
* since the heuristic is ASCII-only by design (a best-effort gap). */
|
||||
while (n < (int)sizeof(buf) && c != 0 && !lexer->eof(lexer)) {
|
||||
buf[n++] = (c < 0x80) ? (char)c : (char)0x7f;
|
||||
if (c == close) break;
|
||||
ADVANCE_C;
|
||||
}
|
||||
|
||||
if (!tsp_intuit_more(buf, n)) {
|
||||
/* If this class/quantifier opener is also the quote's own delimiter
|
||||
* (`m{ \d{2} }`, `m[ [abc] ]`), it opens a nested level — bump the count
|
||||
* so the matching inner close doesn't terminate the quote early. The
|
||||
* content scanner does this when it consumes `{`/`[` inline; we must do
|
||||
* the same here, since we only reach this branch when the opener leads a
|
||||
* scan (e.g. right after an escape) and the content scanner is bypassed. */
|
||||
int32_t qi = lexerstate_is_quote_opener(state, open);
|
||||
if (qi) lexerstate_saw_opener(state, qi);
|
||||
TOKEN(open == '[' ? TOKEN_REGEXP_OPEN_BRACKET : TOKEN_REGEXP_OPEN_BRACE);
|
||||
}
|
||||
/* subscript: let the grammar's immediate '[' / '{' win */
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (valid_symbols[TOKEN_POD]) {
|
||||
int column = lexer->get_column(lexer);
|
||||
if (column == 0 && c == '=') {
|
||||
@@ -648,7 +1230,7 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
|
||||
bool should_indent = false;
|
||||
bool should_interpolate = true;
|
||||
|
||||
TSPString delim;
|
||||
TSPString delim = {0};
|
||||
tspstring_reset(&delim);
|
||||
if (!skipped_whitespace) {
|
||||
if (c == '~') {
|
||||
@@ -694,7 +1276,13 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
|
||||
ADVANCE_C;
|
||||
}
|
||||
}
|
||||
if (delim.length > 0) {
|
||||
// We stop the loop above either at the closing quote or at EOF. Only
|
||||
// commit a heredoc if we actually found the closer; an EMPTY delimiter
|
||||
// (`<<''` / `<<""`) is legal — perl terminates its body at the next blank
|
||||
// line, which the body matcher already handles (an empty delim compares
|
||||
// equal to an empty line). Keying off the closer rather than a non-empty
|
||||
// delim is also more correct: it won't mis-fire on EOF-without-closer.
|
||||
if (c == delim_open) {
|
||||
// gotta eat that delimiter
|
||||
ADVANCE_C;
|
||||
// gotta null terminate up in here
|
||||
@@ -724,10 +1312,28 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
|
||||
}
|
||||
MARK_END;
|
||||
|
||||
// In a paired multi-part quote (s{}{}, tr[][], ...), the replacement pair
|
||||
// opens a fresh quote whose delimiter may differ from the pattern's. The
|
||||
// pattern's quote is still on top of the stack (middle_close consumes its
|
||||
// closer but doesn't pop it), so its closer would wrongly stay active while
|
||||
// we scan the replacement. Now that the second pair is opening, retire it.
|
||||
// MIDDLE_SKIP being a valid symbol here means we're at the middle choice
|
||||
// point (between pattern and replacement) rather than an initial begin.
|
||||
if (valid_symbols[TOKEN_QUOTELIKE_MIDDLE_SKIP] && state->quotes.size) {
|
||||
lexerstate_pop_quote(state, state->quotes.size);
|
||||
}
|
||||
|
||||
lexerstate_push_quote(state, delim);
|
||||
// TODO - fill in this debug print here?
|
||||
// DEBUG("Generic QSTRING open='%c' close='%c'\n", state->delim_open,
|
||||
// state->delim_close);
|
||||
/* Pattern-leading bracket lookahead (see TSPQuote.body_leads_with_delim):
|
||||
* for a paired delimiter, peek past any leading whitespace -- if the body's
|
||||
* first real char is the delimiter again (m{{...}}, qr{ {...} }, m[[...]]),
|
||||
* it's a literal leading group, not a subscript. This is pure lookahead:
|
||||
* MARK_END already covers just the opener, so these advances don't extend
|
||||
* the emitted token and the body is re-scanned from the opener's end. */
|
||||
if (close_for_open(delim)) {
|
||||
while (is_tsp_whitespace(c)) ADVANCE_C;
|
||||
if (c == delim) array_back(&state->quotes)->body_leads_with_delim = true;
|
||||
}
|
||||
TOKEN(TOKEN_QUOTELIKE_BEGIN);
|
||||
}
|
||||
|
||||
@@ -885,6 +1491,23 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
|
||||
}
|
||||
return false;
|
||||
}
|
||||
// A lone non-identifier punctuation variable inside ${...} — ${@}, ${!},
|
||||
// ${%}, ${$}, ${"} … — names the punctuation variable $@/$!/…, NOT a
|
||||
// sigil/operator. The main lexer would otherwise pick the sigil/operator token
|
||||
// (and for @ / % that token swallows the closing brace, decapitating the rest
|
||||
// of the file). Recognize it here with the same '}'-lookahead the bareword
|
||||
// path uses: one var char immediately followed (modulo whitespace) by '}' is
|
||||
// the autoquoted name. Identifiers, digits and ^carets are handled by the
|
||||
// grammar's other varname alternatives; '{' would begin a block, '#' a comment.
|
||||
if (valid_symbols[TOKEN_BRACE_AUTOQUOTED] && !isidfirst(c) &&
|
||||
c > ' ' && c != '}' && c != '{' && c != '^' && c != '#' &&
|
||||
!(c >= '0' && c <= '9')) {
|
||||
ADVANCE_C;
|
||||
MARK_END;
|
||||
while (is_tsp_whitespace(c)) ADVANCE_C;
|
||||
if (c == '}') TOKEN(TOKEN_BRACE_AUTOQUOTED);
|
||||
return false;
|
||||
}
|
||||
if (isidfirst(c) &&
|
||||
(valid_symbols[TOKEN_FAT_COMMA_AUTOQUOTED] || valid_symbols[TOKEN_BRACE_AUTOQUOTED])) {
|
||||
// we zip until the end of the identifier; then we do a lookeahed to see if it's autoquoted
|
||||
@@ -898,17 +1521,26 @@ bool tree_sitter_perl_external_scanner_scan(void *payload, TSLexer *lexer,
|
||||
|
||||
// NOTE - TS is annoying about skipping chars after you've hit done
|
||||
// mark_end, so we have to do the regular advance so our token actually shows up
|
||||
// The contextual-keyword block jumps here (after reading its word + MARK_END)
|
||||
// to reuse this comment-skip + fat_comma_check instead of duplicating them.
|
||||
kw_autoquote:
|
||||
while (is_tsp_whitespace(c) || c == '#') {
|
||||
while (is_tsp_whitespace(c)) ADVANCE_C;
|
||||
// now we need to skip comments - we get in a funny way if we have a quotelike
|
||||
// operator followed by a comment as the quote char
|
||||
if (c == '#') {
|
||||
ADVANCE_C;
|
||||
while (lexer->get_column(lexer)) ADVANCE_C;
|
||||
while (lexer->get_column(lexer) && !lexer->eof(lexer)) ADVANCE_C;
|
||||
}
|
||||
if (lexer->eof(lexer)) return false;
|
||||
// TODO - in theory there could be POD here that we needa skip over (EYES ROLL)
|
||||
}
|
||||
// The keyword peek's fat comma detection gotos here when it finds a
|
||||
// statement keyword followed by '=' on a new line. The peek has already
|
||||
// consumed the word + whitespace; lexer is at '='. MARK_END was set by
|
||||
// the peek's caller (covers the word). Same goto pattern as heredoc vs
|
||||
// diamond operator disambiguation.
|
||||
fat_comma_check:
|
||||
c1 = lexer->lookahead;
|
||||
ADVANCE_C;
|
||||
if (valid_symbols[TOKEN_FAT_COMMA_AUTOQUOTED]) {
|
||||
|
||||
@@ -52,67 +52,96 @@ extern "C" {
|
||||
|
||||
/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
|
||||
/// less than the array's current capacity, this function has no effect.
|
||||
#define array_reserve(self, new_capacity) \
|
||||
_array__reserve((Array *)(self), array_elem_size(self), new_capacity)
|
||||
#define array_reserve(self, new_capacity) \
|
||||
((self)->contents = _array__reserve( \
|
||||
(void *)(self)->contents, &(self)->capacity, \
|
||||
array_elem_size(self), new_capacity) \
|
||||
)
|
||||
|
||||
/// Free any memory allocated for this array. Note that this does not free any
|
||||
/// memory allocated for the array's contents.
|
||||
#define array_delete(self) _array__delete((Array *)(self))
|
||||
#define array_delete(self) \
|
||||
do { \
|
||||
if ((self)->contents) ts_free((self)->contents); \
|
||||
(self)->contents = NULL; \
|
||||
(self)->size = 0; \
|
||||
(self)->capacity = 0; \
|
||||
} while (0)
|
||||
|
||||
/// Push a new `element` onto the end of the array.
|
||||
#define array_push(self, element) \
|
||||
(_array__grow((Array *)(self), 1, array_elem_size(self)), \
|
||||
(self)->contents[(self)->size++] = (element))
|
||||
#define array_push(self, element) \
|
||||
do { \
|
||||
(self)->contents = _array__grow( \
|
||||
(void *)(self)->contents, (self)->size, &(self)->capacity, \
|
||||
1, array_elem_size(self) \
|
||||
); \
|
||||
(self)->contents[(self)->size++] = (element); \
|
||||
} while(0)
|
||||
|
||||
/// Increase the array's size by `count` elements.
|
||||
/// New elements are zero-initialized.
|
||||
#define array_grow_by(self, count) \
|
||||
do { \
|
||||
if ((count) == 0) break; \
|
||||
_array__grow((Array *)(self), count, array_elem_size(self)); \
|
||||
#define array_grow_by(self, count) \
|
||||
do { \
|
||||
if ((count) == 0) break; \
|
||||
(self)->contents = _array__grow( \
|
||||
(self)->contents, (self)->size, &(self)->capacity, \
|
||||
count, array_elem_size(self) \
|
||||
); \
|
||||
memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
|
||||
(self)->size += (count); \
|
||||
(self)->size += (count); \
|
||||
} while (0)
|
||||
|
||||
/// Append all elements from one array to the end of another.
|
||||
#define array_push_all(self, other) \
|
||||
#define array_push_all(self, other) \
|
||||
array_extend((self), (other)->size, (other)->contents)
|
||||
|
||||
/// Append `count` elements to the end of the array, reading their values from the
|
||||
/// `contents` pointer.
|
||||
#define array_extend(self, count, contents) \
|
||||
_array__splice( \
|
||||
(Array *)(self), array_elem_size(self), (self)->size, \
|
||||
0, count, contents \
|
||||
#define array_extend(self, count, other_contents) \
|
||||
(self)->contents = _array__splice( \
|
||||
(void*)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
array_elem_size(self), (self)->size, 0, count, other_contents \
|
||||
)
|
||||
|
||||
/// Remove `old_count` elements from the array starting at the given `index`. At
|
||||
/// the same index, insert `new_count` new elements, reading their values from the
|
||||
/// `new_contents` pointer.
|
||||
#define array_splice(self, _index, old_count, new_count, new_contents) \
|
||||
_array__splice( \
|
||||
(Array *)(self), array_elem_size(self), _index, \
|
||||
old_count, new_count, new_contents \
|
||||
#define array_splice(self, _index, old_count, new_count, new_contents) \
|
||||
(self)->contents = _array__splice( \
|
||||
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
array_elem_size(self), _index, old_count, new_count, new_contents \
|
||||
)
|
||||
|
||||
/// Insert one `element` into the array at the given `index`.
|
||||
#define array_insert(self, _index, element) \
|
||||
_array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
|
||||
#define array_insert(self, _index, element) \
|
||||
(self)->contents = _array__splice( \
|
||||
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
array_elem_size(self), _index, 0, 1, &(element) \
|
||||
)
|
||||
|
||||
/// Remove one element from the array at the given `index`.
|
||||
#define array_erase(self, _index) \
|
||||
_array__erase((Array *)(self), array_elem_size(self), _index)
|
||||
_array__erase((void *)(self)->contents, &(self)->size, array_elem_size(self), _index)
|
||||
|
||||
/// Pop the last element off the array, returning the element by value.
|
||||
#define array_pop(self) ((self)->contents[--(self)->size])
|
||||
|
||||
/// Assign the contents of one array to another, reallocating if necessary.
|
||||
#define array_assign(self, other) \
|
||||
_array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
|
||||
#define array_assign(self, other) \
|
||||
(self)->contents = _array__assign( \
|
||||
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
(const void *)(other)->contents, (other)->size, array_elem_size(self) \
|
||||
)
|
||||
|
||||
/// Swap one array with another
|
||||
#define array_swap(self, other) \
|
||||
_array__swap((Array *)(self), (Array *)(other))
|
||||
#define array_swap(self, other) \
|
||||
do { \
|
||||
void *_array_swap_tmp = (void *)(self)->contents; \
|
||||
(self)->contents = (other)->contents; \
|
||||
(other)->contents = _array_swap_tmp; \
|
||||
_array__swap(&(self)->size, &(self)->capacity, \
|
||||
&(other)->size, &(other)->capacity); \
|
||||
} while (0)
|
||||
|
||||
/// Get the size of the array contents
|
||||
#define array_elem_size(self) (sizeof *(self)->contents)
|
||||
@@ -157,82 +186,90 @@ extern "C" {
|
||||
|
||||
// Private
|
||||
|
||||
typedef Array(void) Array;
|
||||
|
||||
/// This is not what you're looking for, see `array_delete`.
|
||||
static inline void _array__delete(Array *self) {
|
||||
if (self->contents) {
|
||||
ts_free(self->contents);
|
||||
self->contents = NULL;
|
||||
self->size = 0;
|
||||
self->capacity = 0;
|
||||
}
|
||||
}
|
||||
// Pointers to individual `Array` fields (rather than the entire `Array` itself)
|
||||
// are passed to the various `_array__*` functions below to address strict aliasing
|
||||
// violations that arises when the _entire_ `Array` struct is passed as `Array(void)*`.
|
||||
//
|
||||
// The `Array` type itself was not altered as a solution in order to avoid breakage
|
||||
// with existing consumers (in particular, parsers with external scanners).
|
||||
|
||||
/// This is not what you're looking for, see `array_erase`.
|
||||
static inline void _array__erase(Array *self, size_t element_size,
|
||||
uint32_t index) {
|
||||
assert(index < self->size);
|
||||
char *contents = (char *)self->contents;
|
||||
static inline void _array__erase(void* self_contents, uint32_t *size,
|
||||
size_t element_size, uint32_t index) {
|
||||
assert(index < *size);
|
||||
char *contents = (char *)self_contents;
|
||||
memmove(contents + index * element_size, contents + (index + 1) * element_size,
|
||||
(self->size - index - 1) * element_size);
|
||||
self->size--;
|
||||
(*size - index - 1) * element_size);
|
||||
(*size)--;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_reserve`.
|
||||
static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
|
||||
if (new_capacity > self->capacity) {
|
||||
if (self->contents) {
|
||||
self->contents = ts_realloc(self->contents, new_capacity * element_size);
|
||||
static inline void *_array__reserve(void *contents, uint32_t *capacity,
|
||||
size_t element_size, uint32_t new_capacity) {
|
||||
void *new_contents = contents;
|
||||
if (new_capacity > *capacity) {
|
||||
if (contents) {
|
||||
new_contents = ts_realloc(contents, new_capacity * element_size);
|
||||
} else {
|
||||
self->contents = ts_malloc(new_capacity * element_size);
|
||||
new_contents = ts_malloc(new_capacity * element_size);
|
||||
}
|
||||
self->capacity = new_capacity;
|
||||
*capacity = new_capacity;
|
||||
}
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_assign`.
|
||||
static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
|
||||
_array__reserve(self, element_size, other->size);
|
||||
self->size = other->size;
|
||||
memcpy(self->contents, other->contents, self->size * element_size);
|
||||
static inline void *_array__assign(void* self_contents, uint32_t *self_size, uint32_t *self_capacity,
|
||||
const void *other_contents, uint32_t other_size, size_t element_size) {
|
||||
void *new_contents = _array__reserve(self_contents, self_capacity, element_size, other_size);
|
||||
*self_size = other_size;
|
||||
memcpy(new_contents, other_contents, *self_size * element_size);
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_swap`.
|
||||
static inline void _array__swap(Array *self, Array *other) {
|
||||
Array swap = *other;
|
||||
*other = *self;
|
||||
*self = swap;
|
||||
static inline void _array__swap(uint32_t *self_size, uint32_t *self_capacity,
|
||||
uint32_t *other_size, uint32_t *other_capacity) {
|
||||
uint32_t tmp_size = *self_size;
|
||||
uint32_t tmp_capacity = *self_capacity;
|
||||
*self_size = *other_size;
|
||||
*self_capacity = *other_capacity;
|
||||
*other_size = tmp_size;
|
||||
*other_capacity = tmp_capacity;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_push` or `array_grow_by`.
|
||||
static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
|
||||
uint32_t new_size = self->size + count;
|
||||
if (new_size > self->capacity) {
|
||||
uint32_t new_capacity = self->capacity * 2;
|
||||
static inline void *_array__grow(void *contents, uint32_t size, uint32_t *capacity,
|
||||
uint32_t count, size_t element_size) {
|
||||
void *new_contents = contents;
|
||||
uint32_t new_size = size + count;
|
||||
if (new_size > *capacity) {
|
||||
uint32_t new_capacity = *capacity * 2;
|
||||
if (new_capacity < 8) new_capacity = 8;
|
||||
if (new_capacity < new_size) new_capacity = new_size;
|
||||
_array__reserve(self, element_size, new_capacity);
|
||||
new_contents = _array__reserve(contents, capacity, element_size, new_capacity);
|
||||
}
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_splice`.
|
||||
static inline void _array__splice(Array *self, size_t element_size,
|
||||
static inline void *_array__splice(void *self_contents, uint32_t *size, uint32_t *capacity,
|
||||
size_t element_size,
|
||||
uint32_t index, uint32_t old_count,
|
||||
uint32_t new_count, const void *elements) {
|
||||
uint32_t new_size = self->size + new_count - old_count;
|
||||
uint32_t new_size = *size + new_count - old_count;
|
||||
uint32_t old_end = index + old_count;
|
||||
uint32_t new_end = index + new_count;
|
||||
assert(old_end <= self->size);
|
||||
assert(old_end <= *size);
|
||||
|
||||
_array__reserve(self, element_size, new_size);
|
||||
void *new_contents = _array__reserve(self_contents, capacity, element_size, new_size);
|
||||
|
||||
char *contents = (char *)self->contents;
|
||||
if (self->size > old_end) {
|
||||
char *contents = (char *)new_contents;
|
||||
if (*size > old_end) {
|
||||
memmove(
|
||||
contents + new_end * element_size,
|
||||
contents + old_end * element_size,
|
||||
(self->size - old_end) * element_size
|
||||
(*size - old_end) * element_size
|
||||
);
|
||||
}
|
||||
if (new_count > 0) {
|
||||
@@ -250,7 +287,9 @@ static inline void _array__splice(Array *self, size_t element_size,
|
||||
);
|
||||
}
|
||||
}
|
||||
self->size += new_count - old_count;
|
||||
*size += new_count - old_count;
|
||||
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
|
||||
|
||||
@@ -18,6 +18,11 @@ typedef uint16_t TSStateId;
|
||||
typedef uint16_t TSSymbol;
|
||||
typedef uint16_t TSFieldId;
|
||||
typedef struct TSLanguage TSLanguage;
|
||||
typedef struct TSLanguageMetadata {
|
||||
uint8_t major_version;
|
||||
uint8_t minor_version;
|
||||
uint8_t patch_version;
|
||||
} TSLanguageMetadata;
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
@@ -26,10 +31,11 @@ typedef struct {
|
||||
bool inherited;
|
||||
} TSFieldMapEntry;
|
||||
|
||||
// Used to index the field and supertype maps.
|
||||
typedef struct {
|
||||
uint16_t index;
|
||||
uint16_t length;
|
||||
} TSFieldMapSlice;
|
||||
} TSMapSlice;
|
||||
|
||||
typedef struct {
|
||||
bool visible;
|
||||
@@ -79,6 +85,12 @@ typedef struct {
|
||||
uint16_t external_lex_state;
|
||||
} TSLexMode;
|
||||
|
||||
typedef struct {
|
||||
uint16_t lex_state;
|
||||
uint16_t external_lex_state;
|
||||
uint16_t reserved_word_set_id;
|
||||
} TSLexerMode;
|
||||
|
||||
typedef union {
|
||||
TSParseAction action;
|
||||
struct {
|
||||
@@ -93,7 +105,7 @@ typedef struct {
|
||||
} TSCharacterRange;
|
||||
|
||||
struct TSLanguage {
|
||||
uint32_t version;
|
||||
uint32_t abi_version;
|
||||
uint32_t symbol_count;
|
||||
uint32_t alias_count;
|
||||
uint32_t token_count;
|
||||
@@ -109,13 +121,13 @@ struct TSLanguage {
|
||||
const TSParseActionEntry *parse_actions;
|
||||
const char * const *symbol_names;
|
||||
const char * const *field_names;
|
||||
const TSFieldMapSlice *field_map_slices;
|
||||
const TSMapSlice *field_map_slices;
|
||||
const TSFieldMapEntry *field_map_entries;
|
||||
const TSSymbolMetadata *symbol_metadata;
|
||||
const TSSymbol *public_symbol_map;
|
||||
const uint16_t *alias_map;
|
||||
const TSSymbol *alias_sequences;
|
||||
const TSLexMode *lex_modes;
|
||||
const TSLexerMode *lex_modes;
|
||||
bool (*lex_fn)(TSLexer *, TSStateId);
|
||||
bool (*keyword_lex_fn)(TSLexer *, TSStateId);
|
||||
TSSymbol keyword_capture_token;
|
||||
@@ -129,15 +141,23 @@ struct TSLanguage {
|
||||
void (*deserialize)(void *, const char *, unsigned);
|
||||
} external_scanner;
|
||||
const TSStateId *primary_state_ids;
|
||||
const char *name;
|
||||
const TSSymbol *reserved_words;
|
||||
uint16_t max_reserved_word_set_size;
|
||||
uint32_t supertype_count;
|
||||
const TSSymbol *supertype_symbols;
|
||||
const TSMapSlice *supertype_map_slices;
|
||||
const TSSymbol *supertype_map_entries;
|
||||
TSLanguageMetadata metadata;
|
||||
};
|
||||
|
||||
static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
|
||||
static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
|
||||
uint32_t index = 0;
|
||||
uint32_t size = len - index;
|
||||
while (size > 1) {
|
||||
uint32_t half_size = size / 2;
|
||||
uint32_t mid_index = index + half_size;
|
||||
TSCharacterRange *range = &ranges[mid_index];
|
||||
const TSCharacterRange *range = &ranges[mid_index];
|
||||
if (lookahead >= range->start && lookahead <= range->end) {
|
||||
return true;
|
||||
} else if (lookahead > range->end) {
|
||||
@@ -145,7 +165,7 @@ static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t
|
||||
}
|
||||
size -= half_size;
|
||||
}
|
||||
TSCharacterRange *range = &ranges[index];
|
||||
const TSCharacterRange *range = &ranges[index];
|
||||
return (lookahead >= range->start && lookahead <= range->end);
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,210 @@
|
||||
/* tsp_intuit_more.h
|
||||
*
|
||||
* A faithful, stateless port of perl's S_intuit_more() (toke.c) and
|
||||
* S_regcurly() (regcomp.c) -- "the one truly awful dwimmer necessary to
|
||||
* conflate C and sed".
|
||||
*
|
||||
* Inside a pattern, a `[` or `{` following a variable is ambiguous:
|
||||
* /$foo[...]/ could be an array subscript OR a character class
|
||||
* /$foo{...}/ could be a hash subscript OR a {n,m} quantifier
|
||||
* Perl resolves this with a weighting heuristic. We reproduce it so the
|
||||
* external scanner can decide whether to emit a literal regex `[` / `{`.
|
||||
*
|
||||
* tsp_intuit_more(s, len) returns TRUE => "there is more to the expression",
|
||||
* i.e. it's a SUBSCRIPT; FALSE => it's a character class / quantifier. This
|
||||
* matches perl's return-value convention exactly.
|
||||
*
|
||||
* The caller is the scanner; it has already established that we are inside a
|
||||
* pattern and that s[0] is '[' or '{'. `s` points at that bracket and spans
|
||||
* `len` buffered bytes, ideally through the matching close bracket (when one
|
||||
* exists in range). The function never reads past s[len-1].
|
||||
*
|
||||
* Three branches of perl's heuristic depend on state a stateless lexer can't
|
||||
* see; these are intentional best-effort gaps an LSP can refine:
|
||||
* - PL_lex_brackets (parser-stack recursion) -> handled by the scanner
|
||||
* - gv_fetchpvn_flags (runtime symbol table, -100) -> treated as the -10 case
|
||||
* - keyword() (-150) -> backed by our own
|
||||
* regenerated keyword trie (tsp_intuit_keywords.h)
|
||||
*/
|
||||
|
||||
#ifndef TSP_INTUIT_MORE_H
|
||||
#define TSP_INTUIT_MORE_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include "tsp_intuit_keywords.h"
|
||||
|
||||
/* --- ASCII classification (perl's is* macros, ASCII subset) ----------------
|
||||
* UTF-8 word characters beyond ASCII are a best-effort gap. */
|
||||
static inline bool tsp_im_isdigit(int c) { return c >= '0' && c <= '9'; }
|
||||
static inline bool tsp_im_isalpha(int c) {
|
||||
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
|
||||
}
|
||||
static inline bool tsp_im_isword(int c) {
|
||||
return tsp_im_isalpha(c) || tsp_im_isdigit(c) || c == '_';
|
||||
}
|
||||
static inline bool tsp_im_isblank(int c) { return c == ' ' || c == '\t'; }
|
||||
|
||||
/* strchr over a literal set, NUL-safe (a NUL `c` never matches). */
|
||||
static inline bool tsp_im_in(const char *set, int c) {
|
||||
if (c == 0) return false;
|
||||
for (; *set; set++)
|
||||
if ((unsigned char)*set == (unsigned char)c) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/* --- S_regcurly: does s[0..len) begin a {m,n} quantifier? ------------------
|
||||
* Mirrors regcomp.c. We only need the syntactic yes/no, so the result[]
|
||||
* out-param is dropped. Accepts {m}, {m,}, {m,n}, and {,n}; rejects {} . */
|
||||
static bool tsp_regcurly(const char *s, int len) {
|
||||
const char *e = s + len;
|
||||
bool min_present = false; /* perl's min_start: digit(s) before the comma */
|
||||
bool max_present = false; /* perl's max_end: digit(s) after the comma */
|
||||
|
||||
if (s >= e || *s++ != '{') return false;
|
||||
|
||||
while (s < e && tsp_im_isblank(*s)) s++;
|
||||
|
||||
if (s < e && tsp_im_isdigit(*s)) {
|
||||
min_present = true;
|
||||
do { s++; } while (s < e && tsp_im_isdigit(*s));
|
||||
}
|
||||
|
||||
while (s < e && tsp_im_isblank(*s)) s++;
|
||||
|
||||
if (s < e && *s == ',') {
|
||||
s++;
|
||||
while (s < e && tsp_im_isblank(*s)) s++;
|
||||
if (s < e && tsp_im_isdigit(*s)) {
|
||||
max_present = true;
|
||||
do { s++; } while (s < e && tsp_im_isdigit(*s));
|
||||
}
|
||||
while (s < e && tsp_im_isblank(*s)) s++;
|
||||
}
|
||||
|
||||
/* Mirrors regcomp.c: must close with '}' and have at least one number,
|
||||
* before or after the comma ({3}, {3,}, {3,5}, {,5}); '{}' and '{,}' fail. */
|
||||
if (s >= e || *s != '}' || (!min_present && !max_present)) return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/* --- S_intuit_more: subscript (TRUE) vs char-class/quantifier (FALSE) ------ */
|
||||
static bool tsp_intuit_more(const char *s, int len) {
|
||||
const char *e = s + len;
|
||||
|
||||
if (s >= e) return true; /* nothing buffered; be safe */
|
||||
|
||||
/* In a pattern, a leading '{' might be a {n,m} quantifier. */
|
||||
if (s[0] == '{') {
|
||||
return !tsp_regcurly(s, len);
|
||||
}
|
||||
|
||||
if (s[0] != '[') return true; /* caller only sends '[' or '{' */
|
||||
|
||||
s++; /* step past '[' */
|
||||
|
||||
/* '^' implies a char class; empty '[]' isn't legal but means "no more". */
|
||||
if (s >= e) return true; /* '[' with nothing after -> expr */
|
||||
if (s[0] == ']' || s[0] == '^') return false;
|
||||
|
||||
/* Find the matching ']'. No ']' in range => has to be an expression. */
|
||||
const char *send = NULL;
|
||||
for (const char *p = s; p < e; p++) {
|
||||
if (*p == ']') { send = p; break; }
|
||||
}
|
||||
if (!send) return true;
|
||||
|
||||
/* One or two digits only => subscript, e.g. [12]. */
|
||||
if (tsp_im_isdigit(s[0]) && send - s <= 2 &&
|
||||
(send - s == 1 || tsp_im_isdigit(s[1]))) {
|
||||
return true;
|
||||
}
|
||||
|
||||
/* The weighting machine. weight >= 0 => char class, < 0 => subscript. */
|
||||
int weight = (s[0] == '$') ? -1 : 2; /* '$' leans slightly to subscript */
|
||||
|
||||
unsigned char seen[256] = {0};
|
||||
unsigned char un_char = 0;
|
||||
bool first_time = true;
|
||||
|
||||
for (; s < send; s++, first_time = false) {
|
||||
unsigned char prev_un_char = un_char;
|
||||
un_char = (unsigned char)s[0];
|
||||
|
||||
switch (s[0]) {
|
||||
case '@':
|
||||
case '&':
|
||||
case '$':
|
||||
weight -= seen[un_char] * 10;
|
||||
/* A word char following one of these: perl would consult the symbol
|
||||
* table for a -100; we lack it, so always take the -10 path (gap). */
|
||||
if (s + 1 < e && tsp_im_isword((unsigned char)s[1])) {
|
||||
weight -= 10;
|
||||
}
|
||||
else if (s[0] == '$' && s + 1 < e && s[1] &&
|
||||
tsp_im_in("[#!%*<>()-=", (unsigned char)s[1])) {
|
||||
/* Possible punctuation variable. */
|
||||
char s2 = (s + 2 < e) ? s[2] : 0;
|
||||
if (tsp_im_in("])} =", (unsigned char)s2)) weight -= 10;
|
||||
else weight -= 1;
|
||||
}
|
||||
break;
|
||||
|
||||
case '\\':
|
||||
if (s + 1 < e && s[1]) {
|
||||
if (tsp_im_in("wds]", (unsigned char)s[1])) weight += 100;
|
||||
else if (seen[(unsigned char)'\''] || seen[(unsigned char)'"']) weight += 1;
|
||||
else if (tsp_im_in("abcfnrtvx", (unsigned char)s[1])) weight += 40;
|
||||
else if (tsp_im_isdigit((unsigned char)s[1])) {
|
||||
weight += 40;
|
||||
while (s + 1 < send && tsp_im_isdigit((unsigned char)s[1])) s++;
|
||||
}
|
||||
}
|
||||
else weight += 100; /* '\' at end strongly char class */
|
||||
break;
|
||||
|
||||
case '-':
|
||||
if (s + 1 < e && s[1] == '\\') weight += 50;
|
||||
if (!first_time && tsp_im_in("aA01! ", prev_un_char)) weight += 30;
|
||||
if (s + 1 < e && tsp_im_in("zZ79~", (unsigned char)s[1])) weight += 30;
|
||||
if (first_time && s + 1 < e &&
|
||||
(tsp_im_isdigit((unsigned char)s[1]) || s[1] == '$')) {
|
||||
weight -= 5; /* negative subscript */
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
if ((first_time || (!tsp_im_isword(prev_un_char)
|
||||
&& prev_un_char != '$'
|
||||
&& prev_un_char != '@'
|
||||
&& prev_un_char != '&'))
|
||||
&& tsp_im_isalpha((unsigned char)s[0])
|
||||
&& s + 1 < send && tsp_im_isalpha((unsigned char)s[1])) {
|
||||
/* A run of >= 2 alphas at a boundary: if it spells a keyword it's
|
||||
* almost certainly not a character class. (Mirrors perl's pointer
|
||||
* walk: s lands on the first non-alpha, which the loop's s++ then
|
||||
* skips -- intentional, matching toke.c.) */
|
||||
const char *d = s;
|
||||
while (s < send && tsp_im_isalpha((unsigned char)s[0])) s++;
|
||||
if (tsp_is_perl_keyword(d, (int)(s - d))) weight -= 150;
|
||||
}
|
||||
|
||||
/* Ascending runs like ...12... or ...ab... lean char class. */
|
||||
if (!first_time && un_char == (unsigned char)(prev_un_char + 1)) {
|
||||
weight += 5;
|
||||
}
|
||||
|
||||
/* Repeats lean subscript (nobody writes [aba] as a class). */
|
||||
weight -= seen[un_char];
|
||||
break;
|
||||
}
|
||||
|
||||
seen[un_char]++;
|
||||
}
|
||||
|
||||
if (weight >= 0) return false; /* probably a character class */
|
||||
return true; /* subscript */
|
||||
}
|
||||
|
||||
#endif /* TSP_INTUIT_MORE_H */
|
||||
@@ -0,0 +1,207 @@
|
||||
/* tsp_intuit_readline.h
|
||||
*
|
||||
* Disambiguates `<...>` after an ambiguous bareword: fileglob/readline
|
||||
* (`print <*.c>`) vs the relational `<` operator (`CONST < 7 > $x`). Real
|
||||
* perl decides this by PL_expect (was a value or a list-op in front of `<`?),
|
||||
* which the scanner can't see, so we hybridise two heuristics whose blind
|
||||
* spots are disjoint:
|
||||
*
|
||||
* (a) glob-shape: a real target looks like a glob pattern (metachar `*?[]{}~`,
|
||||
* path `/`, filename-dot `foo.c`) or a pure filehandle name; content that
|
||||
* is neither (a number, a spaced scalar, an operand) is relational. This
|
||||
* catches `CONST < $a > foo`, which (b) alone misreads.
|
||||
* (b) orphan lookahead: a glob is a complete term, so a bare term right after
|
||||
* its `>` would be orphaned -> `<` was the operator. This catches
|
||||
* `CONST < $a/$b > $c` (glob-shaped content, but `$c` orphans), which (a)
|
||||
* alone misreads.
|
||||
*
|
||||
* tsp_is_fileglob(content, clen, after, alen): TRUE => emit fileglob; FALSE =>
|
||||
* bail to the `<` operator. See the decision body below. Best-effort and
|
||||
* ASCII-only (an LSP with parser state can refine it); never reads past either
|
||||
* buffer. `after` is a capped window; if it holds only whitespace we treat it
|
||||
* as EOF and allow the glob.
|
||||
*/
|
||||
|
||||
#ifndef TSP_INTUIT_READLINE_H
|
||||
#define TSP_INTUIT_READLINE_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <string.h>
|
||||
|
||||
/* --- ASCII classification (mirrors the subset used elsewhere) -------------- */
|
||||
static inline bool tsp_rl_isdigit(int c) { return c >= '0' && c <= '9'; }
|
||||
static inline bool tsp_rl_isalpha(int c) {
|
||||
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
|
||||
}
|
||||
static inline bool tsp_rl_isword(int c) {
|
||||
return tsp_rl_isalpha(c) || tsp_rl_isdigit(c) || c == '_';
|
||||
}
|
||||
static inline bool tsp_rl_isspace(int c) {
|
||||
return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' ||
|
||||
c == '\v';
|
||||
}
|
||||
|
||||
/* The low-precedence operator / statement-modifier keywords that may legally
|
||||
* FOLLOW a complete `bareword <glob>` term. Seeing one of these after the `>`
|
||||
* means the glob is complete, not orphaning a term. */
|
||||
static inline bool tsp_rl_is_trailing_keyword(const char *w, size_t n) {
|
||||
static const char *const kw[] = {
|
||||
"if", "unless", "while", "until", "for", "foreach", "and", "or", "xor",
|
||||
"not", "cmp", "eq", "ne", "lt", "gt", "le", "ge", "x"};
|
||||
for (size_t i = 0; i < sizeof(kw) / sizeof(kw[0]); i++) {
|
||||
if (strlen(kw[i]) == n && memcmp(w, kw[i], n) == 0) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/* The infix word-operators that, when appearing INSIDE the `<...>` content as a
|
||||
* whitespace-delimited word, prove it is a relational expression rather than a
|
||||
* glob pattern (a glob never contains a bare perl operator word). */
|
||||
static inline bool tsp_rl_is_infix_word(const char *w, size_t n) {
|
||||
static const char *const kw[] = {"and", "or", "xor", "not", "cmp", "eq",
|
||||
"ne", "lt", "gt", "le", "ge", "x"};
|
||||
for (size_t i = 0; i < sizeof(kw) / sizeof(kw[0]); i++) {
|
||||
if (strlen(kw[i]) == n && memcmp(w, kw[i], n) == 0) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Scan the `<...>` content for a whitespace-delimited infix operator word.
|
||||
* Only alpha runs that start at a whitespace boundary (and are not glued to
|
||||
* sigils/digits/punctuation) count; `$x` or `*.c` never qualify. */
|
||||
static inline bool tsp_rl_content_has_infix_op(const char *s, size_t len) {
|
||||
size_t i = 0;
|
||||
while (i < len) {
|
||||
/* skip leading whitespace */
|
||||
while (i < len && tsp_rl_isspace((unsigned char)s[i])) i++;
|
||||
size_t start = i;
|
||||
bool clean = true; /* a clean bare word: only alphas */
|
||||
while (i < len && !tsp_rl_isspace((unsigned char)s[i])) {
|
||||
if (!tsp_rl_isalpha((unsigned char)s[i])) clean = false;
|
||||
i++;
|
||||
}
|
||||
if (clean && i > start && tsp_rl_is_infix_word(s + start, i - start))
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/* The glob metacharacters whose presence in the content is decisive evidence
|
||||
* of a glob pattern (as opposed to a relational operand). */
|
||||
static inline bool tsp_rl_is_glob_meta(int c) {
|
||||
return c == '*' || c == '?' || c == '[' || c == ']' || c == '{' || c == '}' ||
|
||||
c == '~';
|
||||
}
|
||||
|
||||
/* Does the content POSITIVELY look like a glob pattern? True if it contains a
|
||||
* glob metacharacter, a path separator `/`, or a filename-looking `.` (a dot
|
||||
* flanked by word characters, e.g. `foo.c`). A bare number (`7`), a lone
|
||||
* scalar (`$a`), or an operator expression has none of these. */
|
||||
static inline bool tsp_rl_content_is_globshaped(const char *s, size_t len) {
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
unsigned char c = (unsigned char)s[i];
|
||||
if (tsp_rl_is_glob_meta(c) || c == '/') return true;
|
||||
if (c == '.') {
|
||||
bool prev_word = (i > 0) && tsp_rl_isword((unsigned char)s[i - 1]);
|
||||
bool next_word = (i + 1 < len) && tsp_rl_isword((unsigned char)s[i + 1]);
|
||||
if (prev_word && next_word) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Is the content a pure filehandle / scalar NAME (no leading whitespace): an
|
||||
* optional leading `$`, then word characters with `::` or `'` package
|
||||
* separators (e.g. `$fh`, `STDIN`, `Foo::BAR`), tolerating only trailing
|
||||
* whitespace? Such content is a readline target, not a relational operand.
|
||||
* The leading-vs-trailing whitespace asymmetry is load-bearing: a glob target
|
||||
* is written tight against `<` (`<$sner >`), whereas a relational `< EXPR`
|
||||
* carries a separating space (`< $a > $b`). */
|
||||
static inline bool tsp_rl_is_filehandle(const char *buf, size_t len) {
|
||||
size_t i = 0;
|
||||
if (i >= len || tsp_rl_isspace((unsigned char)buf[i])) return false;
|
||||
if (buf[i] == '$') i++;
|
||||
if (i >= len) return false; /* bare `$`: not a name */
|
||||
|
||||
bool last_was_word = false;
|
||||
while (i < len) {
|
||||
if (tsp_rl_isword((unsigned char)buf[i])) {
|
||||
last_was_word = true;
|
||||
i++;
|
||||
} else if (buf[i] == ':') {
|
||||
if (!last_was_word) return false;
|
||||
if (i + 1 >= len || buf[i + 1] != ':') return false;
|
||||
last_was_word = false;
|
||||
i += 2;
|
||||
} else if (buf[i] == '\'') {
|
||||
if (!last_was_word) return false;
|
||||
last_was_word = false;
|
||||
i++;
|
||||
} else if (tsp_rl_isspace((unsigned char)buf[i])) {
|
||||
if (!last_was_word) return false;
|
||||
for (i++; i < len; i++)
|
||||
if (!tsp_rl_isspace((unsigned char)buf[i])) return false;
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return last_was_word;
|
||||
}
|
||||
|
||||
/* --- the decision ---------------------------------------------------------- */
|
||||
static inline bool tsp_is_fileglob(const char *content, size_t clen,
|
||||
const char *after, size_t alen) {
|
||||
/* The empty diamond `<>` (rare here; usually handled before us). */
|
||||
if (clen == 0) return true;
|
||||
|
||||
/* (1) operator-word inside the content => relational expression, not glob. */
|
||||
if (tsp_rl_content_has_infix_op(content, clen)) return false;
|
||||
|
||||
/* (2) GLOB-SHAPE GATE. A real glob/readline target positively looks like a
|
||||
* glob pattern (metachar / path / filename-dot) or a pure filehandle name.
|
||||
* Content that is neither -- a bare number (`< 7 >`), a lone scalar with a
|
||||
* separating space (`< $a > $b`), an arithmetic sub-expression -- is a
|
||||
* relational operand, so bail. This is what keeps `CONST < $a > foo` and
|
||||
* `CONST < 7 > $x` relational without any lookahead. */
|
||||
if (!tsp_rl_content_is_globshaped(content, clen) &&
|
||||
!tsp_rl_is_filehandle(content, clen))
|
||||
return false;
|
||||
|
||||
/* (3) post-`>` orphan lookahead. The content looks glob-shaped, but a real
|
||||
* glob is a COMPLETE term: if a bare term follows the `>` with no connecting
|
||||
* operator it would be orphaned, which means `<` was the relational operator
|
||||
* after all (e.g. `CONST < $a/$b > $c` -- glob-shaped content `$a/$b`, yet
|
||||
* the trailing `$c` orphans). */
|
||||
size_t i = 0;
|
||||
while (i < alen && tsp_rl_isspace((unsigned char)after[i])) i++;
|
||||
|
||||
/* Window exhausted before any non-space byte: treat as EOF-like -> glob ok. */
|
||||
if (i >= alen) return true;
|
||||
|
||||
unsigned char nc = (unsigned char)after[i];
|
||||
|
||||
/* An alphabetic word: keyword => complete term (glob ok); otherwise an
|
||||
* ambiguous bareword, which we lean toward NOT treating as an orphan. */
|
||||
if (tsp_rl_isalpha(nc)) {
|
||||
size_t start = i;
|
||||
while (i < alen && tsp_rl_isword((unsigned char)after[i])) i++;
|
||||
if (tsp_rl_is_trailing_keyword(after + start, i - start)) return true;
|
||||
/* Unknown bareword after the glob: lean toward glob (do not orphan). */
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Bare-term openers with no connecting operator => orphan => bail. */
|
||||
if (nc == '$' || nc == '@' || nc == '%' || nc == '"' || nc == '\'' ||
|
||||
tsp_rl_isdigit(nc) || nc == '(' || nc == '[' || nc == '{') {
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Everything else — operators (`+ - * / . < > ! ~ ? : | & ^`), `=`, `=>`,
|
||||
* comma, `;`, closing brackets `) ] }` — terminates the term cleanly, so the
|
||||
* glob is complete. */
|
||||
return true;
|
||||
}
|
||||
|
||||
#endif /* TSP_INTUIT_READLINE_H */
|
||||
@@ -0,0 +1,33 @@
|
||||
/* THIS FILE IS GENERATED BY generate_keywords.pl
|
||||
*
|
||||
* Statement keywords recognized by peek_is_statement_keyword().
|
||||
* To add a keyword, edit the @keywords list in generate_keywords.pl
|
||||
* and re-run: perl generate_keywords.pl
|
||||
*
|
||||
* Current keywords: class, method, no, package, role, sub, use
|
||||
*/
|
||||
|
||||
// First-char filter: returns PEEK_NO_MATCH immediately if the
|
||||
// lookahead can't start any statement keyword.
|
||||
#define KEYWORD_FIRST_CHAR_FILTER(la) \
|
||||
(!(la == 'c' || la == 'm' || la == 'n' || la == 'p' || la == 'r' || la == 's' || la == 'u'))
|
||||
|
||||
// Word-reading loop: characters that appear in any keyword.
|
||||
#define KEYWORD_WORD_CHAR(la) \
|
||||
(la == 'a' || la == 'b' || la == 'c' || la == 'd' || la == 'e' || la == 'g' || la == 'h' || la == 'k' || la == 'l' || la == 'm' || la == 'n' || la == 'o' || la == 'p' || la == 'r' || la == 's' || la == 't' || la == 'u')
|
||||
|
||||
// Keyword classification: sets `kind` to one of the KwKind values.
|
||||
// KW_NONE — not a statement keyword (a plain bareword / autoquote key)
|
||||
// KW_ALWAYS — always a statement keyword (package, use, no, class, role)
|
||||
// KW_NEEDS_NAME — only a declaration if followed by a name (sub, method)
|
||||
// Unlike a bare match this never returns: the caller decides what to do with a
|
||||
// non-keyword, which may still be a fat-comma autoquote key.
|
||||
#define KEYWORD_MATCH(word, kind) do { \
|
||||
if (strcmp(word, "package") == 0 || strcmp(word, "use") == 0 || strcmp(word, "no") == 0 || strcmp(word, "class") == 0 || strcmp(word, "role") == 0) { \
|
||||
(kind) = KW_ALWAYS; \
|
||||
} else if (strcmp(word, "sub") == 0 || strcmp(word, "method") == 0) { \
|
||||
(kind) = KW_NEEDS_NAME; \
|
||||
} else { \
|
||||
(kind) = KW_NONE; \
|
||||
} \
|
||||
} while(0)
|
||||
@@ -541,8 +541,8 @@ static const struct TSPRange tsp_id_start[] = {
|
||||
{ 201552, 205744 }
|
||||
};
|
||||
|
||||
bool is_tsp_id_start (int32_t codepoint) {
|
||||
return bsearch(&codepoint, tsp_id_start, sizeof(tsp_id_start) / sizeof(tsp_id_start[0]), sizeof(tsp_id_start[0]), tsprange_contains);
|
||||
static bool is_tsp_id_start (int32_t codepoint) {
|
||||
return tsp_bsearch(&codepoint, tsp_id_start, sizeof(tsp_id_start) / sizeof(tsp_id_start[0]), sizeof(tsp_id_start[0]), tsprange_contains);
|
||||
}
|
||||
|
||||
static const struct TSPRange tsp_id_continue[] = {
|
||||
@@ -1205,8 +1205,8 @@ static const struct TSPRange tsp_id_continue[] = {
|
||||
{ 917760, 918000 }
|
||||
};
|
||||
|
||||
bool is_tsp_id_continue (int32_t codepoint) {
|
||||
return bsearch(&codepoint, tsp_id_continue, sizeof(tsp_id_continue) / sizeof(tsp_id_continue[0]), sizeof(tsp_id_continue[0]), tsprange_contains);
|
||||
static bool is_tsp_id_continue (int32_t codepoint) {
|
||||
return tsp_bsearch(&codepoint, tsp_id_continue, sizeof(tsp_id_continue) / sizeof(tsp_id_continue[0]), sizeof(tsp_id_continue[0]), tsprange_contains);
|
||||
}
|
||||
|
||||
static const struct TSPRange tsp_whitespace[] = {
|
||||
@@ -1221,6 +1221,6 @@ static const struct TSPRange tsp_whitespace[] = {
|
||||
{ 8287, 8288 },
|
||||
{ 12288, 12289 }
|
||||
};
|
||||
bool is_tsp_whitespace (int32_t codepoint) {
|
||||
return bsearch(&codepoint, tsp_whitespace, sizeof(tsp_whitespace) / sizeof(tsp_whitespace[0]), sizeof(tsp_whitespace[0]), tsprange_contains);
|
||||
static bool is_tsp_whitespace (int32_t codepoint) {
|
||||
return tsp_bsearch(&codepoint, tsp_whitespace, sizeof(tsp_whitespace) / sizeof(tsp_whitespace[0]), sizeof(tsp_whitespace[0]), tsprange_contains);
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ c5cfb43042b6b72045f4ba997834d0a7786d2793d91680868b5815b39f14fc78 internal/cbm/v
|
||||
b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea internal/cbm/vendored/common/tree_sitter/alloc.h
|
||||
31e60a1bff6f715afacce03b5b70efe42b58371b4f9595dd4af52a577ff9608c internal/cbm/vendored/common/tree_sitter/array.h
|
||||
180b893c8734778fd32f372dfbc27bd6ad1cd2221f26150b31256ff6716320d2 internal/cbm/vendored/common/tree_sitter/parser.h
|
||||
73be629708008f0cb31e0f8d41dc71ae901626baba191c24efd41bf52220c913 internal/cbm/vendored/grammars/MANIFEST.md
|
||||
e1233d81f21d868b8f2d9e8fca10e15f2ec3ae4db4a7ba0301161cc011deef65 internal/cbm/vendored/grammars/MANIFEST.md
|
||||
ad8425038de519f8c4e3e9339feebf99dfad8a6002dcf79227d91402780a32cc internal/cbm/vendored/grammars/ada/LICENSE
|
||||
02805ec13939b749c891567be36bf024b09034e04c80683a1ae667272458d549 internal/cbm/vendored/grammars/ada/parser.c
|
||||
115a75d000bef9c70c4de7dfbb7f2a80b90cbb14ba265056dc7abb1ef3b9b6db internal/cbm/vendored/grammars/ada/tree_sitter/alloc.h
|
||||
@@ -553,13 +553,17 @@ fafd2e286865aacc132e938c1640f51619ec9e7866cd2d8debff39540336c14a internal/cbm/v
|
||||
6db3411bfeff9fea8e64a1436f13b4c3558c52df0c19ea038a6a22fecae1b6eb internal/cbm/vendored/grammars/pascal/parser.c
|
||||
2db740ea1be6b71014d2be1385491f01bf2d15b607e61c5b30b4128535efe68a internal/cbm/vendored/grammars/pascal/tree_sitter/parser.h
|
||||
68a9a526ae357ed5a2f8ca5dabb14131b283554b69639bb04816088d4b1f2fa0 internal/cbm/vendored/grammars/perl/LICENSE
|
||||
cb08206e89750c1fab700b89fc9876afb5cc689827e514ef49a5569c54635b61 internal/cbm/vendored/grammars/perl/bsearch.h
|
||||
25c014ab8ce027992a6cfd76ef7b88bf3a41f63fd6ad43f38a553e218a7a9ea8 internal/cbm/vendored/grammars/perl/parser.c
|
||||
fe4dc0394501b3d6211dcc1e31e9731befae844fcd6db0130e3c16f992d85860 internal/cbm/vendored/grammars/perl/scanner.c
|
||||
6f953a91955c72b1b6b063c34fa877c8b36f5b43c320083ad7361256e67b9484 internal/cbm/vendored/grammars/perl/bsearch.h
|
||||
ab3f9bf80dda8a4b919ffc8b556bd6e6e44995974e71acf4f4c0d6ebf5653625 internal/cbm/vendored/grammars/perl/parser.c
|
||||
117d53b68474bf11e0b207d3e63e359b05be6f98d91d2c31b6abcec3a960eb1b internal/cbm/vendored/grammars/perl/scanner.c
|
||||
b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea internal/cbm/vendored/grammars/perl/tree_sitter/alloc.h
|
||||
5bdf6ed1a78e3409fd443e085ca967a64c188a5d082aaf7f819bccd53a471c94 internal/cbm/vendored/grammars/perl/tree_sitter/array.h
|
||||
a1f6ef161fbaf48a0e10fca90ef5290a062462b307b3898aa562993853b9f80a internal/cbm/vendored/grammars/perl/tree_sitter/parser.h
|
||||
9cbc0731f8c9bd52bd3de9644fd887f20cecea2d17634b20769db4940eadb566 internal/cbm/vendored/grammars/perl/tsp_unicode.h
|
||||
31e60a1bff6f715afacce03b5b70efe42b58371b4f9595dd4af52a577ff9608c internal/cbm/vendored/grammars/perl/tree_sitter/array.h
|
||||
180b893c8734778fd32f372dfbc27bd6ad1cd2221f26150b31256ff6716320d2 internal/cbm/vendored/grammars/perl/tree_sitter/parser.h
|
||||
e65b7139547077b59ad62cb109efcf73d420311706c9dc7b6f9831cb246f2c61 internal/cbm/vendored/grammars/perl/tsp_intuit_keywords.h
|
||||
45ada959185bb4b15f4fe702d52f0aeb5cd992b64cb129248aa843644e6026a2 internal/cbm/vendored/grammars/perl/tsp_intuit_more.h
|
||||
9504b2ef7cce98bd3b22e2226ca05c1b63fd17b56ed3375bf72c4e429af42180 internal/cbm/vendored/grammars/perl/tsp_intuit_readline.h
|
||||
2c86ede8eedbaf7854c2f9eef4b835ae55e563d7a85c05e47d8bfee502e5a78f internal/cbm/vendored/grammars/perl/tsp_keywords.h
|
||||
1c81d6e72511578b6ebcafc3e9002427207f5167a7d916b92e34b5f1c36fa01c internal/cbm/vendored/grammars/perl/tsp_unicode.h
|
||||
664764fabf40c0b318e06324a5a12379092a2df25b8cbf2415f71bc0bb352346 internal/cbm/vendored/grammars/php/LICENSE
|
||||
de8eb36bc8f517ab9f3eaf82e3825d7b3e11b62e9471545f4c906100cfce0e07 internal/cbm/vendored/grammars/php/_common_scanner.h
|
||||
bc796948b625f74e390cfd1254392136c86fafbd2dc0c9d93360ec20bc6b0dfb internal/cbm/vendored/grammars/php/parser.c
|
||||
|
||||
+11
-1
@@ -248,9 +248,19 @@ cbm_mem_budget_t cbm_mem_resolve_budget(size_t total_ram, double ram_fraction,
|
||||
cbm_mem_budget_t cbm_mem_resolve_budget_capped(size_t total_ram, double ram_fraction,
|
||||
const char *budget_mb, size_t hard_cap_bytes) {
|
||||
cbm_mem_budget_t result = cbm_mem_resolve_budget(total_ram, ram_fraction, budget_mb);
|
||||
/* The parent already divided the aggregate budget (env override or
|
||||
* ram_fraction) across job slots. That per-slot share is the hard cap:
|
||||
* N workers × a per-worker absolute override would oversubscribe the host
|
||||
* (#1654). A lower explicit value still wins. Keep CBM_MEM_BUDGET_MB as
|
||||
* the source when the env discriminator fired so the ceiling is visible
|
||||
* as the user's aggregate, not as a silent daemon_worker_cap rewrite. */
|
||||
if (hard_cap_bytes > 0 && (result.budget == 0 || result.budget > hard_cap_bytes)) {
|
||||
bool explicit_override =
|
||||
result.source != NULL && strcmp(result.source, "CBM_MEM_BUDGET_MB") == 0;
|
||||
result.budget = hard_cap_bytes;
|
||||
result.source = "daemon_worker_cap";
|
||||
if (!explicit_override) {
|
||||
result.source = "daemon_worker_cap";
|
||||
}
|
||||
result.hard_capped = true;
|
||||
}
|
||||
return result;
|
||||
|
||||
@@ -31,7 +31,7 @@ void cbm_mem_init_with_cap(double ram_fraction, size_t hard_cap_bytes);
|
||||
* the caller never re-parses the env string. */
|
||||
typedef struct {
|
||||
size_t budget; /* resolved budget in bytes */
|
||||
const char *source; /* log token: "ram_fraction" | "CBM_MEM_BUDGET_MB" */
|
||||
const char *source; /* "ram_fraction" | "CBM_MEM_BUDGET_MB" | "daemon_worker_cap" */
|
||||
bool clamped; /* override was valid but exceeded total_ram → clamped down */
|
||||
bool invalid; /* override was present but unparseable / out-of-range / ≤0 */
|
||||
bool hard_capped; /* internal worker hard cap reduced the resolved budget */
|
||||
|
||||
+250
-5
@@ -130,6 +130,7 @@ enum {
|
||||
#define MCP_MAX_HEADER_SIZE ((size_t)8U * 1024U)
|
||||
#define MCP_SEARCH_OUTPUT_MAX ((size_t)64U * 1024U * 1024U)
|
||||
#define MCP_SEARCH_SCAN_TIMEOUT_MS ((uint64_t)30000U)
|
||||
#define MCP_FILE_OUTLINE_OUTPUT_MAX ((size_t)2U * 1024U * 1024U)
|
||||
|
||||
/* ── Helpers ────────────────────────────────────────────────────── */
|
||||
|
||||
@@ -557,6 +558,21 @@ static const tool_def_t TOOLS[] = {
|
||||
"\"type\":\"string\"},\"include_neighbors\":{"
|
||||
"\"type\":\"boolean\",\"default\":false}},\"required\":[\"qualified_name\",\"project\"]}"},
|
||||
|
||||
{"get_file_outline", "Get file outline",
|
||||
"Return a compact declaration outline for one exact repository-relative file. Results are "
|
||||
"filtered by optional exact labels, ordered deterministically by source position, and "
|
||||
"bounded with exact total/offset/limit pagination metadata. File/folder/container nodes "
|
||||
"are excluded. The query observes request cancellation and fails without partial output.",
|
||||
"{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"},"
|
||||
"\"file_path\":{\"type\":\"string\",\"description\":\"Exact repository-relative "
|
||||
"file path\"},\"labels\":{\"type\":\"array\",\"items\":{\"type\":\"string\"},"
|
||||
"\"maxItems\":16,\"description\":\"Optional exact node-label filter\"},"
|
||||
"\"limit\":{\"type\":\"integer\",\"minimum\":1,\"maximum\":200,\"default\":100},"
|
||||
"\"offset\":{\"type\":\"integer\",\"minimum\":0,\"default\":0},"
|
||||
"\"format\":{\"type\":\"string\",\"enum\":[\"tree\",\"json\"],"
|
||||
"\"default\":\"tree\"}},\"additionalProperties\":false,"
|
||||
"\"required\":[\"project\",\"file_path\"]}"},
|
||||
|
||||
{"get_graph_schema", "Get graph schema",
|
||||
"Get the schema of the knowledge graph (node labels, edge types)",
|
||||
"{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"}},\"required\":["
|
||||
@@ -745,6 +761,7 @@ static const tool_annotation_def_t TOOL_ANNOTATIONS[] = {
|
||||
{"query_graph", false, true, true, false},
|
||||
{"trace_path", false, true, true, false},
|
||||
{"get_code_snippet", false, true, true, false},
|
||||
{"get_file_outline", false, true, true, false},
|
||||
{"get_graph_schema", false, true, true, false},
|
||||
{"compare_graphs", true, false, true, false},
|
||||
{"get_architecture", false, true, true, false},
|
||||
@@ -808,13 +825,14 @@ static void mcp_add_tool_def(yyjson_mut_doc *doc, yyjson_mut_val *tools, int i)
|
||||
|
||||
static bool mcp_tool_allowed(cbm_mcp_tool_profile_t profile, const char *name) {
|
||||
static const char *const analysis_tools[] = {
|
||||
"search_graph", "query_graph", "trace_path", "get_code_snippet",
|
||||
"get_graph_schema", "compare_graphs", "get_architecture", "search_code",
|
||||
"list_projects", "index_status", "check_index_coverage", "detect_changes",
|
||||
"search_graph", "query_graph", "trace_path", "get_code_snippet",
|
||||
"get_file_outline", "get_graph_schema", "compare_graphs", "get_architecture",
|
||||
"search_code", "list_projects", "index_status", "check_index_coverage",
|
||||
"detect_changes",
|
||||
};
|
||||
static const char *const scout_tools[] = {
|
||||
"search_graph", "trace_path", "get_code_snippet", "get_architecture",
|
||||
"list_projects", "index_status", "check_index_coverage",
|
||||
"search_graph", "trace_path", "get_code_snippet", "get_file_outline",
|
||||
"get_architecture", "list_projects", "index_status", "check_index_coverage",
|
||||
};
|
||||
if (!name) {
|
||||
return false;
|
||||
@@ -9303,6 +9321,230 @@ static char *build_snippet_response(cbm_mcp_server_t *srv, cbm_node_t *node,
|
||||
return result;
|
||||
}
|
||||
|
||||
static bool file_outline_request_cancelled(void *context) {
|
||||
return mcp_request_cancelled((const cbm_mcp_server_t *)context);
|
||||
}
|
||||
|
||||
static char *handle_get_file_outline(cbm_mcp_server_t *srv, const char *args) {
|
||||
const char *args_text = args ? args : "{}";
|
||||
yyjson_doc *args_doc = yyjson_read(args_text, strlen(args_text), 0);
|
||||
yyjson_val *root = args_doc ? yyjson_doc_get_root(args_doc) : NULL;
|
||||
if (!root || !yyjson_is_obj(root)) {
|
||||
yyjson_doc_free(args_doc);
|
||||
return cbm_mcp_text_result("get_file_outline arguments must be a JSON object", true);
|
||||
}
|
||||
|
||||
char *project = get_project_arg(args_text);
|
||||
yyjson_val *file_value = yyjson_obj_get(root, "file_path");
|
||||
const char *file_path =
|
||||
file_value && yyjson_is_str(file_value) ? yyjson_get_str(file_value) : NULL;
|
||||
if (!project) {
|
||||
yyjson_doc_free(args_doc);
|
||||
return cbm_mcp_text_result("project is required", true);
|
||||
}
|
||||
if (!file_path || !file_path[0]) {
|
||||
free(project);
|
||||
yyjson_doc_free(args_doc);
|
||||
return cbm_mcp_text_result("file_path is required", true);
|
||||
}
|
||||
|
||||
char normalized_path[CBM_SZ_4K];
|
||||
if (coverage_normalize_rel(file_path, false, normalized_path, sizeof(normalized_path)) !=
|
||||
COVERAGE_PATH_OK) {
|
||||
free(project);
|
||||
yyjson_doc_free(args_doc);
|
||||
return cbm_mcp_text_result("file_path must be a repository-relative path without '..'",
|
||||
true);
|
||||
}
|
||||
|
||||
int limit = 100;
|
||||
yyjson_val *limit_value = yyjson_obj_get(root, "limit");
|
||||
if (limit_value) {
|
||||
if (!yyjson_is_int(limit_value)) {
|
||||
free(project);
|
||||
yyjson_doc_free(args_doc);
|
||||
return cbm_mcp_text_result("limit must be an integer", true);
|
||||
}
|
||||
int64_t parsed = yyjson_get_sint(limit_value);
|
||||
if (parsed < 1 || parsed > CBM_STORE_FILE_OUTLINE_MAX_LIMIT) {
|
||||
free(project);
|
||||
yyjson_doc_free(args_doc);
|
||||
return cbm_mcp_text_result("limit must be between 1 and 200", true);
|
||||
}
|
||||
limit = (int)parsed;
|
||||
}
|
||||
|
||||
int offset = 0;
|
||||
yyjson_val *offset_value = yyjson_obj_get(root, "offset");
|
||||
if (offset_value) {
|
||||
if (!yyjson_is_int(offset_value)) {
|
||||
free(project);
|
||||
yyjson_doc_free(args_doc);
|
||||
return cbm_mcp_text_result("offset must be a non-negative integer", true);
|
||||
}
|
||||
int64_t parsed = yyjson_get_sint(offset_value);
|
||||
if (parsed < 0 || parsed > INT_MAX) {
|
||||
free(project);
|
||||
yyjson_doc_free(args_doc);
|
||||
return cbm_mcp_text_result("offset must be a non-negative integer", true);
|
||||
}
|
||||
offset = (int)parsed;
|
||||
}
|
||||
|
||||
bool json_format = false;
|
||||
yyjson_val *format_value = yyjson_obj_get(root, "format");
|
||||
if (format_value) {
|
||||
const char *format = yyjson_is_str(format_value) ? yyjson_get_str(format_value) : NULL;
|
||||
if (!format || (strcmp(format, "tree") != 0 && strcmp(format, "json") != 0)) {
|
||||
free(project);
|
||||
yyjson_doc_free(args_doc);
|
||||
return cbm_mcp_text_result("format must be either 'tree' or 'json'", true);
|
||||
}
|
||||
json_format = strcmp(format, "json") == 0;
|
||||
}
|
||||
|
||||
const char *labels[CBM_STORE_FILE_OUTLINE_MAX_LABELS];
|
||||
int label_count = 0;
|
||||
yyjson_val *labels_value = yyjson_obj_get(root, "labels");
|
||||
if (labels_value) {
|
||||
if (!yyjson_is_arr(labels_value) ||
|
||||
yyjson_arr_size(labels_value) > CBM_STORE_FILE_OUTLINE_MAX_LABELS) {
|
||||
free(project);
|
||||
yyjson_doc_free(args_doc);
|
||||
return cbm_mcp_text_result("labels must be an array of at most 16 strings", true);
|
||||
}
|
||||
size_t index = 0;
|
||||
size_t max = 0;
|
||||
yyjson_val *item;
|
||||
yyjson_arr_foreach(labels_value, index, max, item) {
|
||||
const char *label = yyjson_is_str(item) ? yyjson_get_str(item) : NULL;
|
||||
if (!label || !label[0] || strlen(label) >= CBM_SZ_128) {
|
||||
free(project);
|
||||
yyjson_doc_free(args_doc);
|
||||
return cbm_mcp_text_result("labels must contain only non-empty bounded strings",
|
||||
true);
|
||||
}
|
||||
labels[label_count++] = label;
|
||||
}
|
||||
}
|
||||
|
||||
cbm_store_t *store = resolve_store(srv, project);
|
||||
if (!store) {
|
||||
char *error = build_project_list_error("project not found or not indexed");
|
||||
char *result = cbm_mcp_text_result(error, true);
|
||||
free(error);
|
||||
free(project);
|
||||
yyjson_doc_free(args_doc);
|
||||
return result;
|
||||
}
|
||||
char *not_indexed = verify_project_indexed(store, project);
|
||||
if (not_indexed) {
|
||||
free(project);
|
||||
yyjson_doc_free(args_doc);
|
||||
return not_indexed;
|
||||
}
|
||||
|
||||
cbm_file_outline_row_t *rows = NULL;
|
||||
int row_count = 0;
|
||||
int total = 0;
|
||||
int rc = cbm_store_get_file_outline(store, project, normalized_path, labels, label_count, limit,
|
||||
offset, file_outline_request_cancelled, srv, &rows,
|
||||
&row_count, &total);
|
||||
if (rc != CBM_STORE_OK) {
|
||||
free(project);
|
||||
yyjson_doc_free(args_doc);
|
||||
if (rc == CBM_STORE_CANCELLED) {
|
||||
return cbm_mcp_text_result("get_file_outline cancelled for this request", true);
|
||||
}
|
||||
if (rc == CBM_STORE_SCAN_LIMIT) {
|
||||
return cbm_mcp_text_result("get_file_outline exceeded its fixed output safety limit",
|
||||
true);
|
||||
}
|
||||
return cbm_mcp_text_result("get_file_outline query failed", true);
|
||||
}
|
||||
|
||||
char *payload = NULL;
|
||||
if (!json_format) {
|
||||
cbm_sb_t sb;
|
||||
cbm_sb_init(&sb);
|
||||
cbm_tree_scalar_str(&sb, "file_path", normalized_path);
|
||||
static const char *const columns[] = {"name", "label", "lines", "qn"};
|
||||
cbm_tree_table_header(&sb, "results", row_count, columns, 4);
|
||||
for (int i = 0; i < row_count; i++) {
|
||||
char lines[CBM_SZ_32];
|
||||
if (rows[i].start_line > 0) {
|
||||
snprintf(lines, sizeof(lines), "%d-%d", rows[i].start_line,
|
||||
rows[i].end_line > rows[i].start_line ? rows[i].end_line
|
||||
: rows[i].start_line);
|
||||
} else {
|
||||
lines[0] = '\0';
|
||||
}
|
||||
cbm_tree_row_begin(&sb);
|
||||
cbm_tree_cell_str(&sb, rows[i].name, true);
|
||||
cbm_tree_cell_str(&sb, rows[i].label, false);
|
||||
cbm_tree_cell_str(&sb, lines, false);
|
||||
cbm_tree_cell_str(&sb, rows[i].qualified_name, false);
|
||||
cbm_tree_row_end(&sb);
|
||||
}
|
||||
cbm_tree_scalar_int(&sb, "total", total);
|
||||
cbm_tree_scalar_int(&sb, "offset", offset);
|
||||
cbm_tree_scalar_int(&sb, "limit", limit);
|
||||
cbm_tree_scalar_int(&sb, "returned", row_count);
|
||||
cbm_tree_scalar_bool(&sb, "has_more", (int64_t)offset + row_count < total);
|
||||
payload = cbm_sb_finish(&sb);
|
||||
} else {
|
||||
yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
|
||||
yyjson_mut_val *object = yyjson_mut_obj(doc);
|
||||
yyjson_mut_doc_set_root(doc, object);
|
||||
yyjson_mut_obj_add_str(doc, object, "file_path", normalized_path);
|
||||
yyjson_mut_val *columns = yyjson_mut_arr(doc);
|
||||
static const char *const column_names[] = {"name", "label", "lines", "qn"};
|
||||
for (size_t i = 0; i < sizeof(column_names) / sizeof(column_names[0]); i++) {
|
||||
yyjson_mut_arr_add_str(doc, columns, column_names[i]);
|
||||
}
|
||||
yyjson_mut_obj_add_val(doc, object, "cols", columns);
|
||||
yyjson_mut_val *json_rows = yyjson_mut_arr(doc);
|
||||
for (int i = 0; i < row_count; i++) {
|
||||
char lines[CBM_SZ_32];
|
||||
if (rows[i].start_line > 0) {
|
||||
snprintf(lines, sizeof(lines), "%d-%d", rows[i].start_line,
|
||||
rows[i].end_line > rows[i].start_line ? rows[i].end_line
|
||||
: rows[i].start_line);
|
||||
} else {
|
||||
lines[0] = '\0';
|
||||
}
|
||||
yyjson_mut_val *row = yyjson_mut_arr(doc);
|
||||
yyjson_mut_arr_add_strcpy(doc, row, rows[i].name);
|
||||
yyjson_mut_arr_add_strcpy(doc, row, rows[i].label);
|
||||
yyjson_mut_arr_add_strcpy(doc, row, lines);
|
||||
yyjson_mut_arr_add_strcpy(doc, row, rows[i].qualified_name);
|
||||
yyjson_mut_arr_add_val(json_rows, row);
|
||||
}
|
||||
yyjson_mut_obj_add_val(doc, object, "rows", json_rows);
|
||||
yyjson_mut_obj_add_int(doc, object, "total", total);
|
||||
yyjson_mut_obj_add_int(doc, object, "offset", offset);
|
||||
yyjson_mut_obj_add_int(doc, object, "limit", limit);
|
||||
yyjson_mut_obj_add_int(doc, object, "returned", row_count);
|
||||
yyjson_mut_obj_add_bool(doc, object, "has_more", (int64_t)offset + row_count < total);
|
||||
payload = yy_doc_to_str(doc);
|
||||
yyjson_mut_doc_free(doc);
|
||||
}
|
||||
|
||||
cbm_store_free_file_outline(rows, row_count);
|
||||
free(project);
|
||||
yyjson_doc_free(args_doc);
|
||||
if (!payload) {
|
||||
return cbm_mcp_text_result("get_file_outline output allocation failed", true);
|
||||
}
|
||||
if (strlen(payload) > MCP_FILE_OUTLINE_OUTPUT_MAX) {
|
||||
free(payload);
|
||||
return cbm_mcp_text_result("get_file_outline exceeded its fixed output safety limit", true);
|
||||
}
|
||||
char *result = cbm_mcp_text_result(payload, false);
|
||||
free(payload);
|
||||
return result;
|
||||
}
|
||||
|
||||
static char *handle_get_code_snippet(cbm_mcp_server_t *srv, const char *args) {
|
||||
char *qn = cbm_mcp_get_string_arg(args, "qualified_name");
|
||||
char *project = get_project_arg(args);
|
||||
@@ -12361,6 +12603,9 @@ static char *dispatch_tool(cbm_mcp_server_t *srv, const char *tool_name, const c
|
||||
if (strcmp(tool_name, "get_code_snippet") == 0) {
|
||||
return handle_get_code_snippet(srv, args_json);
|
||||
}
|
||||
if (strcmp(tool_name, "get_file_outline") == 0) {
|
||||
return handle_get_file_outline(srv, args_json);
|
||||
}
|
||||
if (strcmp(tool_name, "search_code") == 0) {
|
||||
return handle_search_code(srv, args_json);
|
||||
}
|
||||
|
||||
@@ -413,6 +413,73 @@ static int pxc_build_lsp_def(CBMArena *arena, const CBMDefinition *src, const ch
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Go: fold per-field "Field" definitions into their owning struct's
|
||||
* field_defs. extract_defs.c emits one flat CBMDefinition per struct field
|
||||
* (label "Field", parent_class = owning struct QN, name = field name,
|
||||
* return_type = raw type text). Those rows are dropped by pxc_build_lsp_def
|
||||
* (pxc_map_label excludes "Field"), so without this fold every Go struct
|
||||
* registers with zero fields and field-chain calls (h.svc.Handle) can
|
||||
* never resolve. Fields are always declared in the same file as their struct,
|
||||
* so scanning the file's own defs covers every case. Runs inside
|
||||
* cbm_pxc_collect_all_defs — one site covers both the prebuilt-registry path
|
||||
* and the per-file fallback, since both consume all_defs. */
|
||||
static void pxc_fold_go_struct_fields(CBMArena *arena, const CBMFileResult *result, CBMLSPDef *defs,
|
||||
int start, int end) {
|
||||
if (!arena || !result || !defs || start >= end) {
|
||||
return;
|
||||
}
|
||||
for (int si = start; si < end; si++) {
|
||||
CBMLSPDef *dst = &defs[si];
|
||||
if (!dst->label || strcmp(dst->label, "Struct") != 0 || !dst->qualified_name) {
|
||||
continue;
|
||||
}
|
||||
int count = 0;
|
||||
size_t total = 0; /* "name:type" bytes; separators and NUL added below */
|
||||
for (int di = 0; di < result->defs.count; di++) {
|
||||
const CBMDefinition *fd = &result->defs.items[di];
|
||||
if (!fd->label || !fd->parent_class || !fd->name || !fd->name[0] || !fd->return_type ||
|
||||
!fd->return_type[0] || strcmp(fd->label, "Field") != 0 ||
|
||||
strcmp(fd->parent_class, dst->qualified_name) != 0) {
|
||||
continue;
|
||||
}
|
||||
total += strlen(fd->name) + 1 + strlen(fd->return_type);
|
||||
count++;
|
||||
}
|
||||
if (count == 0) {
|
||||
continue;
|
||||
}
|
||||
/* count - 1 separators + NUL. */
|
||||
size_t bufsz = total + (size_t)(count - 1) + 1;
|
||||
char *buf = (char *)cbm_arena_alloc(arena, bufsz);
|
||||
if (!buf) {
|
||||
continue;
|
||||
}
|
||||
char *p = buf;
|
||||
int written = 0;
|
||||
for (int di = 0; di < result->defs.count; di++) {
|
||||
const CBMDefinition *fd = &result->defs.items[di];
|
||||
if (!fd->label || !fd->parent_class || !fd->name || !fd->name[0] || !fd->return_type ||
|
||||
!fd->return_type[0] || strcmp(fd->label, "Field") != 0 ||
|
||||
strcmp(fd->parent_class, dst->qualified_name) != 0) {
|
||||
continue;
|
||||
}
|
||||
size_t n = strlen(fd->name);
|
||||
memcpy(p, fd->name, n);
|
||||
p += n;
|
||||
*p++ = ':';
|
||||
n = strlen(fd->return_type);
|
||||
memcpy(p, fd->return_type, n);
|
||||
p += n;
|
||||
if (written + 1 < count) {
|
||||
*p++ = '|';
|
||||
}
|
||||
written++;
|
||||
}
|
||||
*p = '\0';
|
||||
dst->field_defs = buf;
|
||||
}
|
||||
}
|
||||
|
||||
/* Carry one Rust type-level impl independently of any method definition.
|
||||
* `impl Trait for Type {}` is semantically meaningful even when the block is
|
||||
* empty (the trait may provide defaults), so attaching the relation only to
|
||||
@@ -472,6 +539,7 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMFileResult
|
||||
if (out_def_starts) {
|
||||
out_def_starts[fi] = idx;
|
||||
}
|
||||
const int file_start = idx;
|
||||
if (!cache[fi])
|
||||
continue;
|
||||
if (!def_modules[fi]) {
|
||||
@@ -510,6 +578,9 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMFileResult
|
||||
}
|
||||
}
|
||||
cbm_pxc_free_import_map(imp_keys, imp_vals, imp_count); /* NULL-safe */
|
||||
if (files[fi].language == CBM_LANG_GO) {
|
||||
pxc_fold_go_struct_fields(&cache[fi]->arena, cache[fi], defs, file_start, idx);
|
||||
}
|
||||
if (files[fi].language == CBM_LANG_RUST) {
|
||||
for (int ii = 0; ii < cache[fi]->impl_traits.count; ii++) {
|
||||
if (pxc_build_rust_impl_relation(
|
||||
@@ -1218,8 +1289,16 @@ void cbm_pxc_dispatch_file(CBMLanguage lang, CBMFileResult *result, const char *
|
||||
switch (lang) {
|
||||
case CBM_LANG_GO:
|
||||
/* Tier 3 (metadata-driven): pure lookup over the Tier-1
|
||||
* lsp_unresolved entries — no parse, no AST walk. */
|
||||
* lsp_unresolved entries — no parse, no AST walk. Then the
|
||||
* AST walk on the shared Tier-2 registry (mirroring every
|
||||
* other language) so NAMED receivers evaluated against
|
||||
* project-wide defs also resolve. The walk variant below is
|
||||
* read-only — the sealed registry is safe for parallel
|
||||
* workers. */
|
||||
cbm_go_fast_resolve_qualified_calls(result, prebuilt, imp_keys, imp_vals, imp_count);
|
||||
cbm_run_go_lsp_cross_with_registry(&result->arena, source, source_len, def_module,
|
||||
prebuilt, imp_keys, imp_vals, imp_count,
|
||||
result->cached_tree, &result->resolved_calls);
|
||||
used_prebuilt = true;
|
||||
break;
|
||||
case CBM_LANG_PYTHON: {
|
||||
|
||||
@@ -1792,6 +1792,13 @@ static void detect_url_in_args(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source,
|
||||
const CBMCall *call) {
|
||||
for (int ai = 0; ai < call->arg_count; ai++) {
|
||||
const CBMCallArg *ca = &call->args[ai];
|
||||
/* A slash-prefixed raw expression is not a URL string. In JS/TS this
|
||||
* is notably a regex literal (`/<table/i`); genuine string literals
|
||||
* and propagated constants are carried in `value`, while template
|
||||
* literals keep their leading backtick in `expr`. */
|
||||
if (!ca->value && ca->expr && ca->expr[0] == '/') {
|
||||
continue;
|
||||
}
|
||||
const char *url = ca->value ? ca->value : ca->expr;
|
||||
if (!url || (url[0] != '/' && url[0] != '`')) {
|
||||
continue;
|
||||
@@ -1800,6 +1807,9 @@ static void detect_url_in_args(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source,
|
||||
if (!normalize_url_arg(url, norm, (int)sizeof(norm))) {
|
||||
continue;
|
||||
}
|
||||
if (!cbm_service_pattern_is_http_route_literal(norm, call->callee_name)) {
|
||||
continue;
|
||||
}
|
||||
char route_qn[CBM_ROUTE_QN_SIZE];
|
||||
char cpath[CBM_SZ_256];
|
||||
snprintf(route_qn, sizeof(route_qn), "__route__ANY__%s",
|
||||
|
||||
@@ -33,12 +33,11 @@ enum {
|
||||
#include <stdint.h>
|
||||
#include "graph_buffer/graph_buffer.h"
|
||||
#include "foundation/log.h"
|
||||
#include "service_patterns.h" /* cbm_service_pattern_is_http_route_literal */
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
bool cbm_service_pattern_is_http_route_literal(const char *literal, const char *callee_name);
|
||||
|
||||
/* True for characters that may appear in a ":name" route parameter. */
|
||||
static inline bool is_route_ident_char(char c) {
|
||||
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_';
|
||||
|
||||
@@ -2640,6 +2640,233 @@ int cbm_store_find_nodes_by_file(cbm_store_t *s, const char *project, const char
|
||||
project, file_path, out, count);
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
cbm_store_cancel_fn cancel;
|
||||
void *context;
|
||||
} file_outline_cancel_guard_t;
|
||||
|
||||
static bool file_outline_cancelled(const file_outline_cancel_guard_t *guard) {
|
||||
return guard && guard->cancel && guard->cancel(guard->context);
|
||||
}
|
||||
|
||||
static int file_outline_progress_cancel(void *context) {
|
||||
return file_outline_cancelled((const file_outline_cancel_guard_t *)context) ? 1 : 0;
|
||||
}
|
||||
|
||||
static bool file_outline_build_sql(char *sql, size_t sql_size, bool count_only, int label_count) {
|
||||
int written =
|
||||
snprintf(sql, sql_size,
|
||||
count_only ? "SELECT COUNT(*) FROM nodes WHERE project = ?1 AND file_path = ?2 "
|
||||
"AND label NOT IN ('Module','Package','File','Folder','Project')"
|
||||
: "SELECT name, label, qualified_name, start_line, end_line FROM nodes "
|
||||
"WHERE project = ?1 AND file_path = ?2 "
|
||||
"AND label NOT IN ('Module','Package','File','Folder','Project')");
|
||||
if (written < 0 || (size_t)written >= sql_size) {
|
||||
return false;
|
||||
}
|
||||
size_t used = (size_t)written;
|
||||
if (label_count > 0) {
|
||||
written = snprintf(sql + used, sql_size - used, " AND label IN (");
|
||||
if (written < 0 || (size_t)written >= sql_size - used) {
|
||||
return false;
|
||||
}
|
||||
used += (size_t)written;
|
||||
for (int i = 0; i < label_count; i++) {
|
||||
written = snprintf(sql + used, sql_size - used, "%s?%d", i > 0 ? "," : "", i + 3);
|
||||
if (written < 0 || (size_t)written >= sql_size - used) {
|
||||
return false;
|
||||
}
|
||||
used += (size_t)written;
|
||||
}
|
||||
written = snprintf(sql + used, sql_size - used, ")");
|
||||
if (written < 0 || (size_t)written >= sql_size - used) {
|
||||
return false;
|
||||
}
|
||||
used += (size_t)written;
|
||||
}
|
||||
if (!count_only) {
|
||||
int limit_bind = label_count + 3;
|
||||
int offset_bind = limit_bind + 1;
|
||||
written = snprintf(sql + used, sql_size - used,
|
||||
" ORDER BY start_line, end_line, label COLLATE BINARY, "
|
||||
"name COLLATE BINARY, qualified_name COLLATE BINARY, id "
|
||||
"LIMIT ?%d OFFSET ?%d",
|
||||
limit_bind, offset_bind);
|
||||
if (written < 0 || (size_t)written >= sql_size - used) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static void file_outline_bind_common(sqlite3_stmt *stmt, const char *project, const char *file_path,
|
||||
const char *const *labels, int label_count) {
|
||||
bind_text(stmt, ST_COL_1, project);
|
||||
bind_text(stmt, ST_COL_2, file_path);
|
||||
for (int i = 0; i < label_count; i++) {
|
||||
bind_text(stmt, i + 3, labels[i]);
|
||||
}
|
||||
}
|
||||
|
||||
void cbm_store_free_file_outline(cbm_file_outline_row_t *rows, int count) {
|
||||
if (!rows) {
|
||||
return;
|
||||
}
|
||||
for (int i = 0; i < count; i++) {
|
||||
free((char *)rows[i].name);
|
||||
free((char *)rows[i].label);
|
||||
free((char *)rows[i].qualified_name);
|
||||
}
|
||||
free(rows);
|
||||
}
|
||||
|
||||
int cbm_store_get_file_outline(cbm_store_t *s, const char *project, const char *file_path,
|
||||
const char *const *labels, int label_count, int limit, int offset,
|
||||
cbm_store_cancel_fn cancel, void *cancel_context,
|
||||
cbm_file_outline_row_t **out, int *count, int *total) {
|
||||
if (out) {
|
||||
*out = NULL;
|
||||
}
|
||||
if (count) {
|
||||
*count = 0;
|
||||
}
|
||||
if (total) {
|
||||
*total = 0;
|
||||
}
|
||||
if (!s || !s->db || !project || !project[0] || !file_path || !file_path[0] || !out || !count ||
|
||||
!total || label_count < 0 || label_count > CBM_STORE_FILE_OUTLINE_MAX_LABELS ||
|
||||
(label_count > 0 && !labels) || limit < 1 || limit > CBM_STORE_FILE_OUTLINE_MAX_LIMIT ||
|
||||
offset < 0) {
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
for (int i = 0; i < label_count; i++) {
|
||||
if (!labels[i] || !labels[i][0]) {
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
}
|
||||
|
||||
file_outline_cancel_guard_t guard = {.cancel = cancel, .context = cancel_context};
|
||||
if (file_outline_cancelled(&guard)) {
|
||||
return CBM_STORE_CANCELLED;
|
||||
}
|
||||
if (cancel) {
|
||||
sqlite3_progress_handler(s->db, 1000, file_outline_progress_cancel, &guard);
|
||||
}
|
||||
|
||||
char sql[ST_SQL_BUF];
|
||||
if (!file_outline_build_sql(sql, sizeof(sql), true, label_count)) {
|
||||
sqlite3_progress_handler(s->db, 0, NULL, NULL);
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
sqlite3_stmt *stmt = NULL;
|
||||
int rc = sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL);
|
||||
if (rc != SQLITE_OK) {
|
||||
store_set_error_sqlite(s, "file outline count prepare");
|
||||
sqlite3_progress_handler(s->db, 0, NULL, NULL);
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
file_outline_bind_common(stmt, project, file_path, labels, label_count);
|
||||
rc = sqlite3_step(stmt);
|
||||
if (rc == SQLITE_ROW) {
|
||||
*total = sqlite3_column_int(stmt, 0);
|
||||
}
|
||||
sqlite3_finalize(stmt);
|
||||
if (rc == SQLITE_INTERRUPT || file_outline_cancelled(&guard)) {
|
||||
sqlite3_progress_handler(s->db, 0, NULL, NULL);
|
||||
*total = 0;
|
||||
return CBM_STORE_CANCELLED;
|
||||
}
|
||||
if (rc != SQLITE_ROW) {
|
||||
store_set_error_sqlite(s, "file outline count");
|
||||
sqlite3_progress_handler(s->db, 0, NULL, NULL);
|
||||
*total = 0;
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
|
||||
if (!file_outline_build_sql(sql, sizeof(sql), false, label_count)) {
|
||||
sqlite3_progress_handler(s->db, 0, NULL, NULL);
|
||||
*total = 0;
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
stmt = NULL;
|
||||
rc = sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL);
|
||||
if (rc != SQLITE_OK) {
|
||||
store_set_error_sqlite(s, "file outline prepare");
|
||||
sqlite3_progress_handler(s->db, 0, NULL, NULL);
|
||||
*total = 0;
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
file_outline_bind_common(stmt, project, file_path, labels, label_count);
|
||||
sqlite3_bind_int(stmt, label_count + 3, limit);
|
||||
sqlite3_bind_int(stmt, label_count + 4, offset);
|
||||
|
||||
cbm_file_outline_row_t *rows = calloc((size_t)limit, sizeof(*rows));
|
||||
if (!rows) {
|
||||
sqlite3_finalize(stmt);
|
||||
sqlite3_progress_handler(s->db, 0, NULL, NULL);
|
||||
*total = 0;
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
size_t text_bytes = 0U;
|
||||
int n = 0;
|
||||
bool was_cancelled = false;
|
||||
for (;;) {
|
||||
if (file_outline_cancelled(&guard)) {
|
||||
was_cancelled = true;
|
||||
break;
|
||||
}
|
||||
rc = sqlite3_step(stmt);
|
||||
if (rc != SQLITE_ROW) {
|
||||
break;
|
||||
}
|
||||
const char *name = (const char *)sqlite3_column_text(stmt, 0);
|
||||
const char *label = (const char *)sqlite3_column_text(stmt, 1);
|
||||
const char *qn = (const char *)sqlite3_column_text(stmt, 2);
|
||||
size_t row_bytes = (size_t)sqlite3_column_bytes(stmt, 0) +
|
||||
(size_t)sqlite3_column_bytes(stmt, 1) +
|
||||
(size_t)sqlite3_column_bytes(stmt, 2) + 3U;
|
||||
if (row_bytes > CBM_STORE_FILE_OUTLINE_MAX_TEXT_BYTES - text_bytes) {
|
||||
sqlite3_finalize(stmt);
|
||||
sqlite3_progress_handler(s->db, 0, NULL, NULL);
|
||||
cbm_store_free_file_outline(rows, n);
|
||||
*total = 0;
|
||||
return CBM_STORE_SCAN_LIMIT;
|
||||
}
|
||||
rows[n].name = heap_strdup(name ? name : "");
|
||||
rows[n].label = heap_strdup(label ? label : "");
|
||||
rows[n].qualified_name = heap_strdup(qn ? qn : "");
|
||||
rows[n].start_line = sqlite3_column_int(stmt, 3);
|
||||
rows[n].end_line = sqlite3_column_int(stmt, 4);
|
||||
if (!rows[n].name || !rows[n].label || !rows[n].qualified_name) {
|
||||
sqlite3_finalize(stmt);
|
||||
sqlite3_progress_handler(s->db, 0, NULL, NULL);
|
||||
cbm_store_free_file_outline(rows, n + 1);
|
||||
*total = 0;
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
text_bytes += row_bytes;
|
||||
n++;
|
||||
}
|
||||
was_cancelled = was_cancelled || rc == SQLITE_INTERRUPT || file_outline_cancelled(&guard);
|
||||
sqlite3_finalize(stmt);
|
||||
sqlite3_progress_handler(s->db, 0, NULL, NULL);
|
||||
if (was_cancelled) {
|
||||
cbm_store_free_file_outline(rows, n);
|
||||
*total = 0;
|
||||
return CBM_STORE_CANCELLED;
|
||||
}
|
||||
if (rc != SQLITE_DONE) {
|
||||
store_set_error_sqlite(s, "file outline rows");
|
||||
cbm_store_free_file_outline(rows, n);
|
||||
*total = 0;
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
|
||||
*out = rows;
|
||||
*count = n;
|
||||
return CBM_STORE_OK;
|
||||
}
|
||||
|
||||
int cbm_store_count_nodes(cbm_store_t *s, const char *project) {
|
||||
if (!s || !s->db) {
|
||||
return 0;
|
||||
|
||||
@@ -28,6 +28,10 @@ typedef struct cbm_store cbm_store_t;
|
||||
#define CBM_STORE_SCAN_LIMIT (-4)
|
||||
#define CBM_STORE_CALLBACK_ERR (-5)
|
||||
|
||||
#define CBM_STORE_FILE_OUTLINE_MAX_LIMIT 200
|
||||
#define CBM_STORE_FILE_OUTLINE_MAX_LABELS 16
|
||||
#define CBM_STORE_FILE_OUTLINE_MAX_TEXT_BYTES (256U * 1024U)
|
||||
|
||||
/* ── Data structures ────────────────────────────────────────────── */
|
||||
|
||||
typedef struct {
|
||||
@@ -42,6 +46,18 @@ typedef struct {
|
||||
const char *properties_json; /* JSON string, NULL → "{}" */
|
||||
} cbm_node_t;
|
||||
|
||||
/* Compact declaration row returned by the bounded file-outline query. */
|
||||
typedef struct {
|
||||
const char *name;
|
||||
const char *label;
|
||||
const char *qualified_name;
|
||||
int start_line;
|
||||
int end_line;
|
||||
} cbm_file_outline_row_t;
|
||||
|
||||
/* Optional cancellation callback for bounded store queries. */
|
||||
typedef bool (*cbm_store_cancel_fn)(void *context);
|
||||
|
||||
typedef struct {
|
||||
int64_t id;
|
||||
const char *project;
|
||||
@@ -497,6 +513,18 @@ int cbm_store_find_nodes_by_label(cbm_store_t *s, const char *project, const cha
|
||||
int cbm_store_find_nodes_by_file(cbm_store_t *s, const char *project, const char *file_path,
|
||||
cbm_node_t **out, int *count);
|
||||
|
||||
/* Return a stable, paginated outline for one exact repository-relative file.
|
||||
* File/folder/container nodes are excluded. labels may be NULL when
|
||||
* label_count is zero; otherwise labels are exact-match filters. The query is
|
||||
* capped by CBM_STORE_FILE_OUTLINE_MAX_LIMIT and a fixed aggregate text-byte
|
||||
* budget, and fails without partial rows when cancelled or over budget.
|
||||
* total is the exact filtered count before pagination. */
|
||||
int cbm_store_get_file_outline(cbm_store_t *s, const char *project, const char *file_path,
|
||||
const char *const *labels, int label_count, int limit, int offset,
|
||||
cbm_store_cancel_fn cancel, void *cancel_context,
|
||||
cbm_file_outline_row_t **out, int *count, int *total);
|
||||
void cbm_store_free_file_outline(cbm_file_outline_row_t *rows, int count);
|
||||
|
||||
/* Batch lookup: map qualified names → node IDs.
|
||||
* qns[i] is resolved; out_ids[i] receives the ID or 0 if not found.
|
||||
* Returns number of QNs actually found, or CBM_STORE_ERR. */
|
||||
|
||||
@@ -752,6 +752,12 @@ static const char TLAPLUS_BOUNDED_QUANTIFICATION[] =
|
||||
"Guard(values) == \\A item \\in values : item = item\n"
|
||||
"====\n";
|
||||
|
||||
/* Pkl access expressions double as plain property reads, so the bare fixture
|
||||
* also proves the un-applied reference is not promoted to a call. */
|
||||
static const char PKL_INSIDE[] = "function accept(value: Int): Int = value\n"
|
||||
"function run(watched: Int): Int = accept(watched)\n";
|
||||
static const char PKL_BARE[] = "function run(watched: Int): Int = watched\n";
|
||||
|
||||
static const char APEX_INSIDE[] = "public class Sample {\n"
|
||||
" private static Integer accept(Integer value) {\n"
|
||||
" return value;\n"
|
||||
@@ -988,6 +994,9 @@ static const RoutineArgumentCase LLVM_IR_CASE = ROUTINE_ARGUMENT_CASE(
|
||||
static const RoutineArgumentCase TLAPLUS_CASE = ROUTINE_ARGUMENT_CASE(
|
||||
"TLAPLUS", CBM_LANG_TLAPLUS, "Sample.tla", TLAPLUS_INSIDE, TLAPLUS_BARE, "bound_op", "Guard",
|
||||
"Accept", "values", 1, 1, 0, "TLA+ operator application with a value argument");
|
||||
static const RoutineArgumentCase PKL_CASE = ROUTINE_ARGUMENT_CASE(
|
||||
"PKL", CBM_LANG_PKL, "sample.pkl", PKL_INSIDE, PKL_BARE, "unqualifiedAccessExpr", "run",
|
||||
"accept", "watched", 1, 1, 0, "native Pkl method application and property-read vocabulary");
|
||||
static const RoutineArgumentCase APEX_CASE = ROUTINE_ARGUMENT_CASE(
|
||||
"APEX", CBM_LANG_APEX, "Sample.cls", APEX_INSIDE, APEX_BARE, "method_invocation", "run",
|
||||
"accept", "watched", 1, 1, 0, "native method application");
|
||||
@@ -1203,6 +1212,7 @@ DEFINE_ROUTINE_ARGUMENT_TEST(func, FUNC_CASE)
|
||||
DEFINE_ROUTINE_ARGUMENT_TEST(puppet, PUPPET_CASE)
|
||||
DEFINE_ROUTINE_ARGUMENT_TEST(slang, SLANG_CASE)
|
||||
DEFINE_ROUTINE_ARGUMENT_TEST(llvm_ir, LLVM_IR_CASE)
|
||||
DEFINE_ROUTINE_ARGUMENT_TEST(pkl, PKL_CASE)
|
||||
DEFINE_ROUTINE_ARGUMENT_TEST(apex, APEX_CASE)
|
||||
DEFINE_ROUTINE_ARGUMENT_TEST(pine, PINE_CASE)
|
||||
DEFINE_ROUTINE_ARGUMENT_TEST(qml, QML_CASE)
|
||||
@@ -1282,15 +1292,15 @@ TEST(repro_call_argument_matrix_b_domain_bitbake) {
|
||||
}
|
||||
|
||||
enum {
|
||||
ROUTINE_ARGUMENT_LANGUAGE_COUNT = 38,
|
||||
ROUTINE_ARGUMENT_LANGUAGE_COUNT = 39,
|
||||
MODULE_ARGUMENT_LANGUAGE_COUNT = 4,
|
||||
DOMAIN_CONTROL_LANGUAGE_COUNT = 6,
|
||||
MATRIX_LANGUAGE_COUNT = ROUTINE_ARGUMENT_LANGUAGE_COUNT + MODULE_ARGUMENT_LANGUAGE_COUNT +
|
||||
DOMAIN_CONTROL_LANGUAGE_COUNT,
|
||||
};
|
||||
|
||||
_Static_assert(MATRIX_LANGUAGE_COUNT == 48,
|
||||
"RACKET..CHIALISP call-capable matrix must contain exactly 48 "
|
||||
_Static_assert(MATRIX_LANGUAGE_COUNT == 49,
|
||||
"RACKET..CHIALISP call-capable matrix must contain exactly 49 "
|
||||
"language rows");
|
||||
|
||||
#define MATRIX_B_LANGUAGE_ROWS(X) \
|
||||
@@ -1322,6 +1332,7 @@ _Static_assert(MATRIX_LANGUAGE_COUNT == 48,
|
||||
X(repro_call_argument_matrix_b_routine_slang, SLANG_CASE.identity.language) \
|
||||
X(repro_call_argument_matrix_b_routine_llvm_ir, LLVM_IR_CASE.identity.language) \
|
||||
X(repro_call_argument_matrix_b_routine_tlaplus, TLAPLUS_CASE.identity.language) \
|
||||
X(repro_call_argument_matrix_b_routine_pkl, PKL_CASE.identity.language) \
|
||||
X(repro_call_argument_matrix_b_routine_apex, APEX_CASE.identity.language) \
|
||||
X(repro_call_argument_matrix_b_routine_pine, PINE_CASE.identity.language) \
|
||||
X(repro_call_argument_matrix_b_routine_qml, QML_CASE.identity.language) \
|
||||
|
||||
@@ -16,6 +16,11 @@
|
||||
* TLA+ bound_op is historical direct-call metadata. Bounded quantification is
|
||||
* not a historical ledger row; repro_call_argument_matrix_b is its separate
|
||||
* negative behavior guard.
|
||||
*
|
||||
* The historical-snapshot column is the audited ledger, not an archaeology
|
||||
* record: every registered call node owns a row, so a newly registered kind
|
||||
* (Pkl's access/new expressions) joins the snapshot with the change that
|
||||
* registers it.
|
||||
*/
|
||||
#include "test_framework.h"
|
||||
#include "lang_specs.h"
|
||||
@@ -54,10 +59,10 @@ typedef struct {
|
||||
} CallNodeManifestEntry;
|
||||
|
||||
enum {
|
||||
EXPECTED_HISTORICAL_CALL_NODE_TOTAL = 223,
|
||||
EXPECTED_ACTIVE_PRIMARY_TOTAL = 193,
|
||||
EXPECTED_DIRECT_CALLS = 158,
|
||||
EXPECTED_CONSTRUCTOR_CALLS = 21,
|
||||
EXPECTED_HISTORICAL_CALL_NODE_TOTAL = 226,
|
||||
EXPECTED_ACTIVE_PRIMARY_TOTAL = 196,
|
||||
EXPECTED_DIRECT_CALLS = 160,
|
||||
EXPECTED_CONSTRUCTOR_CALLS = 22,
|
||||
EXPECTED_OPERATOR_CALLS = 12,
|
||||
EXPECTED_IMPLICIT_CALLS = 2,
|
||||
EXPECTED_DSL_INVOCATIONS = 14,
|
||||
@@ -65,7 +70,7 @@ enum {
|
||||
EXPECTED_CALLEE_WRAPPERS = 8,
|
||||
EXPECTED_ARGUMENT_WRAPPERS = 1,
|
||||
EXPECTED_CONTROL_OR_DOCUMENT_NONCALLS = 6,
|
||||
EXPECTED_PRIMARY_OWNERS = 193,
|
||||
EXPECTED_PRIMARY_OWNERS = 196,
|
||||
EXPECTED_SYNTHETIC_OWNERS = 10,
|
||||
EXPECTED_NO_CALL_EDGE_OWNERS = 20,
|
||||
EXPECTED_PRIMARY_OPERATOR_CALLS = 4,
|
||||
@@ -297,6 +302,9 @@ static const CallNodeManifestEntry CALL_NODE_MANIFEST[] = {
|
||||
ENTRY(CBM_LANG_TLAPLUS, "function_evaluation", DIRECT_CALL, PRIMARY_EXTRACTOR, true, true),
|
||||
ENTRY(CBM_LANG_TLAPLUS, "call", DIRECT_CALL, PRIMARY_EXTRACTOR, true, true),
|
||||
ENTRY(CBM_LANG_TLAPLUS, "bound_op", DIRECT_CALL, PRIMARY_EXTRACTOR, true, true),
|
||||
ENTRY(CBM_LANG_PKL, "unqualifiedAccessExpr", DIRECT_CALL, PRIMARY_EXTRACTOR, true, true),
|
||||
ENTRY(CBM_LANG_PKL, "qualifiedAccessExpr", DIRECT_CALL, PRIMARY_EXTRACTOR, true, true),
|
||||
ENTRY(CBM_LANG_PKL, "newExpr", CONSTRUCTOR_CALL, PRIMARY_EXTRACTOR, true, true),
|
||||
ENTRY(CBM_LANG_APEX, "method_invocation", DIRECT_CALL, PRIMARY_EXTRACTOR, true, true),
|
||||
ENTRY(CBM_LANG_PINE, "call", DIRECT_CALL, PRIMARY_EXTRACTOR, true, true),
|
||||
ENTRY(CBM_LANG_MOJO, "call", DIRECT_CALL, PRIMARY_EXTRACTOR, true, true),
|
||||
|
||||
@@ -45,7 +45,8 @@
|
||||
* 6. calls-extracted : inv_has_call(r, callee) == 1.
|
||||
* Only asserted for languages that have non-empty
|
||||
* call_types: HCL (function_call), NICKEL (infix_expr),
|
||||
* JSONNET (functioncall), STARLARK (call).
|
||||
* JSONNET (functioncall), STARLARK (call),
|
||||
* PKL (unqualified/qualifiedAccessExpr, newExpr).
|
||||
*
|
||||
* FULL-PIPELINE (rh_index_files -> cbm_store_t*, via inv_count_* store helpers):
|
||||
* 7. callable-sourcing : inv_count_calls_by_source(store,project,&mod,&call).
|
||||
@@ -85,11 +86,15 @@
|
||||
* Dims 1-5 ("Class"). No calls.
|
||||
* XML -- class_types = element -> "Class". Dims 1-5 ("Class"). No calls.
|
||||
* PROPERTIES -- var_types = property -> "Variable". Dims 1-5 ("Variable"). No calls.
|
||||
* PKL -- func_types = classMethod/objectMethod -> "Function";
|
||||
* class_types = clazz -> "Class"; var_types = classProperty/objectProperty.
|
||||
* call_types = empty_types. Dims 1-5 ("Function", "Class"). No call dim.
|
||||
*
|
||||
* LANGUAGES WITH CALLABLES (dims 1-6 + R, and pipeline dims 7-8 where applicable):
|
||||
* PKL -- func_types = classMethod/objectMethod -> "Function";
|
||||
* class_types = clazz/typeAlias -> "Class";
|
||||
* var_types = classProperty/objectProperty;
|
||||
* call_types = unqualifiedAccessExpr/qualifiedAccessExpr/newExpr.
|
||||
* Dims 1-8. The access-expr call types double as property reads,
|
||||
* so extract_pkl_callee gates them on an `argumentList` child;
|
||||
* the test adds an inline negative assertion for that gate.
|
||||
* HCL -- class_types = block -> "Class"; var_types = attribute;
|
||||
* call_types = function_call. Dims 1-6. No func_types so no pipeline
|
||||
* dim 7 (calls would be module-sourced with no Function anchor).
|
||||
@@ -766,39 +771,72 @@ TEST(repro_grammar_config_ron) {
|
||||
|
||||
/* ── PKL ──────────────────────────────────────────────────────────────────────
|
||||
* Idiomatic PKL (Apple Pkl) module with a class definition
|
||||
* (pkl_class_types = {"clazz"} -> "Class"), a method inside it
|
||||
* (pkl_class_types = {"clazz", "typeAlias"} -> "Class"), methods inside it
|
||||
* (pkl_func_types = {"classMethod", "objectMethod"} -> "Function"), and
|
||||
* class properties (pkl_var_types = {"classProperty", "objectProperty"}).
|
||||
* pkl_call_types = empty_types so no call extraction occurs.
|
||||
* pkl_call_types = {"unqualifiedAccessExpr", "qualifiedAccessExpr", "newExpr"}.
|
||||
*
|
||||
* Dims asserted: 1-5 + R ("Class" for the class def, "Function" for the method).
|
||||
* Dims 6-8 SKIPPED: call_types = empty_types in spec.
|
||||
* Expected GREEN: dims 1-5. Dim 5 RED would indicate clazz->Class or
|
||||
* classMethod->Function mapping is broken in the PKL grammar walker.
|
||||
* Dims asserted: 1-8 (full battery) + R.
|
||||
* Dim 6 GREEN: `makeUrl(host, port)` inside url() extracts callee "makeUrl".
|
||||
* Dim 7 GREEN: every call site in the fixture is inside a classMethod body, so
|
||||
* no CALLS edge is Module-sourced. (Real-world Pkl does call at module level;
|
||||
* the fixture deliberately avoids it because dim 7 treats Module-sourced
|
||||
* in-body calls as the enclosing-func gap.)
|
||||
* Dim 8 GREEN: makeUrl and Server are both defined in-file, so neither the
|
||||
* unqualified call nor the newExpr constructor edge dangles.
|
||||
*
|
||||
* PKL-SPECIFIC REGRESSION (asserted inline below): `unqualifiedAccessExpr` and
|
||||
* `qualifiedAccessExpr` are the same node for a call and for a bare property
|
||||
* read, so the interpolated `host` / `port` reads inside makeUrl must NOT be
|
||||
* emitted as CALLS. extract_pkl_callee gates on an `argumentList` child; without
|
||||
* that gate every property read in every Pkl file becomes a call edge.
|
||||
*/
|
||||
TEST(repro_grammar_config_pkl) {
|
||||
static const char src[] =
|
||||
"module cbm.Config\n"
|
||||
"\n"
|
||||
"function makeUrl(host: String, port: Int): String = \"http://\\(host):\\(port)\"\n"
|
||||
"typealias Port = Int\n"
|
||||
"\n"
|
||||
"function makeUrl(host: String, port: Port): String = \"http://\\(host):\\(port)\"\n"
|
||||
"\n"
|
||||
"class Server {\n"
|
||||
" host: String = \"localhost\"\n"
|
||||
" port: Int = 8080\n"
|
||||
" port: Port = 8080\n"
|
||||
" tls: Boolean = false\n"
|
||||
"\n"
|
||||
" function url(): String = \"http://\\(host):\\(port)\"\n"
|
||||
"}\n"
|
||||
" function url(): String = makeUrl(host, port)\n"
|
||||
"\n"
|
||||
"server = new Server {\n"
|
||||
" host = \"0.0.0.0\"\n"
|
||||
" port = 9000\n"
|
||||
" function clone(): Server = new Server { host = host }\n"
|
||||
"}\n";
|
||||
static const char bad[] = "module cbm.Config\nclass Server {\n host:";
|
||||
if (config_struct_battery("PKL", src, CBM_LANG_PKL, "config.pkl",
|
||||
"Class", "Function") != 0)
|
||||
if (config_callable_battery("PKL", src, CBM_LANG_PKL, "config.pkl",
|
||||
"Function", "makeUrl") != 0)
|
||||
return 1;
|
||||
return config_robustness("PKL", bad, CBM_LANG_PKL, "config.pkl");
|
||||
|
||||
/* Bare property reads must not be calls (see PKL-SPECIFIC REGRESSION above). */
|
||||
CBMFileResult *pr = inv_rx(src, CBM_LANG_PKL, "config.pkl");
|
||||
if (!pr) {
|
||||
printf(" %sFAIL%s [PKL] inv_rx returned NULL\n", tf_red(), tf_reset());
|
||||
return 1;
|
||||
}
|
||||
int bogus = 0;
|
||||
for (int i = 0; i < pr->calls.count; i++) {
|
||||
const char *cn = pr->calls.items[i].callee_name;
|
||||
if (cn && (strcmp(cn, "host") == 0 || strcmp(cn, "port") == 0 ||
|
||||
strcmp(cn, "tls") == 0)) {
|
||||
bogus++;
|
||||
}
|
||||
}
|
||||
cbm_free_result(pr);
|
||||
if (bogus != 0) {
|
||||
printf(" %sFAIL%s [PKL] property-read-not-call: %d bare property read(s) "
|
||||
"emitted as a CALLS edge\n", tf_red(), tf_reset(), bogus);
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (config_robustness("PKL", bad, CBM_LANG_PKL, "config.pkl") != 0)
|
||||
return 1;
|
||||
return config_pipeline_battery("PKL", "config.pkl", src);
|
||||
}
|
||||
|
||||
/* ── NICKEL ───────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -195,7 +195,7 @@ static const LanguageCapabilityEntry LANGUAGE_CAPABILITIES[CBM_LANG_COUNT] = {
|
||||
NO_CALL(SMITHY),
|
||||
NO_CALL(WIT),
|
||||
CALL_WITH_REFERENCE_VOCAB(TLAPLUS),
|
||||
NO_CALL(PKL),
|
||||
CALL_WITH_REFERENCE_VOCAB(PKL),
|
||||
NO_CALL(GOMOD),
|
||||
CALL_WITH_REFERENCE_VOCAB(APEX),
|
||||
NO_CALL(SOQL),
|
||||
@@ -301,8 +301,8 @@ TEST(repro_language_capability_ledger_covers_every_enum) {
|
||||
}
|
||||
}
|
||||
|
||||
if (counts[CAP_CALL_WITH_REFERENCE_VOCAB] != 87 ||
|
||||
counts[CAP_CALL_WITHOUT_REFERENCE_VOCAB] != 27 || counts[CAP_NO_CALL] != 50 ||
|
||||
if (counts[CAP_CALL_WITH_REFERENCE_VOCAB] != 88 ||
|
||||
counts[CAP_CALL_WITHOUT_REFERENCE_VOCAB] != 27 || counts[CAP_NO_CALL] != 49 ||
|
||||
counts[CAP_TRANSFORM_ONLY] != 1 || counts[CAP_UNSUPPORTED] != 1) {
|
||||
fprintf(stderr,
|
||||
" [language-registry] invariant=capability_partition call_ref_vocab=%d ref_gap=%d "
|
||||
@@ -319,9 +319,9 @@ TEST(repro_language_capability_ledger_covers_every_enum) {
|
||||
TEST(repro_call_argument_matrices_equal_call_capability_ledger) {
|
||||
enum {
|
||||
EXPECTED_MATRIX_A_ROWS = 68,
|
||||
EXPECTED_MATRIX_B_ROWS = 48,
|
||||
EXPECTED_CALL_CAPABLE_LANGUAGES = 114,
|
||||
EXPECTED_NON_CALL_LANGUAGES = 52,
|
||||
EXPECTED_MATRIX_B_ROWS = 49,
|
||||
EXPECTED_CALL_CAPABLE_LANGUAGES = 115,
|
||||
EXPECTED_NON_CALL_LANGUAGES = 51,
|
||||
EXPECTED_NON_CALL_DOMAIN_CONTROLS = 2,
|
||||
};
|
||||
CBMLanguage matrix_a_ids[CBM_LANG_COUNT];
|
||||
|
||||
+157
-6
@@ -15250,12 +15250,9 @@ TEST(clsp_tier2_shared_registry_readonly_c) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Direct guard for the finalize-time short-name / embedded-type indexes and their
|
||||
* iterators (type_registry.c), which the Rust trait/free-func fast paths rely on.
|
||||
* Verifies: embed index yields exactly the types whose embedded_types carry a
|
||||
* matching BARE name, in ascending registry order, deduped when a type lists the
|
||||
* same bare twice; free-func index yields only free funcs (receiver_type==NULL) with
|
||||
* the given short_name, not methods. */
|
||||
/* Direct guard for the type-name / embedded-type / free-function registry
|
||||
* indexes and their iterators (type_registry.c). Verifies that every iterator
|
||||
* preserves ascending registry order, which is part of resolver tie-breaking. */
|
||||
TEST(registry_short_name_indexes) {
|
||||
CBMArena arena;
|
||||
cbm_arena_init(&arena);
|
||||
@@ -15288,6 +15285,25 @@ TEST(registry_short_name_indexes) {
|
||||
t.short_name = "C";
|
||||
t.embedded_types = c_emb;
|
||||
cbm_registry_add_type(®, t);
|
||||
memset(&t, 0, sizeof(t));
|
||||
t.qualified_name = "other.Trait";
|
||||
t.short_name = "Trait";
|
||||
cbm_registry_add_type(®, t);
|
||||
memset(&t, 0, sizeof(t));
|
||||
t.qualified_name = "Trait";
|
||||
t.short_name = "Trait";
|
||||
cbm_registry_add_type(®, t);
|
||||
|
||||
/* Populate enough distinct names to force many hash-bucket collisions, with
|
||||
* repeated short names spread across the registry. The differential check
|
||||
* below compares the index against the exact former linear-scan order. */
|
||||
for (int i = 0; i < 512; i++) {
|
||||
memset(&t, 0, sizeof(t));
|
||||
t.qualified_name = cbm_arena_sprintf(&arena, "bulk.mod%d.Name%d", i, i % 23);
|
||||
ASSERT_NOT_NULL(t.qualified_name);
|
||||
t.short_name = strrchr(t.qualified_name, '.') + 1;
|
||||
cbm_registry_add_type(®, t);
|
||||
}
|
||||
|
||||
/* free func "helper" (x2 — different QNs), method "M.helper", free func "other". */
|
||||
CBMRegisteredFunc f;
|
||||
@@ -15310,6 +15326,77 @@ TEST(registry_short_name_indexes) {
|
||||
cbm_registry_add_func(®, f);
|
||||
|
||||
cbm_registry_finalize(®);
|
||||
cbm_registry_build_type_short_index(®);
|
||||
|
||||
/* Qualified-name bare segment "Trait": types 0, 4, then the bare-QN type
|
||||
* 5. The iterator is a hash prefilter, so apply the exact predicate before
|
||||
* asserting its order just like production consumers do. */
|
||||
CBMTypeNameIter nit;
|
||||
cbm_registry_types_by_short_name(®, "Trait", &nit);
|
||||
{
|
||||
int expected[] = {0, 4, 5};
|
||||
int exact_count = 0;
|
||||
int candidate;
|
||||
while ((candidate = cbm_type_name_iter_next(&nit)) >= 0) {
|
||||
const char *qn = reg.types[candidate].qualified_name;
|
||||
const char *last_dot = qn ? strrchr(qn, '.') : NULL;
|
||||
const char *candidate_short = last_dot ? last_dot + 1 : qn;
|
||||
if (!candidate_short || strcmp(candidate_short, "Trait") != 0)
|
||||
continue;
|
||||
ASSERT_TRUE(exact_count < 3);
|
||||
ASSERT_EQ(candidate, expected[exact_count++]);
|
||||
}
|
||||
ASSERT_EQ(exact_count, 3);
|
||||
}
|
||||
cbm_registry_types_by_short_name(®, "Missing", &nit);
|
||||
{
|
||||
int exact_count = 0;
|
||||
int candidate;
|
||||
while ((candidate = cbm_type_name_iter_next(&nit)) >= 0) {
|
||||
const char *qn = reg.types[candidate].qualified_name;
|
||||
const char *last_dot = qn ? strrchr(qn, '.') : NULL;
|
||||
const char *candidate_short = last_dot ? last_dot + 1 : qn;
|
||||
if (candidate_short && strcmp(candidate_short, "Missing") == 0)
|
||||
exact_count++;
|
||||
}
|
||||
ASSERT_EQ(exact_count, 0);
|
||||
}
|
||||
|
||||
for (int name_i = 0; name_i < 23; name_i++) {
|
||||
char short_name[32];
|
||||
snprintf(short_name, sizeof(short_name), "Name%d", name_i);
|
||||
cbm_registry_types_by_short_name(®, short_name, &nit);
|
||||
int linear_i = 0;
|
||||
for (;;) {
|
||||
int expected = -1;
|
||||
while (linear_i < reg.type_count) {
|
||||
int candidate = linear_i++;
|
||||
const char *qn = reg.types[candidate].qualified_name;
|
||||
const char *last_dot = qn ? strrchr(qn, '.') : NULL;
|
||||
const char *candidate_short = last_dot ? last_dot + 1 : qn;
|
||||
if (candidate_short && strcmp(candidate_short, short_name) == 0) {
|
||||
expected = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
int actual;
|
||||
for (;;) {
|
||||
actual = cbm_type_name_iter_next(&nit);
|
||||
if (actual < 0)
|
||||
break;
|
||||
const char *qn = reg.types[actual].qualified_name;
|
||||
const char *last_dot = qn ? strrchr(qn, '.') : NULL;
|
||||
const char *candidate_short = last_dot ? last_dot + 1 : qn;
|
||||
if (candidate_short && strcmp(candidate_short, short_name) == 0)
|
||||
break;
|
||||
}
|
||||
|
||||
ASSERT_EQ(actual, expected);
|
||||
if (expected < 0)
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* Embed index for bare "Trait": types 1 (A) then 2 (B), ascending, B once. */
|
||||
CBMTypeEmbedIter it;
|
||||
@@ -15339,6 +15426,70 @@ TEST(registry_short_name_indexes) {
|
||||
ASSERT_EQ(cbm_free_func_iter_next(&fit), 3);
|
||||
ASSERT_EQ(cbm_free_func_iter_next(&fit), -1);
|
||||
|
||||
/* A type appended after finalize is covered by the iterator tail. If the
|
||||
* auxiliary allocation is unavailable, the iterator falls back to the same
|
||||
* complete linear scan rather than silently dropping candidates. Keep this
|
||||
* after the other iterator assertions because their tails intentionally
|
||||
* expose post-finalize candidates for caller-side exact filtering too. */
|
||||
int tail_type_i = reg.type_count;
|
||||
memset(&t, 0, sizeof(t));
|
||||
t.qualified_name = "tail.Trait";
|
||||
t.short_name = "Trait";
|
||||
cbm_registry_add_type(®, t);
|
||||
cbm_registry_types_by_short_name(®, "Trait", &nit);
|
||||
ASSERT_EQ(cbm_type_name_iter_next(&nit), 0);
|
||||
ASSERT_EQ(cbm_type_name_iter_next(&nit), 4);
|
||||
ASSERT_EQ(cbm_type_name_iter_next(&nit), 5);
|
||||
ASSERT_EQ(cbm_type_name_iter_next(&nit), tail_type_i);
|
||||
ASSERT_EQ(cbm_type_name_iter_next(&nit), -1);
|
||||
|
||||
int *saved_type_short_buckets = reg.type_short_buckets;
|
||||
reg.type_short_buckets = NULL;
|
||||
cbm_registry_types_by_short_name(®, "Trait", &nit);
|
||||
int fallback_expected[] = {0, 4, 5, tail_type_i};
|
||||
int fallback_count = 0;
|
||||
int candidate;
|
||||
while ((candidate = cbm_type_name_iter_next(&nit)) >= 0) {
|
||||
const char *qn = reg.types[candidate].qualified_name;
|
||||
const char *last_dot = qn ? strrchr(qn, '.') : NULL;
|
||||
const char *candidate_short = last_dot ? last_dot + 1 : qn;
|
||||
if (!candidate_short || strcmp(candidate_short, "Trait") != 0)
|
||||
continue;
|
||||
ASSERT_TRUE(fallback_count < 4);
|
||||
ASSERT_EQ(candidate, fallback_expected[fallback_count++]);
|
||||
}
|
||||
ASSERT_EQ(fallback_count, 4);
|
||||
reg.type_short_buckets = saved_type_short_buckets;
|
||||
|
||||
/* A failed rebuild must discard the previous auxiliary index. Re-finalize
|
||||
* after adding the tail type so the QN boundary advances, then force arena
|
||||
* exhaustion for the optional rebuild. The iterator must fall back to the
|
||||
* complete linear scan instead of retaining the stale pre-tail index. */
|
||||
cbm_registry_finalize(®);
|
||||
int saved_nblocks = arena.nblocks;
|
||||
size_t saved_used = arena.used;
|
||||
arena.nblocks = CBM_ARENA_MAX_BLOCKS;
|
||||
arena.used = arena.block_size;
|
||||
cbm_registry_build_type_short_index(®);
|
||||
arena.nblocks = saved_nblocks;
|
||||
arena.used = saved_used;
|
||||
ASSERT_NULL(reg.type_short_buckets);
|
||||
ASSERT_NULL(reg.type_short_entries);
|
||||
ASSERT_EQ(reg.type_short_bucket_count, 0);
|
||||
|
||||
cbm_registry_types_by_short_name(®, "Trait", &nit);
|
||||
fallback_count = 0;
|
||||
while ((candidate = cbm_type_name_iter_next(&nit)) >= 0) {
|
||||
const char *qn = reg.types[candidate].qualified_name;
|
||||
const char *last_dot = qn ? strrchr(qn, '.') : NULL;
|
||||
const char *candidate_short = last_dot ? last_dot + 1 : qn;
|
||||
if (!candidate_short || strcmp(candidate_short, "Trait") != 0)
|
||||
continue;
|
||||
ASSERT_TRUE(fallback_count < 4);
|
||||
ASSERT_EQ(candidate, fallback_expected[fallback_count++]);
|
||||
}
|
||||
ASSERT_EQ(fallback_count, 4);
|
||||
|
||||
cbm_arena_destroy(&arena);
|
||||
PASS();
|
||||
}
|
||||
|
||||
@@ -1285,6 +1285,99 @@ TEST(golsp_crossfile_stdlib_interface) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Cross-package receiver-method resolution when struct field type texts are
|
||||
* written through import aliases. parse_field_defs_into_type qualifies the
|
||||
* text with the defining module ("Svc:svc.Svc" in module test.main becomes
|
||||
* "test.main.svc.Svc") — a QN that exists nowhere in the project-wide
|
||||
* registry, so the dispatch must re-qualify the alias segment through the
|
||||
* calling file's import map and land on the real receiver type. Mirrors a
|
||||
* common cross-module shape: a service struct field typed from an sdk module
|
||||
* and an interface-typed client field from an api module. */
|
||||
TEST(golsp_crossfile_aliased_field_requal) {
|
||||
const char *source = "package main\n\n"
|
||||
"func callSvc(h *Handler) error {\n"
|
||||
"\th.Svc.Ping()\n\treturn nil\n}\n\n"
|
||||
"func callPb(h *Holder) error {\n"
|
||||
"\th.C.Ping()\n\treturn nil\n}\n";
|
||||
|
||||
CBMLSPDef defs[] = {
|
||||
/* myapp/svc — concrete service struct */
|
||||
{.qualified_name = "myapp/svc.Svc",
|
||||
.short_name = "Svc",
|
||||
.label = "Struct",
|
||||
.def_module_qn = "myapp/svc"},
|
||||
/* test.main — receiver structs; field texts use import aliases, the
|
||||
* trigger for the wrong qualification */
|
||||
{.qualified_name = "test.main.Handler",
|
||||
.short_name = "Handler",
|
||||
.label = "Struct",
|
||||
.def_module_qn = "test.main",
|
||||
.field_defs = "Svc:svc.Svc"},
|
||||
{.qualified_name = "test.main.Holder",
|
||||
.short_name = "Holder",
|
||||
.label = "Struct",
|
||||
.def_module_qn = "test.main",
|
||||
.field_defs = "C:pb.Client"},
|
||||
/* myapp/pb — client interface */
|
||||
{.qualified_name = "myapp/pb.Client",
|
||||
.short_name = "Client",
|
||||
.label = "Interface",
|
||||
.def_module_qn = "myapp/pb",
|
||||
.is_interface = true,
|
||||
.method_names_str = "Ping"},
|
||||
/* methods, after their receiver types (extraction order) */
|
||||
{.qualified_name = "myapp/svc.Svc.Ping",
|
||||
.short_name = "Ping",
|
||||
.label = "Method",
|
||||
.def_module_qn = "myapp/svc",
|
||||
.receiver_type = "myapp/svc.Svc",
|
||||
.return_types = "error"},
|
||||
{.qualified_name = "myapp/pb.Client.Ping",
|
||||
.short_name = "Ping",
|
||||
.label = "Method",
|
||||
.def_module_qn = "myapp/pb",
|
||||
.receiver_type = "myapp/pb.Client",
|
||||
.return_types = "error"},
|
||||
};
|
||||
const char *imp_names[] = {"svc", "pb"};
|
||||
const char *imp_qns[] = {"myapp/svc", "myapp/pb"};
|
||||
|
||||
CBMArena arena;
|
||||
cbm_arena_init(&arena);
|
||||
CBMResolvedCallArray out = {0};
|
||||
|
||||
CBMTypeRegistry *reg = cbm_go_build_cross_registry(&arena, defs, 6);
|
||||
ASSERT_NOT_NULL(reg);
|
||||
|
||||
cbm_run_go_lsp_cross_with_registry(&arena, source, (int)strlen(source), "test.main", reg,
|
||||
imp_names, imp_qns, 2, NULL, &out);
|
||||
|
||||
int svc_idx = find_resolved_arr_confident(&out, "callSvc", "Svc.Ping");
|
||||
if (svc_idx < 0) {
|
||||
printf(" cross-registry diagnostics (%d records):\n", out.count);
|
||||
for (int i = 0; i < out.count; i++) {
|
||||
const CBMResolvedCall *rc = &out.items[i];
|
||||
printf(" %s -> %s [%s %.2f]\n", rc->caller_qn ? rc->caller_qn : "(null)",
|
||||
rc->callee_qn ? rc->callee_qn : "(null)",
|
||||
rc->strategy ? rc->strategy : "(null)", rc->confidence);
|
||||
}
|
||||
}
|
||||
ASSERT_GTE(svc_idx, 0);
|
||||
ASSERT_STR_EQ(out.items[svc_idx].callee_qn, "myapp/svc.Svc.Ping");
|
||||
ASSERT_STR_EQ(out.items[svc_idx].strategy, "lsp_type_dispatch");
|
||||
ASSERT_TRUE(out.items[svc_idx].confidence >= 0.9f);
|
||||
|
||||
int pb_idx = find_resolved_arr_confident(&out, "callPb", "Client.Ping");
|
||||
ASSERT_GTE(pb_idx, 0);
|
||||
ASSERT_STR_EQ(out.items[pb_idx].callee_qn, "myapp/pb.Client.Ping");
|
||||
ASSERT_TRUE(strcmp(out.items[pb_idx].strategy, "lsp_interface_dispatch") == 0 ||
|
||||
strcmp(out.items[pb_idx].strategy, "lsp_type_dispatch") == 0);
|
||||
ASSERT_TRUE(out.items[pb_idx].confidence >= 0.8f);
|
||||
|
||||
cbm_arena_destroy(&arena);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(golsp_crossfile_local_interface_single_impl) {
|
||||
const char *source =
|
||||
"package main\n\n"
|
||||
@@ -1448,6 +1541,7 @@ SUITE(go_lsp) {
|
||||
RUN_TEST(golsp_crossfile_return_type_chain);
|
||||
RUN_TEST(golsp_crossfile_interface_dispatch);
|
||||
RUN_TEST(golsp_crossfile_interface_field_chain);
|
||||
RUN_TEST(golsp_crossfile_aliased_field_requal);
|
||||
RUN_TEST(golsp_crossfile_map_index);
|
||||
RUN_TEST(golsp_crossfile_stdlib_interface);
|
||||
RUN_TEST(golsp_crossfile_local_interface_single_impl);
|
||||
|
||||
@@ -26,6 +26,7 @@ TEST(infrascan_http_route_literal_guard_rejects_filesystem_paths) {
|
||||
ASSERT_FALSE(cbm_service_pattern_is_http_route_literal("/var/run/app.json", "requests.get"));
|
||||
ASSERT_FALSE(cbm_service_pattern_is_http_route_literal("/locations/", "str.split"));
|
||||
ASSERT_FALSE(cbm_service_pattern_is_http_route_literal("/api", "os.path.join"));
|
||||
ASSERT_FALSE(cbm_service_pattern_is_http_route_literal("/html/g", "template.replace"));
|
||||
ASSERT_FALSE(cbm_service_pattern_is_http_route_literal(NULL, "requests.get"));
|
||||
ASSERT_FALSE(cbm_service_pattern_is_http_route_literal("", "requests.get"));
|
||||
ASSERT_TRUE(cbm_service_pattern_is_http_route_literal("/api/orders", "requests.get"));
|
||||
|
||||
+182
-14
@@ -851,6 +851,7 @@ TEST(mcp_tools_list) {
|
||||
ASSERT_NOT_NULL(strstr(json, "query_graph"));
|
||||
ASSERT_NOT_NULL(strstr(json, "trace_path"));
|
||||
ASSERT_NOT_NULL(strstr(json, "get_code_snippet"));
|
||||
ASSERT_NOT_NULL(strstr(json, "get_file_outline"));
|
||||
ASSERT_NOT_NULL(strstr(json, "get_graph_schema"));
|
||||
ASSERT_NOT_NULL(strstr(json, "get_architecture"));
|
||||
ASSERT_NOT_NULL(strstr(json, "search_code"));
|
||||
@@ -905,6 +906,7 @@ TEST(mcp_tools_list_latest_metadata) {
|
||||
ASSERT_NOT_NULL(strstr(json, "\"title\":\"Search graph\""));
|
||||
ASSERT_NOT_NULL(strstr(json, "\"title\":\"Index repository\""));
|
||||
ASSERT_NOT_NULL(strstr(json, "\"title\":\"Check index coverage\""));
|
||||
ASSERT_NOT_NULL(strstr(json, "\"title\":\"Get file outline\""));
|
||||
/* No tool may declare an outputSchema. The blanket permissive schema
|
||||
* ({"type":"object","additionalProperties":true}) carried zero information
|
||||
* for clients, but its presence made spec-compliant clients read
|
||||
@@ -939,6 +941,7 @@ TEST(mcp_tools_have_behavior_annotations) {
|
||||
{"query_graph", false, true, true, false},
|
||||
{"trace_path", false, true, true, false},
|
||||
{"get_code_snippet", false, true, true, false},
|
||||
{"get_file_outline", false, true, true, false},
|
||||
{"get_graph_schema", false, true, true, false},
|
||||
{"compare_graphs", true, false, true, false},
|
||||
{"get_architecture", false, true, true, false},
|
||||
@@ -1336,9 +1339,9 @@ static int issue403_initialize_count_calls(const char *session_root, bool approv
|
||||
cbm_setenv("CBM_CACHE_DIR", cache, 1);
|
||||
|
||||
char err[1024];
|
||||
bool approved = !approve_sensitive ||
|
||||
cbm_workspace_grant_add(cache, cbm_workspace_home_dir(), session_root, true,
|
||||
err, sizeof(err));
|
||||
bool approved =
|
||||
!approve_sensitive || cbm_workspace_grant_add(cache, cbm_workspace_home_dir(), session_root,
|
||||
true, err, sizeof(err));
|
||||
cbm_config_t *cfg = approved ? cbm_config_open(cache) : NULL;
|
||||
cbm_mcp_server_t *srv = cfg ? cbm_mcp_server_new(NULL) : NULL;
|
||||
int calls = -2;
|
||||
@@ -1368,8 +1371,8 @@ static int issue403_initialize_count_calls(const char *session_root, bool approv
|
||||
}
|
||||
|
||||
TEST(mcp_issue403_sensitive_root_stops_before_discovery_count) {
|
||||
int sensitive = issue403_initialize_count_calls(
|
||||
"C:/Users/dev/AppData/Local/Programs/Antigravity", false);
|
||||
int sensitive =
|
||||
issue403_initialize_count_calls("C:/Users/dev/AppData/Local/Programs/Antigravity", false);
|
||||
int ordinary = issue403_initialize_count_calls("C:/Users/dev/projects/app", false);
|
||||
ASSERT_EQ(sensitive, 0);
|
||||
ASSERT_EQ(ordinary, 1);
|
||||
@@ -1434,13 +1437,37 @@ TEST(server_handle_tools_list_defaults_to_all_tools_and_accepts_cursor) {
|
||||
ASSERT_NOT_NULL(strstr(resp, "ingest_traces"));
|
||||
free(resp);
|
||||
|
||||
resp = cbm_mcp_server_handle(
|
||||
srv,
|
||||
"{\"jsonrpc\":\"2.0\",\"id\":201,\"method\":\"tools/list\",\"params\":{\"cursor\":\"8\"}}");
|
||||
/* A cursored page advertises nextCursor exactly while tools remain after
|
||||
* it. This used to pass the literal cursor "8", which silently encoded
|
||||
* "there are at most MCP_TOOLS_PAGE_SIZE * 2 tools" -- so registering a
|
||||
* 17th tool broke it for a reason that had nothing to do with pagination.
|
||||
* Derive the offset from the live count instead: the final page, wherever
|
||||
* it falls, is the one that must not advertise more. */
|
||||
resp = cbm_mcp_server_handle(srv, "{\"jsonrpc\":\"2.0\",\"id\":203,\"method\":\"tools/list\"}");
|
||||
ASSERT_NOT_NULL(resp);
|
||||
size_t total_tools = mcp_response_tool_count(resp);
|
||||
free(resp);
|
||||
ASSERT_TRUE(total_tools > 1U);
|
||||
|
||||
char last_page_req[160];
|
||||
snprintf(last_page_req, sizeof(last_page_req),
|
||||
"{\"jsonrpc\":\"2.0\",\"id\":201,\"method\":\"tools/list\","
|
||||
"\"params\":{\"cursor\":\"%zu\"}}",
|
||||
total_tools - 1U);
|
||||
resp = cbm_mcp_server_handle(srv, last_page_req);
|
||||
ASSERT_NOT_NULL(resp);
|
||||
ASSERT_NOT_NULL(strstr(resp, "\"id\":201"));
|
||||
ASSERT_NULL(strstr(resp, "\"nextCursor\""));
|
||||
ASSERT_NOT_NULL(strstr(resp, "manage_adr"));
|
||||
ASSERT_EQ(mcp_response_tool_count(resp), 1U);
|
||||
free(resp);
|
||||
|
||||
/* ...and a page that does have tools after it MUST advertise the cursor,
|
||||
* so the assertion above cannot pass merely because paging never emits. */
|
||||
resp = cbm_mcp_server_handle(
|
||||
srv,
|
||||
"{\"jsonrpc\":\"2.0\",\"id\":204,\"method\":\"tools/list\",\"params\":{\"cursor\":\"0\"}}");
|
||||
ASSERT_NOT_NULL(resp);
|
||||
ASSERT_NOT_NULL(strstr(resp, "\"nextCursor\""));
|
||||
free(resp);
|
||||
|
||||
cbm_mcp_server_free(srv);
|
||||
@@ -1463,9 +1490,10 @@ TEST(server_handle_analysis_profile_filters_and_rejects_mutators) {
|
||||
resp = cbm_mcp_server_handle(srv, "{\"jsonrpc\":\"2.0\",\"id\":220,\"method\":\"tools/list\"}");
|
||||
ASSERT_NOT_NULL(resp);
|
||||
static const char *const analysis_tools[] = {
|
||||
"search_graph", "query_graph", "trace_path", "get_code_snippet",
|
||||
"get_graph_schema", "compare_graphs", "get_architecture", "search_code",
|
||||
"list_projects", "index_status", "check_index_coverage", "detect_changes",
|
||||
"search_graph", "query_graph", "trace_path", "get_code_snippet",
|
||||
"get_file_outline", "get_graph_schema", "compare_graphs", "get_architecture",
|
||||
"search_code", "list_projects", "index_status", "check_index_coverage",
|
||||
"detect_changes",
|
||||
};
|
||||
ASSERT_EQ(mcp_response_tool_count(resp), sizeof(analysis_tools) / sizeof(analysis_tools[0]));
|
||||
for (size_t i = 0U; i < sizeof(analysis_tools) / sizeof(analysis_tools[0]); i++) {
|
||||
@@ -1504,10 +1532,11 @@ TEST(server_handle_scout_profile_exposes_only_the_fast_tier) {
|
||||
|
||||
resp = cbm_mcp_server_handle(srv, "{\"jsonrpc\":\"2.0\",\"id\":223,\"method\":\"tools/list\"}");
|
||||
ASSERT_NOT_NULL(resp);
|
||||
ASSERT_EQ(mcp_response_tool_count(resp), 7U);
|
||||
ASSERT_EQ(mcp_response_tool_count(resp), 8U);
|
||||
ASSERT_TRUE(mcp_response_has_exact_tool(resp, "search_graph"));
|
||||
ASSERT_TRUE(mcp_response_has_exact_tool(resp, "trace_path"));
|
||||
ASSERT_TRUE(mcp_response_has_exact_tool(resp, "get_code_snippet"));
|
||||
ASSERT_TRUE(mcp_response_has_exact_tool(resp, "get_file_outline"));
|
||||
ASSERT_TRUE(mcp_response_has_exact_tool(resp, "get_architecture"));
|
||||
ASSERT_TRUE(mcp_response_has_exact_tool(resp, "list_projects"));
|
||||
ASSERT_TRUE(mcp_response_has_exact_tool(resp, "index_status"));
|
||||
@@ -1730,6 +1759,8 @@ static cbm_mcp_server_t *setup_mcp_with_data(void) {
|
||||
return srv;
|
||||
}
|
||||
|
||||
static char *extract_text_content(const char *mcp_result);
|
||||
|
||||
TEST(tool_list_projects_empty) {
|
||||
cbm_mcp_server_t *srv = setup_mcp_with_data();
|
||||
|
||||
@@ -1788,6 +1819,142 @@ TEST(tool_compare_graphs_registered_issue525) {
|
||||
ASSERT_NOT_NULL(resp);
|
||||
ASSERT_NULL(strstr(resp, "unknown tool: compare_graphs"));
|
||||
free(resp);
|
||||
cbm_mcp_server_free(srv);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(tool_get_file_outline_returns_bounded_filtered_columnar_rows_issue469) {
|
||||
cbm_mcp_server_t *srv = setup_mcp_with_data();
|
||||
cbm_store_t *store = cbm_mcp_server_store(srv);
|
||||
ASSERT_NOT_NULL(store);
|
||||
ASSERT_EQ(cbm_store_upsert_project(store, "outline-project", "/tmp/outline-project"),
|
||||
CBM_STORE_OK);
|
||||
cbm_mcp_server_set_project(srv, "outline-project");
|
||||
|
||||
cbm_node_t nodes[] = {
|
||||
{.project = "outline-project",
|
||||
.label = "Module",
|
||||
.name = "main",
|
||||
.qualified_name = "outline-project.main",
|
||||
.file_path = "src/main.c",
|
||||
.start_line = 1,
|
||||
.end_line = 80},
|
||||
{.project = "outline-project",
|
||||
.label = "Function",
|
||||
.name = "alpha",
|
||||
.qualified_name = "outline-project.src.main.alpha",
|
||||
.file_path = "src/main.c",
|
||||
.start_line = 10,
|
||||
.end_line = 14},
|
||||
{.project = "outline-project",
|
||||
.label = "Class",
|
||||
.name = "IgnoredClass",
|
||||
.qualified_name = "outline-project.src.main.IgnoredClass",
|
||||
.file_path = "src/main.c",
|
||||
.start_line = 15,
|
||||
.end_line = 40},
|
||||
{.project = "outline-project",
|
||||
.label = "Method",
|
||||
.name = "omega",
|
||||
.qualified_name = "outline-project.src.main.omega",
|
||||
.file_path = "src/main.c",
|
||||
.start_line = 30,
|
||||
.end_line = 33},
|
||||
{.project = "outline-project",
|
||||
.label = "Function",
|
||||
.name = "other",
|
||||
.qualified_name = "outline-project.src.other.other",
|
||||
.file_path = "src/other.c",
|
||||
.start_line = 1,
|
||||
.end_line = 2},
|
||||
};
|
||||
for (size_t i = 0; i < sizeof(nodes) / sizeof(nodes[0]); i++) {
|
||||
ASSERT_GT(cbm_store_upsert_node(store, &nodes[i]), 0);
|
||||
}
|
||||
|
||||
char *response =
|
||||
cbm_mcp_handle_tool(srv, "get_file_outline",
|
||||
"{\"project\":\"outline-project\",\"file_path\":\"src/main.c\","
|
||||
"\"labels\":[\"Function\",\"Method\"],\"limit\":1}");
|
||||
ASSERT_NOT_NULL(response);
|
||||
ASSERT_NOT_NULL(strstr(response, "cols"));
|
||||
ASSERT_NOT_NULL(strstr(response, "(cols: name label lines qn)"));
|
||||
ASSERT_NOT_NULL(strstr(response, "alpha"));
|
||||
ASSERT_NULL(strstr(response, "omega"));
|
||||
ASSERT_NULL(strstr(response, "IgnoredClass"));
|
||||
ASSERT_NOT_NULL(strstr(response, "total: 2"));
|
||||
ASSERT_NOT_NULL(strstr(response, "has_more: true"));
|
||||
ASSERT_NULL(strstr(response, "unknown tool"));
|
||||
free(response);
|
||||
|
||||
cbm_mcp_server_free(srv);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(tool_get_file_outline_validates_json_path_limit_and_cancel_issue469) {
|
||||
cbm_mcp_server_t *srv = setup_mcp_with_data();
|
||||
cbm_store_t *store = cbm_mcp_server_store(srv);
|
||||
ASSERT_NOT_NULL(store);
|
||||
ASSERT_EQ(cbm_store_upsert_project(store, "outline-controls", "/tmp/outline-controls"),
|
||||
CBM_STORE_OK);
|
||||
cbm_mcp_server_set_project(srv, "outline-controls");
|
||||
cbm_node_t node = {.project = "outline-controls",
|
||||
.label = "Function",
|
||||
.name = "bounded",
|
||||
.qualified_name = "outline-controls.src.main.bounded",
|
||||
.file_path = "src/main.c",
|
||||
.start_line = 7,
|
||||
.end_line = 9};
|
||||
ASSERT_GT(cbm_store_upsert_node(store, &node), 0);
|
||||
|
||||
char *response =
|
||||
cbm_mcp_handle_tool(srv, "get_file_outline",
|
||||
"{\"project\":\"outline-controls\",\"file_path\":\"../outside.c\"}");
|
||||
ASSERT_NOT_NULL(response);
|
||||
ASSERT_NOT_NULL(strstr(response, "repository-relative"));
|
||||
ASSERT_NOT_NULL(strstr(response, "isError"));
|
||||
free(response);
|
||||
|
||||
response = cbm_mcp_handle_tool(
|
||||
srv, "get_file_outline",
|
||||
"{\"project\":\"outline-controls\",\"file_path\":\"src/main.c\",\"limit\":201}");
|
||||
ASSERT_NOT_NULL(response);
|
||||
ASSERT_NOT_NULL(strstr(response, "between 1 and 200"));
|
||||
ASSERT_NOT_NULL(strstr(response, "isError"));
|
||||
free(response);
|
||||
|
||||
response = cbm_mcp_handle_tool(srv, "get_file_outline",
|
||||
"{\"project\":\"outline-controls\",\"file_path\":\"src/main.c\","
|
||||
"\"format\":\"json\"}");
|
||||
ASSERT_NOT_NULL(response);
|
||||
char *inner = extract_text_content(response);
|
||||
ASSERT_NOT_NULL(inner);
|
||||
yyjson_doc *doc = yyjson_read(inner, strlen(inner), 0);
|
||||
ASSERT_NOT_NULL(doc);
|
||||
yyjson_val *root = yyjson_doc_get_root(doc);
|
||||
ASSERT_TRUE(yyjson_is_arr(yyjson_obj_get(root, "cols")));
|
||||
ASSERT_TRUE(yyjson_is_arr(yyjson_obj_get(root, "rows")));
|
||||
ASSERT_EQ(yyjson_get_int(yyjson_obj_get(root, "total")), 1);
|
||||
yyjson_doc_free(doc);
|
||||
free(inner);
|
||||
free(response);
|
||||
|
||||
ASSERT_TRUE(cbm_mcp_server_request_scope_begin(srv));
|
||||
ASSERT_TRUE(cbm_mcp_server_cancel_active(srv));
|
||||
response = cbm_mcp_handle_tool(
|
||||
srv, "get_file_outline", "{\"project\":\"outline-controls\",\"file_path\":\"src/main.c\"}");
|
||||
ASSERT_NOT_NULL(response);
|
||||
ASSERT_NOT_NULL(strstr(response, "cancelled for this request"));
|
||||
ASSERT_NOT_NULL(strstr(response, "isError"));
|
||||
free(response);
|
||||
cbm_mcp_server_request_scope_end(srv);
|
||||
|
||||
response = cbm_mcp_handle_tool(
|
||||
srv, "get_file_outline", "{\"project\":\"outline-controls\",\"file_path\":\"src/main.c\"}");
|
||||
ASSERT_NOT_NULL(response);
|
||||
ASSERT_NOT_NULL(strstr(response, "bounded"));
|
||||
ASSERT_NULL(strstr(response, "cancelled"));
|
||||
free(response);
|
||||
|
||||
cbm_mcp_server_free(srv);
|
||||
PASS();
|
||||
@@ -1812,7 +1979,6 @@ TEST(tool_search_graph_basic) {
|
||||
/* Forward declarations for helpers defined later in this file */
|
||||
static cbm_mcp_server_t *setup_snippet_server(char *tmp_dir, size_t tmp_sz);
|
||||
static void cleanup_snippet_dir(const char *tmp_dir);
|
||||
static char *extract_text_content(const char *mcp_result);
|
||||
|
||||
TEST(tool_search_graph_semantic_only_skips_structural_results_issue1295) {
|
||||
cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL);
|
||||
@@ -13555,6 +13721,8 @@ SUITE(mcp) {
|
||||
RUN_TEST(tool_compare_graphs_enforces_encoded_budget_issue525);
|
||||
RUN_TEST(tool_compare_graphs_validation_and_scan_cap_are_atomic_issue525);
|
||||
RUN_TEST(tool_compare_graphs_cancel_and_readonly_handles_release_issue525);
|
||||
RUN_TEST(tool_get_file_outline_returns_bounded_filtered_columnar_rows_issue469);
|
||||
RUN_TEST(tool_get_file_outline_validates_json_path_limit_and_cancel_issue469);
|
||||
RUN_TEST(tool_search_graph_basic);
|
||||
RUN_TEST(tool_search_graph_semantic_only_skips_structural_results_issue1295);
|
||||
RUN_TEST(tool_trace_totals_respect_test_filter);
|
||||
|
||||
+14
-4
@@ -516,6 +516,11 @@ TEST(resolve_budget_override_when_total_unknown) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* CBM_MEM_BUDGET_MB is an aggregate ceiling the parent divides across job
|
||||
* slots. A lower explicit value still wins; a raise is clipped to the per-slot
|
||||
* share so N workers cannot oversubscribe the host (#1654). The source stays
|
||||
* CBM_MEM_BUDGET_MB so the clip is the user's aggregate, not a silent
|
||||
* daemon_worker_cap rewrite of a fraction-derived default. */
|
||||
TEST(resolve_budget_worker_cap_preserves_lower_user_override) {
|
||||
size_t total = 8192 * CBM_TEST_MB;
|
||||
size_t worker_cap = 16 * CBM_TEST_MB;
|
||||
@@ -524,10 +529,15 @@ TEST(resolve_budget_worker_cap_preserves_lower_user_override) {
|
||||
ASSERT_STR_EQ(lower.source, "CBM_MEM_BUDGET_MB");
|
||||
ASSERT_FALSE(lower.hard_capped);
|
||||
|
||||
cbm_mem_budget_t capped = cbm_mem_resolve_budget_capped(total, 0.5, "64", worker_cap);
|
||||
ASSERT_EQ(capped.budget, worker_cap);
|
||||
ASSERT_STR_EQ(capped.source, "daemon_worker_cap");
|
||||
ASSERT_TRUE(capped.hard_capped);
|
||||
cbm_mem_budget_t raised = cbm_mem_resolve_budget_capped(total, 0.5, "64", worker_cap);
|
||||
ASSERT_EQ(raised.budget, worker_cap);
|
||||
ASSERT_STR_EQ(raised.source, "CBM_MEM_BUDGET_MB");
|
||||
ASSERT_TRUE(raised.hard_capped);
|
||||
|
||||
cbm_mem_budget_t fraction = cbm_mem_resolve_budget_capped(total, 0.5, NULL, worker_cap);
|
||||
ASSERT_EQ(fraction.budget, worker_cap);
|
||||
ASSERT_STR_EQ(fraction.source, "daemon_worker_cap");
|
||||
ASSERT_TRUE(fraction.hard_capped);
|
||||
PASS();
|
||||
}
|
||||
|
||||
|
||||
@@ -3203,6 +3203,233 @@ TEST(parallel_python_lsp_override_emits_lsp_strategy_edges) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Go cross-package field-chain fixture (field_defs fold) ─────── */
|
||||
|
||||
/* Production's sequential driver seeds Folder nodes via pass_structure; the
|
||||
* compact harness (run_sequential_with_lsp_cross_*) does not. Go imports
|
||||
* resolve to Folder nodes, so a Go import-map fixture must seed File and
|
||||
* Folder nodes itself — replicating the production shape. */
|
||||
static cbm_gbuf_t *run_go_field_chain_sequential(const char *project, const char *repo_path,
|
||||
cbm_file_info_t *files, int file_count) {
|
||||
cbm_gbuf_t *gbuf = cbm_gbuf_new(project, repo_path);
|
||||
cbm_registry_t *reg = cbm_registry_new();
|
||||
CBMFileResult **cache = (CBMFileResult **)calloc((size_t)file_count, sizeof(CBMFileResult *));
|
||||
if (!gbuf || !reg || !cache) {
|
||||
cbm_gbuf_free(gbuf);
|
||||
cbm_registry_free(reg);
|
||||
free(cache);
|
||||
return NULL;
|
||||
}
|
||||
atomic_int cancelled;
|
||||
atomic_init(&cancelled, 0);
|
||||
cbm_pipeline_ctx_t ctx = {
|
||||
.project_name = project,
|
||||
.repo_path = repo_path,
|
||||
.gbuf = gbuf,
|
||||
.registry = reg,
|
||||
.cancelled = &cancelled,
|
||||
.result_cache = cache,
|
||||
};
|
||||
|
||||
seed_test_file_nodes(gbuf, project, files, file_count);
|
||||
char *svc_dir_qn = cbm_pipeline_fqn_folder(project, "svc");
|
||||
if (svc_dir_qn) {
|
||||
cbm_gbuf_upsert_node(gbuf, "Folder", "svc", svc_dir_qn, "svc", 0, 0, "{}");
|
||||
free(svc_dir_qn);
|
||||
}
|
||||
|
||||
cbm_init();
|
||||
cbm_pipeline_pass_definitions(&ctx, files, file_count);
|
||||
cbm_pipeline_pass_lsp_cross(&ctx, files, file_count, cache);
|
||||
cbm_pipeline_pass_calls(&ctx, files, file_count);
|
||||
cbm_pipeline_pass_usages(&ctx, files, file_count);
|
||||
cbm_pipeline_pass_semantic(&ctx, files, file_count);
|
||||
|
||||
/* CBM_GO_FIELD_DIAG dump. NOTE: resolved-call records may borrow QN
|
||||
* strings across file results (cross-file resolution in pass_lsp_cross),
|
||||
* so every result must stay alive while ANY is inspected. Print all
|
||||
* first, free all second. */
|
||||
if (getenv("CBM_GO_FIELD_DIAG")) {
|
||||
for (int i = 0; i < file_count; i++) {
|
||||
if (!cache[i]) {
|
||||
continue;
|
||||
}
|
||||
const CBMFileResult *r = cache[i];
|
||||
printf(" [diag] file %s defs=%d imports=%d calls=%d resolved=%d\n", files[i].rel_path,
|
||||
r->defs.count, r->imports.count, r->calls.count, r->resolved_calls.count);
|
||||
for (int j = 0; j < r->resolved_calls.count; j++) {
|
||||
const CBMResolvedCall *rc = &r->resolved_calls.items[j];
|
||||
printf(" [diag] rc caller=%s callee=%s strategy=%s conf=%.2f kind=%d span=[%u,%u)\n",
|
||||
rc->caller_qn ? rc->caller_qn : "?", rc->callee_qn ? rc->callee_qn : "?",
|
||||
rc->strategy ? rc->strategy : "?", rc->confidence, (int)rc->kind,
|
||||
(unsigned)rc->site_start_byte, (unsigned)rc->site_end_byte);
|
||||
}
|
||||
for (int j = 0; j < r->calls.count; j++) {
|
||||
const CBMCall *c = &r->calls.items[j];
|
||||
printf(" [diag] call callee=%s enclosing=%s span=[%u,%u) req=%d\n",
|
||||
c->callee_name ? c->callee_name : "?", c->enclosing_func_qn ? c->enclosing_func_qn : "?",
|
||||
(unsigned)c->site_start_byte, (unsigned)c->site_end_byte, (int)c->requires_lsp_resolution);
|
||||
}
|
||||
for (int j = 0; j < r->defs.count; j++) {
|
||||
const CBMDefinition *d = &r->defs.items[j];
|
||||
printf(" [diag] def label=%s qn=%s parent=%s ret=%s\n", d->label ? d->label : "?",
|
||||
d->qualified_name ? d->qualified_name : "?", d->parent_class ? d->parent_class : "?",
|
||||
d->return_type ? d->return_type : "?");
|
||||
}
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < file_count; i++) {
|
||||
cbm_free_result(cache[i]);
|
||||
}
|
||||
free(cache);
|
||||
harness_ctx_free_tables(&ctx);
|
||||
cbm_registry_free(reg);
|
||||
if (ctx.seq_cross_arena_live) {
|
||||
cbm_arena_destroy(&ctx.seq_cross_arena);
|
||||
ctx.seq_cross_arena_live = false;
|
||||
}
|
||||
if (ctx.seq_cross_def_modules) {
|
||||
for (int i = 0; i < ctx.seq_cross_def_module_count; i++) {
|
||||
free(ctx.seq_cross_def_modules[i]);
|
||||
}
|
||||
free(ctx.seq_cross_def_modules);
|
||||
ctx.seq_cross_def_modules = NULL;
|
||||
}
|
||||
return gbuf;
|
||||
}
|
||||
|
||||
/* Cross-package field chains (h.S.Ping()) resolve end to end through the
|
||||
* shared prebuilt Go registry. Regression for the field_defs fold: Go struct
|
||||
* fields are extracted as flat "Field" definitions that pxc_map_label drops,
|
||||
* so structs registered with zero fields and field chains never resolved.
|
||||
* Mirrors the real-world handler shape — an app package holds an aliased
|
||||
* cross-package field ("S *s.Svc") and app.Call calls through it. */
|
||||
TEST(parallel_go_cross_package_field_chain_resolves) {
|
||||
char tmpdir[256];
|
||||
snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_par_gofold_XXXXXX");
|
||||
if (!cbm_mkdtemp(tmpdir)) {
|
||||
FAIL("mkdtemp failed");
|
||||
}
|
||||
|
||||
char svc_path[512];
|
||||
char app_path[512];
|
||||
snprintf(svc_path, sizeof(svc_path), "%s/acme-order/internal/service/order_service.go", tmpdir);
|
||||
snprintf(app_path, sizeof(app_path), "%s/acme-order/internal/handler/order_handler.go", tmpdir);
|
||||
if (th_write_file(svc_path, "package service\n"
|
||||
"\n"
|
||||
"import (\n"
|
||||
" \"context\"\n"
|
||||
" pb \"acme-sdk/apis/order\"\n"
|
||||
")\n"
|
||||
"\n"
|
||||
"type OrderService struct{}\n"
|
||||
"\n"
|
||||
"func (s *OrderService) PlaceOrder(ctx context.Context, req *pb.PlaceOrderReq) (*pb.PlaceOrderRsp, error) {\n"
|
||||
" return nil, nil\n"
|
||||
"}\n"
|
||||
"\n"
|
||||
"func (s *OrderService) ListOrders(ctx context.Context, req *pb.ListOrdersReq) (*pb.ListOrdersRsp, error) {\n"
|
||||
" return nil, nil\n"
|
||||
"}\n") != 0 ||
|
||||
th_write_file(app_path, "package handler\n"
|
||||
"\n"
|
||||
"import (\n"
|
||||
" \"context\"\n"
|
||||
"\n"
|
||||
" pb \"acme-sdk/apis/order\"\n"
|
||||
"\n"
|
||||
" \"acme-order/internal/service\"\n"
|
||||
")\n"
|
||||
"\n"
|
||||
"type OrderHandler struct {\n"
|
||||
" pb.UnimplementedOrderServer\n"
|
||||
" cartSvc *service.CartService\n"
|
||||
" userSvc *service.UserService\n"
|
||||
" orderSvc *service.OrderService\n"
|
||||
" inventorySvc *service.InventoryService\n"
|
||||
" paymentSvc *service.PaymentService\n"
|
||||
" shipmentSvc *service.ShipmentService\n"
|
||||
" couponSvc *service.CouponService\n"
|
||||
" searchSvc *service.SearchService\n"
|
||||
"}\n"
|
||||
"\n"
|
||||
"func (h *OrderHandler) ListOrders(ctx context.Context, req *pb.ListOrdersReq) (*pb.ListOrdersRsp, error) {\n"
|
||||
" return h.orderSvc.ListOrders(ctx, req)\n"
|
||||
"}\n"
|
||||
"\n"
|
||||
"func (h *OrderHandler) PlaceOrder(ctx context.Context, req *pb.PlaceOrderReq) (*pb.PlaceOrderRsp, error) {\n"
|
||||
" return h.orderSvc.PlaceOrder(ctx, req)\n"
|
||||
"}\n"
|
||||
"\n"
|
||||
"func (h *OrderHandler) UpdateCart(ctx context.Context, req *pb.UpdateCartReq) (*pb.UpdateCartRsp, error) {\n"
|
||||
" return h.cartSvc.UpdateCart(ctx, req)\n"
|
||||
"}\n") != 0) {
|
||||
th_rmtree(tmpdir);
|
||||
FAIL("write fixture failed");
|
||||
}
|
||||
|
||||
cbm_file_info_t files[2] = {0};
|
||||
files[0].path = svc_path;
|
||||
files[0].rel_path = (char *)"acme-order/internal/service/order_service.go";
|
||||
files[0].language = CBM_LANG_GO;
|
||||
files[1].path = app_path;
|
||||
files[1].rel_path = (char *)"acme-order/internal/handler/order_handler.go";
|
||||
files[1].language = CBM_LANG_GO;
|
||||
|
||||
cbm_gbuf_t *gbuf = run_go_field_chain_sequential("go_field_fold", tmpdir, files, 2);
|
||||
ASSERT_NOT_NULL(gbuf);
|
||||
|
||||
const cbm_gbuf_edge_t *edge = find_call_edge_to_target_fragment(gbuf, "handler.PlaceOrder", ".service.PlaceOrder");
|
||||
const bool found = edge != NULL;
|
||||
const bool dispatch =
|
||||
edge && edge->properties_json &&
|
||||
strstr(edge->properties_json, "\"strategy\":\"lsp_type_dispatch\"");
|
||||
if (!found || !dispatch) {
|
||||
printf(" go field chain diagnostic: found=%d dispatch=%d\n", found, dispatch);
|
||||
if (edge && edge->properties_json) {
|
||||
printf(" go field chain props: %s\n", edge->properties_json);
|
||||
}
|
||||
/* Dump all callable nodes and CALLS edges for cross-referencing */
|
||||
{
|
||||
const cbm_gbuf_node_t **nodes = NULL;
|
||||
int ncount = 0;
|
||||
if (cbm_gbuf_find_by_label(gbuf, "Function", &nodes, &ncount) == 0) {
|
||||
for (int i = 0; i < ncount; i++) {
|
||||
printf(" node Function: %s\n", nodes[i]->qualified_name);
|
||||
}
|
||||
}
|
||||
if (cbm_gbuf_find_by_label(gbuf, "Method", &nodes, &ncount) == 0) {
|
||||
for (int i = 0; i < ncount; i++) {
|
||||
printf(" node Method: %s\n", nodes[i]->qualified_name);
|
||||
}
|
||||
}
|
||||
cbm_gbuf_edge_visitor_fn edge_dump = NULL;
|
||||
(void)edge_dump;
|
||||
/* print every CALLS edge with endpoints */
|
||||
const cbm_gbuf_edge_t **all = NULL;
|
||||
int ecount = 0;
|
||||
if (cbm_gbuf_find_edges_by_type(gbuf, "CALLS", &all, &ecount) == 0) {
|
||||
for (int i = 0; i < ecount; i++) {
|
||||
const cbm_gbuf_node_t *src =
|
||||
all[i] ? cbm_gbuf_find_by_id(gbuf, all[i]->source_id) : NULL;
|
||||
const cbm_gbuf_node_t *dst =
|
||||
all[i] ? cbm_gbuf_find_by_id(gbuf, all[i]->target_id) : NULL;
|
||||
printf(" CALLS edge: %s -> %s props=%s\n",
|
||||
src ? src->qualified_name : "?",
|
||||
dst ? dst->qualified_name : "?",
|
||||
all[i]->properties_json ? all[i]->properties_json : "{}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
cbm_gbuf_free(gbuf);
|
||||
th_rmtree(tmpdir);
|
||||
|
||||
ASSERT_TRUE(found);
|
||||
ASSERT_TRUE(dispatch);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Cross-file regression for the QN-mismatch bug: py_lsp's per-file mode
|
||||
* emits resolved_calls.callee_qn as the raw import-module path (e.g.
|
||||
* `greeter.Greeter` from `from greeter import Greeter`) rather than the
|
||||
@@ -4108,6 +4335,7 @@ SUITE(parallel) {
|
||||
RUN_TEST(parallel_cpp_preprocessed_coordinate_collision_preserves_hidden_target);
|
||||
RUN_TEST(parallel_cuda_preprocessed_coordinate_collision_preserves_hidden_target);
|
||||
RUN_TEST(parallel_python_lsp_override_cross_file_emits_lsp_strategy_edges);
|
||||
RUN_TEST(parallel_go_cross_package_field_chain_resolves);
|
||||
RUN_TEST(parallel_cross_file_reread_preserves_unretained_edges);
|
||||
RUN_TEST(parallel_java_kotlin_lsp_override_cross_file_emits_lsp_strategy_edges);
|
||||
RUN_TEST(parallel_lsp_tail_match_fallbacks_gated_to_jvm);
|
||||
|
||||
@@ -104,6 +104,22 @@ static const char *PY_CLEAN = "def ok():\n"
|
||||
"def ok2():\n"
|
||||
" return 2\n";
|
||||
|
||||
/* Perl formats have a line-oriented body terminated by a lone dot. The
|
||||
* following named sub pins the important recovery boundary: a grammar must
|
||||
* both accept the format and resume normal declaration parsing afterwards. */
|
||||
static const char *PERL_FORMAT_WITH_FOLLOWING_SUB = "package Report;\n"
|
||||
"format REPORT =\n"
|
||||
"@<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n"
|
||||
"$headline\n"
|
||||
".\n"
|
||||
"sub after_format { return 1; }\n";
|
||||
|
||||
/* A grammar refresh must not hide real Perl syntax loss. */
|
||||
static const char *PERL_MALFORMED = "package Broken;\n"
|
||||
"sub before_error { return 1; }\n"
|
||||
"} ] } ]\n"
|
||||
"sub after_error { return 2; }\n";
|
||||
|
||||
/* ── Tests ────────────────────────────────────────────────────────────────── */
|
||||
|
||||
TEST(c_ifdef_split_brace_sets_parse_incomplete) {
|
||||
@@ -418,6 +434,42 @@ TEST(width_bearing_error_at_eof_still_flagged_issue1610) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* #1838: tree-sitter-perl v1.0.0 rejects a valid line-oriented format and
|
||||
* reports the file as partial. The supported upstream v1.2.1 grammar accepts
|
||||
* the format and preserves declaration extraction beyond its dot terminator. */
|
||||
TEST(perl_format_followed_by_named_sub_is_complete_issue1838) {
|
||||
CBMFileResult *r = do_extract(PERL_FORMAT_WITH_FOLLOWING_SUB, CBM_LANG_PERL, "report.pl");
|
||||
ASSERT_NOT_NULL(r);
|
||||
ASSERT_FALSE(r->has_error);
|
||||
bool partial = r->parse_incomplete;
|
||||
int error_regions = r->error_region_count;
|
||||
bool has_error_ranges = r->error_ranges != NULL;
|
||||
bool has_following_sub = has_def(r, "after_format");
|
||||
if (partial || error_regions != 0 || has_error_ranges || !has_following_sub) {
|
||||
fprintf(stderr,
|
||||
" Perl format result: partial=%d regions=%d ranges=%s following_sub=%d\n",
|
||||
partial,
|
||||
error_regions,
|
||||
r->error_ranges ? r->error_ranges : "(none)",
|
||||
has_following_sub);
|
||||
cbm_free_result(r);
|
||||
FAIL("a valid Perl format and its following named sub must parse completely");
|
||||
}
|
||||
cbm_free_result(r);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(perl_malformed_source_remains_partial_issue1838) {
|
||||
CBMFileResult *r = do_extract(PERL_MALFORMED, CBM_LANG_PERL, "broken.pl");
|
||||
ASSERT_NOT_NULL(r);
|
||||
ASSERT_FALSE(r->has_error);
|
||||
ASSERT_TRUE(r->parse_incomplete);
|
||||
ASSERT_GTE(r->error_region_count, 1);
|
||||
ASSERT_NOT_NULL(r->error_ranges);
|
||||
cbm_free_result(r);
|
||||
PASS();
|
||||
}
|
||||
|
||||
SUITE(parse_coverage) {
|
||||
RUN_TEST(c_ifdef_split_brace_sets_parse_incomplete);
|
||||
RUN_TEST(c_ifdef_split_brace_neighbors_still_extracted);
|
||||
@@ -437,4 +489,6 @@ SUITE(parse_coverage) {
|
||||
RUN_TEST(missing_final_newline_not_flagged_across_grammars_issue1610);
|
||||
RUN_TEST(real_error_before_eof_still_flagged_without_final_newline_issue1610);
|
||||
RUN_TEST(width_bearing_error_at_eof_still_flagged_issue1610);
|
||||
RUN_TEST(perl_format_followed_by_named_sub_is_complete_issue1838);
|
||||
RUN_TEST(perl_malformed_source_remains_partial_issue1838);
|
||||
}
|
||||
|
||||
@@ -5352,6 +5352,70 @@ TEST(pipeline_parallel_rust_cross_only_macro_hidden_gets_synthetic_carrier) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Slash-prefixed call arguments are not necessarily HTTP routes. Keep the
|
||||
* parallel arg-url heuristic from minting Route nodes for filesystem paths or
|
||||
* regex-replacement operands, while preserving a genuine API path. */
|
||||
TEST(pipeline_arg_url_rejects_non_http_slash_arguments) {
|
||||
char tmp[256];
|
||||
snprintf(tmp, sizeof(tmp), "/tmp/cbm_arg_url_guard_XXXXXX");
|
||||
if (!cbm_mkdtemp(tmp)) {
|
||||
FAIL("tmpdir");
|
||||
}
|
||||
|
||||
write_temp_file(tmp, "src/args.py",
|
||||
"import requests\n"
|
||||
"TMP = '/tmp/pgv_fuzz.bin'\n"
|
||||
"def run_copy(path):\n"
|
||||
" return path\n"
|
||||
"def write_fixture():\n"
|
||||
" return run_copy(TMP)\n"
|
||||
"def load_api():\n"
|
||||
" return requests.get('/api/data')\n");
|
||||
write_temp_file(tmp, "src/regex.js",
|
||||
"function sink(value) { return value; }\n"
|
||||
"export function sanitize(template) {\n"
|
||||
" sink(/<table/i);\n"
|
||||
" return template.replace('/html/g', '');\n"
|
||||
"}\n");
|
||||
for (int i = 0; i < 52; i++) {
|
||||
char name[64];
|
||||
char body[128];
|
||||
snprintf(name, sizeof(name), "src/filler%d.ts", i);
|
||||
snprintf(body, sizeof(body), "export function filler%d(): number { return %d; }\n", i, i);
|
||||
write_temp_file(tmp, name, body);
|
||||
}
|
||||
|
||||
char *old_workers = getenv("CBM_WORKERS");
|
||||
char *saved = old_workers ? strdup(old_workers) : NULL;
|
||||
cbm_setenv("CBM_WORKERS", "4", 1);
|
||||
|
||||
char db_path[512];
|
||||
snprintf(db_path, sizeof(db_path), "%s/arg_url_guard.db", tmp);
|
||||
cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL);
|
||||
ASSERT_NOT_NULL(p);
|
||||
ASSERT_EQ(cbm_pipeline_run(p), 0);
|
||||
const char *project = cbm_pipeline_project_name(p);
|
||||
|
||||
cbm_store_t *s = cbm_store_open_path(db_path);
|
||||
ASSERT_NOT_NULL(s);
|
||||
|
||||
ASSERT_EQ(count_nodes_named(s, project, "/html/g"), 0);
|
||||
ASSERT_EQ(count_nodes_named(s, project, "/<table/i"), 0);
|
||||
ASSERT_EQ(count_nodes_named(s, project, "/tmp/pgv_fuzz.bin"), 0);
|
||||
ASSERT_GTE(count_nodes_named(s, project, "/api/data"), 1);
|
||||
|
||||
cbm_store_close(s);
|
||||
cbm_pipeline_free(p);
|
||||
if (saved) {
|
||||
cbm_setenv("CBM_WORKERS", saved, 1);
|
||||
free(saved);
|
||||
} else {
|
||||
cbm_unsetenv("CBM_WORKERS");
|
||||
}
|
||||
th_rmtree(tmp);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Native `fetch()` (#856), sequential path (< 50 files → pass_calls.c). A bare
|
||||
* unqualified call to the global fetch API has no import and no local
|
||||
* definition anywhere in this project, so registry resolution comes back
|
||||
@@ -13059,6 +13123,7 @@ SUITE(pipeline) {
|
||||
RUN_TEST(pipeline_python_receiver_parallel_suppresses_weak_method_edges);
|
||||
RUN_TEST(pipeline_parallel_python_cross_only_dunder_gets_synthetic_carrier);
|
||||
RUN_TEST(pipeline_parallel_rust_cross_only_macro_hidden_gets_synthetic_carrier);
|
||||
RUN_TEST(pipeline_arg_url_rejects_non_http_slash_arguments);
|
||||
RUN_TEST(pipeline_native_fetch_classified_as_http_calls);
|
||||
RUN_TEST(pipeline_native_fetch_parallel_classified_as_http_calls);
|
||||
RUN_TEST(pipeline_local_fetch_shadow_not_classified_as_http);
|
||||
|
||||
@@ -13,6 +13,17 @@
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
|
||||
typedef struct {
|
||||
int calls;
|
||||
int cancel_on_call;
|
||||
} file_outline_cancel_probe_t;
|
||||
|
||||
static bool file_outline_cancel_probe(void *context) {
|
||||
file_outline_cancel_probe_t *probe = context;
|
||||
probe->calls++;
|
||||
return probe->calls >= probe->cancel_on_call;
|
||||
}
|
||||
|
||||
/* ── Label allowlist / SQL drift guard ──────────────────────────── */
|
||||
|
||||
/* CONTRACT PIN. `cbm_label_is_type_like()` is documented in cbm.h as the single
|
||||
@@ -330,6 +341,156 @@ TEST(store_node_find_by_file) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(store_file_outline_is_filtered_stable_bounded_and_cancellable_issue469) {
|
||||
cbm_store_t *s = cbm_store_open_memory();
|
||||
ASSERT_NOT_NULL(s);
|
||||
ASSERT_EQ(cbm_store_upsert_project(s, "outline", "/tmp/outline"), CBM_STORE_OK);
|
||||
|
||||
cbm_node_t nodes[] = {
|
||||
{.project = "outline",
|
||||
.label = "Method",
|
||||
.name = "omega",
|
||||
.qualified_name = "outline.main.omega",
|
||||
.file_path = "main.c",
|
||||
.start_line = 30,
|
||||
.end_line = 33},
|
||||
{.project = "outline",
|
||||
.label = "Module",
|
||||
.name = "main",
|
||||
.qualified_name = "outline.main",
|
||||
.file_path = "main.c",
|
||||
.start_line = 1,
|
||||
.end_line = 80},
|
||||
{.project = "outline",
|
||||
.label = "Function",
|
||||
.name = "zeta",
|
||||
.qualified_name = "outline.main.zeta",
|
||||
.file_path = "main.c",
|
||||
.start_line = 10,
|
||||
.end_line = 20},
|
||||
{.project = "outline",
|
||||
.label = "Function",
|
||||
.name = "alpha",
|
||||
.qualified_name = "outline.main.alpha",
|
||||
.file_path = "main.c",
|
||||
.start_line = 10,
|
||||
.end_line = 15},
|
||||
{.project = "outline",
|
||||
.label = "Class",
|
||||
.name = "VisibleWithoutFilter",
|
||||
.qualified_name = "outline.main.VisibleWithoutFilter",
|
||||
.file_path = "main.c",
|
||||
.start_line = 5,
|
||||
.end_line = 40},
|
||||
{.project = "outline",
|
||||
.label = "Function",
|
||||
.name = "other",
|
||||
.qualified_name = "outline.other.other",
|
||||
.file_path = "other.c",
|
||||
.start_line = 1,
|
||||
.end_line = 2},
|
||||
};
|
||||
for (size_t i = 0; i < sizeof(nodes) / sizeof(nodes[0]); i++) {
|
||||
ASSERT_GT(cbm_store_upsert_node(s, &nodes[i]), 0);
|
||||
}
|
||||
|
||||
static const char *const labels[] = {"Function", "Method"};
|
||||
cbm_file_outline_row_t *rows = NULL;
|
||||
int count = 0;
|
||||
int total = 0;
|
||||
ASSERT_EQ(cbm_store_get_file_outline(s, "outline", "main.c", labels, 2, 2, 0, NULL, NULL, &rows,
|
||||
&count, &total),
|
||||
CBM_STORE_OK);
|
||||
ASSERT_EQ(total, 3);
|
||||
ASSERT_EQ(count, 2);
|
||||
ASSERT_STR_EQ(rows[0].name, "alpha");
|
||||
ASSERT_STR_EQ(rows[1].name, "zeta");
|
||||
cbm_store_free_file_outline(rows, count);
|
||||
|
||||
rows = NULL;
|
||||
count = 0;
|
||||
total = 0;
|
||||
ASSERT_EQ(cbm_store_get_file_outline(s, "outline", "main.c", labels, 2, 2, 2, NULL, NULL, &rows,
|
||||
&count, &total),
|
||||
CBM_STORE_OK);
|
||||
ASSERT_EQ(total, 3);
|
||||
ASSERT_EQ(count, 1);
|
||||
ASSERT_STR_EQ(rows[0].name, "omega");
|
||||
cbm_store_free_file_outline(rows, count);
|
||||
|
||||
rows = NULL;
|
||||
count = 0;
|
||||
total = 0;
|
||||
ASSERT_EQ(cbm_store_get_file_outline(s, "outline", "main.c", NULL, 0, 10, 0, NULL, NULL, &rows,
|
||||
&count, &total),
|
||||
CBM_STORE_OK);
|
||||
ASSERT_EQ(total, 4);
|
||||
ASSERT_EQ(count, 4);
|
||||
ASSERT_STR_EQ(rows[0].name, "VisibleWithoutFilter");
|
||||
ASSERT_STR_EQ(rows[1].name, "alpha");
|
||||
ASSERT_STR_EQ(rows[2].name, "zeta");
|
||||
ASSERT_STR_EQ(rows[3].name, "omega");
|
||||
cbm_store_free_file_outline(rows, count);
|
||||
|
||||
file_outline_cancel_probe_t probe = {.cancel_on_call = 3};
|
||||
rows = (cbm_file_outline_row_t *)(uintptr_t)1;
|
||||
count = 99;
|
||||
total = 99;
|
||||
ASSERT_EQ(cbm_store_get_file_outline(s, "outline", "main.c", labels, 2, 2, 0,
|
||||
file_outline_cancel_probe, &probe, &rows, &count, &total),
|
||||
CBM_STORE_CANCELLED);
|
||||
ASSERT_NULL(rows);
|
||||
ASSERT_EQ(count, 0);
|
||||
ASSERT_EQ(total, 0);
|
||||
ASSERT_TRUE(probe.calls >= 3);
|
||||
|
||||
const char *empty_label[] = {""};
|
||||
ASSERT_EQ(cbm_store_get_file_outline(s, "outline", "main.c", empty_label, 1, 1, 0, NULL, NULL,
|
||||
&rows, &count, &total),
|
||||
CBM_STORE_ERR);
|
||||
ASSERT_EQ(cbm_store_get_file_outline(s, "outline", "main.c", NULL, 0,
|
||||
CBM_STORE_FILE_OUTLINE_MAX_LIMIT + 1, 0, NULL, NULL, &rows,
|
||||
&count, &total),
|
||||
CBM_STORE_ERR);
|
||||
|
||||
cbm_store_close(s);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(store_file_outline_fails_closed_on_text_budget_issue469) {
|
||||
cbm_store_t *s = cbm_store_open_memory();
|
||||
ASSERT_NOT_NULL(s);
|
||||
ASSERT_EQ(cbm_store_upsert_project(s, "outline-budget", "/tmp/outline-budget"), CBM_STORE_OK);
|
||||
|
||||
size_t qn_size = CBM_STORE_FILE_OUTLINE_MAX_TEXT_BYTES + 64U;
|
||||
char *large_qn = malloc(qn_size + 1U);
|
||||
ASSERT_NOT_NULL(large_qn);
|
||||
memset(large_qn, 'q', qn_size);
|
||||
large_qn[qn_size] = '\0';
|
||||
cbm_node_t node = {.project = "outline-budget",
|
||||
.label = "Function",
|
||||
.name = "oversized",
|
||||
.qualified_name = large_qn,
|
||||
.file_path = "large.c",
|
||||
.start_line = 1,
|
||||
.end_line = 2};
|
||||
ASSERT_GT(cbm_store_upsert_node(s, &node), 0);
|
||||
free(large_qn);
|
||||
|
||||
cbm_file_outline_row_t *rows = (cbm_file_outline_row_t *)(uintptr_t)1;
|
||||
int count = 99;
|
||||
int total = 99;
|
||||
ASSERT_EQ(cbm_store_get_file_outline(s, "outline-budget", "large.c", NULL, 0, 1, 0, NULL, NULL,
|
||||
&rows, &count, &total),
|
||||
CBM_STORE_SCAN_LIMIT);
|
||||
ASSERT_NULL(rows);
|
||||
ASSERT_EQ(count, 0);
|
||||
ASSERT_EQ(total, 0);
|
||||
|
||||
cbm_store_close(s);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(store_node_find_not_found) {
|
||||
cbm_store_t *s = cbm_store_open_memory();
|
||||
cbm_store_upsert_project(s, "test", "/tmp/test");
|
||||
@@ -2227,6 +2388,8 @@ SUITE(store_nodes) {
|
||||
RUN_TEST(store_node_dedup);
|
||||
RUN_TEST(store_node_find_by_label);
|
||||
RUN_TEST(store_node_find_by_file);
|
||||
RUN_TEST(store_file_outline_is_filtered_stable_bounded_and_cancellable_issue469);
|
||||
RUN_TEST(store_file_outline_fails_closed_on_text_budget_issue469);
|
||||
RUN_TEST(store_node_find_not_found);
|
||||
RUN_TEST(store_node_count_empty);
|
||||
RUN_TEST(store_node_delete_by_file);
|
||||
|
||||
@@ -148,3 +148,34 @@ class McpServer:
|
||||
self.proc.kill()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def wait_projects_with_stats(server, timeout=90.0, poll=1.0):
|
||||
"""Poll list_projects until a project row carries node/edge counts.
|
||||
|
||||
index_repository returns when the pipeline finishes, but the project
|
||||
stats are published asynchronously; on slow CI runners the first
|
||||
list_projects can observe the registration row before its counts land
|
||||
(`nodes` absent/None). That window is environmental, not a product
|
||||
regression (#1952), so wait it out instead of failing setup on it.
|
||||
|
||||
Returns (projects, last_text): the parsed project list (possibly [])
|
||||
and the last raw list_projects payload for diagnostics.
|
||||
"""
|
||||
deadline = time.time() + timeout
|
||||
last_txt = ""
|
||||
while True:
|
||||
resp = server.call_tool("list_projects", {}, timeout=60)
|
||||
txt, err = server.tool_text(resp)
|
||||
projects = []
|
||||
if not err and txt:
|
||||
last_txt = txt
|
||||
try:
|
||||
projects = json.loads(txt).get("projects") or []
|
||||
except ValueError:
|
||||
projects = []
|
||||
if projects and projects[0].get("nodes") is not None:
|
||||
return projects, last_txt
|
||||
if time.time() >= deadline:
|
||||
return projects, last_txt
|
||||
time.sleep(poll)
|
||||
|
||||
@@ -31,6 +31,7 @@ import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
MATH_TS = (
|
||||
"export function add(a: number, b: number): number { return a + b; }\n"
|
||||
@@ -67,13 +68,24 @@ def main():
|
||||
# CLI path itself works and isolating the failure to argv encoding.
|
||||
ascii_repo = os.path.join(work, "ascii_repo")
|
||||
make_fixture(ascii_repo)
|
||||
env = dict(os.environ)
|
||||
env["CBM_CACHE_DIR"] = os.path.join(work, "cache_ascii")
|
||||
ctrl = subprocess.run(
|
||||
[binary, "cli", "index_repository",
|
||||
json.dumps({"repo_path": ascii_repo})],
|
||||
capture_output=True, timeout=120, env=env)
|
||||
ctrl_out = (ctrl.stdout or b"").decode("utf-8", "replace")
|
||||
# The control is setup, not the surface under test: a cold runner can
|
||||
# lose the first one-shot to coordination-daemon startup latency
|
||||
# (#1952). One retry against a fresh cache separates that from a real
|
||||
# CLI indexing failure.
|
||||
ctrl_out = ""
|
||||
for attempt in ("cache_ascii", "cache_ascii_retry"):
|
||||
env = dict(os.environ)
|
||||
env["CBM_CACHE_DIR"] = os.path.join(work, attempt)
|
||||
ctrl = subprocess.run(
|
||||
[binary, "cli", "index_repository",
|
||||
json.dumps({"repo_path": ascii_repo})],
|
||||
capture_output=True, timeout=120, env=env)
|
||||
ctrl_out = (ctrl.stdout or b"").decode("utf-8", "replace")
|
||||
if '"nodes"' in ctrl_out:
|
||||
break
|
||||
print("SETUP: ASCII control attempt %r did not index via CLI:\n%s"
|
||||
% (attempt, ctrl_out[:300]))
|
||||
time.sleep(2)
|
||||
if '"nodes"' not in ctrl_out:
|
||||
print("SETUP FAIL: ASCII control did not index via CLI:\n%s" %
|
||||
ctrl_out[:300])
|
||||
|
||||
@@ -31,6 +31,7 @@ import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
SYMBOL = "someIndexedSymbol"
|
||||
SRC = "export function %s(a: number): number { return a + 1; }\n" % SYMBOL
|
||||
@@ -63,9 +64,19 @@ def main():
|
||||
|
||||
# repo_path / cwd in the forward-slash drive form Claude Code passes.
|
||||
repo_fwd = repo.replace("\\", "/")
|
||||
idx = run_cli(binary, cache, ["cli", "index_repository",
|
||||
json.dumps({"repo_path": repo_fwd})])
|
||||
idx_out = (idx.stdout or b"").decode("utf-8", "replace")
|
||||
# Setup, not the surface under test: retry once so a cold runner's
|
||||
# coordination-daemon startup latency (#1952) is not misread as a
|
||||
# broken CLI index. Reindexing the same cache is idempotent.
|
||||
idx_out = ""
|
||||
for attempt in (1, 2):
|
||||
idx = run_cli(binary, cache, ["cli", "index_repository",
|
||||
json.dumps({"repo_path": repo_fwd})])
|
||||
idx_out = (idx.stdout or b"").decode("utf-8", "replace")
|
||||
if '"nodes"' in idx_out:
|
||||
break
|
||||
print("SETUP: index attempt %d did not run:\n%s"
|
||||
% (attempt, idx_out[:300]))
|
||||
time.sleep(2)
|
||||
if '"nodes"' not in idx_out:
|
||||
print("SETUP FAIL: index did not run:\n%s" % idx_out[:300])
|
||||
return 2
|
||||
@@ -88,10 +99,17 @@ def main():
|
||||
# first — the supported way to keep hooks armed outside MCP sessions —
|
||||
# and retire it afterwards (with a kill-by-pid backstop so a stuck stop
|
||||
# can never hang CI).
|
||||
start = run_cli(binary, cache, ["daemon", "start"], timeout=60)
|
||||
start_out = (start.stdout or b"").decode("utf-8", "replace")
|
||||
print("daemon start rc=%d %r" % (start.returncode, start_out[:120]))
|
||||
if start.returncode != 0:
|
||||
start_out = ""
|
||||
rc = 1
|
||||
for attempt in (1, 2):
|
||||
start = run_cli(binary, cache, ["daemon", "start"], timeout=60)
|
||||
start_out = (start.stdout or b"").decode("utf-8", "replace")
|
||||
rc = start.returncode
|
||||
print("daemon start attempt %d rc=%d %r" % (attempt, rc, start_out[:120]))
|
||||
if rc == 0:
|
||||
break
|
||||
time.sleep(2)
|
||||
if rc != 0:
|
||||
print("SETUP FAIL: permanent daemon did not start")
|
||||
return 2
|
||||
pid_match = re.search(r"pid (\d+)", start_out)
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
|
||||
Guards the fix for issue #636 / #357 (landed on main via #700) at the product
|
||||
surface (real codebase-memory-mcp process, real SQLite DB, real stdio). Two
|
||||
byte-identical TypeScript fixtures are indexed: one under an ASCII parent path,
|
||||
one under a non-ASCII parent path. The invariant under test:
|
||||
byte-identical polyglot fixtures (TypeScript + Go, #1959) are indexed: one under
|
||||
an ASCII parent path, one under a non-ASCII parent path. The invariant under test:
|
||||
|
||||
A byte-identical fixture must produce equivalent graph counts regardless of
|
||||
whether its absolute path contains non-ASCII characters.
|
||||
@@ -34,7 +34,7 @@ import sys
|
||||
import tempfile
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from mcp_stdio import McpServer # noqa: E402
|
||||
from mcp_stdio import McpServer, wait_projects_with_stats # noqa: E402
|
||||
|
||||
MATH_TS = (
|
||||
"export function add(a: number, b: number): number { return a + b; }\n"
|
||||
@@ -54,6 +54,81 @@ MAIN_TS = (
|
||||
"run();\n"
|
||||
)
|
||||
|
||||
# Dummy Go package covering the shapes the Go/cgo extraction work (#1932
|
||||
# family) showed were previously invisible to CI: struct fields and a method,
|
||||
# a build-tag twin pair sharing one function name, a cgo file (C preamble,
|
||||
# `C.` call, `//export` directive), and a channel producer/consumer pair
|
||||
# (#1959). Go qualified names embed the containing directory, so these also
|
||||
# put non-ASCII repo paths through the Go pipeline passes.
|
||||
GO_MOD = "module example.com/fixture\n\ngo 1.22\n"
|
||||
|
||||
GO_STORE = """package gopkg
|
||||
|
||||
type Store struct {
|
||||
Total int
|
||||
Name string
|
||||
}
|
||||
|
||||
func (s *Store) Push(x int) {
|
||||
s.Total = s.Total + x
|
||||
}
|
||||
|
||||
func NewStore(name string) *Store {
|
||||
return &Store{Name: name}
|
||||
}
|
||||
"""
|
||||
|
||||
GO_FLUSH_LINUX = """//go:build linux
|
||||
|
||||
package gopkg
|
||||
|
||||
func FlushDisk() int { return 1 }
|
||||
"""
|
||||
|
||||
GO_FLUSH_WINDOWS = """//go:build windows
|
||||
|
||||
package gopkg
|
||||
|
||||
func FlushDisk() int { return 2 }
|
||||
"""
|
||||
|
||||
GO_BRIDGE = """package gopkg
|
||||
|
||||
/*
|
||||
#include <stdio.h>
|
||||
static void announce(void) { puts("hi"); }
|
||||
*/
|
||||
import "C"
|
||||
|
||||
//export FixtureReady
|
||||
func FixtureReady() int { return 3 }
|
||||
|
||||
func Announce() {
|
||||
C.announce()
|
||||
}
|
||||
"""
|
||||
|
||||
GO_EVENTS = """package gopkg
|
||||
|
||||
func Produce(events chan string) {
|
||||
events <- "tick"
|
||||
}
|
||||
|
||||
func Drain(events chan string) string {
|
||||
v := <-events
|
||||
return v
|
||||
}
|
||||
"""
|
||||
|
||||
GO_FILES = (
|
||||
("go.mod", GO_MOD),
|
||||
("gopkg/store.go", GO_STORE),
|
||||
("gopkg/flush_linux.go", GO_FLUSH_LINUX),
|
||||
("gopkg/flush_windows.go", GO_FLUSH_WINDOWS),
|
||||
("gopkg/bridge.go", GO_BRIDGE),
|
||||
("gopkg/events.go", GO_EVENTS),
|
||||
)
|
||||
|
||||
# Distinct non-ASCII scripts — each must behave like the ASCII baseline.
|
||||
NON_ASCII_SEGMENTS = {
|
||||
"latin1_accents": "café_repo",
|
||||
@@ -244,6 +319,42 @@ def make_fixture(root):
|
||||
for name, text in (("math.ts", MATH_TS), ("main.ts", MAIN_TS)):
|
||||
with open(os.path.join(src, name), "wb") as f:
|
||||
f.write(text.encode("utf-8")) # exact bytes, identical across copies
|
||||
for rel, text in GO_FILES:
|
||||
path = os.path.join(root, rel.replace("/", os.sep))
|
||||
os.makedirs(os.path.dirname(path) or root, exist_ok=True)
|
||||
with open(path, "wb") as f:
|
||||
f.write(text.encode("utf-8"))
|
||||
|
||||
|
||||
def no_project_error(index_txt, repo, cache):
|
||||
"""Assemble the venue diagnostics for an index that left no project row.
|
||||
|
||||
The count summary cannot explain a venue-specific empty listing; carry the
|
||||
index response, the cache contents and the supervisor's worker logs (the
|
||||
only record of the pipeline's own error) into the CI log.
|
||||
"""
|
||||
try:
|
||||
cache_entries = sorted(os.listdir(cache))
|
||||
except OSError as exc:
|
||||
cache_entries = ["<listdir failed: %s>" % exc]
|
||||
log_tails = []
|
||||
logs_dir = os.path.join(cache, "logs")
|
||||
if os.path.isdir(logs_dir):
|
||||
for log_name in sorted(os.listdir(logs_dir)):
|
||||
try:
|
||||
with open(os.path.join(logs_dir, log_name), "rb") as lf:
|
||||
tail = lf.read()[-800:].decode("utf-8", "replace")
|
||||
log_tails.append("%s: %s" % (log_name, tail))
|
||||
except OSError as exc:
|
||||
log_tails.append("%s: <unreadable: %s>" % (log_name, exc))
|
||||
try:
|
||||
repo_entries = sorted(os.listdir(repo))
|
||||
except OSError as exc:
|
||||
repo_entries = ["<listdir failed: %s>" % exc]
|
||||
return {"error": "no project listed after index; index said %r; "
|
||||
"cache holds %r; repo holds %r; worker logs: %s"
|
||||
% (index_txt[:400], cache_entries, repo_entries,
|
||||
" | ".join(log_tails) or "<none>")}
|
||||
|
||||
|
||||
def index_and_count(binary, repo, cache):
|
||||
@@ -255,42 +366,29 @@ def index_and_count(binary, repo, cache):
|
||||
index_txt, err = s.tool_text(resp)
|
||||
if err:
|
||||
return {"error": "index tools/call error: %r" % err}
|
||||
lp = s.call_tool("list_projects", {}, timeout=60)
|
||||
lp_txt, _ = s.tool_text(lp)
|
||||
projects = json.loads(lp_txt).get("projects") or []
|
||||
if not projects:
|
||||
# The count summary cannot explain a venue-specific empty listing;
|
||||
# carry the index response and the cache contents into the log.
|
||||
try:
|
||||
cache_entries = sorted(os.listdir(cache))
|
||||
except OSError as exc:
|
||||
cache_entries = ["<listdir failed: %s>" % exc]
|
||||
# The supervisor's worker logs carry the actual pipeline error.
|
||||
log_tails = []
|
||||
logs_dir = os.path.join(cache, "logs")
|
||||
if os.path.isdir(logs_dir):
|
||||
for log_name in sorted(os.listdir(logs_dir)):
|
||||
try:
|
||||
with open(os.path.join(logs_dir, log_name), "rb") as lf:
|
||||
tail = lf.read()[-800:].decode("utf-8", "replace")
|
||||
log_tails.append("%s: %s" % (log_name, tail))
|
||||
except OSError as exc:
|
||||
log_tails.append("%s: <unreadable: %s>" % (log_name, exc))
|
||||
try:
|
||||
repo_entries = sorted(os.listdir(repo))
|
||||
except OSError as exc:
|
||||
repo_entries = ["<listdir failed: %s>" % exc]
|
||||
return {"error": "no project listed after index; index said %r; "
|
||||
"cache holds %r; repo holds %r; worker logs: %s"
|
||||
% (index_txt[:400], cache_entries, repo_entries,
|
||||
" | ".join(log_tails) or "<none>")}
|
||||
p = projects[0]
|
||||
out = {"name": p.get("name"), "nodes": p.get("nodes"),
|
||||
"edges": p.get("edges")}
|
||||
# The index response itself carries the synchronous, authoritative
|
||||
# counts ("nodes"/"edges"). list_projects publishes its stats columns
|
||||
# asynchronously — on some venues never within a one-shot session — so
|
||||
# gating on it misreads a healthy index as a setup failure (#1952).
|
||||
try:
|
||||
summary = json.loads(index_txt)
|
||||
except ValueError:
|
||||
summary = {}
|
||||
out = {"name": summary.get("project"), "nodes": summary.get("nodes"),
|
||||
"edges": summary.get("edges")}
|
||||
if out["nodes"] is None:
|
||||
# Payload without counts: fall back to list_projects, polled
|
||||
# because its stats row can trail the index on slow runners.
|
||||
projects, _ = wait_projects_with_stats(s)
|
||||
if not projects:
|
||||
return no_project_error(index_txt, repo, cache)
|
||||
p = projects[0]
|
||||
out = {"name": p.get("name"), "nodes": p.get("nodes"),
|
||||
"edges": p.get("edges")}
|
||||
# Definition-level counts prove the parser ran (not just discovery).
|
||||
# query_graph defaults to TOON text; this scripted consumer requests
|
||||
# format="json" ({"columns":[...],"rows":[["<n>"]],...}) explicitly.
|
||||
name = p.get("name")
|
||||
name = out["name"]
|
||||
defs = 0
|
||||
for label in ("Function", "Class", "Method"):
|
||||
q = "MATCH (n:%s) RETURN count(n)" % label
|
||||
@@ -305,6 +403,25 @@ def index_and_count(binary, repo, cache):
|
||||
except Exception:
|
||||
pass
|
||||
out["definition_nodes"] = defs
|
||||
# Go-file definitions separately: the Go passes derive qualified names
|
||||
# from the containing directory, so they meet non-ASCII paths on a
|
||||
# different route than the TS passes; a Go-specific count catches a
|
||||
# regression that total counts could mask (#1959).
|
||||
go_defs = 0
|
||||
for label in ("Function", "Method"):
|
||||
q = ("MATCH (n:%s) WHERE n.file_path CONTAINS '.go' "
|
||||
"RETURN count(n)" % label)
|
||||
r = s.call_tool("query_graph",
|
||||
{"query": q, "project": name, "format": "json"},
|
||||
timeout=60)
|
||||
t, _ = s.tool_text(r)
|
||||
try:
|
||||
rows = json.loads(t).get("rows") or []
|
||||
if rows and rows[0]:
|
||||
go_defs += int(rows[0][0])
|
||||
except Exception:
|
||||
pass
|
||||
out["go_definition_nodes"] = go_defs
|
||||
return out
|
||||
|
||||
|
||||
@@ -331,15 +448,30 @@ def main():
|
||||
|
||||
ascii_repo = os.path.join(work, "ascii_repo")
|
||||
make_fixture(ascii_repo)
|
||||
base = index_and_count(binary, ascii_repo, os.path.join(work, "c_ascii"))
|
||||
# The baseline is setup, not the surface under test: a cold runner can
|
||||
# lose the first index to daemon startup latency (#1952). One retry
|
||||
# against a fresh cache separates that environmental window from a
|
||||
# real indexing failure before the guard declares a precondition skip.
|
||||
base = {}
|
||||
for attempt in ("c_ascii", "c_ascii_retry"):
|
||||
base = index_and_count(binary, ascii_repo, os.path.join(work, attempt))
|
||||
if not base.get("error") and base.get("nodes"):
|
||||
break
|
||||
print("SETUP: ASCII baseline attempt %r did not index: %r"
|
||||
% (attempt, base))
|
||||
if base.get("error") or not base.get("nodes"):
|
||||
print("SETUP FAIL: ASCII baseline did not index: %r" % base)
|
||||
return 2
|
||||
print("baseline (ASCII): nodes=%s edges=%s definitions=%s" %
|
||||
(base["nodes"], base["edges"], base["definition_nodes"]))
|
||||
print("baseline (ASCII): nodes=%s edges=%s definitions=%s go=%s" %
|
||||
(base["nodes"], base["edges"], base["definition_nodes"],
|
||||
base["go_definition_nodes"]))
|
||||
if base["definition_nodes"] < 1:
|
||||
print("SETUP FAIL: ASCII baseline produced no definitions: %r" % base)
|
||||
return 2
|
||||
if base["go_definition_nodes"] < 1:
|
||||
print("SETUP FAIL: ASCII baseline extracted no Go definitions: %r"
|
||||
% base)
|
||||
return 2
|
||||
|
||||
for key, seg in NON_ASCII_SEGMENTS.items():
|
||||
repo = os.path.join(work, seg)
|
||||
@@ -348,13 +480,15 @@ def main():
|
||||
ok = (not got.get("error")
|
||||
and got.get("nodes") == base["nodes"]
|
||||
and got.get("edges") == base["edges"]
|
||||
and got.get("definition_nodes") == base["definition_nodes"])
|
||||
and got.get("definition_nodes") == base["definition_nodes"]
|
||||
and got.get("go_definition_nodes") == base["go_definition_nodes"])
|
||||
status = "PASS" if ok else "FAIL"
|
||||
print("[%s] non-ascii/%-14s nodes=%s edges=%s definitions=%s "
|
||||
"(baseline %s/%s/%s) name=%r" %
|
||||
print("[%s] non-ascii/%-14s nodes=%s edges=%s definitions=%s go=%s "
|
||||
"(baseline %s/%s/%s/%s) name=%r" %
|
||||
(status, key, got.get("nodes"), got.get("edges"),
|
||||
got.get("definition_nodes"), base["nodes"], base["edges"],
|
||||
base["definition_nodes"], got.get("name")))
|
||||
got.get("definition_nodes"), got.get("go_definition_nodes"),
|
||||
base["nodes"], base["edges"], base["definition_nodes"],
|
||||
base["go_definition_nodes"], got.get("name")))
|
||||
if not ok:
|
||||
# The counts alone cannot explain a venue-specific failure;
|
||||
# surface the captured error verbatim so CI logs carry the
|
||||
@@ -372,7 +506,9 @@ def main():
|
||||
if (not retry.get("error")
|
||||
and retry.get("nodes") == base["nodes"]
|
||||
and retry.get("edges") == base["edges"]
|
||||
and retry.get("definition_nodes") == base["definition_nodes"]):
|
||||
and retry.get("definition_nodes") == base["definition_nodes"]
|
||||
and retry.get("go_definition_nodes")
|
||||
== base["go_definition_nodes"]):
|
||||
print(" %s retry matched baseline -- order-dependent, "
|
||||
"not path-dependent" % key)
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user