trending/codebase-memory-mcp
1
0
Fork 0
mirror of https://github.com/DeusData/codebase-memory-mcp.git synced 2026-10-24 06:23:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into feat/649-omp-client-surface

Browse Source
This commit is contained in:
DavidHLP
2026-08-27 21:17:17 +08:00
committed by GitHub
parent 0a74be79a3 e678722746
commit e8d3c809df
59 changed files with 599344 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
THIRD_PARTY.md
+10 -5
View File
@@ -37,7 +37,7 @@ The core runtime headers in `internal/cbm/vendored/common/tree_sitter/`
## Tree-sitter Grammars
159 pre-generated parsers are vendored in `internal/cbm/vendored/grammars/<lang>/`
160 pre-generated parsers are vendored in `internal/cbm/vendored/grammars/<lang>/`
(generated `parser.c` plus `scanner.c` where applicable, compiled statically).
Each grammar is the work of its upstream authors and each grammar directory
contains the upstream `LICENSE` file.
@@ -54,10 +54,15 @@ License summary:
`pine` is **ISC** (declared by its upstream).
- The grammars authored in-house for this project (`cobol`, `form`, `janet`,
`magma`, `protobuf`, `wolfram`) are **MIT** under the project's own license,
(c) DeusData. Six further grammars (`assembly`, `cfml`, `cfscript`,
`dotenv`, `pine`, `qml`) are self-maintained forks that retain their
original upstream authors' licenses — see the manifest for per-grammar
provenance.
(c) DeusData. Seven further grammars (`arkts`, `assembly`, `cfml`,
`cfscript`, `dotenv`, `pine`, `qml`) are self-maintained forks that retain
their original upstream authors' licenses — see the manifest for per-grammar
provenance. `arkts` is a first-party derivative of
[tree-sitter/tree-sitter-typescript](https://github.com/tree-sitter/tree-sitter-typescript)
(MIT, (c) 2017 Max Brunsfeld; on the
[tree-sitter-javascript](https://github.com/tree-sitter/tree-sitter-javascript)
base, MIT, (c) 2014 Max Brunsfeld) with (c) 2026 DeusData ArkTS additions;
its grammar source lives in `tools/tree-sitter-arkts/`.
### tree-sitter-objectscript (UDL + routine)
internal/cbm/cbm.h
+5
View File
@@ -174,6 +174,7 @@ typedef enum {
CBM_LANG_OBJECTSCRIPT_UDL, // InterSystems ObjectScript UDL (.cls class files)
CBM_LANG_OBJECTSCRIPT_ROUTINE, // InterSystems ObjectScript routine (.mac/.int/.rtn/.inc)
CBM_LANG_OBJECTSCRIPT_EXPORT, // InterSystems Studio Export XML (<Export generator="Cache">)
CBM_LANG_ARKTS, // ArkTS (HarmonyOS/OpenHarmony .ets — TypeScript superset + ArkUI)
CBM_LANG_COUNT
} CBMLanguage;
@@ -551,6 +552,7 @@ typedef struct {
typedef struct {
const char *names[CBM_MAX_STRING_CONSTANTS];
const char *values[CBM_MAX_STRING_CONSTANTS];
bool is_url_builder[CBM_MAX_STRING_CONSTANTS];
int count;
} CBMStringConstantMap;
@@ -726,6 +728,9 @@ void cbm_channels_push(CBMChannelArray *arr, CBMArena *a, CBMChannel ch);
// --- Sub-extractor entry points ---
void cbm_extract_definitions(CBMExtractCtx *ctx);
/* Internal companion for embedded-language trees that contribute definitions
* to an existing host-file Module rather than minting a second Module. */
void cbm_extract_definitions_without_module(CBMExtractCtx *ctx);
// dbt lineage for Jinja-templated SQL models: emits a Model def plus one usage
// per ref()/source() call. No-op unless the file parses as SQL and actually
// contains a dbt builtin call. Defined in extract_dbt.c.
internal/cbm/extract_calls.c
+45 -3
View File
@@ -30,14 +30,29 @@ enum { MIN_PRINTABLE = 0x20 };
/* Handler arg scan start index (skip first positional). */
enum { HANDLER_START_IDX = 1 };
/* Look up a module-level string constant by name. */
/* Look up a module-level string constant by name. URL-builder entries share the
* map but are not constants: a bare `thingPath` reference is the function, not
* the URL it would build (issue #1009). */
static const char *lookup_string_constant(const CBMExtractCtx *ctx, const char *name) {
if (!name || !name[0]) {
return NULL;
}
const CBMStringConstantMap *map = &ctx->string_constants;
for (int i = 0; i < map->count; i++) {
if (strcmp(map->names[i], name) == 0) {
if (!map->is_url_builder[i] && strcmp(map->names[i], name) == 0) {
return map->values[i];
}
}
return NULL;
}
static const char *lookup_url_builder(const CBMExtractCtx *ctx, const char *name) {
if (!name || !name[0]) {
return NULL;
}
const CBMStringConstantMap *map = &ctx->string_constants;
for (int i = 0; i < map->count; i++) {
if (map->is_url_builder[i] && strcmp(map->names[i], name) == 0) {
return map->values[i];
}
}
@@ -1855,8 +1870,22 @@ static void extract_call_args(CBMExtractCtx *ctx, TSNode args, CBMCall *call) {
ca->index = positional_idx++;
if (is_string_like(ak) && ca->expr) {
ca->value = strip_quotes(ctx->arena, ca->expr);
} else if (strcmp(ak, "template_string") == 0) {
/* Flattened {} form so downstream url-arg detection joins the
* canonical server route shape (issue #1006/#1009). */
ca->value = cbm_template_string_text(ctx->arena, arg_node, ctx->source);
} else if (strcmp(ak, "identifier") == 0 && ca->expr) {
ca->value = lookup_string_constant(ctx, ca->expr);
} else if (strcmp(ak, "call_expression") == 0) {
/* URL-builder helper call (issue #1009): resolve
* client(buildPath(id)) through the per-file builder map. */
TSNode fn = ts_node_child_by_field_name(arg_node, TS_FIELD("function"));
if (!ts_node_is_null(fn) && strcmp(ts_node_type(fn), "identifier") == 0) {
char *fname = cbm_node_text(ctx->arena, fn, ctx->source);
if (fname) {
ca->value = lookup_url_builder(ctx, fname);
}
}
}
call->arg_count++;
}
@@ -2085,6 +2114,19 @@ static const char *extract_url_or_topic_arg(CBMExtractCtx *ctx, TSNode args) {
}
}
/* URL-builder helper call (issue #1009): `client(buildPath(id))` — the
* builder's returned URL was recorded in the per-file constant map. */
if (strcmp(ak, "call_expression") == 0) {
TSNode fn = ts_node_child_by_field_name(arg, TS_FIELD("function"));
if (!ts_node_is_null(fn) && strcmp(ts_node_type(fn), "identifier") == 0) {
char *fname = cbm_node_text(ctx->arena, fn, ctx->source);
const char *val = fname ? lookup_url_builder(ctx, fname) : NULL;
if (val) {
return val;
}
}
}
if (ai < MAX_POSITIONAL_SCAN) {
const char *val = extract_positional_url(ctx, arg, ak);
if (val) {
@@ -3296,7 +3338,7 @@ CBMInvocationDescriptor handle_calls(CBMExtractCtx *ctx, TSNode node, const CBML
// right. Bare calls (helper()) and new_expression have no member
// receiver, so they keep is_method=false (struct is zero-init).
if ((ctx->language == CBM_LANG_JAVASCRIPT || ctx->language == CBM_LANG_TYPESCRIPT ||
ctx->language == CBM_LANG_TSX) &&
ctx->language == CBM_LANG_TSX || ctx->language == CBM_LANG_ARKTS) &&
strcmp(ts_node_type(node), "call_expression") == 0) {
TSNode fn = ts_node_child_by_field_name(node, TS_FIELD("function"));
if (!ts_node_is_null(fn) && strcmp(ts_node_type(fn), "member_expression") == 0) {
internal/cbm/extract_channels.c
+1
View File
@@ -1058,6 +1058,7 @@ void cbm_extract_channels(CBMExtractCtx *ctx) {
case CBM_LANG_JAVASCRIPT:
case CBM_LANG_TYPESCRIPT:
case CBM_LANG_TSX:
case CBM_LANG_ARKTS:
extract_channels_js(ctx);
break;
case CBM_LANG_PYTHON:
internal/cbm/extract_defs.c
+39 -21
View File
@@ -2550,7 +2550,7 @@ static const char **extract_base_classes(CBMArena *a, TSNode node, const char *s
}
// Languages whose heritage is not exposed via a tree-sitter field need
// dedicated walkers; the generic field/keyword path mis-captures them.
if (lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX) {
if (lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX || lang == CBM_LANG_ARKTS) {
const char **ts_result = extract_ts_bases(a, node, source);
if (ts_result) {
return ts_result;
@@ -3097,7 +3097,7 @@ static char *resolve_param_type_text(CBMArena *a, TSNode param, const char *sour
return ts_node_is_null(typename) ? NULL : cbm_node_text(a, typename, source);
}
if (lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX) {
if (lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX || lang == CBM_LANG_ARKTS) {
if (strcmp(pk, "required_parameter") == 0 || strcmp(pk, "optional_parameter") == 0) {
return extract_ts_param_type(a, param, source);
}
@@ -3230,7 +3230,7 @@ static bool is_signature_implicit_receiver(CBMArena *a, TSNode param, const char
return true;
}
if (lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX) {
if (lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX || lang == CBM_LANG_ARKTS) {
char *name = signature_receiver_name(a, param, source, false);
return name && strcmp(name, "this") == 0;
}
@@ -3658,7 +3658,7 @@ static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec
if (ctx->enclosing_class_qn &&
(ctx->language == CBM_LANG_CPP || ctx->language == CBM_LANG_CUDA ||
ctx->language == CBM_LANG_TYPESCRIPT || ctx->language == CBM_LANG_TSX ||
ctx->language == CBM_LANG_NIX)) {
ctx->language == CBM_LANG_ARKTS || ctx->language == CBM_LANG_NIX)) {
def.qualified_name = cbm_arena_sprintf(a, "%s.%s", ctx->enclosing_class_qn, qn_name);
}
def.label = "Function";
@@ -3792,7 +3792,7 @@ static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec
// JS/TS export detection
if (ctx->language == CBM_LANG_JAVASCRIPT || ctx->language == CBM_LANG_TYPESCRIPT ||
ctx->language == CBM_LANG_TSX) {
ctx->language == CBM_LANG_TSX || ctx->language == CBM_LANG_ARKTS) {
if (is_js_exported(node)) {
def.is_entry_point = true;
}
@@ -4395,16 +4395,17 @@ static void extract_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec
label = "Interface";
}
}
// Rust/Swift/D: a struct is a distinct kind from a class — emit the precise
// "Struct" label rather than collapsing it to "Class". Scoped to these three
// grammar/LSP languages. Rust's struct node is `struct_item`; D's is
// `struct_declaration`. C/C++/Obj-C keep `struct_specifier` → "Class"
// (a C++ struct is class-like). "Struct" is a type-like container: every
// type-resolution / registry / IMPLEMENTS / LSP-registrar consumer routes
// through cbm_label_is_type_like(), so a struct still resolves as a type for
// its methods, fields, inheritance and impls.
// Rust/Swift/D/ArkTS: a struct is a distinct kind from a class — emit the
// precise "Struct" label rather than collapsing it to "Class". Scoped to
// these grammar/LSP languages. Rust's struct node is `struct_item`; D's and
// ArkTS's is `struct_declaration` (an ArkUI @Component custom component).
// C/C++/Obj-C keep `struct_specifier` → "Class" (a C++ struct is
// class-like). "Struct" is a type-like container: every type-resolution /
// registry / IMPLEMENTS / LSP-registrar consumer routes through
// cbm_label_is_type_like(), so a struct still resolves as a type for its
// methods, fields, inheritance and impls.
if (ctx->language == CBM_LANG_RUST || ctx->language == CBM_LANG_SWIFT ||
ctx->language == CBM_LANG_DLANG) {
ctx->language == CBM_LANG_DLANG || ctx->language == CBM_LANG_ARKTS) {
if (strcmp(kind, "struct_item") == 0 || strcmp(kind, "struct_declaration") == 0) {
label = "Struct";
}
@@ -5492,6 +5493,7 @@ static void extract_vars_mainstream(CBMExtractCtx *ctx, TSNode node, CBMArena *a
case CBM_LANG_JAVASCRIPT:
case CBM_LANG_TYPESCRIPT:
case CBM_LANG_TSX:
case CBM_LANG_ARKTS:
extract_js_vars(ctx, node, a);
break;
case CBM_LANG_JAVA: {
@@ -5988,6 +5990,7 @@ static void extract_var_names(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec
case CBM_LANG_JAVASCRIPT:
case CBM_LANG_TYPESCRIPT:
case CBM_LANG_TSX:
case CBM_LANG_ARKTS:
case CBM_LANG_JAVA:
case CBM_LANG_CSHARP:
case CBM_LANG_CPP:
@@ -6683,6 +6686,10 @@ static void extract_class_fields(CBMExtractCtx *ctx, TSNode class_node, const ch
def.start_line = ts_node_start_point(child).row + TS_LINE_OFFSET;
def.end_line = ts_node_end_point(child).row + TS_LINE_OFFSET;
def.is_exported = cbm_is_exported(name, ctx->language);
/* Field decorators/annotations (ArkTS @State/@Prop/@Link state members,
* TS @Input()-style fields, JVM field annotations via the modifiers
* child) — same extraction the class and method paths already run. */
def.decorators = extract_decorators(a, child, ctx->source, ctx->language, spec);
cbm_defs_push(&ctx->result->defs, a, def);
}
@@ -7481,7 +7488,8 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec,
bool descend_into_func =
(ctx->language == CBM_LANG_WOLFRAM || ctx->language == CBM_LANG_TYPESCRIPT ||
ctx->language == CBM_LANG_JAVASCRIPT || ctx->language == CBM_LANG_TSX ||
ctx->language == CBM_LANG_ADA || ctx->language == CBM_LANG_NIX);
ctx->language == CBM_LANG_ARKTS || ctx->language == CBM_LANG_ADA ||
ctx->language == CBM_LANG_NIX);
if (!descend_into_func) {
continue;
}
@@ -7501,7 +7509,8 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec,
* the class/func paths on the namespace node itself. */
if (is_namespace_scope_kind(ctx->language, kind, node)) {
const char *new_enclosing = compute_class_qn(ctx, node, frame.enclosing_class_qn);
if (ctx->language == CBM_LANG_TYPESCRIPT || ctx->language == CBM_LANG_TSX) {
if (ctx->language == CBM_LANG_TYPESCRIPT || ctx->language == CBM_LANG_TSX ||
ctx->language == CBM_LANG_ARKTS) {
extract_typescript_namespace_def(ctx, node, frame.enclosing_class_qn);
}
wd_push_children_reverse(&s, node, new_enclosing);
@@ -7524,6 +7533,19 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec,
free(s.data);
}
void cbm_extract_definitions_without_module(CBMExtractCtx *ctx) {
const CBMLangSpec *spec = cbm_lang_spec(ctx->language);
if (!spec) {
return;
}
// Walk AST for function/class definitions
walk_defs(ctx, ctx->root, spec, 0);
// Extract module-level variables
extract_variables(ctx, ctx->root, spec);
}
void cbm_extract_definitions(CBMExtractCtx *ctx) {
const CBMLangSpec *spec = cbm_lang_spec(ctx->language);
if (!spec) {
@@ -7545,9 +7567,5 @@ void cbm_extract_definitions(CBMExtractCtx *ctx) {
mod.is_test = ctx->result->is_test_file;
cbm_defs_push(&ctx->result->defs, a, mod);
// Walk AST for function/class definitions
walk_defs(ctx, ctx->root, spec, 0);
// Extract module-level variables
extract_variables(ctx, ctx->root, spec);
cbm_extract_definitions_without_module(ctx);
}
internal/cbm/extract_imports.c
+156 -33
View File
@@ -1395,8 +1395,13 @@ static void parse_spec_imports(CBMExtractCtx *ctx) {
// that the main parser uses. Adding another host language is a one-line
// declaration in lang_specs.c.
typedef struct {
TSNode script;
TSNode content;
} CBMEmbeddedBlock;
static void embedded_collect_content_nodes(TSNode root, const CBMEmbeddedLangSpec *spec,
TSNode *out, int *out_count, int max_out) {
CBMEmbeddedBlock *out, int *out_count, int max_out) {
/* Iterative DFS so deeply-nested script blocks are still found. Cap the
* stack to a sane bound (host grammars do not have million-deep markup
* trees) — no need to introduce TSNodeStack here. */
@@ -1412,7 +1417,8 @@ static void embedded_collect_content_nodes(TSNode root, const CBMEmbeddedLangSpe
for (uint32_t k = 0; k < cc; k++) {
TSNode c = ts_node_child(node, k);
if (strcmp(ts_node_type(c), spec->content_node_type) == 0) {
out[(*out_count)++] = c;
out[*out_count] = (CBMEmbeddedBlock){.script = node, .content = c};
(*out_count)++;
if (*out_count >= max_out) {
return;
}
@@ -1429,50 +1435,166 @@ static void embedded_collect_content_nodes(TSNode root, const CBMEmbeddedLangSpe
}
}
static bool ascii_trimmed_equals(const char *value, const char *expected) {
if (!value || !expected) {
return false;
}
while (*value && isspace((unsigned char)*value)) {
value++;
}
size_t len = strlen(value);
while (len > 0 && isspace((unsigned char)value[len - 1])) {
len--;
}
size_t expected_len = strlen(expected);
if (len != expected_len) {
return false;
}
for (size_t i = 0; i < len; i++) {
if (tolower((unsigned char)value[i]) != tolower((unsigned char)expected[i])) {
return false;
}
}
return true;
}
/* Vue is the only host that opts into structural embedded extraction. Restrict
* its inline programs to the language forms whose parsers we can select
* exactly. A src attribute always denotes an external program and therefore
* suppresses any inline extraction, even for malformed mixed markup. */
static bool vue_embedded_language(CBMExtractCtx *ctx, TSNode script, CBMLanguage *language) {
enum { VUE_ATTR_STACK_CAP = 128 };
TSNode stack[VUE_ATTR_STACK_CAP];
int top = 0;
bool has_lang = false;
bool lang_supported = true;
CBMLanguage selected = CBM_LANG_JAVASCRIPT;
stack[top++] = script;
while (top > 0) {
TSNode node = stack[--top];
if (strcmp(ts_node_type(node), "attribute") == 0) {
uint32_t named_count = ts_node_named_child_count(node);
if (named_count == 0) {
return false;
}
TSNode name_node = ts_node_named_child(node, 0);
char *name = cbm_node_text(ctx->arena, name_node, ctx->source);
if (ascii_trimmed_equals(name, "src")) {
return false;
}
if (ascii_trimmed_equals(name, "lang")) {
TSNode value_node = node;
if (!find_first_descendant_of(node, "attribute_value", &value_node)) {
return false;
}
char *value = cbm_node_text(ctx->arena, value_node, ctx->source);
has_lang = true;
if (ascii_trimmed_equals(value, "js") ||
ascii_trimmed_equals(value, "javascript")) {
selected = CBM_LANG_JAVASCRIPT;
} else if (ascii_trimmed_equals(value, "ts") ||
ascii_trimmed_equals(value, "typescript")) {
selected = CBM_LANG_TYPESCRIPT;
} else {
lang_supported = false;
}
}
continue;
}
uint32_t count = ts_node_named_child_count(node);
if ((int)count > VUE_ATTR_STACK_CAP - top) {
return false;
}
for (int i = (int)count - 1; i >= 0; i--) {
stack[top++] = ts_node_named_child(node, (uint32_t)i);
}
}
if (has_lang && !lang_supported) {
return false;
}
*language = selected;
return true;
}
static void parse_one_embedded_block(CBMExtractCtx *ctx, const CBMEmbeddedBlock *block,
CBMLanguage embedded_language, bool extract_structure) {
const TSLanguage *language = cbm_ts_language(embedded_language);
if (!language || ctx->source_len < 0) {
return;
}
TSRange range = {
.start_point = ts_node_start_point(block->content),
.end_point = ts_node_end_point(block->content),
.start_byte = ts_node_start_byte(block->content),
.end_byte = ts_node_end_byte(block->content),
};
if (range.end_byte <= range.start_byte || range.end_byte > (uint32_t)ctx->source_len) {
return;
}
/* One fresh parser/tree per block prevents parser state or error recovery
* from one script from changing the next script's result. Parsing the full
* original buffer through the absolute included range preserves byte and
* point coordinates across CRLF and UTF-8 prefixes. */
TSParser *parser = ts_parser_new();
if (!parser) {
return;
}
if (!ts_parser_set_language(parser, language) ||
!ts_parser_set_included_ranges(parser, &range, 1)) {
ts_parser_delete(parser);
return;
}
TSTree *tree = ts_parser_parse_string(parser, NULL, ctx->source, (uint32_t)ctx->source_len);
if (!tree) {
ts_parser_delete(parser);
return;
}
CBMExtractCtx sub_ctx = {
.arena = ctx->arena,
.result = ctx->result,
.source = ctx->source,
.source_len = ctx->source_len,
.language = embedded_language,
.project = ctx->project,
.rel_path = ctx->rel_path,
.module_qn = ctx->module_qn,
.root = ts_tree_root_node(tree),
.macro_table = ctx->macro_table,
.return_type_table = ctx->return_type_table,
};
if (extract_structure) {
cbm_extract_definitions_without_module(&sub_ctx);
}
walk_es_imports(&sub_ctx, sub_ctx.root);
if (extract_structure) {
cbm_extract_unified(&sub_ctx);
}
ts_tree_delete(tree);
ts_parser_delete(parser);
}
static void parse_embedded_imports(CBMExtractCtx *ctx) {
const CBMLangSpec *spec = cbm_lang_spec(ctx->language);
if (!spec || !spec->embedded_imports) {
return;
}
for (const CBMEmbeddedLangSpec *e = spec->embedded_imports; e->script_node_type != NULL; e++) {
const TSLanguage *embedded_lang = cbm_ts_language(e->embedded_language);
if (!embedded_lang) {
continue; /* embedded grammar not linked in — silently skip */
}
enum { MAX_EMBEDDED_BLOCKS = 16 };
TSNode hits[MAX_EMBEDDED_BLOCKS];
CBMEmbeddedBlock hits[MAX_EMBEDDED_BLOCKS];
int hit_count = 0;
embedded_collect_content_nodes(ctx->root, e, hits, &hit_count, MAX_EMBEDDED_BLOCKS);
if (hit_count == 0) {
continue;
}
TSParser *parser = ts_parser_new();
if (!parser) {
continue;
}
if (!ts_parser_set_language(parser, embedded_lang)) {
ts_parser_delete(parser);
continue;
}
for (int i = 0; i < hit_count; i++) {
uint32_t s = ts_node_start_byte(hits[i]);
uint32_t end = ts_node_end_byte(hits[i]);
if (end <= s) {
CBMLanguage embedded_language = e->embedded_language;
bool extract_structure = ctx->language == CBM_LANG_VUE;
if (extract_structure &&
!vue_embedded_language(ctx, hits[i].script, &embedded_language)) {
continue;
}
const char *sub_src = ctx->source + s;
uint32_t sub_len = end - s;
TSTree *sub_tree = ts_parser_parse_string(parser, NULL, sub_src, sub_len);
if (!sub_tree) {
continue;
}
CBMExtractCtx sub_ctx = *ctx;
sub_ctx.source = sub_src;
sub_ctx.root = ts_tree_root_node(sub_tree);
walk_es_imports(&sub_ctx, sub_ctx.root);
ts_tree_delete(sub_tree);
parse_one_embedded_block(ctx, &hits[i], embedded_language, extract_structure);
}
ts_parser_delete(parser);
}
}
@@ -2764,6 +2886,7 @@ void cbm_extract_imports(CBMExtractCtx *ctx) {
case CBM_LANG_JAVASCRIPT:
case CBM_LANG_TYPESCRIPT:
case CBM_LANG_TSX:
case CBM_LANG_ARKTS:
parse_es_imports(ctx);
break;
case CBM_LANG_JAVA:
internal/cbm/extract_type_refs.c
+1
View File
@@ -219,6 +219,7 @@ static void process_body_type_ref(CBMExtractCtx *ctx, TSNode node, const char *f
break;
case CBM_LANG_TYPESCRIPT:
case CBM_LANG_TSX:
case CBM_LANG_ARKTS:
extract_ts_body_type_refs(ctx, node, kind, func_qn);
break;
case CBM_LANG_JAVA:
internal/cbm/extract_unified.c
+223
View File
@@ -1099,6 +1099,227 @@ static void handle_string_refs(CBMExtractCtx *ctx, TSNode node, const WalkState
cbm_stringref_push(&ctx->result->string_refs, ctx->arena, ref);
}
// --- URL-builder helpers (issue #1009) ---
/* Map-aware template flatten for builder bodies: a ${...} substitution that is
* a bare identifier or a call to an already-recorded name (const or an earlier
* builder in the same file) inlines that value; anything else becomes "{}".
* The query string is not part of a route's identity, so the result is
* truncated at the first '?'. Handles the composed-builder shape
* `return \`${basePath(id)}?${params}\``. */
static const char *builder_template_text(CBMExtractCtx *ctx, TSNode node) {
enum { BLD_BUF = 512 };
char buf[BLD_BUF];
size_t pos = 0;
uint32_t nc = ts_node_named_child_count(node);
for (uint32_t i = 0; i < nc; i++) {
TSNode c = ts_node_named_child(node, i);
const char *k = ts_node_type(c);
const char *piece = NULL;
if (strcmp(k, "string_fragment") == 0) {
piece = cbm_node_text(ctx->arena, c, ctx->source);
} else if (strcmp(k, "template_substitution") == 0) {
piece = "{}";
if (ts_node_named_child_count(c) > 0) {
TSNode expr = ts_node_named_child(c, 0);
/* `${basePath(id)}` inlines a builder, `${BASE}` a const. A bare
* name never resolves to a builder: that reads the function. */
bool want_builder = strcmp(ts_node_type(expr), "call_expression") == 0;
TSNode name_node =
want_builder ? ts_node_child_by_field_name(expr, TS_FIELD("function")) : expr;
if (!ts_node_is_null(name_node) &&
strcmp(ts_node_type(name_node), "identifier") == 0) {
char *nm = cbm_node_text(ctx->arena, name_node, ctx->source);
if (nm) {
const CBMStringConstantMap *map = &ctx->string_constants;
for (int mi = 0; mi < map->count; mi++) {
if (map->values[mi] && map->is_url_builder[mi] == want_builder &&
strcmp(map->names[mi], nm) == 0) {
piece = map->values[mi];
break;
}
}
}
}
}
} else {
continue;
}
if (!piece) {
continue;
}
size_t pl = strlen(piece);
if (pos + pl >= BLD_BUF) {
return NULL;
}
memcpy(buf + pos, piece, pl);
pos += pl;
}
/* Route identity excludes the query string. */
for (size_t qi = 0; qi < pos; qi++) {
if (buf[qi] == '?') {
pos = qi;
break;
}
}
if (pos == 0) {
return NULL;
}
return cbm_arena_strndup(ctx->arena, buf, pos);
}
/* Flatten a string-ish node (plain string or template literal) to text. */
static const char *url_builder_literal_text(CBMExtractCtx *ctx, TSNode value_node) {
const char *kind = ts_node_type(value_node);
if (strcmp(kind, "template_string") == 0) {
return builder_template_text(ctx, value_node);
}
if (!is_string_node(kind)) {
return NULL;
}
char *text = cbm_node_text(ctx->arena, value_node, ctx->source);
if (!text || !text[0]) {
return NULL;
}
int len = (int)strlen(text);
if (len >= CBM_QUOTE_PAIR && (text[0] == '"' || text[0] == '\'')) {
text = cbm_arena_strndup(ctx->arena, text + SKIP_ONE, (size_t)(len - PAIR_LEN));
}
return text;
}
/* A route-shaped URL literal: an absolute path that classifies as a URL. */
static const char *builder_route_url(CBMExtractCtx *ctx, TSNode value_node) {
const char *url = url_builder_literal_text(ctx, value_node);
if (!url || url[0] != '/' || cbm_classify_string(url, (int)strlen(url)) != CBM_STRREF_URL) {
return NULL;
}
return url;
}
static void record_url_builder(CBMExtractCtx *ctx, const char *name, const char *url) {
if (!name || !name[0] || !url) {
return;
}
CBMStringConstantMap *map = &ctx->string_constants;
for (int i = 0; i < map->count; i++) {
if (strcmp(map->names[i], name) == 0) {
if (map->is_url_builder[i] && map->values[i] && strcmp(map->values[i], url) != 0) {
map->values[i] = NULL; /* ambiguous builder */
}
return;
}
}
if (map->count < CBM_MAX_STRING_CONSTANTS) {
map->names[map->count] = (char *)name;
map->values[map->count] = (char *)url;
map->is_url_builder[map->count] = true;
map->count++;
}
}
/* Every `return` this function owns yields a route-shaped URL literal. A body
* that also returns a computed path (`return computePath(kind)`) would attribute
* its one literal to call sites that never produce it, so a mixed builder is
* declined rather than guessed. Returns inside a nested function belong to that
* function, not to this one. */
static bool builder_returns_only_urls(CBMExtractCtx *ctx, TSNode func) {
TSNode body = ts_node_child_by_field_name(func, TS_FIELD("body"));
if (ts_node_is_null(body)) {
return false;
}
bool only_urls = true;
TSTreeCursor cursor = ts_tree_cursor_new(body);
for (;;) {
TSNode node = ts_tree_cursor_current_node(&cursor);
if (strcmp(ts_node_type(node), "return_statement") == 0 &&
ts_node_eq(cbm_find_enclosing_func(node, ctx->language), func)) {
if (ts_node_named_child_count(node) == 0 ||
!builder_route_url(ctx, ts_node_named_child(node, 0))) {
only_urls = false;
break;
}
}
if (ts_tree_cursor_goto_first_child(&cursor)) {
continue;
}
if (ts_tree_cursor_goto_next_sibling(&cursor)) {
continue;
}
bool found = false;
while (ts_tree_cursor_goto_parent(&cursor)) {
if (ts_tree_cursor_goto_next_sibling(&cursor)) {
found = true;
break;
}
}
if (!found) {
break;
}
}
ts_tree_cursor_delete(&cursor);
return only_urls;
}
/* URL-builder helper pattern (issue #1009): a small function whose return value
* is a URL-shaped literal, consumed as `client(buildPath(id))`. The literal
* never appears as a call argument, so first_string_arg resolution cannot see
* it; record `functionName -> url` in the per-file constant map and let the
* call-site call_expression branch resolve it. Covers `return`-statement bodies
* and arrow-function expression bodies.
*
* JS/TS only. The predicate accepts any absolute pathname, and `return` plus a
* string literal is also the shape of every C or Go helper handing back a
* filesystem path (`/etc/...`, `/proc/self/...`); recording those would mint a
* Route node and an HTTP_CALLS edge in a language that speaks no HTTP. */
static void handle_url_builders(CBMExtractCtx *ctx, TSNode node, const WalkState *state) {
if (ctx->language != CBM_LANG_JAVASCRIPT && ctx->language != CBM_LANG_TYPESCRIPT &&
ctx->language != CBM_LANG_TSX) {
return;
}
const char *kind = ts_node_type(node);
if (strcmp(kind, "arrow_function") == 0) {
TSNode body = ts_node_child_by_field_name(node, TS_FIELD("body"));
if (ts_node_is_null(body) || strcmp(ts_node_type(body), "statement_block") == 0) {
return;
}
const char *url = builder_route_url(ctx, body);
if (!url) {
return;
}
TSNode parent = ts_node_parent(node);
if (!ts_node_is_null(parent) && strcmp(ts_node_type(parent), "variable_declarator") == 0) {
TSNode name_node = ts_node_child_by_field_name(parent, TS_FIELD("name"));
if (!ts_node_is_null(name_node)) {
record_url_builder(ctx, cbm_node_text(ctx->arena, name_node, ctx->source), url);
}
}
return;
}
if (strcmp(kind, "return_statement") != 0) {
return;
}
if (!state->enclosing_func_qn || state->enclosing_func_qn == ctx->module_qn) {
return;
}
if (ts_node_named_child_count(node) == 0) {
return;
}
const char *url = builder_route_url(ctx, ts_node_named_child(node, 0));
if (!url) {
return;
}
TSNode func = cbm_find_enclosing_func(node, ctx->language);
if (ts_node_is_null(func) || !builder_returns_only_urls(ctx, func)) {
return;
}
const char *name = strrchr(state->enclosing_func_qn, '.');
name = name ? name + 1 : state->enclosing_func_qn;
record_url_builder(ctx, name, url);
}
// --- YAML nested field extraction (D2) ---
/* Recursively walk YAML block_mapping_pair nodes, building dotted key paths.
@@ -1649,6 +1870,7 @@ static bool is_actual_import_boundary(CBMExtractCtx *ctx, TSNode node, const CBM
case CBM_LANG_JAVASCRIPT:
case CBM_LANG_TYPESCRIPT:
case CBM_LANG_TSX:
case CBM_LANG_ARKTS:
if (strcmp(kind, "export_statement") == 0) {
/* An export is an import CONTEXT only in its re-export forms:
* `export ... from 'mod'` (source field) or a bare specifier list
@@ -2160,6 +2382,7 @@ void cbm_extract_unified(CBMExtractCtx *ctx) {
handle_string_constants(ctx, node, &state);
handle_objectscript_type_map(ctx, node, &state);
handle_url_builders(ctx, node, &state);
CBMInvocationDescriptor invocation = handle_calls(ctx, node, spec, &state);
handle_usages(ctx, node, spec, &state);
handle_throws(ctx, node, spec, &state);
internal/cbm/extract_usages.c
+6 -3
View File
@@ -104,6 +104,7 @@ static bool is_reference_node(TSNode node, CBMLanguage lang) {
case CBM_LANG_JAVASCRIPT:
case CBM_LANG_TYPESCRIPT:
case CBM_LANG_TSX:
case CBM_LANG_ARKTS:
case CBM_LANG_QML:
case CBM_LANG_CFSCRIPT:
return strcmp(kind, "property_identifier") == 0 ||
@@ -1367,6 +1368,7 @@ static bool language_may_stamp_exact_callable_value_candidate(CBMLanguage langua
case CBM_LANG_JAVASCRIPT:
case CBM_LANG_TYPESCRIPT:
case CBM_LANG_TSX:
case CBM_LANG_ARKTS:
case CBM_LANG_GO:
case CBM_LANG_PYTHON:
case CBM_LANG_C:
@@ -1492,7 +1494,7 @@ static TSNode call_reference_candidate_site(CBMExtractCtx *ctx, TSNode node, con
(void)occurrence_parent(cursor, node, &parent, &parent_field);
}
bool ts_family = ctx->language == CBM_LANG_JAVASCRIPT || ctx->language == CBM_LANG_TYPESCRIPT ||
ctx->language == CBM_LANG_TSX;
ctx->language == CBM_LANG_TSX || ctx->language == CBM_LANG_ARKTS;
if (ts_family && strcmp(kind, "property_identifier") == 0 && !ts_node_is_null(parent) &&
strcmp(ts_node_type(parent), "member_expression") == 0) {
TSNode property = ts_node_child_by_field_name(parent, TS_FIELD("property"));
@@ -1995,7 +1997,8 @@ static bool is_import_binding_occurrence(CBMExtractCtx *ctx, TSNode node, const
}
case CBM_LANG_JAVASCRIPT:
case CBM_LANG_TYPESCRIPT:
case CBM_LANG_TSX: {
case CBM_LANG_TSX:
case CBM_LANG_ARKTS: {
if (strcmp(ts_node_type(boundary), "import_statement") != 0) {
return false;
}
@@ -2182,7 +2185,7 @@ static void record_lexical_binding(CBMExtractCtx *ctx, WalkState *state, TSNode
scope_id = lexical_ancestor_of_kind(state, current_id, true, false);
whole_scope = true;
} else if (ctx->language == CBM_LANG_JAVASCRIPT || ctx->language == CBM_LANG_TYPESCRIPT ||
ctx->language == CBM_LANG_TSX) {
ctx->language == CBM_LANG_TSX || ctx->language == CBM_LANG_ARKTS) {
bool is_var = js_var_binding(node);
scope_id = lexical_ancestor_of_kind(state, current_id, is_var, !is_var);
if (scope_id == 0) {
internal/cbm/grammar_arkts.c
+4
View File
@@ -0,0 +1,4 @@
// Vendored tree-sitter grammar: arkts (first-party fork of tree-sitter-typescript)
// Each grammar compiled as separate unit (conflicting static symbols).
#include "vendored/grammars/arkts/parser.c"
#include "vendored/grammars/arkts/scanner.c"
internal/cbm/helpers.c
+7 -3
View File
@@ -211,6 +211,7 @@ bool cbm_is_keyword(const char *name, CBMLanguage lang) {
case CBM_LANG_JAVASCRIPT:
case CBM_LANG_TYPESCRIPT:
case CBM_LANG_TSX:
case CBM_LANG_ARKTS:
keywords = js_keywords;
break;
case CBM_LANG_RUST:
@@ -344,7 +345,8 @@ bool cbm_is_test_file(const char *rel_path, CBMLanguage lang) {
return has_prefix(base, "test_") || has_suffix(base, "_test.py");
case CBM_LANG_JAVASCRIPT:
case CBM_LANG_TYPESCRIPT:
case CBM_LANG_TSX: {
case CBM_LANG_TSX:
case CBM_LANG_ARKTS: {
char noext[NOEXT_BUF];
strip_ext(base, noext, sizeof(noext));
return has_suffix(noext, ".test") || has_suffix(noext, ".spec") ||
@@ -509,7 +511,7 @@ bool cbm_is_namespace_scope_kind(CBMLanguage lang, const char *kind) {
if (lang == CBM_LANG_CPP || lang == CBM_LANG_CUDA) {
return strcmp(kind, "namespace_definition") == 0;
}
if (lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX) {
if (lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX || lang == CBM_LANG_ARKTS) {
return strcmp(kind, "internal_module") == 0;
}
return false;
@@ -751,6 +753,7 @@ static const char **func_kinds_for_lang(CBMLanguage lang) {
case CBM_LANG_JAVASCRIPT:
case CBM_LANG_TYPESCRIPT:
case CBM_LANG_TSX:
case CBM_LANG_ARKTS:
return func_kinds_js;
case CBM_LANG_RUST:
return func_kinds_rust;
@@ -1341,7 +1344,8 @@ bool cbm_is_module_level_p(TSNode parent, CBMLanguage lang) {
if (lang == CBM_LANG_PYTHON) {
return check_script_module_level(parent, pk, "module", "expression_statement");
}
if (lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX) {
if (lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX ||
lang == CBM_LANG_ARKTS) {
return check_script_module_level(parent, pk, "program", "export_statement");
}
if (lang == CBM_LANG_LUA) {
internal/cbm/lang_specs.c
+27
View File
@@ -167,6 +167,7 @@ extern const TSLanguage *tree_sitter_pine(void);
extern const TSLanguage *tree_sitter_mojo(void);
extern const TSLanguage *tree_sitter_objectscript_udl(void);
extern const TSLanguage *tree_sitter_objectscript_routine(void);
extern const TSLanguage *tree_sitter_arkts(void);
// -- Empty sentinel --
static const char *empty_types[] = {NULL};
@@ -250,6 +251,23 @@ static const char *ts_class_types[] = {"class_declaration",
NULL};
static const char *ts_decorator_types[] = {"decorator", NULL};
// ==================== ARKTS (HarmonyOS .ets) ====================
// TypeScript-superset grammar (first-party tree-sitter-typescript fork).
// Reuses the JS/TS arrays and adds the ArkUI constructs: `@Component struct`
// components are class-like containers (struct_declaration), and their
// decorated state members (public_field_definition) are extracted as fields
// so @State/@Prop/@Link/... properties are findable.
static const char *arkts_class_types[] = {"class_declaration",
"class",
"abstract_class_declaration",
"struct_declaration",
"enum_declaration",
"interface_declaration",
"type_alias_declaration",
"internal_module",
NULL};
static const char *arkts_field_types[] = {"public_field_definition", NULL};
// ==================== QML (Qt) ====================
// QMLJS grammar is a TypeScript superset plus declarative ui_* nodes, so the
// JS/TS function, call, branch, variable and module arrays are reused as-is.
@@ -2640,6 +2658,15 @@ static const CBMLangSpec lang_specs[CBM_LANG_COUNT] = {
// each class as CBM_LANG_OBJECTSCRIPT_UDL, so this language never reaches
// cbm_lang_spec()/cbm_ts_language() directly. Left as a zero spec.
// CBM_LANG_ARKTS
[CBM_LANG_ARKTS] = {CBM_LANG_ARKTS, ts_func_types, arkts_class_types, arkts_field_types,
js_module_types, js_call_types, js_import_types, js_import_types,
js_branch_types, js_var_types,
(const char *[]){"assignment_expression", "augmented_assignment_expression",
NULL},
js_throw_types, NULL, ts_decorator_types, NULL,
ts_env_members, tree_sitter_arkts, NULL},
};
_Static_assert(sizeof(lang_specs) / sizeof(lang_specs[0]) == CBM_LANG_COUNT,
internal/cbm/vendored/grammars/MANIFEST.md
+4 -2
View File
@@ -9,12 +9,13 @@ The grammars were originally vendored as bare `parser.c`+`scanner.c` with **no r
## Summary
- Grammars: **159** — vendored-from-upstream: **142**, first-party/self-maintained: **12**, registry-disagreement: **5** (nim removed 2026-06-12; objectscript_udl + objectscript_routine added 2026-06-24; mojo added 2026-07-01 — see notes below)
- ABI distribution: **7×** ABI-13 **85×** ABI-14 **64×** ABI-15 (runtime ceiling is ABI 15; never vendor ABI 16 without a runtime upgrade)
- Grammars: **160** — vendored-from-upstream: **142**, first-party/self-maintained: **13**, registry-disagreement: **5** (nim removed 2026-06-12; objectscript_udl + objectscript_routine added 2026-06-24; mojo added 2026-07-01; arkts added 2026-08-26 — see notes below)
- ABI distribution: **7×** ABI-13 **85×** ABI-14 **65×** ABI-15 (runtime ceiling is ABI 15; never vendor ABI 16 without a runtime upgrade)
- Vendored copies missing LICENSE: **0** — all upstream LICENSE files restored 2026-06-11 (first-party grammars carry the project MIT license; `move` uses the Helix-listed upstream tzakian/tree-sitter-move MIT text, `zsh` uses georgeharker/tree-sitter-zsh MIT)
- `verdict`: VERIFIED-BOTH = our source matches *both* registries; VERIFIED-NVIM/HELIX = matches one; registry-disagreement = registries name a different repo (listed separately); `vendor-maintained` = the language vendor's own grammar, not in nvim/Helix.
- **objectscript_udl / objectscript_routine** (added 2026-06-24): vendored from [intersystems/tree-sitter-objectscript](https://github.com/intersystems/tree-sitter-objectscript) @ `a7ffcdf` — MIT, the InterSystems-official grammars (a niche vendor language, hence `vendor-maintained`, not in nvim-treesitter/Helix). **Re-vendor note:** each `scanner.c`'s upstream `#include "../../common/scanner.h"` is repointed to a per-directory `objectscript_common.h` (a verbatim copy of upstream `common/scanner.h`), because this repo's shared `vendored/common/scanner.h` belongs to the cfml/fsharp grammars and differs. The generated `parser.c`/`scanner.c` are otherwise byte-for-byte upstream — on re-vendor, re-apply only that single include rename. **Local modification (2026-07-16):** in `objectscript_common.h`, two loop counters `uint8_t i` were widened to `int i` (the `reverse_marker` scan and the `html_marker_buffer` reversal) to clear CodeQL `cpp/comparison-with-wider-type` — a false positive in practice (both lengths are hard-bounded by `MARKER_BUFFER_MAX_LEN = 30`, so `uint8_t` could never wrap), fixed for cleanliness. On re-vendor, re-apply this widening too (or upstream it at intersystems/tree-sitter-objectscript).
- **mojo** (added 2026-07-01): vendored from [lsh/tree-sitter-mojo](https://github.com/lsh/tree-sitter-mojo) @ `33193a99afe6` — MIT, ABI 15. Helix tracks `lsh/tree-sitter-mojo` as its Mojo grammar source, but the Helix-pinned commit (`3d7c53b8038f`) no longer resolves in the upstream repository after a force-push, so this vendor uses current upstream `main` rather than the stale registry SHA. Security review covered only the vendored C surface (`parser.c`, `scanner.c`, `tree_sitter/*.h`) plus upstream license/provenance metadata; no package manager hooks, workflow files, prompt/agent instruction files, or generated lockfiles were vendored.
- **arkts** (added 2026-08-26): **first-party derivative** — a fork of [tree-sitter/tree-sitter-typescript](https://github.com/tree-sitter/tree-sitter-typescript)'s `typescript` dialect pinned @ `75b3874edb2d` (v0.23.2, the same commit our vendored typescript/tsx come from), on the tree-sitter-javascript base @ `3a837b6f3658` (v0.23.1, the version upstream's own package-lock pins), extended with ArkTS/ArkUI syntax (`@Component struct` declarations, UI-DSL trailing-closure calls with post-block attribute chains, decorated function declarations, `import lazy`, `@Extend`/`@Styles` leading-dot attribute chains, anonymous `stateStyles` style blocks). Grammar source + corpus tests live in `tools/tree-sitter-arkts/`; regenerate with `npx tree-sitter-cli@0.25.10 generate` (ABI 15). **External scanner:** `scanner.c` is a symbol-rename trampoline; `_common_scanner.h` is byte-identical (same SHA-256) to the reviewed `typescript/_common_scanner.h` already shipped. **LICENSE:** tree-sitter-typescript's MIT text **verbatim and byte-identical** (© 2017 Max Brunsfeld) — which is what MIT requires of a derivative work, and what lets the provenance audit byte-verify it against upstream rather than take a note on trust. The fork is registered in the audit's `FORKS` map. Our own copyright for the ArkTS additions, and the tree-sitter-javascript attribution (© 2014 Max Brunsfeld), live with the SOURCE in `tools/tree-sitter-arkts/grammar.js` and in `THIRD_PARTY.md`, which is what ships in the release archives.
> ⚠️ **Pinned commit = the revision nvim-treesitter/Helix vendor** (battle-tested, canonical source), not bleeding-edge HEAD. When re-vendoring, update the pinned commit here.
@@ -234,6 +235,7 @@ upstream author (correctly retained). The table now records the true origin.
| grammar | cur ABI | original upstream | license |
|---|:---:|---|---|
| arkts | 15 | **self-maintained fork** of tree-sitter/tree-sitter-typescript @ `75b3874edb2d` (v0.23.2; javascript base @ `3a837b6f3658` v0.23.1); grammar source in `tools/tree-sitter-arkts/` | MIT, (c) 2017 Max Brunsfeld — byte-identical to upstream; ArkTS additions (c) 2026 DeusData, see grammar.js + THIRD_PARTY.md |
| assembly | 14 | RubixDev/tree-sitter-assembly (**repo deleted from GitHub** — our retained MIT copy, (c) 2023 RubixDev, is the surviving grant) | MIT |
| cfml | 15 | cfmleditor/tree-sitter-cfml | MIT, (c) 2014 Gareth Edwards & Gavin Baumanis — byte-identical |
| cfscript | 15 | cfmleditor/tree-sitter-cfml | MIT, same — byte-identical |
internal/cbm/vendored/grammars/arkts/LICENSE
+21
View File
@@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2017 Max Brunsfeld
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
internal/cbm/vendored/grammars/arkts/_common_scanner.h
+347
View File
@@ -0,0 +1,347 @@
#include "tree_sitter/parser.h"
#include <wctype.h>
enum TokenType {
AUTOMATIC_SEMICOLON,
TEMPLATE_CHARS,
TERNARY_QMARK,
HTML_COMMENT,
LOGICAL_OR,
ESCAPE_SEQUENCE,
REGEX_PATTERN,
JSX_TEXT,
FUNCTION_SIGNATURE_AUTOMATIC_SEMICOLON,
ERROR_RECOVERY,
};
static void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
static void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
static bool scan_template_chars(TSLexer *lexer) {
lexer->result_symbol = TEMPLATE_CHARS;
for (bool has_content = false;; has_content = true) {
lexer->mark_end(lexer);
switch (lexer->lookahead) {
case '`':
return has_content;
case '\0':
return false;
case '$':
advance(lexer);
if (lexer->lookahead == '{') {
return has_content;
}
break;
case '\\':
return has_content;
default:
advance(lexer);
}
}
}
static bool scan_whitespace_and_comments(TSLexer *lexer, bool *scanned_comment) {
for (;;) {
while (iswspace(lexer->lookahead)) {
skip(lexer);
}
if (lexer->lookahead == '/') {
skip(lexer);
if (lexer->lookahead == '/') {
skip(lexer);
while (lexer->lookahead != 0 && lexer->lookahead != '\n') {
skip(lexer);
}
*scanned_comment = true;
} else if (lexer->lookahead == '*') {
skip(lexer);
while (lexer->lookahead != 0) {
if (lexer->lookahead == '*') {
skip(lexer);
if (lexer->lookahead == '/') {
skip(lexer);
break;
}
} else {
skip(lexer);
}
}
} else {
return false;
}
} else {
return true;
}
}
}
static bool scan_automatic_semicolon(TSLexer *lexer, const bool *valid_symbols, bool *scanned_comment) {
lexer->result_symbol = AUTOMATIC_SEMICOLON;
lexer->mark_end(lexer);
for (;;) {
if (lexer->lookahead == 0) {
return true;
}
if (lexer->lookahead == '}') {
// Automatic semicolon insertion breaks detection of object patterns
// in a typed context:
// type F = ({a}: {a: number}) => number;
// Therefore, disable automatic semicolons when followed by typing
do {
skip(lexer);
} while (iswspace(lexer->lookahead));
if (lexer->lookahead == ':') {
return valid_symbols[LOGICAL_OR]; // Don't return false if we're in a ternary by checking if || is valid
}
return true;
}
if (!iswspace(lexer->lookahead)) {
return false;
}
if (lexer->lookahead == '\n') {
break;
}
skip(lexer);
}
skip(lexer);
if (!scan_whitespace_and_comments(lexer, scanned_comment)) {
return false;
}
switch (lexer->lookahead) {
case '`':
case ',':
case '.':
case ';':
case '*':
case '%':
case '>':
case '<':
case '=':
case '?':
case '^':
case '|':
case '&':
case '/':
case ':':
return false;
case '{':
if (valid_symbols[FUNCTION_SIGNATURE_AUTOMATIC_SEMICOLON]) {
return false;
}
break;
// Don't insert a semicolon before a '[' or '(', unless we're parsing
// a type. Detect whether we're parsing a type or an expression using
// the validity of a binary operator token.
case '(':
case '[':
if (valid_symbols[LOGICAL_OR]) {
return false;
}
break;
// Insert a semicolon before `--` and `++`, but not before binary `+` or `-`.
case '+':
skip(lexer);
return lexer->lookahead == '+';
case '-':
skip(lexer);
return lexer->lookahead == '-';
// Don't insert a semicolon before `!=`, but do insert one before a unary `!`.
case '!':
skip(lexer);
return lexer->lookahead != '=';
// Don't insert a semicolon before `in` or `instanceof`, but do insert one
// before an identifier.
case 'i':
skip(lexer);
if (lexer->lookahead != 'n') {
return true;
}
skip(lexer);
if (!iswalpha(lexer->lookahead)) {
return false;
}
for (unsigned i = 0; i < 8; i++) {
if (lexer->lookahead != "stanceof"[i]) {
return true;
}
skip(lexer);
}
if (!iswalpha(lexer->lookahead)) {
return false;
}
break;
}
return true;
}
static bool scan_ternary_qmark(TSLexer *lexer) {
for (;;) {
if (!iswspace(lexer->lookahead)) {
break;
}
skip(lexer);
}
if (lexer->lookahead == '?') {
advance(lexer);
/* Optional chaining. */
if (lexer->lookahead == '?' || lexer->lookahead == '.') {
return false;
}
lexer->mark_end(lexer);
lexer->result_symbol = TERNARY_QMARK;
/* TypeScript optional arguments contain the ?: sequence, possibly
with whitespace. */
for (;;) {
if (!iswspace(lexer->lookahead)) {
break;
}
advance(lexer);
}
if (lexer->lookahead == ':' || lexer->lookahead == ')' || lexer->lookahead == ',') {
return false;
}
if (lexer->lookahead == '.') {
advance(lexer);
if (iswdigit(lexer->lookahead)) {
return true;
}
return false;
}
return true;
}
return false;
}
static bool scan_closing_comment(TSLexer *lexer) {
while (iswspace(lexer->lookahead) || lexer->lookahead == 0x2028 || lexer->lookahead == 0x2029) {
skip(lexer);
}
const char *comment_start = "<!--";
const char *comment_end = "-->";
if (lexer->lookahead == '<') {
for (unsigned i = 0; i < 4; i++) {
if (lexer->lookahead != comment_start[i]) {
return false;
}
advance(lexer);
}
} else if (lexer->lookahead == '-') {
for (unsigned i = 0; i < 3; i++) {
if (lexer->lookahead != comment_end[i]) {
return false;
}
advance(lexer);
}
} else {
return false;
}
while (lexer->lookahead != 0 && lexer->lookahead != '\n' && lexer->lookahead != 0x2028 &&
lexer->lookahead != 0x2029) {
advance(lexer);
}
lexer->result_symbol = HTML_COMMENT;
lexer->mark_end(lexer);
return true;
}
static bool scan_jsx_text(TSLexer *lexer) {
// saw_text will be true if we see any non-whitespace content, or any whitespace content that is not a newline and
// does not immediately follow a newline.
bool saw_text = false;
// at_newline will be true if we are currently at a newline, or if we are at whitespace that is not a newline but
// immediately follows a newline.
bool at_newline = false;
while (lexer->lookahead != 0 && lexer->lookahead != '<' && lexer->lookahead != '>' && lexer->lookahead != '{' &&
lexer->lookahead != '}' && lexer->lookahead != '&') {
bool is_wspace = iswspace(lexer->lookahead);
if (lexer->lookahead == '\n') {
at_newline = true;
} else {
// If at_newline is already true, and we see some whitespace, then it must stay true.
// Otherwise, it should be false.
//
// See the table below to determine the logic for computing `saw_text`.
//
// |------------------------------------|
// | at_newline | is_wspace | saw_text |
// |------------|-----------|-----------|
// | false (0) | false (0) | true (1) |
// | false (0) | true (1) | true (1) |
// | true (1) | false (0) | true (1) |
// | true (1) | true (1) | false (0) |
// |------------------------------------|
at_newline &= is_wspace;
if (!at_newline) {
saw_text = true;
}
}
advance(lexer);
}
lexer->result_symbol = JSX_TEXT;
return saw_text;
}
static inline bool external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) {
if (valid_symbols[TEMPLATE_CHARS]) {
if (valid_symbols[AUTOMATIC_SEMICOLON]) {
return false;
}
return scan_template_chars(lexer);
}
if (valid_symbols[JSX_TEXT] && scan_jsx_text(lexer)) {
return true;
}
if (valid_symbols[AUTOMATIC_SEMICOLON] || valid_symbols[FUNCTION_SIGNATURE_AUTOMATIC_SEMICOLON]) {
bool scanned_comment = false;
bool ret = scan_automatic_semicolon(lexer, valid_symbols, &scanned_comment);
if (!ret && !scanned_comment && valid_symbols[TERNARY_QMARK] && lexer->lookahead == '?') {
return scan_ternary_qmark(lexer);
}
return ret;
}
if (valid_symbols[TERNARY_QMARK]) {
return scan_ternary_qmark(lexer);
}
if (valid_symbols[HTML_COMMENT] && !valid_symbols[LOGICAL_OR] && !valid_symbols[ESCAPE_SEQUENCE] &&
!valid_symbols[REGEX_PATTERN]) {
return scan_closing_comment(lexer);
}
return false;
}
internal/cbm/vendored/grammars/arkts/parser.c
Generated
+286866
View File
File diff suppressed because it is too large Load Diff
internal/cbm/vendored/grammars/arkts/scanner.c
Generated
+21
View File
@@ -0,0 +1,21 @@
// ArkTS external scanner — verbatim tree-sitter-typescript scanner logic.
//
// The scanner body lives in _common_scanner.h, a byte-identical copy of
// tree-sitter/tree-sitter-typescript common/scanner.h @ 75b3874edb2d
// (v0.23.2). Only the exported symbol names differ (arkts instead of
// typescript) because tree-sitter derives them from the grammar name.
// The external token set and order are identical to the typescript
// dialect's, which is what makes the verbatim reuse correct.
#include "_common_scanner.h"
void *tree_sitter_arkts_external_scanner_create() { return NULL; }
void tree_sitter_arkts_external_scanner_destroy(void *payload) {}
unsigned tree_sitter_arkts_external_scanner_serialize(void *payload, char *buffer) { return 0; }
void tree_sitter_arkts_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) {}
bool tree_sitter_arkts_external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) {
return external_scanner_scan(payload, lexer, valid_symbols);
}
internal/cbm/vendored/grammars/arkts/tree_sitter/alloc.h
+54
View File
@@ -0,0 +1,54 @@
#ifndef TREE_SITTER_ALLOC_H_
#define TREE_SITTER_ALLOC_H_
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
// Allow clients to override allocation functions
#ifdef TREE_SITTER_REUSE_ALLOCATOR
extern void *(*ts_current_malloc)(size_t size);
extern void *(*ts_current_calloc)(size_t count, size_t size);
extern void *(*ts_current_realloc)(void *ptr, size_t size);
extern void (*ts_current_free)(void *ptr);
#ifndef ts_malloc
#define ts_malloc ts_current_malloc
#endif
#ifndef ts_calloc
#define ts_calloc ts_current_calloc
#endif
#ifndef ts_realloc
#define ts_realloc ts_current_realloc
#endif
#ifndef ts_free
#define ts_free ts_current_free
#endif
#else
#ifndef ts_malloc
#define ts_malloc malloc
#endif
#ifndef ts_calloc
#define ts_calloc calloc
#endif
#ifndef ts_realloc
#define ts_realloc realloc
#endif
#ifndef ts_free
#define ts_free free
#endif
#endif
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_ALLOC_H_
internal/cbm/vendored/grammars/arkts/tree_sitter/array.h
+291
View File
@@ -0,0 +1,291 @@
#ifndef TREE_SITTER_ARRAY_H_
#define TREE_SITTER_ARRAY_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "./alloc.h"
#include <assert.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#ifdef _MSC_VER
#pragma warning(push)
#pragma warning(disable : 4101)
#elif defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-variable"
#endif
#define Array(T) \
struct { \
T *contents; \
uint32_t size; \
uint32_t capacity; \
}
/// Initialize an array.
#define array_init(self) \
((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
/// Create an empty array.
#define array_new() \
{ NULL, 0, 0 }
/// Get a pointer to the element at a given `index` in the array.
#define array_get(self, _index) \
(assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index])
/// Get a pointer to the first element in the array.
#define array_front(self) array_get(self, 0)
/// Get a pointer to the last element in the array.
#define array_back(self) array_get(self, (self)->size - 1)
/// Clear the array, setting its size to zero. Note that this does not free any
/// memory allocated for the array's contents.
#define array_clear(self) ((self)->size = 0)
/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
/// less than the array's current capacity, this function has no effect.
#define array_reserve(self, new_capacity) \
_array__reserve((Array *)(self), array_elem_size(self), new_capacity)
/// Free any memory allocated for this array. Note that this does not free any
/// memory allocated for the array's contents.
#define array_delete(self) _array__delete((Array *)(self))
/// Push a new `element` onto the end of the array.
#define array_push(self, element) \
(_array__grow((Array *)(self), 1, array_elem_size(self)), \
(self)->contents[(self)->size++] = (element))
/// Increase the array's size by `count` elements.
/// New elements are zero-initialized.
#define array_grow_by(self, count) \
do { \
if ((count) == 0) break; \
_array__grow((Array *)(self), count, array_elem_size(self)); \
memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
(self)->size += (count); \
} while (0)
/// Append all elements from one array to the end of another.
#define array_push_all(self, other) \
array_extend((self), (other)->size, (other)->contents)
/// Append `count` elements to the end of the array, reading their values from the
/// `contents` pointer.
#define array_extend(self, count, contents) \
_array__splice( \
(Array *)(self), array_elem_size(self), (self)->size, \
0, count, contents \
)
/// Remove `old_count` elements from the array starting at the given `index`. At
/// the same index, insert `new_count` new elements, reading their values from the
/// `new_contents` pointer.
#define array_splice(self, _index, old_count, new_count, new_contents) \
_array__splice( \
(Array *)(self), array_elem_size(self), _index, \
old_count, new_count, new_contents \
)
/// Insert one `element` into the array at the given `index`.
#define array_insert(self, _index, element) \
_array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
/// Remove one element from the array at the given `index`.
#define array_erase(self, _index) \
_array__erase((Array *)(self), array_elem_size(self), _index)
/// Pop the last element off the array, returning the element by value.
#define array_pop(self) ((self)->contents[--(self)->size])
/// Assign the contents of one array to another, reallocating if necessary.
#define array_assign(self, other) \
_array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
/// Swap one array with another
#define array_swap(self, other) \
_array__swap((Array *)(self), (Array *)(other))
/// Get the size of the array contents
#define array_elem_size(self) (sizeof *(self)->contents)
/// Search a sorted array for a given `needle` value, using the given `compare`
/// callback to determine the order.
///
/// If an existing element is found to be equal to `needle`, then the `index`
/// out-parameter is set to the existing value's index, and the `exists`
/// out-parameter is set to true. Otherwise, `index` is set to an index where
/// `needle` should be inserted in order to preserve the sorting, and `exists`
/// is set to false.
#define array_search_sorted_with(self, compare, needle, _index, _exists) \
_array__search_sorted(self, 0, compare, , needle, _index, _exists)
/// Search a sorted array for a given `needle` value, using integer comparisons
/// of a given struct field (specified with a leading dot) to determine the order.
///
/// See also `array_search_sorted_with`.
#define array_search_sorted_by(self, field, needle, _index, _exists) \
_array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists)
/// Insert a given `value` into a sorted array, using the given `compare`
/// callback to determine the order.
#define array_insert_sorted_with(self, compare, value) \
do { \
unsigned _index, _exists; \
array_search_sorted_with(self, compare, &(value), &_index, &_exists); \
if (!_exists) array_insert(self, _index, value); \
} while (0)
/// Insert a given `value` into a sorted array, using integer comparisons of
/// a given struct field (specified with a leading dot) to determine the order.
///
/// See also `array_search_sorted_by`.
#define array_insert_sorted_by(self, field, value) \
do { \
unsigned _index, _exists; \
array_search_sorted_by(self, field, (value) field, &_index, &_exists); \
if (!_exists) array_insert(self, _index, value); \
} while (0)
// Private
typedef Array(void) Array;
/// This is not what you're looking for, see `array_delete`.
static inline void _array__delete(Array *self) {
if (self->contents) {
ts_free(self->contents);
self->contents = NULL;
self->size = 0;
self->capacity = 0;
}
}
/// This is not what you're looking for, see `array_erase`.
static inline void _array__erase(Array *self, size_t element_size,
uint32_t index) {
assert(index < self->size);
char *contents = (char *)self->contents;
memmove(contents + index * element_size, contents + (index + 1) * element_size,
(self->size - index - 1) * element_size);
self->size--;
}
/// This is not what you're looking for, see `array_reserve`.
static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
if (new_capacity > self->capacity) {
if (self->contents) {
self->contents = ts_realloc(self->contents, new_capacity * element_size);
} else {
self->contents = ts_malloc(new_capacity * element_size);
}
self->capacity = new_capacity;
}
}
/// This is not what you're looking for, see `array_assign`.
static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
_array__reserve(self, element_size, other->size);
self->size = other->size;
memcpy(self->contents, other->contents, self->size * element_size);
}
/// This is not what you're looking for, see `array_swap`.
static inline void _array__swap(Array *self, Array *other) {
Array swap = *other;
*other = *self;
*self = swap;
}
/// This is not what you're looking for, see `array_push` or `array_grow_by`.
static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
uint32_t new_size = self->size + count;
if (new_size > self->capacity) {
uint32_t new_capacity = self->capacity * 2;
if (new_capacity < 8) new_capacity = 8;
if (new_capacity < new_size) new_capacity = new_size;
_array__reserve(self, element_size, new_capacity);
}
}
/// This is not what you're looking for, see `array_splice`.
static inline void _array__splice(Array *self, size_t element_size,
uint32_t index, uint32_t old_count,
uint32_t new_count, const void *elements) {
uint32_t new_size = self->size + new_count - old_count;
uint32_t old_end = index + old_count;
uint32_t new_end = index + new_count;
assert(old_end <= self->size);
_array__reserve(self, element_size, new_size);
char *contents = (char *)self->contents;
if (self->size > old_end) {
memmove(
contents + new_end * element_size,
contents + old_end * element_size,
(self->size - old_end) * element_size
);
}
if (new_count > 0) {
if (elements) {
memcpy(
(contents + index * element_size),
elements,
new_count * element_size
);
} else {
memset(
(contents + index * element_size),
0,
new_count * element_size
);
}
}
self->size += new_count - old_count;
}
/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`.
#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \
do { \
*(_index) = start; \
*(_exists) = false; \
uint32_t size = (self)->size - *(_index); \
if (size == 0) break; \
int comparison; \
while (size > 1) { \
uint32_t half_size = size / 2; \
uint32_t mid_index = *(_index) + half_size; \
comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \
if (comparison <= 0) *(_index) = mid_index; \
size -= half_size; \
} \
comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \
if (comparison == 0) *(_exists) = true; \
else if (comparison < 0) *(_index) += 1; \
} while (0)
/// Helper macro for the `_sorted_by` routines below. This takes the left (existing)
/// parameter by reference in order to work with the generic sorting function above.
#define _compare_int(a, b) ((int)*(a) - (int)(b))
#ifdef _MSC_VER
#pragma warning(pop)
#elif defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic pop
#endif
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_ARRAY_H_
internal/cbm/vendored/grammars/arkts/tree_sitter/parser.h
+286
View File
@@ -0,0 +1,286 @@
#ifndef TREE_SITTER_PARSER_H_
#define TREE_SITTER_PARSER_H_
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#define ts_builtin_sym_error ((TSSymbol)-1)
#define ts_builtin_sym_end 0
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
#ifndef TREE_SITTER_API_H_
typedef uint16_t TSStateId;
typedef uint16_t TSSymbol;
typedef uint16_t TSFieldId;
typedef struct TSLanguage TSLanguage;
typedef struct TSLanguageMetadata {
uint8_t major_version;
uint8_t minor_version;
uint8_t patch_version;
} TSLanguageMetadata;
#endif
typedef struct {
TSFieldId field_id;
uint8_t child_index;
bool inherited;
} TSFieldMapEntry;
// Used to index the field and supertype maps.
typedef struct {
uint16_t index;
uint16_t length;
} TSMapSlice;
typedef struct {
bool visible;
bool named;
bool supertype;
} TSSymbolMetadata;
typedef struct TSLexer TSLexer;
struct TSLexer {
int32_t lookahead;
TSSymbol result_symbol;
void (*advance)(TSLexer *, bool);
void (*mark_end)(TSLexer *);
uint32_t (*get_column)(TSLexer *);
bool (*is_at_included_range_start)(const TSLexer *);
bool (*eof)(const TSLexer *);
void (*log)(const TSLexer *, const char *, ...);
};
typedef enum {
TSParseActionTypeShift,
TSParseActionTypeReduce,
TSParseActionTypeAccept,
TSParseActionTypeRecover,
} TSParseActionType;
typedef union {
struct {
uint8_t type;
TSStateId state;
bool extra;
bool repetition;
} shift;
struct {
uint8_t type;
uint8_t child_count;
TSSymbol symbol;
int16_t dynamic_precedence;
uint16_t production_id;
} reduce;
uint8_t type;
} TSParseAction;
typedef struct {
uint16_t lex_state;
uint16_t external_lex_state;
} TSLexMode;
typedef struct {
uint16_t lex_state;
uint16_t external_lex_state;
uint16_t reserved_word_set_id;
} TSLexerMode;
typedef union {
TSParseAction action;
struct {
uint8_t count;
bool reusable;
} entry;
} TSParseActionEntry;
typedef struct {
int32_t start;
int32_t end;
} TSCharacterRange;
struct TSLanguage {
uint32_t abi_version;
uint32_t symbol_count;
uint32_t alias_count;
uint32_t token_count;
uint32_t external_token_count;
uint32_t state_count;
uint32_t large_state_count;
uint32_t production_id_count;
uint32_t field_count;
uint16_t max_alias_sequence_length;
const uint16_t *parse_table;
const uint16_t *small_parse_table;
const uint32_t *small_parse_table_map;
const TSParseActionEntry *parse_actions;
const char * const *symbol_names;
const char * const *field_names;
const TSMapSlice *field_map_slices;
const TSFieldMapEntry *field_map_entries;
const TSSymbolMetadata *symbol_metadata;
const TSSymbol *public_symbol_map;
const uint16_t *alias_map;
const TSSymbol *alias_sequences;
const TSLexerMode *lex_modes;
bool (*lex_fn)(TSLexer *, TSStateId);
bool (*keyword_lex_fn)(TSLexer *, TSStateId);
TSSymbol keyword_capture_token;
struct {
const bool *states;
const TSSymbol *symbol_map;
void *(*create)(void);
void (*destroy)(void *);
bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist);
unsigned (*serialize)(void *, char *);
void (*deserialize)(void *, const char *, unsigned);
} external_scanner;
const TSStateId *primary_state_ids;
const char *name;
const TSSymbol *reserved_words;
uint16_t max_reserved_word_set_size;
uint32_t supertype_count;
const TSSymbol *supertype_symbols;
const TSMapSlice *supertype_map_slices;
const TSSymbol *supertype_map_entries;
TSLanguageMetadata metadata;
};
static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
uint32_t index = 0;
uint32_t size = len - index;
while (size > 1) {
uint32_t half_size = size / 2;
uint32_t mid_index = index + half_size;
const TSCharacterRange *range = &ranges[mid_index];
if (lookahead >= range->start && lookahead <= range->end) {
return true;
} else if (lookahead > range->end) {
index = mid_index;
}
size -= half_size;
}
const TSCharacterRange *range = &ranges[index];
return (lookahead >= range->start && lookahead <= range->end);
}
/*
* Lexer Macros
*/
#ifdef _MSC_VER
#define UNUSED __pragma(warning(suppress : 4101))
#else
#define UNUSED __attribute__((unused))
#endif
#define START_LEXER() \
bool result = false; \
bool skip = false; \
UNUSED \
bool eof = false; \
int32_t lookahead; \
goto start; \
next_state: \
lexer->advance(lexer, skip); \
start: \
skip = false; \
lookahead = lexer->lookahead;
#define ADVANCE(state_value) \
{ \
state = state_value; \
goto next_state; \
}
#define ADVANCE_MAP(...) \
{ \
static const uint16_t map[] = { __VA_ARGS__ }; \
for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \
if (map[i] == lookahead) { \
state = map[i + 1]; \
goto next_state; \
} \
} \
}
#define SKIP(state_value) \
{ \
skip = true; \
state = state_value; \
goto next_state; \
}
#define ACCEPT_TOKEN(symbol_value) \
result = true; \
lexer->result_symbol = symbol_value; \
lexer->mark_end(lexer);
#define END_STATE() return result;
/*
* Parse Table Macros
*/
#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
#define STATE(id) id
#define ACTIONS(id) id
#define SHIFT(state_value) \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = (state_value) \
} \
}}
#define SHIFT_REPEAT(state_value) \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = (state_value), \
.repetition = true \
} \
}}
#define SHIFT_EXTRA() \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.extra = true \
} \
}}
#define REDUCE(symbol_name, children, precedence, prod_id) \
{{ \
.reduce = { \
.type = TSParseActionTypeReduce, \
.symbol = symbol_name, \
.child_count = children, \
.dynamic_precedence = precedence, \
.production_id = prod_id \
}, \
}}
#define RECOVER() \
{{ \
.type = TSParseActionTypeRecover \
}}
#define ACCEPT_INPUT() \
{{ \
.type = TSParseActionTypeAccept \
}}
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_PARSER_H_
scripts/audit-license-provenance.py
+7
View File
@@ -22,6 +22,13 @@ GRAMMARS = os.path.join(ROOT, "internal/cbm/vendored/grammars")
FIRST_PARTY = {"cobol", "form", "janet", "magma", "protobuf", "wolfram"}
FORKS = { # self-maintained forks: vendored LICENSE must match the original upstream
# arkts: our own ArkTS fork of tree-sitter-typescript. The vendored LICENSE is
# upstream's MIT verbatim, which is what MIT requires of a derivative; our own
# copyright and the fork's provenance live where our SOURCE is (grammar.js
# header, MANIFEST.md, THIRD_PARTY.md) rather than inside upstream's notice.
# This is a routing entry, NOT an exemption -- check_upstream still byte-verifies
# against the real upstream repository.
"arkts": "tree-sitter/tree-sitter-typescript",
"cfml": "cfmleditor/tree-sitter-cfml",
"cfscript": "cfmleditor/tree-sitter-cfml",
"dotenv": "pnx/tree-sitter-dotenv",
scripts/vendored-checksums.txt
+8 -1
View File
@@ -5,7 +5,7 @@ c5cfb43042b6b72045f4ba997834d0a7786d2793d91680868b5815b39f14fc78 internal/cbm/v
b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea internal/cbm/vendored/common/tree_sitter/alloc.h
31e60a1bff6f715afacce03b5b70efe42b58371b4f9595dd4af52a577ff9608c internal/cbm/vendored/common/tree_sitter/array.h
180b893c8734778fd32f372dfbc27bd6ad1cd2221f26150b31256ff6716320d2 internal/cbm/vendored/common/tree_sitter/parser.h
351738d52ef14e574aa69d4a9b95f8d7b6eddc2658fef91bab95a72d00687b08 internal/cbm/vendored/grammars/MANIFEST.md
12419cd10df8b0a38f29dfcdf89ce42ea7837711e87e923631a136adbac6685d internal/cbm/vendored/grammars/MANIFEST.md
ad8425038de519f8c4e3e9339feebf99dfad8a6002dcf79227d91402780a32cc internal/cbm/vendored/grammars/ada/LICENSE
02805ec13939b749c891567be36bf024b09034e04c80683a1ae667272458d549 internal/cbm/vendored/grammars/ada/parser.c
115a75d000bef9c70c4de7dfbb7f2a80b90cbb14ba265056dc7abb1ef3b9b6db internal/cbm/vendored/grammars/ada/tree_sitter/alloc.h
@@ -22,6 +22,13 @@ e960fdd8672cea2a93887913017f231153695ca1778ffd3fcdfceac97ef6ffec internal/cbm/v
115a75d000bef9c70c4de7dfbb7f2a80b90cbb14ba265056dc7abb1ef3b9b6db internal/cbm/vendored/grammars/apex/tree_sitter/alloc.h
02d139d44303b0b0b85a899839dbc1346c478e7318d1f2971fad2b38fd57cb73 internal/cbm/vendored/grammars/apex/tree_sitter/array.h
2db740ea1be6b71014d2be1385491f01bf2d15b607e61c5b30b4128535efe68a internal/cbm/vendored/grammars/apex/tree_sitter/parser.h
49bf33cf78ef5897e4e161ce1517df7de1ae5042a65b6bcfd44401e0fc606559 internal/cbm/vendored/grammars/arkts/LICENSE
da66ef2bd14a3f7ea743e25ba068c6c9aae2c3509db200ff80c4a0e6116e564c internal/cbm/vendored/grammars/arkts/_common_scanner.h
a7973090c8c59fcd54f23f10586c67d02164fdedc2f79d12eae8cb144911785f internal/cbm/vendored/grammars/arkts/parser.c
4be8f28ab087be489ede143618fd578732e6719a604af0bb5212d3a5c769e8f9 internal/cbm/vendored/grammars/arkts/scanner.c
b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea internal/cbm/vendored/grammars/arkts/tree_sitter/alloc.h
5bdf6ed1a78e3409fd443e085ca967a64c188a5d082aaf7f819bccd53a471c94 internal/cbm/vendored/grammars/arkts/tree_sitter/array.h
180b893c8734778fd32f372dfbc27bd6ad1cd2221f26150b31256ff6716320d2 internal/cbm/vendored/grammars/arkts/tree_sitter/parser.h
0e9f43573925c2e4b49f328a45c4df22c882191d77d876b0d18ecd60bd21e17c internal/cbm/vendored/grammars/assembly/LICENSE
dbcd8f007af7af67ef58bea0b1f7d3121902915df80ccce23588f0ac48438b35 internal/cbm/vendored/grammars/assembly/parser.c
253b44a7b4313a7afd0c505c2fc6e7ce4b8e78955ebf4be3ea000532ec060673 internal/cbm/vendored/grammars/assembly/tree_sitter/alloc.h
src/cypher/cypher.c
+22 -6
View File
@@ -3180,6 +3180,7 @@ static void process_edges(cbm_store_t *store, cbm_edge_t *edges, int edge_count,
/* C11 _Thread_local directly: cypher.c stays windows.h-free (compat.h pulls
* in windows.h, whose legacy `far` macro breaks this file's identifiers). */
static _Thread_local int g_cypher_depth_clamped = 0;
static _Thread_local int g_cypher_trail_truncated = 0;
static void expand_var_length(cbm_store_t *store, cbm_rel_pattern_t *rel,
cbm_node_pattern_t *target_node, binding_t *b, cbm_node_t *src,
@@ -3202,7 +3203,11 @@ static void expand_var_length(cbm_store_t *store, cbm_rel_pattern_t *rel,
}
cbm_traverse_result_t tr = {0};
const char *dir = rel->direction ? rel->direction : "outbound";
cbm_store_bfs(store, src->id, dir, rel->types, rel->type_count, max_depth, CBM_PERCENT, &tr);
cbm_store_bfs_trail(store, src->id, dir, rel->types, rel->type_count, max_depth, CBM_PERCENT,
&tr);
if (tr.truncated) {
g_cypher_trail_truncated = 1;
}
cbm_node_t *bound_to = binding_get(b, to_var);
int64_t bound_to_id = bound_to ? bound_to->id : 0;
/* Same contract as process_edges: the budget caps materialisation, never
@@ -4893,6 +4898,7 @@ int cbm_cypher_execute(cbm_store_t *store, const char *query, const char *projec
cbm_cypher_result_t *out) {
memset(out, 0, sizeof(*out));
g_cypher_depth_clamped = 0;
g_cypher_trail_truncated = 0;
cypher_deadline_arm(); /* #601: start the wall-clock budget for this query */
if (max_rows <= 0) {
max_rows = CYPHER_RESULT_CEILING;
@@ -4963,12 +4969,22 @@ int cbm_cypher_execute(cbm_store_t *store, const char *query, const char *projec
out->col_count = rb.col_count;
out->rows = rb.rows;
out->row_count = rb.row_count;
if (g_cypher_depth_clamped > 0) {
if (g_cypher_depth_clamped > 0 || g_cypher_trail_truncated) {
char wbuf[CBM_SZ_256];
snprintf(wbuf, sizeof(wbuf),
"variable-length hop range clamped to the engine ceiling (%d) — an empty "
"result may mean \"clamped\", not \"no such path\"",
g_cypher_depth_clamped);
if (g_cypher_depth_clamped > 0 && g_cypher_trail_truncated) {
snprintf(wbuf, sizeof(wbuf),
"variable-length hop range clamped to the engine ceiling (%d) and "
"traversal budget was exhausted — results may be partial",
g_cypher_depth_clamped);
} else if (g_cypher_depth_clamped > 0) {
snprintf(wbuf, sizeof(wbuf),
"variable-length hop range clamped to the engine ceiling (%d) — an empty "
"result may mean \"clamped\", not \"no such path\"",
g_cypher_depth_clamped);
} else {
snprintf(wbuf, sizeof(wbuf),
"variable-length traversal budget was exhausted — results may be partial");
}
out->warning = heap_strdup(wbuf);
}
src/discover/language.c
+4
View File
@@ -90,6 +90,9 @@ static const ext_entry_t EXT_TABLE[] = {
/* Elm */
{".elm", CBM_LANG_ELM},
/* ArkTS (HarmonyOS/OpenHarmony) */
{".ets", CBM_LANG_ARKTS},
/* Emacs Lisp */
{".el", CBM_LANG_EMACSLISP},
@@ -850,6 +853,7 @@ static const char *LANG_NAMES[CBM_LANG_COUNT] = {
[CBM_LANG_OBJECTSCRIPT_UDL] = "ObjectScript UDL",
[CBM_LANG_OBJECTSCRIPT_ROUTINE] = "ObjectScript Routine",
[CBM_LANG_OBJECTSCRIPT_EXPORT] = "ObjectScript Export XML",
[CBM_LANG_ARKTS] = "ArkTS",
};
src/discover/userconfig.c
+1
View File
@@ -75,6 +75,7 @@ static const lang_name_entry_t LANG_NAME_TABLE[] = {
{"javascript", CBM_LANG_JAVASCRIPT},
{"typescript", CBM_LANG_TYPESCRIPT},
{"tsx", CBM_LANG_TSX},
{"arkts", CBM_LANG_ARKTS},
{"rust", CBM_LANG_RUST},
{"java", CBM_LANG_JAVA},
{"c++", CBM_LANG_CPP},
src/pipeline/pass_calls.c
+2 -2
View File
@@ -616,8 +616,8 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call,
* defer to emit_classified_edge and suppress ONLY the plain-CALLS
* fall-through, so every service edge stays main-identical. res.strategy may
* be lsp_* here; the helper's explicit drop-list leaves lsp_* untouched. */
bool is_tsjs =
lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX;
bool is_tsjs = lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT ||
lang == CBM_LANG_TSX || lang == CBM_LANG_ARKTS;
bool tsjs_drop_plain_call =
cbm_tsjs_suppress_weak_method_match(is_tsjs, call->is_method, res.strategy);
src/pipeline/pass_parallel.c
+2 -2
View File
@@ -2468,8 +2468,8 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB
* construction. res.strategy may carry an lsp_* value here (LSP-resolved
* calls keep res through this point); the helper's EXPLICIT drop-list
* leaves lsp_ts_method / lsp_cross untouched. See #606 direction. */
bool is_tsjs =
lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX;
bool is_tsjs = lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT ||
lang == CBM_LANG_TSX || lang == CBM_LANG_ARKTS;
bool tsjs_drop_plain_call =
cbm_tsjs_suppress_weak_method_match(is_tsjs, call->is_method, res.strategy);
src/pipeline/registry.c
+2 -1
View File
@@ -457,7 +457,8 @@ bool cbm_tsjs_suppress_weak_method_match(bool is_tsjs, bool is_method, const cha
}
static bool js_ts_family(CBMLanguage lang) {
return lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX;
return lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX ||
lang == CBM_LANG_ARKTS;
}
static const char *path_basename(const char *path) {
src/store/store.c
+109 -28
View File
@@ -25,6 +25,7 @@ enum {
ST_COL_7 = 7,
ST_COL_8 = 8,
ST_COL_9 = 9,
ST_COL_10 = 10,
ST_FOUND = -1,
ST_BUF_16 = 16,
ST_BUF_64 = 64,
@@ -63,6 +64,8 @@ enum {
ST_METHOD_PROP_LEN = 8,
ST_PATH_PROP_LEN = 6,
ST_HANDLER_PROP_LEN = 9,
ST_BFS_CTE_ROW_MULTIPLIER = 8,
ST_BFS_MAX_CTE_ROWS = 4096,
};
#define SLEN(s) (sizeof(s) - 1)
@@ -4673,8 +4676,35 @@ static void bfs_build_types_clause(int edge_type_count, char *buf, int buf_sz) {
}
}
int cbm_store_bfs(cbm_store_t *s, int64_t start_id, const char *direction, const char **edge_types,
int edge_type_count, int max_depth, int max_results, cbm_traverse_result_t *out) {
static int bfs_cte_row_limit(int max_results) {
int result_budget = max_results > 0 ? max_results : ST_INIT_CAP_16;
long row_budget = (long)result_budget * ST_BFS_CTE_ROW_MULTIPLIER + SKIP_ONE;
if (row_budget > ST_BFS_MAX_CTE_ROWS) {
return ST_BFS_MAX_CTE_ROWS;
}
if (row_budget < ST_INIT_CAP_16) {
return ST_INIT_CAP_16;
}
return (int)row_budget;
}
static int bfs_cte_row_limit_for_depth(int max_results, int max_depth) {
int base = bfs_cte_row_limit(max_results);
/* Reserve rows across the requested depth range. A result-sized cap can
* be exhausted by a wide first hop before any deeper match is generated;
* scaling by depth keeps the bounded traversal useful for *N..N queries
* while the hard ceiling still protects the SQLite worker. */
long depth = max_depth > 0 ? max_depth : ST_INIT_CAP_16;
long scaled = (long)base * depth;
if (scaled > ST_BFS_MAX_CTE_ROWS) {
return ST_BFS_MAX_CTE_ROWS;
}
return (int)scaled;
}
static int store_bfs(cbm_store_t *s, int64_t start_id, const char *direction,
const char **edge_types, int edge_type_count, int max_depth, int max_results,
bool trail, cbm_traverse_result_t *out) {
memset(out, 0, sizeof(*out));
cbm_node_t root = {0};
@@ -4701,32 +4731,59 @@ int cbm_store_bfs(cbm_store_t *s, int64_t start_id, const char *direction, const
next_id = "e.target_id";
}
snprintf(sql, sizeof(sql),
/* SHORTEST-PATH semantics: the UNION dedupes (node, hop) PAIRS,
* so a single self-loop minted every hop level for every node it
* could reach — walk-padding that fabricated *k..k Cypher matches
* of arbitrary length and exploded the row set to nodes x depth
* (#797). MIN(hop) GROUP BY node returns each node once at its
* minimal distance. */
"WITH RECURSIVE bfs(node_id, hop) AS ("
" SELECT %lld, 0"
" UNION"
" SELECT %s, bfs.hop + 1"
" FROM bfs"
" JOIN edges e ON %s"
" WHERE e.type IN (%s) AND bfs.hop < %d"
")"
"SELECT n.id, n.project, n.label, n.name, n.qualified_name, "
"n.file_path, n.start_line, n.end_line, n.properties, MIN(bfs.hop) AS hop "
"FROM bfs "
"JOIN nodes n ON n.id = bfs.node_id "
"WHERE bfs.hop > 0 " /* exclude root at hop 0 (self via a loop still appears) */
"GROUP BY n.id "
/* (hop, id) is a unique total order — deterministic pagination
* watermarks and reproducible trace output depend on it. */
"ORDER BY hop, n.id "
"LIMIT %d;",
(long long)start_id, next_id, join_cond, types_clause, max_depth, max_results);
int cte_row_limit = 0;
if (trail) {
cte_row_limit = bfs_cte_row_limit_for_depth(max_results, max_depth);
snprintf(sql, sizeof(sql),
"WITH RECURSIVE bfs(node_id, hop, edge_path) AS ("
" SELECT %lld, 0, ''"
" UNION"
" SELECT %s, bfs.hop + 1,"
" CASE WHEN bfs.edge_path = '' THEN CAST(e.id AS TEXT)"
" ELSE bfs.edge_path || ',' || e.id END"
" FROM bfs"
" JOIN edges e ON %s"
" WHERE e.type IN (%s) AND bfs.hop < %d"
" AND instr(',' || bfs.edge_path || ',', ',' || e.id || ',') = 0"
/* SQLite's recursive queue is breadth-first by default. A
* bounded trail budget must reach the requested depth before
* a wide shallow layer consumes every row, so prioritize the
* recursive hop (depth-first) while retaining deterministic
* ordering in the outer result query. */
" ORDER BY 2 DESC, 1 ASC, 3 ASC"
" LIMIT %d"
")"
"SELECT DISTINCT n.id, n.project, n.label, n.name, n.qualified_name, "
"n.file_path, n.start_line, n.end_line, n.properties, bfs.hop, "
"(SELECT count(*) FROM bfs) "
"FROM bfs JOIN nodes n ON n.id = bfs.node_id "
"WHERE bfs.hop > 0 ORDER BY bfs.hop, n.id LIMIT %d;",
(long long)start_id, next_id, join_cond, types_clause, max_depth,
cte_row_limit + SKIP_ONE, max_results);
} else {
snprintf(sql, sizeof(sql),
/* SHORTEST-PATH semantics: the UNION dedupes (node, hop) pairs.
* MIN(hop) GROUP BY node returns each node once at its minimal
* distance, while relationship-trail state remains scoped to
* Cypher variable-length expansion. */
"WITH RECURSIVE bfs(node_id, hop) AS ("
" SELECT %lld, 0"
" UNION"
" SELECT %s, bfs.hop + 1"
" FROM bfs"
" JOIN edges e ON %s"
" WHERE e.type IN (%s) AND bfs.hop < %d"
")"
"SELECT n.id, n.project, n.label, n.name, n.qualified_name, "
"n.file_path, n.start_line, n.end_line, n.properties, MIN(bfs.hop) AS hop "
"FROM bfs "
"JOIN nodes n ON n.id = bfs.node_id "
"WHERE bfs.hop > 0 "
"GROUP BY n.id "
"ORDER BY hop, n.id "
"LIMIT %d;",
(long long)start_id, next_id, join_cond, types_clause, max_depth, max_results);
}
sqlite3_stmt *stmt = NULL;
rc = sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL);
@@ -4746,6 +4803,7 @@ int cbm_store_bfs(cbm_store_t *s, int64_t start_id, const char *direction, const
int cap = ST_INIT_CAP_16;
int n = 0;
int cte_rows = 0;
cbm_node_hop_t *visited = malloc(cap * sizeof(cbm_node_hop_t));
int scan_rc15;
@@ -4756,6 +4814,9 @@ int cbm_store_bfs(cbm_store_t *s, int64_t start_id, const char *direction, const
}
scan_node(stmt, &visited[n].node);
visited[n].hop = sqlite3_column_int(stmt, ST_COL_9);
if (trail) {
cte_rows = sqlite3_column_int(stmt, ST_COL_10);
}
n++;
}
if (scan_rc15 != SQLITE_DONE) { /* SCANCHK:15:stmt */
@@ -4768,6 +4829,13 @@ int cbm_store_bfs(cbm_store_t *s, int64_t start_id, const char *direction, const
sqlite3_finalize(stmt);
if (trail && cte_rows > cte_row_limit) {
out->truncated = true;
char limit_buf[16];
snprintf(limit_buf, sizeof(limit_buf), "%d", cte_row_limit);
cbm_log_warn("cypher.trail_truncated", "cte_rows", limit_buf, "result", "partial");
}
out->visited = visited;
out->visited_count = n;
@@ -4783,6 +4851,19 @@ int cbm_store_bfs(cbm_store_t *s, int64_t start_id, const char *direction, const
return CBM_STORE_OK;
}
int cbm_store_bfs(cbm_store_t *s, int64_t start_id, const char *direction, const char **edge_types,
int edge_type_count, int max_depth, int max_results, cbm_traverse_result_t *out) {
return store_bfs(s, start_id, direction, edge_types, edge_type_count, max_depth, max_results,
false, out);
}
int cbm_store_bfs_trail(cbm_store_t *s, int64_t start_id, const char *direction,
const char **edge_types, int edge_type_count, int max_depth,
int max_results, cbm_traverse_result_t *out) {
return store_bfs(s, start_id, direction, edge_types, edge_type_count, max_depth, max_results,
true, out);
}
/* Multi-source BFS: one recursive CTE anchored on ALL seeds (via a temp
* table — never a per-seed loop, which re-walks overlapping hub subgraphs
* seed_count times). Semantics for impact analysis:
src/store/store.h
+7
View File
@@ -211,6 +211,8 @@ typedef struct {
int visited_count;
cbm_edge_info_t *edges;
int edge_count;
/* True when trail expansion hit its recursive-row safety budget. */
bool truncated;
} cbm_traverse_result_t;
/* ── Schema introspection ───────────────────────────────────────── */
@@ -603,6 +605,11 @@ void cbm_store_search_free(cbm_search_output_t *out);
int cbm_store_bfs(cbm_store_t *s, int64_t start_id, const char *direction, const char **edge_types,
int edge_type_count, int max_depth, int max_results, cbm_traverse_result_t *out);
/* Variable-length Cypher traversal with relationship-trail semantics. */
int cbm_store_bfs_trail(cbm_store_t *s, int64_t start_id, const char *direction,
const char **edge_types, int edge_type_count, int max_depth,
int max_results, cbm_traverse_result_t *out);
/* Multi-source BFS from ALL seed ids at once (one CTE, temp-table anchored).
* Seeds are EXCLUDED from the result (impact semantics); MIN(hop) across the
* seed set; canonical (hop,id) order; *truncated set when the max_results
tests/repro/repro_call_argument_matrix_a.c
+12 -3
View File
@@ -605,6 +605,14 @@ SEMANTIC_CASE(tsx, TSX, "main.tsx",
"function bare(handler: number): number { return handler; }\n",
"handler", "call_expression", 2, 2);
SEMANTIC_CASE(arkts, ARKTS, "main.ets",
"function handler(): number { return 0; }\n"
"function accept(value: number): number { return value; }\n"
"function argument(handler: number): number { return accept(handler); }\n"
"function direct(): number { return handler(); }\n"
"function bare(handler: number): number { return handler; }\n",
"handler", "call_expression", 2, 2);
SEMANTIC_CASE(rust, RUST, "main.rs",
"fn handler() -> i32 { 0 }\n"
"fn accept(value: i32) -> i32 { value }\n"
@@ -1244,7 +1252,8 @@ SEMANTIC_CASE(agda, AGDA, "Main.agda",
X(zsh) \
X(tcl) \
X(ada) \
X(agda)
X(agda) \
X(arkts)
#define DEFINE_MATRIX_TEST(tag_) \
TEST(repro_call_argument_matrix_a_##tag_) { \
@@ -1259,8 +1268,8 @@ MATRIX_A_CASES(DEFINE_MATRIX_TEST)
static const MatrixCase *const MATRIX_A_CASE_ROWS[] = {MATRIX_A_CASES(MATRIX_CASE_POINTER)};
#undef MATRIX_CASE_POINTER
_Static_assert(sizeof(MATRIX_A_CASE_ROWS) / sizeof(MATRIX_A_CASE_ROWS[0]) == 67,
"GO..AGDA call-capable matrix must contain exactly 67 language rows");
_Static_assert(sizeof(MATRIX_A_CASE_ROWS) / sizeof(MATRIX_A_CASE_ROWS[0]) == 68,
"GO..ARKTS call-capable matrix must contain exactly 68 language rows");
size_t repro_call_argument_matrix_a_copy_language_ids(CBMLanguage *language_ids, size_t capacity) {
size_t row_count = sizeof(MATRIX_A_CASE_ROWS) / sizeof(MATRIX_A_CASE_ROWS[0]);
tests/repro/repro_call_node_manifest.c
+8 -6
View File
@@ -54,10 +54,10 @@ typedef struct {
} CallNodeManifestEntry;
enum {
EXPECTED_HISTORICAL_CALL_NODE_TOTAL = 219,
EXPECTED_ACTIVE_PRIMARY_TOTAL = 189,
EXPECTED_DIRECT_CALLS = 155,
EXPECTED_CONSTRUCTOR_CALLS = 20,
EXPECTED_HISTORICAL_CALL_NODE_TOTAL = 221,
EXPECTED_ACTIVE_PRIMARY_TOTAL = 191,
EXPECTED_DIRECT_CALLS = 156,
EXPECTED_CONSTRUCTOR_CALLS = 21,
EXPECTED_OPERATOR_CALLS = 12,
EXPECTED_IMPLICIT_CALLS = 2,
EXPECTED_DSL_INVOCATIONS = 14,
@@ -65,7 +65,7 @@ enum {
EXPECTED_CALLEE_WRAPPERS = 8,
EXPECTED_ARGUMENT_WRAPPERS = 1,
EXPECTED_CONTROL_OR_DOCUMENT_NONCALLS = 6,
EXPECTED_PRIMARY_OWNERS = 189,
EXPECTED_PRIMARY_OWNERS = 191,
EXPECTED_SYNTHETIC_OWNERS = 10,
EXPECTED_NO_CALL_EDGE_OWNERS = 20,
EXPECTED_PRIMARY_OPERATOR_CALLS = 4,
@@ -309,6 +309,8 @@ static const CallNodeManifestEntry CALL_NODE_MANIFEST[] = {
true),
ENTRY(CBM_LANG_OBJECTSCRIPT_ROUTINE, "routine_tag_call", DIRECT_CALL, PRIMARY_EXTRACTOR, true,
true),
ENTRY(CBM_LANG_ARKTS, "call_expression", DIRECT_CALL, PRIMARY_EXTRACTOR, true, true),
ENTRY(CBM_LANG_ARKTS, "new_expression", CONSTRUCTOR_CALL, PRIMARY_EXTRACTOR, true, true),
};
#undef ENTRY
@@ -316,7 +318,7 @@ static const CallNodeManifestEntry CALL_NODE_MANIFEST[] = {
#define MANIFEST_COUNT (sizeof(CALL_NODE_MANIFEST) / sizeof(CALL_NODE_MANIFEST[0]))
_Static_assert(MANIFEST_COUNT == EXPECTED_HISTORICAL_CALL_NODE_TOTAL,
"call-node manifest must contain exactly 219 entries");
"call-node manifest must contain exactly 221 entries");
_Static_assert(EXPECTED_DIRECT_CALLS + EXPECTED_CONSTRUCTOR_CALLS + EXPECTED_OPERATOR_CALLS +
EXPECTED_IMPLICIT_CALLS + EXPECTED_DSL_INVOCATIONS +
EXPECTED_BUILD_DEPENDENCIES + EXPECTED_CALLEE_WRAPPERS +
tests/repro/repro_language_registry.c
+4 -3
View File
@@ -210,6 +210,7 @@ static const LanguageCapabilityEntry LANGUAGE_CAPABILITIES[CBM_LANG_COUNT] = {
CALL_WITH_REFERENCE_VOCAB(OBJECTSCRIPT_UDL),
CALL_WITH_REFERENCE_VOCAB(OBJECTSCRIPT_ROUTINE),
TRANSFORM_ONLY(OBJECTSCRIPT_EXPORT),
CALL_WITH_REFERENCE_VOCAB(ARKTS),
};
#undef CALL_WITH_REFERENCE_VOCAB
@@ -298,7 +299,7 @@ TEST(repro_language_capability_ledger_covers_every_enum) {
}
}
if (counts[CAP_CALL_WITH_REFERENCE_VOCAB] != 86 ||
if (counts[CAP_CALL_WITH_REFERENCE_VOCAB] != 87 ||
counts[CAP_CALL_WITHOUT_REFERENCE_VOCAB] != 25 || counts[CAP_NO_CALL] != 50 ||
counts[CAP_TRANSFORM_ONLY] != 1 || counts[CAP_UNSUPPORTED] != 1) {
fprintf(stderr,
@@ -315,9 +316,9 @@ TEST(repro_language_capability_ledger_covers_every_enum) {
TEST(repro_call_argument_matrices_equal_call_capability_ledger) {
enum {
EXPECTED_MATRIX_A_ROWS = 67,
EXPECTED_MATRIX_A_ROWS = 68,
EXPECTED_MATRIX_B_ROWS = 46,
EXPECTED_CALL_CAPABLE_LANGUAGES = 111,
EXPECTED_CALL_CAPABLE_LANGUAGES = 112,
EXPECTED_NON_CALL_LANGUAGES = 52,
EXPECTED_NON_CALL_DOMAIN_CONTROLS = 2,
};
tests/test_cypher.c
+82 -3
View File
@@ -1171,15 +1171,24 @@ TEST(cypher_exec_varlength_path_semantics_issue797) {
ASSERT_EQ(r1.row_count, 1);
cbm_cypher_result_free(&r1);
/* Bug 2: *2..2 from loopy — only the REAL 2-chain (leaf); the self-loop
* must not be reused to pad paths (relationship uniqueness). */
/* Bug 2: *2..2 from loopy has two relationship-unique trails: the
* self-loop followed by e1 reaches mid, and e1 followed by e2 reaches leaf.
* Reusing the self-loop within one trail remains forbidden. */
cbm_cypher_result_t r2 = {0};
ASSERT_EQ(cbm_cypher_execute(s,
"MATCH (a {name: \"loopy\"})-[:CALLS*2..2]->(b) "
"RETURN DISTINCT b.name",
"test", 0, &r2),
0);
ASSERT_EQ(r2.row_count, 1); /* leaf only */
ASSERT_EQ(r2.row_count, 2);
bool saw_mid = false;
bool saw_leaf = false;
for (int i = 0; i < r2.row_count; i++) {
saw_mid |= strcmp(r2.rows[i][0], "mid") == 0;
saw_leaf |= strcmp(r2.rows[i][0], "leaf") == 0;
}
ASSERT_TRUE(saw_mid);
ASSERT_TRUE(saw_leaf);
cbm_cypher_result_free(&r2);
/* Bug 2 amplifier: no directed path of length 5 exists at all. */
@@ -1798,6 +1807,74 @@ TEST(cypher_exec_var_length_explicit_bound_capped) {
PASS();
}
/* Pin the relationship-trail contract: a self-loop edge cannot be reused
* within one variable-length trail, so *2..2 yields no fabricated match. */
TEST(cypher_exec_var_length_no_reuse_self_loop) {
cbm_store_t *s = cbm_store_open_memory();
cbm_store_upsert_project(s, "test", "/tmp/test");
cbm_node_t n = {.project = "test",
.label = "Function",
.name = "Recursive",
.qualified_name = "test.Recursive",
.file_path = "recursive.go"};
int64_t id = cbm_store_upsert_node(s, &n);
cbm_edge_t e = {.project = "test", .source_id = id, .target_id = id, .type = "CALLS"};
cbm_store_insert_edge(s, &e);
cbm_cypher_result_t r = {0};
int rc = cbm_cypher_execute(s,
"MATCH (f:Function {name: \"Recursive\"})-[:CALLS*2..2]"
"->(g:Function) RETURN g.name",
"test", 0, &r);
ASSERT_EQ(rc, 0);
ASSERT_EQ(r.row_count, 0);
cbm_cypher_result_free(&r);
cbm_store_close(s);
PASS();
}
TEST(cypher_exec_var_length_truncation_surfaces_warning) {
cbm_store_t *s = cbm_store_open_memory();
cbm_store_upsert_project(s, "test", "/tmp/test");
int64_t ids[18];
for (int i = 0; i < 18; i++) {
char name[16];
snprintf(name, sizeof(name), "node-%d", i);
cbm_node_t node = {.project = "test",
.label = "Function",
.name = name,
.qualified_name = name,
.file_path = "graph.c"};
ids[i] = cbm_store_upsert_node(s, &node);
}
for (int source = 0; source < 17; source++) {
for (int target = source + 1; target < 18; target++) {
cbm_edge_t edge = {.project = "test",
.source_id = ids[source],
.target_id = ids[target],
.type = "CALLS"};
cbm_store_insert_edge(s, &edge);
}
}
cbm_cypher_result_t result = {0};
int rc = cbm_cypher_execute(s,
"MATCH (a:Function {name: \"node-0\"})-[:CALLS*10..10]->"
"(b:Function) RETURN b.name",
"test", 0, &result);
ASSERT_EQ(rc, 0);
ASSERT_NOT_NULL(result.warning);
ASSERT_TRUE(strstr(result.warning, "traversal budget") != NULL);
ASSERT_TRUE(result.row_count > 0);
cbm_cypher_result_free(&result);
cbm_store_close(s);
PASS();
}
TEST(cypher_exec_defines_edge) {
cbm_store_t *s = setup_cypher_store();
cbm_cypher_result_t r = {0};
@@ -4037,6 +4114,8 @@ SUITE(cypher) {
RUN_TEST(cypher_exec_variable_length);
RUN_TEST(cypher_exec_variable_length_repeated_node_var_unifies);
RUN_TEST(cypher_exec_var_length_explicit_bound_capped);
RUN_TEST(cypher_exec_var_length_no_reuse_self_loop);
RUN_TEST(cypher_exec_var_length_truncation_surfaces_warning);
RUN_TEST(cypher_exec_defines_edge);
RUN_TEST(cypher_exec_no_results);
RUN_TEST(cypher_exec_where_numeric);
tests/test_edge_structural.c
+14
View File
@@ -407,6 +407,19 @@ TEST(es_calls_crossfile_typescript) {
PASS();
}
TEST(es_calls_vue_embedded_issue1410) {
static const ES_LangFile f[] = {
{"App.vue",
"<template><p>Vue</p></template>\n"
"<script setup lang=\"ts\">\n"
"function callee(): number { return 42; }\n"
"function caller(): number { return callee(); }\n"
"</script>\n"},
};
ASSERT_EQ(es_exact_edge_by_name(f, 1, "CALLS", "caller", "callee"), 1);
PASS();
}
/* Java: caller in Main.java calls static method from Util.java (same package). */
TEST(es_calls_crossfile_java) {
static const ES_LangFile f[] = {
@@ -913,6 +926,7 @@ SUITE(edge_structural) {
RUN_TEST(es_calls_crossfile_java);
RUN_TEST(es_calls_crossfile_kotlin);
RUN_TEST(es_calls_crossfile_csharp);
RUN_TEST(es_calls_vue_embedded_issue1410);
/* ── FAMILY 2: INHERITS cross-file ────────────────────────── */
/* GREEN: Java, C#, C++ (extraction confirmed correct). */
tests/test_extraction.c
+440
View File
@@ -74,6 +74,27 @@ static int count_defs_with_label(CBMFileResult *r, const char *label) {
return count;
}
static int count_defs_named(CBMFileResult *r, const char *label, const char *name) {
int count = 0;
for (int i = 0; i < r->defs.count; i++) {
if (strcmp(r->defs.items[i].label, label) == 0 &&
strcmp(r->defs.items[i].name, name) == 0) {
count++;
}
}
return count;
}
static int count_calls_named(CBMFileResult *r, const char *callee) {
int count = 0;
for (int i = 0; i < r->calls.count; i++) {
if (r->calls.items[i].callee_name && strcmp(r->calls.items[i].callee_name, callee) == 0) {
count++;
}
}
return count;
}
/* Convenience: extract, assert no error, return result. Caller frees. */
static CBMFileResult *extract(const char *src, CBMLanguage lang, const char *proj,
const char *path) {
@@ -2881,6 +2902,103 @@ TEST(vue_imports_basic) {
PASS();
}
TEST(vue_embedded_structure_issue1410) {
const char *src = "<template><div>caf\xC3\xA9</div></template>\r\n"
"<script>\r\n"
"import { external } from './dep.js';\r\n"
"function normalFn() { return callee(); }\r\n"
"</script>\r\n"
"<script setup lang=\"ts\">class Widget {}\r\n"
"function setupFn(): void { callee(); }</script>\r\n";
CBMFileResult *r = extract(src, CBM_LANG_VUE, "t", "App.vue");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT_EQ(count_defs_with_label(r, "Module"), 1);
ASSERT_EQ(count_defs_with_label(r, "Function"), 2);
ASSERT_EQ(count_defs_with_label(r, "Class"), 1);
ASSERT_EQ(count_defs_named(r, "Function", "normalFn"), 1);
ASSERT_EQ(count_defs_named(r, "Function", "setupFn"), 1);
ASSERT_EQ(count_defs_named(r, "Class", "Widget"), 1);
ASSERT_EQ(count_calls_named(r, "callee"), 2);
ASSERT_EQ(r->imports.count, 1);
ASSERT(has_import(r, "dep.js"));
const CBMDefinition *normal = NULL;
const CBMDefinition *widget = NULL;
const CBMDefinition *setup = NULL;
for (int i = 0; i < r->defs.count; i++) {
if (strcmp(r->defs.items[i].name, "normalFn") == 0) {
normal = &r->defs.items[i];
} else if (strcmp(r->defs.items[i].name, "Widget") == 0) {
widget = &r->defs.items[i];
} else if (strcmp(r->defs.items[i].name, "setupFn") == 0) {
setup = &r->defs.items[i];
}
}
ASSERT_NOT_NULL(normal);
ASSERT_NOT_NULL(widget);
ASSERT_NOT_NULL(setup);
ASSERT_EQ(normal->start_line, 4);
ASSERT_EQ(widget->start_line, 6);
ASSERT_EQ(setup->start_line, 7);
cbm_free_result(r);
PASS();
}
TEST(vue_embedded_structure_negative_controls_issue1410) {
static const char *sources[] = {
"<script lang=\"coffee\">function hidden() { forbidden(); }</script>\n",
"<script src=\"./external.js\">function hidden() { forbidden(); }</script>\n",
"<template><p>scriptless</p></template>\n",
};
for (int i = 0; i < 3; i++) {
CBMFileResult *r = extract(sources[i], CBM_LANG_VUE, "t", "Negative.vue");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT_EQ(count_defs_with_label(r, "Module"), 1);
ASSERT_EQ(count_defs_with_label(r, "Function"), 0);
ASSERT_EQ(count_defs_with_label(r, "Class"), 0);
ASSERT_EQ(r->calls.count, 0);
ASSERT_EQ(r->imports.count, 0);
cbm_free_result(r);
}
PASS();
}
TEST(vue_embedded_structure_host_controls_issue1410) {
CBMFileResult *plain = extract("function plainTs(): void { target(); }\n", CBM_LANG_TYPESCRIPT,
"t", "plain.ts");
ASSERT_NOT_NULL(plain);
ASSERT_FALSE(plain->has_error);
ASSERT_EQ(count_defs_named(plain, "Function", "plainTs"), 1);
ASSERT_EQ(count_calls_named(plain, "target"), 1);
cbm_free_result(plain);
static const struct {
CBMLanguage language;
const char *path;
const char *source;
} hosts[] = {
{CBM_LANG_SVELTE, "Control.svelte",
"<script>import value from './svelte.js'; function hidden() { target(); }</script>\n"},
{CBM_LANG_HTML, "control.html",
"<script>import value from './html.js'; function hidden() { target(); }</script>\n"},
{CBM_LANG_ASTRO, "Control.astro",
"---\nimport value from './astro.js'; function hidden() { target(); }\n---\n"},
};
for (int i = 0; i < 3; i++) {
CBMFileResult *r = extract(hosts[i].source, hosts[i].language, "t", hosts[i].path);
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT_EQ(count_defs_with_label(r, "Module"), 1);
ASSERT_EQ(count_defs_with_label(r, "Function"), 0);
ASSERT_EQ(r->calls.count, 0);
ASSERT_EQ(r->imports.count, 1);
cbm_free_result(r);
}
PASS();
}
TEST(html_imports_basic) {
/* Plain HTML with inline ES module imports — same generic walker. */
CBMFileResult *r = extract("<!DOCTYPE html><html><head>\n"
@@ -3304,6 +3422,148 @@ TEST(extract_java_method_annotations_issue382) {
PASS();
}
/* ── ArkTS (HarmonyOS .ets) ─────────────────────────────────────── */
TEST(arkts_component_struct) {
CBMFileResult *r = extract(
"@Entry\n@Component\nstruct Index {\n @State message: string = 'Hello'\n\n"
" build() {\n Column() {\n Text(this.message).fontSize(20)\n }\n"
" .width('100%')\n }\n}\n",
CBM_LANG_ARKTS, "t", "Index.ets");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT(has_def(r, "Struct", "Index"));
ASSERT(has_def(r, "Method", "build"));
ASSERT(has_def(r, "Field", "message"));
const CBMDefinition *s = find_def_by_name(r, "Index");
ASSERT(decorators_contain(s, "Component"));
ASSERT(decorators_contain(s, "Entry"));
cbm_free_result(r);
PASS();
}
/* The common shared-component form: `@Component export struct X` puts the
* decorator on the export_statement; extract_decorators reaches it through
* the prev-sibling walk (skipping the anonymous `export` token). */
TEST(arkts_exported_struct_decorators) {
CBMFileResult *r = extract("@Component\nexport struct TitleBar {\n"
" @Prop title: string\n\n build() {\n Row() {\n"
" Text(this.title)\n }\n }\n}\n",
CBM_LANG_ARKTS, "t", "TitleBar.ets");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT(has_def(r, "Struct", "TitleBar"));
ASSERT(decorators_contain(find_def_by_name(r, "TitleBar"), "Component"));
ASSERT(has_def(r, "Field", "title"));
cbm_free_result(r);
PASS();
}
TEST(arkts_member_decorators) {
CBMFileResult *r = extract(
"@Component\nstruct S {\n @State a: number = 0\n @Prop b: string\n"
" @Link c: boolean\n @Provide('k') d: string = ''\n @Consume('k') e: string\n"
" @StorageLink('s') f: number = 1\n @State @Watch('onW') g: boolean = false\n\n"
" build() {\n }\n}\n",
CBM_LANG_ARKTS, "t", "S.ets");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT(decorators_contain(find_def_by_name(r, "a"), "State"));
ASSERT(decorators_contain(find_def_by_name(r, "b"), "Prop"));
ASSERT(decorators_contain(find_def_by_name(r, "c"), "Link"));
ASSERT(decorators_contain(find_def_by_name(r, "d"), "Provide"));
ASSERT(decorators_contain(find_def_by_name(r, "e"), "Consume"));
ASSERT(decorators_contain(find_def_by_name(r, "f"), "StorageLink"));
ASSERT(decorators_contain(find_def_by_name(r, "g"), "Watch"));
cbm_free_result(r);
PASS();
}
/* Spike regression: ArkUI builtins must never be emitted as definitions.
* Routing .ets through the plain TypeScript grammar made `Column() { ... }`
* parse chaos mint Column/Row/ListItem/... as user function DEFINITIONS
* (0/19 components found; 28% of CALLS edges were cross-file fabrications).
* With the arkts grammar they are call expressions — calls, never defs. */
TEST(arkts_no_phantom_builtin_defs) {
CBMFileResult *r = extract(
"@Component\nstruct S {\n build() {\n Column() {\n Row() {\n"
" Text('x').fontSize(10)\n }\n List() {\n ListItem() {\n"
" Text('y')\n }\n }\n ForEach(this.items, (i: string) => {\n"
" Text(i)\n })\n }\n .width('100%')\n }\n}\n",
CBM_LANG_ARKTS, "t", "S.ets");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
static const char *builtins[] = {"Column", "Row", "Text", "List", "ListItem", "ForEach", NULL};
for (int i = 0; builtins[i]; i++) {
ASSERT_FALSE(has_def_any(r, builtins[i]));
}
ASSERT(has_call(r, "Column"));
ASSERT(has_call(r, "ListItem"));
ASSERT(has_call(r, "ForEach"));
cbm_free_result(r);
PASS();
}
TEST(arkts_builder_extend_styles) {
CBMFileResult *r = extract(
"@Builder\nfunction card(t: string) {\n Column() {\n Text(t)\n }\n}\n\n"
"@Extend(Text)\nfunction fancy(size: number) {\n .fontSize(size)\n}\n\n"
"@Styles\nfunction pressed() {\n .backgroundColor('#eee')\n}\n",
CBM_LANG_ARKTS, "t", "b.ets");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT(has_def(r, "Function", "card"));
ASSERT(has_def(r, "Function", "fancy"));
ASSERT(has_def(r, "Function", "pressed"));
ASSERT(decorators_contain(find_def_by_name(r, "card"), "Builder"));
ASSERT(decorators_contain(find_def_by_name(r, "fancy"), "Extend"));
cbm_free_result(r);
PASS();
}
TEST(arkts_lazy_import) {
CBMFileResult *r = extract("import lazy { HeavyModule, Other } from './heavy'\n"
"import { router } from '@kit.ArkUI'\n"
"import lazy from './lazymod'\n",
CBM_LANG_ARKTS, "t", "i.ets");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT(has_import(r, "./heavy"));
ASSERT(has_import(r, "@kit.ArkUI"));
ASSERT(has_import(r, "./lazymod"));
int heavy = 0, other = 0;
for (int i = 0; i < r->imports.count; i++) {
if (r->imports.items[i].local_name) {
if (strcmp(r->imports.items[i].local_name, "HeavyModule") == 0) {
heavy = 1;
}
if (strcmp(r->imports.items[i].local_name, "Other") == 0) {
other = 1;
}
}
}
ASSERT(heavy);
ASSERT(other);
cbm_free_result(r);
PASS();
}
TEST(arkts_ts_compat) {
CBMFileResult *r = extract(
"@Observed\nclass Model {\n count: number = 0\n bump(): void { this.count++ }\n}\n\n"
"interface Props {\n title: string\n}\n\n"
"export function helper(x: number): number {\n return x * 2\n}\n",
CBM_LANG_ARKTS, "t", "m.ets");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT(has_def(r, "Class", "Model"));
ASSERT(has_def(r, "Method", "bump"));
ASSERT(has_def(r, "Interface", "Props"));
ASSERT(has_def(r, "Function", "helper"));
cbm_free_result(r);
PASS();
}
/* Issue #1005: JAX-RS splits a route across two annotations (@GET carries the
* verb, a sibling @Path carries the path). Returning on the first mapping
* annotation dropped every method-level @Path, and the class-level @Path
@@ -3370,6 +3630,169 @@ static const CBMCall *find_call_by_callee(CBMFileResult *r, const char *callee)
return NULL;
}
/* Issue #1009: URL-builder helper pattern — a function returning a URL-shaped
* literal, consumed as client(buildPath(id)). The builder's URL is recorded in
* the per-file constant map and resolved at the call site, for both return
* statements and arrow expression bodies. */
TEST(extract_ts_url_builder_issue1009) {
CBMFileResult *r = extract("function thingDetail(id: string): string {\n"
" return `/api/v1/things/${id}/detail`;\n"
"}\n"
"const arrowPath = (id: string) => `/api/v1/arrows/${id}`;\n"
"export function useThing(id: string) {\n"
" return apiGet(thingDetail(id));\n"
"}\n"
"export function useArrow(id: string) {\n"
" return apiFetch(arrowPath(id));\n"
"}\n",
CBM_LANG_TYPESCRIPT, "t", "builders.ts");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
const CBMCall *c1 = find_call_by_callee(r, "apiGet");
ASSERT_NOT_NULL(c1);
ASSERT_NOT_NULL(c1->first_string_arg);
ASSERT_STR_EQ(c1->first_string_arg, "/api/v1/things/{}/detail");
const CBMCall *c2 = find_call_by_callee(r, "apiFetch");
ASSERT_NOT_NULL(c2);
ASSERT_NOT_NULL(c2->first_string_arg);
ASSERT_STR_EQ(c2->first_string_arg, "/api/v1/arrows/{}");
cbm_free_result(r);
PASS();
}
/* Issue #1009 (composed builders): a builder whose template inlines an earlier
* builder's call plus a query string: `return \`${basePath(id)}?${params}\``.
* The known-substitution is inlined and the query string is truncated, so the
* resolved URL joins the server route exactly. */
TEST(extract_ts_url_builder_composed_issue1009) {
CBMFileResult *r = extract("function activityPath(id: string): string {\n"
" return `/api/v1/team-members/${id}/activity`;\n"
"}\n"
"function buildPath(id: string, cursor: string): string {\n"
" const params = new URLSearchParams();\n"
" params.set('cursor', cursor);\n"
" return `${activityPath(id)}?${params.toString()}`;\n"
"}\n"
"export function useActivity(id: string, cursor: string) {\n"
" return apiGet(buildPath(id, cursor));\n"
"}\n",
CBM_LANG_TYPESCRIPT, "t", "composed.ts");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
const CBMCall *c = find_call_by_callee(r, "apiGet");
ASSERT_NOT_NULL(c);
ASSERT_NOT_NULL(c->first_string_arg);
ASSERT_STR_EQ(c->first_string_arg, "/api/v1/team-members/{}/activity");
cbm_free_result(r);
PASS();
}
static bool any_call_arg_resolves_to(CBMFileResult *r, const char *url) {
for (int i = 0; i < r->calls.count; i++) {
const CBMCall *c = &r->calls.items[i];
if (c->first_string_arg && strcmp(c->first_string_arg, url) == 0) {
return true;
}
for (int a = 0; a < c->arg_count; a++) {
if (c->args[a].value && strcmp(c->args[a].value, url) == 0) {
return true;
}
}
}
return false;
}
TEST(extract_c_url_builder_gated_issue1009) {
CBMFileResult *r = extract("static const char *cfg_path(void) {\n"
" return \"/srv/myapp/conf.d\";\n"
"}\n"
"void init(void) {\n"
" parse_config(cfg_path());\n"
"}\n",
CBM_LANG_C, "t", "conf.c");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT_FALSE(any_call_arg_resolves_to(r, "/srv/myapp/conf.d"));
cbm_free_result(r);
PASS();
}
/* One literal return next to a computed one would attribute the literal to every
* call site, including those taking the computed branch. A builder whose returns
* are not all URL literals is declined. */
TEST(extract_ts_url_builder_mixed_returns_issue1009) {
CBMFileResult *r = extract("function pathFor(kind: string): string {\n"
" if (kind === 'user') return '/api/users';\n"
" return computePath(kind);\n"
"}\n"
"export function load(kind: string) {\n"
" return apiGet(pathFor(kind));\n"
"}\n",
CBM_LANG_TYPESCRIPT, "t", "mixed.ts");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT_FALSE(any_call_arg_resolves_to(r, "/api/users"));
cbm_free_result(r);
PASS();
}
/* Two different literal URLs make the call site's route unknowable, so the
* builder is tombstoned and lookups miss. */
TEST(extract_ts_url_builder_ambiguous_issue1009) {
CBMFileResult *r = extract("function pathFor(kind: string): string {\n"
" if (kind === 'user') return '/api/users';\n"
" return '/api/teams';\n"
"}\n"
"export function load(kind: string) {\n"
" return apiGet(pathFor(kind));\n"
"}\n",
CBM_LANG_TYPESCRIPT, "t", "ambiguous.ts");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT_FALSE(any_call_arg_resolves_to(r, "/api/users"));
ASSERT_FALSE(any_call_arg_resolves_to(r, "/api/teams"));
cbm_free_result(r);
PASS();
}
/* Handing a builder to a callback position builds no URL. Only a call to it
* resolves, so the mapping function does not inherit an HTTP_CALLS edge to the
* route. */
TEST(extract_ts_url_builder_reference_issue1009) {
CBMFileResult *r = extract("function thingPath(id: string): string {\n"
" return `/api/v1/things/${id}`;\n"
"}\n"
"export function loadAll(ids: string[]) {\n"
" return ids.map(thingPath);\n"
"}\n",
CBM_LANG_TYPESCRIPT, "t", "reference.ts");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT_FALSE(any_call_arg_resolves_to(r, "/api/v1/things/{}"));
cbm_free_result(r);
PASS();
}
/* A helper returning an ordinary string is not a URL builder, so its call site
* gets no resolved string argument. */
TEST(extract_ts_url_builder_non_url_issue1009) {
CBMFileResult *r = extract("function label(): string {\n"
" return 'plain text';\n"
"}\n"
"export function render() {\n"
" return send(label());\n"
"}\n",
CBM_LANG_TYPESCRIPT, "t", "label.ts");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
const CBMCall *c = find_call_by_callee(r, "send");
ASSERT_NOT_NULL(c);
ASSERT_NULL(c->first_string_arg);
ASSERT_FALSE(any_call_arg_resolves_to(r, "plain text"));
cbm_free_result(r);
PASS();
}
/* Issue #1006: JS/TS template-literal URLs must flatten ${...} substitutions
* to the canonical "{}" placeholder, both as call arguments (HTTP_CALLS) and
* as URL-shaped string_refs collected from const/return positions. */
@@ -5747,6 +6170,9 @@ SUITE(extraction) {
RUN_TEST(svelte_imports_basic);
RUN_TEST(svelte_imports_no_script);
RUN_TEST(vue_imports_basic);
RUN_TEST(vue_embedded_structure_issue1410);
RUN_TEST(vue_embedded_structure_negative_controls_issue1410);
RUN_TEST(vue_embedded_structure_host_controls_issue1410);
RUN_TEST(html_imports_basic);
/* config_extraction_test.go ports */
@@ -5782,10 +6208,24 @@ SUITE(extraction) {
RUN_TEST(js_index_module_qn_not_collide_with_folder);
RUN_TEST(python_regular_module_qn_unchanged);
RUN_TEST(extract_java_method_annotations_issue382);
RUN_TEST(arkts_component_struct);
RUN_TEST(arkts_exported_struct_decorators);
RUN_TEST(arkts_member_decorators);
RUN_TEST(arkts_no_phantom_builtin_defs);
RUN_TEST(arkts_builder_extend_styles);
RUN_TEST(arkts_lazy_import);
RUN_TEST(arkts_ts_compat);
RUN_TEST(extract_java_jaxrs_path_composition_issue1005);
RUN_TEST(extract_ts_template_string_url_issue1006);
RUN_TEST(extract_go_binary_concat_url_issue1249);
RUN_TEST(extract_go_binary_concat_url_no_literal_suffix_issue1249);
RUN_TEST(extract_ts_url_builder_issue1009);
RUN_TEST(extract_ts_url_builder_composed_issue1009);
RUN_TEST(extract_c_url_builder_gated_issue1009);
RUN_TEST(extract_ts_url_builder_mixed_returns_issue1009);
RUN_TEST(extract_ts_url_builder_ambiguous_issue1009);
RUN_TEST(extract_ts_url_builder_reference_issue1009);
RUN_TEST(extract_ts_url_builder_non_url_issue1009);
RUN_TEST(extract_java_no_double_class_qn);
RUN_TEST(extract_go_no_filename_in_module_qn);
RUN_TEST(extract_large_ts_has_functions_issue213);
tests/test_grammar_imports.c
+3
View File
@@ -25,6 +25,9 @@ typedef struct {
} ImportCase;
static const ImportCase IMPORT_CASES[] = {
{"arkts", CBM_LANG_ARKTS, "a.ets",
"import lazy { HeavyModule } from './heavy'\nimport { router } from '@kit.ArkUI'\nimport curves from '@ohos.curves'\n",
3, NULL},
{"astro", CBM_LANG_ASTRO, "fixture.astro",
"<script>\nimport confetti from \"canvas-confetti\";\nimport dayjs from \"dayjs\";\n\nconfetti();\nconsole.log(dayjs().format());\n</script>\n<div>hi</div>\n",
2, NULL},
tests/test_grammar_labels.c
+1
View File
@@ -152,6 +152,7 @@ static const LabelGolden LABEL_GOLDENS[] = {
{"lean", "Function:2,Module:1"},
{"move", "Function:1,Module:1"},
{"mojo", "Class:1,Function:1,Method:1,Module:1"},
{"arkts", "Field:1,Function:1,Method:1,Module:1,Struct:1"},
{"smali", "Class:1,Function:1,Module:1"},
{"systemverilog", "Class:1,Function:1,Module:1"},
{"verilog", "Class:1,Module:1"},
tests/test_grammar_regression.c
+8
View File
@@ -202,6 +202,14 @@ const GrammarCase CBM_GRAMMAR_CASES[] = {
"fn foo() -> Int:\n return 1\n\nstruct A:\n fn bar(self) -> Int:\n return foo()\n",
2,
{"foo", "A", NULL}},
{"arkts",
CBM_LANG_ARKTS,
"a.ets",
"@Component\nstruct Card {\n @State title: string = ''\n\n build() {\n Column() {\n"
" Text(this.title).fontSize(20)\n }\n }\n}\n\n@Builder\nfunction rowCard() {\n"
" Row() {}\n}\n",
2,
{"Card", "build", "rowCard"}},
{"smali",
CBM_LANG_SMALI,
"A.smali",
tests/test_language.c
+6
View File
@@ -961,6 +961,11 @@ TEST(lang_ext_mojo) {
PASS();
}
TEST(lang_ext_arkts) {
ASSERT_EQ(cbm_language_for_extension(".ets"), CBM_LANG_ARKTS);
PASS();
}
TEST(lang_ext_squirrel) {
ASSERT_EQ(cbm_language_for_extension(".nut"), CBM_LANG_SQUIRREL);
PASS();
@@ -1277,6 +1282,7 @@ SUITE(language) {
RUN_TEST(lang_ext_cairo);
RUN_TEST(lang_ext_move);
RUN_TEST(lang_ext_mojo);
RUN_TEST(lang_ext_arkts);
RUN_TEST(lang_ext_squirrel);
RUN_TEST(lang_ext_func);
RUN_TEST(lang_ext_rst);
tests/test_store_search.c
+152
View File
@@ -4,14 +4,25 @@
* Ported from internal/store/store_test.go (TestSearch, TestBFS, etc.)
*/
#include "../src/foundation/compat.h"
#include "../src/foundation/log.h"
#include "test_framework.h"
#include "test_helpers.h"
#include <store/store.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
static char trail_log[1024];
static void capture_trail_log(const char *line) {
size_t used = strlen(trail_log);
if (used < sizeof(trail_log) - 1) {
snprintf(trail_log + used, sizeof(trail_log) - used, "%s\n", line);
}
}
/* Helper: create a typical graph for search/traversal tests.
*
* Nodes: SubmitOrder (Function), ProcessOrder (Function), OrderService (Class)
@@ -992,6 +1003,144 @@ TEST(store_bfs_with_risk_labels) {
PASS();
}
TEST(store_bfs_reachability_is_not_trail_capped) {
cbm_store_t *s = cbm_store_open_memory();
cbm_store_upsert_project(s, "test", "/tmp/test");
cbm_node_t root = {.project = "test",
.label = "Function",
.name = "Root",
.qualified_name = "test.Root"};
int64_t root_id = cbm_store_upsert_node(s, &root);
enum { NODE_COUNT = 4200 };
for (int i = 0; i < NODE_COUNT; i++) {
char name[32];
snprintf(name, sizeof(name), "Leaf%d", i);
cbm_node_t leaf = {
.project = "test", .label = "Function", .name = name, .qualified_name = name};
int64_t leaf_id = cbm_store_upsert_node(s, &leaf);
cbm_edge_t edge = {
.project = "test", .source_id = root_id, .target_id = leaf_id, .type = "CALLS"};
cbm_store_insert_edge(s, &edge);
}
const char *types[] = {"CALLS"};
cbm_traverse_result_t result = {0};
int rc = cbm_store_bfs(s, root_id, "outbound", types, 1, 1, 5000, &result);
ASSERT_EQ(rc, CBM_STORE_OK);
ASSERT_EQ(result.visited_count, NODE_COUNT);
cbm_store_traverse_free(&result);
cbm_store_close(s);
PASS();
}
TEST(store_bfs_trail_warns_when_path_rows_are_truncated) {
cbm_store_t *s = cbm_store_open_memory();
cbm_store_upsert_project(s, "test", "/tmp/test");
int64_t ids[18];
for (int i = 0; i < 18; i++) {
char name[16];
snprintf(name, sizeof(name), "node-%d", i);
cbm_node_t node = {.project = "test",
.label = "Function",
.name = name,
.qualified_name = name,
.file_path = "graph.c"};
ids[i] = cbm_store_upsert_node(s, &node);
}
for (int source = 0; source < 17; source++) {
for (int target = source + 1; target < 18; target++) {
cbm_edge_t edge = {.project = "test",
.source_id = ids[source],
.target_id = ids[target],
.type = "CALLS"};
cbm_store_insert_edge(s, &edge);
}
}
trail_log[0] = '\0';
cbm_log_set_sink(capture_trail_log);
const char *types[] = {"CALLS"};
cbm_traverse_result_t result = {0};
int rc = cbm_store_bfs_trail(s, ids[0], "outbound", types, 1, 10, 5000, &result);
cbm_log_set_sink(NULL);
ASSERT_EQ(rc, CBM_STORE_OK);
ASSERT_TRUE(result.truncated);
ASSERT_TRUE(strstr(trail_log, "cypher.trail_truncated") != NULL);
ASSERT_TRUE(strstr(trail_log, "result=partial") != NULL);
cbm_store_traverse_free(&result);
cbm_store_close(s);
PASS();
}
TEST(store_bfs_trail_preserves_deeper_match_under_hub_budget) {
cbm_store_t *s = cbm_store_open_memory();
cbm_store_upsert_project(s, "test", "/tmp/test");
cbm_node_t root = {.project = "test",
.label = "Function",
.name = "root",
.qualified_name = "test.root"};
cbm_node_t branch = {.project = "test",
.label = "Function",
.name = "branch",
.qualified_name = "test.branch"};
cbm_node_t target = {.project = "test",
.label = "Function",
.name = "target",
.qualified_name = "test.target"};
int64_t root_id = cbm_store_upsert_node(s, &root);
int64_t branch_id = cbm_store_upsert_node(s, &branch);
int64_t target_id = cbm_store_upsert_node(s, &target);
cbm_edge_t edge = {
.project = "test", .source_id = root_id, .target_id = branch_id, .type = "CALLS"};
cbm_store_insert_edge(s, &edge);
edge.source_id = branch_id;
edge.target_id = target_id;
cbm_store_insert_edge(s, &edge);
/* A high-fanout hub must not consume the bounded CTE before a deeper path
* is considered; the depth-first queue should still surface `target`. */
enum { LEAF_COUNT = 4100 };
for (int i = 0; i < LEAF_COUNT; i++) {
char name[32];
snprintf(name, sizeof(name), "leaf-%d", i);
cbm_node_t leaf = {.project = "test",
.label = "Function",
.name = name,
.qualified_name = name};
int64_t leaf_id = cbm_store_upsert_node(s, &leaf);
edge.source_id = root_id;
edge.target_id = leaf_id;
cbm_store_insert_edge(s, &edge);
}
const char *types[] = {"CALLS"};
cbm_traverse_result_t result = {0};
int rc = cbm_store_bfs_trail(s, root_id, "outbound", types, 1, 2, 5000, &result);
ASSERT_EQ(rc, CBM_STORE_OK);
ASSERT_TRUE(result.truncated);
bool saw_target = false;
for (int i = 0; i < result.visited_count; i++) {
if (result.visited[i].node.id == target_id) {
saw_target = true;
break;
}
}
ASSERT_TRUE(saw_target);
cbm_store_traverse_free(&result);
cbm_store_close(s);
PASS();
}
/* ── BFS cross-service summary ─────────────────────────────────── */
TEST(store_bfs_cross_service_summary) {
@@ -1509,6 +1658,9 @@ SUITE(store_search) {
RUN_TEST(store_cross_service_detection);
RUN_TEST(store_deduplicate_hops);
RUN_TEST(store_bfs_with_risk_labels);
RUN_TEST(store_bfs_reachability_is_not_trail_capped);
RUN_TEST(store_bfs_trail_warns_when_path_rows_are_truncated);
RUN_TEST(store_bfs_trail_preserves_deeper_match_under_hub_budget);
RUN_TEST(store_bfs_cross_service_summary);
RUN_TEST(store_glob_to_like);
RUN_TEST(store_extract_like_hints);
tools/tree-sitter-arkts/grammar.js
+1238
View File
File diff suppressed because it is too large Load Diff
tools/tree-sitter-arkts/javascript-grammar.js
+1249
View File
File diff suppressed because it is too large Load Diff
tools/tree-sitter-arkts/package.json
+16
View File
@@ -0,0 +1,16 @@
{
"name": "tree-sitter-arkts",
"version": "0.1.0",
"description": "ArkTS (HarmonyOS .ets) grammar for tree-sitter — fork of tree-sitter-typescript",
"keywords": ["tree-sitter", "arkts", "ets", "harmonyos", "openharmony", "arkui"],
"license": "MIT",
"devDependencies": {
"tree-sitter-cli": "0.25.10"
},
"tree-sitter": [
{
"scope": "source.arkts",
"file-types": ["ets"]
}
]
}
tools/tree-sitter-arkts/src/_common_scanner.h
+347
View File
@@ -0,0 +1,347 @@
#include "tree_sitter/parser.h"
#include <wctype.h>
enum TokenType {
AUTOMATIC_SEMICOLON,
TEMPLATE_CHARS,
TERNARY_QMARK,
HTML_COMMENT,
LOGICAL_OR,
ESCAPE_SEQUENCE,
REGEX_PATTERN,
JSX_TEXT,
FUNCTION_SIGNATURE_AUTOMATIC_SEMICOLON,
ERROR_RECOVERY,
};
static void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
static void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
static bool scan_template_chars(TSLexer *lexer) {
lexer->result_symbol = TEMPLATE_CHARS;
for (bool has_content = false;; has_content = true) {
lexer->mark_end(lexer);
switch (lexer->lookahead) {
case '`':
return has_content;
case '\0':
return false;
case '$':
advance(lexer);
if (lexer->lookahead == '{') {
return has_content;
}
break;
case '\\':
return has_content;
default:
advance(lexer);
}
}
}
static bool scan_whitespace_and_comments(TSLexer *lexer, bool *scanned_comment) {
for (;;) {
while (iswspace(lexer->lookahead)) {
skip(lexer);
}
if (lexer->lookahead == '/') {
skip(lexer);
if (lexer->lookahead == '/') {
skip(lexer);
while (lexer->lookahead != 0 && lexer->lookahead != '\n') {
skip(lexer);
}
*scanned_comment = true;
} else if (lexer->lookahead == '*') {
skip(lexer);
while (lexer->lookahead != 0) {
if (lexer->lookahead == '*') {
skip(lexer);
if (lexer->lookahead == '/') {
skip(lexer);
break;
}
} else {
skip(lexer);
}
}
} else {
return false;
}
} else {
return true;
}
}
}
static bool scan_automatic_semicolon(TSLexer *lexer, const bool *valid_symbols, bool *scanned_comment) {
lexer->result_symbol = AUTOMATIC_SEMICOLON;
lexer->mark_end(lexer);
for (;;) {
if (lexer->lookahead == 0) {
return true;
}
if (lexer->lookahead == '}') {
// Automatic semicolon insertion breaks detection of object patterns
// in a typed context:
// type F = ({a}: {a: number}) => number;
// Therefore, disable automatic semicolons when followed by typing
do {
skip(lexer);
} while (iswspace(lexer->lookahead));
if (lexer->lookahead == ':') {
return valid_symbols[LOGICAL_OR]; // Don't return false if we're in a ternary by checking if || is valid
}
return true;
}
if (!iswspace(lexer->lookahead)) {
return false;
}
if (lexer->lookahead == '\n') {
break;
}
skip(lexer);
}
skip(lexer);
if (!scan_whitespace_and_comments(lexer, scanned_comment)) {
return false;
}
switch (lexer->lookahead) {
case '`':
case ',':
case '.':
case ';':
case '*':
case '%':
case '>':
case '<':
case '=':
case '?':
case '^':
case '|':
case '&':
case '/':
case ':':
return false;
case '{':
if (valid_symbols[FUNCTION_SIGNATURE_AUTOMATIC_SEMICOLON]) {
return false;
}
break;
// Don't insert a semicolon before a '[' or '(', unless we're parsing
// a type. Detect whether we're parsing a type or an expression using
// the validity of a binary operator token.
case '(':
case '[':
if (valid_symbols[LOGICAL_OR]) {
return false;
}
break;
// Insert a semicolon before `--` and `++`, but not before binary `+` or `-`.
case '+':
skip(lexer);
return lexer->lookahead == '+';
case '-':
skip(lexer);
return lexer->lookahead == '-';
// Don't insert a semicolon before `!=`, but do insert one before a unary `!`.
case '!':
skip(lexer);
return lexer->lookahead != '=';
// Don't insert a semicolon before `in` or `instanceof`, but do insert one
// before an identifier.
case 'i':
skip(lexer);
if (lexer->lookahead != 'n') {
return true;
}
skip(lexer);
if (!iswalpha(lexer->lookahead)) {
return false;
}
for (unsigned i = 0; i < 8; i++) {
if (lexer->lookahead != "stanceof"[i]) {
return true;
}
skip(lexer);
}
if (!iswalpha(lexer->lookahead)) {
return false;
}
break;
}
return true;
}
static bool scan_ternary_qmark(TSLexer *lexer) {
for (;;) {
if (!iswspace(lexer->lookahead)) {
break;
}
skip(lexer);
}
if (lexer->lookahead == '?') {
advance(lexer);
/* Optional chaining. */
if (lexer->lookahead == '?' || lexer->lookahead == '.') {
return false;
}
lexer->mark_end(lexer);
lexer->result_symbol = TERNARY_QMARK;
/* TypeScript optional arguments contain the ?: sequence, possibly
with whitespace. */
for (;;) {
if (!iswspace(lexer->lookahead)) {
break;
}
advance(lexer);
}
if (lexer->lookahead == ':' || lexer->lookahead == ')' || lexer->lookahead == ',') {
return false;
}
if (lexer->lookahead == '.') {
advance(lexer);
if (iswdigit(lexer->lookahead)) {
return true;
}
return false;
}
return true;
}
return false;
}
static bool scan_closing_comment(TSLexer *lexer) {
while (iswspace(lexer->lookahead) || lexer->lookahead == 0x2028 || lexer->lookahead == 0x2029) {
skip(lexer);
}
const char *comment_start = "<!--";
const char *comment_end = "-->";
if (lexer->lookahead == '<') {
for (unsigned i = 0; i < 4; i++) {
if (lexer->lookahead != comment_start[i]) {
return false;
}
advance(lexer);
}
} else if (lexer->lookahead == '-') {
for (unsigned i = 0; i < 3; i++) {
if (lexer->lookahead != comment_end[i]) {
return false;
}
advance(lexer);
}
} else {
return false;
}
while (lexer->lookahead != 0 && lexer->lookahead != '\n' && lexer->lookahead != 0x2028 &&
lexer->lookahead != 0x2029) {
advance(lexer);
}
lexer->result_symbol = HTML_COMMENT;
lexer->mark_end(lexer);
return true;
}
static bool scan_jsx_text(TSLexer *lexer) {
// saw_text will be true if we see any non-whitespace content, or any whitespace content that is not a newline and
// does not immediately follow a newline.
bool saw_text = false;
// at_newline will be true if we are currently at a newline, or if we are at whitespace that is not a newline but
// immediately follows a newline.
bool at_newline = false;
while (lexer->lookahead != 0 && lexer->lookahead != '<' && lexer->lookahead != '>' && lexer->lookahead != '{' &&
lexer->lookahead != '}' && lexer->lookahead != '&') {
bool is_wspace = iswspace(lexer->lookahead);
if (lexer->lookahead == '\n') {
at_newline = true;
} else {
// If at_newline is already true, and we see some whitespace, then it must stay true.
// Otherwise, it should be false.
//
// See the table below to determine the logic for computing `saw_text`.
//
// |------------------------------------|
// | at_newline | is_wspace | saw_text |
// |------------|-----------|-----------|
// | false (0) | false (0) | true (1) |
// | false (0) | true (1) | true (1) |
// | true (1) | false (0) | true (1) |
// | true (1) | true (1) | false (0) |
// |------------------------------------|
at_newline &= is_wspace;
if (!at_newline) {
saw_text = true;
}
}
advance(lexer);
}
lexer->result_symbol = JSX_TEXT;
return saw_text;
}
static inline bool external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) {
if (valid_symbols[TEMPLATE_CHARS]) {
if (valid_symbols[AUTOMATIC_SEMICOLON]) {
return false;
}
return scan_template_chars(lexer);
}
if (valid_symbols[JSX_TEXT] && scan_jsx_text(lexer)) {
return true;
}
if (valid_symbols[AUTOMATIC_SEMICOLON] || valid_symbols[FUNCTION_SIGNATURE_AUTOMATIC_SEMICOLON]) {
bool scanned_comment = false;
bool ret = scan_automatic_semicolon(lexer, valid_symbols, &scanned_comment);
if (!ret && !scanned_comment && valid_symbols[TERNARY_QMARK] && lexer->lookahead == '?') {
return scan_ternary_qmark(lexer);
}
return ret;
}
if (valid_symbols[TERNARY_QMARK]) {
return scan_ternary_qmark(lexer);
}
if (valid_symbols[HTML_COMMENT] && !valid_symbols[LOGICAL_OR] && !valid_symbols[ESCAPE_SEQUENCE] &&
!valid_symbols[REGEX_PATTERN]) {
return scan_closing_comment(lexer);
}
return false;
}
tools/tree-sitter-arkts/src/grammar.json
+12164
View File
File diff suppressed because it is too large Load Diff
tools/tree-sitter-arkts/src/node-types.json
+6186
View File
File diff suppressed because it is too large Load Diff
tools/tree-sitter-arkts/src/parser.c
+286866
View File
File diff suppressed because it is too large Load Diff
tools/tree-sitter-arkts/src/scanner.c
+21
View File
@@ -0,0 +1,21 @@
// ArkTS external scanner — verbatim tree-sitter-typescript scanner logic.
//
// The scanner body lives in _common_scanner.h, a byte-identical copy of
// tree-sitter/tree-sitter-typescript common/scanner.h @ 75b3874edb2d
// (v0.23.2). Only the exported symbol names differ (arkts instead of
// typescript) because tree-sitter derives them from the grammar name.
// The external token set and order are identical to the typescript
// dialect's, which is what makes the verbatim reuse correct.
#include "_common_scanner.h"
void *tree_sitter_arkts_external_scanner_create() { return NULL; }
void tree_sitter_arkts_external_scanner_destroy(void *payload) {}
unsigned tree_sitter_arkts_external_scanner_serialize(void *payload, char *buffer) { return 0; }
void tree_sitter_arkts_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) {}
bool tree_sitter_arkts_external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) {
return external_scanner_scan(payload, lexer, valid_symbols);
}
tools/tree-sitter-arkts/src/tree_sitter/alloc.h
+54
View File
@@ -0,0 +1,54 @@
#ifndef TREE_SITTER_ALLOC_H_
#define TREE_SITTER_ALLOC_H_
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
// Allow clients to override allocation functions
#ifdef TREE_SITTER_REUSE_ALLOCATOR
extern void *(*ts_current_malloc)(size_t size);
extern void *(*ts_current_calloc)(size_t count, size_t size);
extern void *(*ts_current_realloc)(void *ptr, size_t size);
extern void (*ts_current_free)(void *ptr);
#ifndef ts_malloc
#define ts_malloc ts_current_malloc
#endif
#ifndef ts_calloc
#define ts_calloc ts_current_calloc
#endif
#ifndef ts_realloc
#define ts_realloc ts_current_realloc
#endif
#ifndef ts_free
#define ts_free ts_current_free
#endif
#else
#ifndef ts_malloc
#define ts_malloc malloc
#endif
#ifndef ts_calloc
#define ts_calloc calloc
#endif
#ifndef ts_realloc
#define ts_realloc realloc
#endif
#ifndef ts_free
#define ts_free free
#endif
#endif
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_ALLOC_H_
tools/tree-sitter-arkts/src/tree_sitter/array.h
+291
View File
@@ -0,0 +1,291 @@
#ifndef TREE_SITTER_ARRAY_H_
#define TREE_SITTER_ARRAY_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "./alloc.h"
#include <assert.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#ifdef _MSC_VER
#pragma warning(push)
#pragma warning(disable : 4101)
#elif defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-variable"
#endif
#define Array(T) \
struct { \
T *contents; \
uint32_t size; \
uint32_t capacity; \
}
/// Initialize an array.
#define array_init(self) \
((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
/// Create an empty array.
#define array_new() \
{ NULL, 0, 0 }
/// Get a pointer to the element at a given `index` in the array.
#define array_get(self, _index) \
(assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index])
/// Get a pointer to the first element in the array.
#define array_front(self) array_get(self, 0)
/// Get a pointer to the last element in the array.
#define array_back(self) array_get(self, (self)->size - 1)
/// Clear the array, setting its size to zero. Note that this does not free any
/// memory allocated for the array's contents.
#define array_clear(self) ((self)->size = 0)
/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
/// less than the array's current capacity, this function has no effect.
#define array_reserve(self, new_capacity) \
_array__reserve((Array *)(self), array_elem_size(self), new_capacity)
/// Free any memory allocated for this array. Note that this does not free any
/// memory allocated for the array's contents.
#define array_delete(self) _array__delete((Array *)(self))
/// Push a new `element` onto the end of the array.
#define array_push(self, element) \
(_array__grow((Array *)(self), 1, array_elem_size(self)), \
(self)->contents[(self)->size++] = (element))
/// Increase the array's size by `count` elements.
/// New elements are zero-initialized.
#define array_grow_by(self, count) \
do { \
if ((count) == 0) break; \
_array__grow((Array *)(self), count, array_elem_size(self)); \
memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
(self)->size += (count); \
} while (0)
/// Append all elements from one array to the end of another.
#define array_push_all(self, other) \
array_extend((self), (other)->size, (other)->contents)
/// Append `count` elements to the end of the array, reading their values from the
/// `contents` pointer.
#define array_extend(self, count, contents) \
_array__splice( \
(Array *)(self), array_elem_size(self), (self)->size, \
0, count, contents \
)
/// Remove `old_count` elements from the array starting at the given `index`. At
/// the same index, insert `new_count` new elements, reading their values from the
/// `new_contents` pointer.
#define array_splice(self, _index, old_count, new_count, new_contents) \
_array__splice( \
(Array *)(self), array_elem_size(self), _index, \
old_count, new_count, new_contents \
)
/// Insert one `element` into the array at the given `index`.
#define array_insert(self, _index, element) \
_array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
/// Remove one element from the array at the given `index`.
#define array_erase(self, _index) \
_array__erase((Array *)(self), array_elem_size(self), _index)
/// Pop the last element off the array, returning the element by value.
#define array_pop(self) ((self)->contents[--(self)->size])
/// Assign the contents of one array to another, reallocating if necessary.
#define array_assign(self, other) \
_array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
/// Swap one array with another
#define array_swap(self, other) \
_array__swap((Array *)(self), (Array *)(other))
/// Get the size of the array contents
#define array_elem_size(self) (sizeof *(self)->contents)
/// Search a sorted array for a given `needle` value, using the given `compare`
/// callback to determine the order.
///
/// If an existing element is found to be equal to `needle`, then the `index`
/// out-parameter is set to the existing value's index, and the `exists`
/// out-parameter is set to true. Otherwise, `index` is set to an index where
/// `needle` should be inserted in order to preserve the sorting, and `exists`
/// is set to false.
#define array_search_sorted_with(self, compare, needle, _index, _exists) \
_array__search_sorted(self, 0, compare, , needle, _index, _exists)
/// Search a sorted array for a given `needle` value, using integer comparisons
/// of a given struct field (specified with a leading dot) to determine the order.
///
/// See also `array_search_sorted_with`.
#define array_search_sorted_by(self, field, needle, _index, _exists) \
_array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists)
/// Insert a given `value` into a sorted array, using the given `compare`
/// callback to determine the order.
#define array_insert_sorted_with(self, compare, value) \
do { \
unsigned _index, _exists; \
array_search_sorted_with(self, compare, &(value), &_index, &_exists); \
if (!_exists) array_insert(self, _index, value); \
} while (0)
/// Insert a given `value` into a sorted array, using integer comparisons of
/// a given struct field (specified with a leading dot) to determine the order.
///
/// See also `array_search_sorted_by`.
#define array_insert_sorted_by(self, field, value) \
do { \
unsigned _index, _exists; \
array_search_sorted_by(self, field, (value) field, &_index, &_exists); \
if (!_exists) array_insert(self, _index, value); \
} while (0)
// Private
typedef Array(void) Array;
/// This is not what you're looking for, see `array_delete`.
static inline void _array__delete(Array *self) {
if (self->contents) {
ts_free(self->contents);
self->contents = NULL;
self->size = 0;
self->capacity = 0;
}
}
/// This is not what you're looking for, see `array_erase`.
static inline void _array__erase(Array *self, size_t element_size,
uint32_t index) {
assert(index < self->size);
char *contents = (char *)self->contents;
memmove(contents + index * element_size, contents + (index + 1) * element_size,
(self->size - index - 1) * element_size);
self->size--;
}
/// This is not what you're looking for, see `array_reserve`.
static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
if (new_capacity > self->capacity) {
if (self->contents) {
self->contents = ts_realloc(self->contents, new_capacity * element_size);
} else {
self->contents = ts_malloc(new_capacity * element_size);
}
self->capacity = new_capacity;
}
}
/// This is not what you're looking for, see `array_assign`.
static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
_array__reserve(self, element_size, other->size);
self->size = other->size;
memcpy(self->contents, other->contents, self->size * element_size);
}
/// This is not what you're looking for, see `array_swap`.
static inline void _array__swap(Array *self, Array *other) {
Array swap = *other;
*other = *self;
*self = swap;
}
/// This is not what you're looking for, see `array_push` or `array_grow_by`.
static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
uint32_t new_size = self->size + count;
if (new_size > self->capacity) {
uint32_t new_capacity = self->capacity * 2;
if (new_capacity < 8) new_capacity = 8;
if (new_capacity < new_size) new_capacity = new_size;
_array__reserve(self, element_size, new_capacity);
}
}
/// This is not what you're looking for, see `array_splice`.
static inline void _array__splice(Array *self, size_t element_size,
uint32_t index, uint32_t old_count,
uint32_t new_count, const void *elements) {
uint32_t new_size = self->size + new_count - old_count;
uint32_t old_end = index + old_count;
uint32_t new_end = index + new_count;
assert(old_end <= self->size);
_array__reserve(self, element_size, new_size);
char *contents = (char *)self->contents;
if (self->size > old_end) {
memmove(
contents + new_end * element_size,
contents + old_end * element_size,
(self->size - old_end) * element_size
);
}
if (new_count > 0) {
if (elements) {
memcpy(
(contents + index * element_size),
elements,
new_count * element_size
);
} else {
memset(
(contents + index * element_size),
0,
new_count * element_size
);
}
}
self->size += new_count - old_count;
}
/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`.
#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \
do { \
*(_index) = start; \
*(_exists) = false; \
uint32_t size = (self)->size - *(_index); \
if (size == 0) break; \
int comparison; \
while (size > 1) { \
uint32_t half_size = size / 2; \
uint32_t mid_index = *(_index) + half_size; \
comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \
if (comparison <= 0) *(_index) = mid_index; \
size -= half_size; \
} \
comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \
if (comparison == 0) *(_exists) = true; \
else if (comparison < 0) *(_index) += 1; \
} while (0)
/// Helper macro for the `_sorted_by` routines below. This takes the left (existing)
/// parameter by reference in order to work with the generic sorting function above.
#define _compare_int(a, b) ((int)*(a) - (int)(b))
#ifdef _MSC_VER
#pragma warning(pop)
#elif defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic pop
#endif
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_ARRAY_H_
tools/tree-sitter-arkts/src/tree_sitter/parser.h
+286
View File
@@ -0,0 +1,286 @@
#ifndef TREE_SITTER_PARSER_H_
#define TREE_SITTER_PARSER_H_
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#define ts_builtin_sym_error ((TSSymbol)-1)
#define ts_builtin_sym_end 0
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
#ifndef TREE_SITTER_API_H_
typedef uint16_t TSStateId;
typedef uint16_t TSSymbol;
typedef uint16_t TSFieldId;
typedef struct TSLanguage TSLanguage;
typedef struct TSLanguageMetadata {
uint8_t major_version;
uint8_t minor_version;
uint8_t patch_version;
} TSLanguageMetadata;
#endif
typedef struct {
TSFieldId field_id;
uint8_t child_index;
bool inherited;
} TSFieldMapEntry;
// Used to index the field and supertype maps.
typedef struct {
uint16_t index;
uint16_t length;
} TSMapSlice;
typedef struct {
bool visible;
bool named;
bool supertype;
} TSSymbolMetadata;
typedef struct TSLexer TSLexer;
struct TSLexer {
int32_t lookahead;
TSSymbol result_symbol;
void (*advance)(TSLexer *, bool);
void (*mark_end)(TSLexer *);
uint32_t (*get_column)(TSLexer *);
bool (*is_at_included_range_start)(const TSLexer *);
bool (*eof)(const TSLexer *);
void (*log)(const TSLexer *, const char *, ...);
};
typedef enum {
TSParseActionTypeShift,
TSParseActionTypeReduce,
TSParseActionTypeAccept,
TSParseActionTypeRecover,
} TSParseActionType;
typedef union {
struct {
uint8_t type;
TSStateId state;
bool extra;
bool repetition;
} shift;
struct {
uint8_t type;
uint8_t child_count;
TSSymbol symbol;
int16_t dynamic_precedence;
uint16_t production_id;
} reduce;
uint8_t type;
} TSParseAction;
typedef struct {
uint16_t lex_state;
uint16_t external_lex_state;
} TSLexMode;
typedef struct {
uint16_t lex_state;
uint16_t external_lex_state;
uint16_t reserved_word_set_id;
} TSLexerMode;
typedef union {
TSParseAction action;
struct {
uint8_t count;
bool reusable;
} entry;
} TSParseActionEntry;
typedef struct {
int32_t start;
int32_t end;
} TSCharacterRange;
struct TSLanguage {
uint32_t abi_version;
uint32_t symbol_count;
uint32_t alias_count;
uint32_t token_count;
uint32_t external_token_count;
uint32_t state_count;
uint32_t large_state_count;
uint32_t production_id_count;
uint32_t field_count;
uint16_t max_alias_sequence_length;
const uint16_t *parse_table;
const uint16_t *small_parse_table;
const uint32_t *small_parse_table_map;
const TSParseActionEntry *parse_actions;
const char * const *symbol_names;
const char * const *field_names;
const TSMapSlice *field_map_slices;
const TSFieldMapEntry *field_map_entries;
const TSSymbolMetadata *symbol_metadata;
const TSSymbol *public_symbol_map;
const uint16_t *alias_map;
const TSSymbol *alias_sequences;
const TSLexerMode *lex_modes;
bool (*lex_fn)(TSLexer *, TSStateId);
bool (*keyword_lex_fn)(TSLexer *, TSStateId);
TSSymbol keyword_capture_token;
struct {
const bool *states;
const TSSymbol *symbol_map;
void *(*create)(void);
void (*destroy)(void *);
bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist);
unsigned (*serialize)(void *, char *);
void (*deserialize)(void *, const char *, unsigned);
} external_scanner;
const TSStateId *primary_state_ids;
const char *name;
const TSSymbol *reserved_words;
uint16_t max_reserved_word_set_size;
uint32_t supertype_count;
const TSSymbol *supertype_symbols;
const TSMapSlice *supertype_map_slices;
const TSSymbol *supertype_map_entries;
TSLanguageMetadata metadata;
};
static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
uint32_t index = 0;
uint32_t size = len - index;
while (size > 1) {
uint32_t half_size = size / 2;
uint32_t mid_index = index + half_size;
const TSCharacterRange *range = &ranges[mid_index];
if (lookahead >= range->start && lookahead <= range->end) {
return true;
} else if (lookahead > range->end) {
index = mid_index;
}
size -= half_size;
}
const TSCharacterRange *range = &ranges[index];
return (lookahead >= range->start && lookahead <= range->end);
}
/*
* Lexer Macros
*/
#ifdef _MSC_VER
#define UNUSED __pragma(warning(suppress : 4101))
#else
#define UNUSED __attribute__((unused))
#endif
#define START_LEXER() \
bool result = false; \
bool skip = false; \
UNUSED \
bool eof = false; \
int32_t lookahead; \
goto start; \
next_state: \
lexer->advance(lexer, skip); \
start: \
skip = false; \
lookahead = lexer->lookahead;
#define ADVANCE(state_value) \
{ \
state = state_value; \
goto next_state; \
}
#define ADVANCE_MAP(...) \
{ \
static const uint16_t map[] = { __VA_ARGS__ }; \
for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \
if (map[i] == lookahead) { \
state = map[i + 1]; \
goto next_state; \
} \
} \
}
#define SKIP(state_value) \
{ \
skip = true; \
state = state_value; \
goto next_state; \
}
#define ACCEPT_TOKEN(symbol_value) \
result = true; \
lexer->result_symbol = symbol_value; \
lexer->mark_end(lexer);
#define END_STATE() return result;
/*
* Parse Table Macros
*/
#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
#define STATE(id) id
#define ACTIONS(id) id
#define SHIFT(state_value) \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = (state_value) \
} \
}}
#define SHIFT_REPEAT(state_value) \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = (state_value), \
.repetition = true \
} \
}}
#define SHIFT_EXTRA() \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.extra = true \
} \
}}
#define REDUCE(symbol_name, children, precedence, prod_id) \
{{ \
.reduce = { \
.type = TSParseActionTypeReduce, \
.symbol = symbol_name, \
.child_count = children, \
.dynamic_precedence = precedence, \
.production_id = prod_id \
}, \
}}
#define RECOVER() \
{{ \
.type = TSParseActionTypeRecover \
}}
#define ACCEPT_INPUT() \
{{ \
.type = TSParseActionTypeAccept \
}}
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_PARSER_H_
tools/tree-sitter-arkts/test/corpus/components.txt
+199
View File
@@ -0,0 +1,199 @@
==================
Entry component struct
==================
@Entry
@Component
struct Index {
build() {
}
}
---
(program
(struct_declaration
(decorator
(identifier))
(decorator
(identifier))
(type_identifier)
(class_body
(method_definition
(property_identifier)
(formal_parameters)
(statement_block)))))
==================
Custom dialog struct
==================
@CustomDialog
struct ConfirmDialog {
controller: CustomDialogController
build() {
}
}
---
(program
(struct_declaration
(decorator
(identifier))
(type_identifier)
(class_body
(public_field_definition
(property_identifier)
(type_annotation
(type_identifier)))
(method_definition
(property_identifier)
(formal_parameters)
(statement_block)))))
==================
Reusable component struct
==================
@Reusable
@Component
struct Card {
build() {
}
}
---
(program
(struct_declaration
(decorator
(identifier))
(decorator
(identifier))
(type_identifier)
(class_body
(method_definition
(property_identifier)
(formal_parameters)
(statement_block)))))
==================
Exported component struct
==================
@Component
export struct TitleBar {
build() {
}
}
---
(program
(export_statement
(decorator
(identifier))
(struct_declaration
(type_identifier)
(class_body
(method_definition
(property_identifier)
(formal_parameters)
(statement_block))))))
==================
Export default struct
==================
@Component
export default struct Footer {
build() {
}
}
---
(program
(export_statement
(decorator
(identifier))
(struct_declaration
(type_identifier)
(class_body
(method_definition
(property_identifier)
(formal_parameters)
(statement_block))))))
==================
Struct with decorator arguments
==================
@Entry({ routeName: 'home' })
@Component
struct Home {
build() {
}
}
---
(program
(struct_declaration
(decorator
(call_expression
(identifier)
(arguments
(object
(pair
(property_identifier)
(string
(string_fragment)))))))
(decorator
(identifier))
(type_identifier)
(class_body
(method_definition
(property_identifier)
(formal_parameters)
(statement_block)))))
==================
Two structs in one file
==================
@Component
struct A {
build() {
}
}
@Component
struct B {
build() {
}
}
---
(program
(struct_declaration
(decorator
(identifier))
(type_identifier)
(class_body
(method_definition
(property_identifier)
(formal_parameters)
(statement_block))))
(struct_declaration
(decorator
(identifier))
(type_identifier)
(class_body
(method_definition
(property_identifier)
(formal_parameters)
(statement_block)))))
tools/tree-sitter-arkts/test/corpus/global_functions.txt
+124
View File
@@ -0,0 +1,124 @@
==================
Builder function
==================
@Builder
function GlobalCard(title: string) {
Column() {
Text(title)
}
}
---
(program
(function_declaration
(decorator
(identifier))
(identifier)
(formal_parameters
(required_parameter
(identifier)
(type_annotation
(predefined_type))))
(statement_block
(expression_statement
(call_expression
(identifier)
(arguments)
(statement_block
(expression_statement
(call_expression
(identifier)
(arguments
(identifier))))))))))
==================
Extend function with implicit-receiver attributes
==================
@Extend(Text)
function fancy(size: number) {
.fontSize(size)
.fontColor(Color.Red)
}
---
(program
(function_declaration
(decorator
(call_expression
(identifier)
(arguments
(identifier))))
(identifier)
(formal_parameters
(required_parameter
(identifier)
(type_annotation
(predefined_type))))
(statement_block
(extend_attribute_chain
(extend_attribute
(property_identifier)
(arguments
(identifier)))
(extend_attribute
(property_identifier)
(arguments
(member_expression
(identifier)
(property_identifier))))))))
==================
Styles function
==================
@Styles
function pressedStyle() {
.backgroundColor('#eee')
}
---
(program
(function_declaration
(decorator
(identifier))
(identifier)
(formal_parameters)
(statement_block
(extend_attribute_chain
(extend_attribute
(property_identifier)
(arguments
(string
(string_fragment))))))))
==================
Plain function unchanged
==================
export function helper(x: number): number {
return x * 2
}
---
(program
(export_statement
(function_declaration
(identifier)
(formal_parameters
(required_parameter
(identifier)
(type_annotation
(predefined_type))))
(type_annotation
(predefined_type))
(statement_block
(return_statement
(binary_expression
(identifier)
(number)))))))
tools/tree-sitter-arkts/test/corpus/imports.txt
+74
View File
@@ -0,0 +1,74 @@
==================
Lazy named import
==================
import lazy { HeavyModule, Other } from './heavy'
---
(program
(import_statement
(import_clause
(import_lazy)
(named_imports
(import_specifier
(identifier))
(import_specifier
(identifier))))
(string
(string_fragment))))
==================
Default import named lazy still works
==================
import lazy from './lazymod'
---
(program
(import_statement
(import_clause
(identifier))
(string
(string_fragment))))
==================
Type-only import unaffected
==================
import type { Props } from './types'
---
(program
(import_statement
(import_clause
(named_imports
(import_specifier
(identifier))))
(string
(string_fragment))))
==================
Kit and default imports
==================
import { router } from '@kit.ArkUI'
import curves from '@ohos.curves'
---
(program
(import_statement
(import_clause
(named_imports
(import_specifier
(identifier))))
(string
(string_fragment)))
(import_statement
(import_clause
(identifier))
(string
(string_fragment))))
tools/tree-sitter-arkts/test/corpus/members.txt
+275
View File
@@ -0,0 +1,275 @@
==================
State member decorators
==================
@Component
struct S {
@State message: string = 'Hello'
@Prop count: number
@Link linked: Model
build() {
}
}
---
(program
(struct_declaration
(decorator
(identifier))
(type_identifier)
(class_body
(public_field_definition
(decorator
(identifier))
(property_identifier)
(type_annotation
(predefined_type))
(string
(string_fragment)))
(public_field_definition
(decorator
(identifier))
(property_identifier)
(type_annotation
(predefined_type)))
(public_field_definition
(decorator
(identifier))
(property_identifier)
(type_annotation
(type_identifier)))
(method_definition
(property_identifier)
(formal_parameters)
(statement_block)))))
==================
Keyed member decorators
==================
@Component
struct S {
@Provide('key') provided: string = ''
@Consume('key') consumed: string
@StorageLink('store') stored: number = 1
build() {
}
}
---
(program
(struct_declaration
(decorator
(identifier))
(type_identifier)
(class_body
(public_field_definition
(decorator
(call_expression
(identifier)
(arguments
(string
(string_fragment)))))
(property_identifier)
(type_annotation
(predefined_type))
(string))
(public_field_definition
(decorator
(call_expression
(identifier)
(arguments
(string
(string_fragment)))))
(property_identifier)
(type_annotation
(predefined_type)))
(public_field_definition
(decorator
(call_expression
(identifier)
(arguments
(string
(string_fragment)))))
(property_identifier)
(type_annotation
(predefined_type))
(number))
(method_definition
(property_identifier)
(formal_parameters)
(statement_block)))))
==================
Stacked Watch decorator
==================
@Component
struct S {
@State @Watch('onChange') watched: boolean = false
build() {
}
}
---
(program
(struct_declaration
(decorator
(identifier))
(type_identifier)
(class_body
(public_field_definition
(decorator
(identifier))
(decorator
(call_expression
(identifier)
(arguments
(string
(string_fragment)))))
(property_identifier)
(type_annotation
(predefined_type))
(false))
(method_definition
(property_identifier)
(formal_parameters)
(statement_block)))))
==================
Plain and private members
==================
@Component
struct S {
plain: number = 0
private scroller: Scroller = new Scroller()
build() {
}
}
---
(program
(struct_declaration
(decorator
(identifier))
(type_identifier)
(class_body
(public_field_definition
(property_identifier)
(type_annotation
(predefined_type))
(number))
(public_field_definition
(accessibility_modifier)
(property_identifier)
(type_annotation
(type_identifier))
(new_expression
(identifier)
(arguments)))
(method_definition
(property_identifier)
(formal_parameters)
(statement_block)))))
==================
Struct methods and getters
==================
@Component
struct S {
aboutToAppear() {
this.init()
}
get label(): string {
return 'x'
}
private init(): void {
}
build() {
}
}
---
(program
(struct_declaration
(decorator
(identifier))
(type_identifier)
(class_body
(method_definition
(property_identifier)
(formal_parameters)
(statement_block
(expression_statement
(call_expression
(member_expression
(this)
(property_identifier))
(arguments)))))
(method_definition
(property_identifier)
(formal_parameters)
(type_annotation
(predefined_type))
(statement_block
(return_statement
(string
(string_fragment)))))
(method_definition
(accessibility_modifier)
(property_identifier)
(formal_parameters)
(type_annotation
(predefined_type))
(statement_block))
(method_definition
(property_identifier)
(formal_parameters)
(statement_block)))))
==================
BuilderParam member
==================
@Component
struct S {
@BuilderParam content: () => void
build() {
}
}
---
(program
(struct_declaration
(decorator
(identifier))
(type_identifier)
(class_body
(public_field_definition
(decorator
(identifier))
(property_identifier)
(type_annotation
(function_type
(formal_parameters)
(predefined_type))))
(method_definition
(property_identifier)
(formal_parameters)
(statement_block)))))
tools/tree-sitter-arkts/test/corpus/typescript_compat.txt
+232
View File
@@ -0,0 +1,232 @@
==================
Observed class with decorator
==================
@Observed
class Model {
count: number = 0
}
---
(program
(class_declaration
(decorator
(identifier))
(type_identifier)
(class_body
(public_field_definition
(property_identifier)
(type_annotation
(predefined_type))
(number)))))
==================
Class with heritage and methods
==================
class Service extends Base implements Api {
private name: string
constructor(name: string) {
super()
this.name = name
}
getName(): string {
return this.name
}
}
---
(program
(class_declaration
(type_identifier)
(class_heritage
(extends_clause
(identifier))
(implements_clause
(type_identifier)))
(class_body
(public_field_definition
(accessibility_modifier)
(property_identifier)
(type_annotation
(predefined_type)))
(method_definition
(property_identifier)
(formal_parameters
(required_parameter
(identifier)
(type_annotation
(predefined_type))))
(statement_block
(expression_statement
(call_expression
(super)
(arguments)))
(expression_statement
(assignment_expression
(member_expression
(this)
(property_identifier))
(identifier)))))
(method_definition
(property_identifier)
(formal_parameters)
(type_annotation
(predefined_type))
(statement_block
(return_statement
(member_expression
(this)
(property_identifier))))))))
==================
Interface and enum
==================
interface Props {
title: string
onDone?: () => void
}
enum Color {
Red,
Green = 2,
}
---
(program
(interface_declaration
(type_identifier)
(interface_body
(property_signature
(property_identifier)
(type_annotation
(predefined_type)))
(property_signature
(property_identifier)
(type_annotation
(function_type
(formal_parameters)
(predefined_type))))))
(enum_declaration
(identifier)
(enum_body
(property_identifier)
(enum_assignment
(property_identifier)
(number)))))
==================
Type alias and generics
==================
type Handler<T> = (value: T) => void
function identity<T>(value: T): T {
return value
}
---
(program
(type_alias_declaration
(type_identifier)
(type_parameters
(type_parameter
(type_identifier)))
(function_type
(formal_parameters
(required_parameter
(identifier)
(type_annotation
(type_identifier))))
(predefined_type)))
(function_declaration
(identifier)
(type_parameters
(type_parameter
(type_identifier)))
(formal_parameters
(required_parameter
(identifier)
(type_annotation
(type_identifier))))
(type_annotation
(type_identifier))
(statement_block
(return_statement
(identifier)))))
==================
Namespace declaration
==================
namespace Util {
export function helper(): number {
return 1
}
}
---
(program
(expression_statement
(internal_module
(identifier)
(statement_block
(export_statement
(function_declaration
(identifier)
(formal_parameters)
(type_annotation
(predefined_type))
(statement_block
(return_statement
(number)))))))))
==================
Async arrow and await
==================
const load = async (url: string): Promise<string> => {
const res = await fetch(url)
return res.text()
}
---
(program
(lexical_declaration
(variable_declarator
(identifier)
(arrow_function
(formal_parameters
(required_parameter
(identifier)
(type_annotation
(predefined_type))))
(type_annotation
(generic_type
(type_identifier)
(type_arguments
(predefined_type))))
(statement_block
(lexical_declaration
(variable_declarator
(identifier)
(await_expression
(call_expression
(identifier)
(arguments
(identifier))))))
(return_statement
(call_expression
(member_expression
(identifier)
(property_identifier))
(arguments))))))))
tools/tree-sitter-arkts/test/corpus/ui_dsl.txt
+398
View File
@@ -0,0 +1,398 @@
==================
Trailing closure component call
==================
@Component
struct S {
build() {
Column() {
Text('hi')
}
}
}
---
(program
(struct_declaration
(decorator
(identifier))
(type_identifier)
(class_body
(method_definition
(property_identifier)
(formal_parameters)
(statement_block
(expression_statement
(call_expression
(identifier)
(arguments)
(statement_block
(expression_statement
(call_expression
(identifier)
(arguments
(string
(string_fragment)))))))))))))
==================
Attribute chain after closing brace
==================
@Component
struct S {
build() {
Column() {
Text('hi')
.fontSize(20)
}
.width('100%')
.height('100%')
}
}
---
(program
(struct_declaration
(decorator
(identifier))
(type_identifier)
(class_body
(method_definition
(property_identifier)
(formal_parameters)
(statement_block
(expression_statement
(call_expression
(member_expression
(call_expression
(member_expression
(call_expression
(identifier)
(arguments)
(statement_block
(expression_statement
(call_expression
(member_expression
(call_expression
(identifier)
(arguments
(string
(string_fragment))))
(property_identifier))
(arguments
(number))))))
(property_identifier))
(arguments
(string
(string_fragment))))
(property_identifier))
(arguments
(string
(string_fragment))))))))))
==================
Nested component blocks with arguments
==================
@Component
struct S {
build() {
Row({ space: 8 }) {
Button('Go')
.onClick(() => {
this.go()
})
}
.height(56)
}
}
---
(program
(struct_declaration
(decorator
(identifier))
(type_identifier)
(class_body
(method_definition
(property_identifier)
(formal_parameters)
(statement_block
(expression_statement
(call_expression
(member_expression
(call_expression
(identifier)
(arguments
(object
(pair
(property_identifier)
(number))))
(statement_block
(expression_statement
(call_expression
(member_expression
(call_expression
(identifier)
(arguments
(string
(string_fragment))))
(property_identifier))
(arguments
(arrow_function
(formal_parameters)
(statement_block
(expression_statement
(call_expression
(member_expression
(this)
(property_identifier))
(arguments))))))))))
(property_identifier))
(arguments
(number)))))))))
==================
Conditional rendering inside build
==================
@Component
struct S {
build() {
Column() {
if (this.flag) {
Text('on')
} else {
Text('off')
}
}
}
}
---
(program
(struct_declaration
(decorator
(identifier))
(type_identifier)
(class_body
(method_definition
(property_identifier)
(formal_parameters)
(statement_block
(expression_statement
(call_expression
(identifier)
(arguments)
(statement_block
(if_statement
(parenthesized_expression
(member_expression
(this)
(property_identifier)))
(statement_block
(expression_statement
(call_expression
(identifier)
(arguments
(string
(string_fragment))))))
(else_clause
(statement_block
(expression_statement
(call_expression
(identifier)
(arguments
(string
(string_fragment))))))))))))))))
==================
ForEach with item builder
==================
@Component
struct S {
build() {
List() {
ForEach(this.items, (item: string) => {
ListItem() {
Text(item)
}
}, (item: string) => item)
}
}
}
---
(program
(struct_declaration
(decorator
(identifier))
(type_identifier)
(class_body
(method_definition
(property_identifier)
(formal_parameters)
(statement_block
(expression_statement
(call_expression
(identifier)
(arguments)
(statement_block
(expression_statement
(call_expression
(identifier)
(arguments
(member_expression
(this)
(property_identifier))
(arrow_function
(formal_parameters
(required_parameter
(identifier)
(type_annotation
(predefined_type))))
(statement_block
(expression_statement
(call_expression
(identifier)
(arguments)
(statement_block
(expression_statement
(call_expression
(identifier)
(arguments
(identifier)))))))))
(arrow_function
(formal_parameters
(required_parameter
(identifier)
(type_annotation
(predefined_type))))
(identifier)))))))))))))
==================
Next-line block splits via ASI, TS-compatible
==================
@Component
struct S {
build() {
Column()
{
Text('hi')
}
}
}
---
(program
(struct_declaration
(decorator
(identifier))
(type_identifier)
(class_body
(method_definition
(property_identifier)
(formal_parameters)
(statement_block
(expression_statement
(call_expression
(identifier)
(arguments)))
(statement_block
(expression_statement
(call_expression
(identifier)
(arguments
(string
(string_fragment)))))))))))
==================
StateStyles anonymous style block
==================
@Component
struct S {
build() {
Button('x')
.stateStyles({
pressed: pressedStyle,
normal: {
.borderRadius(20)
.borderWidth(1)
}
})
}
}
---
(program
(struct_declaration
(decorator
(identifier))
(type_identifier)
(class_body
(method_definition
(property_identifier)
(formal_parameters)
(statement_block
(expression_statement
(call_expression
(member_expression
(call_expression
(identifier)
(arguments
(string
(string_fragment))))
(property_identifier))
(arguments
(object
(pair
(property_identifier)
(identifier))
(pair
(property_identifier)
(style_block
(extend_attribute
(property_identifier)
(arguments
(number)))
(extend_attribute
(property_identifier)
(arguments
(number))))))))))))))
==================
Mixin extends call keeps its class body
==================
class Mixin extends withTag(Base) {
run(): void {}
}
---
(program
(class_declaration
(type_identifier)
(class_heritage
(extends_clause
(call_expression
(identifier)
(arguments
(identifier)))))
(class_body
(method_definition
(property_identifier)
(formal_parameters)
(type_annotation
(predefined_type))
(statement_block)))))
tools/tree-sitter-arkts/tree-sitter.json
+16
View File
@@ -0,0 +1,16 @@
{
"grammars": [
{
"name": "arkts",
"camelcase": "Arkts",
"scope": "source.arkts",
"path": ".",
"file-types": ["ets"]
}
],
"metadata": {
"version": "0.1.0",
"license": "MIT",
"description": "ArkTS (HarmonyOS .ets) grammar for tree-sitter — fork of tree-sitter-typescript v0.23.2"
}
}