Files
codebase-memory-mcp/tools/tree-sitter-form/src/grammar.json
T
Martin Vogel 18fa9979ee Replace vmem with mimalloc global allocator, add extraction-phase prescan, fix __init__.py QN collision
Memory management:
- Vendor mimalloc v2.1.9 as global allocator (MI_OVERRIDE=1 in prod)
- New mem.h/mem.c: RSS-based budget tracking via mi_process_info()
- Remove vmem.c/vmem.h (mmap-based budget tracking)
- Remove slab tier2 bump allocator (~300 LOC); >64B goes to mimalloc
- Slab tier1 pages from malloc (= mimalloc) instead of vmem
- Arena blocks from malloc instead of vmem
- Budget raised from 35% to 50% RAM (no more untracked C++ heap)

Extraction-phase prescan (eliminates disk re-reads):
- HTTP call sites: keyword check + URL extraction during extraction
- HTTP routes: decorator + source-based extraction during extraction
- Config file refs: regex scan during extraction
- httplinks: 41.8s → 13ms on Linux kernel (3,212x faster)
- configlink: 41.4s → 0.8s on Linux kernel (54x faster)
- Linux kernel fast-mode total: 2m38s → 1m18s

Bug fixes:
- __init__.py Module QN no longer collides with Folder QN
- index.ts same fix for JS/TS packages
- 13 regression tests for QN collision at FQN + extraction layers
- search_graph/search_code default limit raised from 10 to 500k
- Resolve all clang-tidy, cppcheck, and clang-format warnings

Repo cleanup:
- tree-sitter-form, tree-sitter-magma moved to tools/
- .gitignore: build/, node_modules/, graph-ui/dist/, TEST_PLAN.md
2026-03-17 22:05:57 +01:00

1450 lines
31 KiB
JSON

{
"$schema": "https://tree-sitter.github.io/tree-sitter/assets/schemas/grammar.schema.json",
"name": "form",
"word": "identifier",
"rules": {
"source_file": {
"type": "REPEAT",
"content": {
"type": "SYMBOL",
"name": "_statement"
}
},
"_statement": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "declaration_statement"
},
{
"type": "SYMBOL",
"name": "substitution_statement"
},
{
"type": "SYMBOL",
"name": "print_statement"
},
{
"type": "SYMBOL",
"name": "procedure_definition"
},
{
"type": "SYMBOL",
"name": "call_statement"
},
{
"type": "SYMBOL",
"name": "if_statement"
},
{
"type": "SYMBOL",
"name": "repeat_statement"
},
{
"type": "SYMBOL",
"name": "do_loop"
},
{
"type": "SYMBOL",
"name": "argument_block"
},
{
"type": "SYMBOL",
"name": "label_statement"
},
{
"type": "SYMBOL",
"name": "module_directive"
},
{
"type": "SYMBOL",
"name": "include_directive"
},
{
"type": "SYMBOL",
"name": "define_directive"
},
{
"type": "SYMBOL",
"name": "preproc_directive_with_arg"
},
{
"type": "SYMBOL",
"name": "preproc_directive_bare"
},
{
"type": "SYMBOL",
"name": "preproc_line_directive"
},
{
"type": "SYMBOL",
"name": "preproc_do"
},
{
"type": "SYMBOL",
"name": "expression_statement"
}
]
},
"declaration_statement": {
"type": "SEQ",
"members": [
{
"type": "FIELD",
"name": "kind",
"content": {
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "Symbols"
},
{
"type": "STRING",
"value": "Symbol"
},
{
"type": "STRING",
"value": "Functions"
},
{
"type": "STRING",
"value": "Function"
},
{
"type": "STRING",
"value": "CFunctions"
},
{
"type": "STRING",
"value": "CFunction"
},
{
"type": "STRING",
"value": "Tensors"
},
{
"type": "STRING",
"value": "Tensor"
},
{
"type": "STRING",
"value": "Vectors"
},
{
"type": "STRING",
"value": "Vector"
},
{
"type": "STRING",
"value": "Indices"
},
{
"type": "STRING",
"value": "Index"
},
{
"type": "STRING",
"value": "Sets"
},
{
"type": "STRING",
"value": "Set"
},
{
"type": "STRING",
"value": "Tables"
},
{
"type": "STRING",
"value": "Table"
},
{
"type": "STRING",
"value": "NTensors"
},
{
"type": "STRING",
"value": "NTensor"
},
{
"type": "STRING",
"value": "AutoDeclare"
},
{
"type": "STRING",
"value": "Dimension"
},
{
"type": "STRING",
"value": "FixIndex"
},
{
"type": "STRING",
"value": "Off"
},
{
"type": "STRING",
"value": "On"
}
]
}
},
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "REPEAT",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": ","
},
{
"type": "SYMBOL",
"name": "identifier"
}
]
}
}
]
},
{
"type": "STRING",
"value": ";"
}
]
},
"substitution_statement": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "Local"
},
{
"type": "STRING",
"value": "Global"
}
]
},
{
"type": "BLANK"
}
]
},
{
"type": "STRING",
"value": "id"
},
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "STRING",
"value": "="
},
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "STRING",
"value": ";"
}
]
},
"print_statement": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "Print"
},
{
"type": "STRING",
"value": "NoPrint"
},
{
"type": "STRING",
"value": "Format"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "BLANK"
}
]
},
{
"type": "STRING",
"value": ";"
}
]
},
"procedure_definition": {
"type": "PREC",
"value": 10,
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "#procedure"
},
{
"type": "FIELD",
"name": "name",
"content": {
"type": "SYMBOL",
"name": "identifier"
}
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "parameter_list"
},
{
"type": "BLANK"
}
]
},
{
"type": "REPEAT",
"content": {
"type": "SYMBOL",
"name": "_statement"
}
},
{
"type": "STRING",
"value": "#endprocedure"
}
]
}
},
"parameter_list": {
"type": "SEQ",
"members": [
{
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "STRING",
"value": "("
}
},
{
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "ALIAS",
"content": {
"type": "SYMBOL",
"name": "identifier"
},
"named": true,
"value": "parameter"
},
{
"type": "REPEAT",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": ","
},
{
"type": "ALIAS",
"content": {
"type": "SYMBOL",
"name": "identifier"
},
"named": true,
"value": "parameter"
}
]
}
}
]
},
{
"type": "BLANK"
}
]
},
{
"type": "STRING",
"value": ")"
}
]
},
"call_statement": {
"type": "PREC",
"value": 10,
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "#call"
},
{
"type": "FIELD",
"name": "name",
"content": {
"type": "SYMBOL",
"name": "identifier"
}
},
{
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "STRING",
"value": "("
}
},
{
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "REPEAT",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": ","
},
{
"type": "SYMBOL",
"name": "_expression"
}
]
}
}
]
},
{
"type": "BLANK"
}
]
},
{
"type": "STRING",
"value": ")"
}
]
},
{
"type": "BLANK"
}
]
}
]
}
},
"include_directive": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "#include"
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "string_literal"
},
{
"type": "SYMBOL",
"name": "identifier"
}
]
}
]
},
"define_directive": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "#define"
},
{
"type": "STRING",
"value": "#redefine"
}
]
},
{
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "SYMBOL",
"name": "string_literal"
}
]
},
"preproc_directive_with_arg": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "#ifdef"
},
{
"type": "STRING",
"value": "#ifndef"
},
{
"type": "STRING",
"value": "#if"
},
{
"type": "STRING",
"value": "#case"
},
{
"type": "STRING",
"value": "#switch"
},
{
"type": "STRING",
"value": "#undefine"
}
]
},
{
"type": "SYMBOL",
"name": "_expression"
}
]
},
"preproc_directive_bare": {
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "#else"
},
{
"type": "STRING",
"value": "#endif"
},
{
"type": "STRING",
"value": "#break"
},
{
"type": "STRING",
"value": "#endswitch"
}
]
},
"preproc_line_directive": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "#message"
},
{
"type": "STRING",
"value": "#write"
},
{
"type": "STRING",
"value": "#close"
},
{
"type": "STRING",
"value": "#system"
},
{
"type": "STRING",
"value": "#optimize"
},
{
"type": "STRING",
"value": "#Optimize"
},
{
"type": "STRING",
"value": "#toexternal"
},
{
"type": "STRING",
"value": "#fromexternal"
},
{
"type": "STRING",
"value": "#setexternal"
},
{
"type": "STRING",
"value": "#rmexternal"
},
{
"type": "STRING",
"value": "#external"
},
{
"type": "STRING",
"value": "#factdollar"
},
{
"type": "STRING",
"value": "#pipe"
}
]
},
{
"type": "PATTERN",
"value": "[^\\n]*"
}
]
},
"preproc_do": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "#do"
},
{
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "STRING",
"value": "="
},
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": ","
},
{
"type": "SYMBOL",
"name": "_expression"
}
]
},
{
"type": "BLANK"
}
]
},
{
"type": "REPEAT",
"content": {
"type": "SYMBOL",
"name": "_statement"
}
},
{
"type": "STRING",
"value": "#enddo"
}
]
},
"module_directive": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "."
},
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "sort"
},
{
"type": "STRING",
"value": "end"
},
{
"type": "STRING",
"value": "clear"
},
{
"type": "STRING",
"value": "store"
},
{
"type": "STRING",
"value": "global"
}
]
}
]
},
"label_statement": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "label"
},
{
"type": "STRING",
"value": "goto"
}
]
},
{
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "STRING",
"value": ";"
}
]
},
"if_statement": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "if"
},
{
"type": "STRING",
"value": "("
},
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "STRING",
"value": ")"
},
{
"type": "REPEAT",
"content": {
"type": "SYMBOL",
"name": "_statement"
}
},
{
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "else"
},
{
"type": "REPEAT",
"content": {
"type": "SYMBOL",
"name": "_statement"
}
}
]
},
{
"type": "BLANK"
}
]
},
{
"type": "STRING",
"value": "endif"
},
{
"type": "STRING",
"value": ";"
}
]
},
"repeat_statement": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "repeat"
},
{
"type": "STRING",
"value": ";"
},
{
"type": "REPEAT",
"content": {
"type": "SYMBOL",
"name": "_statement"
}
},
{
"type": "STRING",
"value": "endrepeat"
},
{
"type": "STRING",
"value": ";"
}
]
},
"argument_block": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "argument"
},
{
"type": "STRING",
"value": ";"
},
{
"type": "REPEAT",
"content": {
"type": "SYMBOL",
"name": "_statement"
}
},
{
"type": "STRING",
"value": "endargument"
},
{
"type": "STRING",
"value": ";"
}
]
},
"do_loop": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "do"
},
{
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "STRING",
"value": "="
},
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "STRING",
"value": ","
},
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "STRING",
"value": ";"
},
{
"type": "REPEAT",
"content": {
"type": "SYMBOL",
"name": "_statement"
}
},
{
"type": "STRING",
"value": "enddo"
},
{
"type": "STRING",
"value": ";"
}
]
},
"expression_statement": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "STRING",
"value": ";"
}
]
},
"_expression": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "SYMBOL",
"name": "number"
},
{
"type": "SYMBOL",
"name": "string_literal"
},
{
"type": "SYMBOL",
"name": "dollar_variable"
},
{
"type": "SYMBOL",
"name": "wildcard"
},
{
"type": "SYMBOL",
"name": "function_application"
},
{
"type": "SYMBOL",
"name": "binary_expression"
},
{
"type": "SYMBOL",
"name": "unary_expression"
},
{
"type": "SYMBOL",
"name": "parenthesized_expression"
},
{
"type": "SYMBOL",
"name": "index_expression"
}
]
},
"function_application": {
"type": "PREC",
"value": 2,
"content": {
"type": "SEQ",
"members": [
{
"type": "FIELD",
"name": "name",
"content": {
"type": "SYMBOL",
"name": "identifier"
}
},
{
"type": "STRING",
"value": "("
},
{
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "REPEAT",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": ","
},
{
"type": "SYMBOL",
"name": "_expression"
}
]
}
}
]
},
{
"type": "BLANK"
}
]
},
{
"type": "STRING",
"value": ")"
}
]
}
},
"binary_expression": {
"type": "CHOICE",
"members": [
{
"type": "PREC_LEFT",
"value": 1,
"content": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "+"
},
{
"type": "STRING",
"value": "-"
}
]
},
{
"type": "SYMBOL",
"name": "_expression"
}
]
}
},
{
"type": "PREC_LEFT",
"value": 2,
"content": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "*"
},
{
"type": "STRING",
"value": "/"
}
]
},
{
"type": "SYMBOL",
"name": "_expression"
}
]
}
},
{
"type": "PREC_LEFT",
"value": 0,
"content": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "=="
},
{
"type": "STRING",
"value": "!="
},
{
"type": "STRING",
"value": "<="
},
{
"type": "STRING",
"value": ">="
},
{
"type": "STRING",
"value": "<"
},
{
"type": "STRING",
"value": ">"
}
]
},
{
"type": "SYMBOL",
"name": "_expression"
}
]
}
},
{
"type": "PREC_LEFT",
"value": -1,
"content": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "&&"
},
{
"type": "STRING",
"value": "||"
}
]
},
{
"type": "SYMBOL",
"name": "_expression"
}
]
}
},
{
"type": "PREC_RIGHT",
"value": 3,
"content": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "STRING",
"value": "^"
},
{
"type": "SYMBOL",
"name": "_expression"
}
]
}
}
]
},
"unary_expression": {
"type": "PREC",
"value": 4,
"content": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "-"
},
{
"type": "STRING",
"value": "+"
},
{
"type": "STRING",
"value": "!"
}
]
},
{
"type": "SYMBOL",
"name": "_expression"
}
]
}
},
"parenthesized_expression": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "("
},
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "STRING",
"value": ")"
}
]
},
"index_expression": {
"type": "PREC",
"value": 1,
"content": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "STRING",
"value": "["
},
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "REPEAT",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": ","
},
{
"type": "SYMBOL",
"name": "_expression"
}
]
}
}
]
},
{
"type": "STRING",
"value": "]"
}
]
}
},
"identifier": {
"type": "PATTERN",
"value": "[a-zA-Z_][a-zA-Z0-9_]*"
},
"dollar_variable": {
"type": "TOKEN",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "$"
},
{
"type": "PATTERN",
"value": "[a-zA-Z_][a-zA-Z0-9_]*"
}
]
}
},
"wildcard": {
"type": "TOKEN",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "PATTERN",
"value": "[a-zA-Z_][a-zA-Z0-9_]*"
},
{
"type": "STRING",
"value": "?"
}
]
},
{
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "?"
},
{
"type": "PATTERN",
"value": "[a-zA-Z_][a-zA-Z0-9_]*"
}
]
}
]
}
},
"number": {
"type": "PATTERN",
"value": "\\d+(\\.\\d+)?"
},
"string_literal": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "\""
},
{
"type": "PATTERN",
"value": "[^\"]*"
},
{
"type": "STRING",
"value": "\""
}
]
},
"line_comment": {
"type": "TOKEN",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "*"
},
{
"type": "PATTERN",
"value": "[^\\n]*"
}
]
}
},
"block_comment": {
"type": "TOKEN",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "/*"
},
{
"type": "PATTERN",
"value": "[^*]*\\*+([^/*][^*]*\\*+)*"
},
{
"type": "STRING",
"value": "/"
}
]
}
}
},
"extras": [
{
"type": "PATTERN",
"value": "\\s"
},
{
"type": "SYMBOL",
"name": "line_comment"
},
{
"type": "SYMBOL",
"name": "block_comment"
}
],
"conflicts": [
[
"procedure_definition"
],
[
"call_statement"
]
],
"precedences": [],
"externals": [],
"inline": [],
"supertypes": [],
"reserved": {}
}