Files
ruff/crates/ruff_python_ast/generate.py
Micha Reiser 8e8bb225a2 Inline conversions between handles for the same arena node
Every variant of a classified expression carries the same row and arena
pointers. Expose the generated conversions across crate boundaries so
the compiler can return those pointers directly in ExprKind::view.
2026-11-20 17:37:48 +00:00

1321 lines
50 KiB
Python
Executable File

#!/usr/bin/python
# /// script
# requires-python = ">=3.11"
# dependencies = []
#
# [tool.uv]
# exclude-newer = "P7D"
# require-build-hashes = true
# preview-features = ["build-dependency-hashes", "missing-exclude-newer-package-lock"]
# ///
from __future__ import annotations
import re
from dataclasses import dataclass
from pathlib import Path
from subprocess import check_output
from typing import Any
import tomllib
# Types that require `crate::`. We can slowly remove these types as we move them to generate scripts.
types_requiring_crate_prefix = {
"IpyEscapeKind",
"ExprContext",
"Identifier",
"Number",
"BytesValue",
"StringValue",
"FStringValue",
"TStringValue",
"Arguments",
"CmpOp",
"Comprehension",
"MappingPatternItems",
"ParameterItems",
"UnaryOp",
"BoolOp",
"Operator",
"Decorator",
"TypeParams",
"Parameters",
"ElifElseClause",
"WithItem",
"MatchCase",
"Alias",
"Singleton",
"PatternArguments",
}
@dataclass
class VisitorInfo:
name: str
accepts_sequence: bool = False
# Map of AST node types to their corresponding visitor information.
# Only visitors that are different from the default `visit_*` method are included.
# These visitors either have a different name or accept a sequence of items.
type_to_visitor_function: dict[str, VisitorInfo] = {
"TypeParams": VisitorInfo("visit_type_params", True),
"Parameters": VisitorInfo("visit_parameters", True),
"Stmt": VisitorInfo("visit_body", True),
"Arguments": VisitorInfo("visit_arguments", True),
}
def rustfmt(code: str) -> str:
return check_output(["rustfmt", "--emit=stdout"], input=code, text=True)
def to_snake_case(node: str) -> str:
"""Converts CamelCase to snake_case"""
return re.sub(r"([A-Z])", r"_\1", node).lower().lstrip("_")
def write_rustdoc(out: list[str], doc: str) -> None:
for line in doc.split("\n"):
out.append(f"/// {line}")
# ------------------------------------------------------------------------------
# Read AST description
def load_ast(root: Path) -> Ast:
ast_path = root.joinpath("crates", "ruff_python_ast", "ast.toml")
with ast_path.open("rb") as ast_file:
ast = tomllib.load(ast_file)
return Ast(ast)
# ------------------------------------------------------------------------------
# Preprocess
@dataclass
class Ast:
"""
The parsed representation of the `ast.toml` file. Defines all of the Python
AST syntax nodes, and which groups (`Stmt`, `Expr`, etc.) they belong to.
"""
groups: list[Group]
all_nodes: list[Node]
nodes_in_tag_order: list[Node]
def __init__(self, ast: dict[str, Any]) -> None:
groups = [Group(name, group) for name, group in ast.items()]
self.groups = [group for group in groups if group.name != "ungrouped"]
self.all_nodes = [node for group in groups for node in group.nodes]
# With expressions first and statements last, both families can be
# recognized by comparing the row tag against a single boundary.
rank = {"Expr": -1, "Stmt": 1}
tag_groups = sorted(groups, key=lambda group: rank.get(group.name, 0))
self.nodes_in_tag_order = [node for group in tag_groups for node in group.nodes]
@dataclass
class Group:
name: str
nodes: list[Node]
add_suffix_to_is_methods: bool
doc: str | None
def __init__(self, group_name: str, group: dict[str, Any]) -> None:
self.name = group_name
self.add_suffix_to_is_methods = group.get("add_suffix_to_is_methods", False)
self.doc = group.get("doc")
self.nodes = [
Node(self, node_name, node) for node_name, node in group["nodes"].items()
]
@dataclass
class Node:
name: str
variant: str
doc: str | None
fields: list[Field]
source_order: list[str] | None
debug_range_first: bool
def __init__(self, group: Group, node_name: str, node: dict[str, Any]) -> None:
self.name = node_name
self.variant = node.get("variant", node_name.removeprefix(group.name))
self.doc = node.get("doc")
self.fields = [Field(field) for field in node["fields"]]
self.source_order = node.get("source_order")
# Auxiliary nodes print their source range first in parser snapshots.
self.debug_range_first = group.name in {
"ungrouped",
"ExceptHandler",
"InterpolatedStringElement",
}
@dataclass
class Field:
name: str
ty: str
optional: bool
is_list: bool
is_annotation: bool
def __init__(self, field: dict[str, Any]) -> None:
self.name = field["name"]
spec = field["type"]
self.optional = spec.endswith("?")
self.is_list = spec.endswith("*")
self.ty = spec[:-1] if self.optional or self.is_list else spec
if not self.ty.isidentifier():
raise ValueError(f"Invalid type for field {self.name}: {spec}")
self.is_annotation = field.get("is_annotation", False)
# ------------------------------------------------------------------------------
# Flat syntax storage
PARTS = {
"StringValue": "StringLiteral",
"BytesValue": "BytesLiteral",
"FStringValue": "FStringPart",
"TStringValue": "TString",
}
FLAGS = {"StringLiteralFlags", "BytesLiteralFlags", "FStringFlags", "TStringFlags"}
TAGS = {
"ExprContext",
"Operator",
"CmpOp",
"UnaryOp",
"BoolOp",
"Singleton",
"IpyEscapeKind",
"ConversionFlag",
}
@dataclass(frozen=True)
class WireField:
name: str
width: int
result: str
build_type: str
encode: str
read: str
def field_codec(field: Field, nodes: set[str]) -> WireField:
name = field.name
inner = field.ty
value = f"self.{name}"
def codec(
width: int,
result: str,
encode: str,
read: str,
*,
build_type: str | None = None,
) -> WireField:
return WireField(name, width, result, build_type or result, encode, read)
if inner in {"MappingPatternItems", "ParameterItems"}:
return codec(
8,
f"crate::flat::{inner}<'a>",
f"{value}.word()",
f"super::{inner}(List::new(self.syntax_node().ast, word))",
build_type=f"super::super::{inner}Id",
)
if field.is_list or inner in PARTS:
item = PARTS.get(inner, inner)
element = (
f"AstNodeId<crate::kind::{item}>" if item in nodes else f"crate::{item}"
)
read = "List::new(self.syntax_node().ast, word)"
if inner == "StringValue":
result = "crate::flat::StringValue<'a>"
read = (
"crate::flat::StringValue::new(List::new(self.syntax_node().ast, word), "
"u64::from(self.syntax_node().row.expr_string_literal_text_offset()) | "
"(u64::from(self.syntax_node().row.expr_string_literal_text_len()) << 32))"
)
elif inner in PARTS:
wrapper = {
"FStringValue": "FStringValue",
"TStringValue": "TStringValue",
"BytesValue": "BytesValue",
}[inner]
result = f"crate::flat::{wrapper}<'a>"
read = f"crate::flat::{wrapper}::new({read})"
else:
result = (
f"NodeList<'a, crate::kind::{item}>"
if item in nodes
else f"List<'a, {element}>"
)
return codec(
8,
result,
f"{value}.word()",
read,
build_type=f"super::super::ListId<{element}>",
)
if inner in nodes:
view = f"crate::{inner}<'a>"
handle = f"BuildId<crate::kind::{inner}>"
if field.optional:
return codec(
4,
f"Option<{view}>",
f"{value}.map_or(0, |id| u64::from(id.slot))",
"self.syntax_node().ast.optional_child(word)",
build_type=f"Option<{handle}>",
)
return codec(
4,
view,
f"u64::from({value}.slot)",
"self.syntax_node().ast.child(word)",
build_type=handle,
)
if inner == "Name":
return codec(
4,
"&'a crate::name::Name",
f"u64::from({value}.0)",
"&self.syntax_node().ast.names[split_words(word)[0] as usize]",
build_type="super::super::NameId",
)
if inner == "str":
return codec(
8,
"&'a str",
f"builder.store_string(&{value})",
"self.syntax_node().ast.string(word)",
build_type="Box<str>",
)
if inner == "LiteralText":
return codec(
8,
"&'a str",
f"builder.store_string({value})",
"self.syntax_node().ast.string(word)",
)
if inner == "LiteralBytes":
return codec(
8,
"&'a [u8]",
f"builder.store_bytes({value})",
"self.syntax_node().ast.bytes(word)",
)
if inner in TAGS:
return codec(1, f"crate::{inner}", f"{value}.encode()", "Word::decode(word)")
if inner in FLAGS:
return codec(
1,
f"crate::{inner}",
f"u64::from({value}.bits())",
f"crate::{inner}::from_bits(word.to_le_bytes()[0])",
)
if inner == "bool":
return codec(1, "bool", f"u64::from({value})", "word != 0")
if inner == "u32":
return codec(4, "u32", f"u64::from({value})", "split_words(word)[0]")
if inner == "Number":
return codec(
4,
"&'a crate::Number",
f"{{ let slot = offset(builder.ast.numbers.len()); builder.ast.numbers.push({value}); u64::from(slot) }}",
"&self.syntax_node().ast.numbers[split_words(word)[0] as usize]",
build_type="crate::Number",
)
if inner == "DebugText":
assert field.optional
return codec(
4,
"Option<&'a crate::DebugText>",
f"{value}.map_or(u64::from(ABSENT), |text| {{ let slot = offset(builder.ast.debug_text.len()); builder.ast.debug_text.push(text); u64::from(slot) }})",
"(split_words(word)[0] != ABSENT).then(|| &self.syntax_node().ast.debug_text[split_words(word)[0] as usize])",
build_type="Option<crate::DebugText>",
)
raise ValueError(f"Missing flat field codec for {field.name}: {field.ty}")
@dataclass(frozen=True)
class InlineField:
name: str
offset: int
width: int
# These derived values are read without following the node's children.
CACHED_FIELDS = {
"ExprStringLiteral": [
InlineField("text_offset", 8, 4),
InlineField("text_len", 12, 2),
],
}
def layout(
fields: list[WireField], node: str
) -> tuple[dict[str, int], dict[str, int], int | None]:
"""Place naturally aligned fields in fifteen bytes, spilling fields into extra if needed."""
reserved = {
i
for field in CACHED_FIELDS.get(node, [])
for i in range(field.offset, field.offset + field.width)
}
def place(inline: list[WireField], extra: int | None) -> dict[str, int] | None:
used = reserved.copy()
if extra is not None:
span = range(extra, extra + 4)
if not used.isdisjoint(span):
return None
used.update(span)
positions = {}
for field in sorted(inline, key=lambda field: -field.width):
for start in range(0, 16 - field.width, field.width):
span = range(start, start + field.width)
if used.isdisjoint(span):
positions[field.name] = start
used.update(span)
break
else:
return None
return positions
def with_extra(inline: list[WireField]) -> tuple[dict[str, int], int] | None:
# Moving the spill offset can leave an aligned eight-byte slot for a list.
for extra in (0, 4, 8):
if (positions := place(inline, extra)) is not None:
return positions, extra
return None
if (complete := place(fields, None)) is not None:
return complete, {}, None
kept: list[WireField] = []
for field in sorted(fields, key=lambda field: field.width):
if with_extra([*kept, field]) is not None:
kept.append(field)
placed = with_extra(kept)
assert placed is not None
offsets, extra = placed
spills = {}
words = 0
for f in fields:
if f not in kept:
spills[f.name] = words
words += (f.width + 3) // 4
return offsets, spills, extra
def source_order_fields(node: Node) -> list[Field]:
fields = node.fields
if node.source_order is None:
return fields
positions = {name: index for index, name in enumerate(node.source_order)}
return sorted(fields, key=lambda field: positions.get(field.name, len(fields)))
def child_fields(node: Node, nodes: set[str]) -> list[Field]:
return [
field
for field in source_order_fields(node)
if (
field.ty in nodes
or field.ty in PARTS
or field.ty
in {
"ArgumentId",
"MappingPatternItems",
"ParameterItems",
"PatternArgumentId",
}
)
]
def child_sources(
node: Node, fields: list[Field], words: dict[str, str], view: str
) -> list[tuple[str, str]]:
"""The source-ordered fields, each containing a handle or a list."""
if not fields:
return []
result = []
for field in fields:
raw = words[field.name].replace(
"self.syntax_node().ast.extra_word", "self.extra_word"
)
if (node.name, field.name) == ("InterpolatedElement", "format_spec"):
# The source-order visitor exposes the elements of the auxiliary spec.
result.append(
(
"list",
f"{view}.format_spec().map_or(&[][..], |spec| spec.elements().words())",
)
)
elif (
field.is_list
or field.ty in PARTS
or field.ty
in {
"ArgumentId",
"MappingPatternItems",
"ParameterItems",
"PatternArgumentId",
}
):
result.append(
(
"list",
f"List::<AstNodeId<crate::kind::Any>>::new(self, {raw}).words()",
)
)
elif field.optional:
result.append(("optional", f"split_words({raw})[0]"))
else:
result.append(("node", f"split_words({raw})[0]"))
return result
def field_value(kind: str, value: str) -> str:
if kind in {"node", "optional"}:
return f"Field::One({child_handle(kind, value)})"
assert kind == "list"
return f"Field::Rows({value})"
def child_handle(kind: str, value: str) -> str:
if kind == "node":
return f"Some(AstNodeId::from_word({value}))"
assert kind == "optional"
return f"{{ let word = {value}; (word != 0).then(|| AstNodeId::from_word(word)) }}"
def simple_children(
node: Node,
nodes: set[str],
sources: list[tuple[str, str]],
schema: list[InlineField],
) -> str | None:
"""Most nodes can borrow one slice from the row or the extra buffer."""
kinds = tuple("node" if kind == "optional" else kind for kind, _ in sources)
if not sources:
return "Children::empty(self)"
if (
all(kind == "node" for kind in kinds)
and (words := inline_child_slice(child_fields(node, nodes), schema)) is not None
):
return f"Children::inline(self, {words})"
if kinds == ("list",):
return f"Children::rows(self, {sources[0][1]})"
if kinds in (("node", "list"), ("list", "node"), ("node", "list", "node")):
first = child_handle(*sources[0]) if kinds[0] == "node" else "None"
last = child_handle(*sources[-1]) if kinds[-1] == "node" else "None"
return f"Children::sequence(self, {first}, {sources[kinds.index('list')][1]}, {last})"
return None
def child_field(
node: Node, nodes: set[str], words: dict[str, str], schema: list[InlineField]
) -> str:
sources = child_sources(
node, child_fields(node, nodes), words, f"node.retype::<crate::{node.name}>()"
)
if simple_children(node, nodes, sources, schema) is not None:
return "Field::One(None)"
arms = [f"{i} => {field_value(*source)}," for i, source in enumerate(sources)]
return "match field {" + "".join(arms) + "_ => Field::One(None) }"
def direct_children(
node: Node, nodes: set[str], words: dict[str, str], schema: list[InlineField]
) -> str:
view = f"node.retype::<crate::{node.name}>()"
sources = child_sources(node, child_fields(node, nodes), words, view)
return (
simple_children(node, nodes, sources, schema)
or f"Children::fields(node, {len(sources)})"
)
def inline_child_slice(children: list[Field], schema: list[InlineField]) -> str | None:
"""Borrow consecutive inline handles when missing children can be trimmed at the ends."""
offsets = {field.name: field.offset for field in schema if field.width == 4}
if any(field.name not in offsets for field in children):
return None
start = offsets[children[0].name]
if any(offsets[field.name] != start + 4 * i for i, field in enumerate(children)):
return None
if any(field.optional for field in children[1:-1]):
return None
first = str(children[0].optional).lower()
last = str(len(children) > 1 and children[-1].optional).lower()
return f"record.inline_children::<{start // 4}, {len(children)}, {first}, {last}>()"
def next_descendant(
node: Node, nodes: set[str], words: dict[str, str], _schema: list[InlineField]
) -> str:
sources = child_sources(
node, child_fields(node, nodes), words, f"node.retype::<crate::{node.name}>()"
)
resume = "stack.resume()"
steps = []
for i in reversed(range(len(sources))):
kind, value = sources[i]
if kind == "node":
if i == 0:
return "".join(steps) + value
steps.append(f"stack.push_node({value});")
elif kind == "list" and i == 0:
return "".join(steps) + f"stack.descend_list({value})"
elif kind == "optional":
if i == 0:
return (
"".join(steps)
+ f"let word = {value}; if word == 0 {{ {resume} }} else {{ word }}"
)
steps.append(
f"let word = {value}; if word != 0 {{ stack.push_node(word); }}"
)
else:
assert kind == "list"
steps.append(f"stack.push_list({value});")
return "".join(steps) + resume
def walk_fields(node: Node, nodes: set[str]) -> str:
"""Visit the fields of a projected node in Python source order."""
if node.name == "Arguments":
return """
for argument in node.items() {
match argument {
super::ArgOrKeyword::Arg(expr) => visitor.visit_expr(expr),
super::ArgOrKeyword::Keyword(keyword) => visitor.visit_keyword(keyword),
}
}
"""
if node.name == "PatternArguments":
return """
for argument in node.items() {
match argument {
crate::PatternOrKeyword::Pattern(pattern) => visitor.visit_pattern(pattern),
crate::PatternOrKeyword::Keyword(keyword) => visitor.visit_pattern_keyword(keyword),
}
}
"""
if node.name == "PatternMatchMapping":
return """
for item in node.items() {
match item {
crate::MappingPatternItem::Entry { key, pattern } => {
visitor.visit_expr(key);
visitor.visit_pattern(pattern);
}
crate::MappingPatternItem::Rest(rest) => visitor.visit_identifier(rest),
}
}
"""
if node.name == "Parameters":
return """
for parameter in node.iter_source_order() {
match parameter {
super::AnyParameter::NonVariadic(node) => { visitor.visit_parameter_with_default(node); }
super::AnyParameter::Variadic(node) => visitor.visit_parameter(node),
}
}
"""
if node.name == "ExprBoolOp":
return """let mut values = node.values().iter();
if let Some(left) = values.next() { visitor.visit_expr(left); }
visitor.visit_bool_op(node.op());
for value in values { visitor.visit_expr(value); }"""
if node.name == "ExprCompare":
return """visitor.visit_expr(node.first_operand());
for (op, right) in node.operators_and_comparators() { visitor.visit_cmp_op(op); visitor.visit_expr(right); }"""
statements = []
for field in source_order_fields(node):
inner = field.ty
get = f"node.{field.name}()"
method = f"visit_{to_snake_case(PARTS.get(inner, inner))}"
if field.is_annotation:
method = "visit_annotation"
if (node.name, field.name) == ("InterpolatedElement", "format_spec"):
statements.append(
f"if let Some(spec) = {get} {{ spec.visit_source_order(visitor); }}"
)
elif inner == "FStringValue":
statements.append(
f"for part in {get} {{ match part {{ super::FStringPartView::Literal(v) => visitor.visit_string_literal(v), super::FStringPartView::FString(v) => visitor.visit_f_string(v) }} }}"
)
elif inner == "Stmt" and field.is_list:
statements.append(f"visitor.visit_body({get});")
elif inner in PARTS or (inner in nodes and field.is_list):
statements.append(f"for item in {get} {{ visitor.{method}(item); }}")
elif inner in nodes and field.optional:
statements.append(
f"if let Some(item) = {get} {{ visitor.{method}(item); }}"
)
elif inner in nodes or inner in {
"Singleton",
"Operator",
"BoolOp",
"UnaryOp",
"CmpOp",
}:
statements.append(f"visitor.{method}({get});")
return "".join(statements)
def copy_field(field: Field, nodes: set[str]) -> str:
"""Build a field from a view into a different arena."""
inner = field.ty
value = f"node.{field.name}()"
if inner in {"MappingPatternItems", "ParameterItems"}:
return f"{value}.copy_to(self, rewriter)"
if field.is_list or inner in PARTS:
item = PARTS.get(inner, inner)
if item in nodes:
return f"self.map_list({value}, |b, child| b.copy_with(child, rewriter))"
if item == "ArgumentId":
return f"""self.map_list({value}, |b, item| match item {{
super::ArgOrKeyword::Arg(v) => super::ArgumentId::Positional(b.copy_with(v, rewriter)),
super::ArgOrKeyword::Keyword(v) => super::ArgumentId::Keyword(b.copy_with(v, rewriter)),
}})"""
if item == "PatternArgumentId":
return f"""self.map_list({value}, |b, item| match item {{
crate::PatternOrKeyword::Pattern(v) => crate::PatternOrKeyword::Pattern(b.copy_with(v, rewriter)),
crate::PatternOrKeyword::Keyword(v) => crate::PatternOrKeyword::Keyword(b.copy_with(v, rewriter)),
}})"""
if item == "FStringPart":
return f"""self.map_list({value}, |b, part| match part {{
super::FStringPartView::Literal(v) => super::FStringPartId::Literal(b.copy_with(v, rewriter)),
super::FStringPartView::FString(v) => super::FStringPartId::FString(b.copy_with(v, rewriter)),
}})"""
assert item == "CmpOp", item
return f"self.list({value})"
if inner in nodes:
return (
f"{value}.map(|child| self.copy_with(child, rewriter))"
if field.optional
else f"self.copy_with({value}, rewriter)"
)
if inner == "Name":
return f"self.intern_name({value}.as_str())"
if inner == "Number":
return f"{value}.clone()"
if inner == "DebugText":
return f"{value}.cloned()"
if inner == "str":
return f"{value}.into()"
return value
def replacement_hook(ty: str) -> str:
method = {
"Stmt": "replace_stmt",
"Expr": "replace_expr",
"StringLiteral": "replace_string_literal",
"InterpolatedStringElement": "replace_interpolated_string_element",
}.get(ty)
if method is None:
return ""
return f"""fn replacement<R: super::Rewriter + ?Sized>(
node: Self::Borrowed<'_>, builder: &mut Builder, rewriter: &mut R,
) -> Option<BuildId<Self>> {{ rewriter.{method}(builder, node) }}
"""
def generate_flat_patterns(ast: Ast, nodes: set[str]) -> list[str]:
"""Generate field projections for destructuring a node in a match."""
groups = ", ".join(f"{group.name}Kind" for group in ast.groups)
out = [
f"pub(crate) mod fields {{ use super::{{AstNode, SyntaxNode, List, NodeList, Ranged, NodeKind, AnyNode, {groups}}};"
]
for node in ast.all_nodes:
ty = node.name
fields = [field_codec(field, nodes) for field in node.fields]
out.append(f"#[derive(Clone, Copy)] pub struct {ty}Fields<'a> {{")
out.append(f"pub node: crate::{ty}<'a>,")
for field in fields:
out.append(f"pub {field.name}: {field.result},")
out.append("}")
out.append(
f"impl Ranged for {ty}Fields<'_> {{ #[inline] fn range(&self) -> ruff_text_size::TextRange {{ self.node.range() }} }}"
)
out.append(
f"impl<'a> crate::{ty}<'a> {{ #[inline] pub fn fields(self) -> {ty}Fields<'a> {{"
)
out.append(f"{ty}Fields {{ node: self,")
for field in fields:
out.append(f"{field.name}: self.{field.name}(),")
out.append("} } }")
for group in ast.groups:
ty = group.name
out.append(f"#[derive(Clone, Copy)] pub enum {ty}Fields<'a> {{")
for node in group.nodes:
out.append(f"{node.variant}({node.name}Fields<'a>),")
out.append("}")
out.append(
f"impl<'a> {ty}Fields<'a> {{ pub fn view(self) -> crate::{ty}<'a> {{ match self {{"
)
for node in group.nodes:
out.append(f"Self::{node.variant}(fields) => fields.node.into(),")
out.append("} } }")
out.append(
f"impl<'a> crate::{ty}<'a> {{ #[inline] pub fn fields(self) -> {ty}Fields<'a> {{ match self.classify() {{"
)
for node in group.nodes:
out.append(
f"{ty}Kind::{node.variant}(node) => {ty}Fields::{node.variant}(node.fields()),"
)
out.append("} } }")
out.append(f"impl<'a> crate::{ty}<'a> {{")
for node in group.nodes:
method = to_snake_case(
node.variant + (group.name if group.add_suffix_to_is_methods else "")
)
out.append(
f"#[inline] pub fn is_{method}(self) -> bool {{ self.node_kind() == NodeKind::{node.name} }}"
)
out.append(
f"#[inline] pub fn as_{method}(self) -> Option<crate::{node.name}<'a>> {{ self.cast() }}"
)
out.append("}")
out.append(
f"impl<'a> {ty}Kind<'a> {{ #[inline] pub fn view(self) -> crate::{ty}<'a> {{ match self {{"
)
for node in group.nodes:
out.append(f"Self::{node.variant}(node) => node.into(),")
out.append("} } }")
out.append(
f"impl ruff_text_size::Ranged for {ty}Kind<'_> {{ fn range(&self) -> ruff_text_size::TextRange {{ self.view().range() }} }}"
)
out.append(
f"impl crate::HasNodeIndex for {ty}Kind<'_> {{ fn node_index(&self) -> crate::NodeIndex {{ self.view().node_index() }} }}"
)
# All variants wrap the same two pointers. Making their conversions visible
# across crate boundaries lets LLVM eliminate the match in Kind::view.
for node in group.nodes:
out.append(f"impl<'a> From<crate::{node.name}<'a>> for crate::{ty}<'a> {{")
out.append(
f"#[inline] fn from(node: crate::{node.name}<'a>) -> Self {{ node.syntax_node().retype() }} }}"
)
out.append("#[derive(Clone, Copy)] pub enum AnyFields<'a> {")
for node in ast.nodes_in_tag_order:
out.append(f"{node.name}({node.name}Fields<'a>),")
out.append("}")
out.append(
"impl<'a> SyntaxNode<'a> { #[inline] pub fn fields(self) -> AnyFields<'a> { match self.classify() {"
)
for node in ast.all_nodes:
out.append(
f"AnyNode::{node.name}(node) => AnyFields::{node.name}(node.fields()),"
)
out.append("} } }")
out.append("}")
return out
def generate_flat_any(ast: Ast) -> list[str]:
"""Classify an untyped view without decoding its payload."""
out = ["#[derive(Clone, Copy, Debug)] pub enum AnyNode<'a> {"]
for node in ast.nodes_in_tag_order:
out.append(f"{node.name}(crate::{node.name}<'a>),")
out.append("}")
out.append(
"impl<'a> SyntaxNode<'a> { #[inline] pub fn classify(self) -> AnyNode<'a> { match self.node_kind() {"
)
for node in ast.all_nodes:
out.append(f"NodeKind::{node.name} => AnyNode::{node.name}(self.retype()),")
out.append("} } }")
out.append(
"impl SyntaxNode<'_> { pub(super) fn fmt_fields(self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self.kind() {"
)
for node in ast.all_nodes:
out.append(
f"NodeKind::{node.name} => std::fmt::Debug::fmt(&self.retype::<crate::{node.name}>(), f),"
)
out.append("} } }")
out.append("impl<'a> SyntaxNode<'a> {")
for node in ast.all_nodes:
out.append(
f"pub fn as_{to_snake_case(node.name)}(self) -> Option<crate::{node.name}<'a>> {{ self.cast() }}"
)
out.append(
f"pub fn is_{to_snake_case(node.name)}(self) -> bool {{ self.node_kind() == NodeKind::{node.name} }}"
)
out.append("}")
out.append(
"impl<'a> AnyNode<'a> { #[inline] pub fn syntax_node(self) -> SyntaxNode<'a> { match self {"
)
for node in ast.all_nodes:
out.append(f"Self::{node.name}(node) => node.syntax_node(),")
out.append("} } }")
out.append(
"pub(super) fn same_fields(left: SyntaxNode<'_>, right: SyntaxNode<'_>) -> bool { match left.classify() {"
)
for node in ast.all_nodes:
fields = node.fields
if fields:
comparisons = " && ".join(
f"left.{f.name}() == right.{f.name}()" for f in fields
)
out.append(
f"AnyNode::{node.name}(left) => {{ let right = right.retype::<crate::{node.name}>(); {comparisons} }},"
)
else:
out.append(f"AnyNode::{node.name}(_) => true,")
out.append("} }")
for ty in [n.name for n in ast.all_nodes] + [g.name for g in ast.groups]:
out.append(
f"impl<'a> From<crate::{ty}<'a>> for SyntaxNode<'a> {{ #[inline] fn from(node: crate::{ty}<'a>) -> Self {{ node.syntax_node() }} }}"
)
for group in ast.groups:
ty = group.name
out.append(
f"impl<'a> From<{ty}Kind<'a>> for SyntaxNode<'a> {{ #[inline] fn from(node: {ty}Kind<'a>) -> Self {{ node.view().syntax_node() }} }}"
)
out.append(
"impl<'a> From<SyntaxNode<'a>> for AnyNode<'a> { #[inline] fn from(node: SyntaxNode<'a>) -> Self { node.classify() } }"
)
for node in ast.all_nodes:
out.append(
f"impl<'a> From<crate::{node.name}<'a>> for AnyNode<'a> {{ #[inline] fn from(node: crate::{node.name}<'a>) -> Self {{ Self::{node.name}(node) }} }}"
)
for group in ast.groups:
out.append(
f"impl<'a> From<crate::{group.name}<'a>> for AnyNode<'a> {{ #[inline] fn from(node: crate::{group.name}<'a>) -> Self {{ node.syntax_node().classify() }} }}"
)
out.append(
f"impl<'a> From<crate::{group.name}Kind<'a>> for AnyNode<'a> {{ #[inline] fn from(kind: crate::{group.name}Kind<'a>) -> Self {{ match kind {{"
)
for node in group.nodes:
out.append(
f"crate::{group.name}Kind::{node.variant}(node) => Self::{node.name}(node),"
)
out.append("} } }")
out.append(
"impl AnyNode<'_> { #[inline] pub fn node_kind(self) -> NodeKind { match self {"
)
for node in ast.all_nodes:
out.append(f"Self::{node.name}(_) => NodeKind::{node.name},")
out.append("} } }")
return out
def generate_row_fields(schemas: dict[str, list[InlineField]]) -> str:
out = [
"// Generated by crates/ruff_python_ast/generate.py from ast.toml.",
"node_payloads! {",
]
for node, fields in schemas.items():
if not fields:
continue
arm = to_snake_case(node)
out.append(f" {node}({arm}) {{")
end = 0
for field in sorted(fields, key=lambda field: field.offset):
assert end <= field.offset and field.offset + field.width <= 15
padding = (
f"[_pad_{end}; {field.offset - end}] " if end < field.offset else ""
)
getter = f"{arm}_{field.name}"
out.append(
f" {padding}{field.name}: u{8 * field.width} at {field.offset} => {getter}, set_{getter};"
)
end = field.offset + field.width
out.append(" }")
return "\n".join([*out, "}", ""])
def generate_flat(ast: Ast) -> tuple[str, str]:
groups = {group.name for group in ast.groups}
nodes = groups | {node.name for node in ast.all_nodes}
schemas: dict[str, list[InlineField]] = {}
words_by_node: dict[str, dict[str, str]] = {}
out = [
"// Generated by crates/ruff_python_ast/generate.py from ast.toml.",
"use ruff_text_size::Ranged;",
"use super::{ABSENT, AstNode, AstNodeId, BuildId, Builder, Emit, List, NodeType, NodeList, Row, SyntaxNode, offset, split_words};",
"use super::scalar::Word;",
"use crate::NodeKind;",
]
initializers = [
"pub mod build { use super::{BuildId, AstNodeId, Builder, ABSENT, Word, offset}; use super::super::Emit;"
]
for group in ast.groups:
name = group.name
kinds = " | ".join(f"NodeKind::{n.name}" for n in group.nodes)
debug_arms = " ".join(
f'{name}Kind::{n.variant}(v) => f.debug_tuple("{n.variant}").field(&v).finish(),'
for n in group.nodes
)
out.append(f"""impl NodeType for crate::kind::{name} {{
type Borrowed<'ast> = crate::{name}<'ast>;
{replacement_hook(name)}
fn matches(kind: NodeKind) -> bool {{ matches!(kind, {kinds}) }}
}}
impl std::fmt::Debug for crate::{name}<'_> {{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {{
match self.classify() {{ {debug_arms} }}
}}
}}""")
for n in group.nodes:
for id_type in ("AstNodeId", "BuildId"):
initializers.append(
f"impl From<{id_type}<crate::kind::{n.name}>> for {id_type}<crate::kind::{name}> {{ "
f"fn from(value: {id_type}<crate::kind::{n.name}>) -> Self {{ value.cast() }} }}"
)
# Classifying a view can reuse the tag from the arena row.
out.append(
f"#[repr(u8)] #[derive(Debug, Copy, Clone)] pub enum {name}Kind<'a> {{"
)
for n in group.nodes:
out.append(f"{n.variant}(crate::{n.name}<'a>) = NodeKind::{n.name} as u8,")
out.append("}")
if name == "Stmt":
out.append(
"#[cold] #[inline(never)] "
'fn invalid_group() -> ! { unreachable!("typed node belongs to this group") }'
)
if name in ("Expr", "Stmt"):
# Checking the family before the match lets LLVM reduce a caller's
# single-variant match to tests on the row tag.
out.append(f"""impl<'a> crate::{name}<'a> {{
#[expect(clippy::inline_always, reason = "let callers test the row tag directly")]
#[inline(always)]
pub fn classify(self) -> {name}Kind<'a> {{
let tag = self.node_kind();
if !crate::kind::{name}::matches(tag) {{ invalid_group(); }}
match tag {{""")
fallback = "invalid_group()"
else:
out.append(
f"impl<'a> crate::{name}<'a> {{ #[inline] pub fn classify(self) -> {name}Kind<'a> {{ match self.node_kind() {{"
)
fallback = 'unreachable!("typed node belongs to this group")'
for n in group.nodes:
out.append(
f"NodeKind::{n.name} => {name}Kind::{n.variant}(self.syntax_node().retype()),"
)
out.append(f"_ => {fallback}, }} }} }}")
for n in ast.all_nodes:
fields = [field_codec(f, nodes) for f in n.fields]
storage_fields = fields
if n.source_order is not None:
by_name = {field.name: field for field in fields}
storage_fields = [by_name[field.name] for field in source_order_fields(n)]
offsets, spills, extra_offset = layout(storage_fields, n.name)
arm = to_snake_case(n.name)
schemas[n.name] = [
*(
InlineField(f.name, offsets[f.name], f.width)
for f in fields
if f.name in offsets
),
*CACHED_FIELDS.get(n.name, []),
]
if extra_offset is not None:
schemas[n.name].append(InlineField("extra_offset", extra_offset, 4))
words_by_node[n.name] = {}
debug_fields = ["node_index", "range"]
if n.debug_range_first:
debug_fields.reverse()
if n.name == "Identifier":
debug_fields.insert(0, "id")
else:
debug_fields.extend(field.name for field in fields)
debug_fields = "".join(
f'.field("{name}", &v.{name}())' for name in debug_fields
)
out.append(f"""impl NodeType for crate::kind::{n.name} {{
type Borrowed<'ast> = crate::{n.name}<'ast>;
const KIND: Option<NodeKind> = Some(NodeKind::{n.name});
{replacement_hook(n.name)}
#[inline]
fn matches(kind: NodeKind) -> bool {{ kind == NodeKind::{n.name} }}
}}
impl std::fmt::Debug for crate::{n.name}<'_> {{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {{
let v = *self;
f.debug_struct("{n.name}"){debug_fields}.finish()
}}
}}""")
method = f"push_{to_snake_case(n.name)}"
parameters = ", ".join(f.name for f in fields)
borrows_literal = any(
field.ty in {"LiteralText", "LiteralBytes"} for field in n.fields
)
out.append(
f"impl Builder {{ #[inline] fn {method}(&mut self, range: ruff_text_size::TextRange, [{parameters}]: [u64; {len(fields)}]) -> BuildId<crate::kind::{n.name}> {{"
)
default = ", Default" if n.name == "Parameters" else ""
lifetime = "<'a>" if borrows_literal else ""
constructor = n.name + ("<'_>" if borrows_literal else "")
initializers.append(
f"#[derive(Debug{default})] pub struct {n.name}{lifetime} {{pub range: ruff_text_size::TextRange,"
)
for field in fields:
initializers.append(f"pub {field.name}: {field.build_type},")
initializers.append(
f"}} impl Emit for {constructor} {{ type Output = crate::kind::{n.name}; #[inline] fn emit(self, builder: &mut Builder) -> BuildId<Self::Output> {{"
)
initializers.extend(f"let {field.name} = {field.encode};" for field in fields)
initializers.append(f"builder.{method}(self.range, [{parameters}]) }} }}")
mutable = "mut " if fields else ""
out.append(f"let {mutable}row = Row::new(NodeKind::{n.name});")
for f in fields:
if f.name in offsets:
value = {
1: f"{f.name}.to_le_bytes()[0]",
4: f"split_words({f.name})[0]",
8: f.name,
}[f.width]
out.append(f"row.set_{arm}_{f.name}({value});")
if n.name == "ExprStringLiteral":
out.append(
"let (start, len) = self.string_literal_text(value);"
"row.set_expr_string_literal_text_offset(start); row.set_expr_string_literal_text_len(len);"
)
if spills:
out.append(f"row.set_{arm}_extra_offset(offset(self.ast.extra.len()));")
words = []
for field in fields:
if field.name in spills:
words.append(f"split_words({field.name})[0]")
if field.width == 8:
words.append(f"split_words({field.name})[1]")
out.append("self.ast.extra.extend([" + ", ".join(words) + "]);")
out.append("self.push(row, range) } }")
if fields:
generics, lifetime = (
("<'a>", "'a") if any("'a" in f.result for f in fields) else ("", "'_")
)
out.append(f"impl{generics} crate::{n.name}<{lifetime}> {{")
if n.name == "ExprCompare":
out.append(
"""
/// Returns the operands and operator of a single comparison.
#[inline]
pub fn as_single(self) -> Option<(crate::Expr<'a>, crate::CmpOp, crate::Expr<'a>)> {
let [operator] = self.ops().as_array()?;
let [left, right] = self.operands().as_array()?;
Some((left, operator, right))
}
"""
)
for f in fields:
if f.name in offsets:
raw = f"self.syntax_node().row.{arm}_{f.name}()"
if f.width < 8:
raw = f"u64::from({raw})"
shared = f"record.inline_word::<{offsets[f.name]}, {f.width}>()"
else:
base = f"self.syntax_node().row.{arm}_extra_offset() as usize"
shared_base = f"split_words(record.inline_word::<{extra_offset}, 4>())[0] as usize"
index = "base" if not spills[f.name] else f"base + {spills[f.name]}"
read = f"self.syntax_node().ast.extra_word::<{str(f.width == 8).lower()}>({index})"
raw = f"{{ let base = {base}; {read} }}"
shared = f"{{ let base = {shared_base}; {read} }}"
if f.result == "&'a crate::name::Name":
out.append(
f"#[inline] pub fn name_id(self) -> super::NameId {{ super::NameId(split_words({raw})[0]) }}"
)
words_by_node[n.name][f.name] = shared
if n.name in {"StringLiteral", "BytesLiteral"} and f.name == "value":
accessor = (
"byte_range" if n.name == "BytesLiteral" else "text_range"
)
out.append(
f"pub(super) fn {accessor}(self) -> super::Interval {{ super::Interval::decode({raw}) }}"
)
if f.name == "iter":
out.append(
'#[expect(clippy::iter_not_returning_iterator, reason = "Python names the iterable expression in a for loop or comprehension `iter`.")]'
)
out.append(f"""#[inline] pub fn {f.name}(self) -> {f.result} {{
let word = {raw};
{f.read}
}}""")
out.append("}")
row_code = [generate_row_fields(schemas), "impl super::Ast {"]
for generate_step, signature in [
(
direct_children,
"fn direct_children<'a>(&'a self, node: SyntaxNode<'a>, kind: NodeKind) -> Children<'a>",
),
(
child_field,
"fn child_field<'a>(&'a self, node: SyntaxNode<'a>, field: u8) -> Field<'a>",
),
(
next_descendant,
"fn next_descendant<'a>(&'a self, node: SyntaxNode<'a>, stack: &mut Pending<'a>) -> u32",
),
]:
steps: dict[str, list[str]] = {}
for node in ast.all_nodes:
step = generate_step(
node, nodes, words_by_node[node.name], schemas[node.name]
)
steps.setdefault(step, []).append(f"NodeKind::{node.name}")
if generate_step == child_field:
del steps["Field::One(None)"]
if generate_step != child_field:
row_code.append("#[inline]")
kind = "kind" if generate_step == direct_children else "record.kind()"
row_code.append(
f"pub(super) {signature} {{ let record = node.row; match {kind} {{"
)
for step, kinds in steps.items():
row_code.append(f"{' | '.join(kinds)} => {{ {step} }}")
if generate_step == child_field:
row_code.append("_ => Field::One(None),")
row_code.append("} }")
row_code.append("}")
out.append(
"impl Builder { pub(super) fn copy_any<R: super::Rewriter + ?Sized>(&mut self, node: super::SyntaxNode<'_>, rewriter: &mut R) -> BuildId<crate::kind::Any> { match node.classify() {"
)
for node in ast.all_nodes:
fields = [f"{field.name}: {copy_field(field, nodes)}," for field in node.fields]
out.append(
f"AnyNode::{node.name}(node) => build::{node.name} {{ range: node.range(), "
+ "".join(fields)
+ "}.emit(self).cast(),"
)
out.append("} } }")
out.append(
"impl<'a> SyntaxNode<'a> { pub fn visit_source_order(self, visitor: &mut (impl super::visitor::source_order::SourceOrderVisitor<'a> + ?Sized)) { self.classify().visit_source_order(visitor); } }"
)
visitor = "impl super::visitor::source_order::SourceOrderVisitor<'a> + ?Sized"
# Every dispatch path calls the same implementation of a concrete node's child order.
children = {node.name: walk_fields(node, nodes) for node in ast.all_nodes}
for node in ast.all_nodes:
if walk := children[node.name]:
access = (
'#[doc = "Visits the children of the module without entering or leaving the module itself."] pub'
if node.name == "ModModule"
else "pub(super)"
)
out.append(
f"impl<'a> crate::{node.name}<'a> {{ #[inline] {access} fn visit_source_order("
f"self, visitor: &mut ({visitor})) {{ let node = self; {walk} }} }}"
)
def source_order_dispatch(
enum: str, variants: list[tuple[str, Node]], visibility: str
) -> None:
out.append(
f"impl<'a> {enum}<'a> {{ {visibility} fn visit_source_order("
f"self, visitor: &mut ({visitor})) {{ match self {{"
)
leaves = []
for variant, node in variants:
if children[node.name]:
out.append(
f"Self::{variant}(node) => node.visit_source_order(visitor),"
)
else:
leaves.append(f"Self::{variant}(_)")
if leaves:
out.append(f"{' | '.join(leaves)} => {{}},")
out.append("} } }")
source_order_dispatch("AnyNode", [(n.name, n) for n in ast.all_nodes], "pub")
for group in ast.groups:
if group.name != "ExceptHandler":
source_order_dispatch(
f"{group.name}Kind", [(n.variant, n) for n in group.nodes], "pub(super)"
)
# Each string-like expression stores a list of one-word part handles.
part_lists = {
words_by_node[node.name][field.name]
for node in ast.all_nodes
for field in node.fields
if field.ty in PARTS
}
assert len(part_lists) == 1, (
"string-like expressions must share the parts list layout"
)
part_list = part_lists.pop()
row_code.append(f"""impl<'a> super::StringLike<'a> {{
#[inline] pub(super) fn part_nodes(self) -> NodeList<'a, crate::kind::Any> {{
let node: SyntaxNode<'a> = self.into();
let record = node.row;
List::new(node.ast, {part_list})
}}
}}""")
initializers.append("}")
out.extend(initializers)
out.extend(generate_flat_patterns(ast, nodes))
out.extend(generate_flat_any(ast))
return "\n".join(out), "\n".join(row_code)
# ------------------------------------------------------------------------------
# Format and write output
def generate(ast: Ast) -> list[str]:
out = [
"// Generated by crates/ruff_python_ast/generate.py from ast.toml.",
"//! Borrowed Python syntax nodes and their lifetime-free ID markers.",
"#[repr(u8)] #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] pub enum NodeKind {",
*(f"{node.name}," for node in ast.nodes_in_tag_order),
"}",
]
types = [*ast.groups, *ast.all_nodes]
out.append(
"/// Lifetime-free marker types used by AST IDs and the parser.\npub mod kind {"
)
for name in ["Any", *(node.name for node in types)]:
out.append(
f"#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] pub enum {name} {{}}"
)
out.append("}")
for node in types:
if node.doc is not None:
write_rustdoc(out, node.doc)
out.append(
f"#[repr(transparent)] #[derive(Copy, Clone, PartialEq)] pub struct {node.name}<'ast> {{ pub(crate) syntax: crate::SyntaxNode<'ast> }}"
)
out.append(f"crate::flat::impl_ast_node!({node.name});")
return out
def write_output(root: Path, out: list[str]) -> None:
out_path = root.joinpath("crates", "ruff_python_ast", "src", "generated.rs")
out_path.write_text(rustfmt("\n".join(out)))
# ------------------------------------------------------------------------------
# Main
def main() -> None:
root = Path(
check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip()
)
ast = load_ast(root)
out = generate(ast)
write_output(root, out)
flat = root / "crates" / "ruff_python_ast" / "src" / "flat" / "generated.rs"
views, rows = generate_flat(ast)
flat.write_text(rustfmt(views))
flat.with_name("row").joinpath("generated.rs").write_text(rustfmt(rows))
if __name__ == "__main__":
main()