#!/usr/bin/python # /// script # requires-python = ">=3.11" # dependencies = [] # # [tool.uv] # exclude-newer = "P7D" # require-build-hashes = true # preview-features = ["build-dependency-hashes", "missing-exclude-newer-package-lock"] # /// from __future__ import annotations import re from dataclasses import dataclass from pathlib import Path from subprocess import check_output from typing import Any import tomllib # Types that require `crate::`. We can slowly remove these types as we move them to generate scripts. types_requiring_crate_prefix = { "IpyEscapeKind", "ExprContext", "Identifier", "Number", "BytesValue", "StringValue", "FStringValue", "TStringValue", "Arguments", "CmpOp", "Comprehension", "MappingPatternItems", "ParameterItems", "UnaryOp", "BoolOp", "Operator", "Decorator", "TypeParams", "Parameters", "ElifElseClause", "WithItem", "MatchCase", "Alias", "Singleton", "PatternArguments", } @dataclass class VisitorInfo: name: str accepts_sequence: bool = False # Map of AST node types to their corresponding visitor information. # Only visitors that are different from the default `visit_*` method are included. # These visitors either have a different name or accept a sequence of items. type_to_visitor_function: dict[str, VisitorInfo] = { "TypeParams": VisitorInfo("visit_type_params", True), "Parameters": VisitorInfo("visit_parameters", True), "Stmt": VisitorInfo("visit_body", True), "Arguments": VisitorInfo("visit_arguments", True), } def rustfmt(code: str) -> str: return check_output(["rustfmt", "--emit=stdout"], input=code, text=True) def to_snake_case(node: str) -> str: """Converts CamelCase to snake_case""" return re.sub(r"([A-Z])", r"_\1", node).lower().lstrip("_") def write_rustdoc(out: list[str], doc: str) -> None: for line in doc.split("\n"): out.append(f"/// {line}") # ------------------------------------------------------------------------------ # Read AST description def load_ast(root: Path) -> Ast: ast_path = root.joinpath("crates", "ruff_python_ast", "ast.toml") with ast_path.open("rb") as ast_file: ast = tomllib.load(ast_file) return Ast(ast) # ------------------------------------------------------------------------------ # Preprocess @dataclass class Ast: """ The parsed representation of the `ast.toml` file. Defines all of the Python AST syntax nodes, and which groups (`Stmt`, `Expr`, etc.) they belong to. """ groups: list[Group] all_nodes: list[Node] nodes_in_tag_order: list[Node] def __init__(self, ast: dict[str, Any]) -> None: groups = [Group(name, group) for name, group in ast.items()] self.groups = [group for group in groups if group.name != "ungrouped"] self.all_nodes = [node for group in groups for node in group.nodes] # With expressions first and statements last, both families can be # recognized by comparing the row tag against a single boundary. rank = {"Expr": -1, "Stmt": 1} tag_groups = sorted(groups, key=lambda group: rank.get(group.name, 0)) self.nodes_in_tag_order = [node for group in tag_groups for node in group.nodes] @dataclass class Group: name: str nodes: list[Node] add_suffix_to_is_methods: bool doc: str | None def __init__(self, group_name: str, group: dict[str, Any]) -> None: self.name = group_name self.add_suffix_to_is_methods = group.get("add_suffix_to_is_methods", False) self.doc = group.get("doc") self.nodes = [ Node(self, node_name, node) for node_name, node in group["nodes"].items() ] @dataclass class Node: name: str variant: str doc: str | None fields: list[Field] source_order: list[str] | None debug_range_first: bool def __init__(self, group: Group, node_name: str, node: dict[str, Any]) -> None: self.name = node_name self.variant = node.get("variant", node_name.removeprefix(group.name)) self.doc = node.get("doc") self.fields = [Field(field) for field in node["fields"]] self.source_order = node.get("source_order") # Auxiliary nodes print their source range first in parser snapshots. self.debug_range_first = group.name in { "ungrouped", "ExceptHandler", "InterpolatedStringElement", } @dataclass class Field: name: str ty: str optional: bool is_list: bool is_annotation: bool def __init__(self, field: dict[str, Any]) -> None: self.name = field["name"] spec = field["type"] self.optional = spec.endswith("?") self.is_list = spec.endswith("*") self.ty = spec[:-1] if self.optional or self.is_list else spec if not self.ty.isidentifier(): raise ValueError(f"Invalid type for field {self.name}: {spec}") self.is_annotation = field.get("is_annotation", False) # ------------------------------------------------------------------------------ # Flat syntax storage PARTS = { "StringValue": "StringLiteral", "BytesValue": "BytesLiteral", "FStringValue": "FStringPart", "TStringValue": "TString", } FLAGS = {"StringLiteralFlags", "BytesLiteralFlags", "FStringFlags", "TStringFlags"} TAGS = { "ExprContext", "Operator", "CmpOp", "UnaryOp", "BoolOp", "Singleton", "IpyEscapeKind", "ConversionFlag", } @dataclass(frozen=True) class WireField: name: str width: int result: str build_type: str encode: str read: str def field_codec(field: Field, nodes: set[str]) -> WireField: name = field.name inner = field.ty value = f"self.{name}" def codec( width: int, result: str, encode: str, read: str, *, build_type: str | None = None, ) -> WireField: return WireField(name, width, result, build_type or result, encode, read) if inner in {"MappingPatternItems", "ParameterItems"}: return codec( 8, f"crate::flat::{inner}<'a>", f"{value}.word()", f"super::{inner}(List::new(self.syntax_node().ast, word))", build_type=f"super::super::{inner}Id", ) if field.is_list or inner in PARTS: item = PARTS.get(inner, inner) element = ( f"AstNodeId" if item in nodes else f"crate::{item}" ) read = "List::new(self.syntax_node().ast, word)" if inner == "StringValue": result = "crate::flat::StringValue<'a>" read = ( "crate::flat::StringValue::new(List::new(self.syntax_node().ast, word), " "u64::from(self.syntax_node().row.expr_string_literal_text_offset()) | " "(u64::from(self.syntax_node().row.expr_string_literal_text_len()) << 32))" ) elif inner in PARTS: wrapper = { "FStringValue": "FStringValue", "TStringValue": "TStringValue", "BytesValue": "BytesValue", }[inner] result = f"crate::flat::{wrapper}<'a>" read = f"crate::flat::{wrapper}::new({read})" else: result = ( f"NodeList<'a, crate::kind::{item}>" if item in nodes else f"List<'a, {element}>" ) return codec( 8, result, f"{value}.word()", read, build_type=f"super::super::ListId<{element}>", ) if inner in nodes: view = f"crate::{inner}<'a>" handle = f"BuildId" if field.optional: return codec( 4, f"Option<{view}>", f"{value}.map_or(0, |id| u64::from(id.slot))", "self.syntax_node().ast.optional_child(word)", build_type=f"Option<{handle}>", ) return codec( 4, view, f"u64::from({value}.slot)", "self.syntax_node().ast.child(word)", build_type=handle, ) if inner == "Name": return codec( 4, "&'a crate::name::Name", f"u64::from({value}.0)", "&self.syntax_node().ast.names[split_words(word)[0] as usize]", build_type="super::super::NameId", ) if inner == "str": return codec( 8, "&'a str", f"builder.store_string(&{value})", "self.syntax_node().ast.string(word)", build_type="Box", ) if inner == "LiteralText": return codec( 8, "&'a str", f"builder.store_string({value})", "self.syntax_node().ast.string(word)", ) if inner == "LiteralBytes": return codec( 8, "&'a [u8]", f"builder.store_bytes({value})", "self.syntax_node().ast.bytes(word)", ) if inner in TAGS: return codec(1, f"crate::{inner}", f"{value}.encode()", "Word::decode(word)") if inner in FLAGS: return codec( 1, f"crate::{inner}", f"u64::from({value}.bits())", f"crate::{inner}::from_bits(word.to_le_bytes()[0])", ) if inner == "bool": return codec(1, "bool", f"u64::from({value})", "word != 0") if inner == "u32": return codec(4, "u32", f"u64::from({value})", "split_words(word)[0]") if inner == "Number": return codec( 4, "&'a crate::Number", f"{{ let slot = offset(builder.ast.numbers.len()); builder.ast.numbers.push({value}); u64::from(slot) }}", "&self.syntax_node().ast.numbers[split_words(word)[0] as usize]", build_type="crate::Number", ) if inner == "DebugText": assert field.optional return codec( 4, "Option<&'a crate::DebugText>", f"{value}.map_or(u64::from(ABSENT), |text| {{ let slot = offset(builder.ast.debug_text.len()); builder.ast.debug_text.push(text); u64::from(slot) }})", "(split_words(word)[0] != ABSENT).then(|| &self.syntax_node().ast.debug_text[split_words(word)[0] as usize])", build_type="Option", ) raise ValueError(f"Missing flat field codec for {field.name}: {field.ty}") @dataclass(frozen=True) class InlineField: name: str offset: int width: int # These derived values are read without following the node's children. CACHED_FIELDS = { "ExprStringLiteral": [ InlineField("text_offset", 8, 4), InlineField("text_len", 12, 2), ], } def layout( fields: list[WireField], node: str ) -> tuple[dict[str, int], dict[str, int], int | None]: """Place naturally aligned fields in fifteen bytes, spilling fields into extra if needed.""" reserved = { i for field in CACHED_FIELDS.get(node, []) for i in range(field.offset, field.offset + field.width) } def place(inline: list[WireField], extra: int | None) -> dict[str, int] | None: used = reserved.copy() if extra is not None: span = range(extra, extra + 4) if not used.isdisjoint(span): return None used.update(span) positions = {} for field in sorted(inline, key=lambda field: -field.width): for start in range(0, 16 - field.width, field.width): span = range(start, start + field.width) if used.isdisjoint(span): positions[field.name] = start used.update(span) break else: return None return positions def with_extra(inline: list[WireField]) -> tuple[dict[str, int], int] | None: # Moving the spill offset can leave an aligned eight-byte slot for a list. for extra in (0, 4, 8): if (positions := place(inline, extra)) is not None: return positions, extra return None if (complete := place(fields, None)) is not None: return complete, {}, None kept: list[WireField] = [] for field in sorted(fields, key=lambda field: field.width): if with_extra([*kept, field]) is not None: kept.append(field) placed = with_extra(kept) assert placed is not None offsets, extra = placed spills = {} words = 0 for f in fields: if f not in kept: spills[f.name] = words words += (f.width + 3) // 4 return offsets, spills, extra def source_order_fields(node: Node) -> list[Field]: fields = node.fields if node.source_order is None: return fields positions = {name: index for index, name in enumerate(node.source_order)} return sorted(fields, key=lambda field: positions.get(field.name, len(fields))) def child_fields(node: Node, nodes: set[str]) -> list[Field]: return [ field for field in source_order_fields(node) if ( field.ty in nodes or field.ty in PARTS or field.ty in { "ArgumentId", "MappingPatternItems", "ParameterItems", "PatternArgumentId", } ) ] def child_sources( node: Node, fields: list[Field], words: dict[str, str], view: str ) -> list[tuple[str, str]]: """The source-ordered fields, each containing a handle or a list.""" if not fields: return [] result = [] for field in fields: raw = words[field.name].replace( "self.syntax_node().ast.extra_word", "self.extra_word" ) if (node.name, field.name) == ("InterpolatedElement", "format_spec"): # The source-order visitor exposes the elements of the auxiliary spec. result.append( ( "list", f"{view}.format_spec().map_or(&[][..], |spec| spec.elements().words())", ) ) elif ( field.is_list or field.ty in PARTS or field.ty in { "ArgumentId", "MappingPatternItems", "ParameterItems", "PatternArgumentId", } ): result.append( ( "list", f"List::>::new(self, {raw}).words()", ) ) elif field.optional: result.append(("optional", f"split_words({raw})[0]")) else: result.append(("node", f"split_words({raw})[0]")) return result def field_value(kind: str, value: str) -> str: if kind in {"node", "optional"}: return f"Field::One({child_handle(kind, value)})" assert kind == "list" return f"Field::Rows({value})" def child_handle(kind: str, value: str) -> str: if kind == "node": return f"Some(AstNodeId::from_word({value}))" assert kind == "optional" return f"{{ let word = {value}; (word != 0).then(|| AstNodeId::from_word(word)) }}" def simple_children( node: Node, nodes: set[str], sources: list[tuple[str, str]], schema: list[InlineField], ) -> str | None: """Most nodes can borrow one slice from the row or the extra buffer.""" kinds = tuple("node" if kind == "optional" else kind for kind, _ in sources) if not sources: return "Children::empty(self)" if ( all(kind == "node" for kind in kinds) and (words := inline_child_slice(child_fields(node, nodes), schema)) is not None ): return f"Children::inline(self, {words})" if kinds == ("list",): return f"Children::rows(self, {sources[0][1]})" if kinds in (("node", "list"), ("list", "node"), ("node", "list", "node")): first = child_handle(*sources[0]) if kinds[0] == "node" else "None" last = child_handle(*sources[-1]) if kinds[-1] == "node" else "None" return f"Children::sequence(self, {first}, {sources[kinds.index('list')][1]}, {last})" return None def child_field( node: Node, nodes: set[str], words: dict[str, str], schema: list[InlineField] ) -> str: sources = child_sources( node, child_fields(node, nodes), words, f"node.retype::()" ) if simple_children(node, nodes, sources, schema) is not None: return "Field::One(None)" arms = [f"{i} => {field_value(*source)}," for i, source in enumerate(sources)] return "match field {" + "".join(arms) + "_ => Field::One(None) }" def direct_children( node: Node, nodes: set[str], words: dict[str, str], schema: list[InlineField] ) -> str: view = f"node.retype::()" sources = child_sources(node, child_fields(node, nodes), words, view) return ( simple_children(node, nodes, sources, schema) or f"Children::fields(node, {len(sources)})" ) def inline_child_slice(children: list[Field], schema: list[InlineField]) -> str | None: """Borrow consecutive inline handles when missing children can be trimmed at the ends.""" offsets = {field.name: field.offset for field in schema if field.width == 4} if any(field.name not in offsets for field in children): return None start = offsets[children[0].name] if any(offsets[field.name] != start + 4 * i for i, field in enumerate(children)): return None if any(field.optional for field in children[1:-1]): return None first = str(children[0].optional).lower() last = str(len(children) > 1 and children[-1].optional).lower() return f"record.inline_children::<{start // 4}, {len(children)}, {first}, {last}>()" def next_descendant( node: Node, nodes: set[str], words: dict[str, str], _schema: list[InlineField] ) -> str: sources = child_sources( node, child_fields(node, nodes), words, f"node.retype::()" ) resume = "stack.resume()" steps = [] for i in reversed(range(len(sources))): kind, value = sources[i] if kind == "node": if i == 0: return "".join(steps) + value steps.append(f"stack.push_node({value});") elif kind == "list" and i == 0: return "".join(steps) + f"stack.descend_list({value})" elif kind == "optional": if i == 0: return ( "".join(steps) + f"let word = {value}; if word == 0 {{ {resume} }} else {{ word }}" ) steps.append( f"let word = {value}; if word != 0 {{ stack.push_node(word); }}" ) else: assert kind == "list" steps.append(f"stack.push_list({value});") return "".join(steps) + resume def walk_fields(node: Node, nodes: set[str]) -> str: """Visit the fields of a projected node in Python source order.""" if node.name == "Arguments": return """ for argument in node.items() { match argument { super::ArgOrKeyword::Arg(expr) => visitor.visit_expr(expr), super::ArgOrKeyword::Keyword(keyword) => visitor.visit_keyword(keyword), } } """ if node.name == "PatternArguments": return """ for argument in node.items() { match argument { crate::PatternOrKeyword::Pattern(pattern) => visitor.visit_pattern(pattern), crate::PatternOrKeyword::Keyword(keyword) => visitor.visit_pattern_keyword(keyword), } } """ if node.name == "PatternMatchMapping": return """ for item in node.items() { match item { crate::MappingPatternItem::Entry { key, pattern } => { visitor.visit_expr(key); visitor.visit_pattern(pattern); } crate::MappingPatternItem::Rest(rest) => visitor.visit_identifier(rest), } } """ if node.name == "Parameters": return """ for parameter in node.iter_source_order() { match parameter { super::AnyParameter::NonVariadic(node) => { visitor.visit_parameter_with_default(node); } super::AnyParameter::Variadic(node) => visitor.visit_parameter(node), } } """ if node.name == "ExprBoolOp": return """let mut values = node.values().iter(); if let Some(left) = values.next() { visitor.visit_expr(left); } visitor.visit_bool_op(node.op()); for value in values { visitor.visit_expr(value); }""" if node.name == "ExprCompare": return """visitor.visit_expr(node.first_operand()); for (op, right) in node.operators_and_comparators() { visitor.visit_cmp_op(op); visitor.visit_expr(right); }""" statements = [] for field in source_order_fields(node): inner = field.ty get = f"node.{field.name}()" method = f"visit_{to_snake_case(PARTS.get(inner, inner))}" if field.is_annotation: method = "visit_annotation" if (node.name, field.name) == ("InterpolatedElement", "format_spec"): statements.append( f"if let Some(spec) = {get} {{ spec.visit_source_order(visitor); }}" ) elif inner == "FStringValue": statements.append( f"for part in {get} {{ match part {{ super::FStringPartView::Literal(v) => visitor.visit_string_literal(v), super::FStringPartView::FString(v) => visitor.visit_f_string(v) }} }}" ) elif inner == "Stmt" and field.is_list: statements.append(f"visitor.visit_body({get});") elif inner in PARTS or (inner in nodes and field.is_list): statements.append(f"for item in {get} {{ visitor.{method}(item); }}") elif inner in nodes and field.optional: statements.append( f"if let Some(item) = {get} {{ visitor.{method}(item); }}" ) elif inner in nodes or inner in { "Singleton", "Operator", "BoolOp", "UnaryOp", "CmpOp", }: statements.append(f"visitor.{method}({get});") return "".join(statements) def copy_field(field: Field, nodes: set[str]) -> str: """Build a field from a view into a different arena.""" inner = field.ty value = f"node.{field.name}()" if inner in {"MappingPatternItems", "ParameterItems"}: return f"{value}.copy_to(self, rewriter)" if field.is_list or inner in PARTS: item = PARTS.get(inner, inner) if item in nodes: return f"self.map_list({value}, |b, child| b.copy_with(child, rewriter))" if item == "ArgumentId": return f"""self.map_list({value}, |b, item| match item {{ super::ArgOrKeyword::Arg(v) => super::ArgumentId::Positional(b.copy_with(v, rewriter)), super::ArgOrKeyword::Keyword(v) => super::ArgumentId::Keyword(b.copy_with(v, rewriter)), }})""" if item == "PatternArgumentId": return f"""self.map_list({value}, |b, item| match item {{ crate::PatternOrKeyword::Pattern(v) => crate::PatternOrKeyword::Pattern(b.copy_with(v, rewriter)), crate::PatternOrKeyword::Keyword(v) => crate::PatternOrKeyword::Keyword(b.copy_with(v, rewriter)), }})""" if item == "FStringPart": return f"""self.map_list({value}, |b, part| match part {{ super::FStringPartView::Literal(v) => super::FStringPartId::Literal(b.copy_with(v, rewriter)), super::FStringPartView::FString(v) => super::FStringPartId::FString(b.copy_with(v, rewriter)), }})""" assert item == "CmpOp", item return f"self.list({value})" if inner in nodes: return ( f"{value}.map(|child| self.copy_with(child, rewriter))" if field.optional else f"self.copy_with({value}, rewriter)" ) if inner == "Name": return f"self.intern_name({value}.as_str())" if inner == "Number": return f"{value}.clone()" if inner == "DebugText": return f"{value}.cloned()" if inner == "str": return f"{value}.into()" return value def replacement_hook(ty: str) -> str: method = { "Stmt": "replace_stmt", "Expr": "replace_expr", "StringLiteral": "replace_string_literal", "InterpolatedStringElement": "replace_interpolated_string_element", }.get(ty) if method is None: return "" return f"""fn replacement( node: Self::Borrowed<'_>, builder: &mut Builder, rewriter: &mut R, ) -> Option> {{ rewriter.{method}(builder, node) }} """ def generate_flat_patterns(ast: Ast, nodes: set[str]) -> list[str]: """Generate field projections for destructuring a node in a match.""" groups = ", ".join(f"{group.name}Kind" for group in ast.groups) out = [ f"pub(crate) mod fields {{ use super::{{AstNode, SyntaxNode, List, NodeList, Ranged, NodeKind, AnyNode, {groups}}};" ] for node in ast.all_nodes: ty = node.name fields = [field_codec(field, nodes) for field in node.fields] out.append(f"#[derive(Clone, Copy)] pub struct {ty}Fields<'a> {{") out.append(f"pub node: crate::{ty}<'a>,") for field in fields: out.append(f"pub {field.name}: {field.result},") out.append("}") out.append( f"impl Ranged for {ty}Fields<'_> {{ #[inline] fn range(&self) -> ruff_text_size::TextRange {{ self.node.range() }} }}" ) out.append( f"impl<'a> crate::{ty}<'a> {{ #[inline] pub fn fields(self) -> {ty}Fields<'a> {{" ) out.append(f"{ty}Fields {{ node: self,") for field in fields: out.append(f"{field.name}: self.{field.name}(),") out.append("} } }") for group in ast.groups: ty = group.name out.append(f"#[derive(Clone, Copy)] pub enum {ty}Fields<'a> {{") for node in group.nodes: out.append(f"{node.variant}({node.name}Fields<'a>),") out.append("}") out.append( f"impl<'a> {ty}Fields<'a> {{ pub fn view(self) -> crate::{ty}<'a> {{ match self {{" ) for node in group.nodes: out.append(f"Self::{node.variant}(fields) => fields.node.into(),") out.append("} } }") out.append( f"impl<'a> crate::{ty}<'a> {{ #[inline] pub fn fields(self) -> {ty}Fields<'a> {{ match self.classify() {{" ) for node in group.nodes: out.append( f"{ty}Kind::{node.variant}(node) => {ty}Fields::{node.variant}(node.fields())," ) out.append("} } }") out.append(f"impl<'a> crate::{ty}<'a> {{") for node in group.nodes: method = to_snake_case( node.variant + (group.name if group.add_suffix_to_is_methods else "") ) out.append( f"#[inline] pub fn is_{method}(self) -> bool {{ self.node_kind() == NodeKind::{node.name} }}" ) out.append( f"#[inline] pub fn as_{method}(self) -> Option> {{ self.cast() }}" ) out.append("}") out.append( f"impl<'a> {ty}Kind<'a> {{ #[inline] pub fn view(self) -> crate::{ty}<'a> {{ match self {{" ) for node in group.nodes: out.append(f"Self::{node.variant}(node) => node.into(),") out.append("} } }") out.append( f"impl ruff_text_size::Ranged for {ty}Kind<'_> {{ fn range(&self) -> ruff_text_size::TextRange {{ self.view().range() }} }}" ) out.append( f"impl crate::HasNodeIndex for {ty}Kind<'_> {{ fn node_index(&self) -> crate::NodeIndex {{ self.view().node_index() }} }}" ) # All variants wrap the same two pointers. Making their conversions visible # across crate boundaries lets LLVM eliminate the match in Kind::view. for node in group.nodes: out.append(f"impl<'a> From> for crate::{ty}<'a> {{") out.append( f"#[inline] fn from(node: crate::{node.name}<'a>) -> Self {{ node.syntax_node().retype() }} }}" ) out.append("#[derive(Clone, Copy)] pub enum AnyFields<'a> {") for node in ast.nodes_in_tag_order: out.append(f"{node.name}({node.name}Fields<'a>),") out.append("}") out.append( "impl<'a> SyntaxNode<'a> { #[inline] pub fn fields(self) -> AnyFields<'a> { match self.classify() {" ) for node in ast.all_nodes: out.append( f"AnyNode::{node.name}(node) => AnyFields::{node.name}(node.fields())," ) out.append("} } }") out.append("}") return out def generate_flat_any(ast: Ast) -> list[str]: """Classify an untyped view without decoding its payload.""" out = ["#[derive(Clone, Copy, Debug)] pub enum AnyNode<'a> {"] for node in ast.nodes_in_tag_order: out.append(f"{node.name}(crate::{node.name}<'a>),") out.append("}") out.append( "impl<'a> SyntaxNode<'a> { #[inline] pub fn classify(self) -> AnyNode<'a> { match self.node_kind() {" ) for node in ast.all_nodes: out.append(f"NodeKind::{node.name} => AnyNode::{node.name}(self.retype()),") out.append("} } }") out.append( "impl SyntaxNode<'_> { pub(super) fn fmt_fields(self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self.kind() {" ) for node in ast.all_nodes: out.append( f"NodeKind::{node.name} => std::fmt::Debug::fmt(&self.retype::(), f)," ) out.append("} } }") out.append("impl<'a> SyntaxNode<'a> {") for node in ast.all_nodes: out.append( f"pub fn as_{to_snake_case(node.name)}(self) -> Option> {{ self.cast() }}" ) out.append( f"pub fn is_{to_snake_case(node.name)}(self) -> bool {{ self.node_kind() == NodeKind::{node.name} }}" ) out.append("}") out.append( "impl<'a> AnyNode<'a> { #[inline] pub fn syntax_node(self) -> SyntaxNode<'a> { match self {" ) for node in ast.all_nodes: out.append(f"Self::{node.name}(node) => node.syntax_node(),") out.append("} } }") out.append( "pub(super) fn same_fields(left: SyntaxNode<'_>, right: SyntaxNode<'_>) -> bool { match left.classify() {" ) for node in ast.all_nodes: fields = node.fields if fields: comparisons = " && ".join( f"left.{f.name}() == right.{f.name}()" for f in fields ) out.append( f"AnyNode::{node.name}(left) => {{ let right = right.retype::(); {comparisons} }}," ) else: out.append(f"AnyNode::{node.name}(_) => true,") out.append("} }") for ty in [n.name for n in ast.all_nodes] + [g.name for g in ast.groups]: out.append( f"impl<'a> From> for SyntaxNode<'a> {{ #[inline] fn from(node: crate::{ty}<'a>) -> Self {{ node.syntax_node() }} }}" ) for group in ast.groups: ty = group.name out.append( f"impl<'a> From<{ty}Kind<'a>> for SyntaxNode<'a> {{ #[inline] fn from(node: {ty}Kind<'a>) -> Self {{ node.view().syntax_node() }} }}" ) out.append( "impl<'a> From> for AnyNode<'a> { #[inline] fn from(node: SyntaxNode<'a>) -> Self { node.classify() } }" ) for node in ast.all_nodes: out.append( f"impl<'a> From> for AnyNode<'a> {{ #[inline] fn from(node: crate::{node.name}<'a>) -> Self {{ Self::{node.name}(node) }} }}" ) for group in ast.groups: out.append( f"impl<'a> From> for AnyNode<'a> {{ #[inline] fn from(node: crate::{group.name}<'a>) -> Self {{ node.syntax_node().classify() }} }}" ) out.append( f"impl<'a> From> for AnyNode<'a> {{ #[inline] fn from(kind: crate::{group.name}Kind<'a>) -> Self {{ match kind {{" ) for node in group.nodes: out.append( f"crate::{group.name}Kind::{node.variant}(node) => Self::{node.name}(node)," ) out.append("} } }") out.append( "impl AnyNode<'_> { #[inline] pub fn node_kind(self) -> NodeKind { match self {" ) for node in ast.all_nodes: out.append(f"Self::{node.name}(_) => NodeKind::{node.name},") out.append("} } }") return out def generate_row_fields(schemas: dict[str, list[InlineField]]) -> str: out = [ "// Generated by crates/ruff_python_ast/generate.py from ast.toml.", "node_payloads! {", ] for node, fields in schemas.items(): if not fields: continue arm = to_snake_case(node) out.append(f" {node}({arm}) {{") end = 0 for field in sorted(fields, key=lambda field: field.offset): assert end <= field.offset and field.offset + field.width <= 15 padding = ( f"[_pad_{end}; {field.offset - end}] " if end < field.offset else "" ) getter = f"{arm}_{field.name}" out.append( f" {padding}{field.name}: u{8 * field.width} at {field.offset} => {getter}, set_{getter};" ) end = field.offset + field.width out.append(" }") return "\n".join([*out, "}", ""]) def generate_flat(ast: Ast) -> tuple[str, str]: groups = {group.name for group in ast.groups} nodes = groups | {node.name for node in ast.all_nodes} schemas: dict[str, list[InlineField]] = {} words_by_node: dict[str, dict[str, str]] = {} out = [ "// Generated by crates/ruff_python_ast/generate.py from ast.toml.", "use ruff_text_size::Ranged;", "use super::{ABSENT, AstNode, AstNodeId, BuildId, Builder, Emit, List, NodeType, NodeList, Row, SyntaxNode, offset, split_words};", "use super::scalar::Word;", "use crate::NodeKind;", ] initializers = [ "pub mod build { use super::{BuildId, AstNodeId, Builder, ABSENT, Word, offset}; use super::super::Emit;" ] for group in ast.groups: name = group.name kinds = " | ".join(f"NodeKind::{n.name}" for n in group.nodes) debug_arms = " ".join( f'{name}Kind::{n.variant}(v) => f.debug_tuple("{n.variant}").field(&v).finish(),' for n in group.nodes ) out.append(f"""impl NodeType for crate::kind::{name} {{ type Borrowed<'ast> = crate::{name}<'ast>; {replacement_hook(name)} fn matches(kind: NodeKind) -> bool {{ matches!(kind, {kinds}) }} }} impl std::fmt::Debug for crate::{name}<'_> {{ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {{ match self.classify() {{ {debug_arms} }} }} }}""") for n in group.nodes: for id_type in ("AstNodeId", "BuildId"): initializers.append( f"impl From<{id_type}> for {id_type} {{ " f"fn from(value: {id_type}) -> Self {{ value.cast() }} }}" ) # Classifying a view can reuse the tag from the arena row. out.append( f"#[repr(u8)] #[derive(Debug, Copy, Clone)] pub enum {name}Kind<'a> {{" ) for n in group.nodes: out.append(f"{n.variant}(crate::{n.name}<'a>) = NodeKind::{n.name} as u8,") out.append("}") if name == "Stmt": out.append( "#[cold] #[inline(never)] " 'fn invalid_group() -> ! { unreachable!("typed node belongs to this group") }' ) if name in ("Expr", "Stmt"): # Checking the family before the match lets LLVM reduce a caller's # single-variant match to tests on the row tag. out.append(f"""impl<'a> crate::{name}<'a> {{ #[expect(clippy::inline_always, reason = "let callers test the row tag directly")] #[inline(always)] pub fn classify(self) -> {name}Kind<'a> {{ let tag = self.node_kind(); if !crate::kind::{name}::matches(tag) {{ invalid_group(); }} match tag {{""") fallback = "invalid_group()" else: out.append( f"impl<'a> crate::{name}<'a> {{ #[inline] pub fn classify(self) -> {name}Kind<'a> {{ match self.node_kind() {{" ) fallback = 'unreachable!("typed node belongs to this group")' for n in group.nodes: out.append( f"NodeKind::{n.name} => {name}Kind::{n.variant}(self.syntax_node().retype())," ) out.append(f"_ => {fallback}, }} }} }}") for n in ast.all_nodes: fields = [field_codec(f, nodes) for f in n.fields] storage_fields = fields if n.source_order is not None: by_name = {field.name: field for field in fields} storage_fields = [by_name[field.name] for field in source_order_fields(n)] offsets, spills, extra_offset = layout(storage_fields, n.name) arm = to_snake_case(n.name) schemas[n.name] = [ *( InlineField(f.name, offsets[f.name], f.width) for f in fields if f.name in offsets ), *CACHED_FIELDS.get(n.name, []), ] if extra_offset is not None: schemas[n.name].append(InlineField("extra_offset", extra_offset, 4)) words_by_node[n.name] = {} debug_fields = ["node_index", "range"] if n.debug_range_first: debug_fields.reverse() if n.name == "Identifier": debug_fields.insert(0, "id") else: debug_fields.extend(field.name for field in fields) debug_fields = "".join( f'.field("{name}", &v.{name}())' for name in debug_fields ) out.append(f"""impl NodeType for crate::kind::{n.name} {{ type Borrowed<'ast> = crate::{n.name}<'ast>; const KIND: Option = Some(NodeKind::{n.name}); {replacement_hook(n.name)} #[inline] fn matches(kind: NodeKind) -> bool {{ kind == NodeKind::{n.name} }} }} impl std::fmt::Debug for crate::{n.name}<'_> {{ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {{ let v = *self; f.debug_struct("{n.name}"){debug_fields}.finish() }} }}""") method = f"push_{to_snake_case(n.name)}" parameters = ", ".join(f.name for f in fields) borrows_literal = any( field.ty in {"LiteralText", "LiteralBytes"} for field in n.fields ) out.append( f"impl Builder {{ #[inline] fn {method}(&mut self, range: ruff_text_size::TextRange, [{parameters}]: [u64; {len(fields)}]) -> BuildId {{" ) default = ", Default" if n.name == "Parameters" else "" lifetime = "<'a>" if borrows_literal else "" constructor = n.name + ("<'_>" if borrows_literal else "") initializers.append( f"#[derive(Debug{default})] pub struct {n.name}{lifetime} {{pub range: ruff_text_size::TextRange," ) for field in fields: initializers.append(f"pub {field.name}: {field.build_type},") initializers.append( f"}} impl Emit for {constructor} {{ type Output = crate::kind::{n.name}; #[inline] fn emit(self, builder: &mut Builder) -> BuildId {{" ) initializers.extend(f"let {field.name} = {field.encode};" for field in fields) initializers.append(f"builder.{method}(self.range, [{parameters}]) }} }}") mutable = "mut " if fields else "" out.append(f"let {mutable}row = Row::new(NodeKind::{n.name});") for f in fields: if f.name in offsets: value = { 1: f"{f.name}.to_le_bytes()[0]", 4: f"split_words({f.name})[0]", 8: f.name, }[f.width] out.append(f"row.set_{arm}_{f.name}({value});") if n.name == "ExprStringLiteral": out.append( "let (start, len) = self.string_literal_text(value);" "row.set_expr_string_literal_text_offset(start); row.set_expr_string_literal_text_len(len);" ) if spills: out.append(f"row.set_{arm}_extra_offset(offset(self.ast.extra.len()));") words = [] for field in fields: if field.name in spills: words.append(f"split_words({field.name})[0]") if field.width == 8: words.append(f"split_words({field.name})[1]") out.append("self.ast.extra.extend([" + ", ".join(words) + "]);") out.append("self.push(row, range) } }") if fields: generics, lifetime = ( ("<'a>", "'a") if any("'a" in f.result for f in fields) else ("", "'_") ) out.append(f"impl{generics} crate::{n.name}<{lifetime}> {{") if n.name == "ExprCompare": out.append( """ /// Returns the operands and operator of a single comparison. #[inline] pub fn as_single(self) -> Option<(crate::Expr<'a>, crate::CmpOp, crate::Expr<'a>)> { let [operator] = self.ops().as_array()?; let [left, right] = self.operands().as_array()?; Some((left, operator, right)) } """ ) for f in fields: if f.name in offsets: raw = f"self.syntax_node().row.{arm}_{f.name}()" if f.width < 8: raw = f"u64::from({raw})" shared = f"record.inline_word::<{offsets[f.name]}, {f.width}>()" else: base = f"self.syntax_node().row.{arm}_extra_offset() as usize" shared_base = f"split_words(record.inline_word::<{extra_offset}, 4>())[0] as usize" index = "base" if not spills[f.name] else f"base + {spills[f.name]}" read = f"self.syntax_node().ast.extra_word::<{str(f.width == 8).lower()}>({index})" raw = f"{{ let base = {base}; {read} }}" shared = f"{{ let base = {shared_base}; {read} }}" if f.result == "&'a crate::name::Name": out.append( f"#[inline] pub fn name_id(self) -> super::NameId {{ super::NameId(split_words({raw})[0]) }}" ) words_by_node[n.name][f.name] = shared if n.name in {"StringLiteral", "BytesLiteral"} and f.name == "value": accessor = ( "byte_range" if n.name == "BytesLiteral" else "text_range" ) out.append( f"pub(super) fn {accessor}(self) -> super::Interval {{ super::Interval::decode({raw}) }}" ) if f.name == "iter": out.append( '#[expect(clippy::iter_not_returning_iterator, reason = "Python names the iterable expression in a for loop or comprehension `iter`.")]' ) out.append(f"""#[inline] pub fn {f.name}(self) -> {f.result} {{ let word = {raw}; {f.read} }}""") out.append("}") row_code = [generate_row_fields(schemas), "impl super::Ast {"] for generate_step, signature in [ ( direct_children, "fn direct_children<'a>(&'a self, node: SyntaxNode<'a>, kind: NodeKind) -> Children<'a>", ), ( child_field, "fn child_field<'a>(&'a self, node: SyntaxNode<'a>, field: u8) -> Field<'a>", ), ( next_descendant, "fn next_descendant<'a>(&'a self, node: SyntaxNode<'a>, stack: &mut Pending<'a>) -> u32", ), ]: steps: dict[str, list[str]] = {} for node in ast.all_nodes: step = generate_step( node, nodes, words_by_node[node.name], schemas[node.name] ) steps.setdefault(step, []).append(f"NodeKind::{node.name}") if generate_step == child_field: del steps["Field::One(None)"] if generate_step != child_field: row_code.append("#[inline]") kind = "kind" if generate_step == direct_children else "record.kind()" row_code.append( f"pub(super) {signature} {{ let record = node.row; match {kind} {{" ) for step, kinds in steps.items(): row_code.append(f"{' | '.join(kinds)} => {{ {step} }}") if generate_step == child_field: row_code.append("_ => Field::One(None),") row_code.append("} }") row_code.append("}") out.append( "impl Builder { pub(super) fn copy_any(&mut self, node: super::SyntaxNode<'_>, rewriter: &mut R) -> BuildId { match node.classify() {" ) for node in ast.all_nodes: fields = [f"{field.name}: {copy_field(field, nodes)}," for field in node.fields] out.append( f"AnyNode::{node.name}(node) => build::{node.name} {{ range: node.range(), " + "".join(fields) + "}.emit(self).cast()," ) out.append("} } }") out.append( "impl<'a> SyntaxNode<'a> { pub fn visit_source_order(self, visitor: &mut (impl super::visitor::source_order::SourceOrderVisitor<'a> + ?Sized)) { self.classify().visit_source_order(visitor); } }" ) visitor = "impl super::visitor::source_order::SourceOrderVisitor<'a> + ?Sized" # Every dispatch path calls the same implementation of a concrete node's child order. children = {node.name: walk_fields(node, nodes) for node in ast.all_nodes} for node in ast.all_nodes: if walk := children[node.name]: access = ( '#[doc = "Visits the children of the module without entering or leaving the module itself."] pub' if node.name == "ModModule" else "pub(super)" ) out.append( f"impl<'a> crate::{node.name}<'a> {{ #[inline] {access} fn visit_source_order(" f"self, visitor: &mut ({visitor})) {{ let node = self; {walk} }} }}" ) def source_order_dispatch( enum: str, variants: list[tuple[str, Node]], visibility: str ) -> None: out.append( f"impl<'a> {enum}<'a> {{ {visibility} fn visit_source_order(" f"self, visitor: &mut ({visitor})) {{ match self {{" ) leaves = [] for variant, node in variants: if children[node.name]: out.append( f"Self::{variant}(node) => node.visit_source_order(visitor)," ) else: leaves.append(f"Self::{variant}(_)") if leaves: out.append(f"{' | '.join(leaves)} => {{}},") out.append("} } }") source_order_dispatch("AnyNode", [(n.name, n) for n in ast.all_nodes], "pub") for group in ast.groups: if group.name != "ExceptHandler": source_order_dispatch( f"{group.name}Kind", [(n.variant, n) for n in group.nodes], "pub(super)" ) # Each string-like expression stores a list of one-word part handles. part_lists = { words_by_node[node.name][field.name] for node in ast.all_nodes for field in node.fields if field.ty in PARTS } assert len(part_lists) == 1, ( "string-like expressions must share the parts list layout" ) part_list = part_lists.pop() row_code.append(f"""impl<'a> super::StringLike<'a> {{ #[inline] pub(super) fn part_nodes(self) -> NodeList<'a, crate::kind::Any> {{ let node: SyntaxNode<'a> = self.into(); let record = node.row; List::new(node.ast, {part_list}) }} }}""") initializers.append("}") out.extend(initializers) out.extend(generate_flat_patterns(ast, nodes)) out.extend(generate_flat_any(ast)) return "\n".join(out), "\n".join(row_code) # ------------------------------------------------------------------------------ # Format and write output def generate(ast: Ast) -> list[str]: out = [ "// Generated by crates/ruff_python_ast/generate.py from ast.toml.", "//! Borrowed Python syntax nodes and their lifetime-free ID markers.", "#[repr(u8)] #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] pub enum NodeKind {", *(f"{node.name}," for node in ast.nodes_in_tag_order), "}", ] types = [*ast.groups, *ast.all_nodes] out.append( "/// Lifetime-free marker types used by AST IDs and the parser.\npub mod kind {" ) for name in ["Any", *(node.name for node in types)]: out.append( f"#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] pub enum {name} {{}}" ) out.append("}") for node in types: if node.doc is not None: write_rustdoc(out, node.doc) out.append( f"#[repr(transparent)] #[derive(Copy, Clone, PartialEq)] pub struct {node.name}<'ast> {{ pub(crate) syntax: crate::SyntaxNode<'ast> }}" ) out.append(f"crate::flat::impl_ast_node!({node.name});") return out def write_output(root: Path, out: list[str]) -> None: out_path = root.joinpath("crates", "ruff_python_ast", "src", "generated.rs") out_path.write_text(rustfmt("\n".join(out))) # ------------------------------------------------------------------------------ # Main def main() -> None: root = Path( check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() ) ast = load_ast(root) out = generate(ast) write_output(root, out) flat = root / "crates" / "ruff_python_ast" / "src" / "flat" / "generated.rs" views, rows = generate_flat(ast) flat.write_text(rustfmt(views)) flat.with_name("row").joinpath("generated.rs").write_text(rustfmt(rows)) if __name__ == "__main__": main()