Files
Micha Reiser 043802b0d8 Build typed arena nodes in the Python parser
Carry the typed IDs returned by node constructors through parsing and
recovery. Parse source-ordered argument and parameter lists directly.
Keep ownership of the tree with the parsed module.
2026-11-20 16:16:30 +00:00

222 lines
7.4 KiB
Rust

use std::collections::HashSet;
use std::hash::{BuildHasher, RandomState};
use ruff_python_ast::HasNodeIndex;
use ruff_python_ast::comparable::{
ComparableExpr, ComparableLiteral, ComparableMod, ComparableStmt, HashableExpr,
};
use ruff_python_parser::{
Mode, ParseError, ParseOptions, parse_expression, parse_module, parse_unchecked,
};
#[track_caller]
fn assert_comparable(left: &str, right: &str) -> Result<(), ParseError> {
let left_parsed = parse_expression(left)?;
let right_parsed = parse_expression(right)?;
let left_compr = ComparableExpr::from(left_parsed.expr());
let right_compr = ComparableExpr::from(right_parsed.expr());
assert_eq!(left_compr, right_compr);
let hasher = RandomState::new();
assert_eq!(hasher.hash_one(left_compr), hasher.hash_one(right_compr));
Ok(())
}
#[track_caller]
fn assert_noncomparable(left: &str, right: &str) -> Result<(), ParseError> {
let left_parsed = parse_expression(left)?;
let right_parsed = parse_expression(right)?;
let left_compr = ComparableExpr::from(left_parsed.expr());
let right_compr = ComparableExpr::from(right_parsed.expr());
assert_ne!(left_compr, right_compr);
Ok(())
}
#[track_caller]
fn assert_hashable_equal(left: &str, right: &str) -> Result<(), ParseError> {
let left_parsed = parse_expression(left)?;
let right_parsed = parse_expression(right)?;
assert_eq!(
HashableExpr::from(left_parsed.expr()),
HashableExpr::from(right_parsed.expr())
);
Ok(())
}
#[track_caller]
fn assert_hashable_unequal(left: &str, right: &str) -> Result<(), ParseError> {
let left_parsed = parse_expression(left)?;
let right_parsed = parse_expression(right)?;
assert_ne!(
HashableExpr::from(left_parsed.expr()),
HashableExpr::from(right_parsed.expr())
);
Ok(())
}
#[test]
fn call_arguments() -> Result<(), ParseError> {
let call = "factory(item, *items, limit=2, **options)";
assert_comparable(call, "factory(item, *items, limit=0x2, **options)")?;
assert_comparable(call, "factory(item, limit=2, *items, **options)")?;
assert_noncomparable(call, "factory(item, *items, **options, limit=2)")
}
#[test]
fn name_keys_ignore_context_location_and_node_index() -> Result<(), ParseError> {
let mut parsed = parse_module("item = item")?;
let value = parsed.suite().at(0).as_assign_stmt().unwrap().value();
let node_index = value.node_index();
let hasher = RandomState::new();
let hash = hasher.hash_one(ComparableExpr::from(value));
parsed.index_module();
let assignment = parsed.suite().at(0).as_assign_stmt().unwrap();
let value = assignment.value();
assert_ne!(value.node_index(), node_index);
let key = ComparableExpr::from(value);
assert_eq!(hasher.hash_one(key), hash);
let target = assignment.targets().at(0).as_name_expr().unwrap();
assert!(HashSet::from([key]).contains(&ComparableExpr::from(target)));
Ok(())
}
#[test]
fn formatter_ast_diff_shows_only_structural_changes() -> Result<(), ParseError> {
fn debug_module(source: &str) -> Result<String, ParseError> {
let module = parse_module(source)?;
Ok(format!("{:#?}", ComparableMod::from(module.syntax())))
}
let split = debug_module("x=b'a' b'b'")?;
assert_eq!(split, debug_module("x = b'ab'")?);
assert_ne!(split, debug_module("x = b'ac'")?);
Ok(())
}
#[test]
fn match_statement_compares_its_pattern() -> Result<(), ParseError> {
let left = parse_module("match x:\n case [1, 2]: pass")?;
let right = parse_module("match x:\n case [0x1, 2]: pass")?;
let different = parse_module("match x:\n case 1 | 2: pass")?;
let left = ComparableStmt::from(left.suite().at(0));
let right = ComparableStmt::from(right.suite().at(0));
assert_eq!(left, right);
let hasher = RandomState::new();
assert_eq!(hasher.hash_one(left), hasher.hash_one(right));
assert_ne!(left, ComparableStmt::from(different.suite().at(0)));
Ok(())
}
#[test]
fn literal_keys_preserve_part_boundaries() -> Result<(), ParseError> {
fn key(expr: ruff_python_ast::Expr<'_>) -> ComparableLiteral<'_> {
ComparableLiteral::from(expr.as_literal_expr().unwrap())
}
let split = parse_expression("'a' 'b'")?;
let joined = parse_expression("'ab'")?;
assert_ne!(key(split.expr()), key(joined.expr()));
Ok(())
}
#[test]
fn concatenated_strings_compare_equal() -> Result<(), ParseError> {
let split_contents = r#"'a' 'b' r'\n raw'"#;
let value_contents = r#"'ab\\n raw'"#;
assert_comparable(split_contents, value_contents)
}
#[test]
fn bytes_across_part_boundaries() -> Result<(), ParseError> {
let split = "b'a' b'bc' b''";
assert_comparable("b'ab' b'c'", split)?;
assert_noncomparable("b'ab'", split)
}
#[test]
fn concatenated_fstrings_compare_equal() -> Result<(), ParseError> {
let split_contents = r#"f"{foo!r} this" r"\n raw" f" and {bar!s} that""#;
let value_contents = r#"f"{foo!r} this\\n raw and {bar!s} that""#;
assert_comparable(split_contents, value_contents)
}
#[test]
fn concatenated_tstrings_compare_equal() -> Result<(), ParseError> {
let split_contents = r#"t"{foo!r} this" rt"\n raw" t" and {bar!s} that""#;
let value_contents = r#"t"{foo!r} this\\n raw and {bar!s} that""#;
assert_comparable(split_contents, value_contents)
}
#[test]
fn empty_interpolated_string_parts() -> Result<(), ParseError> {
assert_noncomparable("f'{x}' ''", "f'{x}'")?;
assert_comparable("t'{x}' t''", "t'{x}'")
}
#[test]
fn t_strings_literal_order_matters_compare_unequal() -> Result<(), ParseError> {
let interp_then_literal_contents = r#"t"{foo}bar""#;
let literal_then_interp_contents = r#"t"bar{foo}""#;
assert_noncomparable(interp_then_literal_contents, literal_then_interp_contents)
}
#[test]
fn equivalent_numbers_hash_equal() -> Result<(), ParseError> {
assert_hashable_equal("2", "2.0")?;
assert_hashable_equal("-1", "-1.0")?;
assert_hashable_equal("2", "2 + 0j")?;
assert_hashable_equal("1j", "0 + 1j")?;
assert_hashable_equal("9007199254740992", "9007199254740992.0")?;
assert_hashable_equal("9007199254740992", "9007199254740993 + 0j")?;
assert_hashable_equal("(2,)", "(2.0,)")?;
assert_hashable_unequal("9007199254740993", "9007199254740992.0")?;
assert_hashable_unequal("9007199254740993", "9007199254740993 + 0j")?;
assert_hashable_unequal("18446744073709551615", "18446744073709551616.0")?;
Ok(())
}
#[test]
fn large_integers_fall_back_to_structural_comparison() -> Result<(), ParseError> {
assert_hashable_unequal("18446744073709551616", "0x10000000000000000")
}
#[test]
fn dynamic_tuple_elements_do_not_compare_by_value() -> Result<(), ParseError> {
assert_hashable_unequal("(f(), 2)", "(f(), 2.0)")
}
#[test]
fn recovered_mapping_pattern_order() {
let sources = [
"match x:\n case {'a': a, **rest}: pass",
"match x:\n case {**rest, 'a': a}: pass",
"match x:\n case {**rest, 'a': a, **rest}: pass",
"match x:\n case {**other, 'a': a, **rest}: pass",
];
let parses: Vec<_> = sources
.iter()
.map(|s| parse_unchecked(s, ParseOptions::from(Mode::Module)))
.collect();
for (i, left) in parses.iter().enumerate() {
for (j, right) in parses.iter().enumerate() {
let left = ComparableMod::from(left.syntax());
let right = ComparableMod::from(right.syntax());
assert_eq!(left == right, i == j);
}
}
}