mirror of
https://github.com/astral-sh/ruff.git
synced 2026-11-21 12:16:15 +08:00
## Summary Use a shared ty override for Ruff’s standalone scripts now that PEP 723 scripts can inherit project settings. Fix the diagnostics in the two PR automation scripts so they can use the same checks. ## Notes to reviewers All 28 standalone scripts passed ty with locked dependencies, and the changed-file hooks and mocked workflow checks passed. --------- Co-authored-by: Alex Waygood <alex.waygood@gmail.com>
547 lines
16 KiB
Python
547 lines
16 KiB
Python
# /// script
|
|
# requires-python = ">=3.12"
|
|
# dependencies = []
|
|
#
|
|
# [tool.uv]
|
|
# no-build = true
|
|
# exclude-newer = "P7D"
|
|
# require-build-hashes = true
|
|
# preview-features = ["build-dependency-hashes", "missing-exclude-newer-package-lock"]
|
|
# ///
|
|
|
|
"""
|
|
Compare memory usage reports between two ty versions and generate a PR comment.
|
|
|
|
This script can be used in two modes:
|
|
|
|
1. Report comparison mode: Reads pre-generated JSON memory reports and compares them.
|
|
2. Full run mode: Sets up projects, runs memory tests, and generates comparison.
|
|
|
|
Examples:
|
|
# Compare pre-generated memory reports
|
|
%(prog)s compare --old-dir old_reports/ --new-dir new_reports/
|
|
|
|
# Full run: set up projects and their dependencies, then run memory tests
|
|
%(prog)s run --old-ty ./ty-old --new-ty ./ty-new
|
|
|
|
# Write output to a file
|
|
%(prog)s compare --old-dir old_reports/ --new-dir new_reports/ --output memory_diff.md
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any, Final, Self
|
|
|
|
KNOWN_PROJECTS: Final = ("flake8", "sphinx", "prefect", "trio")
|
|
|
|
|
|
@dataclass(slots=True, kw_only=True)
|
|
class MemoryReport:
|
|
"""Memory report from a single ty run."""
|
|
|
|
total_bytes: int
|
|
struct_metadata_bytes: int
|
|
struct_fields_bytes: int
|
|
memo_metadata_bytes: int
|
|
memo_fields_bytes: int
|
|
structs: list[dict[str, Any]] = field(default_factory=list)
|
|
queries: list[dict[str, Any]] = field(default_factory=list)
|
|
|
|
@classmethod
|
|
def from_json(cls, path: Path) -> Self:
|
|
"""Load a memory report from a JSON file."""
|
|
with open(path) as f:
|
|
data = json.load(f)
|
|
return cls(
|
|
total_bytes=data["total_bytes"],
|
|
struct_metadata_bytes=data["struct_metadata_bytes"],
|
|
struct_fields_bytes=data["struct_fields_bytes"],
|
|
memo_metadata_bytes=data["memo_metadata_bytes"],
|
|
memo_fields_bytes=data["memo_fields_bytes"],
|
|
structs=data.get("structs", []),
|
|
queries=data.get("queries", []),
|
|
)
|
|
|
|
|
|
@dataclass(slots=True, kw_only=True)
|
|
class ProjectComparison:
|
|
"""Comparison of memory usage between old and new ty versions for a project."""
|
|
|
|
name: str
|
|
old: MemoryReport
|
|
new: MemoryReport
|
|
|
|
@property
|
|
def total_diff_bytes(self) -> int:
|
|
return self.new.total_bytes - self.old.total_bytes
|
|
|
|
|
|
def format_bytes(bytes: int) -> str:
|
|
"""Format bytes as a human-readable size."""
|
|
bytes: float = float(bytes)
|
|
|
|
for unit in ("B", "kB", "MB"):
|
|
if abs(bytes) < 1024.0:
|
|
return f"{bytes:.2f}{unit}"
|
|
bytes /= 1024.0
|
|
|
|
return f"{bytes:.2f}GB"
|
|
|
|
|
|
def format_diff(*, old_bytes: int, new_bytes: int) -> str:
|
|
"""Format a difference with percentage and direction indicator."""
|
|
diff = new_bytes - old_bytes
|
|
|
|
if diff == 0:
|
|
return "-"
|
|
|
|
sign = "+" if diff > 0 else ""
|
|
|
|
if old_bytes == 0:
|
|
return f"{sign}{format_bytes(diff)} (new)"
|
|
|
|
return f"{sign}{diff / old_bytes:.2%} ({format_bytes(abs(diff))})"
|
|
|
|
|
|
def format_outcome(*, old_bytes: int, new_bytes: int) -> str:
|
|
"""Format the outcome indicator."""
|
|
diff = new_bytes - old_bytes
|
|
if diff > 0:
|
|
return "⏫"
|
|
elif diff < 0:
|
|
return "⬇️"
|
|
else:
|
|
return "✅"
|
|
|
|
|
|
def load_reports_from_directory(directory: Path) -> dict[str, MemoryReport]:
|
|
"""Load all JSON memory reports from a directory."""
|
|
reports = {}
|
|
for path in directory.glob("*.json"):
|
|
project_name = path.stem
|
|
reports[project_name] = MemoryReport.from_json(path)
|
|
return reports
|
|
|
|
|
|
def item_total_bytes(item: dict[str, Any]) -> int:
|
|
"""Get total bytes (metadata + fields) for a struct or query item."""
|
|
result = item.get("metadata_bytes", 0) + item.get("fields_bytes", 0)
|
|
assert isinstance(result, int)
|
|
return result
|
|
|
|
|
|
def diff_items(
|
|
*, old_items: list[dict[str, Any]], new_items: list[dict[str, Any]]
|
|
) -> list[tuple[str, int, int]]:
|
|
"""Diff two lists of struct/query items by name.
|
|
|
|
Returns a list of (name, old_bytes, new_bytes) sorted by absolute diff descending.
|
|
"""
|
|
old_by_name: dict[str, dict[str, Any]] = {item["name"]: item for item in old_items}
|
|
new_by_name: dict[str, dict[str, Any]] = {item["name"]: item for item in new_items}
|
|
|
|
all_names = old_by_name.keys() | new_by_name.keys()
|
|
|
|
diffs = []
|
|
for name in all_names:
|
|
old_bytes = item_total_bytes(old_by_name[name]) if name in old_by_name else 0
|
|
new_bytes = item_total_bytes(new_by_name[name]) if name in new_by_name else 0
|
|
if old_bytes != new_bytes:
|
|
diffs.append((name.replace(" ", ""), old_bytes, new_bytes))
|
|
|
|
diffs.sort(key=lambda x: abs(x[2] - x[1]), reverse=True)
|
|
return diffs
|
|
|
|
|
|
# Maximum number of changed items to show per category in the detailed breakdown
|
|
MAX_CHANGED_ITEMS: Final = 15
|
|
|
|
|
|
def render_summary(projects: list[ProjectComparison]) -> str:
|
|
"""Render a summary of all project comparisons."""
|
|
if not projects:
|
|
return "No memory reports to compare."
|
|
|
|
projects.sort(key=lambda p: p.total_diff_bytes, reverse=True)
|
|
|
|
# Suppress the memory report if no project had any top-line changes >10KB
|
|
any_increased = any(p.total_diff_bytes > 10_000 for p in projects)
|
|
any_decreased = any(p.total_diff_bytes < -10_000 for p in projects)
|
|
any_changed = any_increased or any_decreased
|
|
|
|
lines = ["## Memory usage report", ""]
|
|
|
|
if any_changed:
|
|
lines.extend(
|
|
[
|
|
"### Summary",
|
|
"",
|
|
"| Project | Old | New | Diff | Outcome |",
|
|
"|---------|-----|-----|------|---------|",
|
|
]
|
|
)
|
|
|
|
for proj in projects:
|
|
outcome = format_outcome(
|
|
old_bytes=proj.old.total_bytes, new_bytes=proj.new.total_bytes
|
|
)
|
|
|
|
lines.append(
|
|
f"| {proj.name} | {format_bytes(proj.old.total_bytes)} | "
|
|
f"{format_bytes(proj.new.total_bytes)} | "
|
|
f"{format_diff(old_bytes=proj.old.total_bytes, new_bytes=proj.new.total_bytes)} | {outcome} |"
|
|
)
|
|
|
|
lines.extend(
|
|
[
|
|
"",
|
|
"### Significant changes",
|
|
"",
|
|
"<details>",
|
|
"<summary>Click to expand detailed breakdown</summary>",
|
|
"",
|
|
]
|
|
)
|
|
|
|
for proj in projects:
|
|
item_diffs = diff_items(
|
|
old_items=proj.old.structs + proj.old.queries,
|
|
new_items=proj.new.structs + proj.new.queries,
|
|
)
|
|
|
|
if not item_diffs:
|
|
continue
|
|
|
|
lines.extend(
|
|
[
|
|
f"### {proj.name}",
|
|
"",
|
|
"| Name | Old | New | Diff | Outcome |",
|
|
"|------|-----|-----|------|---------|",
|
|
]
|
|
)
|
|
|
|
for name, old_bytes, new_bytes in item_diffs[:MAX_CHANGED_ITEMS]:
|
|
outcome = format_outcome(old_bytes=old_bytes, new_bytes=new_bytes)
|
|
|
|
lines.append(
|
|
f"| `{name}` | {format_bytes(old_bytes)} | "
|
|
f"{format_bytes(new_bytes)} | "
|
|
f"{format_diff(old_bytes=old_bytes, new_bytes=new_bytes)} |"
|
|
f"{outcome} |"
|
|
)
|
|
|
|
remaining = len(item_diffs) - MAX_CHANGED_ITEMS
|
|
if remaining > 0:
|
|
lines.append(f"| ... | | | *{remaining} more* |")
|
|
|
|
lines.append("")
|
|
|
|
lines.extend(
|
|
[
|
|
"</details>",
|
|
"",
|
|
]
|
|
)
|
|
else:
|
|
lines.append("Memory usage unchanged ✅")
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
def setup_project(*, name: str, dest: Path) -> tuple[Path, list[str]]:
|
|
"""Clone a project and install its mypy-primer dependencies."""
|
|
project_path = dest / name
|
|
setup_script = Path(__file__).with_name("setup_primer_project.py")
|
|
setup_command = [
|
|
"uv",
|
|
"run",
|
|
"--locked",
|
|
"--python",
|
|
sys.executable,
|
|
"--script",
|
|
str(setup_script),
|
|
]
|
|
|
|
if project_path.exists():
|
|
print(f"Project {name} already exists at {project_path}", file=sys.stderr)
|
|
else:
|
|
print(f"Setting up {name} and its dependencies...", file=sys.stderr)
|
|
subprocess.run(
|
|
[*setup_command, name, str(project_path)],
|
|
check=True,
|
|
stdout=sys.stderr,
|
|
)
|
|
|
|
ty_command = subprocess.run(
|
|
[*setup_command, "--print-ty-command", name, str(project_path)],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
return project_path, shlex.split(ty_command.stdout)
|
|
|
|
|
|
def run_ty_memory_check(
|
|
*,
|
|
ty_path: str,
|
|
ty_command: list[str],
|
|
project_path: Path,
|
|
output_path: Path,
|
|
) -> None:
|
|
"""Run ty on a project and capture memory report to a file."""
|
|
env = os.environ.copy()
|
|
env["TY_MEMORY_REPORT"] = "json"
|
|
env["TY_MAX_PARALLELISM"] = "1" # For deterministic memory numbers
|
|
|
|
print(f"Running {ty_path} on {project_path.name}...", file=sys.stderr)
|
|
command = [
|
|
str(Path(ty_path).resolve()) if argument == "{ty}" else argument
|
|
for argument in ty_command
|
|
]
|
|
result = subprocess.run(
|
|
command,
|
|
cwd=project_path,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
env=env,
|
|
)
|
|
# Write stdout (JSON memory report) to file
|
|
output_path.write_text(result.stdout, encoding="utf-8")
|
|
|
|
|
|
def run_memory_tests(
|
|
*,
|
|
old_ty: str,
|
|
new_ty: str,
|
|
projects_dir: Path,
|
|
old_reports_dir: Path,
|
|
new_reports_dir: Path,
|
|
) -> None:
|
|
"""Run memory tests for all projects with both ty versions."""
|
|
old_reports_dir.mkdir(parents=True, exist_ok=True)
|
|
new_reports_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
for project_name in KNOWN_PROJECTS:
|
|
project_path, ty_command = setup_project(name=project_name, dest=projects_dir)
|
|
|
|
# Run old ty
|
|
old_report_path = old_reports_dir / f"{project_name}.json"
|
|
run_ty_memory_check(
|
|
ty_path=old_ty,
|
|
ty_command=ty_command,
|
|
project_path=project_path,
|
|
output_path=old_report_path,
|
|
)
|
|
|
|
# Run new ty
|
|
new_report_path = new_reports_dir / f"{project_name}.json"
|
|
run_ty_memory_check(
|
|
ty_path=new_ty,
|
|
ty_command=ty_command,
|
|
project_path=project_path,
|
|
output_path=new_report_path,
|
|
)
|
|
|
|
|
|
def cmd_compare(args: argparse.Namespace) -> None:
|
|
"""Handle the 'compare' subcommand."""
|
|
comparisons = []
|
|
|
|
if args.old and args.new:
|
|
# Single file comparison
|
|
old_report = MemoryReport.from_json(args.old)
|
|
new_report = MemoryReport.from_json(args.new)
|
|
project_name = args.new.stem
|
|
comparisons.append(
|
|
ProjectComparison(name=project_name, old=old_report, new=new_report)
|
|
)
|
|
elif args.old_dir and args.new_dir:
|
|
# Directory comparison
|
|
old_reports = load_reports_from_directory(args.old_dir)
|
|
new_reports = load_reports_from_directory(args.new_dir)
|
|
|
|
# Find common projects
|
|
common_projects = set(old_reports.keys()) & set(new_reports.keys())
|
|
if not common_projects:
|
|
print(
|
|
"Error: No common projects found between old and new directories",
|
|
file=sys.stderr,
|
|
)
|
|
sys.exit(1)
|
|
|
|
for project in sorted(common_projects):
|
|
comparisons.append(
|
|
ProjectComparison(
|
|
name=project,
|
|
old=old_reports[project],
|
|
new=new_reports[project],
|
|
)
|
|
)
|
|
else:
|
|
print(
|
|
"Error: Must specify either --old/--new or --old-dir/--new-dir",
|
|
file=sys.stderr,
|
|
)
|
|
sys.exit(1)
|
|
|
|
rendered = render_summary(comparisons)
|
|
|
|
if args.output:
|
|
args.output.write_text(rendered, encoding="utf-8")
|
|
print(f"Output written to {args.output}", file=sys.stderr)
|
|
print(rendered, file=sys.stderr)
|
|
else:
|
|
print(rendered)
|
|
|
|
|
|
def cmd_run(args: argparse.Namespace) -> None:
|
|
"""Handle the 'run' subcommand."""
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
# Set up directories
|
|
projects_dir = args.projects_dir or Path(tmpdir) / "ty_memory_projects"
|
|
old_reports_dir = args.old_reports_dir or Path(tmpdir) / "ty_memory_old"
|
|
new_reports_dir = args.new_reports_dir or Path(tmpdir) / "ty_memory_new"
|
|
|
|
print(f"Projects directory: {projects_dir}", file=sys.stderr)
|
|
print(f"Old reports directory: {old_reports_dir}", file=sys.stderr)
|
|
print(f"New reports directory: {new_reports_dir}", file=sys.stderr)
|
|
|
|
# Run memory tests
|
|
run_memory_tests(
|
|
old_ty=args.old_ty,
|
|
new_ty=args.new_ty,
|
|
projects_dir=projects_dir,
|
|
old_reports_dir=old_reports_dir,
|
|
new_reports_dir=new_reports_dir,
|
|
)
|
|
|
|
# Load and compare reports
|
|
old_reports = load_reports_from_directory(old_reports_dir)
|
|
new_reports = load_reports_from_directory(new_reports_dir)
|
|
|
|
comparisons = []
|
|
for project in sorted(old_reports.keys() & new_reports.keys()):
|
|
comparisons.append(
|
|
ProjectComparison(
|
|
name=project,
|
|
old=old_reports[project],
|
|
new=new_reports[project],
|
|
)
|
|
)
|
|
|
|
rendered = render_summary(comparisons)
|
|
|
|
if args.output:
|
|
args.output.write_text(rendered, encoding="utf-8")
|
|
print(f"Output written to {args.output}", file=sys.stderr)
|
|
print(rendered, file=sys.stderr)
|
|
else:
|
|
print(rendered)
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
)
|
|
|
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
|
|
# Compare subcommand
|
|
compare_parser = subparsers.add_parser(
|
|
"compare",
|
|
help="Compare pre-generated memory reports",
|
|
)
|
|
compare_parser.add_argument(
|
|
"--old",
|
|
type=Path,
|
|
help="Path to old memory report JSON file",
|
|
)
|
|
compare_parser.add_argument(
|
|
"--new",
|
|
type=Path,
|
|
help="Path to new memory report JSON file",
|
|
)
|
|
compare_parser.add_argument(
|
|
"--old-dir",
|
|
type=Path,
|
|
help="Directory containing old memory report JSON files",
|
|
)
|
|
compare_parser.add_argument(
|
|
"--new-dir",
|
|
type=Path,
|
|
help="Directory containing new memory report JSON files",
|
|
)
|
|
compare_parser.add_argument(
|
|
"--output",
|
|
type=Path,
|
|
help="Write output to file instead of stdout",
|
|
)
|
|
|
|
# Run subcommand
|
|
run_parser = subparsers.add_parser(
|
|
"run",
|
|
help="Set up projects, run ty, and compare memory usage",
|
|
)
|
|
run_parser.add_argument(
|
|
"--old-ty",
|
|
required=True,
|
|
help="Path to old ty executable",
|
|
)
|
|
run_parser.add_argument(
|
|
"--new-ty",
|
|
required=True,
|
|
help="Path to new ty executable",
|
|
)
|
|
run_parser.add_argument(
|
|
"--projects-dir",
|
|
type=Path,
|
|
help="Directory to clone projects into (default: temp directory)",
|
|
)
|
|
run_parser.add_argument(
|
|
"--old-reports-dir",
|
|
type=Path,
|
|
help="Directory for old memory reports (default: temp directory)",
|
|
)
|
|
run_parser.add_argument(
|
|
"--new-reports-dir",
|
|
type=Path,
|
|
help="Directory for new memory reports (default: temp directory)",
|
|
)
|
|
run_parser.add_argument(
|
|
"--output",
|
|
type=Path,
|
|
help="Write output to file instead of stdout",
|
|
)
|
|
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> None:
|
|
args = parse_args()
|
|
|
|
match args.command:
|
|
case "compare":
|
|
cmd_compare(args)
|
|
case "run":
|
|
cmd_run(args)
|
|
case _:
|
|
assert False, (
|
|
f"Unknown subcommand {args.command!r}; is the script out of date?"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|