refactor(transforms): drop the Python mirror of the Rust log selector

LogCompressor.compress() runs entirely in Rust. The class still carried a Python re-implementation of the line parser, scorer, selector, deduper, context adder and formatter, plus the pytest short-summary helpers and a _store_in_ccr shim, kept only so old tests could call them. Production never ran them, and _store_in_ccr still had the #816 bug that the live path fixed (no explicit_hash).

Removes those private helpers. Tests now target what ships:
- format detection tests call headroom._core.detect_log_format, which compress() uses;
- level counts and the JS/rustc stack-trace shapes become compress()-level tests;
- two Rust unit tests are added for behaviour no Rust test pinned yet: the full level keyword table (FAIL, DEBUG, TRACE, unknown) and score ordering/boosts;
- helper tests whose behaviour Rust already unit-tests (dedupe, pytest short-summary recognition, cap precedence and reservation, omission naming, new trace flavours) are deleted.

adaptive_sizer.py stays: main's SearchCompressor helpers still import it until the search-mirror PR lands.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Tejas Chopra
2026-10-09 17:20:42 -07:00
co-authored by Claude Opus 5.5
parent a9815bbb5f
commit b82ef1b73a
5 changed files with 106 additions and 996 deletions
@@ -1780,6 +1780,54 @@ mod tests {
assert_eq!(lines[2].level, LogLevel::Unknown);
}
#[test]
fn level_classifier_covers_every_level() {
let c = cmp();
let cases: &[(&str, LogLevel)] = &[
("ERROR: something went wrong", LogLevel::Error),
("error: file not found", LogLevel::Error),
("Error: Invalid input", LogLevel::Error),
("FATAL: system crash", LogLevel::Error),
("fatal error occurred", LogLevel::Error),
("CRITICAL: database down", LogLevel::Error),
("FAIL tests/test_foo.py", LogLevel::Fail),
("FAILED to connect", LogLevel::Fail),
("Test failed", LogLevel::Fail),
("WARN: deprecated function", LogLevel::Warn),
("WARNING: low disk space", LogLevel::Warn),
("warning: unused variable", LogLevel::Warn),
("INFO: starting process", LogLevel::Info),
("info starting", LogLevel::Info),
("DEBUG: variable x = 5", LogLevel::Debug),
("debug mode enabled", LogLevel::Debug),
("TRACE: entering function", LogLevel::Trace),
("Just some regular text", LogLevel::Unknown),
];
for (line, expected) in cases {
let parsed = c.parse_lines(&[line]);
assert_eq!(parsed[0].level, *expected, "level for {line:?}");
}
}
#[test]
fn score_line_orders_levels_and_boosts() {
let line = |level: LogLevel, is_stack_trace: bool, is_summary: bool| LogLine {
line_number: 0,
content: String::new(),
level,
is_stack_trace,
is_summary,
score: 0.0,
};
let info = score_log_line(&line(LogLevel::Info, false, false));
assert!(score_log_line(&line(LogLevel::Error, false, false)) > info);
assert!(score_log_line(&line(LogLevel::Fail, false, false)) > info);
assert_eq!(score_log_line(&line(LogLevel::Warn, false, false)), 0.5);
let plain = score_log_line(&line(LogLevel::Unknown, false, false));
assert!(score_log_line(&line(LogLevel::Unknown, true, false)) > plain);
assert!(score_log_line(&line(LogLevel::Unknown, false, true)) > plain);
}
#[test]
fn recognizes_pytest_short_summary_entries_by_section_and_line_number() {
let lines = [
+1 -422
View File
@@ -11,11 +11,6 @@ is now a thin shim that:
2. Routes `LogCompressor.compress()` entirely through the Rust
implementation, picking up the bug fixes (chained-exception trace
survival, conservative warning dedupe, loud CCR failures).
3. Implements legacy internal helpers (`_detect_format`, `_parse_lines`,
`_score_line`, `_select_lines`, `_select_with_first_last`,
`_dedupe_similar`, `_format_output`) on top of the Rust building
blocks where a Rust delegation makes sense; otherwise keeps Python
logic that mirrors Rust scoring.
# Bug fixes the Rust port carries (and this shim therefore inherits)
@@ -93,89 +88,6 @@ class LogLine:
return hash(self.line_number)
def _scan_pytest_short_summary(
lines: list[tuple[int, str]],
) -> tuple[dict[int, str], set[int], int | None]:
"""Scan for pytest's "short test summary info" sections.
Mirrors Rust's ``scan_pytest_short_summary``: a section opens on a
complete ``=== ... short test summary info ... ===`` separator (a trailing
``\\r`` is ignored) and closes at the next ``===`` line or EOF. For
``FAILED <nodeid> - <msg>`` the id is the text before ``" - "``.
Args:
lines: ``(line_number, content)`` pairs in log order.
Returns:
``(entries, totals_lines, first_header)``: line number -> node id for
each entry, the line numbers of the ``===`` lines closing a section
(pytest's run totals), and the first section header's line number.
"""
entries: dict[int, str] = {}
totals_lines: set[int] = set()
first_header: int | None = None
in_short_summary = False
for line_number, line in lines:
recognition_line = line.removesuffix("\r")
if recognition_line.startswith("==="):
opens = (
recognition_line.endswith("===") and "short test summary info" in recognition_line
)
if opens:
if first_header is None:
first_header = line_number
elif in_short_summary:
totals_lines.add(line_number)
in_short_summary = opens
continue
if not in_short_summary:
continue
rest = next(
(
line.removeprefix(prefix)
for prefix in ("FAILED ", "ERROR ")
if line.startswith(prefix)
),
None,
)
if rest is not None:
node_id = rest.split(" - ", 1)[0].strip()
if node_id:
entries[line_number] = node_id
return entries, totals_lines, first_header
def _pytest_short_summary_entries(lines: list[tuple[int, str]]) -> dict[int, str]:
"""Map line numbers to node ids of pytest short-summary FAILED/ERROR entries.
See ``_scan_pytest_short_summary``.
"""
return _scan_pytest_short_summary(lines)[0]
def _pytest_cap_reserve(
log_lines: list[LogLine], totals_lines: set[int], first_header: int | None
) -> set[int]:
"""Return the lines the global cap must keep while a short summary competes.
Mirrors Rust's ``pytest_cap_reserve``: each section's totals line and the
first ``E `` assertion line before the summary. Short-summary entries win
equal-score ties, so without this reserve a long summary crowds out the run
totals and every error message.
"""
reserved = set(totals_lines)
for line in log_lines:
if first_header is not None and line.line_number >= first_header:
break
if line.content.startswith("E "):
reserved.add(line.line_number)
break
return reserved
@dataclass
class LogCompressorConfig:
"""Configuration for log compression."""
@@ -241,9 +153,7 @@ class LogCompressor:
"""Rust-backed log compressor.
Drop-in replacement for the retired Python class. `compress()`
delegates to Rust end-to-end; internal helpers used by the
existing test surface keep working but route through the same
Rust building blocks where they exist.
delegates to Rust end-to-end.
"""
def __init__(self, config: LogCompressorConfig | None = None) -> None:
@@ -305,337 +215,6 @@ class LogCompressor:
stats=stats_dict,
)
# ─── Legacy internal helpers (test surface compat) ──────────────────
def _detect_format(self, lines: list[str]) -> LogFormat:
"""Delegate to the Rust format detector."""
from headroom._core import detect_log_format
return _format_from_str(detect_log_format(list(lines)))
def _parse_lines(self, lines: list[str]) -> list[LogLine]:
"""Parse + categorize lines, mirroring Rust's classification.
Stays Python so the legacy direct-call test surface keeps
working without rebuilding through Rust on every test. Rust
unit tests pin Rust's behavior; this implementation must
mirror Rust's level/stack-trace/summary classification rules.
"""
import re
# Mirror of Rust's level classifier: aho-corasick with
# word-boundary post-filter. Python's `re` is fast enough for
# the test path; the Rust path uses aho-corasick. Both share
# the same keyword set.
level_patterns = [
(
LogLevel.ERROR,
re.compile(r"\b(?:ERROR|error|Error|FATAL|fatal|Fatal|CRITICAL|critical)\b"),
),
(LogLevel.FAIL, re.compile(r"\b(?:FAIL|FAILED|fail|failed|Fail|Failed)\b")),
(LogLevel.WARN, re.compile(r"\b(?:WARN|WARNING|warn|warning|Warn|Warning)\b")),
(LogLevel.INFO, re.compile(r"\b(?:INFO|info|Info)\b")),
(LogLevel.DEBUG, re.compile(r"\b(?:DEBUG|debug|Debug)\b")),
(LogLevel.TRACE, re.compile(r"\b(?:TRACE|trace|Trace)\b")),
]
stack_trace_patterns = [
re.compile(r"^\s*Traceback \(most recent call last\)"),
re.compile(r'^\s*File ".+", line \d+'),
re.compile(r"^\s*at .+\(.+:\d+:\d+\)"),
re.compile(r"^\s+at [\w.$]+\("),
re.compile(r"^\s*--> .+:\d+:\d+"),
re.compile(r"^\s*\d+:\s+0x[0-9a-f]+"),
# Rust panics (RustBacktrace flavor)
re.compile(r"^thread '[^']*' panicked at"),
re.compile(r"^stack backtrace:"),
re.compile(r"^\s+\d+: \S"),
# Go panics / goroutine dumps (GoPanic flavor)
re.compile(r"^(?:panic|fatal error): "),
re.compile(r"^goroutine \d+ \["),
re.compile(r"^\t\S+\.go:\d+(?: \+0x[0-9a-f]+)?$"),
# .NET (DotNet flavor)
re.compile(r"^Unhandled exception\."),
re.compile(r"^\s*at .+\) in .+:line \d+"),
# Java chained-exception continuations
re.compile(r"^Caused by: "),
re.compile(r"^\s*\.\.\. \d+ more$"),
]
summary_patterns = [
re.compile(r"^={3,}"),
re.compile(r"^-{3,}"),
re.compile(r"^\d+ (passed|failed|skipped|error|warning)"),
re.compile(r"^(?:Tests?|Suites?):?\s+\d+"),
re.compile(r"^(?:TOTAL|Total|Summary)"),
re.compile(r"^(?:Build|Compile|Test).*(?:succeeded|failed|complete)"),
]
log_lines: list[LogLine] = []
in_stack_trace = False
stack_trace_lines = 0
short_summary_entries = _pytest_short_summary_entries(list(enumerate(lines)))
for i, line in enumerate(lines):
log_line = LogLine(line_number=i, content=line)
for level, pattern in level_patterns:
if pattern.search(line):
log_line.level = level
break
for pattern in stack_trace_patterns:
if pattern.search(line):
in_stack_trace = True
stack_trace_lines = 0
break
if in_stack_trace:
log_line.is_stack_trace = True
stack_trace_lines += 1
if stack_trace_lines > self.config.stack_trace_max_lines or not line.strip():
in_stack_trace = False
for pattern in summary_patterns:
if pattern.search(line):
log_line.is_summary = True
break
log_line.is_summary = log_line.is_summary or i in short_summary_entries
log_line.score = self._score_line(log_line)
log_lines.append(log_line)
return log_lines
def _score_line(self, log_line: LogLine) -> float:
"""Per-line importance scoring."""
level_scores = {
LogLevel.ERROR: 1.0,
LogLevel.FAIL: 1.0,
LogLevel.WARN: 0.5,
LogLevel.INFO: 0.1,
LogLevel.DEBUG: 0.05,
LogLevel.TRACE: 0.02,
LogLevel.UNKNOWN: 0.1,
}
score = level_scores.get(log_line.level, 0.1)
if log_line.is_stack_trace:
score += 0.3
if log_line.is_summary:
score += 0.4
return min(1.0, score)
def _select_lines(self, log_lines: list[LogLine], bias: float = 1.0) -> list[LogLine]:
"""Select important lines using the same algorithm Rust uses."""
from headroom.transforms.adaptive_sizer import compute_optimal_k
all_strings = [line.content for line in log_lines]
short_summary_entries, totals_lines, first_header = _scan_pytest_short_summary(
[(line.line_number, line.content) for line in log_lines]
)
adaptive_max = compute_optimal_k(
all_strings, bias=bias, min_k=10, max_k=self.config.max_total_lines
)
errors: list[LogLine] = []
fails: list[LogLine] = []
warnings: list[LogLine] = []
stack_traces: list[list[LogLine]] = []
summaries: list[LogLine] = []
current_stack: list[LogLine] = []
for log_line in log_lines:
if log_line.level == LogLevel.ERROR:
errors.append(log_line)
elif log_line.level == LogLevel.FAIL:
fails.append(log_line)
elif log_line.level == LogLevel.WARN:
warnings.append(log_line)
if log_line.is_stack_trace:
current_stack.append(log_line)
elif current_stack:
stack_traces.append(current_stack)
current_stack = []
if log_line.is_summary:
summaries.append(log_line)
if current_stack:
stack_traces.append(current_stack)
selected: list[LogLine] = []
if errors:
selected.extend(self._select_with_first_last(errors, self.config.max_errors))
if fails:
selected.extend(self._select_with_first_last(fails, self.config.max_errors))
if warnings:
if self.config.dedupe_warnings:
warnings = self._dedupe_similar(warnings)
selected.extend(warnings[: self.config.max_warnings])
for stack in stack_traces[: self.config.max_stack_traces]:
selected.extend(stack[: self.config.stack_trace_max_lines])
if self.config.keep_summary_lines:
selected.extend(summaries)
selected = self._add_context(log_lines, selected)
selected = sorted(set(selected), key=lambda x: x.line_number)
if len(selected) > adaptive_max:
if self.config.keep_summary_lines:
reserved: set[int] = set()
if short_summary_entries:
reserved = _pytest_cap_reserve(log_lines, totals_lines, first_header)
present = {line.line_number for line in selected}
selected.extend(
line
for line in log_lines
if line.line_number in reserved and line.line_number not in present
)
selected = sorted(
selected,
key=lambda line: (
-(line.line_number in reserved),
-line.score,
-(line.line_number in short_summary_entries),
line.line_number,
),
)
else:
selected = sorted(selected, key=lambda line: (-line.score, line.line_number))
selected = selected[:adaptive_max]
selected = sorted(selected, key=lambda x: x.line_number)
return selected
def _select_with_first_last(self, lines: list[LogLine], max_count: int) -> list[LogLine]:
if len(lines) <= max_count:
return lines
selected: list[LogLine] = []
if self.config.keep_first_error and lines:
selected.append(lines[0])
if self.config.keep_last_error and lines and lines[-1] not in selected:
selected.append(lines[-1])
remaining = max_count - len(selected)
if remaining > 0:
candidates = sorted(
(line for line in lines if line not in selected),
key=lambda x: x.score,
reverse=True,
)
selected.extend(candidates[:remaining])
return selected
def _dedupe_similar(self, lines: list[LogLine]) -> list[LogLine]:
"""Conservative dedupe — preserves message prefix, only
normalises trailing variable region (digits, hex, paths).
Mirrors Rust `normalize_for_dedupe`."""
import re
seen: set[str] = set()
deduped: list[LogLine] = []
digit_re = re.compile(r"\d+")
hex_re = re.compile(r"0x[0-9a-fA-F]+")
path_re = re.compile(r"/[\w/]+/")
for line in lines:
content = line.content
split_at = next((i for i, c in enumerate(content) if c in (":", "=")), len(content))
prefix = content[:split_at]
suffix = content[split_at:]
suffix = digit_re.sub("N", suffix)
suffix = hex_re.sub("ADDR", suffix)
suffix = path_re.sub("/PATH/", suffix)
normalized = prefix + suffix
if normalized not in seen:
seen.add(normalized)
deduped.append(line)
return deduped
def _add_context(self, all_lines: list[LogLine], selected: list[LogLine]) -> list[LogLine]:
selected_indices = {line.line_number for line in selected}
context_indices: set[int] = set()
for idx in selected_indices:
for i in range(max(0, idx - self.config.error_context_lines), idx):
context_indices.add(i)
for i in range(
idx + 1,
min(len(all_lines), idx + self.config.error_context_lines + 1),
):
context_indices.add(i)
for idx in context_indices:
if idx not in selected_indices and idx < len(all_lines):
selected.append(all_lines[idx])
return selected
def _format_output(
self, selected: list[LogLine], all_lines: list[LogLine]
) -> tuple[str, dict[str, int]]:
short_summary_entries = _pytest_short_summary_entries(
[(line.line_number, line.content) for line in all_lines]
)
selected_numbers = {line.line_number for line in selected}
omitted_short_summary_ids = [
node_id
for line_number, node_id in short_summary_entries.items()
if line_number not in selected_numbers
]
stats: dict[str, int] = {
"errors": sum(1 for line in all_lines if line.level == LogLevel.ERROR),
"fails": sum(1 for line in all_lines if line.level == LogLevel.FAIL),
"warnings": sum(1 for line in all_lines if line.level == LogLevel.WARN),
"info": sum(1 for line in all_lines if line.level == LogLevel.INFO),
"total": len(all_lines),
"selected": len(selected),
}
output_lines = [line.content for line in selected]
omitted = len(all_lines) - len(selected)
if omitted > 0:
summary_parts: list[str] = []
for label, key in (
("ERROR", "errors"),
("FAIL", "fails"),
("WARN", "warnings"),
("INFO", "info"),
):
count = stats[key]
if count > 0:
summary_parts.append(f"{count} {label}")
if summary_parts:
omitted_names = ""
if omitted_short_summary_ids:
omitted_names = "; omitted: " + ", ".join(omitted_short_summary_ids[:5])
overflow = len(omitted_short_summary_ids) - 5
if overflow > 0:
omitted_names += f", +{overflow} more"
output_lines.append(
f"[{omitted} lines omitted: {', '.join(summary_parts)}{omitted_names}]"
)
return "\n".join(output_lines), stats
def _store_in_ccr(self, original: str, compressed: str, original_count: int) -> str | None:
"""Backwards-compat shim — the legacy callsite name. Now
delegates to `_persist_to_python_ccr`. Returns the stored
cache_key if persistence succeeded, else None.
"""
# Compute the same cache key the Rust path would (MD5 of
# original truncated to 24 hex chars).
import hashlib
cache_key = hashlib.md5(original.encode()).hexdigest()[:24]
try:
from ..cache.compression_store import get_compression_store
except ImportError as e:
logger.warning("CCR store import failed; cache_key %s not persisted: %s", cache_key, e)
return None
try:
store: Any = get_compression_store()
return cast(
"str | None",
store.store(original, compressed, original_item_count=original_count),
)
except Exception as e:
logger.warning("CCR store write failed; cache_key %s not persisted: %s", cache_key, e)
return None
def _persist_to_python_ccr(self, original: str, compressed: str, cache_key: str) -> None:
"""Promote a Rust-emitted cache_key into the production Python
CompressionStore. Failures are logged at warning level."""
+56 -213
View File
@@ -7,6 +7,7 @@ Tests cover:
4. Edge cases
"""
from headroom._core import detect_log_format
from headroom.transforms.log_compressor import (
LogCompressionResult,
LogCompressor,
@@ -17,6 +18,21 @@ from headroom.transforms.log_compressor import (
)
def _detect(lines: list[str]) -> LogFormat:
"""The Rust format detector that compress() uses."""
return LogFormat(detect_log_format(list(lines)))
_FILLER = [f"step {i} ok" for i in range(100)]
def _compress_amid_filler(lines: list[str], **config) -> LogCompressionResult:
"""Compress ``lines`` between 100 filler lines on each side, so selection
actually has to choose."""
content = "\n".join([*_FILLER, *lines, *_FILLER])
return LogCompressor(LogCompressorConfig(enable_ccr=False, **config)).compress(content)
class TestLogFormatDetection:
"""Tests for detecting different log formats."""
@@ -36,9 +52,8 @@ tests/test_foo.py::test_edge - AssertionError
FAILED tests/test_foo.py::test_edge
========================= 1 failed, 14 passed =========================
"""
compressor = LogCompressor()
lines = content.split("\n")
detected = compressor._detect_format(lines)
detected = _detect(lines)
assert detected == LogFormat.PYTEST
def test_detect_npm_format(self):
@@ -50,9 +65,8 @@ npm ERR! ERESOLVE unable to resolve dependency tree
npm info using npm@9.0.0
> added 150 packages in 5s
"""
compressor = LogCompressor()
lines = content.split("\n")
detected = compressor._detect_format(lines)
detected = _detect(lines)
assert detected == LogFormat.NPM
def test_detect_cargo_format(self):
@@ -70,9 +84,8 @@ error[E0382]: borrow of moved value: `s`
Finished dev [unoptimized + debuginfo] target(s) in 0.50s
Running `target/debug/myproject`
"""
compressor = LogCompressor()
lines = content.split("\n")
detected = compressor._detect_format(lines)
detected = _detect(lines)
assert detected == LogFormat.CARGO
def test_detect_make_format(self):
@@ -84,9 +97,8 @@ make[1]: *** [Makefile:10: utils.o] Error 1
make: *** [Makefile:5: all] Error 2
g++ -Wall -o program main.cpp utils.cpp
"""
compressor = LogCompressor()
lines = content.split("\n")
detected = compressor._detect_format(lines)
detected = _detect(lines)
assert detected == LogFormat.MAKE
def test_detect_jest_format(self):
@@ -96,9 +108,8 @@ FAIL src/utils/helpers.test.ts
Test Suites: 1 failed, 1 passed, 2 total
Tests: 2 failed, 10 passed, 12 total
"""
compressor = LogCompressor()
lines = content.split("\n")
detected = compressor._detect_format(lines)
detected = _detect(lines)
assert detected == LogFormat.JEST
def test_detect_generic_format(self):
@@ -109,238 +120,70 @@ WARNING Low memory
ERROR Connection timeout
CRITICAL System failure
"""
compressor = LogCompressor()
lines = content.split("\n")
detected = compressor._detect_format(lines)
detected = _detect(lines)
assert detected == LogFormat.GENERIC
def test_detect_empty_returns_generic(self):
"""Empty or minimal input returns GENERIC."""
compressor = LogCompressor()
assert compressor._detect_format([]) == LogFormat.GENERIC
assert compressor._detect_format(["random line"]) == LogFormat.GENERIC
assert _detect([]) == LogFormat.GENERIC
assert _detect(["random line"]) == LogFormat.GENERIC
class TestLogLevelDetection:
"""Tests for log level detection in lines."""
"""Level counts reported by compress(). DEBUG/TRACE/UNKNOWN and the full
keyword table are unit-tested in Rust (level_classifier_covers_every_level)."""
def test_detect_error_levels(self):
"""ERROR, FATAL, CRITICAL are detected."""
compressor = LogCompressor()
error_lines = [
def test_level_counts(self):
lines = [
"ERROR: something went wrong",
"error: file not found",
"Error: Invalid input",
"FATAL: system crash",
"fatal error occurred",
"CRITICAL: database down",
]
for line in error_lines:
log_lines = compressor._parse_lines([line])
assert log_lines[0].level == LogLevel.ERROR, f"Failed for: {line}"
def test_detect_fail_levels(self):
"""FAIL, FAILED are detected."""
compressor = LogCompressor()
fail_lines = [
"FAIL tests/test_foo.py",
"FAILED to connect",
"Test failed",
]
for line in fail_lines:
log_lines = compressor._parse_lines([line])
assert log_lines[0].level == LogLevel.FAIL, f"Failed for: {line}"
def test_detect_warn_levels(self):
"""WARN, WARNING are detected."""
compressor = LogCompressor()
warn_lines = [
"WARN: deprecated function",
"WARNING: low disk space",
"warning: unused variable",
"INFO: starting process",
"info starting",
"DEBUG: variable x = 5",
"TRACE: entering function",
]
for line in warn_lines:
log_lines = compressor._parse_lines([line])
assert log_lines[0].level == LogLevel.WARN, f"Failed for: {line}"
def test_detect_info_debug_trace(self):
"""INFO, DEBUG, TRACE are detected."""
compressor = LogCompressor()
test_cases = [
("INFO: starting process", LogLevel.INFO),
("info starting", LogLevel.INFO),
("DEBUG: variable x = 5", LogLevel.DEBUG),
("debug mode enabled", LogLevel.DEBUG),
("TRACE: entering function", LogLevel.TRACE),
]
for line, expected_level in test_cases:
log_lines = compressor._parse_lines([line])
assert log_lines[0].level == expected_level, f"Failed for: {line}"
def test_unknown_level_default(self):
"""Lines without level markers default to UNKNOWN."""
compressor = LogCompressor()
log_lines = compressor._parse_lines(["Just some regular text"])
assert log_lines[0].level == LogLevel.UNKNOWN
stats = _compress_amid_filler(lines).stats
assert stats["errors"] == 6
assert stats["fails"] == 3
assert stats["warnings"] == 3
assert stats["info"] == 2
class TestStackTraceDetection:
"""Tests for stack trace detection."""
"""Stack-trace lines survive compression. Python, Go, Rust-panic, .NET and
Java traces are unit-tested in Rust; these pin the JS and rustc shapes."""
def test_detect_python_traceback(self):
"""Python traceback is detected."""
content = """Traceback (most recent call last):
File "main.py", line 42, in process
result = compute(data)
File "utils.py", line 15, in compute
return data / 0
ZeroDivisionError: division by zero
"""
compressor = LogCompressor()
log_lines = compressor._parse_lines(content.split("\n"))
# First several lines should be marked as stack trace
stack_trace_count = sum(1 for line in log_lines if line.is_stack_trace)
assert stack_trace_count > 0
def test_detect_javascript_stack_trace(self):
"""JavaScript stack trace is detected."""
content = """Error: Connection failed
at Connection.connect (src/db.js:42:15)
at async main (src/index.js:10:5)
"""
compressor = LogCompressor()
log_lines = compressor._parse_lines(content.split("\n"))
stack_trace_count = sum(1 for line in log_lines if line.is_stack_trace)
assert stack_trace_count > 0
def test_detect_rust_error_location(self):
"""Rust error location is detected."""
content = """error[E0382]: borrow of moved value: `s`
--> src/main.rs:5:13
|
3 | let s = String::from("hello");
| - move occurs
"""
compressor = LogCompressor()
log_lines = compressor._parse_lines(content.split("\n"))
stack_trace_count = sum(1 for line in log_lines if line.is_stack_trace)
assert stack_trace_count > 0
class TestLineDeduplication:
"""Tests for warning/line deduplication."""
def test_dedupe_identical_warnings(self):
"""Identical warnings are deduplicated."""
compressor = LogCompressor()
lines = [
LogLine(line_number=1, content="WARNING: unused variable 'x'", level=LogLevel.WARN),
LogLine(line_number=2, content="WARNING: unused variable 'x'", level=LogLevel.WARN),
LogLine(line_number=3, content="WARNING: unused variable 'x'", level=LogLevel.WARN),
def test_javascript_stack_trace_kept(self):
trace = [
"Error: Connection failed",
" at Connection.connect (src/db.js:42:15)",
" at async main (src/index.js:10:5)",
]
# No context lines, so only stack-trace detection can keep the frames.
compressed = _compress_amid_filler(trace, error_context_lines=0).compressed
for line in trace:
assert line in compressed
deduped = compressor._dedupe_similar(lines)
assert len(deduped) == 1
def test_dedupe_similar_with_numbers(self):
"""Similar warnings with different numbers are deduplicated."""
compressor = LogCompressor()
lines = [
LogLine(line_number=1, content="WARNING: error at line 10", level=LogLevel.WARN),
LogLine(line_number=2, content="WARNING: error at line 20", level=LogLevel.WARN),
LogLine(line_number=3, content="WARNING: error at line 30", level=LogLevel.WARN),
def test_rust_error_location_kept(self):
trace = [
"error[E0382]: borrow of moved value: `s`",
" --> src/main.rs:5:13",
]
deduped = compressor._dedupe_similar(lines)
# Numbers normalized to "N", so all three are treated as identical pattern
assert len(deduped) == 1
def test_dedupe_similar_with_paths(self):
"""Similar warnings with different paths are deduplicated.
Note: The path regex /[\\w/]+/ requires paths to end with '/'.
Paths like '/path/to/' will be normalized, but '/path/to/file' won't
be fully normalized because 'file' doesn't end with '/'.
"""
compressor = LogCompressor()
# Paths ending with / are normalized
lines = [
LogLine(line_number=1, content="WARNING: in /path/to/ error", level=LogLevel.WARN),
LogLine(line_number=2, content="WARNING: in /other/dir/ error", level=LogLevel.WARN),
LogLine(line_number=3, content="WARNING: in /another/path/ error", level=LogLevel.WARN),
]
deduped = compressor._dedupe_similar(lines)
# Paths normalized to /PATH/, so all three are treated as identical pattern
assert len(deduped) == 1
def test_keeps_different_warnings(self):
"""Different warnings are preserved."""
compressor = LogCompressor()
lines = [
LogLine(line_number=1, content="WARNING: unused variable", level=LogLevel.WARN),
LogLine(line_number=2, content="WARNING: deprecated function", level=LogLevel.WARN),
LogLine(line_number=3, content="WARNING: missing docstring", level=LogLevel.WARN),
]
deduped = compressor._dedupe_similar(lines)
assert len(deduped) == 3
class TestLineScoring:
"""Tests for line importance scoring."""
def test_error_lines_score_highest(self):
"""ERROR and FAIL lines get highest scores."""
compressor = LogCompressor()
error_line = LogLine(line_number=1, content="ERROR: critical", level=LogLevel.ERROR)
fail_line = LogLine(line_number=2, content="FAILED test", level=LogLevel.FAIL)
info_line = LogLine(line_number=3, content="INFO: normal", level=LogLevel.INFO)
error_score = compressor._score_line(error_line)
fail_score = compressor._score_line(fail_line)
info_score = compressor._score_line(info_line)
assert error_score > info_score
assert fail_score > info_score
def test_stack_trace_boost(self):
"""Stack trace lines get boosted score."""
compressor = LogCompressor()
regular = LogLine(line_number=1, content="some line", level=LogLevel.UNKNOWN)
stack_trace = LogLine(
line_number=2, content=" File 'x.py'", level=LogLevel.UNKNOWN, is_stack_trace=True
)
assert compressor._score_line(stack_trace) > compressor._score_line(regular)
def test_summary_line_boost(self):
"""Summary lines get boosted score."""
compressor = LogCompressor()
regular = LogLine(line_number=1, content="some line", level=LogLevel.UNKNOWN)
summary = LogLine(
line_number=2, content="10 passed, 2 failed", level=LogLevel.UNKNOWN, is_summary=True
)
assert compressor._score_line(summary) > compressor._score_line(regular)
# No context lines, so only stack-trace detection can keep the frames.
compressed = _compress_amid_filler(trace, error_context_lines=0).compressed
for line in trace:
assert line in compressed
class TestCompressionBehavior:
+1 -336
View File
@@ -1,7 +1,5 @@
from __future__ import annotations
from types import SimpleNamespace
import pytest
from headroom.transforms.log_compressor import (
@@ -9,93 +7,9 @@ from headroom.transforms.log_compressor import (
LogCompressor,
LogCompressorConfig,
LogFormat,
LogLevel,
LogLine,
)
def test_detect_parse_and_score_log_lines() -> None:
compressor = LogCompressor(LogCompressorConfig(stack_trace_max_lines=2))
pytest_lines = [
"============================= test session starts =============================",
"collected 2 items",
"ERROR critical failure",
"Traceback (most recent call last)",
' File "app.py", line 10',
"",
"2 failed, 1 warning",
]
assert compressor._detect_format(pytest_lines) is LogFormat.PYTEST
assert compressor._detect_format(["npm ERR! missing script"]) is LogFormat.NPM
assert compressor._detect_format(["Compiling app", "warning: check this"]) is LogFormat.CARGO
assert (
compressor._detect_format(["PASS src/app.test.js", "Test Suites: 1 failed"])
is LogFormat.JEST
)
assert compressor._detect_format(["make: *** fail", "gcc -o app app.c"]) is LogFormat.MAKE
assert compressor._detect_format(["unclassified line"]) is LogFormat.GENERIC
parsed = compressor._parse_lines(pytest_lines)
assert parsed[0].is_summary is True
assert parsed[2].level is LogLevel.ERROR
assert parsed[3].is_stack_trace is True
assert parsed[4].is_stack_trace is True
assert parsed[6].is_summary is True
assert compressor._score_line(LogLine(1, "warn", level=LogLevel.WARN)) == 0.5
assert (
compressor._score_line(
LogLine(2, "error summary", level=LogLevel.ERROR, is_stack_trace=True, is_summary=True)
)
== 1.0
)
def test_pytest_short_summary_helper_recognizes_only_section_entries() -> None:
compressor = LogCompressor()
lines = [
"FAILED outside.py::test_before",
"=== short test summary info ===",
"FAILED tests/test_a.py::test_short",
"ERROR tests/test_b.py::test_long - RuntimeError: boom",
"=== 2 failed in 0.1s ===",
"FAILED outside.py::test_after",
"=== short test summary info ===",
"FAILED tests/test_c.py::test_eof - AssertionError",
]
parsed = compressor._parse_lines(lines)
assert parsed[0].is_summary is False
assert parsed[2].is_summary is True
assert parsed[3].is_summary is True
assert parsed[5].is_summary is False
assert parsed[7].is_summary is True
def test_pytest_short_summary_helper_accepts_crlf_complete_separators() -> None:
compressor = LogCompressor()
lines = [
"=== diagnostic: short test summary info unavailable\r",
"FAILED outside.py::test_before\r",
"=== short test summary info ===\r",
"FAILED tests/test_a.py::test_short\r",
"ERROR tests/test_b.py::test_long - RuntimeError: boom\r",
"=== 2 failed in 0.1s ===\r",
"FAILED outside.py::test_after\r",
"=== short test summary info ===\r",
"FAILED tests/test_c.py::test_eof - AssertionError\r",
]
parsed = compressor._parse_lines(lines)
assert parsed[1].is_summary is False
assert parsed[3].is_summary is True
assert parsed[4].is_summary is True
assert parsed[6].is_summary is False
assert parsed[8].is_summary is True
assert parsed[3].content == lines[3]
def _malformed_short_summary_diagnostic_log() -> list[str]:
lines = [f"INFO setup output {i}" for i in range(50)]
lines.append("=== diagnostic: short test summary info unavailable")
@@ -104,233 +18,6 @@ def _malformed_short_summary_diagnostic_log() -> list[str]:
return lines
@pytest.mark.parametrize("keep_summary_lines", [False, True])
def test_malformed_short_summary_diagnostic_does_not_affect_legacy_helpers(
monkeypatch: pytest.MonkeyPatch, keep_summary_lines: bool
) -> None:
compressor = LogCompressor(
LogCompressorConfig(
max_errors=2,
error_context_lines=0,
keep_summary_lines=keep_summary_lines,
max_total_lines=1_000,
enable_ccr=False,
)
)
monkeypatch.setitem(
__import__("sys").modules,
"headroom.transforms.adaptive_sizer",
SimpleNamespace(compute_optimal_k=lambda items, **kwargs: 1_000),
)
parsed = compressor._parse_lines(_malformed_short_summary_diagnostic_log())
assert len(parsed) == 71
assert all(not line.is_summary for line in parsed[51:])
selected = compressor._select_lines(parsed)
selected_lookalikes = [
line.line_number for line in selected if line.level in (LogLevel.ERROR, LogLevel.FAIL)
]
assert selected_lookalikes == [51, 60, 61, 70]
output, _ = compressor._format_output(selected, parsed)
assert "; omitted: " not in output
if not keep_summary_lines:
assert output == (
"FAILED outside.py::test_0\n"
"FAILED outside.py::test_9\n"
"ERROR outside.py::test_10\n"
"ERROR outside.py::test_19\n"
"[67 lines omitted: 10 ERROR, 10 FAIL, 51 INFO]"
)
def test_select_dedupe_add_context_and_format_output(monkeypatch: pytest.MonkeyPatch) -> None:
compressor = LogCompressor(
LogCompressorConfig(
max_errors=2,
max_warnings=1,
error_context_lines=1,
max_stack_traces=1,
stack_trace_max_lines=2,
)
)
monkeypatch.setitem(
__import__("sys").modules,
"headroom.transforms.adaptive_sizer",
SimpleNamespace(compute_optimal_k=lambda items, **kwargs: 6),
)
log_lines = [
LogLine(0, "info line", level=LogLevel.INFO, score=0.1),
LogLine(1, "ERROR first", level=LogLevel.ERROR, score=1.0),
LogLine(2, "context after first", level=LogLevel.UNKNOWN, score=0.1),
LogLine(3, "WARNING /tmp/a/123 issue", level=LogLevel.WARN, score=0.5),
LogLine(4, "WARNING /tmp/b/999 issue", level=LogLevel.WARN, score=0.5),
LogLine(5, "FAIL final", level=LogLevel.FAIL, score=1.0),
LogLine(6, "Traceback (most recent call last)", is_stack_trace=True, score=0.4),
LogLine(7, ' File "app.py", line 2', is_stack_trace=True, score=0.4),
LogLine(8, "1 failed, 1 warning", is_summary=True, score=0.5),
]
selected = compressor._select_lines(log_lines)
assert [line.line_number for line in selected] == [1, 3, 4, 5, 6, 8]
assert compressor._select_with_first_last(log_lines[:2], max_count=5) == log_lines[:2]
many_errors = [
LogLine(10, "first", level=LogLevel.ERROR, score=0.1),
LogLine(11, "mid", level=LogLevel.ERROR, score=0.9),
LogLine(12, "last", level=LogLevel.ERROR, score=0.2),
]
trimmed = compressor._select_with_first_last(many_errors, max_count=2)
assert trimmed == [many_errors[0], many_errors[2]]
# fixed_in_3e5: conservative dedupe preserves message prefix (everything
# before the first `:` or `=`), so warnings without a colon keep their
# full content as the dedupe key. The two lines below have different
# paths/numbers and no `:`, so they DON'T collapse anymore — Python's
# pre-3e5 aggressive normalization treated them as duplicates, masking
# distinct error categories.
distinct = compressor._dedupe_similar(log_lines[3:5])
assert len(distinct) == 2
# Same dedupe IS triggered when the prefix matches (lines have a colon).
similar = compressor._dedupe_similar(
[
LogLine(20, "warning: file /tmp/a/123 issue", level=LogLevel.WARN),
LogLine(21, "warning: file /tmp/b/999 issue", level=LogLevel.WARN),
]
)
assert len(similar) == 1
output, stats = compressor._format_output(selected, log_lines)
assert stats == {
"errors": 1,
"fails": 1,
"warnings": 2,
"info": 1,
"total": 9,
"selected": 6,
}
assert output.endswith("[3 lines omitted: 1 ERROR, 1 FAIL, 2 WARN, 1 INFO]")
@pytest.mark.parametrize(
("keep_summary_lines", "expected"),
# True: the totals line (7) is reserved, and entries win the remaining ties.
[(True, [4, 5, 7]), (False, [0, 1, 4])],
)
def test_short_summary_global_cap_precedence_matches_rust(
monkeypatch: pytest.MonkeyPatch, keep_summary_lines: bool, expected: list[int]
) -> None:
compressor = LogCompressor(
LogCompressorConfig(
max_errors=10,
error_context_lines=0,
keep_summary_lines=keep_summary_lines,
max_total_lines=3,
)
)
monkeypatch.setitem(
__import__("sys").modules,
"headroom.transforms.adaptive_sizer",
SimpleNamespace(compute_optimal_k=lambda items, **kwargs: 3),
)
parsed = compressor._parse_lines(
[
"FAILED traceback.py::test_early_one\r",
"FAILED traceback.py::test_early_two\r",
"ordinary output\r",
"=== short test summary info ===\r",
"FAILED tests/test_summary.py::test_one\r",
"ERROR tests/test_summary.py::test_two\r",
"FAILED tests/test_summary.py::test_three\r",
"=== 3 failed in 0.1s ===\r",
]
)
assert [line.line_number for line in compressor._select_lines(parsed)] == expected
def test_legacy_cap_reserves_totals_and_first_error_line(monkeypatch: pytest.MonkeyPatch) -> None:
compressor = LogCompressor(
LogCompressorConfig(max_errors=10, error_context_lines=0, max_total_lines=4)
)
monkeypatch.setitem(
__import__("sys").modules,
"headroom.transforms.adaptive_sizer",
SimpleNamespace(compute_optimal_k=lambda items, **kwargs: 4),
)
parsed = compressor._parse_lines(
[
"E AssertionError: first",
"E AssertionError: second",
"=== short test summary info ===",
"FAILED t.py::a",
"FAILED t.py::b",
"FAILED t.py::c",
"FAILED t.py::d",
"=== 4 failed in 0.1s ===",
]
)
# The first E line (0) and the totals line (7) are reserved; entries fill the rest.
assert [line.line_number for line in compressor._select_lines(parsed)] == [0, 3, 4, 7]
@pytest.mark.parametrize(
("count", "suffix"),
[
(
3,
"; omitted: tests/test_ids.py::test_0, tests/test_ids.py::test_1, "
"tests/test_ids.py::test_2",
),
(
5,
"; omitted: tests/test_ids.py::test_0, tests/test_ids.py::test_1, "
"tests/test_ids.py::test_2, tests/test_ids.py::test_3, tests/test_ids.py::test_4",
),
(
7,
"; omitted: tests/test_ids.py::test_0, tests/test_ids.py::test_1, "
"tests/test_ids.py::test_2, tests/test_ids.py::test_3, tests/test_ids.py::test_4, "
"+2 more",
),
],
)
def test_short_summary_omission_naming_is_bounded(count: int, suffix: str) -> None:
compressor = LogCompressor(LogCompressorConfig(keep_summary_lines=False))
contents = ["=== short test summary info ===\r"] + [
f"FAILED tests/test_ids.py::test_{i}\r" for i in range(count)
]
contents.append("=== failures complete ===\r")
all_lines = compressor._parse_lines(contents)
selected = [all_lines[0], all_lines[-1]]
output, _ = compressor._format_output(selected, all_lines)
assert output.endswith(f"[{count} lines omitted: {count} FAIL, 1 INFO{suffix}]")
def test_short_summary_omission_naming_uses_ids_and_preserves_duplicates() -> None:
compressor = LogCompressor(LogCompressorConfig(keep_summary_lines=False))
all_lines = compressor._parse_lines(
[
"=== short test summary info ===",
"ERROR tests/test_ids.py::test_error - RuntimeError: boom",
"FAILED tests/test_ids.py::test_repeat",
"FAILED tests/test_ids.py::test_repeat - AssertionError",
"FAILED tests/test_ids.py::test_kept",
"=== failures complete ===",
]
)
output, stats = compressor._format_output([all_lines[0], all_lines[4], all_lines[5]], all_lines)
assert output.endswith(
"[3 lines omitted: 1 ERROR, 3 FAIL, 1 INFO; omitted: tests/test_ids.py::test_error, "
"tests/test_ids.py::test_repeat, tests/test_ids.py::test_repeat]"
)
assert stats["errors"] == 1
assert stats["fails"] == 3
def test_log_compressor_compress_and_ccr_paths() -> None:
"""Phase 3e.5: `compress()` is now a single Rust call, so this test
exercises end-to-end behavior instead of monkeypatching internal
@@ -502,29 +189,7 @@ def test_real_compress_ignores_malformed_short_summary_diagnostic() -> None:
)
def test_store_in_ccr_and_result_properties(monkeypatch: pytest.MonkeyPatch) -> None:
compressor = LogCompressor()
monkeypatch.setitem(
__import__("sys").modules,
"headroom.cache.compression_store",
SimpleNamespace(
get_compression_store=lambda: SimpleNamespace(
store=lambda original, compressed, original_item_count=0: "stored-log"
)
),
)
assert compressor._store_in_ccr("orig", "comp", 10) == "stored-log"
def broken_store():
raise RuntimeError("boom")
monkeypatch.setitem(
__import__("sys").modules,
"headroom.cache.compression_store",
SimpleNamespace(get_compression_store=broken_store),
)
assert compressor._store_in_ccr("orig", "comp", 10) is None
def test_result_properties() -> None:
result = LogCompressionResult(
compressed="small",
original="this is a substantially longer log body",
-25
View File
@@ -116,31 +116,6 @@ def test_node_async_frames_count_toward_log_detection() -> None:
assert result.content_type is ContentType.BUILD_OUTPUT
# Shim pattern mirror ----------------------------------------------------------
def test_parse_lines_marks_new_flavor_openers() -> None:
compressor = LogCompressor()
lines = [
"panic: boom",
"goroutine 7 [select]:",
"\t/app/main.go:10 +0x20",
"thread 'main' panicked at src/lib.rs:1:1:",
"stack backtrace:",
" 0: rust_begin_unwind",
"Unhandled exception. System.Exception: x",
" at App.Main(String[] a) in /src/P.cs:line 3",
"Caused by: java.io.IOException: y",
" ... 3 more",
"", # blank resets the shim's legacy in-trace latch
"plain line",
]
parsed = compressor._parse_lines(lines)
flags = [ln.is_stack_trace for ln in parsed]
assert all(flags[:-2]), f"unmarked opener among {flags}"
assert not flags[-1]
# End-to-end compression --------------------------------------------------------