mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
refactor(transforms): drop the Python mirror of the Rust log selector
LogCompressor.compress() runs entirely in Rust. The class still carried a Python re-implementation of the line parser, scorer, selector, deduper, context adder and formatter, plus the pytest short-summary helpers and a _store_in_ccr shim, kept only so old tests could call them. Production never ran them, and _store_in_ccr still had the #816 bug that the live path fixed (no explicit_hash). Removes those private helpers. Tests now target what ships: - format detection tests call headroom._core.detect_log_format, which compress() uses; - level counts and the JS/rustc stack-trace shapes become compress()-level tests; - two Rust unit tests are added for behaviour no Rust test pinned yet: the full level keyword table (FAIL, DEBUG, TRACE, unknown) and score ordering/boosts; - helper tests whose behaviour Rust already unit-tests (dedupe, pytest short-summary recognition, cap precedence and reservation, omission naming, new trace flavours) are deleted. adaptive_sizer.py stays: main's SearchCompressor helpers still import it until the search-mirror PR lands. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a9815bbb5f
commit
b82ef1b73a
@@ -1780,6 +1780,54 @@ mod tests {
|
||||
assert_eq!(lines[2].level, LogLevel::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn level_classifier_covers_every_level() {
|
||||
let c = cmp();
|
||||
let cases: &[(&str, LogLevel)] = &[
|
||||
("ERROR: something went wrong", LogLevel::Error),
|
||||
("error: file not found", LogLevel::Error),
|
||||
("Error: Invalid input", LogLevel::Error),
|
||||
("FATAL: system crash", LogLevel::Error),
|
||||
("fatal error occurred", LogLevel::Error),
|
||||
("CRITICAL: database down", LogLevel::Error),
|
||||
("FAIL tests/test_foo.py", LogLevel::Fail),
|
||||
("FAILED to connect", LogLevel::Fail),
|
||||
("Test failed", LogLevel::Fail),
|
||||
("WARN: deprecated function", LogLevel::Warn),
|
||||
("WARNING: low disk space", LogLevel::Warn),
|
||||
("warning: unused variable", LogLevel::Warn),
|
||||
("INFO: starting process", LogLevel::Info),
|
||||
("info starting", LogLevel::Info),
|
||||
("DEBUG: variable x = 5", LogLevel::Debug),
|
||||
("debug mode enabled", LogLevel::Debug),
|
||||
("TRACE: entering function", LogLevel::Trace),
|
||||
("Just some regular text", LogLevel::Unknown),
|
||||
];
|
||||
for (line, expected) in cases {
|
||||
let parsed = c.parse_lines(&[line]);
|
||||
assert_eq!(parsed[0].level, *expected, "level for {line:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn score_line_orders_levels_and_boosts() {
|
||||
let line = |level: LogLevel, is_stack_trace: bool, is_summary: bool| LogLine {
|
||||
line_number: 0,
|
||||
content: String::new(),
|
||||
level,
|
||||
is_stack_trace,
|
||||
is_summary,
|
||||
score: 0.0,
|
||||
};
|
||||
let info = score_log_line(&line(LogLevel::Info, false, false));
|
||||
assert!(score_log_line(&line(LogLevel::Error, false, false)) > info);
|
||||
assert!(score_log_line(&line(LogLevel::Fail, false, false)) > info);
|
||||
assert_eq!(score_log_line(&line(LogLevel::Warn, false, false)), 0.5);
|
||||
let plain = score_log_line(&line(LogLevel::Unknown, false, false));
|
||||
assert!(score_log_line(&line(LogLevel::Unknown, true, false)) > plain);
|
||||
assert!(score_log_line(&line(LogLevel::Unknown, false, true)) > plain);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recognizes_pytest_short_summary_entries_by_section_and_line_number() {
|
||||
let lines = [
|
||||
|
||||
@@ -11,11 +11,6 @@ is now a thin shim that:
|
||||
2. Routes `LogCompressor.compress()` entirely through the Rust
|
||||
implementation, picking up the bug fixes (chained-exception trace
|
||||
survival, conservative warning dedupe, loud CCR failures).
|
||||
3. Implements legacy internal helpers (`_detect_format`, `_parse_lines`,
|
||||
`_score_line`, `_select_lines`, `_select_with_first_last`,
|
||||
`_dedupe_similar`, `_format_output`) on top of the Rust building
|
||||
blocks where a Rust delegation makes sense; otherwise keeps Python
|
||||
logic that mirrors Rust scoring.
|
||||
|
||||
# Bug fixes the Rust port carries (and this shim therefore inherits)
|
||||
|
||||
@@ -93,89 +88,6 @@ class LogLine:
|
||||
return hash(self.line_number)
|
||||
|
||||
|
||||
def _scan_pytest_short_summary(
|
||||
lines: list[tuple[int, str]],
|
||||
) -> tuple[dict[int, str], set[int], int | None]:
|
||||
"""Scan for pytest's "short test summary info" sections.
|
||||
|
||||
Mirrors Rust's ``scan_pytest_short_summary``: a section opens on a
|
||||
complete ``=== ... short test summary info ... ===`` separator (a trailing
|
||||
``\\r`` is ignored) and closes at the next ``===`` line or EOF. For
|
||||
``FAILED <nodeid> - <msg>`` the id is the text before ``" - "``.
|
||||
|
||||
Args:
|
||||
lines: ``(line_number, content)`` pairs in log order.
|
||||
|
||||
Returns:
|
||||
``(entries, totals_lines, first_header)``: line number -> node id for
|
||||
each entry, the line numbers of the ``===`` lines closing a section
|
||||
(pytest's run totals), and the first section header's line number.
|
||||
"""
|
||||
entries: dict[int, str] = {}
|
||||
totals_lines: set[int] = set()
|
||||
first_header: int | None = None
|
||||
in_short_summary = False
|
||||
|
||||
for line_number, line in lines:
|
||||
recognition_line = line.removesuffix("\r")
|
||||
if recognition_line.startswith("==="):
|
||||
opens = (
|
||||
recognition_line.endswith("===") and "short test summary info" in recognition_line
|
||||
)
|
||||
if opens:
|
||||
if first_header is None:
|
||||
first_header = line_number
|
||||
elif in_short_summary:
|
||||
totals_lines.add(line_number)
|
||||
in_short_summary = opens
|
||||
continue
|
||||
if not in_short_summary:
|
||||
continue
|
||||
|
||||
rest = next(
|
||||
(
|
||||
line.removeprefix(prefix)
|
||||
for prefix in ("FAILED ", "ERROR ")
|
||||
if line.startswith(prefix)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if rest is not None:
|
||||
node_id = rest.split(" - ", 1)[0].strip()
|
||||
if node_id:
|
||||
entries[line_number] = node_id
|
||||
|
||||
return entries, totals_lines, first_header
|
||||
|
||||
|
||||
def _pytest_short_summary_entries(lines: list[tuple[int, str]]) -> dict[int, str]:
|
||||
"""Map line numbers to node ids of pytest short-summary FAILED/ERROR entries.
|
||||
|
||||
See ``_scan_pytest_short_summary``.
|
||||
"""
|
||||
return _scan_pytest_short_summary(lines)[0]
|
||||
|
||||
|
||||
def _pytest_cap_reserve(
|
||||
log_lines: list[LogLine], totals_lines: set[int], first_header: int | None
|
||||
) -> set[int]:
|
||||
"""Return the lines the global cap must keep while a short summary competes.
|
||||
|
||||
Mirrors Rust's ``pytest_cap_reserve``: each section's totals line and the
|
||||
first ``E `` assertion line before the summary. Short-summary entries win
|
||||
equal-score ties, so without this reserve a long summary crowds out the run
|
||||
totals and every error message.
|
||||
"""
|
||||
reserved = set(totals_lines)
|
||||
for line in log_lines:
|
||||
if first_header is not None and line.line_number >= first_header:
|
||||
break
|
||||
if line.content.startswith("E "):
|
||||
reserved.add(line.line_number)
|
||||
break
|
||||
return reserved
|
||||
|
||||
|
||||
@dataclass
|
||||
class LogCompressorConfig:
|
||||
"""Configuration for log compression."""
|
||||
@@ -241,9 +153,7 @@ class LogCompressor:
|
||||
"""Rust-backed log compressor.
|
||||
|
||||
Drop-in replacement for the retired Python class. `compress()`
|
||||
delegates to Rust end-to-end; internal helpers used by the
|
||||
existing test surface keep working but route through the same
|
||||
Rust building blocks where they exist.
|
||||
delegates to Rust end-to-end.
|
||||
"""
|
||||
|
||||
def __init__(self, config: LogCompressorConfig | None = None) -> None:
|
||||
@@ -305,337 +215,6 @@ class LogCompressor:
|
||||
stats=stats_dict,
|
||||
)
|
||||
|
||||
# ─── Legacy internal helpers (test surface compat) ──────────────────
|
||||
|
||||
def _detect_format(self, lines: list[str]) -> LogFormat:
|
||||
"""Delegate to the Rust format detector."""
|
||||
from headroom._core import detect_log_format
|
||||
|
||||
return _format_from_str(detect_log_format(list(lines)))
|
||||
|
||||
def _parse_lines(self, lines: list[str]) -> list[LogLine]:
|
||||
"""Parse + categorize lines, mirroring Rust's classification.
|
||||
|
||||
Stays Python so the legacy direct-call test surface keeps
|
||||
working without rebuilding through Rust on every test. Rust
|
||||
unit tests pin Rust's behavior; this implementation must
|
||||
mirror Rust's level/stack-trace/summary classification rules.
|
||||
"""
|
||||
import re
|
||||
|
||||
# Mirror of Rust's level classifier: aho-corasick with
|
||||
# word-boundary post-filter. Python's `re` is fast enough for
|
||||
# the test path; the Rust path uses aho-corasick. Both share
|
||||
# the same keyword set.
|
||||
level_patterns = [
|
||||
(
|
||||
LogLevel.ERROR,
|
||||
re.compile(r"\b(?:ERROR|error|Error|FATAL|fatal|Fatal|CRITICAL|critical)\b"),
|
||||
),
|
||||
(LogLevel.FAIL, re.compile(r"\b(?:FAIL|FAILED|fail|failed|Fail|Failed)\b")),
|
||||
(LogLevel.WARN, re.compile(r"\b(?:WARN|WARNING|warn|warning|Warn|Warning)\b")),
|
||||
(LogLevel.INFO, re.compile(r"\b(?:INFO|info|Info)\b")),
|
||||
(LogLevel.DEBUG, re.compile(r"\b(?:DEBUG|debug|Debug)\b")),
|
||||
(LogLevel.TRACE, re.compile(r"\b(?:TRACE|trace|Trace)\b")),
|
||||
]
|
||||
stack_trace_patterns = [
|
||||
re.compile(r"^\s*Traceback \(most recent call last\)"),
|
||||
re.compile(r'^\s*File ".+", line \d+'),
|
||||
re.compile(r"^\s*at .+\(.+:\d+:\d+\)"),
|
||||
re.compile(r"^\s+at [\w.$]+\("),
|
||||
re.compile(r"^\s*--> .+:\d+:\d+"),
|
||||
re.compile(r"^\s*\d+:\s+0x[0-9a-f]+"),
|
||||
# Rust panics (RustBacktrace flavor)
|
||||
re.compile(r"^thread '[^']*' panicked at"),
|
||||
re.compile(r"^stack backtrace:"),
|
||||
re.compile(r"^\s+\d+: \S"),
|
||||
# Go panics / goroutine dumps (GoPanic flavor)
|
||||
re.compile(r"^(?:panic|fatal error): "),
|
||||
re.compile(r"^goroutine \d+ \["),
|
||||
re.compile(r"^\t\S+\.go:\d+(?: \+0x[0-9a-f]+)?$"),
|
||||
# .NET (DotNet flavor)
|
||||
re.compile(r"^Unhandled exception\."),
|
||||
re.compile(r"^\s*at .+\) in .+:line \d+"),
|
||||
# Java chained-exception continuations
|
||||
re.compile(r"^Caused by: "),
|
||||
re.compile(r"^\s*\.\.\. \d+ more$"),
|
||||
]
|
||||
summary_patterns = [
|
||||
re.compile(r"^={3,}"),
|
||||
re.compile(r"^-{3,}"),
|
||||
re.compile(r"^\d+ (passed|failed|skipped|error|warning)"),
|
||||
re.compile(r"^(?:Tests?|Suites?):?\s+\d+"),
|
||||
re.compile(r"^(?:TOTAL|Total|Summary)"),
|
||||
re.compile(r"^(?:Build|Compile|Test).*(?:succeeded|failed|complete)"),
|
||||
]
|
||||
|
||||
log_lines: list[LogLine] = []
|
||||
in_stack_trace = False
|
||||
stack_trace_lines = 0
|
||||
short_summary_entries = _pytest_short_summary_entries(list(enumerate(lines)))
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
log_line = LogLine(line_number=i, content=line)
|
||||
|
||||
for level, pattern in level_patterns:
|
||||
if pattern.search(line):
|
||||
log_line.level = level
|
||||
break
|
||||
|
||||
for pattern in stack_trace_patterns:
|
||||
if pattern.search(line):
|
||||
in_stack_trace = True
|
||||
stack_trace_lines = 0
|
||||
break
|
||||
|
||||
if in_stack_trace:
|
||||
log_line.is_stack_trace = True
|
||||
stack_trace_lines += 1
|
||||
if stack_trace_lines > self.config.stack_trace_max_lines or not line.strip():
|
||||
in_stack_trace = False
|
||||
|
||||
for pattern in summary_patterns:
|
||||
if pattern.search(line):
|
||||
log_line.is_summary = True
|
||||
break
|
||||
log_line.is_summary = log_line.is_summary or i in short_summary_entries
|
||||
|
||||
log_line.score = self._score_line(log_line)
|
||||
log_lines.append(log_line)
|
||||
|
||||
return log_lines
|
||||
|
||||
def _score_line(self, log_line: LogLine) -> float:
|
||||
"""Per-line importance scoring."""
|
||||
level_scores = {
|
||||
LogLevel.ERROR: 1.0,
|
||||
LogLevel.FAIL: 1.0,
|
||||
LogLevel.WARN: 0.5,
|
||||
LogLevel.INFO: 0.1,
|
||||
LogLevel.DEBUG: 0.05,
|
||||
LogLevel.TRACE: 0.02,
|
||||
LogLevel.UNKNOWN: 0.1,
|
||||
}
|
||||
score = level_scores.get(log_line.level, 0.1)
|
||||
if log_line.is_stack_trace:
|
||||
score += 0.3
|
||||
if log_line.is_summary:
|
||||
score += 0.4
|
||||
return min(1.0, score)
|
||||
|
||||
def _select_lines(self, log_lines: list[LogLine], bias: float = 1.0) -> list[LogLine]:
|
||||
"""Select important lines using the same algorithm Rust uses."""
|
||||
from headroom.transforms.adaptive_sizer import compute_optimal_k
|
||||
|
||||
all_strings = [line.content for line in log_lines]
|
||||
short_summary_entries, totals_lines, first_header = _scan_pytest_short_summary(
|
||||
[(line.line_number, line.content) for line in log_lines]
|
||||
)
|
||||
adaptive_max = compute_optimal_k(
|
||||
all_strings, bias=bias, min_k=10, max_k=self.config.max_total_lines
|
||||
)
|
||||
|
||||
errors: list[LogLine] = []
|
||||
fails: list[LogLine] = []
|
||||
warnings: list[LogLine] = []
|
||||
stack_traces: list[list[LogLine]] = []
|
||||
summaries: list[LogLine] = []
|
||||
current_stack: list[LogLine] = []
|
||||
|
||||
for log_line in log_lines:
|
||||
if log_line.level == LogLevel.ERROR:
|
||||
errors.append(log_line)
|
||||
elif log_line.level == LogLevel.FAIL:
|
||||
fails.append(log_line)
|
||||
elif log_line.level == LogLevel.WARN:
|
||||
warnings.append(log_line)
|
||||
if log_line.is_stack_trace:
|
||||
current_stack.append(log_line)
|
||||
elif current_stack:
|
||||
stack_traces.append(current_stack)
|
||||
current_stack = []
|
||||
if log_line.is_summary:
|
||||
summaries.append(log_line)
|
||||
if current_stack:
|
||||
stack_traces.append(current_stack)
|
||||
|
||||
selected: list[LogLine] = []
|
||||
if errors:
|
||||
selected.extend(self._select_with_first_last(errors, self.config.max_errors))
|
||||
if fails:
|
||||
selected.extend(self._select_with_first_last(fails, self.config.max_errors))
|
||||
if warnings:
|
||||
if self.config.dedupe_warnings:
|
||||
warnings = self._dedupe_similar(warnings)
|
||||
selected.extend(warnings[: self.config.max_warnings])
|
||||
for stack in stack_traces[: self.config.max_stack_traces]:
|
||||
selected.extend(stack[: self.config.stack_trace_max_lines])
|
||||
if self.config.keep_summary_lines:
|
||||
selected.extend(summaries)
|
||||
|
||||
selected = self._add_context(log_lines, selected)
|
||||
selected = sorted(set(selected), key=lambda x: x.line_number)
|
||||
|
||||
if len(selected) > adaptive_max:
|
||||
if self.config.keep_summary_lines:
|
||||
reserved: set[int] = set()
|
||||
if short_summary_entries:
|
||||
reserved = _pytest_cap_reserve(log_lines, totals_lines, first_header)
|
||||
present = {line.line_number for line in selected}
|
||||
selected.extend(
|
||||
line
|
||||
for line in log_lines
|
||||
if line.line_number in reserved and line.line_number not in present
|
||||
)
|
||||
selected = sorted(
|
||||
selected,
|
||||
key=lambda line: (
|
||||
-(line.line_number in reserved),
|
||||
-line.score,
|
||||
-(line.line_number in short_summary_entries),
|
||||
line.line_number,
|
||||
),
|
||||
)
|
||||
else:
|
||||
selected = sorted(selected, key=lambda line: (-line.score, line.line_number))
|
||||
selected = selected[:adaptive_max]
|
||||
selected = sorted(selected, key=lambda x: x.line_number)
|
||||
|
||||
return selected
|
||||
|
||||
def _select_with_first_last(self, lines: list[LogLine], max_count: int) -> list[LogLine]:
|
||||
if len(lines) <= max_count:
|
||||
return lines
|
||||
|
||||
selected: list[LogLine] = []
|
||||
if self.config.keep_first_error and lines:
|
||||
selected.append(lines[0])
|
||||
if self.config.keep_last_error and lines and lines[-1] not in selected:
|
||||
selected.append(lines[-1])
|
||||
|
||||
remaining = max_count - len(selected)
|
||||
if remaining > 0:
|
||||
candidates = sorted(
|
||||
(line for line in lines if line not in selected),
|
||||
key=lambda x: x.score,
|
||||
reverse=True,
|
||||
)
|
||||
selected.extend(candidates[:remaining])
|
||||
|
||||
return selected
|
||||
|
||||
def _dedupe_similar(self, lines: list[LogLine]) -> list[LogLine]:
|
||||
"""Conservative dedupe — preserves message prefix, only
|
||||
normalises trailing variable region (digits, hex, paths).
|
||||
Mirrors Rust `normalize_for_dedupe`."""
|
||||
import re
|
||||
|
||||
seen: set[str] = set()
|
||||
deduped: list[LogLine] = []
|
||||
digit_re = re.compile(r"\d+")
|
||||
hex_re = re.compile(r"0x[0-9a-fA-F]+")
|
||||
path_re = re.compile(r"/[\w/]+/")
|
||||
|
||||
for line in lines:
|
||||
content = line.content
|
||||
split_at = next((i for i, c in enumerate(content) if c in (":", "=")), len(content))
|
||||
prefix = content[:split_at]
|
||||
suffix = content[split_at:]
|
||||
suffix = digit_re.sub("N", suffix)
|
||||
suffix = hex_re.sub("ADDR", suffix)
|
||||
suffix = path_re.sub("/PATH/", suffix)
|
||||
normalized = prefix + suffix
|
||||
if normalized not in seen:
|
||||
seen.add(normalized)
|
||||
deduped.append(line)
|
||||
return deduped
|
||||
|
||||
def _add_context(self, all_lines: list[LogLine], selected: list[LogLine]) -> list[LogLine]:
|
||||
selected_indices = {line.line_number for line in selected}
|
||||
context_indices: set[int] = set()
|
||||
for idx in selected_indices:
|
||||
for i in range(max(0, idx - self.config.error_context_lines), idx):
|
||||
context_indices.add(i)
|
||||
for i in range(
|
||||
idx + 1,
|
||||
min(len(all_lines), idx + self.config.error_context_lines + 1),
|
||||
):
|
||||
context_indices.add(i)
|
||||
for idx in context_indices:
|
||||
if idx not in selected_indices and idx < len(all_lines):
|
||||
selected.append(all_lines[idx])
|
||||
return selected
|
||||
|
||||
def _format_output(
|
||||
self, selected: list[LogLine], all_lines: list[LogLine]
|
||||
) -> tuple[str, dict[str, int]]:
|
||||
short_summary_entries = _pytest_short_summary_entries(
|
||||
[(line.line_number, line.content) for line in all_lines]
|
||||
)
|
||||
selected_numbers = {line.line_number for line in selected}
|
||||
omitted_short_summary_ids = [
|
||||
node_id
|
||||
for line_number, node_id in short_summary_entries.items()
|
||||
if line_number not in selected_numbers
|
||||
]
|
||||
stats: dict[str, int] = {
|
||||
"errors": sum(1 for line in all_lines if line.level == LogLevel.ERROR),
|
||||
"fails": sum(1 for line in all_lines if line.level == LogLevel.FAIL),
|
||||
"warnings": sum(1 for line in all_lines if line.level == LogLevel.WARN),
|
||||
"info": sum(1 for line in all_lines if line.level == LogLevel.INFO),
|
||||
"total": len(all_lines),
|
||||
"selected": len(selected),
|
||||
}
|
||||
output_lines = [line.content for line in selected]
|
||||
omitted = len(all_lines) - len(selected)
|
||||
if omitted > 0:
|
||||
summary_parts: list[str] = []
|
||||
for label, key in (
|
||||
("ERROR", "errors"),
|
||||
("FAIL", "fails"),
|
||||
("WARN", "warnings"),
|
||||
("INFO", "info"),
|
||||
):
|
||||
count = stats[key]
|
||||
if count > 0:
|
||||
summary_parts.append(f"{count} {label}")
|
||||
if summary_parts:
|
||||
omitted_names = ""
|
||||
if omitted_short_summary_ids:
|
||||
omitted_names = "; omitted: " + ", ".join(omitted_short_summary_ids[:5])
|
||||
overflow = len(omitted_short_summary_ids) - 5
|
||||
if overflow > 0:
|
||||
omitted_names += f", +{overflow} more"
|
||||
output_lines.append(
|
||||
f"[{omitted} lines omitted: {', '.join(summary_parts)}{omitted_names}]"
|
||||
)
|
||||
return "\n".join(output_lines), stats
|
||||
|
||||
def _store_in_ccr(self, original: str, compressed: str, original_count: int) -> str | None:
|
||||
"""Backwards-compat shim — the legacy callsite name. Now
|
||||
delegates to `_persist_to_python_ccr`. Returns the stored
|
||||
cache_key if persistence succeeded, else None.
|
||||
"""
|
||||
# Compute the same cache key the Rust path would (MD5 of
|
||||
# original truncated to 24 hex chars).
|
||||
import hashlib
|
||||
|
||||
cache_key = hashlib.md5(original.encode()).hexdigest()[:24]
|
||||
try:
|
||||
from ..cache.compression_store import get_compression_store
|
||||
except ImportError as e:
|
||||
logger.warning("CCR store import failed; cache_key %s not persisted: %s", cache_key, e)
|
||||
return None
|
||||
try:
|
||||
store: Any = get_compression_store()
|
||||
return cast(
|
||||
"str | None",
|
||||
store.store(original, compressed, original_item_count=original_count),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("CCR store write failed; cache_key %s not persisted: %s", cache_key, e)
|
||||
return None
|
||||
|
||||
def _persist_to_python_ccr(self, original: str, compressed: str, cache_key: str) -> None:
|
||||
"""Promote a Rust-emitted cache_key into the production Python
|
||||
CompressionStore. Failures are logged at warning level."""
|
||||
|
||||
+56
-213
@@ -7,6 +7,7 @@ Tests cover:
|
||||
4. Edge cases
|
||||
"""
|
||||
|
||||
from headroom._core import detect_log_format
|
||||
from headroom.transforms.log_compressor import (
|
||||
LogCompressionResult,
|
||||
LogCompressor,
|
||||
@@ -17,6 +18,21 @@ from headroom.transforms.log_compressor import (
|
||||
)
|
||||
|
||||
|
||||
def _detect(lines: list[str]) -> LogFormat:
|
||||
"""The Rust format detector that compress() uses."""
|
||||
return LogFormat(detect_log_format(list(lines)))
|
||||
|
||||
|
||||
_FILLER = [f"step {i} ok" for i in range(100)]
|
||||
|
||||
|
||||
def _compress_amid_filler(lines: list[str], **config) -> LogCompressionResult:
|
||||
"""Compress ``lines`` between 100 filler lines on each side, so selection
|
||||
actually has to choose."""
|
||||
content = "\n".join([*_FILLER, *lines, *_FILLER])
|
||||
return LogCompressor(LogCompressorConfig(enable_ccr=False, **config)).compress(content)
|
||||
|
||||
|
||||
class TestLogFormatDetection:
|
||||
"""Tests for detecting different log formats."""
|
||||
|
||||
@@ -36,9 +52,8 @@ tests/test_foo.py::test_edge - AssertionError
|
||||
FAILED tests/test_foo.py::test_edge
|
||||
========================= 1 failed, 14 passed =========================
|
||||
"""
|
||||
compressor = LogCompressor()
|
||||
lines = content.split("\n")
|
||||
detected = compressor._detect_format(lines)
|
||||
detected = _detect(lines)
|
||||
assert detected == LogFormat.PYTEST
|
||||
|
||||
def test_detect_npm_format(self):
|
||||
@@ -50,9 +65,8 @@ npm ERR! ERESOLVE unable to resolve dependency tree
|
||||
npm info using npm@9.0.0
|
||||
> added 150 packages in 5s
|
||||
"""
|
||||
compressor = LogCompressor()
|
||||
lines = content.split("\n")
|
||||
detected = compressor._detect_format(lines)
|
||||
detected = _detect(lines)
|
||||
assert detected == LogFormat.NPM
|
||||
|
||||
def test_detect_cargo_format(self):
|
||||
@@ -70,9 +84,8 @@ error[E0382]: borrow of moved value: `s`
|
||||
Finished dev [unoptimized + debuginfo] target(s) in 0.50s
|
||||
Running `target/debug/myproject`
|
||||
"""
|
||||
compressor = LogCompressor()
|
||||
lines = content.split("\n")
|
||||
detected = compressor._detect_format(lines)
|
||||
detected = _detect(lines)
|
||||
assert detected == LogFormat.CARGO
|
||||
|
||||
def test_detect_make_format(self):
|
||||
@@ -84,9 +97,8 @@ make[1]: *** [Makefile:10: utils.o] Error 1
|
||||
make: *** [Makefile:5: all] Error 2
|
||||
g++ -Wall -o program main.cpp utils.cpp
|
||||
"""
|
||||
compressor = LogCompressor()
|
||||
lines = content.split("\n")
|
||||
detected = compressor._detect_format(lines)
|
||||
detected = _detect(lines)
|
||||
assert detected == LogFormat.MAKE
|
||||
|
||||
def test_detect_jest_format(self):
|
||||
@@ -96,9 +108,8 @@ FAIL src/utils/helpers.test.ts
|
||||
Test Suites: 1 failed, 1 passed, 2 total
|
||||
Tests: 2 failed, 10 passed, 12 total
|
||||
"""
|
||||
compressor = LogCompressor()
|
||||
lines = content.split("\n")
|
||||
detected = compressor._detect_format(lines)
|
||||
detected = _detect(lines)
|
||||
assert detected == LogFormat.JEST
|
||||
|
||||
def test_detect_generic_format(self):
|
||||
@@ -109,238 +120,70 @@ WARNING Low memory
|
||||
ERROR Connection timeout
|
||||
CRITICAL System failure
|
||||
"""
|
||||
compressor = LogCompressor()
|
||||
lines = content.split("\n")
|
||||
detected = compressor._detect_format(lines)
|
||||
detected = _detect(lines)
|
||||
assert detected == LogFormat.GENERIC
|
||||
|
||||
def test_detect_empty_returns_generic(self):
|
||||
"""Empty or minimal input returns GENERIC."""
|
||||
compressor = LogCompressor()
|
||||
assert compressor._detect_format([]) == LogFormat.GENERIC
|
||||
assert compressor._detect_format(["random line"]) == LogFormat.GENERIC
|
||||
assert _detect([]) == LogFormat.GENERIC
|
||||
assert _detect(["random line"]) == LogFormat.GENERIC
|
||||
|
||||
|
||||
class TestLogLevelDetection:
|
||||
"""Tests for log level detection in lines."""
|
||||
"""Level counts reported by compress(). DEBUG/TRACE/UNKNOWN and the full
|
||||
keyword table are unit-tested in Rust (level_classifier_covers_every_level)."""
|
||||
|
||||
def test_detect_error_levels(self):
|
||||
"""ERROR, FATAL, CRITICAL are detected."""
|
||||
compressor = LogCompressor()
|
||||
|
||||
error_lines = [
|
||||
def test_level_counts(self):
|
||||
lines = [
|
||||
"ERROR: something went wrong",
|
||||
"error: file not found",
|
||||
"Error: Invalid input",
|
||||
"FATAL: system crash",
|
||||
"fatal error occurred",
|
||||
"CRITICAL: database down",
|
||||
]
|
||||
|
||||
for line in error_lines:
|
||||
log_lines = compressor._parse_lines([line])
|
||||
assert log_lines[0].level == LogLevel.ERROR, f"Failed for: {line}"
|
||||
|
||||
def test_detect_fail_levels(self):
|
||||
"""FAIL, FAILED are detected."""
|
||||
compressor = LogCompressor()
|
||||
|
||||
fail_lines = [
|
||||
"FAIL tests/test_foo.py",
|
||||
"FAILED to connect",
|
||||
"Test failed",
|
||||
]
|
||||
|
||||
for line in fail_lines:
|
||||
log_lines = compressor._parse_lines([line])
|
||||
assert log_lines[0].level == LogLevel.FAIL, f"Failed for: {line}"
|
||||
|
||||
def test_detect_warn_levels(self):
|
||||
"""WARN, WARNING are detected."""
|
||||
compressor = LogCompressor()
|
||||
|
||||
warn_lines = [
|
||||
"WARN: deprecated function",
|
||||
"WARNING: low disk space",
|
||||
"warning: unused variable",
|
||||
"INFO: starting process",
|
||||
"info starting",
|
||||
"DEBUG: variable x = 5",
|
||||
"TRACE: entering function",
|
||||
]
|
||||
|
||||
for line in warn_lines:
|
||||
log_lines = compressor._parse_lines([line])
|
||||
assert log_lines[0].level == LogLevel.WARN, f"Failed for: {line}"
|
||||
|
||||
def test_detect_info_debug_trace(self):
|
||||
"""INFO, DEBUG, TRACE are detected."""
|
||||
compressor = LogCompressor()
|
||||
|
||||
test_cases = [
|
||||
("INFO: starting process", LogLevel.INFO),
|
||||
("info starting", LogLevel.INFO),
|
||||
("DEBUG: variable x = 5", LogLevel.DEBUG),
|
||||
("debug mode enabled", LogLevel.DEBUG),
|
||||
("TRACE: entering function", LogLevel.TRACE),
|
||||
]
|
||||
|
||||
for line, expected_level in test_cases:
|
||||
log_lines = compressor._parse_lines([line])
|
||||
assert log_lines[0].level == expected_level, f"Failed for: {line}"
|
||||
|
||||
def test_unknown_level_default(self):
|
||||
"""Lines without level markers default to UNKNOWN."""
|
||||
compressor = LogCompressor()
|
||||
log_lines = compressor._parse_lines(["Just some regular text"])
|
||||
assert log_lines[0].level == LogLevel.UNKNOWN
|
||||
stats = _compress_amid_filler(lines).stats
|
||||
assert stats["errors"] == 6
|
||||
assert stats["fails"] == 3
|
||||
assert stats["warnings"] == 3
|
||||
assert stats["info"] == 2
|
||||
|
||||
|
||||
class TestStackTraceDetection:
|
||||
"""Tests for stack trace detection."""
|
||||
"""Stack-trace lines survive compression. Python, Go, Rust-panic, .NET and
|
||||
Java traces are unit-tested in Rust; these pin the JS and rustc shapes."""
|
||||
|
||||
def test_detect_python_traceback(self):
|
||||
"""Python traceback is detected."""
|
||||
content = """Traceback (most recent call last):
|
||||
File "main.py", line 42, in process
|
||||
result = compute(data)
|
||||
File "utils.py", line 15, in compute
|
||||
return data / 0
|
||||
ZeroDivisionError: division by zero
|
||||
"""
|
||||
compressor = LogCompressor()
|
||||
log_lines = compressor._parse_lines(content.split("\n"))
|
||||
|
||||
# First several lines should be marked as stack trace
|
||||
stack_trace_count = sum(1 for line in log_lines if line.is_stack_trace)
|
||||
assert stack_trace_count > 0
|
||||
|
||||
def test_detect_javascript_stack_trace(self):
|
||||
"""JavaScript stack trace is detected."""
|
||||
content = """Error: Connection failed
|
||||
at Connection.connect (src/db.js:42:15)
|
||||
at async main (src/index.js:10:5)
|
||||
"""
|
||||
compressor = LogCompressor()
|
||||
log_lines = compressor._parse_lines(content.split("\n"))
|
||||
|
||||
stack_trace_count = sum(1 for line in log_lines if line.is_stack_trace)
|
||||
assert stack_trace_count > 0
|
||||
|
||||
def test_detect_rust_error_location(self):
|
||||
"""Rust error location is detected."""
|
||||
content = """error[E0382]: borrow of moved value: `s`
|
||||
--> src/main.rs:5:13
|
||||
|
|
||||
3 | let s = String::from("hello");
|
||||
| - move occurs
|
||||
"""
|
||||
compressor = LogCompressor()
|
||||
log_lines = compressor._parse_lines(content.split("\n"))
|
||||
|
||||
stack_trace_count = sum(1 for line in log_lines if line.is_stack_trace)
|
||||
assert stack_trace_count > 0
|
||||
|
||||
|
||||
class TestLineDeduplication:
|
||||
"""Tests for warning/line deduplication."""
|
||||
|
||||
def test_dedupe_identical_warnings(self):
|
||||
"""Identical warnings are deduplicated."""
|
||||
compressor = LogCompressor()
|
||||
|
||||
lines = [
|
||||
LogLine(line_number=1, content="WARNING: unused variable 'x'", level=LogLevel.WARN),
|
||||
LogLine(line_number=2, content="WARNING: unused variable 'x'", level=LogLevel.WARN),
|
||||
LogLine(line_number=3, content="WARNING: unused variable 'x'", level=LogLevel.WARN),
|
||||
def test_javascript_stack_trace_kept(self):
|
||||
trace = [
|
||||
"Error: Connection failed",
|
||||
" at Connection.connect (src/db.js:42:15)",
|
||||
" at async main (src/index.js:10:5)",
|
||||
]
|
||||
# No context lines, so only stack-trace detection can keep the frames.
|
||||
compressed = _compress_amid_filler(trace, error_context_lines=0).compressed
|
||||
for line in trace:
|
||||
assert line in compressed
|
||||
|
||||
deduped = compressor._dedupe_similar(lines)
|
||||
assert len(deduped) == 1
|
||||
|
||||
def test_dedupe_similar_with_numbers(self):
|
||||
"""Similar warnings with different numbers are deduplicated."""
|
||||
compressor = LogCompressor()
|
||||
|
||||
lines = [
|
||||
LogLine(line_number=1, content="WARNING: error at line 10", level=LogLevel.WARN),
|
||||
LogLine(line_number=2, content="WARNING: error at line 20", level=LogLevel.WARN),
|
||||
LogLine(line_number=3, content="WARNING: error at line 30", level=LogLevel.WARN),
|
||||
def test_rust_error_location_kept(self):
|
||||
trace = [
|
||||
"error[E0382]: borrow of moved value: `s`",
|
||||
" --> src/main.rs:5:13",
|
||||
]
|
||||
|
||||
deduped = compressor._dedupe_similar(lines)
|
||||
# Numbers normalized to "N", so all three are treated as identical pattern
|
||||
assert len(deduped) == 1
|
||||
|
||||
def test_dedupe_similar_with_paths(self):
|
||||
"""Similar warnings with different paths are deduplicated.
|
||||
|
||||
Note: The path regex /[\\w/]+/ requires paths to end with '/'.
|
||||
Paths like '/path/to/' will be normalized, but '/path/to/file' won't
|
||||
be fully normalized because 'file' doesn't end with '/'.
|
||||
"""
|
||||
compressor = LogCompressor()
|
||||
|
||||
# Paths ending with / are normalized
|
||||
lines = [
|
||||
LogLine(line_number=1, content="WARNING: in /path/to/ error", level=LogLevel.WARN),
|
||||
LogLine(line_number=2, content="WARNING: in /other/dir/ error", level=LogLevel.WARN),
|
||||
LogLine(line_number=3, content="WARNING: in /another/path/ error", level=LogLevel.WARN),
|
||||
]
|
||||
|
||||
deduped = compressor._dedupe_similar(lines)
|
||||
# Paths normalized to /PATH/, so all three are treated as identical pattern
|
||||
assert len(deduped) == 1
|
||||
|
||||
def test_keeps_different_warnings(self):
|
||||
"""Different warnings are preserved."""
|
||||
compressor = LogCompressor()
|
||||
|
||||
lines = [
|
||||
LogLine(line_number=1, content="WARNING: unused variable", level=LogLevel.WARN),
|
||||
LogLine(line_number=2, content="WARNING: deprecated function", level=LogLevel.WARN),
|
||||
LogLine(line_number=3, content="WARNING: missing docstring", level=LogLevel.WARN),
|
||||
]
|
||||
|
||||
deduped = compressor._dedupe_similar(lines)
|
||||
assert len(deduped) == 3
|
||||
|
||||
|
||||
class TestLineScoring:
|
||||
"""Tests for line importance scoring."""
|
||||
|
||||
def test_error_lines_score_highest(self):
|
||||
"""ERROR and FAIL lines get highest scores."""
|
||||
compressor = LogCompressor()
|
||||
|
||||
error_line = LogLine(line_number=1, content="ERROR: critical", level=LogLevel.ERROR)
|
||||
fail_line = LogLine(line_number=2, content="FAILED test", level=LogLevel.FAIL)
|
||||
info_line = LogLine(line_number=3, content="INFO: normal", level=LogLevel.INFO)
|
||||
|
||||
error_score = compressor._score_line(error_line)
|
||||
fail_score = compressor._score_line(fail_line)
|
||||
info_score = compressor._score_line(info_line)
|
||||
|
||||
assert error_score > info_score
|
||||
assert fail_score > info_score
|
||||
|
||||
def test_stack_trace_boost(self):
|
||||
"""Stack trace lines get boosted score."""
|
||||
compressor = LogCompressor()
|
||||
|
||||
regular = LogLine(line_number=1, content="some line", level=LogLevel.UNKNOWN)
|
||||
stack_trace = LogLine(
|
||||
line_number=2, content=" File 'x.py'", level=LogLevel.UNKNOWN, is_stack_trace=True
|
||||
)
|
||||
|
||||
assert compressor._score_line(stack_trace) > compressor._score_line(regular)
|
||||
|
||||
def test_summary_line_boost(self):
|
||||
"""Summary lines get boosted score."""
|
||||
compressor = LogCompressor()
|
||||
|
||||
regular = LogLine(line_number=1, content="some line", level=LogLevel.UNKNOWN)
|
||||
summary = LogLine(
|
||||
line_number=2, content="10 passed, 2 failed", level=LogLevel.UNKNOWN, is_summary=True
|
||||
)
|
||||
|
||||
assert compressor._score_line(summary) > compressor._score_line(regular)
|
||||
# No context lines, so only stack-trace detection can keep the frames.
|
||||
compressed = _compress_amid_filler(trace, error_context_lines=0).compressed
|
||||
for line in trace:
|
||||
assert line in compressed
|
||||
|
||||
|
||||
class TestCompressionBehavior:
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from headroom.transforms.log_compressor import (
|
||||
@@ -9,93 +7,9 @@ from headroom.transforms.log_compressor import (
|
||||
LogCompressor,
|
||||
LogCompressorConfig,
|
||||
LogFormat,
|
||||
LogLevel,
|
||||
LogLine,
|
||||
)
|
||||
|
||||
|
||||
def test_detect_parse_and_score_log_lines() -> None:
|
||||
compressor = LogCompressor(LogCompressorConfig(stack_trace_max_lines=2))
|
||||
pytest_lines = [
|
||||
"============================= test session starts =============================",
|
||||
"collected 2 items",
|
||||
"ERROR critical failure",
|
||||
"Traceback (most recent call last)",
|
||||
' File "app.py", line 10',
|
||||
"",
|
||||
"2 failed, 1 warning",
|
||||
]
|
||||
assert compressor._detect_format(pytest_lines) is LogFormat.PYTEST
|
||||
assert compressor._detect_format(["npm ERR! missing script"]) is LogFormat.NPM
|
||||
assert compressor._detect_format(["Compiling app", "warning: check this"]) is LogFormat.CARGO
|
||||
assert (
|
||||
compressor._detect_format(["PASS src/app.test.js", "Test Suites: 1 failed"])
|
||||
is LogFormat.JEST
|
||||
)
|
||||
assert compressor._detect_format(["make: *** fail", "gcc -o app app.c"]) is LogFormat.MAKE
|
||||
assert compressor._detect_format(["unclassified line"]) is LogFormat.GENERIC
|
||||
|
||||
parsed = compressor._parse_lines(pytest_lines)
|
||||
assert parsed[0].is_summary is True
|
||||
assert parsed[2].level is LogLevel.ERROR
|
||||
assert parsed[3].is_stack_trace is True
|
||||
assert parsed[4].is_stack_trace is True
|
||||
assert parsed[6].is_summary is True
|
||||
assert compressor._score_line(LogLine(1, "warn", level=LogLevel.WARN)) == 0.5
|
||||
assert (
|
||||
compressor._score_line(
|
||||
LogLine(2, "error summary", level=LogLevel.ERROR, is_stack_trace=True, is_summary=True)
|
||||
)
|
||||
== 1.0
|
||||
)
|
||||
|
||||
|
||||
def test_pytest_short_summary_helper_recognizes_only_section_entries() -> None:
|
||||
compressor = LogCompressor()
|
||||
lines = [
|
||||
"FAILED outside.py::test_before",
|
||||
"=== short test summary info ===",
|
||||
"FAILED tests/test_a.py::test_short",
|
||||
"ERROR tests/test_b.py::test_long - RuntimeError: boom",
|
||||
"=== 2 failed in 0.1s ===",
|
||||
"FAILED outside.py::test_after",
|
||||
"=== short test summary info ===",
|
||||
"FAILED tests/test_c.py::test_eof - AssertionError",
|
||||
]
|
||||
|
||||
parsed = compressor._parse_lines(lines)
|
||||
|
||||
assert parsed[0].is_summary is False
|
||||
assert parsed[2].is_summary is True
|
||||
assert parsed[3].is_summary is True
|
||||
assert parsed[5].is_summary is False
|
||||
assert parsed[7].is_summary is True
|
||||
|
||||
|
||||
def test_pytest_short_summary_helper_accepts_crlf_complete_separators() -> None:
|
||||
compressor = LogCompressor()
|
||||
lines = [
|
||||
"=== diagnostic: short test summary info unavailable\r",
|
||||
"FAILED outside.py::test_before\r",
|
||||
"=== short test summary info ===\r",
|
||||
"FAILED tests/test_a.py::test_short\r",
|
||||
"ERROR tests/test_b.py::test_long - RuntimeError: boom\r",
|
||||
"=== 2 failed in 0.1s ===\r",
|
||||
"FAILED outside.py::test_after\r",
|
||||
"=== short test summary info ===\r",
|
||||
"FAILED tests/test_c.py::test_eof - AssertionError\r",
|
||||
]
|
||||
|
||||
parsed = compressor._parse_lines(lines)
|
||||
|
||||
assert parsed[1].is_summary is False
|
||||
assert parsed[3].is_summary is True
|
||||
assert parsed[4].is_summary is True
|
||||
assert parsed[6].is_summary is False
|
||||
assert parsed[8].is_summary is True
|
||||
assert parsed[3].content == lines[3]
|
||||
|
||||
|
||||
def _malformed_short_summary_diagnostic_log() -> list[str]:
|
||||
lines = [f"INFO setup output {i}" for i in range(50)]
|
||||
lines.append("=== diagnostic: short test summary info unavailable")
|
||||
@@ -104,233 +18,6 @@ def _malformed_short_summary_diagnostic_log() -> list[str]:
|
||||
return lines
|
||||
|
||||
|
||||
@pytest.mark.parametrize("keep_summary_lines", [False, True])
|
||||
def test_malformed_short_summary_diagnostic_does_not_affect_legacy_helpers(
|
||||
monkeypatch: pytest.MonkeyPatch, keep_summary_lines: bool
|
||||
) -> None:
|
||||
compressor = LogCompressor(
|
||||
LogCompressorConfig(
|
||||
max_errors=2,
|
||||
error_context_lines=0,
|
||||
keep_summary_lines=keep_summary_lines,
|
||||
max_total_lines=1_000,
|
||||
enable_ccr=False,
|
||||
)
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
__import__("sys").modules,
|
||||
"headroom.transforms.adaptive_sizer",
|
||||
SimpleNamespace(compute_optimal_k=lambda items, **kwargs: 1_000),
|
||||
)
|
||||
parsed = compressor._parse_lines(_malformed_short_summary_diagnostic_log())
|
||||
|
||||
assert len(parsed) == 71
|
||||
assert all(not line.is_summary for line in parsed[51:])
|
||||
selected = compressor._select_lines(parsed)
|
||||
selected_lookalikes = [
|
||||
line.line_number for line in selected if line.level in (LogLevel.ERROR, LogLevel.FAIL)
|
||||
]
|
||||
assert selected_lookalikes == [51, 60, 61, 70]
|
||||
|
||||
output, _ = compressor._format_output(selected, parsed)
|
||||
assert "; omitted: " not in output
|
||||
if not keep_summary_lines:
|
||||
assert output == (
|
||||
"FAILED outside.py::test_0\n"
|
||||
"FAILED outside.py::test_9\n"
|
||||
"ERROR outside.py::test_10\n"
|
||||
"ERROR outside.py::test_19\n"
|
||||
"[67 lines omitted: 10 ERROR, 10 FAIL, 51 INFO]"
|
||||
)
|
||||
|
||||
|
||||
def test_select_dedupe_add_context_and_format_output(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
compressor = LogCompressor(
|
||||
LogCompressorConfig(
|
||||
max_errors=2,
|
||||
max_warnings=1,
|
||||
error_context_lines=1,
|
||||
max_stack_traces=1,
|
||||
stack_trace_max_lines=2,
|
||||
)
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
__import__("sys").modules,
|
||||
"headroom.transforms.adaptive_sizer",
|
||||
SimpleNamespace(compute_optimal_k=lambda items, **kwargs: 6),
|
||||
)
|
||||
log_lines = [
|
||||
LogLine(0, "info line", level=LogLevel.INFO, score=0.1),
|
||||
LogLine(1, "ERROR first", level=LogLevel.ERROR, score=1.0),
|
||||
LogLine(2, "context after first", level=LogLevel.UNKNOWN, score=0.1),
|
||||
LogLine(3, "WARNING /tmp/a/123 issue", level=LogLevel.WARN, score=0.5),
|
||||
LogLine(4, "WARNING /tmp/b/999 issue", level=LogLevel.WARN, score=0.5),
|
||||
LogLine(5, "FAIL final", level=LogLevel.FAIL, score=1.0),
|
||||
LogLine(6, "Traceback (most recent call last)", is_stack_trace=True, score=0.4),
|
||||
LogLine(7, ' File "app.py", line 2', is_stack_trace=True, score=0.4),
|
||||
LogLine(8, "1 failed, 1 warning", is_summary=True, score=0.5),
|
||||
]
|
||||
selected = compressor._select_lines(log_lines)
|
||||
assert [line.line_number for line in selected] == [1, 3, 4, 5, 6, 8]
|
||||
|
||||
assert compressor._select_with_first_last(log_lines[:2], max_count=5) == log_lines[:2]
|
||||
many_errors = [
|
||||
LogLine(10, "first", level=LogLevel.ERROR, score=0.1),
|
||||
LogLine(11, "mid", level=LogLevel.ERROR, score=0.9),
|
||||
LogLine(12, "last", level=LogLevel.ERROR, score=0.2),
|
||||
]
|
||||
trimmed = compressor._select_with_first_last(many_errors, max_count=2)
|
||||
assert trimmed == [many_errors[0], many_errors[2]]
|
||||
# fixed_in_3e5: conservative dedupe preserves message prefix (everything
|
||||
# before the first `:` or `=`), so warnings without a colon keep their
|
||||
# full content as the dedupe key. The two lines below have different
|
||||
# paths/numbers and no `:`, so they DON'T collapse anymore — Python's
|
||||
# pre-3e5 aggressive normalization treated them as duplicates, masking
|
||||
# distinct error categories.
|
||||
distinct = compressor._dedupe_similar(log_lines[3:5])
|
||||
assert len(distinct) == 2
|
||||
# Same dedupe IS triggered when the prefix matches (lines have a colon).
|
||||
similar = compressor._dedupe_similar(
|
||||
[
|
||||
LogLine(20, "warning: file /tmp/a/123 issue", level=LogLevel.WARN),
|
||||
LogLine(21, "warning: file /tmp/b/999 issue", level=LogLevel.WARN),
|
||||
]
|
||||
)
|
||||
assert len(similar) == 1
|
||||
|
||||
output, stats = compressor._format_output(selected, log_lines)
|
||||
assert stats == {
|
||||
"errors": 1,
|
||||
"fails": 1,
|
||||
"warnings": 2,
|
||||
"info": 1,
|
||||
"total": 9,
|
||||
"selected": 6,
|
||||
}
|
||||
assert output.endswith("[3 lines omitted: 1 ERROR, 1 FAIL, 2 WARN, 1 INFO]")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("keep_summary_lines", "expected"),
|
||||
# True: the totals line (7) is reserved, and entries win the remaining ties.
|
||||
[(True, [4, 5, 7]), (False, [0, 1, 4])],
|
||||
)
|
||||
def test_short_summary_global_cap_precedence_matches_rust(
|
||||
monkeypatch: pytest.MonkeyPatch, keep_summary_lines: bool, expected: list[int]
|
||||
) -> None:
|
||||
compressor = LogCompressor(
|
||||
LogCompressorConfig(
|
||||
max_errors=10,
|
||||
error_context_lines=0,
|
||||
keep_summary_lines=keep_summary_lines,
|
||||
max_total_lines=3,
|
||||
)
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
__import__("sys").modules,
|
||||
"headroom.transforms.adaptive_sizer",
|
||||
SimpleNamespace(compute_optimal_k=lambda items, **kwargs: 3),
|
||||
)
|
||||
parsed = compressor._parse_lines(
|
||||
[
|
||||
"FAILED traceback.py::test_early_one\r",
|
||||
"FAILED traceback.py::test_early_two\r",
|
||||
"ordinary output\r",
|
||||
"=== short test summary info ===\r",
|
||||
"FAILED tests/test_summary.py::test_one\r",
|
||||
"ERROR tests/test_summary.py::test_two\r",
|
||||
"FAILED tests/test_summary.py::test_three\r",
|
||||
"=== 3 failed in 0.1s ===\r",
|
||||
]
|
||||
)
|
||||
|
||||
assert [line.line_number for line in compressor._select_lines(parsed)] == expected
|
||||
|
||||
|
||||
def test_legacy_cap_reserves_totals_and_first_error_line(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
compressor = LogCompressor(
|
||||
LogCompressorConfig(max_errors=10, error_context_lines=0, max_total_lines=4)
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
__import__("sys").modules,
|
||||
"headroom.transforms.adaptive_sizer",
|
||||
SimpleNamespace(compute_optimal_k=lambda items, **kwargs: 4),
|
||||
)
|
||||
parsed = compressor._parse_lines(
|
||||
[
|
||||
"E AssertionError: first",
|
||||
"E AssertionError: second",
|
||||
"=== short test summary info ===",
|
||||
"FAILED t.py::a",
|
||||
"FAILED t.py::b",
|
||||
"FAILED t.py::c",
|
||||
"FAILED t.py::d",
|
||||
"=== 4 failed in 0.1s ===",
|
||||
]
|
||||
)
|
||||
|
||||
# The first E line (0) and the totals line (7) are reserved; entries fill the rest.
|
||||
assert [line.line_number for line in compressor._select_lines(parsed)] == [0, 3, 4, 7]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("count", "suffix"),
|
||||
[
|
||||
(
|
||||
3,
|
||||
"; omitted: tests/test_ids.py::test_0, tests/test_ids.py::test_1, "
|
||||
"tests/test_ids.py::test_2",
|
||||
),
|
||||
(
|
||||
5,
|
||||
"; omitted: tests/test_ids.py::test_0, tests/test_ids.py::test_1, "
|
||||
"tests/test_ids.py::test_2, tests/test_ids.py::test_3, tests/test_ids.py::test_4",
|
||||
),
|
||||
(
|
||||
7,
|
||||
"; omitted: tests/test_ids.py::test_0, tests/test_ids.py::test_1, "
|
||||
"tests/test_ids.py::test_2, tests/test_ids.py::test_3, tests/test_ids.py::test_4, "
|
||||
"+2 more",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_short_summary_omission_naming_is_bounded(count: int, suffix: str) -> None:
|
||||
compressor = LogCompressor(LogCompressorConfig(keep_summary_lines=False))
|
||||
contents = ["=== short test summary info ===\r"] + [
|
||||
f"FAILED tests/test_ids.py::test_{i}\r" for i in range(count)
|
||||
]
|
||||
contents.append("=== failures complete ===\r")
|
||||
all_lines = compressor._parse_lines(contents)
|
||||
selected = [all_lines[0], all_lines[-1]]
|
||||
|
||||
output, _ = compressor._format_output(selected, all_lines)
|
||||
|
||||
assert output.endswith(f"[{count} lines omitted: {count} FAIL, 1 INFO{suffix}]")
|
||||
|
||||
|
||||
def test_short_summary_omission_naming_uses_ids_and_preserves_duplicates() -> None:
|
||||
compressor = LogCompressor(LogCompressorConfig(keep_summary_lines=False))
|
||||
all_lines = compressor._parse_lines(
|
||||
[
|
||||
"=== short test summary info ===",
|
||||
"ERROR tests/test_ids.py::test_error - RuntimeError: boom",
|
||||
"FAILED tests/test_ids.py::test_repeat",
|
||||
"FAILED tests/test_ids.py::test_repeat - AssertionError",
|
||||
"FAILED tests/test_ids.py::test_kept",
|
||||
"=== failures complete ===",
|
||||
]
|
||||
)
|
||||
|
||||
output, stats = compressor._format_output([all_lines[0], all_lines[4], all_lines[5]], all_lines)
|
||||
|
||||
assert output.endswith(
|
||||
"[3 lines omitted: 1 ERROR, 3 FAIL, 1 INFO; omitted: tests/test_ids.py::test_error, "
|
||||
"tests/test_ids.py::test_repeat, tests/test_ids.py::test_repeat]"
|
||||
)
|
||||
assert stats["errors"] == 1
|
||||
assert stats["fails"] == 3
|
||||
|
||||
|
||||
def test_log_compressor_compress_and_ccr_paths() -> None:
|
||||
"""Phase 3e.5: `compress()` is now a single Rust call, so this test
|
||||
exercises end-to-end behavior instead of monkeypatching internal
|
||||
@@ -502,29 +189,7 @@ def test_real_compress_ignores_malformed_short_summary_diagnostic() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_store_in_ccr_and_result_properties(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
compressor = LogCompressor()
|
||||
monkeypatch.setitem(
|
||||
__import__("sys").modules,
|
||||
"headroom.cache.compression_store",
|
||||
SimpleNamespace(
|
||||
get_compression_store=lambda: SimpleNamespace(
|
||||
store=lambda original, compressed, original_item_count=0: "stored-log"
|
||||
)
|
||||
),
|
||||
)
|
||||
assert compressor._store_in_ccr("orig", "comp", 10) == "stored-log"
|
||||
|
||||
def broken_store():
|
||||
raise RuntimeError("boom")
|
||||
|
||||
monkeypatch.setitem(
|
||||
__import__("sys").modules,
|
||||
"headroom.cache.compression_store",
|
||||
SimpleNamespace(get_compression_store=broken_store),
|
||||
)
|
||||
assert compressor._store_in_ccr("orig", "comp", 10) is None
|
||||
|
||||
def test_result_properties() -> None:
|
||||
result = LogCompressionResult(
|
||||
compressed="small",
|
||||
original="this is a substantially longer log body",
|
||||
|
||||
@@ -116,31 +116,6 @@ def test_node_async_frames_count_toward_log_detection() -> None:
|
||||
assert result.content_type is ContentType.BUILD_OUTPUT
|
||||
|
||||
|
||||
# Shim pattern mirror ----------------------------------------------------------
|
||||
|
||||
|
||||
def test_parse_lines_marks_new_flavor_openers() -> None:
|
||||
compressor = LogCompressor()
|
||||
lines = [
|
||||
"panic: boom",
|
||||
"goroutine 7 [select]:",
|
||||
"\t/app/main.go:10 +0x20",
|
||||
"thread 'main' panicked at src/lib.rs:1:1:",
|
||||
"stack backtrace:",
|
||||
" 0: rust_begin_unwind",
|
||||
"Unhandled exception. System.Exception: x",
|
||||
" at App.Main(String[] a) in /src/P.cs:line 3",
|
||||
"Caused by: java.io.IOException: y",
|
||||
" ... 3 more",
|
||||
"", # blank resets the shim's legacy in-trace latch
|
||||
"plain line",
|
||||
]
|
||||
parsed = compressor._parse_lines(lines)
|
||||
flags = [ln.is_stack_trace for ln in parsed]
|
||||
assert all(flags[:-2]), f"unmarked opener among {flags}"
|
||||
assert not flags[-1]
|
||||
|
||||
|
||||
# End-to-end compression --------------------------------------------------------
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user