mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
refactor(transforms): drop the Python mirror of the Rust search selector
SearchCompressor.compress() runs entirely in Rust. The class still carried a Python re-implementation of the scorer, selector and formatter (_score_matches, _select_matches, _format_output, _cjk_bigrams, _is_cjk_char) plus a _parse_search_results wrapper, kept only so old tests could call them. Production never ran them, and the module's own docstring noted the Python error-boost keywords had drifted from Rust. Removes those helpers. Tests now target what ships: - parser tests call headroom._core.parse_search_lines (the live Rust parser) through a small test helper; - scoring tests (context words, error/warning/TODO boost, context_keywords, short-word filter, boost_errors=False, CJK query bigrams) become compress()-level tests, because Rust only unit-tests error boosting and bigram extraction; - the score-cap test is dropped: the cap is not observable through compress(). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Assisted-by: claude-code:claude-opus-5-5 Headline-Session: 6e77a807-8dd4-4a77-a2fa-ddce6dc2515e Headline-AI-Share: 21% strict / 21% weighted
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a9815bbb5f
commit
0384d6106c
@@ -71,8 +71,7 @@ use md5::{Digest, Md5};
|
||||
use crate::ccr::CcrStore;
|
||||
use crate::signals::{ImportanceContext, LineImportanceDetector};
|
||||
|
||||
/// True for CJK ideographs, kana, and Hangul. Code-point ranges kept
|
||||
/// byte-identical with the Python `_is_cjk_char` for search-compressor parity.
|
||||
/// True for CJK ideographs, kana, and Hangul.
|
||||
fn is_cjk_char(c: char) -> bool {
|
||||
matches!(
|
||||
c as u32,
|
||||
@@ -81,7 +80,7 @@ fn is_cjk_char(c: char) -> bool {
|
||||
}
|
||||
|
||||
/// CJK character bigrams from the CJK runs of a (lowercased) query, so a
|
||||
/// spaceless CJK query can match content. Mirrors the Python `_cjk_bigrams`.
|
||||
/// spaceless CJK query can match content.
|
||||
fn cjk_bigrams(text: &str) -> BTreeSet<String> {
|
||||
let mut out = BTreeSet::new();
|
||||
let mut run: Vec<char> = Vec::new();
|
||||
@@ -452,9 +451,8 @@ impl SearchCompressor {
|
||||
bias: f64,
|
||||
stats: &mut SearchCompressorStats,
|
||||
) -> BTreeMap<String, FileMatches> {
|
||||
// Python `_select_matches` sorts files by total match score
|
||||
// descending. `BTreeMap` iterates in key order, so we collect
|
||||
// and sort explicitly.
|
||||
// Sort files by total match score, descending. `BTreeMap`
|
||||
// iterates in key order, so we collect and sort explicitly.
|
||||
let mut by_score: Vec<(&String, &FileMatches)> = files.iter().collect();
|
||||
by_score.sort_by(|a, b| {
|
||||
b.1.total_score()
|
||||
@@ -1363,8 +1361,8 @@ src/main.py-44-context line";
|
||||
|
||||
#[test]
|
||||
fn select_respects_per_file_cap_and_global_cap() {
|
||||
// Note: compute_optimal_k enforces a hard `min_k=5` floor (matches
|
||||
// Python `_select_matches`), so `max_total_matches` is a soft cap
|
||||
// Note: compute_optimal_k enforces a hard `min_k=5` floor, so
|
||||
// `max_total_matches` is a soft cap
|
||||
// that bites only above that floor. Configure 6 here to exercise
|
||||
// the cap path.
|
||||
let compressor = SearchCompressor::new(SearchCompressorConfig {
|
||||
|
||||
@@ -11,10 +11,6 @@ is now a thin shim that:
|
||||
2. Routes `SearchCompressor.compress()` entirely through the Rust
|
||||
implementation, picking up the parser bug fixes and the
|
||||
`signals::LineImportanceDetector` trait consumer pattern.
|
||||
3. Implements legacy internals (`_parse_search_results`,
|
||||
`_score_matches`, `_select_matches`, `_format_output`) on top of
|
||||
the same Rust building blocks so the existing 49 unit tests keep
|
||||
covering meaningful code paths.
|
||||
|
||||
# Bug fixes the Rust port carries (and this shim therefore inherits)
|
||||
|
||||
@@ -50,36 +46,6 @@ from typing import Any, cast
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _is_cjk_char(c: str) -> bool:
|
||||
"""True for CJK ideographs, kana, and Hangul. Code-point ranges kept
|
||||
byte-identical with the Rust `is_cjk_char` for search-compressor parity."""
|
||||
o = ord(c)
|
||||
return (
|
||||
0x3040 <= o <= 0x30FF
|
||||
or 0x3400 <= o <= 0x4DBF
|
||||
or 0x4E00 <= o <= 0x9FFF
|
||||
or 0xAC00 <= o <= 0xD7AF
|
||||
or 0xF900 <= o <= 0xFAFF
|
||||
)
|
||||
|
||||
|
||||
def _cjk_bigrams(text: str) -> set[str]:
|
||||
"""CJK character bigrams from the CJK runs of a (lowercased) query, so a
|
||||
spaceless CJK query can match content. Mirrors the Rust `cjk_bigrams`."""
|
||||
out: set[str] = set()
|
||||
run: list[str] = []
|
||||
for c in text:
|
||||
if _is_cjk_char(c):
|
||||
run.append(c)
|
||||
else:
|
||||
for i in range(len(run) - 1):
|
||||
out.add(run[i] + run[i + 1])
|
||||
run = []
|
||||
for i in range(len(run) - 1):
|
||||
out.add(run[i] + run[i + 1])
|
||||
return out
|
||||
|
||||
|
||||
# ─── Public dataclasses (preserve existing import surface) ──────────────────
|
||||
|
||||
|
||||
@@ -159,11 +125,8 @@ class SearchCompressionResult:
|
||||
class SearchCompressor:
|
||||
"""Compresses grep/ripgrep search results via the Rust port.
|
||||
|
||||
Drop-in replacement for the retired Python class. The main
|
||||
`compress()` method delegates to Rust end-to-end. The internal
|
||||
helpers used by the existing test surface are preserved and route
|
||||
through the same Rust parser so the bug fixes (Windows paths,
|
||||
dashes-in-filename) land everywhere.
|
||||
Drop-in replacement for the retired Python class. `compress()`
|
||||
delegates to Rust end-to-end.
|
||||
"""
|
||||
|
||||
def __init__(self, config: SearchCompressorConfig | None = None) -> None:
|
||||
@@ -229,153 +192,6 @@ class SearchCompressor:
|
||||
summaries=summaries,
|
||||
)
|
||||
|
||||
# ─── Legacy internal helpers (test surface compat) ──────────────────
|
||||
|
||||
def _parse_search_results(self, content: str) -> dict[str, FileMatches]:
|
||||
"""Parse via the Rust parser, build legacy Python dataclasses."""
|
||||
from headroom._core import parse_search_lines
|
||||
|
||||
out: dict[str, FileMatches] = {}
|
||||
for file_path, line_no, body in parse_search_lines(content):
|
||||
if file_path not in out:
|
||||
out[file_path] = FileMatches(file=file_path)
|
||||
out[file_path].matches.append(
|
||||
SearchMatch(file=file_path, line_number=int(line_no), content=body)
|
||||
)
|
||||
return out
|
||||
|
||||
def _score_matches(
|
||||
self,
|
||||
file_matches: dict[str, FileMatches],
|
||||
context: str,
|
||||
) -> None:
|
||||
"""Score matches by relevance to context.
|
||||
|
||||
Stays Python so the legacy direct-call test surface keeps
|
||||
working without rebuilding through Rust on every test. The
|
||||
scoring constants mirror Rust `SearchCompressor::score_matches`,
|
||||
pinned by Rust unit tests and Python tests over the same inputs:
|
||||
word-overlap and CJK-bigram scoring are byte-equal. (The error-
|
||||
boost keyword set still diverges for a few terms fixed only on
|
||||
the Rust side -- see keyword_detector; there is no cross-impl
|
||||
assertion, so this equality is test-pinned, not mechanically
|
||||
enforced.)
|
||||
"""
|
||||
from headroom.transforms.error_detection import PRIORITY_PATTERNS_SEARCH
|
||||
|
||||
context_lower = context.lower()
|
||||
# Dedup whitespace words (len>2 by codepoints), and add CJK char bigrams
|
||||
# so a spaceless CJK query can match content.
|
||||
context_words = {w for w in context_lower.split() if len(w) > 2}
|
||||
context_words |= _cjk_bigrams(context_lower)
|
||||
|
||||
for fm in file_matches.values():
|
||||
for match in fm.matches:
|
||||
score = 0.0
|
||||
content_lower = match.content.lower()
|
||||
|
||||
for word in context_words:
|
||||
if word in content_lower:
|
||||
score += 0.3
|
||||
|
||||
if self.config.boost_errors:
|
||||
for i, pattern in enumerate(PRIORITY_PATTERNS_SEARCH):
|
||||
if pattern.search(match.content):
|
||||
score += 0.5 - (i * 0.1)
|
||||
break # only one priority boost per line, matches Rust
|
||||
|
||||
for keyword in self.config.context_keywords:
|
||||
if keyword.lower() in content_lower:
|
||||
score += 0.4
|
||||
|
||||
match.score = min(1.0, score)
|
||||
|
||||
def _select_matches(
|
||||
self,
|
||||
file_matches: dict[str, FileMatches],
|
||||
bias: float = 1.0,
|
||||
) -> dict[str, FileMatches]:
|
||||
"""Select top matches per file and globally."""
|
||||
from headroom.transforms.adaptive_sizer import compute_optimal_k
|
||||
|
||||
sorted_files = sorted(
|
||||
file_matches.items(),
|
||||
key=lambda x: sum(m.score for m in x[1].matches),
|
||||
reverse=True,
|
||||
)[: self.config.max_files]
|
||||
|
||||
all_match_strings = [
|
||||
f"{file_path}:{m.line_number}:{m.content}"
|
||||
for file_path, fm in sorted_files
|
||||
for m in fm.matches
|
||||
]
|
||||
adaptive_total = compute_optimal_k(
|
||||
all_match_strings,
|
||||
bias=bias,
|
||||
min_k=5,
|
||||
max_k=self.config.max_total_matches,
|
||||
)
|
||||
|
||||
selected: dict[str, FileMatches] = {}
|
||||
total_selected = 0
|
||||
for file_path, fm in sorted_files:
|
||||
if total_selected >= adaptive_total:
|
||||
break
|
||||
|
||||
sorted_matches = sorted(fm.matches, key=lambda m: m.score, reverse=True)
|
||||
|
||||
file_selected: list[SearchMatch] = []
|
||||
remaining_slots = min(
|
||||
self.config.max_matches_per_file,
|
||||
adaptive_total - total_selected,
|
||||
)
|
||||
|
||||
if self.config.always_keep_first and fm.first:
|
||||
file_selected.append(fm.first)
|
||||
remaining_slots -= 1
|
||||
|
||||
if (
|
||||
self.config.always_keep_last
|
||||
and fm.last
|
||||
and fm.last is not fm.first
|
||||
and remaining_slots > 0
|
||||
):
|
||||
file_selected.append(fm.last)
|
||||
remaining_slots -= 1
|
||||
|
||||
for match in sorted_matches:
|
||||
if remaining_slots <= 0:
|
||||
break
|
||||
if match not in file_selected:
|
||||
file_selected.append(match)
|
||||
remaining_slots -= 1
|
||||
|
||||
file_selected.sort(key=lambda m: m.line_number)
|
||||
selected[file_path] = FileMatches(file=file_path, matches=file_selected)
|
||||
total_selected += len(file_selected)
|
||||
|
||||
return selected
|
||||
|
||||
def _format_output(
|
||||
self,
|
||||
selected: dict[str, FileMatches],
|
||||
original: dict[str, FileMatches],
|
||||
) -> tuple[str, dict[str, str]]:
|
||||
lines: list[str] = []
|
||||
summaries: dict[str, str] = {}
|
||||
|
||||
for file_path, fm in sorted(selected.items()):
|
||||
for match in fm.matches:
|
||||
lines.append(f"{match.file}:{match.line_number}:{match.content}")
|
||||
original_fm = original.get(file_path)
|
||||
if original_fm and len(original_fm.matches) > len(fm.matches):
|
||||
omitted = len(original_fm.matches) - len(fm.matches)
|
||||
summary = f"[... and {omitted} more matches in {file_path}]"
|
||||
lines.append(summary)
|
||||
summaries[file_path] = summary
|
||||
|
||||
return "\n".join(lines), summaries
|
||||
|
||||
# ─── Internal CCR persistence ───────────────────────────────────────
|
||||
|
||||
def _persist_to_python_ccr(self, original: str, compressed: str, cache_key: str) -> None:
|
||||
|
||||
+79
-123
@@ -7,6 +7,7 @@ Tests cover:
|
||||
4. Edge cases
|
||||
"""
|
||||
|
||||
from headroom._core import parse_search_lines
|
||||
from headroom.transforms.search_compressor import (
|
||||
FileMatches,
|
||||
SearchCompressionResult,
|
||||
@@ -16,6 +17,31 @@ from headroom.transforms.search_compressor import (
|
||||
)
|
||||
|
||||
|
||||
def _parse(content: str) -> dict[str, FileMatches]:
|
||||
"""Group the Rust parser's (file, line, body) rows by file."""
|
||||
out: dict[str, FileMatches] = {}
|
||||
for file_path, line_no, body in parse_search_lines(content):
|
||||
out.setdefault(file_path, FileMatches(file=file_path)).matches.append(
|
||||
SearchMatch(file=file_path, line_number=int(line_no), content=body)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _keep_best_one(**config) -> SearchCompressor:
|
||||
"""A compressor that keeps only the single highest-scored match per file,
|
||||
so a compress() call shows which line the scorer ranked first. Ties keep
|
||||
the earliest line."""
|
||||
return SearchCompressor(
|
||||
SearchCompressorConfig(
|
||||
max_matches_per_file=1,
|
||||
always_keep_first=False,
|
||||
always_keep_last=False,
|
||||
enable_ccr=False,
|
||||
**config,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class TestGrepOutputParsing:
|
||||
"""Tests for parsing grep/ripgrep style output."""
|
||||
|
||||
@@ -25,8 +51,7 @@ class TestGrepOutputParsing:
|
||||
src/main.py:43: \"\"\"Process items.\"\"\"
|
||||
src/utils.py:15:def validate(data):
|
||||
"""
|
||||
compressor = SearchCompressor()
|
||||
file_matches = compressor._parse_search_results(content)
|
||||
file_matches = _parse(content)
|
||||
|
||||
assert "src/main.py" in file_matches
|
||||
assert "src/utils.py" in file_matches
|
||||
@@ -39,8 +64,7 @@ src/utils.py:15:def validate(data):
|
||||
src/main.py:42:def process_data(items):
|
||||
src/main.py-43-some context after
|
||||
"""
|
||||
compressor = SearchCompressor()
|
||||
file_matches = compressor._parse_search_results(content)
|
||||
file_matches = _parse(content)
|
||||
|
||||
assert "src/main.py" in file_matches
|
||||
# All three lines should be parsed (both : and - separators)
|
||||
@@ -51,8 +75,7 @@ src/main.py-43-some context after
|
||||
content = """src/config.py:10:DATABASE_URL = "postgres://user:pass@host:5432/db"
|
||||
src/config.py:20:REDIS_URL = "redis://localhost:6379"
|
||||
"""
|
||||
compressor = SearchCompressor()
|
||||
file_matches = compressor._parse_search_results(content)
|
||||
file_matches = _parse(content)
|
||||
|
||||
assert "src/config.py" in file_matches
|
||||
matches = file_matches["src/config.py"].matches
|
||||
@@ -65,8 +88,7 @@ src/config.py:20:REDIS_URL = "redis://localhost:6379"
|
||||
content = """C:\\Users\\dev\\src\\main.py:10:def main():
|
||||
C:\\Users\\dev\\src\\utils.py:20:def helper():
|
||||
"""
|
||||
compressor = SearchCompressor()
|
||||
file_matches = compressor._parse_search_results(content)
|
||||
file_matches = _parse(content)
|
||||
|
||||
# Windows paths may not parse correctly due to : in path
|
||||
# This tests current behavior
|
||||
@@ -74,15 +96,13 @@ C:\\Users\\dev\\src\\utils.py:20:def helper():
|
||||
|
||||
def test_parse_empty_content(self):
|
||||
"""Empty input returns empty result."""
|
||||
compressor = SearchCompressor()
|
||||
file_matches = compressor._parse_search_results("")
|
||||
file_matches = _parse("")
|
||||
|
||||
assert file_matches == {}
|
||||
|
||||
def test_parse_whitespace_only(self):
|
||||
"""Whitespace-only input returns empty result."""
|
||||
compressor = SearchCompressor()
|
||||
file_matches = compressor._parse_search_results(" \n\n \n")
|
||||
file_matches = _parse(" \n\n \n")
|
||||
|
||||
assert file_matches == {}
|
||||
|
||||
@@ -92,8 +112,7 @@ C:\\Users\\dev\\src\\utils.py:20:def helper():
|
||||
without any grep-style formatting
|
||||
just normal lines here"""
|
||||
|
||||
compressor = SearchCompressor()
|
||||
file_matches = compressor._parse_search_results(content)
|
||||
file_matches = _parse(content)
|
||||
|
||||
assert file_matches == {}
|
||||
|
||||
@@ -104,8 +123,7 @@ this is not a grep line
|
||||
src/utils.py:20:another valid line
|
||||
more random text
|
||||
"""
|
||||
compressor = SearchCompressor()
|
||||
file_matches = compressor._parse_search_results(content)
|
||||
file_matches = _parse(content)
|
||||
|
||||
assert "src/main.py" in file_matches
|
||||
assert "src/utils.py" in file_matches
|
||||
@@ -126,8 +144,7 @@ class TestContextLineBodyReference:
|
||||
|
||||
def test_body_reference_does_not_become_the_line_number(self):
|
||||
content = "app/settings.py-476-a:7:b:8:c"
|
||||
compressor = SearchCompressor()
|
||||
file_matches = compressor._parse_search_results(content)
|
||||
file_matches = _parse(content)
|
||||
|
||||
assert list(file_matches) == ["app/settings.py"]
|
||||
matches = file_matches["app/settings.py"].matches
|
||||
@@ -137,8 +154,7 @@ class TestContextLineBodyReference:
|
||||
|
||||
def test_extensionless_context_path_keeps_its_own_line_number(self):
|
||||
content = "CHANGELOG-12-a:99:b"
|
||||
compressor = SearchCompressor()
|
||||
file_matches = compressor._parse_search_results(content)
|
||||
file_matches = _parse(content)
|
||||
|
||||
assert list(file_matches) == ["CHANGELOG"]
|
||||
matches = file_matches["CHANGELOG"].matches
|
||||
@@ -149,8 +165,7 @@ class TestContextLineBodyReference:
|
||||
def test_colon_row_with_dashed_path_is_unaffected(self):
|
||||
"""The dash marker sits in the *path* here, so the colon tier still wins."""
|
||||
content = "logs/2026-05-03/app.log:12:ERROR"
|
||||
compressor = SearchCompressor()
|
||||
file_matches = compressor._parse_search_results(content)
|
||||
file_matches = _parse(content)
|
||||
|
||||
assert list(file_matches) == ["logs/2026-05-03/app.log"]
|
||||
matches = file_matches["logs/2026-05-03/app.log"].matches
|
||||
@@ -169,8 +184,7 @@ class TestContextLineBodyReference:
|
||||
``app.py-476-foo.rs`` at line 12.
|
||||
"""
|
||||
content = "app.py-476-foo.rs:12:ref"
|
||||
compressor = SearchCompressor()
|
||||
file_matches = compressor._parse_search_results(content)
|
||||
file_matches = _parse(content)
|
||||
|
||||
assert list(file_matches) == ["app.py"]
|
||||
matches = file_matches["app.py"].matches
|
||||
@@ -181,8 +195,7 @@ class TestContextLineBodyReference:
|
||||
def test_filename_style_body_reference_behind_a_directory(self):
|
||||
"""Same shape with a directory component, which also looks path-like."""
|
||||
content = "pkg/server.ts-91-lib/index.js:7:import"
|
||||
compressor = SearchCompressor()
|
||||
file_matches = compressor._parse_search_results(content)
|
||||
file_matches = _parse(content)
|
||||
|
||||
assert list(file_matches) == ["pkg/server.ts"]
|
||||
matches = file_matches["pkg/server.ts"].matches
|
||||
@@ -193,8 +206,7 @@ class TestContextLineBodyReference:
|
||||
def test_bare_path_body_keeps_the_context_coordinates(self):
|
||||
"""The body needs no reference at all to be path-like."""
|
||||
content = "app.py-476-./vendor/other.py"
|
||||
compressor = SearchCompressor()
|
||||
file_matches = compressor._parse_search_results(content)
|
||||
file_matches = _parse(content)
|
||||
|
||||
assert list(file_matches) == ["app.py"]
|
||||
matches = file_matches["app.py"].matches
|
||||
@@ -205,8 +217,7 @@ class TestContextLineBodyReference:
|
||||
def test_many_context_rows_keep_their_own_coordinates(self):
|
||||
"""Every coordinate an agent would act on survives across many rows."""
|
||||
content = "\n".join(f"pkg/mod/file.py-{476 + i * 7}-hits:{i}:of:9" for i in range(12))
|
||||
compressor = SearchCompressor()
|
||||
file_matches = compressor._parse_search_results(content)
|
||||
file_matches = _parse(content)
|
||||
|
||||
assert list(file_matches) == ["pkg/mod/file.py"]
|
||||
matches = file_matches["pkg/mod/file.py"].matches
|
||||
@@ -227,8 +238,7 @@ c.py:4:line 4
|
||||
b.py:5:line 5
|
||||
a.py:6:line 6
|
||||
"""
|
||||
compressor = SearchCompressor()
|
||||
file_matches = compressor._parse_search_results(content)
|
||||
file_matches = _parse(content)
|
||||
|
||||
assert len(file_matches) == 3
|
||||
assert len(file_matches["a.py"].matches) == 3
|
||||
@@ -275,95 +285,49 @@ class TestMatchScoring:
|
||||
"""Tests for match relevance scoring."""
|
||||
|
||||
def test_score_context_word_overlap(self):
|
||||
"""Matches containing context words get higher scores."""
|
||||
"""Matches containing context words rank higher."""
|
||||
content = """src/main.py:10:def process_data():
|
||||
src/main.py:20:def calculate_result():
|
||||
src/main.py:30:def handle_error():
|
||||
"""
|
||||
compressor = SearchCompressor()
|
||||
file_matches = compressor._parse_search_results(content)
|
||||
compressor._score_matches(file_matches, context="error handling")
|
||||
|
||||
matches = file_matches["src/main.py"].matches
|
||||
error_match = next(m for m in matches if "error" in m.content)
|
||||
data_match = next(m for m in matches if "data" in m.content)
|
||||
|
||||
# Error match should score higher with "error" context
|
||||
assert error_match.score > data_match.score
|
||||
result = _keep_best_one(boost_errors=False).compress(content, context="error handling")
|
||||
assert result.compressed.startswith("src/main.py:30:def handle_error():")
|
||||
|
||||
def test_score_error_patterns_boosted(self):
|
||||
"""Error/exception patterns get boosted scores."""
|
||||
"""Error/exception patterns rank higher."""
|
||||
content = """src/main.py:10:def normal_function():
|
||||
src/main.py:20:raise ValueError("error occurred")
|
||||
src/main.py:30:# TODO: fix this
|
||||
src/main.py:30:def other_function():
|
||||
"""
|
||||
compressor = SearchCompressor(config=SearchCompressorConfig(boost_errors=True))
|
||||
file_matches = compressor._parse_search_results(content)
|
||||
compressor._score_matches(file_matches, context="")
|
||||
|
||||
matches = file_matches["src/main.py"].matches
|
||||
error_match = next(m for m in matches if "error" in m.content.lower())
|
||||
normal_match = next(m for m in matches if "normal" in m.content)
|
||||
|
||||
assert error_match.score > normal_match.score
|
||||
result = _keep_best_one(boost_errors=True).compress(content)
|
||||
assert result.compressed.startswith('src/main.py:20:raise ValueError("error occurred")')
|
||||
|
||||
def test_score_warning_patterns(self):
|
||||
"""Warning patterns get boosted scores."""
|
||||
"""Warning patterns rank higher."""
|
||||
content = """src/main.py:10:def normal():
|
||||
src/main.py:20:# WARNING: deprecated
|
||||
src/main.py:30:def other():
|
||||
"""
|
||||
compressor = SearchCompressor()
|
||||
file_matches = compressor._parse_search_results(content)
|
||||
compressor._score_matches(file_matches, context="")
|
||||
|
||||
matches = file_matches["src/main.py"].matches
|
||||
warning_match = next(m for m in matches if "WARNING" in m.content)
|
||||
normal_match = next(m for m in matches if "normal" in m.content)
|
||||
|
||||
assert warning_match.score > normal_match.score
|
||||
result = _keep_best_one().compress(content)
|
||||
assert result.compressed.startswith("src/main.py:20:# WARNING: deprecated")
|
||||
|
||||
def test_score_todo_patterns(self):
|
||||
"""TODO/FIXME patterns get boosted scores."""
|
||||
"""TODO/FIXME patterns rank higher."""
|
||||
content = """src/main.py:10:def normal():
|
||||
src/main.py:20:# FIXME: this needs work
|
||||
src/main.py:30:# TODO: implement later
|
||||
src/main.py:30:def other():
|
||||
"""
|
||||
compressor = SearchCompressor()
|
||||
file_matches = compressor._parse_search_results(content)
|
||||
compressor._score_matches(file_matches, context="")
|
||||
|
||||
matches = file_matches["src/main.py"].matches
|
||||
fixme_match = next(m for m in matches if "FIXME" in m.content)
|
||||
normal_match = next(m for m in matches if "normal" in m.content)
|
||||
|
||||
assert fixme_match.score > normal_match.score
|
||||
result = _keep_best_one().compress(content)
|
||||
assert result.compressed.startswith("src/main.py:20:# FIXME: this needs work")
|
||||
|
||||
def test_score_context_keywords_config(self):
|
||||
"""context_keywords configuration boosts matching lines."""
|
||||
content = """src/main.py:10:def auth_handler():
|
||||
src/main.py:20:def data_processor():
|
||||
content = """src/main.py:10:def data_processor():
|
||||
src/main.py:20:def auth_handler():
|
||||
src/main.py:30:def other():
|
||||
"""
|
||||
config = SearchCompressorConfig(context_keywords=["auth", "security"])
|
||||
compressor = SearchCompressor(config=config)
|
||||
file_matches = compressor._parse_search_results(content)
|
||||
compressor._score_matches(file_matches, context="")
|
||||
|
||||
matches = file_matches["src/main.py"].matches
|
||||
auth_match = next(m for m in matches if "auth" in m.content)
|
||||
data_match = next(m for m in matches if "data" in m.content)
|
||||
|
||||
assert auth_match.score > data_match.score
|
||||
|
||||
def test_score_capped_at_one(self):
|
||||
"""Scores are capped at 1.0."""
|
||||
content = """src/main.py:10:ERROR FATAL exception fail warning TODO FIXME
|
||||
"""
|
||||
compressor = SearchCompressor()
|
||||
file_matches = compressor._parse_search_results(content)
|
||||
compressor._score_matches(file_matches, context="error fatal exception")
|
||||
|
||||
match = file_matches["src/main.py"].matches[0]
|
||||
assert match.score <= 1.0
|
||||
result = _keep_best_one(context_keywords=["auth", "security"]).compress(content)
|
||||
assert result.compressed.startswith("src/main.py:20:def auth_handler():")
|
||||
|
||||
|
||||
class TestMatchSelection:
|
||||
@@ -679,8 +643,7 @@ src/file (1).py:40:content
|
||||
"""Negative line numbers don't match the pattern."""
|
||||
content = "src/file.py:-1:invalid"
|
||||
|
||||
compressor = SearchCompressor()
|
||||
file_matches = compressor._parse_search_results(content)
|
||||
file_matches = _parse(content)
|
||||
|
||||
# Pattern requires \d+ which is positive integers only
|
||||
assert len(file_matches) == 0
|
||||
@@ -715,16 +678,16 @@ class TestContextIntegration:
|
||||
|
||||
def test_short_context_words_ignored(self):
|
||||
"""Context words <= 2 chars are ignored for scoring."""
|
||||
content = """src/file.py:10:a = 1
|
||||
src/file.py:20:do something important
|
||||
content = """src/file.py:10:plain line one
|
||||
src/file.py:20:has an xy token
|
||||
src/file.py:30:plain line three
|
||||
"""
|
||||
compressor = SearchCompressor()
|
||||
file_matches = compressor._parse_search_results(content)
|
||||
compressor._score_matches(file_matches, context="a")
|
||||
|
||||
# Short context word "a" shouldn't cause errors or abnormal scoring
|
||||
matches = file_matches["src/file.py"].matches
|
||||
assert all(m.score <= 1.0 for m in matches)
|
||||
# "xy" is too short to score, so nothing outranks the first line.
|
||||
result = _keep_best_one(boost_errors=False).compress(content, context="xy")
|
||||
assert result.compressed.startswith("src/file.py:10:plain line one")
|
||||
# A longer word from the same line does score.
|
||||
result = _keep_best_one(boost_errors=False).compress(content, context="token")
|
||||
assert result.compressed.startswith("src/file.py:20:has an xy token")
|
||||
|
||||
|
||||
class TestOutputFormatting:
|
||||
@@ -839,21 +802,14 @@ class TestConfigOptions:
|
||||
|
||||
def test_disable_error_boost(self):
|
||||
"""boost_errors=False doesn't prioritize error patterns."""
|
||||
content = """src/file.py:1:ERROR critical failure
|
||||
src/file.py:2:normal code line
|
||||
content = """src/file.py:1:normal code line
|
||||
src/file.py:2:ERROR critical failure
|
||||
src/file.py:3:more normal code
|
||||
"""
|
||||
compressor = SearchCompressor(
|
||||
config=SearchCompressorConfig(
|
||||
boost_errors=False,
|
||||
)
|
||||
)
|
||||
file_matches = compressor._parse_search_results(content)
|
||||
compressor._score_matches(file_matches, context="")
|
||||
|
||||
matches = file_matches["src/file.py"].matches
|
||||
# Without boost, both should have similar (low) scores
|
||||
error_match = next(m for m in matches if "ERROR" in m.content)
|
||||
assert error_match.score == 0.0 # No boost applied
|
||||
boosted = _keep_best_one(boost_errors=True).compress(content)
|
||||
assert boosted.compressed.startswith("src/file.py:2:ERROR critical failure")
|
||||
unboosted = _keep_best_one(boost_errors=False).compress(content)
|
||||
assert unboosted.compressed.startswith("src/file.py:1:normal code line")
|
||||
|
||||
def test_min_matches_for_ccr(self):
|
||||
"""min_matches_for_ccr threshold is respected."""
|
||||
|
||||
@@ -2,35 +2,39 @@
|
||||
|
||||
The relevance scorer tokenized the query on whitespace, so a spaceless CJK query
|
||||
matched content only when the WHOLE query was a literal substring of a line. CJK
|
||||
char bigrams now let a longer CJK query boost lines that share a substring. The
|
||||
Rust<->Python parity was also hardened (dedup like Python's set; char-length
|
||||
filter instead of bytes). These exercise the Python legacy scorer that mirrors
|
||||
Rust.
|
||||
char bigrams now let a longer CJK query boost lines that share a substring.
|
||||
Bigram extraction itself is unit-tested in Rust (``cjk_bigrams_from_runs``);
|
||||
this pins the end-to-end ranking through ``compress()``.
|
||||
"""
|
||||
|
||||
from headroom.transforms.search_compressor import (
|
||||
SearchCompressor,
|
||||
SearchCompressorConfig,
|
||||
_cjk_bigrams,
|
||||
)
|
||||
|
||||
|
||||
def test_cjk_bigrams_from_runs():
|
||||
assert _cjk_bigrams("认证令牌") == {"认证", "证令", "令牌"}
|
||||
assert _cjk_bigrams("hello world") == set() # ASCII -> no CJK bigrams
|
||||
assert _cjk_bigrams("a认b证") == set() # isolated CJK chars -> no adjacent pair
|
||||
|
||||
|
||||
def test_score_matches_cjk_query_bigrams_boost():
|
||||
compressor = SearchCompressor(SearchCompressorConfig(boost_errors=False, context_keywords=[]))
|
||||
def test_cjk_query_bigrams_boost_ranking():
|
||||
# Keep only the top-scored match per file, so the output shows the ranking.
|
||||
compressor = SearchCompressor(
|
||||
SearchCompressorConfig(
|
||||
max_matches_per_file=1,
|
||||
always_keep_first=False,
|
||||
always_keep_last=False,
|
||||
boost_errors=False,
|
||||
context_keywords=[],
|
||||
enable_ccr=False,
|
||||
)
|
||||
)
|
||||
content = "\n".join(
|
||||
[
|
||||
"src/a.py:10:认证令牌已过期需要重新登录",
|
||||
"src/b.py:2:plain ascii content here",
|
||||
"src/a.py:10:plain ascii content here",
|
||||
"src/a.py:20:认证令牌已过期需要重新登录",
|
||||
"src/a.py:30:more plain content",
|
||||
]
|
||||
)
|
||||
parsed = compressor._parse_search_results(content)
|
||||
# the whole query is NOT a substring of the content line, but its bigrams are
|
||||
compressor._score_matches(parsed, "认证令牌缓存淘汰策略")
|
||||
assert parsed["src/a.py"].matches[0].score > 0 # 认证/证令/令牌 bigrams match
|
||||
assert parsed["src/b.py"].matches[0].score == 0
|
||||
# The whole query is NOT a substring of the CJK line, but its bigrams are.
|
||||
result = compressor.compress(content, context="认证令牌缓存淘汰策略")
|
||||
assert result.compressed.startswith("src/a.py:20:认证令牌已过期需要重新登录")
|
||||
# Without a matching query the earliest line wins the tie.
|
||||
baseline = compressor.compress(content, context="缓存淘汰策略")
|
||||
assert baseline.compressed.startswith("src/a.py:10:plain ascii content here")
|
||||
|
||||
@@ -5,21 +5,20 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
|
||||
from headroom.transforms.search_compressor import (
|
||||
FileMatches,
|
||||
SearchCompressionResult,
|
||||
SearchCompressor,
|
||||
SearchCompressorConfig,
|
||||
SearchMatch,
|
||||
)
|
||||
|
||||
|
||||
def test_parse_score_select_and_format_search_results(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def test_compress_scores_caps_and_summarizes_search_results() -> None:
|
||||
compressor = SearchCompressor(
|
||||
SearchCompressorConfig(
|
||||
max_matches_per_file=3,
|
||||
max_total_matches=4,
|
||||
max_files=2,
|
||||
context_keywords=["auth"],
|
||||
enable_ccr=False,
|
||||
)
|
||||
)
|
||||
content = "\n".join(
|
||||
@@ -27,45 +26,21 @@ def test_parse_score_select_and_format_search_results(monkeypatch: pytest.Monkey
|
||||
"src/auth.py:10:ERROR auth failed",
|
||||
"src/auth.py-11-warning auth retry",
|
||||
"src/auth.py:12:plain auth line",
|
||||
"src/auth.py:13:another auth line",
|
||||
"src/db.py:2:warning token expired",
|
||||
"not a match",
|
||||
]
|
||||
)
|
||||
parsed = compressor._parse_search_results(content)
|
||||
assert set(parsed) == {"src/auth.py", "src/db.py"}
|
||||
assert parsed["src/auth.py"].first == SearchMatch(
|
||||
file="src/auth.py", line_number=10, content="ERROR auth failed"
|
||||
)
|
||||
assert parsed["src/auth.py"].last.line_number == 12
|
||||
result = compressor.compress(content, context="find auth error")
|
||||
|
||||
compressor._score_matches(parsed, "find auth error")
|
||||
assert parsed["src/auth.py"].matches[0].score == 1.0
|
||||
assert parsed["src/db.py"].matches[0].score > 0
|
||||
|
||||
monkeypatch.setitem(
|
||||
__import__("sys").modules,
|
||||
"headroom.transforms.adaptive_sizer",
|
||||
SimpleNamespace(compute_optimal_k=lambda items, **kwargs: 4),
|
||||
)
|
||||
selected = compressor._select_matches(parsed, bias=1.2)
|
||||
assert list(selected) == ["src/auth.py", "src/db.py"]
|
||||
assert [m.line_number for m in selected["src/auth.py"].matches] == [10, 11, 12]
|
||||
|
||||
formatted, summaries = compressor._format_output(
|
||||
selected,
|
||||
{
|
||||
**parsed,
|
||||
"src/db.py": FileMatches(
|
||||
file="src/db.py",
|
||||
matches=[
|
||||
SearchMatch(file="src/db.py", line_number=2, content="warning token expired"),
|
||||
SearchMatch(file="src/db.py", line_number=3, content="another line"),
|
||||
],
|
||||
),
|
||||
},
|
||||
)
|
||||
assert "src/auth.py:10:ERROR auth failed" in formatted
|
||||
assert summaries["src/db.py"] == "[... and 1 more matches in src/db.py]"
|
||||
# "not a match" is not a search row; both files are parsed.
|
||||
assert result.original_match_count == 5
|
||||
assert result.files_affected == 2
|
||||
# The error line ranks first and is kept; auth.py is capped at 3 matches.
|
||||
assert "src/auth.py:10:ERROR auth failed" in result.compressed
|
||||
assert "src/db.py:2:warning token expired" in result.compressed
|
||||
assert result.summaries == {"src/auth.py": "[... and 1 more matches in src/auth.py]"}
|
||||
assert "[... and 1 more matches in src/auth.py]" in result.compressed
|
||||
|
||||
|
||||
def test_search_compressor_compress_paths_and_ccr() -> None:
|
||||
|
||||
Reference in New Issue
Block a user