mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
test(router): repin the code-to-Kompress tests and add the #4214 fence regression
oxura's change removes the CODE_AWARE -> Kompress fallbacks. Existing tests pinned the old fallback behaviour; rewrite them to assert the new one (code is left unchanged and Kompress never sees it) instead of deleting them, and add a regression for the fenced script quoted in #4214 with enable_code_aware both off and on. Update wiki/proxy.md and docs/content/docs/code-compression.mdx, which still said detected code falls back to Kompress. Co-authored-by: oxura <grozafroza44@gmail.com>
This commit is contained in:
@@ -169,5 +169,5 @@ unload_tree_sitter()
|
||||
| Syntax validity | Guaranteed |
|
||||
|
||||
<Callout type="info" title="Routing differs between the proxy and the library API">
|
||||
ContentRouter always *detects* source code automatically, but whether it routes that code to CodeAwareCompressor depends on the entry point. `headroom proxy` enables code-aware routing by default (`HEADROOM_CODE_AWARE_ENABLED` defaults to on; disable with `--no-code-aware` or `HEADROOM_CODE_AWARE_ENABLED=0`). The one-function `compress()` API and `HeadroomClient`, however, build `ContentRouterConfig` with its own default, `enable_code_aware=False` -- there, detected code falls back to Kompress unless you construct your own pipeline with `ContentRouterConfig(enable_code_aware=True)`. Direct usage of `CodeAwareCompressor`, as shown on this page, is unaffected either way. See [How Compression Works](/docs/how-compression-works#content-type-detection).
|
||||
ContentRouter always *detects* source code automatically, but whether it routes that code to CodeAwareCompressor depends on the entry point. `headroom proxy` enables code-aware routing by default (`HEADROOM_CODE_AWARE_ENABLED` defaults to on; disable with `--no-code-aware` or `HEADROOM_CODE_AWARE_ENABLED=0`). The one-function `compress()` API and `HeadroomClient`, however, build `ContentRouterConfig` with its own default, `enable_code_aware=False` -- there, detected code is left unchanged (it is never handed to the prose model, Kompress, which drops keywords) unless you construct your own pipeline with `ContentRouterConfig(enable_code_aware=True)`. Direct usage of `CodeAwareCompressor`, as shown on this page, is unaffected either way. See [How Compression Works](/docs/how-compression-works#content-type-detection).
|
||||
</Callout>
|
||||
|
||||
@@ -29,7 +29,7 @@ The router auto-detects content type by analyzing structure and patterns. No man
|
||||
| Content Type | Detection Signal | Compressor | Typical Savings |
|
||||
|---|---|---|---|
|
||||
| JSON arrays | Valid JSON with array elements | SmartCrusher | Varies with redundancy -- measured 26-54% on realistic (non-duplicate) tool-output arrays of 100-10,000 items; highly repetitive arrays compress far more. See `benchmarks/compression_benchmark.py`. |
|
||||
| Source code | Syntax patterns, indentation, keywords | Kompress via the library `compress()`/`HeadroomClient` path (`enable_code_aware` defaults to `False` there); CodeAwareCompressor via `headroom proxy` (`HEADROOM_CODE_AWARE_ENABLED` defaults to on) | workload-dependent |
|
||||
| Source code | Syntax patterns, indentation, keywords | Left unchanged (passthrough) via the library `compress()`/`HeadroomClient` path (`enable_code_aware` defaults to `False` there); CodeAwareCompressor via `headroom proxy` (`HEADROOM_CODE_AWARE_ENABLED` defaults to on). Source code is never sent to Kompress. | workload-dependent |
|
||||
| Search results | `file:line:content` format | SearchCompressor | 80-95% (measured ~91% on a synthetic 40-file ripgrep-style corpus) |
|
||||
| Build/test logs | Timestamps, log levels, pytest/npm markers | LogCompressor | 85-95% (measured ~93% on a synthetic 1,000-line log with clustered errors) |
|
||||
| Diffs | Unified diff format | DiffCompressor | 60-80% (measured ~70% on a multi-file diff with default `max_context_lines`/`max_hunks_per_file` limits) |
|
||||
|
||||
@@ -1589,6 +1589,37 @@ class TestSmartCrusherFallback:
|
||||
f"{strategy_chain}"
|
||||
)
|
||||
|
||||
def test_code_aware_declining_never_falls_back_to_kompress(self, router, monkeypatch):
|
||||
"""Code-aware declining a block leaves it unchanged; prose ML never runs (#4214).
|
||||
|
||||
Replaces the old "no double Kompress" check: with the CODE_AWARE ->
|
||||
Kompress fallbacks removed, Kompress must not run even once.
|
||||
"""
|
||||
import headroom.transforms.content_router as crm
|
||||
|
||||
ml_calls: list[str] = []
|
||||
|
||||
def spy(self, content, context="", question=None):
|
||||
ml_calls.append(content)
|
||||
return content, len(content.split())
|
||||
|
||||
monkeypatch.setattr(crm.ContentRouter, "_try_ml_compressor", spy)
|
||||
|
||||
plain = "This is just plain text. " * 200
|
||||
|
||||
compressed, _compressed_tokens, strategy_chain, _reversible = (
|
||||
router._apply_strategy_to_content(
|
||||
plain,
|
||||
CompressionStrategy.CODE_AWARE,
|
||||
context="",
|
||||
)
|
||||
)
|
||||
|
||||
assert "code_aware" in strategy_chain
|
||||
assert "kompress" not in strategy_chain
|
||||
assert ml_calls == []
|
||||
assert compressed == plain
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TestCompressBlockContent — PR #704 shared-path regression
|
||||
|
||||
@@ -2829,6 +2829,14 @@ def test_json_in_html_preserves_records_and_markup() -> None:
|
||||
assert not result.unrecoverable_sections
|
||||
|
||||
|
||||
# A Claude Code Read of Rust: line-numbered code that code_aware hands back
|
||||
# unchanged (it cannot parse the number column) while reporting compressed=True.
|
||||
# Code runs ~2 tokens per whitespace word, so a word count badly undercounts it.
|
||||
_NUMBERED_RUST = "\n".join(
|
||||
f"{i}\tlet value_{i} = compute(&items[{i}..], Some(opts.clone()))?;" for i in range(1, 121)
|
||||
)
|
||||
|
||||
|
||||
def _noop_code_aware_router(monkeypatch: pytest.MonkeyPatch, kompress_out: str) -> ContentRouter:
|
||||
monkeypatch.delenv("HEADROOM_LOSSLESS_THEN_LOSSY", raising=False)
|
||||
router = ContentRouter(ContentRouterConfig(enable_code_aware=True))
|
||||
@@ -2850,6 +2858,52 @@ def _noop_code_aware_router(monkeypatch: pytest.MonkeyPatch, kompress_out: str)
|
||||
return router
|
||||
|
||||
|
||||
@pytest.mark.parametrize("layer_lossy", [False, True], ids=["default", "lossless-then-lossy"])
|
||||
def test_code_aware_noop_never_hands_code_to_kompress(
|
||||
monkeypatch: pytest.MonkeyPatch, layer_lossy: bool
|
||||
) -> None:
|
||||
"""A code_aware no-op leaves the block unchanged even if Kompress would shrink it (#4214).
|
||||
|
||||
Kompress is a prose model: it drops keywords, so it never sees code, neither as
|
||||
the CODE_AWARE fallback nor as the lossless-then-lossy inline attempt.
|
||||
"""
|
||||
kompressed = "\n".join(f"{i}\tvalue_{i} compute(&items[{i}..])" for i in range(1, 121))
|
||||
assert _estimate_tokens(kompressed) < _estimate_tokens(_NUMBERED_RUST)
|
||||
router = _noop_code_aware_router(monkeypatch, kompressed)
|
||||
router._lossless_then_lossy = layer_lossy
|
||||
calls: list[str] = []
|
||||
kompress = router._try_ml_compressor
|
||||
|
||||
def counting(content: str, *args: object, **kwargs: object) -> tuple[str, int]:
|
||||
calls.append(content)
|
||||
return kompress(content, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(router, "_try_ml_compressor", counting)
|
||||
|
||||
compressed, compressed_tokens, strategy_chain, _reversible = router._apply_strategy_to_content(
|
||||
_NUMBERED_RUST, CompressionStrategy.CODE_AWARE, context=""
|
||||
)
|
||||
|
||||
assert compressed == _NUMBERED_RUST
|
||||
assert compressed_tokens == _estimate_tokens(_NUMBERED_RUST)
|
||||
assert strategy_chain == ["code_aware"]
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_code_aware_noop_reports_tokens_without_kompress(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""An unchanged block is reported at its own token count, never its word count."""
|
||||
router = _noop_code_aware_router(monkeypatch, _NUMBERED_RUST)
|
||||
|
||||
compressed, compressed_tokens, _chain, _reversible = router._apply_strategy_to_content(
|
||||
_NUMBERED_RUST, CompressionStrategy.CODE_AWARE, context=""
|
||||
)
|
||||
|
||||
assert compressed == _NUMBERED_RUST
|
||||
assert compressed_tokens == _estimate_tokens(_NUMBERED_RUST)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("enabled", "preferred", "layer_lossy"),
|
||||
[(False, True, False), (True, True, False), (True, False, False), (True, True, True)],
|
||||
@@ -2881,6 +2935,75 @@ def test_fenced_code_keeps_keywords_when_code_compression_declines(
|
||||
assert f"```python\n{code}\n```" in result.compressed
|
||||
|
||||
|
||||
# The ctx_execute shape from #4214: a fenced Python script followed by its output.
|
||||
# The reporter's own 1324-character fence is not published, so this is a
|
||||
# reconstruction: it keeps the four lines quoted in the issue, parses, and is
|
||||
# exactly 1324 characters. The fake Kompress deletes the keywords the issue lists.
|
||||
_ISSUE_4214_SCRIPT = """import json, collections
|
||||
|
||||
d = json.load(open('turns.json'))
|
||||
segs=[s for s in open('segs.txt').read().split() if s]
|
||||
def load(path):
|
||||
try:
|
||||
return json.load(open(path))
|
||||
except Exception as e: return None
|
||||
for i,s in enumerate(segs):
|
||||
t0=(d or {}).get('turns') or []
|
||||
if not t0 or len(t0) < 2:
|
||||
continue
|
||||
else:
|
||||
label = s if s else 'unknown'
|
||||
counts = collections.Counter(x.get('role') for x in t0)
|
||||
for role, n in counts.items():
|
||||
if n > 1 or role == 'tool':
|
||||
print(i, label, role, n)
|
||||
else:
|
||||
print(i, label, role)
|
||||
total = sum(counts.values())
|
||||
if total > 10 or label == 'big':
|
||||
print('large session', label, total)
|
||||
out = [k for k in counts if counts[k] > 0]
|
||||
print(json.dumps(out))
|
||||
summary = {s: i for i, s in enumerate(segs)}
|
||||
for k, v in summary.items():
|
||||
if v % 2 == 0 or k.startswith('a'):
|
||||
print(k, v)
|
||||
print('done', len(segs))"""
|
||||
_ISSUE_4214_FENCE = _ISSUE_4214_SCRIPT + "\n#" + "x" * (1324 - len(_ISSUE_4214_SCRIPT) - 2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enabled", [False, True], ids=["code-aware-off", "code-aware-on"])
|
||||
def test_issue_4214_fenced_script_survives_kompress_unchanged(
|
||||
monkeypatch: pytest.MonkeyPatch, enabled: bool
|
||||
) -> None:
|
||||
import ast
|
||||
import re
|
||||
|
||||
assert len(_ISSUE_4214_FENCE) == 1324
|
||||
ast.parse(_ISSUE_4214_FENCE)
|
||||
monkeypatch.delenv("HEADROOM_LOSSLESS_THEN_LOSSY", raising=False)
|
||||
router = ContentRouter(ContentRouterConfig(enable_code_aware=enabled))
|
||||
seen: list[str] = []
|
||||
|
||||
def drop_keywords(content: str, *_args: object, **_kwargs: object) -> tuple[str, int]:
|
||||
seen.append(content)
|
||||
dropped = re.sub(r"\b(for|in|if|as|or|else) ?", "", content)
|
||||
return dropped, _estimate_tokens(dropped)
|
||||
|
||||
monkeypatch.setattr(router, "_try_ml_compressor", drop_keywords)
|
||||
output = "\n".join(
|
||||
f"Segment {i}: The job read the batch, checked each record against the schema "
|
||||
"and wrote a summary line for the report because the operator asked."
|
||||
for i in range(40)
|
||||
)
|
||||
content = f"```python\n{_ISSUE_4214_FENCE}\n```\n\n{output}"
|
||||
|
||||
result = router.compress(content, context="")
|
||||
|
||||
assert f"```python\n{_ISSUE_4214_FENCE}\n```" in result.compressed
|
||||
assert not any("def load" in section for section in seen)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("as_block", [False, True], ids=["string-tool", "tool-result-block"])
|
||||
@pytest.mark.parametrize("other_marker", [False, True], ids=["no-marker", "unrelated-marker"])
|
||||
@pytest.mark.parametrize("ccr_enabled", [False, True], ids=["explicit-no-ccr", "reversible-tools"])
|
||||
|
||||
Reference in New Issue
Block a user