mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix: clarify CCR marker content preservation (#3175)
## Description Closes #3098 CCR compression markers currently tell agents that content was compressed and can be retrieved, but do not explicitly state that the original content is preserved. This can cause agents to interpret the marker as data corruption or truncation and perform unnecessary verification work. This change adds `Original content preserved.` to the Kompress CCR marker while preserving the existing `Retrieve more: hash=<24-hex>` retrieval protocol. ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - Added `Original content preserved.` to CCR markers emitted by: - `KompressCompressor.compress()` - `KompressCompressor.compress_batch()` - `RemoteKompressCompressor.compress()` - Updated the primary `CCRToolInjector` marker regex to recognize the new marker wording while retaining compatibility with existing marker formats. - Added regression tests covering all three producer paths and new marker detection. - No changes to compression, CCR storage, hashing, retrieval, TTL, or eviction behavior. ## Testing - [ ] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - [ ] Type checking passes (`mypy headroom`) - [x] New tests added for new functionality - [x] Manual testing performed ### Test Output ```text Targeted pytest suites: tests/test_ccr_marker_policy.py tests/test_ccr_tool_injection.py tests/test_remote_kompress_dropin.py 64 passed in 3.84s ruff check: All checks passed! ruff format --check: 1356 files already formatted ```` ## Real Behavior Proof - Environment: Windows 11, Python 3.14.2, Node 24.14.0, uv 0.11.32. - Exact command / steps: Ran the targeted CCR pytest suite with `python -m pytest tests/test_ccr_marker_policy.py tests/test_ccr_tool_injection.py tests/test_remote_kompress_dropin.py`; ran `uv tool run ruff check headroom tests` and `uv tool run ruff format --check headroom tests`; additionally exercised CCR compression, marker generation, `CCRToolInjector` detection, and `CompressionStore` retrieval with a synthetic tool result. - Observed result: The targeted suite passed with 64 tests passing; Ruff check passed; formatting check passed; the generated marker contained `Original content preserved.`, `CCRToolInjector` detected its hash, and `CompressionStore` retrieved the original 730-character input with a verbatim match. - Not tested: The full repository test suite, upstream cloud LLM billing calls, and neural network retraining/inference quality were not tested; this change only modifies CCR marker wording and parser compatibility. ## Runtime Rollout Safety - Rollout-managed feature(s): None; this is a static CCR marker wording and parser compatibility change. - Minimum rollout channel: Standard release; no staged rollout is required. - Stable/default behavior changed: Yes; the default Kompress CCR marker now explicitly states that original content is preserved. - Kill switch / disable path: None required; reverting the change restores the previous marker wording. - Unsafe override required: No. - Qualification impact: None; compression, CCR storage, hashing, retrieval, TTL, and eviction behavior are unchanged. - Rollback path: Revert commit `4bef0f3a` to restore the previous marker wording. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist * [x] My code follows the project's style guidelines * [ ] I have performed a self-review of my code * [ ] I have commented my code, particularly in hard-to-understand areas * [ ] I have made corresponding changes to the documentation * [x] My changes generate no new warnings * [x] I have added tests that prove my fix is effective or that my feature works * [x] New and existing targeted unit tests pass locally with my changes * [x] I did **not** edit `CHANGELOG.md` — it is generated by release-please from my Conventional Commit PR title (a CI guard enforces this) ## Screenshots (if applicable) N/A — this is a model-facing marker/protocol change. ## Additional Notes The compressed representation itself remains intentionally condensed. The integrity statement refers to the original content preserved in CCR storage, which remains retrievable through the existing hash-based retrieval mechanism. Co-authored-by: JerrettDavis <mxjerrett@gmail.com>
This commit is contained in:
co-authored by
JerrettDavis
parent
f0ec2bb376
commit
005a4e146f
@@ -235,7 +235,9 @@ class CCRToolInjector:
|
||||
#
|
||||
# Standard format: [N <type> compressed to M. Retrieve more: hash=xxx]
|
||||
# Matches items, lines, matches, or any other type
|
||||
re.compile(r"\[(\d+) \w+ compressed to (\d+)\. Retrieve more: hash=([a-f0-9]{24})\]"),
|
||||
re.compile(
|
||||
r"\[(\d+) \w+ compressed to (\d+)(?: \([^)]+\))?\.(?: Original content preserved\.)? Retrieve more: hash=([a-f0-9]{24})\]"
|
||||
),
|
||||
# Legacy format without "to M" or "Retrieve more:" (old TextCompressor)
|
||||
re.compile(r"\[(\d+) \w+ compressed\. hash=([a-f0-9]{24})\]"),
|
||||
# Generic fallback: any bracket compression marker with hash (exactly 24 chars)
|
||||
|
||||
@@ -1370,6 +1370,7 @@ def ccr_retrieval_marker(
|
||||
return (
|
||||
f"\n[{n_words} words compressed to {compressed_count}"
|
||||
f" (from {source_lines} source {line_word})."
|
||||
" Original content preserved."
|
||||
f" Retrieve more: hash={cache_key}]"
|
||||
)
|
||||
|
||||
|
||||
@@ -34,6 +34,125 @@ def test_source_line_span_marker_is_still_detected() -> None:
|
||||
assert _hashes(marker) == ["c00eb437e5e5c00eb437e5e5"]
|
||||
|
||||
|
||||
def test_new_marker_with_integrity_clause_is_detected() -> None:
|
||||
# Issue #3098: marker includes "Original content preserved."
|
||||
marker = (
|
||||
"[122 items compressed to 27 (from 5 source lines). "
|
||||
"Original content preserved. Retrieve more: hash=c00eb437e5e5c00eb437e5e5]"
|
||||
)
|
||||
assert _hashes(marker) == ["c00eb437e5e5c00eb437e5e5"]
|
||||
|
||||
|
||||
class _FakeEncoding:
|
||||
def __init__(self, word_lists: list[list[str]]):
|
||||
self._word_lists = word_lists
|
||||
self._ids = [[0] * len(w) for w in word_lists]
|
||||
|
||||
def __getitem__(self, key: str):
|
||||
return {"input_ids": self._ids, "attention_mask": self._ids}[key]
|
||||
|
||||
def word_ids(self, batch_index: int = 0) -> list[int]:
|
||||
return list(range(len(self._word_lists[batch_index])))
|
||||
|
||||
|
||||
class _FakeTokenizer:
|
||||
def __call__(self, words: list[str] | list[list[str]], **_kwargs) -> _FakeEncoding:
|
||||
batch = words if (words and isinstance(words[0], list)) else [words] # type: ignore[list-item]
|
||||
return _FakeEncoding(batch) # type: ignore[arg-type]
|
||||
|
||||
|
||||
class _FakeModel:
|
||||
"""Keep the first two words of each chunk -> ratio < 0.8 (CCR fires)."""
|
||||
|
||||
def get_keep_mask(
|
||||
self, input_ids: list[list[int]], _attention_mask: object
|
||||
) -> list[list[bool]]:
|
||||
return [[i < 2 for i in range(len(input_ids[0]))]]
|
||||
|
||||
def get_scores(self, input_ids: list[list[int]], _attention_mask: object) -> list[list[float]]:
|
||||
return [[1.0 if i < 2 else 0.0 for i in range(len(row))] for row in input_ids]
|
||||
|
||||
|
||||
def test_remote_kompress_compressor_emits_original_content_preserved(monkeypatch) -> None:
|
||||
from headroom.transforms.kompress_remote import RemoteKompressCompressor
|
||||
|
||||
class _FakeResponse:
|
||||
status_code = 200
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
return None
|
||||
|
||||
def json(self) -> dict:
|
||||
return {"compressed": "short text", "compression_ratio": 0.2, "original_tokens": 100}
|
||||
|
||||
class _FakeClient:
|
||||
def post(self, url, headers=None, json=None): # noqa: A002, ANN001
|
||||
return _FakeResponse()
|
||||
|
||||
def close(self) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(
|
||||
"headroom.transforms.kompress_remote.store_kompress_in_ccr",
|
||||
lambda original, compressed, tokens: "cafebabecafebabecafebabe",
|
||||
)
|
||||
c = RemoteKompressCompressor("https://ml.example.invalid")
|
||||
c._client = _FakeClient()
|
||||
c.config.enable_ccr = True
|
||||
|
||||
result = c.compress("line one\nline two\nline three " * 20)
|
||||
assert (
|
||||
"Original content preserved. Retrieve more: hash=cafebabecafebabecafebabe"
|
||||
in result.compressed
|
||||
)
|
||||
|
||||
|
||||
def test_kompress_compressor_emits_original_content_preserved(monkeypatch) -> None:
|
||||
from headroom.transforms.kompress_compressor import KompressCompressor
|
||||
|
||||
compressor = KompressCompressor()
|
||||
compressor.config.enable_ccr = True
|
||||
monkeypatch.setattr(
|
||||
compressor,
|
||||
"_store_in_ccr",
|
||||
lambda original, compressed, tokens: "cafebabecafebabecafebabe",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"headroom.transforms.kompress_compressor._load_kompress",
|
||||
lambda *a, **k: (_FakeModel(), _FakeTokenizer(), "onnx"),
|
||||
)
|
||||
|
||||
result = compressor.compress("line one\nline two\nline three " * 20)
|
||||
assert (
|
||||
"Original content preserved. Retrieve more: hash=cafebabecafebabecafebabe"
|
||||
in result.compressed
|
||||
)
|
||||
|
||||
|
||||
def test_kompress_compressor_batch_emits_original_content_preserved(monkeypatch) -> None:
|
||||
from headroom.transforms.kompress_compressor import KompressCompressor
|
||||
|
||||
compressor = KompressCompressor()
|
||||
compressor.config.enable_ccr = True
|
||||
monkeypatch.setattr(compressor, "_should_use_sequential_fallback", lambda: False)
|
||||
monkeypatch.setattr(
|
||||
compressor,
|
||||
"_store_in_ccr",
|
||||
lambda original, compressed, tokens: "cafebabecafebabecafebabe",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"headroom.transforms.kompress_compressor._load_kompress",
|
||||
lambda *a, **k: (_FakeModel(), _FakeTokenizer(), "onnx"),
|
||||
)
|
||||
|
||||
results = compressor.compress_batch(["line one\nline two\nline three " * 20])
|
||||
assert len(results) == 1
|
||||
assert (
|
||||
"Original content preserved. Retrieve more: hash=cafebabecafebabecafebabe"
|
||||
in results[0].compressed
|
||||
)
|
||||
|
||||
|
||||
def test_has_new_ccr_markers_detects_hash_not_seen_in_previous_forward() -> None:
|
||||
old = "[100 items compressed to 10. Retrieve more: hash=abc123def456abc123def456]"
|
||||
new = "[50 items compressed to 5. Retrieve more: hash=deadbeefdeadbeefdeadbeef]"
|
||||
|
||||
@@ -97,7 +97,7 @@ def _tok(text: str) -> int:
|
||||
def test_payload_tokens_is_cl100k_or_one_per_character(monkeypatch):
|
||||
marker = ccr_retrieval_marker(100, 59, SINGLE_TOKEN_SOURCE, _real_key(SINGLE_TOKEN_SOURCE))
|
||||
assert _real_key(SINGLE_TOKEN_SOURCE) == "7dbb8f8de9f3e1d7c6f3a6e1"
|
||||
assert payload_tokens(marker) == _tok(marker) == 43
|
||||
assert payload_tokens(marker) == _tok(marker) == 47
|
||||
assert payload_tokens(SINGLE_TOKEN_SOURCE) == _tok(SINGLE_TOKEN_SOURCE) == 100
|
||||
monkeypatch.setattr(kc, "_payload_encoder", False)
|
||||
assert payload_tokens(marker) == len(marker)
|
||||
|
||||
Reference in New Issue
Block a user