mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 18:41:07 +08:00
## Description The router could reuse an earlier compressed result even when the next request disabled Kompress or requested a different model, bias, or context. Include the effective request inputs in cache identity so results and skip decisions stay within the policy that produced them. Refs #4477, part (a). Reversibility-gate changes in part (b) are outside this patch. ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) ## Changes Made - Add model, force/skip flags, effective bias, and context to deterministic cache keys; JSON framing prevents delimiter collisions - Use one helper for string, tool-result, and text-block keys, covering memory/disk results, skips, and frozen verdicts - Add public `apply()` regressions for policy changes and same-policy reuse, actual recorded frozen verdicts, tool/hook bias, enriched relevance context, process determinism, and reopened SQLite cache ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - [ ] Type checking passes (`mypy headroom`) - [x] New tests added for new functionality - [x] Manual testing performed ### Test Output ```text # Before patch, initial 51-case regression file: 45 failed, 6 passed in 1.47s # After patch, including 3 explicit frozen-verdict regressions: 54 passed HEADROOM_OFFLINE=1 TIKTOKEN_CACHE_DIR=/private/tmp/headroom-tiktoken PYTHONPATH=. ../headroom/.venv/bin/python -m pytest tests/test_router_cache_policy.py tests/test_router_result_cache.py tests/test_transforms_content_router.py tests/test_content_router_concurrency.py tests/test_content_router_user_blocks.py -q -rs --tb=short 172 passed, 2 skipped in 12.14s # Both skips: trafilatura is not installed (existing HTML tests). ../headroom/.venv/bin/ruff check . All checks passed! ../headroom/.venv/bin/ruff format --check headroom/transforms/content_router.py tests/test_router_cache_policy.py 2 files already formatted git diff --check # exit 0 ``` ## Real Behavior Proof - Environment: macOS 26.4 arm64, Python 3.12.14, official 0.42 native wheel; Python source imported from this worktree (asserted by script). Offline, SQLite result cache enabled, CCR markers off, no provider/model call - Exact command / steps: Ran the standalone replay below as `HEADROOM_OFFLINE=1 TIKTOKEN_CACHE_DIR=/private/tmp/headroom-tiktoken PYTHONPATH=. ../headroom/.venv/bin/python ../evidence/router-policy/repro.py` before and after the patch. Seed a prior ML-result sentinel, reopen the real disk cache through a fresh router, then call real `apply()` with default, disabled, and skip policies. No mocked router/cache/compressor in this replay; the earlier ML output is deliberately seeded - Observed result: Same-policy reuse remained a disk hit. Before: disabled and skip requests both forwarded the sentinel. After: both preserved the original text, with separate skip entries (output below) - Not tested: Live provider requests, ML inference/downloads, full test suite, mypy, native rebuild, or the separate reversibility fix. Runtime used the existing pure-Python detector fallback because ONNX Runtime 1.24+ is unavailable ```text before after same_policy_reused_seeded_result true true same_policy_disk_hits 1 1 disabled_reused_incompatible_result true false skip_reused_incompatible_result true false disabled_output_equals_original false true skip_output_equals_original false true cache_entries 1 1 skip_entries 0 2 ``` <details> <summary>Standalone reproduction script</summary> ```python import json import os import tempfile from pathlib import Path os.environ.update(HEADROOM_BEACON="off", HEADROOM_CCR_BACKEND="memory", HEADROOM_OFFLINE="1") import headroom.transforms.content_router as module from headroom.transforms.content_router import ContentRouter, ContentRouterConfig, content_cache_key assert Path(module.__file__).resolve() == Path.cwd() / "headroom/transforms/content_router.py" class Tokenizer: def count_text(self, text): return max(1, len(str(text)) // 4) text = "The service reports useful observations about the current deployment. " * 80 sentinel = "A cached result produced with ML enabled." payload = [{"role": "tool", "content": text}] with tempfile.TemporaryDirectory() as directory: os.environ["HEADROOM_WORKSPACE_DIR"] = directory os.environ["HEADROOM_COMPRESSION_CACHE_PATH"] = str(Path(directory) / "results.db") config = ContentRouterConfig(result_cache_disk=True, ccr_inject_marker=False) config.read_lifecycle.enabled = False writer = ContentRouter(config) writer._cache.put(content_cache_key(text, None, gated=True), sentinel, 0.1, "kompress") reader = ContentRouter(config) same = reader.apply(payload, Tokenizer()).messages[0]["content"] hits = reader._cache.stats["cache_disk_hits"] disabled = reader.apply(payload, Tokenizer(), kompress_model="disabled").messages[0]["content"] skipped = reader.apply(payload, Tokenizer(), skip_kompress=True).messages[0]["content"] print(json.dumps({ "same_policy_reused_seeded_result": same == sentinel, "same_policy_disk_hits": hits, "disabled_reused_incompatible_result": disabled == sentinel, "skip_reused_incompatible_result": skipped == sentinel, "disabled_output_equals_original": disabled == text, "skip_output_equals_original": skipped == text, "cache_entries": reader._cache.size, "skip_entries": reader._cache.skip_size, }, indent=2)) ``` </details> ## Runtime Rollout Safety - Rollout-managed feature(s): None; cache correctness fix - Minimum rollout channel: Stable - Stable/default behavior changed: Different request policies stop sharing results, skips, and frozen decisions; the new key format causes a one-time cache miss for older entries - Kill switch / disable path: No new switch; revert the patch to restore old behavior - Unsafe override required: No - Qualification impact: No compressor or reversibility-gate behavior changes; context-sensitive keys may reduce reuse when context changes - Rollback path: Revert this commit; cached data requires no schema migration ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [x] I did **not** edit `CHANGELOG.md` — it is generated by release-please from my Conventional Commit PR title (a CI guard enforces this) ## Additional Notes AI assistance: Codex investigated, implemented, and ran the checks above. No new dependency or public configuration change. Documentation N/A for this internal cache correction. Native rebuild not run because Cargo is unavailable. Self-review and independent review are complete with no outstanding findings; the frozen-verdict test coverage recommendation was addressed and independently rerun (54 passed). All configured commit hooks, including scoped mypy and Commitlint, passed; full-project mypy was not run. <!-- devin-review-badge-begin --> --- <a href="https://app.devin.ai/review/headroomlabs-ai/headroom/pull/4590" target="_blank"><picture><source media="(prefers-color-scheme: dark)" srcset="https://static.devin.ai/assets/gh-devin-review-dark.svg?v=4"><img src="https://static.devin.ai/assets/gh-devin-review-light.svg?v=4" alt="Devin Review"></picture></a> <!-- devin-review-badge-end --> Co-authored-by: JD Davis <mxjerrett@gmail.com>
240 lines
9.1 KiB
Python
240 lines
9.1 KiB
Python
"""Request policy must scope every tier of the router's content cache."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from headroom.transforms.content_router import (
|
|
CompressionStrategy,
|
|
ContentRouter,
|
|
ContentRouterConfig,
|
|
content_cache_key,
|
|
)
|
|
|
|
TEXT = "The service reports useful observations about the current deployment. " * 80
|
|
|
|
|
|
class Tokenizer:
|
|
def count_text(self, text: str) -> int:
|
|
return max(1, len(str(text)) // 4)
|
|
|
|
|
|
def messages(shape, tool_name="inspect", arguments="{}"):
|
|
call = {
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_1",
|
|
"type": "function",
|
|
"function": {"name": tool_name, "arguments": arguments},
|
|
}
|
|
],
|
|
}
|
|
if shape == "tool_result":
|
|
result = {
|
|
"role": "user",
|
|
"content": [{"type": "tool_result", "tool_use_id": "call_1", "content": TEXT}],
|
|
}
|
|
else:
|
|
content = TEXT if shape == "string" else [{"type": "text", "text": TEXT}]
|
|
result = {"role": "tool", "tool_call_id": "call_1", "content": content}
|
|
return [call, result]
|
|
|
|
|
|
def output(result, shape):
|
|
content = result.messages[-1]["content"]
|
|
if shape == "string":
|
|
return content
|
|
return content[0]["content" if shape == "tool_result" else "text"]
|
|
|
|
|
|
@pytest.fixture
|
|
def router(monkeypatch):
|
|
# Only the compressor is replaced: apply(), admission, all cache tiers,
|
|
# runtime policy binding and frozen verdict handling remain real.
|
|
router = ContentRouter(ContentRouterConfig(ccr_inject_marker=False))
|
|
monkeypatch.setattr(router, "_kompress_model_ready", lambda: True)
|
|
return router
|
|
|
|
|
|
def install_compressor(monkeypatch, router, *, skip_first=False, marked=False):
|
|
calls = []
|
|
|
|
def compress(content, context="", bias=1.0, **kwargs):
|
|
calls.append((router._runtime_kompress_model, context, bias))
|
|
disabled = router._runtime_kompress_model == "disabled" or router._runtime_skip_kompress
|
|
skip = disabled or (skip_first and len(calls) == 1)
|
|
compressed = content if skip else f"compressed under policy {len(calls)}"
|
|
if marked and not skip:
|
|
compressed += " <<ccr:0123456789abcdef>>"
|
|
return SimpleNamespace(
|
|
original=content,
|
|
compressed=compressed,
|
|
compression_ratio=1.0 if skip else 0.1,
|
|
strategy_used=CompressionStrategy.KOMPRESS,
|
|
strategy_chain=[],
|
|
)
|
|
|
|
monkeypatch.setattr(router, "compress", compress)
|
|
return calls
|
|
|
|
|
|
@pytest.mark.parametrize("shape", ["string", "tool_result", "text"])
|
|
@pytest.mark.parametrize("freeze", [False, True])
|
|
@pytest.mark.parametrize(
|
|
"changed",
|
|
[
|
|
{"kompress_model": "disabled"},
|
|
{"kompress_model": "another-model"},
|
|
{"skip_kompress": True},
|
|
{"force_kompress": True},
|
|
{"context": "a different question"},
|
|
{"target_ratio": 0.3},
|
|
],
|
|
)
|
|
def test_apply_reuses_results_only_under_the_same_policy(
|
|
monkeypatch, router, shape, freeze, changed
|
|
):
|
|
monkeypatch.setenv("HEADROOM_FREEZE_BLOCK_DECISION", str(int(freeze)))
|
|
calls = install_compressor(monkeypatch, router, marked=freeze)
|
|
payload = messages(shape)
|
|
first = output(router.apply(payload, Tokenizer()), shape)
|
|
assert first != TEXT
|
|
assert output(router.apply(payload, Tokenizer()), shape) == first
|
|
assert len(calls) == 1
|
|
|
|
second = output(router.apply(payload, Tokenizer(), **changed), shape)
|
|
assert len(calls) == 2, "changed policy must not reuse the previous result"
|
|
assert second != first
|
|
if changed.get("kompress_model") == "disabled" or changed.get("skip_kompress"):
|
|
assert second == TEXT
|
|
assert output(router.apply(payload, Tokenizer(), **changed), shape) == second
|
|
assert len(calls) == 2
|
|
# Returning to the original policy still finds its own cached output.
|
|
assert output(router.apply(payload, Tokenizer()), shape) == first
|
|
assert len(calls) == 2
|
|
|
|
|
|
@pytest.mark.parametrize("shape", ["string", "tool_result", "text"])
|
|
def test_frozen_verdicts_are_recorded_separately_for_each_policy(monkeypatch, router, shape):
|
|
monkeypatch.setenv("HEADROOM_FREEZE_BLOCK_DECISION", "1")
|
|
# Lossy output needs a retrieval marker before a verdict is safe to freeze.
|
|
calls = install_compressor(monkeypatch, router, marked=True)
|
|
payload = messages(shape)
|
|
|
|
first = output(router.apply(payload, Tokenizer()), shape)
|
|
initial_verdicts = dict(router._frozen_verdicts)
|
|
assert len(initial_verdicts) == 1
|
|
assert list(initial_verdicts.values()) == [True]
|
|
assert output(router.apply(payload, Tokenizer()), shape) == first
|
|
assert router._frozen_verdicts == initial_verdicts
|
|
assert len(calls) == 1
|
|
|
|
second = output(router.apply(payload, Tokenizer(), kompress_model="another-model"), shape)
|
|
assert second != first
|
|
assert len(calls) == 2
|
|
assert len(router._frozen_verdicts) == 2
|
|
assert all(verdict is True for verdict in router._frozen_verdicts.values())
|
|
assert initial_verdicts.items() <= router._frozen_verdicts.items()
|
|
assert output(router.apply(payload, Tokenizer()), shape) == first
|
|
assert len(calls) == 2
|
|
|
|
|
|
@pytest.mark.parametrize("shape", ["string", "tool_result", "text"])
|
|
@pytest.mark.parametrize("changed", [{"kompress_model": "another-model"}, {"force_kompress": True}])
|
|
def test_a_skip_under_one_policy_does_not_suppress_another(monkeypatch, router, shape, changed):
|
|
calls = install_compressor(monkeypatch, router, skip_first=True)
|
|
payload = messages(shape)
|
|
assert output(router.apply(payload, Tokenizer()), shape) == TEXT
|
|
assert output(router.apply(payload, Tokenizer()), shape) == TEXT
|
|
assert len(calls) == 1
|
|
assert output(router.apply(payload, Tokenizer(), **changed), shape) != TEXT
|
|
assert len(calls) == 2
|
|
|
|
|
|
def test_hook_bias_partitions_string_results(monkeypatch, router):
|
|
calls = install_compressor(monkeypatch, router)
|
|
payload = messages("string")
|
|
first = output(router.apply(payload, Tokenizer()), "string")
|
|
second = output(router.apply(payload, Tokenizer(), biases={1: 2.0}), "string")
|
|
assert second != first
|
|
assert [call[2] for call in calls] == [1.0, 2.0]
|
|
|
|
|
|
@pytest.mark.parametrize("shape", ["string", "tool_result"])
|
|
def test_effective_tool_bias_partitions_results(monkeypatch, router, shape):
|
|
router.config.tool_profiles = {
|
|
"inspect": SimpleNamespace(bias=1.0),
|
|
"summarize": SimpleNamespace(bias=2.0),
|
|
}
|
|
calls = install_compressor(monkeypatch, router)
|
|
first = output(router.apply(messages(shape), Tokenizer()), shape)
|
|
second = output(router.apply(messages(shape, tool_name="summarize"), Tokenizer()), shape)
|
|
assert second != first
|
|
assert [call[2] for call in calls] == [1.0, 2.0]
|
|
|
|
|
|
def test_tool_result_relevance_uses_the_enriched_context(monkeypatch, router):
|
|
router.config.relevance_split = True
|
|
calls = install_compressor(monkeypatch, router)
|
|
first = router.apply(messages("tool_result", arguments='{"query":"alpha"}'), Tokenizer())
|
|
second = router.apply(messages("tool_result", arguments='{"query":"beta"}'), Tokenizer())
|
|
assert output(first, "tool_result") != output(second, "tool_result")
|
|
assert len(calls) == 2
|
|
assert calls[0][1] != calls[1][1]
|
|
|
|
|
|
def test_policy_keys_are_deterministic_across_processes():
|
|
options = {
|
|
"kompress_model": "custom|model",
|
|
"force_kompress": True,
|
|
"skip_kompress": True,
|
|
"bias": 1.5,
|
|
"context": "question|with\x00delimiters",
|
|
}
|
|
expected = content_cache_key(TEXT, 0.5, gated=True, **options)
|
|
code = (
|
|
"from headroom.transforms.content_router import content_cache_key;"
|
|
f"print(content_cache_key({TEXT!r}, 0.5, gated=True, **{options!r}))"
|
|
)
|
|
for seed in ("1", "2"):
|
|
found = subprocess.run(
|
|
[sys.executable, "-c", code],
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
env={**os.environ, "PYTHONHASHSEED": seed},
|
|
).stdout.strip()
|
|
assert found == expected
|
|
|
|
|
|
def test_context_and_content_have_unambiguous_boundaries():
|
|
assert content_cache_key("b|c", None, context="a") != content_cache_key(
|
|
"c", None, context="a|b"
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("shape", ["string", "tool_result", "text"])
|
|
def test_reopened_disk_results_are_scoped_to_the_request_policy(tmp_path, monkeypatch, shape):
|
|
monkeypatch.setenv("HEADROOM_COMPRESSION_CACHE_PATH", str(tmp_path / "policy.db"))
|
|
config = ContentRouterConfig(ccr_inject_marker=False, result_cache_disk=True)
|
|
first = ContentRouter(config)
|
|
calls = install_compressor(monkeypatch, first)
|
|
payload = messages(shape)
|
|
original = output(first.apply(payload, Tokenizer()), shape)
|
|
assert len(calls) == 1
|
|
|
|
reopened = ContentRouter(config)
|
|
new_calls = install_compressor(monkeypatch, reopened)
|
|
assert output(reopened.apply(payload, Tokenizer()), shape) == original
|
|
assert new_calls == []
|
|
assert reopened._cache.stats["cache_disk_hits"] == 1
|
|
assert output(reopened.apply(payload, Tokenizer(), kompress_model="disabled"), shape) == TEXT
|
|
assert len(new_calls) == 1
|