mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 15:10:00 +08:00
## Description This PR deletes 22 TOIN tests (15 skip markers, about 590 lines) that cannot run, because each one is unconditionally skipped. PR-B5 (6819b7e5, 2026-05-02) made TOIN observation-only. Since then `ToolIntelligenceNetwork.get_recommendation()` returns `None` and emits a `DeprecationWarning`. The tests that exercised the old request-time hints were marked `@pytest.mark.skip(reason="PR-B5: ...")` instead of being deleted. The current contract is covered by `tests/test_toin_observation_only.py`, which is unchanged. Evidence: every removed test or class carries an unconditional `@pytest.mark.skip` whose reason starts with "PR-B5". No test that runs today was removed: the pass count is the same before and after. Refs #4136 ## Type of Change - [ ] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [x] Code refactoring (no functional changes) ## Changes Made - `tests/test_toin_fixes.py`: delete `TestStrategySuccessRates`, `TestFieldRetrievalFrequencyWeighting` and `TestQueryContextUsage`. - `tests/test_toin_full_integration.py`: delete `TestTOINFullFeedbackLoop` and `TestTOINProgressiveConfidence`. - `tests/test_toin_feedback.py`: delete `TestGetRecommendationObservations`, and the `ToolPattern` import it alone used. - `tests/test_toin.py`: delete six `test_get_recommendation_*` stubs whose body is `pass`, and their comment. Update the module docstring, which said these tests were skipped. - `tests/test_critical_gaps.py`: delete `test_toin_preserves_fields_returns_list`. ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - [ ] Type checking passes (`mypy headroom`) - [ ] New tests added for new functionality - [ ] Manual testing performed ### Test Output ```text $ pytest -q -p no:cacheprovider tests/test_toin.py tests/test_toin_fixes.py tests/test_toin_full_integration.py tests/test_toin_feedback.py tests/test_critical_gaps.py tests/test_toin_observation_only.py # on origin/main (a9815bbb): 118 passed, 23 skipped, 1 warning in 3.00s # on this branch: 118 passed, 1 skipped, 1 warning in 3.27s # the remaining skip: tests/test_toin_observation_only.py:453 needs 'hypothesis' $ uvx ruff@0.16.8 check tests/test_toin.py tests/test_toin_fixes.py tests/test_toin_full_integration.py tests/test_toin_feedback.py tests/test_critical_gaps.py All checks passed! ``` ## Real Behavior Proof - Environment: macOS, Python 3.12.13, this branch checked out in a worktree, repo .venv; temp HOME and HEADROOM_WORKSPACE_DIR, offline. - Exact command / steps: (1) The shared smoke harness `smoke.sh <worktree>`: it starts a real proxy from this checkout against a local fake upstream and sends one Anthropic and one OpenAI request with a compressible tool result. (2) A direct command on this PR's own path: `HOME=$TMP/home HEADROOM_WORKSPACE_DIR=$TMP/ws python -W always::DeprecationWarning -c "from headroom.telemetry import ToolIntelligenceNetwork, ToolSignature; print(ToolIntelligenceNetwork().get_recommendation(ToolSignature.from_items([{'id': 1}])))"; pytest -q tests/test_toin.py tests/test_toin_fixes.py tests/test_toin_full_integration.py tests/test_toin_feedback.py tests/test_critical_gaps.py tests/test_toin_observation_only.py tests/test_toin_publish.py`. - Observed result: Both requests return 200 and `/stats` shows savings, with only the expected offline-mode warnings in proxy.log. The direct command's output is below. - Not tested: Nothing at runtime changes; this PR touches tests only. ```text $ bash smoke.sh <worktree> branch: cleanup/core-misc-toin-skipped-tests @37d4666chealth: 200 POST /v1/messages -> HTTP200 smoke-ok POST /v1/chat/completions -> HTTP200 smoke-ok stats: requests=2 failed=0 tokens_saved=13574 savings_percent=99.82 proxy.log WARNING/ERROR lines ($SMOKE_TMP/ws/logs/proxy-49628.log): 2 headroom.proxy - WARNING - event=proxy_offline_mode air-gap active — every connection Headroom initiates is refused (telemetry, update check, license reporter, model/tokenizer/binary/dataset downloads, remote Kompress, OTLP/Langfuse export, Copilot auth, subscription polling, OpenAI embedders, Headroom Cloud compression). Still allowed on purpose: forwarding your requests to the upstream you configured, operator-configured local endpoints, loopback health probes. 4 headroom.transforms.kompress_compressor - WARNING - Kompress: artifact prefetch found no usable file for chopratejas/kompress-v2-base 4 headroom.transforms.kompress_compressor - WARNING - Kompress: artifact prefetch refused for chopratejas/kompress-v2-base (HEADROOM_OFFLINE is set): HEADROOM_OFFLINE is set: refusing outbound network access for HuggingFace download of chopratejas/kompress-v2-base/onnx/kompress-int8-wo.onnx to huggingface.co. Unset HEADROOM_OFFLINE, or turn off the feature that needs this connection. (smoke dir: $SMOKE_TMP) $ HOME=$TMP/home HEADROOM_WORKSPACE_DIR=$TMP/ws python -W always::DeprecationWarning -c "from headroom.telemetry import ToolIntelligenceNetwork, ToolSignature; print(ToolIntelligenceNetwork().get_recommendation(ToolSignature.from_items([{'id': 1}])))"; pytest -q tests/test_toin.py tests/test_toin_fixes.py tests/test_toin_full_integration.py tests/test_toin_feedback.py tests/test_critical_gaps.py tests/test_toin_observation_only.py tests/test_toin_publish.py <string>:4: DeprecationWarning: ToolIntelligenceNetwork.get_recommendation() is deprecated and now returns None. PR-B5 retired the request-time hint API; recommendations come from recommendations.toml at startup. See headroom/telemetry/toin.py module docstring. None ================== 126 passed, 1 skipped, 1 warning in 5.89s =================== ``` ## Runtime Rollout Safety - Rollout-managed feature(s): None. - Minimum rollout channel: N/A; test-only change. - Stable/default behavior changed: No. No product code changes. - Kill switch / disable path: N/A. - Unsafe override required: No. - Qualification impact: None. - Rollback path: Revert this commit. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [x] I did **not** edit `CHANGELOG.md` — it is generated by release-please from my Conventional Commit PR title (a CI guard enforces this) ## Screenshots (if applicable) N/A ## Additional Notes Part of maintainer-requested cleanup #4136 (dead code, duplication, debuggability), split into small PRs for review. `get_recommendation()` itself stays: it is a deprecated public method, and removing it is a separate decision. A few live tests still assert that it returns `None`, in `tests/test_proxy_ccr.py`, `tests/test_adapter_hooks.py` and `tests/test_critical_gaps.py`. They stay until the method goes. 🤖 Generated with [Claude Code](https://claude.com/claude-code)
364 lines
13 KiB
Python
364 lines
13 KiB
Python
"""Tests for TOIN feedback loop: headroom_retrieve calls flow back to TOIN."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from headroom.cache.compression_store import get_compression_store, reset_compression_store
|
|
from headroom.telemetry import (
|
|
TOINConfig,
|
|
ToolIntelligenceNetwork,
|
|
ToolSignature,
|
|
get_toin,
|
|
reset_toin,
|
|
)
|
|
from headroom.transforms.kompress_compressor import KompressCompressor
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def reset_globals(monkeypatch, tmp_path):
|
|
"""Reset global state before each test."""
|
|
temp_toin_path = str(tmp_path / "toin_test.json")
|
|
monkeypatch.setenv("HEADROOM_TOIN_PATH", temp_toin_path)
|
|
reset_toin()
|
|
reset_compression_store()
|
|
yield
|
|
reset_compression_store()
|
|
reset_toin()
|
|
|
|
|
|
def _make_config(min_samples: int = 5) -> TOINConfig:
|
|
"""Create a TOIN config for testing."""
|
|
return TOINConfig(
|
|
enabled=True,
|
|
min_samples_for_recommendation=min_samples,
|
|
storage_path="", # disable persistence
|
|
)
|
|
|
|
|
|
def _make_signature(structure_hash: str = "test_hash_123") -> ToolSignature:
|
|
"""Create a minimal ToolSignature for testing."""
|
|
return ToolSignature(
|
|
structure_hash=structure_hash,
|
|
field_count=3,
|
|
has_nested_objects=False,
|
|
has_arrays=True,
|
|
max_depth=1,
|
|
)
|
|
|
|
|
|
def test_kompress_ccr_retrieval_updates_toin():
|
|
"""Kompress CCR entries should be first-class TOIN patterns."""
|
|
original = "\n".join(
|
|
[
|
|
"HEADROOM_MODE=debug PATH=/tmp/headroom",
|
|
"ordinary line without the target token",
|
|
"another ordinary line",
|
|
]
|
|
)
|
|
compressed = "HEADROOM_MODE=debug"
|
|
|
|
compressor = KompressCompressor()
|
|
hash_key = compressor._store_in_ccr(
|
|
original,
|
|
compressed,
|
|
original_tokens=len(original.split()),
|
|
)
|
|
|
|
assert hash_key is not None
|
|
store = get_compression_store()
|
|
entry = store.retrieve(hash_key)
|
|
assert entry is not None
|
|
assert entry.tool_signature_hash is not None
|
|
# Retrieval is by hash and returns the full original content.
|
|
assert "HEADROOM" in entry.original_content
|
|
|
|
stats = get_toin().get_stats()
|
|
assert stats["total_compressions"] == 1
|
|
assert stats["total_retrievals"] == 1
|
|
|
|
|
|
class TestRecordRetrievalPopulatesFields:
|
|
"""Bug 1 related: record_retrieval with query_fields populates field data."""
|
|
|
|
def test_record_retrieval_populates_fields(self):
|
|
"""record_retrieval with query_fields should populate field_retrieval_frequency."""
|
|
config = _make_config()
|
|
toin = ToolIntelligenceNetwork(config=config)
|
|
sig_hash = "retrieval_field_test"
|
|
|
|
# Record some compressions first to create the pattern
|
|
sig = _make_signature(sig_hash)
|
|
for _ in range(5):
|
|
toin.record_compression(
|
|
tool_signature=sig,
|
|
original_count=50,
|
|
compressed_count=10,
|
|
original_tokens=2000,
|
|
compressed_tokens=500,
|
|
strategy="top_n",
|
|
)
|
|
|
|
# Record multiple retrievals with same field
|
|
for _ in range(5):
|
|
toin.record_retrieval(
|
|
sig_hash,
|
|
"search",
|
|
query="error_message:timeout",
|
|
query_fields=["error_message"],
|
|
)
|
|
|
|
pattern = toin._patterns[("global", "unknown", "unknown", sig_hash)]
|
|
assert pattern.total_retrievals == 5
|
|
assert pattern.search_retrievals == 5
|
|
assert len(pattern.field_retrieval_frequency) > 0
|
|
|
|
|
|
class TestCCRFeedbackExtraction:
|
|
"""Bug 2: _record_ccr_feedback_from_response extracts headroom_retrieve calls."""
|
|
|
|
def test_extract_headroom_retrieve_from_response(self):
|
|
"""Should detect headroom_retrieve tool_use blocks in response content."""
|
|
response = {
|
|
"content": [
|
|
{"type": "text", "text": "Let me retrieve that."},
|
|
{
|
|
"type": "tool_use",
|
|
"id": "toolu_123",
|
|
"name": "headroom_retrieve",
|
|
"input": {"hash": "abc123def456"},
|
|
},
|
|
]
|
|
}
|
|
|
|
# Extract tool calls the same way _record_ccr_feedback_from_response does
|
|
content = response.get("content", [])
|
|
retrieve_calls = []
|
|
for block in content:
|
|
if not isinstance(block, dict):
|
|
continue
|
|
if block.get("type") == "tool_use" and block.get("name") == "headroom_retrieve":
|
|
input_data = block.get("input", {})
|
|
if input_data.get("hash"):
|
|
retrieve_calls.append(input_data)
|
|
|
|
assert len(retrieve_calls) == 1
|
|
assert retrieve_calls[0]["hash"] == "abc123def456"
|
|
|
|
def test_ignore_non_retrieve_tool_calls(self):
|
|
"""Should ignore tool_use blocks that are not headroom_retrieve."""
|
|
response = {
|
|
"content": [
|
|
{
|
|
"type": "tool_use",
|
|
"id": "toolu_456",
|
|
"name": "some_other_tool",
|
|
"input": {"data": "something"},
|
|
},
|
|
{
|
|
"type": "tool_use",
|
|
"id": "toolu_789",
|
|
"name": "headroom_retrieve",
|
|
"input": {"hash": "xyz789"},
|
|
},
|
|
]
|
|
}
|
|
|
|
content = response.get("content", [])
|
|
retrieve_calls = []
|
|
for block in content:
|
|
if not isinstance(block, dict):
|
|
continue
|
|
if block.get("type") == "tool_use" and block.get("name") == "headroom_retrieve":
|
|
input_data = block.get("input", {})
|
|
if input_data.get("hash"):
|
|
retrieve_calls.append(input_data)
|
|
|
|
assert len(retrieve_calls) == 1
|
|
assert retrieve_calls[0]["hash"] == "xyz789"
|
|
|
|
def test_empty_content_does_not_crash(self):
|
|
"""Should handle empty or missing content gracefully."""
|
|
for response in [
|
|
{"content": []},
|
|
{"content": "not a list"},
|
|
{},
|
|
]:
|
|
content = response.get("content", [])
|
|
if not isinstance(content, list):
|
|
continue
|
|
# Should not raise
|
|
for _block in content:
|
|
pass
|
|
|
|
def test_missing_hash_skipped(self):
|
|
"""Should skip headroom_retrieve calls without a hash."""
|
|
response = {
|
|
"content": [
|
|
{
|
|
"type": "tool_use",
|
|
"id": "toolu_000",
|
|
"name": "headroom_retrieve",
|
|
"input": {"query": "some query"}, # No hash
|
|
},
|
|
]
|
|
}
|
|
|
|
content = response.get("content", [])
|
|
retrieve_calls = []
|
|
for block in content:
|
|
if not isinstance(block, dict):
|
|
continue
|
|
if block.get("type") == "tool_use" and block.get("name") == "headroom_retrieve":
|
|
input_data = block.get("input", {})
|
|
if input_data.get("hash"):
|
|
retrieve_calls.append(input_data)
|
|
|
|
assert len(retrieve_calls) == 0
|
|
|
|
|
|
class TestStreamingFeedbackIntegration:
|
|
"""Bug 2: Full feedback loop — streaming headroom_retrieve reaches TOIN."""
|
|
|
|
def test_record_ccr_feedback_calls_store_retrieve(self):
|
|
"""_record_ccr_feedback_from_response calls store.retrieve by hash.
|
|
|
|
Retrieval is by hash only — any legacy ``query`` in the tool input is
|
|
ignored, and the full content is fetched for the feedback side effect.
|
|
"""
|
|
from headroom.proxy.server import HeadroomProxy
|
|
|
|
response = {
|
|
"content": [
|
|
{
|
|
"type": "tool_use",
|
|
"id": "toolu_001",
|
|
"name": "headroom_retrieve",
|
|
"input": {"hash": "feedbackhash1", "query": "error details"},
|
|
},
|
|
]
|
|
}
|
|
|
|
mock_store = MagicMock()
|
|
with patch(
|
|
"headroom.cache.compression_store.get_compression_store",
|
|
return_value=mock_store,
|
|
):
|
|
# Create a minimal proxy to test the method
|
|
proxy = HeadroomProxy.__new__(HeadroomProxy)
|
|
proxy.config = MagicMock()
|
|
proxy.config.ccr_inject_tool = True
|
|
|
|
proxy._record_ccr_feedback_from_response(response, "anthropic", "req-test-001")
|
|
|
|
mock_store.retrieve.assert_called_once_with("feedbackhash1")
|
|
mock_store.search.assert_not_called()
|
|
|
|
def test_record_ccr_feedback_calls_store_retrieve_no_query(self):
|
|
"""_record_ccr_feedback_from_response calls store.retrieve by hash."""
|
|
from headroom.proxy.server import HeadroomProxy
|
|
|
|
response = {
|
|
"content": [
|
|
{
|
|
"type": "tool_use",
|
|
"id": "toolu_002",
|
|
"name": "headroom_retrieve",
|
|
"input": {"hash": "feedbackhash2"},
|
|
},
|
|
]
|
|
}
|
|
|
|
mock_store = MagicMock()
|
|
with patch(
|
|
"headroom.cache.compression_store.get_compression_store",
|
|
return_value=mock_store,
|
|
):
|
|
proxy = HeadroomProxy.__new__(HeadroomProxy)
|
|
proxy.config = MagicMock()
|
|
proxy.config.ccr_inject_tool = True
|
|
|
|
proxy._record_ccr_feedback_from_response(response, "anthropic", "req-test-002")
|
|
|
|
mock_store.retrieve.assert_called_once_with("feedbackhash2")
|
|
|
|
def test_record_ccr_feedback_handles_store_exception(self):
|
|
"""_record_ccr_feedback_from_response should not raise on store errors."""
|
|
from headroom.proxy.server import HeadroomProxy
|
|
|
|
response = {
|
|
"content": [
|
|
{
|
|
"type": "tool_use",
|
|
"id": "toolu_003",
|
|
"name": "headroom_retrieve",
|
|
"input": {"hash": "feedbackhash3"},
|
|
},
|
|
]
|
|
}
|
|
|
|
mock_store = MagicMock()
|
|
mock_store.retrieve.side_effect = RuntimeError("store unavailable")
|
|
with patch(
|
|
"headroom.cache.compression_store.get_compression_store",
|
|
return_value=mock_store,
|
|
):
|
|
proxy = HeadroomProxy.__new__(HeadroomProxy)
|
|
proxy.config = MagicMock()
|
|
proxy.config.ccr_inject_tool = True
|
|
|
|
# Should not raise
|
|
proxy._record_ccr_feedback_from_response(response, "anthropic", "req-test-003")
|
|
|
|
|
|
class TestParseSSEToolUse:
|
|
"""Bug 3: _parse_sse_to_response correctly handles tool_use blocks."""
|
|
|
|
def test_parse_sse_extracts_tool_use(self):
|
|
"""SSE with tool_use content_block should be parsed correctly."""
|
|
from headroom.proxy.server import HeadroomProxy
|
|
|
|
sse_data = (
|
|
'data: {"type":"message_start","message":{"id":"msg_01","model":"claude-3-5-sonnet-20241022","role":"assistant","stop_reason":null,"usage":{"input_tokens":100,"output_tokens":0}}}\n'
|
|
"\n"
|
|
'data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}\n'
|
|
"\n"
|
|
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Let me retrieve that."}}\n'
|
|
"\n"
|
|
'data: {"type":"content_block_stop","index":0}\n'
|
|
"\n"
|
|
'data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_abc","name":"headroom_retrieve"}}\n'
|
|
"\n"
|
|
'data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\\"hash\\": \\"abc123\\"}"}}\n'
|
|
"\n"
|
|
'data: {"type":"content_block_stop","index":1}\n'
|
|
"\n"
|
|
'data: {"type":"message_delta","delta":{"stop_reason":"tool_use"},"usage":{"output_tokens":50}}\n'
|
|
)
|
|
|
|
proxy = HeadroomProxy.__new__(HeadroomProxy)
|
|
result = proxy._parse_sse_to_response(sse_data, "anthropic")
|
|
|
|
assert result is not None
|
|
assert len(result["content"]) == 2
|
|
|
|
text_block = result["content"][0]
|
|
assert text_block["type"] == "text"
|
|
assert "retrieve" in text_block["text"]
|
|
|
|
tool_block = result["content"][1]
|
|
assert tool_block["type"] == "tool_use"
|
|
assert tool_block["name"] == "headroom_retrieve"
|
|
assert tool_block["id"] == "toolu_abc"
|
|
assert tool_block["input"]["hash"] == "abc123"
|
|
|
|
def test_parse_sse_non_anthropic_returns_none(self):
|
|
"""Non-anthropic provider should return None."""
|
|
from headroom.proxy.server import HeadroomProxy
|
|
|
|
proxy = HeadroomProxy.__new__(HeadroomProxy)
|
|
result = proxy._parse_sse_to_response("data: {}", "openai")
|
|
assert result is None
|