mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 15:10:00 +08:00
## Description This PR deletes 22 TOIN tests (15 skip markers, about 590 lines) that cannot run, because each one is unconditionally skipped. PR-B5 (6819b7e5, 2026-05-02) made TOIN observation-only. Since then `ToolIntelligenceNetwork.get_recommendation()` returns `None` and emits a `DeprecationWarning`. The tests that exercised the old request-time hints were marked `@pytest.mark.skip(reason="PR-B5: ...")` instead of being deleted. The current contract is covered by `tests/test_toin_observation_only.py`, which is unchanged. Evidence: every removed test or class carries an unconditional `@pytest.mark.skip` whose reason starts with "PR-B5". No test that runs today was removed: the pass count is the same before and after. Refs #4136 ## Type of Change - [ ] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [x] Code refactoring (no functional changes) ## Changes Made - `tests/test_toin_fixes.py`: delete `TestStrategySuccessRates`, `TestFieldRetrievalFrequencyWeighting` and `TestQueryContextUsage`. - `tests/test_toin_full_integration.py`: delete `TestTOINFullFeedbackLoop` and `TestTOINProgressiveConfidence`. - `tests/test_toin_feedback.py`: delete `TestGetRecommendationObservations`, and the `ToolPattern` import it alone used. - `tests/test_toin.py`: delete six `test_get_recommendation_*` stubs whose body is `pass`, and their comment. Update the module docstring, which said these tests were skipped. - `tests/test_critical_gaps.py`: delete `test_toin_preserves_fields_returns_list`. ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - [ ] Type checking passes (`mypy headroom`) - [ ] New tests added for new functionality - [ ] Manual testing performed ### Test Output ```text $ pytest -q -p no:cacheprovider tests/test_toin.py tests/test_toin_fixes.py tests/test_toin_full_integration.py tests/test_toin_feedback.py tests/test_critical_gaps.py tests/test_toin_observation_only.py # on origin/main (a9815bbb): 118 passed, 23 skipped, 1 warning in 3.00s # on this branch: 118 passed, 1 skipped, 1 warning in 3.27s # the remaining skip: tests/test_toin_observation_only.py:453 needs 'hypothesis' $ uvx ruff@0.16.8 check tests/test_toin.py tests/test_toin_fixes.py tests/test_toin_full_integration.py tests/test_toin_feedback.py tests/test_critical_gaps.py All checks passed! ``` ## Real Behavior Proof - Environment: macOS, Python 3.12.13, this branch checked out in a worktree, repo .venv; temp HOME and HEADROOM_WORKSPACE_DIR, offline. - Exact command / steps: (1) The shared smoke harness `smoke.sh <worktree>`: it starts a real proxy from this checkout against a local fake upstream and sends one Anthropic and one OpenAI request with a compressible tool result. (2) A direct command on this PR's own path: `HOME=$TMP/home HEADROOM_WORKSPACE_DIR=$TMP/ws python -W always::DeprecationWarning -c "from headroom.telemetry import ToolIntelligenceNetwork, ToolSignature; print(ToolIntelligenceNetwork().get_recommendation(ToolSignature.from_items([{'id': 1}])))"; pytest -q tests/test_toin.py tests/test_toin_fixes.py tests/test_toin_full_integration.py tests/test_toin_feedback.py tests/test_critical_gaps.py tests/test_toin_observation_only.py tests/test_toin_publish.py`. - Observed result: Both requests return 200 and `/stats` shows savings, with only the expected offline-mode warnings in proxy.log. The direct command's output is below. - Not tested: Nothing at runtime changes; this PR touches tests only. ```text $ bash smoke.sh <worktree> branch: cleanup/core-misc-toin-skipped-tests @37d4666chealth: 200 POST /v1/messages -> HTTP200 smoke-ok POST /v1/chat/completions -> HTTP200 smoke-ok stats: requests=2 failed=0 tokens_saved=13574 savings_percent=99.82 proxy.log WARNING/ERROR lines ($SMOKE_TMP/ws/logs/proxy-49628.log): 2 headroom.proxy - WARNING - event=proxy_offline_mode air-gap active — every connection Headroom initiates is refused (telemetry, update check, license reporter, model/tokenizer/binary/dataset downloads, remote Kompress, OTLP/Langfuse export, Copilot auth, subscription polling, OpenAI embedders, Headroom Cloud compression). Still allowed on purpose: forwarding your requests to the upstream you configured, operator-configured local endpoints, loopback health probes. 4 headroom.transforms.kompress_compressor - WARNING - Kompress: artifact prefetch found no usable file for chopratejas/kompress-v2-base 4 headroom.transforms.kompress_compressor - WARNING - Kompress: artifact prefetch refused for chopratejas/kompress-v2-base (HEADROOM_OFFLINE is set): HEADROOM_OFFLINE is set: refusing outbound network access for HuggingFace download of chopratejas/kompress-v2-base/onnx/kompress-int8-wo.onnx to huggingface.co. Unset HEADROOM_OFFLINE, or turn off the feature that needs this connection. (smoke dir: $SMOKE_TMP) $ HOME=$TMP/home HEADROOM_WORKSPACE_DIR=$TMP/ws python -W always::DeprecationWarning -c "from headroom.telemetry import ToolIntelligenceNetwork, ToolSignature; print(ToolIntelligenceNetwork().get_recommendation(ToolSignature.from_items([{'id': 1}])))"; pytest -q tests/test_toin.py tests/test_toin_fixes.py tests/test_toin_full_integration.py tests/test_toin_feedback.py tests/test_critical_gaps.py tests/test_toin_observation_only.py tests/test_toin_publish.py <string>:4: DeprecationWarning: ToolIntelligenceNetwork.get_recommendation() is deprecated and now returns None. PR-B5 retired the request-time hint API; recommendations come from recommendations.toml at startup. See headroom/telemetry/toin.py module docstring. None ================== 126 passed, 1 skipped, 1 warning in 5.89s =================== ``` ## Runtime Rollout Safety - Rollout-managed feature(s): None. - Minimum rollout channel: N/A; test-only change. - Stable/default behavior changed: No. No product code changes. - Kill switch / disable path: N/A. - Unsafe override required: No. - Qualification impact: None. - Rollback path: Revert this commit. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [x] I did **not** edit `CHANGELOG.md` — it is generated by release-please from my Conventional Commit PR title (a CI guard enforces this) ## Screenshots (if applicable) N/A ## Additional Notes Part of maintainer-requested cleanup #4136 (dead code, duplication, debuggability), split into small PRs for review. `get_recommendation()` itself stays: it is a deprecated public method, and removing it is a separate decision. A few live tests still assert that it returns `None`, in `tests/test_proxy_ccr.py`, `tests/test_adapter_hooks.py` and `tests/test_critical_gaps.py`. They stay until the method goes. 🤖 Generated with [Claude Code](https://claude.com/claude-code)
442 lines
15 KiB
Python
442 lines
15 KiB
Python
"""Comprehensive tests for TOIN implementation fixes.
|
|
|
|
This file tests all the fixes made to the TOIN implementation:
|
|
1. toin_hint.recommended_strategy is used in SmartCrusher
|
|
2. strategy_success_rates are used in recommendations
|
|
3. preserve_fields are merged in federated learning
|
|
4. tool_signature_hash and strategy are passed to feedback system
|
|
5. user_count is tracked via instance_id
|
|
6. field_retrieval_frequency weights preserve_fields
|
|
7. query_context keywords and patterns are detected
|
|
"""
|
|
|
|
import json
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from headroom.cache.compression_feedback import (
|
|
get_compression_feedback,
|
|
reset_compression_feedback,
|
|
)
|
|
from headroom.cache.compression_store import (
|
|
RetrievalEvent,
|
|
get_compression_store,
|
|
reset_compression_store,
|
|
)
|
|
from headroom.telemetry import ToolSignature
|
|
from headroom.telemetry.toin import (
|
|
TOINConfig,
|
|
ToolIntelligenceNetwork,
|
|
get_toin,
|
|
reset_toin,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def fresh_toin():
|
|
"""Create a fresh TOIN instance with temporary storage."""
|
|
reset_toin()
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
storage_path = str(Path(tmpdir) / "toin_test.json")
|
|
toin = get_toin(
|
|
TOINConfig(
|
|
storage_path=storage_path,
|
|
auto_save_interval=0,
|
|
)
|
|
)
|
|
yield toin
|
|
reset_toin()
|
|
|
|
|
|
@pytest.fixture
|
|
def fresh_feedback():
|
|
"""Create a fresh feedback instance."""
|
|
reset_compression_feedback()
|
|
feedback = get_compression_feedback()
|
|
yield feedback
|
|
reset_compression_feedback()
|
|
|
|
|
|
@pytest.fixture
|
|
def fresh_store():
|
|
"""Create a fresh compression store."""
|
|
reset_compression_store()
|
|
store = get_compression_store(max_entries=100, default_ttl=300)
|
|
yield store
|
|
reset_compression_store()
|
|
|
|
|
|
class TestPreserveFieldsMerging:
|
|
"""Test preserve_fields merging in federated learning."""
|
|
|
|
def test_preserve_fields_merged_on_import(self, fresh_toin):
|
|
"""Imported preserve_fields should be merged with existing."""
|
|
items = [{"id": i, "name": f"item_{i}"} for i in range(10)]
|
|
signature = ToolSignature.from_items(items)
|
|
sig_hash = signature.structure_hash
|
|
|
|
# Create local pattern with some preserve_fields
|
|
fresh_toin.record_compression(
|
|
tool_signature=signature,
|
|
original_count=10,
|
|
compressed_count=5,
|
|
original_tokens=1000,
|
|
compressed_tokens=500,
|
|
strategy="smart_sample",
|
|
)
|
|
local_pattern = fresh_toin._patterns[("global", "unknown", "unknown", sig_hash)]
|
|
local_pattern.preserve_fields = ["field_a", "field_b"]
|
|
|
|
# Import pattern with different preserve_fields
|
|
import_data = {
|
|
"patterns": {
|
|
sig_hash: {
|
|
"tool_signature_hash": sig_hash,
|
|
"total_compressions": 100,
|
|
"total_retrievals": 20,
|
|
"sample_size": 100,
|
|
"preserve_fields": ["field_c", "field_d"],
|
|
}
|
|
}
|
|
}
|
|
|
|
fresh_toin.import_patterns(import_data)
|
|
|
|
# Verify merge
|
|
pattern = fresh_toin._patterns[("global", "unknown", "unknown", sig_hash)]
|
|
assert "field_a" in pattern.preserve_fields
|
|
assert "field_b" in pattern.preserve_fields
|
|
assert "field_c" in pattern.preserve_fields
|
|
assert "field_d" in pattern.preserve_fields
|
|
|
|
def test_preserve_fields_limited_to_10(self, fresh_toin):
|
|
"""preserve_fields should be capped at 10 entries."""
|
|
items = [{"id": i} for i in range(10)]
|
|
signature = ToolSignature.from_items(items)
|
|
sig_hash = signature.structure_hash
|
|
|
|
# Create pattern with 8 fields
|
|
fresh_toin.record_compression(
|
|
tool_signature=signature,
|
|
original_count=10,
|
|
compressed_count=5,
|
|
original_tokens=1000,
|
|
compressed_tokens=500,
|
|
strategy="smart_sample",
|
|
)
|
|
pattern = fresh_toin._patterns[("global", "unknown", "unknown", sig_hash)]
|
|
pattern.preserve_fields = [f"field_{i}" for i in range(8)]
|
|
|
|
# Import with 5 more fields
|
|
import_data = {
|
|
"patterns": {
|
|
sig_hash: {
|
|
"tool_signature_hash": sig_hash,
|
|
"total_compressions": 50,
|
|
"sample_size": 50,
|
|
"preserve_fields": [f"imported_{i}" for i in range(5)],
|
|
}
|
|
}
|
|
}
|
|
|
|
fresh_toin.import_patterns(import_data)
|
|
|
|
# Should be capped at 10
|
|
pattern = fresh_toin._patterns[("global", "unknown", "unknown", sig_hash)]
|
|
assert len(pattern.preserve_fields) <= 10
|
|
|
|
|
|
class TestUserCountTracking:
|
|
"""Test user_count tracking via instance_id."""
|
|
|
|
def test_user_count_increments_for_new_instance(self, fresh_toin):
|
|
"""user_count should increment when a new instance is seen."""
|
|
items = [{"id": i} for i in range(10)]
|
|
signature = ToolSignature.from_items(items)
|
|
|
|
# Record compression (first instance)
|
|
fresh_toin.record_compression(
|
|
tool_signature=signature,
|
|
original_count=10,
|
|
compressed_count=5,
|
|
original_tokens=1000,
|
|
compressed_tokens=500,
|
|
strategy="smart_sample",
|
|
)
|
|
|
|
pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
|
|
assert pattern.user_count == 1
|
|
assert len(pattern._seen_instance_hashes) == 1
|
|
assert fresh_toin._instance_id in pattern._seen_instance_hashes
|
|
|
|
def test_user_count_stable_for_same_instance(self, fresh_toin):
|
|
"""user_count should not increase for same instance."""
|
|
items = [{"id": i} for i in range(10)]
|
|
signature = ToolSignature.from_items(items)
|
|
|
|
# Record multiple compressions from same instance
|
|
for _ in range(10):
|
|
fresh_toin.record_compression(
|
|
tool_signature=signature,
|
|
original_count=10,
|
|
compressed_count=5,
|
|
original_tokens=1000,
|
|
compressed_tokens=500,
|
|
strategy="smart_sample",
|
|
)
|
|
|
|
pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
|
|
assert pattern.user_count == 1 # Still 1
|
|
|
|
def test_instance_hashes_serialized_and_loaded(self):
|
|
"""_seen_instance_hashes should survive save/load cycle."""
|
|
reset_toin()
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
storage_path = str(Path(tmpdir) / "toin_persist.json")
|
|
toin1 = ToolIntelligenceNetwork(TOINConfig(storage_path=storage_path))
|
|
|
|
items = [{"id": i} for i in range(10)]
|
|
signature = ToolSignature.from_items(items)
|
|
|
|
# Record compression
|
|
toin1.record_compression(
|
|
tool_signature=signature,
|
|
original_count=10,
|
|
compressed_count=5,
|
|
original_tokens=1000,
|
|
compressed_tokens=500,
|
|
strategy="smart_sample",
|
|
)
|
|
|
|
# Save
|
|
toin1.save()
|
|
|
|
# Load in new instance
|
|
toin2 = ToolIntelligenceNetwork(TOINConfig(storage_path=storage_path))
|
|
|
|
pattern = toin2._patterns.get(
|
|
("global", "unknown", "unknown", signature.structure_hash)
|
|
)
|
|
assert pattern is not None
|
|
assert pattern.user_count >= 1
|
|
assert len(pattern._seen_instance_hashes) >= 1
|
|
|
|
def test_user_count_merged_on_import(self, fresh_toin):
|
|
"""user_count should reflect merged instance hashes."""
|
|
items = [{"id": i} for i in range(10)]
|
|
signature = ToolSignature.from_items(items)
|
|
sig_hash = signature.structure_hash
|
|
|
|
# Create local pattern
|
|
fresh_toin.record_compression(
|
|
tool_signature=signature,
|
|
original_count=10,
|
|
compressed_count=5,
|
|
original_tokens=1000,
|
|
compressed_tokens=500,
|
|
strategy="smart_sample",
|
|
)
|
|
|
|
# Import pattern with different instance hashes
|
|
import_data = {
|
|
"patterns": {
|
|
sig_hash: {
|
|
"tool_signature_hash": sig_hash,
|
|
"total_compressions": 50,
|
|
"sample_size": 50,
|
|
"seen_instance_hashes": ["other_instance_1", "other_instance_2"],
|
|
"user_count": 2,
|
|
}
|
|
}
|
|
}
|
|
|
|
fresh_toin.import_patterns(import_data)
|
|
|
|
pattern = fresh_toin._patterns[("global", "unknown", "unknown", sig_hash)]
|
|
# Should have local + 2 imported = 3
|
|
assert pattern.user_count >= 3
|
|
|
|
|
|
class TestFeedbackStrategyTracking:
|
|
"""Test strategy tracking in compression feedback."""
|
|
|
|
def test_record_compression_tracks_strategy(self, fresh_feedback):
|
|
"""record_compression should track strategy."""
|
|
fresh_feedback.record_compression(
|
|
tool_name="test_tool",
|
|
original_count=100,
|
|
compressed_count=20,
|
|
strategy="smart_sample",
|
|
tool_signature_hash="abc123",
|
|
)
|
|
|
|
pattern = fresh_feedback._tool_patterns.get("test_tool")
|
|
assert pattern is not None
|
|
assert "smart_sample" in pattern.strategy_compressions
|
|
assert pattern.strategy_compressions["smart_sample"] == 1
|
|
|
|
def test_record_retrieval_tracks_strategy(self, fresh_feedback):
|
|
"""record_retrieval should track strategy retrievals."""
|
|
# First record a compression
|
|
fresh_feedback.record_compression(
|
|
tool_name="test_tool",
|
|
original_count=100,
|
|
compressed_count=20,
|
|
strategy="smart_sample",
|
|
)
|
|
|
|
# Then record a retrieval with strategy
|
|
event = RetrievalEvent(
|
|
hash="test_hash",
|
|
query="test query",
|
|
items_retrieved=100,
|
|
total_items=100,
|
|
tool_name="test_tool",
|
|
timestamp=1234567890.0,
|
|
retrieval_type="full",
|
|
)
|
|
|
|
fresh_feedback.record_retrieval(event, strategy="smart_sample")
|
|
|
|
pattern = fresh_feedback._tool_patterns.get("test_tool")
|
|
assert "smart_sample" in pattern.strategy_retrievals
|
|
assert pattern.strategy_retrievals["smart_sample"] == 1
|
|
|
|
def test_strategy_retrieval_rate_calculation(self, fresh_feedback):
|
|
"""strategy_retrieval_rate should calculate correctly."""
|
|
# Record 10 compressions
|
|
for _ in range(10):
|
|
fresh_feedback.record_compression(
|
|
tool_name="test_tool",
|
|
original_count=100,
|
|
compressed_count=20,
|
|
strategy="smart_sample",
|
|
)
|
|
|
|
# Record 3 retrievals
|
|
for _ in range(3):
|
|
event = RetrievalEvent(
|
|
hash="test_hash",
|
|
query="test query",
|
|
items_retrieved=100,
|
|
total_items=100,
|
|
tool_name="test_tool",
|
|
timestamp=1234567890.0,
|
|
retrieval_type="full",
|
|
)
|
|
fresh_feedback.record_retrieval(event, strategy="smart_sample")
|
|
|
|
pattern = fresh_feedback._tool_patterns.get("test_tool")
|
|
rate = pattern.strategy_retrieval_rate("smart_sample")
|
|
assert rate == 0.3 # 3 retrievals / 10 compressions
|
|
|
|
def test_best_strategy_selection(self, fresh_feedback):
|
|
"""best_strategy should return strategy with lowest retrieval rate."""
|
|
# Record compressions for multiple strategies
|
|
for _ in range(10):
|
|
fresh_feedback.record_compression(
|
|
tool_name="test_tool",
|
|
original_count=100,
|
|
compressed_count=20,
|
|
strategy="bad_strategy",
|
|
)
|
|
for _ in range(10):
|
|
fresh_feedback.record_compression(
|
|
tool_name="test_tool",
|
|
original_count=100,
|
|
compressed_count=20,
|
|
strategy="good_strategy",
|
|
)
|
|
|
|
# Record more retrievals for bad strategy
|
|
for _ in range(8):
|
|
event = RetrievalEvent(
|
|
hash="test_hash",
|
|
query=None,
|
|
items_retrieved=100,
|
|
total_items=100,
|
|
tool_name="test_tool",
|
|
timestamp=1234567890.0,
|
|
retrieval_type="full",
|
|
)
|
|
fresh_feedback.record_retrieval(event, strategy="bad_strategy")
|
|
|
|
# Record few retrievals for good strategy
|
|
for _ in range(2):
|
|
event = RetrievalEvent(
|
|
hash="test_hash",
|
|
query=None,
|
|
items_retrieved=100,
|
|
total_items=100,
|
|
tool_name="test_tool",
|
|
timestamp=1234567890.0,
|
|
retrieval_type="full",
|
|
)
|
|
fresh_feedback.record_retrieval(event, strategy="good_strategy")
|
|
|
|
pattern = fresh_feedback._tool_patterns.get("test_tool")
|
|
# good_strategy has 20% retrieval rate, bad_strategy has 80%
|
|
best = pattern.best_strategy()
|
|
assert best == "good_strategy"
|
|
|
|
|
|
class TestSignatureHashTracking:
|
|
"""Test tool_signature_hash tracking in feedback."""
|
|
|
|
def test_signature_hash_recorded(self, fresh_feedback):
|
|
"""record_compression should track signature hash."""
|
|
fresh_feedback.record_compression(
|
|
tool_name="test_tool",
|
|
original_count=100,
|
|
compressed_count=20,
|
|
strategy="smart_sample",
|
|
tool_signature_hash="unique_sig_hash",
|
|
)
|
|
|
|
pattern = fresh_feedback._tool_patterns.get("test_tool")
|
|
assert "unique_sig_hash" in pattern.signature_hashes
|
|
|
|
def test_multiple_signature_hashes_tracked(self, fresh_feedback):
|
|
"""Multiple different signature hashes should be tracked."""
|
|
hashes = ["hash_1", "hash_2", "hash_3"]
|
|
|
|
for h in hashes:
|
|
fresh_feedback.record_compression(
|
|
tool_name="test_tool",
|
|
original_count=100,
|
|
compressed_count=20,
|
|
tool_signature_hash=h,
|
|
)
|
|
|
|
pattern = fresh_feedback._tool_patterns.get("test_tool")
|
|
for h in hashes:
|
|
assert h in pattern.signature_hashes
|
|
|
|
|
|
class TestIntegration:
|
|
"""Integration tests for the full feedback loop."""
|
|
|
|
def test_store_passes_strategy_to_feedback(self, fresh_store, fresh_feedback):
|
|
"""CompressionStore should pass strategy to feedback on retrieval."""
|
|
# Store with strategy
|
|
hash_key = fresh_store.store(
|
|
original=json.dumps([{"id": i} for i in range(50)]),
|
|
compressed=json.dumps([{"id": i} for i in range(10)]),
|
|
original_item_count=50,
|
|
compressed_item_count=10,
|
|
tool_name="test_tool",
|
|
tool_signature_hash="test_sig_hash",
|
|
compression_strategy="smart_sample",
|
|
)
|
|
|
|
# Retrieve triggers feedback
|
|
fresh_store.retrieve(hash_key, query="test query")
|
|
|
|
# Verify feedback received the strategy
|
|
pattern = fresh_feedback._tool_patterns.get("test_tool")
|
|
if pattern:
|
|
# Strategy should be tracked
|
|
assert pattern.total_retrievals >= 1
|