test(toin): delete the permanently skipped PR-B5 get_recommendation tests

PR-B5 (6819b7e5) made ToolIntelligenceNetwork.get_recommendation() a stub that returns None. The tests for the old request-time hints were marked skip instead of deleted, so 15 tests (about 590 lines) have not run since May. test_toin_observation_only.py covers the current contract.

Evidence: every removed test carries an unconditional @pytest.mark.skip(reason="PR-B5: ..."); no live test was touched.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Assisted-by: claude-code:claude-opus-5-5
Headline-Session: 6e77a807-8dd4-4a77-a2fa-ddce6dc2515e
Headline-AI-Share: 100% strict / 100% weighted
This commit is contained in:
Tejas Chopra
2026-10-09 16:43:46 -07:00
co-authored by Claude Opus 5.5
parent a9815bbb5f
commit 37d4666cc9
5 changed files with 3 additions and 595 deletions
-6
View File
@@ -845,12 +845,6 @@ class TestTOINHighPriorityFixes:
# (auto-save happens inside record_compression)
assert not toin._dirty
@pytest.mark.skip(
reason="PR-B5: get_recommendation retired; preserve_fields lives on the aggregated ToolPattern instead"
)
def test_toin_preserves_fields_returns_list(self):
"""Retired in PR-B5 along with the request-time hint API."""
class TestCompressionStoreHighPriorityFixes:
"""Additional HIGH priority tests for CompressionStore."""
+3 -45
View File
@@ -1,9 +1,8 @@
"""Tests for Tool Output Intelligence Network (TOIN).
PR-B5 retired the request-time hint API. Tests that exercised the old
`get_recommendation()` / `CompressionHint` shape are skipped at module
level — the new observation-only contract is covered by
`tests/test_toin_observation_only.py` and `tests/test_toin_publish.py`.
PR-B5 retired the request-time hint API. The observation-only contract
is covered by `tests/test_toin_observation_only.py` and
`tests/test_toin_publish.py`.
"""
import os
@@ -331,47 +330,6 @@ class TestToolIntelligenceNetwork:
# Field should be in commonly_retrieved_fields after 3+ retrievals
assert len(pattern.commonly_retrieved_fields) > 0
# PR-B5: the following tests exercised the request-time hint API
# that's now retired. They're skipped wholesale; the new contract
# ("get_recommendation always returns None and emits a deprecation
# warning") is covered by tests/test_toin_observation_only.py.
@pytest.mark.skip(
reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"
)
def test_get_recommendation_no_data(self):
pass
@pytest.mark.skip(
reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"
)
def test_get_recommendation_insufficient_samples(self):
pass
@pytest.mark.skip(
reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"
)
def test_get_recommendation_aggressive_compression(self):
pass
@pytest.mark.skip(
reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"
)
def test_get_recommendation_conservative_compression(self):
pass
@pytest.mark.skip(
reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"
)
def test_get_recommendation_skip_compression(self):
pass
@pytest.mark.skip(
reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"
)
def test_get_recommendation_disabled(self):
pass
def test_get_stats(self):
"""get_stats returns overall statistics."""
toin = ToolIntelligenceNetwork()
-89
View File
@@ -10,7 +10,6 @@ from headroom.cache.compression_store import get_compression_store, reset_compre
from headroom.telemetry import (
TOINConfig,
ToolIntelligenceNetwork,
ToolPattern,
ToolSignature,
get_toin,
reset_toin,
@@ -81,94 +80,6 @@ def test_kompress_ccr_retrieval_updates_toin():
assert stats["total_retrievals"] == 1
@pytest.mark.skip(reason="PR-B5: observations counter and request-time hint API retired")
class TestGetRecommendationObservations:
"""Bug 1: get_recommendation() should increment observations counter."""
def test_increments_observations_when_pattern_exists(self):
"""get_recommendation() should increment observations when pattern exists."""
config = _make_config(min_samples=5)
toin = ToolIntelligenceNetwork(config=config)
sig = _make_signature("obs_test_hash")
# Record enough compressions to create a pattern with sufficient samples
for _ in range(15):
toin.record_compression(
tool_signature=sig,
original_count=100,
compressed_count=20,
original_tokens=5000,
compressed_tokens=1000,
strategy="top_n",
)
# Get recommendation
toin.get_recommendation(sig)
# Check observations incremented
pattern = toin._patterns[("global", "unknown", "unknown", sig.structure_hash)]
assert pattern.observations == 1
# Call again
toin.get_recommendation(sig)
assert pattern.observations == 2
def test_increments_observations_even_below_min_samples(self):
"""observations increments even when sample_size < min_samples."""
config = _make_config(min_samples=100)
toin = ToolIntelligenceNetwork(config=config)
sig = _make_signature("low_sample_hash")
# Record just a few compressions (below min_samples)
for _ in range(3):
toin.record_compression(
tool_signature=sig,
original_count=50,
compressed_count=10,
original_tokens=2000,
compressed_tokens=500,
strategy="top_n",
)
result = toin.get_recommendation(sig)
assert result.source == "local" # Not enough samples
pattern = toin._patterns[("global", "unknown", "unknown", sig.structure_hash)]
assert pattern.observations == 1
def test_no_increment_for_unknown_pattern(self):
"""get_recommendation() should NOT increment for unknown patterns."""
config = _make_config()
toin = ToolIntelligenceNetwork(config=config)
sig = _make_signature("nonexistent_hash")
result = toin.get_recommendation(sig)
assert result.source == "default"
assert result.reason == "No pattern data for this tool type"
# No pattern exists, nothing to increment
assert "nonexistent_hash" not in toin._patterns
def test_observations_survives_serialization(self):
"""observations field should serialize and deserialize correctly."""
pattern = ToolPattern(
tool_signature_hash="serial_test",
total_compressions=10,
observations=42,
)
d = pattern.to_dict()
assert d["observations"] == 42
restored = ToolPattern.from_dict(d)
assert restored.observations == 42
def test_observations_defaults_to_zero(self):
"""observations defaults to 0 for new patterns."""
pattern = ToolPattern(tool_signature_hash="new_pattern")
assert pattern.observations == 0
class TestRecordRetrievalPopulatesFields:
"""Bug 1 related: record_retrieval with query_fields populates field data."""
-253
View File
@@ -68,99 +68,6 @@ def fresh_store():
reset_compression_store()
@pytest.mark.skip(
reason="PR-B5: strategy-recommendation API retired (get_recommendation returns None)"
)
class TestStrategySuccessRates:
"""Test that strategy_success_rates are used in recommendations."""
def test_recommends_strategy_with_high_success_rate(self, fresh_toin):
"""Strategy with success rate >= 0.5 should be recommended."""
items = [{"id": i, "score": 100 - i} for i in range(20)]
signature = ToolSignature.from_items(items)
# Record compressions to build pattern
for _ in range(10):
fresh_toin.record_compression(
tool_signature=signature,
original_count=20,
compressed_count=10,
original_tokens=2000,
compressed_tokens=1000,
strategy="smart_sample",
)
# Set high success rate
pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
pattern.strategy_success_rates["smart_sample"] = 0.8
pattern.optimal_strategy = "smart_sample"
# Get recommendation
hint = fresh_toin.get_recommendation(signature, "test query")
assert hint.recommended_strategy == "smart_sample"
def test_rejects_strategy_with_low_success_rate(self, fresh_toin):
"""Strategy with success rate < 0.5 should NOT be recommended."""
items = [{"id": i, "score": 100 - i} for i in range(20)]
signature = ToolSignature.from_items(items)
# Record compressions
for _ in range(10):
fresh_toin.record_compression(
tool_signature=signature,
original_count=20,
compressed_count=10,
original_tokens=2000,
compressed_tokens=1000,
strategy="bad_strategy",
)
# Set low success rate
pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
pattern.strategy_success_rates["bad_strategy"] = 0.2
pattern.optimal_strategy = "bad_strategy"
# Get recommendation
hint = fresh_toin.get_recommendation(signature, "test query")
# Should not recommend the bad strategy
assert hint.recommended_strategy != "bad_strategy"
# Confidence should be reduced
assert "low success" in hint.reason.lower()
def test_finds_best_strategy_when_optimal_is_bad(self, fresh_toin):
"""When optimal_strategy has low success, find a better alternative."""
items = [{"id": i, "score": 100 - i} for i in range(20)]
signature = ToolSignature.from_items(items)
# Record compressions
for _ in range(10):
fresh_toin.record_compression(
tool_signature=signature,
original_count=20,
compressed_count=10,
original_tokens=2000,
compressed_tokens=1000,
strategy="smart_sample",
)
# Set up multiple strategies with different success rates
pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
pattern.strategy_success_rates = {
"bad_strategy": 0.2,
"good_strategy": 0.9,
}
pattern.optimal_strategy = "bad_strategy"
# Get recommendation
hint = fresh_toin.get_recommendation(signature, "test query")
# Should recommend the better strategy
assert hint.recommended_strategy == "good_strategy"
assert "using good_strategy instead" in hint.reason
class TestPreserveFieldsMerging:
"""Test preserve_fields merging in federated learning."""
@@ -352,166 +259,6 @@ class TestUserCountTracking:
assert pattern.user_count >= 3
@pytest.mark.skip(
reason="PR-B5: get_recommendation retired; field-weighting now consumed only by toin publish"
)
class TestFieldRetrievalFrequencyWeighting:
"""Test field_retrieval_frequency weighting in preserve_fields."""
def test_query_fields_prioritized_in_preserve_fields(self, fresh_toin):
"""Fields mentioned in query should be prioritized."""
items = [{"id": i, "status": "ok", "category": f"cat_{i}"} for i in range(20)]
signature = ToolSignature.from_items(items)
# Build pattern with field retrieval data
for _ in range(10):
fresh_toin.record_compression(
tool_signature=signature,
original_count=20,
compressed_count=10,
original_tokens=2000,
compressed_tokens=1000,
strategy="smart_sample",
)
# Record retrievals for "status" field
status_hash = fresh_toin._hash_field_name("status")
pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
pattern.field_retrieval_frequency = {
status_hash: 50,
fresh_toin._hash_field_name("category"): 10,
}
pattern.preserve_fields = [status_hash]
# Get recommendation with query mentioning "status"
hint = fresh_toin.get_recommendation(signature, "status:error")
# status hash should be in preserve_fields
assert status_hash in hint.preserve_fields
def test_preserve_fields_sorted_by_frequency(self, fresh_toin):
"""preserve_fields should be sorted by retrieval frequency."""
items = [{"id": i} for i in range(20)]
signature = ToolSignature.from_items(items)
# Build pattern
for _ in range(10):
fresh_toin.record_compression(
tool_signature=signature,
original_count=20,
compressed_count=10,
original_tokens=2000,
compressed_tokens=1000,
strategy="smart_sample",
)
pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
field_a = fresh_toin._hash_field_name("field_a")
field_b = fresh_toin._hash_field_name("field_b")
field_c = fresh_toin._hash_field_name("field_c")
pattern.field_retrieval_frequency = {
field_a: 10,
field_b: 50, # Most frequent
field_c: 30,
}
pattern.preserve_fields = [field_a, field_b, field_c]
# Get recommendation (no query context)
hint = fresh_toin.get_recommendation(signature, "")
# Should be sorted by frequency
if len(hint.preserve_fields) >= 3:
# field_b should come before field_c which should come before field_a
b_idx = hint.preserve_fields.index(field_b) if field_b in hint.preserve_fields else -1
c_idx = hint.preserve_fields.index(field_c) if field_c in hint.preserve_fields else -1
hint.preserve_fields.index(field_a) if field_a in hint.preserve_fields else -1
if b_idx >= 0 and c_idx >= 0:
assert b_idx < c_idx, "Higher frequency field should come first"
@pytest.mark.skip(reason="PR-B5: get_recommendation retired (returns None / DeprecationWarning)")
class TestQueryContextUsage:
"""Test query_context usage in recommendations."""
def test_exhaustive_query_keywords_detected(self, fresh_toin):
"""Exhaustive query keywords should trigger conservative compression."""
items = [{"id": i, "score": 100 - i} for i in range(50)]
signature = ToolSignature.from_items(items)
# Build pattern with aggressive compression normally
for _ in range(10):
fresh_toin.record_compression(
tool_signature=signature,
original_count=50,
compressed_count=10,
original_tokens=5000,
compressed_tokens=1000,
strategy="smart_sample",
)
# Low retrieval rate = aggressive compression
pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
pattern.total_retrievals = 0
# Query with exhaustive keyword
hint = fresh_toin.get_recommendation(signature, "list all items in category")
# Should be more conservative
assert hint.max_items >= 40
assert "exhaustive query" in hint.reason.lower()
assert hint.compression_level == "conservative"
def test_every_keyword_triggers_conservative(self, fresh_toin):
"""'every' keyword should trigger conservative compression."""
items = [{"id": i} for i in range(50)]
signature = ToolSignature.from_items(items)
for _ in range(10):
fresh_toin.record_compression(
tool_signature=signature,
original_count=50,
compressed_count=10,
original_tokens=5000,
compressed_tokens=1000,
strategy="smart_sample",
)
pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
pattern.total_retrievals = 0
hint = fresh_toin.get_recommendation(signature, "find every user")
assert "exhaustive query" in hint.reason.lower()
def test_partial_pattern_matching(self, fresh_toin):
"""Partial pattern matching should boost max_items."""
items = [{"id": i, "status": "ok"} for i in range(50)]
signature = ToolSignature.from_items(items)
for _ in range(10):
fresh_toin.record_compression(
tool_signature=signature,
original_count=50,
compressed_count=10,
original_tokens=5000,
compressed_tokens=1000,
strategy="smart_sample",
)
pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
pattern.total_retrievals = 0
# Add a problematic query pattern
pattern.common_query_patterns = ["status:*"]
# Query that uses the same field
hint = fresh_toin.get_recommendation(signature, "status:error")
# Should match the pattern
assert hint.max_items >= 25 or "retrieval pattern" in hint.reason
class TestFeedbackStrategyTracking:
"""Test strategy tracking in compression feedback."""
-202
View File
@@ -247,208 +247,6 @@ class TestTOINPersistenceAcrossInstances:
print("\n[PASS] TOIN persistence works correctly")
@pytest.mark.skip(
reason="PR-B5: get_recommendation retired; feedback-loop covered by test_toin_observation_only.py"
)
class TestTOINFullFeedbackLoop:
"""Test 3: Verify TOIN feedback loop with recommendations."""
def test_toin_full_feedback_loop(self, sample_tool_signature):
"""Verify TOIN learns from high retrieval rate and recommends skip."""
print("\n" + "=" * 60)
print("TEST: test_toin_full_feedback_loop")
print("=" * 60)
with tempfile.TemporaryDirectory() as tmpdir:
storage_path = str(Path(tmpdir) / "toin_feedback_test.json")
config = TOINConfig(
storage_path=storage_path,
min_samples_for_recommendation=5, # Lower threshold for test
high_retrieval_threshold=0.5, # 50% retrieval = high
)
toin = ToolIntelligenceNetwork(config)
print("\n--- Phase 1: Record compressions ---")
# Record 5 compressions with same tool signature
for i in range(5):
toin.record_compression(
tool_signature=sample_tool_signature,
original_count=100,
compressed_count=15,
original_tokens=5000,
compressed_tokens=750,
strategy="smart_sample",
)
print(f" Recorded compression {i + 1}")
print("\n--- Phase 2: Record retrievals (simulating high retrieval rate) ---")
# Record 3 full retrievals (60% retrieval rate = high)
for i in range(3):
toin.record_retrieval(
tool_signature_hash=sample_tool_signature.structure_hash,
retrieval_type="full", # Full retrieval = compression too aggressive
strategy="smart_sample",
)
print(f" Recorded full retrieval {i + 1}")
# Get pattern stats
pattern = toin.get_pattern(sample_tool_signature.structure_hash)
print("\n--- Pattern Stats ---")
print(f" total_compressions: {pattern.total_compressions}")
print(f" total_retrievals: {pattern.total_retrievals}")
print(f" retrieval_rate: {pattern.retrieval_rate:.1%}")
print(f" full_retrieval_rate: {pattern.full_retrieval_rate:.1%}")
print(f" skip_compression_recommended: {pattern.skip_compression_recommended}")
# Get recommendation
print("\n--- Getting Recommendation ---")
hint = toin.get_recommendation(sample_tool_signature)
print(f" source: {hint.source}")
print(f" skip_compression: {hint.skip_compression}")
print(f" compression_level: {hint.compression_level}")
print(f" max_items: {hint.max_items}")
print(f" confidence: {hint.confidence:.3f}")
print(f" reason: {hint.reason}")
print(f" based_on_samples: {hint.based_on_samples}")
# Verify high retrieval rate triggers skip recommendation
# With 60% retrieval rate (3/5) and full_retrieval_rate of 100% (3/3),
# TOIN should recommend skipping compression
retrieval_rate = pattern.retrieval_rate
assert retrieval_rate >= 0.5, (
f"Expected retrieval rate >= 50%, got {retrieval_rate:.1%}"
)
# With high retrieval rate and high full retrieval rate, should skip
if pattern.full_retrieval_rate > 0.8:
assert hint.skip_compression or hint.compression_level in (
"none",
"conservative",
), (
f"High full retrieval rate should trigger skip or conservative, "
f"got compression_level={hint.compression_level}"
)
print("\n[PASS] High retrieval rate correctly influences recommendation")
else:
print("\n[INFO] Full retrieval rate not high enough for skip recommendation")
print(f" full_retrieval_rate: {pattern.full_retrieval_rate:.1%}")
print("\n[PASS] TOIN feedback loop works correctly")
@pytest.mark.skip(
reason="PR-B5: get_recommendation retired; confidence-progression validated via record + get_pattern instead"
)
class TestTOINProgressiveConfidence:
"""Test 4: Verify TOIN confidence increases with sample size."""
def test_toin_progressive_confidence(self, sample_tool_signature):
"""Verify confidence increases with more samples."""
print("\n" + "=" * 60)
print("TEST: test_toin_progressive_confidence")
print("=" * 60)
with tempfile.TemporaryDirectory() as tmpdir:
storage_path = str(Path(tmpdir) / "toin_confidence_test.json")
config = TOINConfig(
storage_path=storage_path,
min_samples_for_recommendation=3,
)
toin = ToolIntelligenceNetwork(config)
confidence_history = []
# Batch 1: Record 1 compression
print("\n--- Batch 1: 1 compression ---")
toin.record_compression(
tool_signature=sample_tool_signature,
original_count=100,
compressed_count=15,
original_tokens=5000,
compressed_tokens=750,
strategy="smart_sample",
)
pattern = toin.get_pattern(sample_tool_signature.structure_hash)
hint = toin.get_recommendation(sample_tool_signature)
confidence_history.append(pattern.confidence)
print(f" sample_size: {pattern.sample_size}")
print(f" confidence: {pattern.confidence:.3f}")
print(f" hint.source: {hint.source}")
# Batch 2: Record 2 more compressions
print("\n--- Batch 2: +2 compressions (total: 3) ---")
for _ in range(2):
toin.record_compression(
tool_signature=sample_tool_signature,
original_count=100,
compressed_count=15,
original_tokens=5000,
compressed_tokens=750,
strategy="smart_sample",
)
pattern = toin.get_pattern(sample_tool_signature.structure_hash)
hint = toin.get_recommendation(sample_tool_signature)
confidence_history.append(pattern.confidence)
print(f" sample_size: {pattern.sample_size}")
print(f" confidence: {pattern.confidence:.3f}")
print(f" hint.source: {hint.source}")
# Batch 3: Record 2 more compressions
print("\n--- Batch 3: +2 compressions (total: 5) ---")
for _ in range(2):
toin.record_compression(
tool_signature=sample_tool_signature,
original_count=100,
compressed_count=15,
original_tokens=5000,
compressed_tokens=750,
strategy="smart_sample",
)
pattern = toin.get_pattern(sample_tool_signature.structure_hash)
hint = toin.get_recommendation(sample_tool_signature)
confidence_history.append(pattern.confidence)
print(f" sample_size: {pattern.sample_size}")
print(f" confidence: {pattern.confidence:.3f}")
print(f" hint.source: {hint.source}")
# Batch 4: Add many more to boost confidence
print("\n--- Batch 4: +15 compressions (total: 20) ---")
for _ in range(15):
toin.record_compression(
tool_signature=sample_tool_signature,
original_count=100,
compressed_count=15,
original_tokens=5000,
compressed_tokens=750,
strategy="smart_sample",
)
pattern = toin.get_pattern(sample_tool_signature.structure_hash)
hint = toin.get_recommendation(sample_tool_signature)
confidence_history.append(pattern.confidence)
print(f" sample_size: {pattern.sample_size}")
print(f" confidence: {pattern.confidence:.3f}")
print(f" hint.source: {hint.source}")
# Print confidence progression
print("\n--- Confidence Progression ---")
for i, conf in enumerate(confidence_history):
print(f" Stage {i + 1}: confidence = {conf:.3f}")
# Verify confidence increases with sample size
# Confidence should generally increase (may plateau at high values)
assert confidence_history[-1] >= confidence_history[0], (
f"Confidence should increase: start={confidence_history[0]:.3f}, "
f"end={confidence_history[-1]:.3f}"
)
# With 20 samples, should have meaningful confidence
assert confidence_history[-1] >= 0.1, (
f"With 20 samples, confidence should be >= 0.1, got {confidence_history[-1]:.3f}"
)
print("\n[PASS] Confidence increases with sample size")
class TestTOINWithSmartCrusher:
"""Test 5: Verify TOIN integration with SmartCrusher."""