mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 15:07:29 +08:00
test(toin): delete the permanently skipped PR-B5 get_recommendation tests
PR-B5 (6819b7e5) made ToolIntelligenceNetwork.get_recommendation() a stub that returns None. The tests for the old request-time hints were marked skip instead of deleted, so 15 tests (about 590 lines) have not run since May. test_toin_observation_only.py covers the current contract.
Evidence: every removed test carries an unconditional @pytest.mark.skip(reason="PR-B5: ..."); no live test was touched.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Assisted-by: claude-code:claude-opus-5-5
Headline-Session: 6e77a807-8dd4-4a77-a2fa-ddce6dc2515e
Headline-AI-Share: 100% strict / 100% weighted
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a9815bbb5f
commit
37d4666cc9
@@ -845,12 +845,6 @@ class TestTOINHighPriorityFixes:
|
||||
# (auto-save happens inside record_compression)
|
||||
assert not toin._dirty
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="PR-B5: get_recommendation retired; preserve_fields lives on the aggregated ToolPattern instead"
|
||||
)
|
||||
def test_toin_preserves_fields_returns_list(self):
|
||||
"""Retired in PR-B5 along with the request-time hint API."""
|
||||
|
||||
|
||||
class TestCompressionStoreHighPriorityFixes:
|
||||
"""Additional HIGH priority tests for CompressionStore."""
|
||||
|
||||
+3
-45
@@ -1,9 +1,8 @@
|
||||
"""Tests for Tool Output Intelligence Network (TOIN).
|
||||
|
||||
PR-B5 retired the request-time hint API. Tests that exercised the old
|
||||
`get_recommendation()` / `CompressionHint` shape are skipped at module
|
||||
level — the new observation-only contract is covered by
|
||||
`tests/test_toin_observation_only.py` and `tests/test_toin_publish.py`.
|
||||
PR-B5 retired the request-time hint API. The observation-only contract
|
||||
is covered by `tests/test_toin_observation_only.py` and
|
||||
`tests/test_toin_publish.py`.
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -331,47 +330,6 @@ class TestToolIntelligenceNetwork:
|
||||
# Field should be in commonly_retrieved_fields after 3+ retrievals
|
||||
assert len(pattern.commonly_retrieved_fields) > 0
|
||||
|
||||
# PR-B5: the following tests exercised the request-time hint API
|
||||
# that's now retired. They're skipped wholesale; the new contract
|
||||
# ("get_recommendation always returns None and emits a deprecation
|
||||
# warning") is covered by tests/test_toin_observation_only.py.
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"
|
||||
)
|
||||
def test_get_recommendation_no_data(self):
|
||||
pass
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"
|
||||
)
|
||||
def test_get_recommendation_insufficient_samples(self):
|
||||
pass
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"
|
||||
)
|
||||
def test_get_recommendation_aggressive_compression(self):
|
||||
pass
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"
|
||||
)
|
||||
def test_get_recommendation_conservative_compression(self):
|
||||
pass
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"
|
||||
)
|
||||
def test_get_recommendation_skip_compression(self):
|
||||
pass
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="PR-B5: get_recommendation retired — see test_toin_observation_only.py"
|
||||
)
|
||||
def test_get_recommendation_disabled(self):
|
||||
pass
|
||||
|
||||
def test_get_stats(self):
|
||||
"""get_stats returns overall statistics."""
|
||||
toin = ToolIntelligenceNetwork()
|
||||
|
||||
@@ -10,7 +10,6 @@ from headroom.cache.compression_store import get_compression_store, reset_compre
|
||||
from headroom.telemetry import (
|
||||
TOINConfig,
|
||||
ToolIntelligenceNetwork,
|
||||
ToolPattern,
|
||||
ToolSignature,
|
||||
get_toin,
|
||||
reset_toin,
|
||||
@@ -81,94 +80,6 @@ def test_kompress_ccr_retrieval_updates_toin():
|
||||
assert stats["total_retrievals"] == 1
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="PR-B5: observations counter and request-time hint API retired")
|
||||
class TestGetRecommendationObservations:
|
||||
"""Bug 1: get_recommendation() should increment observations counter."""
|
||||
|
||||
def test_increments_observations_when_pattern_exists(self):
|
||||
"""get_recommendation() should increment observations when pattern exists."""
|
||||
config = _make_config(min_samples=5)
|
||||
toin = ToolIntelligenceNetwork(config=config)
|
||||
sig = _make_signature("obs_test_hash")
|
||||
|
||||
# Record enough compressions to create a pattern with sufficient samples
|
||||
for _ in range(15):
|
||||
toin.record_compression(
|
||||
tool_signature=sig,
|
||||
original_count=100,
|
||||
compressed_count=20,
|
||||
original_tokens=5000,
|
||||
compressed_tokens=1000,
|
||||
strategy="top_n",
|
||||
)
|
||||
|
||||
# Get recommendation
|
||||
toin.get_recommendation(sig)
|
||||
|
||||
# Check observations incremented
|
||||
pattern = toin._patterns[("global", "unknown", "unknown", sig.structure_hash)]
|
||||
assert pattern.observations == 1
|
||||
|
||||
# Call again
|
||||
toin.get_recommendation(sig)
|
||||
assert pattern.observations == 2
|
||||
|
||||
def test_increments_observations_even_below_min_samples(self):
|
||||
"""observations increments even when sample_size < min_samples."""
|
||||
config = _make_config(min_samples=100)
|
||||
toin = ToolIntelligenceNetwork(config=config)
|
||||
sig = _make_signature("low_sample_hash")
|
||||
|
||||
# Record just a few compressions (below min_samples)
|
||||
for _ in range(3):
|
||||
toin.record_compression(
|
||||
tool_signature=sig,
|
||||
original_count=50,
|
||||
compressed_count=10,
|
||||
original_tokens=2000,
|
||||
compressed_tokens=500,
|
||||
strategy="top_n",
|
||||
)
|
||||
|
||||
result = toin.get_recommendation(sig)
|
||||
assert result.source == "local" # Not enough samples
|
||||
|
||||
pattern = toin._patterns[("global", "unknown", "unknown", sig.structure_hash)]
|
||||
assert pattern.observations == 1
|
||||
|
||||
def test_no_increment_for_unknown_pattern(self):
|
||||
"""get_recommendation() should NOT increment for unknown patterns."""
|
||||
config = _make_config()
|
||||
toin = ToolIntelligenceNetwork(config=config)
|
||||
sig = _make_signature("nonexistent_hash")
|
||||
|
||||
result = toin.get_recommendation(sig)
|
||||
assert result.source == "default"
|
||||
assert result.reason == "No pattern data for this tool type"
|
||||
|
||||
# No pattern exists, nothing to increment
|
||||
assert "nonexistent_hash" not in toin._patterns
|
||||
|
||||
def test_observations_survives_serialization(self):
|
||||
"""observations field should serialize and deserialize correctly."""
|
||||
pattern = ToolPattern(
|
||||
tool_signature_hash="serial_test",
|
||||
total_compressions=10,
|
||||
observations=42,
|
||||
)
|
||||
|
||||
d = pattern.to_dict()
|
||||
assert d["observations"] == 42
|
||||
|
||||
restored = ToolPattern.from_dict(d)
|
||||
assert restored.observations == 42
|
||||
|
||||
def test_observations_defaults_to_zero(self):
|
||||
"""observations defaults to 0 for new patterns."""
|
||||
pattern = ToolPattern(tool_signature_hash="new_pattern")
|
||||
assert pattern.observations == 0
|
||||
|
||||
|
||||
class TestRecordRetrievalPopulatesFields:
|
||||
"""Bug 1 related: record_retrieval with query_fields populates field data."""
|
||||
|
||||
|
||||
@@ -68,99 +68,6 @@ def fresh_store():
|
||||
reset_compression_store()
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="PR-B5: strategy-recommendation API retired (get_recommendation returns None)"
|
||||
)
|
||||
class TestStrategySuccessRates:
|
||||
"""Test that strategy_success_rates are used in recommendations."""
|
||||
|
||||
def test_recommends_strategy_with_high_success_rate(self, fresh_toin):
|
||||
"""Strategy with success rate >= 0.5 should be recommended."""
|
||||
items = [{"id": i, "score": 100 - i} for i in range(20)]
|
||||
signature = ToolSignature.from_items(items)
|
||||
|
||||
# Record compressions to build pattern
|
||||
for _ in range(10):
|
||||
fresh_toin.record_compression(
|
||||
tool_signature=signature,
|
||||
original_count=20,
|
||||
compressed_count=10,
|
||||
original_tokens=2000,
|
||||
compressed_tokens=1000,
|
||||
strategy="smart_sample",
|
||||
)
|
||||
|
||||
# Set high success rate
|
||||
pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
|
||||
pattern.strategy_success_rates["smart_sample"] = 0.8
|
||||
pattern.optimal_strategy = "smart_sample"
|
||||
|
||||
# Get recommendation
|
||||
hint = fresh_toin.get_recommendation(signature, "test query")
|
||||
|
||||
assert hint.recommended_strategy == "smart_sample"
|
||||
|
||||
def test_rejects_strategy_with_low_success_rate(self, fresh_toin):
|
||||
"""Strategy with success rate < 0.5 should NOT be recommended."""
|
||||
items = [{"id": i, "score": 100 - i} for i in range(20)]
|
||||
signature = ToolSignature.from_items(items)
|
||||
|
||||
# Record compressions
|
||||
for _ in range(10):
|
||||
fresh_toin.record_compression(
|
||||
tool_signature=signature,
|
||||
original_count=20,
|
||||
compressed_count=10,
|
||||
original_tokens=2000,
|
||||
compressed_tokens=1000,
|
||||
strategy="bad_strategy",
|
||||
)
|
||||
|
||||
# Set low success rate
|
||||
pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
|
||||
pattern.strategy_success_rates["bad_strategy"] = 0.2
|
||||
pattern.optimal_strategy = "bad_strategy"
|
||||
|
||||
# Get recommendation
|
||||
hint = fresh_toin.get_recommendation(signature, "test query")
|
||||
|
||||
# Should not recommend the bad strategy
|
||||
assert hint.recommended_strategy != "bad_strategy"
|
||||
# Confidence should be reduced
|
||||
assert "low success" in hint.reason.lower()
|
||||
|
||||
def test_finds_best_strategy_when_optimal_is_bad(self, fresh_toin):
|
||||
"""When optimal_strategy has low success, find a better alternative."""
|
||||
items = [{"id": i, "score": 100 - i} for i in range(20)]
|
||||
signature = ToolSignature.from_items(items)
|
||||
|
||||
# Record compressions
|
||||
for _ in range(10):
|
||||
fresh_toin.record_compression(
|
||||
tool_signature=signature,
|
||||
original_count=20,
|
||||
compressed_count=10,
|
||||
original_tokens=2000,
|
||||
compressed_tokens=1000,
|
||||
strategy="smart_sample",
|
||||
)
|
||||
|
||||
# Set up multiple strategies with different success rates
|
||||
pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
|
||||
pattern.strategy_success_rates = {
|
||||
"bad_strategy": 0.2,
|
||||
"good_strategy": 0.9,
|
||||
}
|
||||
pattern.optimal_strategy = "bad_strategy"
|
||||
|
||||
# Get recommendation
|
||||
hint = fresh_toin.get_recommendation(signature, "test query")
|
||||
|
||||
# Should recommend the better strategy
|
||||
assert hint.recommended_strategy == "good_strategy"
|
||||
assert "using good_strategy instead" in hint.reason
|
||||
|
||||
|
||||
class TestPreserveFieldsMerging:
|
||||
"""Test preserve_fields merging in federated learning."""
|
||||
|
||||
@@ -352,166 +259,6 @@ class TestUserCountTracking:
|
||||
assert pattern.user_count >= 3
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="PR-B5: get_recommendation retired; field-weighting now consumed only by toin publish"
|
||||
)
|
||||
class TestFieldRetrievalFrequencyWeighting:
|
||||
"""Test field_retrieval_frequency weighting in preserve_fields."""
|
||||
|
||||
def test_query_fields_prioritized_in_preserve_fields(self, fresh_toin):
|
||||
"""Fields mentioned in query should be prioritized."""
|
||||
items = [{"id": i, "status": "ok", "category": f"cat_{i}"} for i in range(20)]
|
||||
signature = ToolSignature.from_items(items)
|
||||
|
||||
# Build pattern with field retrieval data
|
||||
for _ in range(10):
|
||||
fresh_toin.record_compression(
|
||||
tool_signature=signature,
|
||||
original_count=20,
|
||||
compressed_count=10,
|
||||
original_tokens=2000,
|
||||
compressed_tokens=1000,
|
||||
strategy="smart_sample",
|
||||
)
|
||||
|
||||
# Record retrievals for "status" field
|
||||
status_hash = fresh_toin._hash_field_name("status")
|
||||
pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
|
||||
pattern.field_retrieval_frequency = {
|
||||
status_hash: 50,
|
||||
fresh_toin._hash_field_name("category"): 10,
|
||||
}
|
||||
pattern.preserve_fields = [status_hash]
|
||||
|
||||
# Get recommendation with query mentioning "status"
|
||||
hint = fresh_toin.get_recommendation(signature, "status:error")
|
||||
|
||||
# status hash should be in preserve_fields
|
||||
assert status_hash in hint.preserve_fields
|
||||
|
||||
def test_preserve_fields_sorted_by_frequency(self, fresh_toin):
|
||||
"""preserve_fields should be sorted by retrieval frequency."""
|
||||
items = [{"id": i} for i in range(20)]
|
||||
signature = ToolSignature.from_items(items)
|
||||
|
||||
# Build pattern
|
||||
for _ in range(10):
|
||||
fresh_toin.record_compression(
|
||||
tool_signature=signature,
|
||||
original_count=20,
|
||||
compressed_count=10,
|
||||
original_tokens=2000,
|
||||
compressed_tokens=1000,
|
||||
strategy="smart_sample",
|
||||
)
|
||||
|
||||
pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
|
||||
field_a = fresh_toin._hash_field_name("field_a")
|
||||
field_b = fresh_toin._hash_field_name("field_b")
|
||||
field_c = fresh_toin._hash_field_name("field_c")
|
||||
|
||||
pattern.field_retrieval_frequency = {
|
||||
field_a: 10,
|
||||
field_b: 50, # Most frequent
|
||||
field_c: 30,
|
||||
}
|
||||
pattern.preserve_fields = [field_a, field_b, field_c]
|
||||
|
||||
# Get recommendation (no query context)
|
||||
hint = fresh_toin.get_recommendation(signature, "")
|
||||
|
||||
# Should be sorted by frequency
|
||||
if len(hint.preserve_fields) >= 3:
|
||||
# field_b should come before field_c which should come before field_a
|
||||
b_idx = hint.preserve_fields.index(field_b) if field_b in hint.preserve_fields else -1
|
||||
c_idx = hint.preserve_fields.index(field_c) if field_c in hint.preserve_fields else -1
|
||||
hint.preserve_fields.index(field_a) if field_a in hint.preserve_fields else -1
|
||||
|
||||
if b_idx >= 0 and c_idx >= 0:
|
||||
assert b_idx < c_idx, "Higher frequency field should come first"
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="PR-B5: get_recommendation retired (returns None / DeprecationWarning)")
|
||||
class TestQueryContextUsage:
|
||||
"""Test query_context usage in recommendations."""
|
||||
|
||||
def test_exhaustive_query_keywords_detected(self, fresh_toin):
|
||||
"""Exhaustive query keywords should trigger conservative compression."""
|
||||
items = [{"id": i, "score": 100 - i} for i in range(50)]
|
||||
signature = ToolSignature.from_items(items)
|
||||
|
||||
# Build pattern with aggressive compression normally
|
||||
for _ in range(10):
|
||||
fresh_toin.record_compression(
|
||||
tool_signature=signature,
|
||||
original_count=50,
|
||||
compressed_count=10,
|
||||
original_tokens=5000,
|
||||
compressed_tokens=1000,
|
||||
strategy="smart_sample",
|
||||
)
|
||||
|
||||
# Low retrieval rate = aggressive compression
|
||||
pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
|
||||
pattern.total_retrievals = 0
|
||||
|
||||
# Query with exhaustive keyword
|
||||
hint = fresh_toin.get_recommendation(signature, "list all items in category")
|
||||
|
||||
# Should be more conservative
|
||||
assert hint.max_items >= 40
|
||||
assert "exhaustive query" in hint.reason.lower()
|
||||
assert hint.compression_level == "conservative"
|
||||
|
||||
def test_every_keyword_triggers_conservative(self, fresh_toin):
|
||||
"""'every' keyword should trigger conservative compression."""
|
||||
items = [{"id": i} for i in range(50)]
|
||||
signature = ToolSignature.from_items(items)
|
||||
|
||||
for _ in range(10):
|
||||
fresh_toin.record_compression(
|
||||
tool_signature=signature,
|
||||
original_count=50,
|
||||
compressed_count=10,
|
||||
original_tokens=5000,
|
||||
compressed_tokens=1000,
|
||||
strategy="smart_sample",
|
||||
)
|
||||
|
||||
pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
|
||||
pattern.total_retrievals = 0
|
||||
|
||||
hint = fresh_toin.get_recommendation(signature, "find every user")
|
||||
|
||||
assert "exhaustive query" in hint.reason.lower()
|
||||
|
||||
def test_partial_pattern_matching(self, fresh_toin):
|
||||
"""Partial pattern matching should boost max_items."""
|
||||
items = [{"id": i, "status": "ok"} for i in range(50)]
|
||||
signature = ToolSignature.from_items(items)
|
||||
|
||||
for _ in range(10):
|
||||
fresh_toin.record_compression(
|
||||
tool_signature=signature,
|
||||
original_count=50,
|
||||
compressed_count=10,
|
||||
original_tokens=5000,
|
||||
compressed_tokens=1000,
|
||||
strategy="smart_sample",
|
||||
)
|
||||
|
||||
pattern = fresh_toin._patterns[("global", "unknown", "unknown", signature.structure_hash)]
|
||||
pattern.total_retrievals = 0
|
||||
# Add a problematic query pattern
|
||||
pattern.common_query_patterns = ["status:*"]
|
||||
|
||||
# Query that uses the same field
|
||||
hint = fresh_toin.get_recommendation(signature, "status:error")
|
||||
|
||||
# Should match the pattern
|
||||
assert hint.max_items >= 25 or "retrieval pattern" in hint.reason
|
||||
|
||||
|
||||
class TestFeedbackStrategyTracking:
|
||||
"""Test strategy tracking in compression feedback."""
|
||||
|
||||
|
||||
@@ -247,208 +247,6 @@ class TestTOINPersistenceAcrossInstances:
|
||||
print("\n[PASS] TOIN persistence works correctly")
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="PR-B5: get_recommendation retired; feedback-loop covered by test_toin_observation_only.py"
|
||||
)
|
||||
class TestTOINFullFeedbackLoop:
|
||||
"""Test 3: Verify TOIN feedback loop with recommendations."""
|
||||
|
||||
def test_toin_full_feedback_loop(self, sample_tool_signature):
|
||||
"""Verify TOIN learns from high retrieval rate and recommends skip."""
|
||||
print("\n" + "=" * 60)
|
||||
print("TEST: test_toin_full_feedback_loop")
|
||||
print("=" * 60)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
storage_path = str(Path(tmpdir) / "toin_feedback_test.json")
|
||||
config = TOINConfig(
|
||||
storage_path=storage_path,
|
||||
min_samples_for_recommendation=5, # Lower threshold for test
|
||||
high_retrieval_threshold=0.5, # 50% retrieval = high
|
||||
)
|
||||
toin = ToolIntelligenceNetwork(config)
|
||||
|
||||
print("\n--- Phase 1: Record compressions ---")
|
||||
# Record 5 compressions with same tool signature
|
||||
for i in range(5):
|
||||
toin.record_compression(
|
||||
tool_signature=sample_tool_signature,
|
||||
original_count=100,
|
||||
compressed_count=15,
|
||||
original_tokens=5000,
|
||||
compressed_tokens=750,
|
||||
strategy="smart_sample",
|
||||
)
|
||||
print(f" Recorded compression {i + 1}")
|
||||
|
||||
print("\n--- Phase 2: Record retrievals (simulating high retrieval rate) ---")
|
||||
# Record 3 full retrievals (60% retrieval rate = high)
|
||||
for i in range(3):
|
||||
toin.record_retrieval(
|
||||
tool_signature_hash=sample_tool_signature.structure_hash,
|
||||
retrieval_type="full", # Full retrieval = compression too aggressive
|
||||
strategy="smart_sample",
|
||||
)
|
||||
print(f" Recorded full retrieval {i + 1}")
|
||||
|
||||
# Get pattern stats
|
||||
pattern = toin.get_pattern(sample_tool_signature.structure_hash)
|
||||
print("\n--- Pattern Stats ---")
|
||||
print(f" total_compressions: {pattern.total_compressions}")
|
||||
print(f" total_retrievals: {pattern.total_retrievals}")
|
||||
print(f" retrieval_rate: {pattern.retrieval_rate:.1%}")
|
||||
print(f" full_retrieval_rate: {pattern.full_retrieval_rate:.1%}")
|
||||
print(f" skip_compression_recommended: {pattern.skip_compression_recommended}")
|
||||
|
||||
# Get recommendation
|
||||
print("\n--- Getting Recommendation ---")
|
||||
hint = toin.get_recommendation(sample_tool_signature)
|
||||
print(f" source: {hint.source}")
|
||||
print(f" skip_compression: {hint.skip_compression}")
|
||||
print(f" compression_level: {hint.compression_level}")
|
||||
print(f" max_items: {hint.max_items}")
|
||||
print(f" confidence: {hint.confidence:.3f}")
|
||||
print(f" reason: {hint.reason}")
|
||||
print(f" based_on_samples: {hint.based_on_samples}")
|
||||
|
||||
# Verify high retrieval rate triggers skip recommendation
|
||||
# With 60% retrieval rate (3/5) and full_retrieval_rate of 100% (3/3),
|
||||
# TOIN should recommend skipping compression
|
||||
retrieval_rate = pattern.retrieval_rate
|
||||
assert retrieval_rate >= 0.5, (
|
||||
f"Expected retrieval rate >= 50%, got {retrieval_rate:.1%}"
|
||||
)
|
||||
|
||||
# With high retrieval rate and high full retrieval rate, should skip
|
||||
if pattern.full_retrieval_rate > 0.8:
|
||||
assert hint.skip_compression or hint.compression_level in (
|
||||
"none",
|
||||
"conservative",
|
||||
), (
|
||||
f"High full retrieval rate should trigger skip or conservative, "
|
||||
f"got compression_level={hint.compression_level}"
|
||||
)
|
||||
print("\n[PASS] High retrieval rate correctly influences recommendation")
|
||||
else:
|
||||
print("\n[INFO] Full retrieval rate not high enough for skip recommendation")
|
||||
print(f" full_retrieval_rate: {pattern.full_retrieval_rate:.1%}")
|
||||
|
||||
print("\n[PASS] TOIN feedback loop works correctly")
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="PR-B5: get_recommendation retired; confidence-progression validated via record + get_pattern instead"
|
||||
)
|
||||
class TestTOINProgressiveConfidence:
|
||||
"""Test 4: Verify TOIN confidence increases with sample size."""
|
||||
|
||||
def test_toin_progressive_confidence(self, sample_tool_signature):
|
||||
"""Verify confidence increases with more samples."""
|
||||
print("\n" + "=" * 60)
|
||||
print("TEST: test_toin_progressive_confidence")
|
||||
print("=" * 60)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
storage_path = str(Path(tmpdir) / "toin_confidence_test.json")
|
||||
config = TOINConfig(
|
||||
storage_path=storage_path,
|
||||
min_samples_for_recommendation=3,
|
||||
)
|
||||
toin = ToolIntelligenceNetwork(config)
|
||||
|
||||
confidence_history = []
|
||||
|
||||
# Batch 1: Record 1 compression
|
||||
print("\n--- Batch 1: 1 compression ---")
|
||||
toin.record_compression(
|
||||
tool_signature=sample_tool_signature,
|
||||
original_count=100,
|
||||
compressed_count=15,
|
||||
original_tokens=5000,
|
||||
compressed_tokens=750,
|
||||
strategy="smart_sample",
|
||||
)
|
||||
pattern = toin.get_pattern(sample_tool_signature.structure_hash)
|
||||
hint = toin.get_recommendation(sample_tool_signature)
|
||||
confidence_history.append(pattern.confidence)
|
||||
print(f" sample_size: {pattern.sample_size}")
|
||||
print(f" confidence: {pattern.confidence:.3f}")
|
||||
print(f" hint.source: {hint.source}")
|
||||
|
||||
# Batch 2: Record 2 more compressions
|
||||
print("\n--- Batch 2: +2 compressions (total: 3) ---")
|
||||
for _ in range(2):
|
||||
toin.record_compression(
|
||||
tool_signature=sample_tool_signature,
|
||||
original_count=100,
|
||||
compressed_count=15,
|
||||
original_tokens=5000,
|
||||
compressed_tokens=750,
|
||||
strategy="smart_sample",
|
||||
)
|
||||
pattern = toin.get_pattern(sample_tool_signature.structure_hash)
|
||||
hint = toin.get_recommendation(sample_tool_signature)
|
||||
confidence_history.append(pattern.confidence)
|
||||
print(f" sample_size: {pattern.sample_size}")
|
||||
print(f" confidence: {pattern.confidence:.3f}")
|
||||
print(f" hint.source: {hint.source}")
|
||||
|
||||
# Batch 3: Record 2 more compressions
|
||||
print("\n--- Batch 3: +2 compressions (total: 5) ---")
|
||||
for _ in range(2):
|
||||
toin.record_compression(
|
||||
tool_signature=sample_tool_signature,
|
||||
original_count=100,
|
||||
compressed_count=15,
|
||||
original_tokens=5000,
|
||||
compressed_tokens=750,
|
||||
strategy="smart_sample",
|
||||
)
|
||||
pattern = toin.get_pattern(sample_tool_signature.structure_hash)
|
||||
hint = toin.get_recommendation(sample_tool_signature)
|
||||
confidence_history.append(pattern.confidence)
|
||||
print(f" sample_size: {pattern.sample_size}")
|
||||
print(f" confidence: {pattern.confidence:.3f}")
|
||||
print(f" hint.source: {hint.source}")
|
||||
|
||||
# Batch 4: Add many more to boost confidence
|
||||
print("\n--- Batch 4: +15 compressions (total: 20) ---")
|
||||
for _ in range(15):
|
||||
toin.record_compression(
|
||||
tool_signature=sample_tool_signature,
|
||||
original_count=100,
|
||||
compressed_count=15,
|
||||
original_tokens=5000,
|
||||
compressed_tokens=750,
|
||||
strategy="smart_sample",
|
||||
)
|
||||
pattern = toin.get_pattern(sample_tool_signature.structure_hash)
|
||||
hint = toin.get_recommendation(sample_tool_signature)
|
||||
confidence_history.append(pattern.confidence)
|
||||
print(f" sample_size: {pattern.sample_size}")
|
||||
print(f" confidence: {pattern.confidence:.3f}")
|
||||
print(f" hint.source: {hint.source}")
|
||||
|
||||
# Print confidence progression
|
||||
print("\n--- Confidence Progression ---")
|
||||
for i, conf in enumerate(confidence_history):
|
||||
print(f" Stage {i + 1}: confidence = {conf:.3f}")
|
||||
|
||||
# Verify confidence increases with sample size
|
||||
# Confidence should generally increase (may plateau at high values)
|
||||
assert confidence_history[-1] >= confidence_history[0], (
|
||||
f"Confidence should increase: start={confidence_history[0]:.3f}, "
|
||||
f"end={confidence_history[-1]:.3f}"
|
||||
)
|
||||
|
||||
# With 20 samples, should have meaningful confidence
|
||||
assert confidence_history[-1] >= 0.1, (
|
||||
f"With 20 samples, confidence should be >= 0.1, got {confidence_history[-1]:.3f}"
|
||||
)
|
||||
|
||||
print("\n[PASS] Confidence increases with sample size")
|
||||
|
||||
|
||||
class TestTOINWithSmartCrusher:
|
||||
"""Test 5: Verify TOIN integration with SmartCrusher."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user