mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 15:10:00 +08:00
## Description Overwriting a SemanticCache entry refreshed its response and timestamps but left its old position in the OrderedDict. With capacity two, put(A), put(B), update(A), put(C) evicted the freshly updated A and retained the older B. Move a successfully stored entry to the end of the LRU. Cache state operations now share an RLock, including semantic scans and touches, while embedding callbacks execute outside it. This happens after embedding generation and entry construction, so a failed update still preserves the original eviction order. It complements the query-only retrieval fix in #4709: a successfully refreshed response should remain retrievable under subsequent capacity pressure. ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) ## Changes Made - Refresh LRU position after successful insertion/replacement. - Cover query-only and context-hash entries, with and without namespaces. - Keep failed embedding updates from refreshing recency, and retain the no-eviction-on-update contract from #2094. - Synchronize cache/index/statistics operations and the complete similarity-scan/touch sequence. Embedding callbacks run outside the state lock; add two-thread scan/update and nonblocking exact-hit regressions. ## Testing - [x] Relevant unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - [x] Formatting passes (`ruff format --check .`) - [x] Type checking passes (original full-package mypy hook) - [x] Regression tests added - [x] Standalone runtime validation completed ### Test Output ```text Initial new regressions against original production code: 4 failed, 1 passed pytest tests/test_cache tests/test_cold_prefix.py: 398 passed, 3 skipped Review follow-up on the initial PR head: 1 failed Original pre-commit and commit-msg hooks (including follow-up): passed Whole-tree Ruff check / format --check: passed (1932 files) Original pre-push make ci-precheck: passed Rust workspace: 1749 passed, 3 ignored Native extension: built, installed and import-verified Critical Python: 177 passed, 8 skipped commitlint: passed ``` The three cache skips require optional spaCy/embedding models. Critical-suite skips require optional relevance models. The existing token-count approximation warning remains visible in the gate log. Initial broader runs failed because the new worktrees lacked the native extension; these were rerun successfully after building it. ## Real Behavior Proof - Environment: Windows 11, CPython 3.13.9, existing development venv; clean baseline `200c838c7393d20ae56a0c338f832ed27133c398`, patch `ffa5198d3e4a7ab634e49efb8b6c6b59e4315acf`. `HEADROOM_OFFLINE=1`, `PYTHONPATH=.` for the selected checkout. No paid/provider credentials. - Exact command / steps: Save the following identical script as `proof.py` and run `python proof.py` from either checkout. Before and after runs were outside pytest. ```python """A real SemanticCache overwrite followed by a capacity-triggered eviction.""" import json from headroom.cache.semantic import SemanticCache, SemanticCacheConfig reports = [] for context in (False, True): for namespace in ("", "provider/model"): cache = SemanticCache(SemanticCacheConfig(max_entries=2)) hashes = ["context-a", "context-b", "context-c"] if context else [None] * 3 cache.put("first", {"answer": "old"}, hashes[0], namespace) cache.put("second", {"answer": "older"}, hashes[1], namespace) cache.put("first", {"answer": "updated"}, hashes[0], namespace) before = cache.get_stats() cache.put("third", {"answer": "new"}, hashes[2], namespace) first = cache.get("first", hashes[0], namespace) second = cache.get("second", hashes[1], namespace) reports.append({"context": context, "namespace": namespace, "update_evictions": before["evictions"], "updated_response": first.response if first else None, "older_response": second.response if second else None, "entries": cache.get_stats()["entries"], "evictions": cache.get_stats()["evictions"]}) print(json.dumps(reports, indent=2)) ``` Review follow-up: run `python concurrency_proof.py` with the script below on the initial PR head and the revised head. It coordinates two real cache callers at a similarity-scan boundary. ```python """A real two-thread scan/update interleaving, without mocked cache results.""" import json import threading from headroom.cache.semantic import SemanticCache scanning, release, writer_progress = threading.Event(), threading.Event(), threading.Event() class PausedCache(SemanticCache): def _cosine_similarity(self, a, b): if not scanning.is_set(): scanning.set() if not release.wait(10): raise RuntimeError("Scan release missing") return super()._cosine_similarity(a, b) cache = PausedCache(embedding_fn=lambda query: [1.0, 0.0]) cache.put("first", "old", "first") cache.put("second", "two", "second") real = threading.RLock() class SignalsWriterLock: def __enter__(self): if threading.current_thread().name == "cache-update": writer_progress.set() return real.__enter__() def __exit__(self, *exc): return real.__exit__(*exc) cache._lock = SignalsWriterLock() errors, results = [], [] def read(): try: entry = cache.get("third", "third") results.append(entry.response if entry else None) except Exception as exc: errors.append(type(exc).__name__ + ": " + str(exc)) def update(): try: cache.put("first", "updated", "first") except Exception as exc: errors.append(type(exc).__name__ + ": " + str(exc)) finally: writer_progress.set() reader = threading.Thread(target=read) writer = threading.Thread(target=update, name="cache-update") reader.start() try: assert scanning.wait(10) writer.start() assert writer_progress.wait(10) finally: release.set() reader.join(10) if writer.ident is not None: writer.join(10) assert not reader.is_alive() and not writer.is_alive() print(json.dumps({"errors": errors, "lookup_responses": results, "updated_response": cache.get("first", "first").response})) # Existing tie behavior: an ordinary exact hit already changes scan order. # No put/update occurs between the two semantic lookups in this control. ties = SemanticCache(embedding_fn=lambda query: [1.0, 0.0]) ties.put("first", "A", "first") ties.put("second", "B", "second") first = ties.get("one unseen query", "unseen-1").response ties.get("first", "first") second = ties.get("another unseen query", "unseen-2").response print(json.dumps({"equal_score_control": [first, second], "policy": "first match in current LRU order; no most-recent tie guarantee"})) ``` - Observed result: The real SemanticCache API reproduces the sequence for four context/namespace combinations. Before the patch, the updated response is missing and the older response survives. After it, updated_response is updated and older_response is None; each case still has two entries and exactly one eviction. No get() is made between update(A) and put(C), which would mask the defect by touching A. The concurrent proof raises OrderedDict mutated during iteration on the initial PR head and completes without errors on the revised head. The equal-score control returns A then B on both original main and revised code: ties still follow current LRU scan order, with no added freshness guarantee. - Not tested: Paid/live embedding services, model quality, production hit-rate/latency improvements and the complete Python suite. The runtime proof uses exact matching without an embedding dependency. ## Runtime Rollout Safety - Rollout-managed feature(s): Existing cache behavior; no new feature flag or channel expansion. - Minimum rollout channel: Normal release after maintainer review and CI qualification. - Stable/default behavior changed: Successful replacement counts as recent use. Capacity, exact/context isolation, TTL and failed-store behavior remain unchanged. - Kill switch / disable path: Disable the existing optional semantic cache in its caller, or revert this commit. - Unsafe override required: No. - Qualification impact: Related cache tests and original local gates passed; hosted CI is tracked separately. - Rollback path: Revert this commit. No dependency or storage-schema change. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have added tests that prove the fix - [x] Relevant new and existing tests pass locally - [x] I did **not** edit `CHANGELOG.md` ## Additional Notes Implementation and local validation were completed with Codex assistance. The author reviewed the initial code and validation record; the review follow-up was checked with Codex. <!-- devin-review-badge-begin --> --- <a href="https://app.devin.ai/review/headroomlabs-ai/headroom/pull/4767" target="_blank"><picture><source media="(prefers-color-scheme: dark)" srcset="https://static.devin.ai/assets/gh-devin-review-dark.svg?v=4"><img src="https://static.devin.ai/assets/gh-devin-review-light.svg?v=4" alt="Devin Review"></picture></a> <!-- devin-review-badge-end -->
883 lines
35 KiB
Python
883 lines
35 KiB
Python
"""Tests for SemanticCache and SemanticCacheLayer."""
|
|
|
|
import hashlib
|
|
import time
|
|
from datetime import datetime
|
|
|
|
import pytest
|
|
|
|
from headroom.cache import (
|
|
AnthropicCacheOptimizer,
|
|
OptimizationContext,
|
|
SemanticCache,
|
|
SemanticCacheLayer,
|
|
)
|
|
from headroom.cache.semantic import SemanticCacheConfig
|
|
|
|
|
|
class TestSemanticCacheConfig:
|
|
"""Test SemanticCacheConfig."""
|
|
|
|
def test_default_values(self):
|
|
"""Test default configuration values."""
|
|
config = SemanticCacheConfig()
|
|
assert config.similarity_threshold == 0.95
|
|
assert config.max_entries == 1000
|
|
assert config.ttl_seconds == 300
|
|
assert config.use_exact_matching is True
|
|
|
|
|
|
class TestSemanticCache:
|
|
"""Test SemanticCache functionality."""
|
|
|
|
@pytest.fixture
|
|
def cache(self):
|
|
"""Create cache instance."""
|
|
config = SemanticCacheConfig(
|
|
max_entries=10,
|
|
ttl_seconds=60,
|
|
)
|
|
return SemanticCache(config)
|
|
|
|
def test_put_and_get_exact_match(self, cache):
|
|
"""Test storing and retrieving with exact hash matching."""
|
|
response = {"text": "Hello, how can I help?"}
|
|
cache.put("What is the weather?", response, messages_hash="hash123")
|
|
|
|
entry = cache.get("What is the weather?", messages_hash="hash123")
|
|
assert entry is not None
|
|
assert entry.response == response
|
|
|
|
def test_get_miss(self, cache):
|
|
"""Test cache miss."""
|
|
entry = cache.get("Unknown query", messages_hash="unknown")
|
|
assert entry is None
|
|
|
|
@pytest.mark.parametrize("messages_hash", [None, ""])
|
|
@pytest.mark.parametrize("namespace", ["", "provider/model"])
|
|
def test_query_only_round_trip(self, messages_hash, namespace):
|
|
cache = SemanticCache()
|
|
cache.put("What failed?", "disk full", messages_hash, namespace)
|
|
|
|
entry = cache.get("What failed?", messages_hash, namespace)
|
|
|
|
assert entry is not None and entry.response == "disk full"
|
|
assert entry.access_count == 2
|
|
assert cache.get_stats()["hits"] == 1
|
|
assert cache.get("What failed?", namespace="other") is None
|
|
assert cache.get("What failed?", messages_hash="context", namespace=namespace) is None
|
|
assert cache.get("What else failed?", namespace=namespace) is None
|
|
|
|
def test_query_only_exact_hit_does_not_call_embedder_again(self):
|
|
def embed(text):
|
|
return [1.0, 0.0]
|
|
|
|
cache = SemanticCache(embedding_fn=embed)
|
|
cache.put("question", "answer")
|
|
|
|
def unavailable(text):
|
|
raise RuntimeError("embedding service offline")
|
|
|
|
cache._embedding_fn = unavailable
|
|
assert cache.get("question").response == "answer"
|
|
|
|
def test_query_only_hit_refreshes_lru(self):
|
|
cache = SemanticCache(SemanticCacheConfig(max_entries=2))
|
|
cache.put("first", "one")
|
|
cache.put("second", "two")
|
|
assert cache.get("first") is not None
|
|
cache.put("third", "three")
|
|
assert cache.get("first").response == "one"
|
|
assert cache.get("second") is None
|
|
|
|
def test_query_only_expiration_and_disabled_exact_matching(self, monkeypatch):
|
|
monkeypatch.setattr(time, "time", lambda: 1000.0)
|
|
cache = SemanticCache(SemanticCacheConfig(ttl_seconds=10))
|
|
cache.put("query", "answer")
|
|
assert cache.get("query") is not None
|
|
cache.config.use_exact_matching = False
|
|
assert cache.get("query") is None
|
|
cache.config.use_exact_matching = True
|
|
monkeypatch.setattr(time, "time", lambda: 1011.0)
|
|
assert cache.get("query") is None
|
|
|
|
@pytest.mark.parametrize("query", ["", " \n\t"])
|
|
def test_blank_query_without_context_never_hits(self, query):
|
|
cache = SemanticCache()
|
|
cache.put(query, "answer")
|
|
assert cache.get(query) is None
|
|
|
|
def test_query_hash_cannot_read_a_context_key(self):
|
|
query = "continue"
|
|
query_hash = hashlib.sha256(query.encode()).hexdigest()[:16]
|
|
cache = SemanticCache()
|
|
cache.put(query, "context answer", messages_hash=query_hash)
|
|
assert cache.get(query) is None
|
|
|
|
@pytest.mark.parametrize("max_entries", [0, -1])
|
|
def test_put_rejects_nonpositive_capacity(self, max_entries, monkeypatch):
|
|
cache = SemanticCache(SemanticCacheConfig(max_entries=max_entries))
|
|
# Fail promptly instead of hanging if the unguarded eviction loop returns.
|
|
monkeypatch.setattr(
|
|
cache, "_evict_oldest", lambda: pytest.fail("Invalid capacity must not evict entries")
|
|
)
|
|
|
|
with pytest.raises(ValueError, match="max_entries must be positive"):
|
|
cache.put("query", "response", messages_hash="request")
|
|
|
|
assert cache.get_stats()["entries"] == 0
|
|
|
|
@pytest.mark.parametrize("max_entries", [0, -1])
|
|
@pytest.mark.parametrize("messages_hash", ["existing", "new"])
|
|
def test_invalid_capacity_change_preserves_entries(
|
|
self, max_entries, messages_hash, monkeypatch
|
|
):
|
|
cache = SemanticCache(SemanticCacheConfig(max_entries=1))
|
|
cache.put("query", "original", messages_hash="existing")
|
|
cache.config.max_entries = max_entries
|
|
stats = cache.get_stats()
|
|
monkeypatch.setattr(
|
|
cache, "_evict_oldest", lambda: pytest.fail("Invalid capacity must not evict entries")
|
|
)
|
|
|
|
with pytest.raises(ValueError, match="max_entries must be positive"):
|
|
cache.put("query", "replacement", messages_hash=messages_hash)
|
|
|
|
assert cache.get_stats() == stats
|
|
assert cache.get("query", messages_hash="existing").response == "original"
|
|
assert cache.get("query", messages_hash="new") is None
|
|
|
|
cache.config.max_entries = 1
|
|
cache.put("query", "updated", messages_hash="existing")
|
|
assert cache.get("query", messages_hash="existing").response == "updated"
|
|
|
|
def test_same_query_different_context_does_not_collide(self, cache):
|
|
"""Two requests that share a trailing user message but differ in earlier
|
|
context (distinct messages_hash) must not overwrite each other. Before the
|
|
fix both were keyed by sha256(query), so the second clobbered the first and
|
|
the first's hash resolved to the second's response."""
|
|
cache.put("run the tests", {"text": "response A"}, messages_hash="ctxA")
|
|
cache.put("run the tests", {"text": "response B"}, messages_hash="ctxB")
|
|
|
|
got_a = cache.get("run the tests", messages_hash="ctxA")
|
|
got_b = cache.get("run the tests", messages_hash="ctxB")
|
|
|
|
assert got_a is not None and got_a.response == {"text": "response A"}
|
|
assert got_b is not None and got_b.response == {"text": "response B"}
|
|
|
|
def test_exact_match_verifies_messages_hash(self, cache):
|
|
"""A stored entry is only returned when its messages_hash matches the
|
|
looked-up hash — never another conversation's cached response."""
|
|
cache.put("continue", {"text": "A"}, messages_hash="hA")
|
|
# A lookup for a hash that isn't stored is a miss, not a wrong hit.
|
|
assert cache.get("continue", messages_hash="hB") is None
|
|
|
|
@pytest.mark.parametrize("context", [False, True])
|
|
@pytest.mark.parametrize("namespace", ["", "provider/model"])
|
|
def test_successful_update_refreshes_lru(self, context, namespace):
|
|
cache = SemanticCache(SemanticCacheConfig(max_entries=2))
|
|
first_hash, second_hash = ("first-context", "second-context") if context else (None, None)
|
|
cache.put("first", "old", first_hash, namespace)
|
|
cache.put("second", "two", second_hash, namespace)
|
|
cache.put("first", "updated", first_hash, namespace)
|
|
assert cache.get_stats()["evictions"] == 0
|
|
|
|
# Looking up first before this insertion would refresh it and hide the bug.
|
|
cache.put("third", "three", "third-context" if context else None, namespace)
|
|
|
|
first = cache.get("first", first_hash, namespace)
|
|
assert first is not None and first.response == "updated"
|
|
assert cache.get("second", second_hash, namespace) is None
|
|
assert cache.get_stats()["entries"] == 2
|
|
assert cache.get_stats()["evictions"] == 1
|
|
|
|
def test_failed_update_does_not_refresh_lru(self):
|
|
def embed(query):
|
|
return [1.0, 0.0]
|
|
|
|
cache = SemanticCache(SemanticCacheConfig(max_entries=2), embedding_fn=embed)
|
|
cache.put("first", "one", "first")
|
|
cache.put("second", "two", "second")
|
|
|
|
def unavailable(query):
|
|
raise RuntimeError("embedding service offline")
|
|
|
|
cache._embedding_fn = unavailable
|
|
with pytest.raises(RuntimeError, match="embedding service offline"):
|
|
cache.put("first", "replacement", "first")
|
|
cache._embedding_fn = None
|
|
cache.put("third", "three", "third")
|
|
|
|
assert cache.get("first", "first") is None
|
|
assert cache.get("second", "second").response == "two"
|
|
|
|
def test_concurrent_update_does_not_interrupt_similarity_scan(self, monkeypatch):
|
|
import threading
|
|
|
|
cache = SemanticCache(embedding_fn=lambda query: [1.0, 0.0])
|
|
cache.put("first", "old", "first")
|
|
cache.put("second", "two", "second")
|
|
scanning, release, writer_progress = threading.Event(), threading.Event(), threading.Event()
|
|
real_lock = threading.RLock()
|
|
|
|
class SignalsWriterLock:
|
|
def __enter__(self):
|
|
if threading.current_thread().name == "cache-update":
|
|
writer_progress.set()
|
|
return real_lock.__enter__()
|
|
|
|
def __exit__(self, *exc):
|
|
return real_lock.__exit__(*exc)
|
|
|
|
# Also works on the old implementation, which does not use this lock.
|
|
cache._lock = SignalsWriterLock()
|
|
real_similarity = cache._cosine_similarity
|
|
|
|
def paused_similarity(a, b):
|
|
if not scanning.is_set():
|
|
scanning.set()
|
|
assert release.wait(5)
|
|
return real_similarity(a, b)
|
|
|
|
monkeypatch.setattr(cache, "_cosine_similarity", paused_similarity)
|
|
errors, results = [], []
|
|
|
|
def read():
|
|
try:
|
|
results.append(cache.get("third", "third"))
|
|
except Exception as exc:
|
|
errors.append(exc)
|
|
|
|
def update():
|
|
try:
|
|
cache.put("first", "updated", "first")
|
|
except Exception as exc:
|
|
errors.append(exc)
|
|
finally:
|
|
# Old code reaches here during the scan; synchronized code
|
|
# signals at lock acquisition and waits for the scan instead.
|
|
writer_progress.set()
|
|
|
|
reader = threading.Thread(target=read)
|
|
writer = threading.Thread(target=update, name="cache-update")
|
|
reader.start()
|
|
try:
|
|
assert scanning.wait(5)
|
|
writer.start()
|
|
assert writer_progress.wait(5)
|
|
finally:
|
|
release.set()
|
|
reader.join(5)
|
|
if writer.ident is not None:
|
|
writer.join(5)
|
|
assert not reader.is_alive() and not writer.is_alive()
|
|
assert errors == []
|
|
assert len(results) == 1 and results[0] is not None
|
|
assert results[0].response in ("old", "updated")
|
|
assert cache.get("first", "first").response == "updated"
|
|
|
|
def test_embedding_callback_does_not_block_exact_hits(self):
|
|
import threading
|
|
|
|
started, release = threading.Event(), threading.Event()
|
|
|
|
def embed(query):
|
|
if query == "slow":
|
|
started.set()
|
|
assert release.wait(5)
|
|
return [1.0, 0.0]
|
|
|
|
cache = SemanticCache(embedding_fn=embed)
|
|
cache.put("first", "one", "first")
|
|
errors = []
|
|
|
|
def lookup():
|
|
try:
|
|
cache.get("slow", "slow")
|
|
except Exception as exc:
|
|
errors.append(exc)
|
|
|
|
reader = threading.Thread(target=lookup)
|
|
reader.start()
|
|
try:
|
|
assert started.wait(5)
|
|
assert cache.get("first", "first").response == "one"
|
|
finally:
|
|
release.set()
|
|
reader.join(5)
|
|
assert not reader.is_alive() and errors == []
|
|
|
|
def test_lru_eviction(self):
|
|
"""Test LRU eviction when at capacity."""
|
|
config = SemanticCacheConfig(max_entries=3)
|
|
cache = SemanticCache(config)
|
|
|
|
# Fill cache
|
|
cache.put("query1", "response1", messages_hash="h1")
|
|
cache.put("query2", "response2", messages_hash="h2")
|
|
cache.put("query3", "response3", messages_hash="h3")
|
|
|
|
# Access query1 to make it recently used
|
|
cache.get("query1", messages_hash="h1")
|
|
|
|
# Add new entry, should evict query2 (oldest unused)
|
|
cache.put("query4", "response4", messages_hash="h4")
|
|
|
|
# query1 should still be there (recently accessed)
|
|
assert cache.get("query1", messages_hash="h1") is not None
|
|
# query2 should be evicted
|
|
assert cache.get("query2", messages_hash="h2") is None
|
|
# query3 and query4 should be there
|
|
assert cache.get("query3", messages_hash="h3") is not None
|
|
assert cache.get("query4", messages_hash="h4") is not None
|
|
|
|
def test_update_at_capacity_does_not_evict_unrelated_entry(self):
|
|
"""Re-storing an existing key at capacity must not drop another entry.
|
|
|
|
The eviction loop used to run before the cache key was computed, so
|
|
overwriting a key that was already present (a retried/duplicate store)
|
|
still evicted the LRU-oldest distinct entry even though the update grows
|
|
nothing. That silently dropped a live entry and turned a later lookup for
|
|
it into a false miss.
|
|
"""
|
|
config = SemanticCacheConfig(max_entries=2)
|
|
cache = SemanticCache(config)
|
|
|
|
cache.put("query1", "response1", messages_hash="h1")
|
|
cache.put("query2", "response2", messages_hash="h2")
|
|
|
|
# Re-store the already-present h2 (e.g. a duplicate/retried request).
|
|
cache.put("query2", "response2b", messages_hash="h2")
|
|
|
|
# h1 must still be there — updating h2 must not evict it.
|
|
got1 = cache.get("query1", messages_hash="h1")
|
|
assert got1 is not None and got1.response == "response1"
|
|
# h2 reflects the update.
|
|
got2 = cache.get("query2", messages_hash="h2")
|
|
assert got2 is not None and got2.response == "response2b"
|
|
|
|
def test_put_failure_after_eviction_keeps_live_entry(self):
|
|
"""A failed embedding must not silently drop an already-evicted entry.
|
|
|
|
The eviction loop used to run before the embedding call, so if the
|
|
embedding function raised, an old live entry was already evicted while
|
|
the new entry was never stored — a permanent data loss on a transient
|
|
error. The exception must still propagate.
|
|
"""
|
|
config = SemanticCacheConfig(max_entries=1, similarity_threshold=0.9)
|
|
cache = SemanticCache(config)
|
|
|
|
call_count = {"n": 0}
|
|
|
|
def flaky_embedding(text: str) -> list[float]:
|
|
call_count["n"] += 1
|
|
if call_count["n"] == 1:
|
|
return [1.0, 0.0]
|
|
raise RuntimeError("embedding backend unavailable")
|
|
|
|
cache._embedding_fn = flaky_embedding
|
|
|
|
# First put succeeds and fills the cache to capacity.
|
|
cache.put("a", "response-a", messages_hash="ha")
|
|
assert cache.get("a", messages_hash="ha") is not None
|
|
assert len(cache._cache) == 1
|
|
|
|
# Second put raises, but the live entry must survive untouched.
|
|
with pytest.raises(RuntimeError, match="embedding backend unavailable"):
|
|
cache.put("b", "response-b", messages_hash="hb")
|
|
|
|
assert cache.get("a", messages_hash="ha") is not None
|
|
assert len(cache._cache) == 1
|
|
|
|
# With the embedding working again, a new put evicts "a" normally.
|
|
def stable_embedding(text: str) -> list[float]:
|
|
# Distinct vectors per query so "a" and "c" never false-hit.
|
|
if text == "a":
|
|
return [1.0, 0.0]
|
|
return [0.0, 1.0]
|
|
|
|
cache._embedding_fn = stable_embedding
|
|
cache.put("c", "response-c", messages_hash="hc")
|
|
assert cache.get("a", messages_hash="ha") is None
|
|
assert cache.get("c", messages_hash="hc") is not None
|
|
assert len(cache._cache) == 1
|
|
|
|
def test_ttl_expiration(self):
|
|
"""Test TTL expiration."""
|
|
config = SemanticCacheConfig(ttl_seconds=1)
|
|
cache = SemanticCache(config)
|
|
|
|
cache.put("expiring query", "response", messages_hash="exp1")
|
|
|
|
# Should be available immediately
|
|
assert cache.get("expiring query", messages_hash="exp1") is not None
|
|
|
|
# Wait for TTL
|
|
time.sleep(1.1)
|
|
|
|
# Should be expired
|
|
assert cache.get("expiring query", messages_hash="exp1") is None
|
|
|
|
def test_invalidate(self, cache):
|
|
"""Test invalidating an entry."""
|
|
key = cache.put("query", "response", messages_hash="inv1")
|
|
|
|
assert cache.get("query", messages_hash="inv1") is not None
|
|
|
|
cache.invalidate(key)
|
|
|
|
assert cache.get("query", messages_hash="inv1") is None
|
|
|
|
def test_clear(self, cache):
|
|
"""Test clearing cache."""
|
|
cache.put("query1", "response1", messages_hash="c1")
|
|
cache.put("query2", "response2", messages_hash="c2")
|
|
|
|
cache.clear()
|
|
|
|
stats = cache.get_stats()
|
|
assert stats["entries"] == 0
|
|
|
|
def test_stats(self, cache):
|
|
"""Test statistics."""
|
|
cache.put("query", "response", messages_hash="s1")
|
|
cache.get("query", messages_hash="s1") # hit
|
|
cache.get("unknown", messages_hash="unknown") # miss
|
|
|
|
stats = cache.get_stats()
|
|
|
|
assert stats["entries"] == 1
|
|
assert stats["hits"] == 1
|
|
assert stats["misses"] == 1
|
|
assert stats["hit_rate"] == 0.5
|
|
|
|
def test_access_count(self, cache):
|
|
"""Test that access count is tracked."""
|
|
cache.put("query", "response", messages_hash="ac1")
|
|
|
|
# Access multiple times
|
|
for _ in range(5):
|
|
entry = cache.get("query", messages_hash="ac1")
|
|
|
|
# Initial count is 1, plus 5 accesses = 6
|
|
assert entry.access_count == 6
|
|
|
|
def test_semantic_similarity_with_embedding_fn(self):
|
|
"""Test semantic similarity with custom embedding function."""
|
|
|
|
def mock_embedding(text: str) -> list[float]:
|
|
# Simple mock: return consistent embedding for similar queries
|
|
if "weather" in text.lower():
|
|
return [1.0, 0.0, 0.0]
|
|
elif "time" in text.lower():
|
|
return [0.0, 1.0, 0.0]
|
|
else:
|
|
return [0.0, 0.0, 1.0]
|
|
|
|
config = SemanticCacheConfig(similarity_threshold=0.9)
|
|
cache = SemanticCache(config, embedding_fn=mock_embedding)
|
|
|
|
# Store a weather query
|
|
cache.put("What is the weather today?", "It's sunny", messages_hash="w1")
|
|
|
|
# Similar weather query should hit
|
|
entry = cache.get("How is the weather?")
|
|
assert entry is not None
|
|
assert entry.response == "It's sunny"
|
|
|
|
# Different query should miss
|
|
entry = cache.get("What time is it?")
|
|
assert entry is None
|
|
|
|
def test_empty_query_never_semantic_matches(self):
|
|
"""An empty extracted query must not trigger cross-context false hits.
|
|
|
|
The query is the last user message; in agent/tool traffic most turns are
|
|
tool_result continuations whose extracted query is "". A real embedder
|
|
maps "" to a fixed non-zero vector, so without a guard every empty-query
|
|
turn would be ~identical to every other and serve one conversation's
|
|
response to an unrelated one. An empty query may only ever hit via the
|
|
exact messages_hash (which is context-complete).
|
|
"""
|
|
|
|
def const_embedding(text: str) -> list[float]:
|
|
# Realistic: a non-zero, identical vector for every input (incl. "").
|
|
return [0.5, 0.5, 0.5]
|
|
|
|
config = SemanticCacheConfig(similarity_threshold=0.9)
|
|
cache = SemanticCache(config, embedding_fn=const_embedding)
|
|
|
|
# Conversation A: an empty-query turn (unique full-context hash).
|
|
cache.put("", "response-A", messages_hash="ctxA")
|
|
|
|
# Conversation B: a different empty-query turn — must NOT get A's answer.
|
|
assert cache.get("", messages_hash="ctxB") is None
|
|
# Its own exact hash still works.
|
|
assert cache.get("", messages_hash="ctxA").response == "response-A"
|
|
|
|
# A whitespace-only query is treated the same as empty.
|
|
cache.put(" \n\t", "response-C", messages_hash="ctxC")
|
|
assert cache.get(" ", messages_hash="ctxD") is None
|
|
|
|
def test_namespace_isolation_exact_match(self, cache):
|
|
"""Same messages_hash in different namespaces must not collide."""
|
|
cache.put("continue", "response A", messages_hash="shared", namespace="ns-a")
|
|
cache.put("continue", "response B", messages_hash="shared", namespace="ns-b")
|
|
|
|
assert (
|
|
cache.get("continue", messages_hash="shared", namespace="ns-a").response == "response A"
|
|
)
|
|
assert (
|
|
cache.get("continue", messages_hash="shared", namespace="ns-b").response == "response B"
|
|
)
|
|
# A third namespace that never stored the hash is a miss.
|
|
assert cache.get("continue", messages_hash="shared", namespace="ns-c") is None
|
|
|
|
def test_namespace_isolation_semantic(self):
|
|
"""Semantic matches never cross namespace boundaries."""
|
|
|
|
def embedding(text: str) -> list[float]:
|
|
return [1.0, 0.0, 0.0] if "weather" in text.lower() else [0.0, 1.0, 0.0]
|
|
|
|
config = SemanticCacheConfig(similarity_threshold=0.9)
|
|
cache = SemanticCache(config, embedding_fn=embedding)
|
|
|
|
cache.put("What is the weather?", "sunny", namespace="ns-a")
|
|
|
|
# Same query, different namespace: no hit.
|
|
assert cache.get("How is the weather?", namespace="ns-b") is None
|
|
# Same namespace: still hits.
|
|
assert cache.get("How is the weather?", namespace="ns-a").response == "sunny"
|
|
|
|
def test_query_derived_key_includes_namespace(self, cache):
|
|
"""Without messages_hash the query-derived key is still namespace-scoped."""
|
|
key_a = cache.put("run the tests", "response A", namespace="ns-a")
|
|
key_b = cache.put("run the tests", "response B", namespace="ns-b")
|
|
|
|
# Different namespaces produce different keys for the same query.
|
|
assert key_a != key_b
|
|
# Re-storing in the same namespace reuses the key (in-place update).
|
|
key_a_again = cache.put("run the tests", "response A2", namespace="ns-a")
|
|
assert key_a_again == key_a
|
|
|
|
def test_max_entries_enforced_globally_across_namespaces(self):
|
|
"""Eviction counts entries across all namespaces, not per namespace."""
|
|
config = SemanticCacheConfig(max_entries=2)
|
|
cache = SemanticCache(config)
|
|
|
|
cache.put("query1", "response1", messages_hash="h1", namespace="ns-a")
|
|
cache.put("query2", "response2", messages_hash="h2", namespace="ns-b")
|
|
|
|
# Cache is full: a new entry in a third namespace evicts the LRU entry
|
|
# even though that entry belongs to a different namespace.
|
|
cache.put("query3", "response3", messages_hash="h3", namespace="ns-c")
|
|
|
|
assert cache.get("query1", messages_hash="h1", namespace="ns-a") is None
|
|
assert cache.get("query2", messages_hash="h2", namespace="ns-b") is not None
|
|
assert cache.get("query3", messages_hash="h3", namespace="ns-c") is not None
|
|
|
|
|
|
class TestSemanticCacheLayer:
|
|
"""Test SemanticCacheLayer functionality."""
|
|
|
|
@pytest.fixture
|
|
def layer(self):
|
|
"""Create cache layer with Anthropic optimizer."""
|
|
optimizer = AnthropicCacheOptimizer()
|
|
return SemanticCacheLayer(
|
|
optimizer,
|
|
similarity_threshold=0.95,
|
|
max_entries=100,
|
|
ttl_seconds=60,
|
|
)
|
|
|
|
@pytest.fixture
|
|
def context(self):
|
|
"""Create optimization context."""
|
|
return OptimizationContext(
|
|
provider="anthropic",
|
|
model="claude-3-opus",
|
|
)
|
|
|
|
def test_process_no_cache_hit(self, layer, context):
|
|
"""Test processing with no cache hit."""
|
|
messages = [
|
|
{"role": "system", "content": "You are helpful."},
|
|
{"role": "user", "content": "Hello!"},
|
|
]
|
|
|
|
result = layer.process(messages, context)
|
|
|
|
assert result.semantic_cache_hit is False
|
|
assert result.cached_response is None
|
|
|
|
def test_process_with_cache_hit(self, layer, context):
|
|
"""Test processing with cache hit."""
|
|
messages = [
|
|
{"role": "system", "content": "You are helpful."},
|
|
{"role": "user", "content": "What is 2+2?"},
|
|
]
|
|
|
|
# First, store a response
|
|
layer.store_response(messages, {"text": "4"}, context)
|
|
|
|
# Now process same messages
|
|
result = layer.process(messages, context)
|
|
|
|
assert result.semantic_cache_hit is True
|
|
assert result.cached_response == {"text": "4"}
|
|
|
|
def test_store_response(self, layer, context):
|
|
"""Test storing a response."""
|
|
messages = [
|
|
{"role": "system", "content": "You are helpful."},
|
|
{"role": "user", "content": "Tell me a joke"},
|
|
]
|
|
|
|
key = layer.store_response(messages, {"text": "Why did..."}, context)
|
|
|
|
assert key is not None
|
|
assert len(key) > 0
|
|
|
|
def test_get_stats(self, layer, context):
|
|
"""Test getting statistics."""
|
|
messages = [
|
|
{"role": "system", "content": "You are helpful."},
|
|
{"role": "user", "content": "Hello"},
|
|
]
|
|
|
|
layer.process(messages, context)
|
|
stats = layer.get_stats()
|
|
|
|
assert "semantic_cache" in stats
|
|
assert "provider_optimizer" in stats
|
|
assert stats["provider_optimizer"] == "anthropic-cache-optimizer"
|
|
|
|
def test_query_extraction(self, layer, context):
|
|
"""Test query extraction from messages."""
|
|
messages = [
|
|
{"role": "system", "content": "System"},
|
|
{"role": "user", "content": "First question"},
|
|
{"role": "assistant", "content": "Answer"},
|
|
{"role": "user", "content": "Second question"},
|
|
]
|
|
|
|
# Store response
|
|
layer.store_response(messages, {"text": "Response"}, context)
|
|
|
|
# The query should be the last user message
|
|
result = layer.process(messages, context)
|
|
assert result.semantic_cache_hit is True
|
|
|
|
def test_query_from_context(self, layer):
|
|
"""Test using query from context."""
|
|
messages = [
|
|
{"role": "user", "content": "Some message"},
|
|
]
|
|
context = OptimizationContext(
|
|
query="Specific query for caching",
|
|
)
|
|
|
|
layer.store_response(messages, {"text": "Response"}, context)
|
|
result = layer.process(messages, context)
|
|
|
|
assert result.semantic_cache_hit is True
|
|
|
|
def test_provider_optimizer_fallback(self, layer, context):
|
|
"""Test that provider optimizer is used on cache miss."""
|
|
messages = [
|
|
{"role": "system", "content": "You are helpful. " * 500},
|
|
{"role": "user", "content": "New uncached question"},
|
|
]
|
|
|
|
result = layer.process(messages, context)
|
|
|
|
# Should have used provider optimizer
|
|
assert result.semantic_cache_hit is False
|
|
# Provider optimizer should have processed
|
|
assert result.metrics.stable_prefix_hash != ""
|
|
|
|
def test_content_block_query_extraction(self, layer, context):
|
|
"""Test query extraction from content block format."""
|
|
messages = [
|
|
{"role": "system", "content": "System"},
|
|
{
|
|
"role": "user",
|
|
"content": [{"type": "text", "text": "Block format question"}],
|
|
},
|
|
]
|
|
|
|
layer.store_response(messages, {"text": "Response"}, context)
|
|
result = layer.process(messages, context)
|
|
|
|
assert result.semantic_cache_hit is True
|
|
|
|
def test_different_model_no_hit(self, layer):
|
|
"""Cached responses must not replay across models."""
|
|
messages = [{"role": "user", "content": "Find alpha."}]
|
|
context_a = OptimizationContext(provider="anthropic", model="claude-3-opus")
|
|
context_b = OptimizationContext(provider="anthropic", model="claude-3-sonnet")
|
|
|
|
layer.store_response(messages, {"answer": "opus response"}, context_a)
|
|
|
|
result = layer.process(messages, context_b)
|
|
assert result.semantic_cache_hit is False
|
|
|
|
def test_different_provider_no_hit(self, layer):
|
|
"""Cached responses must not replay across providers."""
|
|
messages = [{"role": "user", "content": "Find alpha."}]
|
|
context_a = OptimizationContext(provider="anthropic", model="claude-3-opus")
|
|
context_b = OptimizationContext(provider="bedrock", model="claude-3-opus")
|
|
|
|
layer.store_response(messages, {"answer": "opus response"}, context_a)
|
|
|
|
result = layer.process(messages, context_b)
|
|
assert result.semantic_cache_hit is False
|
|
|
|
def test_different_metadata_no_hit(self, layer):
|
|
"""Cached responses must not replay across different metadata."""
|
|
messages = [{"role": "user", "content": "Find alpha."}]
|
|
context_a = OptimizationContext(
|
|
provider="anthropic",
|
|
model="claude-3-opus",
|
|
metadata={"temperature": 0.7},
|
|
)
|
|
context_b = OptimizationContext(
|
|
provider="anthropic",
|
|
model="claude-3-opus",
|
|
metadata={"temperature": 0.9},
|
|
)
|
|
|
|
layer.store_response(messages, {"answer": "opus response"}, context_a)
|
|
|
|
result = layer.process(messages, context_b)
|
|
assert result.semantic_cache_hit is False
|
|
|
|
def test_embedding_different_model_no_hit_same_context_hit(self):
|
|
"""Semantic embedding lookup is namespace-scoped too."""
|
|
messages = [{"role": "user", "content": "What is 2+2?"}]
|
|
|
|
def deterministic_embedding(text: str) -> list[float]:
|
|
digest = hashlib.sha256(text.encode()).digest()
|
|
return [float(b) / 255.0 for b in digest[:4]]
|
|
|
|
layer = SemanticCacheLayer(
|
|
AnthropicCacheOptimizer(),
|
|
similarity_threshold=0.9,
|
|
max_entries=100,
|
|
ttl_seconds=60,
|
|
embedding_fn=deterministic_embedding,
|
|
)
|
|
|
|
context_a = OptimizationContext(provider="anthropic", model="claude-3-opus")
|
|
context_b = OptimizationContext(provider="anthropic", model="claude-3-sonnet")
|
|
|
|
layer.store_response(messages, {"text": "4"}, context_a)
|
|
|
|
# Same context: exact messages_hash hit via the embedding path's store.
|
|
assert layer.process(messages, context_a).semantic_cache_hit is True
|
|
# Different model: namespace differs, so no hit even though the query
|
|
# embedding is identical.
|
|
assert layer.process(messages, context_b).semantic_cache_hit is False
|
|
|
|
def test_request_id_and_timestamp_do_not_affect_matching(self, layer):
|
|
"""request_id and timestamp are not part of the namespace."""
|
|
messages = [{"role": "user", "content": "Hello!"}]
|
|
context_a = OptimizationContext(
|
|
provider="anthropic",
|
|
model="claude-3-opus",
|
|
request_id="req-1",
|
|
timestamp=datetime(2024, 1, 1),
|
|
)
|
|
context_b = OptimizationContext(
|
|
provider="anthropic",
|
|
model="claude-3-opus",
|
|
request_id="req-2",
|
|
timestamp=datetime(2025, 6, 15),
|
|
)
|
|
|
|
layer.store_response(messages, {"text": "hi"}, context_a)
|
|
|
|
result = layer.process(messages, context_b)
|
|
assert result.semantic_cache_hit is True
|
|
assert result.cached_response == {"text": "hi"}
|
|
|
|
def test_unserializable_metadata_store_returns_empty_and_no_hit(self, layer):
|
|
"""Unserializable metadata yields a None namespace: store refuses and
|
|
lookup skips the cache entirely."""
|
|
messages = [{"role": "user", "content": "Hello!"}]
|
|
context = OptimizationContext(
|
|
provider="anthropic",
|
|
model="claude-3-opus",
|
|
metadata={"x": object()},
|
|
)
|
|
|
|
key = layer.store_response(messages, {"text": "hi"}, context)
|
|
assert key == ""
|
|
|
|
result = layer.process(messages, context)
|
|
assert result.semantic_cache_hit is False
|
|
|
|
@pytest.fixture(params=["object", "set", "cycle"])
|
|
def unhashable_messages(self, request):
|
|
if request.param == "cycle":
|
|
opaque = []
|
|
opaque.append(opaque)
|
|
else:
|
|
opaque = object() if request.param == "object" else {"opaque"}
|
|
return [{"role": "user", "content": "Hello!", "opaque": opaque}]
|
|
|
|
def test_unhashable_messages_are_not_stored(self, layer, context, unhashable_messages):
|
|
assert layer.store_response(unhashable_messages, "private answer", context) == ""
|
|
assert layer.cache.get_stats()["entries"] == 0
|
|
|
|
def test_unhashable_messages_skip_query_only_entries(self, layer, context, unhashable_messages):
|
|
namespace = layer._compute_namespace(context)
|
|
layer.cache.put("Hello!", "another context's answer", namespace=namespace)
|
|
|
|
result = layer.process(unhashable_messages, context)
|
|
|
|
assert result.semantic_cache_hit is False
|
|
assert result.cached_response is None
|
|
assert result.messages[-1]["content"] == "Hello!"
|
|
|
|
|
|
class TestSemanticCoverage:
|
|
def test_process_namespace_none_delegates(self):
|
|
from headroom.cache import AnthropicCacheOptimizer, OptimizationContext, SemanticCacheLayer
|
|
|
|
layer = SemanticCacheLayer(AnthropicCacheOptimizer())
|
|
messages = [{"role": "user", "content": "hi"}]
|
|
context = OptimizationContext(provider="anthropic", model="m", metadata={"x": object()})
|
|
result = layer.process(messages, context)
|
|
assert result.semantic_cache_hit is False
|
|
|
|
def test_invalidate_namespaced_entry(self):
|
|
from headroom.cache.semantic import SemanticCache, SemanticCacheConfig
|
|
|
|
cache = SemanticCache(SemanticCacheConfig())
|
|
key = cache.put("q", "r", messages_hash="h1", namespace="ns-a")
|
|
assert cache.get("q", messages_hash="h1", namespace="ns-a") is not None
|
|
assert cache.invalidate(key) is True
|
|
assert cache.get("q", messages_hash="h1", namespace="ns-a") is None
|
|
cache.put("q", "r2", messages_hash="h1", namespace="ns-a")
|
|
assert cache.get("q", messages_hash="h1", namespace="ns-a").response == "r2"
|
|
|
|
def test_evict_oldest_removes_namespaced_index(self):
|
|
from headroom.cache.semantic import SemanticCache, SemanticCacheConfig
|
|
|
|
cache = SemanticCache(SemanticCacheConfig(max_entries=1))
|
|
cache.put("q1", "r1", messages_hash="h1", namespace="ns-a")
|
|
cache.put("q2", "r2", messages_hash="h2", namespace="ns-b")
|
|
assert cache.get("q1", messages_hash="h1", namespace="ns-a") is None
|
|
|
|
def test_context_free_store_policy(self):
|
|
from headroom.cache import AnthropicCacheOptimizer, OptimizationContext, SemanticCacheLayer
|
|
|
|
layer = SemanticCacheLayer(AnthropicCacheOptimizer())
|
|
messages = [{"role": "user", "content": "Hello!"}]
|
|
layer.store_response(messages, {"text": "hi"})
|
|
assert layer.process(messages, OptimizationContext()).semantic_cache_hit is True
|
|
ctx = OptimizationContext(provider="anthropic", model="claude-3-opus")
|
|
assert layer.process(messages, ctx).semantic_cache_hit is False
|