mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(tokenizers): don't cache token-count inputs larger than max_chars
Fixes #4264 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
3fef15d376
commit
1ffe0993fe
@@ -78,12 +78,15 @@ class TokenCountCache:
|
||||
must not run the eviction check (a dict hit would clear the whole
|
||||
cache and force a full re-encode of every cached message on the
|
||||
next miss — the storm this exists to prevent).
|
||||
Oversized texts are never admitted, so warm entries survive.
|
||||
"""
|
||||
if text in self._counts:
|
||||
self._counts[text] = count
|
||||
return
|
||||
if len(text) < self._min_chars:
|
||||
return
|
||||
if len(text) > self._max_chars:
|
||||
return
|
||||
if len(self._counts) >= self._max_entries or self._chars >= self._max_chars:
|
||||
self._counts.clear()
|
||||
self._chars = 0
|
||||
|
||||
@@ -157,3 +157,22 @@ def test_recput_of_below_min_key_never_admits_and_never_evicts() -> None:
|
||||
assert cache._counts == {"aaaaaaaaaa-1": 10}
|
||||
cache.put("x", 5) # re-put: still no admission, no eviction
|
||||
assert cache._counts == {"aaaaaaaaaa-1": 10}
|
||||
|
||||
|
||||
def test_oversized_text_is_not_cached() -> None:
|
||||
cache = TokenCountCache(min_chars=1, max_chars=10)
|
||||
cache.put("x" * 20, 5)
|
||||
assert cache.get("x" * 20) is None
|
||||
|
||||
|
||||
def test_warm_entry_survives_oversized_put() -> None:
|
||||
cache = TokenCountCache(min_chars=1, max_chars=10)
|
||||
cache.put("warm", 7)
|
||||
cache.put("x" * 20, 5)
|
||||
assert cache.get("warm") == 7
|
||||
|
||||
|
||||
def test_text_of_exactly_max_chars_is_admitted() -> None:
|
||||
cache = TokenCountCache(min_chars=1, max_chars=10)
|
||||
cache.put("x" * 10, 5)
|
||||
assert cache.get("x" * 10) == 5
|
||||
|
||||
Reference in New Issue
Block a user