mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
## Description
`TokenCountCache.put` only checked the size budget before admitting a
new key, never the size of the key itself. A single text longer than
`max_chars` (8M chars by default) was stored and stayed alive until the
next new-key put cleared the cache, pinning a large request body in
memory for the tokenizer's lifetime.
Fixes #4264
## Type of Change
- [x] Bug fix (non-breaking change that fixes an issue)
## Changes Made
- `headroom/tokenizers/base.py`: `put` returns early when `len(text) >
max_chars`. The check sits before the eviction clear, so warm entries
are kept. Re-count branch and soft budget for ordinary entries are
unchanged.
- `tests/test_token_count_cache.py`: oversized text is not cached, warm
entry survives an oversized put, text of exactly `max_chars` is still
admitted.
## Testing
- [x] Unit tests added/updated
- [x] Ran tests locally
```
$ python -m pytest tests/test_token_count_cache.py -q
============================= 21 passed in 2.57s ==============================
$ ruff check headroom/tokenizers/base.py tests/test_token_count_cache.py
All checks passed!
$ ruff format --check headroom/tokenizers/base.py tests/test_token_count_cache.py
2 files already formatted
```
## Real Behavior Proof
- Environment: Windows 11, Python 3.13, real `EstimatingTokenCounter`,
branch at 1ffe0993 vs upstream main `base.py`
- Exact command / steps: count a 9,600,000-char string via
`EstimatingTokenCounter().count_text(s)` under `tracemalloc`, then `del
s; gc.collect()` and read retained bytes
- Observed result: upstream main: count 2400000, retained_bytes 9600105
(string pinned by the cache). This branch: count 2400000, retained_bytes
32. Same token count, oversized body released.
- Not tested: full proxy end-to-end memory profile under live traffic
```
$ python repro.py # upstream main base.py
count 2400000 retained_bytes 9600105
$ python repro.py # this branch
count 2400000 retained_bytes 32
```
## Runtime Rollout Safety
- Rollout-managed feature(s): none
- Minimum rollout channel: stable
- Stable/default behavior changed: only oversized texts skip the count
cache; returned counts are identical
- Kill switch / disable path: none needed; revert the commit
- Unsafe override required: no
- Qualification impact: none
- Rollback path: revert this commit
## Review Readiness
- [x] I have performed a self-review
- [x] This PR is ready for human review
🤖 Generated with [Claude Code](https://claude.com/claude-code)
<!-- devin-review-badge-begin -->
---
<a href="https://app.devin.ai/review/headroomlabs-ai/headroom/pull/4469"
target="_blank"><picture><source media="(prefers-color-scheme: dark)"
srcset="https://static.devin.ai/assets/gh-devin-review-dark.svg?v=4"><img
src="https://static.devin.ai/assets/gh-devin-review-light.svg?v=4"
alt="Devin Review"></picture></a>
<!-- devin-review-badge-end -->
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: JD Davis <mxjerrett@gmail.com>
179 lines
7.0 KiB
Python
179 lines
7.0 KiB
Python
"""The token-count memo must be invisible: same integers, or it is a bug.
|
|
|
|
These counts feed context_pressure -> min_ratio -> which blocks get compressed,
|
|
so "the cache returned a different number" is a compression regression, not a
|
|
cache miss. Every test here is an equality test for that reason.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from headroom.providers.anthropic import AnthropicProvider
|
|
from headroom.tokenizers.base import TokenCountCache
|
|
from headroom.tokenizers.estimator import EstimatingTokenCounter
|
|
from headroom.tokenizers.tiktoken_counter import TiktokenCounter
|
|
|
|
BODIES = [
|
|
"word " * 500,
|
|
json.dumps([{"id": i, "name": f"item-{i}", "ok": i % 2 == 0} for i in range(300)]),
|
|
"def f(x):\n return x + 1\n" * 200,
|
|
"2026-08-06 13:00:00 INFO worker did a thing\n" * 400,
|
|
"日本語のテキストをここに置きます。" * 200,
|
|
"<|endoftext|> literal special token marker " * 100, # forces the ValueError path
|
|
"x" * 300,
|
|
]
|
|
|
|
|
|
def _counters():
|
|
return [
|
|
("anthropic", AnthropicProvider().get_token_counter("claude-sonnet-5")),
|
|
("tiktoken", TiktokenCounter(model="gpt-4o")),
|
|
("estimator-auto", EstimatingTokenCounter()),
|
|
("estimator-fixed", EstimatingTokenCounter(chars_per_token=3.5)),
|
|
]
|
|
|
|
|
|
@pytest.mark.filterwarnings("ignore::UserWarning")
|
|
@pytest.mark.parametrize("body", BODIES)
|
|
def test_cached_count_equals_uncached(body: str) -> None:
|
|
for name, counter in _counters():
|
|
counter._count_cache.clear()
|
|
first = counter.count_text(body) # miss, populates
|
|
second = counter.count_text(body) # hit
|
|
counter._count_cache.clear()
|
|
third = counter.count_text(body) # miss again
|
|
assert first == second == third, f"{name}: {first} != {second} != {third}"
|
|
|
|
|
|
@pytest.mark.filterwarnings("ignore::UserWarning")
|
|
def test_empty_and_tiny_text_still_correct() -> None:
|
|
for _name, counter in _counters():
|
|
assert counter.count_text("") == 0
|
|
assert counter.count_text("hi") == counter.count_text("hi")
|
|
|
|
|
|
def test_cache_clears_when_full_rather_than_growing() -> None:
|
|
cache = TokenCountCache(min_chars=1, max_entries=4, max_chars=10**9)
|
|
for i in range(10):
|
|
cache.put(f"text-number-{i}", i)
|
|
assert len(cache._counts) <= 4
|
|
|
|
|
|
def test_cache_respects_the_character_budget() -> None:
|
|
cache = TokenCountCache(min_chars=1, max_entries=10**6, max_chars=1000)
|
|
for i in range(50):
|
|
cache.put("x" * 100 + str(i), i)
|
|
assert cache._chars <= 1000 + 200 # one entry may straddle the cap
|
|
|
|
|
|
def test_small_strings_are_not_cached() -> None:
|
|
"""They encode in microseconds; caching them would evict the entries that matter."""
|
|
cache = TokenCountCache(min_chars=256)
|
|
cache.put("short", 1)
|
|
assert cache.get("short") is None
|
|
|
|
|
|
def test_distinct_texts_do_not_collide() -> None:
|
|
cache = TokenCountCache(min_chars=1)
|
|
cache.put("alpha", 1)
|
|
cache.put("beta", 2)
|
|
assert (cache.get("alpha"), cache.get("beta"), cache.get("gamma")) == (1, 2, None)
|
|
|
|
|
|
@pytest.mark.filterwarnings("ignore::UserWarning")
|
|
def test_counters_do_not_share_a_cache_across_encodings() -> None:
|
|
"""cl100k and o200k are both live in one process; a shared memo would mix them."""
|
|
a = TiktokenCounter(encoding="cl100k_base")
|
|
b = TiktokenCounter(encoding="o200k_base")
|
|
body = "tokenization differs between these two encodings. " * 100
|
|
assert a.count_text(body) == a.count_text(body)
|
|
assert b.count_text(body) == b.count_text(body)
|
|
assert a._count_cache is not b._count_cache
|
|
|
|
|
|
@pytest.mark.filterwarnings("ignore::UserWarning")
|
|
def test_concurrent_counting_is_consistent() -> None:
|
|
"""The pipeline runs on a thread pool and shares one counter."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
counter = AnthropicProvider().get_token_counter("claude-sonnet-5")
|
|
bodies = [f"{b}\n{i}" for i, b in enumerate(BODIES * 3)]
|
|
expected = {b: counter.count_text(b) for b in bodies}
|
|
counter._count_cache.clear()
|
|
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
got = list(pool.map(counter.count_text, bodies))
|
|
assert got == [expected[b] for b in bodies]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# O2 (2026-09-27 perf audit): a re-count of an entry already in the cache is a
|
|
# hit, not an admission — it must update the stored value without running the
|
|
# eviction check. Before this change the admission path ran for every put(),
|
|
# so re-counting one cached message while the cache sat at capacity cleared
|
|
# the WHOLE cache and forced a full re-encode of every cached message on the
|
|
# next miss (the storm the cache exists to prevent).
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_recput_of_existing_key_updates_value_without_evicting() -> None:
|
|
"""Re-counting a cached key at capacity must not clear the cache."""
|
|
cache = TokenCountCache(min_chars=1, max_entries=2, max_chars=1000)
|
|
cache.put("alpha", 10)
|
|
cache.put("beta", 20) # cache now at max_entries
|
|
cache.put("alpha", 99) # hit: update in place, skip admission
|
|
assert cache.get("alpha") == 99
|
|
assert cache.get("beta") == 20 # not evicted
|
|
assert len(cache._counts) == 2
|
|
|
|
|
|
def test_recput_of_existing_key_at_char_budget_does_not_evict() -> None:
|
|
"""The char-budget branch of admission must be skipped on a hit too."""
|
|
cache = TokenCountCache(min_chars=1, max_entries=100, max_chars=40)
|
|
cache.put("a" * 20, 10)
|
|
cache.put("b" * 20, 20) # _chars == 40 == max_chars
|
|
cache.put("a" * 20, 99) # hit: no admission, no clear
|
|
assert cache.get("a" * 20) == 99
|
|
assert cache.get("b" * 20) == 20
|
|
assert len(cache._counts) == 2
|
|
|
|
|
|
def test_new_key_at_capacity_still_admits_via_clear() -> None:
|
|
"""The hit-path early return must not change admission for NEW keys."""
|
|
cache = TokenCountCache(min_chars=1, max_entries=2, max_chars=1000)
|
|
cache.put("alpha", 10)
|
|
cache.put("beta", 20)
|
|
cache.put("gamma", 30) # new key at capacity: documented clear + admit
|
|
assert cache._counts == {"gamma": 30}
|
|
|
|
|
|
def test_recput_of_below_min_key_never_admits_and_never_evicts() -> None:
|
|
"""Keys below min_chars are rejected by admission; re-puts must not evict."""
|
|
cache = TokenCountCache(min_chars=10, max_entries=2, max_chars=1000)
|
|
cache.put("aaaaaaaaaa-1", 10)
|
|
cache.put("x", 1) # below min_chars: rejected
|
|
assert cache._counts == {"aaaaaaaaaa-1": 10}
|
|
cache.put("x", 5) # re-put: still no admission, no eviction
|
|
assert cache._counts == {"aaaaaaaaaa-1": 10}
|
|
|
|
|
|
def test_oversized_text_is_not_cached() -> None:
|
|
cache = TokenCountCache(min_chars=1, max_chars=10)
|
|
cache.put("x" * 20, 5)
|
|
assert cache.get("x" * 20) is None
|
|
|
|
|
|
def test_warm_entry_survives_oversized_put() -> None:
|
|
cache = TokenCountCache(min_chars=1, max_chars=10)
|
|
cache.put("warm", 7)
|
|
cache.put("x" * 20, 5)
|
|
assert cache.get("warm") == 7
|
|
|
|
|
|
def test_text_of_exactly_max_chars_is_admitted() -> None:
|
|
cache = TokenCountCache(min_chars=1, max_chars=10)
|
|
cache.put("x" * 10, 5)
|
|
assert cache.get("x" * 10) == 5
|