mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(tokenizers): coerce structured text values in message token counters (#4472)
## Description
A dict-valued `text` in a content block reached
`TiktokenCounter.count_text` unchanged and raised `TypeError: unhashable
type: 'dict'` from the string cache, failing `count_messages`. The base
estimator message walker had the same `str` assumption.
Fixes #4273
## Type of Change
- [x] Bug fix (non-breaking change that fixes an issue)
## Changes Made
- `headroom/tokenizers/tiktoken_counter.py`: route content `text` blocks
and non-string/non-list `content` through the existing
`coerce_countable_text`.
- `headroom/tokenizers/base.py`: same for role, name,
non-string/non-list content, `text` blocks, Strands `text`,
`tool_use`/`toolUse` names, reasoning text and `thinking`.
Image/document estimation untouched. Inputs are not mutated.
- `tests/test_tokenizers.py`: `TestDictValuedTextFields`, parametrized
over `TiktokenCounter` and `EstimatingTokenCounter`; each structured
message must count the same as its explicit coerced-text equivalent, and
input must be unchanged.
## Testing
- [x] Unit tests added/updated
- [x] Tests fail before the fix and pass after
```
$ pytest tests/test_tokenizers.py -q -k DictValued # without fix
9 failed, 3 passed, 69 deselected
$ pytest tests/test_tokenizers.py -q # with fix
67 passed, 14 skipped
$ ruff check headroom/tokenizers tests/test_tokenizers.py && ruff format --check ...
All checks passed! / 8 files already formatted
```
## Real Behavior Proof
- Environment: Windows 11, Python 3.12, tiktoken gpt-4o encoding, branch
rebased on upstream/main 3fef15d3
- Exact command / steps: ran the reproducer from #4273
(`TiktokenCounter('gpt-4o').count_messages([{'role':'user','content':[{'type':'text','text':{'message':'non-string
provider text'}}]}])`)
- Observed result: main raises `TypeError: unhashable type: 'dict'`;
this branch prints `{"tokens": 16, "input_unchanged": true}`
- Not tested: mistral-common tokenizer cases (optional dependency not
installed, 14 skipped)
## Runtime Rollout Safety
- Rollout-managed feature(s): none
- Minimum rollout channel: n/a, bug fix in token counting
- Stable/default behavior changed: only for non-string text values,
which previously crashed; string inputs count identically
- Kill switch / disable path: none needed
- Unsafe override required: no
- Qualification impact: none
- Rollback path: revert this commit
## Review Readiness
- [x] I have performed a self-review
- [x] This PR is ready for human review
🤖 Generated with [Claude Code](https://claude.com/claude-code)
<!-- devin-review-badge-begin -->
---
<a href="https://app.devin.ai/review/headroomlabs-ai/headroom/pull/4472"
target="_blank"><picture><source media="(prefers-color-scheme: dark)"
srcset="https://static.devin.ai/assets/gh-devin-review-dark.svg?v=4"><img
src="https://static.devin.ai/assets/gh-devin-review-light.svg?v=4"
alt="Devin Review"></picture></a>
<!-- devin-review-badge-end -->
---------
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: JD Davis <mxjerrett@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
JD Davis
parent
3c1fcbdfde
commit
6cc27a453d
@@ -214,16 +214,22 @@ class BaseTokenizer(ABC):
|
||||
|
||||
# Count role
|
||||
role = message.get("role", "")
|
||||
total += self.count_text(role)
|
||||
total += self.count_text(coerce_countable_text(role))
|
||||
|
||||
# Count content
|
||||
content = message.get("content")
|
||||
if isinstance(content, dict):
|
||||
# A single block sent without its list wrapper is priced like
|
||||
# the wrapped block, so images keep their fixed estimate (#4273).
|
||||
content = [content]
|
||||
if content is not None:
|
||||
if isinstance(content, str):
|
||||
total += self.count_text(content)
|
||||
elif isinstance(content, list):
|
||||
# Multi-part content (images, tool results, etc.)
|
||||
total += self._count_content_parts(content)
|
||||
else:
|
||||
total += self.count_text(coerce_countable_text(content))
|
||||
|
||||
# Count tool calls
|
||||
tool_calls = message.get("tool_calls")
|
||||
@@ -238,7 +244,7 @@ class BaseTokenizer(ABC):
|
||||
# Count name field
|
||||
name = message.get("name")
|
||||
if name:
|
||||
total += self.count_text(name)
|
||||
total += self.count_text(coerce_countable_text(name))
|
||||
total += 1 # Name field overhead
|
||||
|
||||
# Reply start overhead
|
||||
@@ -258,7 +264,7 @@ class BaseTokenizer(ABC):
|
||||
part_type = part.get("type", "")
|
||||
|
||||
if part_type == "text":
|
||||
total += self.count_text(part.get("text", ""))
|
||||
total += self.count_text(coerce_countable_text(part.get("text", "")))
|
||||
elif part_type in ("image_url", "image", "input_image"):
|
||||
# Images are NOT tokenized as text — they have a pixel-based cost.
|
||||
# Anthropic: tokens = (width * height) / 750, max ~1600 after resize.
|
||||
@@ -284,15 +290,15 @@ class BaseTokenizer(ABC):
|
||||
else:
|
||||
total += self._count_serialized(content)
|
||||
elif part_type == "tool_use":
|
||||
total += self.count_text(part.get("name", ""))
|
||||
total += self.count_text(coerce_countable_text(part.get("name", "")))
|
||||
total += self._count_serialized(part.get("input", {}))
|
||||
elif not part_type and "text" in part:
|
||||
# Strands SDK format: {"text": "..."} without "type" field
|
||||
total += self.count_text(part["text"])
|
||||
total += self.count_text(coerce_countable_text(part["text"]))
|
||||
elif not part_type and "toolUse" in part:
|
||||
# Strands SDK tool_use: {"toolUse": {"name": ..., "input": ...}}
|
||||
tool_use = part["toolUse"]
|
||||
total += self.count_text(tool_use.get("name", ""))
|
||||
total += self.count_text(coerce_countable_text(tool_use.get("name", "")))
|
||||
total += self._count_serialized(tool_use.get("input", {}))
|
||||
elif not part_type and "toolResult" in part:
|
||||
# Strands SDK tool_result: {"toolResult": {"content": [...]}}
|
||||
@@ -311,7 +317,9 @@ class BaseTokenizer(ABC):
|
||||
reasoning = part["reasoningContent"]
|
||||
reasoning_text = reasoning.get("reasoningText", {})
|
||||
if isinstance(reasoning_text, dict):
|
||||
total += self.count_text(reasoning_text.get("text", ""))
|
||||
total += self.count_text(
|
||||
coerce_countable_text(reasoning_text.get("text", ""))
|
||||
)
|
||||
elif isinstance(reasoning_text, str):
|
||||
total += self.count_text(reasoning_text)
|
||||
elif not part_type and "document" in part:
|
||||
@@ -361,7 +369,7 @@ class BaseTokenizer(ABC):
|
||||
# ~4 bytes/token (len * 3/4 / 4), which lands that session
|
||||
# near the provider figure and never prices an omitted
|
||||
# block at zero.
|
||||
total += self.count_text(part.get("thinking", "") or "")
|
||||
total += self.count_text(coerce_countable_text(part.get("thinking", "") or ""))
|
||||
total += len(part.get("signature") or "") * 3 // 16
|
||||
else:
|
||||
# Unknown type - estimate from JSON
|
||||
|
||||
@@ -297,6 +297,9 @@ class TiktokenCounter(BaseTokenizer):
|
||||
continue
|
||||
|
||||
if key == "content":
|
||||
if isinstance(value, dict):
|
||||
# Unwrapped single block: price it like the list form (#4273).
|
||||
value = [value]
|
||||
if isinstance(value, str):
|
||||
total += self.count_text(value)
|
||||
elif isinstance(value, list):
|
||||
@@ -304,7 +307,9 @@ class TiktokenCounter(BaseTokenizer):
|
||||
for part in value:
|
||||
if isinstance(part, dict):
|
||||
if part.get("type") == "text":
|
||||
total += self.count_text(part.get("text", ""))
|
||||
total += self.count_text(
|
||||
coerce_countable_text(part.get("text", ""))
|
||||
)
|
||||
elif part.get("type") == "image_url":
|
||||
# Image tokens vary by detail level
|
||||
detail = part.get("image_url", {}).get("detail", "auto")
|
||||
@@ -324,6 +329,8 @@ class TiktokenCounter(BaseTokenizer):
|
||||
total += self._count_content_parts([part])
|
||||
elif isinstance(part, str):
|
||||
total += self.count_text(part)
|
||||
else:
|
||||
total += self.count_text(coerce_countable_text(value))
|
||||
elif key == "role":
|
||||
total += self.count_text(coerce_countable_text(value))
|
||||
elif key == "name":
|
||||
|
||||
@@ -17,6 +17,7 @@ from headroom.tokenizers import (
|
||||
list_supported_models,
|
||||
register_tokenizer,
|
||||
)
|
||||
from headroom.tokenizers.base import coerce_countable_text
|
||||
|
||||
|
||||
class TestTiktokenCounter:
|
||||
@@ -771,3 +772,120 @@ class TestThinkingBlockCounting:
|
||||
- count_content_blocks(unsigned, counter.count_text)
|
||||
== 750
|
||||
)
|
||||
|
||||
|
||||
class TestDictValuedTextFields:
|
||||
"""Structured (non-string) text fields must not crash token counting.
|
||||
|
||||
Regression (GH #4273): a dict-valued ``text`` block reached
|
||||
``tiktoken.encode`` unchanged and raised ``TypeError: unhashable type:
|
||||
'dict'``, aborting compression for the whole request. The fix reuses
|
||||
:func:`coerce_countable_text` so a non-string text-ish field is priced
|
||||
like the JSON string it should have been.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"counter_cls",
|
||||
[TiktokenCounter, EstimatingTokenCounter],
|
||||
ids=["tiktoken", "estimator"],
|
||||
)
|
||||
def test_dict_text_block_counts_like_coerced_string(self, counter_cls):
|
||||
counter = counter_cls()
|
||||
text_dict = {"message": "non-string provider text"}
|
||||
expected = coerce_countable_text(text_dict)
|
||||
|
||||
bad = {"role": "user", "content": [{"type": "text", "text": text_dict}]}
|
||||
good = {"role": "user", "content": [{"type": "text", "text": expected}]}
|
||||
|
||||
assert counter.count_messages([bad]) == counter.count_messages([good])
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"counter_cls",
|
||||
[TiktokenCounter, EstimatingTokenCounter],
|
||||
ids=["tiktoken", "estimator"],
|
||||
)
|
||||
def test_dict_text_block_does_not_mutate_input(self, counter_cls):
|
||||
import copy
|
||||
|
||||
counter = counter_cls()
|
||||
message = {
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": {"message": "non-string provider text"}}],
|
||||
}
|
||||
original = copy.deepcopy(message)
|
||||
|
||||
counter.count_messages([message])
|
||||
|
||||
assert message == original
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"counter_cls",
|
||||
[TiktokenCounter, EstimatingTokenCounter],
|
||||
ids=["tiktoken", "estimator"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"block",
|
||||
[
|
||||
{"type": "text", "text": {"message": "non-string provider text"}},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64," + "A" * 20000}},
|
||||
],
|
||||
ids=["text", "image"],
|
||||
)
|
||||
def test_unwrapped_content_block_counts_like_list_form(self, counter_cls, block):
|
||||
# An image must keep its fixed estimate, not be priced as base64 text.
|
||||
counter = counter_cls()
|
||||
bare = {"role": "user", "content": block}
|
||||
wrapped = {"role": "user", "content": [block]}
|
||||
|
||||
assert counter.count_messages([bare]) == counter.count_messages([wrapped])
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"counter_cls",
|
||||
[TiktokenCounter, EstimatingTokenCounter],
|
||||
ids=["tiktoken", "estimator"],
|
||||
)
|
||||
def test_dict_role_counts_like_coerced_string(self, counter_cls):
|
||||
counter = counter_cls()
|
||||
role_dict = {"role": "user"}
|
||||
expected = coerce_countable_text(role_dict)
|
||||
|
||||
bad = {"role": role_dict, "content": "hi"}
|
||||
good = {"role": expected, "content": "hi"}
|
||||
|
||||
assert counter.count_messages([bad]) == counter.count_messages([good])
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"counter_cls",
|
||||
[TiktokenCounter, EstimatingTokenCounter],
|
||||
ids=["tiktoken", "estimator"],
|
||||
)
|
||||
def test_dict_tool_result_content_counts_like_coerced_string(self, counter_cls):
|
||||
counter = counter_cls()
|
||||
tr_dict = {"output": "non-string provider text"}
|
||||
expected = coerce_countable_text(tr_dict)
|
||||
|
||||
bad = {
|
||||
"role": "user",
|
||||
"content": [{"type": "tool_result", "content": tr_dict}],
|
||||
}
|
||||
good = {
|
||||
"role": "user",
|
||||
"content": [{"type": "tool_result", "content": expected}],
|
||||
}
|
||||
|
||||
assert counter.count_messages([bad]) == counter.count_messages([good])
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"counter_cls",
|
||||
[TiktokenCounter, EstimatingTokenCounter],
|
||||
ids=["tiktoken", "estimator"],
|
||||
)
|
||||
def test_dict_thinking_counts_like_coerced_string(self, counter_cls):
|
||||
counter = counter_cls()
|
||||
thinking_dict = {"reasoning": "non-string provider text"}
|
||||
expected = coerce_countable_text(thinking_dict)
|
||||
|
||||
bad = {"role": "assistant", "content": [{"type": "thinking", "thinking": thinking_dict}]}
|
||||
good = {"role": "assistant", "content": [{"type": "thinking", "thinking": expected}]}
|
||||
|
||||
assert counter.count_messages([bad]) == counter.count_messages([good])
|
||||
|
||||
Reference in New Issue
Block a user