mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 12:05:34 +08:00
fix(tokenizers): resolve chatgpt-4o-* aliases to o200k_base (#4672)
## Description `chatgpt-4o-latest` (and dated `chatgpt-4o-*` aliases) are GPT-4o but matched no prefix. `TokenizerRegistry` returned the estimating counter, and `get_encoding_for_model` fell back to `cl100k_base` instead of `o200k_base`. This adds the `chatgpt-4o-` prefix to both tables. No dependency change. Closes #4268 ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) ## Changes Made - `headroom/tokenizers/registry.py`: add `^chatgpt-4o-` -> tiktoken. - `headroom/tokenizers/tiktoken_counter.py`: add `chatgpt-4o-` -> `o200k_base`. - `tests/test_tokenizer_encoding_resolution.py`: new cases (lowercase, mixed case, cold-cache registry path). They fail on `main` and pass with the fix. ## Testing - [x] Unit tests pass (`pytest`): `tests/test_tokenizer_encoding_resolution.py` only, 31 passed (3 failed before the fix) - [x] Linting passes (`ruff check` / `ruff format --check` on touched files) - [ ] Type checking passes (`mypy headroom`): not run - [x] New tests added ## Real Behavior Proof - Environment: macOS, Python 3.12, real `tiktoken`, no mocks. - Exact command / steps: script comparing `tiktoken.encoding_for_model("chatgpt-4o-latest")` with `get_tokenizer(...)` and `TiktokenCounter(...)` on 30 repeats of a Chinese sentence. - Observed result: reference 450 tokens. Before: registry 400 (`EstimatingTokenCounter`), explicit `TiktokenCounter` 600. After: both 450. - Not tested: full test suite, mypy, CI hooks (commitlint/pre-push), Rust build, non-macOS platforms. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] I did **not** edit `CHANGELOG.md` This change was prepared with AI assistance (Claude Code). Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5.5
parent
0ddaa56c21
commit
9648d2d867
@@ -27,6 +27,7 @@ MODEL_PATTERNS: list[tuple[str, str]] = [
|
||||
# OpenAI models -> tiktoken
|
||||
(r"^gpt-5", "tiktoken"),
|
||||
(r"^gpt-4o", "tiktoken"),
|
||||
(r"^chatgpt-4o-", "tiktoken"),
|
||||
(r"^gpt-4", "tiktoken"),
|
||||
(r"^gpt-3\.5", "tiktoken"),
|
||||
(r"^o1", "tiktoken"),
|
||||
|
||||
@@ -187,6 +187,8 @@ def get_encoding_for_model(model: str) -> str:
|
||||
# o200k_base instead of cl100k_base for unknown gpt-4 snapshots.
|
||||
for prefix, encoding in (
|
||||
("gpt-4o", "o200k_base"),
|
||||
# chatgpt-4o-latest and dated chatgpt-4o-* aliases are GPT-4o.
|
||||
("chatgpt-4o-", "o200k_base"),
|
||||
# gpt-4.1 / gpt-4.5 use o200k_base and MUST precede the "gpt-4" prefix,
|
||||
# which they would otherwise match and be mis-encoded as cl100k_base.
|
||||
("gpt-4.1", "o200k_base"),
|
||||
|
||||
@@ -42,6 +42,9 @@ CJK = "这是一个测试文档,用于验证分词器的差异。" * 30
|
||||
("GPT-4o", "o200k_base"),
|
||||
("GPT-4.1", "o200k_base"),
|
||||
("Gpt-4O-Mini", "o200k_base"),
|
||||
# chatgpt-4o-* aliases are GPT-4o (#4268).
|
||||
("chatgpt-4o-latest", "o200k_base"),
|
||||
("ChatGPT-4o-Latest", "o200k_base"),
|
||||
("GPT-5", "o200k_base"),
|
||||
("O4-Mini", "o200k_base"),
|
||||
("GPT-4", "cl100k_base"),
|
||||
@@ -65,7 +68,7 @@ def test_encoding_resolution(model: str, expected: str) -> None:
|
||||
assert get_encoding_for_model(model) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["gpt-5", "GPT-4o", "GPT-4.1"])
|
||||
@pytest.mark.parametrize("model", ["gpt-5", "GPT-4o", "GPT-4.1", "chatgpt-4o-latest"])
|
||||
def test_cold_cache_uppercase_still_gets_the_right_encoding(model: str) -> None:
|
||||
"""End-to-end through the registry, with the uppercase spelling resolved first.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user