mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(tokenizers): look for a page tree only in PDF bytes
A Strands txt/csv/docx document whose text quotes PDF syntax was priced by that quoted page count instead of its size. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -451,6 +451,10 @@ class BaseTokenizer(ABC):
|
||||
@staticmethod
|
||||
def _estimate_document_tokens(doc_bytes: bytes) -> int:
|
||||
"""Estimate a document file's tokens from its page count (see the constants)."""
|
||||
if b"%PDF-" not in doc_bytes[:1024]:
|
||||
# Only a PDF has a page tree: other bytes (a Strands txt/csv/docx
|
||||
# document) keep the size estimate even when their text quotes PDF syntax.
|
||||
return max(1, len(doc_bytes) // _DOCUMENT_BYTES_PER_PAGE) * _DOCUMENT_TOKENS_PER_PAGE
|
||||
searchable = [doc_bytes]
|
||||
# One inflate budget for the whole file, not per stream: a PDF with
|
||||
# many object streams must not multiply the request path's memory.
|
||||
|
||||
@@ -159,6 +159,10 @@ def test_pdf_page_count_falls_back_from_tree_to_page_objects_to_size() -> None:
|
||||
# Page tree hidden (compressed object streams): ~3KB of file per page.
|
||||
opaque = bytes(range(256)) * 4096
|
||||
assert count_content_blocks([_base64_document(opaque)], len) == len(opaque) // 3000 * 1500
|
||||
# Text that quotes PDF syntax is not a PDF (a Strands txt document): size estimate.
|
||||
quoted = b"Notes on /Type /Pages /Count 1 and /Type /Page objects.\n" + b"x" * 300_000
|
||||
strands = {"document": {"format": "txt", "name": "notes", "source": {"bytes": quoted}}}
|
||||
assert count_content_blocks([strands], len) == len(quoted) // 3000 * 1500
|
||||
# Undecodable base64 is still a document: one page, no exception.
|
||||
broken = {"type": "document", "source": {"type": "base64", "data": "not base64!"}}
|
||||
assert count_content_blocks([broken], len) == 1500
|
||||
|
||||
Reference in New Issue
Block a user