fix(tokenizers): look for a page tree only in PDF bytes

A Strands txt/csv/docx document whose text quotes PDF syntax was priced by
that quoted page count instead of its size.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Garm
2026-10-23 11:24:11 +02:00
co-authored by Claude Opus 5.5
parent c5e0d0ce98
commit c4d503051d
2 changed files with 8 additions and 0 deletions
+4
View File
@@ -451,6 +451,10 @@ class BaseTokenizer(ABC):
@staticmethod
def _estimate_document_tokens(doc_bytes: bytes) -> int:
"""Estimate a document file's tokens from its page count (see the constants)."""
if b"%PDF-" not in doc_bytes[:1024]:
# Only a PDF has a page tree: other bytes (a Strands txt/csv/docx
# document) keep the size estimate even when their text quotes PDF syntax.
return max(1, len(doc_bytes) // _DOCUMENT_BYTES_PER_PAGE) * _DOCUMENT_TOKENS_PER_PAGE
searchable = [doc_bytes]
# One inflate budget for the whole file, not per stream: a PDF with
# many object streams must not multiply the request path's memory.
@@ -159,6 +159,10 @@ def test_pdf_page_count_falls_back_from_tree_to_page_objects_to_size() -> None:
# Page tree hidden (compressed object streams): ~3KB of file per page.
opaque = bytes(range(256)) * 4096
assert count_content_blocks([_base64_document(opaque)], len) == len(opaque) // 3000 * 1500
# Text that quotes PDF syntax is not a PDF (a Strands txt document): size estimate.
quoted = b"Notes on /Type /Pages /Count 1 and /Type /Page objects.\n" + b"x" * 300_000
strands = {"document": {"format": "txt", "name": "notes", "source": {"bytes": quoted}}}
assert count_content_blocks([strands], len) == len(quoted) // 3000 * 1500
# Undecodable base64 is still a document: one page, no exception.
broken = {"type": "document", "source": {"type": "base64", "data": "not base64!"}}
assert count_content_blocks([broken], len) == 1500