fix(tokenizers): read the PDF page count from the active page tree only

The page-tree regex scanned every byte of the file, content streams
included, and int()ed every digit run it matched. Page text that quoted
"/Type /Pages /Count 999999" priced a one-page PDF at 1,499,998,500
tokens, a 5,000-digit run raised ValueError out of count_messages (the
proxy answered 500), and an incremental update that dropped pages kept
the obsolete tree's larger count because the max won.

The count now comes from the active tree: the last trailer (or xref
stream dict) names /Root, each object is read at its latest definition
(top level or inside an inflated object stream), and the catalog's
/Pages node gives /Count. Objects are walked one at a time so stream
bodies are skipped by their /Length, and comments and string literals
are cut from object syntax before any key is read. /Count is at most 5
digits and at most 10,000; anything else falls back to the live page
objects, then to the size estimate. The walk stops after 100,000 objects
(size estimate) so a pathological file cannot stall the request path.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Garm
2026-10-24 09:49:27 +02:00
co-authored by Claude Opus 5.5
parent 7a8468be57
commit 0ecaceb984
2 changed files with 227 additions and 61 deletions
+98 -25
View File
@@ -32,21 +32,110 @@ _COUNT_CACHE_MAX_CHARS = 8_000_000
#: Document pricing (see ``BaseTokenizer._estimate_document_tokens``). Anthropic
#: documents ~1,500-3,000 text tokens per PDF page; the low end is used. Pages
#: come from the PDF's own page tree when it is readable: the root ``/Pages``
#: node's ``/Count`` is the total (inner nodes count subsets, hence the max),
#: else the ``/Type /Page`` objects. PDF 1.5+ often keeps both inside compressed
#: object streams, which are inflated for the search (one budget per file). Only
#: when neither is found does ~3KB of file per page stand in.
#: come from the PDF's active page tree: the last trailer's ``/Root`` catalog,
#: its ``/Pages`` node and that node's ``/Count``, each object read at its latest
#: definition (an incremental update appends the new one). Only object syntax is
#: read: comments, string literals and stream bodies are skipped, so page text
#: never passes for structure. PDF 1.5+ often keeps these objects inside
#: compressed object streams, which are inflated (one budget per file). With no
#: usable tree the ``/Type /Page`` objects count, and with none of those ~3KB of
#: file per page stands in.
_DOCUMENT_TOKENS_PER_PAGE = 1500
_DOCUMENT_BYTES_PER_PAGE = 3000
_PDF_MAX_PAGES = 10_000 # a larger /Count is treated as malformed, not priced
_PDF_OBJECT_STREAM_MAX_BYTES = 4_000_000
_PDF_OBJECT_STREAM = re.compile(rb"/Type\s*/ObjStm\b[^>]*>>\s*stream\r?\n")
_PDF_PAGE_TREE_COUNT = re.compile(
rb"/Type\s*/Pages\b[^>]*?/Count\s+(\d+)|/Count\s+(\d+)[^>]*?/Type\s*/Pages\b"
_PDF_MAX_OBJECTS = 100_000 # bounds the walk on the request path
# Each pattern opens with a lookahead on its first byte so the scan skips binary fast.
_PDF_TOP_LEVEL = re.compile(
rb"(?=[%\dt])(?:%[^\r\n]*|(?<!\d)(\d{1,10})\s+\d{1,5}\s+obj\b|(?<![A-Za-z/])trailer\b)"
)
_PDF_SYNTAX_STOP = re.compile(
rb"(?=[%(se])(?:%[^\r\n]*|\(|(?<![A-Za-z/])(?:stream\r?\n|endobj|startxref))"
)
_PDF_STRING_TOKEN = re.compile(rb"\\.|[()]", re.S)
_PDF_STREAM_LENGTH = re.compile(rb"/Length\s+(\d{1,10})(?!\d|\s+\d+\s+R)")
_PDF_STREAM_END = re.compile(rb"\s*endstream")
_PDF_ROOT = re.compile(rb"/Root\s+(\d{1,10})\s+\d{1,5}\s+R")
_PDF_PAGES = re.compile(rb"/Pages\s+(\d{1,10})\s+\d{1,5}\s+R")
# A longer digit run never reaches int(): it is malformed, and the page objects count instead.
_PDF_COUNT = re.compile(rb"/Count\s+(\d{1,5})(?!\d)")
_PDF_FIRST = re.compile(rb"/First\s+(\d{1,10})")
_PDF_PAGE_OBJECT = re.compile(rb"/Type\s*/Page(?![A-Za-z])")
def _pdf_syntax(data: bytes, pos: int = 0) -> tuple[bytes, int, bool]:
"""PDF syntax from ``pos`` to the next ``stream``/``endobj``/``startxref``, minus
comments and string literals; also where it stopped and whether a stream starts there."""
parts = []
while match := _PDF_SYNTAX_STOP.search(data, pos):
parts.append(data[pos : match.start()])
pos = match.end()
if match.group() == b"(":
depth = 0 # a string literal may nest balanced parentheses
pos = len(data)
for token in _PDF_STRING_TOKEN.finditer(data, match.start()):
depth += {b"(": 1, b")": -1}.get(token.group(), 0)
if not depth:
pos = token.end()
break
elif not match.group().startswith(b"%"):
return b"".join(parts), pos, match.group().startswith(b"stream")
return b"".join(parts) + data[pos:], len(data), False
def _pdf_page_count(doc: bytes) -> int:
"""Pages in the PDF's active page tree, else its page objects, else 0."""
objects: dict[int, bytes] = {}
trailers: list[bytes] = []
budget = _PDF_OBJECT_STREAM_MAX_BYTES
pos = 0
eof = doc.rfind(b"%%EOF") # bytes after the last revision hold no objects
while len(objects) <= _PDF_MAX_OBJECTS and (
match := _PDF_TOP_LEVEL.search(doc, pos, len(doc) if eof < 0 else eof)
):
pos = match.end()
if match.group().startswith(b"%"):
continue
syntax, pos, is_stream = _pdf_syntax(doc, pos)
if match.group(1) is None:
trailers.append(syntax)
continue
objects[int(match.group(1))] = syntax # a later definition replaces an earlier one
if not is_stream:
continue
length = _PDF_STREAM_LENGTH.search(syntax)
end = pos + int(length.group(1)) if length else -1
if not (length and _PDF_STREAM_END.match(doc, end)):
end = doc.find(b"endstream", pos)
start, pos = pos, len(doc) if end < 0 else end
if re.search(rb"/Type\s*/XRef\b", syntax):
trailers.append(syntax) # a cross-reference stream's dict is its trailer
first = _PDF_FIRST.search(syntax)
if first and budget > 0 and re.search(rb"/Type\s*/ObjStm\b", syntax):
# One inflate budget for the whole file, not per stream: a PDF with
# many object streams must not multiply the request path's memory.
with contextlib.suppress(zlib.error):
inflated = zlib.decompressobj().decompress(memoryview(doc)[start:pos], budget)
budget -= len(inflated)
head = re.findall(rb"\d{1,10}", inflated[: int(first.group(1))])
offsets = [int(first.group(1)) + int(offset) for offset in head[1::2]]
for number, at, stop in zip(head[::2], offsets, [*offsets[1:], len(inflated)]):
objects[int(number)] = inflated[at:stop]
if len(objects) > _PDF_MAX_OBJECTS:
return 0 # too many objects to walk: the size estimate stands in
def syntax_of(number: int) -> bytes:
return _pdf_syntax(objects.get(number, b""))[0]
root = next((m for t in reversed(trailers) if (m := _PDF_ROOT.search(t))), None)
tree = _PDF_PAGES.search(syntax_of(int(root.group(1)))) if root else None
count = _PDF_COUNT.search(syntax_of(int(tree.group(1)))) if tree else None
if count and 0 < int(count.group(1)) <= _PDF_MAX_PAGES:
return int(count.group(1))
return sum(1 for number in objects if _PDF_PAGE_OBJECT.search(syntax_of(number)))
class TokenCountCache:
"""Exact memo for ``count_text``.
@@ -455,23 +544,7 @@ class BaseTokenizer(ABC):
# Only a PDF has a page tree: other bytes (a Strands txt/csv/docx
# document) keep the size estimate even when their text quotes PDF syntax.
return max(1, len(doc_bytes) // _DOCUMENT_BYTES_PER_PAGE) * _DOCUMENT_TOKENS_PER_PAGE
searchable = [doc_bytes]
# One inflate budget for the whole file, not per stream: a PDF with
# many object streams must not multiply the request path's memory.
budget = _PDF_OBJECT_STREAM_MAX_BYTES
for match in _PDF_OBJECT_STREAM.finditer(doc_bytes):
if budget <= 0:
break
end = doc_bytes.find(b"endstream", match.end())
with contextlib.suppress(zlib.error):
inflated = zlib.decompressobj().decompress(doc_bytes[match.end() : end], budget)
budget -= len(inflated)
searchable.append(inflated)
text = b"\n".join(searchable)
tree_counts = [int(a or b) for a, b in _PDF_PAGE_TREE_COUNT.findall(text)]
pages = max(tree_counts) if tree_counts else len(_PDF_PAGE_OBJECT.findall(text))
if not pages:
pages = len(doc_bytes) // _DOCUMENT_BYTES_PER_PAGE
pages = _pdf_page_count(doc_bytes) or len(doc_bytes) // _DOCUMENT_BYTES_PER_PAGE
return max(1, pages) * _DOCUMENT_TOKENS_PER_PAGE
@staticmethod
+129 -36
View File
@@ -110,18 +110,70 @@ def test_shared_walker_counts_nested_tool_result_blocks() -> None:
assert abs(count_content_blocks([nested], len) - count_content_blocks([flat], len)) < 100
def _pdf(page_count: int, *, tree: bool = True, pad: int = 1_000_000) -> bytes:
"""A PDF-shaped blob: its page tree (or only page objects), then opaque bytes."""
pages = b"".join(
b"%d 0 obj << /Type /Page /Parent 2 0 R >> endobj\n" % (3 + i) for i in range(page_count)
)
root = b"2 0 obj << /Type /Pages /Count %d >> endobj\n" % page_count if tree else b""
return (
b"%PDF-1.7\n1 0 obj << /Type /Catalog >> endobj\n"
+ root
+ pages
+ bytes(range(256)) * (pad // 256)
)
def _stream(data: bytes, entries: bytes = b"") -> bytes:
return b"<< /Length %d%s >>\nstream\n%s\nendstream" % (len(data), entries, data)
def _pdf_file(objects: dict[int, bytes | dict[int, bytes]], prior: bytes = b"") -> bytes:
"""A PDF with catalog 1, indexed by an xref table and trailer, or by an xref stream
when a dict value packs those objects into an object stream. With ``prior``, an
incremental update of that file."""
out = prior or b"%PDF-1.7\n"
rows = {} if prior else {0: (0, 0, 65535)}
for number, body in objects.items():
if isinstance(body, dict):
head, data = b"", b""
for index, (packed, obj) in enumerate(body.items()):
head, data = head + b"%d %d " % (packed, len(data)), data + obj + b"\n"
rows[packed] = (2, number, index)
entries = b" /Type /ObjStm /N %d /First %d /Filter /FlateDecode" % (
len(body),
len(head),
)
body = _stream(zlib.compress(head + data), entries)
rows[number] = (1, len(out), 0)
out += b"%d 0 obj\n%s\nendobj\n" % (number, body)
prev = b" /Prev %s" % prior.split(b"startxref")[-1].split()[0] if prior else b""
xref_at = len(out)
if any(kind == 2 for kind, _, _ in rows.values()):
rows[max(rows) + 1] = (1, xref_at, 0)
index = b" ".join(b"%d 1" % number for number in sorted(rows))
table = b"".join(
bytes([kind]) + a.to_bytes(4, "big") + b.to_bytes(2, "big")
for _, (kind, a, b) in sorted(rows.items())
)
entries = b" /Type /XRef /Size %d /W [1 4 2] /Index [%s] /Root 1 0 R%s" % (
max(rows) + 1,
index,
prev,
)
out += b"%d 0 obj\n%s\nendobj\n" % (max(rows), _stream(table, entries))
else:
out += b"xref\n" + b"".join(
b"%d 1\n%010d %05d %s \n" % (number, a, b, b"n" if kind else b"f")
for number, (kind, a, b) in sorted(rows.items())
)
out += b"trailer\n<< /Size %d /Root 1 0 R%s >>\n" % (max(rows) + 1, prev)
return out + b"startxref\n%d\n%%%%EOF\n" % xref_at
def _page_tree(page_count: int) -> dict[int, bytes]:
"""Catalog 1, page tree 2 and ``page_count`` blank pages from object 3 on."""
kids = range(3, 3 + page_count)
return {
1: b"<< /Type /Catalog /Pages 2 0 R >>",
2: b"<< /Type /Pages /Kids [%s] /Count %d /MediaBox [0 0 612 792] >>"
% (b" ".join(b"%d 0 R" % kid for kid in kids), page_count),
**dict.fromkeys(kids, b"<< /Type /Page /Parent 2 0 R >>"),
}
def _pdf(page_count: int, *, trailer: bool = True, pad: int = 1_000_000) -> bytes:
"""Blank pages plus ``pad`` bytes of image data; ``trailer=False`` cuts the file
before its xref and trailer, as a truncated download would."""
image = _stream(bytes(range(256)) * (pad // 256))
pdf = _pdf_file({**_page_tree(page_count), 3 + page_count: image})
return pdf if trailer else pdf[: pdf.rindex(b"xref")]
def _base64_document(raw: bytes) -> dict:
@@ -146,15 +198,10 @@ def test_base64_pdf_document_is_priced_by_pages_not_as_text() -> None:
def test_pdf_page_count_falls_back_from_tree_to_page_objects_to_size() -> None:
assert count_content_blocks([_base64_document(_pdf(5, tree=False))], len) == 5 * 1500
# A truncated file has no trailer to find its page tree from: its page objects count.
assert count_content_blocks([_base64_document(_pdf(5, trailer=False))], len) == 5 * 1500
# PDF 1.5+ keeps the page tree inside a compressed object stream.
packed = zlib.compress(b"<< /Type /Pages /Kids [3 0 R] /Count 7 >>")
hidden = (
b"%PDF-1.7\n4 0 obj << /Type /ObjStm /N 1 /First 4 /Filter /FlateDecode >>\nstream\n"
+ packed
+ b"\nendstream\nendobj\n"
+ bytes(range(256)) * 4096
)
hidden = _pdf_file({10: _page_tree(7), 11: _stream(bytes(range(256)) * 4096)})
assert count_content_blocks([_base64_document(hidden)], len) == 7 * 1500
# Page tree hidden (compressed object streams): ~3KB of file per page.
opaque = bytes(range(256)) * 4096
@@ -169,26 +216,72 @@ def test_pdf_page_count_falls_back_from_tree_to_page_objects_to_size() -> None:
def test_object_stream_inflation_shares_one_budget_per_file(monkeypatch) -> None:
def object_stream(body: bytes) -> bytes:
return (
b"1 0 obj << /Type /ObjStm /N 1 /First 4 /Filter /FlateDecode >>\nstream\n"
+ zlib.compress(body)
+ b"\nendstream\nendobj\n"
)
filler = b"<< /Type /Font >>" * 4
pdf = (
b"%PDF-1.7\n"
+ object_stream(filler)
+ object_stream(b"<< /Type /Pages /Kids [3 0 R] /Count 7 >>")
+ bytes(range(256)) * 4096
)
fonts = dict.fromkeys(range(10, 30), b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>")
pdf = _pdf_file({30: fonts, 31: _page_tree(7), 32: _stream(bytes(range(256)) * 4096)})
assert count_content_blocks([_base64_document(pdf)], len) == 7 * 1500
# The first stream spends the whole budget, so the page tree after it stays packed.
monkeypatch.setattr(base, "_PDF_OBJECT_STREAM_MAX_BYTES", len(filler))
# The fonts stream (larger than the page tree's) spends the whole budget, so the
# page tree after it stays packed.
fonts_stream = pdf[pdf.index(b"stream\n") + 7 : pdf.index(b"\nendstream")]
monkeypatch.setattr(base, "_PDF_OBJECT_STREAM_MAX_BYTES", len(zlib.decompress(fonts_stream)))
assert count_content_blocks([_base64_document(pdf)], len) == len(pdf) // 3000 * 1500
def test_a_pdf_with_too_many_objects_to_walk_gets_the_size_estimate(monkeypatch) -> None:
pdf = _pdf(5)
monkeypatch.setattr(base, "_PDF_MAX_OBJECTS", 4)
assert count_content_blocks([_base64_document(pdf)], len) == len(pdf) // 3000 * 1500
def _one_page_pdf(text: bytes, count: bytes) -> bytes:
"""A minimal valid one-page PDF showing ``text``; ``count`` ends its page tree's dict."""
return _pdf_file(
{
1: b"<< /Type /Catalog /Pages 2 0 R >>",
2: b"<< /Type /Pages /Kids [3 0 R] /MediaBox [0 0 612 792] %s >>" % count,
3: b"<< /Type /Page /Parent 2 0 R /Contents 4 0 R"
b" /Resources << /Font << /F1 5 0 R >> >> >>",
4: _stream(b"BT /F1 12 Tf 72 720 Td (%s) Tj ET" % text),
5: b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
}
)
@pytest.mark.parametrize(
("text", "count"),
[
(b"hello", b"/Count 1"),
# Page text quoting page-tree syntax is content, not structure.
(b"/Type /Pages /Count 999999", b"/Count 1"),
(b"/Type /Pages /Count " + b"9" * 5000, b"/Count 1"),
(b"2 0 obj << /Type /Pages /Count 999 >> endobj", b"/Count 1"),
# Neither is a comment or a string inside the page tree's own dict.
(b"hello", b"% /Count 999\n/Title (a (nested) /Count 999) /Count 1"),
# A malformed or absurd count falls back to the page objects.
(b"hello", b"/Count 1000000"),
(b"hello", b"/Count " + b"9" * 5000),
],
ids=["plain", "quoted-tree", "long-number", "fake-object", "comment-string", "absurd", "long"],
)
def test_page_count_comes_only_from_the_page_tree_dict(text: bytes, count: bytes) -> None:
assert base.BaseTokenizer._estimate_document_tokens(_one_page_pdf(text, count)) == 1500
def test_incremental_update_prices_the_active_page_tree_not_an_obsolete_one() -> None:
original = _pdf_file(_page_tree(5))
# An edit keeps two pages under a new tree; the old tree (/Count 5) stays in the file.
edited = _pdf_file(
{
1: b"<< /Type /Catalog /Pages 8 0 R >>",
3: b"<< /Type /Page /Parent 8 0 R >>",
4: b"<< /Type /Page /Parent 8 0 R >>",
8: b"<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 /MediaBox [0 0 612 792] >>",
},
prior=original,
)
assert base.BaseTokenizer._estimate_document_tokens(original) == 5 * 1500
assert base.BaseTokenizer._estimate_document_tokens(edited) == 2 * 1500
def test_text_and_content_document_sources_count_their_text() -> None:
text = {
"type": "document",