mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(tokenizers): read the PDF page count from the active page tree only
The page-tree regex scanned every byte of the file, content streams included, and int()ed every digit run it matched. Page text that quoted "/Type /Pages /Count 999999" priced a one-page PDF at 1,499,998,500 tokens, a 5,000-digit run raised ValueError out of count_messages (the proxy answered 500), and an incremental update that dropped pages kept the obsolete tree's larger count because the max won. The count now comes from the active tree: the last trailer (or xref stream dict) names /Root, each object is read at its latest definition (top level or inside an inflated object stream), and the catalog's /Pages node gives /Count. Objects are walked one at a time so stream bodies are skipped by their /Length, and comments and string literals are cut from object syntax before any key is read. /Count is at most 5 digits and at most 10,000; anything else falls back to the live page objects, then to the size estimate. The walk stops after 100,000 objects (size estimate) so a pathological file cannot stall the request path. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+98
-25
@@ -32,21 +32,110 @@ _COUNT_CACHE_MAX_CHARS = 8_000_000
|
||||
|
||||
#: Document pricing (see ``BaseTokenizer._estimate_document_tokens``). Anthropic
|
||||
#: documents ~1,500-3,000 text tokens per PDF page; the low end is used. Pages
|
||||
#: come from the PDF's own page tree when it is readable: the root ``/Pages``
|
||||
#: node's ``/Count`` is the total (inner nodes count subsets, hence the max),
|
||||
#: else the ``/Type /Page`` objects. PDF 1.5+ often keeps both inside compressed
|
||||
#: object streams, which are inflated for the search (one budget per file). Only
|
||||
#: when neither is found does ~3KB of file per page stand in.
|
||||
#: come from the PDF's active page tree: the last trailer's ``/Root`` catalog,
|
||||
#: its ``/Pages`` node and that node's ``/Count``, each object read at its latest
|
||||
#: definition (an incremental update appends the new one). Only object syntax is
|
||||
#: read: comments, string literals and stream bodies are skipped, so page text
|
||||
#: never passes for structure. PDF 1.5+ often keeps these objects inside
|
||||
#: compressed object streams, which are inflated (one budget per file). With no
|
||||
#: usable tree the ``/Type /Page`` objects count, and with none of those ~3KB of
|
||||
#: file per page stands in.
|
||||
_DOCUMENT_TOKENS_PER_PAGE = 1500
|
||||
_DOCUMENT_BYTES_PER_PAGE = 3000
|
||||
_PDF_MAX_PAGES = 10_000 # a larger /Count is treated as malformed, not priced
|
||||
_PDF_OBJECT_STREAM_MAX_BYTES = 4_000_000
|
||||
_PDF_OBJECT_STREAM = re.compile(rb"/Type\s*/ObjStm\b[^>]*>>\s*stream\r?\n")
|
||||
_PDF_PAGE_TREE_COUNT = re.compile(
|
||||
rb"/Type\s*/Pages\b[^>]*?/Count\s+(\d+)|/Count\s+(\d+)[^>]*?/Type\s*/Pages\b"
|
||||
_PDF_MAX_OBJECTS = 100_000 # bounds the walk on the request path
|
||||
# Each pattern opens with a lookahead on its first byte so the scan skips binary fast.
|
||||
_PDF_TOP_LEVEL = re.compile(
|
||||
rb"(?=[%\dt])(?:%[^\r\n]*|(?<!\d)(\d{1,10})\s+\d{1,5}\s+obj\b|(?<![A-Za-z/])trailer\b)"
|
||||
)
|
||||
_PDF_SYNTAX_STOP = re.compile(
|
||||
rb"(?=[%(se])(?:%[^\r\n]*|\(|(?<![A-Za-z/])(?:stream\r?\n|endobj|startxref))"
|
||||
)
|
||||
_PDF_STRING_TOKEN = re.compile(rb"\\.|[()]", re.S)
|
||||
_PDF_STREAM_LENGTH = re.compile(rb"/Length\s+(\d{1,10})(?!\d|\s+\d+\s+R)")
|
||||
_PDF_STREAM_END = re.compile(rb"\s*endstream")
|
||||
_PDF_ROOT = re.compile(rb"/Root\s+(\d{1,10})\s+\d{1,5}\s+R")
|
||||
_PDF_PAGES = re.compile(rb"/Pages\s+(\d{1,10})\s+\d{1,5}\s+R")
|
||||
# A longer digit run never reaches int(): it is malformed, and the page objects count instead.
|
||||
_PDF_COUNT = re.compile(rb"/Count\s+(\d{1,5})(?!\d)")
|
||||
_PDF_FIRST = re.compile(rb"/First\s+(\d{1,10})")
|
||||
_PDF_PAGE_OBJECT = re.compile(rb"/Type\s*/Page(?![A-Za-z])")
|
||||
|
||||
|
||||
def _pdf_syntax(data: bytes, pos: int = 0) -> tuple[bytes, int, bool]:
|
||||
"""PDF syntax from ``pos`` to the next ``stream``/``endobj``/``startxref``, minus
|
||||
comments and string literals; also where it stopped and whether a stream starts there."""
|
||||
parts = []
|
||||
while match := _PDF_SYNTAX_STOP.search(data, pos):
|
||||
parts.append(data[pos : match.start()])
|
||||
pos = match.end()
|
||||
if match.group() == b"(":
|
||||
depth = 0 # a string literal may nest balanced parentheses
|
||||
pos = len(data)
|
||||
for token in _PDF_STRING_TOKEN.finditer(data, match.start()):
|
||||
depth += {b"(": 1, b")": -1}.get(token.group(), 0)
|
||||
if not depth:
|
||||
pos = token.end()
|
||||
break
|
||||
elif not match.group().startswith(b"%"):
|
||||
return b"".join(parts), pos, match.group().startswith(b"stream")
|
||||
return b"".join(parts) + data[pos:], len(data), False
|
||||
|
||||
|
||||
def _pdf_page_count(doc: bytes) -> int:
|
||||
"""Pages in the PDF's active page tree, else its page objects, else 0."""
|
||||
objects: dict[int, bytes] = {}
|
||||
trailers: list[bytes] = []
|
||||
budget = _PDF_OBJECT_STREAM_MAX_BYTES
|
||||
pos = 0
|
||||
eof = doc.rfind(b"%%EOF") # bytes after the last revision hold no objects
|
||||
while len(objects) <= _PDF_MAX_OBJECTS and (
|
||||
match := _PDF_TOP_LEVEL.search(doc, pos, len(doc) if eof < 0 else eof)
|
||||
):
|
||||
pos = match.end()
|
||||
if match.group().startswith(b"%"):
|
||||
continue
|
||||
syntax, pos, is_stream = _pdf_syntax(doc, pos)
|
||||
if match.group(1) is None:
|
||||
trailers.append(syntax)
|
||||
continue
|
||||
objects[int(match.group(1))] = syntax # a later definition replaces an earlier one
|
||||
if not is_stream:
|
||||
continue
|
||||
length = _PDF_STREAM_LENGTH.search(syntax)
|
||||
end = pos + int(length.group(1)) if length else -1
|
||||
if not (length and _PDF_STREAM_END.match(doc, end)):
|
||||
end = doc.find(b"endstream", pos)
|
||||
start, pos = pos, len(doc) if end < 0 else end
|
||||
if re.search(rb"/Type\s*/XRef\b", syntax):
|
||||
trailers.append(syntax) # a cross-reference stream's dict is its trailer
|
||||
first = _PDF_FIRST.search(syntax)
|
||||
if first and budget > 0 and re.search(rb"/Type\s*/ObjStm\b", syntax):
|
||||
# One inflate budget for the whole file, not per stream: a PDF with
|
||||
# many object streams must not multiply the request path's memory.
|
||||
with contextlib.suppress(zlib.error):
|
||||
inflated = zlib.decompressobj().decompress(memoryview(doc)[start:pos], budget)
|
||||
budget -= len(inflated)
|
||||
head = re.findall(rb"\d{1,10}", inflated[: int(first.group(1))])
|
||||
offsets = [int(first.group(1)) + int(offset) for offset in head[1::2]]
|
||||
for number, at, stop in zip(head[::2], offsets, [*offsets[1:], len(inflated)]):
|
||||
objects[int(number)] = inflated[at:stop]
|
||||
|
||||
if len(objects) > _PDF_MAX_OBJECTS:
|
||||
return 0 # too many objects to walk: the size estimate stands in
|
||||
|
||||
def syntax_of(number: int) -> bytes:
|
||||
return _pdf_syntax(objects.get(number, b""))[0]
|
||||
|
||||
root = next((m for t in reversed(trailers) if (m := _PDF_ROOT.search(t))), None)
|
||||
tree = _PDF_PAGES.search(syntax_of(int(root.group(1)))) if root else None
|
||||
count = _PDF_COUNT.search(syntax_of(int(tree.group(1)))) if tree else None
|
||||
if count and 0 < int(count.group(1)) <= _PDF_MAX_PAGES:
|
||||
return int(count.group(1))
|
||||
return sum(1 for number in objects if _PDF_PAGE_OBJECT.search(syntax_of(number)))
|
||||
|
||||
|
||||
class TokenCountCache:
|
||||
"""Exact memo for ``count_text``.
|
||||
|
||||
@@ -455,23 +544,7 @@ class BaseTokenizer(ABC):
|
||||
# Only a PDF has a page tree: other bytes (a Strands txt/csv/docx
|
||||
# document) keep the size estimate even when their text quotes PDF syntax.
|
||||
return max(1, len(doc_bytes) // _DOCUMENT_BYTES_PER_PAGE) * _DOCUMENT_TOKENS_PER_PAGE
|
||||
searchable = [doc_bytes]
|
||||
# One inflate budget for the whole file, not per stream: a PDF with
|
||||
# many object streams must not multiply the request path's memory.
|
||||
budget = _PDF_OBJECT_STREAM_MAX_BYTES
|
||||
for match in _PDF_OBJECT_STREAM.finditer(doc_bytes):
|
||||
if budget <= 0:
|
||||
break
|
||||
end = doc_bytes.find(b"endstream", match.end())
|
||||
with contextlib.suppress(zlib.error):
|
||||
inflated = zlib.decompressobj().decompress(doc_bytes[match.end() : end], budget)
|
||||
budget -= len(inflated)
|
||||
searchable.append(inflated)
|
||||
text = b"\n".join(searchable)
|
||||
tree_counts = [int(a or b) for a, b in _PDF_PAGE_TREE_COUNT.findall(text)]
|
||||
pages = max(tree_counts) if tree_counts else len(_PDF_PAGE_OBJECT.findall(text))
|
||||
if not pages:
|
||||
pages = len(doc_bytes) // _DOCUMENT_BYTES_PER_PAGE
|
||||
pages = _pdf_page_count(doc_bytes) or len(doc_bytes) // _DOCUMENT_BYTES_PER_PAGE
|
||||
return max(1, pages) * _DOCUMENT_TOKENS_PER_PAGE
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -110,18 +110,70 @@ def test_shared_walker_counts_nested_tool_result_blocks() -> None:
|
||||
assert abs(count_content_blocks([nested], len) - count_content_blocks([flat], len)) < 100
|
||||
|
||||
|
||||
def _pdf(page_count: int, *, tree: bool = True, pad: int = 1_000_000) -> bytes:
|
||||
"""A PDF-shaped blob: its page tree (or only page objects), then opaque bytes."""
|
||||
pages = b"".join(
|
||||
b"%d 0 obj << /Type /Page /Parent 2 0 R >> endobj\n" % (3 + i) for i in range(page_count)
|
||||
)
|
||||
root = b"2 0 obj << /Type /Pages /Count %d >> endobj\n" % page_count if tree else b""
|
||||
return (
|
||||
b"%PDF-1.7\n1 0 obj << /Type /Catalog >> endobj\n"
|
||||
+ root
|
||||
+ pages
|
||||
+ bytes(range(256)) * (pad // 256)
|
||||
)
|
||||
def _stream(data: bytes, entries: bytes = b"") -> bytes:
|
||||
return b"<< /Length %d%s >>\nstream\n%s\nendstream" % (len(data), entries, data)
|
||||
|
||||
|
||||
def _pdf_file(objects: dict[int, bytes | dict[int, bytes]], prior: bytes = b"") -> bytes:
|
||||
"""A PDF with catalog 1, indexed by an xref table and trailer, or by an xref stream
|
||||
when a dict value packs those objects into an object stream. With ``prior``, an
|
||||
incremental update of that file."""
|
||||
out = prior or b"%PDF-1.7\n"
|
||||
rows = {} if prior else {0: (0, 0, 65535)}
|
||||
for number, body in objects.items():
|
||||
if isinstance(body, dict):
|
||||
head, data = b"", b""
|
||||
for index, (packed, obj) in enumerate(body.items()):
|
||||
head, data = head + b"%d %d " % (packed, len(data)), data + obj + b"\n"
|
||||
rows[packed] = (2, number, index)
|
||||
entries = b" /Type /ObjStm /N %d /First %d /Filter /FlateDecode" % (
|
||||
len(body),
|
||||
len(head),
|
||||
)
|
||||
body = _stream(zlib.compress(head + data), entries)
|
||||
rows[number] = (1, len(out), 0)
|
||||
out += b"%d 0 obj\n%s\nendobj\n" % (number, body)
|
||||
prev = b" /Prev %s" % prior.split(b"startxref")[-1].split()[0] if prior else b""
|
||||
xref_at = len(out)
|
||||
if any(kind == 2 for kind, _, _ in rows.values()):
|
||||
rows[max(rows) + 1] = (1, xref_at, 0)
|
||||
index = b" ".join(b"%d 1" % number for number in sorted(rows))
|
||||
table = b"".join(
|
||||
bytes([kind]) + a.to_bytes(4, "big") + b.to_bytes(2, "big")
|
||||
for _, (kind, a, b) in sorted(rows.items())
|
||||
)
|
||||
entries = b" /Type /XRef /Size %d /W [1 4 2] /Index [%s] /Root 1 0 R%s" % (
|
||||
max(rows) + 1,
|
||||
index,
|
||||
prev,
|
||||
)
|
||||
out += b"%d 0 obj\n%s\nendobj\n" % (max(rows), _stream(table, entries))
|
||||
else:
|
||||
out += b"xref\n" + b"".join(
|
||||
b"%d 1\n%010d %05d %s \n" % (number, a, b, b"n" if kind else b"f")
|
||||
for number, (kind, a, b) in sorted(rows.items())
|
||||
)
|
||||
out += b"trailer\n<< /Size %d /Root 1 0 R%s >>\n" % (max(rows) + 1, prev)
|
||||
return out + b"startxref\n%d\n%%%%EOF\n" % xref_at
|
||||
|
||||
|
||||
def _page_tree(page_count: int) -> dict[int, bytes]:
|
||||
"""Catalog 1, page tree 2 and ``page_count`` blank pages from object 3 on."""
|
||||
kids = range(3, 3 + page_count)
|
||||
return {
|
||||
1: b"<< /Type /Catalog /Pages 2 0 R >>",
|
||||
2: b"<< /Type /Pages /Kids [%s] /Count %d /MediaBox [0 0 612 792] >>"
|
||||
% (b" ".join(b"%d 0 R" % kid for kid in kids), page_count),
|
||||
**dict.fromkeys(kids, b"<< /Type /Page /Parent 2 0 R >>"),
|
||||
}
|
||||
|
||||
|
||||
def _pdf(page_count: int, *, trailer: bool = True, pad: int = 1_000_000) -> bytes:
|
||||
"""Blank pages plus ``pad`` bytes of image data; ``trailer=False`` cuts the file
|
||||
before its xref and trailer, as a truncated download would."""
|
||||
image = _stream(bytes(range(256)) * (pad // 256))
|
||||
pdf = _pdf_file({**_page_tree(page_count), 3 + page_count: image})
|
||||
return pdf if trailer else pdf[: pdf.rindex(b"xref")]
|
||||
|
||||
|
||||
def _base64_document(raw: bytes) -> dict:
|
||||
@@ -146,15 +198,10 @@ def test_base64_pdf_document_is_priced_by_pages_not_as_text() -> None:
|
||||
|
||||
|
||||
def test_pdf_page_count_falls_back_from_tree_to_page_objects_to_size() -> None:
|
||||
assert count_content_blocks([_base64_document(_pdf(5, tree=False))], len) == 5 * 1500
|
||||
# A truncated file has no trailer to find its page tree from: its page objects count.
|
||||
assert count_content_blocks([_base64_document(_pdf(5, trailer=False))], len) == 5 * 1500
|
||||
# PDF 1.5+ keeps the page tree inside a compressed object stream.
|
||||
packed = zlib.compress(b"<< /Type /Pages /Kids [3 0 R] /Count 7 >>")
|
||||
hidden = (
|
||||
b"%PDF-1.7\n4 0 obj << /Type /ObjStm /N 1 /First 4 /Filter /FlateDecode >>\nstream\n"
|
||||
+ packed
|
||||
+ b"\nendstream\nendobj\n"
|
||||
+ bytes(range(256)) * 4096
|
||||
)
|
||||
hidden = _pdf_file({10: _page_tree(7), 11: _stream(bytes(range(256)) * 4096)})
|
||||
assert count_content_blocks([_base64_document(hidden)], len) == 7 * 1500
|
||||
# Page tree hidden (compressed object streams): ~3KB of file per page.
|
||||
opaque = bytes(range(256)) * 4096
|
||||
@@ -169,26 +216,72 @@ def test_pdf_page_count_falls_back_from_tree_to_page_objects_to_size() -> None:
|
||||
|
||||
|
||||
def test_object_stream_inflation_shares_one_budget_per_file(monkeypatch) -> None:
|
||||
def object_stream(body: bytes) -> bytes:
|
||||
return (
|
||||
b"1 0 obj << /Type /ObjStm /N 1 /First 4 /Filter /FlateDecode >>\nstream\n"
|
||||
+ zlib.compress(body)
|
||||
+ b"\nendstream\nendobj\n"
|
||||
)
|
||||
|
||||
filler = b"<< /Type /Font >>" * 4
|
||||
pdf = (
|
||||
b"%PDF-1.7\n"
|
||||
+ object_stream(filler)
|
||||
+ object_stream(b"<< /Type /Pages /Kids [3 0 R] /Count 7 >>")
|
||||
+ bytes(range(256)) * 4096
|
||||
)
|
||||
fonts = dict.fromkeys(range(10, 30), b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>")
|
||||
pdf = _pdf_file({30: fonts, 31: _page_tree(7), 32: _stream(bytes(range(256)) * 4096)})
|
||||
assert count_content_blocks([_base64_document(pdf)], len) == 7 * 1500
|
||||
# The first stream spends the whole budget, so the page tree after it stays packed.
|
||||
monkeypatch.setattr(base, "_PDF_OBJECT_STREAM_MAX_BYTES", len(filler))
|
||||
# The fonts stream (larger than the page tree's) spends the whole budget, so the
|
||||
# page tree after it stays packed.
|
||||
fonts_stream = pdf[pdf.index(b"stream\n") + 7 : pdf.index(b"\nendstream")]
|
||||
monkeypatch.setattr(base, "_PDF_OBJECT_STREAM_MAX_BYTES", len(zlib.decompress(fonts_stream)))
|
||||
assert count_content_blocks([_base64_document(pdf)], len) == len(pdf) // 3000 * 1500
|
||||
|
||||
|
||||
def test_a_pdf_with_too_many_objects_to_walk_gets_the_size_estimate(monkeypatch) -> None:
|
||||
pdf = _pdf(5)
|
||||
monkeypatch.setattr(base, "_PDF_MAX_OBJECTS", 4)
|
||||
assert count_content_blocks([_base64_document(pdf)], len) == len(pdf) // 3000 * 1500
|
||||
|
||||
|
||||
def _one_page_pdf(text: bytes, count: bytes) -> bytes:
|
||||
"""A minimal valid one-page PDF showing ``text``; ``count`` ends its page tree's dict."""
|
||||
return _pdf_file(
|
||||
{
|
||||
1: b"<< /Type /Catalog /Pages 2 0 R >>",
|
||||
2: b"<< /Type /Pages /Kids [3 0 R] /MediaBox [0 0 612 792] %s >>" % count,
|
||||
3: b"<< /Type /Page /Parent 2 0 R /Contents 4 0 R"
|
||||
b" /Resources << /Font << /F1 5 0 R >> >> >>",
|
||||
4: _stream(b"BT /F1 12 Tf 72 720 Td (%s) Tj ET" % text),
|
||||
5: b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "count"),
|
||||
[
|
||||
(b"hello", b"/Count 1"),
|
||||
# Page text quoting page-tree syntax is content, not structure.
|
||||
(b"/Type /Pages /Count 999999", b"/Count 1"),
|
||||
(b"/Type /Pages /Count " + b"9" * 5000, b"/Count 1"),
|
||||
(b"2 0 obj << /Type /Pages /Count 999 >> endobj", b"/Count 1"),
|
||||
# Neither is a comment or a string inside the page tree's own dict.
|
||||
(b"hello", b"% /Count 999\n/Title (a (nested) /Count 999) /Count 1"),
|
||||
# A malformed or absurd count falls back to the page objects.
|
||||
(b"hello", b"/Count 1000000"),
|
||||
(b"hello", b"/Count " + b"9" * 5000),
|
||||
],
|
||||
ids=["plain", "quoted-tree", "long-number", "fake-object", "comment-string", "absurd", "long"],
|
||||
)
|
||||
def test_page_count_comes_only_from_the_page_tree_dict(text: bytes, count: bytes) -> None:
|
||||
assert base.BaseTokenizer._estimate_document_tokens(_one_page_pdf(text, count)) == 1500
|
||||
|
||||
|
||||
def test_incremental_update_prices_the_active_page_tree_not_an_obsolete_one() -> None:
|
||||
original = _pdf_file(_page_tree(5))
|
||||
# An edit keeps two pages under a new tree; the old tree (/Count 5) stays in the file.
|
||||
edited = _pdf_file(
|
||||
{
|
||||
1: b"<< /Type /Catalog /Pages 8 0 R >>",
|
||||
3: b"<< /Type /Page /Parent 8 0 R >>",
|
||||
4: b"<< /Type /Page /Parent 8 0 R >>",
|
||||
8: b"<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 /MediaBox [0 0 612 792] >>",
|
||||
},
|
||||
prior=original,
|
||||
)
|
||||
assert base.BaseTokenizer._estimate_document_tokens(original) == 5 * 1500
|
||||
assert base.BaseTokenizer._estimate_document_tokens(edited) == 2 * 1500
|
||||
|
||||
|
||||
def test_text_and_content_document_sources_count_their_text() -> None:
|
||||
text = {
|
||||
"type": "document",
|
||||
|
||||
Reference in New Issue
Block a user