fix(tokenizers): read PDF keys only from their own dictionary's top level

The page-count lookups matched /Root, /Pages and /Count anywhere in an
object's syntax, nested dictionaries included. A valid page tree with
`/PieceInfo << /Example << /Private << /Count 9999 >> >> >> /Count 1`
priced 14,998,500 tokens instead of 1,500 (JerrettDavis's repro), and a
nested /Pages in the catalog or /Root in the trailer misled the walk the
same way.

_pdf_own_keys now reduces a dictionary to its own top level, blanking
every nested dict, array and hex string, and every key lookup reads only
that: /Root in the trailer or xref stream dict, /Pages in the catalog,
/Count in the page tree, /Type /Page in the page-object fallback, and
/Length, /Type /XRef, /Type /ObjStm and /First in stream dicts. The last
group matters too: a private /Type /ObjStm made a content stream's data
redefine the page tree, and a private /Length 0 ended a stream skip at
its first byte so the data was walked as objects.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Garm
2026-10-24 10:16:30 +02:00
co-authored by Claude Opus 5.5
parent 0ecaceb984
commit 6d8e9d269d
2 changed files with 107 additions and 20 deletions
+36 -15
View File
@@ -36,10 +36,11 @@ _COUNT_CACHE_MAX_CHARS = 8_000_000
#: its ``/Pages`` node and that node's ``/Count``, each object read at its latest
#: definition (an incremental update appends the new one). Only object syntax is
#: read: comments, string literals and stream bodies are skipped, so page text
#: never passes for structure. PDF 1.5+ often keeps these objects inside
#: compressed object streams, which are inflated (one budget per file). With no
#: usable tree the ``/Type /Page`` objects count, and with none of those ~3KB of
#: file per page stands in.
#: never passes for structure, and each key comes from its own dictionary's top
#: level, never from a dict, array or hex string nested in it. PDF 1.5+ often
#: keeps these objects inside compressed object streams, which are inflated (one
#: budget per file). With no usable tree the ``/Type /Page`` objects count, and
#: with none of those ~3KB of file per page stands in.
_DOCUMENT_TOKENS_PER_PAGE = 1500
_DOCUMENT_BYTES_PER_PAGE = 3000
_PDF_MAX_PAGES = 10_000 # a larger /Count is treated as malformed, not priced
@@ -61,6 +62,7 @@ _PDF_PAGES = re.compile(rb"/Pages\s+(\d{1,10})\s+\d{1,5}\s+R")
_PDF_COUNT = re.compile(rb"/Count\s+(\d{1,5})(?!\d)")
_PDF_FIRST = re.compile(rb"/First\s+(\d{1,10})")
_PDF_PAGE_OBJECT = re.compile(rb"/Type\s*/Page(?![A-Za-z])")
_PDF_NESTING = re.compile(rb"<<|>>|<[^<>]*>|\[|\]")
def _pdf_syntax(data: bytes, pos: int = 0) -> tuple[bytes, int, bool]:
@@ -83,6 +85,24 @@ def _pdf_syntax(data: bytes, pos: int = 0) -> tuple[bytes, int, bool]:
return b"".join(parts) + data[pos:], len(data), False
def _pdf_own_keys(syntax: bytes) -> bytes:
"""The top level of the dictionary ``syntax`` opens with, each nested dict, array
and hex string cut to ``_``: a key read from it is that dictionary's own."""
body = syntax.lstrip(b"\0\t\n\x0c\r ")
if not body.startswith(b"<<"):
return b""
parts: list[bytes] = []
depth = pos = 0
for token in _PDF_NESTING.finditer(body):
if depth == 1:
parts += [body[pos : token.start()], b" _ "]
depth += {b"<<": 1, b"[": 1, b">>": -1, b"]": -1}.get(token.group(), 0)
if not depth:
break
pos = token.end()
return b"".join(parts)
def _pdf_page_count(doc: bytes) -> int:
"""Pages in the PDF's active page tree, else its page objects, else 0."""
objects: dict[int, bytes] = {}
@@ -98,20 +118,21 @@ def _pdf_page_count(doc: bytes) -> int:
continue
syntax, pos, is_stream = _pdf_syntax(doc, pos)
if match.group(1) is None:
trailers.append(syntax)
trailers.append(_pdf_own_keys(syntax))
continue
objects[int(match.group(1))] = syntax # a later definition replaces an earlier one
if not is_stream:
continue
length = _PDF_STREAM_LENGTH.search(syntax)
keys = _pdf_own_keys(syntax)
length = _PDF_STREAM_LENGTH.search(keys)
end = pos + int(length.group(1)) if length else -1
if not (length and _PDF_STREAM_END.match(doc, end)):
end = doc.find(b"endstream", pos)
start, pos = pos, len(doc) if end < 0 else end
if re.search(rb"/Type\s*/XRef\b", syntax):
trailers.append(syntax) # a cross-reference stream's dict is its trailer
first = _PDF_FIRST.search(syntax)
if first and budget > 0 and re.search(rb"/Type\s*/ObjStm\b", syntax):
if re.search(rb"/Type\s*/XRef\b", keys):
trailers.append(keys) # a cross-reference stream's dict is its trailer
first = _PDF_FIRST.search(keys)
if first and budget > 0 and re.search(rb"/Type\s*/ObjStm\b", keys):
# One inflate budget for the whole file, not per stream: a PDF with
# many object streams must not multiply the request path's memory.
with contextlib.suppress(zlib.error):
@@ -125,15 +146,15 @@ def _pdf_page_count(doc: bytes) -> int:
if len(objects) > _PDF_MAX_OBJECTS:
return 0 # too many objects to walk: the size estimate stands in
def syntax_of(number: int) -> bytes:
return _pdf_syntax(objects.get(number, b""))[0]
def keys_of(number: int) -> bytes:
return _pdf_own_keys(_pdf_syntax(objects.get(number, b""))[0])
root = next((m for t in reversed(trailers) if (m := _PDF_ROOT.search(t))), None)
tree = _PDF_PAGES.search(syntax_of(int(root.group(1)))) if root else None
count = _PDF_COUNT.search(syntax_of(int(tree.group(1)))) if tree else None
tree = _PDF_PAGES.search(keys_of(int(root.group(1)))) if root else None
count = _PDF_COUNT.search(keys_of(int(tree.group(1)))) if tree else None
if count and 0 < int(count.group(1)) <= _PDF_MAX_PAGES:
return int(count.group(1))
return sum(1 for number in objects if _PDF_PAGE_OBJECT.search(syntax_of(number)))
return sum(1 for number in objects if _PDF_PAGE_OBJECT.search(keys_of(number)))
class TokenCountCache:
+71 -5
View File
@@ -114,10 +114,12 @@ def _stream(data: bytes, entries: bytes = b"") -> bytes:
return b"<< /Length %d%s >>\nstream\n%s\nendstream" % (len(data), entries, data)
def _pdf_file(objects: dict[int, bytes | dict[int, bytes]], prior: bytes = b"") -> bytes:
def _pdf_file(
objects: dict[int, bytes | dict[int, bytes]], prior: bytes = b"", trailer: bytes = b""
) -> bytes:
"""A PDF with catalog 1, indexed by an xref table and trailer, or by an xref stream
when a dict value packs those objects into an object stream. With ``prior``, an
incremental update of that file."""
incremental update of that file; ``trailer`` opens the trailer's dict."""
out = prior or b"%PDF-1.7\n"
rows = {} if prior else {0: (0, 0, 65535)}
for number, body in objects.items():
@@ -142,7 +144,8 @@ def _pdf_file(objects: dict[int, bytes | dict[int, bytes]], prior: bytes = b"")
bytes([kind]) + a.to_bytes(4, "big") + b.to_bytes(2, "big")
for _, (kind, a, b) in sorted(rows.items())
)
entries = b" /Type /XRef /Size %d /W [1 4 2] /Index [%s] /Root 1 0 R%s" % (
entries = b" %s/Type /XRef /Size %d /W [1 4 2] /Index [%s] /Root 1 0 R%s" % (
trailer,
max(rows) + 1,
index,
prev,
@@ -153,7 +156,7 @@ def _pdf_file(objects: dict[int, bytes | dict[int, bytes]], prior: bytes = b"")
b"%d 1\n%010d %05d %s \n" % (number, a, b, b"n" if kind else b"f")
for number, (kind, a, b) in sorted(rows.items())
)
out += b"trailer\n<< /Size %d /Root 1 0 R%s >>\n" % (max(rows) + 1, prev)
out += b"trailer\n<< %s/Size %d /Root 1 0 R%s >>\n" % (trailer, max(rows) + 1, prev)
return out + b"startxref\n%d\n%%%%EOF\n" % xref_at
@@ -259,13 +262,76 @@ def _one_page_pdf(text: bytes, count: bytes) -> bytes:
# A malformed or absurd count falls back to the page objects.
(b"hello", b"/Count 1000000"),
(b"hello", b"/Count " + b"9" * 5000),
# A count in a dict, array or hex string nested in the tree's dict is not its own.
(b"hello", b"/PieceInfo << /Example << /Private << /Count 9999 >> >> >> /Count 1"),
(b"hello", b"/Private [<00ff> << /Count 9999 >>] /Count 1"),
# Nor is a nested /Type /Page a page object when the count falls back to those.
(b"hello", b"/PieceInfo << /Private << /Type /Page >> >> /Count 1000000"),
],
ids=[
"plain",
"quoted-tree",
"long-number",
"fake-object",
"comment-string",
"absurd",
"long",
"nested-dict",
"nested-array",
"nested-page",
],
ids=["plain", "quoted-tree", "long-number", "fake-object", "comment-string", "absurd", "long"],
)
def test_page_count_comes_only_from_the_page_tree_dict(text: bytes, count: bytes) -> None:
assert base.BaseTokenizer._estimate_document_tokens(_one_page_pdf(text, count)) == 1500
_FAKE_TREE = b"endstream\n2 0 obj << /Type /Pages /Count 9999 >> endobj"
@pytest.mark.parametrize(
("objects", "trailer"),
[
# The catalog nests a private /Pages ahead of its own.
(
{1: b"<< /PieceInfo << /Private << /Pages 7 0 R >> >> /Type /Catalog /Pages 2 0 R >>"},
b"",
),
# The trailer nests a private /Root ahead of its own.
({}, b"/Private << /Root 6 0 R >> "),
# Laid out like an object stream redefining page tree 2, but /Type /ObjStm is
# private data, not a key of the stream's own dict.
(
{
8: _stream(
zlib.compress(b"2 0 << /Type /Pages /Count 9999 >>"),
b" /Filter /FlateDecode /Private << /Type /ObjStm /N 1 /First 4 >>",
)
},
b"",
),
# A private /Length 0 ahead of the stream's own would end the skip at the
# stream's first byte, and its data would be walked as objects.
(
{
8: b"<< /DecodeParms << /Length 0 >> /Length %d >>\nstream\n%s\nendstream"
% (len(_FAKE_TREE), _FAKE_TREE)
},
b"",
),
],
ids=["catalog-pages", "trailer-root", "object-stream", "stream-length"],
)
def test_root_pages_and_stream_keys_come_only_from_their_own_dicts(
objects: dict[int, bytes], trailer: bytes
) -> None:
decoys = {
6: b"<< /Type /Catalog /Pages 7 0 R >>",
7: b"<< /Type /Pages /Kids [] /Count 9999 >>",
}
pdf = _pdf_file({**_page_tree(1), **decoys, **objects}, trailer=trailer)
assert base.BaseTokenizer._estimate_document_tokens(pdf) == 1500
def test_incremental_update_prices_the_active_page_tree_not_an_obsolete_one() -> None:
original = _pdf_file(_page_tree(5))
# An edit keeps two pages under a new tree; the old tree (/Count 5) stays in the file.