mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(tokenizers): read PDF keys only from their own dictionary's top level
The page-count lookups matched /Root, /Pages and /Count anywhere in an object's syntax, nested dictionaries included. A valid page tree with `/PieceInfo << /Example << /Private << /Count 9999 >> >> >> /Count 1` priced 14,998,500 tokens instead of 1,500 (JerrettDavis's repro), and a nested /Pages in the catalog or /Root in the trailer misled the walk the same way. _pdf_own_keys now reduces a dictionary to its own top level, blanking every nested dict, array and hex string, and every key lookup reads only that: /Root in the trailer or xref stream dict, /Pages in the catalog, /Count in the page tree, /Type /Page in the page-object fallback, and /Length, /Type /XRef, /Type /ObjStm and /First in stream dicts. The last group matters too: a private /Type /ObjStm made a content stream's data redefine the page tree, and a private /Length 0 ended a stream skip at its first byte so the data was walked as objects. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+36
-15
@@ -36,10 +36,11 @@ _COUNT_CACHE_MAX_CHARS = 8_000_000
|
||||
#: its ``/Pages`` node and that node's ``/Count``, each object read at its latest
|
||||
#: definition (an incremental update appends the new one). Only object syntax is
|
||||
#: read: comments, string literals and stream bodies are skipped, so page text
|
||||
#: never passes for structure. PDF 1.5+ often keeps these objects inside
|
||||
#: compressed object streams, which are inflated (one budget per file). With no
|
||||
#: usable tree the ``/Type /Page`` objects count, and with none of those ~3KB of
|
||||
#: file per page stands in.
|
||||
#: never passes for structure, and each key comes from its own dictionary's top
|
||||
#: level, never from a dict, array or hex string nested in it. PDF 1.5+ often
|
||||
#: keeps these objects inside compressed object streams, which are inflated (one
|
||||
#: budget per file). With no usable tree the ``/Type /Page`` objects count, and
|
||||
#: with none of those ~3KB of file per page stands in.
|
||||
_DOCUMENT_TOKENS_PER_PAGE = 1500
|
||||
_DOCUMENT_BYTES_PER_PAGE = 3000
|
||||
_PDF_MAX_PAGES = 10_000 # a larger /Count is treated as malformed, not priced
|
||||
@@ -61,6 +62,7 @@ _PDF_PAGES = re.compile(rb"/Pages\s+(\d{1,10})\s+\d{1,5}\s+R")
|
||||
_PDF_COUNT = re.compile(rb"/Count\s+(\d{1,5})(?!\d)")
|
||||
_PDF_FIRST = re.compile(rb"/First\s+(\d{1,10})")
|
||||
_PDF_PAGE_OBJECT = re.compile(rb"/Type\s*/Page(?![A-Za-z])")
|
||||
_PDF_NESTING = re.compile(rb"<<|>>|<[^<>]*>|\[|\]")
|
||||
|
||||
|
||||
def _pdf_syntax(data: bytes, pos: int = 0) -> tuple[bytes, int, bool]:
|
||||
@@ -83,6 +85,24 @@ def _pdf_syntax(data: bytes, pos: int = 0) -> tuple[bytes, int, bool]:
|
||||
return b"".join(parts) + data[pos:], len(data), False
|
||||
|
||||
|
||||
def _pdf_own_keys(syntax: bytes) -> bytes:
|
||||
"""The top level of the dictionary ``syntax`` opens with, each nested dict, array
|
||||
and hex string cut to ``_``: a key read from it is that dictionary's own."""
|
||||
body = syntax.lstrip(b"\0\t\n\x0c\r ")
|
||||
if not body.startswith(b"<<"):
|
||||
return b""
|
||||
parts: list[bytes] = []
|
||||
depth = pos = 0
|
||||
for token in _PDF_NESTING.finditer(body):
|
||||
if depth == 1:
|
||||
parts += [body[pos : token.start()], b" _ "]
|
||||
depth += {b"<<": 1, b"[": 1, b">>": -1, b"]": -1}.get(token.group(), 0)
|
||||
if not depth:
|
||||
break
|
||||
pos = token.end()
|
||||
return b"".join(parts)
|
||||
|
||||
|
||||
def _pdf_page_count(doc: bytes) -> int:
|
||||
"""Pages in the PDF's active page tree, else its page objects, else 0."""
|
||||
objects: dict[int, bytes] = {}
|
||||
@@ -98,20 +118,21 @@ def _pdf_page_count(doc: bytes) -> int:
|
||||
continue
|
||||
syntax, pos, is_stream = _pdf_syntax(doc, pos)
|
||||
if match.group(1) is None:
|
||||
trailers.append(syntax)
|
||||
trailers.append(_pdf_own_keys(syntax))
|
||||
continue
|
||||
objects[int(match.group(1))] = syntax # a later definition replaces an earlier one
|
||||
if not is_stream:
|
||||
continue
|
||||
length = _PDF_STREAM_LENGTH.search(syntax)
|
||||
keys = _pdf_own_keys(syntax)
|
||||
length = _PDF_STREAM_LENGTH.search(keys)
|
||||
end = pos + int(length.group(1)) if length else -1
|
||||
if not (length and _PDF_STREAM_END.match(doc, end)):
|
||||
end = doc.find(b"endstream", pos)
|
||||
start, pos = pos, len(doc) if end < 0 else end
|
||||
if re.search(rb"/Type\s*/XRef\b", syntax):
|
||||
trailers.append(syntax) # a cross-reference stream's dict is its trailer
|
||||
first = _PDF_FIRST.search(syntax)
|
||||
if first and budget > 0 and re.search(rb"/Type\s*/ObjStm\b", syntax):
|
||||
if re.search(rb"/Type\s*/XRef\b", keys):
|
||||
trailers.append(keys) # a cross-reference stream's dict is its trailer
|
||||
first = _PDF_FIRST.search(keys)
|
||||
if first and budget > 0 and re.search(rb"/Type\s*/ObjStm\b", keys):
|
||||
# One inflate budget for the whole file, not per stream: a PDF with
|
||||
# many object streams must not multiply the request path's memory.
|
||||
with contextlib.suppress(zlib.error):
|
||||
@@ -125,15 +146,15 @@ def _pdf_page_count(doc: bytes) -> int:
|
||||
if len(objects) > _PDF_MAX_OBJECTS:
|
||||
return 0 # too many objects to walk: the size estimate stands in
|
||||
|
||||
def syntax_of(number: int) -> bytes:
|
||||
return _pdf_syntax(objects.get(number, b""))[0]
|
||||
def keys_of(number: int) -> bytes:
|
||||
return _pdf_own_keys(_pdf_syntax(objects.get(number, b""))[0])
|
||||
|
||||
root = next((m for t in reversed(trailers) if (m := _PDF_ROOT.search(t))), None)
|
||||
tree = _PDF_PAGES.search(syntax_of(int(root.group(1)))) if root else None
|
||||
count = _PDF_COUNT.search(syntax_of(int(tree.group(1)))) if tree else None
|
||||
tree = _PDF_PAGES.search(keys_of(int(root.group(1)))) if root else None
|
||||
count = _PDF_COUNT.search(keys_of(int(tree.group(1)))) if tree else None
|
||||
if count and 0 < int(count.group(1)) <= _PDF_MAX_PAGES:
|
||||
return int(count.group(1))
|
||||
return sum(1 for number in objects if _PDF_PAGE_OBJECT.search(syntax_of(number)))
|
||||
return sum(1 for number in objects if _PDF_PAGE_OBJECT.search(keys_of(number)))
|
||||
|
||||
|
||||
class TokenCountCache:
|
||||
|
||||
@@ -114,10 +114,12 @@ def _stream(data: bytes, entries: bytes = b"") -> bytes:
|
||||
return b"<< /Length %d%s >>\nstream\n%s\nendstream" % (len(data), entries, data)
|
||||
|
||||
|
||||
def _pdf_file(objects: dict[int, bytes | dict[int, bytes]], prior: bytes = b"") -> bytes:
|
||||
def _pdf_file(
|
||||
objects: dict[int, bytes | dict[int, bytes]], prior: bytes = b"", trailer: bytes = b""
|
||||
) -> bytes:
|
||||
"""A PDF with catalog 1, indexed by an xref table and trailer, or by an xref stream
|
||||
when a dict value packs those objects into an object stream. With ``prior``, an
|
||||
incremental update of that file."""
|
||||
incremental update of that file; ``trailer`` opens the trailer's dict."""
|
||||
out = prior or b"%PDF-1.7\n"
|
||||
rows = {} if prior else {0: (0, 0, 65535)}
|
||||
for number, body in objects.items():
|
||||
@@ -142,7 +144,8 @@ def _pdf_file(objects: dict[int, bytes | dict[int, bytes]], prior: bytes = b"")
|
||||
bytes([kind]) + a.to_bytes(4, "big") + b.to_bytes(2, "big")
|
||||
for _, (kind, a, b) in sorted(rows.items())
|
||||
)
|
||||
entries = b" /Type /XRef /Size %d /W [1 4 2] /Index [%s] /Root 1 0 R%s" % (
|
||||
entries = b" %s/Type /XRef /Size %d /W [1 4 2] /Index [%s] /Root 1 0 R%s" % (
|
||||
trailer,
|
||||
max(rows) + 1,
|
||||
index,
|
||||
prev,
|
||||
@@ -153,7 +156,7 @@ def _pdf_file(objects: dict[int, bytes | dict[int, bytes]], prior: bytes = b"")
|
||||
b"%d 1\n%010d %05d %s \n" % (number, a, b, b"n" if kind else b"f")
|
||||
for number, (kind, a, b) in sorted(rows.items())
|
||||
)
|
||||
out += b"trailer\n<< /Size %d /Root 1 0 R%s >>\n" % (max(rows) + 1, prev)
|
||||
out += b"trailer\n<< %s/Size %d /Root 1 0 R%s >>\n" % (trailer, max(rows) + 1, prev)
|
||||
return out + b"startxref\n%d\n%%%%EOF\n" % xref_at
|
||||
|
||||
|
||||
@@ -259,13 +262,76 @@ def _one_page_pdf(text: bytes, count: bytes) -> bytes:
|
||||
# A malformed or absurd count falls back to the page objects.
|
||||
(b"hello", b"/Count 1000000"),
|
||||
(b"hello", b"/Count " + b"9" * 5000),
|
||||
# A count in a dict, array or hex string nested in the tree's dict is not its own.
|
||||
(b"hello", b"/PieceInfo << /Example << /Private << /Count 9999 >> >> >> /Count 1"),
|
||||
(b"hello", b"/Private [<00ff> << /Count 9999 >>] /Count 1"),
|
||||
# Nor is a nested /Type /Page a page object when the count falls back to those.
|
||||
(b"hello", b"/PieceInfo << /Private << /Type /Page >> >> /Count 1000000"),
|
||||
],
|
||||
ids=[
|
||||
"plain",
|
||||
"quoted-tree",
|
||||
"long-number",
|
||||
"fake-object",
|
||||
"comment-string",
|
||||
"absurd",
|
||||
"long",
|
||||
"nested-dict",
|
||||
"nested-array",
|
||||
"nested-page",
|
||||
],
|
||||
ids=["plain", "quoted-tree", "long-number", "fake-object", "comment-string", "absurd", "long"],
|
||||
)
|
||||
def test_page_count_comes_only_from_the_page_tree_dict(text: bytes, count: bytes) -> None:
|
||||
assert base.BaseTokenizer._estimate_document_tokens(_one_page_pdf(text, count)) == 1500
|
||||
|
||||
|
||||
_FAKE_TREE = b"endstream\n2 0 obj << /Type /Pages /Count 9999 >> endobj"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("objects", "trailer"),
|
||||
[
|
||||
# The catalog nests a private /Pages ahead of its own.
|
||||
(
|
||||
{1: b"<< /PieceInfo << /Private << /Pages 7 0 R >> >> /Type /Catalog /Pages 2 0 R >>"},
|
||||
b"",
|
||||
),
|
||||
# The trailer nests a private /Root ahead of its own.
|
||||
({}, b"/Private << /Root 6 0 R >> "),
|
||||
# Laid out like an object stream redefining page tree 2, but /Type /ObjStm is
|
||||
# private data, not a key of the stream's own dict.
|
||||
(
|
||||
{
|
||||
8: _stream(
|
||||
zlib.compress(b"2 0 << /Type /Pages /Count 9999 >>"),
|
||||
b" /Filter /FlateDecode /Private << /Type /ObjStm /N 1 /First 4 >>",
|
||||
)
|
||||
},
|
||||
b"",
|
||||
),
|
||||
# A private /Length 0 ahead of the stream's own would end the skip at the
|
||||
# stream's first byte, and its data would be walked as objects.
|
||||
(
|
||||
{
|
||||
8: b"<< /DecodeParms << /Length 0 >> /Length %d >>\nstream\n%s\nendstream"
|
||||
% (len(_FAKE_TREE), _FAKE_TREE)
|
||||
},
|
||||
b"",
|
||||
),
|
||||
],
|
||||
ids=["catalog-pages", "trailer-root", "object-stream", "stream-length"],
|
||||
)
|
||||
def test_root_pages_and_stream_keys_come_only_from_their_own_dicts(
|
||||
objects: dict[int, bytes], trailer: bytes
|
||||
) -> None:
|
||||
decoys = {
|
||||
6: b"<< /Type /Catalog /Pages 7 0 R >>",
|
||||
7: b"<< /Type /Pages /Kids [] /Count 9999 >>",
|
||||
}
|
||||
pdf = _pdf_file({**_page_tree(1), **decoys, **objects}, trailer=trailer)
|
||||
assert base.BaseTokenizer._estimate_document_tokens(pdf) == 1500
|
||||
|
||||
|
||||
def test_incremental_update_prices_the_active_page_tree_not_an_obsolete_one() -> None:
|
||||
original = _pdf_file(_page_tree(5))
|
||||
# An edit keeps two pages under a new tree; the old tree (/Count 5) stays in the file.
|
||||
|
||||
Reference in New Issue
Block a user