mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(tokenizers): read the catalog and page tree where the xref puts them
The page count came from a linear walk where the latest definition of an object number in file order won. An unreferenced later "2 0 obj" (after %%EOF, or packed in an object stream the xref never points at) replaced the real page tree: a 1-page file that pypdf strict opens at 1 page was priced at 14,998,500 tokens. _pdf_page_count now reads the PDF the way a reader does. The trailer chain from the last startxref (tables, xref streams, hybrid /XRefStm, /Prev) gives /Root and each object's location, the newest entry per number winning. The catalog and its /Pages node are read only there: a type 1 entry must hold "N G obj" with the matching number and generation, a type 2 entry is index i of object stream M, read through M's own entry, verified, and inflated within the file's shared budget. Xref streams with PNG predictors (10-15, each row's filter byte) are decoded. Every budget from the previous commit stays shared across the whole traversal. A missing, damaged or unverifiable xref, an unsupported filter, or a spent budget gives the size estimate. The linear walk and the /Type /Page fallback are deleted: content streams are never read, so nothing in stream data or an unreferenced object can reach the count. On 2,207 local PDFs with Spotlight page counts, 2,200 are exact against 2,201 before; 7 (0.32%) take the size estimate: 5 encrypted files whose page tree sits in an encrypted object stream, and 2 with a damaged xref stream that pypdf strict also rejects. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+258
-191
@@ -8,7 +8,6 @@ from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import contextlib
|
||||
import json
|
||||
import re
|
||||
import zlib
|
||||
@@ -32,28 +31,24 @@ _COUNT_CACHE_MAX_CHARS = 8_000_000
|
||||
|
||||
#: Document pricing (see ``BaseTokenizer._estimate_document_tokens``). Anthropic
|
||||
#: documents ~1,500-3,000 text tokens per PDF page; the low end is used. Pages
|
||||
#: come from the PDF's active page tree: the last trailer's ``/Root`` catalog,
|
||||
#: its ``/Pages`` node and that node's ``/Count``, each object read at its latest
|
||||
#: definition (an incremental update appends the new one). Only object syntax is
|
||||
#: read: comments, string literals and stream bodies are skipped, so page text
|
||||
#: never passes for structure, and each key comes from its own dictionary's top
|
||||
#: level, never from a dict, array or hex string nested in it. A stream body ends
|
||||
#: at its ``/Length`` only where ``endstream`` follows; an indirect ``/Length``
|
||||
#: is read at the offset the cross-reference data gives, and a stream end that
|
||||
#: cannot be verified stops the walk (size estimate). PDF 1.5+ often
|
||||
#: keeps these objects inside compressed object streams, which are inflated (one
|
||||
#: budget per file). With no usable tree the ``/Type /Page`` objects count, and
|
||||
#: with none of those ~3KB of file per page stands in.
|
||||
#: come from the PDF's active page tree, read the way a PDF reader reads it: the
|
||||
#: cross-reference data the last ``startxref`` chains to (tables and xref streams,
|
||||
#: through ``/Prev``) gives ``/Root`` and where each object lives, and the catalog
|
||||
#: and its ``/Pages`` node are read there (an ``N G obj`` at that offset, or entry
|
||||
#: i of an object stream), each verified, never wherever their text happens to
|
||||
#: appear in the file. Comments and string literals are cut from an object's
|
||||
#: syntax, and each key comes from its own dictionary's top level, never from a
|
||||
#: dict, array or hex string nested in it. A missing, damaged or unverifiable xref,
|
||||
#: an absurd ``/Count``, or a spent budget gives ~3KB of file per page instead.
|
||||
_DOCUMENT_TOKENS_PER_PAGE = 1500
|
||||
_DOCUMENT_BYTES_PER_PAGE = 3000
|
||||
_PDF_MAX_PAGES = 10_000 # a larger /Count is treated as malformed, not priced
|
||||
# One budget per file of bytes scanned or inflated, and one of xref rows read
|
||||
# (repeats included), so no chain of sections or streams can multiply the work.
|
||||
_PDF_OBJECT_STREAM_MAX_BYTES = 4_000_000
|
||||
_PDF_MAX_OBJECTS = 100_000 # bounds the walk on the request path
|
||||
_PDF_MAX_OBJECTS = 100_000
|
||||
_PDF_MAX_XREF_SECTIONS = 1_000 # a longer /Prev chain is treated as malformed
|
||||
# Each pattern opens with a lookahead on its first byte so the scan skips binary fast.
|
||||
_PDF_TOP_LEVEL = re.compile(
|
||||
rb"(?=[%\dt])(?:%[^\r\n]*|(?<!\d)(\d{1,10})\s+\d{1,5}\s+obj\b|(?<![A-Za-z/])trailer\b)"
|
||||
)
|
||||
# A stream's data starts after its keyword, any spaces and an EOL if present (a
|
||||
# lone CR too): the spec requires CRLF or LF, but readers accept the others.
|
||||
_PDF_SYNTAX_STOP = re.compile(
|
||||
@@ -62,222 +57,294 @@ _PDF_SYNTAX_STOP = re.compile(
|
||||
_PDF_STRING_TOKEN = re.compile(rb"\\.|[()]", re.S)
|
||||
_PDF_STREAM_LENGTH = re.compile(rb"/Length\s+(\d{1,10})(?!\d|\s+\d+\s+R)")
|
||||
_PDF_STREAM_END = re.compile(rb"\s*endstream")
|
||||
_PDF_ROOT = re.compile(rb"/Root\s+(\d{1,10})\s+\d{1,5}\s+R")
|
||||
_PDF_PAGES = re.compile(rb"/Pages\s+(\d{1,10})\s+\d{1,5}\s+R")
|
||||
# A longer digit run never reaches int(): it is malformed, and the page objects count instead.
|
||||
_PDF_ROOT = re.compile(rb"/Root\s+(\d{1,10})\s+(\d{1,5})\s+R")
|
||||
_PDF_PAGES = re.compile(rb"/Pages\s+(\d{1,10})\s+(\d{1,5})\s+R")
|
||||
# A longer digit run never reaches int(): it is malformed, and the size estimate stands in.
|
||||
_PDF_COUNT = re.compile(rb"/Count\s+(\d{1,5})(?!\d)")
|
||||
_PDF_FIRST = re.compile(rb"/First\s+(\d{1,10})")
|
||||
_PDF_PAGE_OBJECT = re.compile(rb"/Type\s*/Page(?![A-Za-z])")
|
||||
_PDF_NESTING = re.compile(rb"<<|>>|<[^<>]*>|(\[[\d\s]*\])|\[|\]")
|
||||
_PDF_LENGTH_REF = re.compile(rb"/Length\s+(\d{1,10})\s+(\d{1,5})\s+R")
|
||||
_PDF_LENGTH_OBJECT = re.compile(rb"\s*(\d{1,10})\s+(\d{1,5})\s+obj\s*(\d{1,10})\s*endobj")
|
||||
_PDF_OBJECT_HEADER = re.compile(rb"\s*(\d{1,10})\s+(\d{1,5})\s+obj\b")
|
||||
_PDF_STARTXREF = re.compile(rb"startxref\s+(\d{1,10})")
|
||||
_PDF_XREF_TABLE = re.compile(rb"\s*xref\b")
|
||||
_PDF_XREF_SUBSECTION = re.compile(rb"\s*(\d{1,10})\s+(\d{1,5})\s+")
|
||||
_PDF_XREF_ROW = re.compile(rb"(\d{10}) \d{5} ([nf])")
|
||||
_PDF_XREF_ROW = re.compile(rb"(\d{10}) (\d{5}) ([nf])")
|
||||
_PDF_XREF_TRAILER = re.compile(rb"\s*trailer\b")
|
||||
_PDF_XREF_STREAM = re.compile(rb"\s*\d{1,10}\s+\d{1,5}\s+obj\b")
|
||||
_PDF_XREF_WIDTHS = re.compile(rb"/W\s*\[\s*(\d)\s+(\d)\s+(\d)\s*\]")
|
||||
|
||||
|
||||
def _pdf_syntax(data: bytes, pos: int = 0) -> tuple[bytes, int, bool]:
|
||||
"""PDF syntax from ``pos`` to the next ``stream``/``endobj``/``startxref``, minus
|
||||
comments and string literals; also where it stopped and whether a stream starts there."""
|
||||
def _pdf_syntax(data: bytes, pos: int = 0, end: int | None = None) -> tuple[bytes, int, bool]:
|
||||
"""PDF syntax from ``pos`` to the next ``stream``/``endobj``/``startxref`` (or ``end``),
|
||||
minus comments and string literals; also where it stopped and whether a stream
|
||||
starts there."""
|
||||
end = len(data) if end is None else min(end, len(data))
|
||||
parts = []
|
||||
while match := _PDF_SYNTAX_STOP.search(data, pos):
|
||||
while match := _PDF_SYNTAX_STOP.search(data, pos, end):
|
||||
parts.append(data[pos : match.start()])
|
||||
pos = match.end()
|
||||
if match.group() == b"(":
|
||||
depth = 0 # a string literal may nest balanced parentheses
|
||||
pos = len(data)
|
||||
for token in _PDF_STRING_TOKEN.finditer(data, match.start()):
|
||||
pos = end
|
||||
for token in _PDF_STRING_TOKEN.finditer(data, match.start(), end):
|
||||
depth += {b"(": 1, b")": -1}.get(token.group(), 0)
|
||||
if not depth:
|
||||
pos = token.end()
|
||||
break
|
||||
elif not match.group().startswith(b"%"):
|
||||
return b"".join(parts), pos, match.group().startswith(b"stream")
|
||||
return b"".join(parts) + data[pos:], len(data), False
|
||||
return b"".join(parts) + data[pos:end], end, False
|
||||
|
||||
|
||||
def _pdf_own_keys(syntax: bytes) -> bytes:
|
||||
def _pdf_own_keys(syntax: bytes, nested: list[bytes] | None = None) -> bytes:
|
||||
"""The top level of the dictionary ``syntax`` opens with, each nested dict, array
|
||||
and hex string cut to ``_``: a key read from it is that dictionary's own. An array
|
||||
of plain integers holds no key, so it stays (an xref stream's ``/W`` and ``/Index``)."""
|
||||
and hex string cut to ``_<i>``: a key read from it is that dictionary's own. An
|
||||
array of plain integers holds no key, so it stays (an xref stream's ``/W`` and
|
||||
``/Index``). ``nested`` collects what was cut, ``_<i>`` being entry i."""
|
||||
body = syntax.lstrip(b"\0\t\n\x0c\r ")
|
||||
if not body.startswith(b"<<"):
|
||||
return b""
|
||||
cut: list[bytes] = [] if nested is None else nested
|
||||
parts: list[bytes] = []
|
||||
depth = pos = 0
|
||||
depth = pos = opened = 0
|
||||
for token in _PDF_NESTING.finditer(body):
|
||||
step = {b"<<": 1, b"[": 1, b">>": -1, b"]": -1}.get(token.group(), 0)
|
||||
if depth == 1:
|
||||
parts += [body[pos : token.start()], token.group(1) or b" _ "]
|
||||
depth += {b"<<": 1, b"[": 1, b">>": -1, b"]": -1}.get(token.group(), 0)
|
||||
parts.append(body[pos : token.start()])
|
||||
if step < 0:
|
||||
break # the dictionary's own end
|
||||
if token.group(1):
|
||||
parts.append(token.group(1))
|
||||
else:
|
||||
parts.append(b" _%d " % len(cut))
|
||||
opened = token.start()
|
||||
if not step:
|
||||
cut.append(token.group()) # a hex string
|
||||
depth += step
|
||||
if step < 0 and depth == 1:
|
||||
cut.append(body[opened : token.end()])
|
||||
if not depth:
|
||||
break
|
||||
pos = token.end()
|
||||
return b"".join(parts)
|
||||
|
||||
|
||||
def _pdf_xref(doc: bytes, end: int, budget: int) -> tuple[dict[int, int], int] | None:
|
||||
"""Offset of each uncompressed object, from the cross-reference sections the last
|
||||
``startxref`` before ``end`` chains to (a table's ``/XRefStm``, then ``/Prev``);
|
||||
the newest entry wins. Read the way a PDF reader reads it, never by searching the
|
||||
file for an object's text, which page content can fake. Returns what is left of
|
||||
``budget`` too, or None once a budget runs out: the whole traversal shares one
|
||||
budget of rows read and one of bytes scanned or inflated, repeats included, so
|
||||
no chain of sections can multiply the work."""
|
||||
offsets: dict[int, int] = {}
|
||||
rows = _PDF_MAX_OBJECTS # one per row read, plus one per section and subsection
|
||||
last = doc.rfind(b"startxref", 0, end)
|
||||
start = _PDF_STARTXREF.match(doc, last) if last >= 0 else None
|
||||
todo = [int(start.group(1))] if start else []
|
||||
seen: set[int] = set()
|
||||
while todo:
|
||||
at = todo.pop(0)
|
||||
if at in seen:
|
||||
continue
|
||||
seen.add(at)
|
||||
rows -= 1
|
||||
if len(seen) > _PDF_MAX_XREF_SECTIONS or rows < 0:
|
||||
return None
|
||||
if table := _PDF_XREF_TABLE.match(doc, at):
|
||||
pos = table.end()
|
||||
while sub := _PDF_XREF_SUBSECTION.match(doc, pos):
|
||||
rows -= int(sub.group(2)) + 1 # also caps the distinct counts compiled below
|
||||
if rows < 0:
|
||||
return None
|
||||
table_rows = re.compile(rb"(?:\d{10} \d{5} [nf]\s{1,2}){%s}" % sub.group(2))
|
||||
if not (block := table_rows.match(doc, sub.end())):
|
||||
break
|
||||
for number, (offset, kind) in enumerate(
|
||||
_PDF_XREF_ROW.findall(block.group()), int(sub.group(1))
|
||||
):
|
||||
if kind == b"n":
|
||||
offsets.setdefault(number, int(offset))
|
||||
pos = block.end()
|
||||
trailer = _PDF_XREF_TRAILER.match(doc, pos)
|
||||
syntax, stop, _ = _pdf_syntax(doc, trailer.end()) if trailer else (b"", pos, False)
|
||||
budget -= stop - pos
|
||||
elif header := _PDF_XREF_STREAM.match(doc, at):
|
||||
syntax, pos, is_stream = _pdf_syntax(doc, header.end())
|
||||
budget -= pos - header.end()
|
||||
else:
|
||||
continue
|
||||
if budget < 0:
|
||||
return None
|
||||
keys = _pdf_own_keys(syntax)
|
||||
if not table:
|
||||
length = _PDF_STREAM_LENGTH.search(keys)
|
||||
widths = _PDF_XREF_WIDTHS.search(keys)
|
||||
kind, field, gen = map(int, widths.groups()) if widths else (0, 0, 0)
|
||||
row = kind + field + gen
|
||||
size = re.search(rb"/Size\s+(\d{1,10})", keys)
|
||||
# shortcut: a section with a predictor (/DecodeParms) or a filter other than
|
||||
# Flate stays unread, so a /Length only it locates gets the size estimate;
|
||||
# decode the PNG predictor if such files turn up.
|
||||
if not (
|
||||
is_stream
|
||||
and length
|
||||
and row
|
||||
and size
|
||||
and re.search(rb"/Type\s*/XRef\b", keys)
|
||||
and _PDF_STREAM_END.match(doc, pos + int(length.group(1)))
|
||||
and not re.search(rb"/DecodeParms|/Filter(?!\s*/FlateDecode\b)", keys)
|
||||
):
|
||||
def _pdf_png_rows(data: bytes, columns: int) -> bytes:
|
||||
"""``data`` decoded from PNG predictor rows: each a filter byte, then ``columns``
|
||||
bytes, one per sample (an xref stream's /Colors 1 /BitsPerComponent 8)."""
|
||||
out, prior = bytearray(), bytes(columns)
|
||||
for at in range(0, len(data), columns + 1):
|
||||
kind, row = data[at], bytearray(data[at + 1 : at + 1 + columns])
|
||||
if kind == 2:
|
||||
row = bytearray((a + b) & 255 for a, b in zip(row, prior))
|
||||
elif kind:
|
||||
for i in range(len(row)):
|
||||
left, up, corner = row[i - 1] if i else 0, prior[i], prior[i - 1] if i else 0
|
||||
if kind == 1:
|
||||
row[i] = (row[i] + left) & 255
|
||||
elif kind == 3:
|
||||
row[i] = (row[i] + (left + up) // 2) & 255
|
||||
elif kind == 4:
|
||||
guess = left + up - corner # Paeth: the neighbour nearest the guess
|
||||
pa, pb, pc = abs(guess - left), abs(guess - up), abs(guess - corner)
|
||||
row[i] = (
|
||||
row[i] + (left if pa <= pb and pa <= pc else up if pb <= pc else corner)
|
||||
) & 255
|
||||
else:
|
||||
raise _PdfUnreadable
|
||||
out += row
|
||||
prior = bytes(row)
|
||||
return bytes(out)
|
||||
|
||||
|
||||
class _PdfUnreadable(Exception):
|
||||
"""The PDF fails verification or a budget runs out: its size estimate stands in."""
|
||||
|
||||
|
||||
class _PdfReader:
|
||||
"""Reads objects at the locations the PDF's cross-reference data gives for them."""
|
||||
|
||||
def __init__(self, doc: bytes) -> None:
|
||||
self.doc = doc
|
||||
self.budget = _PDF_OBJECT_STREAM_MAX_BYTES
|
||||
self.rows = _PDF_MAX_OBJECTS
|
||||
self.entries: dict[int, tuple[int, int, int]] = {} # number: (type, field 2, field 3)
|
||||
self.packed: dict[int, tuple[list[bytes], int, bytes]] = {} # header, /First, data
|
||||
self.root: re.Match[bytes] | None = None
|
||||
eof = doc.rfind(b"%%EOF") # bytes after the last revision hold no objects
|
||||
last = doc.rfind(b"startxref", 0, len(doc) if eof < 0 else eof)
|
||||
start = _PDF_STARTXREF.match(doc, last) if last >= 0 else None
|
||||
todo = [int(start.group(1))] if start else []
|
||||
seen: set[int] = set()
|
||||
while todo:
|
||||
at = todo.pop(0)
|
||||
if at in seen:
|
||||
continue
|
||||
data = doc[pos : pos + int(length.group(1))]
|
||||
if b"/Filter" in keys:
|
||||
if budget <= 0: # max_length 0 would mean no limit
|
||||
return None
|
||||
inflate = zlib.decompressobj()
|
||||
try:
|
||||
data = inflate.decompress(data, budget)
|
||||
except zlib.error:
|
||||
continue
|
||||
if inflate.unconsumed_tail:
|
||||
return None
|
||||
budget -= len(data)
|
||||
index = re.search(rb"/Index\s*\[([\d\s]*)\]", keys)
|
||||
bounds = [int(n) for n in (index.group(1) if index else b"0 " + size.group(1)).split()]
|
||||
rows -= min(sum(bounds[1::2]), len(data) // row)
|
||||
if rows < 0:
|
||||
return None
|
||||
numbers = (
|
||||
n for a, count in zip(bounds[::2], bounds[1::2]) for n in range(a, a + count)
|
||||
)
|
||||
for number, at_row in zip(numbers, range(0, len(data) - row + 1, row)):
|
||||
entry = data[at_row : at_row + row]
|
||||
if int.from_bytes(entry[:kind] or b"\x01", "big") == 1: # type 1: at an offset
|
||||
offsets.setdefault(number, int.from_bytes(entry[kind : kind + field], "big"))
|
||||
for key in (rb"/XRefStm", rb"/Prev"):
|
||||
if found := re.search(key + rb"\s+(\d{1,10})(?!\d)", keys):
|
||||
todo.append(int(found.group(1)))
|
||||
return offsets, budget
|
||||
seen.add(at)
|
||||
if len(seen) > _PDF_MAX_XREF_SECTIONS:
|
||||
raise _PdfUnreadable
|
||||
self.spend(rows=1)
|
||||
table = _PDF_XREF_TABLE.match(doc, at)
|
||||
keys = self.read_table(table.end()) if table else self.read_xref_stream(at)
|
||||
self.root = self.root or _PDF_ROOT.search(keys) # the newest trailer's
|
||||
for key in (rb"/XRefStm", rb"/Prev"): # a table's stream entries, then older sections
|
||||
if found := re.search(key + rb"\s+(\d{1,10})(?!\d)", keys):
|
||||
todo.append(int(found.group(1)))
|
||||
if not self.root:
|
||||
raise _PdfUnreadable
|
||||
|
||||
def spend(self, scanned: int = 0, rows: int = 0) -> None:
|
||||
self.budget -= scanned
|
||||
self.rows -= rows
|
||||
if self.budget < 0 or self.rows < 0:
|
||||
raise _PdfUnreadable
|
||||
|
||||
def keys_at(self, pos: int, nested: list[bytes] | None = None) -> tuple[bytes, int, bool]:
|
||||
"""Own keys of the dictionary at ``pos``, where its syntax stops, and whether a
|
||||
stream starts there; the scan is charged to the budget."""
|
||||
syntax, stop, is_stream = _pdf_syntax(self.doc, pos, pos + self.budget + 1)
|
||||
self.spend(stop - pos) # a scan past the budget stops one byte beyond it
|
||||
return _pdf_own_keys(syntax, nested), stop, is_stream
|
||||
|
||||
def read_table(self, pos: int) -> bytes:
|
||||
while sub := _PDF_XREF_SUBSECTION.match(self.doc, pos):
|
||||
self.spend(rows=int(sub.group(2)) + 1) # also caps the distinct counts compiled
|
||||
rows = re.compile(rb"(?:\d{10} \d{5} [nf]\s{1,2}){%s}" % sub.group(2))
|
||||
if not (block := rows.match(self.doc, sub.end())):
|
||||
raise _PdfUnreadable
|
||||
for number, (offset, gen, kind) in enumerate(
|
||||
_PDF_XREF_ROW.findall(block.group()), int(sub.group(1))
|
||||
):
|
||||
if kind == b"n": # the newest section is read first, so its entry wins
|
||||
self.entries.setdefault(number, (1, int(offset), int(gen)))
|
||||
pos = block.end()
|
||||
if not (trailer := _PDF_XREF_TRAILER.match(self.doc, pos)):
|
||||
raise _PdfUnreadable
|
||||
return self.keys_at(trailer.end())[0]
|
||||
|
||||
def read_xref_stream(self, at: int) -> bytes:
|
||||
header = _PDF_OBJECT_HEADER.match(self.doc, at)
|
||||
nested: list[bytes] = []
|
||||
keys, data = self.stream(header.end(), nested) if header else (b"", b"")
|
||||
widths = _PDF_XREF_WIDTHS.search(keys)
|
||||
size = re.search(rb"/Size\s+(\d{1,10})", keys)
|
||||
if not (widths and size and re.search(rb"/Type\s*/XRef\b", keys)):
|
||||
raise _PdfUnreadable
|
||||
kind, field, gen = map(int, widths.groups())
|
||||
width = kind + field + gen
|
||||
predicted = 0
|
||||
if parms := re.search(rb"/DecodeParms\s+_(\d{1,4})(?!\d)", keys):
|
||||
raw = nested[int(parms.group(1))] if int(parms.group(1)) < len(nested) else b""
|
||||
if not raw.startswith(b"<<"):
|
||||
raise _PdfUnreadable # an array, or "_<i>" typed into the dict itself
|
||||
values = _pdf_own_keys(raw)
|
||||
predictor = re.search(rb"/Predictor\s+(\d{1,2})(?!\d)", values)
|
||||
columns = re.search(rb"/Columns\s+(\d{1,2})(?!\d)", values)
|
||||
if re.search(rb"/Colors\s+(?!1(?!\d))|/BitsPerComponent\s+(?!8(?!\d))", values):
|
||||
raise _PdfUnreadable
|
||||
if predictor and int(predictor.group(1)) >= 10:
|
||||
predicted = 1
|
||||
if int(columns.group(1) if columns else 1) != width:
|
||||
raise _PdfUnreadable
|
||||
elif predictor and int(predictor.group(1)) != 1:
|
||||
raise _PdfUnreadable # TIFF prediction: not written for xref streams
|
||||
if not width:
|
||||
raise _PdfUnreadable
|
||||
index = re.search(rb"/Index\s*\[([\d\s]*)\]", keys)
|
||||
bounds = (index.group(1) if index else b"0 " + size.group(1)).split()
|
||||
if any(len(n) > 10 for n in bounds):
|
||||
raise _PdfUnreadable
|
||||
starts, lengths = [int(n) for n in bounds[::2]], [int(n) for n in bounds[1::2]]
|
||||
count = min(sum(lengths), len(data) // (width + predicted))
|
||||
self.spend(rows=count)
|
||||
data = data[: count * (width + predicted)]
|
||||
if predicted:
|
||||
data = _pdf_png_rows(data, width)
|
||||
numbers = (n for a, length in zip(starts, lengths) for n in range(a, a + length))
|
||||
for number, at_row in zip(numbers, range(0, count * width, width)):
|
||||
entry = data[at_row : at_row + width]
|
||||
entry_type = int.from_bytes(entry[:kind] or b"\x01", "big")
|
||||
if entry_type in (1, 2):
|
||||
second = int.from_bytes(entry[kind : kind + field], "big")
|
||||
self.entries.setdefault(
|
||||
number, (entry_type, second, int.from_bytes(entry[kind + field :], "big"))
|
||||
)
|
||||
return keys
|
||||
|
||||
def stream(self, pos: int, nested: list[bytes]) -> tuple[bytes, bytes]:
|
||||
"""Own keys and inflated data of the stream object whose dict starts at ``pos``.
|
||||
Its end must be verified: ``endstream`` right after ``/Length`` bytes."""
|
||||
keys, start, is_stream = self.keys_at(pos, nested)
|
||||
length = _PDF_STREAM_LENGTH.search(keys)
|
||||
size = int(length.group(1)) if length else self.integer(_PDF_LENGTH_REF.search(keys))
|
||||
if not (is_stream and _PDF_STREAM_END.match(self.doc, start + size)):
|
||||
raise _PdfUnreadable
|
||||
data = self.doc[start : start + size]
|
||||
if b"/Filter" not in keys:
|
||||
if b"/DecodeParms" in keys:
|
||||
raise _PdfUnreadable
|
||||
return keys, data
|
||||
if not re.search(rb"/Filter\s*/FlateDecode\b", keys) or self.budget <= 0:
|
||||
raise _PdfUnreadable # (max_length 0 would mean no limit)
|
||||
inflate = zlib.decompressobj()
|
||||
try:
|
||||
data = inflate.decompress(data, self.budget)
|
||||
except zlib.error:
|
||||
raise _PdfUnreadable from None
|
||||
if inflate.unconsumed_tail:
|
||||
raise _PdfUnreadable
|
||||
self.spend(len(data))
|
||||
return keys, data
|
||||
|
||||
def integer(self, ref: re.Match[bytes] | None) -> int:
|
||||
"""The integer an indirect ``/Length`` names, at its xref offset."""
|
||||
entry = self.entries.get(int(ref.group(1))) if ref else None
|
||||
value = _PDF_LENGTH_OBJECT.match(self.doc, entry[1]) if entry and entry[0] == 1 else None
|
||||
if not (ref and entry and value and value.group(1, 2) == ref.group(1, 2)):
|
||||
raise _PdfUnreadable
|
||||
if int(ref.group(2)) != entry[2]: # the generation the xref holds
|
||||
raise _PdfUnreadable
|
||||
return int(value.group(3))
|
||||
|
||||
def get(self, ref: re.Match[bytes] | None) -> bytes:
|
||||
"""Own keys of the object ``ref`` names, read where the xref says it lives."""
|
||||
number, gen = (int(ref.group(1)), int(ref.group(2))) if ref else (-1, -1)
|
||||
kind, where, extra = self.entries.get(number, (0, 0, 0))
|
||||
if kind == 1 and extra == gen:
|
||||
header = _PDF_OBJECT_HEADER.match(self.doc, where)
|
||||
if header and (int(header.group(1)), int(header.group(2))) == (number, gen):
|
||||
return self.keys_at(header.end())[0]
|
||||
elif kind == 2 and gen == 0:
|
||||
head, first, data = self.packed.get(where) or self.unpack(where)
|
||||
i = 2 * extra # entry i of the stream's header of (number, offset) pairs
|
||||
if i + 1 < len(head) and int(head[i]) == number:
|
||||
stop = first + int(head[i + 3]) if i + 3 < len(head) else len(data)
|
||||
return _pdf_own_keys(_pdf_syntax(data[first + int(head[i + 1]) : stop])[0])
|
||||
raise _PdfUnreadable
|
||||
|
||||
def unpack(self, number: int) -> tuple[list[bytes], int, bytes]:
|
||||
"""An object stream's header numbers, its /First and its inflated data."""
|
||||
kind, where, gen = self.entries.get(number, (0, 0, 0))
|
||||
header = _PDF_OBJECT_HEADER.match(self.doc, where) if kind == 1 else None
|
||||
if not (header and (int(header.group(1)), int(header.group(2))) == (number, gen)):
|
||||
raise _PdfUnreadable
|
||||
keys, data = self.stream(header.end(), [])
|
||||
first = _PDF_FIRST.search(keys)
|
||||
if not (first and re.search(rb"/Type\s*/ObjStm\b", keys)) or b"/DecodeParms" in keys:
|
||||
raise _PdfUnreadable
|
||||
head = re.findall(rb"\d{1,10}", data[: int(first.group(1))])
|
||||
self.packed[number] = head, int(first.group(1)), data
|
||||
return self.packed[number]
|
||||
|
||||
|
||||
def _pdf_page_count(doc: bytes) -> int:
|
||||
"""Pages in the PDF's active page tree, else its page objects, else 0."""
|
||||
objects: dict[int, bytes] = {}
|
||||
trailers: list[bytes] = []
|
||||
budget = _PDF_OBJECT_STREAM_MAX_BYTES
|
||||
xref: dict[int, int] | None = None # read once, when a stream's /Length is indirect
|
||||
pos = 0
|
||||
eof = doc.rfind(b"%%EOF") # bytes after the last revision hold no objects
|
||||
while len(objects) <= _PDF_MAX_OBJECTS and (
|
||||
match := _PDF_TOP_LEVEL.search(doc, pos, len(doc) if eof < 0 else eof)
|
||||
):
|
||||
pos = match.end()
|
||||
if match.group().startswith(b"%"):
|
||||
continue
|
||||
syntax, pos, is_stream = _pdf_syntax(doc, pos)
|
||||
if match.group(1) is None:
|
||||
trailers.append(_pdf_own_keys(syntax))
|
||||
continue
|
||||
objects[int(match.group(1))] = syntax # a later definition replaces an earlier one
|
||||
if not is_stream:
|
||||
continue
|
||||
keys = _pdf_own_keys(syntax)
|
||||
length = _PDF_STREAM_LENGTH.search(keys)
|
||||
size = int(length.group(1)) if length else -1
|
||||
if not length and (ref := _PDF_LENGTH_REF.search(keys)):
|
||||
if xref is None:
|
||||
found = _pdf_xref(doc, len(doc) if eof < 0 else eof, budget)
|
||||
if not found:
|
||||
return 0 # reading the xref ran out of budget: the size estimate
|
||||
xref, budget = found
|
||||
value = _PDF_LENGTH_OBJECT.match(doc, xref.get(int(ref.group(1)), len(doc)))
|
||||
if value and value.group(1, 2) == ref.group(1, 2):
|
||||
size = int(value.group(3))
|
||||
if size < 0 or not _PDF_STREAM_END.match(doc, pos + size):
|
||||
return 0 # an unverified stream end: its data could pass for objects
|
||||
start, pos = pos, pos + size
|
||||
if re.search(rb"/Type\s*/XRef\b", keys):
|
||||
trailers.append(keys) # a cross-reference stream's dict is its trailer
|
||||
first = _PDF_FIRST.search(keys)
|
||||
if first and budget > 0 and re.search(rb"/Type\s*/ObjStm\b", keys):
|
||||
# One inflate budget for the whole file, not per stream: a PDF with
|
||||
# many object streams must not multiply the request path's memory.
|
||||
with contextlib.suppress(zlib.error):
|
||||
inflated = zlib.decompressobj().decompress(memoryview(doc)[start:pos], budget)
|
||||
budget -= len(inflated)
|
||||
head = re.findall(rb"\d{1,10}", inflated[: int(first.group(1))])
|
||||
offsets = [int(first.group(1)) + int(offset) for offset in head[1::2]]
|
||||
for number, at, stop in zip(head[::2], offsets, [*offsets[1:], len(inflated)]):
|
||||
objects[int(number)] = inflated[at:stop]
|
||||
|
||||
if len(objects) > _PDF_MAX_OBJECTS:
|
||||
return 0 # too many objects to walk: the size estimate stands in
|
||||
|
||||
def keys_of(number: int) -> bytes:
|
||||
return _pdf_own_keys(_pdf_syntax(objects.get(number, b""))[0])
|
||||
|
||||
root = next((m for t in reversed(trailers) if (m := _PDF_ROOT.search(t))), None)
|
||||
tree = _PDF_PAGES.search(keys_of(int(root.group(1)))) if root else None
|
||||
count = _PDF_COUNT.search(keys_of(int(tree.group(1)))) if tree else None
|
||||
if count and 0 < int(count.group(1)) <= _PDF_MAX_PAGES:
|
||||
return int(count.group(1))
|
||||
return sum(1 for number in objects if _PDF_PAGE_OBJECT.search(keys_of(number)))
|
||||
"""Pages in the PDF's active page tree, else 0."""
|
||||
try:
|
||||
reader = _PdfReader(doc)
|
||||
count = _PDF_COUNT.search(reader.get(_PDF_PAGES.search(reader.get(reader.root))))
|
||||
except _PdfUnreadable:
|
||||
return 0
|
||||
return int(count.group(1)) if count and 0 < int(count.group(1)) <= _PDF_MAX_PAGES else 0
|
||||
|
||||
|
||||
class TokenCountCache:
|
||||
|
||||
@@ -114,12 +114,32 @@ def _stream(data: bytes, entries: bytes = b"") -> bytes:
|
||||
return b"<< /Length %d%s >>\nstream\n%s\nendstream" % (len(data), entries, data)
|
||||
|
||||
|
||||
def _png_encoded(data: bytes, width: int) -> bytes:
|
||||
"""``data`` as PNG predictor rows of ``width`` bytes, cycling the five filter types."""
|
||||
out, prior = b"", bytes(width)
|
||||
for number, at in enumerate(range(0, len(data), width)):
|
||||
row, coded = data[at : at + width], bytearray()
|
||||
for i, value in enumerate(row):
|
||||
left, up, corner = row[i - 1] if i else 0, prior[i], prior[i - 1] if i else 0
|
||||
guess = left + up - corner
|
||||
pa, pb, pc = abs(guess - left), abs(guess - up), abs(guess - corner)
|
||||
paeth = left if pa <= pb and pa <= pc else up if pb <= pc else corner
|
||||
coded.append((value - (0, left, up, (left + up) // 2, paeth)[number % 5]) & 255)
|
||||
out += bytes([number % 5]) + coded
|
||||
prior = row
|
||||
return out
|
||||
|
||||
|
||||
def _pdf_file(
|
||||
objects: dict[int, bytes | dict[int, bytes]], prior: bytes = b"", trailer: bytes = b""
|
||||
objects: dict[int, bytes | dict[int, bytes]],
|
||||
prior: bytes = b"",
|
||||
trailer: bytes = b"",
|
||||
predictor: bool = False,
|
||||
) -> bytes:
|
||||
"""A PDF with catalog 1, indexed by an xref table and trailer, or by an xref stream
|
||||
when a dict value packs those objects into an object stream. With ``prior``, an
|
||||
incremental update of that file; ``trailer`` opens the trailer's dict."""
|
||||
when a dict value packs those objects into an object stream or ``predictor`` asks
|
||||
for a Flate xref stream with PNG predictor rows. With ``prior``, an incremental
|
||||
update of that file; ``trailer`` opens the trailer's dict."""
|
||||
out = prior or b"%PDF-1.7\n"
|
||||
rows = {} if prior else {0: (0, 0, 65535)}
|
||||
for number, body in objects.items():
|
||||
@@ -137,7 +157,7 @@ def _pdf_file(
|
||||
out += b"%d 0 obj\n%s\nendobj\n" % (number, body)
|
||||
prev = b" /Prev %s" % prior.split(b"startxref")[-1].split()[0] if prior else b""
|
||||
xref_at = len(out)
|
||||
if any(kind == 2 for kind, _, _ in rows.values()):
|
||||
if predictor or any(kind == 2 for kind, _, _ in rows.values()):
|
||||
rows[max(rows) + 1] = (1, xref_at, 0)
|
||||
index = b" ".join(b"%d 1" % number for number in sorted(rows))
|
||||
table = b"".join(
|
||||
@@ -150,6 +170,9 @@ def _pdf_file(
|
||||
index,
|
||||
prev,
|
||||
)
|
||||
if predictor:
|
||||
table = zlib.compress(_png_encoded(table, 7))
|
||||
entries += b" /Filter /FlateDecode /DecodeParms << /Columns 7 /Predictor 12 >>"
|
||||
out += b"%d 0 obj\n%s\nendobj\n" % (max(rows), _stream(table, entries))
|
||||
else:
|
||||
out += b"xref\n" + b"".join(
|
||||
@@ -200,9 +223,10 @@ def test_base64_pdf_document_is_priced_by_pages_not_as_text() -> None:
|
||||
assert 12 * 1500 <= got < 12 * 1500 + 100, f"{name} priced a 12-page PDF at {got}"
|
||||
|
||||
|
||||
def test_pdf_page_count_falls_back_from_tree_to_page_objects_to_size() -> None:
|
||||
# A truncated file has no trailer to find its page tree from: its page objects count.
|
||||
assert count_content_blocks([_base64_document(_pdf(5, trailer=False))], len) == 5 * 1500
|
||||
def test_pdf_page_count_falls_back_from_the_tree_to_the_size_estimate() -> None:
|
||||
# A truncated file has no xref to find its page tree through: ~3KB of file per page.
|
||||
truncated = _pdf(5, trailer=False)
|
||||
assert count_content_blocks([_base64_document(truncated)], len) == len(truncated) // 3000 * 1500
|
||||
# PDF 1.5+ keeps the page tree inside a compressed object stream.
|
||||
hidden = _pdf_file({10: _page_tree(7), 11: _stream(bytes(range(256)) * 4096)})
|
||||
assert count_content_blocks([_base64_document(hidden)], len) == 7 * 1500
|
||||
@@ -218,18 +242,20 @@ def test_pdf_page_count_falls_back_from_tree_to_page_objects_to_size() -> None:
|
||||
assert count_content_blocks([broken], len) == 1500
|
||||
|
||||
|
||||
def test_object_stream_inflation_shares_one_budget_per_file(monkeypatch) -> None:
|
||||
def test_only_the_page_trees_object_stream_is_inflated_within_the_budget(monkeypatch) -> None:
|
||||
fonts = dict.fromkeys(range(10, 30), b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>")
|
||||
pdf = _pdf_file({30: fonts, 31: _page_tree(7), 32: _stream(bytes(range(256)) * 4096)})
|
||||
inflates = []
|
||||
real = zlib.decompressobj
|
||||
monkeypatch.setattr(base.zlib, "decompressobj", lambda *a: inflates.append(1) or real(*a))
|
||||
assert count_content_blocks([_base64_document(pdf)], len) == 7 * 1500
|
||||
# The fonts stream (larger than the page tree's) spends the whole budget, so the
|
||||
# page tree after it stays packed.
|
||||
fonts_stream = pdf[pdf.index(b"stream\n") + 7 : pdf.index(b"\nendstream")]
|
||||
monkeypatch.setattr(base, "_PDF_OBJECT_STREAM_MAX_BYTES", len(zlib.decompress(fonts_stream)))
|
||||
assert len(inflates) == 1 # the fonts' stream is never read
|
||||
# A budget the page tree's stream does not fit in gives the size estimate.
|
||||
monkeypatch.setattr(base, "_PDF_OBJECT_STREAM_MAX_BYTES", 600)
|
||||
assert count_content_blocks([_base64_document(pdf)], len) == len(pdf) // 3000 * 1500
|
||||
|
||||
|
||||
def test_a_pdf_with_too_many_objects_to_walk_gets_the_size_estimate(monkeypatch) -> None:
|
||||
def test_a_pdf_with_more_xref_rows_than_the_budget_gets_the_size_estimate(monkeypatch) -> None:
|
||||
pdf = _pdf(5)
|
||||
monkeypatch.setattr(base, "_PDF_MAX_OBJECTS", 4)
|
||||
assert count_content_blocks([_base64_document(pdf)], len) == len(pdf) // 3000 * 1500
|
||||
@@ -267,14 +293,12 @@ def _one_page_pdf(text: bytes, count: bytes) -> bytes:
|
||||
(b"2 0 obj << /Type /Pages /Count 999 >> endobj", b"/Count 1"),
|
||||
# Neither is a comment or a string inside the page tree's own dict.
|
||||
(b"hello", b"% /Count 999\n/Title (a (nested) /Count 999) /Count 1"),
|
||||
# A malformed or absurd count falls back to the page objects.
|
||||
# A malformed or absurd count gives the size estimate (1 page at this size).
|
||||
(b"hello", b"/Count 1000000"),
|
||||
(b"hello", b"/Count " + b"9" * 5000),
|
||||
# A count in a dict, array or hex string nested in the tree's dict is not its own.
|
||||
(b"hello", b"/PieceInfo << /Example << /Private << /Count 9999 >> >> >> /Count 1"),
|
||||
(b"hello", b"/Private [<00ff> << /Count 9999 >>] /Count 1"),
|
||||
# Nor is a nested /Type /Page a page object when the count falls back to those.
|
||||
(b"hello", b"/PieceInfo << /Private << /Type /Page >> >> /Count 1000000"),
|
||||
],
|
||||
ids=[
|
||||
"plain",
|
||||
@@ -286,7 +310,6 @@ def _one_page_pdf(text: bytes, count: bytes) -> bytes:
|
||||
"long",
|
||||
"nested-dict",
|
||||
"nested-array",
|
||||
"nested-page",
|
||||
],
|
||||
)
|
||||
def test_page_count_comes_only_from_the_page_tree_dict(text: bytes, count: bytes) -> None:
|
||||
@@ -376,13 +399,12 @@ _LONE_CR = b"BT /F1 12 Tf 72 720 Td (hi) Tj ET endobj 2 0 obj << /Type /Pages /C
|
||||
),
|
||||
1,
|
||||
),
|
||||
# A stream end that cannot be verified stops the walk, and 0 means the size
|
||||
# estimate: the length packed in an object stream, a wrong one, a reference to
|
||||
# no object, no /Length at all.
|
||||
(_pdf_file(_packed(_INDIRECT, 6)), 0),
|
||||
(_pdf_file({**_INDIRECT, 6: b"40"}), 0),
|
||||
(_pdf_file(_one_page_objects(_FAKE_TREE, length=b"/Length 9 0 R")), 0),
|
||||
(_pdf_file(_one_page_objects(_FAKE_TREE, length=b"")), 0),
|
||||
# The page count never reads a content stream, so neither does its length: one
|
||||
# packed in an object stream, a wrong one, a reference to no object, or none.
|
||||
(_pdf_file(_packed(_INDIRECT, 6)), 1),
|
||||
(_pdf_file({**_INDIRECT, 6: b"40"}), 1),
|
||||
(_pdf_file(_one_page_objects(_FAKE_TREE, length=b"/Length 9 0 R")), 1),
|
||||
(_pdf_file(_one_page_objects(_FAKE_TREE, length=b"")), 1),
|
||||
],
|
||||
ids=[
|
||||
"indirect",
|
||||
@@ -457,6 +479,47 @@ def test_xref_dicts_share_one_scan_budget_across_sections() -> None:
|
||||
assert _priced_after(chain + b">> " * 300 + b"\nstartxref\n9\n%%EOF\n") == 0
|
||||
|
||||
|
||||
def _unreferenced(pdf: bytes, obj: bytes) -> bytes:
|
||||
"""``pdf`` (an xref table file) with ``obj`` added before its table, which does not list it."""
|
||||
at = pdf.rindex(b"\nxref\n") + 1
|
||||
xref_at = int(pdf.rsplit(b"startxref\n", 1)[1].split()[0])
|
||||
tail = pdf[at:].replace(b"startxref\n%d" % xref_at, b"startxref\n%d" % (xref_at + len(obj)))
|
||||
return pdf[:at] + obj + tail
|
||||
|
||||
|
||||
_DECOY_TREE = b"<< /Type /Pages /Kids [3 0 R] /Count 9999 >>"
|
||||
_DECOY_STREAM = _stream(
|
||||
zlib.compress(b"2 0 " + _DECOY_TREE), b" /Type /ObjStm /N 1 /First 4 /Filter /FlateDecode"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pdf",
|
||||
[
|
||||
# A later "2 0 obj" outside any stream that the xref does not list.
|
||||
_unreferenced(_one_page_pdf(b"hello", b"/Count 1"), b"2 0 obj\n%s\nendobj\n" % _DECOY_TREE),
|
||||
# A later object stream holding an object 2, unlisted or listed: either way the
|
||||
# xref puts object 2 at its own offset.
|
||||
_unreferenced(
|
||||
_one_page_pdf(b"hello", b"/Count 1"), b"9 0 obj\n%s\nendobj\n" % _DECOY_STREAM
|
||||
),
|
||||
_pdf_file({**_one_page_objects(b"hello"), 9: _DECOY_STREAM}),
|
||||
],
|
||||
ids=["object", "unlisted-object-stream", "listed-object-stream"],
|
||||
)
|
||||
def test_only_the_definition_the_xref_points_at_counts(pdf: bytes) -> None:
|
||||
assert base.BaseTokenizer._estimate_document_tokens(pdf) == 1500
|
||||
|
||||
|
||||
def test_a_predictor_12_xref_stream_is_decoded_row_by_row() -> None:
|
||||
# Its rows cycle the five PNG filter types; the tree is top level, or packed.
|
||||
for pdf in (
|
||||
_pdf_file(_page_tree(12), predictor=True),
|
||||
_pdf_file({20: _page_tree(12)}, predictor=True),
|
||||
):
|
||||
assert base.BaseTokenizer._estimate_document_tokens(pdf) == 12 * 1500
|
||||
|
||||
|
||||
def test_incremental_update_prices_the_active_page_tree_not_an_obsolete_one() -> None:
|
||||
original = _pdf_file(_page_tree(5))
|
||||
# An edit keeps two pages under a new tree; the old tree (/Count 5) stays in the file.
|
||||
|
||||
Reference in New Issue
Block a user