fix(tokenizers): give the xref traversal shared per-document budgets

_pdf_xref bounded rows and inflation per section only, while its entries
accumulated across the whole /Prev chain of up to 1,000 sections. Two
compressed xref streams declaring 100,000 entries each built 200,000
entries from a 2.5 KB file (JerrettDavis's repro), 1,000 chained streams
each inflating to 4 MB of rows took 18 s, and 1,000 table sections whose
trailer dicts stay open rescanned most of the file per section (7 s).

The traversal now draws on budgets shared by the whole document: one of
rows read (table and stream rows, repeats included, plus one per section
and subsection, which also caps the distinct row counts compiled), and
the file's one 4 MB inflate budget, now also spent on every byte of
section dict syntax scanned. Running out, or past the section cap,
returns None, and _pdf_page_count prices the size estimate rather than
a partial xref.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Garm
2026-10-24 11:15:23 +02:00
co-authored by Claude Opus 5.5
parent adcc4f1d83
commit 78c22acc85
2 changed files with 97 additions and 13 deletions
+40 -13
View File
@@ -49,7 +49,7 @@ _DOCUMENT_BYTES_PER_PAGE = 3000
_PDF_MAX_PAGES = 10_000 # a larger /Count is treated as malformed, not priced
_PDF_OBJECT_STREAM_MAX_BYTES = 4_000_000
_PDF_MAX_OBJECTS = 100_000 # bounds the walk on the request path
_PDF_MAX_XREF_SECTIONS = 1_000 # bounds the /Prev chain an indirect /Length is read through
_PDF_MAX_XREF_SECTIONS = 1_000 # a longer /Prev chain is treated as malformed
# Each pattern opens with a lookahead on its first byte so the scan skips binary fast.
_PDF_TOP_LEVEL = re.compile(
rb"(?=[%\dt])(?:%[^\r\n]*|(?<!\d)(\d{1,10})\s+\d{1,5}\s+obj\b|(?<![A-Za-z/])trailer\b)"
@@ -119,24 +119,34 @@ def _pdf_own_keys(syntax: bytes) -> bytes:
return b"".join(parts)
def _pdf_xref(doc: bytes, end: int) -> dict[int, int]:
def _pdf_xref(doc: bytes, end: int, budget: int) -> tuple[dict[int, int], int] | None:
"""Offset of each uncompressed object, from the cross-reference sections the last
``startxref`` before ``end`` chains to (a table's ``/XRefStm``, then ``/Prev``);
the newest entry wins. Read the way a PDF reader reads it, never by searching the
file for an object's text, which page content can fake."""
file for an object's text, which page content can fake. Returns what is left of
``budget`` too, or None once a budget runs out: the whole traversal shares one
budget of rows read and one of bytes scanned or inflated, repeats included, so
no chain of sections can multiply the work."""
offsets: dict[int, int] = {}
rows = _PDF_MAX_OBJECTS # one per row read, plus one per section and subsection
last = doc.rfind(b"startxref", 0, end)
start = _PDF_STARTXREF.match(doc, last) if last >= 0 else None
todo = [int(start.group(1))] if start else []
seen: set[int] = set()
while todo and len(seen) < _PDF_MAX_XREF_SECTIONS:
while todo:
at = todo.pop(0)
if at in seen:
continue
seen.add(at)
rows -= 1
if len(seen) > _PDF_MAX_XREF_SECTIONS or rows < 0:
return None
if table := _PDF_XREF_TABLE.match(doc, at):
pos = table.end()
while sub := _PDF_XREF_SUBSECTION.match(doc, pos):
rows -= int(sub.group(2)) + 1 # also caps the distinct counts compiled below
if rows < 0:
return None
table_rows = re.compile(rb"(?:\d{10} \d{5} [nf]\s{1,2}){%s}" % sub.group(2))
if not (block := table_rows.match(doc, sub.end())):
break
@@ -147,10 +157,17 @@ def _pdf_xref(doc: bytes, end: int) -> dict[int, int]:
offsets.setdefault(number, int(offset))
pos = block.end()
trailer = _PDF_XREF_TRAILER.match(doc, pos)
keys = _pdf_own_keys(_pdf_syntax(doc, trailer.end())[0]) if trailer else b""
syntax, stop, _ = _pdf_syntax(doc, trailer.end()) if trailer else (b"", pos, False)
budget -= stop - pos
elif header := _PDF_XREF_STREAM.match(doc, at):
syntax, pos, is_stream = _pdf_syntax(doc, header.end())
keys = _pdf_own_keys(syntax)
budget -= pos - header.end()
else:
continue
if budget < 0:
return None
keys = _pdf_own_keys(syntax)
if not table:
length = _PDF_STREAM_LENGTH.search(keys)
widths = _PDF_XREF_WIDTHS.search(keys)
kind, field, gen = map(int, widths.groups()) if widths else (0, 0, 0)
@@ -171,26 +188,32 @@ def _pdf_xref(doc: bytes, end: int) -> dict[int, int]:
continue
data = doc[pos : pos + int(length.group(1))]
if b"/Filter" in keys:
if budget <= 0: # max_length 0 would mean no limit
return None
inflate = zlib.decompressobj()
try:
data = zlib.decompressobj().decompress(data, _PDF_OBJECT_STREAM_MAX_BYTES)
data = inflate.decompress(data, budget)
except zlib.error:
continue
if inflate.unconsumed_tail:
return None
budget -= len(data)
index = re.search(rb"/Index\s*\[([\d\s]*)\]", keys)
bounds = [int(n) for n in (index.group(1) if index else b"0 " + size.group(1)).split()]
rows -= min(sum(bounds[1::2]), len(data) // row)
if rows < 0:
return None
numbers = (
n for a, count in zip(bounds[::2], bounds[1::2]) for n in range(a, a + count)
)
rows = range(0, min(len(data), _PDF_MAX_OBJECTS * row) - row + 1, row)
for number, at_row in zip(numbers, rows):
for number, at_row in zip(numbers, range(0, len(data) - row + 1, row)):
entry = data[at_row : at_row + row]
if int.from_bytes(entry[:kind] or b"\x01", "big") == 1: # type 1: at an offset
offsets.setdefault(number, int.from_bytes(entry[kind : kind + field], "big"))
else:
continue
for key in (rb"/XRefStm", rb"/Prev"):
if found := re.search(key + rb"\s+(\d{1,10})(?!\d)", keys):
todo.append(int(found.group(1)))
return offsets
return offsets, budget
def _pdf_page_count(doc: bytes) -> int:
@@ -218,7 +241,11 @@ def _pdf_page_count(doc: bytes) -> int:
length = _PDF_STREAM_LENGTH.search(keys)
size = int(length.group(1)) if length else -1
if not length and (ref := _PDF_LENGTH_REF.search(keys)):
xref = _pdf_xref(doc, len(doc) if eof < 0 else eof) if xref is None else xref
if xref is None:
found = _pdf_xref(doc, len(doc) if eof < 0 else eof, budget)
if not found:
return 0 # reading the xref ran out of budget: the size estimate
xref, budget = found
value = _PDF_LENGTH_OBJECT.match(doc, xref.get(int(ref.group(1)), len(doc)))
if value and value.group(1, 2) == ref.group(1, 2):
size = int(value.group(3))
@@ -400,6 +400,63 @@ def test_stream_data_is_never_walked_as_objects(pdf: bytes, pages: int) -> None:
assert base._pdf_page_count(pdf) == pages
def _xref_chain(*sections: bytes) -> bytes:
"""A file of xref streams chained by /Prev, each of ``sections`` the end of one's dict
and its stream, ready as an update's ``prior``."""
out, prev = b"%PDF-1.7\n", b""
for number, body in enumerate(sections, 100):
at = len(out)
out += b"%d 0 obj\n<< /Type /XRef /Size 999999 /W [1 4 2]%s%s\nendobj\n" % (
number,
prev,
body,
)
prev = b" /Prev %d" % at
return out + b"startxref\n%d\n%%%%EOF\n" % at
def _flate_rows(index: bytes, data: bytes) -> bytes:
packed = zlib.compress(data, 9)
return b" /Index [%s] /Filter /FlateDecode /Length %d >>\nstream\n%s\nendstream" % (
index,
len(packed),
packed,
)
def _priced_after(prior: bytes) -> int:
return base._pdf_page_count(_pdf_file(_INDIRECT, prior=prior))
def test_xref_rows_share_one_budget_across_sections() -> None:
# Two compressed sections of 100,000 entries each: 200,000 rows read is past the
# 100,000 the whole traversal may read, so the size estimate stands in.
rows = bytes(7 * 100_000)
chain = _xref_chain(_flate_rows(b"0 100000", rows), _flate_rows(b"100000 100000", rows))
assert _priced_after(chain) == 0
# A normal update chain through a compressed section is well inside the budgets.
assert _priced_after(_xref_chain(_flate_rows(b"0 1", bytes(7)))) == 1
def test_xref_streams_share_the_files_one_inflate_budget(monkeypatch) -> None:
# 100 sections each inflating to 4 MB of rows: the first runs out the file's one
# 4 MB budget, so the size estimate stands in and no other section is inflated.
inflates = []
real = zlib.decompressobj
monkeypatch.setattr(base.zlib, "decompressobj", lambda *a: inflates.append(1) or real(*a))
assert _priced_after(_xref_chain(*[_flate_rows(b"0 571428", bytes(4_000_000))] * 100)) == 0
assert len(inflates) == 1
def test_xref_dicts_share_one_scan_budget_across_sections() -> None:
# 300 table sections whose trailer dicts stay open, so each holds every later one:
# reading the chain would rescan most of the file 300 times.
section = b"xref\n0 0\ntrailer\n<< /Prev %010d " + b"/A " * 200
width = len(section % 0)
chain = b"%PDF-1.7\n" + b"".join(section % (9 + (i + 1) * width) for i in range(300))
assert _priced_after(chain + b">> " * 300 + b"\nstartxref\n9\n%%EOF\n") == 0
def test_incremental_update_prices_the_active_page_tree_not_an_obsolete_one() -> None:
original = _pdf_file(_page_tree(5))
# An edit keeps two pages under a new tree; the old tree (/Count 5) stays in the file.