fix(anydoc): 升级到 0.1.9，为恶意 PDF 的解析开销加上上界 (#2720)

* build(anydoc): 构建脚本从 Cargo.toml 读取待打补丁的 crate 版本原先 0.1.8 硬编码在脚本的 6 处，升级要同时改动它们；现在统一从 `anydoc = "=X.Y.Z"` 这一处 pin 推导。另外补两道拦截：patched-anydoc/ 的复用标记记录已打补丁的版本，避免版本变更后沿用上一版的副本、最终在 cargo 的 patch 解析里报出难以定位的错误；构建前校验 version.go 与该 pin 一致，因为这个常量会作为 anydoc_version 写进每一篇已解析文档的元数据。 * fix(anydoc): 升级到 0.1.9，为恶意 PDF 的解析开销加上上界 anydoc 0.1.9 只把 pdf-inspector 从 0.1.8 提到 1.14.2，但那一版含 9 个修复，其中 7 个是给此前完全无上界的解析工作加限额：每页 Form XObject 展开、CID /W 区间、Encoding cidrange 与 ToUnicode bfrange 展开、分配操作符前的内容流解码、detector 的 Tj/TJ 操作数回溯、以及不相交矩形的表格聚类。它们全都落在 process_pdf_mem 这条路径上，而这是 anydoc 转换 PDF 的唯一入口，且只由上传文件的字节驱动。这正是 guarded() 拦不住的那一类问题：耗尽 CPU 不会 panic，分配失败直接 abort。以 detector 回溯为例，缺少操作数的 TJ 会重扫整个内容流，因此一串裸 `] TJ` 的开销是长度的平方：977 KB 的 PDF 要 26.7 秒，1.9 MB 要 111.9 秒；升级后分别是 5.8 毫秒和 11.0 毫秒。 TestDetectorLookbackStaysLinear 用 15 秒预算把这个上界钉住——它在 0.1.8 上会超时失败，在 0.1.9 上 0.01 秒通过。另外 2 个修复改善提取质量，直接影响入库内容：Form XObject 内的文字现在跟踪 text line matrix 并处理 T*/TL/'/"/Tc/Tw，嵌套表单文字不再丢换行和字间距；小型大写字母（small-caps）的文字段会合并，不再被当成额外的表格列而产出错位的 markdown 表格。升级本身不需要改绑定代码：anydoc 0.1.8 与 0.1.9 的 src/ 字节级一致， C ABI、文档模型和 GFM 序列化器都没变，6 项本地修改原样保留。依赖树也只动了 3 行版本和 1 个 checksum，lopdf 仍是 0.42，cargo audit 结果与升级前相同（仅 ttf-parser 未维护这一条已允许的警告）。 * docs(anydoc): 文档与注释中的耗时数字对齐到实测记录表格与注释里的数字改为与随 PR 附上的实测日志同一次运行的结果，便于核对。
2026-10-02 05:54:33 +08:00 · 2026-08-16 12:30:55 +08:00
parent ad2f1d1450
commit 7f0e1a91e9
6 changed files with 165 additions and 29 deletions
@@ -10,6 +10,7 @@ import (
 	"sync"
 	"sync/atomic"
 	"testing"
+	"time"
 )

 // These tests exercise the linked Rust converter, so they only build with the
@@ -191,6 +192,36 @@ func TestDeeplyNestedPDFFailsWithoutKillingTheProcess(t *testing.T) {
 	}
 }

+// A show-text operator with no operand made the page-classification scan walk
+// back over the whole content stream looking for one, so a stream of bare
+// `] TJ` tokens cost time quadratic in its length: before pdf-inspector 1.14.2
+// this input took 26 seconds of CPU, and a 2 MB one took nearly two minutes.
+// Unbounded work is the half of the hostile-PDF problem that guarded() cannot
+// catch — it never panics, it just holds the core — so the bound is pinned
+// here. The budget is deliberately far above the ~7ms a bounded lookback needs:
+// what it has to distinguish is linear from quadratic, not fast from slow.
+func TestDetectorLookbackStaysLinear(t *testing.T) {
+	const (
+		operators = 200000
+		budget    = 15 * time.Second
+	)
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		// The result is irrelevant: the page carries one real text run, so
+		// this converts either way. Only how long it takes is under test.
+		_, _ = Convert(unmatchedShowTextPDF(operators), Options{Format: "pdf"})
+	}()
+
+	select {
+	case <-done:
+	case <-time.After(budget):
+		t.Fatalf("converting a PDF with %d unmatched TJ operators took over %s; "+
+			"the detector's operand lookback is no longer bounded", operators, budget)
+	}
+}
+
 // buildDocx writes the smallest OOXML package that carries a heading, a
 // paragraph, and one embedded image.
 func buildDocx(t *testing.T) []byte {
@@ -304,6 +335,21 @@ func nestedPDF(nested string) []byte {
 	})
 }

+// unmatchedShowTextPDF is a one-page PDF whose content stream shows text once
+// and then repeats a TJ operator whose array operand is never opened.
+func unmatchedShowTextPDF(operators int) []byte {
+	content := "BT /F1 12 Tf 20 100 Td (Shipping summary) Tj ET\n" +
+		strings.Repeat("] TJ\n", operators)
+	return writePDF([]string{
+		"<< /Type /Catalog /Pages 2 0 R >>",
+		"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
+		"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R " +
+			"/Resources << /Font << /F1 5 0 R >> >> >>",
+		fmt.Sprintf("<< /Length %d >>\nstream\n%sendstream", len(content), content),
+		"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
+	})
+}
+
 // textlessPDF is a one-page PDF with no text stream: the shape of a scanned
 // exam paper as far as pdf-inspector is concerned.
 func textlessPDF() []byte {
@@ -41,31 +41,42 @@ case "$target" in
  *) lib_name=libanydoc_go.a ;;
 esac

-# document_to_markdown is private in published anydoc. Copy 0.1.8 and re-export
-# that one function so anydoc-go can keep the official serializer.
+# The crate version to patch is read from Cargo.toml rather than repeated here,
+# so an upgrade is a one-line change to the `anydoc = "=X.Y.Z"` pin.
+anydoc_version=$(sed -n 's/^anydoc = "=\([0-9][^"]*\)".*/\1/p' "$crate_dir/Cargo.toml" | head -1)
+if [ -z "$anydoc_version" ]; then
+  echo "error: no pinned 'anydoc = \"=X.Y.Z\"' dependency in $crate_dir/Cargo.toml" >&2
+  exit 1
+fi
+
+# document_to_markdown is private in published anydoc. Copy the pinned version
+# and re-export that one function so anydoc-go can keep the official serializer.
 prepare_patched_anydoc() {
  local dest="$crate_dir/patched-anydoc"
-  if [ -f "$dest/.weknora-patched" ]; then
+  # The marker records which version was patched: after a version bump the
+  # copy left by the previous build is the wrong crate, and reusing it would
+  # fail deep inside cargo's patch resolution instead of here.
+  if [ -f "$dest/.weknora-patched" ] && [ "$(cat "$dest/.weknora-patched")" = "$anydoc_version" ]; then
    return
  fi

  local src=""
  local cargo_home="${CARGO_HOME:-$HOME/.cargo}"
-  src=$(find "$cargo_home/registry/src" -maxdepth 2 -type d -name 'anydoc-0.1.8' 2>/dev/null | head -1 || true)
+  src=$(find "$cargo_home/registry/src" -maxdepth 2 -type d -name "anydoc-$anydoc_version" 2>/dev/null | head -1 || true)

  if [ -z "$src" ]; then
-    local tarball="$crate_dir/.anydoc-0.1.8.crate"
-    echo "Fetching anydoc 0.1.8 to patch document_to_markdown into the public API"
-    if ! curl -fsSL "https://rsproxy.cn/api/v1/crates/anydoc/0.1.8/download" -o "$tarball"; then
-      curl -fsSL "https://static.crates.io/crates/anydoc/anydoc-0.1.8.crate" -o "$tarball"
+    local tarball="$crate_dir/.anydoc-$anydoc_version.crate"
+    echo "Fetching anydoc $anydoc_version to patch document_to_markdown into the public API"
+    if ! curl -fsSL "https://rsproxy.cn/api/v1/crates/anydoc/$anydoc_version/download" -o "$tarball"; then
+      curl -fsSL "https://static.crates.io/crates/anydoc/anydoc-$anydoc_version.crate" -o "$tarball"
    fi
    local unpack="$crate_dir/.anydoc-unpack"
    rm -rf "$unpack"
    mkdir -p "$unpack"
    tar -xzf "$tarball" -C "$unpack"
-    src=$(find "$unpack" -maxdepth 1 -type d -name 'anydoc-0.1.8' | head -1)
+    src=$(find "$unpack" -maxdepth 1 -type d -name "anydoc-$anydoc_version" | head -1)
    if [ -z "$src" ]; then
-      echo "error: unpacked anydoc-0.1.8 crate is missing" >&2
+      echo "error: unpacked anydoc-$anydoc_version crate is missing" >&2
      exit 1
    fi
  fi
@@ -73,20 +84,30 @@ prepare_patched_anydoc() {
  rm -rf "$dest"
  cp -R "$src" "$dest"
  if ! grep -q '^use render::markdown::document_to_markdown;$' "$dest/src/lib.rs"; then
-    echo "error: anydoc 0.1.8 lib.rs no longer has the expected document_to_markdown import" >&2
+    echo "error: anydoc $anydoc_version lib.rs no longer has the expected document_to_markdown import" >&2
    exit 1
  fi
  sed -i.bak 's/^use render::markdown::document_to_markdown;/pub use render::markdown::document_to_markdown;/' "$dest/src/lib.rs"
  rm -f "$dest/src/lib.rs.bak"
-  touch "$dest/.weknora-patched"
-  rm -rf "$crate_dir/.anydoc-unpack" "$crate_dir/.anydoc-0.1.8.crate"
+  printf '%s\n' "$anydoc_version" > "$dest/.weknora-patched"
+  rm -rf "$crate_dir/.anydoc-unpack" "$crate_dir/.anydoc-$anydoc_version.crate"
 }

+# version.go is what the parser reports as `anydoc_version` on every parsed
+# document, so a bump that misses it would mislabel the whole knowledge base.
+go_version=$(sed -n 's/^const Version = "\([^"]*\)".*/\1/p' "$crate_dir/version.go" | head -1)
+if [ "$go_version" != "$anydoc_version" ]; then
+  echo "error: version.go says '$go_version' but Cargo.toml pins anydoc '$anydoc_version'." >&2
+  echo "Bump both: the Go constant is the version WeKnora records for parsed documents." >&2
+  exit 1
+fi
+
+echo "Building anydoc $anydoc_version archive for $target"
+prepare_patched_anydoc
+
 # --locked: build exactly the dependency versions in the committed Cargo.lock.
 # That lockfile is what pins the patched pdf-inspector/lopdf, so a silent
 # resolver drift must fail the build rather than ship an unaudited tree.
-echo "Building anydoc archive for $target"
-prepare_patched_anydoc
 cargo build --release --locked --manifest-path "$crate_dir/Cargo.toml" --target "$target"

 dest="$crate_dir/lib/$lib_dir"
@@ -89,7 +89,7 @@ dependencies = [

 [[package]]
 name = "anydoc"
-version = "0.1.8"
+version = "0.1.9"
 dependencies = [
 "calamine",
 "cfb",
@@ -104,7 +104,7 @@ dependencies = [

 [[package]]
 name = "anydoc-go"
-version = "0.1.8"
+version = "0.1.9"
 dependencies = [
 "anydoc",
 "cbindgen",
@@ -868,9 +868,9 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"

 [[package]]
 name = "pdf-inspector"
-version = "0.1.8"
+version = "1.14.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd2f755e49ad38eafbc82ba2bec1c59d57b5bf829b13a8f35e4263bc8913df3a"
+checksum = "1e024ae242c514e2adf6aee186678e0eabdc2e5ecfbb2159186881b4498593cb"
 dependencies = [
 "env_logger",
 "include_dir",
@@ -5,7 +5,7 @@
 # symbols from, not a loadable shared object.
 [package]
 name = "anydoc-go"
-version = "0.1.8"
+version = "0.1.9"
 edition = "2024"
 description = "Go bindings (C ABI) for anydoc"
 license = "MIT"
@@ -19,14 +19,14 @@ crate-type = ["staticlib"]
 # workspace, so it takes the published crate rather than a path dependency.
 # Pinned exactly: the C ABI mirrors this version's error and format enums.
 [dependencies]
-anydoc = "=0.1.8"
+anydoc = "=0.1.9"

 [build-dependencies]
 cbindgen = "0.29"

 # document_to_markdown is crate-private in published anydoc. The build script
-# copies 0.1.8 and re-exports that one function so we can keep the official
-# GFM serializer after rewriting asset images to External URLs.
+# copies the pinned version above and re-exports that one function so we can
+# keep the official GFM serializer after rewriting asset images to External URLs.
 [patch.crates-io]
 anydoc = { path = "patched-anydoc" }

@@ -29,7 +29,7 @@ only `internal/infrastructure/docparser/anydoc/backend_cgo.go` imports it.
 | Upstream PR | firecrawl/anydoc#30 ("feat: add Go bindings") |
 | PR head | `1a7a6c0` |
 | Rebased onto | `4e3089b` (`chore: release v0.1.8`) |
-| anydoc crate | `0.1.8` (from crates.io, pinned with `=`) |
+| anydoc crate | `0.1.9` (from crates.io, pinned with `=`) |
 | License | MIT (see LICENSE) |

 The PR branched before anydoc 0.1.7, so it was rebased onto v0.1.8 before
@@ -40,12 +40,73 @@ version, bumped from 0.1.3 to 0.1.8. Taking v0.1.8 also picks up the
 `pdf-inspector` 0.1.8 bump that fixes [RUSTSEC-2026-0187](https://rustsec.org/advisories/RUSTSEC-2026-0187.html),
 the `lopdf` stack overflow that aborts the process on a hostile PDF.

+The pin then moved to 0.1.9, which is that same binding source against a newer
+PDF stack: anydoc's `src/` is byte-identical between the two releases, so the C
+ABI, the document model, and the GFM serializer this binding reaches into are
+unchanged, and none of the local modifications below had to be re-applied. What
+0.1.9 does change is `pdf-inspector`, from the crate published as 0.1.8 to
+1.14.2 (the jump in the number is that project unifying its Rust, Python, and
+Node versions, not fourteen major releases). See "Why 0.1.9 matters here" below.
+
+## Why 0.1.9 matters here
+
+anydoc's changelog lists 0.1.9 as a single dependency bump, which reads like
+housekeeping. It is nine `pdf-inspector` fixes, and seven of them are
+denial-of-service bounds on PDF parsing.
+
+Those seven bound work that had no bound at all: Form XObject expansion per
+page, CID `/W` ranges, Encoding `cidrange` and ToUnicode `bfrange` expansion,
+content-stream decode before operators are allocated, the detector's `Tj`/`TJ`
+operand lookback, and disjoint-rect table clustering. Every one sits on the path
+`pdf_inspector::process_pdf_mem` takes — the only way anydoc converts a PDF —
+and every one is driven by nothing but the bytes of the uploaded file.
+
+This is the class of bug the `guarded()` panic catcher explicitly cannot
+contain, for the same reason `RUSTSEC-2026-0187` could not be: unbounded CPU
+never panics, and an allocation failure aborts. The detector lookback is the
+cheapest to demonstrate — before 1.14.2, each `Tj`/`TJ` whose operand was
+missing rescanned the whole content stream, so a stream of bare `] TJ` tokens
+was quadratic:
+
+| `] TJ` tokens | PDF size | 0.1.8 | 0.1.9 |
+| --- | --- | --- | --- |
+| 40,000 | 195 KB | 1.0 s | 2.0 ms |
+| 120,000 | 586 KB | 9.7 s | 4.1 ms |
+| 200,000 | 977 KB | 25.9 s | 6.5 ms |
+| 400,000 | 1.9 MB | 105.6 s | 12.2 ms |
+
+One run on a 4-core VM, same Go code either side, only the linked archive
+differing. Quadrupling the input cost 16× the time before the fix and roughly
+4× after. A 2 MB upload holding a core for nearly two minutes needs no
+privilege and no malformed container — the file parses fine, it just takes
+forever, and every concurrent upload takes a core of its own.
+`TestDetectorLookbackStaysLinear` in `internal/infrastructure/docparser/anydoc`
+fails if a bump reintroduces it.
+
+Two commits improve extraction rather than bound it, and both change what gets
+indexed: text inside Form XObjects now tracks the text line matrix and honors
+`T*`, `TL`, `'`, `"`, `Tc`, and `Tw`, so nested form text keeps its line breaks
+and spacing instead of running together, and small-caps runs merge instead of
+being read as extra table columns — a spurious column reaches the model as a
+misaligned markdown table.
+
+Nothing else in the range reaches WeKnora, which is worth recording so the next
+upgrade does not go looking for it. The crate published as 0.1.8 was cut at
+`pdf-inspector`'s `packages-2026-08-10` tag, one release behind 1.14.1, so 0.1.9
+formally spans that release too — but its two functional commits both miss this
+binding. One only touches the Node and Python surfaces. The other serves
+invisible (`Tr 3`) OCR text layers instead of reporting `needs_ocr`, which
+sounds like it would spare WeKnora an OCR round trip, except that it landed in
+`extract_text_in_regions`; anydoc uses the positioned-text path, which already
+retried with invisible text included. Scanned PDFs still come back as "OCR is
+required" and still fall back to the docreader.
+
 ## Local modifications

 Keep this list current: it is the diff a future upgrade has to re-apply. Items
 2–4 are bugs in the upstream PR and are worth sending back to it.

-1. `Cargo.toml` — depends on the published `anydoc = "=0.1.8"` crate instead of
+1. `Cargo.toml` — depends on the published `anydoc = "=0.1.9"` crate instead of
   the workspace path dependency, declares its own empty `[workspace]`, and
   repeats the upstream release profile (`lto`, `strip`), which it would
   otherwise inherit from the anydoc workspace.
@@ -72,9 +133,12 @@ Keep this list current: it is the diff a future upgrade has to re-apply. Items
 5. Removed the upstream CLI (`cmd/anydoc`) and the binding test suite, which
   reads fixtures from the anydoc repository. WeKnora's own tests live in
   `internal/infrastructure/docparser/anydoc`.
-6. `scripts/build-anydoc-lib.sh` copies anydoc 0.1.8 to `patched-anydoc/`
-   (gitignored) and re-exports `document_to_markdown`, which is crate-private
-   in the published crate. `src/asset_links.rs` then rewrites `ImageSource::Asset`
+6. `scripts/build-anydoc-lib.sh` copies the pinned anydoc release to
+   `patched-anydoc/` (gitignored) and re-exports `document_to_markdown`, which
+   is crate-private in the published crate. It reads the version from the
+   `anydoc = "=X.Y.Z"` pin above and checks it against `version.go`, so a bump
+   is one line and cannot half-land.
+   `src/asset_links.rs` then rewrites `ImageSource::Asset`
   to `External("images/image-N.ext")` so the official serializer emits in-place
   image links. `anydoc_to_markdown_with_asset_links` is the ABI for that path.

@@ -92,6 +156,11 @@ kills the process with a stack overflow (`RUSTSEC-2026-0187`), which neither
 and the same input comes back as an ordinary error;
 `TestDeeplyNestedPDFFailsWithoutKillingTheProcess` keeps it that way.

+Moving the pin to 0.1.9 added no crate and removed none: `lopdf` stays at 0.42
+and the whole tree below `pdf-inspector` is unchanged, so the lockfile diff is
+three version lines and one checksum. The audit result is therefore the same
+one described below.
+
 `cargo audit` currently reports one allowed warning: `ttf-parser` 0.25.1 is
 unmaintained (`RUSTSEC-2026-0192`), pulled in transitively by the PDF stack. It
 is not a vulnerability and nothing here can fix it, so warnings report without
@@ -3,4 +3,4 @@ package anydoc
 // Version is the version of the anydoc-go module, bumped in lockstep with the
 // Rust crate version (go/Cargo.toml) and the GitHub release tag. The
 // packaged static libraries.
-const Version = "0.1.8"
+const Version = "0.1.9"