mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-10-21 05:59:35 +08:00
Martin Vogel
c1da1924b2
The PDF text layer carries data tables (D3). Their sources and notices are
now in THIRD_PARTY.md (shipped in every archive through
THIRD_PARTY_NOTICES.md) and in the SBOM:
- the Adobe Glyph List subset (BSD-3-Clause, Copyright 2002-2019 Adobe),
- the printable-ASCII advance widths of Helvetica and Times-Roman from
Adobe's core-14 font metrics (the AFM files are not distributed),
- the Unicode Character Database data of pdf_unicode.c (Unicode
License v3, full text included).
scripts/gen-pdf-unicode.py (stdlib CPython only) regenerates
pdf_unicode.c byte for byte with CPython 3.14.6 (Unicode 16.0.0);
pdf_tables.c is maintained data from here on. License texts were taken
from their sources (adobe-type-tools/agl-aglfn, unicode.org).
Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
104 lines
4.2 KiB
Python
104 lines
4.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Generate internal/cbm/pdf/pdf_unicode.c: the Unicode facts the PDF mention
|
|
scanner needs, exactly as CPython sees them (the field test's scanner ran on
|
|
CPython's str and re).
|
|
|
|
usage: scripts/gen-pdf-unicode.py internal/cbm/pdf/pdf_unicode.c (then clang-format -i it)
|
|
The output depends on the CPython version (its Unicode data): the committed
|
|
tables were made with CPython 3.14.6 (Unicode 16.0.0).
|
|
"""
|
|
import sys
|
|
import unicodedata
|
|
|
|
MAXCP = 0x110000
|
|
|
|
|
|
def is_surrogate(c):
|
|
return 0xD800 <= c <= 0xDFFF
|
|
|
|
|
|
def ranges(pred):
|
|
out = []
|
|
start = None
|
|
for c in range(MAXCP):
|
|
ok = not is_surrogate(c) and pred(chr(c))
|
|
if ok and start is None:
|
|
start = c
|
|
elif not ok and start is not None:
|
|
out.append((start, c - 1))
|
|
start = None
|
|
if start is not None:
|
|
out.append((start, MAXCP - 1))
|
|
return out
|
|
|
|
|
|
def emit_ranges(f, name, rs):
|
|
f.write("const pdf_urange_t %s[] = {\n" % name)
|
|
for i in range(0, len(rs), 4):
|
|
f.write(" " + " ".join("{0x%X, 0x%X}," % r for r in rs[i:i + 4]) + "\n")
|
|
f.write("};\nconst int %s_COUNT = (int)(sizeof(%s) / sizeof(%s[0]));\n\n" % (name, name, name))
|
|
|
|
|
|
def main():
|
|
out = sys.argv[1]
|
|
# per-character NFKC
|
|
nfkc = []
|
|
seq = []
|
|
for c in range(MAXCP):
|
|
if is_surrogate(c):
|
|
continue
|
|
ch = chr(c)
|
|
m = unicodedata.normalize("NFKC", ch)
|
|
if m != ch:
|
|
nfkc.append((c, len(seq), len(m)))
|
|
seq.extend(ord(x) for x in m)
|
|
# canonical composition pairs: a + b -> c when NFC composes them
|
|
comp = []
|
|
for c in range(MAXCP):
|
|
if is_surrogate(c):
|
|
continue
|
|
d = unicodedata.decomposition(chr(c))
|
|
if not d or d.startswith("<"):
|
|
continue
|
|
parts = [int(x, 16) for x in d.split()]
|
|
if len(parts) != 2:
|
|
continue
|
|
a, b = parts
|
|
if unicodedata.normalize("NFC", chr(a) + chr(b)) == chr(c):
|
|
comp.append((a, b, c))
|
|
comp.sort()
|
|
with open(out, "w", encoding="utf-8") as f:
|
|
f.write("/*\n * pdf_unicode.c -- Unicode %s facts for the PDF mention scanner (generated).\n"
|
|
" *\n * The field test's scanner ran on CPython's str and re; these tables are what\n"
|
|
" * CPython %s answers: NFKC of each code point, the canonical compositions,\n"
|
|
" * str.isspace, the re module's \\w (str.isalnum or '_'), str.isalpha, and the\n"
|
|
" * one-character str.isupper / str.islower / titlecase. Generated by\n"
|
|
" * scripts/gen-pdf-unicode.py; do not edit by hand. Data of the Unicode\n"
|
|
" * Character Database, (c) Unicode, Inc., Unicode License v3\n"
|
|
" * (THIRD_PARTY.md).\n */\n"
|
|
% (unicodedata.unidata_version, sys.version.split()[0]))
|
|
f.write("#include \"pdf_unicode.h\"\n\n")
|
|
f.write("const pdf_nfkc_t PDF_NFKC[] = {\n")
|
|
for i in range(0, len(nfkc), 4):
|
|
f.write(" " + " ".join("{0x%X, %d, %d}," % e for e in nfkc[i:i + 4]) + "\n")
|
|
f.write("};\nconst int PDF_NFKC_COUNT = (int)(sizeof(PDF_NFKC) / sizeof(PDF_NFKC[0]));\n\n")
|
|
f.write("const uint32_t PDF_NFKC_SEQ[] = {\n")
|
|
for i in range(0, len(seq), 10):
|
|
f.write(" " + " ".join("0x%X," % x for x in seq[i:i + 10]) + "\n")
|
|
f.write("};\n\n")
|
|
f.write("const pdf_ucomp_t PDF_UCOMP[] = {\n")
|
|
for i in range(0, len(comp), 3):
|
|
f.write(" " + " ".join("{0x%X, 0x%X, 0x%X}," % e for e in comp[i:i + 3]) + "\n")
|
|
f.write("};\nconst int PDF_UCOMP_COUNT = (int)(sizeof(PDF_UCOMP) / sizeof(PDF_UCOMP[0]));\n\n")
|
|
emit_ranges(f, "PDF_U_SPACE", ranges(lambda c: c.isspace()))
|
|
emit_ranges(f, "PDF_U_WORD", ranges(lambda c: c.isalnum() or c == "_"))
|
|
emit_ranges(f, "PDF_U_ALPHA", ranges(lambda c: c.isalpha()))
|
|
emit_ranges(f, "PDF_U_UPPER", ranges(lambda c: c.isupper()))
|
|
emit_ranges(f, "PDF_U_LOWER", ranges(lambda c: c.islower()))
|
|
emit_ranges(f, "PDF_U_TITLE", ranges(lambda c: c.istitle() and not c.isupper()))
|
|
print("nfkc", len(nfkc), "seq", len(seq), "comp", len(comp))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|