From 902e8b4396a7927620b6d5a1150844cc61ff253c Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 29 Oct 2026 22:17:47 -0500 Subject: [PATCH] Ship ICU data without the categories no Intl form reads The runtime pack shipped ICU's full icudt78l.dat (33.1 MB), and every Intl executable gets a copy beside it. scr_intl.c answers its Intl forms with ICU's break iterators, number and date formatters, calendars and locale IDs, which never read the charset converters or the collation, language, region, time zone name, unit, spell-out and transliterator trees: 21.5 MB of the file. - The pack build writes icudt78l.dat without those items (ICU_DATA_DROPPED in icu-inputs.mjs). filterIcuData rewrites the data file's table of contents to the remaining items, and the result is checked against a pinned sha256 and size (11,653,984 B). - tests/harness/icu-data-filter.test.ts runs every @requires-icu corpus program under 47 locales and 3 time zones with the pack's data, with the full data and in Node, and fails when the outputs differ, so an Intl form that reads a dropped category fails there. - intl-word-segmenter-dictionaries.ts segments Chinese, Japanese, Thai, Lao, Khmer and Burmese text, which ICU breaks with the dictionaries in brkitr/. --- packages/compiler/src/backend/linker.ts | 2 +- .../compiler/src/backend/runtime-pack-core.ts | 4 +- .../src/backend/runtime-pack-native.ts | 2 +- .../compiler/src/backend/vendor-inputs.ts | 3 +- .../intl-word-segmenter-dictionaries.ts.json | 6 + .../runtime-pack-common/scripts/build.mjs | 7 +- .../runtime-pack-common/scripts/icu-data.mjs | 112 +++++++++- .../scripts/icu-inputs.mjs | 26 +++ .../runtime-pack-common/test/icu-data.test.ts | 80 +++++++ packages/runtime/vendor/README.md | 4 +- .../intl-word-segmenter-dictionaries.ts | 25 +++ .../intl-word-segmenter-dictionaries.ts.json | 1 + tests/harness/icu-data-filter.test.ts | 206 ++++++++++++++++++ 13 files changed, 460 insertions(+), 18 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/intl-word-segmenter-dictionaries.ts.json create mode 100644 packages/runtime-pack-common/test/icu-data.test.ts create mode 100644 tests/corpus/intl-word-segmenter-dictionaries.ts create mode 100644 tests/harness/corpus-costs/intl-word-segmenter-dictionaries.ts.json create mode 100644 tests/harness/icu-data-filter.test.ts diff --git a/packages/compiler/src/backend/linker.ts b/packages/compiler/src/backend/linker.ts index 8d4976403..66a417582 100644 --- a/packages/compiler/src/backend/linker.ts +++ b/packages/compiler/src/backend/linker.ts @@ -235,7 +235,7 @@ async function objectLinkerDependencyPaths( * file next to the binary is what the runtime looks for first. * * A file already present at the pinned size is left alone: the pack ships - * one data set per file name, and re-copying 31.6 MiB on every build would + * one data set per file name, and re-copying 11.1 MiB on every build would * be pure cost. The copy is atomic (private temporary, then rename) so two * concurrent builds of the same program never observe a partial file. */ export async function installRuntimeDataFiles( diff --git a/packages/compiler/src/backend/runtime-pack-core.ts b/packages/compiler/src/backend/runtime-pack-core.ts index 41bfb1236..e9a74a7a6 100644 --- a/packages/compiler/src/backend/runtime-pack-core.ts +++ b/packages/compiler/src/backend/runtime-pack-core.ts @@ -75,7 +75,7 @@ export interface RuntimePackManifest { flavors: Partial>; archives: RuntimePackArchive[]; /** Files the linked executable needs at RUN time rather than at link time, - * installed beside it on every build. ICU's data file (icudt78l.dat, 31.6 + * installed beside it on every build. ICU's data file (icudt78l.dat, 11.1 * MiB) is the only one; Intl programs copy it, every other program links * neither ICU nor its data. */ data?: RuntimePackArtifact[]; @@ -492,7 +492,7 @@ export function selectRuntimePackArtifacts( archives: manifest.archives.filter((archive) => evaluateRuntimePredicate(archive.predicate, features), ), - // ICU's data file is 31.6 MiB: only a program that links ICU installs it. + // ICU's data file is 11.1 MiB: only a program that links ICU installs it. data: features.intl === true ? (manifest.data ?? []) : [], systemLibraries: manifest.system_libraries .filter((entry) => evaluateRuntimePredicate(entry.predicate, features)) diff --git a/packages/compiler/src/backend/runtime-pack-native.ts b/packages/compiler/src/backend/runtime-pack-native.ts index 850258ebb..26968b396 100644 --- a/packages/compiler/src/backend/runtime-pack-native.ts +++ b/packages/compiler/src/backend/runtime-pack-native.ts @@ -429,7 +429,7 @@ export function stageNativeRuntimeSelection( * own selection (backend/linker.ts); this host writes executables * synchronously and has no link plan, so it installs them here. * - * A file already present at the pinned size is left alone (31.6 MiB of pure + * A file already present at the pinned size is left alone (11.1 MiB of pure * cost otherwise), and the copy is atomic — a private temporary, then a * rename — so two concurrent builds of one program never read a partial * file. */ diff --git a/packages/compiler/src/backend/vendor-inputs.ts b/packages/compiler/src/backend/vendor-inputs.ts index 4e87272d8..b03bfc4c1 100644 --- a/packages/compiler/src/backend/vendor-inputs.ts +++ b/packages/compiler/src/backend/vendor-inputs.ts @@ -10,7 +10,8 @@ export const BROTLI_VERSION = "1.2.0"; * (`process.versions.icu` there), from the `release-78.3` tag. ICU is not * vendored: the build fetches the release's own source tarball and data zip * from unicode-org/icu's GitHub release, pinning each by version and sha256, - * and caches the download. Its data file icudt78l.dat (33,107,232 B) ships + * and caches the download. The runtime packs ship its data file icudt78l.dat + * (33,107,232 B) without the categories no Intl form reads (11,653,984 B), * beside the executable; see packages/runtime-pack-common/scripts/ * icu-data.mjs. */ export const ICU_VERSION = "78.3"; diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/intl-word-segmenter-dictionaries.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/intl-word-segmenter-dictionaries.ts.json new file mode 100644 index 000000000..31197f414 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/intl-word-segmenter-dictionaries.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/intl-word-segmenter-dictionaries.ts" + ], + "diags": [] +} diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index c4191fec6..c3cf2b7b0 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -20,7 +20,7 @@ import { installRuntimePack, withBuildLock } from "./build-state.mjs"; import { shareIdenticalArtifacts } from "./share-artifacts.mjs"; import { parallelMap } from "./parallel-map.mjs"; import { zlibInputs } from "./zlib-inputs.mjs"; -import { ICU_DATA_FILE, ensureIcuData, ensureIcuSources, icuSourceList } from "./icu-data.mjs"; +import { ICU_DATA_FILE, ensurePackIcuData, ensureIcuSources, icuSourceList } from "./icu-data.mjs"; const run = promisify(execFile); @@ -668,11 +668,12 @@ async function build() { // ICU's data is a RUN-time input, not a link input: the executable loads // icudt78l.dat from beside itself (or ICU_DATA). `data` tells the // compiler where the pack keeps it, so every Intl build installs it - // beside the binary it links. + // beside the binary it links. The pack's copy leaves out the categories + // no Intl form reads (ICU_DATA_DROPPED in icu-inputs.mjs). const data = []; if (requestedArchives.has("icu")) { process.stdout.write(`building ${packageManifest.name} icu data\n`); - const source = await ensureIcuData( + const source = await ensurePackIcuData( join(repoRoot, "node_modules", ".cache", "scriptc-icu-data"), ); const destination = join(stagedOutputRoot, "icu", ICU_DATA_FILE); diff --git a/packages/runtime-pack-common/scripts/icu-data.mjs b/packages/runtime-pack-common/scripts/icu-data.mjs index 2d5e14e38..7d1e2c704 100644 --- a/packages/runtime-pack-common/scripts/icu-data.mjs +++ b/packages/runtime-pack-common/scripts/icu-data.mjs @@ -2,8 +2,11 @@ * * Intl answers from ICU, so the pack builds ICU's C library from the release * source tarball and ships its data as a FILE the executable loads at run - * time (31.6 MiB of CLDR patterns, locale metadata, break rules and - * time-zone shapes) — never embedded in the binary. Neither is vendored: + * time — never embedded in the binary. The release's data file is 31.6 MiB; + * the pack's copy keeps the locale data, break rules and dictionaries, + * currency data and time-zone rules the Intl forms read, 11.1 MiB, and + * leaves out the categories in ICU_DATA_DROPPED (filterIcuData below, + * pinned by ICU_PACK_DATA_SHA256). Neither input is vendored: * both come from unicode-org/icu's official GitHub release for the pinned * version, each downloaded once into a cache and reused after that, and each * verified against a pinned sha256 on every use (the source by the tarball's, @@ -11,9 +14,10 @@ * here with both hashes, never later in a build. Offline builds work from a * pre-populated cache. * - * The pack build drops `icudt78l.dat` into artifacts/icu/ and lists it in - * runtime-pack.json's `data`, so the compiler can install it beside every - * executable that links ICU (packages/compiler/src/backend/linker.ts). The + * The pack build writes that copy of `icudt78l.dat` to artifacts/icu/ and + * lists it in runtime-pack.json's `data`, so the compiler can install it + * beside every executable that links ICU (packages/compiler/src/backend/ + * linker.ts). The * data zip is unpacked by the small reader below rather than a dependency: * it holds three stored/deflated members and nothing else. The source * tarball is unpacked by the system `tar`, as the libnode build does. @@ -21,7 +25,7 @@ import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { mkdir, mkdtemp, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { promisify } from "node:util"; import { inflateRawSync, crc32 } from "node:zlib"; import { @@ -31,6 +35,9 @@ import { ICU_DATA_URL, ICU_DATA_ZIP_SHA256, ICU_DATA_ZIP_SIZE, + ICU_DATA_DROPPED, + ICU_PACK_DATA_SHA256, + ICU_PACK_DATA_SIZE, ICU_SOURCE_SHA256, ICU_SOURCE_SIZE, ICU_SOURCE_URL, @@ -180,7 +187,7 @@ export function unzipMember(zip, name) { export async function ensureIcuData(cacheDir) { const zipPath = join(cacheDir, `icu4c-${ICU_DATA_VERSION}-data-bin-l.zip`); const dataPath = join(cacheDir, `${ICU_DATA_VERSION}`, ICU_DATA_FILE); - if (await validFile(dataPath, ICU_DATA_SHA256)) return dataPath; + if (await validFile(dataPath, ICU_DATA_SHA256, ICU_DATA_SIZE)) return dataPath; let zip; try { zip = await readFile(zipPath); @@ -211,11 +218,98 @@ export async function ensureIcuData(cacheDir) { return dataPath; } -async function validFile(path, expectedSha256) { +async function validFile(path, expectedSha256, expectedSize) { try { - if ((await stat(path)).size !== ICU_DATA_SIZE) return false; + if ((await stat(path)).size !== expectedSize) return false; return sha256(await readFile(path)) === expectedSha256; } catch { return false; } } + +/** The items of an ICU common data file (a `.dat`: a header, then a table of + * contents of name and data offsets, sorted by name, then the items): each + * item's name (`icudt78l/coll/de.res`) and its bytes. */ +export function icuDataItems(bytes) { + const headerSize = bytes.readUInt16LE(0); + if (bytes[2] !== 0xda || bytes[3] !== 0x27) throw new Error("not an ICU data file"); + const count = bytes.readUInt32LE(headerSize); + const entries = []; + for (let i = 0; i < count; i++) { + const nameStart = headerSize + bytes.readUInt32LE(headerSize + 4 + i * 8); + const nameEnd = bytes.indexOf(0, nameStart); + entries.push({ + name: bytes.toString("latin1", nameStart, nameEnd), + start: headerSize + bytes.readUInt32LE(headerSize + 8 + i * 8), + }); + } + return entries.map((entry, i) => ({ + name: entry.name, + bytes: bytes.subarray( + entry.start, + i + 1 < entries.length ? entries[i + 1].start : bytes.length, + ), + })); +} + +/** Whether a data item falls under one of the ICU_DATA_DROPPED patterns. */ +export function icuDataItemDropped(name, dropped = ICU_DATA_DROPPED) { + const path = name.slice(name.indexOf("/") + 1); + return dropped.some((pattern) => + pattern.startsWith("*.") + ? !path.includes("/") && path.endsWith(pattern.slice(1)) + : path.startsWith(pattern), + ); +} + +/** An ICU common data file holding the items of `bytes` that `keep` accepts, + * in their order, laid out as icupkg writes one: the original header, the + * table of contents, the names, then each item at a 16-byte boundary. ICU + * finds items by a binary search of the sorted table, so a subset of a valid + * file in its order is a valid file. */ +export function filterIcuData(bytes, keep) { + const headerSize = bytes.readUInt16LE(0); + const items = icuDataItems(bytes).filter((item) => keep(item.name)); + const names = items.map((item) => Buffer.from(`${item.name}\0`, "latin1")); + const tocSize = 4 + items.length * 8; + const align = (n) => (n + 15) & ~15; + let nameOffset = tocSize; + let dataOffset = align(tocSize + names.reduce((sum, name) => sum + name.length, 0)); + const layout = items.map((item, i) => { + const entry = { nameOffset, dataOffset }; + nameOffset += names[i].length; + dataOffset = align(dataOffset + item.bytes.length); + return entry; + }); + const out = Buffer.alloc(headerSize + dataOffset); + bytes.copy(out, 0, 0, headerSize); + out.writeUInt32LE(items.length, headerSize); + items.forEach((item, i) => { + out.writeUInt32LE(layout[i].nameOffset, headerSize + 4 + i * 8); + out.writeUInt32LE(layout[i].dataOffset, headerSize + 8 + i * 8); + names[i].copy(out, headerSize + layout[i].nameOffset); + item.bytes.copy(out, headerSize + layout[i].dataOffset); + }); + return out; +} + +/** The runtime pack's data file: the pinned icudt78l.dat without the + * ICU_DATA_DROPPED items, written into `cacheDir` once and verified against + * its own pin on every call. Returns its path. */ +export async function ensurePackIcuData(cacheDir) { + const dataPath = join(cacheDir, `${ICU_DATA_VERSION}-pack`, ICU_DATA_FILE); + if (await validFile(dataPath, ICU_PACK_DATA_SHA256, ICU_PACK_DATA_SIZE)) return dataPath; + const full = await readFile(await ensureIcuData(cacheDir)); + const bytes = filterIcuData(full, (name) => !icuDataItemDropped(name)); + if (bytes.length !== ICU_PACK_DATA_SIZE || sha256(bytes) !== ICU_PACK_DATA_SHA256) + throw new Error( + `filtered ICU data is ${bytes.length} bytes with sha256 ${sha256(bytes)}, ` + + `expected ${ICU_PACK_DATA_SIZE} bytes with sha256 ${ICU_PACK_DATA_SHA256} ` + + "(update ICU_PACK_DATA_SHA256 and ICU_PACK_DATA_SIZE in icu-inputs.mjs with ICU_DATA_DROPPED)", + ); + await mkdir(dirname(dataPath), { recursive: true }); + const temporary = `${dataPath}.${process.pid}.tmp`; + await writeFile(temporary, bytes); + await rename(temporary, dataPath); + return dataPath; +} diff --git a/packages/runtime-pack-common/scripts/icu-inputs.mjs b/packages/runtime-pack-common/scripts/icu-inputs.mjs index 0aef3ba37..7dfe8379d 100644 --- a/packages/runtime-pack-common/scripts/icu-inputs.mjs +++ b/packages/runtime-pack-common/scripts/icu-inputs.mjs @@ -22,3 +22,29 @@ export const ICU_DATA_ZIP_SIZE = 12_836_659; export const ICU_DATA_FILE = "icudt78l.dat"; export const ICU_DATA_SHA256 = "d5cf2a40dccbe471781ec7af85693bff542ff12f0b670c9630c4e72d60714b8b"; export const ICU_DATA_SIZE = 33_107_232; + +/** The data categories the runtime pack's copy of icudt78l.dat leaves out: + * item-name prefixes under `icudt78l/` (a trailing `/` names a tree) and + * `*.cnv` for the charset converters. scr_intl.c answers its Intl forms with + * ICU's break iterators, number and date formatters, calendars and locale + * IDs, which read the locale data, `brkitr/` and `curr/` and never these: + * converters, collation, display names of languages, regions and time + * zones, unit names, spell-out rules and transliterators. An Intl form that + * needs one of them removes it from this list and updates the two pins + * below in the same change. tests/harness/icu-data-filter.test.ts runs every + * `@requires-icu` corpus program with the pack's file and with the full one + * and fails when their outputs differ. */ +export const ICU_DATA_DROPPED = [ + "*.cnv", + "coll/", + "lang/", + "rbnf/", + "region/", + "translit/", + "unit/", + "zone/", +]; +/** icudt78l.dat without the ICU_DATA_DROPPED items (icu-data.mjs, filterIcuData). */ +export const ICU_PACK_DATA_SHA256 = + "513a72821c8ee68c7e8a24c346cadc2f8de22efaa19fb68c14957f313e74f35b"; +export const ICU_PACK_DATA_SIZE = 11_653_984; diff --git a/packages/runtime-pack-common/test/icu-data.test.ts b/packages/runtime-pack-common/test/icu-data.test.ts new file mode 100644 index 000000000..43c524f31 --- /dev/null +++ b/packages/runtime-pack-common/test/icu-data.test.ts @@ -0,0 +1,80 @@ +import { expect, test } from "vitest"; +import { filterIcuData, icuDataItemDropped, icuDataItems } from "../scripts/icu-data.mjs"; + +/** A common data file in ICU's layout: a 32-byte header with the 0xda27 + * magic, the table of contents, the names, and each item at a 16-byte + * boundary (padded with 0xaa, as ICU's tools pad). */ +function dataFile(items: [string, string][]): Buffer { + const header = Buffer.alloc(32); + header.writeUInt16LE(32, 0); + header[2] = 0xda; + header[3] = 0x27; + const names = Buffer.concat(items.map(([name]) => Buffer.from(`${name}\0`, "latin1"))); + const tocSize = 4 + items.length * 8; + const toc = Buffer.alloc(tocSize); + toc.writeUInt32LE(items.length, 0); + let nameOffset = tocSize; + let offset = (tocSize + names.length + 15) & ~15; + const body: Buffer[] = []; + items.forEach(([name, contents], i) => { + toc.writeUInt32LE(nameOffset, 4 + i * 8); + toc.writeUInt32LE(offset, 8 + i * 8); + nameOffset += name.length + 1; + const padded = Buffer.alloc((contents.length + 15) & ~15, 0xaa); + padded.write(contents, "latin1"); + body.push(padded); + offset += padded.length; + }); + const lead = Buffer.alloc(((tocSize + names.length + 15) & ~15) - tocSize - names.length, 0xaa); + return Buffer.concat([header, toc, names, lead, ...body]); +} + +const items: [string, string][] = [ + ["icudt78l/brkitr/word.brk", "word rules"], + ["icudt78l/coll/de.res", "collation"], + ["icudt78l/curr/de.res", "currencies"], + ["icudt78l/de.res", "locale"], + ["icudt78l/ibm-1047_P100-1995.cnv", "converter"], + ["icudt78l/zone/de.res", "zone names"], + ["icudt78l/zoneinfo64.res", "time zones"], +]; + +test("the dropped categories match trees and root-level converters only", () => { + const dropped = items.map(([name]) => icuDataItemDropped(name)); + expect(dropped).toEqual([false, true, false, false, true, true, false]); + expect(icuDataItemDropped("icudt78l/brkitr/x.cnv")).toBe(false); +}); + +test("a filtered file keeps the accepted items' names, order and bytes", () => { + const file = dataFile(items); + const filtered = filterIcuData(file, (name) => !icuDataItemDropped(name)); + expect(filtered.subarray(0, 32).equals(file.subarray(0, 32))).toBe(true); + const kept = icuDataItems(filtered); + expect(kept.map((item) => item.name)).toEqual([ + "icudt78l/brkitr/word.brk", + "icudt78l/curr/de.res", + "icudt78l/de.res", + "icudt78l/zoneinfo64.res", + ]); + for (const item of kept) { + const [, contents] = items.find(([name]) => name === item.name)!; + expect(item.bytes.subarray(0, contents.length).toString("latin1")).toBe(contents); + } + for (const [i, item] of kept.entries()) + expect(filtered.readUInt32LE(32 + 8 + i * 8) % 16, item.name).toBe(0); +}); + +test("keeping every item reproduces each item", () => { + const file = dataFile(items); + const same = icuDataItems(filterIcuData(file, () => true)); + const original = icuDataItems(file); + expect(same.length).toBe(original.length); + for (const [i, item] of same.entries()) { + expect(item.name).toBe(original[i]!.name); + expect(item.bytes.subarray(0, 10).equals(original[i]!.bytes.subarray(0, 10))).toBe(true); + } +}); + +test("a file without ICU's magic is refused", () => { + expect(() => icuDataItems(Buffer.alloc(64))).toThrow("not an ICU data file"); +}); diff --git a/packages/runtime/vendor/README.md b/packages/runtime/vendor/README.md index e032f1a53..228ea5233 100644 --- a/packages/runtime/vendor/README.md +++ b/packages/runtime/vendor/README.md @@ -91,7 +91,9 @@ The locale/CLDR engine behind the Intl forms only it can answer (see packages/ru ICU is not vendored: 970 source files plus 33 MiB of compiled CLDR tables are a download, not a reviewable tree in this repository. Both build inputs come from unicode-org/icu's official GitHub release for the pinned version, and both are pinned by version and sha256 in packages/compiler/src/backend/vendor-inputs.ts (mirrored in packages/runtime-pack-common/scripts/icu-inputs.mjs for the runtime packs, with a unit test that keeps the two identical): - `icu4c-78.3-sources.tgz` (27,977,255 B, sha256 `3a2e7a47604ba702f345878308e6fefeca612ee895cf4a5f222e7955fabfe0c0`), the release's own source tarball: `common/`, `i18n/` and `stubdata/` (the empty data-symbol stub ICU needs when its data ships outside the archive), plus LICENSE. `source/common/sources.txt` and `source/i18n/sources.txt` are the *lists of record*: a build compiles exactly the units ICU's own autoconf build compiles, so `icuSourceList` (vendor-inputs.ts for the source-build path, packages/runtime-pack-common/scripts/icu-data.mjs for the runtime packs) reads them rather than re-deriving the set. -- `icu4c-78.3-data-bin-l.zip` (12,836,659 B, sha256 `982619632b78887f1895b063e96e8c3cc7f99283337c8abbd05aa71635de613c`), whose `icudt78l.dat` (33,107,232 B, sha256 `d5cf2a40dccbe471781ec7af85693bff542ff12f0b670c9630c4e72d60714b8b`) is not compiled: it is shipped as `artifacts/icu/icudt78l.dat` and installed beside every executable that links ICU, where the runtime looks first (`$ORIGIN`-style: the executable's own directory), then `ICU_DATA`. +- `icu4c-78.3-data-bin-l.zip` (12,836,659 B, sha256 `982619632b78887f1895b063e96e8c3cc7f99283337c8abbd05aa71635de613c`), whose `icudt78l.dat` (33,107,232 B, sha256 `d5cf2a40dccbe471781ec7af85693bff542ff12f0b670c9630c4e72d60714b8b`) is not compiled. The pack build writes a copy without the data categories no Intl form reads, ships it as `artifacts/icu/icudt78l.dat` and installs it beside every executable that links ICU, where the runtime looks first (`$ORIGIN`-style: the executable's own directory), then `ICU_DATA`. + +The pack's copy of `icudt78l.dat` (11,653,984 B, sha256 `513a72821c8ee68c7e8a24c346cadc2f8de22efaa19fb68c14957f313e74f35b`) leaves out the items `ICU_DATA_DROPPED` in icu-inputs.mjs names: the charset converters (`*.cnv`) and the `coll/`, `lang/`, `rbnf/`, `region/`, `translit/`, `unit/` and `zone/` trees, 21.5 MB that scr_intl.c's break iterators, number and date formatters, calendars and locale IDs never read. `filterIcuData` in icu-data.mjs rewrites the data file's table of contents to the remaining items, and the build checks the result against the pinned sha256 and size. tests/harness/icu-data-filter.test.ts runs every `@requires-icu` corpus program under 47 locales and 3 time zones with the pack's data, with the full data and in Node, and fails when the outputs differ. An Intl form that reads a dropped category (time zone names, display names, collation, unit names) removes it from `ICU_DATA_DROPPED` and updates the two pins in the same change. Each download is cached (the runtime-pack build under `node_modules/.cache/scriptc-icu-*`, the source-build path under the per-user build cache's `vendor/`), verified against its pinned sha256 on every use, and unpacked by the small zip reader in icu-data.mjs (the data) or the system `tar` (the sources). A hash mismatch fails the build with both hashes; an offline build works from a pre-populated cache. To update, bump `ICU_VERSION` with the new release's two assets' sizes and sha256 in vendor-inputs.ts and icu-inputs.mjs. diff --git a/tests/corpus/intl-word-segmenter-dictionaries.ts b/tests/corpus/intl-word-segmenter-dictionaries.ts new file mode 100644 index 000000000..cbb5034c5 --- /dev/null +++ b/tests/corpus/intl-word-segmenter-dictionaries.ts @@ -0,0 +1,25 @@ +// @requires-icu +// @env: LC_ALL=en_US.UTF-8 +// Word segmentation of scripts written without spaces: Chinese, Japanese, +// Thai, Lao, Khmer and Burmese. ICU breaks them with the dictionaries in its +// data file (brkitr/*.dict), as Node does; each line lists a text's word-like +// segments and the boundaries of all its segments. +const texts = [ + "我们今天去北京大学参观博物馆。", + "東京都の天気は晴れです。日本語の文章を単語に分けます。", + "ภาษาไทยเป็นภาษาที่ไม่มีการเว้นวรรคระหว่างคำ", + "ພາສາລາວເປັນພາສາທີ່ສວຍງາມ", + "ភាសាខ្មែរគឺជាភាសាផ្លូវការរបស់ប្រទេសកម្ពុជា", + "မြန်မာဘာသာစကားသည်မြန်မာနိုင်ငံ၏ရုံးသုံးဘာသာဖြစ်သည်", + "한국어는 띄어쓰기를 합니다.", +]; +const segmenter = new Intl.Segmenter(undefined, { granularity: "word" }); +for (const text of texts) { + const words: string[] = []; + const boundaries: number[] = []; + for (const { segment, index, isWordLike } of segmenter.segment(text)) { + boundaries.push(index); + if (isWordLike === true) words.push(segment); + } + console.log(words.join("|"), boundaries.join(",")); +} diff --git a/tests/harness/corpus-costs/intl-word-segmenter-dictionaries.ts.json b/tests/harness/corpus-costs/intl-word-segmenter-dictionaries.ts.json new file mode 100644 index 000000000..b18912827 --- /dev/null +++ b/tests/harness/corpus-costs/intl-word-segmenter-dictionaries.ts.json @@ -0,0 +1 @@ +{ "plain": 8690, "san": 67970 } diff --git a/tests/harness/icu-data-filter.test.ts b/tests/harness/icu-data-filter.test.ts new file mode 100644 index 000000000..2e3788124 --- /dev/null +++ b/tests/harness/icu-data-filter.test.ts @@ -0,0 +1,206 @@ +/* The runtime pack ships icudt78l.dat without the categories no Intl form + * reads (ICU_DATA_DROPPED in packages/runtime-pack-common/scripts/ + * icu-inputs.mjs). Every `@requires-icu` corpus program runs under a sweep + * of locales and time zones three ways: with the pack's data beside it, with + * ICU's full data file beside it, and in Node. The three outputs must be + * equal. A later Intl form that reads a dropped category gets ICU's root + * fallback or a missing-data error from the pack's file and fails here; + * that change removes the category from ICU_DATA_DROPPED and updates the + * pinned sha256 and size beside it. */ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { copyFileSync, globSync, mkdirSync, readFileSync, rmSync } from "node:fs"; +import { availableParallelism } from "node:os"; +import { basename, dirname, join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { afterAll, describe, expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; + +const execFileAsync = promisify(execFile); +const repoRoot = join(import.meta.dirname, "../.."); +const corpusDir = join(repoRoot, "tests/corpus"); +const workDir = join(repoRoot, "node_modules/.cache/scriptc-tests/icu-data-filter"); + +const icuData = (await import( + pathToFileURL(join(repoRoot, "packages/runtime-pack-common/scripts/icu-data.mjs")).href +)) as { + ensureIcuData(cacheDir: string): Promise; + filterIcuData(bytes: Buffer, keep: (name: string) => boolean): Buffer; + icuDataItemDropped(name: string): boolean; +}; +const icuInputs = (await import( + pathToFileURL(join(repoRoot, "packages/runtime-pack-common/scripts/icu-inputs.mjs")).href +)) as { ICU_DATA_DROPPED: string[]; ICU_PACK_DATA_SHA256: string }; + +const LOCALES = [ + "en_US.UTF-8", + "de_DE.UTF-8", + "fr_FR.UTF-8", + "fr_CA.UTF-8", + "es_MX.UTF-8", + "pt_BR.UTF-8", + "it_IT.UTF-8", + "nl_NL.UTF-8", + "sv_SE.UTF-8", + "fi_FI.UTF-8", + "pl_PL.UTF-8", + "ru_RU.UTF-8", + "uk_UA.UTF-8", + "tr_TR.UTF-8", + "el_GR.UTF-8", + "he_IL.UTF-8", + "ar_EG.UTF-8", + "ar_SA.UTF-8", + "fa_IR.UTF-8", + "ur_PK.UTF-8", + "hi_IN.UTF-8", + "bn_BD.UTF-8", + "mr_IN.UTF-8", + "ne_NP.UTF-8", + "ta_IN.UTF-8", + "th_TH.UTF-8", + "lo_LA.UTF-8", + "km_KH.UTF-8", + "my_MM.UTF-8", + "ja_JP.UTF-8", + "ko_KR.UTF-8", + "zh_CN.UTF-8", + "zh_TW.UTF-8", + "zh_HK.UTF-8", + "vi_VN.UTF-8", + "id_ID.UTF-8", + "am_ET.UTF-8", + "sw_KE.UTF-8", + "ps_AF.UTF-8", + "sr_RS.UTF-8", + "sr_RS@latin", + "ca_ES@valencia", + "de_CH.UTF-8", + "en_IN.UTF-8", + "en_GB.UTF-8", + "C.UTF-8", + "POSIX", +]; +const TIME_ZONES = ["UTC", "Asia/Kolkata", "America/New_York"]; + +const programs = [...globSync(join(corpusDir, "*.ts")), ...globSync(join(corpusDir, "*/main.ts"))] + .filter((file) => + readFileSync(file, "utf8") + .split("\n", 2) + .some((line) => /^\/\/ @requires-icu\s*$/.test(line)), + ) + .sort(); + +/** The program's `// @env:` variables other than TZ and LC_ALL, which the + * sweep sets. */ +function programEnv(file: string): Record { + const env: Record = {}; + for (const line of readFileSync(file, "utf8").split("\n", 2)) { + const m = /^\/\/ @env:\s*(.+?)\s*$/.exec(line); + if (!m) continue; + for (const pair of m[1]!.split(",")) { + const eq = pair.indexOf("="); + if (eq > 0) env[pair.slice(0, eq).trim()] = pair.slice(eq + 1).trim(); + } + } + delete env["TZ"]; + delete env["LC_ALL"]; + return env; +} + +async function run(cmd: string, args: string[], env: Record): Promise { + try { + const { stdout, stderr } = await execFileAsync(cmd, args, { + env: { ...process.env, LANG: "", LC_MESSAGES: "", ...env }, + maxBuffer: 16 << 20, + }); + return `${stdout}\n--- stderr\n${stderr}`; + } catch (error) { + const e = error as { stdout?: string; stderr?: string; code?: number }; + return `${e.stdout ?? ""}\n--- stderr\n${e.stderr ?? ""}\n--- exit ${e.code}`; + } +} + +/** Runs `tasks` with at most `limit` in flight. */ +async function pool(tasks: (() => Promise)[], limit: number): Promise { + const results: T[] = Array.from({ length: tasks.length }); + let next = 0; + await Promise.all( + Array.from({ length: Math.min(limit, tasks.length) }, async () => { + while (next < tasks.length) { + const i = next++; + results[i] = await tasks[i]!(); + } + }), + ); + return results; +} + +const skip = process.platform !== "linux" || process.arch !== "x64"; + +describe.skipIf(skip)("the runtime pack's filtered ICU data", () => { + afterAll(() => rmSync(workDir, { recursive: true, force: true })); + + test("is the pinned full data file without the dropped categories", async () => { + mkdirSync(workDir, { recursive: true }); + const full = readFileSync( + await icuData.ensureIcuData(join(repoRoot, "node_modules/.cache/scriptc-icu-data")), + ); + const filtered = icuData.filterIcuData(full, (name) => !icuData.icuDataItemDropped(name)); + expect(createHash("sha256").update(filtered).digest("hex")).toBe( + icuInputs.ICU_PACK_DATA_SHA256, + ); + expect(icuInputs.ICU_DATA_DROPPED.length).toBeGreaterThan(0); + }, 120_000); + + test.for(programs.map((file) => [file.slice(corpusDir.length + 1), file] as const))( + "%s gives the same output with the pack's data, the full data and Node", + async ([rel, file]) => { + const out = join(workDir, rel.replace(/[/.]/g, "_")); + const result = await compile(file, { + outPath: join(out, "filtered", "program"), + outDir: join(out, "filtered"), + }); + expect(result.ok, !result.ok ? JSON.stringify(result.diagnostics, null, 2) : "").toBe(true); + if (!result.ok) return; + const installed = readFileSync(join(dirname(result.binaryPath), "icudt78l.dat")); + expect(createHash("sha256").update(installed).digest("hex")).toBe( + icuInputs.ICU_PACK_DATA_SHA256, + ); + // The same executable with ICU's full data file beside it, where the + // runtime looks first. + const fullBinary = join(out, "full", basename(result.binaryPath)); + mkdirSync(dirname(fullBinary), { recursive: true }); + copyFileSync(result.binaryPath, fullBinary); + copyFileSync( + await icuData.ensureIcuData(join(repoRoot, "node_modules/.cache/scriptc-icu-data")), + join(dirname(fullBinary), "icudt78l.dat"), + ); + const env = programEnv(file); + const cases = TIME_ZONES.flatMap((tz) => + LOCALES.map((locale) => ({ ...env, TZ: tz, LC_ALL: locale })), + ); + const outputs = await pool( + cases.map((caseEnv) => async () => ({ + env: `TZ=${caseEnv.TZ} LC_ALL=${caseEnv.LC_ALL}`, + filtered: await run(result.binaryPath, [], caseEnv), + full: await run(fullBinary, [], caseEnv), + node: await run(process.execPath, [file], caseEnv), + })), + Math.max(2, availableParallelism() >> 1), + ); + for (const o of outputs) { + expect( + o.filtered, + `${rel} under ${o.env}: the pack's ICU data answers differently from the full data; ` + + `an Intl form reads a category in ICU_DATA_DROPPED (${icuInputs.ICU_DATA_DROPPED.join(", ")})`, + ).toBe(o.full); + expect(o.full, `${rel} under ${o.env}: scriptc with the full ICU data and Node`).toBe( + o.node, + ); + } + }, + 600_000, + ); +});