diff --git a/packages/compiler/src/backend/linker.ts b/packages/compiler/src/backend/linker.ts index 8d4976403..66a417582 100644 --- a/packages/compiler/src/backend/linker.ts +++ b/packages/compiler/src/backend/linker.ts @@ -235,7 +235,7 @@ async function objectLinkerDependencyPaths( * file next to the binary is what the runtime looks for first. * * A file already present at the pinned size is left alone: the pack ships - * one data set per file name, and re-copying 31.6 MiB on every build would + * one data set per file name, and re-copying 11.1 MiB on every build would * be pure cost. The copy is atomic (private temporary, then rename) so two * concurrent builds of the same program never observe a partial file. */ export async function installRuntimeDataFiles( diff --git a/packages/compiler/src/backend/runtime-pack-core.ts b/packages/compiler/src/backend/runtime-pack-core.ts index 41bfb1236..e9a74a7a6 100644 --- a/packages/compiler/src/backend/runtime-pack-core.ts +++ b/packages/compiler/src/backend/runtime-pack-core.ts @@ -75,7 +75,7 @@ export interface RuntimePackManifest { flavors: Partial>; archives: RuntimePackArchive[]; /** Files the linked executable needs at RUN time rather than at link time, - * installed beside it on every build. ICU's data file (icudt78l.dat, 31.6 + * installed beside it on every build. ICU's data file (icudt78l.dat, 11.1 * MiB) is the only one; Intl programs copy it, every other program links * neither ICU nor its data. */ data?: RuntimePackArtifact[]; @@ -492,7 +492,7 @@ export function selectRuntimePackArtifacts( archives: manifest.archives.filter((archive) => evaluateRuntimePredicate(archive.predicate, features), ), - // ICU's data file is 31.6 MiB: only a program that links ICU installs it. + // ICU's data file is 11.1 MiB: only a program that links ICU installs it. data: features.intl === true ? (manifest.data ?? []) : [], systemLibraries: manifest.system_libraries .filter((entry) => evaluateRuntimePredicate(entry.predicate, features)) diff --git a/packages/compiler/src/backend/runtime-pack-native.ts b/packages/compiler/src/backend/runtime-pack-native.ts index 850258ebb..26968b396 100644 --- a/packages/compiler/src/backend/runtime-pack-native.ts +++ b/packages/compiler/src/backend/runtime-pack-native.ts @@ -429,7 +429,7 @@ export function stageNativeRuntimeSelection( * own selection (backend/linker.ts); this host writes executables * synchronously and has no link plan, so it installs them here. * - * A file already present at the pinned size is left alone (31.6 MiB of pure + * A file already present at the pinned size is left alone (11.1 MiB of pure * cost otherwise), and the copy is atomic — a private temporary, then a * rename — so two concurrent builds of one program never read a partial * file. */ diff --git a/packages/compiler/src/backend/vendor-inputs.ts b/packages/compiler/src/backend/vendor-inputs.ts index 4e87272d8..b03bfc4c1 100644 --- a/packages/compiler/src/backend/vendor-inputs.ts +++ b/packages/compiler/src/backend/vendor-inputs.ts @@ -10,7 +10,8 @@ export const BROTLI_VERSION = "1.2.0"; * (`process.versions.icu` there), from the `release-78.3` tag. ICU is not * vendored: the build fetches the release's own source tarball and data zip * from unicode-org/icu's GitHub release, pinning each by version and sha256, - * and caches the download. Its data file icudt78l.dat (33,107,232 B) ships + * and caches the download. The runtime packs ship its data file icudt78l.dat + * (33,107,232 B) without the categories no Intl form reads (11,653,984 B), * beside the executable; see packages/runtime-pack-common/scripts/ * icu-data.mjs. */ export const ICU_VERSION = "78.3"; diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/intl-word-segmenter-dictionaries.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/intl-word-segmenter-dictionaries.ts.json new file mode 100644 index 000000000..31197f414 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/intl-word-segmenter-dictionaries.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/intl-word-segmenter-dictionaries.ts" + ], + "diags": [] +} diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index c4191fec6..c3cf2b7b0 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -20,7 +20,7 @@ import { installRuntimePack, withBuildLock } from "./build-state.mjs"; import { shareIdenticalArtifacts } from "./share-artifacts.mjs"; import { parallelMap } from "./parallel-map.mjs"; import { zlibInputs } from "./zlib-inputs.mjs"; -import { ICU_DATA_FILE, ensureIcuData, ensureIcuSources, icuSourceList } from "./icu-data.mjs"; +import { ICU_DATA_FILE, ensurePackIcuData, ensureIcuSources, icuSourceList } from "./icu-data.mjs"; const run = promisify(execFile); @@ -668,11 +668,12 @@ async function build() { // ICU's data is a RUN-time input, not a link input: the executable loads // icudt78l.dat from beside itself (or ICU_DATA). `data` tells the // compiler where the pack keeps it, so every Intl build installs it - // beside the binary it links. + // beside the binary it links. The pack's copy leaves out the categories + // no Intl form reads (ICU_DATA_DROPPED in icu-inputs.mjs). const data = []; if (requestedArchives.has("icu")) { process.stdout.write(`building ${packageManifest.name} icu data\n`); - const source = await ensureIcuData( + const source = await ensurePackIcuData( join(repoRoot, "node_modules", ".cache", "scriptc-icu-data"), ); const destination = join(stagedOutputRoot, "icu", ICU_DATA_FILE); diff --git a/packages/runtime-pack-common/scripts/icu-data.mjs b/packages/runtime-pack-common/scripts/icu-data.mjs index 2d5e14e38..7d1e2c704 100644 --- a/packages/runtime-pack-common/scripts/icu-data.mjs +++ b/packages/runtime-pack-common/scripts/icu-data.mjs @@ -2,8 +2,11 @@ * * Intl answers from ICU, so the pack builds ICU's C library from the release * source tarball and ships its data as a FILE the executable loads at run - * time (31.6 MiB of CLDR patterns, locale metadata, break rules and - * time-zone shapes) — never embedded in the binary. Neither is vendored: + * time — never embedded in the binary. The release's data file is 31.6 MiB; + * the pack's copy keeps the locale data, break rules and dictionaries, + * currency data and time-zone rules the Intl forms read, 11.1 MiB, and + * leaves out the categories in ICU_DATA_DROPPED (filterIcuData below, + * pinned by ICU_PACK_DATA_SHA256). Neither input is vendored: * both come from unicode-org/icu's official GitHub release for the pinned * version, each downloaded once into a cache and reused after that, and each * verified against a pinned sha256 on every use (the source by the tarball's, @@ -11,9 +14,10 @@ * here with both hashes, never later in a build. Offline builds work from a * pre-populated cache. * - * The pack build drops `icudt78l.dat` into artifacts/icu/ and lists it in - * runtime-pack.json's `data`, so the compiler can install it beside every - * executable that links ICU (packages/compiler/src/backend/linker.ts). The + * The pack build writes that copy of `icudt78l.dat` to artifacts/icu/ and + * lists it in runtime-pack.json's `data`, so the compiler can install it + * beside every executable that links ICU (packages/compiler/src/backend/ + * linker.ts). The * data zip is unpacked by the small reader below rather than a dependency: * it holds three stored/deflated members and nothing else. The source * tarball is unpacked by the system `tar`, as the libnode build does. @@ -21,7 +25,7 @@ import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { mkdir, mkdtemp, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { promisify } from "node:util"; import { inflateRawSync, crc32 } from "node:zlib"; import { @@ -31,6 +35,9 @@ import { ICU_DATA_URL, ICU_DATA_ZIP_SHA256, ICU_DATA_ZIP_SIZE, + ICU_DATA_DROPPED, + ICU_PACK_DATA_SHA256, + ICU_PACK_DATA_SIZE, ICU_SOURCE_SHA256, ICU_SOURCE_SIZE, ICU_SOURCE_URL, @@ -180,7 +187,7 @@ export function unzipMember(zip, name) { export async function ensureIcuData(cacheDir) { const zipPath = join(cacheDir, `icu4c-${ICU_DATA_VERSION}-data-bin-l.zip`); const dataPath = join(cacheDir, `${ICU_DATA_VERSION}`, ICU_DATA_FILE); - if (await validFile(dataPath, ICU_DATA_SHA256)) return dataPath; + if (await validFile(dataPath, ICU_DATA_SHA256, ICU_DATA_SIZE)) return dataPath; let zip; try { zip = await readFile(zipPath); @@ -211,11 +218,98 @@ export async function ensureIcuData(cacheDir) { return dataPath; } -async function validFile(path, expectedSha256) { +async function validFile(path, expectedSha256, expectedSize) { try { - if ((await stat(path)).size !== ICU_DATA_SIZE) return false; + if ((await stat(path)).size !== expectedSize) return false; return sha256(await readFile(path)) === expectedSha256; } catch { return false; } } + +/** The items of an ICU common data file (a `.dat`: a header, then a table of + * contents of name and data offsets, sorted by name, then the items): each + * item's name (`icudt78l/coll/de.res`) and its bytes. */ +export function icuDataItems(bytes) { + const headerSize = bytes.readUInt16LE(0); + if (bytes[2] !== 0xda || bytes[3] !== 0x27) throw new Error("not an ICU data file"); + const count = bytes.readUInt32LE(headerSize); + const entries = []; + for (let i = 0; i < count; i++) { + const nameStart = headerSize + bytes.readUInt32LE(headerSize + 4 + i * 8); + const nameEnd = bytes.indexOf(0, nameStart); + entries.push({ + name: bytes.toString("latin1", nameStart, nameEnd), + start: headerSize + bytes.readUInt32LE(headerSize + 8 + i * 8), + }); + } + return entries.map((entry, i) => ({ + name: entry.name, + bytes: bytes.subarray( + entry.start, + i + 1 < entries.length ? entries[i + 1].start : bytes.length, + ), + })); +} + +/** Whether a data item falls under one of the ICU_DATA_DROPPED patterns. */ +export function icuDataItemDropped(name, dropped = ICU_DATA_DROPPED) { + const path = name.slice(name.indexOf("/") + 1); + return dropped.some((pattern) => + pattern.startsWith("*.") + ? !path.includes("/") && path.endsWith(pattern.slice(1)) + : path.startsWith(pattern), + ); +} + +/** An ICU common data file holding the items of `bytes` that `keep` accepts, + * in their order, laid out as icupkg writes one: the original header, the + * table of contents, the names, then each item at a 16-byte boundary. ICU + * finds items by a binary search of the sorted table, so a subset of a valid + * file in its order is a valid file. */ +export function filterIcuData(bytes, keep) { + const headerSize = bytes.readUInt16LE(0); + const items = icuDataItems(bytes).filter((item) => keep(item.name)); + const names = items.map((item) => Buffer.from(`${item.name}\0`, "latin1")); + const tocSize = 4 + items.length * 8; + const align = (n) => (n + 15) & ~15; + let nameOffset = tocSize; + let dataOffset = align(tocSize + names.reduce((sum, name) => sum + name.length, 0)); + const layout = items.map((item, i) => { + const entry = { nameOffset, dataOffset }; + nameOffset += names[i].length; + dataOffset = align(dataOffset + item.bytes.length); + return entry; + }); + const out = Buffer.alloc(headerSize + dataOffset); + bytes.copy(out, 0, 0, headerSize); + out.writeUInt32LE(items.length, headerSize); + items.forEach((item, i) => { + out.writeUInt32LE(layout[i].nameOffset, headerSize + 4 + i * 8); + out.writeUInt32LE(layout[i].dataOffset, headerSize + 8 + i * 8); + names[i].copy(out, headerSize + layout[i].nameOffset); + item.bytes.copy(out, headerSize + layout[i].dataOffset); + }); + return out; +} + +/** The runtime pack's data file: the pinned icudt78l.dat without the + * ICU_DATA_DROPPED items, written into `cacheDir` once and verified against + * its own pin on every call. Returns its path. */ +export async function ensurePackIcuData(cacheDir) { + const dataPath = join(cacheDir, `${ICU_DATA_VERSION}-pack`, ICU_DATA_FILE); + if (await validFile(dataPath, ICU_PACK_DATA_SHA256, ICU_PACK_DATA_SIZE)) return dataPath; + const full = await readFile(await ensureIcuData(cacheDir)); + const bytes = filterIcuData(full, (name) => !icuDataItemDropped(name)); + if (bytes.length !== ICU_PACK_DATA_SIZE || sha256(bytes) !== ICU_PACK_DATA_SHA256) + throw new Error( + `filtered ICU data is ${bytes.length} bytes with sha256 ${sha256(bytes)}, ` + + `expected ${ICU_PACK_DATA_SIZE} bytes with sha256 ${ICU_PACK_DATA_SHA256} ` + + "(update ICU_PACK_DATA_SHA256 and ICU_PACK_DATA_SIZE in icu-inputs.mjs with ICU_DATA_DROPPED)", + ); + await mkdir(dirname(dataPath), { recursive: true }); + const temporary = `${dataPath}.${process.pid}.tmp`; + await writeFile(temporary, bytes); + await rename(temporary, dataPath); + return dataPath; +} diff --git a/packages/runtime-pack-common/scripts/icu-inputs.mjs b/packages/runtime-pack-common/scripts/icu-inputs.mjs index 0aef3ba37..7dfe8379d 100644 --- a/packages/runtime-pack-common/scripts/icu-inputs.mjs +++ b/packages/runtime-pack-common/scripts/icu-inputs.mjs @@ -22,3 +22,29 @@ export const ICU_DATA_ZIP_SIZE = 12_836_659; export const ICU_DATA_FILE = "icudt78l.dat"; export const ICU_DATA_SHA256 = "d5cf2a40dccbe471781ec7af85693bff542ff12f0b670c9630c4e72d60714b8b"; export const ICU_DATA_SIZE = 33_107_232; + +/** The data categories the runtime pack's copy of icudt78l.dat leaves out: + * item-name prefixes under `icudt78l/` (a trailing `/` names a tree) and + * `*.cnv` for the charset converters. scr_intl.c answers its Intl forms with + * ICU's break iterators, number and date formatters, calendars and locale + * IDs, which read the locale data, `brkitr/` and `curr/` and never these: + * converters, collation, display names of languages, regions and time + * zones, unit names, spell-out rules and transliterators. An Intl form that + * needs one of them removes it from this list and updates the two pins + * below in the same change. tests/harness/icu-data-filter.test.ts runs every + * `@requires-icu` corpus program with the pack's file and with the full one + * and fails when their outputs differ. */ +export const ICU_DATA_DROPPED = [ + "*.cnv", + "coll/", + "lang/", + "rbnf/", + "region/", + "translit/", + "unit/", + "zone/", +]; +/** icudt78l.dat without the ICU_DATA_DROPPED items (icu-data.mjs, filterIcuData). */ +export const ICU_PACK_DATA_SHA256 = + "513a72821c8ee68c7e8a24c346cadc2f8de22efaa19fb68c14957f313e74f35b"; +export const ICU_PACK_DATA_SIZE = 11_653_984; diff --git a/packages/runtime-pack-common/test/icu-data.test.ts b/packages/runtime-pack-common/test/icu-data.test.ts new file mode 100644 index 000000000..43c524f31 --- /dev/null +++ b/packages/runtime-pack-common/test/icu-data.test.ts @@ -0,0 +1,80 @@ +import { expect, test } from "vitest"; +import { filterIcuData, icuDataItemDropped, icuDataItems } from "../scripts/icu-data.mjs"; + +/** A common data file in ICU's layout: a 32-byte header with the 0xda27 + * magic, the table of contents, the names, and each item at a 16-byte + * boundary (padded with 0xaa, as ICU's tools pad). */ +function dataFile(items: [string, string][]): Buffer { + const header = Buffer.alloc(32); + header.writeUInt16LE(32, 0); + header[2] = 0xda; + header[3] = 0x27; + const names = Buffer.concat(items.map(([name]) => Buffer.from(`${name}\0`, "latin1"))); + const tocSize = 4 + items.length * 8; + const toc = Buffer.alloc(tocSize); + toc.writeUInt32LE(items.length, 0); + let nameOffset = tocSize; + let offset = (tocSize + names.length + 15) & ~15; + const body: Buffer[] = []; + items.forEach(([name, contents], i) => { + toc.writeUInt32LE(nameOffset, 4 + i * 8); + toc.writeUInt32LE(offset, 8 + i * 8); + nameOffset += name.length + 1; + const padded = Buffer.alloc((contents.length + 15) & ~15, 0xaa); + padded.write(contents, "latin1"); + body.push(padded); + offset += padded.length; + }); + const lead = Buffer.alloc(((tocSize + names.length + 15) & ~15) - tocSize - names.length, 0xaa); + return Buffer.concat([header, toc, names, lead, ...body]); +} + +const items: [string, string][] = [ + ["icudt78l/brkitr/word.brk", "word rules"], + ["icudt78l/coll/de.res", "collation"], + ["icudt78l/curr/de.res", "currencies"], + ["icudt78l/de.res", "locale"], + ["icudt78l/ibm-1047_P100-1995.cnv", "converter"], + ["icudt78l/zone/de.res", "zone names"], + ["icudt78l/zoneinfo64.res", "time zones"], +]; + +test("the dropped categories match trees and root-level converters only", () => { + const dropped = items.map(([name]) => icuDataItemDropped(name)); + expect(dropped).toEqual([false, true, false, false, true, true, false]); + expect(icuDataItemDropped("icudt78l/brkitr/x.cnv")).toBe(false); +}); + +test("a filtered file keeps the accepted items' names, order and bytes", () => { + const file = dataFile(items); + const filtered = filterIcuData(file, (name) => !icuDataItemDropped(name)); + expect(filtered.subarray(0, 32).equals(file.subarray(0, 32))).toBe(true); + const kept = icuDataItems(filtered); + expect(kept.map((item) => item.name)).toEqual([ + "icudt78l/brkitr/word.brk", + "icudt78l/curr/de.res", + "icudt78l/de.res", + "icudt78l/zoneinfo64.res", + ]); + for (const item of kept) { + const [, contents] = items.find(([name]) => name === item.name)!; + expect(item.bytes.subarray(0, contents.length).toString("latin1")).toBe(contents); + } + for (const [i, item] of kept.entries()) + expect(filtered.readUInt32LE(32 + 8 + i * 8) % 16, item.name).toBe(0); +}); + +test("keeping every item reproduces each item", () => { + const file = dataFile(items); + const same = icuDataItems(filterIcuData(file, () => true)); + const original = icuDataItems(file); + expect(same.length).toBe(original.length); + for (const [i, item] of same.entries()) { + expect(item.name).toBe(original[i]!.name); + expect(item.bytes.subarray(0, 10).equals(original[i]!.bytes.subarray(0, 10))).toBe(true); + } +}); + +test("a file without ICU's magic is refused", () => { + expect(() => icuDataItems(Buffer.alloc(64))).toThrow("not an ICU data file"); +}); diff --git a/packages/runtime/vendor/README.md b/packages/runtime/vendor/README.md index e032f1a53..228ea5233 100644 --- a/packages/runtime/vendor/README.md +++ b/packages/runtime/vendor/README.md @@ -91,7 +91,9 @@ The locale/CLDR engine behind the Intl forms only it can answer (see packages/ru ICU is not vendored: 970 source files plus 33 MiB of compiled CLDR tables are a download, not a reviewable tree in this repository. Both build inputs come from unicode-org/icu's official GitHub release for the pinned version, and both are pinned by version and sha256 in packages/compiler/src/backend/vendor-inputs.ts (mirrored in packages/runtime-pack-common/scripts/icu-inputs.mjs for the runtime packs, with a unit test that keeps the two identical): - `icu4c-78.3-sources.tgz` (27,977,255 B, sha256 `3a2e7a47604ba702f345878308e6fefeca612ee895cf4a5f222e7955fabfe0c0`), the release's own source tarball: `common/`, `i18n/` and `stubdata/` (the empty data-symbol stub ICU needs when its data ships outside the archive), plus LICENSE. `source/common/sources.txt` and `source/i18n/sources.txt` are the *lists of record*: a build compiles exactly the units ICU's own autoconf build compiles, so `icuSourceList` (vendor-inputs.ts for the source-build path, packages/runtime-pack-common/scripts/icu-data.mjs for the runtime packs) reads them rather than re-deriving the set. -- `icu4c-78.3-data-bin-l.zip` (12,836,659 B, sha256 `982619632b78887f1895b063e96e8c3cc7f99283337c8abbd05aa71635de613c`), whose `icudt78l.dat` (33,107,232 B, sha256 `d5cf2a40dccbe471781ec7af85693bff542ff12f0b670c9630c4e72d60714b8b`) is not compiled: it is shipped as `artifacts/icu/icudt78l.dat` and installed beside every executable that links ICU, where the runtime looks first (`$ORIGIN`-style: the executable's own directory), then `ICU_DATA`. +- `icu4c-78.3-data-bin-l.zip` (12,836,659 B, sha256 `982619632b78887f1895b063e96e8c3cc7f99283337c8abbd05aa71635de613c`), whose `icudt78l.dat` (33,107,232 B, sha256 `d5cf2a40dccbe471781ec7af85693bff542ff12f0b670c9630c4e72d60714b8b`) is not compiled. The pack build writes a copy without the data categories no Intl form reads, ships it as `artifacts/icu/icudt78l.dat` and installs it beside every executable that links ICU, where the runtime looks first (`$ORIGIN`-style: the executable's own directory), then `ICU_DATA`. + +The pack's copy of `icudt78l.dat` (11,653,984 B, sha256 `513a72821c8ee68c7e8a24c346cadc2f8de22efaa19fb68c14957f313e74f35b`) leaves out the items `ICU_DATA_DROPPED` in icu-inputs.mjs names: the charset converters (`*.cnv`) and the `coll/`, `lang/`, `rbnf/`, `region/`, `translit/`, `unit/` and `zone/` trees, 21.5 MB that scr_intl.c's break iterators, number and date formatters, calendars and locale IDs never read. `filterIcuData` in icu-data.mjs rewrites the data file's table of contents to the remaining items, and the build checks the result against the pinned sha256 and size. tests/harness/icu-data-filter.test.ts runs every `@requires-icu` corpus program under 47 locales and 3 time zones with the pack's data, with the full data and in Node, and fails when the outputs differ. An Intl form that reads a dropped category (time zone names, display names, collation, unit names) removes it from `ICU_DATA_DROPPED` and updates the two pins in the same change. Each download is cached (the runtime-pack build under `node_modules/.cache/scriptc-icu-*`, the source-build path under the per-user build cache's `vendor/`), verified against its pinned sha256 on every use, and unpacked by the small zip reader in icu-data.mjs (the data) or the system `tar` (the sources). A hash mismatch fails the build with both hashes; an offline build works from a pre-populated cache. To update, bump `ICU_VERSION` with the new release's two assets' sizes and sha256 in vendor-inputs.ts and icu-inputs.mjs. diff --git a/tests/corpus/intl-word-segmenter-dictionaries.ts b/tests/corpus/intl-word-segmenter-dictionaries.ts new file mode 100644 index 000000000..cbb5034c5 --- /dev/null +++ b/tests/corpus/intl-word-segmenter-dictionaries.ts @@ -0,0 +1,25 @@ +// @requires-icu +// @env: LC_ALL=en_US.UTF-8 +// Word segmentation of scripts written without spaces: Chinese, Japanese, +// Thai, Lao, Khmer and Burmese. ICU breaks them with the dictionaries in its +// data file (brkitr/*.dict), as Node does; each line lists a text's word-like +// segments and the boundaries of all its segments. +const texts = [ + "我们今天去北京大学参观博物馆。", + "東京都の天気は晴れです。日本語の文章を単語に分けます。", + "ภาษาไทยเป็นภาษาที่ไม่มีการเว้นวรรคระหว่างคำ", + "ພາສາລາວເປັນພາສາທີ່ສວຍງາມ", + "ភាសាខ្មែរគឺជាភាសាផ្លូវការរបស់ប្រទេសកម្ពុជា", + "မြန်မာဘာသာစကားသည်မြန်မာနိုင်ငံ၏ရုံးသုံးဘာသာဖြစ်သည်", + "한국어는 띄어쓰기를 합니다.", +]; +const segmenter = new Intl.Segmenter(undefined, { granularity: "word" }); +for (const text of texts) { + const words: string[] = []; + const boundaries: number[] = []; + for (const { segment, index, isWordLike } of segmenter.segment(text)) { + boundaries.push(index); + if (isWordLike === true) words.push(segment); + } + console.log(words.join("|"), boundaries.join(",")); +} diff --git a/tests/harness/corpus-costs/intl-word-segmenter-dictionaries.ts.json b/tests/harness/corpus-costs/intl-word-segmenter-dictionaries.ts.json new file mode 100644 index 000000000..b18912827 --- /dev/null +++ b/tests/harness/corpus-costs/intl-word-segmenter-dictionaries.ts.json @@ -0,0 +1 @@ +{ "plain": 8690, "san": 67970 } diff --git a/tests/harness/icu-data-filter.test.ts b/tests/harness/icu-data-filter.test.ts new file mode 100644 index 000000000..2e3788124 --- /dev/null +++ b/tests/harness/icu-data-filter.test.ts @@ -0,0 +1,206 @@ +/* The runtime pack ships icudt78l.dat without the categories no Intl form + * reads (ICU_DATA_DROPPED in packages/runtime-pack-common/scripts/ + * icu-inputs.mjs). Every `@requires-icu` corpus program runs under a sweep + * of locales and time zones three ways: with the pack's data beside it, with + * ICU's full data file beside it, and in Node. The three outputs must be + * equal. A later Intl form that reads a dropped category gets ICU's root + * fallback or a missing-data error from the pack's file and fails here; + * that change removes the category from ICU_DATA_DROPPED and updates the + * pinned sha256 and size beside it. */ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { copyFileSync, globSync, mkdirSync, readFileSync, rmSync } from "node:fs"; +import { availableParallelism } from "node:os"; +import { basename, dirname, join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { afterAll, describe, expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; + +const execFileAsync = promisify(execFile); +const repoRoot = join(import.meta.dirname, "../.."); +const corpusDir = join(repoRoot, "tests/corpus"); +const workDir = join(repoRoot, "node_modules/.cache/scriptc-tests/icu-data-filter"); + +const icuData = (await import( + pathToFileURL(join(repoRoot, "packages/runtime-pack-common/scripts/icu-data.mjs")).href +)) as { + ensureIcuData(cacheDir: string): Promise; + filterIcuData(bytes: Buffer, keep: (name: string) => boolean): Buffer; + icuDataItemDropped(name: string): boolean; +}; +const icuInputs = (await import( + pathToFileURL(join(repoRoot, "packages/runtime-pack-common/scripts/icu-inputs.mjs")).href +)) as { ICU_DATA_DROPPED: string[]; ICU_PACK_DATA_SHA256: string }; + +const LOCALES = [ + "en_US.UTF-8", + "de_DE.UTF-8", + "fr_FR.UTF-8", + "fr_CA.UTF-8", + "es_MX.UTF-8", + "pt_BR.UTF-8", + "it_IT.UTF-8", + "nl_NL.UTF-8", + "sv_SE.UTF-8", + "fi_FI.UTF-8", + "pl_PL.UTF-8", + "ru_RU.UTF-8", + "uk_UA.UTF-8", + "tr_TR.UTF-8", + "el_GR.UTF-8", + "he_IL.UTF-8", + "ar_EG.UTF-8", + "ar_SA.UTF-8", + "fa_IR.UTF-8", + "ur_PK.UTF-8", + "hi_IN.UTF-8", + "bn_BD.UTF-8", + "mr_IN.UTF-8", + "ne_NP.UTF-8", + "ta_IN.UTF-8", + "th_TH.UTF-8", + "lo_LA.UTF-8", + "km_KH.UTF-8", + "my_MM.UTF-8", + "ja_JP.UTF-8", + "ko_KR.UTF-8", + "zh_CN.UTF-8", + "zh_TW.UTF-8", + "zh_HK.UTF-8", + "vi_VN.UTF-8", + "id_ID.UTF-8", + "am_ET.UTF-8", + "sw_KE.UTF-8", + "ps_AF.UTF-8", + "sr_RS.UTF-8", + "sr_RS@latin", + "ca_ES@valencia", + "de_CH.UTF-8", + "en_IN.UTF-8", + "en_GB.UTF-8", + "C.UTF-8", + "POSIX", +]; +const TIME_ZONES = ["UTC", "Asia/Kolkata", "America/New_York"]; + +const programs = [...globSync(join(corpusDir, "*.ts")), ...globSync(join(corpusDir, "*/main.ts"))] + .filter((file) => + readFileSync(file, "utf8") + .split("\n", 2) + .some((line) => /^\/\/ @requires-icu\s*$/.test(line)), + ) + .sort(); + +/** The program's `// @env:` variables other than TZ and LC_ALL, which the + * sweep sets. */ +function programEnv(file: string): Record { + const env: Record = {}; + for (const line of readFileSync(file, "utf8").split("\n", 2)) { + const m = /^\/\/ @env:\s*(.+?)\s*$/.exec(line); + if (!m) continue; + for (const pair of m[1]!.split(",")) { + const eq = pair.indexOf("="); + if (eq > 0) env[pair.slice(0, eq).trim()] = pair.slice(eq + 1).trim(); + } + } + delete env["TZ"]; + delete env["LC_ALL"]; + return env; +} + +async function run(cmd: string, args: string[], env: Record): Promise { + try { + const { stdout, stderr } = await execFileAsync(cmd, args, { + env: { ...process.env, LANG: "", LC_MESSAGES: "", ...env }, + maxBuffer: 16 << 20, + }); + return `${stdout}\n--- stderr\n${stderr}`; + } catch (error) { + const e = error as { stdout?: string; stderr?: string; code?: number }; + return `${e.stdout ?? ""}\n--- stderr\n${e.stderr ?? ""}\n--- exit ${e.code}`; + } +} + +/** Runs `tasks` with at most `limit` in flight. */ +async function pool(tasks: (() => Promise)[], limit: number): Promise { + const results: T[] = Array.from({ length: tasks.length }); + let next = 0; + await Promise.all( + Array.from({ length: Math.min(limit, tasks.length) }, async () => { + while (next < tasks.length) { + const i = next++; + results[i] = await tasks[i]!(); + } + }), + ); + return results; +} + +const skip = process.platform !== "linux" || process.arch !== "x64"; + +describe.skipIf(skip)("the runtime pack's filtered ICU data", () => { + afterAll(() => rmSync(workDir, { recursive: true, force: true })); + + test("is the pinned full data file without the dropped categories", async () => { + mkdirSync(workDir, { recursive: true }); + const full = readFileSync( + await icuData.ensureIcuData(join(repoRoot, "node_modules/.cache/scriptc-icu-data")), + ); + const filtered = icuData.filterIcuData(full, (name) => !icuData.icuDataItemDropped(name)); + expect(createHash("sha256").update(filtered).digest("hex")).toBe( + icuInputs.ICU_PACK_DATA_SHA256, + ); + expect(icuInputs.ICU_DATA_DROPPED.length).toBeGreaterThan(0); + }, 120_000); + + test.for(programs.map((file) => [file.slice(corpusDir.length + 1), file] as const))( + "%s gives the same output with the pack's data, the full data and Node", + async ([rel, file]) => { + const out = join(workDir, rel.replace(/[/.]/g, "_")); + const result = await compile(file, { + outPath: join(out, "filtered", "program"), + outDir: join(out, "filtered"), + }); + expect(result.ok, !result.ok ? JSON.stringify(result.diagnostics, null, 2) : "").toBe(true); + if (!result.ok) return; + const installed = readFileSync(join(dirname(result.binaryPath), "icudt78l.dat")); + expect(createHash("sha256").update(installed).digest("hex")).toBe( + icuInputs.ICU_PACK_DATA_SHA256, + ); + // The same executable with ICU's full data file beside it, where the + // runtime looks first. + const fullBinary = join(out, "full", basename(result.binaryPath)); + mkdirSync(dirname(fullBinary), { recursive: true }); + copyFileSync(result.binaryPath, fullBinary); + copyFileSync( + await icuData.ensureIcuData(join(repoRoot, "node_modules/.cache/scriptc-icu-data")), + join(dirname(fullBinary), "icudt78l.dat"), + ); + const env = programEnv(file); + const cases = TIME_ZONES.flatMap((tz) => + LOCALES.map((locale) => ({ ...env, TZ: tz, LC_ALL: locale })), + ); + const outputs = await pool( + cases.map((caseEnv) => async () => ({ + env: `TZ=${caseEnv.TZ} LC_ALL=${caseEnv.LC_ALL}`, + filtered: await run(result.binaryPath, [], caseEnv), + full: await run(fullBinary, [], caseEnv), + node: await run(process.execPath, [file], caseEnv), + })), + Math.max(2, availableParallelism() >> 1), + ); + for (const o of outputs) { + expect( + o.filtered, + `${rel} under ${o.env}: the pack's ICU data answers differently from the full data; ` + + `an Intl form reads a category in ICU_DATA_DROPPED (${icuInputs.ICU_DATA_DROPPED.join(", ")})`, + ).toBe(o.full); + expect(o.full, `${rel} under ${o.env}: scriptc with the full ICU data and Node`).toBe( + o.node, + ); + } + }, + 600_000, + ); +});