Ship ICU data without the categories no Intl form reads (#2598)

packages/compiler/src/backend/linker.ts

@@ -235,7 +235,7 @@ async function objectLinkerDependencyPaths(
  * file next to the binary is what the runtime looks for first.
  *
  * A file already present at the pinned size is left alone: the pack ships
- * one data set per file name, and re-copying 31.6 MiB on every build would
+ * one data set per file name, and re-copying 11.1 MiB on every build would
  * be pure cost. The copy is atomic (private temporary, then rename) so two
  * concurrent builds of the same program never observe a partial file. */
 export async function installRuntimeDataFiles(

packages/compiler/src/backend/runtime-pack-core.ts

@@ -75,7 +75,7 @@ export interface RuntimePackManifest {
   flavors: Partial<Record<RuntimePackFlavor, RuntimePackFlavorManifest>>;
   archives: RuntimePackArchive[];
   /** Files the linked executable needs at RUN time rather than at link time,
-   * installed beside it on every build. ICU's data file (icudt78l.dat, 31.6
+   * installed beside it on every build. ICU's data file (icudt78l.dat, 11.1
    * MiB) is the only one; Intl programs copy it, every other program links
    * neither ICU nor its data. */
   data?: RuntimePackArtifact[];
@@ -492,7 +492,7 @@ export function selectRuntimePackArtifacts(
     archives: manifest.archives.filter((archive) =>
       evaluateRuntimePredicate(archive.predicate, features),
     ),
-    // ICU's data file is 31.6 MiB: only a program that links ICU installs it.
+    // ICU's data file is 11.1 MiB: only a program that links ICU installs it.
     data: features.intl === true ? (manifest.data ?? []) : [],
     systemLibraries: manifest.system_libraries
       .filter((entry) => evaluateRuntimePredicate(entry.predicate, features))

packages/compiler/src/backend/runtime-pack-native.ts

@@ -429,7 +429,7 @@ export function stageNativeRuntimeSelection(
  * own selection (backend/linker.ts); this host writes executables
  * synchronously and has no link plan, so it installs them here.
  *
- * A file already present at the pinned size is left alone (31.6 MiB of pure
+ * A file already present at the pinned size is left alone (11.1 MiB of pure
  * cost otherwise), and the copy is atomic — a private temporary, then a
  * rename — so two concurrent builds of one program never read a partial
  * file. */

packages/compiler/src/backend/vendor-inputs.ts

@@ -10,7 +10,8 @@ export const BROTLI_VERSION = "1.2.0";
  * (`process.versions.icu` there), from the `release-78.3` tag. ICU is not
  * vendored: the build fetches the release's own source tarball and data zip
  * from unicode-org/icu's GitHub release, pinning each by version and sha256,
- * and caches the download. Its data file icudt78l.dat (33,107,232 B) ships
+ * and caches the download. The runtime packs ship its data file icudt78l.dat
+ * (33,107,232 B) without the categories no Intl form reads (11,653,984 B),
  * beside the executable; see packages/runtime-pack-common/scripts/
  * icu-data.mjs. */
 export const ICU_VERSION = "78.3";

packages/compiler/test/ts7/baselines/order-parity/tests/corpus/intl-word-segmenter-dictionaries.ts.json

@@ -0,0 +1,6 @@
+{
+ "order": [
+  "<repo>/tests/corpus/intl-word-segmenter-dictionaries.ts"
+ ],
+ "diags": []
+}

packages/runtime-pack-common/scripts/build.mjs

@@ -20,7 +20,7 @@ import { installRuntimePack, withBuildLock } from "./build-state.mjs";
 import { shareIdenticalArtifacts } from "./share-artifacts.mjs";
 import { parallelMap } from "./parallel-map.mjs";
 import { zlibInputs } from "./zlib-inputs.mjs";
-import { ICU_DATA_FILE, ensureIcuData, ensureIcuSources, icuSourceList } from "./icu-data.mjs";
+import { ICU_DATA_FILE, ensurePackIcuData, ensureIcuSources, icuSourceList } from "./icu-data.mjs";
 
 const run = promisify(execFile);
 
@@ -668,11 +668,12 @@ async function build() {
     // ICU's data is a RUN-time input, not a link input: the executable loads
     // icudt78l.dat from beside itself (or ICU_DATA). `data` tells the
     // compiler where the pack keeps it, so every Intl build installs it
-    // beside the binary it links.
+    // beside the binary it links. The pack's copy leaves out the categories
+    // no Intl form reads (ICU_DATA_DROPPED in icu-inputs.mjs).
     const data = [];
     if (requestedArchives.has("icu")) {
       process.stdout.write(`building ${packageManifest.name} icu data\n`);
-      const source = await ensureIcuData(
+      const source = await ensurePackIcuData(
         join(repoRoot, "node_modules", ".cache", "scriptc-icu-data"),
       );
       const destination = join(stagedOutputRoot, "icu", ICU_DATA_FILE);

packages/runtime-pack-common/scripts/icu-data.mjs

@@ -2,8 +2,11 @@
  *
  * Intl answers from ICU, so the pack builds ICU's C library from the release
  * source tarball and ships its data as a FILE the executable loads at run
- * time (31.6 MiB of CLDR patterns, locale metadata, break rules and
- * time-zone shapes) — never embedded in the binary. Neither is vendored:
+ * time — never embedded in the binary. The release's data file is 31.6 MiB;
+ * the pack's copy keeps the locale data, break rules and dictionaries,
+ * currency data and time-zone rules the Intl forms read, 11.1 MiB, and
+ * leaves out the categories in ICU_DATA_DROPPED (filterIcuData below,
+ * pinned by ICU_PACK_DATA_SHA256). Neither input is vendored:
  * both come from unicode-org/icu's official GitHub release for the pinned
  * version, each downloaded once into a cache and reused after that, and each
  * verified against a pinned sha256 on every use (the source by the tarball's,
@@ -11,9 +14,10 @@
  * here with both hashes, never later in a build. Offline builds work from a
  * pre-populated cache.
  *
- * The pack build drops `icudt78l.dat` into artifacts/icu/ and lists it in
- * runtime-pack.json's `data`, so the compiler can install it beside every
- * executable that links ICU (packages/compiler/src/backend/linker.ts). The
+ * The pack build writes that copy of `icudt78l.dat` to artifacts/icu/ and
+ * lists it in runtime-pack.json's `data`, so the compiler can install it
+ * beside every executable that links ICU (packages/compiler/src/backend/
+ * linker.ts). The
  * data zip is unpacked by the small reader below rather than a dependency:
  * it holds three stored/deflated members and nothing else. The source
  * tarball is unpacked by the system `tar`, as the libnode build does.
@@ -21,7 +25,7 @@
 import { execFile } from "node:child_process";
 import { createHash } from "node:crypto";
 import { mkdir, mkdtemp, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
-import { join } from "node:path";
+import { dirname, join } from "node:path";
 import { promisify } from "node:util";
 import { inflateRawSync, crc32 } from "node:zlib";
 import {
@@ -31,6 +35,9 @@ import {
   ICU_DATA_URL,
   ICU_DATA_ZIP_SHA256,
   ICU_DATA_ZIP_SIZE,
+  ICU_DATA_DROPPED,
+  ICU_PACK_DATA_SHA256,
+  ICU_PACK_DATA_SIZE,
   ICU_SOURCE_SHA256,
   ICU_SOURCE_SIZE,
   ICU_SOURCE_URL,
@@ -180,7 +187,7 @@ export function unzipMember(zip, name) {
 export async function ensureIcuData(cacheDir) {
   const zipPath = join(cacheDir, `icu4c-${ICU_DATA_VERSION}-data-bin-l.zip`);
   const dataPath = join(cacheDir, `${ICU_DATA_VERSION}`, ICU_DATA_FILE);
-  if (await validFile(dataPath, ICU_DATA_SHA256)) return dataPath;
+  if (await validFile(dataPath, ICU_DATA_SHA256, ICU_DATA_SIZE)) return dataPath;
   let zip;
   try {
     zip = await readFile(zipPath);
@@ -211,11 +218,98 @@ export async function ensureIcuData(cacheDir) {
   return dataPath;
 }
 
-async function validFile(path, expectedSha256) {
+async function validFile(path, expectedSha256, expectedSize) {
   try {
-    if ((await stat(path)).size !== ICU_DATA_SIZE) return false;
+    if ((await stat(path)).size !== expectedSize) return false;
     return sha256(await readFile(path)) === expectedSha256;
   } catch {
     return false;
   }
 }
+
+/** The items of an ICU common data file (a `.dat`: a header, then a table of
+ * contents of name and data offsets, sorted by name, then the items): each
+ * item's name (`icudt78l/coll/de.res`) and its bytes. */
+export function icuDataItems(bytes) {
+  const headerSize = bytes.readUInt16LE(0);
+  if (bytes[2] !== 0xda || bytes[3] !== 0x27) throw new Error("not an ICU data file");
+  const count = bytes.readUInt32LE(headerSize);
+  const entries = [];
+  for (let i = 0; i < count; i++) {
+    const nameStart = headerSize + bytes.readUInt32LE(headerSize + 4 + i * 8);
+    const nameEnd = bytes.indexOf(0, nameStart);
+    entries.push({
+      name: bytes.toString("latin1", nameStart, nameEnd),
+      start: headerSize + bytes.readUInt32LE(headerSize + 8 + i * 8),
+    });
+  }
+  return entries.map((entry, i) => ({
+    name: entry.name,
+    bytes: bytes.subarray(
+      entry.start,
+      i + 1 < entries.length ? entries[i + 1].start : bytes.length,
+    ),
+  }));
+}
+
+/** Whether a data item falls under one of the ICU_DATA_DROPPED patterns. */
+export function icuDataItemDropped(name, dropped = ICU_DATA_DROPPED) {
+  const path = name.slice(name.indexOf("/") + 1);
+  return dropped.some((pattern) =>
+    pattern.startsWith("*.")
+      ? !path.includes("/") && path.endsWith(pattern.slice(1))
+      : path.startsWith(pattern),
+  );
+}
+
+/** An ICU common data file holding the items of `bytes` that `keep` accepts,
+ * in their order, laid out as icupkg writes one: the original header, the
+ * table of contents, the names, then each item at a 16-byte boundary. ICU
+ * finds items by a binary search of the sorted table, so a subset of a valid
+ * file in its order is a valid file. */
+export function filterIcuData(bytes, keep) {
+  const headerSize = bytes.readUInt16LE(0);
+  const items = icuDataItems(bytes).filter((item) => keep(item.name));
+  const names = items.map((item) => Buffer.from(`${item.name}\0`, "latin1"));
+  const tocSize = 4 + items.length * 8;
+  const align = (n) => (n + 15) & ~15;
+  let nameOffset = tocSize;
+  let dataOffset = align(tocSize + names.reduce((sum, name) => sum + name.length, 0));
+  const layout = items.map((item, i) => {
+    const entry = { nameOffset, dataOffset };
+    nameOffset += names[i].length;
+    dataOffset = align(dataOffset + item.bytes.length);
+    return entry;
+  });
+  const out = Buffer.alloc(headerSize + dataOffset);
+  bytes.copy(out, 0, 0, headerSize);
+  out.writeUInt32LE(items.length, headerSize);
+  items.forEach((item, i) => {
+    out.writeUInt32LE(layout[i].nameOffset, headerSize + 4 + i * 8);
+    out.writeUInt32LE(layout[i].dataOffset, headerSize + 8 + i * 8);
+    names[i].copy(out, headerSize + layout[i].nameOffset);
+    item.bytes.copy(out, headerSize + layout[i].dataOffset);
+  });
+  return out;
+}
+
+/** The runtime pack's data file: the pinned icudt78l.dat without the
+ * ICU_DATA_DROPPED items, written into `cacheDir` once and verified against
+ * its own pin on every call. Returns its path. */
+export async function ensurePackIcuData(cacheDir) {
+  const dataPath = join(cacheDir, `${ICU_DATA_VERSION}-pack`, ICU_DATA_FILE);
+  if (await validFile(dataPath, ICU_PACK_DATA_SHA256, ICU_PACK_DATA_SIZE)) return dataPath;
+  const full = await readFile(await ensureIcuData(cacheDir));
+  const bytes = filterIcuData(full, (name) => !icuDataItemDropped(name));
+  if (bytes.length !== ICU_PACK_DATA_SIZE || sha256(bytes) !== ICU_PACK_DATA_SHA256)
+    throw new Error(
+      `filtered ICU data is ${bytes.length} bytes with sha256 ${sha256(bytes)}, ` +
+        `expected ${ICU_PACK_DATA_SIZE} bytes with sha256 ${ICU_PACK_DATA_SHA256} ` +
+        "(update ICU_PACK_DATA_SHA256 and ICU_PACK_DATA_SIZE in icu-inputs.mjs with ICU_DATA_DROPPED)",
+    );
+  await mkdir(dirname(dataPath), { recursive: true });
+  const temporary = `${dataPath}.${process.pid}.tmp`;
+  await writeFile(temporary, bytes);
+  await rename(temporary, dataPath);
+  return dataPath;
+}

packages/runtime-pack-common/scripts/icu-inputs.mjs

@@ -22,3 +22,29 @@ export const ICU_DATA_ZIP_SIZE = 12_836_659;
 export const ICU_DATA_FILE = "icudt78l.dat";
 export const ICU_DATA_SHA256 = "d5cf2a40dccbe471781ec7af85693bff542ff12f0b670c9630c4e72d60714b8b";
 export const ICU_DATA_SIZE = 33_107_232;
+
+/** The data categories the runtime pack's copy of icudt78l.dat leaves out:
+ * item-name prefixes under `icudt78l/` (a trailing `/` names a tree) and
+ * `*.cnv` for the charset converters. scr_intl.c answers its Intl forms with
+ * ICU's break iterators, number and date formatters, calendars and locale
+ * IDs, which read the locale data, `brkitr/` and `curr/` and never these:
+ * converters, collation, display names of languages, regions and time
+ * zones, unit names, spell-out rules and transliterators. An Intl form that
+ * needs one of them removes it from this list and updates the two pins
+ * below in the same change. tests/harness/icu-data-filter.test.ts runs every
+ * `@requires-icu` corpus program with the pack's file and with the full one
+ * and fails when their outputs differ. */
+export const ICU_DATA_DROPPED = [
+  "*.cnv",
+  "coll/",
+  "lang/",
+  "rbnf/",
+  "region/",
+  "translit/",
+  "unit/",
+  "zone/",
+];
+/** icudt78l.dat without the ICU_DATA_DROPPED items (icu-data.mjs, filterIcuData). */
+export const ICU_PACK_DATA_SHA256 =
+  "513a72821c8ee68c7e8a24c346cadc2f8de22efaa19fb68c14957f313e74f35b";
+export const ICU_PACK_DATA_SIZE = 11_653_984;

packages/runtime-pack-common/test/icu-data.test.ts

@@ -0,0 +1,80 @@
+import { expect, test } from "vitest";
+import { filterIcuData, icuDataItemDropped, icuDataItems } from "../scripts/icu-data.mjs";
+
+/** A common data file in ICU's layout: a 32-byte header with the 0xda27
+ * magic, the table of contents, the names, and each item at a 16-byte
+ * boundary (padded with 0xaa, as ICU's tools pad). */
+function dataFile(items: [string, string][]): Buffer {
+  const header = Buffer.alloc(32);
+  header.writeUInt16LE(32, 0);
+  header[2] = 0xda;
+  header[3] = 0x27;
+  const names = Buffer.concat(items.map(([name]) => Buffer.from(`${name}\0`, "latin1")));
+  const tocSize = 4 + items.length * 8;
+  const toc = Buffer.alloc(tocSize);
+  toc.writeUInt32LE(items.length, 0);
+  let nameOffset = tocSize;
+  let offset = (tocSize + names.length + 15) & ~15;
+  const body: Buffer[] = [];
+  items.forEach(([name, contents], i) => {
+    toc.writeUInt32LE(nameOffset, 4 + i * 8);
+    toc.writeUInt32LE(offset, 8 + i * 8);
+    nameOffset += name.length + 1;
+    const padded = Buffer.alloc((contents.length + 15) & ~15, 0xaa);
+    padded.write(contents, "latin1");
+    body.push(padded);
+    offset += padded.length;
+  });
+  const lead = Buffer.alloc(((tocSize + names.length + 15) & ~15) - tocSize - names.length, 0xaa);
+  return Buffer.concat([header, toc, names, lead, ...body]);
+}
+
+const items: [string, string][] = [
+  ["icudt78l/brkitr/word.brk", "word rules"],
+  ["icudt78l/coll/de.res", "collation"],
+  ["icudt78l/curr/de.res", "currencies"],
+  ["icudt78l/de.res", "locale"],
+  ["icudt78l/ibm-1047_P100-1995.cnv", "converter"],
+  ["icudt78l/zone/de.res", "zone names"],
+  ["icudt78l/zoneinfo64.res", "time zones"],
+];
+
+test("the dropped categories match trees and root-level converters only", () => {
+  const dropped = items.map(([name]) => icuDataItemDropped(name));
+  expect(dropped).toEqual([false, true, false, false, true, true, false]);
+  expect(icuDataItemDropped("icudt78l/brkitr/x.cnv")).toBe(false);
+});
+
+test("a filtered file keeps the accepted items' names, order and bytes", () => {
+  const file = dataFile(items);
+  const filtered = filterIcuData(file, (name) => !icuDataItemDropped(name));
+  expect(filtered.subarray(0, 32).equals(file.subarray(0, 32))).toBe(true);
+  const kept = icuDataItems(filtered);
+  expect(kept.map((item) => item.name)).toEqual([
+    "icudt78l/brkitr/word.brk",
+    "icudt78l/curr/de.res",
+    "icudt78l/de.res",
+    "icudt78l/zoneinfo64.res",
+  ]);
+  for (const item of kept) {
+    const [, contents] = items.find(([name]) => name === item.name)!;
+    expect(item.bytes.subarray(0, contents.length).toString("latin1")).toBe(contents);
+  }
+  for (const [i, item] of kept.entries())
+    expect(filtered.readUInt32LE(32 + 8 + i * 8) % 16, item.name).toBe(0);
+});
+
+test("keeping every item reproduces each item", () => {
+  const file = dataFile(items);
+  const same = icuDataItems(filterIcuData(file, () => true));
+  const original = icuDataItems(file);
+  expect(same.length).toBe(original.length);
+  for (const [i, item] of same.entries()) {
+    expect(item.name).toBe(original[i]!.name);
+    expect(item.bytes.subarray(0, 10).equals(original[i]!.bytes.subarray(0, 10))).toBe(true);
+  }
+});
+
+test("a file without ICU's magic is refused", () => {
+  expect(() => icuDataItems(Buffer.alloc(64))).toThrow("not an ICU data file");
+});

packages/runtime/vendor/README.md

@@ -91,7 +91,9 @@ The locale/CLDR engine behind the Intl forms only it can answer (see packages/ru
 ICU is not vendored: 970 source files plus 33 MiB of compiled CLDR tables are a download, not a reviewable tree in this repository. Both build inputs come from unicode-org/icu's official GitHub release for the pinned version, and both are pinned by version and sha256 in packages/compiler/src/backend/vendor-inputs.ts (mirrored in packages/runtime-pack-common/scripts/icu-inputs.mjs for the runtime packs, with a unit test that keeps the two identical):
 
 - `icu4c-78.3-sources.tgz` (27,977,255 B, sha256 `3a2e7a47604ba702f345878308e6fefeca612ee895cf4a5f222e7955fabfe0c0`), the release's own source tarball: `common/`, `i18n/` and `stubdata/` (the empty data-symbol stub ICU needs when its data ships outside the archive), plus LICENSE. `source/common/sources.txt` and `source/i18n/sources.txt` are the *lists of record*: a build compiles exactly the units ICU's own autoconf build compiles, so `icuSourceList` (vendor-inputs.ts for the source-build path, packages/runtime-pack-common/scripts/icu-data.mjs for the runtime packs) reads them rather than re-deriving the set.
-- `icu4c-78.3-data-bin-l.zip` (12,836,659 B, sha256 `982619632b78887f1895b063e96e8c3cc7f99283337c8abbd05aa71635de613c`), whose `icudt78l.dat` (33,107,232 B, sha256 `d5cf2a40dccbe471781ec7af85693bff542ff12f0b670c9630c4e72d60714b8b`) is not compiled: it is shipped as `artifacts/icu/icudt78l.dat` and installed beside every executable that links ICU, where the runtime looks first (`$ORIGIN`-style: the executable's own directory), then `ICU_DATA`.
+- `icu4c-78.3-data-bin-l.zip` (12,836,659 B, sha256 `982619632b78887f1895b063e96e8c3cc7f99283337c8abbd05aa71635de613c`), whose `icudt78l.dat` (33,107,232 B, sha256 `d5cf2a40dccbe471781ec7af85693bff542ff12f0b670c9630c4e72d60714b8b`) is not compiled. The pack build writes a copy without the data categories no Intl form reads, ships it as `artifacts/icu/icudt78l.dat` and installs it beside every executable that links ICU, where the runtime looks first (`$ORIGIN`-style: the executable's own directory), then `ICU_DATA`.
+
+The pack's copy of `icudt78l.dat` (11,653,984 B, sha256 `513a72821c8ee68c7e8a24c346cadc2f8de22efaa19fb68c14957f313e74f35b`) leaves out the items `ICU_DATA_DROPPED` in icu-inputs.mjs names: the charset converters (`*.cnv`) and the `coll/`, `lang/`, `rbnf/`, `region/`, `translit/`, `unit/` and `zone/` trees, 21.5 MB that scr_intl.c's break iterators, number and date formatters, calendars and locale IDs never read. `filterIcuData` in icu-data.mjs rewrites the data file's table of contents to the remaining items, and the build checks the result against the pinned sha256 and size. tests/harness/icu-data-filter.test.ts runs every `@requires-icu` corpus program under 47 locales and 3 time zones with the pack's data, with the full data and in Node, and fails when the outputs differ. An Intl form that reads a dropped category (time zone names, display names, collation, unit names) removes it from `ICU_DATA_DROPPED` and updates the two pins in the same change.
 
 Each download is cached (the runtime-pack build under `node_modules/.cache/scriptc-icu-*`, the source-build path under the per-user build cache's `vendor/`), verified against its pinned sha256 on every use, and unpacked by the small zip reader in icu-data.mjs (the data) or the system `tar` (the sources). A hash mismatch fails the build with both hashes; an offline build works from a pre-populated cache. To update, bump `ICU_VERSION` with the new release's two assets' sizes and sha256 in vendor-inputs.ts and icu-inputs.mjs.
 

tests/corpus/intl-word-segmenter-dictionaries.ts

@@ -0,0 +1,25 @@
+// @requires-icu
+// @env: LC_ALL=en_US.UTF-8
+// Word segmentation of scripts written without spaces: Chinese, Japanese,
+// Thai, Lao, Khmer and Burmese. ICU breaks them with the dictionaries in its
+// data file (brkitr/*.dict), as Node does; each line lists a text's word-like
+// segments and the boundaries of all its segments.
+const texts = [
+  "我们今天去北京大学参观博物馆。",
+  "東京都の天気は晴れです。日本語の文章を単語に分けます。",
+  "ภาษาไทยเป็นภาษาที่ไม่มีการเว้นวรรคระหว่างคำ",
+  "ພາສາລາວເປັນພາສາທີ່ສວຍງາມ",
+  "ភាសាខ្មែរគឺជាភាសាផ្លូវការរបស់ប្រទេសកម្ពុជា",
+  "မြန်မာဘာသာစကားသည်မြန်မာနိုင်ငံ၏ရုံးသုံးဘာသာဖြစ်သည်",
+  "한국어는 띄어쓰기를 합니다.",
+];
+const segmenter = new Intl.Segmenter(undefined, { granularity: "word" });
+for (const text of texts) {
+  const words: string[] = [];
+  const boundaries: number[] = [];
+  for (const { segment, index, isWordLike } of segmenter.segment(text)) {
+    boundaries.push(index);
+    if (isWordLike === true) words.push(segment);
+  }
+  console.log(words.join("|"), boundaries.join(","));
+}

tests/harness/corpus-costs/intl-word-segmenter-dictionaries.ts.json

@@ -0,0 +1 @@
+{ "plain": 8690, "san": 67970 }

tests/harness/icu-data-filter.test.ts

@@ -0,0 +1,206 @@
+/* The runtime pack ships icudt78l.dat without the categories no Intl form
+ * reads (ICU_DATA_DROPPED in packages/runtime-pack-common/scripts/
+ * icu-inputs.mjs). Every `@requires-icu` corpus program runs under a sweep
+ * of locales and time zones three ways: with the pack's data beside it, with
+ * ICU's full data file beside it, and in Node. The three outputs must be
+ * equal. A later Intl form that reads a dropped category gets ICU's root
+ * fallback or a missing-data error from the pack's file and fails here;
+ * that change removes the category from ICU_DATA_DROPPED and updates the
+ * pinned sha256 and size beside it. */
+import { execFile } from "node:child_process";
+import { createHash } from "node:crypto";
+import { copyFileSync, globSync, mkdirSync, readFileSync, rmSync } from "node:fs";
+import { availableParallelism } from "node:os";
+import { basename, dirname, join } from "node:path";
+import { pathToFileURL } from "node:url";
+import { promisify } from "node:util";
+import { afterAll, describe, expect, test } from "vitest";
+import { compile } from "@scriptc/compiler";
+
+const execFileAsync = promisify(execFile);
+const repoRoot = join(import.meta.dirname, "../..");
+const corpusDir = join(repoRoot, "tests/corpus");
+const workDir = join(repoRoot, "node_modules/.cache/scriptc-tests/icu-data-filter");
+
+const icuData = (await import(
+  pathToFileURL(join(repoRoot, "packages/runtime-pack-common/scripts/icu-data.mjs")).href
+)) as {
+  ensureIcuData(cacheDir: string): Promise<string>;
+  filterIcuData(bytes: Buffer, keep: (name: string) => boolean): Buffer;
+  icuDataItemDropped(name: string): boolean;
+};
+const icuInputs = (await import(
+  pathToFileURL(join(repoRoot, "packages/runtime-pack-common/scripts/icu-inputs.mjs")).href
+)) as { ICU_DATA_DROPPED: string[]; ICU_PACK_DATA_SHA256: string };
+
+const LOCALES = [
+  "en_US.UTF-8",
+  "de_DE.UTF-8",
+  "fr_FR.UTF-8",
+  "fr_CA.UTF-8",
+  "es_MX.UTF-8",
+  "pt_BR.UTF-8",
+  "it_IT.UTF-8",
+  "nl_NL.UTF-8",
+  "sv_SE.UTF-8",
+  "fi_FI.UTF-8",
+  "pl_PL.UTF-8",
+  "ru_RU.UTF-8",
+  "uk_UA.UTF-8",
+  "tr_TR.UTF-8",
+  "el_GR.UTF-8",
+  "he_IL.UTF-8",
+  "ar_EG.UTF-8",
+  "ar_SA.UTF-8",
+  "fa_IR.UTF-8",
+  "ur_PK.UTF-8",
+  "hi_IN.UTF-8",
+  "bn_BD.UTF-8",
+  "mr_IN.UTF-8",
+  "ne_NP.UTF-8",
+  "ta_IN.UTF-8",
+  "th_TH.UTF-8",
+  "lo_LA.UTF-8",
+  "km_KH.UTF-8",
+  "my_MM.UTF-8",
+  "ja_JP.UTF-8",
+  "ko_KR.UTF-8",
+  "zh_CN.UTF-8",
+  "zh_TW.UTF-8",
+  "zh_HK.UTF-8",
+  "vi_VN.UTF-8",
+  "id_ID.UTF-8",
+  "am_ET.UTF-8",
+  "sw_KE.UTF-8",
+  "ps_AF.UTF-8",
+  "sr_RS.UTF-8",
+  "sr_RS@latin",
+  "ca_ES@valencia",
+  "de_CH.UTF-8",
+  "en_IN.UTF-8",
+  "en_GB.UTF-8",
+  "C.UTF-8",
+  "POSIX",
+];
+const TIME_ZONES = ["UTC", "Asia/Kolkata", "America/New_York"];
+
+const programs = [...globSync(join(corpusDir, "*.ts")), ...globSync(join(corpusDir, "*/main.ts"))]
+  .filter((file) =>
+    readFileSync(file, "utf8")
+      .split("\n", 2)
+      .some((line) => /^\/\/ @requires-icu\s*$/.test(line)),
+  )
+  .sort();
+
+/** The program's `// @env:` variables other than TZ and LC_ALL, which the
+ * sweep sets. */
+function programEnv(file: string): Record<string, string> {
+  const env: Record<string, string> = {};
+  for (const line of readFileSync(file, "utf8").split("\n", 2)) {
+    const m = /^\/\/ @env:\s*(.+?)\s*$/.exec(line);
+    if (!m) continue;
+    for (const pair of m[1]!.split(",")) {
+      const eq = pair.indexOf("=");
+      if (eq > 0) env[pair.slice(0, eq).trim()] = pair.slice(eq + 1).trim();
+    }
+  }
+  delete env["TZ"];
+  delete env["LC_ALL"];
+  return env;
+}
+
+async function run(cmd: string, args: string[], env: Record<string, string>): Promise<string> {
+  try {
+    const { stdout, stderr } = await execFileAsync(cmd, args, {
+      env: { ...process.env, LANG: "", LC_MESSAGES: "", ...env },
+      maxBuffer: 16 << 20,
+    });
+    return `${stdout}\n--- stderr\n${stderr}`;
+  } catch (error) {
+    const e = error as { stdout?: string; stderr?: string; code?: number };
+    return `${e.stdout ?? ""}\n--- stderr\n${e.stderr ?? ""}\n--- exit ${e.code}`;
+  }
+}
+
+/** Runs `tasks` with at most `limit` in flight. */
+async function pool<T>(tasks: (() => Promise<T>)[], limit: number): Promise<T[]> {
+  const results: T[] = Array.from({ length: tasks.length });
+  let next = 0;
+  await Promise.all(
+    Array.from({ length: Math.min(limit, tasks.length) }, async () => {
+      while (next < tasks.length) {
+        const i = next++;
+        results[i] = await tasks[i]!();
+      }
+    }),
+  );
+  return results;
+}
+
+const skip = process.platform !== "linux" || process.arch !== "x64";
+
+describe.skipIf(skip)("the runtime pack's filtered ICU data", () => {
+  afterAll(() => rmSync(workDir, { recursive: true, force: true }));
+
+  test("is the pinned full data file without the dropped categories", async () => {
+    mkdirSync(workDir, { recursive: true });
+    const full = readFileSync(
+      await icuData.ensureIcuData(join(repoRoot, "node_modules/.cache/scriptc-icu-data")),
+    );
+    const filtered = icuData.filterIcuData(full, (name) => !icuData.icuDataItemDropped(name));
+    expect(createHash("sha256").update(filtered).digest("hex")).toBe(
+      icuInputs.ICU_PACK_DATA_SHA256,
+    );
+    expect(icuInputs.ICU_DATA_DROPPED.length).toBeGreaterThan(0);
+  }, 120_000);
+
+  test.for(programs.map((file) => [file.slice(corpusDir.length + 1), file] as const))(
+    "%s gives the same output with the pack's data, the full data and Node",
+    async ([rel, file]) => {
+      const out = join(workDir, rel.replace(/[/.]/g, "_"));
+      const result = await compile(file, {
+        outPath: join(out, "filtered", "program"),
+        outDir: join(out, "filtered"),
+      });
+      expect(result.ok, !result.ok ? JSON.stringify(result.diagnostics, null, 2) : "").toBe(true);
+      if (!result.ok) return;
+      const installed = readFileSync(join(dirname(result.binaryPath), "icudt78l.dat"));
+      expect(createHash("sha256").update(installed).digest("hex")).toBe(
+        icuInputs.ICU_PACK_DATA_SHA256,
+      );
+      // The same executable with ICU's full data file beside it, where the
+      // runtime looks first.
+      const fullBinary = join(out, "full", basename(result.binaryPath));
+      mkdirSync(dirname(fullBinary), { recursive: true });
+      copyFileSync(result.binaryPath, fullBinary);
+      copyFileSync(
+        await icuData.ensureIcuData(join(repoRoot, "node_modules/.cache/scriptc-icu-data")),
+        join(dirname(fullBinary), "icudt78l.dat"),
+      );
+      const env = programEnv(file);
+      const cases = TIME_ZONES.flatMap((tz) =>
+        LOCALES.map((locale) => ({ ...env, TZ: tz, LC_ALL: locale })),
+      );
+      const outputs = await pool(
+        cases.map((caseEnv) => async () => ({
+          env: `TZ=${caseEnv.TZ} LC_ALL=${caseEnv.LC_ALL}`,
+          filtered: await run(result.binaryPath, [], caseEnv),
+          full: await run(fullBinary, [], caseEnv),
+          node: await run(process.execPath, [file], caseEnv),
+        })),
+        Math.max(2, availableParallelism() >> 1),
+      );
+      for (const o of outputs) {
+        expect(
+          o.filtered,
+          `${rel} under ${o.env}: the pack's ICU data answers differently from the full data; ` +
+            `an Intl form reads a category in ICU_DATA_DROPPED (${icuInputs.ICU_DATA_DROPPED.join(", ")})`,
+        ).toBe(o.full);
+        expect(o.full, `${rel} under ${o.env}: scriptc with the full ICU data and Node`).toBe(
+          o.node,
+        );
+      }
+    },
+    600_000,
+  );
+});