diff --git a/packages/compiler/src/backend/native/executable.ts b/packages/compiler/src/backend/native/executable.ts index c4750ba06..f797eec9a 100644 --- a/packages/compiler/src/backend/native/executable.ts +++ b/packages/compiler/src/backend/native/executable.ts @@ -718,7 +718,17 @@ export async function compileCInternal( let icuArchive = icu ? (icuPack?.archive ?? icuArchivePath(driver, vendorBuildIdentity, vendorCacheRoot)) : null; - let icuIncludeDirs: readonly string[] = icuPack === null ? [] : [icuPack.include]; + // scr_intl.c's ICU headers. They are known before the runtime-object flags + // are: those flags compile every runtime unit, so without them a cached + // scr_intl.o would take whatever ICU headers the host has. + const icuSource = + icu && icuPack === null ? join(await ensureIcuSources(vendorCacheRoot), "source") : null; + const icuIncludeDirs: readonly string[] = + icuPack !== null + ? [icuPack.include] + : icuSource !== null + ? [join(icuSource, "common"), join(icuSource, "i18n")] + : []; // ours would collide; see scr_regex.c) — the standalone objects are for // static builds only. @@ -784,8 +794,6 @@ export async function compileCInternal( ? await materialize() : await stageVendorInputs(materialize, join(stageRoot, "icu")); icuArchive = paths[0]!; - const icuSource = join(await ensureIcuSources(materializeCacheRoot), "source"); - icuIncludeDirs = [join(icuSource, "common"), join(icuSource, "i18n")]; } if (regex) { for (const object of lreObjectPaths( @@ -924,6 +932,7 @@ export async function compileCInternal( ...(regex ? ["-I", vendorEngineDir()] : []), ...(zlibObjects.length > 0 ? ["-I", vendorZlibDir()] : []), ...(tlsArchive !== null ? ["-I", join(vendorTlsDir(), "include")] : []), + ...icuIncludeDirs.flatMap((dir) => ["-I", dir]), ]; // Keep sanitizer runtime objects optimized in both postures. Large dev // programs still compile at O0, avoiding an expensive LLVM optimization diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/intl-worker-first-use.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/intl-worker-first-use.ts.json new file mode 100644 index 000000000..b5a08b90a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/intl-worker-first-use.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/intl-worker-first-use.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_intl.c b/packages/runtime/src/scr_intl.c index cbe160e88..28264f3b5 100644 --- a/packages/runtime/src/scr_intl.c +++ b/packages/runtime/src/scr_intl.c @@ -23,6 +23,7 @@ * That is what lets the zero-cost gate assert that a non-Intl binary links * no ICU symbol at all. */ #include "scr_runtime.h" +#include "scr_intl_utf16.h" #include "scr_number.h" #include @@ -36,6 +37,7 @@ #include #include +#include #include #include #include @@ -51,9 +53,12 @@ /* ── data ───────────────────────────────────────────────────────────── * ICU keeps the mapped data alive for the process lifetime, so the load - * happens exactly once and never unmaps. */ + * happens exactly once and never unmaps. Worker threads can make the first + * ICU call at the same time: pthread_once runs the load on one of them and + * holds the others until it has finished, so none reads a partial result. */ -static int scr_intl_data_state; /* 0 = not tried, 1 = ready, 2 = failed */ +static pthread_once_t scr_intl_data_once = PTHREAD_ONCE_INIT; +static bool scr_intl_data_ready; /* The default locale every formatter and the default word segmenter use, * resolved with the data (scr_intl_default_locale). */ @@ -117,12 +122,10 @@ static void scr_intl_default_locale(void) { snprintf(scr_intl_locale, sizeof scr_intl_locale, "%s", name); } -/** True once ICU can resolve data: the file beside the executable, else - * whatever ICU's own search finds (ICU_DATA). Never throws; the callers - * do, with the same message. */ -static bool scr_intl_data(void) { - if (scr_intl_data_state != 0) return scr_intl_data_state == 1; - scr_intl_data_state = 2; +/** Points ICU at its data (the file beside the executable, else whatever + * ICU's own search finds, ICU_DATA), probes it and resolves the default + * locale. Runs once per process (scr_intl_data). */ +static void scr_intl_data_load(void) { char dir[PATH_MAX]; if (scr_intl_exe_dir(dir, sizeof dir)) { char path[PATH_MAX]; @@ -136,11 +139,17 @@ static bool scr_intl_data(void) { * Everything else this unit formats resolves through the same data. */ UErrorCode status = U_ZERO_ERROR; UNumberFormat *probe = unum_open(UNUM_DECIMAL, NULL, 0, NULL, NULL, &status); - if (U_FAILURE(status) || probe == NULL) return false; + if (U_FAILURE(status) || probe == NULL) return; unum_close(probe); scr_intl_default_locale(); - scr_intl_data_state = 1; - return true; + scr_intl_data_ready = true; +} + +/** True once ICU can resolve data. Never throws; the callers do, with the + * same message. */ +static bool scr_intl_data(void) { + (void)pthread_once(&scr_intl_data_once, scr_intl_data_load); + return scr_intl_data_ready; } /** Throw the missing-data Error (Node cannot reach this: its ICU data is @@ -155,62 +164,22 @@ static void scr_intl_no_data(void) { /* ── WTF-8 ↔ UTF-16 ─────────────────────────────────────────────────── * ScrStr storage is canonical WTF-8 (a lone surrogate is its 3-byte * generalized form), so both directions preserve lone surrogates and - * every public offset stays a UTF-16 unit count. */ + * every public offset stays a UTF-16 unit count. Bytes that are not WTF-8 + * decode to U+FFFD (scr_intl_utf16.h), so the UTF-16 copy holds exactly the + * units it reports. */ -static int32_t scr_intl_units(const char *data, size_t len) { - int32_t units = 0; - for (size_t at = 0; at < len;) { - unsigned char byte = (unsigned char)data[at]; - if (byte < 0x80) at += 1; - else if (byte < 0xe0) at += 2; - else if (byte < 0xf0) at += 3; - else { - at += 4; - units++; /* a non-BMP code point is two units */ - } - units++; - } - return units; -} - -/** A UTF-16 copy of `s`, NUL-terminated; *units receives its length. - * Never NULL for a successful call (an empty string allocates one unit). */ +/** A UTF-16 copy of `s`, NUL-terminated; *units receives the number of + * units written. Never NULL for a successful call (an empty string + * allocates one unit). */ static UChar *scr_intl_to_utf16(const ScrStr *s, int32_t *units) { - const char *data = s->data; - size_t len = s->len; - int32_t count = scr_intl_units(data, len); - UChar *text = malloc(((size_t)count + 1) * sizeof *text); + const unsigned char *data = (const unsigned char *)s->data; + size_t count = scr_intl_utf16_units(data, s->len); + if (count > INT32_MAX - 1) scr_trap("scriptc: string too long for Intl\n"); + UChar *text = malloc((count + 1) * sizeof *text); if (text == NULL) scr_trap("scriptc: out of memory\n"); - int32_t at = 0; - for (size_t pos = 0; pos < len;) { - unsigned char byte = (unsigned char)data[pos]; - uint32_t cp; - size_t size; - if (byte < 0x80) { - cp = byte; - size = 1; - } else if (byte < 0xe0) { - cp = ((uint32_t)(byte & 0x1f) << 6) | ((unsigned char)data[pos + 1] & 0x3fu); - size = 2; - } else if (byte < 0xf0) { - cp = ((uint32_t)(byte & 0x0f) << 12) | (((unsigned char)data[pos + 1] & 0x3fu) << 6) | - ((unsigned char)data[pos + 2] & 0x3fu); - size = 3; - } else { - cp = ((uint32_t)(byte & 0x07) << 18) | (((unsigned char)data[pos + 1] & 0x3fu) << 12) | - (((unsigned char)data[pos + 2] & 0x3fu) << 6) | ((unsigned char)data[pos + 3] & 0x3fu); - size = 4; - } - if (cp > 0xffff) { - text[at++] = (UChar)(0xd800 + ((cp - 0x10000) >> 10)); - text[at++] = (UChar)(0xdc00 + ((cp - 0x10000) & 0x3ff)); - } else { - text[at++] = (UChar)cp; - } - pos += size; - } - text[at] = 0; - *units = count; + size_t written = scr_intl_utf16_fill(data, s->len, (uint16_t *)text); + text[written] = 0; + *units = (int32_t)written; return text; } diff --git a/packages/runtime/src/scr_intl_utf16.h b/packages/runtime/src/scr_intl_utf16.h new file mode 100644 index 000000000..3ab368146 --- /dev/null +++ b/packages/runtime/src/scr_intl_utf16.h @@ -0,0 +1,82 @@ +/* scriptc runtime — WTF-8 to UTF-16 for the ICU unit (scr_intl.c). Kept + * apart from ICU's headers so packages/runtime/test can check the decode + * on its own (test_intl_utf16.c). */ +#ifndef SCR_INTL_UTF16_H +#define SCR_INTL_UTF16_H + +#include +#include + +/** The code point of the WTF-8 sequence at `p` (before `end`), and its byte + * count in *size. A malformed sequence (an overlong form, a value above + * U+10FFFF, a stray or missing continuation byte) is decoded one maximal + * subpart at a time to U+FFFD, as the WHATWG UTF-8 decoder does. The 3-byte + * form of a surrogate is WTF-8's lone surrogate and decodes to that unit. */ +static inline uint32_t scr_intl_decode(const unsigned char *p, const unsigned char *end, size_t *size) { + unsigned char lead = p[0]; + *size = 1; + if (lead < 0x80) return lead; + size_t need; + unsigned char lo = 0x80, hi = 0xbf; + uint32_t cp; + if (lead >= 0xc2 && lead <= 0xdf) { + need = 1; + cp = lead & 0x1fu; + } else if (lead >= 0xe0 && lead <= 0xef) { + need = 2; + cp = lead & 0x0fu; + if (lead == 0xe0) lo = 0xa0; + } else if (lead >= 0xf0 && lead <= 0xf4) { + need = 3; + cp = lead & 0x07u; + if (lead == 0xf0) lo = 0x90; + if (lead == 0xf4) hi = 0x8f; + } else { + return 0xfffd; + } + for (size_t i = 1; i <= need; i++) { + if (p + i >= end || p[i] < lo || p[i] > hi) { + *size = i; + return 0xfffd; + } + cp = (cp << 6) | (p[i] & 0x3fu); + lo = 0x80; + hi = 0xbf; + } + *size = need + 1; + return cp; +} + +/** The number of UTF-16 units the `len` bytes at `data` decode to. */ +static inline size_t scr_intl_utf16_units(const unsigned char *data, size_t len) { + const unsigned char *end = data + len; + size_t count = 0; + for (const unsigned char *p = data; p < end;) { + size_t size; + count += scr_intl_decode(p, end, &size) > 0xffff ? 2 : 1; + p += size; + } + return count; +} + +/** Writes the UTF-16 units of the `len` bytes at `data` to `out`, which has + * room for scr_intl_utf16_units(data, len) of them, and returns how many it + * wrote: always that count. */ +static inline size_t scr_intl_utf16_fill(const unsigned char *data, size_t len, uint16_t *out) { + const unsigned char *end = data + len; + size_t at = 0; + for (const unsigned char *p = data; p < end;) { + size_t size; + uint32_t cp = scr_intl_decode(p, end, &size); + if (cp > 0xffff) { + out[at++] = (uint16_t)(0xd800 + ((cp - 0x10000) >> 10)); + out[at++] = (uint16_t)(0xdc00 + ((cp - 0x10000) & 0x3ff)); + } else { + out[at++] = (uint16_t)cp; + } + p += size; + } + return at; +} + +#endif /* SCR_INTL_UTF16_H */ diff --git a/packages/runtime/src/scr_lib.c b/packages/runtime/src/scr_lib.c index 67f18c0d8..a46868b20 100644 --- a/packages/runtime/src/scr_lib.c +++ b/packages/runtime/src/scr_lib.c @@ -626,9 +626,11 @@ ScrArr *scr_process_argv(void) { int argc = scr_lib_arg_count(); scr_argv_arr = scr_arr_new(SCR_ELEM_STR, (size_t)argc + 1); scr_arr_push_ref(scr_argv_arr, scr_str_new("scriptc", 7)); + /* Node decodes each argument as UTF-8, replacing malformed bytes with + * U+FFFD. */ for (int i = 0; i < argc; i++) { const char *a = scr_lib_arg(i); - scr_arr_push_ref(scr_argv_arr, scr_str_new(a, strlen(a))); + scr_arr_push_ref(scr_argv_arr, scr_str_from_utf8_lossy((const uint8_t *)a, strlen(a))); } } return scr_arr_retain(scr_argv_arr); @@ -2828,7 +2830,7 @@ void scr_process_set_title(const ScrStr *title) { ScrStr *scr_process_argv0(void) { if (scr_lib_arg_count() < 1) return scr_str_new("", 0); const char *a = scr_lib_arg(0); - return scr_str_new(a, strlen(a)); + return scr_str_from_utf8_lossy((const uint8_t *)a, strlen(a)); } static SCR_TL ScrDyn *scr_release_object; @@ -6650,7 +6652,8 @@ ScrScandir *scr_fs_scandir(ScrStr *path) { scr_trap("scriptc: out of memory\n"); } } - s->names[s->len] = scr_str_new(ent->d_name, strlen(ent->d_name)); + /* Decoded as UTF-8 with U+FFFD for malformed bytes, as Node does. */ + s->names[s->len] = scr_str_from_utf8_lossy((const uint8_t *)ent->d_name, strlen(ent->d_name)); s->kinds[s->len] = kind; s->len++; } @@ -6896,7 +6899,9 @@ bool scr_dir_read(ScrDir *d, ScrStr **name, double *kind) { return false; } if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0) continue; - *name = scr_str_new(ent->d_name, strlen(ent->d_name)); + /* Node decodes an entry name as UTF-8, replacing malformed bytes with + * U+FFFD. */ + *name = scr_str_from_utf8_lossy((const uint8_t *)ent->d_name, strlen(ent->d_name)); *kind = scr_dir_entry_kind(d, ent); return true; } diff --git a/packages/runtime/test/intl-utf16.test.ts b/packages/runtime/test/intl-utf16.test.ts new file mode 100644 index 000000000..7c4b564a0 --- /dev/null +++ b/packages/runtime/test/intl-utf16.test.ts @@ -0,0 +1,75 @@ +import { execFile, execFileSync } from "node:child_process"; +import { mkdir } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { beforeAll, expect, test } from "vitest"; + +const execFileAsync = promisify(execFile); +const testDir = import.meta.dirname; +const bin = join(testDir, "build", "test_intl_utf16"); + +// The ICU unit's WTF-8 decode (scr_intl_utf16.h), built with ASan so a +// unit written past the count it reports fails the run. +beforeAll(async () => { + await mkdir(join(testDir, "build"), { recursive: true }); + await execFileAsync("clang", [ + "-std=c11", + "-O1", + "-Wall", + "-Wextra", + "-fsanitize=address", + "-I", + join(testDir, "../src"), + "-o", + bin, + join(testDir, "test_intl_utf16.c"), + ]); +}, 120_000); + +const CASES: number[][] = [ + [0x61, 0x62], + [0xc3, 0xa9, 0xe2, 0x82, 0xac, 0xf0, 0x9f, 0x98, 0x80], + [0x61, 0xff, 0x62], + [0xc0, 0xaf], + [0xc1, 0xbf, 0x41], + [0xe0, 0x80, 0x80], + [0xe0, 0x9f, 0xbf], + [0xf0, 0x80, 0x80, 0x41], + [0xf0, 0x8f, 0xbf, 0xbf], + [0xf4, 0x90, 0x80, 0x80], + [0xf5, 0x80, 0x80, 0x80], + [0x80, 0xbf, 0x20], + [0xe2, 0x82], + [0xe2, 0x82, 0x41], + [0xf0, 0x9f, 0x98], + [0xf0, 0x9f], + [0xc3], + [0x74, 0x72, 0x75, 0x6e, 0x63, 0xe2, 0x82], + [0x67, 0x6f, 0xff, 0xe0, 0x80, 0x77, 0x6f], +]; + +/** A WTF-8 lone surrogate (ED A0..BF xx) decodes to its unit, and its + * truncated prefix is one maximal subpart; every other sequence decodes as + * Node's TextDecoder decodes it. */ +const SURROGATES: [number[], string][] = [ + [[0x73, 0xed, 0xa0, 0x80, 0x78], "0073 d800 0078"], + [[0xed, 0xbf, 0xbf], "dfff"], + [[0xed, 0xa0], "fffd"], +]; + +function hexUnits(text: string): string { + return [...Array(text.length).keys()] + .map((i) => text.charCodeAt(i).toString(16).padStart(4, "0")) + .join(" "); +} + +test("malformed sequences decode to U+FFFD as Node's UTF-8 decoder does, and the count is the units written", () => { + const all = [...CASES, ...SURROGATES.map(([bytes]) => bytes)]; + const input = all.map((bytes) => Buffer.from(bytes).toString("hex")).join("\n") + "\n"; + const lines = execFileSync(bin, [], { input, encoding: "utf8" }).trimEnd().split("\n"); + const decoder = new TextDecoder(); + expect(lines).toEqual([ + ...CASES.map((bytes) => hexUnits(decoder.decode(Uint8Array.from(bytes)))), + ...SURROGATES.map(([, units]) => units), + ]); +}); diff --git a/packages/runtime/test/test_intl_utf16.c b/packages/runtime/test/test_intl_utf16.c new file mode 100644 index 000000000..08f88ef73 --- /dev/null +++ b/packages/runtime/test/test_intl_utf16.c @@ -0,0 +1,33 @@ +/* scr_intl_utf16.h over byte sequences from stdin, one per line as hex + * (`61ff62`). Prints each sequence's UTF-16 units as hex, space-separated, + * and exits 1 when the count and the units written disagree. */ +#include "scr_intl_utf16.h" + +#include +#include +#include + +int main(void) { + char line[4096]; + while (fgets(line, sizeof line, stdin) != NULL) { + unsigned char bytes[2048]; + size_t len = 0; + for (const char *p = line; p[0] != '\0' && p[0] != '\n' && p[1] != '\0'; p += 2) { + unsigned int byte; + if (sscanf(p, "%2x", &byte) != 1) return 2; + bytes[len++] = (unsigned char)byte; + } + size_t count = scr_intl_utf16_units(bytes, len); + uint16_t *units = malloc((count + 1) * sizeof *units); + if (units == NULL) return 2; + size_t written = scr_intl_utf16_fill(bytes, len, units); + if (written != count) { + printf("count %zu written %zu\n", count, written); + return 1; + } + for (size_t i = 0; i < written; i++) printf(i == 0 ? "%04x" : " %04x", units[i]); + printf("\n"); + free(units); + } + return 0; +} diff --git a/tests/corpus/intl-worker-first-use.ts b/tests/corpus/intl-worker-first-use.ts new file mode 100644 index 000000000..8e8621ac5 --- /dev/null +++ b/tests/corpus/intl-worker-first-use.ts @@ -0,0 +1,36 @@ +// @requires-icu +// @env: TZ=UTC,LC_ALL=en_US.UTF-8 +// The program's first ICU calls come from eight worker threads at once: +// each worker formats a number and a date as its first Intl use. Every +// worker answers, and none reports the ICU data as missing while another +// thread is still loading it. +import { Worker, isMainThread, parentPort, workerData } from "node:worker_threads"; + +if (isMainThread) { + const results: string[] = []; + let pending = 8; + for (let i = 0; i < 8; i++) { + const worker = new Worker(new URL(import.meta.url), { workerData: i }); + worker.on("message", (text: string) => { + results.push(text); + }); + worker.on("exit", () => { + pending--; + if (pending === 0) { + results.sort(); + for (const text of results) console.log(text); + } + }); + } +} else { + const index = workerData as number; + let text: string; + try { + const number = (1234567.5 + index).toLocaleString(); + const date = new Date(Date.UTC(2024, 0, 1 + index, 9, 30)).toLocaleString(); + text = `${index} ${number} ${date}`; + } catch (error) { + text = `${index} threw ${error instanceof Error ? error.message : String(error)}`; + } + parentPort!.postMessage(text); +} diff --git a/tests/harness/corpus-costs/intl-worker-first-use.ts.json b/tests/harness/corpus-costs/intl-worker-first-use.ts.json new file mode 100644 index 000000000..29984d425 --- /dev/null +++ b/tests/harness/corpus-costs/intl-worker-first-use.ts.json @@ -0,0 +1 @@ +{ "plain": 4393, "san": 189179 } diff --git a/tests/harness/intl-malformed-utf8.test.ts b/tests/harness/intl-malformed-utf8.test.ts new file mode 100644 index 000000000..2b6abaddd --- /dev/null +++ b/tests/harness/intl-malformed-utf8.test.ts @@ -0,0 +1,73 @@ +/* Strings that come from bytes the OS hands over (directory entry names, + * command-line arguments) decode as UTF-8 with U+FFFD for each malformed + * subsequence, as Node decodes them, and a word segmenter over them matches + * Node: overlong forms, truncated sequences, stray continuation bytes and + * UTF-8-encoded surrogates included. */ +import { execFile } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; + +const execFileAsync = promisify(execFile); + +const PROGRAM = ` +import { readdirSync } from "node:fs"; +const words = new Intl.Segmenter(undefined, { granularity: "word" }); +function show(label: string, text: string): void { + const parts = [...words.segment(text)].map((part) => JSON.stringify(part.segment) + "@" + part.index); + console.log(label, JSON.stringify(text), text.length, parts.join(" ")); +} +const names = readdirSync(process.argv[2]!).sort(); +for (const name of names) show("name", name); +show("arg", process.argv[3]!); +`; + +const NAMES = [ + [0x61, 0xff, 0x62, 0x2e, 0x74, 0x78, 0x74], + [0x6f, 0x76, 0x65, 0x72, 0xf0, 0x80, 0x80, 0x41, 0x6c, 0x6f, 0x6e, 0x67], + [0x74, 0x72, 0x75, 0x6e, 0x63, 0xe2, 0x82], + [0x73, 0x75, 0x72, 0xed, 0xa0, 0x80, 0x78], + [0x6f, 0x6b, 0x2d, 0xc3, 0xa9, 0x2e, 0x74, 0x78, 0x74], + [0x63, 0x6f, 0x6e, 0x74, 0x80, 0xbf, 0x20, 0x77, 0x6f, 0x72, 0x64], +]; + +test.skipIf(process.platform !== "linux" || process.arch !== "x64")( + "malformed UTF-8 from directory entries and arguments segments as in Node", + async () => { + const dir = mkdtempSync(join(tmpdir(), "scriptc-intl-malformed-")); + try { + const entry = join(dir, "main.ts"); + writeFileSync(entry, PROGRAM); + const files = join(dir, "files"); + mkdirSync(files); + for (const name of NAMES) + writeFileSync(Buffer.concat([Buffer.from(files + "/"), Buffer.from(name)]), ""); + const result = await compile(entry, { + outPath: join(dir, "out", "main"), + outDir: join(dir, "out"), + sanitize: process.env["SCRIPTC_SAN"] === "1", + }); + expect(result.ok, !result.ok ? JSON.stringify(result.diagnostics, null, 2) : "").toBe(true); + if (!result.ok) return; + // The argument's bytes are not valid UTF-8, so the shell passes them. + const run = (argv0: string[]) => + execFileAsync( + "/bin/sh", + ["-c", `"$@" "$(printf 'go\\377\\340\\200wo rd')"`, "sh", ...argv0, files], + { + env: { ...process.env, LC_ALL: "en_US.UTF-8" }, + }, + ); + const node = await run([process.execPath, entry]); + const native = await run([result.binaryPath]); + expect(native.stdout).toBe(node.stdout); + expect(node.stdout).toContain("�"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, + 300_000, +);