Decode malformed UTF-8 to U+FFFD in the ICU converter and at directory and argument boundaries

The ICU unit's WTF-8 to UTF-16 conversion counted units from lead bytes
alone. A 4-byte-shaped encoding of a BMP value counted two units and wrote
one, so ICU received a length past the units written, and word segments
built from those ranges could return prior heap contents. The conversion now
decodes each maximal ill-formed subsequence to U+FFFD, as the WHATWG UTF-8
decoder does, keeps WTF-8's 3-byte lone surrogates as their unit, and
reports the units it wrote. The decoder lives in scr_intl_utf16.h.

Directory entry names (readdir, opendir) and process.argv entries now decode
as UTF-8 with U+FFFD for malformed bytes, as Node decodes them, instead of
entering strings as raw bytes.

test_intl_utf16.c checks the decoder against Node's TextDecoder over
malformed sequences. intl-malformed-utf8.test.ts compares a program that
segments malformed directory names and an argument against Node.

packages/compiler/src/backend/native/executable.ts

@@ -718,7 +718,17 @@ export async function compileCInternal(
   let icuArchive = icu
     ? (icuPack?.archive ?? icuArchivePath(driver, vendorBuildIdentity, vendorCacheRoot))
     : null;
-  let icuIncludeDirs: readonly string[] = icuPack === null ? [] : [icuPack.include];
+  // scr_intl.c's ICU headers. They are known before the runtime-object flags
+  // are: those flags compile every runtime unit, so without them a cached
+  // scr_intl.o would take whatever ICU headers the host has.
+  const icuSource =
+    icu && icuPack === null ? join(await ensureIcuSources(vendorCacheRoot), "source") : null;
+  const icuIncludeDirs: readonly string[] =
+    icuPack !== null
+      ? [icuPack.include]
+      : icuSource !== null
+        ? [join(icuSource, "common"), join(icuSource, "i18n")]
+        : [];
 
   // ours would collide; see scr_regex.c) — the standalone objects are for
   // static builds only.
@@ -784,8 +794,6 @@ export async function compileCInternal(
           ? await materialize()
           : await stageVendorInputs(materialize, join(stageRoot, "icu"));
       icuArchive = paths[0]!;
-      const icuSource = join(await ensureIcuSources(materializeCacheRoot), "source");
-      icuIncludeDirs = [join(icuSource, "common"), join(icuSource, "i18n")];
     }
     if (regex) {
       for (const object of lreObjectPaths(
@@ -924,6 +932,7 @@ export async function compileCInternal(
     ...(regex ? ["-I", vendorEngineDir()] : []),
     ...(zlibObjects.length > 0 ? ["-I", vendorZlibDir()] : []),
     ...(tlsArchive !== null ? ["-I", join(vendorTlsDir(), "include")] : []),
+    ...icuIncludeDirs.flatMap((dir) => ["-I", dir]),
   ];
   // Keep sanitizer runtime objects optimized in both postures. Large dev
   // programs still compile at O0, avoiding an expensive LLVM optimization

packages/compiler/test/ts7/baselines/order-parity/tests/corpus/intl-worker-first-use.ts.json

@@ -0,0 +1,6 @@
+{
+ "order": [
+  "<repo>/tests/corpus/intl-worker-first-use.ts"
+ ],
+ "diags": []
+}

packages/runtime/src/scr_intl.c

@@ -23,6 +23,7 @@
  * That is what lets the zero-cost gate assert that a non-Intl binary links
  * no ICU symbol at all. */
 #include "scr_runtime.h"
+#include "scr_intl_utf16.h"
 #include "scr_number.h"
 
 #include <unicode/ubrk.h>
@@ -36,6 +37,7 @@
 
 #include <limits.h>
 #include <math.h>
+#include <pthread.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -51,9 +53,12 @@
 
 /* ── data ─────────────────────────────────────────────────────────────
  * ICU keeps the mapped data alive for the process lifetime, so the load
- * happens exactly once and never unmaps. */
+ * happens exactly once and never unmaps. Worker threads can make the first
+ * ICU call at the same time: pthread_once runs the load on one of them and
+ * holds the others until it has finished, so none reads a partial result. */
 
-static int scr_intl_data_state; /* 0 = not tried, 1 = ready, 2 = failed */
+static pthread_once_t scr_intl_data_once = PTHREAD_ONCE_INIT;
+static bool scr_intl_data_ready;
 
 /* The default locale every formatter and the default word segmenter use,
  * resolved with the data (scr_intl_default_locale). */
@@ -117,12 +122,10 @@ static void scr_intl_default_locale(void) {
   snprintf(scr_intl_locale, sizeof scr_intl_locale, "%s", name);
 }
 
-/** True once ICU can resolve data: the file beside the executable, else
- * whatever ICU's own search finds (ICU_DATA). Never throws; the callers
- * do, with the same message. */
-static bool scr_intl_data(void) {
-  if (scr_intl_data_state != 0) return scr_intl_data_state == 1;
-  scr_intl_data_state = 2;
+/** Points ICU at its data (the file beside the executable, else whatever
+ * ICU's own search finds, ICU_DATA), probes it and resolves the default
+ * locale. Runs once per process (scr_intl_data). */
+static void scr_intl_data_load(void) {
   char dir[PATH_MAX];
   if (scr_intl_exe_dir(dir, sizeof dir)) {
     char path[PATH_MAX];
@@ -136,11 +139,17 @@ static bool scr_intl_data(void) {
    * Everything else this unit formats resolves through the same data. */
   UErrorCode status = U_ZERO_ERROR;
   UNumberFormat *probe = unum_open(UNUM_DECIMAL, NULL, 0, NULL, NULL, &status);
-  if (U_FAILURE(status) || probe == NULL) return false;
+  if (U_FAILURE(status) || probe == NULL) return;
   unum_close(probe);
   scr_intl_default_locale();
-  scr_intl_data_state = 1;
-  return true;
+  scr_intl_data_ready = true;
+}
+
+/** True once ICU can resolve data. Never throws; the callers do, with the
+ * same message. */
+static bool scr_intl_data(void) {
+  (void)pthread_once(&scr_intl_data_once, scr_intl_data_load);
+  return scr_intl_data_ready;
 }
 
 /** Throw the missing-data Error (Node cannot reach this: its ICU data is
@@ -155,62 +164,22 @@ static void scr_intl_no_data(void) {
 /* ── WTF-8 ↔ UTF-16 ───────────────────────────────────────────────────
  * ScrStr storage is canonical WTF-8 (a lone surrogate is its 3-byte
  * generalized form), so both directions preserve lone surrogates and
- * every public offset stays a UTF-16 unit count. */
+ * every public offset stays a UTF-16 unit count. Bytes that are not WTF-8
+ * decode to U+FFFD (scr_intl_utf16.h), so the UTF-16 copy holds exactly the
+ * units it reports. */
 
-static int32_t scr_intl_units(const char *data, size_t len) {
-  int32_t units = 0;
-  for (size_t at = 0; at < len;) {
-    unsigned char byte = (unsigned char)data[at];
-    if (byte < 0x80) at += 1;
-    else if (byte < 0xe0) at += 2;
-    else if (byte < 0xf0) at += 3;
-    else {
-      at += 4;
-      units++; /* a non-BMP code point is two units */
-    }
-    units++;
-  }
-  return units;
-}
-
-/** A UTF-16 copy of `s`, NUL-terminated; *units receives its length.
- * Never NULL for a successful call (an empty string allocates one unit). */
+/** A UTF-16 copy of `s`, NUL-terminated; *units receives the number of
+ * units written. Never NULL for a successful call (an empty string
+ * allocates one unit). */
 static UChar *scr_intl_to_utf16(const ScrStr *s, int32_t *units) {
-  const char *data = s->data;
-  size_t len = s->len;
-  int32_t count = scr_intl_units(data, len);
-  UChar *text = malloc(((size_t)count + 1) * sizeof *text);
+  const unsigned char *data = (const unsigned char *)s->data;
+  size_t count = scr_intl_utf16_units(data, s->len);
+  if (count > INT32_MAX - 1) scr_trap("scriptc: string too long for Intl\n");
+  UChar *text = malloc((count + 1) * sizeof *text);
   if (text == NULL) scr_trap("scriptc: out of memory\n");
-  int32_t at = 0;
-  for (size_t pos = 0; pos < len;) {
-    unsigned char byte = (unsigned char)data[pos];
-    uint32_t cp;
-    size_t size;
-    if (byte < 0x80) {
-      cp = byte;
-      size = 1;
-    } else if (byte < 0xe0) {
-      cp = ((uint32_t)(byte & 0x1f) << 6) | ((unsigned char)data[pos + 1] & 0x3fu);
-      size = 2;
-    } else if (byte < 0xf0) {
-      cp = ((uint32_t)(byte & 0x0f) << 12) | (((unsigned char)data[pos + 1] & 0x3fu) << 6) |
-           ((unsigned char)data[pos + 2] & 0x3fu);
-      size = 3;
-    } else {
-      cp = ((uint32_t)(byte & 0x07) << 18) | (((unsigned char)data[pos + 1] & 0x3fu) << 12) |
-           (((unsigned char)data[pos + 2] & 0x3fu) << 6) | ((unsigned char)data[pos + 3] & 0x3fu);
-      size = 4;
-    }
-    if (cp > 0xffff) {
-      text[at++] = (UChar)(0xd800 + ((cp - 0x10000) >> 10));
-      text[at++] = (UChar)(0xdc00 + ((cp - 0x10000) & 0x3ff));
-    } else {
-      text[at++] = (UChar)cp;
-    }
-    pos += size;
-  }
-  text[at] = 0;
-  *units = count;
+  size_t written = scr_intl_utf16_fill(data, s->len, (uint16_t *)text);
+  text[written] = 0;
+  *units = (int32_t)written;
   return text;
 }
 

packages/runtime/src/scr_intl_utf16.h

@@ -0,0 +1,82 @@
+/* scriptc runtime — WTF-8 to UTF-16 for the ICU unit (scr_intl.c). Kept
+ * apart from ICU's headers so packages/runtime/test can check the decode
+ * on its own (test_intl_utf16.c). */
+#ifndef SCR_INTL_UTF16_H
+#define SCR_INTL_UTF16_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+/** The code point of the WTF-8 sequence at `p` (before `end`), and its byte
+ * count in *size. A malformed sequence (an overlong form, a value above
+ * U+10FFFF, a stray or missing continuation byte) is decoded one maximal
+ * subpart at a time to U+FFFD, as the WHATWG UTF-8 decoder does. The 3-byte
+ * form of a surrogate is WTF-8's lone surrogate and decodes to that unit. */
+static inline uint32_t scr_intl_decode(const unsigned char *p, const unsigned char *end, size_t *size) {
+  unsigned char lead = p[0];
+  *size = 1;
+  if (lead < 0x80) return lead;
+  size_t need;
+  unsigned char lo = 0x80, hi = 0xbf;
+  uint32_t cp;
+  if (lead >= 0xc2 && lead <= 0xdf) {
+    need = 1;
+    cp = lead & 0x1fu;
+  } else if (lead >= 0xe0 && lead <= 0xef) {
+    need = 2;
+    cp = lead & 0x0fu;
+    if (lead == 0xe0) lo = 0xa0;
+  } else if (lead >= 0xf0 && lead <= 0xf4) {
+    need = 3;
+    cp = lead & 0x07u;
+    if (lead == 0xf0) lo = 0x90;
+    if (lead == 0xf4) hi = 0x8f;
+  } else {
+    return 0xfffd;
+  }
+  for (size_t i = 1; i <= need; i++) {
+    if (p + i >= end || p[i] < lo || p[i] > hi) {
+      *size = i;
+      return 0xfffd;
+    }
+    cp = (cp << 6) | (p[i] & 0x3fu);
+    lo = 0x80;
+    hi = 0xbf;
+  }
+  *size = need + 1;
+  return cp;
+}
+
+/** The number of UTF-16 units the `len` bytes at `data` decode to. */
+static inline size_t scr_intl_utf16_units(const unsigned char *data, size_t len) {
+  const unsigned char *end = data + len;
+  size_t count = 0;
+  for (const unsigned char *p = data; p < end;) {
+    size_t size;
+    count += scr_intl_decode(p, end, &size) > 0xffff ? 2 : 1;
+    p += size;
+  }
+  return count;
+}
+
+/** Writes the UTF-16 units of the `len` bytes at `data` to `out`, which has
+ * room for scr_intl_utf16_units(data, len) of them, and returns how many it
+ * wrote: always that count. */
+static inline size_t scr_intl_utf16_fill(const unsigned char *data, size_t len, uint16_t *out) {
+  const unsigned char *end = data + len;
+  size_t at = 0;
+  for (const unsigned char *p = data; p < end;) {
+    size_t size;
+    uint32_t cp = scr_intl_decode(p, end, &size);
+    if (cp > 0xffff) {
+      out[at++] = (uint16_t)(0xd800 + ((cp - 0x10000) >> 10));
+      out[at++] = (uint16_t)(0xdc00 + ((cp - 0x10000) & 0x3ff));
+    } else {
+      out[at++] = (uint16_t)cp;
+    }
+    p += size;
+  }
+  return at;
+}
+
+#endif /* SCR_INTL_UTF16_H */

packages/runtime/src/scr_lib.c

@@ -626,9 +626,11 @@ ScrArr *scr_process_argv(void) {
     int argc = scr_lib_arg_count();
     scr_argv_arr = scr_arr_new(SCR_ELEM_STR, (size_t)argc + 1);
     scr_arr_push_ref(scr_argv_arr, scr_str_new("scriptc", 7));
+    /* Node decodes each argument as UTF-8, replacing malformed bytes with
+     * U+FFFD. */
     for (int i = 0; i < argc; i++) {
       const char *a = scr_lib_arg(i);
-      scr_arr_push_ref(scr_argv_arr, scr_str_new(a, strlen(a)));
+      scr_arr_push_ref(scr_argv_arr, scr_str_from_utf8_lossy((const uint8_t *)a, strlen(a)));
     }
   }
   return scr_arr_retain(scr_argv_arr);
@@ -2828,7 +2830,7 @@ void scr_process_set_title(const ScrStr *title) {
 ScrStr *scr_process_argv0(void) {
   if (scr_lib_arg_count() < 1) return scr_str_new("", 0);
   const char *a = scr_lib_arg(0);
-  return scr_str_new(a, strlen(a));
+  return scr_str_from_utf8_lossy((const uint8_t *)a, strlen(a));
 }
 
 static SCR_TL ScrDyn *scr_release_object;
@@ -6650,7 +6652,8 @@ ScrScandir *scr_fs_scandir(ScrStr *path) {
         scr_trap("scriptc: out of memory\n");
       }
     }
-    s->names[s->len] = scr_str_new(ent->d_name, strlen(ent->d_name));
+    /* Decoded as UTF-8 with U+FFFD for malformed bytes, as Node does. */
+    s->names[s->len] = scr_str_from_utf8_lossy((const uint8_t *)ent->d_name, strlen(ent->d_name));
     s->kinds[s->len] = kind;
     s->len++;
   }
@@ -6896,7 +6899,9 @@ bool scr_dir_read(ScrDir *d, ScrStr **name, double *kind) {
       return false;
     }
     if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0) continue;
-    *name = scr_str_new(ent->d_name, strlen(ent->d_name));
+    /* Node decodes an entry name as UTF-8, replacing malformed bytes with
+     * U+FFFD. */
+    *name = scr_str_from_utf8_lossy((const uint8_t *)ent->d_name, strlen(ent->d_name));
     *kind = scr_dir_entry_kind(d, ent);
     return true;
   }

packages/runtime/test/intl-utf16.test.ts

@@ -0,0 +1,75 @@
+import { execFile, execFileSync } from "node:child_process";
+import { mkdir } from "node:fs/promises";
+import { join } from "node:path";
+import { promisify } from "node:util";
+import { beforeAll, expect, test } from "vitest";
+
+const execFileAsync = promisify(execFile);
+const testDir = import.meta.dirname;
+const bin = join(testDir, "build", "test_intl_utf16");
+
+// The ICU unit's WTF-8 decode (scr_intl_utf16.h), built with ASan so a
+// unit written past the count it reports fails the run.
+beforeAll(async () => {
+  await mkdir(join(testDir, "build"), { recursive: true });
+  await execFileAsync("clang", [
+    "-std=c11",
+    "-O1",
+    "-Wall",
+    "-Wextra",
+    "-fsanitize=address",
+    "-I",
+    join(testDir, "../src"),
+    "-o",
+    bin,
+    join(testDir, "test_intl_utf16.c"),
+  ]);
+}, 120_000);
+
+const CASES: number[][] = [
+  [0x61, 0x62],
+  [0xc3, 0xa9, 0xe2, 0x82, 0xac, 0xf0, 0x9f, 0x98, 0x80],
+  [0x61, 0xff, 0x62],
+  [0xc0, 0xaf],
+  [0xc1, 0xbf, 0x41],
+  [0xe0, 0x80, 0x80],
+  [0xe0, 0x9f, 0xbf],
+  [0xf0, 0x80, 0x80, 0x41],
+  [0xf0, 0x8f, 0xbf, 0xbf],
+  [0xf4, 0x90, 0x80, 0x80],
+  [0xf5, 0x80, 0x80, 0x80],
+  [0x80, 0xbf, 0x20],
+  [0xe2, 0x82],
+  [0xe2, 0x82, 0x41],
+  [0xf0, 0x9f, 0x98],
+  [0xf0, 0x9f],
+  [0xc3],
+  [0x74, 0x72, 0x75, 0x6e, 0x63, 0xe2, 0x82],
+  [0x67, 0x6f, 0xff, 0xe0, 0x80, 0x77, 0x6f],
+];
+
+/** A WTF-8 lone surrogate (ED A0..BF xx) decodes to its unit, and its
+ * truncated prefix is one maximal subpart; every other sequence decodes as
+ * Node's TextDecoder decodes it. */
+const SURROGATES: [number[], string][] = [
+  [[0x73, 0xed, 0xa0, 0x80, 0x78], "0073 d800 0078"],
+  [[0xed, 0xbf, 0xbf], "dfff"],
+  [[0xed, 0xa0], "fffd"],
+];
+
+function hexUnits(text: string): string {
+  return [...Array(text.length).keys()]
+    .map((i) => text.charCodeAt(i).toString(16).padStart(4, "0"))
+    .join(" ");
+}
+
+test("malformed sequences decode to U+FFFD as Node's UTF-8 decoder does, and the count is the units written", () => {
+  const all = [...CASES, ...SURROGATES.map(([bytes]) => bytes)];
+  const input = all.map((bytes) => Buffer.from(bytes).toString("hex")).join("\n") + "\n";
+  const lines = execFileSync(bin, [], { input, encoding: "utf8" }).trimEnd().split("\n");
+  const decoder = new TextDecoder();
+  expect(lines).toEqual([
+    ...CASES.map((bytes) => hexUnits(decoder.decode(Uint8Array.from(bytes)))),
+    ...SURROGATES.map(([, units]) => units),
+  ]);
+});

packages/runtime/test/test_intl_utf16.c

@@ -0,0 +1,33 @@
+/* scr_intl_utf16.h over byte sequences from stdin, one per line as hex
+ * (`61ff62`). Prints each sequence's UTF-16 units as hex, space-separated,
+ * and exits 1 when the count and the units written disagree. */
+#include "scr_intl_utf16.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+int main(void) {
+  char line[4096];
+  while (fgets(line, sizeof line, stdin) != NULL) {
+    unsigned char bytes[2048];
+    size_t len = 0;
+    for (const char *p = line; p[0] != '\0' && p[0] != '\n' && p[1] != '\0'; p += 2) {
+      unsigned int byte;
+      if (sscanf(p, "%2x", &byte) != 1) return 2;
+      bytes[len++] = (unsigned char)byte;
+    }
+    size_t count = scr_intl_utf16_units(bytes, len);
+    uint16_t *units = malloc((count + 1) * sizeof *units);
+    if (units == NULL) return 2;
+    size_t written = scr_intl_utf16_fill(bytes, len, units);
+    if (written != count) {
+      printf("count %zu written %zu\n", count, written);
+      return 1;
+    }
+    for (size_t i = 0; i < written; i++) printf(i == 0 ? "%04x" : " %04x", units[i]);
+    printf("\n");
+    free(units);
+  }
+  return 0;
+}

tests/corpus/intl-worker-first-use.ts

@@ -0,0 +1,36 @@
+// @requires-icu
+// @env: TZ=UTC,LC_ALL=en_US.UTF-8
+// The program's first ICU calls come from eight worker threads at once:
+// each worker formats a number and a date as its first Intl use. Every
+// worker answers, and none reports the ICU data as missing while another
+// thread is still loading it.
+import { Worker, isMainThread, parentPort, workerData } from "node:worker_threads";
+
+if (isMainThread) {
+  const results: string[] = [];
+  let pending = 8;
+  for (let i = 0; i < 8; i++) {
+    const worker = new Worker(new URL(import.meta.url), { workerData: i });
+    worker.on("message", (text: string) => {
+      results.push(text);
+    });
+    worker.on("exit", () => {
+      pending--;
+      if (pending === 0) {
+        results.sort();
+        for (const text of results) console.log(text);
+      }
+    });
+  }
+} else {
+  const index = workerData as number;
+  let text: string;
+  try {
+    const number = (1234567.5 + index).toLocaleString();
+    const date = new Date(Date.UTC(2024, 0, 1 + index, 9, 30)).toLocaleString();
+    text = `${index} ${number} ${date}`;
+  } catch (error) {
+    text = `${index} threw ${error instanceof Error ? error.message : String(error)}`;
+  }
+  parentPort!.postMessage(text);
+}

tests/harness/corpus-costs/intl-worker-first-use.ts.json

@@ -0,0 +1 @@
+{ "plain": 4393, "san": 189179 }

tests/harness/intl-malformed-utf8.test.ts

@@ -0,0 +1,73 @@
+/* Strings that come from bytes the OS hands over (directory entry names,
+ * command-line arguments) decode as UTF-8 with U+FFFD for each malformed
+ * subsequence, as Node decodes them, and a word segmenter over them matches
+ * Node: overlong forms, truncated sequences, stray continuation bytes and
+ * UTF-8-encoded surrogates included. */
+import { execFile } from "node:child_process";
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { promisify } from "node:util";
+import { expect, test } from "vitest";
+import { compile } from "@scriptc/compiler";
+
+const execFileAsync = promisify(execFile);
+
+const PROGRAM = `
+import { readdirSync } from "node:fs";
+const words = new Intl.Segmenter(undefined, { granularity: "word" });
+function show(label: string, text: string): void {
+  const parts = [...words.segment(text)].map((part) => JSON.stringify(part.segment) + "@" + part.index);
+  console.log(label, JSON.stringify(text), text.length, parts.join(" "));
+}
+const names = readdirSync(process.argv[2]!).sort();
+for (const name of names) show("name", name);
+show("arg", process.argv[3]!);
+`;
+
+const NAMES = [
+  [0x61, 0xff, 0x62, 0x2e, 0x74, 0x78, 0x74],
+  [0x6f, 0x76, 0x65, 0x72, 0xf0, 0x80, 0x80, 0x41, 0x6c, 0x6f, 0x6e, 0x67],
+  [0x74, 0x72, 0x75, 0x6e, 0x63, 0xe2, 0x82],
+  [0x73, 0x75, 0x72, 0xed, 0xa0, 0x80, 0x78],
+  [0x6f, 0x6b, 0x2d, 0xc3, 0xa9, 0x2e, 0x74, 0x78, 0x74],
+  [0x63, 0x6f, 0x6e, 0x74, 0x80, 0xbf, 0x20, 0x77, 0x6f, 0x72, 0x64],
+];
+
+test.skipIf(process.platform !== "linux" || process.arch !== "x64")(
+  "malformed UTF-8 from directory entries and arguments segments as in Node",
+  async () => {
+    const dir = mkdtempSync(join(tmpdir(), "scriptc-intl-malformed-"));
+    try {
+      const entry = join(dir, "main.ts");
+      writeFileSync(entry, PROGRAM);
+      const files = join(dir, "files");
+      mkdirSync(files);
+      for (const name of NAMES)
+        writeFileSync(Buffer.concat([Buffer.from(files + "/"), Buffer.from(name)]), "");
+      const result = await compile(entry, {
+        outPath: join(dir, "out", "main"),
+        outDir: join(dir, "out"),
+        sanitize: process.env["SCRIPTC_SAN"] === "1",
+      });
+      expect(result.ok, !result.ok ? JSON.stringify(result.diagnostics, null, 2) : "").toBe(true);
+      if (!result.ok) return;
+      // The argument's bytes are not valid UTF-8, so the shell passes them.
+      const run = (argv0: string[]) =>
+        execFileAsync(
+          "/bin/sh",
+          ["-c", `"$@" "$(printf 'go\\377\\340\\200wo rd')"`, "sh", ...argv0, files],
+          {
+            env: { ...process.env, LC_ALL: "en_US.UTF-8" },
+          },
+        );
+      const node = await run([process.execPath, entry]);
+      const native = await run([result.binaryPath]);
+      expect(native.stdout).toBe(node.stdout);
+      expect(node.stdout).toContain("�");
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  },
+  300_000,
+);