Match Node 24's node:string_decoder for values known only at run time (#2584)

packages/compiler/ambient/scriptc-node-fallback.d.ts

@@ -932,6 +932,7 @@ interface BufferConstructor {
    * optional byte offset/length shares x's storage (the subarray rule). */
   from(data: ArrayBuffer | SharedArrayBuffer, byteOffset?: number, length?: number): Buffer;
   from(data: Uint8Array | readonly number[]): Buffer;
+  of(...items: number[]): Buffer;
   alloc(size: number, fill?: string | number | Uint8Array, encoding?: BufferEncoding): Buffer;
   compare(buf1: Uint8Array, buf2: Uint8Array): number;
   /* alloc without the zero-fill guarantee. Below Buffer.poolSize >>> 1

packages/compiler/src/backend/llvm/lib-shared.ts

@@ -627,6 +627,7 @@ export const LIB_FN_SYMS: Record<string, string> = {
   "strdec.next": "scr_strdec_next",
   "strdec.end": "scr_strdec_end",
   "strdec.encoding": "scr_strdec_encoding",
+  "strdec.encodingValue": "scr_strdec_encoding_value",
   "stats.isFile": "scr_stats_is_file",
   "stats.isDirectory": "scr_stats_is_dir",
   "stats.isSymbolicLink": "scr_stats_is_symlink",

packages/compiler/src/frontend/lowering/builtins/text-codecs.ts

@@ -159,11 +159,16 @@ export function lowerStringDecoderMethodCall(
     };
   }
   if (!(chunk.type.kind === "bytes" && chunk.type.elem === "u8")) {
-    lowerer.noLowering(
-      `StringDecoder.${name} of '${lowerer.fmt(chunk.type)}' data`,
-      node,
-      "pass a string or typed-array view (narrow unions first)",
-    );
+    // Any other value decides at run time, as Node's write does: a string
+    // passes through, an ArrayBufferView decodes, and anything else throws.
+    const op = name === "end" ? "endDyn" : "writeDyn";
+    return {
+      kind: "call",
+      callee: stringDecoderHelper(lowerer, op, receiver.type.shapeId, loc),
+      args: [receiver, lowerer.coerceInto(node, chunk, DYN)],
+      type: STRING,
+      loc,
+    };
   }
   const helper = stringDecoderHelper(
     lowerer,

packages/compiler/src/frontend/lowering/containers/bytes.ts

@@ -2353,6 +2353,9 @@ function lowerBufferStaticValue(
   const member = access.name.text;
   const loc = locOf(call);
   const args = call.arguments.nodes;
+  // Buffer.of(...items): Node's own `of`, a new unpooled Buffer of the items
+  // converted in order, as Uint8Array.of converts them.
+  if (member === "of") return lowerBytesOf(lowerer, call, "u8");
   if (member === "from") {
     if (
       args.length >= 1 &&

packages/compiler/src/frontend/lowering/lower-classes.ts

@@ -11059,8 +11059,9 @@ export function lowerNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr {
     // starting at 0 (nothing buffered). A literal Buffer encoding folds
     // its alias at compile time; any other encoding argument is
     // normalized at run time (Node's normalizeEncoding, which throws
-    // ERR_UNKNOWN_ENCODING for an unknown name). Omitted means utf8,
-    // Node's default.
+    // ERR_UNKNOWN_ENCODING for an unknown name). An encoding that may not
+    // be a string takes strdec.encodingValue, which formats the error
+    // with %s. Omitted means utf8, Node's default.
     if (symbol && ts.symbolName(symbol) === "StringDecoder" && lowerer.isStdlibSymbol(symbol)) {
       const args = expr.arguments?.nodes ?? [];
       if (args.length > 1) {
@@ -11073,12 +11074,18 @@ export function lowerNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr {
         : encT!.isStringLiteralType()
           ? knownBufEncoding(encT!.value)
           : undefined;
+      const textual = (type: ts.Type): boolean =>
+        type.isUnionType()
+          ? ts.constituentTypes(type).every(textual)
+          : (type.flags &
+              (ts.TypeFlags.StringLike | ts.TypeFlags.Undefined | ts.TypeFlags.Null)) !==
+            0;
       const encoding: IrExpr =
         encName !== undefined
           ? { kind: "strLit", value: encName, type: STRING, loc }
           : {
               kind: "libCall",
-              fn: "strdec.encoding",
+              fn: textual(encT!) ? "strdec.encoding" : "strdec.encodingValue",
               args: [lowerer.lowerExprExpecting(encNode!, DYN)],
               type: STRING,
               loc,

packages/compiler/src/ir/builtin-effects.ts

@@ -91,6 +91,7 @@ export const MAY_THROW_LIB_FNS: ReadonlySet<IrLibFn> = new Set([
   "child.onDyn",
   "text.decoderEncoding",
   "strdec.encoding",
+  "strdec.encodingValue",
   "strdec.writeDyn",
   "strdec.nextDyn",
   "dyn.construct",

packages/compiler/src/ir/builtin-signatures.ts

@@ -5063,6 +5063,10 @@ function buildLibFnSignatures() {
     /** A runtime encoding argument's canonical name (Node's normalizeEncoding:
      * nullish and "" are utf8); an unknown name throws ERR_UNKNOWN_ENCODING. */
     "strdec.encoding": { argTypes: [DYN], result: STRING },
+    /** The same for an encoding that may not be a string (scr_inspect.c):
+     * an object whose `length` is 3 to 9 matches by its string in lower
+     * case; other values throw ERR_UNKNOWN_ENCODING formatted with %s. */
+    "strdec.encodingValue": { argTypes: [DYN], result: STRING },
     // node:readline: the interface handle is f64; the callbacks' func types
     // are program-dependent (the emitter picks the adapter by parameter
     // count), so those slots are null like child.onExit's.

packages/compiler/src/ir/runtime-features.ts

@@ -1231,6 +1231,8 @@ const RUNTIME_FEATURE_RULES = {
       "global.native",
       // ERR_ARG_NOT_ITERABLE renders the value as util.format's %s does.
       "webstream.class",
+      // ERR_UNKNOWN_ENCODING formats an encoding that is not a string with %s.
+      "strdec.encodingValue",
       "webstream.install",
       "util.styleText",
       // ERR_UNHANDLED_ERROR quotes util.inspect of a non-Error payload.

packages/compiler/test/ts7/baselines/order-parity/tests/corpus/string-decoder-untyped-values.js.json

@@ -0,0 +1,6 @@
+{
+ "order": [
+  "<repo>/tests/corpus/string-decoder-untyped-values.js"
+ ],
+ "diags": []
+}

packages/runtime/src/scr_bytes.c

@@ -2873,8 +2873,8 @@ ScrStr *scr_bytes_normalize_encoding(const ScrStr *raw) {
 /* new StringDecoder(encoding) with an encoding known only at run time:
  * Node's normalizeEncoding. undefined, null and "" are utf8; a Buffer
  * encoding name in any ASCII case answers its canonical name (+1); any
- * other string throws ERR_UNKNOWN_ENCODING (NULL). Encodings that are not
- * strings have no lowering. */
+ * other string throws ERR_UNKNOWN_ENCODING (NULL). Encodings that may not
+ * be strings go through scr_strdec_encoding_value (scr_inspect.c). */
 ScrStr *scr_strdec_encoding(const ScrDyn *enc) {
   if (enc->kind == SCR_DYN_UNDEF || enc->kind == SCR_DYN_NULL) return scr_str_new("utf8", 4);
   if (enc->kind != SCR_DYN_STR) {

packages/runtime/src/scr_bytes_strdec_dyn.c

@@ -17,9 +17,17 @@
 static ScrBytes *scr_strdec_dyn_view(const ScrDyn *chunk, bool *is_string) {
   *is_string = chunk->kind == SCR_DYN_STR;
   if (*is_string) return NULL;
+  /* A capsule answers by its materialized value: a typed array or Buffer
+   * decodes, and an array or record throws as below. */
+  if (chunk->kind == SCR_DYN_TYPED_REF) {
+    ScrDyn *materialized = scr_dyn_typed_ref_materialize(chunk);
+    if (!materialized) return NULL;
+    ScrBytes *view = scr_strdec_dyn_view(materialized, is_string);
+    scr_dyn_release(materialized);
+    return view;
+  }
   /* An ArrayBuffer or SharedArrayBuffer is not a view: Node throws. */
-  if (chunk->kind == SCR_DYN_TYPED_REF || chunk->kind == SCR_DYN_BYTES)
-    return scr_bytes_buffer_source(chunk);
+  if (chunk->kind == SCR_DYN_BYTES) return scr_bytes_buffer_source(chunk);
   scr_dyn_arg_type_fail("buf", "an instance of Buffer, TypedArray, or DataView", chunk);
   return NULL;
 }

packages/runtime/src/scr_inspect.c

@@ -3182,6 +3182,63 @@ ScrStr *scr_insp_format(ScrDyn *const *args, size_t argc, const ScrInspFormatOpt
   return insp_format_loop(args, argc, o, &ops);
 }
 
+/* new StringDecoder(encoding) with an encoding that may not be a string:
+ * Node's normalizeEncoding. Strings, undefined and null go through
+ * scr_strdec_encoding. Another value matches only as an object whose
+ * `length` is 3 to 9: its string, in lower case, against the names of that
+ * length (slowCases converts it once). Everything else throws
+ * ERR_UNKNOWN_ENCODING, formatted as Node formats it ("Unknown encoding:
+ * %s"). The canonical name (+1), or NULL with the exception pending. */
+ScrStr *scr_strdec_encoding_value(const ScrDyn *enc) {
+  if (enc->kind == SCR_DYN_STR || enc->kind == SCR_DYN_UNDEF || enc->kind == SCR_DYN_NULL)
+    return scr_strdec_encoding(enc);
+  if (insp_fmt_object(enc)) {
+    ScrStr *name = scr_str_new("length", 6);
+    ScrDyn *key = scr_dyn_new_str(name);
+    scr_str_release(name);
+    ScrDyn *length = scr_dyn_reflect_get((ScrDyn *)enc, key, (ScrDyn *)enc);
+    scr_dyn_release(key);
+    if (!length) return NULL;
+    double count = length->kind == SCR_DYN_NUM ? length->v.num : 0;
+    scr_dyn_release(length);
+    static const struct { int length; const char *from; const char *to; } names[] = {
+      {3, "hex", "hex"},         {4, "utf8", "utf8"},           {4, "ucs2", "utf16le"},
+      {5, "utf-8", "utf8"},      {5, "ascii", "ascii"},         {5, "ucs-2", "utf16le"},
+      {6, "base64", "base64"},   {6, "latin1", "latin1"},       {6, "binary", "latin1"},
+      {7, "utf16le", "utf16le"}, {8, "utf-16le", "utf16le"},    {9, "base64url", "base64url"},
+    };
+    if (count >= 3 && count <= 9 && count == (int)count) {
+      ScrStr *text = scr_dyn_string_coerce_js(enc);
+      if (!text) return NULL;
+      const char *match = NULL;
+      for (size_t i = 0; i < sizeof names / sizeof names[0] && !match; i++) {
+        size_t n = strlen(names[i].from);
+        if (names[i].length != (int)count || text->len != n) continue;
+        bool same = true;
+        for (size_t k = 0; k < n && same; k++) {
+          char c = text->data[k];
+          if (c >= 'A' && c <= 'Z') c = (char)(c - 'A' + 'a');
+          same = c == names[i].from[k];
+        }
+        if (same) match = names[i].to;
+      }
+      scr_str_release(text);
+      if (match) return scr_str_new(match, strlen(match));
+    }
+  }
+  ScrStr *format = scr_str_new("Unknown encoding: %s", 20);
+  ScrDyn *args[2] = {scr_dyn_new_str(format), (ScrDyn *)enc};
+  scr_str_release(format);
+  ScrInspFormatOptions options;
+  scr_insp_format_defaults(&options, "new StringDecoder");
+  ScrStr *message = scr_insp_format(args, 2, &options);
+  scr_dyn_release(args[0]);
+  if (!message) return NULL;
+  scr_throw_error_msg_code(SCR_ERR_TYPE, message->data, message->len, "ERR_UNKNOWN_ENCODING");
+  scr_str_release(message);
+  return NULL;
+}
+
 /* The formatter over arguments that are all strings, numbers, booleans,
  * null or undefined (the compiler proves it from their types): the same
  * rules for those kinds, without the checked-dynamic walk the other

packages/runtime/src/scr_inspect.h

@@ -77,6 +77,9 @@ void scr_insp_format_defaults(ScrInspFormatOptions *o, const char *surface);
  * SC2020 refusal pending for values that have no lowering. */
 bool scr_insp_format_read_options(const ScrDyn *options, ScrInspFormatOptions *o);
 ScrStr *scr_insp_format(ScrDyn *const *args, size_t argc, const ScrInspFormatOptions *o);
+/* new StringDecoder(encoding) for an encoding of any type: Node's
+ * normalizeEncoding, ERR_UNKNOWN_ENCODING formatted with %s (+1 or NULL). */
+ScrStr *scr_strdec_encoding_value(const ScrDyn *enc);
 /* util.format(...pack) and util.formatWithOptions(options, ...pack) over a
  * dyn array of the arguments (+1). */
 ScrStr *scr_insp_format_pack(ScrDyn *pack);

tests/corpus/string-decoder-untyped-values.js

@@ -0,0 +1,63 @@
+// StringDecoder with values whose types are known only at run time, as
+// Node's write, end and normalizeEncoding take them: strings pass through,
+// any ArrayBufferView decodes as its bytes, other inputs throw
+// ERR_INVALID_ARG_TYPE, and encodings that are not strings throw
+// ERR_UNKNOWN_ENCODING formatted with %s. Buffer.of builds the inputs.
+import { StringDecoder } from "node:string_decoder";
+
+function feed(decoder, chunks) {
+  let out = "";
+  for (const chunk of chunks) out += decoder.write(chunk);
+  return out + decoder.end();
+}
+
+function attempt(label, fn) {
+  try {
+    console.log(label, JSON.stringify(fn()));
+  } catch (e) {
+    console.log(label, e.name, e.code, e.message);
+  }
+}
+
+const euro = Buffer.of(0xe2, 0x82, 0xac);
+console.log(euro, euro.length, Buffer.isBuffer(euro), euro.byteOffset);
+console.log(Buffer.of(), Buffer.of(1, 256, -1, 2.9, "7"));
+
+// Strings, Buffers, typed arrays and DataViews in one decoder.
+console.log(
+  JSON.stringify(
+    feed(new StringDecoder("utf8"), [
+      euro.subarray(0, 1),
+      euro.subarray(1),
+      "x",
+      new Uint16Array([0x41, 0x42]),
+      new DataView(Buffer.of(0xc3, 0xa9).buffer),
+    ]),
+  ),
+);
+console.log(JSON.stringify(feed(new StringDecoder("hex"), [new Int8Array([1, -1]), "zz", Buffer.of(16)])));
+console.log(JSON.stringify(feed(new StringDecoder("utf16le"), [Buffer.of(0x3d), Buffer.of(0xd8, 0x4d), Buffer.of(0xdc)])));
+
+// end(chunk): a string precedes the flushed partial; undefined only flushes.
+const decoder = new StringDecoder("utf8");
+decoder.write(euro.subarray(0, 2));
+attempt("end string", () => decoder.end("tail"));
+attempt("end undefined", () => decoder.end(undefined));
+decoder.write(euro.subarray(0, 2));
+attempt("end view", () => decoder.end(euro.subarray(2)));
+
+for (const bad of [123, null, {}, [1, 2], new ArrayBuffer(2), undefined, true]) {
+  attempt(`write ${typeof bad}`, () => decoder.write(bad));
+}
+attempt("end number", () => decoder.end(42));
+
+const encodings = [1, -0, NaN, true, 10n, Symbol("s"), {}, [], ["h", "e", "x"], { a: 1, b: { c: 2 } }, "", null];
+for (const name of [undefined, "test", "Latin1", "UCS-2", "BASE64URL"]) encodings.push(name);
+for (const encoding of encodings) {
+  attempt(`encoding ${typeof encoding}`, () => new StringDecoder(encoding).encoding);
+}
+// An object whose length is 3 to 9 matches by its string in lower case.
+const named = (length, text) => ({ length, toString: () => text });
+attempt("object hex", () => new StringDecoder(named(3, "HEX")).encoding);
+attempt("object ucs2", () => new StringDecoder(named(4, "Ucs2")).encoding);
+attempt("object base64url", () => new StringDecoder(named(9, "base64URL")).encoding);

tests/harness/corpus-costs/string-decoder-untyped-values.js.json

@@ -0,0 +1 @@
+{ "plain": 8500, "san": 90200 }

tests/harness/npm-suite.ts

@@ -94,7 +94,9 @@ const KNOWN: Record<string, Known> = {
   "node24-no-error-assertions": { run: "SC2020" },
   "node24-parseargs": { run: "SC1090" },
   "node24-promise-timers": { run: "SC2020" },
-  "node24-string-decoder": { run: "SC2020" },
+  // The decoding, encoding and input-error lines match Node; the closing
+  // `d.encoding = 'hex'` assignment refuses.
+  "node24-string-decoder": { run: "SC1090" },
   "node24-style-text": { run: "SC2020" },
   "node24-system-errors": { run: "SC1090" },
   "node24-text-encoding": { run: "SC2020" },

tests/node-parallel/common/index.cjs

@@ -8,6 +8,7 @@ const os = require("os");
 const { inspect } = require("util");
 
 const hasIntl = true;
+const enoughTestMem = os.totalmem() > 0x70000000; /* 1.75 Gb */
 const isDebug = false;
 const isWindows = process.platform === "win32";
 const isAIX = os.type() === "AIX";
@@ -127,11 +128,37 @@ function invalidArgTypeHelper(input) {
   return ` Received type ${typeof input} (${inspected})`;
 }
 
-function spawnPromisified(...args) {
+function getArrayBufferViews(buf) {
+  const { buffer, byteOffset, byteLength } = buf;
+  const out = [];
+  const arrayBufferViews = [
+    Int8Array,
+    Uint8Array,
+    Uint8ClampedArray,
+    Int16Array,
+    Uint16Array,
+    Int32Array,
+    Uint32Array,
+    Float16Array,
+    Float32Array,
+    Float64Array,
+    BigInt64Array,
+    BigUint64Array,
+    DataView,
+  ];
+  for (const type of arrayBufferViews) {
+    const { BYTES_PER_ELEMENT = 1 } = type;
+    if (byteLength % BYTES_PER_ELEMENT === 0)
+      out.push(new type(buffer, byteOffset, byteLength / BYTES_PER_ELEMENT));
+  }
+  return out;
+}
+
+function spawnPromisified(command, args, options) {
   const { spawn } = require("child_process");
   let stderr = "";
   let stdout = "";
-  const child = spawn(...args);
+  const child = spawn(command, args, options);
   child.stderr.setEncoding("utf8");
   child.stderr.on("data", (data) => {
     stderr += data;
@@ -147,6 +174,7 @@ function spawnPromisified(...args) {
 }
 
 module.exports = {
+  enoughTestMem,
   hasCrypto,
   hasIntl,
   isAIX,
@@ -160,6 +188,7 @@ module.exports = {
   mustSucceed,
   skip,
   expectWarning,
+  getArrayBufferViews,
   expectsError,
   invalidArgTypeHelper,
   platformTimeout,