Match Node 24's node:string_decoder for values known only at run time

packages/compiler/ambient/scriptc-node-fallback.d.ts

@@ -920,6 +920,7 @@ interface BufferConstructor {
    * optional byte offset/length shares x's storage (the subarray rule). */
   from(data: ArrayBuffer | SharedArrayBuffer, byteOffset?: number, length?: number): Buffer;
   from(data: Uint8Array | readonly number[]): Buffer;
+  of(...items: number[]): Buffer;
   alloc(size: number, fill?: string | number | Uint8Array, encoding?: BufferEncoding): Buffer;
   compare(buf1: Uint8Array, buf2: Uint8Array): number;
   /* alloc without the zero-fill guarantee. Below Buffer.poolSize >>> 1

packages/compiler/src/backend/llvm/lib-shared.ts

@@ -624,6 +624,8 @@ export const LIB_FN_SYMS: Record<string, string> = {
   "strdec.next": "scr_strdec_next",
   "strdec.end": "scr_strdec_end",
   "strdec.encoding": "scr_strdec_encoding",
+  "strdec.view": "scr_strdec_view",
+  "strdec.encodingValue": "scr_strdec_encoding_value",
   "stats.isFile": "scr_stats_is_file",
   "stats.isDirectory": "scr_stats_is_dir",
   "stats.isSymbolicLink": "scr_stats_is_symlink",

packages/compiler/src/frontend/lowering/builtins/text-codecs.ts

@@ -132,11 +132,16 @@ export function lowerStringDecoderMethodCall(
     };
   }
   if (!(chunk.type.kind === "bytes" && chunk.type.elem === "u8")) {
-    lowerer.noLowering(
-      `StringDecoder.${name} of '${lowerer.fmt(chunk.type)}' data`,
-      node,
-      "pass a string or typed-array view (narrow unions first)",
-    );
+    // Any other value decides at run time, as Node's write does: a string
+    // passes through, an ArrayBufferView decodes, and anything else throws.
+    const op = name === "end" ? "endDyn" : "writeDyn";
+    return {
+      kind: "call",
+      callee: stringDecoderHelper(lowerer, op, receiver.type.shapeId, loc),
+      args: [receiver, lowerer.coerceInto(node, chunk, DYN)],
+      type: STRING,
+      loc,
+    };
   }
   const helper = stringDecoderHelper(
     lowerer,
@@ -153,13 +158,15 @@ export function lowerStringDecoderMethodCall(
  * the pure strdec.* libCalls. */
 function stringDecoderHelper(
   lowerer: Lowerer,
-  op: "write" | "end" | "endChunk" | "endString" | "writeString",
+  op: "write" | "end" | "endChunk" | "endString" | "writeString" | "writeDyn" | "endDyn",
   shapeId: string,
   loc: SrcLoc,
 ): string {
   const key = `strdec.${op}`;
   const existing = lowerer.valueHelpers.get(key);
   if (existing) return existing;
+  if (op === "writeDyn" || op === "endDyn")
+    return stringDecoderDynHelper(lowerer, op, key, shapeId, loc);
   const name = keyedHelperName(lowerer, `%strdec.${op}`, key);
   lowerer.valueHelpers.set(key, name);
   const recT: IrType = { kind: "record", shapeId };
@@ -306,6 +313,76 @@ function stringDecoderHelper(
   return name;
 }
 
+/** The %strdec helpers of a chunk known only at run time (Node's write and
+ * end): a string passes through write unchanged, and end returns it
+ * followed by the flushed partial; an ArrayBufferView decodes as a u8 view
+ * of its bytes; end(undefined) only flushes; any other value throws
+ * ERR_INVALID_ARG_TYPE (strdec.view). */
+function stringDecoderDynHelper(
+  lowerer: Lowerer,
+  op: "writeDyn" | "endDyn",
+  key: string,
+  shapeId: string,
+  loc: SrcLoc,
+): string {
+  const name = keyedHelperName(lowerer, `%strdec.${op}`, key);
+  lowerer.valueHelpers.set(key, name);
+  const recT: IrType = { kind: "record", shapeId };
+  const d = (): IrExpr => varRef("d.0", recT, loc);
+  const chunk = (): IrExpr => varRef("chunk.0", DYN, loc);
+  const call = (helper: Parameters<typeof stringDecoderHelper>[1], args: IrExpr[]): IrExpr => ({
+    kind: "call",
+    callee: stringDecoderHelper(lowerer, helper, shapeId, loc),
+    args,
+    type: STRING,
+    loc,
+  });
+  const text: IrExpr = { kind: "dynCheck", value: chunk(), type: STRING, loc };
+  const view: IrExpr = { kind: "libCall", fn: "strdec.view", args: [chunk()], type: BYTES_U8, loc };
+  const isString: IrExpr = { kind: "dynTest", test: "string", value: chunk(), type: BOOL, loc };
+  const body: IrStmt[] = [];
+  if (op === "endDyn")
+    body.push({
+      kind: "if",
+      cond: { kind: "dynTest", test: "undefined", value: chunk(), type: BOOL, loc },
+      then: [{ kind: "return", value: call("end", [d()]), loc }],
+      else_: null,
+      loc,
+    });
+  body.push(
+    {
+      kind: "if",
+      cond: isString,
+      then: [
+        {
+          kind: "return",
+          value:
+            op === "endDyn" ? call("endString", [d(), text]) : call("writeString", [d(), text]),
+          loc,
+        },
+      ],
+      else_: null,
+      loc,
+    },
+    { kind: "return", value: call(op === "endDyn" ? "endChunk" : "write", [d(), view]), loc },
+  );
+  lowerer.liftedFns.push({
+    name,
+    params: [
+      { localId: "d.0", name: "d", type: recT },
+      { localId: "chunk.0", name: "chunk", type: DYN },
+    ],
+    returnType: STRING,
+    locals: [
+      { id: "d.0", name: "d", type: recT, mutable: false },
+      { id: "chunk.0", name: "chunk", type: DYN, mutable: false },
+    ],
+    body,
+    loc,
+  });
+  return name;
+}
+
 type TextCodecCtor = {
   cls: "TextDecoder" | "TextEncoder";
   ctor: ts.NewExpression;

packages/compiler/src/frontend/lowering/containers/bytes.ts

@@ -2353,6 +2353,9 @@ function lowerBufferStaticValue(
   const member = access.name.text;
   const loc = locOf(call);
   const args = call.arguments.nodes;
+  // Buffer.of(...items): Node's own `of`, a new unpooled Buffer of the items
+  // converted in order, as Uint8Array.of converts them.
+  if (member === "of") return lowerBytesOf(lowerer, call, "u8");
   if (member === "from") {
     if (
       args.length >= 1 &&

packages/compiler/src/frontend/lowering/lower-classes.ts

@@ -11014,8 +11014,9 @@ export function lowerNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr {
     // starting at 0 (nothing buffered). A literal Buffer encoding folds
     // its alias at compile time; any other encoding argument is
     // normalized at run time (Node's normalizeEncoding, which throws
-    // ERR_UNKNOWN_ENCODING for an unknown name). Omitted means utf8,
-    // Node's default.
+    // ERR_UNKNOWN_ENCODING for an unknown name). An encoding that may not
+    // be a string takes strdec.encodingValue, which formats the error
+    // with %s. Omitted means utf8, Node's default.
     if (symbol && ts.symbolName(symbol) === "StringDecoder" && lowerer.isStdlibSymbol(symbol)) {
       const args = expr.arguments?.nodes ?? [];
       if (args.length > 1) {
@@ -11028,12 +11029,18 @@ export function lowerNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr {
         : encT!.isStringLiteralType()
           ? knownBufEncoding(encT!.value)
           : undefined;
+      const textual = (type: ts.Type): boolean =>
+        type.isUnionType()
+          ? ts.constituentTypes(type).every(textual)
+          : (type.flags &
+              (ts.TypeFlags.StringLike | ts.TypeFlags.Undefined | ts.TypeFlags.Null)) !==
+            0;
       const encoding: IrExpr =
         encName !== undefined
           ? { kind: "strLit", value: encName, type: STRING, loc }
           : {
               kind: "libCall",
-              fn: "strdec.encoding",
+              fn: textual(encT!) ? "strdec.encoding" : "strdec.encodingValue",
               args: [lowerer.lowerExprExpecting(encNode!, DYN)],
               type: STRING,
               loc,

packages/compiler/src/ir/builtin-effects.ts

@@ -91,6 +91,8 @@ export const MAY_THROW_LIB_FNS: ReadonlySet<IrLibFn> = new Set([
   "child.onDyn",
   "text.decoderEncoding",
   "strdec.encoding",
+  "strdec.view",
+  "strdec.encodingValue",
   "dyn.construct",
   "dyn.reflectApply",
   "dyn.reflectGet",

packages/compiler/src/ir/builtin-signatures.ts

@@ -4996,6 +4996,14 @@ function buildLibFnSignatures() {
     /** A runtime encoding argument's canonical name (Node's normalizeEncoding:
      * nullish and "" are utf8); an unknown name throws ERR_UNKNOWN_ENCODING. */
     "strdec.encoding": { argTypes: [DYN], result: STRING },
+    /** The same for an encoding that may not be a string (scr_inspect.c):
+     * an object whose `length` is 3 to 9 matches by its string in lower
+     * case; other values throw ERR_UNKNOWN_ENCODING formatted with %s. */
+    "strdec.encodingValue": { argTypes: [DYN], result: STRING },
+    /** A write or end input that is not a string: a u8 view of an
+     * ArrayBufferView's bytes (+1); any other value throws
+     * ERR_INVALID_ARG_TYPE. */
+    "strdec.view": { argTypes: [DYN], result: BYTES_U8 },
     // node:readline: the interface handle is f64; the callbacks' func types
     // are program-dependent (the emitter picks the adapter by parameter
     // count), so those slots are null like child.onExit's.

packages/compiler/src/ir/runtime-features.ts

@@ -1119,6 +1119,8 @@ const RUNTIME_FEATURE_RULES = {
       "global.native",
       // ERR_ARG_NOT_ITERABLE renders the value as util.format's %s does.
       "webstream.class",
+      // ERR_UNKNOWN_ENCODING formats an encoding that is not a string with %s.
+      "strdec.encodingValue",
       "webstream.install",
       "util.styleText",
       "crypto.cipherNew",

packages/compiler/test/ts7/baselines/order-parity/tests/corpus/string-decoder-untyped-values.js.json

@@ -0,0 +1,6 @@
+{
+ "order": [
+  "<repo>/tests/corpus/string-decoder-untyped-values.js"
+ ],
+ "diags": []
+}

packages/runtime/src/scr_bytes.c

@@ -1524,6 +1524,25 @@ static ScrStr *scr_strdec_step(const ScrStr *enc, double pending, const ScrBytes
   return s;
 }
 
+/* The bytes of a StringDecoder write(buf) or end(buf) input that is not a
+ * string: a u8 view of an ArrayBufferView's bytes. Any other value throws
+ * ERR_INVALID_ARG_TYPE for "buf", as Node's write does. */
+ScrBytes *scr_strdec_view(const ScrDyn *value) {
+  if (value->kind == SCR_DYN_TYPED_REF) {
+    ScrDyn *materialized = scr_dyn_typed_ref_materialize(value);
+    if (!materialized) return NULL;
+    ScrBytes *result = scr_strdec_view(materialized);
+    scr_dyn_release(materialized);
+    return result;
+  }
+  if (value->kind == SCR_DYN_BYTES) {
+    ScrBytes *bytes = value->v.bytes;
+    return scr_bytes_buffer_view(bytes, SCR_BYTES_U8, scr_bytes_byte_offset(bytes), true, scr_bytes_byte_len(bytes));
+  }
+  scr_dyn_arg_type_fail("buf", "an instance of Buffer, TypedArray, or DataView", value);
+  return NULL;
+}
+
 /* decoder.write(chunk): the decoded complete prefix (+1). */
 ScrStr *scr_strdec_write(const ScrStr *enc, double pending, const ScrBytes *chunk) {
   SCR_BYTES_SNAPSHOT(chunk);
@@ -2754,8 +2773,8 @@ ScrStr *scr_bytes_normalize_encoding(const ScrStr *raw) {
 /* new StringDecoder(encoding) with an encoding known only at run time:
  * Node's normalizeEncoding. undefined, null and "" are utf8; a Buffer
  * encoding name in any ASCII case answers its canonical name (+1); any
- * other string throws ERR_UNKNOWN_ENCODING (NULL). Encodings that are not
- * strings have no lowering. */
+ * other string throws ERR_UNKNOWN_ENCODING (NULL). Encodings that may not
+ * be strings go through scr_strdec_encoding_value (scr_inspect.c). */
 ScrStr *scr_strdec_encoding(const ScrDyn *enc) {
   if (enc->kind == SCR_DYN_UNDEF || enc->kind == SCR_DYN_NULL) return scr_str_new("utf8", 4);
   if (enc->kind != SCR_DYN_STR) {

packages/runtime/src/scr_bytes.h

@@ -192,6 +192,7 @@ ScrStr *scr_strdec_write(const ScrStr *enc, double pending, const ScrBytes *chun
 double scr_strdec_next(const ScrStr *enc, double pending, const ScrBytes *chunk);
 ScrStr *scr_strdec_end(const ScrStr *enc, double pending);
 ScrStr *scr_strdec_encoding(const ScrDyn *enc); /* +1, or NULL with ERR_UNKNOWN_ENCODING pending */
+ScrBytes *scr_strdec_view(const ScrDyn *value); /* +1, or NULL with ERR_INVALID_ARG_TYPE pending */
 /* A Buffer encoding name known only at run time, Node's getEncodingOps: the
  * canonical name of an encoding in any ASCII case (+1), or NULL with
  * ERR_UNKNOWN_ENCODING ("Unknown encoding: <raw>") pending. Borrows raw. */

packages/runtime/src/scr_inspect.c

@@ -3161,6 +3161,63 @@ ScrStr *scr_insp_format(ScrDyn *const *args, size_t argc, const ScrInspFormatOpt
   return insp_format_loop(args, argc, o, &ops);
 }
 
+/* new StringDecoder(encoding) with an encoding that may not be a string:
+ * Node's normalizeEncoding. Strings, undefined and null go through
+ * scr_strdec_encoding. Another value matches only as an object whose
+ * `length` is 3 to 9: its string, in lower case, against the names of that
+ * length (slowCases converts it once). Everything else throws
+ * ERR_UNKNOWN_ENCODING, formatted as Node formats it ("Unknown encoding:
+ * %s"). The canonical name (+1), or NULL with the exception pending. */
+ScrStr *scr_strdec_encoding_value(const ScrDyn *enc) {
+  if (enc->kind == SCR_DYN_STR || enc->kind == SCR_DYN_UNDEF || enc->kind == SCR_DYN_NULL)
+    return scr_strdec_encoding(enc);
+  if (insp_fmt_object(enc)) {
+    ScrStr *name = scr_str_new("length", 6);
+    ScrDyn *key = scr_dyn_new_str(name);
+    scr_str_release(name);
+    ScrDyn *length = scr_dyn_reflect_get((ScrDyn *)enc, key, (ScrDyn *)enc);
+    scr_dyn_release(key);
+    if (!length) return NULL;
+    double count = length->kind == SCR_DYN_NUM ? length->v.num : 0;
+    scr_dyn_release(length);
+    static const struct { int length; const char *from; const char *to; } names[] = {
+      {3, "hex", "hex"},         {4, "utf8", "utf8"},           {4, "ucs2", "utf16le"},
+      {5, "utf-8", "utf8"},      {5, "ascii", "ascii"},         {5, "ucs-2", "utf16le"},
+      {6, "base64", "base64"},   {6, "latin1", "latin1"},       {6, "binary", "latin1"},
+      {7, "utf16le", "utf16le"}, {8, "utf-16le", "utf16le"},    {9, "base64url", "base64url"},
+    };
+    if (count >= 3 && count <= 9 && count == (int)count) {
+      ScrStr *text = scr_dyn_string_coerce_js(enc);
+      if (!text) return NULL;
+      const char *match = NULL;
+      for (size_t i = 0; i < sizeof names / sizeof names[0] && !match; i++) {
+        size_t n = strlen(names[i].from);
+        if (names[i].length != (int)count || text->len != n) continue;
+        bool same = true;
+        for (size_t k = 0; k < n && same; k++) {
+          char c = text->data[k];
+          if (c >= 'A' && c <= 'Z') c = (char)(c - 'A' + 'a');
+          same = c == names[i].from[k];
+        }
+        if (same) match = names[i].to;
+      }
+      scr_str_release(text);
+      if (match) return scr_str_new(match, strlen(match));
+    }
+  }
+  ScrStr *format = scr_str_new("Unknown encoding: %s", 20);
+  ScrDyn *args[2] = {scr_dyn_new_str(format), (ScrDyn *)enc};
+  scr_str_release(format);
+  ScrInspFormatOptions options;
+  scr_insp_format_defaults(&options, "new StringDecoder");
+  ScrStr *message = scr_insp_format(args, 2, &options);
+  scr_dyn_release(args[0]);
+  if (!message) return NULL;
+  scr_throw_error_msg_code(SCR_ERR_TYPE, message->data, message->len, "ERR_UNKNOWN_ENCODING");
+  scr_str_release(message);
+  return NULL;
+}
+
 /* The formatter over arguments that are all strings, numbers, booleans,
  * null or undefined (the compiler proves it from their types): the same
  * rules for those kinds, without the checked-dynamic walk the other

packages/runtime/src/scr_inspect.h

@@ -77,6 +77,9 @@ void scr_insp_format_defaults(ScrInspFormatOptions *o, const char *surface);
  * SC2020 refusal pending for values that have no lowering. */
 bool scr_insp_format_read_options(const ScrDyn *options, ScrInspFormatOptions *o);
 ScrStr *scr_insp_format(ScrDyn *const *args, size_t argc, const ScrInspFormatOptions *o);
+/* new StringDecoder(encoding) for an encoding of any type: Node's
+ * normalizeEncoding, ERR_UNKNOWN_ENCODING formatted with %s (+1 or NULL). */
+ScrStr *scr_strdec_encoding_value(const ScrDyn *enc);
 /* util.format(...pack) and util.formatWithOptions(options, ...pack) over a
  * dyn array of the arguments (+1). */
 ScrStr *scr_insp_format_pack(ScrDyn *pack);

tests/corpus/string-decoder-untyped-values.js

@@ -0,0 +1,63 @@
+// StringDecoder with values whose types are known only at run time, as
+// Node's write, end and normalizeEncoding take them: strings pass through,
+// any ArrayBufferView decodes as its bytes, other inputs throw
+// ERR_INVALID_ARG_TYPE, and encodings that are not strings throw
+// ERR_UNKNOWN_ENCODING formatted with %s. Buffer.of builds the inputs.
+import { StringDecoder } from "node:string_decoder";
+
+function feed(decoder, chunks) {
+  let out = "";
+  for (const chunk of chunks) out += decoder.write(chunk);
+  return out + decoder.end();
+}
+
+function attempt(label, fn) {
+  try {
+    console.log(label, JSON.stringify(fn()));
+  } catch (e) {
+    console.log(label, e.name, e.code, e.message);
+  }
+}
+
+const euro = Buffer.of(0xe2, 0x82, 0xac);
+console.log(euro, euro.length, Buffer.isBuffer(euro), euro.byteOffset);
+console.log(Buffer.of(), Buffer.of(1, 256, -1, 2.9, "7"));
+
+// Strings, Buffers, typed arrays and DataViews in one decoder.
+console.log(
+  JSON.stringify(
+    feed(new StringDecoder("utf8"), [
+      euro.subarray(0, 1),
+      euro.subarray(1),
+      "x",
+      new Uint16Array([0x41, 0x42]),
+      new DataView(Buffer.of(0xc3, 0xa9).buffer),
+    ]),
+  ),
+);
+console.log(JSON.stringify(feed(new StringDecoder("hex"), [new Int8Array([1, -1]), "zz", Buffer.of(16)])));
+console.log(JSON.stringify(feed(new StringDecoder("utf16le"), [Buffer.of(0x3d), Buffer.of(0xd8, 0x4d), Buffer.of(0xdc)])));
+
+// end(chunk): a string precedes the flushed partial; undefined only flushes.
+const decoder = new StringDecoder("utf8");
+decoder.write(euro.subarray(0, 2));
+attempt("end string", () => decoder.end("tail"));
+attempt("end undefined", () => decoder.end(undefined));
+decoder.write(euro.subarray(0, 2));
+attempt("end view", () => decoder.end(euro.subarray(2)));
+
+for (const bad of [123, null, {}, [1, 2], new ArrayBuffer(2), undefined, true]) {
+  attempt(`write ${typeof bad}`, () => decoder.write(bad));
+}
+attempt("end number", () => decoder.end(42));
+
+const encodings = [1, -0, NaN, true, 10n, Symbol("s"), {}, [], ["h", "e", "x"], { a: 1, b: { c: 2 } }, "", null];
+for (const name of [undefined, "test", "Latin1", "UCS-2", "BASE64URL"]) encodings.push(name);
+for (const encoding of encodings) {
+  attempt(`encoding ${typeof encoding}`, () => new StringDecoder(encoding).encoding);
+}
+// An object whose length is 3 to 9 matches by its string in lower case.
+const named = (length, text) => ({ length, toString: () => text });
+attempt("object hex", () => new StringDecoder(named(3, "HEX")).encoding);
+attempt("object ucs2", () => new StringDecoder(named(4, "Ucs2")).encoding);
+attempt("object base64url", () => new StringDecoder(named(9, "base64URL")).encoding);

tests/harness/corpus-costs/string-decoder-untyped-values.js.json

@@ -0,0 +1 @@
+{ "plain": 8500, "san": 90200 }

tests/node-parallel/common/index.cjs

@@ -8,6 +8,7 @@ const os = require("os");
 const { inspect } = require("util");
 
 const hasIntl = true;
+const enoughTestMem = require("os").totalmem() > 0x70000000; /* 1.75 Gb */
 const isDebug = false;
 const isWindows = process.platform === "win32";
 const isAIX = os.type() === "AIX";
@@ -102,11 +103,37 @@ function invalidArgTypeHelper(input) {
   return ` Received type ${typeof input} (${inspected})`;
 }
 
-function spawnPromisified(...args) {
+function getArrayBufferViews(buf) {
+  const { buffer, byteOffset, byteLength } = buf;
+  const out = [];
+  const arrayBufferViews = [
+    Int8Array,
+    Uint8Array,
+    Uint8ClampedArray,
+    Int16Array,
+    Uint16Array,
+    Int32Array,
+    Uint32Array,
+    Float16Array,
+    Float32Array,
+    Float64Array,
+    BigInt64Array,
+    BigUint64Array,
+    DataView,
+  ];
+  for (const type of arrayBufferViews) {
+    const { BYTES_PER_ELEMENT = 1 } = type;
+    if (byteLength % BYTES_PER_ELEMENT === 0)
+      out.push(new type(buffer, byteOffset, byteLength / BYTES_PER_ELEMENT));
+  }
+  return out;
+}
+
+function spawnPromisified(command, args, options) {
   const { spawn } = require("child_process");
   let stderr = "";
   let stdout = "";
-  const child = spawn(...args);
+  const child = spawn(command, args, options);
   child.stderr.setEncoding("utf8");
   child.stderr.on("data", (data) => {
     stderr += data;
@@ -122,6 +149,7 @@ function spawnPromisified(...args) {
 }
 
 module.exports = {
+  enoughTestMem,
   hasIntl,
   isAIX,
   isDebug,
@@ -133,6 +161,7 @@ module.exports = {
   mustNotCall,
   skip,
   expectWarning,
+  getArrayBufferViews,
   invalidArgTypeHelper,
   spawnPromisified,
 };