diff --git a/internal/compatibility/dynamic-support.json b/internal/compatibility/dynamic-support.json index 7663283c5..21460717a 100644 --- a/internal/compatibility/dynamic-support.json +++ b/internal/compatibility/dynamic-support.json @@ -60,7 +60,7 @@ "module": { "exports": ["createRequire", "builtinModules", "isBuiltin", "syncBuiltinESMExports", "register", "findSourceMap"], "refused": ["register"], "evidence": ["tests/fixtures/npm/cases/create-require/main.ts", "tests/fixtures/npm/cases/misc-shims/main.ts"] }, "url": { "exports": ["URL", "URLSearchParams", "fileURLToPath", "pathToFileURL", "parse", "format", "resolve", "domainToASCII", "domainToUnicode", "urlToHttpOptions"], "members": ["url.toString", "url.toJSON", "URL.canParse", "URL.parse", "urlSearchParams.append", "urlSearchParams.delete", "urlSearchParams.entries", "urlSearchParams.forEach", "urlSearchParams.get", "urlSearchParams.getAll", "urlSearchParams.has", "urlSearchParams.keys", "urlSearchParams.set", "urlSearchParams.sort", "urlSearchParams.toString", "urlSearchParams.values", "urlSearchParams.size"], "evidence": ["tests/fixtures/npm/cases/misc-shims/main.ts", "tests/fixtures/npm/cases/island-web-plumbing/main.ts", "tests/fixtures/npm/cases/node24-url-factories/main.ts"] }, "buffer": { "exports": ["Buffer", "SlowBuffer", "INSPECT_MAX_BYTES", "kMaxLength", "kStringMaxLength", "constants", "isAscii", "isUtf8", "atob", "btoa", "Blob", "File", "transcode", "resolveObjectURL"], "members": ["Buffer.alloc", "Buffer.allocUnsafe", "Buffer.allocUnsafeSlow", "Buffer.from", "Buffer.isBuffer", "Buffer.isEncoding", "Buffer.byteLength", "Buffer.concat", "Buffer.compare", "buf.compare", "buf.copy", "buf.equals", "buf.fill", "buf.includes", "buf.indexOf", "buf.lastIndexOf", "buf.subarray", "buf.slice", "buf.swap16", "buf.swap32", "buf.swap64", "buf.toJSON", "buf.toString", "buf.write", "buf.readBigInt64BE", "buf.readBigInt64LE", "buf.readBigUInt64BE", "buf.readBigUInt64LE", "buf.readDoubleBE", "buf.readDoubleLE", "buf.readFloatBE", "buf.readFloatLE", "buf.readInt8", "buf.readInt16BE", "buf.readInt16LE", "buf.readInt32BE", "buf.readInt32LE", "buf.readIntBE", "buf.readIntLE", "buf.readUInt8", "buf.readUInt16BE", "buf.readUInt16LE", "buf.readUInt32BE", "buf.readUInt32LE", "buf.readUIntBE", "buf.readUIntLE", "buf.writeBigInt64BE", "buf.writeBigInt64LE", "buf.writeBigUInt64BE", "buf.writeBigUInt64LE", "buf.writeDoubleBE", "buf.writeDoubleLE", "buf.writeFloatBE", "buf.writeFloatLE", "buf.writeInt8", "buf.writeInt16BE", "buf.writeInt16LE", "buf.writeInt32BE", "buf.writeInt32LE", "buf.writeIntBE", "buf.writeIntLE", "buf.writeUInt8", "buf.writeUInt16BE", "buf.writeUInt16LE", "buf.writeUInt32BE", "buf.writeUInt32LE", "buf.writeUIntBE", "buf.writeUIntLE"], "evidence": ["tests/fixtures/npm/cases/buffer-shims/main.ts"] }, - "string_decoder": { "exports": ["StringDecoder"], "members": ["stringDecoder.write", "stringDecoder.end"], "evidence": ["tests/fixtures/npm/cases/buffer-shims/main.ts"] }, + "string_decoder": { "exports": ["StringDecoder"], "members": ["stringDecoder.write", "stringDecoder.end"], "evidence": ["tests/fixtures/npm/cases/buffer-shims/main.ts", "tests/fixtures/npm/cases/node24-string-decoder/main.ts"] }, "assert": { "exports": ["AssertionError", "ok", "fail", "equal", "notEqual", "strictEqual", "notStrictEqual", "deepEqual", "notDeepEqual", "deepStrictEqual", "notDeepStrictEqual", "throws", "doesNotThrow", "rejects", "doesNotReject", "match", "doesNotMatch", "ifError", "strict"], "evidence": ["tests/fixtures/npm/cases/assert-shims/main.ts"] }, "assert/strict": { "exports": ["AssertionError", "ok", "fail", "equal", "notEqual", "strictEqual", "notStrictEqual", "deepEqual", "notDeepEqual", "deepStrictEqual", "notDeepStrictEqual", "throws", "doesNotThrow", "rejects", "doesNotReject", "match", "doesNotMatch", "ifError", "strict"], "evidence": ["tests/fixtures/npm/cases/assert-shims/main.ts"] }, "domain": { "exports": ["create", "createDomain", "Domain", "active"], "members": ["domain.add", "domain.bind", "domain.enter", "domain.exit", "domain.intercept", "domain.remove", "domain.run"], "evidence": ["tests/fixtures/npm/cases/island-web-plumbing/main.ts"] }, diff --git a/internal/compatibility/generated/node-v24-backlog.json b/internal/compatibility/generated/node-v24-backlog.json index c39230f38..59b851bfc 100644 --- a/internal/compatibility/generated/node-v24-backlog.json +++ b/internal/compatibility/generated/node-v24-backlog.json @@ -82971,7 +82971,8 @@ "confidence": "high", "source": "compiler-dedicated:StringDecoder", "tests": [ - "tests/corpus/1662-string-decoder-encodings.ts" + "tests/corpus/1662-string-decoder-encodings.ts", + "tests/corpus/node24-string-decoder.ts" ] }, "dynamic": { @@ -82981,7 +82982,8 @@ "confidence": "high", "source": "island-export:node:string_decoder.StringDecoder", "tests": [ - "tests/fixtures/npm/cases/buffer-shims/main.ts" + "tests/fixtures/npm/cases/buffer-shims/main.ts", + "tests/fixtures/npm/cases/node24-string-decoder/main.ts" ] } } @@ -83008,7 +83010,8 @@ "confidence": "high", "source": "compiler-dedicated:stringDecoder.end", "tests": [ - "tests/corpus/1662-string-decoder-encodings.ts" + "tests/corpus/1662-string-decoder-encodings.ts", + "tests/corpus/node24-string-decoder.ts" ] }, "dynamic": { @@ -83018,7 +83021,8 @@ "confidence": "high", "source": "island-member:node:string_decoder.stringDecoder.end", "tests": [ - "tests/fixtures/npm/cases/buffer-shims/main.ts" + "tests/fixtures/npm/cases/buffer-shims/main.ts", + "tests/fixtures/npm/cases/node24-string-decoder/main.ts" ] } } @@ -83045,7 +83049,8 @@ "confidence": "high", "source": "compiler-dedicated:stringDecoder.write", "tests": [ - "tests/corpus/1662-string-decoder-encodings.ts" + "tests/corpus/1662-string-decoder-encodings.ts", + "tests/corpus/node24-string-decoder.ts" ] }, "dynamic": { @@ -83055,7 +83060,8 @@ "confidence": "high", "source": "island-member:node:string_decoder.stringDecoder.write", "tests": [ - "tests/fixtures/npm/cases/buffer-shims/main.ts" + "tests/fixtures/npm/cases/buffer-shims/main.ts", + "tests/fixtures/npm/cases/node24-string-decoder/main.ts" ] } } diff --git a/internal/compatibility/generated/node-v24-internal.json b/internal/compatibility/generated/node-v24-internal.json index 0f9f50ccf..770cdbc15 100644 --- a/internal/compatibility/generated/node-v24-internal.json +++ b/internal/compatibility/generated/node-v24-internal.json @@ -100291,14 +100291,16 @@ "status": "partial", "evidence": "derived:descendants", "tests": [ - "tests/corpus/1662-string-decoder-encodings.ts" + "tests/corpus/1662-string-decoder-encodings.ts", + "tests/corpus/node24-string-decoder.ts" ] }, "dynamic": { "status": "partial", "evidence": "derived:descendants", "tests": [ - "tests/fixtures/npm/cases/buffer-shims/main.ts" + "tests/fixtures/npm/cases/buffer-shims/main.ts", + "tests/fixtures/npm/cases/node24-string-decoder/main.ts" ] }, "anchorSource": "chapter" @@ -100324,14 +100326,16 @@ "status": "partial", "evidence": "compiler-dedicated:StringDecoder", "tests": [ - "tests/corpus/1662-string-decoder-encodings.ts" + "tests/corpus/1662-string-decoder-encodings.ts", + "tests/corpus/node24-string-decoder.ts" ] }, "dynamic": { "status": "partial", "evidence": "island-export:node:string_decoder.StringDecoder", "tests": [ - "tests/fixtures/npm/cases/buffer-shims/main.ts" + "tests/fixtures/npm/cases/buffer-shims/main.ts", + "tests/fixtures/npm/cases/node24-string-decoder/main.ts" ] }, "anchorSource": "exact" @@ -100357,14 +100361,16 @@ "status": "partial", "evidence": "compiler-dedicated:stringDecoder.end", "tests": [ - "tests/corpus/1662-string-decoder-encodings.ts" + "tests/corpus/1662-string-decoder-encodings.ts", + "tests/corpus/node24-string-decoder.ts" ] }, "dynamic": { "status": "partial", "evidence": "island-member:node:string_decoder.stringDecoder.end", "tests": [ - "tests/fixtures/npm/cases/buffer-shims/main.ts" + "tests/fixtures/npm/cases/buffer-shims/main.ts", + "tests/fixtures/npm/cases/node24-string-decoder/main.ts" ] }, "anchorSource": "exact" @@ -100390,14 +100396,16 @@ "status": "partial", "evidence": "compiler-dedicated:stringDecoder.write", "tests": [ - "tests/corpus/1662-string-decoder-encodings.ts" + "tests/corpus/1662-string-decoder-encodings.ts", + "tests/corpus/node24-string-decoder.ts" ] }, "dynamic": { "status": "partial", "evidence": "island-member:node:string_decoder.stringDecoder.write", "tests": [ - "tests/fixtures/npm/cases/buffer-shims/main.ts" + "tests/fixtures/npm/cases/buffer-shims/main.ts", + "tests/fixtures/npm/cases/node24-string-decoder/main.ts" ] }, "anchorSource": "exact" diff --git a/internal/compatibility/static-support.json b/internal/compatibility/static-support.json index 799ab1dba..4df0fdacb 100644 --- a/internal/compatibility/static-support.json +++ b/internal/compatibility/static-support.json @@ -79,7 +79,7 @@ "node-builtin.querystring": ["tests/corpus/957-builtins-namespace.ts", "tests/corpus/2917-builtin-callable-widths.ts"], "node-builtin.readline": ["tests/corpus/1475-readline-closed-stdin.ts"], "node-builtin.stream": ["tests/corpus/1685-stream-readable-basics.ts", "tests/corpus/1746-stream-for-await.ts", "tests/corpus/1814-stream-pipeline.cjs", "tests/corpus/2629-stream-consumers.ts", "tests/corpus/2861-readable-iterator-close.ts"], - "node-builtin.string_decoder": ["tests/corpus/1662-string-decoder-encodings.ts"], + "node-builtin.string_decoder": ["tests/corpus/1662-string-decoder-encodings.ts", "tests/corpus/node24-string-decoder.ts"], "node-builtin.test": ["tests/harness/node-test.test.ts"], "node-builtin.timers": ["tests/corpus/1804-timers-module.ts", "tests/corpus/2093-timers-promises.ts"], "node-builtin.tls": ["tests/corpus/2331-tls-server-options-typed.ts", "tests/corpus/2557-tls-ca-store.ts"], @@ -198,7 +198,7 @@ { "symbols": ["crypto.hkdf", "crypto.hkdfSync", "crypto.scrypt", "crypto.scryptSync"], "status": "partial", "evidence": ["tests/corpus/node24-crypto-kdf.ts", "tests/corpus/node24-crypto-kdf-errors.ts", "tests/corpus/node24-crypto-kdf-options.ts"] }, { "symbols": ["EventEmitter", "emitter.on", "emitter.addListener", "emitter.once", "emitter.prependListener", "emitter.prependOnceListener", "emitter.off", "emitter.removeListener", "emitter.removeAllListeners", "emitter.emit", "emitter.listenerCount", "emitter.listeners", "emitter.rawListeners", "emitter.eventNames", "emitter.setMaxListeners", "emitter.getMaxListeners", "events.getEventListeners", "events.listenerCount", "events.once", "events.on", "events.setMaxListeners"], "status": "partial", "evidence": ["tests/corpus/1677-emitter-listeners.ts", "tests/corpus/2620-ee-override-once-order.ts"] }, { "symbols": ["AsyncLocalStorage", "asyncLocalStorage.run", "asyncLocalStorage.exit", "asyncLocalStorage.getStore", "asyncLocalStorage.enterWith", "asyncLocalStorage.disable"], "status": "partial", "evidence": ["tests/corpus/2213-async-local-storage.cjs"] }, - { "symbols": ["StringDecoder", "stringDecoder.write", "stringDecoder.end"], "status": "partial", "evidence": ["tests/corpus/1662-string-decoder-encodings.ts"] }, + { "symbols": ["StringDecoder", "stringDecoder.write", "stringDecoder.end"], "status": "partial", "evidence": ["tests/corpus/1662-string-decoder-encodings.ts", "tests/corpus/node24-string-decoder.ts"] }, { "symbols": ["setImmediate", "setInterval", "setTimeout", "clearImmediate", "clearInterval", "clearTimeout", "timersPromises.setTimeout", "timersPromises.setImmediate", "timersPromises.setInterval", "immediate.hasRef", "immediate.ref", "immediate.unref", "timeout.hasRef", "timeout.ref", "timeout.unref", "timeout.refresh"], "status": "partial", "evidence": ["tests/corpus/1804-timers-module.ts", "tests/corpus/1805-timer-callback-args.ts", "tests/corpus/2093-timers-promises.ts", "tests/corpus/2852-timers-promises-interval.ts"] }, { "symbols": ["stream.finished", "stream.pipeline", "stream.getDefaultHighWaterMark", "readable.push", "readable.unshift", "readable.read", "readable.pause", "readable.resume", "readable.isPaused", "readable.setEncoding", "readable.pipe", "readable.unpipe", "readable.destroy", "writable.write", "writable.end", "writable.cork", "writable.uncork", "writable.setDefaultEncoding"], "status": "partial", "evidence": ["tests/corpus/1685-stream-readable-basics.ts", "tests/corpus/1688-stream-writable-basics.ts", "tests/corpus/1814-stream-pipeline.cjs"] }, { "symbols": ["net.createServer", "net.connect", "net.createConnection", "net.getDefaultAutoSelectFamilyAttemptTimeout", "net.setDefaultAutoSelectFamilyAttemptTimeout", "server.address", "server.close", "server.listen", "socket.destroy", "socket.end", "socket.pause", "socket.resume", "socket.setEncoding", "socket.setTimeout", "socket.write"], "status": "partial", "evidence": ["tests/corpus/2500-net-autosel-timeout.ts", "tests/corpus/2596-net-arg-ladders.cjs"] }, diff --git a/packages/compiler/src/frontend/lowering/lower-builtins.ts b/packages/compiler/src/frontend/lowering/lower-builtins.ts index 627076d3c..8dbfad982 100644 --- a/packages/compiler/src/frontend/lowering/lower-builtins.ts +++ b/packages/compiler/src/frontend/lowering/lower-builtins.ts @@ -4779,13 +4779,8 @@ export function lowerForkCall(lowerer: Lowerer, expr: ts.CallExpression, loc: Sr ); } -/** Method calls on StringDecoder receivers: `d.write(chunk)` decodes the - * complete prefix of pending+chunk and re-buffers the trailing partial - * sequence; `d.end()` flushes the buffered partial as its replacement - * chars — Node's utf8 StringDecoder exactly (SEMANTICS.md), through the - * interned %strdec helpers over the packed-f64 pending field. end(buf) - * and the rest of @types/node's surface fence per member. Null for - * non-decoder receivers. */ +/** StringDecoder writes preserve pending bytes; end(input) performs the + * final write and flushes the state. String input passes through unchanged. */ export function lowerStringDecoderMethodCall(lowerer: Lowerer, call: ts.CallExpression, access: ts.PropertyAccessExpression,): IrExpr | null { if (call.questionDotToken || access.questionDotToken) return null; @@ -4793,47 +4788,43 @@ export function lowerForkCall(lowerer: Lowerer, expr: ts.CallExpression, loc: Sr if (!lowerer.isStdlibMember(access)) return null; const name = access.name.text; const loc = locOf(call); - if (name === "write" && call.arguments.length === 1) { - const receiver = lowerer.lowerExpr(access.expression); - if (receiver.type.kind !== "record") lowerer.badType(access.expression, lowerer.typeOf(access.expression)); - let chunk = lowerBuiltinValuePreservingUndefined(lowerer, call.arguments[0]!); - chunk = checkedOptionalBuiltinArm(lowerer, chunk, BYTES_U8) ?? chunk; - if (!(chunk.type.kind === "bytes" && chunk.type.elem === "u8")) { - lowerer.noLowering( - `StringDecoder.write of '${lowerer.fmt(chunk.type)}' data`, - call.arguments[0]!, - "Buffer/Uint8Array chunks decode (narrow unions first)", - ); - } - const helper = lowerer.strdecHelper("write", receiver.type.shapeId, loc); - return { kind: "call", callee: helper, args: [receiver, chunk], type: STRING, loc }; + if ((name !== "write" && name !== "end") || call.arguments.length > 1 || + (name === "write" && call.arguments.length !== 1) || call.arguments.some(ts.isSpreadElement)) { + lowerer.noLowering(`StringDecoder.${name}`, call, "use write(input) or end(input?)", lowerer.checker.getSymbolAtLocation(access.name)); } - if (name === "end") { - if (call.arguments.length !== 0) { - lowerer.noLowering( - "StringDecoder.end with a buffer argument", - call, - "write the buffer, then end(): d.write(buf) + d.end() is Node's own equivalence", - ); - } - const receiver = lowerer.lowerExpr(access.expression); - if (receiver.type.kind !== "record") lowerer.badType(access.expression, lowerer.typeOf(access.expression)); - const helper = lowerer.strdecHelper("end", receiver.type.shapeId, loc); - return { kind: "call", callee: helper, args: [receiver], type: STRING, loc }; + const receiver = lowerer.lowerExpr(access.expression); + if (receiver.type.kind !== "record") lowerer.badType(access.expression, lowerer.typeOf(access.expression)); + if (call.arguments.length === 0) { + return { kind: "call", callee: lowerer.strdecHelper("end", receiver.type.shapeId, loc), args: [receiver], type: STRING, loc }; } - lowerer.noLowering( - `StringDecoder.${name}`, - call, - "write(buffer) and end() are the supported StringDecoder members", - lowerer.checker.getSymbolAtLocation(access.name), - ); + const node = call.arguments[0]!; + const undefinedArg = name === "end" ? lowerStaticallyUndefinedArgument(lowerer, node) : null; + if (undefinedArg) { + const saved = lowerer.declareHiddenLocal("%strdecRecv", receiver.type); + const result: IrExpr = { kind: "call", callee: lowerer.strdecHelper("end", receiver.type.shapeId, loc), args: [varRef(saved.id, receiver.type, loc)], type: STRING, loc }; + return { kind: "seqExpr", stmts: [{ kind: "varDecl", localId: saved.id, init: receiver, loc }], result: defaultAfterUndefined(undefinedArg, result), type: STRING, loc }; + } + let chunk = lowerBuiltinValuePreservingUndefined(lowerer, node); + if (chunk.type.kind === "string") { + const op = name === "end" ? "endString" : "writeString"; + return { kind: "call", callee: lowerer.strdecHelper(op, receiver.type.shapeId, loc), args: [receiver, chunk], type: STRING, loc }; + } + chunk = checkedOptionalBuiltinArm(lowerer, chunk, BYTES_U8) ?? chunk; + if (chunk.type.kind === "bytes" && chunk.type.elem !== "u8") { + chunk = { kind: "libCall", fn: "bytes.bufferSource", args: [lowerer.coerceInto(node, chunk, DYN)], type: BYTES_U8, loc }; + } + if (!(chunk.type.kind === "bytes" && chunk.type.elem === "u8")) { + lowerer.noLowering(`StringDecoder.${name} of '${lowerer.fmt(chunk.type)}' data`, node, "pass a string or typed-array view (narrow unions first)"); + } + const helper = lowerer.strdecHelper(name === "end" ? "endChunk" : "write", receiver.type.shapeId, loc); + return { kind: "call", callee: helper, args: [receiver, chunk], type: STRING, loc }; } /** The interned %strdec helpers: write(d, chunk) returns the decoded * complete prefix and re-buffers the trailing partial into the pending * field; end(d) flushes it. Both thread the packed-f64 state through * the pure strdec.* libCalls. */ - export function strdecHelper(lowerer: Lowerer, op: "write" | "end", shapeId: string, loc: SrcLoc): string { + export function strdecHelper(lowerer: Lowerer, op: "write" | "end" | "endChunk" | "endString" | "writeString", shapeId: string, loc: SrcLoc): string { const key = `strdec.${op}`; const existing = lowerer.widthHelpers.get(key); if (existing) return existing; @@ -4865,7 +4856,7 @@ export function lowerForkCall(lowerer: Lowerer, expr: ts.CallExpression, loc: Sr { id: "s.0", name: "s", type: STRING, mutable: false }, ]; let body: IrStmt[]; - if (op === "write") { + if (op === "write" || op === "endChunk") { params.push({ localId: "chunk.0", name: "chunk", type: BYTES_U8 }); locals.splice(1, 0, { id: "chunk.0", name: "chunk", type: BYTES_U8, mutable: false }); body = [ @@ -4904,6 +4895,24 @@ export function lowerForkCall(lowerer: Lowerer, expr: ts.CallExpression, loc: Sr { kind: "return", value: varRef("s.0", STRING, loc), loc }, ]; } + if (op === "endString" || op === "writeString") { + const inputT = STRING; + params.push({ localId: "chunk.0", name: "chunk", type: inputT }); + locals.push({ id: "chunk.0", name: "chunk", type: inputT, mutable: false }); + if (op === "writeString") body = [{ kind: "return", value: varRef("chunk.0", STRING, loc), loc }]; + if (op === "endString") body[body.length - 1] = { kind: "return", value: { + kind: "strConcat", left: varRef("chunk.0", STRING, loc), right: varRef("s.0", STRING, loc), type: STRING, loc, + }, loc }; + } + if (op === "endChunk") { + locals.push({ id: "tail.0", name: "tail", type: STRING, mutable: false }); + body.splice(body.length - 1, 0, + { kind: "varDecl", localId: "tail.0", init: { kind: "libCall", fn: "strdec.end", args: [encRead(), pendingRead()], type: STRING, loc }, loc }, + { kind: "recordSet", obj: varRef("d.0", recT, loc), shapeId, field: "%pending", value: { kind: "numLit", value: 0, type: F64, loc }, loc }); + body[body.length - 1] = { kind: "return", value: { + kind: "strConcat", left: varRef("s.0", STRING, loc), right: varRef("tail.0", STRING, loc), type: STRING, loc, + }, loc }; + } lowerer.liftedFns.push({ name, params, returnType: STRING, locals, body, loc }); return name; } diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 3079a44f1..5932ad8c5 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -11024,7 +11024,7 @@ export class Lowerer { return lowerDcTracingChannelProperty(this, access); } - strdecHelper(op: "write" | "end", shapeId: string, loc: SrcLoc): string { + strdecHelper(op: "write" | "end" | "endChunk" | "endString" | "writeString", shapeId: string, loc: SrcLoc): string { return strdecHelper(this, op, shapeId, loc); } diff --git a/packages/compiler/test/ts7/baselines/order-parity.json b/packages/compiler/test/ts7/baselines/order-parity.json index 9e4bfe7d9..a2c9e31af 100644 --- a/packages/compiler/test/ts7/baselines/order-parity.json +++ b/packages/compiler/test/ts7/baselines/order-parity.json @@ -13384,6 +13384,18 @@ "/tests/corpus/js-open-binding-writes.js" ], "diags": [] + }, + "/tests/corpus/node24-string-decoder.ts": { + "order": [ + "/tests/corpus/node24-string-decoder.ts" + ], + "diags": [] + }, + "/tests/fixtures/npm/cases/node24-string-decoder/main.ts": { + "order": [ + "/tests/fixtures/npm/cases/node24-string-decoder/main.ts" + ], + "diags": [] } } } diff --git a/packages/runtime/src/scr_island.c b/packages/runtime/src/scr_island.c index 7ae19bd3b..428f0e847 100644 --- a/packages/runtime/src/scr_island.c +++ b/packages/runtime/src/scr_island.c @@ -4905,7 +4905,7 @@ static const char isl_modules_bootstrap[] = " const K_STRING_MAX_LENGTH = 536870888;\n" " const INSPECT_MAX_BYTES = 50;\n" " const utf8Enc = new TextEncoder();\n" - " const utf8Dec = new TextDecoder();\n" + " const utf8Dec = new TextDecoder(\"utf-8\", { ignoreBOM: true });\n" " const B64 = \"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/\";\n" " const B64U = \"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_\";\n" " const b64Val = (() => {\n" @@ -5667,97 +5667,89 @@ static const char isl_modules_bootstrap[] = " mod.default = mod;\n" " return mod;\n" " });\n" - /* node:string_decoder — StringDecoder over the Buffer shim: utf8 - * rides the prelude TextDecoder's streaming state machine, - * utf16le holds byte parity, base64 carries mod-3 remainders. */ + /* node:string_decoder — preserve Node's byte buffering independently + * of WHATWG TextDecoder: BOMs, invalid prefixes, UTF-16 pairs, and + * base64 groups retain their chunk-boundary behavior. */ " builtins.string_decoder = memo(() => {\n" "function makeStringDecoder(Buffer) {\n" " const normEnc = (enc) => {\n" - " if (enc === undefined || enc === null) return \"utf8\";\n" - " const e = String(enc).toLowerCase();\n" - " if (e === \"utf8\" || e === \"utf-8\") return \"utf8\";\n" - " if (e === \"hex\") return \"hex\";\n" - " if (e === \"base64\") return \"base64\";\n" - " if (e === \"base64url\") return \"base64url\";\n" - " if (e === \"latin1\" || e === \"binary\") return \"latin1\";\n" - " if (e === \"ascii\") return \"ascii\";\n" - " if (e === \"utf16le\" || e === \"utf-16le\" || e === \"ucs2\" || e === \"ucs-2\") return \"utf16le\";\n" - " const err = new TypeError(\"Unknown encoding: \" + enc);\n" - " err.code = \"ERR_UNKNOWN_ENCODING\";\n" + " if (enc === undefined || enc === null || enc === '') return 'utf8';\n" + " const e = typeof enc === 'string' ? enc.toLowerCase() : enc;\n" + " if (e === 'utf8' || e === 'utf-8') return 'utf8';\n" + " if (e === 'hex' || e === 'base64' || e === 'base64url' || e === 'ascii') return e;\n" + " if (e === 'latin1' || e === 'binary') return 'latin1';\n" + " if (e === 'utf16le' || e === 'utf-16le' || e === 'ucs2' || e === 'ucs-2') return 'utf16le';\n" + " const err = new TypeError('Unknown encoding: ' + String(enc));\n" + " err.code = 'ERR_UNKNOWN_ENCODING';\n" " throw err;\n" " };\n" " class StringDecoder {\n" " constructor(encoding) {\n" " this.encoding = normEnc(encoding);\n" - " if (this.encoding === \"utf8\") {\n" - " this._dec = new TextDecoder();\n" - " } else if (this.encoding === \"utf16le\") {\n" - " this._oddByte = -1;\n" - " } else if (this.encoding === \"base64\" || this.encoding === \"base64url\") {\n" - " this._rem = null;\n" - " }\n" + " this._encoding = this.encoding;\n" + " this._pending = new Uint8Array(0);\n" + " this._missing = 0;\n" " }\n" " write(buf) {\n" - " if (typeof buf === \"string\") return buf;\n" - " const u8 = buf instanceof Uint8Array ? buf : new Uint8Array(buf);\n" - " switch (this.encoding) {\n" - " case \"utf8\":\n" - " return this._dec.decode(u8, { stream: true });\n" - " case \"utf16le\": {\n" - " let bytes = u8;\n" - " if (this._oddByte >= 0) {\n" - " const joined = new Uint8Array(u8.length + 1);\n" - " joined[0] = this._oddByte;\n" - " joined.set(u8, 1);\n" - " bytes = joined;\n" - " this._oddByte = -1;\n" - " }\n" - " if (bytes.length % 2 !== 0) {\n" - " this._oddByte = bytes[bytes.length - 1];\n" - " bytes = bytes.subarray(0, bytes.length - 1);\n" - " }\n" - " return Buffer.from(bytes).toString(\"utf16le\");\n" - " }\n" - " case \"base64\":\n" - " case \"base64url\": {\n" - " let bytes = u8;\n" - " if (this._rem !== null) {\n" - " const joined = new Uint8Array(this._rem.length + u8.length);\n" - " joined.set(this._rem, 0);\n" - " joined.set(u8, this._rem.length);\n" - " bytes = joined;\n" - " this._rem = null;\n" - " }\n" - " const usable = bytes.length - (bytes.length % 3);\n" - " if (usable < bytes.length) {\n" - " this._rem = bytes.slice(usable);\n" - " bytes = bytes.subarray(0, usable);\n" - " }\n" - " return Buffer.from(bytes).toString(this.encoding);\n" - " }\n" - " default:\n" - " return Buffer.from(u8).toString(this.encoding);\n" + " if (typeof buf === 'string') return buf;\n" + " if (!ArrayBuffer.isView(buf)) {\n" + " const received = buf === null ? 'null' : buf === undefined ? 'undefined' : typeof buf === 'object' ? 'an instance of ' + (buf.constructor && buf.constructor.name || 'Object') : 'type ' + typeof buf + ' (' + String(buf) + ')';\n" + " const err = new TypeError('The \"buf\" argument must be an instance of Buffer, TypedArray, or DataView. Received ' + received);\n" + " err.code = 'ERR_INVALID_ARG_TYPE';\n" + " throw err;\n" " }\n" + " const bytes = new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);\n" + " const enc = this._encoding;\n" + " let offset = 0;\n" + " let out = '';\n" + " if (this._missing > 0) {\n" + " let take = Math.min(bytes.length, this._missing);\n" + " if (enc === 'utf8') {\n" + " for (let i = 0; i < take; i++) {\n" + " if ((bytes[i] & 0xc0) !== 0x80) { take = i; this._missing = i; break; }\n" + " }\n" + " }\n" + " const pending = new Uint8Array(this._pending.length + take);\n" + " pending.set(this._pending);\n" + " pending.set(bytes.subarray(0, take), this._pending.length);\n" + " this._pending = pending;\n" + " this._missing -= take;\n" + " offset = take;\n" + " if (this._missing === 0) {\n" + " out = Buffer.from(pending).toString(enc);\n" + " this._pending = new Uint8Array(0);\n" + " }\n" + " if (offset === bytes.length) return out;\n" + " }\n" + " const n = bytes.length - offset;\n" + " let held = 0;\n" + " let total = 0;\n" + " if (enc === 'utf8') {\n" + " for (let back = 1; back <= Math.min(n, 3); back++) {\n" + " const b = bytes[bytes.length - back];\n" + " if ((b & 0xc0) === 0x80) continue;\n" + " total = (b & 0xe0) === 0xc0 ? 2 : (b & 0xf0) === 0xe0 ? 3 : (b & 0xf8) === 0xf0 ? 4 : 0;\n" + " if (back < total) held = back;\n" + " break;\n" + " }\n" + " } else if (enc === 'utf16le') {\n" + " if (n % 2) { held = 1; total = 2; }\n" + " else if (n >= 2 && (bytes[bytes.length - 1] & 0xfc) === 0xd8) { held = 2; total = 4; }\n" + " } else if (enc === 'base64' || enc === 'base64url') {\n" + " held = n % 3;\n" + " total = 3;\n" + " }\n" + " const end = bytes.length - held;\n" + " this._pending = bytes.slice(end);\n" + " this._missing = held ? total - held : 0;\n" + " return out + Buffer.from(bytes.subarray(offset, end)).toString(enc);\n" " }\n" " end(buf) {\n" - " let out = buf !== undefined ? this.write(buf) : \"\";\n" - " switch (this.encoding) {\n" - " case \"utf8\":\n" - " out += this._dec.decode();\n" - " this._dec = new TextDecoder();\n" - " break;\n" - " case \"utf16le\":\n" - " this._oddByte = -1;\n" - " break;\n" - " case \"base64\":\n" - " case \"base64url\":\n" - " if (this._rem !== null) {\n" - " out += Buffer.from(this._rem).toString(this.encoding);\n" - " this._rem = null;\n" - " }\n" - " break;\n" - " }\n" - " return out;\n" + " const out = buf === undefined ? '' : this.write(buf);\n" + " const tail = Buffer.from(this._pending).toString(this._encoding);\n" + " this._pending = new Uint8Array(0);\n" + " this._missing = 0;\n" + " return out + tail;\n" " }\n" " }\n" " return { StringDecoder };\n" diff --git a/tests/corpus/node24-string-decoder.ts b/tests/corpus/node24-string-decoder.ts new file mode 100644 index 000000000..bbdf263d5 --- /dev/null +++ b/tests/corpus/node24-string-decoder.ts @@ -0,0 +1,32 @@ +import { StringDecoder } from "node:string_decoder"; + +const utf8 = new StringDecoder("utf8"); +console.log("split", JSON.stringify(utf8.write(Buffer.from([0xe2]))), JSON.stringify(utf8.write(Buffer.from([0x82]))), JSON.stringify(utf8.end(Buffer.from([0xac])))); +console.log("reuse", JSON.stringify(utf8.end(Buffer.from("again"))), JSON.stringify(utf8.end())); +console.log("bom", JSON.stringify(utf8.write(Buffer.from([0xef]))), JSON.stringify(utf8.end(Buffer.from([0xbb, 0xbf, 0x61])))); +// @ts-expect-error Node accepts strings; its declarations only name Uint8Array. +console.log("string", JSON.stringify(utf8.write(Buffer.from([0xe2]))), JSON.stringify(utf8.write("middle")), JSON.stringify(utf8.end("last"))); +console.log("undefined", JSON.stringify(utf8.write(Buffer.from([0xc3]))), JSON.stringify(utf8.end(undefined)), JSON.stringify(utf8.end())); +console.log("invalid", JSON.stringify(utf8.write(Buffer.from([0xe0]))), JSON.stringify(utf8.write(Buffer.from([0x80]))), JSON.stringify(utf8.end(Buffer.from([0x61])))); +console.log("view", JSON.stringify(utf8.end(new Uint8Array([0, 0x68, 0x69, 0]).subarray(1, 3)))); +// @ts-expect-error Node accepts every ArrayBufferView, including Uint16Array. +console.log("wide", JSON.stringify(utf8.end(new Uint16Array([0x6968])))); +// @ts-expect-error Node accepts DataView input as raw bytes. +console.log("data-view", utf8.end(new DataView(new Uint8Array([0, 0x68, 0x69, 0]).buffer, 1, 2))); +const utf16 = new StringDecoder("utf16le"); +console.log("utf16", JSON.stringify(utf16.write(Buffer.from([0x3d, 0xd8]))), JSON.stringify(utf16.write(Buffer.from([0x00]))), JSON.stringify(utf16.end(Buffer.from([0xde, 0x61, 0])))); +console.log("odd", JSON.stringify(utf16.write(Buffer.from([0x61]))), JSON.stringify(utf16.end(Buffer.from([0, 0x62]))), JSON.stringify(utf16.end(Buffer.from([0x63, 0])))); +const base64 = new StringDecoder("base64"); +console.log("base64", JSON.stringify(base64.write(Buffer.from([1]))), JSON.stringify(base64.end(Buffer.from([2, 3, 4]))), JSON.stringify(base64.end(Buffer.from([5])))); +const url = new StringDecoder("base64url"); +console.log("base64url", JSON.stringify(url.write(Buffer.from([251]))), JSON.stringify(url.end(Buffer.from([255, 254, 251])))); +const hex = new StringDecoder("hex"); +// @ts-expect-error Node StringDecoder.write accepts string passthrough. +console.log("hex", hex.write("unchanged"), hex.end(Buffer.from([0xab, 0xcd])), hex.end()); +let order = ""; +function receiver(): StringDecoder { order += "r"; return utf8; } +function chunk(): Buffer { order += "c"; return Buffer.from([0x61]); } +console.log("order", receiver().end(chunk()), order); + +function absent(): undefined { order += "u"; return undefined; } +console.log("undefined-order", receiver().end(absent()), order); diff --git a/tests/fixtures/npm/cases/node24-string-decoder/main.ts b/tests/fixtures/npm/cases/node24-string-decoder/main.ts new file mode 100644 index 000000000..6bfd82939 --- /dev/null +++ b/tests/fixtures/npm/cases/node24-string-decoder/main.ts @@ -0,0 +1,4 @@ +// @dynamic +// @stderr +import { report } from "node24-string-decoder-fixture"; +report(); diff --git a/tests/fixtures/npm/cases/node24-string-decoder/package.json b/tests/fixtures/npm/cases/node24-string-decoder/package.json new file mode 100644 index 000000000..089153bcb --- /dev/null +++ b/tests/fixtures/npm/cases/node24-string-decoder/package.json @@ -0,0 +1 @@ +{"type":"module"} diff --git a/tests/fixtures/npm/node_modules/node24-string-decoder-fixture/index.d.ts b/tests/fixtures/npm/node_modules/node24-string-decoder-fixture/index.d.ts new file mode 100644 index 000000000..826d89dbd --- /dev/null +++ b/tests/fixtures/npm/node_modules/node24-string-decoder-fixture/index.d.ts @@ -0,0 +1 @@ +export function report(): void; diff --git a/tests/fixtures/npm/node_modules/node24-string-decoder-fixture/index.js b/tests/fixtures/npm/node_modules/node24-string-decoder-fixture/index.js new file mode 100644 index 000000000..d49be5545 --- /dev/null +++ b/tests/fixtures/npm/node_modules/node24-string-decoder-fixture/index.js @@ -0,0 +1,43 @@ +import { StringDecoder } from "node:string_decoder"; +import { Buffer } from "node:buffer"; + +export function report() { + const bom = Buffer.from([0xef, 0xbb, 0xbf, 0x61]); + console.log('buffer-bom', JSON.stringify(bom.toString('utf8')), JSON.stringify(bom.toString('utf8', 0, 3)), JSON.stringify(bom.toString('utf8', 3))); + const inputs = [ + [0xef, 0xbb, 0xbf, 0x61], [0xc2, 0xa2], [0xe2, 0x82, 0xac], + [0xf0, 0x9f, 0x98, 0x80], [0xe0, 0x80, 0x61], [0xed, 0xa0, 0x80], + [0xf4, 0x90, 0x80, 0x80], [0xc0, 0xaf], [0xff, 0x80, 0xe2, 0x82], + [0x61, 0, 0x3d, 0xd8, 0, 0xde, 0x62, 0], [0x3d, 0xd8], [0x61], + [0x3d, 0xd8, 0, 0xde, 0x3d, 0xd8, 0, 0xde], + ]; + for (const enc of ['utf8', 'utf16le', 'base64', 'base64url', 'latin1', 'ascii', 'hex']) { + for (const input of inputs) { + const bytes = Buffer.from(input); + for (let a = 0; a <= bytes.length; a++) { + for (let b = a; b <= bytes.length; b++) { + const d = new StringDecoder(enc); + console.log(enc, a, b, JSON.stringify([ + d.write(bytes.subarray(0, a)), d.write(bytes.subarray(a, b)), + d.end(bytes.subarray(b)), d.end(), d.end(Buffer.from('ok')), + ])); + } + } + } + } + const d = new StringDecoder(); + console.log('strings', JSON.stringify([d.write(Buffer.from([0xe2])), d.write('middle'), d.end('last'), d.end()])); + console.log('empty', JSON.stringify([d.write(Buffer.from([0xe0])), d.write(Buffer.alloc(0)), d.write(Buffer.from([0x80])), d.end(undefined)])); + const backing = new Uint8Array([0, 0x68, 0x69, 0]); + console.log('views', d.end(new DataView(backing.buffer, 1, 2)), d.end(new Uint16Array([0x6968])), d.end(backing.subarray(1, 3))); + for (const encoding of [undefined, null, '', 'UTF-8', 'UCS-2', 'BINARY', 'nope', false, 0]) { + try { console.log('encoding', String(encoding), new StringDecoder(encoding).encoding); } + catch (e) { console.log('encoding-error', e.name, e.code, e.message); } + } + for (const input of [undefined, null, 1, {}, [], new ArrayBuffer(2)]) { + try { d.write(input); console.log('unexpected'); } + catch (e) { console.log('input-error', e.name, e.code, e.message); } + } + d.encoding = 'hex'; + console.log('encoding-property', d.encoding, d.end(Buffer.from('hi'))); +} diff --git a/tests/fixtures/npm/node_modules/node24-string-decoder-fixture/package.json b/tests/fixtures/npm/node_modules/node24-string-decoder-fixture/package.json new file mode 100644 index 000000000..b25eb224b --- /dev/null +++ b/tests/fixtures/npm/node_modules/node24-string-decoder-fixture/package.json @@ -0,0 +1 @@ +{"name":"node24-string-decoder-fixture","version":"1.0.0","type":"module","main":"index.js","types":"index.d.ts"}