mirror of
https://github.com/vercel-labs/scriptc.git
synced 2026-10-26 06:34:20 +08:00
Expand Node 24 StringDecoder input and streaming support
This commit is contained in:
Chris Tate
@@ -60,7 +60,7 @@
|
||||
"module": { "exports": ["createRequire", "builtinModules", "isBuiltin", "syncBuiltinESMExports", "register", "findSourceMap"], "refused": ["register"], "evidence": ["tests/fixtures/npm/cases/create-require/main.ts", "tests/fixtures/npm/cases/misc-shims/main.ts"] },
|
||||
"url": { "exports": ["URL", "URLSearchParams", "fileURLToPath", "pathToFileURL", "parse", "format", "resolve", "domainToASCII", "domainToUnicode", "urlToHttpOptions"], "members": ["url.toString", "url.toJSON", "URL.canParse", "URL.parse", "urlSearchParams.append", "urlSearchParams.delete", "urlSearchParams.entries", "urlSearchParams.forEach", "urlSearchParams.get", "urlSearchParams.getAll", "urlSearchParams.has", "urlSearchParams.keys", "urlSearchParams.set", "urlSearchParams.sort", "urlSearchParams.toString", "urlSearchParams.values", "urlSearchParams.size"], "evidence": ["tests/fixtures/npm/cases/misc-shims/main.ts", "tests/fixtures/npm/cases/island-web-plumbing/main.ts", "tests/fixtures/npm/cases/node24-url-factories/main.ts"] },
|
||||
"buffer": { "exports": ["Buffer", "SlowBuffer", "INSPECT_MAX_BYTES", "kMaxLength", "kStringMaxLength", "constants", "isAscii", "isUtf8", "atob", "btoa", "Blob", "File", "transcode", "resolveObjectURL"], "members": ["Buffer.alloc", "Buffer.allocUnsafe", "Buffer.allocUnsafeSlow", "Buffer.from", "Buffer.isBuffer", "Buffer.isEncoding", "Buffer.byteLength", "Buffer.concat", "Buffer.compare", "buf.compare", "buf.copy", "buf.equals", "buf.fill", "buf.includes", "buf.indexOf", "buf.lastIndexOf", "buf.subarray", "buf.slice", "buf.swap16", "buf.swap32", "buf.swap64", "buf.toJSON", "buf.toString", "buf.write", "buf.readBigInt64BE", "buf.readBigInt64LE", "buf.readBigUInt64BE", "buf.readBigUInt64LE", "buf.readDoubleBE", "buf.readDoubleLE", "buf.readFloatBE", "buf.readFloatLE", "buf.readInt8", "buf.readInt16BE", "buf.readInt16LE", "buf.readInt32BE", "buf.readInt32LE", "buf.readIntBE", "buf.readIntLE", "buf.readUInt8", "buf.readUInt16BE", "buf.readUInt16LE", "buf.readUInt32BE", "buf.readUInt32LE", "buf.readUIntBE", "buf.readUIntLE", "buf.writeBigInt64BE", "buf.writeBigInt64LE", "buf.writeBigUInt64BE", "buf.writeBigUInt64LE", "buf.writeDoubleBE", "buf.writeDoubleLE", "buf.writeFloatBE", "buf.writeFloatLE", "buf.writeInt8", "buf.writeInt16BE", "buf.writeInt16LE", "buf.writeInt32BE", "buf.writeInt32LE", "buf.writeIntBE", "buf.writeIntLE", "buf.writeUInt8", "buf.writeUInt16BE", "buf.writeUInt16LE", "buf.writeUInt32BE", "buf.writeUInt32LE", "buf.writeUIntBE", "buf.writeUIntLE"], "evidence": ["tests/fixtures/npm/cases/buffer-shims/main.ts"] },
|
||||
"string_decoder": { "exports": ["StringDecoder"], "members": ["stringDecoder.write", "stringDecoder.end"], "evidence": ["tests/fixtures/npm/cases/buffer-shims/main.ts"] },
|
||||
"string_decoder": { "exports": ["StringDecoder"], "members": ["stringDecoder.write", "stringDecoder.end"], "evidence": ["tests/fixtures/npm/cases/buffer-shims/main.ts", "tests/fixtures/npm/cases/node24-string-decoder/main.ts"] },
|
||||
"assert": { "exports": ["AssertionError", "ok", "fail", "equal", "notEqual", "strictEqual", "notStrictEqual", "deepEqual", "notDeepEqual", "deepStrictEqual", "notDeepStrictEqual", "throws", "doesNotThrow", "rejects", "doesNotReject", "match", "doesNotMatch", "ifError", "strict"], "evidence": ["tests/fixtures/npm/cases/assert-shims/main.ts"] },
|
||||
"assert/strict": { "exports": ["AssertionError", "ok", "fail", "equal", "notEqual", "strictEqual", "notStrictEqual", "deepEqual", "notDeepEqual", "deepStrictEqual", "notDeepStrictEqual", "throws", "doesNotThrow", "rejects", "doesNotReject", "match", "doesNotMatch", "ifError", "strict"], "evidence": ["tests/fixtures/npm/cases/assert-shims/main.ts"] },
|
||||
"domain": { "exports": ["create", "createDomain", "Domain", "active"], "members": ["domain.add", "domain.bind", "domain.enter", "domain.exit", "domain.intercept", "domain.remove", "domain.run"], "evidence": ["tests/fixtures/npm/cases/island-web-plumbing/main.ts"] },
|
||||
|
||||
@@ -82971,7 +82971,8 @@
|
||||
"confidence": "high",
|
||||
"source": "compiler-dedicated:StringDecoder",
|
||||
"tests": [
|
||||
"tests/corpus/1662-string-decoder-encodings.ts"
|
||||
"tests/corpus/1662-string-decoder-encodings.ts",
|
||||
"tests/corpus/node24-string-decoder.ts"
|
||||
]
|
||||
},
|
||||
"dynamic": {
|
||||
@@ -82981,7 +82982,8 @@
|
||||
"confidence": "high",
|
||||
"source": "island-export:node:string_decoder.StringDecoder",
|
||||
"tests": [
|
||||
"tests/fixtures/npm/cases/buffer-shims/main.ts"
|
||||
"tests/fixtures/npm/cases/buffer-shims/main.ts",
|
||||
"tests/fixtures/npm/cases/node24-string-decoder/main.ts"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -83008,7 +83010,8 @@
|
||||
"confidence": "high",
|
||||
"source": "compiler-dedicated:stringDecoder.end",
|
||||
"tests": [
|
||||
"tests/corpus/1662-string-decoder-encodings.ts"
|
||||
"tests/corpus/1662-string-decoder-encodings.ts",
|
||||
"tests/corpus/node24-string-decoder.ts"
|
||||
]
|
||||
},
|
||||
"dynamic": {
|
||||
@@ -83018,7 +83021,8 @@
|
||||
"confidence": "high",
|
||||
"source": "island-member:node:string_decoder.stringDecoder.end",
|
||||
"tests": [
|
||||
"tests/fixtures/npm/cases/buffer-shims/main.ts"
|
||||
"tests/fixtures/npm/cases/buffer-shims/main.ts",
|
||||
"tests/fixtures/npm/cases/node24-string-decoder/main.ts"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -83045,7 +83049,8 @@
|
||||
"confidence": "high",
|
||||
"source": "compiler-dedicated:stringDecoder.write",
|
||||
"tests": [
|
||||
"tests/corpus/1662-string-decoder-encodings.ts"
|
||||
"tests/corpus/1662-string-decoder-encodings.ts",
|
||||
"tests/corpus/node24-string-decoder.ts"
|
||||
]
|
||||
},
|
||||
"dynamic": {
|
||||
@@ -83055,7 +83060,8 @@
|
||||
"confidence": "high",
|
||||
"source": "island-member:node:string_decoder.stringDecoder.write",
|
||||
"tests": [
|
||||
"tests/fixtures/npm/cases/buffer-shims/main.ts"
|
||||
"tests/fixtures/npm/cases/buffer-shims/main.ts",
|
||||
"tests/fixtures/npm/cases/node24-string-decoder/main.ts"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100291,14 +100291,16 @@
|
||||
"status": "partial",
|
||||
"evidence": "derived:descendants",
|
||||
"tests": [
|
||||
"tests/corpus/1662-string-decoder-encodings.ts"
|
||||
"tests/corpus/1662-string-decoder-encodings.ts",
|
||||
"tests/corpus/node24-string-decoder.ts"
|
||||
]
|
||||
},
|
||||
"dynamic": {
|
||||
"status": "partial",
|
||||
"evidence": "derived:descendants",
|
||||
"tests": [
|
||||
"tests/fixtures/npm/cases/buffer-shims/main.ts"
|
||||
"tests/fixtures/npm/cases/buffer-shims/main.ts",
|
||||
"tests/fixtures/npm/cases/node24-string-decoder/main.ts"
|
||||
]
|
||||
},
|
||||
"anchorSource": "chapter"
|
||||
@@ -100324,14 +100326,16 @@
|
||||
"status": "partial",
|
||||
"evidence": "compiler-dedicated:StringDecoder",
|
||||
"tests": [
|
||||
"tests/corpus/1662-string-decoder-encodings.ts"
|
||||
"tests/corpus/1662-string-decoder-encodings.ts",
|
||||
"tests/corpus/node24-string-decoder.ts"
|
||||
]
|
||||
},
|
||||
"dynamic": {
|
||||
"status": "partial",
|
||||
"evidence": "island-export:node:string_decoder.StringDecoder",
|
||||
"tests": [
|
||||
"tests/fixtures/npm/cases/buffer-shims/main.ts"
|
||||
"tests/fixtures/npm/cases/buffer-shims/main.ts",
|
||||
"tests/fixtures/npm/cases/node24-string-decoder/main.ts"
|
||||
]
|
||||
},
|
||||
"anchorSource": "exact"
|
||||
@@ -100357,14 +100361,16 @@
|
||||
"status": "partial",
|
||||
"evidence": "compiler-dedicated:stringDecoder.end",
|
||||
"tests": [
|
||||
"tests/corpus/1662-string-decoder-encodings.ts"
|
||||
"tests/corpus/1662-string-decoder-encodings.ts",
|
||||
"tests/corpus/node24-string-decoder.ts"
|
||||
]
|
||||
},
|
||||
"dynamic": {
|
||||
"status": "partial",
|
||||
"evidence": "island-member:node:string_decoder.stringDecoder.end",
|
||||
"tests": [
|
||||
"tests/fixtures/npm/cases/buffer-shims/main.ts"
|
||||
"tests/fixtures/npm/cases/buffer-shims/main.ts",
|
||||
"tests/fixtures/npm/cases/node24-string-decoder/main.ts"
|
||||
]
|
||||
},
|
||||
"anchorSource": "exact"
|
||||
@@ -100390,14 +100396,16 @@
|
||||
"status": "partial",
|
||||
"evidence": "compiler-dedicated:stringDecoder.write",
|
||||
"tests": [
|
||||
"tests/corpus/1662-string-decoder-encodings.ts"
|
||||
"tests/corpus/1662-string-decoder-encodings.ts",
|
||||
"tests/corpus/node24-string-decoder.ts"
|
||||
]
|
||||
},
|
||||
"dynamic": {
|
||||
"status": "partial",
|
||||
"evidence": "island-member:node:string_decoder.stringDecoder.write",
|
||||
"tests": [
|
||||
"tests/fixtures/npm/cases/buffer-shims/main.ts"
|
||||
"tests/fixtures/npm/cases/buffer-shims/main.ts",
|
||||
"tests/fixtures/npm/cases/node24-string-decoder/main.ts"
|
||||
]
|
||||
},
|
||||
"anchorSource": "exact"
|
||||
|
||||
@@ -79,7 +79,7 @@
|
||||
"node-builtin.querystring": ["tests/corpus/957-builtins-namespace.ts", "tests/corpus/2917-builtin-callable-widths.ts"],
|
||||
"node-builtin.readline": ["tests/corpus/1475-readline-closed-stdin.ts"],
|
||||
"node-builtin.stream": ["tests/corpus/1685-stream-readable-basics.ts", "tests/corpus/1746-stream-for-await.ts", "tests/corpus/1814-stream-pipeline.cjs", "tests/corpus/2629-stream-consumers.ts", "tests/corpus/2861-readable-iterator-close.ts"],
|
||||
"node-builtin.string_decoder": ["tests/corpus/1662-string-decoder-encodings.ts"],
|
||||
"node-builtin.string_decoder": ["tests/corpus/1662-string-decoder-encodings.ts", "tests/corpus/node24-string-decoder.ts"],
|
||||
"node-builtin.test": ["tests/harness/node-test.test.ts"],
|
||||
"node-builtin.timers": ["tests/corpus/1804-timers-module.ts", "tests/corpus/2093-timers-promises.ts"],
|
||||
"node-builtin.tls": ["tests/corpus/2331-tls-server-options-typed.ts", "tests/corpus/2557-tls-ca-store.ts"],
|
||||
@@ -198,7 +198,7 @@
|
||||
{ "symbols": ["crypto.hkdf", "crypto.hkdfSync", "crypto.scrypt", "crypto.scryptSync"], "status": "partial", "evidence": ["tests/corpus/node24-crypto-kdf.ts", "tests/corpus/node24-crypto-kdf-errors.ts", "tests/corpus/node24-crypto-kdf-options.ts"] },
|
||||
{ "symbols": ["EventEmitter", "emitter.on", "emitter.addListener", "emitter.once", "emitter.prependListener", "emitter.prependOnceListener", "emitter.off", "emitter.removeListener", "emitter.removeAllListeners", "emitter.emit", "emitter.listenerCount", "emitter.listeners", "emitter.rawListeners", "emitter.eventNames", "emitter.setMaxListeners", "emitter.getMaxListeners", "events.getEventListeners", "events.listenerCount", "events.once", "events.on", "events.setMaxListeners"], "status": "partial", "evidence": ["tests/corpus/1677-emitter-listeners.ts", "tests/corpus/2620-ee-override-once-order.ts"] },
|
||||
{ "symbols": ["AsyncLocalStorage", "asyncLocalStorage.run", "asyncLocalStorage.exit", "asyncLocalStorage.getStore", "asyncLocalStorage.enterWith", "asyncLocalStorage.disable"], "status": "partial", "evidence": ["tests/corpus/2213-async-local-storage.cjs"] },
|
||||
{ "symbols": ["StringDecoder", "stringDecoder.write", "stringDecoder.end"], "status": "partial", "evidence": ["tests/corpus/1662-string-decoder-encodings.ts"] },
|
||||
{ "symbols": ["StringDecoder", "stringDecoder.write", "stringDecoder.end"], "status": "partial", "evidence": ["tests/corpus/1662-string-decoder-encodings.ts", "tests/corpus/node24-string-decoder.ts"] },
|
||||
{ "symbols": ["setImmediate", "setInterval", "setTimeout", "clearImmediate", "clearInterval", "clearTimeout", "timersPromises.setTimeout", "timersPromises.setImmediate", "timersPromises.setInterval", "immediate.hasRef", "immediate.ref", "immediate.unref", "timeout.hasRef", "timeout.ref", "timeout.unref", "timeout.refresh"], "status": "partial", "evidence": ["tests/corpus/1804-timers-module.ts", "tests/corpus/1805-timer-callback-args.ts", "tests/corpus/2093-timers-promises.ts", "tests/corpus/2852-timers-promises-interval.ts"] },
|
||||
{ "symbols": ["stream.finished", "stream.pipeline", "stream.getDefaultHighWaterMark", "readable.push", "readable.unshift", "readable.read", "readable.pause", "readable.resume", "readable.isPaused", "readable.setEncoding", "readable.pipe", "readable.unpipe", "readable.destroy", "writable.write", "writable.end", "writable.cork", "writable.uncork", "writable.setDefaultEncoding"], "status": "partial", "evidence": ["tests/corpus/1685-stream-readable-basics.ts", "tests/corpus/1688-stream-writable-basics.ts", "tests/corpus/1814-stream-pipeline.cjs"] },
|
||||
{ "symbols": ["net.createServer", "net.connect", "net.createConnection", "net.getDefaultAutoSelectFamilyAttemptTimeout", "net.setDefaultAutoSelectFamilyAttemptTimeout", "server.address", "server.close", "server.listen", "socket.destroy", "socket.end", "socket.pause", "socket.resume", "socket.setEncoding", "socket.setTimeout", "socket.write"], "status": "partial", "evidence": ["tests/corpus/2500-net-autosel-timeout.ts", "tests/corpus/2596-net-arg-ladders.cjs"] },
|
||||
|
||||
@@ -4779,13 +4779,8 @@ export function lowerForkCall(lowerer: Lowerer, expr: ts.CallExpression, loc: Sr
|
||||
);
|
||||
}
|
||||
|
||||
/** Method calls on StringDecoder receivers: `d.write(chunk)` decodes the
|
||||
* complete prefix of pending+chunk and re-buffers the trailing partial
|
||||
* sequence; `d.end()` flushes the buffered partial as its replacement
|
||||
* chars — Node's utf8 StringDecoder exactly (SEMANTICS.md), through the
|
||||
* interned %strdec helpers over the packed-f64 pending field. end(buf)
|
||||
* and the rest of @types/node's surface fence per member. Null for
|
||||
* non-decoder receivers. */
|
||||
/** StringDecoder writes preserve pending bytes; end(input) performs the
|
||||
* final write and flushes the state. String input passes through unchanged. */
|
||||
export function lowerStringDecoderMethodCall(lowerer: Lowerer, call: ts.CallExpression,
|
||||
access: ts.PropertyAccessExpression,): IrExpr | null {
|
||||
if (call.questionDotToken || access.questionDotToken) return null;
|
||||
@@ -4793,47 +4788,43 @@ export function lowerForkCall(lowerer: Lowerer, expr: ts.CallExpression, loc: Sr
|
||||
if (!lowerer.isStdlibMember(access)) return null;
|
||||
const name = access.name.text;
|
||||
const loc = locOf(call);
|
||||
if (name === "write" && call.arguments.length === 1) {
|
||||
const receiver = lowerer.lowerExpr(access.expression);
|
||||
if (receiver.type.kind !== "record") lowerer.badType(access.expression, lowerer.typeOf(access.expression));
|
||||
let chunk = lowerBuiltinValuePreservingUndefined(lowerer, call.arguments[0]!);
|
||||
chunk = checkedOptionalBuiltinArm(lowerer, chunk, BYTES_U8) ?? chunk;
|
||||
if (!(chunk.type.kind === "bytes" && chunk.type.elem === "u8")) {
|
||||
lowerer.noLowering(
|
||||
`StringDecoder.write of '${lowerer.fmt(chunk.type)}' data`,
|
||||
call.arguments[0]!,
|
||||
"Buffer/Uint8Array chunks decode (narrow unions first)",
|
||||
);
|
||||
}
|
||||
const helper = lowerer.strdecHelper("write", receiver.type.shapeId, loc);
|
||||
return { kind: "call", callee: helper, args: [receiver, chunk], type: STRING, loc };
|
||||
if ((name !== "write" && name !== "end") || call.arguments.length > 1 ||
|
||||
(name === "write" && call.arguments.length !== 1) || call.arguments.some(ts.isSpreadElement)) {
|
||||
lowerer.noLowering(`StringDecoder.${name}`, call, "use write(input) or end(input?)", lowerer.checker.getSymbolAtLocation(access.name));
|
||||
}
|
||||
if (name === "end") {
|
||||
if (call.arguments.length !== 0) {
|
||||
lowerer.noLowering(
|
||||
"StringDecoder.end with a buffer argument",
|
||||
call,
|
||||
"write the buffer, then end(): d.write(buf) + d.end() is Node's own equivalence",
|
||||
);
|
||||
}
|
||||
const receiver = lowerer.lowerExpr(access.expression);
|
||||
if (receiver.type.kind !== "record") lowerer.badType(access.expression, lowerer.typeOf(access.expression));
|
||||
const helper = lowerer.strdecHelper("end", receiver.type.shapeId, loc);
|
||||
return { kind: "call", callee: helper, args: [receiver], type: STRING, loc };
|
||||
const receiver = lowerer.lowerExpr(access.expression);
|
||||
if (receiver.type.kind !== "record") lowerer.badType(access.expression, lowerer.typeOf(access.expression));
|
||||
if (call.arguments.length === 0) {
|
||||
return { kind: "call", callee: lowerer.strdecHelper("end", receiver.type.shapeId, loc), args: [receiver], type: STRING, loc };
|
||||
}
|
||||
lowerer.noLowering(
|
||||
`StringDecoder.${name}`,
|
||||
call,
|
||||
"write(buffer) and end() are the supported StringDecoder members",
|
||||
lowerer.checker.getSymbolAtLocation(access.name),
|
||||
);
|
||||
const node = call.arguments[0]!;
|
||||
const undefinedArg = name === "end" ? lowerStaticallyUndefinedArgument(lowerer, node) : null;
|
||||
if (undefinedArg) {
|
||||
const saved = lowerer.declareHiddenLocal("%strdecRecv", receiver.type);
|
||||
const result: IrExpr = { kind: "call", callee: lowerer.strdecHelper("end", receiver.type.shapeId, loc), args: [varRef(saved.id, receiver.type, loc)], type: STRING, loc };
|
||||
return { kind: "seqExpr", stmts: [{ kind: "varDecl", localId: saved.id, init: receiver, loc }], result: defaultAfterUndefined(undefinedArg, result), type: STRING, loc };
|
||||
}
|
||||
let chunk = lowerBuiltinValuePreservingUndefined(lowerer, node);
|
||||
if (chunk.type.kind === "string") {
|
||||
const op = name === "end" ? "endString" : "writeString";
|
||||
return { kind: "call", callee: lowerer.strdecHelper(op, receiver.type.shapeId, loc), args: [receiver, chunk], type: STRING, loc };
|
||||
}
|
||||
chunk = checkedOptionalBuiltinArm(lowerer, chunk, BYTES_U8) ?? chunk;
|
||||
if (chunk.type.kind === "bytes" && chunk.type.elem !== "u8") {
|
||||
chunk = { kind: "libCall", fn: "bytes.bufferSource", args: [lowerer.coerceInto(node, chunk, DYN)], type: BYTES_U8, loc };
|
||||
}
|
||||
if (!(chunk.type.kind === "bytes" && chunk.type.elem === "u8")) {
|
||||
lowerer.noLowering(`StringDecoder.${name} of '${lowerer.fmt(chunk.type)}' data`, node, "pass a string or typed-array view (narrow unions first)");
|
||||
}
|
||||
const helper = lowerer.strdecHelper(name === "end" ? "endChunk" : "write", receiver.type.shapeId, loc);
|
||||
return { kind: "call", callee: helper, args: [receiver, chunk], type: STRING, loc };
|
||||
}
|
||||
|
||||
/** The interned %strdec helpers: write(d, chunk) returns the decoded
|
||||
* complete prefix and re-buffers the trailing partial into the pending
|
||||
* field; end(d) flushes it. Both thread the packed-f64 state through
|
||||
* the pure strdec.* libCalls. */
|
||||
export function strdecHelper(lowerer: Lowerer, op: "write" | "end", shapeId: string, loc: SrcLoc): string {
|
||||
export function strdecHelper(lowerer: Lowerer, op: "write" | "end" | "endChunk" | "endString" | "writeString", shapeId: string, loc: SrcLoc): string {
|
||||
const key = `strdec.${op}`;
|
||||
const existing = lowerer.widthHelpers.get(key);
|
||||
if (existing) return existing;
|
||||
@@ -4865,7 +4856,7 @@ export function lowerForkCall(lowerer: Lowerer, expr: ts.CallExpression, loc: Sr
|
||||
{ id: "s.0", name: "s", type: STRING, mutable: false },
|
||||
];
|
||||
let body: IrStmt[];
|
||||
if (op === "write") {
|
||||
if (op === "write" || op === "endChunk") {
|
||||
params.push({ localId: "chunk.0", name: "chunk", type: BYTES_U8 });
|
||||
locals.splice(1, 0, { id: "chunk.0", name: "chunk", type: BYTES_U8, mutable: false });
|
||||
body = [
|
||||
@@ -4904,6 +4895,24 @@ export function lowerForkCall(lowerer: Lowerer, expr: ts.CallExpression, loc: Sr
|
||||
{ kind: "return", value: varRef("s.0", STRING, loc), loc },
|
||||
];
|
||||
}
|
||||
if (op === "endString" || op === "writeString") {
|
||||
const inputT = STRING;
|
||||
params.push({ localId: "chunk.0", name: "chunk", type: inputT });
|
||||
locals.push({ id: "chunk.0", name: "chunk", type: inputT, mutable: false });
|
||||
if (op === "writeString") body = [{ kind: "return", value: varRef("chunk.0", STRING, loc), loc }];
|
||||
if (op === "endString") body[body.length - 1] = { kind: "return", value: {
|
||||
kind: "strConcat", left: varRef("chunk.0", STRING, loc), right: varRef("s.0", STRING, loc), type: STRING, loc,
|
||||
}, loc };
|
||||
}
|
||||
if (op === "endChunk") {
|
||||
locals.push({ id: "tail.0", name: "tail", type: STRING, mutable: false });
|
||||
body.splice(body.length - 1, 0,
|
||||
{ kind: "varDecl", localId: "tail.0", init: { kind: "libCall", fn: "strdec.end", args: [encRead(), pendingRead()], type: STRING, loc }, loc },
|
||||
{ kind: "recordSet", obj: varRef("d.0", recT, loc), shapeId, field: "%pending", value: { kind: "numLit", value: 0, type: F64, loc }, loc });
|
||||
body[body.length - 1] = { kind: "return", value: {
|
||||
kind: "strConcat", left: varRef("s.0", STRING, loc), right: varRef("tail.0", STRING, loc), type: STRING, loc,
|
||||
}, loc };
|
||||
}
|
||||
lowerer.liftedFns.push({ name, params, returnType: STRING, locals, body, loc });
|
||||
return name;
|
||||
}
|
||||
|
||||
@@ -11024,7 +11024,7 @@ export class Lowerer {
|
||||
return lowerDcTracingChannelProperty(this, access);
|
||||
}
|
||||
|
||||
strdecHelper(op: "write" | "end", shapeId: string, loc: SrcLoc): string {
|
||||
strdecHelper(op: "write" | "end" | "endChunk" | "endString" | "writeString", shapeId: string, loc: SrcLoc): string {
|
||||
return strdecHelper(this, op, shapeId, loc);
|
||||
}
|
||||
|
||||
|
||||
@@ -13384,6 +13384,18 @@
|
||||
"<repo>/tests/corpus/js-open-binding-writes.js"
|
||||
],
|
||||
"diags": []
|
||||
},
|
||||
"<repo>/tests/corpus/node24-string-decoder.ts": {
|
||||
"order": [
|
||||
"<repo>/tests/corpus/node24-string-decoder.ts"
|
||||
],
|
||||
"diags": []
|
||||
},
|
||||
"<repo>/tests/fixtures/npm/cases/node24-string-decoder/main.ts": {
|
||||
"order": [
|
||||
"<repo>/tests/fixtures/npm/cases/node24-string-decoder/main.ts"
|
||||
],
|
||||
"diags": []
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4905,7 +4905,7 @@ static const char isl_modules_bootstrap[] =
|
||||
" const K_STRING_MAX_LENGTH = 536870888;\n"
|
||||
" const INSPECT_MAX_BYTES = 50;\n"
|
||||
" const utf8Enc = new TextEncoder();\n"
|
||||
" const utf8Dec = new TextDecoder();\n"
|
||||
" const utf8Dec = new TextDecoder(\"utf-8\", { ignoreBOM: true });\n"
|
||||
" const B64 = \"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/\";\n"
|
||||
" const B64U = \"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_\";\n"
|
||||
" const b64Val = (() => {\n"
|
||||
@@ -5667,97 +5667,89 @@ static const char isl_modules_bootstrap[] =
|
||||
" mod.default = mod;\n"
|
||||
" return mod;\n"
|
||||
" });\n"
|
||||
/* node:string_decoder — StringDecoder over the Buffer shim: utf8
|
||||
* rides the prelude TextDecoder's streaming state machine,
|
||||
* utf16le holds byte parity, base64 carries mod-3 remainders. */
|
||||
/* node:string_decoder — preserve Node's byte buffering independently
|
||||
* of WHATWG TextDecoder: BOMs, invalid prefixes, UTF-16 pairs, and
|
||||
* base64 groups retain their chunk-boundary behavior. */
|
||||
" builtins.string_decoder = memo(() => {\n"
|
||||
"function makeStringDecoder(Buffer) {\n"
|
||||
" const normEnc = (enc) => {\n"
|
||||
" if (enc === undefined || enc === null) return \"utf8\";\n"
|
||||
" const e = String(enc).toLowerCase();\n"
|
||||
" if (e === \"utf8\" || e === \"utf-8\") return \"utf8\";\n"
|
||||
" if (e === \"hex\") return \"hex\";\n"
|
||||
" if (e === \"base64\") return \"base64\";\n"
|
||||
" if (e === \"base64url\") return \"base64url\";\n"
|
||||
" if (e === \"latin1\" || e === \"binary\") return \"latin1\";\n"
|
||||
" if (e === \"ascii\") return \"ascii\";\n"
|
||||
" if (e === \"utf16le\" || e === \"utf-16le\" || e === \"ucs2\" || e === \"ucs-2\") return \"utf16le\";\n"
|
||||
" const err = new TypeError(\"Unknown encoding: \" + enc);\n"
|
||||
" err.code = \"ERR_UNKNOWN_ENCODING\";\n"
|
||||
" if (enc === undefined || enc === null || enc === '') return 'utf8';\n"
|
||||
" const e = typeof enc === 'string' ? enc.toLowerCase() : enc;\n"
|
||||
" if (e === 'utf8' || e === 'utf-8') return 'utf8';\n"
|
||||
" if (e === 'hex' || e === 'base64' || e === 'base64url' || e === 'ascii') return e;\n"
|
||||
" if (e === 'latin1' || e === 'binary') return 'latin1';\n"
|
||||
" if (e === 'utf16le' || e === 'utf-16le' || e === 'ucs2' || e === 'ucs-2') return 'utf16le';\n"
|
||||
" const err = new TypeError('Unknown encoding: ' + String(enc));\n"
|
||||
" err.code = 'ERR_UNKNOWN_ENCODING';\n"
|
||||
" throw err;\n"
|
||||
" };\n"
|
||||
" class StringDecoder {\n"
|
||||
" constructor(encoding) {\n"
|
||||
" this.encoding = normEnc(encoding);\n"
|
||||
" if (this.encoding === \"utf8\") {\n"
|
||||
" this._dec = new TextDecoder();\n"
|
||||
" } else if (this.encoding === \"utf16le\") {\n"
|
||||
" this._oddByte = -1;\n"
|
||||
" } else if (this.encoding === \"base64\" || this.encoding === \"base64url\") {\n"
|
||||
" this._rem = null;\n"
|
||||
" }\n"
|
||||
" this._encoding = this.encoding;\n"
|
||||
" this._pending = new Uint8Array(0);\n"
|
||||
" this._missing = 0;\n"
|
||||
" }\n"
|
||||
" write(buf) {\n"
|
||||
" if (typeof buf === \"string\") return buf;\n"
|
||||
" const u8 = buf instanceof Uint8Array ? buf : new Uint8Array(buf);\n"
|
||||
" switch (this.encoding) {\n"
|
||||
" case \"utf8\":\n"
|
||||
" return this._dec.decode(u8, { stream: true });\n"
|
||||
" case \"utf16le\": {\n"
|
||||
" let bytes = u8;\n"
|
||||
" if (this._oddByte >= 0) {\n"
|
||||
" const joined = new Uint8Array(u8.length + 1);\n"
|
||||
" joined[0] = this._oddByte;\n"
|
||||
" joined.set(u8, 1);\n"
|
||||
" bytes = joined;\n"
|
||||
" this._oddByte = -1;\n"
|
||||
" }\n"
|
||||
" if (bytes.length % 2 !== 0) {\n"
|
||||
" this._oddByte = bytes[bytes.length - 1];\n"
|
||||
" bytes = bytes.subarray(0, bytes.length - 1);\n"
|
||||
" }\n"
|
||||
" return Buffer.from(bytes).toString(\"utf16le\");\n"
|
||||
" }\n"
|
||||
" case \"base64\":\n"
|
||||
" case \"base64url\": {\n"
|
||||
" let bytes = u8;\n"
|
||||
" if (this._rem !== null) {\n"
|
||||
" const joined = new Uint8Array(this._rem.length + u8.length);\n"
|
||||
" joined.set(this._rem, 0);\n"
|
||||
" joined.set(u8, this._rem.length);\n"
|
||||
" bytes = joined;\n"
|
||||
" this._rem = null;\n"
|
||||
" }\n"
|
||||
" const usable = bytes.length - (bytes.length % 3);\n"
|
||||
" if (usable < bytes.length) {\n"
|
||||
" this._rem = bytes.slice(usable);\n"
|
||||
" bytes = bytes.subarray(0, usable);\n"
|
||||
" }\n"
|
||||
" return Buffer.from(bytes).toString(this.encoding);\n"
|
||||
" }\n"
|
||||
" default:\n"
|
||||
" return Buffer.from(u8).toString(this.encoding);\n"
|
||||
" if (typeof buf === 'string') return buf;\n"
|
||||
" if (!ArrayBuffer.isView(buf)) {\n"
|
||||
" const received = buf === null ? 'null' : buf === undefined ? 'undefined' : typeof buf === 'object' ? 'an instance of ' + (buf.constructor && buf.constructor.name || 'Object') : 'type ' + typeof buf + ' (' + String(buf) + ')';\n"
|
||||
" const err = new TypeError('The \"buf\" argument must be an instance of Buffer, TypedArray, or DataView. Received ' + received);\n"
|
||||
" err.code = 'ERR_INVALID_ARG_TYPE';\n"
|
||||
" throw err;\n"
|
||||
" }\n"
|
||||
" const bytes = new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);\n"
|
||||
" const enc = this._encoding;\n"
|
||||
" let offset = 0;\n"
|
||||
" let out = '';\n"
|
||||
" if (this._missing > 0) {\n"
|
||||
" let take = Math.min(bytes.length, this._missing);\n"
|
||||
" if (enc === 'utf8') {\n"
|
||||
" for (let i = 0; i < take; i++) {\n"
|
||||
" if ((bytes[i] & 0xc0) !== 0x80) { take = i; this._missing = i; break; }\n"
|
||||
" }\n"
|
||||
" }\n"
|
||||
" const pending = new Uint8Array(this._pending.length + take);\n"
|
||||
" pending.set(this._pending);\n"
|
||||
" pending.set(bytes.subarray(0, take), this._pending.length);\n"
|
||||
" this._pending = pending;\n"
|
||||
" this._missing -= take;\n"
|
||||
" offset = take;\n"
|
||||
" if (this._missing === 0) {\n"
|
||||
" out = Buffer.from(pending).toString(enc);\n"
|
||||
" this._pending = new Uint8Array(0);\n"
|
||||
" }\n"
|
||||
" if (offset === bytes.length) return out;\n"
|
||||
" }\n"
|
||||
" const n = bytes.length - offset;\n"
|
||||
" let held = 0;\n"
|
||||
" let total = 0;\n"
|
||||
" if (enc === 'utf8') {\n"
|
||||
" for (let back = 1; back <= Math.min(n, 3); back++) {\n"
|
||||
" const b = bytes[bytes.length - back];\n"
|
||||
" if ((b & 0xc0) === 0x80) continue;\n"
|
||||
" total = (b & 0xe0) === 0xc0 ? 2 : (b & 0xf0) === 0xe0 ? 3 : (b & 0xf8) === 0xf0 ? 4 : 0;\n"
|
||||
" if (back < total) held = back;\n"
|
||||
" break;\n"
|
||||
" }\n"
|
||||
" } else if (enc === 'utf16le') {\n"
|
||||
" if (n % 2) { held = 1; total = 2; }\n"
|
||||
" else if (n >= 2 && (bytes[bytes.length - 1] & 0xfc) === 0xd8) { held = 2; total = 4; }\n"
|
||||
" } else if (enc === 'base64' || enc === 'base64url') {\n"
|
||||
" held = n % 3;\n"
|
||||
" total = 3;\n"
|
||||
" }\n"
|
||||
" const end = bytes.length - held;\n"
|
||||
" this._pending = bytes.slice(end);\n"
|
||||
" this._missing = held ? total - held : 0;\n"
|
||||
" return out + Buffer.from(bytes.subarray(offset, end)).toString(enc);\n"
|
||||
" }\n"
|
||||
" end(buf) {\n"
|
||||
" let out = buf !== undefined ? this.write(buf) : \"\";\n"
|
||||
" switch (this.encoding) {\n"
|
||||
" case \"utf8\":\n"
|
||||
" out += this._dec.decode();\n"
|
||||
" this._dec = new TextDecoder();\n"
|
||||
" break;\n"
|
||||
" case \"utf16le\":\n"
|
||||
" this._oddByte = -1;\n"
|
||||
" break;\n"
|
||||
" case \"base64\":\n"
|
||||
" case \"base64url\":\n"
|
||||
" if (this._rem !== null) {\n"
|
||||
" out += Buffer.from(this._rem).toString(this.encoding);\n"
|
||||
" this._rem = null;\n"
|
||||
" }\n"
|
||||
" break;\n"
|
||||
" }\n"
|
||||
" return out;\n"
|
||||
" const out = buf === undefined ? '' : this.write(buf);\n"
|
||||
" const tail = Buffer.from(this._pending).toString(this._encoding);\n"
|
||||
" this._pending = new Uint8Array(0);\n"
|
||||
" this._missing = 0;\n"
|
||||
" return out + tail;\n"
|
||||
" }\n"
|
||||
" }\n"
|
||||
" return { StringDecoder };\n"
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import { StringDecoder } from "node:string_decoder";
|
||||
|
||||
const utf8 = new StringDecoder("utf8");
|
||||
console.log("split", JSON.stringify(utf8.write(Buffer.from([0xe2]))), JSON.stringify(utf8.write(Buffer.from([0x82]))), JSON.stringify(utf8.end(Buffer.from([0xac]))));
|
||||
console.log("reuse", JSON.stringify(utf8.end(Buffer.from("again"))), JSON.stringify(utf8.end()));
|
||||
console.log("bom", JSON.stringify(utf8.write(Buffer.from([0xef]))), JSON.stringify(utf8.end(Buffer.from([0xbb, 0xbf, 0x61]))));
|
||||
// @ts-expect-error Node accepts strings; its declarations only name Uint8Array.
|
||||
console.log("string", JSON.stringify(utf8.write(Buffer.from([0xe2]))), JSON.stringify(utf8.write("middle")), JSON.stringify(utf8.end("last")));
|
||||
console.log("undefined", JSON.stringify(utf8.write(Buffer.from([0xc3]))), JSON.stringify(utf8.end(undefined)), JSON.stringify(utf8.end()));
|
||||
console.log("invalid", JSON.stringify(utf8.write(Buffer.from([0xe0]))), JSON.stringify(utf8.write(Buffer.from([0x80]))), JSON.stringify(utf8.end(Buffer.from([0x61]))));
|
||||
console.log("view", JSON.stringify(utf8.end(new Uint8Array([0, 0x68, 0x69, 0]).subarray(1, 3))));
|
||||
// @ts-expect-error Node accepts every ArrayBufferView, including Uint16Array.
|
||||
console.log("wide", JSON.stringify(utf8.end(new Uint16Array([0x6968]))));
|
||||
// @ts-expect-error Node accepts DataView input as raw bytes.
|
||||
console.log("data-view", utf8.end(new DataView(new Uint8Array([0, 0x68, 0x69, 0]).buffer, 1, 2)));
|
||||
const utf16 = new StringDecoder("utf16le");
|
||||
console.log("utf16", JSON.stringify(utf16.write(Buffer.from([0x3d, 0xd8]))), JSON.stringify(utf16.write(Buffer.from([0x00]))), JSON.stringify(utf16.end(Buffer.from([0xde, 0x61, 0]))));
|
||||
console.log("odd", JSON.stringify(utf16.write(Buffer.from([0x61]))), JSON.stringify(utf16.end(Buffer.from([0, 0x62]))), JSON.stringify(utf16.end(Buffer.from([0x63, 0]))));
|
||||
const base64 = new StringDecoder("base64");
|
||||
console.log("base64", JSON.stringify(base64.write(Buffer.from([1]))), JSON.stringify(base64.end(Buffer.from([2, 3, 4]))), JSON.stringify(base64.end(Buffer.from([5]))));
|
||||
const url = new StringDecoder("base64url");
|
||||
console.log("base64url", JSON.stringify(url.write(Buffer.from([251]))), JSON.stringify(url.end(Buffer.from([255, 254, 251]))));
|
||||
const hex = new StringDecoder("hex");
|
||||
// @ts-expect-error Node StringDecoder.write accepts string passthrough.
|
||||
console.log("hex", hex.write("unchanged"), hex.end(Buffer.from([0xab, 0xcd])), hex.end());
|
||||
let order = "";
|
||||
function receiver(): StringDecoder { order += "r"; return utf8; }
|
||||
function chunk(): Buffer { order += "c"; return Buffer.from([0x61]); }
|
||||
console.log("order", receiver().end(chunk()), order);
|
||||
|
||||
function absent(): undefined { order += "u"; return undefined; }
|
||||
console.log("undefined-order", receiver().end(absent()), order);
|
||||
@@ -0,0 +1,4 @@
|
||||
// @dynamic
|
||||
// @stderr
|
||||
import { report } from "node24-string-decoder-fixture";
|
||||
report();
|
||||
@@ -0,0 +1 @@
|
||||
{"type":"module"}
|
||||
+1
@@ -0,0 +1 @@
|
||||
export function report(): void;
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
import { StringDecoder } from "node:string_decoder";
|
||||
import { Buffer } from "node:buffer";
|
||||
|
||||
export function report() {
|
||||
const bom = Buffer.from([0xef, 0xbb, 0xbf, 0x61]);
|
||||
console.log('buffer-bom', JSON.stringify(bom.toString('utf8')), JSON.stringify(bom.toString('utf8', 0, 3)), JSON.stringify(bom.toString('utf8', 3)));
|
||||
const inputs = [
|
||||
[0xef, 0xbb, 0xbf, 0x61], [0xc2, 0xa2], [0xe2, 0x82, 0xac],
|
||||
[0xf0, 0x9f, 0x98, 0x80], [0xe0, 0x80, 0x61], [0xed, 0xa0, 0x80],
|
||||
[0xf4, 0x90, 0x80, 0x80], [0xc0, 0xaf], [0xff, 0x80, 0xe2, 0x82],
|
||||
[0x61, 0, 0x3d, 0xd8, 0, 0xde, 0x62, 0], [0x3d, 0xd8], [0x61],
|
||||
[0x3d, 0xd8, 0, 0xde, 0x3d, 0xd8, 0, 0xde],
|
||||
];
|
||||
for (const enc of ['utf8', 'utf16le', 'base64', 'base64url', 'latin1', 'ascii', 'hex']) {
|
||||
for (const input of inputs) {
|
||||
const bytes = Buffer.from(input);
|
||||
for (let a = 0; a <= bytes.length; a++) {
|
||||
for (let b = a; b <= bytes.length; b++) {
|
||||
const d = new StringDecoder(enc);
|
||||
console.log(enc, a, b, JSON.stringify([
|
||||
d.write(bytes.subarray(0, a)), d.write(bytes.subarray(a, b)),
|
||||
d.end(bytes.subarray(b)), d.end(), d.end(Buffer.from('ok')),
|
||||
]));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const d = new StringDecoder();
|
||||
console.log('strings', JSON.stringify([d.write(Buffer.from([0xe2])), d.write('middle'), d.end('last'), d.end()]));
|
||||
console.log('empty', JSON.stringify([d.write(Buffer.from([0xe0])), d.write(Buffer.alloc(0)), d.write(Buffer.from([0x80])), d.end(undefined)]));
|
||||
const backing = new Uint8Array([0, 0x68, 0x69, 0]);
|
||||
console.log('views', d.end(new DataView(backing.buffer, 1, 2)), d.end(new Uint16Array([0x6968])), d.end(backing.subarray(1, 3)));
|
||||
for (const encoding of [undefined, null, '', 'UTF-8', 'UCS-2', 'BINARY', 'nope', false, 0]) {
|
||||
try { console.log('encoding', String(encoding), new StringDecoder(encoding).encoding); }
|
||||
catch (e) { console.log('encoding-error', e.name, e.code, e.message); }
|
||||
}
|
||||
for (const input of [undefined, null, 1, {}, [], new ArrayBuffer(2)]) {
|
||||
try { d.write(input); console.log('unexpected'); }
|
||||
catch (e) { console.log('input-error', e.name, e.code, e.message); }
|
||||
}
|
||||
d.encoding = 'hex';
|
||||
console.log('encoding-property', d.encoding, d.end(Buffer.from('hi')));
|
||||
}
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"name":"node24-string-decoder-fixture","version":"1.0.0","type":"module","main":"index.js","types":"index.d.ts"}
|
||||
Reference in New Issue
Block a user