From fa98044a5cae785c92374ff0413eeb76c1257539 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 06:11:52 -0500 Subject: [PATCH 001/184] Run completion callbacks in their call's AsyncLocalStorage context, and add bind, snapshot and store options Node runs the callback of an operation that a callback-style API starts with the AsyncLocalStorage stores active when the call started it. The runtime invokes these callbacks from the event loop, where the context is empty, so getStore() answered undefined in fs, crypto, zlib, dns, child_process, net, http and dgram callbacks. In programs whose scheduling carries contexts (RuntimeFeatures asyncContext), the lowering now binds each completion-callback argument to the context active at the call (completion-callbacks.ts): typed closures through als.bindCallback, whose bound closure enters through a thunk the backend emits once per callback signature, and dyn functions through als.bindCallbackDyn. The fs callback family binds in the runtime, where scr_fs_cb_schedule queues the callback. Other programs compile as before. AsyncLocalStorage.bind(fn) binds fn to the active context, also when it is empty (a typed function keeps its type; in JavaScript files the bound function is named "bound" and forwards its receiver, as in Node). AsyncLocalStorage.snapshot() answers a function that runs its callback in the snapshot's context. new AsyncLocalStorage({ defaultValue, name }) validates its options as Node does; getStore() answers the default value where the store has no entry, exit() runs its callback with the store undefined, and the name property answers the name option or "". Event listeners on sockets, requests, responses and streams, stream callbacks, server.close(callback) and https and tls callbacks still run with no store; the limitations page lists them. Corpus 7110-7112. --- .../builtin-apis/asynclocalstorage.mdx | 4 +- .../generated/node-v24-internal.json | 43 +++++- internal/compatibility/static-support.json | 2 +- .../ambient/scriptc-node-fallback.d.ts | 13 +- .../compiler/src/backend/llvm/lib-process.ts | 52 ++++++- .../compiler/src/backend/llvm/lib-shared.ts | 4 + .../lowering/builtins/async-context.ts | 79 +++++++++- .../frontend/lowering/completion-callbacks.ts | 130 +++++++++++++++++ .../lowering/expressions/property-access.ts | 3 + .../src/frontend/lowering/lower-classes.ts | 23 +-- .../compiler/src/frontend/lowering/lowerer.ts | 2 + packages/compiler/src/frontend/type-mapper.ts | 23 +++ packages/compiler/src/ir/builtin-effects.ts | 2 + .../compiler/src/ir/builtin-signatures.ts | 14 ++ packages/compiler/src/ir/runtime-features.ts | 4 + packages/compiler/src/ir/validate.ts | 7 + .../test/completion-callbacks.test.ts | 60 ++++++++ .../7110-als-completion-callbacks.ts.json | 6 + .../7111-als-bind-snapshot-options.ts.json | 6 + .../7112-als-bind-snapshot-dynamic.mjs.json | 6 + packages/runtime/src/scr_async.c | 41 ++++++ packages/runtime/src/scr_async.h | 15 ++ packages/runtime/src/scr_async_dyn.c | 138 +++++++++++++++++- tests/corpus/7110-als-completion-callbacks.ts | 135 +++++++++++++++++ .../corpus/7111-als-bind-snapshot-options.ts | 59 ++++++++ .../corpus/7112-als-bind-snapshot-dynamic.mjs | 51 +++++++ .../7110-als-completion-callbacks.ts.json | 1 + .../7111-als-bind-snapshot-options.ts.json | 1 + .../7112-als-bind-snapshot-dynamic.mjs.json | 1 + 29 files changed, 898 insertions(+), 27 deletions(-) create mode 100644 packages/compiler/src/frontend/lowering/completion-callbacks.ts create mode 100644 packages/compiler/test/completion-callbacks.test.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7110-als-completion-callbacks.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7111-als-bind-snapshot-options.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7112-als-bind-snapshot-dynamic.mjs.json create mode 100644 tests/corpus/7110-als-completion-callbacks.ts create mode 100644 tests/corpus/7111-als-bind-snapshot-options.ts create mode 100644 tests/corpus/7112-als-bind-snapshot-dynamic.mjs create mode 100644 tests/harness/corpus-costs/7110-als-completion-callbacks.ts.json create mode 100644 tests/harness/corpus-costs/7111-als-bind-snapshot-options.ts.json create mode 100644 tests/harness/corpus-costs/7112-als-bind-snapshot-dynamic.mjs.json diff --git a/docs/content/limitations/builtin-apis/asynclocalstorage.mdx b/docs/content/limitations/builtin-apis/asynclocalstorage.mdx index 341994faa..99f89d65c 100644 --- a/docs/content/limitations/builtin-apis/asynclocalstorage.mdx +++ b/docs/content/limitations/builtin-apis/asynclocalstorage.mdx @@ -1,2 +1,4 @@ - **AsyncLocalStorage:** timer, interval, `setImmediate`, `process.nextTick` and `queueMicrotask` callbacks run with the stores that were active when they were scheduled, as in Node. - Completion callbacks of callback-style I/O APIs, such as `fs.readFile` and `crypto.randomBytes`, run with no store, where Node runs them with the stores that were active when the operation started. + The completion callbacks of `fs`, `crypto` and `zlib` callback APIs, `dns.lookup`, `child_process.execFile`, `server.listen`, `net.connect`, `http.request`, `http.get` and `dgram` run with the stores that were active at the call, as in Node. + Event listeners on sockets, requests, responses and streams, stream callbacks such as `write(chunk, callback)`, `server.close(callback)`, and the callbacks of `https` and `tls` calls run with no store, where Node runs them in the context of the resource or of the code that triggers them. + `AsyncResource` has no lowering yet. diff --git a/internal/compatibility/generated/node-v24-internal.json b/internal/compatibility/generated/node-v24-internal.json index d40bb7bfc..34fe2a843 100644 --- a/internal/compatibility/generated/node-v24-internal.json +++ b/internal/compatibility/generated/node-v24-internal.json @@ -1387,7 +1387,10 @@ "evidence": "derived:descendants", "tests": [ "tests/corpus/2213-async-local-storage.cjs", - "tests/corpus/5815-als-callback-context.ts" + "tests/corpus/5815-als-callback-context.ts", + "tests/corpus/7110-als-completion-callbacks.ts", + "tests/corpus/7111-als-bind-snapshot-options.ts", + "tests/corpus/7112-als-bind-snapshot-dynamic.mjs" ] }, "anchorSource": "chapter" @@ -1437,7 +1440,10 @@ "evidence": "compiler-dedicated:AsyncLocalStorage", "tests": [ "tests/corpus/2213-async-local-storage.cjs", - "tests/corpus/5815-als-callback-context.ts" + "tests/corpus/5815-als-callback-context.ts", + "tests/corpus/7110-als-completion-callbacks.ts", + "tests/corpus/7111-als-bind-snapshot-options.ts", + "tests/corpus/7112-als-bind-snapshot-dynamic.mjs" ] }, "anchorSource": "exact" @@ -1510,7 +1516,10 @@ "evidence": "compiler-dedicated:asyncLocalStorage.disable", "tests": [ "tests/corpus/2213-async-local-storage.cjs", - "tests/corpus/5815-als-callback-context.ts" + "tests/corpus/5815-als-callback-context.ts", + "tests/corpus/7110-als-completion-callbacks.ts", + "tests/corpus/7111-als-bind-snapshot-options.ts", + "tests/corpus/7112-als-bind-snapshot-dynamic.mjs" ] }, "anchorSource": "exact" @@ -1537,7 +1546,10 @@ "evidence": "compiler-dedicated:asyncLocalStorage.getStore", "tests": [ "tests/corpus/2213-async-local-storage.cjs", - "tests/corpus/5815-als-callback-context.ts" + "tests/corpus/5815-als-callback-context.ts", + "tests/corpus/7110-als-completion-callbacks.ts", + "tests/corpus/7111-als-bind-snapshot-options.ts", + "tests/corpus/7112-als-bind-snapshot-dynamic.mjs" ] }, "anchorSource": "exact" @@ -1564,7 +1576,10 @@ "evidence": "compiler-dedicated:asyncLocalStorage.enterWith", "tests": [ "tests/corpus/2213-async-local-storage.cjs", - "tests/corpus/5815-als-callback-context.ts" + "tests/corpus/5815-als-callback-context.ts", + "tests/corpus/7110-als-completion-callbacks.ts", + "tests/corpus/7111-als-bind-snapshot-options.ts", + "tests/corpus/7112-als-bind-snapshot-dynamic.mjs" ] }, "anchorSource": "exact" @@ -1591,7 +1606,10 @@ "evidence": "compiler-dedicated:asyncLocalStorage.run", "tests": [ "tests/corpus/2213-async-local-storage.cjs", - "tests/corpus/5815-als-callback-context.ts" + "tests/corpus/5815-als-callback-context.ts", + "tests/corpus/7110-als-completion-callbacks.ts", + "tests/corpus/7111-als-bind-snapshot-options.ts", + "tests/corpus/7112-als-bind-snapshot-dynamic.mjs" ] }, "anchorSource": "exact" @@ -1618,7 +1636,10 @@ "evidence": "compiler-dedicated:asyncLocalStorage.exit", "tests": [ "tests/corpus/2213-async-local-storage.cjs", - "tests/corpus/5815-als-callback-context.ts" + "tests/corpus/5815-als-callback-context.ts", + "tests/corpus/7110-als-completion-callbacks.ts", + "tests/corpus/7111-als-bind-snapshot-options.ts", + "tests/corpus/7112-als-bind-snapshot-dynamic.mjs" ] }, "anchorSource": "exact" @@ -1922,6 +1943,9 @@ "tests": [ "tests/corpus/2213-async-local-storage.cjs", "tests/corpus/5815-als-callback-context.ts", + "tests/corpus/7110-als-completion-callbacks.ts", + "tests/corpus/7111-als-bind-snapshot-options.ts", + "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", "tests/harness/node-test.test.ts" ] }, @@ -2484,7 +2508,10 @@ "evidence": "compiler-dedicated:AsyncLocalStorage", "tests": [ "tests/corpus/2213-async-local-storage.cjs", - "tests/corpus/5815-als-callback-context.ts" + "tests/corpus/5815-als-callback-context.ts", + "tests/corpus/7110-als-completion-callbacks.ts", + "tests/corpus/7111-als-bind-snapshot-options.ts", + "tests/corpus/7112-als-bind-snapshot-dynamic.mjs" ] }, "anchorSource": "exact" diff --git a/internal/compatibility/static-support.json b/internal/compatibility/static-support.json index 6fc2c563c..efc60f263 100644 --- a/internal/compatibility/static-support.json +++ b/internal/compatibility/static-support.json @@ -429,7 +429,7 @@ { "symbols": ["crypto.hkdf", "crypto.hkdfSync", "crypto.scrypt", "crypto.scryptSync"], "status": "partial", "evidence": ["tests/corpus/node24-crypto-kdf.ts", "tests/corpus/node24-crypto-kdf-errors.ts", "tests/corpus/node24-crypto-kdf-options.ts", "tests/corpus/5438-crypto-kdf-digests.ts"] }, { "symbols": ["EventEmitter", "emitter.on", "emitter.addListener", "emitter.once", "emitter.prependListener", "emitter.prependOnceListener", "emitter.off", "emitter.removeListener", "emitter.removeAllListeners", "emitter.emit", "emitter.listenerCount", "emitter.listeners", "emitter.rawListeners", "emitter.eventNames", "emitter.setMaxListeners", "emitter.getMaxListeners", "events.once"], "status": "partial", "evidence": ["tests/corpus/1677-emitter-listeners.ts", "tests/corpus/2620-ee-override-once-order.ts"] }, { "symbols": ["events.getEventListeners", "events.getMaxListeners", "events.listenerCount", "events.setMaxListeners", "events.defaultMaxListeners"], "status": "partial", "evidence": ["tests/corpus/5885-events-max-listener-helpers.ts", "tests/corpus/5846-events-esm-default-max.ts", "tests/corpus/5847-events-cjs-default-max.cjs"] }, - { "symbols": ["AsyncLocalStorage", "asyncLocalStorage.run", "asyncLocalStorage.exit", "asyncLocalStorage.getStore", "asyncLocalStorage.enterWith", "asyncLocalStorage.disable"], "status": "partial", "evidence": ["tests/corpus/2213-async-local-storage.cjs", "tests/corpus/5815-als-callback-context.ts"] }, + { "symbols": ["AsyncLocalStorage", "asyncLocalStorage.run", "asyncLocalStorage.exit", "asyncLocalStorage.getStore", "asyncLocalStorage.enterWith", "asyncLocalStorage.disable", "asyncLocalStorage.name", "AsyncLocalStorage.bind", "AsyncLocalStorage.snapshot"], "status": "partial", "evidence": ["tests/corpus/2213-async-local-storage.cjs", "tests/corpus/5815-als-callback-context.ts", "tests/corpus/7110-als-completion-callbacks.ts", "tests/corpus/7111-als-bind-snapshot-options.ts", "tests/corpus/7112-als-bind-snapshot-dynamic.mjs"] }, { "symbols": ["StringDecoder", "stringDecoder.write", "stringDecoder.end"], "status": "partial", "evidence": ["tests/corpus/1662-string-decoder-encodings.ts", "tests/corpus/node24-string-decoder.ts", "tests/corpus/5882-string-decoder-runtime-encoding.ts"] }, { "symbols": ["setImmediate", "setInterval", "setTimeout", "clearImmediate", "clearInterval", "clearTimeout", "timersPromises.setTimeout", "timersPromises.setImmediate", "timersPromises.setInterval", "immediate.hasRef", "immediate.ref", "immediate.unref", "timeout.hasRef", "timeout.ref", "timeout.unref", "timeout.refresh"], "status": "partial", "evidence": ["tests/corpus/1804-timers-module.ts", "tests/corpus/1805-timer-callback-args.ts", "tests/corpus/2093-timers-promises.ts", "tests/corpus/2852-timers-promises-interval.ts", "tests/corpus/5830-timers-promises-signal.ts"] }, { "symbols": ["timersPromises.scheduler.wait", "timersPromises.scheduler.yield"], "status": "partial", "evidence": ["tests/corpus/5839-timers-promises-scheduler.ts", "tests/corpus/5889-timers-scheduler-signal.ts"] }, diff --git a/packages/compiler/ambient/scriptc-node-fallback.d.ts b/packages/compiler/ambient/scriptc-node-fallback.d.ts index b3d2954a8..ac30fde4a 100644 --- a/packages/compiler/ambient/scriptc-node-fallback.d.ts +++ b/packages/compiler/ambient/scriptc-node-fallback.d.ts @@ -6887,9 +6887,18 @@ declare module "node:tty" { declare module "async_hooks" { /* AsyncLocalStorage: fiber-carried context snapshots — run/getStore/ * exit/enterWith/disable lower onto the runtime's active-slot machinery - * (als.* libCalls); values cross as dyn values. */ + * (als.* libCalls); values cross as dyn values. bind and snapshot bind + * functions to the active context; the options carry the store's + * default value and name. */ + export interface AsyncLocalStorageOptions { + defaultValue?: any; + name?: string | undefined; + } export class AsyncLocalStorage { - constructor(); + constructor(options?: AsyncLocalStorageOptions); + static bind any>(fn: Func): Func; + static snapshot(): (fn: (...args: TArgs) => R, ...args: TArgs) => R; + readonly name: string; run(store: T, callback: (...args: any[]) => R, ...args: any[]): R; exit(callback: (...args: any[]) => R, ...args: any[]): R; getStore(): T | undefined; diff --git a/packages/compiler/src/backend/llvm/lib-process.ts b/packages/compiler/src/backend/llvm/lib-process.ts index 3e8a9bff5..1a3973b07 100644 --- a/packages/compiler/src/backend/llvm/lib-process.ts +++ b/packages/compiler/src/backend/llvm/lib-process.ts @@ -1,10 +1,11 @@ /* Focused LLVM library-call emission extracted from emitter.ts. */ import { InternalCompilerError } from "../../errors.js"; import { undefinedArmTag } from "../../ir/analysis.js"; -import { F64, RUNTIME_ERROR_CLASSES, STRING } from "../../ir/ir.js"; +import { F64, type IrType, RUNTIME_ERROR_CLASSES, STRING, typeKey } from "../../ir/ir.js"; import type { LlvmEmitterContext, LibCallExpr, LlValue } from "./expr-context.js"; import { f64Lit } from "./common.js"; import { mockTimerSym } from "./lib-shared.js"; +import { FN_ATTRS } from "./shapes.js"; export function emitChildProcessLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue { const B = host.B; @@ -460,9 +461,58 @@ export function emitAsyncContextLibCall(host: LlvmEmitterContext, e: LibCallExpr for (const a of e.args) host.emitExpr(a); return { name: "", type: e.type }; } + if (e.fn === "als.bindCallback") { + // The closure moves into the bound closure, whose entry is the + // per-signature thunk; with nothing to bind the runtime answers the + // closure itself. + if (e.type.kind !== "func") + throw new InternalCompilerError("llvm emitter bug: als.bindCallback on non-func"); + const cb = host.emitExpr(e.args[0]!); + const always = host.emitExpr(e.args[1]!); + host.toHeap(cb); + const thunk = boundCallbackThunk(host, e.type); + host.declare("declare ptr @scr_als_bind_closure(ptr, ptr, i1 zeroext)"); + const t = B.tmp(); + B.line( + `${t} = call ptr @scr_als_bind_closure(ptr ${cb.name}, ptr @${thunk}, i1 ${always.name})`, + ); + return host.own({ name: t, type: e.type }); + } return host.emitGenericLibCall(e); } +/** The entry of a callback bound to an AsyncLocalStorage context, one per + * callback signature: enters the captured context, calls the target with + * the same arguments (the callValue ABI) and restores the caller's + * context, also when the target leaves an exception pending. */ +function boundCallbackThunk(host: LlvmEmitterContext, cbT: IrType & { kind: "func" }): string { + const key = `alsbound:${typeKey(cbT)}`; + let sym = host.resolveThunks.get(key); + if (sym) return sym; + sym = `sc_alsbound_${host.resolveThunks.size}`; + host.resolveThunks.set(key, sym); + host.declare("declare ptr @scr_als_bound_target(ptr)"); + host.declare("declare ptr @scr_als_bound_enter(ptr)"); + host.declare("declare void @scr_als_bound_leave(ptr, ptr)"); + const ret = host.llType(cbT.ret); + const params = cbT.params.map((p, i) => `${host.llType(p)} %a${i}`); + const call = `call ${ret} %fn(${["ptr %cb", ...params].join(", ")})`; + host.resolveThunkDefs.push( + `define internal ${ret} @${sym}(${["ptr %self", ...params].join(", ")}) ${FN_ATTRS} { ; AsyncLocalStorage-bound callback ${typeKey(cbT)}`, + `entry:`, + ` %cb = call ptr @scr_als_bound_target(ptr %self)`, + ` %prev = call ptr @scr_als_bound_enter(ptr %self)`, + ` %fnp = getelementptr inbounds %ScrClosure, ptr %cb, i64 0, i32 1`, + ` %fn = load ptr, ptr %fnp`, + ret === "void" ? ` ${call}` : ` %r = ${call}`, + ` call void @scr_als_bound_leave(ptr %prev, ptr %cb)`, + ret === "void" ? ` ret void` : ` ret ${ret} %r`, + `}`, + ``, + ); + return sym; +} + export function emitProcessLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue { const B = host.B; if (e.fn === "process.forkTarget") { diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 53c715108..5ed131567 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1438,6 +1438,10 @@ export const LIB_FN_SYMS: Record = { "als.exitRun": "scr_als_exit_run", "als.enterWith": "scr_als_enter_with", "als.disable": "scr_als_disable", + "als.newOptions": "scr_als_new_options", + "als.name": "scr_als_name", + "als.bindCallbackDyn": "scr_als_bind_dyn", + "als.snapshot": "scr_als_snapshot", "dc.chanBindStore": "scr_dc_chan_bind_store", "dc.chanUnbindStore": "scr_dc_chan_unbind_store", "dc.chanRunStores": "scr_dc_chan_run_stores", diff --git a/packages/compiler/src/frontend/lowering/builtins/async-context.ts b/packages/compiler/src/frontend/lowering/builtins/async-context.ts index 18bd93a86..a130df59b 100644 --- a/packages/compiler/src/frontend/lowering/builtins/async-context.ts +++ b/packages/compiler/src/frontend/lowering/builtins/async-context.ts @@ -2,7 +2,7 @@ import { isBuiltinClassInstance } from "./receiver-types.js"; import { dynUndefinedExpr } from "../../../ir/build.js"; import * as ts from "../../ts7/adapter.js"; import { type Lowerer } from "../lowerer.js"; -import { locOf } from "../../program.js"; +import { isJsSourceFile, locOf } from "../../program.js"; import { BOOL, DYN, @@ -184,16 +184,89 @@ export function lowerDiagnosticsChannelMethodCall( ); } +/** Whether `node` names the AsyncLocalStorage class itself. */ +function isAsyncLocalStorageClass(lowerer: Lowerer, node: ts.Expression): boolean { + if (!ts.isIdentifier(node)) return false; + const symbol = lowerer.resolveValueSymbol(node); + return ( + symbol !== null && + lowerer.isStdlibSymbol(symbol) && + ts.symbolName(symbol) === "AsyncLocalStorage" + ); +} + +/** The static AsyncLocalStorage members: bind(fn) answers fn bound to the + * active context (its own type for a typed function; a dyn function in + * JavaScript files, where the bound function's name and receiver are + * observable, and for dyn values); snapshot() answers a dyn function + * (cb, ...args) that runs cb in the context active at the snapshot. */ +function lowerAsyncLocalStorageStaticCall( + lowerer: Lowerer, + call: ts.CallExpression, + access: ts.PropertyAccessExpression, +): IrExpr { + const name = access.name.text; + const loc = locOf(call); + const always: IrExpr = { kind: "boolLit", value: true, type: BOOL, loc }; + if (name === "bind" && call.arguments.nodes.length === 1) { + const arg = call.arguments.nodes[0]!; + if (!isJsSourceFile(ts.getSourceFileOfNode(call))) { + const fn = lowerer.lowerExpr(arg); + if (fn.type.kind === "func") + return { kind: "libCall", fn: "als.bindCallback", args: [fn, always], type: fn.type, loc }; + } + return asyncLocalStorageResult(lowerer, call, { + kind: "libCall", + fn: "als.bindCallbackDyn", + args: [lowerDiagnosticsMessage(lowerer, arg), always], + type: DYN, + loc, + }); + } + if (name === "snapshot" && call.arguments.nodes.length === 0) + return { kind: "libCall", fn: "als.snapshot", args: [], type: DYN, loc }; + lowerer.noLowering( + `AsyncLocalStorage.${name}`, + call, + "AsyncLocalStorage.bind(fn) and AsyncLocalStorage.snapshot() are the supported static members", + lowerer.checker.getSymbolAtLocation(access.name), + ); +} + +/** new AsyncLocalStorage(options): the options object as a dyn value + * (plain property literals build it member by member). */ +export function lowerAsyncLocalStorageOptions(lowerer: Lowerer, node: ts.Expression): IrExpr { + return lowerTracingHandlers(lowerer, node); +} + +/** `.name` on AsyncLocalStorage receivers: the constructor's name option + * (als.name), converted to the property's type. Null otherwise. */ +export function lowerAsyncLocalStorageProperty( + lowerer: Lowerer, + access: ts.PropertyAccessExpression, +): IrExpr | null { + if (access.questionDotToken || access.name.text !== "name") return null; + if (!isBuiltinClassInstance(lowerer, access.expression, "AsyncLocalStorage")) return null; + if (!lowerer.isStdlibMember(access)) return null; + const loc = locOf(access); + const receiver = lowerer.lowerExprExpecting(access.expression, F64); + const value: IrExpr = { kind: "libCall", fn: "als.name", args: [receiver], type: DYN, loc }; + const expected = lowerer.mapTypeOf(lowerer.typeOf(access)); + return !expected || expected.kind === "dyn" ? value : lowerer.coerceInto(access, value, expected); +} + /** Method calls on AsyncLocalStorage receivers: run(store, fn, ...args), * exit(fn, ...args), getStore(), enterWith(store), disable() — over the - * f64 store handle (als.* libCalls; values cross as dyn values). Null - * for non-ALS receivers. */ + * f64 store handle (als.* libCalls; values cross as dyn values) — and the + * static bind/snapshot members. Null for other receivers. */ export function lowerAsyncLocalStorageMethodCall( lowerer: Lowerer, call: ts.CallExpression, access: ts.PropertyAccessExpression, ): IrExpr | null { if (call.questionDotToken || access.questionDotToken) return null; + if (isAsyncLocalStorageClass(lowerer, access.expression)) + return lowerAsyncLocalStorageStaticCall(lowerer, call, access); if (!isBuiltinClassInstance(lowerer, access.expression, "AsyncLocalStorage")) return null; if (!lowerer.isStdlibMember(access)) return null; const name = access.name.text; diff --git a/packages/compiler/src/frontend/lowering/completion-callbacks.ts b/packages/compiler/src/frontend/lowering/completion-callbacks.ts new file mode 100644 index 000000000..402741da6 --- /dev/null +++ b/packages/compiler/src/frontend/lowering/completion-callbacks.ts @@ -0,0 +1,130 @@ +/* Completion callbacks in programs that use AsyncLocalStorage. Node runs + * the callback of an operation a callback-style API starts (fs.readFile, + * crypto.randomBytes, zlib.gzip, dns.lookup, server.listen, net.connect, + * http.get, ...) with the stores active when the call started it. The runtime invokes these callbacks from + * the event loop, where the context is empty, so in programs whose + * scheduling carries contexts (RuntimeFeatures asyncContext) each such + * argument is bound to the active context at the call: als.bindCallback + * for typed closures, als.bindCallbackDyn for dyn functions. Other + * programs are left unchanged. */ +import { BOOL, type IrExpr, type IrModule, type IrStmt } from "../../ir/ir.js"; +import type { IrLibFn } from "../../ir/builtin-signatures.js"; +import { mapExprChildren, mapStmtChildren } from "../../ir/traverse.js"; +import { moduleRuntimeFeatures } from "../../ir/runtime-features.js"; + +/** The argument that is the completion callback: its index; "last" for + * calls whose callback is the final argument when one is passed; or + * "arguments" for the fs callback calls, whose argument is the call's + * dyn argument vector, where the runtime finds the callback. */ +type CallbackPosition = number | "last" | "arguments"; + +const COMPLETION_CALLBACKS: ReadonlyMap = new Map< + IrLibFn, + CallbackPosition +>([ + ["cp.execFile", 2], + ["crypto.generateKeyCb", 2], + ["crypto.hkdfCb", 5], + ["crypto.pbkdf2Cb", 5], + ["crypto.randomBytesCb", 1], + ["crypto.randomFillCb", 4], + ["crypto.randomFillCbDyn", 4], + ["crypto.randomIntCb", 3], + ["crypto.scryptCb", 4], + ["dgram.bindCb", 3], + ["dgram.closeCb", 1], + ["dgram.connectCb", 3], + ["dns.lookupDyn", 2], + ["fs.callbackCall", "arguments"], + ["fs.callbackGlob", "arguments"], + ["fs.renameCb", 2], + ["http.requestAgentCb", "last"], + ["http.requestCb", "last"], + ["http.requestConnCb", "last"], + ["http.requestDynCb", "last"], + ["http.requestUrlCb", "last"], + ["net.connectCb", 2], + ["net.listenCb", 2], + ["net.listenOptsCb", 4], + ["net.listenOptsReusePortCb", 5], + ["net.listenPathCb", 2], + ["process.stderrWriteBytesCb", 2], + ["process.stdoutWriteBytesCb", 2], + ["zlib.cbOpts", 3], + ["zlib.cbOptsDyn", 3], + ["zlib.deflateCb", 1], + ["zlib.deflateRawCb", 1], + ["zlib.gunzipCb", 1], + ["zlib.gzipCb", 1], + ["zlib.inflateCb", 1], + ["zlib.inflateRawCb", 1], + ["zlib.unzipCb", 1], +]); + +/** `cb` bound to the context active where it is evaluated. An empty + * context leaves it unbound (`always` false): the loop already runs + * callbacks with an empty context. */ +function bindToActiveContext(cb: IrExpr): IrExpr { + const always: IrExpr = { kind: "boolLit", value: false, type: BOOL, loc: cb.loc }; + if (cb.type.kind === "func") + return { + kind: "libCall", + fn: "als.bindCallback", + args: [cb, always], + type: cb.type, + loc: cb.loc, + }; + if (cb.type.kind === "dyn") + return { + kind: "libCall", + fn: "als.bindCallbackDyn", + args: [cb, always], + type: cb.type, + loc: cb.loc, + }; + return cb; +} + +/** An element of a dyn argument vector that can be the callback: a boxed + * closure binds as a typed closure; any other dyn value that is not a + * boxed primitive binds as a dyn function (non-functions pass through). */ +function bindArgumentElement(e: IrExpr): IrExpr { + if (e.kind === "dynFrom") { + return e.value.type.kind === "func" ? { ...e, value: bindToActiveContext(e.value) } : e; + } + return e.type.kind === "dyn" ? bindToActiveContext(e) : e; +} + +function bindCall(e: IrExpr): IrExpr { + if (e.kind !== "libCall") return e; + const position = COMPLETION_CALLBACKS.get(e.fn); + if (position === undefined) return e; + if (position === "arguments") { + const index = e.fn === "fs.callbackCall" ? 1 : 0; + const vector = e.args[index]; + if (vector === undefined || vector.kind !== "dynArrLit") return e; + const args = e.args.slice(); + args[index] = { + kind: "dynArrLit", + elems: vector.elems.map(bindArgumentElement), + type: vector.type, + loc: vector.loc, + }; + return { ...e, args }; + } + const index = position === "last" ? e.args.length - 1 : position; + const cb = e.args[index]; + if (cb === undefined || (position === "last" && cb.type.kind !== "func")) return e; + const args = e.args.slice(); + args[index] = bindToActiveContext(cb); + return { ...e, args }; +} + +/** Binds the completion callbacks of `mod` when its scheduling carries + * AsyncLocalStorage contexts. */ +export function bindCompletionCallbacks(mod: IrModule): void { + if (!moduleRuntimeFeatures(mod).asyncContext) return; + const rewriteExpr = (e: IrExpr): IrExpr => bindCall(mapExprChildren(e, rewriteExpr, rewriteStmt)); + const rewriteStmt = (s: IrStmt): IrStmt => mapStmtChildren(s, rewriteExpr, rewriteStmt); + for (const fn of mod.functions) fn.body = fn.body.map(rewriteStmt); +} diff --git a/packages/compiler/src/frontend/lowering/expressions/property-access.ts b/packages/compiler/src/frontend/lowering/expressions/property-access.ts index 7f3910c15..0777e8fb9 100644 --- a/packages/compiler/src/frontend/lowering/expressions/property-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/property-access.ts @@ -134,6 +134,7 @@ import { import { expandoMemberRead } from "../lower-expando.js"; import { lowerProtocolPropertyRead } from "../class-protocol-dispatch.js"; import { lowerReadlineProperty } from "../builtins/readline.js"; +import { lowerAsyncLocalStorageProperty } from "../builtins/async-context.js"; import { lowerEventStaticProperty } from "../builtins/numbers.js"; import { lowerStreamProperty, lowerStreamStateProperty, streamSidesOf } from "../lower-stream.js"; import { lowerValueStreamProperty } from "../builtins/hash-stream.js"; @@ -814,6 +815,8 @@ export function lowerPropertyAccessExpr( // diagnostics_channel Channel receivers — name/hasSubscribers // over the f64 channel handle. lowerer.lowerDiagnosticsChannelProperty(expr) ?? + // AsyncLocalStorage receivers — the name option. + lowerAsyncLocalStorageProperty(lowerer, expr) ?? // readline Interface receivers — the history and option members // over the f64 interface handle. lowerReadlineProperty(lowerer, expr) ?? diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index d8ccc15b7..14371f46b 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -1,3 +1,4 @@ +import { lowerAsyncLocalStorageOptions } from "./builtins/async-context.js"; import { DATE_TIME_FIELD, dateRootOf, @@ -10471,21 +10472,23 @@ export function lowerNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr { lowerer.badType(expr, tsType); } // `new AsyncLocalStorage()` (node:async_hooks): a fresh store id — - // an f64 handle (type-mapper.ts), the Channel story. Construction options - // ({ defaultValue, name }) have no lowering yet. + // an f64 handle (type-mapper.ts), the Channel story. The options form + // ({ defaultValue, name }) passes the options object as a dyn value. if ( symbol !== null && ts.symbolName(symbol) === "AsyncLocalStorage" && lowerer.isStdlibSymbol(symbol) ) { - if ((expr.arguments?.nodes.length ?? 0) > 0) { - lowerer.noLowering( - "new AsyncLocalStorage(options)", - expr, - "the zero-argument constructor is the supported form (defaultValue/name options have no lowering yet)", - ); - } - return { kind: "libCall", fn: "als.new", args: [], type: F64, loc }; + const options = expr.arguments?.nodes[0]; + if (options === undefined) + return { kind: "libCall", fn: "als.new", args: [], type: F64, loc }; + return { + kind: "libCall", + fn: "als.newOptions", + args: [lowerAsyncLocalStorageOptions(lowerer, options)], + type: F64, + loc, + }; } // `new Promise((resolve) => ...)`: the ambient Promise constructor. if (symbol !== null && ts.symbolName(symbol) === "Promise" && lowerer.isStdlibSymbol(symbol)) { diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index edf0a694b..fd2f44d73 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -29,6 +29,7 @@ import { refuseUncheckedCycleReads, type CycleInitFlag, } from "./cycle-initialization.js"; +import { bindCompletionCallbacks } from "./completion-callbacks.js"; import { isJsonRequireDecl, type JsonRequireModule } from "./json-require.js"; import { checkRequireBindingAccesses, @@ -7678,6 +7679,7 @@ export class Lowerer { }; if (module) sanitizeUnregisteredClassTypes(module, (name) => this.classes.has(name)); if (module) checkCycleBindingAccesses(this, module); + if (module) bindCompletionCallbacks(module); // Execution tiers from the final bodies (ir/tiers.ts). Assigned with or // without a module: coverage reports the dynamic tier of programs that // still have blockers. diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index e6c5c4c93..535c2b584 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -4674,6 +4674,9 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { if (hybrid) return hybrid; } const callSigs = checker.getCallSignatures(widened); + // AsyncLocalStorage.snapshot()'s generic runner is a dyn function value + // (als.snapshot), called through the dyn call path. + if (callSigs.length === 1 && isAsyncLocalStorageSnapshot(callSigs[0]!, ctx)) return DYN; if (callSigs.length === 1) return mapCallSignature(callSigs[0]!, ctx); // Records: object types whose members are all data properties (shorthand // methods in type position count — they're func-typed fields) with @@ -5004,6 +5007,26 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { /** One non-overloaded call signature's func type (mapTypeInner's * single-signature rule; erasedOverloadMemberType maps each overload of a * member through it). */ +/** Whether `sig` is the function type AsyncLocalStorage.snapshot() returns: + * declared as that static method's return type in the async_hooks module + * of the standard library declarations. */ +function isAsyncLocalStorageSnapshot(sig: ts.Signature, ctx: TypeMapperCtx): boolean { + const decl = sig.declaration; + if (decl === undefined || !ts.isFunctionTypeNode(decl)) return false; + const method = decl.parent; + return ( + method !== undefined && + ts.isMethodDeclaration(method) && + ts.isIdentifier(method.name) && + method.name.text === "snapshot" && + method.parent !== undefined && + ts.isClassDeclaration(method.parent) && + method.parent.name?.text === "AsyncLocalStorage" && + ctx.isStdlibFile(ts.getSourceFileOfNode(decl)) && + isDeclaredInAmbientModule(decl, "async_hooks") + ); +} + function mapCallSignature(sig: ts.Signature, ctx: TypeMapperCtx): IrType | null { const { checker } = ctx; if (sig.getTypeParameters().length && ctx.erasingSignature !== sig) return null; diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index c202b8538..34a0a2b14 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -194,6 +194,8 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "async.awaitDyn", "als.run", "als.exitRun", + "als.newOptions", + "als.bindCallbackDyn", "dc.chanBindStore", "dc.chanRunStores", "http.resWriteHeadDyn", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index f37e8809f..f426cb6c7 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -4592,6 +4592,20 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "als.exitRun": { argTypes: [F64, DYN, DYN], result: DYN }, "als.enterWith": { argTypes: [F64, DYN], result: VOID }, "als.disable": { argTypes: [F64], result: VOID }, + /** new AsyncLocalStorage(options) (Node's object check throws; MAY + * THROW) and the store's name option (dyn; "" without one). */ + "als.newOptions": { argTypes: [DYN], result: F64 }, + "als.name": { argTypes: [F64], result: DYN }, + /** A function bound to the active context: the completion callbacks of + * callback-style APIs (always false: an empty context leaves the + * callback unbound) and AsyncLocalStorage.bind (always true). The typed + * form answers its closure argument's type; the dyn form throws Node's + * ERR_INVALID_ARG_TYPE for a non-function when always (MAY THROW). */ + "als.bindCallback": { argTypes: [null, BOOL], result: VOID }, + "als.bindCallbackDyn": { argTypes: [DYN, BOOL], result: DYN }, + /** AsyncLocalStorage.snapshot(): a dyn function (cb, ...args) that runs + * cb in the context active at the snapshot. */ + "als.snapshot": { argTypes: [], result: DYN }, /** Channel.bindStore/unbindStore/runStores (scr_dc.c): the * AsyncLocalStorage integration — set-semantics bindings per store, * runStores entering every bound store with transform(data) around the diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index 5c30b0f69..05ebe2496 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -115,12 +115,16 @@ const DYN_ASYNC_LIB_FNS: ReadonlySet = new Set([ "perf.measure", // Blob and File (scr_blob.c, linked with this unit, which boxes their // reads as dyn promises). + "als.bindCallbackDyn", "als.disable", "als.enterWith", "als.exitRun", "als.get", + "als.name", "als.new", + "als.newOptions", "als.run", + "als.snapshot", "blob.arrayBuffer", "blob.bytes", "blob.fileNew", diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index bb36c7007..915f7a4af 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -5310,6 +5310,13 @@ function validateFunction( } function checkLibSpecialCall(e: IrExpr & { kind: "libCall" }): boolean { + if (e.fn === "als.bindCallback") { + // The bound function, at its closure argument's own type. + const fn = e.args[0]?.type; + if (fn?.kind !== "func" || !typeEquals(e.type, fn)) + err(`libCall ${e.fn} must return its function argument's type`, e.loc); + return true; + } if ( e.fn === "crypto.randomFillView" || e.fn === "crypto.randomFillViewRest" || diff --git a/packages/compiler/test/completion-callbacks.test.ts b/packages/compiler/test/completion-callbacks.test.ts new file mode 100644 index 000000000..817472c19 --- /dev/null +++ b/packages/compiler/test/completion-callbacks.test.ts @@ -0,0 +1,60 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { compile } from "../src/index.js"; + +const directories: string[] = []; +afterEach(async () => { + await Promise.all(directories.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); +}); + +async function emit(source: string): Promise { + const outDir = await mkdtemp(join(tmpdir(), "scriptc-completion-callbacks-")); + directories.push(outDir); + const entry = join(outDir, "main.ts"); + await writeFile(entry, source); + const outPath = join(outDir, "program.ll"); + const result = await compile(entry, { entry, outDir, outPath, outputKind: "llvm" }); + expect(result.ok, JSON.stringify(result)).toBe(true); + return readFile(outPath, "utf8"); +} + +const randomBytes = ` +import { randomBytes } from "node:crypto"; +randomBytes(4, (error, buffer) => console.log(error === null, buffer.length)); +`; + +test("programs without AsyncLocalStorage pass completion callbacks unchanged", async () => { + const llvm = await emit(randomBytes); + expect(llvm).not.toContain("@scr_als_bind_closure"); + expect(llvm).not.toContain("sc_alsbound_"); +}); + +test("programs that enter stores bind each completion callback to the active context", async () => { + const llvm = await emit(` +import { AsyncLocalStorage } from "node:async_hooks"; +${randomBytes} +const store = new AsyncLocalStorage(); +store.run("a", () => randomBytes(1, (error, buffer) => console.log(store.getStore(), buffer.length))); +`); + const binds = llvm.match( + /call ptr @scr_als_bind_closure\(ptr [^,]+, ptr @sc_alsbound_\d+, i1 false\)/g, + ); + expect(binds?.length).toBe(2); + // One thunk per callback signature. + expect(llvm.match(/^define internal void @sc_alsbound_\d+\(/gm)?.length).toBe(1); +}); + +test("AsyncLocalStorage.bind binds a typed function with an empty context too", async () => { + const llvm = await emit(` +import { AsyncLocalStorage } from "node:async_hooks"; +const store = new AsyncLocalStorage(); +const read = AsyncLocalStorage.bind((n: number) => \`\${store.getStore()} \${n}\`); +console.log(store.run("x", () => read(1))); +`); + expect(llvm).toMatch( + /call ptr @scr_als_bind_closure\(ptr [^,]+, ptr @sc_alsbound_\d+, i1 true\)/, + ); + expect(llvm).toMatch(/^define internal ptr @sc_alsbound_\d+\(ptr %self, double %a0\)/m); +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7110-als-completion-callbacks.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7110-als-completion-callbacks.ts.json new file mode 100644 index 000000000..5645ac5df --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7110-als-completion-callbacks.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7110-als-completion-callbacks.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7111-als-bind-snapshot-options.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7111-als-bind-snapshot-options.ts.json new file mode 100644 index 000000000..27aa14a8d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7111-als-bind-snapshot-options.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7111-als-bind-snapshot-options.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7112-als-bind-snapshot-dynamic.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7112-als-bind-snapshot-dynamic.mjs.json new file mode 100644 index 000000000..1915cae63 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7112-als-bind-snapshot-dynamic.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7112-als-bind-snapshot-dynamic.mjs" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_async.c b/packages/runtime/src/scr_async.c index 9e3365fbe..992ec3767 100644 --- a/packages/runtime/src/scr_async.c +++ b/packages/runtime/src/scr_async.c @@ -4012,6 +4012,47 @@ double scr_set_immediate_als(ScrClosure *cb) { return scr_set_immediate(scr_als_ void scr_next_tick_als(ScrClosure *cb) { scr_next_tick(scr_als_bind(cb)); } void scr_queue_microtask_als(ScrClosure *cb) { scr_queue_microtask(scr_als_bind(cb)); } +/* A typed callback bound to the context active when it was bound: the + * completion callbacks of callback-style APIs in programs that use + * AsyncLocalStorage, and AsyncLocalStorage.bind. The bound closure's entry + * is a thunk the compiler emits per callback signature; it reads the + * target (scr_als_bound_target), enters the captured context + * (scr_als_bound_enter), calls the target with its own arguments and + * restores the caller's context (scr_als_bound_leave). With `always` + * false an empty context leaves the callback unbound, since the loop runs + * callbacks with an empty context; AsyncLocalStorage.bind passes true, so + * a bound function called later inside run() still sees no store. */ +ScrClosure *scr_als_bind_closure(ScrClosure *cb /*moves*/, void *thunk, bool always) { + ScrAlsCtx *active = *SCR_ALS_SLOT(); + if (cb == NULL || (active == NULL && !always)) return cb; + ScrClosure *bound = scr_closure_new(thunk, 2); + bound->caps[0] = scr_box_new(SCR_BOX_FUNC); + scr_box_set_ref(bound->caps[0], cb); + bound->caps[1] = scr_box_new_obj(scr_als_ctx_retain_v, scr_als_ctx_release_v, NULL); + scr_box_set_ref(bound->caps[1], scr_als_ctx_retain(active)); + return bound; +} + +/* The bound target (+1, released by scr_als_bound_leave). */ +ScrClosure *scr_als_bound_target(ScrClosure *self) { + return scr_box_get_ref(self->caps[0]); +} + +/* Enters the captured context; returns the caller's (owned). */ +ScrAlsCtx *scr_als_bound_enter(ScrClosure *self) { + ScrAlsCtx **slot = SCR_ALS_SLOT(); + ScrAlsCtx *prev = *slot; + *slot = scr_box_get_ref(self->caps[1]); /* the slot owns this reference */ + return prev; +} + +/* Restores the caller's context, replacing whatever the target entered + * (an enterWith inside it), and releases the target. */ +void scr_als_bound_leave(ScrAlsCtx *prev, ScrClosure *target) { + scr_als_restore(prev); + scr_closure_release(target); +} + /* The checked-dynamic argument form (JS files — common.mustCall wrappers * and the suite's invalid-input probes): a non-function throws Node's * ERR_INVALID_ARG_TYPE synchronously; a function value is called with diff --git a/packages/runtime/src/scr_async.h b/packages/runtime/src/scr_async.h index fc0128294..003bf3be8 100644 --- a/packages/runtime/src/scr_async.h +++ b/packages/runtime/src/scr_async.h @@ -198,6 +198,14 @@ void scr_als_disable(double id); * argument vector, restore (the finally). Result +1 or NULL pending. */ ScrDyn *scr_als_run(double id, ScrDyn *value, ScrDyn *fn, ScrDyn *args); ScrDyn *scr_als_exit_run(double id, ScrDyn *fn, ScrDyn *args); +/* new AsyncLocalStorage(options) (options borrowed; NaN with the exception + * pending for a non-object) and the name it carries (+1). */ +double scr_als_new_options(const ScrDyn *options); +ScrDyn *scr_als_name(double id); +/* AsyncLocalStorage.bind over a dyn function and the dyn completion + * callbacks (fn borrowed; +1), and AsyncLocalStorage.snapshot() (+1). */ +ScrDyn *scr_als_bind_dyn(ScrDyn *fn, bool always); +ScrDyn *scr_als_snapshot(void); /* process.on/once('unhandledRejection', fn): dyn listeners dispatched * per never-observed rejection at the end of its nextTick/microtask @@ -473,6 +481,13 @@ double scr_set_timeout_handle_als(ScrClosure *cb, double ms); double scr_set_immediate_als(ScrClosure *cb); void scr_next_tick_als(ScrClosure *cb); void scr_queue_microtask_als(ScrClosure *cb); +/* A typed callback bound to the active context (scr_async.c): `thunk` is + * the compiler's per-signature entry, which brackets the call with + * scr_als_bound_target/enter/leave. cb moves in; the result is +1. */ +ScrClosure *scr_als_bind_closure(ScrClosure *cb, void *thunk, bool always); +ScrClosure *scr_als_bound_target(ScrClosure *self); +ScrAlsCtx *scr_als_bound_enter(ScrClosure *self); +void scr_als_bound_leave(ScrAlsCtx *prev, ScrClosure *target); /* A successful process.stdout/stderr.write completion on that same queue. * The callback and its program-shaped Error | null adapter both MOVE into * the entry; the adapter is invoked with NULL (Node's success argument). */ diff --git a/packages/runtime/src/scr_async_dyn.c b/packages/runtime/src/scr_async_dyn.c index a6cd813de..c47626557 100644 --- a/packages/runtime/src/scr_async_dyn.c +++ b/packages/runtime/src/scr_async_dyn.c @@ -11,6 +11,7 @@ * 'warning' listeners + Node's stderr report). */ #include "scr_runtime.h" +#include #include #include #include @@ -30,6 +31,21 @@ static void scr_ad_oom(void) { static SCR_TL double scr_als_counter = 0; +/* The constructor options of each store, indexed by id - 1: the default + * value getStore() answers where the store has no entry, and the name. + * NULL until a store with options exists. */ +typedef struct { + ScrDyn *default_value; /* NULL: no default */ + ScrDyn *name; /* NULL: "" */ +} ScrAlsOptions; +static SCR_TL ScrAlsOptions *scr_als_options = NULL; +static SCR_TL size_t scr_als_options_len = 0; + +static const ScrAlsOptions *scr_als_options_of(double id) { + size_t i = (size_t)id - 1; + return id >= 1 && i < scr_als_options_len ? &scr_als_options[i] : NULL; +} + /* Main-slot teardown (atexit, LIFO after scr_init's audit registration — * the dc-registry precedent): a top-level enterWith leaves a context in * the main slot at exit; release it before the RC audit counts. */ @@ -37,6 +53,13 @@ static void scr_als_teardown(void) { /* atexit runs on the main context — the active slot IS main's. */ scr_als_ctx_release(*SCR_ALS_SLOT()); *SCR_ALS_SLOT() = NULL; + for (size_t i = 0; i < scr_als_options_len; i++) { + scr_dyn_release(scr_als_options[i].default_value); + scr_dyn_release(scr_als_options[i].name); + } + free(scr_als_options); + scr_als_options = NULL; + scr_als_options_len = 0; } double scr_als_new(void) { @@ -48,6 +71,48 @@ double scr_als_new(void) { return ++scr_als_counter; } +/* new AsyncLocalStorage(options): Node validates that options is an + * object, then keeps options.defaultValue and options.name. NaN with the + * exception pending when options is not an object. */ +double scr_als_new_options(const ScrDyn *options) { + if (options->kind == SCR_DYN_UNDEF) return scr_als_new(); + /* A typed record or class instance reads through its snapshot. */ + ScrDyn *snapshot = options->kind == SCR_DYN_TYPED_REF ? scr_dyn_typed_ref_materialize(options) : NULL; + if (snapshot) options = snapshot; + if (options->kind != SCR_DYN_OBJ) { + scr_dyn_arg_type_fail("options", "of type object", options); + scr_dyn_release(snapshot); + return NAN; + } + double id = scr_als_new(); + size_t index = (size_t)id - 1; + if (index >= scr_als_options_len) { + size_t len = index + 1; + ScrAlsOptions *grown = realloc(scr_als_options, len * sizeof *grown); + if (!grown) scr_ad_oom(); + for (size_t i = scr_als_options_len; i < len; i++) grown[i] = (ScrAlsOptions){NULL, NULL}; + scr_als_options = grown; + scr_als_options_len = len; + } + ScrDyn *value = scr_dyn_obj_get(options, "defaultValue", 12); + ScrDyn *name = scr_dyn_obj_get(options, "name", 4); + scr_als_options[index].default_value = scr_dyn_retain(value ? value : scr_dyn_undefined()); + scr_als_options[index].name = + name && name->kind != SCR_DYN_UNDEF ? scr_dyn_retain(name) : NULL; + scr_dyn_release(snapshot); + return id; +} + +/* asyncLocalStorage.name (+1): the constructor's name option, or "". */ +ScrDyn *scr_als_name(double id) { + const ScrAlsOptions *options = scr_als_options_of(id); + if (options && options->name) return scr_dyn_retain(options->name); + ScrStr *empty = scr_str_new("", 0); + ScrDyn *result = scr_dyn_new_str(empty); + scr_str_release(empty); + return result; +} + ScrDyn *scr_als_get(double id) { const ScrAlsCtx *c = *SCR_ALS_SLOT(); if (c) { @@ -55,6 +120,8 @@ ScrDyn *scr_als_get(double id) { if (c->entries[i].id == id) return scr_dyn_retain(c->entries[i].value); } } + const ScrAlsOptions *options = scr_als_options_of(id); + if (options && options->default_value) return scr_dyn_retain(options->default_value); return scr_dyn_retain(scr_dyn_undefined()); } @@ -104,8 +171,77 @@ ScrDyn *scr_als_run(double id, ScrDyn *value, ScrDyn *fn, ScrDyn *args) { return scr_als_call_in(scr_als_enter(id, value), fn, args); } +/* exit() runs fn with the store undefined: a store with a default value + * holds an undefined entry for the call (getStore() answers undefined, + * not the default, as in Node); other stores drop their entry. */ ScrDyn *scr_als_exit_run(double id, ScrDyn *fn, ScrDyn *args) { - return scr_als_call_in(scr_als_enter_absent(id), fn, args); + const ScrAlsOptions *options = scr_als_options_of(id); + ScrAlsCtx *prev = options && options->default_value + ? scr_als_enter(id, scr_dyn_undefined()) + : scr_als_enter_absent(id); + return scr_als_call_in(prev, fn, args); +} + +/* AsyncLocalStorage.bind(fn) and the dyn completion callbacks of + * callback-style APIs: a function that calls fn in the context active + * when it was bound, with its own arguments and receiver. caps[0] holds + * fn, caps[1] the context, as scr_als_bind_closure lays them out. */ +static void *scr_als_ctx_retain_v(void *c) { return scr_als_ctx_retain(c); } +static void scr_als_ctx_release_v(void *c) { scr_als_ctx_release(c); } + +static ScrDyn *scr_als_bound_dyn_call(ScrClosure *self, ScrDyn *const *args, size_t argc) { + ScrDyn *fn = scr_box_get_ref(self->caps[0]); + ScrAlsCtx *prev = scr_als_bound_enter(self); + ScrDyn *result = scr_dyn_call(fn, args, argc, "callback"); + scr_als_restore(prev); + scr_dyn_release(fn); + return result; +} + +static ScrClosure *scr_als_context_closure(void *fn, size_t ncaps) { + ScrClosure *closure = scr_closure_new(fn, ncaps); + closure->caps[ncaps - 1] = scr_box_new_obj(scr_als_ctx_retain_v, scr_als_ctx_release_v, NULL); + scr_box_set_ref(closure->caps[ncaps - 1], scr_als_ctx_retain(*SCR_ALS_SLOT())); + return closure; +} + +/* fn borrowed; +1. With `always` false an empty context answers fn + * itself (the completion-callback form). AsyncLocalStorage.bind throws + * Node's ERR_INVALID_ARG_TYPE for a non-function. */ +ScrDyn *scr_als_bind_dyn(ScrDyn *fn, bool always) { + if (fn->kind != SCR_DYN_FUNC) { + if (always) { + scr_dyn_arg_type_fail("fn", "of type function", fn); + return NULL; + } + return scr_dyn_retain(fn); + } + if (!always && *SCR_ALS_SLOT() == NULL) return scr_dyn_retain(fn); + ScrClosure *closure = scr_als_context_closure(NULL, 2); + closure->caps[0] = scr_box_new_obj(scr_dyn_retain_v, scr_dyn_release_v, scr_dyn_trace_v); + scr_box_set_ref(closure->caps[0], scr_dyn_retain(fn)); + return scr_dyn_new_func(closure, scr_als_bound_dyn_call, fn->v.fn.arity, "(...args)", "bound"); +} + +/* AsyncLocalStorage.snapshot(): a function (cb, ...args) that calls cb + * with args in the context active at the snapshot. */ +static ScrDyn *scr_als_snapshot_call(ScrClosure *self, ScrDyn *const *args, size_t argc) { + const ScrDyn *cb = argc > 0 ? args[0] : scr_dyn_undefined(); + if (cb->kind != SCR_DYN_FUNC) { + static const char message[] = "cb is not a function"; + scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); + return NULL; + } + ScrAlsCtx **slot = SCR_ALS_SLOT(); + ScrAlsCtx *prev = *slot; + *slot = scr_box_get_ref(self->caps[0]); /* the slot owns this reference */ + ScrDyn *result = scr_dyn_call(cb, args + 1, argc - 1, "callback"); + scr_als_restore(prev); + return result; +} + +ScrDyn *scr_als_snapshot(void) { + return scr_dyn_new_func(scr_als_context_closure(NULL, 1), scr_als_snapshot_call, 1, "(cb, ...args)", "bound"); } /* The two rejection-event registries share one shape: listeners with a diff --git a/tests/corpus/7110-als-completion-callbacks.ts b/tests/corpus/7110-als-completion-callbacks.ts new file mode 100644 index 000000000..4b4e97e0d --- /dev/null +++ b/tests/corpus/7110-als-completion-callbacks.ts @@ -0,0 +1,135 @@ +// The completion callback of a callback-style API runs with the +// AsyncLocalStorage stores that were active when the call started the +// operation, as in Node. +import { AsyncLocalStorage } from "node:async_hooks"; +import { execFile } from "node:child_process"; +import * as crypto from "node:crypto"; +import * as dgram from "node:dgram"; +import * as dns from "node:dns"; +import { readFile, rename, writeFileSync, unlinkSync } from "node:fs"; +import * as http from "node:http"; +import * as net from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as zlib from "node:zlib"; + +const store = new AsyncLocalStorage(); + +// Starts one operation inside run(name) and waits for its callback. +function step(name: string, start: (done: () => void) => void): Promise { + return new Promise((resolve) => { + store.run(name, () => + start(() => { + console.log(name, store.getStore()); + resolve(); + }), + ); + }); +} + +const dir = tmpdir(); +const file = join(dir, `scriptc-als-${process.pid}.txt`); +writeFileSync(file, "hello"); + +await step("fs.readFile", (done) => readFile(file, "utf8", () => done())); +await step("fs.rename", (done) => rename(file, `${file}.moved`, () => done())); +unlinkSync(`${file}.moved`); + +await step("crypto.randomBytes", (done) => crypto.randomBytes(4, () => done())); +await step("crypto.randomInt", (done) => crypto.randomInt(10, () => done())); +await step("crypto.randomFill", (done) => crypto.randomFill(new Uint8Array(4), () => done())); +await step("crypto.pbkdf2", (done) => crypto.pbkdf2("p", "s", 1, 8, "sha256", () => done())); +await step("crypto.scrypt", (done) => crypto.scrypt("p", "s", 8, () => done())); +await step("crypto.hkdf", (done) => crypto.hkdf("sha256", "k", "s", "i", 8, () => done())); +await step("crypto.generateKey", (done) => + crypto.generateKey("hmac", { length: 64 }, () => done()), +); + +await step("zlib.gzip", (done) => zlib.gzip(Buffer.from("hello"), () => done())); +await step("zlib.deflate options", (done) => + zlib.deflate(Buffer.from("hello"), { level: 1 }, () => done()), +); + +await step("dns.lookup", (done) => dns.lookup("localhost", () => done())); +await step("child_process.execFile", (done) => execFile("true", [], () => done())); + +// A server's listen callback, a client connection and an HTTP response +// each run in the stores of their own call. +const server = net.createServer((socket) => socket.end()); +await step("net listen", (done) => server.listen(0, "127.0.0.1", () => done())); +const port = (server.address() as net.AddressInfo).port; +await step("net.connect", (done) => { + const socket = net.connect(port, "127.0.0.1", () => { + socket.destroy(); + done(); + }); +}); +server.close(); + +const web = http.createServer((_req, res) => res.end("ok")); +await new Promise((resolve) => web.listen(0, "127.0.0.1", () => resolve())); +const webPort = (web.address() as net.AddressInfo).port; +await step("http.get", (done) => + http.get(`http://127.0.0.1:${webPort}/`, (res) => { + res.resume(); + done(); + }), +); +await step("http.request", (done) => + http + .request({ host: "127.0.0.1", port: webPort, path: "/", method: "GET" }, (res) => { + res.resume(); + done(); + }) + .end(), +); +web.close(); + +const optionsServer = net.createServer(); +await step("net listen options", (done) => + optionsServer.listen({ port: 0, host: "127.0.0.1" }, () => done()), +); +optionsServer.close(); +const pathServer = net.createServer(); +const socketPath = join(dir, `scriptc-als-${process.pid}.sock`); +await step("net listen path", (done) => pathServer.listen(socketPath, () => done())); +pathServer.close(); + +const udp = dgram.createSocket("udp4"); +await step("dgram bind", (done) => udp.bind(0, "127.0.0.1", () => done())); +const peer = dgram.createSocket("udp4"); +await new Promise((resolve) => peer.bind(0, "127.0.0.1", () => resolve())); +await step("dgram connect", (done) => udp.connect(peer.address().port, "127.0.0.1", () => done())); +peer.close(); +await step("dgram close", (done) => udp.close(() => done())); + +await step("process.stdout.write", (done) => process.stdout.write("", () => done())); + +// Nested: a callback starts another operation in a different store. +await step("outer", (done) => + crypto.randomBytes(1, () => { + console.log("outer callback", store.getStore()); + store.run("inner", () => + crypto.randomBytes(1, () => { + console.log("inner callback", store.getStore()); + done(); + }), + ); + }), +); + +// A callback whose call ran with no store sees no store, and a store the +// callback enters does not reach the next callback. +await new Promise((resolve) => + crypto.randomBytes(1, () => { + console.log("no store", store.getStore()); + store.enterWith("entered"); + resolve(); + }), +); +await new Promise((resolve) => + crypto.randomBytes(1, () => { + console.log("after enterWith", store.getStore()); + resolve(); + }), +); diff --git a/tests/corpus/7111-als-bind-snapshot-options.ts b/tests/corpus/7111-als-bind-snapshot-options.ts new file mode 100644 index 000000000..d1c6dbbf2 --- /dev/null +++ b/tests/corpus/7111-als-bind-snapshot-options.ts @@ -0,0 +1,59 @@ +// AsyncLocalStorage.bind, AsyncLocalStorage.snapshot and the constructor +// options, as in Node. +import { AsyncLocalStorage } from "node:async_hooks"; + +const store = new AsyncLocalStorage(); + +// bind(fn) runs fn in the context active at the bind, with its arguments. +const add = store.run("bound", () => + AsyncLocalStorage.bind((a: number, b: number) => `${store.getStore()}:${a + b}`), +); +console.log(add(1, 2)); +console.log(store.run("other", () => add(3, 4))); + +// A function bound outside any store sees no store inside run(). +const plain = AsyncLocalStorage.bind(() => store.getStore()); +console.log(store.run("ignored", () => plain())); + +// The bound function keeps its result type. +const length = store.run("len", () => AsyncLocalStorage.bind((s: string) => s.length)); +console.log(length("four") * 2); + +// snapshot() runs a function in the context of the snapshot. +const snap = store.run("snapshot", () => AsyncLocalStorage.snapshot()); +console.log(snap(() => store.getStore())); +console.log(store.run("later", () => snap(() => store.getStore()))); +console.log(snap((x: number, y: number) => x * y, 6, 7)); + +// A store entered inside the bound call ends with it. +const enter = store.run("start", () => + AsyncLocalStorage.bind(() => { + store.enterWith("entered"); + return store.getStore(); + }), +); +console.log(enter(), store.getStore()); + +// The default value and the name. +const config = new AsyncLocalStorage<{ level: number }>({ + defaultValue: { level: 1 }, + name: "config", +}); +console.log(config.name, config.getStore()?.level); +config.run({ level: 2 }, () => console.log("run", config.getStore()?.level)); +config.exit(() => console.log("exit", config.getStore())); +config.disable(); +console.log("disabled", config.getStore()?.level); +const unnamed = new AsyncLocalStorage(); +console.log(JSON.stringify(unnamed.name), unnamed.getStore()); +const empty = new AsyncLocalStorage({}); +console.log(JSON.stringify(empty.name), empty.getStore()); + +// Bound functions and snapshots carry the context across asynchronous work. +const later = store.run("timer", () => AsyncLocalStorage.bind(() => store.getStore())); +await new Promise((resolve) => + setTimeout(() => { + console.log("in timer", later(), store.getStore()); + resolve(); + }, 1), +); diff --git a/tests/corpus/7112-als-bind-snapshot-dynamic.mjs b/tests/corpus/7112-als-bind-snapshot-dynamic.mjs new file mode 100644 index 000000000..22e3238b8 --- /dev/null +++ b/tests/corpus/7112-als-bind-snapshot-dynamic.mjs @@ -0,0 +1,51 @@ +// AsyncLocalStorage.bind, snapshot and the constructor options over +// untyped values, as in Node. +import { AsyncLocalStorage } from "node:async_hooks"; +import { readFile } from "node:fs"; + +const store = new AsyncLocalStorage(); + +function sum(a, b) { + return [this?.tag, a + b, store.getStore()]; +} +const bound = store.run("bound", () => AsyncLocalStorage.bind(sum)); +console.log(bound.name, bound.length, bound === sum, typeof bound); +console.log(JSON.stringify(bound.call({ tag: "receiver" }, 1, 2))); +console.log(JSON.stringify(store.run("other", () => bound(3, 4)))); + +const snap = store.run("snapshot", () => AsyncLocalStorage.snapshot()); +console.log(snap.name, snap.length, typeof snap); +console.log(snap((a, b) => `${store.getStore()} ${a + b}`, 2, 3)); + +for (const attempt of [ + () => AsyncLocalStorage.bind(5), + () => AsyncLocalStorage.bind("fn"), + () => snap(5), + () => snap(), +]) { + try { + console.log("ok", attempt()); + } catch (error) { + console.log(error.name, error.code, error.message); + } +} + +for (const options of [1, "x", null, true]) { + try { + new AsyncLocalStorage(options); + console.log("ok", options); + } catch (error) { + console.log(error.name, error.code, error.message); + } +} +const named = new AsyncLocalStorage({ name: "named", defaultValue: 0 }); +console.log(named.name, named.getStore()); +named.run(5, () => console.log(named.getStore())); +console.log(new AsyncLocalStorage(undefined).name === ""); + +// An untyped callback runs in the stores of its call. +store.run("readFile", () => + readFile(import.meta.filename, "utf8", (error, text) => + console.log("readFile", store.getStore(), error === null, text.length > 0), + ), +); diff --git a/tests/harness/corpus-costs/7110-als-completion-callbacks.ts.json b/tests/harness/corpus-costs/7110-als-completion-callbacks.ts.json new file mode 100644 index 000000000..d8f3f6d12 --- /dev/null +++ b/tests/harness/corpus-costs/7110-als-completion-callbacks.ts.json @@ -0,0 +1 @@ +{ "plain": 6500, "san": 16000 } diff --git a/tests/harness/corpus-costs/7111-als-bind-snapshot-options.ts.json b/tests/harness/corpus-costs/7111-als-bind-snapshot-options.ts.json new file mode 100644 index 000000000..3f7384f20 --- /dev/null +++ b/tests/harness/corpus-costs/7111-als-bind-snapshot-options.ts.json @@ -0,0 +1 @@ +{ "plain": 3500, "san": 6000 } diff --git a/tests/harness/corpus-costs/7112-als-bind-snapshot-dynamic.mjs.json b/tests/harness/corpus-costs/7112-als-bind-snapshot-dynamic.mjs.json new file mode 100644 index 000000000..3fd934f0e --- /dev/null +++ b/tests/harness/corpus-costs/7112-als-bind-snapshot-dynamic.mjs.json @@ -0,0 +1 @@ +{ "plain": 3500, "san": 12000 } From ef8cc7f74f13470c21d3c47b92df75ad9055808a Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 09:48:55 -0500 Subject: [PATCH 002/184] Settle Promise.all over different entry types on Node's turns `Promise.all([a, b])` whose entries have different types (the checker's tuple overload) ran through an async helper that awaited the combinator and then each fulfilled entry, so its awaits resumed three microtask turns after Node's. The combinator now stores the entry promises, and an adapter run inside its settlement reads them into the tuple, so the result settles on the combinator's turn: awaiting it, storing it first, chaining it with then and returning it from an async function all land on Node's turns. Plain entries join the combinator through Promise.resolve, as in Node, and unit-typed entries compile. --- .../compiler/src/backend/llvm/lib-dispatch.ts | 88 ++++++ .../frontend/lowering/builtins/promises.ts | 273 +++++++++--------- .../compiler/src/ir/builtin-signatures.ts | 12 + packages/compiler/src/ir/validate.ts | 26 ++ .../7035-promise-all-tuple-turns.ts.json | 6 + packages/runtime/src/scr_async.c | 6 + packages/runtime/src/scr_async.h | 2 + tests/corpus/7035-promise-all-tuple-turns.ts | 64 ++++ .../7035-promise-all-tuple-turns.ts.json | 1 + 9 files changed, 338 insertions(+), 140 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7035-promise-all-tuple-turns.ts.json create mode 100644 tests/corpus/7035-promise-all-tuple-turns.ts create mode 100644 tests/harness/corpus-costs/7035-promise-all-tuple-turns.ts.json diff --git a/packages/compiler/src/backend/llvm/lib-dispatch.ts b/packages/compiler/src/backend/llvm/lib-dispatch.ts index 24f2bf019..207132d05 100644 --- a/packages/compiler/src/backend/llvm/lib-dispatch.ts +++ b/packages/compiler/src/backend/llvm/lib-dispatch.ts @@ -5,6 +5,7 @@ import { emitTlsLibCall } from "./lib-tls.js"; import { emitHttp2LibCall } from "./lib-http2.js"; import { emitDatagramLibCall } from "./lib-datagram.js"; import { emitTestLibCall } from "./lib-test.js"; +import { FN_ATTRS, arrNewCall, traceArg, vAdapters } from "./shapes.js"; /* Focused LLVM library-call emission extracted from emitter.ts. */ import { InternalCompilerError } from "../../errors.js"; @@ -236,6 +237,32 @@ export function emitGenericLibCall(host: LlvmEmitterContext, e: LibCallExpr): Ll host.emitPendingCheck(); return output; } + if (e.fn === "promise.allTuple") return emitPromiseAllTuple(host, e); + if (e.fn === "promise.fulfilledValue") { + // The payload of a fulfilled promise (borrowed operand): numbers and + // booleans by value, strings and references +1. + const promise = host.emitExpr(e.args[0]!); + const out = B.tmp(); + switch (e.type.kind) { + case "f64": + case "procStream": + host.declare(`declare double @scr_promise_payload_f64(ptr)`); + B.line(`${out} = call double @scr_promise_payload_f64(ptr ${promise.name})`); + return { name: out, type: e.type }; + case "bool": + host.declare(`declare zeroext i1 @scr_promise_payload_bool(ptr)`); + B.line(`${out} = call zeroext i1 @scr_promise_payload_bool(ptr ${promise.name})`); + return { name: out, type: e.type }; + case "string": + host.declare(`declare ptr @scr_promise_payload_str(ptr)`); + B.line(`${out} = call ptr @scr_promise_payload_str(ptr ${promise.name}) ; +1`); + return host.own({ name: out, type: e.type }); + default: + host.declare(`declare ptr @scr_promise_payload_ref(ptr)`); + B.line(`${out} = call ptr @scr_promise_payload_ref(ptr ${promise.name}) ; +1`); + return host.own({ name: out, type: e.type }); + } + } if (e.fn === "dyn.typedRefIs" || e.fn === "dyn.typedRefKeyIs") { const literal = e.args[1]!.kind === "strLit" ? e.args[1]!.value : null; if (e.fn === "dyn.typedRefKeyIs" && literal === null) @@ -877,6 +904,7 @@ export function emitLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue { case "disposable": case "punycode": case "constants": + case "promise": return host.emitGenericLibCall(e); case "test": return emitTestLibCall(host, e); @@ -891,3 +919,63 @@ export function emitLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue { } } } + +/** Promise.all's tuple form (promise.allTuple): the combinator over the + * entries stores each fulfilled entry promise, and a settlement adapter on + * the combinator calls the builder with them and fulfills the result with + * the tuple inside the combinator's settlement; a rejection settles the + * result the same way. The combinator itself counts as handled. */ +function emitPromiseAllTuple(host: LlvmEmitterContext, e: LibCallExpr): LlValue { + const B = host.B; + const build = e.args[1]!; + if (build.kind !== "closure" || e.type.kind !== "promise") + throw new InternalCompilerError("llvm emitter bug: promise.allTuple shape"); + const entriesT = e.args[0]!.type; + const entries = host.emitExpr(e.args[0]!); + const tuple = e.type.inner; + const key = `allTuple:${build.fnName}`; + let adapter = host.resolveThunks.get(key); + if (adapter === undefined) { + adapter = `sc_all_tuple_${host.resolveThunks.size}`; + host.resolveThunks.set(key, adapter); + const rc = vAdapters(host.shapeHost, tuple); + host.declare(`declare ptr @scr_promise_payload_ref(ptr)`); + host.declare(`declare void @scr_promise_fulfill_ref(ptr, ptr, ptr, ptr, ptr)`); + host.resolveThunkDefs.push( + `define internal void @${adapter}(ptr %dst, ptr %src) ${FN_ATTRS} { ; Promise.all tuple`, + `entry:`, + ` %entries = call ptr @scr_promise_payload_ref(ptr %src) ; +1, moves into the builder`, + ` %tuple = call ptr @${host.callTarget(build.fnName)}(ptr %entries)`, + ` call void @scr_promise_fulfill_ref(ptr %dst, ptr %tuple, ptr ${rc.retain}, ptr ${rc.release}, ptr ${traceArg(host.shapeHost, tuple)})`, + ` ret void`, + `}`, + ``, + ); + } + host.declare(`declare ptr @scr_promise_all(ptr, ptr, ptr)`); + host.declare(`declare void @scr_promise_all_store_promise(ptr, double, ptr)`); + host.declare(`declare ptr @scr_promise_new()`); + host.declare(`declare void @scr_promise_race_add(ptr, ptr, ptr)`); + host.declare(`declare void @scr_promise_mark_handled(ptr)`); + host.declare(`declare void @scr_promise_release(ptr)`); + host.declare(`declare double @scr_arr_len(ptr)`); + if (entriesT.kind !== "array") + throw new InternalCompilerError("llvm emitter bug: promise.allTuple entries"); + const len = B.tmp(); + const cap = B.tmp(); + B.line(`${len} = call double @scr_arr_len(ptr ${entries.name})`); + B.line(`${cap} = fptoui double ${len} to ${host.sizeType}`); + const values = B.tmp(); + B.line(`${values} = ${arrNewCall(host.shapeHost, entriesT.elem, cap)}`); + host.own({ name: values, type: entriesT }); + const all = B.tmp(); + B.line( + `${all} = call ptr @scr_promise_all(ptr ${entries.name}, ptr ${values}, ptr @scr_promise_all_store_promise)`, + ); + B.line(`call void @scr_promise_mark_handled(ptr ${all})`); + const result = B.tmp(); + B.line(`${result} = call ptr @scr_promise_new()`); + B.line(`call void @scr_promise_race_add(ptr ${result}, ptr ${all}, ptr @${adapter})`); + B.line(`call void @scr_promise_release(ptr ${all})`); + return host.own({ name: result, type: e.type }); +} diff --git a/packages/compiler/src/frontend/lowering/builtins/promises.ts b/packages/compiler/src/frontend/lowering/builtins/promises.ts index 75e527aa5..c30590619 100644 --- a/packages/compiler/src/frontend/lowering/builtins/promises.ts +++ b/packages/compiler/src/frontend/lowering/builtins/promises.ts @@ -2,7 +2,7 @@ import { keyedHelperName } from "../stable-names.js"; import { checkedPromiseAll, thenReader } from "../checked-promise-all.js"; import { InternalCompilerError } from "../../../errors.js"; import * as ts from "../../ts7/adapter.js"; -import { newFnCtx, type Lowerer } from "../lowerer.js"; +import type { Lowerer } from "../lowerer.js"; import { varRef } from "../../../ir/build.js"; import { maybePromiseResolve, maybePromiseUnionOf } from "../maybe-promise.js"; import { isJsSourceFile, locOf } from "../../program.js"; @@ -15,32 +15,25 @@ import { type IrType, UNDEFINED_T, VOID, + F64, arrayOf, + funcOf, isUnitType, typeEquals, typeKey, } from "../../../ir/ir.js"; /** `Promise.all([p, q, r])` over DIFFERENT promise types — the checker's - * tuple overload, `Promise<[A, B, C]>`. One interned async helper per - * entry/result signature takes the entries as arguments, awaits the - * runtime combinator over the same promises (each widened to - * Promise, a type-only view), so settlement is Node's exactly: the - * first rejection in settlement order wins and every later one counts as - * handled; then it reads each (now fulfilled) entry and returns the - * tuple. Entries keep their identity — no value converts through the - * checked-dynamic tree. Non-promise entries (numbers, strings, booleans, - * unit values — never thenable) pass straight into their slot; a - * checked-dynamic entry (a JavaScript function's result) joins through - * PromiseResolve: the promise it holds, a promise that adopts a thenable, - * or a promise fulfilled with its plain value. The tuple - * fulfills one microtask turn after the combinator does. Null when the - * result type or an entry has no typed home here. */ -function lowerTuplePromiseAll( + * tuple overload, `Promise<[A, B, C]>`: the result type and tuple shape, + * or null when the result type or an entry has no typed home here. + * Entries are promises, checked-dynamic values (a JavaScript function's + * result) and plain numbers, strings, booleans and unit values, which are + * never thenable. */ +function tuplePromiseAllShape( lowerer: Lowerer, call: ts.CallExpression, - elems: IrExpr[], -): IrExpr | null { + elems: readonly IrExpr[], +): { resultT: IrType & { kind: "promise" }; tuple: IrType & { kind: "record" } } | null { const resultT = lowerer.mapTypeOf(lowerer.typeOf(call)); if (resultT?.kind !== "promise" || resultT.inner.kind !== "record") return null; const shape = lowerer.shapes.get(resultT.inner.shapeId); @@ -52,135 +45,135 @@ function lowerTuplePromiseAll( const settles = (t: IrType): boolean => t.kind === "promise" || t.kind === "dyn"; if (!elems.some((e) => settles(e.type))) return null; if (!elems.every((e) => settles(e.type) || plainValue(e.type))) return null; + return { resultT, tuple: resultT.inner }; +} + +/** `Promise.all([p, q, r])` over different promise types. Each entry + * becomes a promise in input order (a plain value through Promise.resolve, + * a checked-dynamic one through its PromiseResolve view: the promise it + * holds, a promise that adopts a thenable, or a promise fulfilled with the + * value), widened to Promise, a type-only view. The runtime + * combinator over them gives Node's settlement: the first rejection in + * settlement order wins and every later one counts as handled. The + * result settles inside the combinator's own settlement, so it and its + * awaits land on Node's microtask turns; a fulfillment runs one interned + * function per entry/result signature that reads the fulfilled entries + * into the tuple. Entries keep their identity: no value converts through + * the checked-dynamic tree. */ +function lowerTuplePromiseAll( + lowerer: Lowerer, + call: ts.CallExpression, + elems: IrExpr[], +): IrExpr | null { + const plan = tuplePromiseAllShape(lowerer, call, elems); + if (!plan) return null; + const { resultT, tuple } = plan; const loc = locOf(call); - const key = `${elems.map((e) => typeKey(e.type)).join(",")}->${typeKey(resultT.inner)}`; + const voidPromise: IrType & { kind: "promise" } = { kind: "promise", inner: VOID }; + const entriesT = arrayOf(voidPromise); + const dynPromise: IrType = { kind: "promise", inner: DYN }; + const entries = elems.map((e): IrExpr => { + let promise: IrExpr; + if (e.type.kind === "promise") promise = e; + else if (e.type.kind === "dyn") + promise = { + kind: "libCall", + fn: "dyn.promiseResolve", + args: [e, thenReader(lowerer, loc)], + type: dynPromise, + loc, + }; + else if (isUnitType(e.type)) { + // A unit entry's value is its type's one value; the promise only + // takes its place in the combinator. + const resolved: IrExpr = { + kind: "intrinsic", + name: "promise.resolve", + args: [], + type: voidPromise, + loc, + }; + promise = + e.kind === "unitLit" + ? resolved + : { + kind: "seqExpr", + stmts: [{ kind: "exprStmt", expr: e, loc }], + result: resolved, + type: voidPromise, + loc, + }; + } else + promise = { + kind: "intrinsic", + name: "promise.resolve", + args: [e], + type: { kind: "promise", inner: e.type }, + loc, + }; + return promise.type.kind === "promise" && promise.type.inner.kind === "void" + ? promise + : { kind: "promiseVoidWiden", value: promise, type: voidPromise, loc }; + }); + const key = `${elems.map((e) => typeKey(e.type)).join(",")}->${typeKey(tuple)}`; let name = lowerer.arrHofHelpers.get(`promise.allTuple:${key}`); if (name === undefined) { name = keyedHelperName(lowerer, "%promise.allTuple", `promise.allTuple:${key}`); lowerer.arrHofHelpers.set(`promise.allTuple:${key}`, name); - const ctx = newFnCtx(false, null, null, resultT.inner); - ctx.isAsync = true; - lowerer.fnStack.push(ctx); - try { - const params = elems.map((e, i) => ({ - localId: `entry.${i}`, - name: `entry${i}`, - type: e.type, - })); - const ref = (i: number): IrExpr => varRef(`entry.${i}`, elems[i]!.type, loc); - const dynPromise: IrType = { kind: "promise", inner: DYN }; - const body: IrStmt[] = []; - // Each dyn entry's promise view, in entry order: the promise it - // holds, a promise that adopts a thenable's settlement, or a promise - // fulfilled with the plain value (PromiseResolve's rule). - const views = new Map(); - elems.forEach((e, i) => { - if (e.type.kind !== "dyn") return; - const view = lowerer.declareHiddenLocal("%allEntry", dynPromise); - body.push({ - kind: "varDecl", - localId: view.id, - init: { - kind: "libCall", - fn: "dyn.promiseResolve", - args: [ref(i), thenReader(lowerer, loc)], - type: dynPromise, - loc, - }, - loc, - }); - views.set(i, varRef(view.id, dynPromise, loc)); - }); - const widened: IrExpr[] = []; - elems.forEach((e, i) => { - const view = views.get(i); - if (view !== undefined) { - widened.push({ - kind: "promiseVoidWiden", - value: view, - type: { kind: "promise", inner: VOID }, - loc, - }); - return; - } - if (e.type.kind !== "promise") return; - widened.push( - e.type.inner.kind === "void" - ? ref(i) - : { - kind: "promiseVoidWiden", - value: ref(i), - type: { kind: "promise", inner: VOID }, + const shape = lowerer.shapes.get(tuple.shapeId)!; + const entriesRef = varRef("entries.0", entriesT, loc); + // Shape fields are sorted by name ("10" before "2"); a tuple field's + // name is its position. + const fields = shape.fields.map((field) => { + const i = Number(field.name); + const type = elems[i]!.type; + const inner = type.kind === "promise" ? type.inner : type; + const value: IrExpr = + inner.kind === "void" + ? { kind: "unitLit", unit: "undefined", type: UNDEFINED_T, loc } + : isUnitType(inner) + ? { + kind: "unitLit", + unit: inner.kind === "nullT" ? "null" : "undefined", + type: inner, loc, - }, - ); - }); - const settled: IrExpr = { - kind: "awaitExpr", - value: { - kind: "intrinsic", - name: "promise.all", - args: [ - { - kind: "arrayLit", - elems: widened, - type: arrayOf({ kind: "promise", inner: VOID }), - loc, - }, - ], - type: { kind: "promise", inner: VOID }, - loc, - }, - type: VOID, - loc, - }; - // Shape fields are sorted by name ("10" before "2"); a tuple field's - // name is its position. - const fields = shape.fields.map((field) => { - const i = Number(field.name); - const entry = elems[i]!; - let value: IrExpr; - const view = views.get(i); - if (view !== undefined) { - value = { kind: "awaitExpr", value: view, type: DYN, loc }; - } else if (entry.type.kind === "promise") { - value = - entry.type.inner.kind === "void" - ? { kind: "unitLit", unit: "undefined", type: UNDEFINED_T, loc } - : { kind: "awaitExpr", value: ref(i), type: entry.type.inner, loc }; - } else { - value = ref(i); - } - return { name: field.name, value: lowerer.coerceToExpected(value, field.type) }; - }); - body.push( - { kind: "exprStmt", expr: settled, loc }, - { - kind: "return", - value: { kind: "recordLit", fields, type: resultT.inner, loc }, - loc, - }, - ); - lowerer.liftedFns.push({ - name, - params, - returnType: resultT.inner, - locals: [ - ...params.map((p) => ({ id: p.localId, name: p.name, type: p.type, mutable: false })), - ...lowerer.ctx.locals, - ], - body, - async: true, - loc, - }); - } catch (e) { - lowerer.arrHofHelpers.delete(`promise.allTuple:${key}`); - throw e; - } finally { - lowerer.fnStack.pop(); - } + } + : { + kind: "libCall", + fn: "promise.fulfilledValue", + args: [ + { + kind: "arrayGet", + arr: entriesRef, + index: { kind: "numLit", value: i, type: F64, loc }, + type: voidPromise, + loc, + }, + ], + type: inner, + loc, + }; + return { name: field.name, value: lowerer.coerceToExpected(value, field.type) }; + }); + lowerer.liftedFns.push({ + name, + params: [{ localId: "entries.0", name: "entries", type: entriesT }], + returnType: tuple, + locals: [{ id: "entries.0", name: "entries", type: entriesT, mutable: false }], + body: [{ kind: "return", value: { kind: "recordLit", fields, type: tuple, loc }, loc }], + loc, + }); } - return { kind: "call", callee: name, args: elems, type: resultT, loc }; + return { + kind: "libCall", + fn: "promise.allTuple", + args: [ + { kind: "arrayLit", elems: entries, type: entriesT, loc }, + { kind: "closure", fnName: name, captures: [], type: funcOf([entriesT], tuple), loc }, + ], + type: resultT, + loc, + }; } /** `Promise.race([...])` on THE Promise global: the entries lower diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index fcdbe8694..d44f3ebbf 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -3139,6 +3139,18 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ // or an optional Error | null slot (same success shape as fs.rename). "process.stdoutWriteBytesCb": { argTypes: [BYTES_U8, STRING, null], result: BOOL }, "process.stderrWriteBytesCb": { argTypes: [BYTES_U8, STRING, null], result: BOOL }, + /** The value of a FULFILLED promise, read without a microtask turn (+1 + * for references): the tuple form of Promise.all reads its entries this + * way once the combinator fulfills, each entry widened to Promise + * and read at the type it was built with (program-dependent). */ + "promise.fulfilledValue": { argTypes: [null], result: VOID }, + /** Promise.all's tuple form as a value: the entries (each widened to + * Promise) join the runtime combinator, and when it fulfills the + * function reads the fulfilled entries into the tuple inside that + * settlement, so the result settles on the combinator's turn; a + * rejection settles it the same way. The result is the tuple's promise + * (program-dependent). */ + "promise.allTuple": { argTypes: [null, null], result: VOID }, "fsp.readFile": { argTypes: [STRING, STRING], result: { kind: "promise", inner: STRING } }, "fsp.writeFile": { argTypes: [STRING, STRING], result: { kind: "promise", inner: VOID } }, /** fs.promises.writeFile(path, data, { mode }): the settled-promise diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index bb36c7007..08c237fe2 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -5310,6 +5310,32 @@ function validateFunction( } function checkLibSpecialCall(e: IrExpr & { kind: "libCall" }): boolean { + if (e.fn === "promise.fulfilledValue") { + const promise = e.args[0]?.type; + if ( + promise?.kind !== "promise" || + e.type.kind === "void" || + isUnitType(e.type) || + promise.inner.kind !== "void" + ) + err(`libCall ${e.fn} must read a Promise entry at a non-void type`, e.loc); + return true; + } + if (e.fn === "promise.allTuple") { + const [entries, build] = e.args; + if ( + entries?.type.kind !== "array" || + entries.type.elem.kind !== "promise" || + entries.type.elem.inner.kind !== "void" || + build?.kind !== "closure" || + build.captures.length > 0 || + build.type.kind !== "func" || + e.type.kind !== "promise" || + !typeEquals(build.type.ret, e.type.inner) + ) + err(`libCall ${e.fn} must join Promise entries into its builder's tuple`, e.loc); + return true; + } if ( e.fn === "crypto.randomFillView" || e.fn === "crypto.randomFillViewRest" || diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7035-promise-all-tuple-turns.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7035-promise-all-tuple-turns.ts.json new file mode 100644 index 000000000..29b935e93 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7035-promise-all-tuple-turns.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7035-promise-all-tuple-turns.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_async.c b/packages/runtime/src/scr_async.c index 9e3365fbe..5bdfe87a5 100644 --- a/packages/runtime/src/scr_async.c +++ b/packages/runtime/src/scr_async.c @@ -1839,6 +1839,12 @@ void scr_promise_all_store_ref(ScrArr *a, double i, ScrPromise *src) { scr_arr_set_ref(a, i, scr_promise_payload_ref(src)); } +/* The entry promise itself: Promise.all's tuple form reads each fulfilled + * entry at its own type once the combinator fulfills. */ +void scr_promise_all_store_promise(ScrArr *a, double i, ScrPromise *src) { + scr_arr_set_ref(a, i, scr_promise_retain(src)); +} + /* Fulfillment payload accessors for the emitted race adapters — all * retained/by-value (the source keeps its settlement). */ double scr_promise_payload_f64(ScrPromise *p) { return p->f64; } diff --git a/packages/runtime/src/scr_async.h b/packages/runtime/src/scr_async.h index fc0128294..cc79ee1a1 100644 --- a/packages/runtime/src/scr_async.h +++ b/packages/runtime/src/scr_async.h @@ -369,6 +369,8 @@ void scr_promise_all_store_f64(ScrArr *a, double i, ScrPromise *src); void scr_promise_all_store_bool(ScrArr *a, double i, ScrPromise *src); void scr_promise_all_store_str(ScrArr *a, double i, ScrPromise *src); void scr_promise_all_store_ref(ScrArr *a, double i, ScrPromise *src); +/* Stores the fulfilled entry promise itself (+1), for Promise.all's tuple form. */ +void scr_promise_all_store_promise(ScrArr *a, double i, ScrPromise *src); void scr_promise_adapt_copy(ScrPromise *dst, ScrPromise *src); double scr_promise_payload_f64(ScrPromise *p); bool scr_promise_payload_bool(ScrPromise *p); diff --git a/tests/corpus/7035-promise-all-tuple-turns.ts b/tests/corpus/7035-promise-all-tuple-turns.ts new file mode 100644 index 000000000..6989df9b9 --- /dev/null +++ b/tests/corpus/7035-promise-all-tuple-turns.ts @@ -0,0 +1,64 @@ +// Promise.all over entries of different types settles, and resumes its +// awaits and then callbacks, on the same microtask turns as Node, whether +// the result is awaited at once, stored first, chained with then, or +// returned from an async function. The first rejection in settlement +// order wins, later ones count as handled, and record entries keep their +// identity. +const log: string[] = []; +function later(label: string, turns: number, value: T, fail = false): Promise { + return new Promise((resolve, reject) => { + let left = turns; + const step = (): void => { + if (left-- > 0) return queueMicrotask(step); + log.push(`settle ${label}`); + if (fail) reject(new Error(label)); + else resolve(value); + }; + step(); + }); +} +async function num(): Promise { + return 1; +} +async function text(): Promise { + return "s"; +} +async function both(): Promise<[number, string]> { + return Promise.all([num(), text()]); +} +interface Item { + id: number; +} +const item: Item = { id: 7 }; + +async function main(): Promise { + let ticking = true; + void (async () => { + for (let i = 0; i < 40 && ticking; i++) { + log.push(`t${i}`); + await null; + } + })(); + const [a, b] = await Promise.all([num(), text()]); + log.push(`awaited ${a}${b}`); + const stored = Promise.all([text(), num(), Promise.resolve(item)]); + const [c, d, e] = await stored; + log.push(`stored ${c}${d} ${e === item}`); + Promise.all([text(), num()]).then(([f, g]) => log.push(`then ${f}${g}`)); + const [h, i] = await both(); + log.push(`returned ${h}${i}`); + const mixed = await Promise.all([later("x", 3, "x"), 5, later("y", 1, true), undefined]); + log.push(`mixed ${mixed[0]} ${mixed[1]} ${mixed[2]} ${mixed[3]}`); + const raw: any = JSON.parse('{"value":41}'); + const [dynamic, counted] = await Promise.all([raw.value, later("z", 2, 2)]); + log.push(`dynamic ${dynamic + counted}`); + try { + await Promise.all([later("late", 4, 1, true), later("early", 2, "e", true), num()]); + } catch (err) { + log.push(`rejected ${(err as Error).message}`); + } + await later("tail", 6, 0); + ticking = false; + console.log(log.join("\n")); +} +void main(); diff --git a/tests/harness/corpus-costs/7035-promise-all-tuple-turns.ts.json b/tests/harness/corpus-costs/7035-promise-all-tuple-turns.ts.json new file mode 100644 index 000000000..9e7986c22 --- /dev/null +++ b/tests/harness/corpus-costs/7035-promise-all-tuple-turns.ts.json @@ -0,0 +1 @@ +{ "plain": 2910, "san": 4860 } From d14c2ee05f7c6cb720ef6721433df5351ab2f4b8 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 09:59:13 -0500 Subject: [PATCH 003/184] Read AsyncLocalStorage options as JavaScript does and call snapshot callbacks without a receiver new AsyncLocalStorage(options) read defaultValue and name as own data properties, so inherited values and getters were ignored, and a name that was not a string reached the compiled code unconverted. The options are now read with a full property get (prototypes, getters, proxies, the snapshot of a typed record or class instance) in Node's order: defaultValue once, name to see whether it is defined and again for its value. The name is kept as String(name); a Symbol throws Node's TypeError, and an exception from a getter propagates. A class instance whose class itself declares one of the two members, which a capsule does not expose to the runtime, is refused with a "not supported yet" error. A snapshot's runner called its callback with the runner's own receiver; Node applies the callback to an undefined receiver. --- packages/runtime/src/scr_async_dyn.c | 70 +++++++++++++++---- .../corpus/7111-als-bind-snapshot-options.ts | 7 ++ .../corpus/7112-als-bind-snapshot-dynamic.mjs | 43 ++++++++++++ 3 files changed, 107 insertions(+), 13 deletions(-) diff --git a/packages/runtime/src/scr_async_dyn.c b/packages/runtime/src/scr_async_dyn.c index c47626557..58a4ac3f8 100644 --- a/packages/runtime/src/scr_async_dyn.c +++ b/packages/runtime/src/scr_async_dyn.c @@ -71,17 +71,61 @@ double scr_als_new(void) { return ++scr_als_counter; } +/* options[key] as JavaScript reads it (+1, or NULL with the exception + * pending): inherited properties and getters included. A class instance + * reads its own fields through its snapshot; a member its class declares + * (a getter or a method) has no runtime view here and is refused. */ +static ScrDyn *scr_als_option(ScrDyn *options, const char *key, size_t key_len) { + ScrStr *name = scr_str_new(key, key_len); + if (options->kind == SCR_DYN_TYPED_REF && scr_dyn_typed_ref_class_has(options, name)) { + char message[160]; + snprintf(message, sizeof message, + "AsyncLocalStorage options whose class declares '%s' are not supported yet", key); + scr_str_release(name); + scr_throw_error_msg(SCR_ERR_ERROR, message, strlen(message)); + return NULL; + } + ScrDyn *key_value = scr_dyn_new_str(name); + scr_str_release(name); + ScrDyn *value = scr_dyn_reflect_get(options, key_value, options); + scr_dyn_release(key_value); + if (value == NULL && !scr_exc_pending()) value = scr_dyn_retain(scr_dyn_undefined()); + return value; +} + /* new AsyncLocalStorage(options): Node validates that options is an - * object, then keeps options.defaultValue and options.name. NaN with the - * exception pending when options is not an object. */ + * object, then reads options.defaultValue once and options.name twice + * (whether it is defined, then its value), keeping String(name). NaN with + * the exception pending when options is not an object or a read throws. */ double scr_als_new_options(const ScrDyn *options) { if (options->kind == SCR_DYN_UNDEF) return scr_als_new(); - /* A typed record or class instance reads through its snapshot. */ - ScrDyn *snapshot = options->kind == SCR_DYN_TYPED_REF ? scr_dyn_typed_ref_materialize(options) : NULL; - if (snapshot) options = snapshot; - if (options->kind != SCR_DYN_OBJ) { + if (options->kind != SCR_DYN_OBJ && options->kind != SCR_DYN_TYPED_REF && options->kind != SCR_DYN_PROXY) { scr_dyn_arg_type_fail("options", "of type object", options); - scr_dyn_release(snapshot); + return NAN; + } + ScrDyn *value = scr_als_option((ScrDyn *)options, "defaultValue", 12); + if (!value) return NAN; + ScrDyn *name = scr_als_option((ScrDyn *)options, "name", 4); + if (name && name->kind != SCR_DYN_UNDEF) { + scr_dyn_release(name); + name = scr_als_option((ScrDyn *)options, "name", 4); + if (name && name->kind == SCR_DYN_SYMBOL) { + static const char message[] = "Cannot convert a Symbol value to a string"; + scr_dyn_release(name); + name = NULL; + scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); + } else if (name) { + ScrStr *text = scr_dyn_string_coerce_js(name); + scr_dyn_release(name); + name = text ? scr_dyn_new_str(text) : NULL; + scr_str_release(text); + } + } else { + scr_dyn_release(name); + name = scr_exc_pending() ? NULL : scr_dyn_retain(scr_dyn_undefined()); + } + if (!name) { + scr_dyn_release(value); return NAN; } double id = scr_als_new(); @@ -94,12 +138,9 @@ double scr_als_new_options(const ScrDyn *options) { scr_als_options = grown; scr_als_options_len = len; } - ScrDyn *value = scr_dyn_obj_get(options, "defaultValue", 12); - ScrDyn *name = scr_dyn_obj_get(options, "name", 4); - scr_als_options[index].default_value = scr_dyn_retain(value ? value : scr_dyn_undefined()); - scr_als_options[index].name = - name && name->kind != SCR_DYN_UNDEF ? scr_dyn_retain(name) : NULL; - scr_dyn_release(snapshot); + scr_als_options[index].default_value = value; /* the read's +1 moves */ + scr_als_options[index].name = name->kind != SCR_DYN_UNDEF ? name : NULL; + if (name->kind == SCR_DYN_UNDEF) scr_dyn_release(name); return id; } @@ -235,7 +276,10 @@ static ScrDyn *scr_als_snapshot_call(ScrClosure *self, ScrDyn *const *args, size ScrAlsCtx **slot = SCR_ALS_SLOT(); ScrAlsCtx *prev = *slot; *slot = scr_box_get_ref(self->caps[0]); /* the slot owns this reference */ + /* Node applies cb to an undefined receiver, whatever the runner's own. */ + scr_dyn_this_push_dyn(scr_dyn_undefined()); ScrDyn *result = scr_dyn_call(cb, args + 1, argc - 1, "callback"); + scr_dyn_this_pop(); scr_als_restore(prev); return result; } diff --git a/tests/corpus/7111-als-bind-snapshot-options.ts b/tests/corpus/7111-als-bind-snapshot-options.ts index d1c6dbbf2..cad93cd5c 100644 --- a/tests/corpus/7111-als-bind-snapshot-options.ts +++ b/tests/corpus/7111-als-bind-snapshot-options.ts @@ -48,6 +48,13 @@ const unnamed = new AsyncLocalStorage(); console.log(JSON.stringify(unnamed.name), unnamed.getStore()); const empty = new AsyncLocalStorage({}); console.log(JSON.stringify(empty.name), empty.getStore()); +// A class instance's fields are options too. +class StoreOptions { + defaultValue = 7; + name = "from-class"; +} +const fromClass = new AsyncLocalStorage(new StoreOptions()); +console.log(fromClass.name, fromClass.getStore()); // Bound functions and snapshots carry the context across asynchronous work. const later = store.run("timer", () => AsyncLocalStorage.bind(() => store.getStore())); diff --git a/tests/corpus/7112-als-bind-snapshot-dynamic.mjs b/tests/corpus/7112-als-bind-snapshot-dynamic.mjs index 22e3238b8..1e20aefd3 100644 --- a/tests/corpus/7112-als-bind-snapshot-dynamic.mjs +++ b/tests/corpus/7112-als-bind-snapshot-dynamic.mjs @@ -43,6 +43,49 @@ console.log(named.name, named.getStore()); named.run(5, () => console.log(named.getStore())); console.log(new AsyncLocalStorage(undefined).name === ""); +// The options are read as JavaScript reads properties: inherited values +// and getters count, defaultValue is read once and name twice, and the +// name becomes a string. +const inherited = new AsyncLocalStorage(Object.create({ defaultValue: "inherited", name: "proto" })); +console.log(inherited.getStore(), inherited.name); +const reads = []; +const viaGetters = new AsyncLocalStorage({ + get defaultValue() { + reads.push("defaultValue"); + return "getter"; + }, + get name() { + reads.push("name"); + return "getter-name"; + }, +}); +console.log(viaGetters.getStore(), viaGetters.name, reads.join(",")); +const coerced = [5, null, { toString: () => "object" }].map((name) => new AsyncLocalStorage({ name }).name); +console.log(JSON.stringify(coerced)); +for (const options of [ + { + get defaultValue() { + throw new Error("from a getter"); + }, + }, + { name: Symbol("s") }, +]) { + try { + new AsyncLocalStorage(options); + console.log("constructed"); + } catch (error) { + console.log(error.name, error.message); + } +} + +// A snapshot calls its callback with an undefined receiver, also when the +// snapshot itself is called as a method. +const kind = function () { + return this === undefined ? "undefined" : typeof this; +}; +const holder = { snap, tag: "holder" }; +console.log(snap(kind), holder.snap(kind), snap.call({ tag: "receiver" }, kind)); + // An untyped callback runs in the stores of its call. store.run("readFile", () => readFile(import.meta.filename, "utf8", (error, text) => From de6dd133708f11851621487aa6ddc49c9252666c Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 10:45:31 -0500 Subject: [PATCH 004/184] Run net, tls, http and https events in Node's AsyncLocalStorage contexts The runtime fired every socket, server, request and response event from the event loop with no store. Node runs them in the context of the resource or request that produced them, so getStore() answered undefined in listeners where Node answers the caller's store. Handles now keep the context Node uses and the runtime enters it around each dispatch: - a socket the context of its creation (net.connect, tls.connect, an http request's socket), for 'connect', 'data', 'end', 'close', 'error' and 'timeout'; - a server the contexts of its listen() call ('listening', 'connection', listen errors) and its close() call ('close' and close callbacks when no connection is open; a close that waits for the last connection settles in that connection's dispatch, as in Node); - socket write() callbacks and end() callbacks with 'finish' the contexts of their calls; - http server requests and responses, which are created in their connection's dispatch, the server's listen() context, and client requests and their responses the http.request() context, including a keep-alive socket an agent hands to a later request. An accepted net socket runs its own events with no store, as in Node; the sockets of an HTTP or TLS server run in the server's listen() context. Events and callbacks of other streams are unchanged. The compiler is unchanged. Programs without net or http link the same runtime code; net and http programs gain the capture and dispatch helpers (about 1.7 KB of text in a net program, 2.2 KB in an http server), and bench:runtime http-api is neutral. Corpus 7119 and 7380. --- .../builtin-apis/asynclocalstorage.mdx | 5 +- .../generated/node-v24-internal.json | 34 ++- internal/compatibility/static-support.json | 2 +- .../7119-als-net-event-context.mjs.json | 6 + .../7380-als-http-event-context.mjs.json | 6 + packages/runtime/src/scr_async.c | 11 + packages/runtime/src/scr_async.h | 9 + packages/runtime/src/scr_http.c | 40 +++ packages/runtime/src/scr_net.c | 252 +++++++++++++----- packages/runtime/src/scr_net.h | 4 + tests/corpus/7119-als-net-event-context.mjs | 124 +++++++++ tests/corpus/7380-als-http-event-context.mjs | 151 +++++++++++ .../7119-als-net-event-context.mjs.json | 1 + .../7380-als-http-event-context.mjs.json | 1 + 14 files changed, 573 insertions(+), 73 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7119-als-net-event-context.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7380-als-http-event-context.mjs.json create mode 100644 tests/corpus/7119-als-net-event-context.mjs create mode 100644 tests/corpus/7380-als-http-event-context.mjs create mode 100644 tests/harness/corpus-costs/7119-als-net-event-context.mjs.json create mode 100644 tests/harness/corpus-costs/7380-als-http-event-context.mjs.json diff --git a/docs/content/limitations/builtin-apis/asynclocalstorage.mdx b/docs/content/limitations/builtin-apis/asynclocalstorage.mdx index 99f89d65c..98fc92ea1 100644 --- a/docs/content/limitations/builtin-apis/asynclocalstorage.mdx +++ b/docs/content/limitations/builtin-apis/asynclocalstorage.mdx @@ -1,4 +1,7 @@ - **AsyncLocalStorage:** timer, interval, `setImmediate`, `process.nextTick` and `queueMicrotask` callbacks run with the stores that were active when they were scheduled, as in Node. The completion callbacks of `fs`, `crypto` and `zlib` callback APIs, `dns.lookup`, `child_process.execFile`, `server.listen`, `net.connect`, `http.request`, `http.get` and `dgram` run with the stores that were active at the call, as in Node. - Event listeners on sockets, requests, responses and streams, stream callbacks such as `write(chunk, callback)`, `server.close(callback)`, and the callbacks of `https` and `tls` calls run with no store, where Node runs them in the context of the resource or of the code that triggers them. + Events of `net`, `tls`, `http` and `https` sockets, servers, requests and responses run with the stores Node runs them with. + A socket's or client request's events see the context of the call that created it, and a server's events and its requests' and responses' events see the context of its `listen()` call. + Socket `write` and `end` callbacks, `'finish'` and `server.close(callback)` see the context of their call. + Events and callbacks of other streams run with no store, where Node runs them in the context of the code that triggers them. `AsyncResource` has no lowering yet. diff --git a/internal/compatibility/generated/node-v24-internal.json b/internal/compatibility/generated/node-v24-internal.json index 34fe2a843..6046daf0e 100644 --- a/internal/compatibility/generated/node-v24-internal.json +++ b/internal/compatibility/generated/node-v24-internal.json @@ -1390,7 +1390,9 @@ "tests/corpus/5815-als-callback-context.ts", "tests/corpus/7110-als-completion-callbacks.ts", "tests/corpus/7111-als-bind-snapshot-options.ts", - "tests/corpus/7112-als-bind-snapshot-dynamic.mjs" + "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", + "tests/corpus/7119-als-net-event-context.mjs", + "tests/corpus/7380-als-http-event-context.mjs" ] }, "anchorSource": "chapter" @@ -1443,7 +1445,9 @@ "tests/corpus/5815-als-callback-context.ts", "tests/corpus/7110-als-completion-callbacks.ts", "tests/corpus/7111-als-bind-snapshot-options.ts", - "tests/corpus/7112-als-bind-snapshot-dynamic.mjs" + "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", + "tests/corpus/7119-als-net-event-context.mjs", + "tests/corpus/7380-als-http-event-context.mjs" ] }, "anchorSource": "exact" @@ -1519,7 +1523,9 @@ "tests/corpus/5815-als-callback-context.ts", "tests/corpus/7110-als-completion-callbacks.ts", "tests/corpus/7111-als-bind-snapshot-options.ts", - "tests/corpus/7112-als-bind-snapshot-dynamic.mjs" + "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", + "tests/corpus/7119-als-net-event-context.mjs", + "tests/corpus/7380-als-http-event-context.mjs" ] }, "anchorSource": "exact" @@ -1549,7 +1555,9 @@ "tests/corpus/5815-als-callback-context.ts", "tests/corpus/7110-als-completion-callbacks.ts", "tests/corpus/7111-als-bind-snapshot-options.ts", - "tests/corpus/7112-als-bind-snapshot-dynamic.mjs" + "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", + "tests/corpus/7119-als-net-event-context.mjs", + "tests/corpus/7380-als-http-event-context.mjs" ] }, "anchorSource": "exact" @@ -1579,7 +1587,9 @@ "tests/corpus/5815-als-callback-context.ts", "tests/corpus/7110-als-completion-callbacks.ts", "tests/corpus/7111-als-bind-snapshot-options.ts", - "tests/corpus/7112-als-bind-snapshot-dynamic.mjs" + "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", + "tests/corpus/7119-als-net-event-context.mjs", + "tests/corpus/7380-als-http-event-context.mjs" ] }, "anchorSource": "exact" @@ -1609,7 +1619,9 @@ "tests/corpus/5815-als-callback-context.ts", "tests/corpus/7110-als-completion-callbacks.ts", "tests/corpus/7111-als-bind-snapshot-options.ts", - "tests/corpus/7112-als-bind-snapshot-dynamic.mjs" + "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", + "tests/corpus/7119-als-net-event-context.mjs", + "tests/corpus/7380-als-http-event-context.mjs" ] }, "anchorSource": "exact" @@ -1639,7 +1651,9 @@ "tests/corpus/5815-als-callback-context.ts", "tests/corpus/7110-als-completion-callbacks.ts", "tests/corpus/7111-als-bind-snapshot-options.ts", - "tests/corpus/7112-als-bind-snapshot-dynamic.mjs" + "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", + "tests/corpus/7119-als-net-event-context.mjs", + "tests/corpus/7380-als-http-event-context.mjs" ] }, "anchorSource": "exact" @@ -1946,6 +1960,8 @@ "tests/corpus/7110-als-completion-callbacks.ts", "tests/corpus/7111-als-bind-snapshot-options.ts", "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", + "tests/corpus/7119-als-net-event-context.mjs", + "tests/corpus/7380-als-http-event-context.mjs", "tests/harness/node-test.test.ts" ] }, @@ -2511,7 +2527,9 @@ "tests/corpus/5815-als-callback-context.ts", "tests/corpus/7110-als-completion-callbacks.ts", "tests/corpus/7111-als-bind-snapshot-options.ts", - "tests/corpus/7112-als-bind-snapshot-dynamic.mjs" + "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", + "tests/corpus/7119-als-net-event-context.mjs", + "tests/corpus/7380-als-http-event-context.mjs" ] }, "anchorSource": "exact" diff --git a/internal/compatibility/static-support.json b/internal/compatibility/static-support.json index efc60f263..c57d6013e 100644 --- a/internal/compatibility/static-support.json +++ b/internal/compatibility/static-support.json @@ -429,7 +429,7 @@ { "symbols": ["crypto.hkdf", "crypto.hkdfSync", "crypto.scrypt", "crypto.scryptSync"], "status": "partial", "evidence": ["tests/corpus/node24-crypto-kdf.ts", "tests/corpus/node24-crypto-kdf-errors.ts", "tests/corpus/node24-crypto-kdf-options.ts", "tests/corpus/5438-crypto-kdf-digests.ts"] }, { "symbols": ["EventEmitter", "emitter.on", "emitter.addListener", "emitter.once", "emitter.prependListener", "emitter.prependOnceListener", "emitter.off", "emitter.removeListener", "emitter.removeAllListeners", "emitter.emit", "emitter.listenerCount", "emitter.listeners", "emitter.rawListeners", "emitter.eventNames", "emitter.setMaxListeners", "emitter.getMaxListeners", "events.once"], "status": "partial", "evidence": ["tests/corpus/1677-emitter-listeners.ts", "tests/corpus/2620-ee-override-once-order.ts"] }, { "symbols": ["events.getEventListeners", "events.getMaxListeners", "events.listenerCount", "events.setMaxListeners", "events.defaultMaxListeners"], "status": "partial", "evidence": ["tests/corpus/5885-events-max-listener-helpers.ts", "tests/corpus/5846-events-esm-default-max.ts", "tests/corpus/5847-events-cjs-default-max.cjs"] }, - { "symbols": ["AsyncLocalStorage", "asyncLocalStorage.run", "asyncLocalStorage.exit", "asyncLocalStorage.getStore", "asyncLocalStorage.enterWith", "asyncLocalStorage.disable", "asyncLocalStorage.name", "AsyncLocalStorage.bind", "AsyncLocalStorage.snapshot"], "status": "partial", "evidence": ["tests/corpus/2213-async-local-storage.cjs", "tests/corpus/5815-als-callback-context.ts", "tests/corpus/7110-als-completion-callbacks.ts", "tests/corpus/7111-als-bind-snapshot-options.ts", "tests/corpus/7112-als-bind-snapshot-dynamic.mjs"] }, + { "symbols": ["AsyncLocalStorage", "asyncLocalStorage.run", "asyncLocalStorage.exit", "asyncLocalStorage.getStore", "asyncLocalStorage.enterWith", "asyncLocalStorage.disable", "asyncLocalStorage.name", "AsyncLocalStorage.bind", "AsyncLocalStorage.snapshot"], "status": "partial", "evidence": ["tests/corpus/2213-async-local-storage.cjs", "tests/corpus/5815-als-callback-context.ts", "tests/corpus/7110-als-completion-callbacks.ts", "tests/corpus/7111-als-bind-snapshot-options.ts", "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", "tests/corpus/7119-als-net-event-context.mjs", "tests/corpus/7380-als-http-event-context.mjs"] }, { "symbols": ["StringDecoder", "stringDecoder.write", "stringDecoder.end"], "status": "partial", "evidence": ["tests/corpus/1662-string-decoder-encodings.ts", "tests/corpus/node24-string-decoder.ts", "tests/corpus/5882-string-decoder-runtime-encoding.ts"] }, { "symbols": ["setImmediate", "setInterval", "setTimeout", "clearImmediate", "clearInterval", "clearTimeout", "timersPromises.setTimeout", "timersPromises.setImmediate", "timersPromises.setInterval", "immediate.hasRef", "immediate.ref", "immediate.unref", "timeout.hasRef", "timeout.ref", "timeout.unref", "timeout.refresh"], "status": "partial", "evidence": ["tests/corpus/1804-timers-module.ts", "tests/corpus/1805-timer-callback-args.ts", "tests/corpus/2093-timers-promises.ts", "tests/corpus/2852-timers-promises-interval.ts", "tests/corpus/5830-timers-promises-signal.ts"] }, { "symbols": ["timersPromises.scheduler.wait", "timersPromises.scheduler.yield"], "status": "partial", "evidence": ["tests/corpus/5839-timers-promises-scheduler.ts", "tests/corpus/5889-timers-scheduler-signal.ts"] }, diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7119-als-net-event-context.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7119-als-net-event-context.mjs.json new file mode 100644 index 000000000..7ff81ea34 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7119-als-net-event-context.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7119-als-net-event-context.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7380-als-http-event-context.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7380-als-http-event-context.mjs.json new file mode 100644 index 000000000..1087d25fe --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7380-als-http-event-context.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7380-als-http-event-context.mjs" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_async.c b/packages/runtime/src/scr_async.c index 992ec3767..90e7dd04f 100644 --- a/packages/runtime/src/scr_async.c +++ b/packages/runtime/src/scr_async.c @@ -774,6 +774,17 @@ void scr_als_restore(ScrAlsCtx *prev) { *SCR_ALS_SLOT() = prev; /* ownership moves back in */ } +ScrAlsCtx *scr_als_dispatch_enter(ScrAlsCtx *ctx) { + ScrAlsCtx **slot = SCR_ALS_SLOT(); + ScrAlsCtx *prev = *slot; + *slot = scr_als_ctx_retain(ctx); + return prev; +} + +void scr_als_dispatch_leave(ScrAlsCtx *prev) { scr_als_restore(prev); } + +ScrAlsCtx *scr_als_capture(void) { return scr_als_ctx_retain(*SCR_ALS_SLOT()); } + /* The value of `id` in the active snapshot (borrowed), or NULL. */ ScrDyn *scr_als_peek(double id) { const ScrAlsCtx *c = *SCR_ALS_SLOT(); diff --git a/packages/runtime/src/scr_async.h b/packages/runtime/src/scr_async.h index 003bf3be8..25f25a842 100644 --- a/packages/runtime/src/scr_async.h +++ b/packages/runtime/src/scr_async.h @@ -192,6 +192,15 @@ ScrDyn *scr_als_get(double id); /* +1; the undefined value when unset */ ScrAlsCtx *scr_als_enter(double id, ScrDyn *value); /* borrows value; prev out (owned) */ ScrAlsCtx *scr_als_enter_absent(double id); /* exit()'s cleared arm */ void scr_als_restore(ScrAlsCtx *prev); /* moves prev back in */ +/* A runtime dispatch (an I/O event the loop delivers, a deferred emit) + * runs its listeners in the AsyncLocalStorage context of the object or + * request Node would run them in: enter installs `ctx` (borrowed, NULL + * for no store) and answers the previous context, which leave puts back. + * A store a listener enters (enterWith) ends with the dispatch. */ +ScrAlsCtx *scr_als_dispatch_enter(ScrAlsCtx *ctx); +void scr_als_dispatch_leave(ScrAlsCtx *prev); +/* The active context, +1 (NULL outside a store): what a handle keeps. */ +ScrAlsCtx *scr_als_capture(void); void scr_als_enter_with(double id, ScrDyn *value); /* no restore point (enterWith) */ void scr_als_disable(double id); /* run/exit: enter (or clear), call the dyn function with the forwarded diff --git a/packages/runtime/src/scr_http.c b/packages/runtime/src/scr_http.c index a5847e59b..18d997fb5 100644 --- a/packages/runtime/src/scr_http.c +++ b/packages/runtime/src/scr_http.c @@ -209,6 +209,11 @@ void scr_http_set_h2_upgrade_hook(void (*hook)(void *h2ctx, ScrClosure *cb, void struct ScrHttpReq { size_t rc; + /* The AsyncLocalStorage context of its creation (+1, NULL outside a + * store): the deferred emits run in it (scr_http_proto_sweep). A server + * request is created in its connection's dispatch, which runs in the + * server's listen() context; a client response in its request's. */ + ScrAlsCtx *als; int status; /* -1 on server requests (Node's undefined statusCode) */ ScrStr *method; ScrStr *url; @@ -301,6 +306,7 @@ void scr_http_req_set_encoding(ScrHttpReq *r, ScrStr *enc /*borrowed*/) { void scr_http_req_release(ScrHttpReq *r) { if (!r || r->rc == SIZE_MAX) return; if (--r->rc == 0) { + scr_als_ctx_release(r->als); scr_str_release(r->method); scr_str_release(r->url); for (size_t i = 0; i < r->nheaders; i++) { @@ -928,6 +934,7 @@ static bool scr_http_trailers_replace(ScrHttpTrailers *t, ScrArr *pairs /*borrow struct ScrHttpRes { size_t rc; + ScrAlsCtx *als; /* +1: as ScrHttpReq's */ ScrNetSocket *sock; /* +1; writes after the connection died are no-ops */ void *h2_stream; /* +1 through scr_http_h2_ops — set on h2 compat * responses; every write path routes to frames */ @@ -985,6 +992,7 @@ ScrHttpRes *scr_http_res_retain(ScrHttpRes *r) { void scr_http_res_release(ScrHttpRes *r) { if (!r || r->rc == SIZE_MAX) return; if (--r->rc == 0) { + scr_als_ctx_release(r->als); for (size_t i = 0; i < r->nheaders; i++) { scr_str_release(r->hnames[i]); scr_str_release(r->hvalues[i]); @@ -2257,9 +2265,25 @@ void scr_http_res_on_write_flush(ScrHttpRes *r, ScrClosure *cb /*moves*/) { static bool scr_http_proto_pending(void) { return scr_http_emits_head < scr_http_emits_len; } +static ScrAlsCtx *scr_http_client_ctx(struct ScrHttpClientReq *c); + +/* The context an emit's handle was created in (borrowed). */ +static ScrAlsCtx *scr_http_emit_ctx(const ScrHttpEmit *e) { + switch (e->kind) { + case SCR_HTTP_EMIT_RES_CLOSE: + case SCR_HTTP_EMIT_RES_WCB: + case SCR_HTTP_EMIT_RES_FINISH: return ((ScrHttpRes *)e->h)->als; + case SCR_HTTP_EMIT_REQ_CLOSE: + case SCR_HTTP_EMIT_REQ_DRAIN: + case SCR_HTTP_EMIT_REQ_ABORTED: return ((ScrHttpReq *)e->h)->als; + default: return scr_http_client_ctx((struct ScrHttpClientReq *)e->h); + } +} + static void scr_http_proto_sweep(void) { while (scr_http_emits_head < scr_http_emits_len) { ScrHttpEmit e = scr_http_emits[scr_http_emits_head++]; + ScrAlsCtx *prev = scr_als_dispatch_enter(scr_http_emit_ctx(&e)); switch (e.kind) { case SCR_HTTP_EMIT_RES_CLOSE: { ScrHttpRes *res = (ScrHttpRes *)e.h; @@ -2349,6 +2373,7 @@ static void scr_http_proto_sweep(void) { break; } } + scr_als_dispatch_leave(prev); scr_http_emit_release(&e); if (scr_exc_pending()) return; } @@ -2870,6 +2895,7 @@ static bool scr_http_conn_parse_head(ScrHttpConn *conn, size_t head_len, bool *o ScrHttpReq *req = calloc(1, sizeof *req); if (!req) scr_http_oom(); req->rc = 1; + req->als = scr_als_capture(); req->status = -1; /* Node: statusCode is undefined on server requests */ req->gated = true; req->join_dup = conn->srv != NULL && conn->srv->join_dup; @@ -3038,6 +3064,7 @@ static bool scr_http_conn_parse_head(ScrHttpConn *conn, size_t head_len, bool *o ScrHttpRes *res = calloc(1, sizeof *res); if (!res) scr_http_oom(); res->rc = 1; + res->als = scr_als_capture(); #ifdef SCR_RC_AUDIT scr_http_live++; #endif @@ -3590,6 +3617,7 @@ ScrHttpReq *scr_http_h2_req_new(ScrNetSocket *sock /*borrowed, nullable*/, ScrHttpReq *req = calloc(1, sizeof *req); if (!req) scr_http_oom(); req->rc = 1; + req->als = scr_als_capture(); req->status = -1; /* server requests: statusCode is Node's undefined */ req->http2 = true; req->sock = sock ? scr_net_sock_retain(sock) : NULL; @@ -3661,6 +3689,7 @@ ScrHttpRes *scr_http_h2_res_new(ScrNetSocket *sock /*borrowed, nullable*/, ScrHttpRes *res = calloc(1, sizeof *res); if (!res) scr_http_oom(); res->rc = 1; + res->als = scr_als_capture(); res->sock = sock ? scr_net_sock_retain(sock) : NULL; res->h2_stream = scr_http_h2_ops->retain(stream); res->keep_alive = true; /* inert on h2 — no Connection header exists */ @@ -3770,6 +3799,9 @@ void scr_http_handler_thunk0(ScrClosure *cb, ScrHttpReq *req, ScrHttpRes *res) { struct ScrHttpClientReq { size_t rc; + /* The http.request() call's AsyncLocalStorage context (+1): the + * request's deferred emits ('socket', 'finish', 'close') run in it. */ + ScrAlsCtx *als; ScrNetSocket *sock; /* +1 */ ScrHttpConn *conn; /* borrowed backref (the socket's ctx); NULL after free */ ScrStr *host, *path, *method; @@ -3834,6 +3866,7 @@ ScrHttpClientReq *scr_http_client_retain(ScrHttpClientReq *c) { void scr_http_client_release(ScrHttpClientReq *c) { if (!c || c->rc == SIZE_MAX) return; if (--c->rc == 0) { + scr_als_ctx_release(c->als); scr_net_ls_drop(&c->resp_ls); scr_net_ls_drop(&c->err_ls); scr_net_ls_drop(&c->timeout_ls); @@ -3872,6 +3905,8 @@ static void scr_http_client_release_internal(struct ScrHttpClientReq *c) { scr_http_client_release(c); } +static ScrAlsCtx *scr_http_client_ctx(struct ScrHttpClientReq *c) { return c->als; } + void *scr_http_client_retain_v(void *p) { return scr_http_client_retain((ScrHttpClientReq *)p); } void scr_http_client_release_v(void *p) { scr_http_client_release((ScrHttpClientReq *)p); } @@ -4648,6 +4683,7 @@ static bool scr_http_client_parse_head(ScrHttpConn *conn, size_t head_len) { ScrHttpReq *res = calloc(1, sizeof *res); if (!res) scr_http_oom(); res->rc = 1; + res->als = scr_als_capture(); res->status = status; res->sock = scr_net_sock_retain(conn->sock); #ifdef SCR_RC_AUDIT @@ -5041,6 +5077,7 @@ static ScrHttpClientReq *scr_http_request_impl(ScrStr *host /*borrowed*/, double ScrHttpClientReq *c = calloc(1, sizeof *c); if (!c) scr_http_oom(); c->rc = 1; + c->als = scr_als_capture(); #ifdef SCR_RC_AUDIT scr_http_live++; #endif @@ -5086,6 +5123,7 @@ static ScrHttpClientReq *scr_http_request_impl(ScrStr *host /*borrowed*/, double if (presock != NULL) { c->sock = presock; + scr_net_sock_set_context(c->sock, c->als); } else { ScrStr *dial = scr_net_blocking_lookup(host); c->sock = scr_net_connect(port, dial, NULL); @@ -5344,6 +5382,8 @@ static void scr_http_client_rebind(ScrHttpClientReq *c, ScrHttpAgentFree *f, boo scr_net_sock_release(old); ScrHttpConn *conn = f->conn; c->sock = f->sock; /* the free entry's +1 moves */ + /* Node runs a pooled socket's events for this request in its context. */ + scr_net_sock_set_context(c->sock, c->als); c->conn = conn; conn->client = scr_http_client_retain(c); conn->pool_agent = NULL; diff --git a/packages/runtime/src/scr_net.c b/packages/runtime/src/scr_net.c index 29a4af393..33374b97c 100644 --- a/packages/runtime/src/scr_net.c +++ b/packages/runtime/src/scr_net.c @@ -420,6 +420,11 @@ enum { SCR_NET_K_SERVER = 1, SCR_NET_K_SOCKET = 2 }; struct ScrNetServer { int kind; /* FIRST: poller udata routes on it */ size_t rc; + /* The AsyncLocalStorage contexts of the listen() and close() calls + * (+1, NULL outside a store): 'listening', 'connection' and 'error' run + * in the first, 'close' and the close() callbacks in the second, as + * Node runs them in its handle's and its close request's contexts. */ + ScrAlsCtx *listen_ctx, *close_ctx; int fd; /* -1 while not listening */ int port; bool listening; @@ -486,6 +491,14 @@ struct ScrNetServer { struct ScrNetSocket { int kind; /* FIRST: poller udata routes on it */ size_t rc; + /* The AsyncLocalStorage context of the socket's creation (+1, NULL + * outside a store): its I/O events ('connect', 'data', 'end', 'close', + * 'error', 'timeout') run in it, as Node runs them in its handle's + * context. An accepted socket has none, except under an HTTP or TLS + * server, whose sockets take the server's listen() context. end_ctx is + * the end() call's context, for 'finish' and the end() callback (Node's + * shutdown request). */ + ScrAlsCtx *als, *end_ctx; int fd; /* -1 once closed */ bool unrefd; /* socket.unref(): does not keep the loop alive */ bool connecting; @@ -668,6 +681,13 @@ static SCR_TL ScrPoller *scr_net_poller = NULL; static void scr_net_close_fd_raw(int fd); /* forget-then-close, defined below */ +/* The listen() call's context: the server's 'listening', 'connection' + * and 'error' events run in it. */ +static void scr_net_server_capture_listen(ScrNetServer *s) { + scr_als_ctx_release(s->listen_ctx); + s->listen_ctx = scr_als_capture(); +} + ScrNetServer *scr_net_server_retain(ScrNetServer *s) { if (s->rc != SIZE_MAX) s->rc++; return s; @@ -683,6 +703,8 @@ void scr_net_server_release(ScrNetServer *s) { scr_net_ls_drop(&s->listening_cbs); scr_net_ls_drop(&s->timeout_ls); scr_net_ls_drop(&s->drop_ls); + scr_als_ctx_release(s->listen_ctx); + scr_als_ctx_release(s->close_ctx); scr_closure_release(s->close_override); for (size_t i = 0; i < 5; i++) scr_dyn_release(s->http_timeout_dyn[i]); scr_dyn_release(s->drop_max_connection); @@ -721,6 +743,8 @@ void scr_net_sock_release(ScrNetSocket *s) { scr_net_ls_drop(&s->finish_ls); scr_net_ls_drop(&s->wcb_ls); scr_net_ls_drop(&s->ready_ls); + scr_als_ctx_release(s->als); + scr_als_ctx_release(s->end_ctx); scr_str_release(s->pending_err); scr_str_release(s->remote_cache); for (size_t i = 0; i < s->dial_n; i++) scr_str_release(s->dial_ips[i]); @@ -965,6 +989,7 @@ static ScrNetSocket *scr_net_sock_new(void) { s->kind = SCR_NET_K_SOCKET; s->rc = 1; s->fd = -1; + s->als = scr_als_capture(); #ifdef SCR_RC_AUDIT scr_net_live++; #endif @@ -1675,6 +1700,7 @@ ScrNetServer *scr_net_create_server(ScrClosure *handler /*moves, nullable*/, Scr * listen: IPv6 any with dual-stack when the kernel allows, IPv4 fallback. */ static void scr_net_listen_any(ScrNetServer *s, double port, bool reuse_port, ScrClosure *cb /*moves, nullable*/) { + scr_net_server_capture_listen(s); if (cb) scr_net_ls_add(&s->listening_cbs, cb, NULL, true); if (s->listening || s->close_emitted) return; if (!scr_net_poller_init()) { @@ -1753,6 +1779,7 @@ void scr_net_listen(ScrNetServer *s, double port, ScrClosure *cb /*moves, nullab static void scr_net_listen_opts_impl(ScrNetServer *s, double port, ScrStr *host /*borrowed*/, bool ipv6_only, bool reuse_port, ScrClosure *cb /*moves, nullable*/) { + scr_net_server_capture_listen(s); if (host == NULL || host->len == 0) { scr_net_listen_any(s, port, reuse_port, cb); return; @@ -1968,6 +1995,8 @@ void scr_net_server_close_direct(ScrNetServer *s, ScrClosure *cb /*moves, nullab scr_closure_release(cb); return; } + scr_als_ctx_release(s->close_ctx); + s->close_ctx = scr_als_capture(); if (cb) scr_net_ls_add(&s->close_ls, cb, NULL, true); if (s->listening) { scr_net_close_fd_raw(s->fd); /* stops accepting NOW (registrations dropped first) */ @@ -2225,6 +2254,13 @@ static void scr_net_server_accept(ScrNetServer *srv) { continue; } ScrNetSocket *sock = scr_net_sock_new(); + /* Node runs an accepted socket's own events with no store, except + * under the protocol layer (an HTTP or TLS server), whose connections + * run in the listen() context this accept runs in. */ + if (!srv->native_conn && !srv->defer_conn) { + scr_als_ctx_release(sock->als); + sock->als = NULL; + } sock->fd = fd; sock->server = scr_net_server_retain(srv); srv->nconns++; @@ -2776,6 +2812,7 @@ void scr_net_server_address_check(ScrNetServer *s) { * deferred 'error', success the deferred 'listening', as for a port. */ void scr_net_listen_path(ScrNetServer *s, ScrStr *path /*borrowed*/, ScrClosure *cb /*moves, nullable*/) { + scr_net_server_capture_listen(s); /* Node's isPipeName: a string that reads as a non-negative number is a port */ double port = scr_string_to_number(path); if (port >= 0) { @@ -3078,6 +3115,8 @@ void scr_net_sock_end(ScrNetSocket *s) { } if (s->fd < 0 || s->wr_ending) return; s->wr_ending = true; + scr_als_ctx_release(s->end_ctx); + s->end_ctx = scr_als_capture(); scr_net_sock_maybe_finish_write(s); if (s->rd_eof && s->wr_done) scr_net_sock_close_fd(s); } @@ -3175,6 +3214,62 @@ void scr_net_sock_on_finish(ScrNetSocket *s, ScrClosure *cb /*moves*/) { if (s->wr_done) s->finish_pending = true; /* already finished: next sweep */ } +/* The context the socket's events run in from now on (borrowed): an HTTP + * agent hands a pooled socket to another request. */ +void scr_net_sock_set_context(ScrNetSocket *s, ScrAlsCtx *ctx) { + ScrAlsCtx *next = scr_als_ctx_retain(ctx); + scr_als_ctx_release(s->als); + s->als = next; +} + +/* A write callback runs in the write() call's AsyncLocalStorage context + * (Node's write request): with a store active, the list holds a closure + * that enters it around the callback. caps[0] is the callback, caps[1] + * the context. */ +static void *scr_net_ctx_retain_v(void *c) { return scr_als_ctx_retain(c); } +static void scr_net_ctx_release_v(void *c) { scr_als_ctx_release(c); } + +static void scr_net_ctx_fire0(ScrClosure *self) { + ScrClosure *cb = scr_box_get_ref(self->caps[0]); + ScrAlsCtx *ctx = scr_box_get_ref(self->caps[1]); + ScrAlsCtx *prev = scr_als_dispatch_enter(ctx); + ((void (*)(ScrClosure *))cb->fn)(cb); + scr_als_dispatch_leave(prev); + scr_als_ctx_release(ctx); + scr_closure_release(cb); +} + +static void scr_net_dyn_fire_arg(ScrClosure *cb, const ScrDyn *arg); + +static void scr_net_ctx_fire_arg(ScrClosure *self, const ScrDyn *arg) { + ScrClosure *cb = scr_box_get_ref(self->caps[0]); + ScrAlsCtx *ctx = scr_box_get_ref(self->caps[1]); + ScrAlsCtx *prev = scr_als_dispatch_enter(ctx); + scr_net_dyn_fire_arg(cb, arg); + scr_als_dispatch_leave(prev); + scr_als_ctx_release(ctx); + scr_closure_release(cb); +} + +/* Adds a write callback (moves) with its firing adapter (NULL: a + * zero-argument closure; scr_net_dyn_fire_arg: a dyn listener). */ +static __attribute__((noinline)) ScrClosure *scr_net_bind_write_cb(ScrClosure *cb /*moves*/, void **fn, + ScrAlsCtx *active) { + ScrClosure *bound = scr_closure_new(*fn == NULL ? (void *)scr_net_ctx_fire0 : NULL, 2); + bound->caps[0] = scr_box_new(SCR_BOX_FUNC); + scr_box_set_ref(bound->caps[0], cb); + bound->caps[1] = scr_box_new_obj(scr_net_ctx_retain_v, scr_net_ctx_release_v, NULL); + scr_box_set_ref(bound->caps[1], scr_als_ctx_retain(active)); + if (*fn != NULL) *fn = (void *)&scr_net_ctx_fire_arg; + return bound; +} + +static void scr_net_sock_add_write_cb(ScrNetSocket *s, ScrClosure *cb, void *fn) { + ScrAlsCtx *active = *SCR_ALS_SLOT(); + if (active != NULL) cb = scr_net_bind_write_cb(cb, &fn, active); + scr_net_ls_add(&s->wcb_ls, cb, fn, true); +} + /* write(chunk, callback): fires when the write buffer drains (this * surface's flush moment), from the sweep. */ void scr_net_sock_on_write_flush(ScrNetSocket *s, ScrClosure *cb /*moves*/) { @@ -3182,7 +3277,7 @@ void scr_net_sock_on_write_flush(ScrNetSocket *s, ScrClosure *cb /*moves*/) { scr_closure_release(cb); return; } - scr_net_ls_add(&s->wcb_ls, cb, NULL, true); + scr_net_sock_add_write_cb(s, cb, NULL); } double scr_net_sock_bytes_written(ScrNetSocket *s) { return (double)s->bytes_written; } @@ -4584,8 +4679,8 @@ void scr_net_server_close_dyn(ScrNetServer *s, const ScrDyn *listener /*borrowed void scr_net_sock_on_write_flush_dyn(ScrNetSocket *s, const ScrDyn *listener /*borrowed*/) { if (s->close_emitted) return; - scr_net_ls_add(&s->wcb_ls, scr_dyn_listener_closure_fn(listener, (void *)&scr_net_dyn_fire_arg), - (void *)&scr_net_dyn_fire_arg, true); + scr_net_sock_add_write_cb(s, scr_dyn_listener_closure_fn(listener, (void *)&scr_net_dyn_fire_arg), + (void *)&scr_net_dyn_fire_arg); } /* socket.setTimeout(ms[, callback]) as the program calls it: Node's @@ -5382,7 +5477,9 @@ static void scr_net_sweep(void) { ((sock->tops && sock->t_est && sock->tops->pending && sock->tops->pending(sock->tctx)) || (!sock->tops && sock->rlen > sock->rhead))) { /* deliver bytes the kernel can't re-signal (see flags_pending) */ + ScrAlsCtx *prev = scr_als_dispatch_enter(sock->als); scr_net_sock_read(sock); + scr_als_dispatch_leave(prev); if (scr_exc_pending()) { scr_net_sock_release(sock); return; @@ -5404,14 +5501,18 @@ static void scr_net_sweep(void) { } if (sock->finish_pending) { sock->finish_pending = false; + ScrAlsCtx *prev = scr_als_dispatch_enter(sock->end_ctx); scr_net_fire0_this(&sock->finish_ls, sock, SCR_DYNH_NET_SOCKET); + scr_als_dispatch_leave(prev); if (scr_exc_pending()) { scr_net_sock_release(sock); return; } } if (sock->lookup_evs != NULL && sock->pending_err) { + ScrAlsCtx *prev = scr_als_dispatch_enter(sock->als); scr_net_lookup_ops->flush(sock); + scr_als_dispatch_leave(prev); if (scr_exc_pending()) { scr_net_sock_release(sock); return; @@ -5425,6 +5526,7 @@ static void scr_net_sweep(void) { * teardown without an event); unconsumed errors fall through to the socket's own * listeners — and the unhandled-'error' exit */ bool consumed = false; + ScrAlsCtx *prev = scr_als_dispatch_enter(sock->als); if (sock->native_err) consumed = sock->native_err(sock->native_ctx, msg); if (!consumed && !scr_exc_pending()) { ScrError *coded = NULL; @@ -5438,6 +5540,7 @@ static void scr_net_sweep(void) { scr_event_error_payload = saved; if (coded != NULL) scr_error_release(coded); } + scr_als_dispatch_leave(prev); sock->pending_code = NULL; scr_str_release(msg); if (scr_exc_pending()) { @@ -5446,7 +5549,10 @@ static void scr_net_sweep(void) { } } if (sock->emit_close && !sock->close_emitted && sock->fd < 0) { - if (!scr_net_sock_settle(sock)) { + ScrAlsCtx *prev = scr_als_dispatch_enter(sock->als); + bool settled = scr_net_sock_settle(sock); + scr_als_dispatch_leave(prev); + if (!settled) { scr_net_sock_release(sock); return; } @@ -5460,7 +5566,9 @@ static void scr_net_sweep(void) { scr_net_server_retain(srv); if (srv->emit_listening) { srv->emit_listening = false; + ScrAlsCtx *prev = scr_als_dispatch_enter(srv->listen_ctx); scr_net_fire0_this(&srv->listening_cbs, srv, SCR_DYNH_NET_SERVER); + scr_als_dispatch_leave(prev); if (scr_exc_pending()) { scr_net_server_release(srv); return; @@ -5469,7 +5577,9 @@ static void scr_net_sweep(void) { if (srv->pending_err) { ScrStr *msg = srv->pending_err; srv->pending_err = NULL; + ScrAlsCtx *prev = scr_als_dispatch_enter(srv->listen_ctx); scr_net_fire_err_this(&srv->err_ls, msg, srv, SCR_DYNH_NET_SERVER); + scr_als_dispatch_leave(prev); scr_str_release(msg); /* A failed listen SETTLES the handle: listeners drop (an error * listener capturing its own server would otherwise cycle forever — @@ -5505,7 +5615,9 @@ static void scr_net_sweep(void) { } if (due == NULL) return; scr_net_server_retain(due); + ScrAlsCtx *prev = scr_als_dispatch_enter(due->close_ctx); scr_net_server_settle(due); + scr_als_dispatch_leave(prev); scr_net_server_release(due); if (scr_exc_pending()) return; } @@ -5539,6 +5651,69 @@ bool scr_loop_io_take(void); bool scr_loop_io_more(void); void scr_loop_io_reopen(void); +/* One poller event of a socket; false with an exception pending. */ +static bool scr_net_sock_event(ScrNetSocket *s, unsigned events) { + if (events & SCRP_TIMER) { + /* the idle period elapsed: 'timeout' fires once (one-shot — the + * next activity re-arms); the socket stays open, like Node */ + s->timeout_armed = false; + if (s->native_timeout) { + s->native_timeout(s->native_ctx); + if (scr_exc_pending()) return false; + } + if (s->server && s->server->http_timeout_surface && + s->native_http_timeout) { + ScrNetServer *srv = scr_net_server_retain(s->server); + bool handled = s->native_http_timeout(s->native_ctx); + if (scr_exc_pending()) { + scr_net_server_release(srv); + return false; + } + bool server_handled = srv->timeout_ls.n > 0; + if (server_handled) { + ScrNetL *snap; + size_t count = scr_net_ls_snapshot(&srv->timeout_ls, &snap); + scr_dyn_this_push(srv, SCR_DYNH_NET_SERVER); + for (size_t j = 0; j < count; j++) { + if (!scr_exc_pending()) ((ScrNetConnFn)snap[j].fn)(snap[j].cb, scr_net_sock_retain(s)); + scr_closure_release(snap[j].cb); + } + scr_dyn_this_pop(); + free(snap); + } + if (!handled && !server_handled && !scr_exc_pending()) + scr_net_sock_destroy(s); /* Node's unhandled server timeout */ + scr_net_server_release(srv); + if (scr_exc_pending()) return false; + } + scr_net_fire0_this(&s->timeout_ls, s, SCR_DYNH_NET_SOCKET); + if (scr_exc_pending()) return false; + if (s->fd < 0) return true; + } + /* kqueue delivers one direction per event (today's exact order); + * epoll may coalesce both into one — writes first (connect + * completion, flushes), then reads, re-checking the fd a write + * failure may have closed. */ + if (events & SCRP_WRITABLE) { + if (s->connecting) scr_net_sock_connect_done(s); + else if (s->tunnel != NULL) scr_net_sock_tunnel_pump(s); + else if (s->tops && !s->t_est) { + s->t_want_write = false; + scr_net_sock_transport_pump(s); + if (!scr_exc_pending() && s->fd >= 0) scr_net_sock_update_write(s); + } else { + s->t_want_write = false; + scr_net_sock_flush(s); + } + } + if ((events & SCRP_READABLE) && !scr_exc_pending() && s->fd >= 0) { + if (s->tunnel != NULL) scr_net_sock_tunnel_pump(s); + else if (s->tops && !s->t_est) scr_net_sock_transport_pump(s); + else scr_net_sock_read(s); + } + return true; +} + static void scr_net_dispatch(void) { if (!scr_net_servers && !scr_net_socks && !scr_net_proto_pending()) { @@ -5564,71 +5739,22 @@ static void scr_net_dispatch(void) { if (!udata) continue; int kind = *(int *)udata; if (kind == SCR_NET_K_SERVER) { - scr_net_server_accept((ScrNetServer *)udata); + /* 'connection' runs in the context of the listen() call. */ + ScrNetServer *srv = (ScrNetServer *)udata; + ScrAlsCtx *prev = scr_als_dispatch_enter(srv->listen_ctx); + scr_net_server_accept(srv); + scr_als_dispatch_leave(prev); } else if (kind == SCR_NET_K_SOCKET) { ScrNetSocket *s = (ScrNetSocket *)udata; /* A callback earlier in this batch may have closed this fd * (destroy inside a data listener) — its remaining events are * stale; the handle itself stays alive until the sweep. */ if (s->fd < 0) continue; - if (evs[i].events & SCRP_TIMER) { - /* the idle period elapsed: 'timeout' fires once (one-shot — the - * next activity re-arms); the socket stays open, like Node */ - s->timeout_armed = false; - if (s->native_timeout) { - s->native_timeout(s->native_ctx); - if (scr_exc_pending()) return; - } - if (s->server && s->server->http_timeout_surface && - s->native_http_timeout) { - ScrNetServer *srv = scr_net_server_retain(s->server); - bool handled = s->native_http_timeout(s->native_ctx); - if (scr_exc_pending()) { - scr_net_server_release(srv); - return; - } - bool server_handled = srv->timeout_ls.n > 0; - if (server_handled) { - ScrNetL *snap; - size_t count = scr_net_ls_snapshot(&srv->timeout_ls, &snap); - scr_dyn_this_push(srv, SCR_DYNH_NET_SERVER); - for (size_t j = 0; j < count; j++) { - if (!scr_exc_pending()) ((ScrNetConnFn)snap[j].fn)(snap[j].cb, scr_net_sock_retain(s)); - scr_closure_release(snap[j].cb); - } - scr_dyn_this_pop(); - free(snap); - } - if (!handled && !server_handled && !scr_exc_pending()) - scr_net_sock_destroy(s); /* Node's unhandled server timeout */ - scr_net_server_release(srv); - if (scr_exc_pending()) return; - } - scr_net_fire0_this(&s->timeout_ls, s, SCR_DYNH_NET_SOCKET); - if (scr_exc_pending()) return; - if (s->fd < 0) continue; - } - /* kqueue delivers one direction per event (today's exact order); - * epoll may coalesce both into one — writes first (connect - * completion, flushes), then reads, re-checking the fd a write - * failure may have closed. */ - if (evs[i].events & SCRP_WRITABLE) { - if (s->connecting) scr_net_sock_connect_done(s); - else if (s->tunnel != NULL) scr_net_sock_tunnel_pump(s); - else if (s->tops && !s->t_est) { - s->t_want_write = false; - scr_net_sock_transport_pump(s); - if (!scr_exc_pending() && s->fd >= 0) scr_net_sock_update_write(s); - } else { - s->t_want_write = false; - scr_net_sock_flush(s); - } - } - if ((evs[i].events & SCRP_READABLE) && !scr_exc_pending() && s->fd >= 0) { - if (s->tunnel != NULL) scr_net_sock_tunnel_pump(s); - else if (s->tops && !s->t_est) scr_net_sock_transport_pump(s); - else scr_net_sock_read(s); - } + /* The socket's events run in its AsyncLocalStorage context. */ + ScrAlsCtx *prev = scr_als_dispatch_enter(s->als); + bool ok = scr_net_sock_event(s, evs[i].events); + scr_als_dispatch_leave(prev); + if (!ok) return; } if (scr_exc_pending()) return; } diff --git a/packages/runtime/src/scr_net.h b/packages/runtime/src/scr_net.h index ffa9b2f72..d31168a33 100644 --- a/packages/runtime/src/scr_net.h +++ b/packages/runtime/src/scr_net.h @@ -36,6 +36,10 @@ typedef struct { } ScrNetLs; void scr_net_ls_add(ScrNetLs *l, ScrClosure *cb, void *fn, bool once); +/* The AsyncLocalStorage context a socket's events run in from now on + * (borrowed; scr_net.c keeps the creation context otherwise). */ +struct ScrAlsCtx; +void scr_net_sock_set_context(ScrNetSocket *s, struct ScrAlsCtx *ctx); ScrNetOnce *scr_net_once_new(void); /* +1 */ ScrNetOnce *scr_net_once_retain(ScrNetOnce *once); /* +1 */ void scr_net_once_release(ScrNetOnce *once); diff --git a/tests/corpus/7119-als-net-event-context.mjs b/tests/corpus/7119-als-net-event-context.mjs new file mode 100644 index 000000000..3fb95ffa8 --- /dev/null +++ b/tests/corpus/7119-als-net-event-context.mjs @@ -0,0 +1,124 @@ +// AsyncLocalStorage contexts of node:net events. A client socket's events +// run in the context of the net.connect() call, whatever context the +// listener was registered in, also after a destroy() from elsewhere. The +// server's 'listening', 'connection' and 'error' events run in the +// context of the listen() call. An accepted socket's own events run with +// no store. write() and end() callbacks, and 'finish', run in the context +// of their call. server.close(callback) runs in the close() call's +// context when no connection is open, and with no store when it waits for +// the last connection. Lines are sorted at the end: events on different +// sockets may interleave. +import { AsyncLocalStorage } from "node:async_hooks"; +import net from "node:net"; + +const store = new AsyncLocalStorage(); +const lines = []; +const log = (what) => lines.push(`${what}: ${store.getStore() ?? "none"}`); + +function echoScenario() { + return new Promise((resolve) => { + const server = store.run("create", () => + net.createServer((sock) => { + log("server connection callback"); + sock.on("data", () => { + log("accepted data"); + store.run("in-data", () => sock.write("pong")); + }); + sock.on("end", () => { + log("accepted end"); + sock.end(); + }); + sock.on("close", () => log("accepted close")); + }), + ); + server.on("connection", () => log("server connection event")); + server.on("close", () => { + log("server close event"); + resolve(); + }); + store.run("listen", () => + server.listen(0, () => { + log("listen callback"); + const client = store.run("connect", () => net.connect(server.address().port)); + store.run("register", () => { + client.on("connect", () => { + log("client connect"); + store.run("write", () => client.write("ping", () => log("write callback"))); + }); + client.on("data", () => { + log("client data"); + store.run("end-call", () => client.end(() => log("end callback"))); + }); + client.on("finish", () => log("client finish")); + client.on("end", () => log("client end")); + client.on("close", () => { + log("client close"); + store.run("close-call", () => server.close(() => log("server close callback"))); + }); + }); + }), + ); + }); +} + +function destroyScenario() { + return new Promise((resolve) => { + const server = net.createServer((sock) => sock.on("data", () => {})); + server.listen(0, () => { + const client = store.run("connect-2", () => net.connect(server.address().port)); + client.on("error", () => {}); + client.on("connect", () => { + client.on("close", () => { + log("close after a destroy elsewhere"); + server.close(() => resolve()); + }); + setTimeout(() => store.run("destroy-call", () => client.destroy()), 10); + }); + }); + }); +} + +function closeScenarios() { + return new Promise((resolve) => { + let pending = 3; + const done = () => { + if (--pending === 0) resolve(); + }; + const waiting = net.createServer((sock) => { + store.run("conn", () => waiting.close(() => { + log("close callback that waited"); + done(); + })); + setTimeout(() => store.run("end", () => sock.end()), 20); + }); + store.run("listen-3", () => + waiting.listen(0, () => { + const client = net.connect(waiting.address().port); + client.on("end", () => client.end()); + }), + ); + const idle = net.createServer(); + store.run("listen-4", () => + idle.listen(0, () => { + store.run("close-now", () => idle.close(() => { + log("close callback of an idle server"); + done(); + })); + }), + ); + const taken = net.createServer(); + taken.listen(0, () => { + const clash = net.createServer(); + clash.on("error", (e) => { + log(`listen error ${e.code}`); + taken.close(() => done()); + }); + store.run("clash-listen", () => clash.listen(taken.address().port)); + }); + }); +} + +await echoScenario(); +await destroyScenario(); +await closeScenarios(); +for (const line of lines.sort()) console.log(line); diff --git a/tests/corpus/7380-als-http-event-context.mjs b/tests/corpus/7380-als-http-event-context.mjs new file mode 100644 index 000000000..a881cbe7e --- /dev/null +++ b/tests/corpus/7380-als-http-event-context.mjs @@ -0,0 +1,151 @@ +// AsyncLocalStorage contexts of node:http, node:https and node:tls events. +// A server's 'request' event and its requests' and responses' events run +// in the context of the server's listen() call. A client request's +// events, its response callback and its response's events run in the +// context of the http.request() call, also on a keep-alive socket that +// an earlier request used. TLS client sockets run in the tls.connect() +// call's context, and a TLS server's connections in its listen() +// context. Lines are sorted at the end: events on different sockets may +// interleave. +import { AsyncLocalStorage } from "node:async_hooks"; +import * as fs from "node:fs"; +import http from "node:http"; +import https from "node:https"; +import tls from "node:tls"; + +const store = new AsyncLocalStorage(); +const lines = []; +const log = (what) => lines.push(`${what}: ${store.getStore() ?? "none"}`); + +function httpScenario() { + return new Promise((resolve) => { + const server = store.run("create", () => + http.createServer((req, res) => { + log("request callback"); + req.on("data", () => log("request data")); + req.on("end", () => { + log("request end"); + res.end("ok"); + }); + res.on("finish", () => log("response finish")); + res.on("close", () => log("response close")); + }), + ); + server.on("request", () => log("request event")); + store.run("listen", () => + server.listen(0, () => { + const port = server.address().port; + const req = store.run("request-call", () => + http.request({ port, method: "POST" }, (res) => { + log("response callback"); + res.on("data", () => log("client response data")); + res.on("end", () => log("client response end")); + }), + ); + store.run("register", () => { + req.on("socket", () => log("client socket")); + req.on("finish", () => log("client finish")); + req.on("close", () => { + log("client close"); + server.close(() => resolve()); + }); + }); + store.run("end-call", () => req.end("body")); + }), + ); + }); +} + +function keepAliveScenario() { + return new Promise((resolve) => { + const agent = new http.Agent({ keepAlive: true, maxSockets: 1 }); + const server = http.createServer((req, res) => res.end("ok")); + server.listen(0, () => { + const port = server.address().port; + let done = 0; + for (const name of ["first", "second"]) { + store.run(name, () => { + const req = http.get({ port, agent }, (res) => { + log(`${name} response`); + res.on("data", () => {}); + res.on("end", () => { + log(`${name} response end`); + if (++done === 2) { + agent.destroy(); + server.close(() => resolve()); + } + }); + }); + req.on("finish", () => log(`${name} finish`)); + req.on("close", () => log(`${name} close`)); + }); + } + }); + }); +} + +const key = fs.readFileSync("tests/fixtures/server/certs/localhost-key.pem"); +const cert = fs.readFileSync("tests/fixtures/server/certs/localhost.pem"); +const ca = fs.readFileSync("tests/fixtures/server/certs/ca.pem", "utf8"); + +function httpsScenario() { + return new Promise((resolve) => { + const server = https.createServer({ key, cert }, (req, res) => { + log("https request"); + req.resume(); + res.on("finish", () => log("https response finish")); + res.end("ok"); + }); + store.run("https-listen", () => + server.listen(0, () => { + const port = server.address().port; + store.run("https-get", () => + https + .get({ port, host: "localhost", ca }, (res) => { + log("https response"); + res.on("data", () => {}); + res.on("end", () => log("https response end")); + }) + .on("close", () => { + log("https client close"); + server.close(() => resolve()); + }), + ); + }), + ); + }); +} + +function tlsScenario() { + return new Promise((resolve) => { + const server = tls.createServer({ key, cert }, (sock) => { + log("tls secureConnection"); + sock.on("data", () => { + log("tls server data"); + sock.end("bye"); + }); + }); + store.run("tls-listen", () => + server.listen(0, () => { + const client = store.run("tls-connect", () => + tls.connect({ port: server.address().port, host: "localhost", ca }, () => { + log("tls secureConnect"); + client.write("hi"); + }), + ); + client.on("data", () => log("tls client data")); + client.on("end", () => log("tls client end")); + client.on("close", () => { + log("tls client close"); + server.close(() => resolve()); + }); + }), + ); + }); +} + +await httpScenario(); +await keepAliveScenario(); +await httpsScenario(); +await tlsScenario(); +for (const line of lines.sort()) console.log(line); diff --git a/tests/harness/corpus-costs/7119-als-net-event-context.mjs.json b/tests/harness/corpus-costs/7119-als-net-event-context.mjs.json new file mode 100644 index 000000000..a57dddb65 --- /dev/null +++ b/tests/harness/corpus-costs/7119-als-net-event-context.mjs.json @@ -0,0 +1 @@ +{ "plain": 4500, "san": 12000 } diff --git a/tests/harness/corpus-costs/7380-als-http-event-context.mjs.json b/tests/harness/corpus-costs/7380-als-http-event-context.mjs.json new file mode 100644 index 000000000..a57dddb65 --- /dev/null +++ b/tests/harness/corpus-costs/7380-als-http-event-context.mjs.json @@ -0,0 +1 @@ +{ "plain": 4500, "san": 12000 } From 84cf0ec19fd9288034678b8ce9aa5c09fcdba916 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 10:54:02 -0500 Subject: [PATCH 005/184] Inspect dynamic classes, functions and %o arguments as Node does - The inspector throws the 'not supported yet' Error for checked-dynamic values it does not render (a promise, a URL, a RegExp reached through `unknown`). Its calls were not marked as throwing, so `console.log("x", p)` printed "x " and the program went on; records and arrays with such a field printed `{ a: }` and `[ ]`. insp.dyn and insp.dynS are now may-throw library calls, so the Error propagates from the call that inspects the value, before anything is written. - A class reached through `unknown` prints `[class C]`, `[class C extends B]` or `[class (anonymous)]` instead of `[Function: C]`. Functions and classes reached as dynamic values print their enumerable own properties (`[Function: f] { x: 1 }`, `[class S extends A] { count: 3 }`), cycles through them as `[Circular *N]`, and `[Function]` past the depth limit. Class objects carry their base class's name. A compiled class's static fields are module globals that the class object does not carry, so inspecting a class that declares static fields through `unknown` throws the 'not supported yet' Error. - util.format's %o renders a checked-dynamic argument with showHidden: an array's `[length]` entry and an object's non-enumerable own properties, string- and symbol-keyed, in brackets. An array with more than 100 output entries keeps the last one out of the grid, as Node's groupArrayElements does. %o of a function, an Error, a typed array or a native handle other than Map, Set and Date throws the 'not supported yet' Error. ScrClassObj has two new fields, so RUNTIME_ABI_VERSION is now 27. --- packages/compiler/src/backend/llvm/classes.ts | 10 +- .../compiler/src/backend/llvm/emit-module.ts | 2 +- packages/compiler/src/backend/llvm/emitter.ts | 9 +- packages/compiler/src/backend/llvm/layouts.ts | 2 +- .../compiler/src/backend/llvm/lib-shared.ts | 1 + .../src/frontend/lowering/lower-classes.ts | 8 + .../src/frontend/lowering/lower-console.ts | 3 +- .../src/frontend/lowering/lower-inspect.ts | 32 ++- packages/compiler/src/ir/builtin-effects.ts | 7 + .../compiler/src/ir/builtin-signatures.ts | 4 + packages/compiler/src/ir/int32-slots.test.ts | 1 + packages/compiler/src/ir/int32-slots.ts | 1 + packages/compiler/src/ir/ir.ts | 4 + .../corpus/6343-inspect-class-values.ts.json | 6 + .../6344-inspect-function-properties.cjs.json | 6 + .../tests/corpus/6357-format-o-hidden.ts.json | 6 + .../corpus/6358-format-o-hidden-js.cjs.json | 6 + packages/runtime/src/scr_error.h | 6 + packages/runtime/src/scr_inspect.c | 187 +++++++++++++++--- packages/runtime/src/scr_inspect.h | 1 + packages/runtime/src/scr_object.c | 2 + tests/corpus/6343-inspect-class-values.ts | 31 +++ .../6344-inspect-function-properties.cjs | 34 ++++ tests/corpus/6357-format-o-hidden.ts | 19 ++ tests/corpus/6358-format-o-hidden-js.cjs | 9 + tests/fixtures/self-hosting/llvm-layouts.ts | 3 +- .../6343-inspect-class-values.ts.json | 1 + .../6344-inspect-function-properties.cjs.json | 1 + .../corpus-costs/6357-format-o-hidden.ts.json | 1 + .../6358-format-o-hidden-js.cjs.json | 1 + tests/harness/dyncheck.test.ts | 88 +++++++++ 31 files changed, 447 insertions(+), 45 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6343-inspect-class-values.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6344-inspect-function-properties.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6357-format-o-hidden.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6358-format-o-hidden-js.cjs.json create mode 100644 tests/corpus/6343-inspect-class-values.ts create mode 100644 tests/corpus/6344-inspect-function-properties.cjs create mode 100644 tests/corpus/6357-format-o-hidden.ts create mode 100644 tests/corpus/6358-format-o-hidden-js.cjs create mode 100644 tests/harness/corpus-costs/6343-inspect-class-values.ts.json create mode 100644 tests/harness/corpus-costs/6344-inspect-function-properties.cjs.json create mode 100644 tests/harness/corpus-costs/6357-format-o-hidden.ts.json create mode 100644 tests/harness/corpus-costs/6358-format-o-hidden-js.cjs.json diff --git a/packages/compiler/src/backend/llvm/classes.ts b/packages/compiler/src/backend/llvm/classes.ts index 3073bd1bd..8b20ec264 100644 --- a/packages/compiler/src/backend/llvm/classes.ts +++ b/packages/compiler/src/backend/llvm/classes.ts @@ -1363,18 +1363,18 @@ export function emitClassShapes( * statics plus their construct thunks — allocate, run the constructor over * a +1 `this`, and hand the remaining +1 out. No pending check is needed: * in-tier constructors are throw-free by construction. - * `classObjs` maps className → { nameSym } registered during body emission - * (the literal interned while the table was open). */ + * `classObjs` maps className → { nameSym, baseNameSym } registered during + * body emission (the literals interned while the table was open). */ export function emitClassObjDefs( host: ClassHost, metaMap: Map, - classObjs: Map, + classObjs: Map, fnByName: Map, llType: (t: IrType) => string, threadLocal = false, ): string[] { const out: string[] = []; - for (const [className, { nameSym }] of classObjs) { + for (const [className, { nameSym, baseNameSym }] of classObjs) { const meta = metaMap.get(className); if (!meta) throw new InternalCompilerError( @@ -1400,7 +1400,7 @@ export function emitClassObjDefs( const ctorArgs = params.map((p, i) => `${llType(p.type)} %a${i}`); const classObj = `@${mangleClassObj(className)} = internal ${threadLocal ? "thread_local " : ""}global %ScrClassObj ` + - `{ ${host.sizeType} -1, ${host.sizeType} ${intervalMeta.pre}, ${host.sizeType} ${intervalMeta.post}, ptr @${mangleCtorThunk(className)}, ptr ${nameSym}, ${host.sizeType} 0, ${host.sizeType} ${meta.def.jsLength ?? (meta.def.runtime ? 1 : 0)}, ptr null, ptr null } ; class ${className}`; + `{ ${host.sizeType} -1, ${host.sizeType} ${intervalMeta.pre}, ${host.sizeType} ${intervalMeta.post}, ptr @${mangleCtorThunk(className)}, ptr ${nameSym}, ${host.sizeType} 0, ${host.sizeType} ${meta.def.jsLength ?? (meta.def.runtime ? 1 : 0)}, ptr null, ptr null, ptr ${baseNameSym ?? "null"}, ${host.sizeType} ${meta.def.ownStaticFields ? 1 : 0} } ; class ${className}`; if (meta.def.runtime) { // A runtime-provided class: %C.construct allocates through the // runtime and returns the instance. diff --git a/packages/compiler/src/backend/llvm/emit-module.ts b/packages/compiler/src/backend/llvm/emit-module.ts index 3d9d41072..8da4bb140 100644 --- a/packages/compiler/src/backend/llvm/emit-module.ts +++ b/packages/compiler/src/backend/llvm/emit-module.ts @@ -483,7 +483,7 @@ export function emitModuleParts(emitter: LlEmitter): string[] { // class-object shape { rc, pre, post, ctor, name } — field reads on // builtin errors and classval loads GEP through these. `%ScrError = type { ${emitter.sizeType}, ptr, ptr, ptr, ptr, ptr, i8, i8, i8, i8, i8, ptr, ptr }`, - `%ScrClassObj = type { ${emitter.sizeType}, ${emitter.sizeType}, ${emitter.sizeType}, ptr, ptr, ${emitter.sizeType}, ${emitter.sizeType}, ptr, ptr }`, + `%ScrClassObj = type { ${emitter.sizeType}, ${emitter.sizeType}, ${emitter.sizeType}, ptr, ptr, ${emitter.sizeType}, ${emitter.sizeType}, ptr, ptr, ptr, ${emitter.sizeType} }`, // The runtime emitter prefix { rc, vt, reg, cls } — user subclasses // embed it (classes.ts), and bare-emitter GEPs address through it. `%ScrEmitter = type { ${emitter.sizeType}, ptr, ptr, ptr }`, diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index bfde59a30..0b4ce184a 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -1000,7 +1000,7 @@ export class LlEmitter { /** Class objects (classes as first-class values): className → the * interned .name literal ref — registered during body emission, the * statics and construct thunks assemble around the bodies. */ - readonly classObjs = new Map(); + readonly classObjs = new Map(); private readonly unionFieldReadGroups = new Map(); /** Preorder intervals of the runtime error classes under THIS module's * class-forest numbering (main() stamps scr_error_vts with them, exactly @@ -3638,7 +3638,12 @@ export class LlEmitter { classObjSym(className: string): string { if (!this.classObjs.has(className)) { const meta = this.classMetaOf(className); - this.classObjs.set(className, { nameSym: this.internLiteral(classJsName(meta.def)) }); + // util.inspect prints `[class C extends B]` with the base's name. + const baseName = meta.base ? classJsName(meta.base.def) : ""; + this.classObjs.set(className, { + nameSym: this.internLiteral(classJsName(meta.def)), + baseNameSym: baseName ? this.internLiteral(baseName) : null, + }); } return mangleClassObj(className); } diff --git a/packages/compiler/src/backend/llvm/layouts.ts b/packages/compiler/src/backend/llvm/layouts.ts index 8e1f0a4c1..5c426e910 100644 --- a/packages/compiler/src/backend/llvm/layouts.ts +++ b/packages/compiler/src/backend/llvm/layouts.ts @@ -17,7 +17,7 @@ export function emitLlvmLayouts( host: ClassHost, mod: IrModule, classes: Map, - classObjects: Map, + classObjects: Map, functions: Map, llType: (type: IrType) => string, nullable?: NullableRefFields, diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 9a65d9a24..be6595658 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1029,6 +1029,7 @@ export const LIB_FN_SYMS: Record = { "dyn.generatorThis": "scr_gen_receiver", "insp.dyn": "scr_insp_dyn", "insp.dynS": "scr_insp_dyn_s", + "insp.dynHidden": "scr_insp_dyn_hidden", "fs.readFileSyncDyn": "scr_fs_read_file_sync_dyn", "fs.readFileEncoded": "scr_fs_read_file_encoded", "fs.readFdEncoded": "scr_fs_read_fd_encoded", diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index d8ccc15b7..3c294bb66 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -4395,6 +4395,14 @@ export function collectClassShapeInner( // INSTANTIATION prints its family's name — JS has one `Box`. jsName: jsNameOverride ?? decl.name?.text ?? "", ...(base ? { base: base.def.name } : {}), + ...(decl.members.nodes.some( + (member) => + ts.isPropertyDeclaration(member) && + !ts.isPrivateIdentifier(member.name) && + member.modifiers?.nodes.some((m) => m.kind === ts.SyntaxKind.StaticKeyword), + ) + ? { ownStaticFields: true as const } + : {}), jsLength: ctor?.parameters?.nodes.findIndex( (parameter) => parameter.initializer || parameter.dotDotDotToken, diff --git a/packages/compiler/src/frontend/lowering/lower-console.ts b/packages/compiler/src/frontend/lowering/lower-console.ts index eec308342..b45218bd0 100644 --- a/packages/compiler/src/frontend/lowering/lower-console.ts +++ b/packages/compiler/src/frontend/lowering/lower-console.ts @@ -249,7 +249,8 @@ export function lowerConsoleArg( // inspect at the rest-args depth 2 (formatWithOptions) — scalar // kinds byte-exactly, boxed functions as [Function: name] / // [Function (anonymous)], composites through the dyn walk - // (insp.dyn). Never throws — Node's console.log never does. + // (insp.dyn). A value the inspector does not render throws the + // 'not supported yet' Error before anything is written. if (lowered.type.kind === "dyn") { return { kind: "libCall", diff --git a/packages/compiler/src/frontend/lowering/lower-inspect.ts b/packages/compiler/src/frontend/lowering/lower-inspect.ts index 47f490c7b..71a7ea327 100644 --- a/packages/compiler/src/frontend/lowering/lower-inspect.ts +++ b/packages/compiler/src/frontend/lowering/lower-inspect.ts @@ -2607,10 +2607,21 @@ function formatSArg(lowerer: Lowerer, node: ts.Expression, depth: number, loc: S } /** %O (depth 2) / %o (showHidden, depth 4) — inspect with the spec's - * option deltas; %o fences over arrays (showHidden adds their hidden - * [length] entry, which has no lowering). */ + * option deltas. A checked-dynamic argument renders through the runtime's + * showHidden walk; static %o fences over arrays and nested dynamic values + * (showHidden adds an array's hidden [length] entry, which the static + * rendering has no lowering for). */ function formatOArg(lowerer: Lowerer, node: ts.Expression, depth: number, loc: SrcLoc): IrExpr { const value = lowerer.lowerExpr(node); + if (depth === 4 && value.type.kind === "dyn") { + return { + kind: "libCall", + fn: "insp.dynHidden", + args: [value, numLit(4, loc)], + type: STRING, + loc, + }; + } if (value.type.kind === "bytes") { lowerer.noLowering( `util.format %${depth === 4 ? "o" : "O"} of typed-array values`, @@ -2626,11 +2637,11 @@ function formatOArg(lowerer: Lowerer, node: ts.Expression, depth: number, loc: S node, reason, ); - if (depth === 4 && typeTreeHasArray(lowerer, value.type, new Set())) { + if (depth === 4 && typeTreeHasHidden(lowerer, value.type, new Set())) { lowerer.noLowering( "util.format %o over arrays", node, - "%o renders with showHidden (arrays gain a hidden [length] entry) — use %O for the default rendering", + "%o renders with showHidden (arrays gain a hidden [length] entry) — use %O for the default rendering, or pass the value as unknown", ); } return inspectExpr(lowerer, value.type, value, numLit(0, loc), numLit(depth, loc), loc); @@ -3031,35 +3042,36 @@ export function lowerFormatCall( } /** Arrays anywhere in the tree — the %o (showHidden) gate. */ -function typeTreeHasArray(lowerer: Lowerer, t: IrType, visiting: Set): boolean { +function typeTreeHasHidden(lowerer: Lowerer, t: IrType, visiting: Set): boolean { switch (t.kind) { case "array": + case "dyn": return true; case "record": { if (visiting.has(t.shapeId)) return false; visiting.add(t.shapeId); const shape = lowerer.shapes.get(t.shapeId); if (shape?.tuple) return true; - return (shape?.fields ?? []).some((f) => typeTreeHasArray(lowerer, f.type, visiting)); + return (shape?.fields ?? []).some((f) => typeTreeHasHidden(lowerer, f.type, visiting)); } case "map": return ( - typeTreeHasArray(lowerer, t.key, visiting) || typeTreeHasArray(lowerer, t.value, visiting) + typeTreeHasHidden(lowerer, t.key, visiting) || typeTreeHasHidden(lowerer, t.value, visiting) ); case "set": - return typeTreeHasArray(lowerer, t.elem, visiting); + return typeTreeHasHidden(lowerer, t.elem, visiting); case "union": { if (visiting.has(t.unionId)) return false; visiting.add(t.unionId); return (lowerer.unions.get(t.unionId)?.arms ?? []).some((a) => - typeTreeHasArray(lowerer, a, visiting), + typeTreeHasHidden(lowerer, a, visiting), ); } case "object": { if (visiting.has(t.className)) return false; visiting.add(t.className); return (lowerer.classes.get(t.className)?.def.fields ?? []).some((f) => - typeTreeHasArray(lowerer, f.type, visiting), + typeTreeHasHidden(lowerer, f.type, visiting), ); } case "bytes": diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index 16db45b6c..229fd8c28 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -174,6 +174,13 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "num.toStringRadix", "num.toStringRadixF64", "insp.jsonDyn", + // A checked-dynamic value the inspector does not render (a promise, a + // URL, a RegExp) throws the 'not supported yet' Error. + "insp.dyn", + "insp.dynS", + // %o's showHidden walk refuses the kinds whose hidden entries it cannot + // render. + "insp.dynHidden", // diagnostics_channel: publish runs subscribers synchronously (a throw // propagates — the documented divergence from triggerUncaughtException); // the subscribe/unsubscribe forms throw Node's ERR_INVALID_ARG_TYPE for diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index fcdbe8694..9f224c72a 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -4290,6 +4290,10 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "insp.errorCause": { argTypes: [null, F64, F64], result: STRING }, "insp.dyn": { argTypes: [DYN, F64, F64], result: STRING }, "insp.dynS": { argTypes: [DYN, F64], result: STRING }, + /** util.format's %o over a checked-dynamic value: inspect with + * showHidden. Kinds whose hidden entries have no rendering throw the + * 'not supported yet' Error. */ + "insp.dynHidden": { argTypes: [DYN, F64], result: STRING }, "insp.begin": { argTypes: [F64], result: VOID }, /** util.inspect's breakLength, compact (0 for false) and sorted for the * renders that follow; the call site restores 80, 3 and false afterwards. */ diff --git a/packages/compiler/src/ir/int32-slots.test.ts b/packages/compiler/src/ir/int32-slots.test.ts index be933ab38..f64e2de1f 100644 --- a/packages/compiler/src/ir/int32-slots.test.ts +++ b/packages/compiler/src/ir/int32-slots.test.ts @@ -281,6 +281,7 @@ test("dynamic stores disqualify the field names they can reach", () => { "dyn.typeof", "insp.dyn", "insp.dynS", + "insp.dynHidden", "dyn.objCreate", "dyn.getPrototype", "dyn.nameClassPrototype", diff --git a/packages/compiler/src/ir/int32-slots.ts b/packages/compiler/src/ir/int32-slots.ts index 21223862a..221692e3f 100644 --- a/packages/compiler/src/ir/int32-slots.ts +++ b/packages/compiler/src/ir/int32-slots.ts @@ -407,6 +407,7 @@ const DYN_READ_ONLY = new Set([ "fs.utimesSync", "insp.dyn", "insp.dynS", + "insp.dynHidden", // The dynamic-tier operators run their dyn twins (scr_js_value.c); the // conversions move a value between dyn and jsval, and js.toF64 is // dynCheck's number check. diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 4a395feb2..5852eb8a6 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -1432,6 +1432,10 @@ export interface IrClassDef { jsName?: string; /** JavaScript constructor arity, ending before the first default or rest parameter. */ jsLength?: number; + /** The class declares public static fields. util.inspect prints them + * after the class tag; a class object reached through `unknown` does + * not carry them unless the runtime holds its static table. */ + ownStaticFields?: true; /** Zero-argument native helper returning the shared prototype data object. * Used when materializing an instance's own-property view. */ prototypeDataHelper?: string; diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6343-inspect-class-values.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6343-inspect-class-values.ts.json new file mode 100644 index 000000000..4fa8625ac --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6343-inspect-class-values.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6343-inspect-class-values.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6344-inspect-function-properties.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6344-inspect-function-properties.cjs.json new file mode 100644 index 000000000..003075140 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6344-inspect-function-properties.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6344-inspect-function-properties.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6357-format-o-hidden.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6357-format-o-hidden.ts.json new file mode 100644 index 000000000..cf0cd31d1 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6357-format-o-hidden.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6357-format-o-hidden.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6358-format-o-hidden-js.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6358-format-o-hidden-js.cjs.json new file mode 100644 index 000000000..73d4eefa6 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6358-format-o-hidden-js.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6358-format-o-hidden-js.cjs" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_error.h b/packages/runtime/src/scr_error.h index a7c921f23..17d217cdc 100644 --- a/packages/runtime/src/scr_error.h +++ b/packages/runtime/src/scr_error.h @@ -48,6 +48,12 @@ typedef struct ScrClassObj { size_t length; struct ScrBox *prototype_data; struct ScrBox *static_data; + /* The base class's name, which util.inspect prints as + * `[class C extends B]`; NULL without a named base. */ + const ScrStr *base_name; + /* Nonzero when the class declares public static fields, which the + * class object does not carry without static_data. */ + size_t own_static_fields; struct ScrBox *caps[]; } ScrClassObj; diff --git a/packages/runtime/src/scr_inspect.c b/packages/runtime/src/scr_inspect.c index a4d124ac3..214648802 100644 --- a/packages/runtime/src/scr_inspect.c +++ b/packages/runtime/src/scr_inspect.c @@ -324,6 +324,9 @@ typedef struct { static SCR_TL InspSeenEnt *g_seen; static SCR_TL size_t g_nseen; +/* util.format's %o (scr_insp_dyn_hidden): this render shows hidden own + * properties, like util.inspect's showHidden. */ +static SCR_TL bool g_show_hidden; static SCR_TL size_t g_seen_cap; /* Detection-numbered circular targets (persist across the call). */ static SCR_TL const void *g_circ[64]; @@ -1375,18 +1378,25 @@ static ScrStr *scr_insp_dyn_collection(ScrDyn *d, double recurse, double depth, return out; } -/* An own enumerable symbol-keyed property of `d` (borrowed), or NULL. */ +/* An own symbol-keyed property of `d` (borrowed) this render shows, or + * NULL: enumerable ones, and under showHidden the non-enumerable ones too. */ static ScrDynEntry *scr_insp_symbol_entry(ScrDyn *d, size_t i) { ScrDynEntry *ent = scr_dyn_own_symbol_entry(d, d->symbol_keys->v.arr.items[i]); - return ent && ent->enumerable ? ent : NULL; + return ent && (ent->enumerable || g_show_hidden) ? ent : NULL; } /* An ordinary object: `Name { key: value }` with its constructor name * `cls` (NULL for a plain object). Node's default view lists the own * enumerable string keys, then the own enumerable symbol keys. */ +/* An own entry this render shows: enumerable ones, and under showHidden + * the non-enumerable ones too (never the encoding's `%` markers). */ +static bool insp_obj_shows(const ScrDynEntry *ent) { + return ent->enumerable || (g_show_hidden && !(ent->key_len && ent->key[0] == '%')); +} + static ScrStr *scr_insp_obj(ScrDyn *d, double recurse, double depth, const char *cls) { size_t shown = 0; - for (size_t i = 0; i < d->v.obj.len; i++) shown += d->v.obj.entries[i].enumerable; + for (size_t i = 0; i < d->v.obj.len; i++) shown += insp_obj_shows(&d->v.obj.entries[i]); size_t nsymbols = d->symbol_keys ? d->symbol_keys->v.arr.len : 0; for (size_t i = 0; i < nsymbols; i++) shown += scr_insp_symbol_entry(d, i) != NULL; if (shown == 0) { @@ -1402,10 +1412,17 @@ static ScrStr *scr_insp_obj(ScrDyn *d, double recurse, double depth, const char scr_insp_begin(recurse + 1); for (size_t i = 0; i < d->v.obj.len; i++) { ScrDynEntry *ent = &d->v.obj.entries[i]; - if (!ent->enumerable) continue; + if (!insp_obj_shows(ent)) continue; ScrStr *val = scr_insp_dyn(ent->value, recurse + 1, depth); InspBuf eb = {0}; - insp_key_into(&eb, ent->key, ent->key_len); + if (ent->enumerable) { + insp_key_into(&eb, ent->key, ent->key_len); + } else { + /* formatProperty's hidden key: the key's rendering in brackets. */ + ib_char(&eb, '['); + insp_key_into(&eb, ent->key, ent->key_len); + ib_char(&eb, ']'); + } ib_cstr(&eb, ": "); ib_bytes(&eb, val->data, val->len); scr_str_release(val); @@ -1419,7 +1436,10 @@ static ScrStr *scr_insp_obj(ScrDyn *d, double recurse, double depth, const char ScrStr *key = scr_insp_dyn(d->symbol_keys->v.arr.items[i], recurse + 1, depth); ScrStr *val = scr_insp_dyn(ent->value, recurse + 1, depth); InspBuf eb = {0}; + /* A hidden symbol key is bracketed like a hidden string key. */ + if (!ent->enumerable) ib_char(&eb, '['); ib_bytes(&eb, key->data, key->len); + if (!ent->enumerable) ib_char(&eb, ']'); ib_cstr(&eb, ": "); ib_bytes(&eb, val->data, val->len); scr_str_release(key); @@ -1480,8 +1500,114 @@ ScrStr *scr_insp_typed_array(ScrBytes *b, double recurse, double depth) { return out; } +/* Node's function rendering: [Function: name] / [Function (anonymous)], + * and [class C extends B] / [class (anonymous)] for a class. The boxed + * name is the compiler's best-effort static spelling (SEMANTICS.md — + * reference-site naming). Enumerable own properties (a class's static + * fields, properties assigned to a function) follow in braces; past the + * depth limit such a function is [Function]. Borrows d; +1. */ +static ScrStr *scr_insp_dyn_func(ScrDyn *d, double recurse, double depth) { + const char *name = d->v.fn.name; + const ScrClassObj *cls = d->v.fn.class_obj; + ScrClosure *owner = scr_closure_identity(d->v.fn.clo); + ScrBox *props = owner->props ? owner->props : cls ? cls->static_data : NULL; + ScrDyn *table = props ? (ScrDyn *)scr_box_get_ref(props) : NULL; /* +1 */ + InspBuf out = {0}; + if (cls) { + ib_cstr(&out, "[class "); + ib_cstr(&out, name && name[0] ? name : "(anonymous)"); + const ScrDyn *base = table ? table->prototype : NULL; + if (cls->base_name && cls->base_name->len) { + ib_cstr(&out, " extends "); + ib_bytes(&out, cls->base_name->data, cls->base_name->len); + } else if (base && base->kind == SCR_DYN_FUNC && base->v.fn.name && base->v.fn.name[0]) { + ib_cstr(&out, " extends "); + ib_cstr(&out, base->v.fn.name); + } + ib_char(&out, ']'); + } else if (name && name[0]) { + ib_cstr(&out, "[Function: "); + ib_cstr(&out, name); + ib_char(&out, ']'); + } else { + ib_cstr(&out, "[Function (anonymous)]"); + } + ScrStr *tag = ib_take(&out); + if (cls && !props && cls->own_static_fields) { + /* Node prints the static fields after the tag; the class object + * does not hold them (the reflection fence's stance). */ + scr_str_release(tag); + static const char message[] = "util.inspect of a class with static fields through unknown is not supported yet"; + scr_throw_error_msg(SCR_ERR_ERROR, message, sizeof message - 1); + return scr_str_new("", 0); + } + size_t shown = 0; + if (table && table->kind == SCR_DYN_OBJ) { + for (size_t i = 0; i < table->v.obj.len; i++) shown += table->v.obj.entries[i].enumerable; + size_t nsymbols = table->symbol_keys ? table->symbol_keys->v.arr.len : 0; + for (size_t i = 0; i < nsymbols; i++) shown += scr_insp_symbol_entry(table, i) != NULL; + } + if (shown == 0) { + scr_dyn_release(table); + return tag; + } + if (recurse > depth) { + scr_dyn_release(table); + scr_str_release(tag); + return scr_str_new("[Function]", 10); + } + /* A function already on the traversal stack renders as [Circular *N], + * keyed by the closure every box of the function shares. */ + double circular = scr_insp_circ_check(owner); + if (circular > 0) { + scr_dyn_release(table); + scr_str_release(tag); + return scr_insp_circular(circular); + } + char *prefix = malloc(tag->len + 1); + if (!prefix) scr_trap("scriptc: out of memory\n"); + memcpy(prefix, tag->data, tag->len); + prefix[tag->len] = '\0'; + scr_str_release(tag); + size_t nseen = g_nseen; + g_pending_seen = owner; + ScrStr *rendered = scr_insp_obj(table, recurse, depth, prefix); + g_pending_seen = NULL; + if (g_nseen > nseen) g_nseen = nseen; + free(prefix); + scr_dyn_release(table); + return rendered; +} + +/* Under showHidden, the kinds whose hidden entries this inspector does not + * render (a function's length/name/prototype, an error's stack, a typed + * array's byte fields, a native handle's internals): the kind's name, NULL + * for the kinds it renders. */ +static const char *insp_hidden_refusal(const ScrDyn *d) { + switch (d->kind) { + case SCR_DYN_FUNC: return "function"; + case SCR_DYN_BYTES: return d->buffer ? "Buffer" : d->v.bytes->is_data_view ? "DataView" : "typed array"; + case SCR_DYN_OBJ: return scr_dyn_obj_get(d, "%error", 6) != NULL ? "Error" : NULL; + case SCR_DYN_HANDLE: + return d->v.handle.tag == SCR_DYNH_MAP || d->v.handle.tag == SCR_DYNH_SET || + d->v.handle.tag == SCR_DYNH_DATE + ? NULL + : scr_dyn_handle_cls(d); + default: return NULL; + } +} + ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { if (d->kind == SCR_DYN_PROXY) return scr_insp_dyn(d->v.proxy.target, recurse, depth); + const char *refused = g_show_hidden ? insp_hidden_refusal(d) : NULL; + if (refused) { + InspBuf out = {0}; + ib_cstr(&out, "util.format %o of "); + ib_cstr(&out, refused); + ib_cstr(&out, " values is not supported yet"); + scr_throw_error(SCR_ERR_ERROR, ib_take(&out)); + return scr_str_new("", 0); + } switch (d->kind) { case SCR_DYN_NULL: return scr_str_new("null", 4); @@ -1499,14 +1625,14 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { return scr_insp_str(d->v.str); case SCR_DYN_ARR: { size_t len = d->v.arr.len; - if (len == 0) return scr_str_new("[]", 2); + if (len == 0 && !g_show_hidden) return scr_str_new("[]", 2); if (recurse > depth) return scr_str_new("[Array]", 7); scr_insp_begin(recurse + 1); /* Node's maxArrayLength (100) counts output entries; a run of holes * is one `` entry (formatSpecialArray). */ const unsigned char *presence = d->v.arr.presence; - size_t i = 0; - for (size_t shown = 0; i < len && shown < 100; shown++) { + size_t i = 0, shown = 0; + for (; i < len && shown < 100; shown++) { if (presence && !presence[i]) { size_t next = i + 1; while (next < len && !presence[next]) next++; @@ -1529,10 +1655,26 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { scr_insp_entry(m, d->v.arr.items[i]->kind == SCR_DYN_NUM); scr_str_release(m); } + if (g_show_hidden) { + /* The own non-enumerable `length`, after the items. Node's + * grouping reads value[k] for every output position k, this one + * included. */ + size_t at = shown + (more ? 1 : 0); + char line[48]; + int n = snprintf(line, sizeof line, "[length]: %zu", len); + ScrStr *s = scr_str_new(line, (size_t)n); + scr_insp_entry(s, at < len && (!presence || presence[at]) && + d->v.arr.items[at]->kind == SCR_DYN_NUM); + scr_str_release(s); + } ScrStr *base = scr_str_new("", 0); ScrStr *b0 = scr_str_new("[", 1); ScrStr *b1 = scr_str_new("]", 1); - ScrStr *out = scr_insp_end(base, b0, b1, recurse + 1, true, more); + /* groupArrayElements leaves the last entry out of the grid when + * there are more than maxArrayLength (100) entries: the "... N more + * items" tail, or under showHidden the `[length]` entry. */ + bool trailing = shown + (more ? 1 : 0) + (g_show_hidden ? 1 : 0) > 100; + ScrStr *out = scr_insp_end(base, b0, b1, recurse + 1, true, trailing); scr_str_release(base); scr_str_release(b0); scr_str_release(b1); @@ -1549,21 +1691,8 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { * the bare marker (where a plain object says [Object]). */ return scr_insp_obj(d, recurse, depth, scr_dyn_class_name(d)); } - case SCR_DYN_FUNC: { - /* Node's function rendering: [Function: name] / [Function - * (anonymous)]. The boxed name is the compiler's best-effort static - * spelling (SEMANTICS.md — reference-site naming). */ - const char *name = d->v.fn.name; - InspBuf out = {0}; - if (name && name[0]) { - ib_cstr(&out, "[Function: "); - ib_cstr(&out, name); - ib_char(&out, ']'); - } else { - ib_cstr(&out, "[Function (anonymous)]"); - } - return ib_take(&out); - } + case SCR_DYN_FUNC: + return scr_insp_dyn_func(d, recurse, depth); case SCR_DYN_HANDLE: { if (d->v.handle.tag == SCR_DYNH_ARRAY_BUFFER || d->v.handle.tag == SCR_DYNH_SHARED_ARRAY_BUFFER) { @@ -1734,6 +1863,16 @@ ScrStr *scr_insp_dyn_s(ScrDyn *d, double depth) { return scr_insp_dyn(d, 0, depth); } +/* util.format's %o over a dyn value: util.inspect with showHidden at the + * given depth. */ +ScrStr *scr_insp_dyn_hidden(ScrDyn *d, double depth) { + bool outer = g_show_hidden; + g_show_hidden = true; + ScrStr *out = scr_insp_dyn(d, 0, depth); + g_show_hidden = outer; + return out; +} + /* ── console.table cells ─────────────────────────────────────────────── */ size_t scr_insp_display_width(const char *data, size_t len) { return insp_width(data, len); } diff --git a/packages/runtime/src/scr_inspect.h b/packages/runtime/src/scr_inspect.h index 5ac3aa9dd..9bf18a23c 100644 --- a/packages/runtime/src/scr_inspect.h +++ b/packages/runtime/src/scr_inspect.h @@ -47,6 +47,7 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth); ScrStr *scr_insp_typed_array(ScrBytes *b, double recurse, double depth); /* format's %s / rest-args twin: dyn strings pass verbatim. */ ScrStr *scr_insp_dyn_s(ScrDyn *d, double depth); +ScrStr *scr_insp_dyn_hidden(ScrDyn *d, double depth); /* %o: showHidden */ void scr_insp_begin(double recurse); /* util.inspect's breakLength, compact (0 for false) and sorted for the * renders that follow; the call site restores the defaults (80, 3, false) diff --git a/packages/runtime/src/scr_object.c b/packages/runtime/src/scr_object.c index 7cafbeca3..2f47a1c7b 100644 --- a/packages/runtime/src/scr_object.c +++ b/packages/runtime/src/scr_object.c @@ -33,6 +33,8 @@ ScrClassObj *scr_classobj_new(const ScrClassObj *template, size_t ncaps) { c->length = template->length; c->prototype_data = NULL; c->static_data = NULL; + c->base_name = template->base_name; + c->own_static_fields = template->own_static_fields; scr_obj_alloc_note(); return c; } diff --git a/tests/corpus/6343-inspect-class-values.ts b/tests/corpus/6343-inspect-class-values.ts new file mode 100644 index 000000000..5e9931c8c --- /dev/null +++ b/tests/corpus/6343-inspect-class-values.ts @@ -0,0 +1,31 @@ +// Classes reached through `unknown` print as Node prints them: +// `[class C]`, `[class C extends B]` with the runtime class's base, and +// `[class (anonymous)]`, at the top level and inside arrays and records. +import { inspect } from "node:util"; + +class A { + x = 1; +} +class B extends A { + y = 2; +} +class C extends B {} +class MyErr extends Error {} +const Anon = [class {}][0]!; +const Sub = class extends A {}; + +const a: unknown = A; +const b: unknown = B; +console.log(a, b); +console.log([a], { b }); +const values: unknown[] = [C, MyErr, Anon, Sub]; +console.log(values); +console.log(inspect({ deep: { values } }, { depth: 0 }), inspect(values, { depth: 0 })); +console.log(String(typeof values[0])); + +// A slot typed as the base holds a subclass at runtime. +const pick = (k: number): typeof A => (k === 0 ? A : k === 1 ? B : C); +for (let k = 0; k < 3; k++) { + const picked: unknown = pick(k); + console.log(k, picked); +} diff --git a/tests/corpus/6344-inspect-function-properties.cjs b/tests/corpus/6344-inspect-function-properties.cjs new file mode 100644 index 000000000..94db45a40 --- /dev/null +++ b/tests/corpus/6344-inspect-function-properties.cjs @@ -0,0 +1,34 @@ +'use strict'; +// Functions and classes reached as dynamic values print their enumerable +// own properties after the tag, as Node prints them: a class's static +// properties, properties assigned to a function, cycles through them, +// and `[Function]` past the depth limit. +const util = require('util'); + +function f() {} +f.x = 1; +f.label = 'f'; +function g() {} +g.self = g; +function plain() {} +class A {} +class S extends A { + static get hidden() { + return 1; + } +} +S.count = 3; +function show(value) { + console.log(value); +} +show(f); +show(g); +show(S); +show(A); +show(plain); +function shallow(value) { + console.log(util.inspect({ value }, { depth: 0 })); +} +shallow(f); +shallow(S); +shallow(A); diff --git a/tests/corpus/6357-format-o-hidden.ts b/tests/corpus/6357-format-o-hidden.ts new file mode 100644 index 000000000..9e48ed4ba --- /dev/null +++ b/tests/corpus/6357-format-o-hidden.ts @@ -0,0 +1,19 @@ +// util.format's %o over values reached through `unknown` renders with +// showHidden, as Node does: arrays show their hidden `[length]` (with +// Node's grouping), at every depth, inside Maps and records too. +import { format } from "node:util"; +const cases: [string, unknown][] = [ + ["ten", Array.from({ length: 10 }, (_, i) => i)], + ["wide", Array.from({ length: 120 }, (_, i) => i)], + ["hundred", Array.from({ length: 100 }, (_, i) => i)], + ["hundred-one", Array.from({ length: 101 }, (_, i) => i)], + ["hundred-strs", Array.from({ length: 100 }, (_, i) => "s" + i)], + ["seven", ["aaaaaaaaaaaaaaaaaaaa", "b", "c", "d", "e", "f", "g"]], + ["strs", Array.from({ length: 30 }, (_, i) => "s" + i)], + ["holes", [1, , 3]], + ["empty", []], + ["nested", { list: [[]], m: new Map([[1, [2]]]) }], + ["deep", [[[[[[1]]]]]]], + ["date", [new Date(0)]], +]; +for (const [k, v] of cases) console.log(k, format("%o", v)); diff --git a/tests/corpus/6358-format-o-hidden-js.cjs b/tests/corpus/6358-format-o-hidden-js.cjs new file mode 100644 index 000000000..df54b2714 --- /dev/null +++ b/tests/corpus/6358-format-o-hidden-js.cjs @@ -0,0 +1,9 @@ +'use strict'; +// %o shows an object's non-enumerable own properties as `[key]`, symbol +// keys included. +const util = require('util'); +const hidden = Object.defineProperty(JSON.parse('{"shown":1}'), 'secret', { value: [2], enumerable: false }); +Object.defineProperty(hidden, 'a-b', { value: 'x', enumerable: false }); +Object.defineProperty(hidden, Symbol('tag'), { value: 3, enumerable: false }); +console.log(util.format('%o and %O', hidden, hidden)); +console.log(util.format('%o', [hidden])); diff --git a/tests/fixtures/self-hosting/llvm-layouts.ts b/tests/fixtures/self-hosting/llvm-layouts.ts index 6ddf3ca3a..36d55bddf 100644 --- a/tests/fixtures/self-hosting/llvm-layouts.ts +++ b/tests/fixtures/self-hosting/llvm-layouts.ts @@ -119,7 +119,8 @@ try { }, ); virtualBorrows = new VirtualBorrows(mod, graph, functions, lifetimes.borrowed); - const classObjects = new Map(request.classObjects.map((name) => [name, { nameSym: `@name_${mangleClassObj(name)}` }])); + const classObjects: Map = new Map(); + for (const name of request.classObjects) classObjects.set(name, { nameSym: `@name_${mangleClassObj(name)}`, baseNameSym: null }); const typeName = (type: IrType): string => type.kind === "void" ? "void" : type.kind === "f64" || type.kind === "date" ? "double" : type.kind === "bool" ? "i1" : "ptr"; // Mirrors the emitter: whole-program int32 fields store the exact i32. const layouts = emitLlvmLayouts(host, mod, graph, classObjects, functions, typeName, undefined, analyzeInt32Slots(mod)); diff --git a/tests/harness/corpus-costs/6343-inspect-class-values.ts.json b/tests/harness/corpus-costs/6343-inspect-class-values.ts.json new file mode 100644 index 000000000..fa2590573 --- /dev/null +++ b/tests/harness/corpus-costs/6343-inspect-class-values.ts.json @@ -0,0 +1 @@ +{ "plain": 3700, "san": 4700 } diff --git a/tests/harness/corpus-costs/6344-inspect-function-properties.cjs.json b/tests/harness/corpus-costs/6344-inspect-function-properties.cjs.json new file mode 100644 index 000000000..71bf780f8 --- /dev/null +++ b/tests/harness/corpus-costs/6344-inspect-function-properties.cjs.json @@ -0,0 +1 @@ +{ "plain": 2700, "san": 3500 } diff --git a/tests/harness/corpus-costs/6357-format-o-hidden.ts.json b/tests/harness/corpus-costs/6357-format-o-hidden.ts.json new file mode 100644 index 000000000..f351a5044 --- /dev/null +++ b/tests/harness/corpus-costs/6357-format-o-hidden.ts.json @@ -0,0 +1 @@ +{ "plain": 2300, "san": 2100 } diff --git a/tests/harness/corpus-costs/6358-format-o-hidden-js.cjs.json b/tests/harness/corpus-costs/6358-format-o-hidden-js.cjs.json new file mode 100644 index 000000000..1f8ff72b2 --- /dev/null +++ b/tests/harness/corpus-costs/6358-format-o-hidden-js.cjs.json @@ -0,0 +1 @@ +{ "plain": 2100, "san": 2000 } diff --git a/tests/harness/dyncheck.test.ts b/tests/harness/dyncheck.test.ts index 6a2d4dc61..dab51a430 100644 --- a/tests/harness/dyncheck.test.ts +++ b/tests/harness/dyncheck.test.ts @@ -1278,4 +1278,92 @@ console.log("done", caught); expect(r.exitCode).toBe(0); expect(r.stdout).toBe("done 100\n"); }); + + test("an inspection refusal throws before console.log writes", async () => { + // A promise reached through `unknown` has no dynamic rendering; the + // refusal must stop the call, not print a partial line and continue. + const r = await compileAndRun( + "inspect-refusal-unwind", + `import { inspect } from "node:util"; +const p: unknown = Promise.resolve(3); +function attempt(label: string, f: () => void): void { + try { + f(); + console.log(label, "returned"); + } catch (e) { + console.log(label, (e as Error).message); + } +} +attempt("log", () => console.log("x", p)); +attempt("inspect", () => console.log(inspect(p).length)); +attempt("record", () => console.log({ a: p })); +attempt("array", () => console.log([p])); +console.log("before"); +console.log(p); +console.log("after"); +`, + ); + const refusal = "util.inspect of a promise value is not supported yet"; + expect(r.exitCode).toBe(1); + expect(r.stdout).toBe( + ["log", "inspect", "record", "array"].map((l) => `${l} ${refusal}\n`).join("") + "before\n", + ); + expect(r.stderr).toContain(`Uncaught Error: ${refusal}`); + }); + + test("inspecting a compiled class with static fields through unknown refuses", async () => { + // Node prints `[class S] { count: 3 }`; the class object does not + // carry compiled static fields, so the inspector refuses rather than + // print `[class S]`. A class without static fields still prints. + const r = await compileAndRun( + "inspect-class-statics", + `class Plain {} +class S extends Plain { + static count = 3; +} +const plain: unknown = Plain; +const s: unknown = S; +console.log(plain); +try { + console.log(s); +} catch (e) { + console.log((e as Error).message); +} +`, + ); + expect(r.exitCode).toBe(0); + expect(r.stdout).toBe( + "[class Plain]\nutil.inspect of a class with static fields through unknown is not supported yet\n", + ); + }); + + test("%o refuses the dynamic kinds whose hidden entries have no rendering", async () => { + // Node's showHidden adds a function's length/name/prototype, an + // error's stack and a typed array's byte fields; %o throws the 'not + // supported yet' Error for them instead of printing without them. + const r = await compileAndRun( + "format-o-hidden-refusals", + `import { format } from "node:util"; +function f(): void {} +const values: [string, unknown][] = [["fn", f], ["err", new Error("e")], ["u8", new Uint8Array(1)], ["arr", [1]]]; +for (const [label, value] of values) { + try { + console.log(label, format("%o", value)); + } catch (e) { + console.log(label, (e as Error).message); + } +} +`, + ); + expect(r.exitCode).toBe(0); + expect(r.stdout).toBe( + [ + "fn util.format %o of function values is not supported yet", + "err util.format %o of Error values is not supported yet", + "u8 util.format %o of typed array values is not supported yet", + "arr [ 1, [length]: 1 ]", + "", + ].join("\n"), + ); + }); }); From 65a68470cf0e5dba6e51e473faebeaf9f0f2e7b0 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 11:12:23 -0500 Subject: [PATCH 006/184] Free a typed teardown's blocks in the order it met them A typed teardown freed each block onto its free list as it went, and the free lists, which are LIFO, handed a torn-down structure's blocks to the next allocations in the reverse order. A structure that a loop rebuilds the same way each time was therefore laid out mirrored every other generation, and every walk over it, its teardown included, ran through memory backwards, which the hardware prefetchers handle three to four times worse. A program that rebuilds a tree of 2^21 - 1 object literals 12 times, summing each generation before dropping it, took 26 to 28 ns per node to build, 24 to sum and 16 to drop in those generations, against 6.8, 6.3 and 5.3 in the others. With this change, every generation runs at the second set of times, and the program takes 0.50 s instead of 1.18 s (Node: 0.91 s). The teardown of a class or record type with children of its own type now keeps the blocks it frees in a segment, linked through their first words and built in order through a tail pointer. A node releases its own block first, places it ahead of its children's when its address is the lower (object literals allocate before their fields) and behind them otherwise (constructor arguments build the children first), and the chain's first teardown frees the whole segment onto the free list in one step. A teardown past DIRECT_DESTROY_CHAIN appends through scr_rc_destroy_seg, which keeps scr_rc_destroy's depth budget and queue. Blocks outside the calling thread's arena are freed as before, and targets without the inline allocation paths keep the old teardown. --- packages/compiler/src/backend/llvm/alloc.ts | 77 +++++- .../compiler/src/backend/llvm/classes.test.ts | 59 +++++ packages/compiler/src/backend/llvm/classes.ts | 73 ++++-- packages/compiler/src/backend/llvm/shapes.ts | 242 ++++++++++++++++-- .../corpus/7260-teardown-segments.ts.json | 6 + packages/runtime/src/scr_cycle.c | 56 ++-- packages/runtime/src/scr_rc.h | 8 + scripts/ci-test-costs.json | 2 + tests/corpus/7260-teardown-segments.ts | 179 +++++++++++++ .../7260-teardown-segments.ts.json | 1 + tests/harness/teardown-segments.test.ts | 97 +++++++ 11 files changed, 732 insertions(+), 68 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7260-teardown-segments.ts.json create mode 100644 tests/corpus/7260-teardown-segments.ts create mode 100644 tests/harness/corpus-costs/7260-teardown-segments.ts.json create mode 100644 tests/harness/teardown-segments.test.ts diff --git a/packages/compiler/src/backend/llvm/alloc.ts b/packages/compiler/src/backend/llvm/alloc.ts index aa6345dab..2efeb96ac 100644 --- a/packages/compiler/src/backend/llvm/alloc.ts +++ b/packages/compiler/src/backend/llvm/alloc.ts @@ -459,18 +459,72 @@ export type FreeObservers = "weak" | "weak-or-source"; * type's disposals), and the block — `scr_cyc_free` for collector-headered * objects, `scr_rt_free` otherwise. Labels use the `sf.` prefix; the lines * end in an open block. + * + * With `keep` (an alloca of ptr holding NULL; inline allocation paths only), + * a block that would go onto the calling thread's free list is stored there + * instead, for a teardown segment (emitSegmentFree); a block the system or + * another arena owns is still freed. */ export function emitObjectFree( host: AllocHost, cyc: boolean, observers: FreeObservers = "weak", + keep: string | null = null, ): string[] { - const lines = threadLocalState(host, "sf", freeLines(host, cyc, observers)); + const lines = threadLocalState(host, "sf", freeLines(host, cyc, observers, keep)); if (host.objectAudit === false) return lines.filter((line) => line !== FREE_NOTE); host.declare(`declare void @scr_obj_free_note()`); return lines; } +/** Lines that put the blocks `first` to `last` (linked through their first + * words; one block when they are equal) of the class at reservation offset + * `off` on the calling thread's free list. */ +function freeListPushLines(prefix: string, off: string, first: string, last: string): string[] { + return [ + ` %${prefix}.shp = getelementptr inbounds i8, ptr @scr_sa, i64 ${SA_SHIFT}`, + ` %${prefix}.sh32 = load i32, ptr %${prefix}.shp`, + ` %${prefix}.sh = zext i32 %${prefix}.sh32 to i64`, + ` %${prefix}.c = lshr i64 ${off}, %${prefix}.sh`, + ` %${prefix}.c8 = shl i64 %${prefix}.c, 3`, + ` %${prefix}.fo = add i64 %${prefix}.c8, ${SA_FREE}`, + ` %${prefix}.fp = getelementptr inbounds i8, ptr @scr_sa, i64 %${prefix}.fo`, + ` %${prefix}.head = load ptr, ptr %${prefix}.fp`, + ` store ptr %${prefix}.head, ptr ${last}`, + ` store ptr ${first}, ptr %${prefix}.fp`, + ]; +} + +/** + * Lines that free a teardown segment: the blocks `first` to `last` (ptr + * values, both NULL for an empty segment), linked through their first words, + * which emitObjectFree kept. Every block of a segment is in the calling + * thread's arena and of one size class, so the segment goes onto that + * class's free list in one step, and the next allocations of the class take + * its blocks in segment order. Labels use `prefix`; the lines end in the + * open block `${prefix}.done`. + */ +export function emitSegmentFree( + host: AllocHost, + prefix: string, + first: string, + last: string, +): string[] { + declareState(host); + const lines = [ + ` %${prefix}.none = icmp eq ptr ${first}, null`, + ` br i1 %${prefix}.none, label %${prefix}.done, label %${prefix}.push`, + `${prefix}.push:`, + ` %${prefix}.base = load i64, ptr @scr_sa`, + ` %${prefix}.bi = ptrtoint ptr ${first} to i64`, + ` %${prefix}.off = sub i64 %${prefix}.bi, %${prefix}.base`, + ...freeListPushLines(prefix, `%${prefix}.off`, first, last), + ` br label %${prefix}.done`, + `${prefix}.done:`, + ]; + return threadLocalState(host, prefix, lines); +} + /** Lines that leave in `%.obs` whether the program can observe a * disposal of this type (folds to a constant). */ function observedLines(prefix: string, observers: FreeObservers): string[] { @@ -482,8 +536,14 @@ function observedLines(prefix: string, observers: FreeObservers): string[] { ]; } -function freeLines(host: AllocHost, cyc: boolean, observers: FreeObservers): string[] { +function freeLines( + host: AllocHost, + cyc: boolean, + observers: FreeObservers, + keep: string | null, +): string[] { if (host.inlineAlloc !== true) { + if (keep !== null) throw new Error("a kept block needs the inline allocation paths"); if (cyc) { host.declare(`declare void @scr_cyc_free(ptr)`); return [` call void @scr_obj_free_note()`, ` call void @scr_cyc_free(ptr %o)`]; @@ -548,16 +608,9 @@ function freeLines(host: AllocHost, cyc: boolean, observers: FreeObservers): str ` %sf.inx = call i1 @llvm.expect.i1(i1 %sf.in, i1 true)`, ` br i1 %sf.inx, label %sf.push, label %sf.sys`, `sf.push:`, - ` %sf.shp = getelementptr inbounds i8, ptr @scr_sa, i64 ${SA_SHIFT}`, - ` %sf.sh32 = load i32, ptr %sf.shp`, - ` %sf.sh = zext i32 %sf.sh32 to i64`, - ` %sf.c = lshr i64 %sf.off, %sf.sh`, - ` %sf.c8 = shl i64 %sf.c, 3`, - ` %sf.fo = add i64 %sf.c8, ${SA_FREE}`, - ` %sf.fp = getelementptr inbounds i8, ptr @scr_sa, i64 %sf.fo`, - ` %sf.head = load ptr, ptr %sf.fp`, - ` store ptr %sf.head, ptr ${blk}`, - ` store ptr ${blk}, ptr %sf.fp`, + ...(keep !== null + ? [` store ptr ${blk}, ptr ${keep}`] + : freeListPushLines("sf", "%sf.off", blk, blk)), ` br label %sf.done`, `sf.sys: ; outside the reservation, or the allocator is compiled out`, ` call void @scr_obj_free_note()`, diff --git a/packages/compiler/src/backend/llvm/classes.test.ts b/packages/compiler/src/backend/llvm/classes.test.ts index 2d95e942c..0c0b73aeb 100644 --- a/packages/compiler/src/backend/llvm/classes.test.ts +++ b/packages/compiler/src/backend/llvm/classes.test.ts @@ -68,6 +68,65 @@ test("leaf layouts keep direct teardown while recursive layouts share the depth } }); +test("teardowns of types with children of their own type free their blocks as one segment", () => { + const linked = { kind: "object" as const, className: "Linked" }; + const mod: IrModule = { + irVersion: 17, + sourceFile: loc.file, + entry: "main", + functions: [{ name: "main", params: [], returnType: VOID, locals: [], body: [], loc }], + classes: [ + { + name: "Linked", + fields: [ + { name: "next", type: linked }, + { name: "other", type: linked }, + ], + methods: [], + loc, + }, + ], + records: [{ id: "branch", fields: [{ name: "next", type: linked }] }], + }; + const define = (llvm: string, name: string): string => + new RegExp(`^define internal [^@]*@${name}\\([^]*?^}`, "m").exec(llvm)?.[0] ?? ""; + const linkedDestroy = `${mangleClassRelease("Linked")}_destroy`; + const branchDestroy = `${mangleRecordRelease("branch")}_destroy`; + for (const workers of [false, true]) { + const llvm = emitLlvmModule({ ...mod, workers }, { pointerBits: 64 }); + // Linked's teardown keeps its block and its children's in a segment: it + // releases its own block first and places it by address against its + // first child, a child torn down directly appends to the segment, one + // past the direct chain appends through the runtime, and the chain's + // first teardown frees the segment onto the free list in one step. + const seg = define(llvm, `${linkedDestroy}_s`); + expect(seg.startsWith("define internal ptr"), "segment form").toBe(true); + expect(seg).toMatch( + /store ptr %(?:sf\.blk|o), ptr %sg\.own[^]*call ptr @sc_release_Linked_destroy_s\(/, + ); + // The children's releases have a copy for each place of the own block. + for (const copy of ["ta", "tb"]) + expect(seg).toContain( + `call ptr @scr_rc_destroy_seg(ptr %v0, ptr @${linkedDestroy}_sc, ptr @${linkedDestroy}, ptr %${copy}0.bt)`, + ); + expect(seg).not.toContain("call void @scr_rc_destroy("); + expect(seg).toContain("icmp ult ptr %sg.o, %sg.r1"); + const first = define(llvm, `${linkedDestroy}_d`); + expect(first).toContain(`call ptr @${linkedDestroy}_s(ptr %o, i32 %d, ptr %head)`); + expect(first).toContain("store ptr %first, ptr %sgf.fp"); + expect(first.includes("llvm.threadlocal.address"), "thread-local state").toBe(workers); + expect(define(llvm, linkedDestroy)).toContain(`call void @${linkedDestroy}_d(ptr %o, i32 0)`); + // A record whose child is another type frees its own block as before. + expect(define(llvm, `${branchDestroy}_s`)).toBe(""); + expect(define(llvm, branchDestroy)).toContain(`call void @${mangleClassRelease("Linked")}(`); + } + // Without the inline allocation paths (32-bit targets) a teardown frees + // through the runtime, block by block. + const llvm32 = emitLlvmModule(mod, { pointerBits: 32 }); + expect(define(llvm32, `${linkedDestroy}_s`)).toBe(""); + expect(define(llvm32, `${linkedDestroy}_d`)).toContain(`call void @${linkedDestroy}_d(`); +}); + test("releases split into an inlinable fast path and an out-of-line remainder", () => { // An array of the class itself makes the hierarchy cycle-capable. const kids = { kind: "array" as const, elem: { kind: "object" as const, className: "Node" } }; diff --git a/packages/compiler/src/backend/llvm/classes.ts b/packages/compiler/src/backend/llvm/classes.ts index 3073bd1bd..3d67fbfd3 100644 --- a/packages/compiler/src/backend/llvm/classes.ts +++ b/packages/compiler/src/backend/llvm/classes.ts @@ -43,6 +43,9 @@ import { destroyDefinitions, directChildReleaseLines, directDestroyTarget, + segmentPrologueLines, + segmentTeardownLines, + segmentedTeardown, usedDirectTargets, type DirectDestroy, type RegionHooks, @@ -1078,25 +1081,62 @@ export function emitClassShapes( bounded && (directTargets.has(`object:${cls.name}`) || refFields.some((f) => directDestroyTarget(host, f.type, directTargets) !== null)); + // The teardown's own symbol, and whether it keeps a segment (shapes.ts + // segmentedTeardown). + const destroy = `${meta.hierarchy ? mangleClassReleaseDirect(cls.name) : mangleClassRelease(cls.name)}_destroy`; + const segmented = + chained && + !meta.compact && + runtimePrefix === null && + !isStreamRooted && + !meta.threadSlot && + segmentedTeardown( + host, + destroy, + refFields.map((f) => f.type), + directTargets, + ); // The field-releasing teardown body shared by both release shapes // (the public one on standalone classes, the DIRECT one on hierarchy // members). Runs at rc == 0. const teardown = (lines: string[]): void => { - refFields.forEach((f, t) => { - lines.push( - ` %f${t} = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${f.index}`, - ` %v${t} = load ptr, ptr %f${t}`, - ); + const release = (prefix: string, f: (typeof refFields)[number], t: number): string[] => { const direct = bounded ? directDestroyTarget(host, f.type, directTargets) : null; - if (direct) - lines.push( - ...directChildReleaseLines(host, `td${t}`, `%v${t}`, direct, llvmCommentText(f.name)), - ); - else - lines.push( + if (direct === null) + return [ ` call void ${releaseSym(host, f.type)}(ptr %v${t}) ; ${llvmCommentText(f.name)}`, - ); - }); + ]; + const own = segmented && direct.destroy === destroy; + return directChildReleaseLines( + host, + `${prefix}${t}`, + `%v${t}`, + direct, + llvmCommentText(f.name), + own, + ); + }; + const loads = refFields.map((f, t) => [ + ` %f${t} = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${f.index}`, + ` %v${t} = load ptr, ptr %f${t}`, + ]); + if (segmented) { + // A segmented teardown loads its fields, releases its own block, and + // then releases the fields (shapes.ts segmentTeardownLines). + const ownType = refFields.flatMap((f, t) => + directDestroyTarget(host, f.type, directTargets)?.destroy === destroy ? [`%v${t}`] : [], + ); + lines.push( + ...segmentPrologueLines(), + ...loads.flat(), + ...emitObjectFree(host, traced, "weak", "%sg.own"), + ...segmentTeardownLines(ownType, (prefix) => + refFields.flatMap((f, t) => release(prefix, f, t)), + ), + ); + return; + } + refFields.forEach((f, t) => lines.push(...loads[t]!, ...release("td", f, t))); if (runtimePrefix) lines.push(...regCall("td", runtimePrefix.drop, "")); if (isStreamRooted) lines.push(...stCall("tds", "scr_stream_st_release", "")); if (meta.threadSlot) lines.push(...slotForgetLines(host, struct, meta, "tdw")); @@ -1178,10 +1218,12 @@ export function emitClassShapes( ]; if (traced) reld.push(...cycleDeadLines(host, "teardown")); if (bounded) { - const destroy = `${mangleClassReleaseDirect(cls.name)}_destroy`; const destroyBody: string[] = []; teardown(destroyBody); - defs.push(...destroyDefinitions(destroy, destroyBody, chained), ``); + defs.push( + ...destroyDefinitions(destroy, destroyBody, chained, segmented ? host : null), + ``, + ); host.declare(`declare void @scr_rc_destroy(ptr, ptr)`); reld.push(` call void @scr_rc_destroy(ptr %o, ptr @${destroy})`); } else teardown(reld); @@ -1203,6 +1245,7 @@ export function emitClassShapes( bounded, lifetime, chained, + segmented, ), ``, ); diff --git a/packages/compiler/src/backend/llvm/shapes.ts b/packages/compiler/src/backend/llvm/shapes.ts index 6de92f667..ae217a76c 100644 --- a/packages/compiler/src/backend/llvm/shapes.ts +++ b/packages/compiler/src/backend/llvm/shapes.ts @@ -34,6 +34,7 @@ import { cycTypeDescriptor, emitObjectAlloc, emitObjectFree, + emitSegmentFree, CYC_APPEND, cycleAppendDecls, cycleAppendHelper, @@ -1340,17 +1341,23 @@ export interface DirectDestroy { /** The release of child `value` (a loaded field, possibly NULL) inside a * teardown whose chain length is `%d`: the child's own release for a shared * or immortal reference, its teardown for the last one. Labels use `tag`; - * the lines end in the open block `${tag}.next`. */ + * the lines end in the open block `${tag}.next`. + * + * With `segment` (the child has the teardown's own type, in a segmented + * teardown; segmentPrologueLines), the child's teardown appends its blocks + * to the teardown's segment. */ export function directChildReleaseLines( host: ShapeHost, tag: string, value: string, target: DirectDestroy, comment: string, + segment = false, ): string[] { const S = host.sizeType; const C = target.count ?? S; - host.declare(`declare void @scr_rc_destroy(ptr, ptr)`); + if (segment) host.declare(`declare ptr @scr_rc_destroy_seg(ptr, ptr, ptr, ptr)`); + else host.declare(`declare void @scr_rc_destroy(ptr, ptr)`); const lines = [ ` %${tag}.nn = icmp ne ptr ${value}, null`, ` br i1 %${tag}.nn, label %${tag}.chk, label %${tag}.next`, @@ -1382,16 +1389,139 @@ export function directChildReleaseLines( ` br i1 %${tag}.short, label %${tag}.direct, label %${tag}.bounded`, `${tag}.direct:`, ` %${tag}.d = add i32 %d, 1`, - ` call void @${target.destroy}_d(ptr ${value}, i32 %${tag}.d)`, + ); + if (!segment) { + lines.push( + ` call void @${target.destroy}_d(ptr ${value}, i32 %${tag}.d)`, + ` br label %${tag}.next`, + `${tag}.bounded:`, + ` call void @scr_rc_destroy(ptr ${value}, ptr @${target.destroy})`, + ` br label %${tag}.next`, + `${tag}.next:`, + ); + return lines; + } + lines.push( + ` %${tag}.t = load ptr, ptr %sg.t`, + ` %${tag}.dt = call ptr @${target.destroy}_s(ptr ${value}, i32 %${tag}.d, ptr %${tag}.t)`, + ` store ptr %${tag}.dt, ptr %sg.t`, ` br label %${tag}.next`, `${tag}.bounded:`, - ` call void @scr_rc_destroy(ptr ${value}, ptr @${target.destroy})`, + ` %${tag}.bt = load ptr, ptr %sg.t`, + ` %${tag}.rt = call ptr @scr_rc_destroy_seg(ptr ${value}, ptr @${target.destroy}_sc, ptr @${target.destroy}, ptr %${tag}.bt)`, + ` store ptr %${tag}.rt, ptr %sg.t`, ` br label %${tag}.next`, `${tag}.next:`, ); return lines; } +/* Teardown segments. A typed teardown that frees each block onto its free + * list as it goes frees a tree in walk order, and the free lists, which are + * LIFO, hand the blocks to the next allocations in the reverse order: a tree + * built again in the same order as the last (each generation of a structure + * a loop rebuilds) is laid out mirrored, and every walk over it, the next + * teardown included, runs through memory backwards, which defeats the + * hardware prefetchers. The runtime frees large dead sets of the cycle + * collector in address order for the same reason (scr_mem_order_for_free). + * + * So the teardown of a type with children of its own type (the `_s` form, + * segmentedTeardown) keeps the blocks it frees in a segment, linked through + * their first words, which it builds in order through a tail pointer: the + * `_s` form takes the segment's last link (the first word of its last block, + * or the head slot of an empty segment) and returns the new one. A node + * first releases its own block (emitObjectFree, keeping it), then appends it + * ahead of its children's blocks when its address is the lower, behind them + * otherwise (segmentTeardownLines), and the children append theirs in field + * order, so a structure + * built parent first (object literals allocate before their fields) or + * children first (constructor arguments) comes back in ascending order. The + * chain's first teardown (`_d`) frees the whole segment onto the class's + * free list in one step (emitSegmentFree), and the walks over the next + * generation run forward. A teardown past DIRECT_DESTROY_CHAIN appends + * through scr_rc_destroy_seg, under the same depth budget as + * scr_rc_destroy. Blocks outside the calling thread's arena are freed as + * before; one type's blocks are all of one class. */ + +/** Whether the teardown of a type whose reference fields have `fieldTypes` + * (with direct targets `targets`) and whose own teardown is `destroy` keeps + * a segment: a child field has its own type, and the inline allocation paths + * are in use. */ +export function segmentedTeardown( + host: ShapeHost, + destroy: string, + fieldTypes: readonly IrType[], + targets: ReadonlyMap, +): boolean { + if (host.inlineAlloc !== true) return false; + return fieldTypes.some((t) => directDestroyTarget(host, t, targets)?.destroy === destroy); +} + +/** The first lines of a segmented teardown, in its entry block: the + * segment's last link (`%sg.t`, from the `%tail` argument) and the kept own + * block (`%sg.own`, for emitObjectFree). */ +export function segmentPrologueLines(): string[] { + return [ + ` %sg.t = alloca ptr`, + ` %sg.own = alloca ptr`, + ` store ptr %tail, ptr %sg.t`, + ` store ptr null, ptr %sg.own`, + ]; +} + +/** The rest of a segmented teardown after emitObjectFree kept (or freed) + * its own block: the block goes ahead of its children's when its address is + * below the first non-NULL one of `children` (loaded fields of the + * teardown's own type, which stand for their blocks) or there is none, and + * behind them otherwise; then the children's releases (`releases`, given a + * label prefix), and the segment's last link is returned. Each order has its + * own copy of the releases, so neither keeps the decision, or the block once + * it is appended, across the children's teardowns. */ +export function segmentTeardownLines( + children: readonly string[], + releases: (prefix: string) => string[], +): string[] { + const lines = [` %sg.o = load ptr, ptr %sg.own`, ` %sg.kept = icmp ne ptr %sg.o, null`]; + let ref = "null"; + [...children].reverse().forEach((child, i) => { + lines.push( + ` %sg.nn${i} = icmp ne ptr ${child}, null`, + ` %sg.r${i} = select i1 %sg.nn${i}, ptr ${child}, ptr ${ref}`, + ); + ref = `%sg.r${i}`; + }); + lines.push( + ` %sg.leaf = icmp eq ptr ${ref}, null`, + ` %sg.lower = icmp ult ptr %sg.o, ${ref}`, + ` %sg.first = or i1 %sg.leaf, %sg.lower`, + ` %sg.ahead = and i1 %sg.kept, %sg.first`, + ` br i1 %sg.ahead, label %sg.front, label %sg.back`, + `sg.front:`, + ...segmentAppendLines("a"), + ...releases("ta"), + ` br label %sg.ret`, + `sg.back:`, + ...releases("tb"), + ` %sg.keep = icmp ne ptr %sg.o, null`, + ` br i1 %sg.keep, label %sg.last, label %sg.ret`, + `sg.last:`, + ...segmentAppendLines("l"), + ` br label %sg.ret`, + `sg.ret:`, + ` %sg.tr = load ptr, ptr %sg.t`, + ` ret ptr %sg.tr`, + ); + return lines; +} + +function segmentAppendLines(tag: string): string[] { + return [ + ` %sg.t${tag} = load ptr, ptr %sg.t`, + ` store ptr %sg.o, ptr %sg.t${tag}`, + ` store ptr %sg.o, ptr %sg.t`, + ]; +} + /** The direct-teardown target of a reference of `type`, if `targets` (keyed * `record:` / `object:`) has one. Nullable unions hold the arm's * pointer, NULL or an immortal sentinel, which the child release handles. */ @@ -1434,12 +1564,42 @@ export function usedDirectTargets( /** The teardown definitions for `destroy`: with `chained`, the body takes * the chain length `%d` (directChildReleaseLines) and the runtime's - * one-argument entry starts a chain. */ + * one-argument entry starts a chain. With `segment` (the host, for a + * segmented teardown), the body is a segment teardown's + * (segmentPrologueLines to segmentTeardownLines), defined as `_s`, with + * `_sc` for scr_rc_destroy_seg, and `_d` starts a segment and frees it. */ export function destroyDefinitions( destroy: string, body: readonly string[], chained: boolean, + segment: ShapeHost | null = null, ): string[] { + if (segment !== null) + return [ + `define internal ptr @${destroy}_s(ptr %o, i32 %d, ptr %tail) ${FN_ATTRS} {`, + `entry:`, + ...body, + `}`, + `define internal ptr @${destroy}_sc(ptr %o, ptr %tail) ${FN_ATTRS} {`, + `entry:`, + ` %t = call ptr @${destroy}_s(ptr %o, i32 0, ptr %tail)`, + ` ret ptr %t`, + `}`, + `define internal void @${destroy}_d(ptr %o, i32 %d) ${FN_ATTRS} {`, + `entry:`, + ` %head = alloca ptr`, + ` store ptr null, ptr %head`, + ` %last = call ptr @${destroy}_s(ptr %o, i32 %d, ptr %head)`, + ` %first = load ptr, ptr %head`, + ...emitSegmentFree(segment, "sgf", "%first", "%last"), + ` ret void`, + `}`, + `define internal void @${destroy}(ptr %o) ${FN_ATTRS} {`, + `entry:`, + ` call void @${destroy}_d(ptr %o, i32 0)`, + ` ret void`, + `}`, + ]; if (!chained) return [ `define internal void @${destroy}(ptr %o) ${FN_ATTRS} {`, @@ -1478,12 +1638,13 @@ export function releaseBody( bounded = true, immortal = false, chained = false, + segmented = false, ): string[] { const S = host.sizeType; if (bounded) host.declare(`declare void @scr_rc_destroy(ptr, ptr)`); const destroy = `${fnName}_destroy`; const lines = [ - ...(bounded ? destroyDefinitions(destroy, freeBody, chained) : []), + ...(bounded ? destroyDefinitions(destroy, freeBody, chained, segmented ? host : null) : []), ...releaseFastPath(host, fnName, `${fnName}_slow`, traced, comment, immortal), `define internal void @${fnName}_slow(ptr %o) noinline ${FN_ATTRS} {${comment ? ` ; ${comment}` : ""}`, `entry:`, @@ -1715,26 +1876,58 @@ export function emitRecordShapes( const chained = directTargets.has(`record:${shape.id}`) || refMembers.some((m) => directDestroyTarget(host, m.type, directTargets) !== null); - for (const m of refMembers) { - freeBody.push( - ` %f${t} = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${m.index}`, - ` %v${t} = load ptr, ptr %f${t}`, + const destroy = `${mangleRecordRelease(shape.id)}_destroy`; + const bounded = refMembers.some((member) => needsBoundedRelease(member.type)); + const segmented = + bounded && + segmentedTeardown( + host, + destroy, + refMembers.map((m) => m.type), + directTargets, ); - const direct = directDestroyTarget(host, m.type, directTargets); - if (direct) { - freeBody.push( - ...directChildReleaseLines(host, `td${t}`, `%v${t}`, direct, llvmCommentText(m.name)), - ); - } else { - freeBody.push( - ` call void ${releaseSym(host, m.type)}(ptr %v${t}) ; ${llvmCommentText(m.name)}`, - ); - } - t++; - } // Closed layouts can carry a kept record source (record-sources.ts). const observers = recordMayKeepSource(shape) ? "weak-or-source" : "weak"; - freeBody.push(...emitObjectFree(host, traced, observers)); + // A segmented teardown loads its fields, releases its own block, and + // then releases the fields (segmentTeardownLines). + const loads: string[] = []; + const ownType: string[] = []; + const release = (prefix: string, m: (typeof refMembers)[number], i: number): string[] => { + const direct = directDestroyTarget(host, m.type, directTargets); + if (direct === null) + return [` call void ${releaseSym(host, m.type)}(ptr %v${i}) ; ${llvmCommentText(m.name)}`]; + const own = segmented && direct.destroy === destroy; + return directChildReleaseLines( + host, + `${prefix}${i}`, + `%v${i}`, + direct, + llvmCommentText(m.name), + own, + ); + }; + for (const m of refMembers) { + const fieldLines = [ + ` %f${t} = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${m.index}`, + ` %v${t} = load ptr, ptr %f${t}`, + ]; + if (segmented) { + loads.push(...fieldLines); + if (directDestroyTarget(host, m.type, directTargets)?.destroy === destroy) + ownType.push(`%v${t}`); + } else freeBody.push(...fieldLines, ...release("td", m, t)); + t++; + } + if (segmented) + freeBody.push( + ...segmentPrologueLines(), + ...loads, + ...emitObjectFree(host, traced, observers, "%sg.own"), + ...segmentTeardownLines(ownType, (prefix) => + refMembers.flatMap((m, i) => release(prefix, m, i)), + ), + ); + else freeBody.push(...emitObjectFree(host, traced, observers)); defs.push( ...releaseBody( host, @@ -1742,9 +1935,10 @@ export function emitRecordShapes( traced, freeBody, "", - refMembers.some((member) => needsBoundedRelease(member.type)), + bounded, false, chained, + segmented, ), ``, ); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7260-teardown-segments.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7260-teardown-segments.ts.json new file mode 100644 index 000000000..a7eac099f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7260-teardown-segments.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7260-teardown-segments.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_cycle.c b/packages/runtime/src/scr_cycle.c index 5e5029e14..1bd0ff7ac 100644 --- a/packages/runtime/src/scr_cycle.c +++ b/packages/runtime/src/scr_cycle.c @@ -293,24 +293,22 @@ static SCR_TL ScrDestroyEntry *scr_destroy_pending; static SCR_TL size_t scr_destroy_count, scr_destroy_capacity; #endif -void scr_rc_destroy(void *obj, void (*destroy)(void *)) { - if (scr_destroy_depth == SCR_DESTROY_DEPTH) { - if (scr_destroy_count == scr_destroy_capacity) { - size_t capacity = scr_destroy_capacity ? scr_destroy_capacity * 2 : 64; - if (capacity < scr_destroy_capacity || capacity > SIZE_MAX / sizeof(ScrDestroyEntry)) - scr_cyc_oom(); - ScrDestroyEntry *entries = realloc(scr_destroy_pending, capacity * sizeof(*entries)); - if (!entries) scr_cyc_oom(); - scr_destroy_pending = entries; - scr_destroy_capacity = capacity; - } - scr_destroy_pending[scr_destroy_count++] = (ScrDestroyEntry){ obj, destroy }; - return; +/* Queues a disposal past the depth budget; the outermost disposal runs it. */ +static void scr_rc_destroy_defer(void *obj, void (*destroy)(void *)) { + if (scr_destroy_count == scr_destroy_capacity) { + size_t capacity = scr_destroy_capacity ? scr_destroy_capacity * 2 : 64; + if (capacity < scr_destroy_capacity || capacity > SIZE_MAX / sizeof(ScrDestroyEntry)) + scr_cyc_oom(); + ScrDestroyEntry *entries = realloc(scr_destroy_pending, capacity * sizeof(*entries)); + if (!entries) scr_cyc_oom(); + scr_destroy_pending = entries; + scr_destroy_capacity = capacity; } - scr_destroy_depth++; - destroy(obj); - scr_destroy_depth--; - if (scr_destroy_depth) return; + scr_destroy_pending[scr_destroy_count++] = (ScrDestroyEntry){ obj, destroy }; +} + +/* Runs the queued disposals once the outermost one has returned. */ +static void scr_rc_destroy_drain(void) { while (scr_destroy_count) { ScrDestroyEntry entry = scr_destroy_pending[--scr_destroy_count]; scr_destroy_depth = 1; @@ -326,6 +324,30 @@ void scr_rc_destroy(void *obj, void (*destroy)(void *)) { } } +void scr_rc_destroy(void *obj, void (*destroy)(void *)) { + if (scr_destroy_depth == SCR_DESTROY_DEPTH) { + scr_rc_destroy_defer(obj, destroy); + return; + } + scr_destroy_depth++; + destroy(obj); + scr_destroy_depth--; + if (scr_destroy_depth == 0) scr_rc_destroy_drain(); +} + +void *scr_rc_destroy_seg(void *obj, void *(*destroy_seg)(void *, void *), + void (*destroy)(void *), void *tail) { + if (scr_destroy_depth == SCR_DESTROY_DEPTH) { + scr_rc_destroy_defer(obj, destroy); + return tail; + } + scr_destroy_depth++; + tail = destroy_seg(obj, tail); + scr_destroy_depth--; + if (scr_destroy_depth == 0) scr_rc_destroy_drain(); + return tail; +} + /* Live cycle-headered objects and the running count of OLD objects freed. * The inline allocation/free paths (scr_runtime.h, llvm/alloc.ts) maintain * both counters so full passes can measure old-generation reclamation. */ diff --git a/packages/runtime/src/scr_rc.h b/packages/runtime/src/scr_rc.h index 7a921b451..83d0589c1 100644 --- a/packages/runtime/src/scr_rc.h +++ b/packages/runtime/src/scr_rc.h @@ -552,6 +552,14 @@ _Static_assert(sizeof(((ScrCycHdr *)0)->gen) == 1 && SCR_CYC_OLD == 2, * depth; deeper objects wait until the outer disposal drains the worklist. * The callback owns the object and must release its children and free it. */ void scr_rc_destroy(void *obj, void (*destroy)(void *)); +/* scr_rc_destroy for an emitted teardown that keeps the blocks it frees in + * a segment, linked through their first words, instead of freeing them onto + * the free list (llvm/shapes.ts): destroy_seg tears obj down, appending its + * blocks after the segment's last link `tail`, and returns the new last + * link, which this returns. A disposal past the depth budget is queued with + * `destroy`, which frees its own blocks, and leaves `tail` as it was. */ +void *scr_rc_destroy_seg(void *obj, void *(*destroy_seg)(void *, void *), + void (*destroy)(void *), void *tail); /* RC hooks for cycle-headered types. on_release: rc was decremented and * stayed above zero — buffer the object as a possible cycle root (may run diff --git a/scripts/ci-test-costs.json b/scripts/ci-test-costs.json index 4c4964579..99d47e972 100644 --- a/scripts/ci-test-costs.json +++ b/scripts/ci-test-costs.json @@ -409,6 +409,7 @@ "tests/harness/stdlib-surface.test.ts": 48, "tests/harness/strip-vt-linear.test.ts": 1795, "tests/harness/surface-manifest.test.ts": 5309, + "tests/harness/teardown-segments.test.ts": 2600, "tests/harness/terminal-geometry.test.ts": 2730, "tests/harness/test262-runner.test.ts": 207, "tests/harness/three-library.test.ts": 251122, @@ -828,6 +829,7 @@ "tests/harness/stdlib-surface.test.ts": 46, "tests/harness/strip-vt-linear.test.ts": 2546, "tests/harness/surface-manifest.test.ts": 5951, + "tests/harness/teardown-segments.test.ts": 5, "tests/harness/terminal-geometry.test.ts": 3270, "tests/harness/test262-runner.test.ts": 230, "tests/harness/three-library.test.ts": 361470, diff --git a/tests/corpus/7260-teardown-segments.ts b/tests/corpus/7260-teardown-segments.ts new file mode 100644 index 000000000..cbd2b144d --- /dev/null +++ b/tests/corpus/7260-teardown-segments.ts @@ -0,0 +1,179 @@ +// Teardowns that hand their blocks back in order. +// +// A typed teardown of a type with children of its own type keeps the blocks +// it frees in one segment and frees the segment when the chain's first +// teardown returns, so the next structure built of that type reuses them in +// order. This program rebuilds structures of such types for many +// generations and reads every value of each generation before dropping it: +// a block handed out twice, or one freed while still in use, shows up as a +// wrong checksum. The shapes cover the ways a segment is put together: +// record literals (allocated before their fields) and constructors whose +// arguments build the children first, lists built by prepending and by +// appending, trees far deeper than one direct teardown chain and lists +// longer than the runtime's whole depth budget (whose tails wait in its +// queue), subtrees still owned elsewhere when their parent goes, records of +// one type holding records of another, and nodes of a cycle-capable class +// that are still candidates of the cycle collector when the last reference +// goes, and rings of them that only the collector frees. + +interface Node { + left: Node | null; + right: Node | null; + value: number; +} + +function literalTree(depth: number, value: number): Node { + return depth === 0 + ? { left: null, right: null, value } + : { left: literalTree(depth - 1, value * 2), right: literalTree(depth - 1, value * 2 + 1), value }; +} + +function sum(node: Node | null): number { + return node === null ? 0 : node.value + sum(node.left) + sum(node.right); +} + +class Branch { + readonly left: Branch | null; + readonly right: Branch | null; + readonly label: string; + constructor(left: Branch | null, right: Branch | null, label: string) { + this.left = left; + this.right = right; + this.label = label; + } +} + +function classTree(depth: number, label: string): Branch { + return depth === 0 + ? new Branch(null, null, label) + : new Branch(classTree(depth - 1, label + "l"), classTree(depth - 1, label + "r"), label); +} + +function labels(branch: Branch | null): number { + return branch === null ? 0 : branch.label.length + labels(branch.left) + labels(branch.right); +} + +interface Cell { + next: Cell | null; + n: number; +} + +function prepended(count: number, base: number): Cell | null { + let head: Cell | null = null; + for (let i = 0; i < count; i++) head = { next: head, n: base + i }; + return head; +} + +function appended(count: number, base: number): Cell | null { + if (count === 0) return null; + const head: Cell = { next: null, n: base }; + let tail = head; + for (let i = 1; i < count; i++) { + const cell: Cell = { next: null, n: base + i }; + tail.next = cell; + tail = cell; + } + return head; +} + +function cells(head: Cell | null): [number, number] { + let count = 0; + let total = 0; + for (let cell = head; cell !== null; cell = cell.next) { + count++; + total += cell.n; + } + return [count, total]; +} + +// Generations of literal trees: shallow, one chain deep, and several chains +// deep. Each generation is checked against the sum the shape must have. +for (const depth of [0, 1, 5, 16, 17, 18]) { + let ok = true; + for (let generation = 0; generation < 6; generation++) { + const tree = literalTree(depth, 1); + const expected = ((2 ** (depth + 1)) * (2 ** (depth + 1) - 1)) / 2; + if (sum(tree) !== expected) ok = false; + } + console.log(`literal trees of depth ${depth}: ${ok}`); +} + +// Generations of class trees built children first. +for (const depth of [3, 12, 17]) { + let total = 0; + for (let generation = 0; generation < 4; generation++) total += labels(classTree(depth, "t")); + console.log(`class trees of depth ${depth}: ${total}`); +} + +// Lists both ways, alternating, so each reuses the other's blocks; the long +// ones outlast the runtime's depth budget. +for (const count of [1, 15, 16, 17, 1000, 3000, 40000]) { + const seen: string[] = []; + for (let generation = 0; generation < 4; generation++) { + const list = generation % 2 === 0 ? prepended(count, generation) : appended(count, generation); + const [length, total] = cells(list); + seen.push(`${length}/${total}`); + } + console.log(`lists of ${count}: ${seen.join(" ")}`); +} + +// Subtrees owned elsewhere survive their parent's teardown, and the blocks +// freed around them are not handed out while they live. +const kept: Node[] = []; +for (let generation = 0; generation < 5; generation++) { + const tree = literalTree(10, generation + 1); + if (tree.left !== null && tree.left.right !== null) kept.push(tree.left.right); + if (tree.right !== null) kept.push(tree.right); + // Building the next tree reuses the dropped blocks. + literalTree(10, 7); +} +console.log(`kept subtrees: ${kept.length} ${kept.map(sum).join(",")}`); + +// A record of one type holding trees of another: the outer teardown frees +// its own blocks as before and tears each inner tree down as a segment. +interface Forest { + first: Node; + second: Node | null; + rest: Forest | null; +} + +function forest(count: number): Forest | null { + let out: Forest | null = null; + for (let i = 0; i < count; i++) out = { first: literalTree(4, i), second: i % 3 === 0 ? null : literalTree(2, i), rest: out }; + return out; +} + +let forestTotal = 0; +for (let generation = 0; generation < 3; generation++) { + for (let f = forest(500); f !== null; f = f.rest) forestTotal += sum(f.first) + sum(f.second); +} +console.log(`forests: ${forestTotal}`); + +// A cycle-capable class: releases of shared links make nodes candidates of +// the cycle collector, which are then torn down while still buffered, and +// the peer links close some rings, which the collector frees. +class Ring { + next: Ring | null = null; + peer: Ring | null = null; + id: number; + constructor(id: number) { + this.id = id; + } +} + +let ringTotal = 0; +for (let generation = 0; generation < 3; generation++) { + let head: Ring | null = null; + const peers: Ring[] = []; + for (let i = 0; i < 2000; i++) { + const ring: Ring = new Ring(i); + ring.next = head; + if (i % 7 === 0) peers.push(ring); + head = ring; + } + for (let r = head; r !== null; r = r.next) if (r.id % 5 === 0) r.peer = peers[r.id % peers.length] ?? null; + for (const peer of peers) ringTotal += peer.id; + peers.length = 0; + for (let r = head; r !== null; r = r.next) ringTotal += r.peer === null ? 0 : r.peer.id; +} +console.log(`rings: ${ringTotal}`); diff --git a/tests/harness/corpus-costs/7260-teardown-segments.ts.json b/tests/harness/corpus-costs/7260-teardown-segments.ts.json new file mode 100644 index 000000000..7f7ce1c2b --- /dev/null +++ b/tests/harness/corpus-costs/7260-teardown-segments.ts.json @@ -0,0 +1 @@ +{ "plain": 5343, "san": 11960 } diff --git a/tests/harness/teardown-segments.test.ts b/tests/harness/teardown-segments.test.ts new file mode 100644 index 000000000..95a2d2795 --- /dev/null +++ b/tests/harness/teardown-segments.test.ts @@ -0,0 +1,97 @@ +/* Teardown segments (llvm/shapes.ts): a typed teardown of a type with + * children of its own type keeps the blocks it frees in a segment and frees + * the segment onto the free list when the chain's first teardown returns. + * Corpus program 7260 pins what such programs print; this suite checks the + * memory side natively, which Node parity cannot: a program that rebuilds + * large trees and lists for a hundred generations reuses every block of the + * last generation, so its resident set stays at about one generation's + * worth. A segment that lost blocks would grow it by a generation each time. + * The sanitized lane frees through the system allocator (no segments) and + * keeps freed memory in quarantine, so the bound is checked in the plain + * lane only. */ +import { execFile } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; + +const exec = promisify(execFile); +const sanitize = process.env["SCRIPTC_SAN"] === "1"; + +const PROGRAM = ` +interface Node { + left: Node | null; + right: Node | null; + value: number; +} +function tree(depth: number): Node { + return depth === 0 + ? { left: null, right: null, value: 1 } + : { left: tree(depth - 1), right: tree(depth - 1), value: depth }; +} +function size(node: Node | null): number { + return node === null ? 0 : 1 + size(node.left) + size(node.right); +} +class Branch { + readonly left: Branch | null; + readonly right: Branch | null; + constructor(left: Branch | null, right: Branch | null) { + this.left = left; + this.right = right; + } +} +function branches(depth: number): Branch { + return depth === 0 ? new Branch(null, null) : new Branch(branches(depth - 1), branches(depth - 1)); +} +function count(branch: Branch | null): number { + return branch === null ? 0 : 1 + count(branch.left) + count(branch.right); +} +interface Cell { + next: Cell | null; + n: number; +} +function cells(n: number): Cell | null { + let head: Cell | null = null; + for (let i = 0; i < n; i++) head = { next: head, n: i }; + return head; +} +let total = 0; +let settled = 0; +for (let generation = 0; generation < 100; generation++) { + total += size(tree(16)) + count(branches(15)); + for (let c = cells(100000); c !== null; c = c.next) total += 1; + if (generation === 2) settled = process.memoryUsage.rss(); +} +const growth = Math.round((process.memoryUsage.rss() - settled) / (1024 * 1024)); +console.log(total, growth); +`; + +test.skipIf(sanitize)( + "rebuilding large trees and lists reuses every block their teardowns free", + { timeout: 300_000 }, + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-segments-")); + try { + const source = join(dir, "churn.ts"); + await writeFile(source, PROGRAM); + const result = await compile(source, { + backend: "llvm", + outDir: dir, + outPath: join(dir, "churn"), + }); + expect(result.ok, JSON.stringify(result.diagnostics)).toBe(true); + if (!result.ok) return; + const { stdout } = await exec(result.binaryPath, [], { timeout: 120_000 }); + const [total, growth] = stdout.trim().split(" ").map(Number); + // 100 generations of 131071 + 65535 nodes and 100000 cells. + expect(total).toBe(100 * (131071 + 65535 + 100000)); + // One generation is about 10 MiB; a lost generation per round would be + // hundreds. + expect(growth).toBeLessThan(8); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, +); From 74fcda0ad93deb463249c9a61c244f5e9d57ded2 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 11:17:42 -0500 Subject: [PATCH 007/184] Evaluate every Promise.all entry before a dynamic entry's then lookup A checked-dynamic entry's PromiseResolve reads its `then` property, which can be a getter. The tuple form resolved each entry as the array was built, so that getter ran before later entries evaluated; Node evaluates the whole array first. With a checked-dynamic entry, every entry now evaluates into a hidden local before the entries resolve. --- .../frontend/lowering/builtins/promises.ts | 28 +++++++++++++++++-- tests/corpus/7035-promise-all-tuple-turns.ts | 13 +++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/builtins/promises.ts b/packages/compiler/src/frontend/lowering/builtins/promises.ts index c30590619..3aecf87e3 100644 --- a/packages/compiler/src/frontend/lowering/builtins/promises.ts +++ b/packages/compiler/src/frontend/lowering/builtins/promises.ts @@ -72,7 +72,28 @@ function lowerTuplePromiseAll( const voidPromise: IrType & { kind: "promise" } = { kind: "promise", inner: VOID }; const entriesT = arrayOf(voidPromise); const dynPromise: IrType = { kind: "promise", inner: DYN }; - const entries = elems.map((e): IrExpr => { + // A checked-dynamic entry's PromiseResolve can run a `then` getter, which + // Node runs after the whole array literal evaluated. With such an entry, + // every entry evaluates into a hidden local first. + const staged: IrStmt[] = []; + const values = !elems.some((e) => e.type.kind === "dyn") + ? elems + : elems.map((e): IrExpr => { + if (e.kind === "unitLit") return e; + if (isUnitType(e.type)) { + staged.push({ kind: "exprStmt", expr: e, loc }); + return { + kind: "unitLit", + unit: e.type.kind === "nullT" ? "null" : "undefined", + type: e.type, + loc, + }; + } + const local = lowerer.declareHiddenLocal("%allValue", e.type); + staged.push({ kind: "varDecl", localId: local.id, init: e, loc }); + return varRef(local.id, e.type, loc); + }); + const entries = values.map((e): IrExpr => { let promise: IrExpr; if (e.type.kind === "promise") promise = e; else if (e.type.kind === "dyn") @@ -164,7 +185,7 @@ function lowerTuplePromiseAll( loc, }); } - return { + const all: IrExpr = { kind: "libCall", fn: "promise.allTuple", args: [ @@ -174,6 +195,9 @@ function lowerTuplePromiseAll( type: resultT, loc, }; + return staged.length === 0 + ? all + : { kind: "seqExpr", stmts: staged, result: all, type: resultT, loc }; } /** `Promise.race([...])` on THE Promise global: the entries lower diff --git a/tests/corpus/7035-promise-all-tuple-turns.ts b/tests/corpus/7035-promise-all-tuple-turns.ts index 6989df9b9..1f8a8803d 100644 --- a/tests/corpus/7035-promise-all-tuple-turns.ts +++ b/tests/corpus/7035-promise-all-tuple-turns.ts @@ -26,6 +26,16 @@ async function text(): Promise { async function both(): Promise<[number, string]> { return Promise.all([num(), text()]); } +const thenable: any = { + get then() { + log.push("then getter"); + return (resolve: (v: number) => void) => resolve(1); + }, +}; +function evaluated(): Promise { + log.push("second entry evaluated"); + return Promise.resolve("s"); +} interface Item { id: number; } @@ -49,6 +59,9 @@ async function main(): Promise { log.push(`returned ${h}${i}`); const mixed = await Promise.all([later("x", 3, "x"), 5, later("y", 1, true), undefined]); log.push(`mixed ${mixed[0]} ${mixed[1]} ${mixed[2]} ${mixed[3]}`); + // A dynamic entry's `then` getter runs after the whole array evaluated. + const [fromThenable, second] = await Promise.all([thenable, evaluated()]); + log.push(`thenable ${fromThenable} ${second}`); const raw: any = JSON.parse('{"value":41}'); const [dynamic, counted] = await Promise.all([raw.value, later("z", 2, 2)]); log.push(`dynamic ${dynamic + counted}`); From 9e1027e13a35c1e6503f4e1b72e9c1aa3bb1af69 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 11:46:44 -0500 Subject: [PATCH 008/184] Run async generator steps in the AsyncLocalStorage context of their resumer An async generator kept the context of its creation for its whole life, and a store its body entered carried into every later step. Node runs each step in the execution that resumes it: a step starts with the context of the next(), return() or throw() call, a store it enters before its first await is the caller's after the call, and after an await the step continues in its own copy. Async generators now start each new step from the resumer's context and hand the prefix's context back at the step's first suspension, as synchronous generator steps and async function prefixes already do; the continuation after an await keeps the generator's own. Two requests that start from the settlement job, which runs from the loop, take Node's contexts too: a request queued behind a running step starts in that step's context, and a return() at a yield, whose unwind waits a turn, runs in the return() call's context. Corpus 7381 and 7382. --- .../7381-als-async-generator-steps.ts.json | 6 + .../7382-als-async-generator-steps.mjs.json | 6 + packages/runtime/src/scr_async.c | 80 +++++++++-- .../corpus/7381-als-async-generator-steps.ts | 133 ++++++++++++++++++ .../corpus/7382-als-async-generator-steps.mjs | 43 ++++++ .../7381-als-async-generator-steps.ts.json | 1 + .../7382-als-async-generator-steps.mjs.json | 1 + 7 files changed, 257 insertions(+), 13 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7381-als-async-generator-steps.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7382-als-async-generator-steps.mjs.json create mode 100644 tests/corpus/7381-als-async-generator-steps.ts create mode 100644 tests/corpus/7382-als-async-generator-steps.mjs create mode 100644 tests/harness/corpus-costs/7381-als-async-generator-steps.ts.json create mode 100644 tests/harness/corpus-costs/7382-als-async-generator-steps.mjs.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7381-als-async-generator-steps.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7381-als-async-generator-steps.ts.json new file mode 100644 index 000000000..47f93314d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7381-als-async-generator-steps.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7381-als-async-generator-steps.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7382-als-async-generator-steps.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7382-als-async-generator-steps.mjs.json new file mode 100644 index 000000000..5f55ba7bd --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7382-als-async-generator-steps.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7382-als-async-generator-steps.mjs" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_async.c b/packages/runtime/src/scr_async.c index 90e7dd04f..e6184ed9b 100644 --- a/packages/runtime/src/scr_async.c +++ b/packages/runtime/src/scr_async.c @@ -715,11 +715,13 @@ static void scr_als_prefix_end(ScrFiber *f) { } } -/* A synchronous generator has no context of its own: each step runs in its - * resumer's execution, as Node runs it. A fiber-backed one switches to its - * own slot, so its step starts from the resumer's store and a store it - * enters is the resumer's after the step (scr_als_prefix_end on the way - * back); a stackless one runs in the resumer's slot (scr_gen_coro_hop). */ +/* A generator step runs in its resumer's execution, as Node runs it. A + * fiber-backed generator switches to its own slot, so its step starts from + * the resumer's store and a store it enters is the resumer's after the + * step (scr_als_prefix_end on the way back); a stackless synchronous one + * runs in the resumer's slot (scr_gen_coro_hop). An async generator's step + * starts the same way, and after an await it continues in its own copy of + * the context, as an async function's body does after its prefix. */ static void scr_als_step_begin(ScrFiber *f) { ScrAlsCtx **slot = SCR_ALS_SLOT(); if (f->als != *slot) { @@ -5303,6 +5305,10 @@ struct ScrGen { * since that return takes its own turn to await x. */ bool return_queued; bool return_awaited; + /* The AsyncLocalStorage context of a return() request deferred at a + * yield (+1): the body's unwind runs in it, as Node resumes it from the + * return value's await. */ + ScrAlsCtx *return_ctx; #ifdef SCR_WORKERS bool context_abandoned; #endif @@ -5406,6 +5412,7 @@ void scr_gen_release(ScrGen *g) { scr_gen_slot_reset(&g->in); scr_gen_slot_reset(&g->ret); scr_gen_exc_reset(&g->async_error); + scr_als_ctx_release(g->return_ctx); if (g->fiber != NULL) { if (g->state == SCR_GEN_UNSTARTED) { /* Never ran: nothing on the stack owns anything — clean teardown. @@ -5760,6 +5767,12 @@ __attribute__((always_inline)) static inline void scr_gen_switched_back(ScrGen * if (release) scr_gen_release(g); } +/* Out of line, so the hop that synchronous generators share stays as it + * was: an async generator's step begins from its resumer's context and + * hands the prefix's context back at its first suspension. */ +static __attribute__((noinline)) void scr_async_gen_step_begin(ScrFiber *f) { scr_als_step_begin(f); } +static __attribute__((noinline)) void scr_async_gen_step_end(ScrFiber *f) { scr_als_prefix_end(f); } + #ifndef __wasi__ /* The stackless form of the hop: the coroutine frame runs on this stack * until its next yield or its final suspension returns here. `retain` @@ -5768,14 +5781,17 @@ __attribute__((always_inline)) static inline void scr_gen_switched_back(ScrGen * __attribute__((always_inline)) static inline void scr_gen_coro_hop(ScrGen *g, bool retain) { ScrFiber *f = g->fiber; if (retain) scr_gen_retain(g); + /* A synchronous generator has no AsyncLocalStorage context of its own: + * its step runs in the resumer's slot, as Node runs it in the execution + * that called next(). An async generator's step runs in its own slot, + * which a new step (not the continuation after an await) starts from the + * resumer's context (scr_als_step_begin). */ + if (g->is_async && g->state != SCR_GEN_AWAITING) scr_async_gen_step_begin(f); g->state = SCR_GEN_RUNNING; f->resumer = scr_current; ScrAlsCtx **resumer_slot = scr_als_active; scr_current = f; scr_exc_swap_cell(&f->exc); - /* A synchronous generator has no AsyncLocalStorage context of its own: - * its step runs in the resumer's slot, as Node runs it in the execution - * that called next(). An async generator's step runs in its own. */ if (g->is_async) scr_als_active = &f->als; if (f->coro == NULL) f->entry(f, f->argpack); else scr_coro_resume(f->coro); @@ -5786,6 +5802,7 @@ __attribute__((always_inline)) static inline void scr_gen_coro_hop(ScrGen *g, bo scr_current = me; scr_exc_swap_cell(me != NULL ? &me->exc : NULL); scr_als_active = resumer_slot; + if (g->is_async) scr_async_gen_step_end(f); scr_gen_switched_back(g, f, retain, true); } @@ -5806,8 +5823,10 @@ static void scr_gen_switch_in(ScrGen *g) { } #endif scr_gen_retain(g); + /* Each step starts from the resumer's context; an async generator's + * continuation after an await keeps its own. */ + if (!g->is_async || g->state != SCR_GEN_AWAITING) scr_als_step_begin(f); g->state = SCR_GEN_RUNNING; - if (!g->is_async) scr_als_step_begin(f); #ifdef _WIN32 ScrCtx here = scr_win_self(); #elif defined(__wasi__) @@ -5979,6 +5998,25 @@ void scr_gen_resume_throw(ScrGen *g) { static void scr_async_gen_start_queued(ScrGen *g); +/* Runs fn(g) with `ctx` (borrowed) as the active AsyncLocalStorage + * context. A settlement job runs from the loop: a request queued behind + * another starts in the context of the step before it (Node drains the + * queue in the generator's own execution), and a deferred return() in its + * call's. */ +static __attribute__((noinline)) void scr_async_gen_in_context(ScrGen *g, ScrAlsCtx *ctx, + void (*fn)(ScrGen *)) { + ScrAlsCtx **slot = SCR_ALS_SLOT(); + ScrAlsCtx *prev = *slot; + *slot = scr_als_ctx_retain(ctx); + fn(g); + scr_als_restore(prev); +} + +/* Starts the queued requests in the context of the step before them. */ +static __attribute__((noinline)) void scr_async_gen_start_queued_in_own(ScrGen *g) { + scr_async_gen_in_context(g, g->fiber != NULL ? g->fiber->als : NULL, scr_async_gen_start_queued); +} + static void scr_async_gen_schedule_settle(ScrGen *g) { if (g->settle_queued) return; ScrFiber *job = scr_mem_calloc(sizeof *job); @@ -6026,6 +6064,11 @@ static bool scr_async_gen_settles_now(ScrGen *g) { return false; } +static __attribute__((noinline)) void scr_async_gen_capture_return_ctx(ScrGen *g) { + scr_als_ctx_release(g->return_ctx); + g->return_ctx = scr_als_ctx_retain(*SCR_ALS_SLOT()); +} + static void scr_async_gen_run_active(ScrGen *g, ScrAsyncGenRequest *req) { const bool was_done = g->state == SCR_GEN_DONE; if (req->mode == SCR_AG_NEXT) { @@ -6041,6 +6084,7 @@ static void scr_async_gen_run_active(ScrGen *g, ScrAsyncGenRequest *req) { /* At a yield, AsyncGeneratorYield awaits the return value before * the body resumes, so finally blocks run one turn later. */ g->return_queued = true; + scr_async_gen_capture_return_ctx(g); scr_async_gen_schedule_settle(g); return; } @@ -6071,6 +6115,16 @@ static void scr_async_gen_run_active(ScrGen *g, ScrAsyncGenRequest *req) { } } +static void scr_async_gen_resume_returning(ScrGen *g); + +/* The deferred return() resumes in its call's context. */ +static __attribute__((noinline)) void scr_async_gen_resume_returning_in_context(ScrGen *g) { + ScrAlsCtx *ctx = g->return_ctx; + g->return_ctx = NULL; + scr_async_gen_in_context(g, ctx, scr_async_gen_resume_returning); + scr_als_ctx_release(ctx); +} + /* The return() request's await has taken its turn: resume the body with * the return completion. */ static void scr_async_gen_resume_returning(ScrGen *g) { @@ -6203,12 +6257,12 @@ static void scr_async_gen_job(ScrGen *g) { } if (g->return_queued) { scr_gen_retain(g); - scr_async_gen_resume_returning(g); + scr_async_gen_resume_returning_in_context(g); scr_gen_release(g); return; } scr_async_gen_finish_active(g, false); - scr_async_gen_start_queued(g); + scr_async_gen_start_queued_in_own(g); } static void scr_async_gen_resume_ready(ScrGen *g) { @@ -6222,11 +6276,11 @@ static void scr_async_gen_resume_ready(ScrGen *g) { scr_gen_switch_in(g); if (g->async_error.kind != SCR_EXC_NONE) { scr_async_gen_finish_active(g, true); - scr_async_gen_start_queued(g); + scr_async_gen_start_queued_in_own(g); } else if (g->state != SCR_GEN_AWAITING) { if (scr_async_gen_settles_now(g)) { scr_async_gen_finish_active(g, false); - scr_async_gen_start_queued(g); + scr_async_gen_start_queued_in_own(g); } else { scr_async_gen_schedule_settle(g); } diff --git a/tests/corpus/7381-als-async-generator-steps.ts b/tests/corpus/7381-als-async-generator-steps.ts new file mode 100644 index 000000000..4f6ace58d --- /dev/null +++ b/tests/corpus/7381-als-async-generator-steps.ts @@ -0,0 +1,133 @@ +// AsyncLocalStorage contexts of async generator steps. Each step starts in +// the context of the next(), return() or throw() call that resumes it, +// whatever context created the generator; a store the step enters before +// its first await is the caller's after the call, and after an await the +// step keeps its own copy. A request queued behind a running step starts +// in that step's context, and a return() at a yield unwinds in the +// return() call's context. Covers yield*, for await with break, and two +// generators resumed in turn. +import { AsyncLocalStorage } from "node:async_hooks"; + +const store = new AsyncLocalStorage(); +const lines: string[] = []; +const log = (what: string): void => { + lines.push(`${what}: ${store.getStore() ?? "none"}`); +}; + +async function* steps(): AsyncGenerator { + log("body start"); + yield 1; + log("after the first yield"); + await null; + log("after an await"); + store.enterWith("inside"); + yield 2; + log("after the second yield"); + yield 3; + log("body end"); +} +const it = store.run("create", () => steps()); +await store.run("next-1", () => it.next()); +log("caller after next-1"); +await store.run("next-2", () => it.next()); +log("caller after next-2"); +await store.run("next-3", () => it.next()); +await store.run("next-4", () => it.next()); + +async function* prefixes(): AsyncGenerator { + store.enterWith("prefix"); + yield 1; + store.enterWith("step-2"); + await null; + log("after an await in step 2"); + yield 2; +} +await store.run("outer", async () => { + const gen = prefixes(); + const first = gen.next(); + log("synchronously after next"); + await first; + log("after awaiting next"); + const second = gen.next(); + log("synchronously after the second next"); + await second; + log("after awaiting the second next"); +}); + +async function* queued(): AsyncGenerator { + log("queued start"); + yield 1; + log("queued resumed"); + yield 2; +} +const q = queued(); +await Promise.all([store.run("q-1", () => q.next()), store.run("q-2", () => q.next())]); + +async function* inner(): AsyncGenerator { + log("inner start"); + yield "a"; + log("inner resumed"); + yield "b"; +} +async function* outerGen(): AsyncGenerator { + log("outer start"); + yield* inner(); + log("outer after yield*"); +} +const delegating = outerGen(); +await store.run("o-1", () => delegating.next()); +await store.run("o-2", () => delegating.next()); +await store.run("o-3", () => delegating.next()); + +async function* catching(): AsyncGenerator { + try { + yield 1; + } catch (e) { + log(`caught ${(e as Error).message}`); + yield 2; + } +} +const c = catching(); +await store.run("t-1", () => c.next()); +await store.run("t-2", () => c.throw(new Error("boom"))); + +async function* finallyGen(): AsyncGenerator { + try { + yield 1; + } finally { + log("finally"); + } +} +const f = finallyGen(); +await store.run("n", () => f.next()); +await store.run("return-call", () => f.return()); + +async function* endless(): AsyncGenerator { + let i = 0; + try { + while (true) { + log(`endless ${i}`); + yield i++; + } + } finally { + log("endless finally"); + } +} +await store.run("loop", async () => { + for await (const v of endless()) { + if (v === 1) break; + } +}); + +async function* named(name: string): AsyncGenerator { + log(`${name} 1`); + yield 1; + log(`${name} 2`); + yield 2; +} +const a = named("A"); +const b = named("B"); +await Promise.all([store.run("a-1", () => a.next()), store.run("b-1", () => b.next())]); +await Promise.all([store.run("b-2", () => b.next()), store.run("a-2", () => a.next())]); + +for (const line of lines) console.log(line); diff --git a/tests/corpus/7382-als-async-generator-steps.mjs b/tests/corpus/7382-als-async-generator-steps.mjs new file mode 100644 index 000000000..bbabdb592 --- /dev/null +++ b/tests/corpus/7382-als-async-generator-steps.mjs @@ -0,0 +1,43 @@ +// AsyncLocalStorage contexts of async generator steps from JavaScript: +// steps start in the context of the call that resumes them, a store a step +// enters before its first await is the caller's, queued requests start in +// the running step's context, and return() at a yield unwinds in the +// return() call's context. +import { AsyncLocalStorage } from "node:async_hooks"; +const s = new AsyncLocalStorage(); +const lines = []; +const g = (t) => lines.push(`${t}: ${s.getStore() ?? "none"}`); +async function* gen() { + g("body start"); + yield 1; + g("after first yield"); + await null; + g("after await"); + s.enterWith("inside"); + yield 2; + g("after second yield"); + yield 3; + g("body end"); +} +const it = s.run("create", () => gen()); +await s.run("next1", () => it.next()); +g("caller after next1"); +await s.run("next2", () => it.next()); +g("caller after next2"); +await s.run("next3", () => it.next()); +await s.run("next4", () => it.next()); +// concurrent next calls queued +async function* gen2() { g("g2 start"); yield 1; g("g2 resumed"); yield 2; } +const it2 = gen2(); +const p1 = s.run("q1", () => it2.next()); +const p2 = s.run("q2", () => it2.next()); +await Promise.all([p1, p2]); +// for await +async function* gen3() { for (let i = 0; i < 2; i++) { g("g3 step " + i); yield i; } } +await s.run("for-await", async () => { for await (const v of gen3()) g("loop body " + v); }); +// return/throw +async function* gen4() { try { yield 1; } finally { g("g4 finally"); } } +const it4 = gen4(); +await s.run("n", () => it4.next()); +await s.run("ret", () => it4.return()); +for (const line of lines) console.log(line); diff --git a/tests/harness/corpus-costs/7381-als-async-generator-steps.ts.json b/tests/harness/corpus-costs/7381-als-async-generator-steps.ts.json new file mode 100644 index 000000000..3f7384f20 --- /dev/null +++ b/tests/harness/corpus-costs/7381-als-async-generator-steps.ts.json @@ -0,0 +1 @@ +{ "plain": 3500, "san": 6000 } diff --git a/tests/harness/corpus-costs/7382-als-async-generator-steps.mjs.json b/tests/harness/corpus-costs/7382-als-async-generator-steps.mjs.json new file mode 100644 index 000000000..3f7384f20 --- /dev/null +++ b/tests/harness/corpus-costs/7382-als-async-generator-steps.mjs.json @@ -0,0 +1 @@ +{ "plain": 3500, "san": 6000 } From 35f2d8f791022b779972e54a55d2b369ab657fef Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 13:07:47 -0500 Subject: [PATCH 009/184] Report dictionary copies of `{}` views, and keep method calls direct after writes into dictionaries of dictionaries Fixes #1408 Co-authored-by: Chris Ainsley --- .../record-identity-and-property-order.mdx | 1 + .../compiler/src/diagnostics/diagnostic.ts | 10 +- .../lowering/class-method-mutations.ts | 16 ++- .../src/frontend/lowering/divergences.ts | 110 +++++++++++++++++- .../test/class-method-mutations.test.ts | 42 +++++++ ...onary-of-dictionaries-keep-methods.ts.json | 6 + ...dictionary-of-dictionaries-keep-methods.ts | 38 ++++++ .../dictionary-view-copies.ts | 29 +++++ tests/coverage-fixtures/no-divergences.ts | 11 +- .../coverage-dictionary-view-copies.txt | 21 ++++ ...onary-of-dictionaries-keep-methods.ts.json | 1 + tests/harness/coverage.test.ts | 14 +++ 12 files changed, 288 insertions(+), 11 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7010-dictionary-of-dictionaries-keep-methods.ts.json create mode 100644 tests/corpus/7010-dictionary-of-dictionaries-keep-methods.ts create mode 100644 tests/coverage-fixtures/dictionary-view-copies.ts create mode 100644 tests/harness/__snapshots__/coverage-dictionary-view-copies.txt create mode 100644 tests/harness/corpus-costs/7010-dictionary-of-dictionaries-keep-methods.ts.json diff --git a/docs/content/limitations/record-identity-and-property-order.mdx b/docs/content/limitations/record-identity-and-property-order.mdx index 0381803f6..224757d52 100644 --- a/docs/content/limitations/record-identity-and-property-order.mdx +++ b/docs/content/limitations/record-identity-and-property-order.mdx @@ -1,6 +1,7 @@ A record converted to a record type with a different set of properties, such as a narrower shape or one that adds optional properties, is copied. Mutations through the converted reference do not affect the original, and the copy is not `===` to it. Empty record layouts are also snapshots because they have no property storage. +A class instance or record held as `{}`, `object` or `unknown` and then converted to a dictionary type such as `Record` is copied as well; a dictionary of `unknown` values keeps the original object. `scriptc coverage` and `scriptc build` report `SC6001` where a copied value is then written or compared, whether the record was passed as an argument, stored in a typed binding, asserted with `as`, assigned to a typed binding later, or stored as an element of a typed array literal. Typed records with declared fields, indexed records, and arrays retain identity and mutations through native checked storage. diff --git a/packages/compiler/src/diagnostics/diagnostic.ts b/packages/compiler/src/diagnostics/diagnostic.ts index 17cb2efc0..47db2193b 100644 --- a/packages/compiler/src/diagnostics/diagnostic.ts +++ b/packages/compiler/src/diagnostics/diagnostic.ts @@ -129,6 +129,7 @@ export function widthCopyDivergenceDiag( useText: string, site: "call" | "declaration" | "cast" | "element", loc: SrcLoc, + dictionary = false, ): ScrDiagnostic { const how = site === "call" @@ -142,11 +143,16 @@ export function widthCopyDivergenceDiag( use === "write" ? `the write to '${useText}' updates only the copy, while under Node it updates the original object` : `'${useText}' compares the copy, which is never the original object, while under Node it is the same object`; + const cause = dictionary + ? "converting an object held as '{}', 'object' or 'unknown' to a dictionary type" + : "converting to a record type with other properties"; return { code: "SC6001", - message: `${how} copies the object (converting to a record type with other properties compiles as a field copy): ${effect}`, + message: `${how} copies the object (${cause} compiles as a field copy): ${effect}`, loc, - hint: `declare the parameter or binding with the value's full type ('${sourceType}'), or make the shape a class — class instances keep their identity when passed as a base class`, + hint: dictionary + ? `keep the value at its own type ('${sourceType}'), or declare the dictionary's values as 'unknown', which keeps the original object` + : `declare the parameter or binding with the value's full type ('${sourceType}'), or make the shape a class — class instances keep their identity when passed as a base class`, }; } diff --git a/packages/compiler/src/frontend/lowering/class-method-mutations.ts b/packages/compiler/src/frontend/lowering/class-method-mutations.ts index abbaf3a75..7629377d4 100644 --- a/packages/compiler/src/frontend/lowering/class-method-mutations.ts +++ b/packages/compiler/src/frontend/lowering/class-method-mutations.ts @@ -125,9 +125,12 @@ export function collectClassMethodMutations( }; // Whether no function value can inhabit `type`. Primitives qualify, and so // do non-callable object types a function is not assignable to: one that - // requires a property functions lack, or a weak (all-optional) type that + // requires a property functions lack, a weak (all-optional) type that // shares no property with functions (TypeScript's weak-type rule rejects - // callable sources too). + // callable sources too), or a dictionary whose string index values are + // neither `any` nor a type parameter: a function type has no index + // signature, and only an `any` index admits a source without one (an + // instantiation can make a type parameter `any`). const functionFree = (type: ts.Type): boolean => { if (type.isUnionType()) return ts.constituentTypes(type).every(functionFree); if ( @@ -149,7 +152,14 @@ export function collectClassMethodMutations( ) return false; const properties = lowerer.checker.getPropertiesOfType(type); - if (properties.length === 0) return false; + if (properties.length === 0) + return lowerer.checker + .getIndexInfosOfType(type) + .some( + (index) => + (index.keyType.flags & ts.TypeFlags.StringLike) !== 0 && + (index.type.flags & (ts.TypeFlags.Any | ts.TypeFlags.Instantiable)) === 0, + ); const required = properties.filter( (property) => (property.flags & ts.SymbolFlags.Optional) === 0, ); diff --git a/packages/compiler/src/frontend/lowering/divergences.ts b/packages/compiler/src/frontend/lowering/divergences.ts index 1c362cb09..f766aba14 100644 --- a/packages/compiler/src/frontend/lowering/divergences.ts +++ b/packages/compiler/src/frontend/lowering/divergences.ts @@ -180,7 +180,15 @@ export function scanDivergences( const copy = safely(() => widthCopyBetween(lowerer, arg, param.type!)); if (copy === undefined || copy === null) return; out.push( - widthCopyDivergenceDiag(copy.source, copy.target, use.kind, use.text, "call", locOf(arg)), + widthCopyDivergenceDiag( + copy.source, + copy.target, + use.kind, + use.text, + "call", + locOf(arg), + copy.dictionary, + ), ); }); } @@ -200,6 +208,7 @@ export function scanDivergences( use.text, "declaration", locOf(decl.initializer!), + copy.dictionary, ), ); } @@ -215,7 +224,15 @@ export function scanDivergences( const copy = safely(() => widthCopyBetween(lowerer, source, type)); if (copy === undefined || copy === null) continue; out.push( - widthCopyDivergenceDiag(copy.source, copy.target, use.kind, use.text, "cast", locOf(source)), + widthCopyDivergenceDiag( + copy.source, + copy.target, + use.kind, + use.text, + "cast", + locOf(source), + copy.dictionary, + ), ); } for (const assignment of assignments) { @@ -240,6 +257,7 @@ export function scanDivergences( use.text, "declaration", locOf(assignment.right), + copy.dictionary, ), ); } @@ -254,7 +272,15 @@ export function scanDivergences( const copy = safely(() => widthCopyBetween(lowerer, item, element)); if (copy === undefined || copy === null) continue; out.push( - widthCopyDivergenceDiag(copy.source, copy.target, "write", use, "element", locOf(item)), + widthCopyDivergenceDiag( + copy.source, + copy.target, + "write", + use, + "element", + locOf(item), + copy.dictionary, + ), ); break; } @@ -653,14 +679,23 @@ function enclosingScope(node: ts.Node): ts.Node | undefined { * instance flowing into a plain record type that does not keep class * identity (ClassProtocols marks the targets a class instance visibly * flows into; a value reaching another record type is a field copy). */ +/** The types a width copy converts between, and whether the target is a + * dictionary. */ +interface WidthCopy { + source: string; + target: string; + dictionary?: boolean; +} + function widthCopyBetween( lowerer: Lowerer, source: ts.Expression, targetNode: ts.TypeNode, -): { source: string; target: string } | null { +): WidthCopy | null { const checker = lowerer.checker; const s = checker.getTypeAtLocation(source); const t = checker.getTypeFromTypeNode(targetNode); + if (isDictionaryType(checker, t)) return dictionaryViewCopy(lowerer, source, s, t); if (!isPlainRecordType(checker, t)) return null; const classSource = isProgramClassType(checker, s); if (!classSource && !isPlainRecordType(checker, s)) return null; @@ -680,6 +715,73 @@ function widthCopyBetween( return { source: checker.typeToString(s), target: checker.typeToString(t) }; } +/** A copy made by converting a checked-dynamic view (`{}`, `object`, + * `unknown`) to a dictionary type that compiles to a native record (a + * dictionary of `unknown` values stays checked-dynamic and keeps the + * object). A dictionary of that type comes back out of the view as itself, + * so a copy is reported only when the view is a constant binding + * initialized with a class instance or a record: that object is converted + * field by field into a new dictionary. */ +function dictionaryViewCopy( + lowerer: Lowerer, + source: ts.Expression, + s: ts.Type, + t: ts.Type, +): WidthCopy | null { + const checker = lowerer.checker; + if (!isDynamicViewType(checker, s)) return null; + if (lowerer.mapTypeOf(t)?.kind !== "record") return null; + while (ts.isParenthesizedExpression(source) || ts.isNonNullExpression(source)) + source = source.expression; + if (!ts.isIdentifier(source)) return null; + const decl = checker.getSymbolAtLocation(source)?.valueDeclaration; + if ( + decl === undefined || + !ts.isVariableDeclaration(decl) || + decl.initializer === undefined || + !ts.isVariableDeclarationList(decl.parent) || + !(decl.parent.flags & ts.NodeFlags.Const) + ) + return null; + let origin = decl.initializer; + while ( + ts.isParenthesizedExpression(origin) || + ts.isNonNullExpression(origin) || + ts.isAsExpression(origin) || + ts.isTypeAssertion(origin) + ) + origin = origin.expression; + const o = checker.getTypeAtLocation(origin); + if (!isProgramClassType(checker, o) && !isPlainRecordType(checker, o)) return null; + return { source: checker.typeToString(o), target: checker.typeToString(t), dictionary: true }; +} + +/** `{}`, `object` and `unknown`: types whose values lower to checked + * dynamic storage that keeps the original object. */ +function isDynamicViewType(checker: ts.TypeChecker, type: ts.Type): boolean { + if (type.flags & (ts.TypeFlags.Unknown | ts.TypeFlags.NonPrimitive)) return true; + return ( + (type.flags & ts.TypeFlags.Object) !== 0 && + checker.getPropertiesOfType(type).length === 0 && + checker.getIndexInfosOfType(type).length === 0 && + checker.getCallSignatures(type).length === 0 && + checker.getConstructSignatures(type).length === 0 + ); +} + +/** A string-keyed dictionary with no declared properties + * (`Record`, `{ [key: string]: number }`). */ +function isDictionaryType(checker: ts.TypeChecker, type: ts.Type): boolean { + if ((type.flags & ts.TypeFlags.Object) === 0) return false; + if (checker.isArrayType(type) || checker.isTupleType(type)) return false; + if (checker.getCallSignatures(type).length > 0) return false; + if (checker.getConstructSignatures(type).length > 0) return false; + if (checker.getPropertiesOfType(type).length > 0) return false; + return checker + .getIndexInfosOfType(type) + .some((index) => (index.keyType.flags & ts.TypeFlags.String) !== 0); +} + function isProgramClassType(checker: ts.TypeChecker, type: ts.Type): boolean { if ((type.flags & ts.TypeFlags.Object) === 0) return false; const symbol = type.getSymbol(); diff --git a/packages/compiler/test/class-method-mutations.test.ts b/packages/compiler/test/class-method-mutations.test.ts index b508a3949..3221e44ee 100644 --- a/packages/compiler/test/class-method-mutations.test.ts +++ b/packages/compiler/test/class-method-mutations.test.ts @@ -135,3 +135,45 @@ test("a reflection call with a well-known symbol key keeps checked dispatch", as const callers = await callersOf(`${range}${write}`, "%Range.sym:iterator"); expect(callers.filter((name) => !name.startsWith("%method.value:"))).toEqual([]); }); + +const groups = ` +class Tally { + n = 0; + add(): void { this.n++; } +} +interface Settings { groups: VALUE } +function addGroup(settings: Settings, name: string): void { + settings.groups[name] = {}; +} +const settings: Settings = { groups: {} }; +const tally = new Tally(); +addGroup(settings, String(tally.n)); +tally.add(); +console.log(tally.n, Object.keys(settings.groups)); +`; + +test("a computed write into a dictionary of dictionaries leaves method calls direct", async () => { + const source = groups.replace("VALUE", "Record>"); + expect(await callersOf(source, "%dyn.class.properties")).toEqual([]); +}); + +test("a computed write into a dictionary that can hold functions keeps checked dispatch", async () => { + for (const value of [ + "Record | (() => void)>", + "Record>", + ]) { + const source = groups.replace("VALUE", value); + expect(await callersOf(source, "%dyn.class.properties")).not.toEqual([]); + } +}); + +test("a computed write into a generic dictionary of dictionaries keeps checked dispatch", async () => { + const source = groups + .replace( + "interface Settings { groups: VALUE }", + "interface Settings { groups: Record> }", + ) + .replace("function addGroup(settings: Settings,", "function addGroup(settings: Settings,") + .replace("const settings: Settings =", "const settings: Settings ="); + expect(await callersOf(source, "%dyn.class.properties")).not.toEqual([]); +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7010-dictionary-of-dictionaries-keep-methods.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7010-dictionary-of-dictionaries-keep-methods.ts.json new file mode 100644 index 000000000..cc3c9dabe --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7010-dictionary-of-dictionaries-keep-methods.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7010-dictionary-of-dictionaries-keep-methods.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7010-dictionary-of-dictionaries-keep-methods.ts b/tests/corpus/7010-dictionary-of-dictionaries-keep-methods.ts new file mode 100644 index 000000000..a53e82f83 --- /dev/null +++ b/tests/corpus/7010-dictionary-of-dictionaries-keep-methods.ts @@ -0,0 +1,38 @@ +// A string-keyed dictionary whose values are dictionaries cannot hold a +// function, so storing into it under a key known only at runtime leaves +// class methods in place: their calls stay direct, and a record a method +// returns is the record the instance holds, so writes through it are kept. +interface Data { + items: string[]; +} +class Store { + data: Data = { items: [] }; + get(): Data { + return this.data; + } +} +class Counter { + count = 0; + bump(): number { + return ++this.count; + } +} +interface Settings { + groups: Record>; +} +function addGroup(settings: Settings, name: string): void { + settings.groups[name] = {}; +} + +const settings: Settings = { groups: {} }; +const name = process.argv.length > 99 ? "never" : "colours"; +addGroup(settings, name); +const group = settings.groups[name]; +if (group !== undefined) group["sky"] = "blue"; +const store = new Store(); +store.get().items.push("a"); +store.get().items.push("b"); +const counter = new Counter(); +let total = 0; +for (let i = 0; i < 1000; i++) total += counter.bump(); +console.log(JSON.stringify(settings.groups), store.data.items.join(","), total); diff --git a/tests/coverage-fixtures/dictionary-view-copies.ts b/tests/coverage-fixtures/dictionary-view-copies.ts new file mode 100644 index 000000000..bfcfa61f4 --- /dev/null +++ b/tests/coverage-fixtures/dictionary-view-copies.ts @@ -0,0 +1,29 @@ +// A class instance or record held as `{}`, `object` or `unknown` keeps the +// original object, and converting it to a dictionary type copies it field +// by field. Each site below then writes through or compares the copy. +class Counter { + label = "c"; + add(): number { + return this.label.length; + } +} +const key = process.argv.length > 5 ? "z" : "b"; + +const counter = new Counter(); +const view: {} = counter; +const table: Record = view; +table[key] = "y"; + +const point = { a: "x" }; +const loose: object = point; +const dict = loose as Record; +dict[key] = "y"; + +const held: unknown = { a: "x" }; +const fromUnknown = held as Record; +console.log(fromUnknown === held); + +const empty: {} = {}; +const filled: Record = empty; +filled[key] = "y"; +console.log(JSON.stringify(counter), JSON.stringify(point), JSON.stringify(empty)); diff --git a/tests/coverage-fixtures/no-divergences.ts b/tests/coverage-fixtures/no-divergences.ts index 3739336e3..ca76bcc32 100644 --- a/tests/coverage-fixtures/no-divergences.ts +++ b/tests/coverage-fixtures/no-divergences.ts @@ -1,6 +1,6 @@ // Ordinary code: identity-preserving calls, declaration-ordered literals, -// Latin-script comparisons, related-class casts, ISO dates and views of the -// global object report no divergence. +// Latin-script comparisons, related-class casts, ISO dates, views of the +// global object and a dictionary held as `{}` report no divergence. interface Item { name: string; qty: number; @@ -38,3 +38,10 @@ const COUNT = Symbol.for("no-divergences.count"); const globalState = globalThis as { [COUNT]?: { n: number } }; globalState[COUNT] = { n: 1 }; console.log(globalState[COUNT]?.n); + +// A dictionary held as `{}` comes back out as itself. +const counts: Record = { a: "x" }; +const opaque: {} = counts; +const again: Record = opaque; +again[counts["a"] ?? "a"] = "y"; +console.log(JSON.stringify(counts)); diff --git a/tests/harness/__snapshots__/coverage-dictionary-view-copies.txt b/tests/harness/__snapshots__/coverage-dictionary-view-copies.txt new file mode 100644 index 000000000..2359a4ffa --- /dev/null +++ b/tests/harness/__snapshots__/coverage-dictionary-view-copies.txt @@ -0,0 +1,21 @@ +scriptc coverage tests/coverage-fixtures/dictionary-view-copies.ts + + statements analyzed 18 + compile statically 18 (100%) + in dynamic tier 3 (operate on values typed only at runtime; --explain-dynamic lists them) + + fully static — all analyzed statements compile to native code. + + differs from Node 4 sites (compiles; the native program can behave differently) + ×1 SC6001 storing a 'Counter' in a 'Record<…>' binding copies the object (converting an object held as '{}', 'object' or 'unknown' to a dictionary type compiles as a field copy): the write to 'table[key]' upda… + at tests/coverage-fixtures/dictionary-view-copies.ts:14:39 + hint: keep the value at its own type ('Counter'), or declare the dictionary's values as 'unknown', which keeps the original object + ×1 SC6001 storing a '{}' in a 'Record<…>' binding copies the object (converting an object held as '{}', 'object' or 'unknown' to a dictionary type compiles as a field copy): the write to 'filled[key]' updates … + at tests/coverage-fixtures/dictionary-view-copies.ts:27:40 + hint: keep the value at its own type ('{}'), or declare the dictionary's values as 'unknown', which keeps the original object + ×1 SC6001 asserting a '{ a: string; }' as 'Record<…>' copies the object (converting an object held as '{}', 'object' or 'unknown' to a dictionary type compiles as a field copy): the write to 'dict[key]' update… + at tests/coverage-fixtures/dictionary-view-copies.ts:19:14 + hint: keep the value at its own type ('{ a: string; }'), or declare the dictionary's values as 'unknown', which keeps the original object + ×1 SC6001 asserting a '{ a: string; }' as 'Record<…>' copies the object (converting an object held as '{}', 'object' or 'unknown' to a dictionary type compiles as a field copy): 'fromUnknown === held' compares… + at tests/coverage-fixtures/dictionary-view-copies.ts:23:21 + hint: keep the value at its own type ('{ a: string; }'), or declare the dictionary's values as 'unknown', which keeps the original object \ No newline at end of file diff --git a/tests/harness/corpus-costs/7010-dictionary-of-dictionaries-keep-methods.ts.json b/tests/harness/corpus-costs/7010-dictionary-of-dictionaries-keep-methods.ts.json new file mode 100644 index 000000000..cc07613e7 --- /dev/null +++ b/tests/harness/corpus-costs/7010-dictionary-of-dictionaries-keep-methods.ts.json @@ -0,0 +1 @@ +{ "plain": 2184, "san": 1198 } diff --git a/tests/harness/coverage.test.ts b/tests/harness/coverage.test.ts index 86d202b46..f3958c5fe 100644 --- a/tests/harness/coverage.test.ts +++ b/tests/harness/coverage.test.ts @@ -107,6 +107,20 @@ test("record width copies report at every storing site", async () => { ); }); +test("dictionary conversions of checked-dynamic views report copies", async () => { + const { coverage } = analyze(fixture("dictionary-view-copies.ts")); + expect(coverage.diagnostics).toEqual([]); + expect((coverage.divergences ?? []).map((d) => d.code)).toEqual([ + "SC6001", + "SC6001", + "SC6001", + "SC6001", + ]); + await expect(report(fixture("dictionary-view-copies.ts"))).toMatchFileSnapshot( + "__snapshots__/coverage-dictionary-view-copies.txt", + ); +}); + test("ordinary code reports no divergence", () => { const { coverage } = analyze(fixture("no-divergences.ts")); expect(coverage.diagnostics).toEqual([]); From 14098794d08c09b7bbaf75566de7b0073a39b7df Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 15:46:43 -0500 Subject: [PATCH 010/184] Type the scenario promises of corpus 7119 and 7380 with JSDoc tsgo rejects a call of `resolve()` without an argument when the `new Promise()` that produced it has no type, and asks for a JSDoc hint; tsc-ts accepts it. The two programs compiled under tsc-ts only, so the checker-identity check failed on them. Each scenario function now declares `@returns {Promise}`, as corpus 1594 does. Their output is unchanged. --- tests/corpus/7119-als-net-event-context.mjs | 3 +++ tests/corpus/7380-als-http-event-context.mjs | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/tests/corpus/7119-als-net-event-context.mjs b/tests/corpus/7119-als-net-event-context.mjs index 3fb95ffa8..d17d1c31c 100644 --- a/tests/corpus/7119-als-net-event-context.mjs +++ b/tests/corpus/7119-als-net-event-context.mjs @@ -15,6 +15,7 @@ const store = new AsyncLocalStorage(); const lines = []; const log = (what) => lines.push(`${what}: ${store.getStore() ?? "none"}`); +/** @returns {Promise} */ function echoScenario() { return new Promise((resolve) => { const server = store.run("create", () => @@ -61,6 +62,7 @@ function echoScenario() { }); } +/** @returns {Promise} */ function destroyScenario() { return new Promise((resolve) => { const server = net.createServer((sock) => sock.on("data", () => {})); @@ -78,6 +80,7 @@ function destroyScenario() { }); } +/** @returns {Promise} */ function closeScenarios() { return new Promise((resolve) => { let pending = 3; diff --git a/tests/corpus/7380-als-http-event-context.mjs b/tests/corpus/7380-als-http-event-context.mjs index a881cbe7e..fd6ef6225 100644 --- a/tests/corpus/7380-als-http-event-context.mjs +++ b/tests/corpus/7380-als-http-event-context.mjs @@ -17,6 +17,7 @@ const store = new AsyncLocalStorage(); const lines = []; const log = (what) => lines.push(`${what}: ${store.getStore() ?? "none"}`); +/** @returns {Promise} */ function httpScenario() { return new Promise((resolve) => { const server = store.run("create", () => @@ -56,6 +57,7 @@ function httpScenario() { }); } +/** @returns {Promise} */ function keepAliveScenario() { return new Promise((resolve) => { const agent = new http.Agent({ keepAlive: true, maxSockets: 1 }); @@ -88,6 +90,7 @@ const key = fs.readFileSync("tests/fixtures/server/certs/localhost-key.pem"); const cert = fs.readFileSync("tests/fixtures/server/certs/localhost.pem"); const ca = fs.readFileSync("tests/fixtures/server/certs/ca.pem", "utf8"); +/** @returns {Promise} */ function httpsScenario() { return new Promise((resolve) => { const server = https.createServer({ key, cert }, (req, res) => { @@ -116,6 +119,7 @@ function httpsScenario() { }); } +/** @returns {Promise} */ function tlsScenario() { return new Promise((resolve) => { const server = tls.createServer({ key, cert }, (sock) => { From 1d8d19e634982f238ccbfd204f4d7355d2a69efa Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 17:05:08 -0500 Subject: [PATCH 011/184] Run stream events and callbacks in Node's AsyncLocalStorage contexts Stream events and callbacks ran with no store. Node's streams run each step in the context of the code that triggers it: a step deferred with process.nextTick (resume, emitReadable, endReadable, afterWrite, the destroy and finish steps) in the context that scheduled it, and other events in the context of the push(), write(), end(), destroy() or pipe() call that emits them. Native-backed streams run their events in the context of their handle. The runtime now captures the active context where it defers a stream step and enters it around the step's dispatch: - the stream tick queue keeps each tick's scheduling context; - the runtime's own process.nextTick jobs (zlib completions, pipeline and finished steps, readline) run in the context they were queued in, as user ticks already did; - fs read and write stream requests keep the context of the call that started them, so a stream's open, reads, writes and close follow its creation; - zlib streams keep their creation context and run each engine step's completion in it. In programs that use AsyncLocalStorage, stream.finished and stream.pipeline callbacks are bound to the context of their call, like the other completion callbacks. Programs that do not use AsyncLocalStorage compile to the same IR. child_process and http2 events are unchanged. Corpus 7383 and 7384. --- .../builtin-apis/asynclocalstorage.mdx | 5 +- .../generated/node-v24-internal.json | 34 ++- internal/compatibility/static-support.json | 2 +- .../frontend/lowering/completion-callbacks.ts | 13 +- .../corpus/7383-als-stream-events.ts.json | 6 + .../corpus/7384-als-stream-events.mjs.json | 6 + packages/runtime/src/scr_async.c | 8 +- packages/runtime/src/scr_fs_stream.c | 13 +- packages/runtime/src/scr_stream.c | 39 ++-- packages/runtime/src/scr_zlib_stream.c | 7 + tests/corpus/7383-als-stream-events.ts | 204 +++++++++++++++++ tests/corpus/7384-als-stream-events.mjs | 214 ++++++++++++++++++ .../7383-als-stream-events.ts.json | 1 + .../7384-als-stream-events.mjs.json | 1 + 14 files changed, 525 insertions(+), 28 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7383-als-stream-events.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7384-als-stream-events.mjs.json create mode 100644 tests/corpus/7383-als-stream-events.ts create mode 100644 tests/corpus/7384-als-stream-events.mjs create mode 100644 tests/harness/corpus-costs/7383-als-stream-events.ts.json create mode 100644 tests/harness/corpus-costs/7384-als-stream-events.mjs.json diff --git a/docs/content/limitations/builtin-apis/asynclocalstorage.mdx b/docs/content/limitations/builtin-apis/asynclocalstorage.mdx index 98fc92ea1..016b712f6 100644 --- a/docs/content/limitations/builtin-apis/asynclocalstorage.mdx +++ b/docs/content/limitations/builtin-apis/asynclocalstorage.mdx @@ -3,5 +3,8 @@ Events of `net`, `tls`, `http` and `https` sockets, servers, requests and responses run with the stores Node runs them with. A socket's or client request's events see the context of the call that created it, and a server's events and its requests' and responses' events see the context of its `listen()` call. Socket `write` and `end` callbacks, `'finish'` and `server.close(callback)` see the context of their call. - Events and callbacks of other streams run with no store, where Node runs them in the context of the code that triggers them. + Stream events and callbacks run with the stores of the code that triggers them, as in Node. + Stream steps that Node defers with `process.nextTick` see the context that scheduled them, and `stream.finished` and `stream.pipeline` callbacks see the context of their call. + `fs` read and write streams and `zlib` streams run their events and callbacks in the context that created them. + Events of `child_process` children and of `http2` sessions and streams do not yet run in the contexts Node gives them. `AsyncResource` has no lowering yet. diff --git a/internal/compatibility/generated/node-v24-internal.json b/internal/compatibility/generated/node-v24-internal.json index 6046daf0e..b5542944d 100644 --- a/internal/compatibility/generated/node-v24-internal.json +++ b/internal/compatibility/generated/node-v24-internal.json @@ -1392,7 +1392,9 @@ "tests/corpus/7111-als-bind-snapshot-options.ts", "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", "tests/corpus/7119-als-net-event-context.mjs", - "tests/corpus/7380-als-http-event-context.mjs" + "tests/corpus/7380-als-http-event-context.mjs", + "tests/corpus/7383-als-stream-events.ts", + "tests/corpus/7384-als-stream-events.mjs" ] }, "anchorSource": "chapter" @@ -1447,7 +1449,9 @@ "tests/corpus/7111-als-bind-snapshot-options.ts", "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", "tests/corpus/7119-als-net-event-context.mjs", - "tests/corpus/7380-als-http-event-context.mjs" + "tests/corpus/7380-als-http-event-context.mjs", + "tests/corpus/7383-als-stream-events.ts", + "tests/corpus/7384-als-stream-events.mjs" ] }, "anchorSource": "exact" @@ -1525,7 +1529,9 @@ "tests/corpus/7111-als-bind-snapshot-options.ts", "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", "tests/corpus/7119-als-net-event-context.mjs", - "tests/corpus/7380-als-http-event-context.mjs" + "tests/corpus/7380-als-http-event-context.mjs", + "tests/corpus/7383-als-stream-events.ts", + "tests/corpus/7384-als-stream-events.mjs" ] }, "anchorSource": "exact" @@ -1557,7 +1563,9 @@ "tests/corpus/7111-als-bind-snapshot-options.ts", "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", "tests/corpus/7119-als-net-event-context.mjs", - "tests/corpus/7380-als-http-event-context.mjs" + "tests/corpus/7380-als-http-event-context.mjs", + "tests/corpus/7383-als-stream-events.ts", + "tests/corpus/7384-als-stream-events.mjs" ] }, "anchorSource": "exact" @@ -1589,7 +1597,9 @@ "tests/corpus/7111-als-bind-snapshot-options.ts", "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", "tests/corpus/7119-als-net-event-context.mjs", - "tests/corpus/7380-als-http-event-context.mjs" + "tests/corpus/7380-als-http-event-context.mjs", + "tests/corpus/7383-als-stream-events.ts", + "tests/corpus/7384-als-stream-events.mjs" ] }, "anchorSource": "exact" @@ -1621,7 +1631,9 @@ "tests/corpus/7111-als-bind-snapshot-options.ts", "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", "tests/corpus/7119-als-net-event-context.mjs", - "tests/corpus/7380-als-http-event-context.mjs" + "tests/corpus/7380-als-http-event-context.mjs", + "tests/corpus/7383-als-stream-events.ts", + "tests/corpus/7384-als-stream-events.mjs" ] }, "anchorSource": "exact" @@ -1653,7 +1665,9 @@ "tests/corpus/7111-als-bind-snapshot-options.ts", "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", "tests/corpus/7119-als-net-event-context.mjs", - "tests/corpus/7380-als-http-event-context.mjs" + "tests/corpus/7380-als-http-event-context.mjs", + "tests/corpus/7383-als-stream-events.ts", + "tests/corpus/7384-als-stream-events.mjs" ] }, "anchorSource": "exact" @@ -1962,6 +1976,8 @@ "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", "tests/corpus/7119-als-net-event-context.mjs", "tests/corpus/7380-als-http-event-context.mjs", + "tests/corpus/7383-als-stream-events.ts", + "tests/corpus/7384-als-stream-events.mjs", "tests/harness/node-test.test.ts" ] }, @@ -2529,7 +2545,9 @@ "tests/corpus/7111-als-bind-snapshot-options.ts", "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", "tests/corpus/7119-als-net-event-context.mjs", - "tests/corpus/7380-als-http-event-context.mjs" + "tests/corpus/7380-als-http-event-context.mjs", + "tests/corpus/7383-als-stream-events.ts", + "tests/corpus/7384-als-stream-events.mjs" ] }, "anchorSource": "exact" diff --git a/internal/compatibility/static-support.json b/internal/compatibility/static-support.json index c57d6013e..640345a02 100644 --- a/internal/compatibility/static-support.json +++ b/internal/compatibility/static-support.json @@ -429,7 +429,7 @@ { "symbols": ["crypto.hkdf", "crypto.hkdfSync", "crypto.scrypt", "crypto.scryptSync"], "status": "partial", "evidence": ["tests/corpus/node24-crypto-kdf.ts", "tests/corpus/node24-crypto-kdf-errors.ts", "tests/corpus/node24-crypto-kdf-options.ts", "tests/corpus/5438-crypto-kdf-digests.ts"] }, { "symbols": ["EventEmitter", "emitter.on", "emitter.addListener", "emitter.once", "emitter.prependListener", "emitter.prependOnceListener", "emitter.off", "emitter.removeListener", "emitter.removeAllListeners", "emitter.emit", "emitter.listenerCount", "emitter.listeners", "emitter.rawListeners", "emitter.eventNames", "emitter.setMaxListeners", "emitter.getMaxListeners", "events.once"], "status": "partial", "evidence": ["tests/corpus/1677-emitter-listeners.ts", "tests/corpus/2620-ee-override-once-order.ts"] }, { "symbols": ["events.getEventListeners", "events.getMaxListeners", "events.listenerCount", "events.setMaxListeners", "events.defaultMaxListeners"], "status": "partial", "evidence": ["tests/corpus/5885-events-max-listener-helpers.ts", "tests/corpus/5846-events-esm-default-max.ts", "tests/corpus/5847-events-cjs-default-max.cjs"] }, - { "symbols": ["AsyncLocalStorage", "asyncLocalStorage.run", "asyncLocalStorage.exit", "asyncLocalStorage.getStore", "asyncLocalStorage.enterWith", "asyncLocalStorage.disable", "asyncLocalStorage.name", "AsyncLocalStorage.bind", "AsyncLocalStorage.snapshot"], "status": "partial", "evidence": ["tests/corpus/2213-async-local-storage.cjs", "tests/corpus/5815-als-callback-context.ts", "tests/corpus/7110-als-completion-callbacks.ts", "tests/corpus/7111-als-bind-snapshot-options.ts", "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", "tests/corpus/7119-als-net-event-context.mjs", "tests/corpus/7380-als-http-event-context.mjs"] }, + { "symbols": ["AsyncLocalStorage", "asyncLocalStorage.run", "asyncLocalStorage.exit", "asyncLocalStorage.getStore", "asyncLocalStorage.enterWith", "asyncLocalStorage.disable", "asyncLocalStorage.name", "AsyncLocalStorage.bind", "AsyncLocalStorage.snapshot"], "status": "partial", "evidence": ["tests/corpus/2213-async-local-storage.cjs", "tests/corpus/5815-als-callback-context.ts", "tests/corpus/7110-als-completion-callbacks.ts", "tests/corpus/7111-als-bind-snapshot-options.ts", "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", "tests/corpus/7119-als-net-event-context.mjs", "tests/corpus/7380-als-http-event-context.mjs", "tests/corpus/7383-als-stream-events.ts", "tests/corpus/7384-als-stream-events.mjs"] }, { "symbols": ["StringDecoder", "stringDecoder.write", "stringDecoder.end"], "status": "partial", "evidence": ["tests/corpus/1662-string-decoder-encodings.ts", "tests/corpus/node24-string-decoder.ts", "tests/corpus/5882-string-decoder-runtime-encoding.ts"] }, { "symbols": ["setImmediate", "setInterval", "setTimeout", "clearImmediate", "clearInterval", "clearTimeout", "timersPromises.setTimeout", "timersPromises.setImmediate", "timersPromises.setInterval", "immediate.hasRef", "immediate.ref", "immediate.unref", "timeout.hasRef", "timeout.ref", "timeout.unref", "timeout.refresh"], "status": "partial", "evidence": ["tests/corpus/1804-timers-module.ts", "tests/corpus/1805-timer-callback-args.ts", "tests/corpus/2093-timers-promises.ts", "tests/corpus/2852-timers-promises-interval.ts", "tests/corpus/5830-timers-promises-signal.ts"] }, { "symbols": ["timersPromises.scheduler.wait", "timersPromises.scheduler.yield"], "status": "partial", "evidence": ["tests/corpus/5839-timers-promises-scheduler.ts", "tests/corpus/5889-timers-scheduler-signal.ts"] }, diff --git a/packages/compiler/src/frontend/lowering/completion-callbacks.ts b/packages/compiler/src/frontend/lowering/completion-callbacks.ts index 402741da6..a00e4924d 100644 --- a/packages/compiler/src/frontend/lowering/completion-callbacks.ts +++ b/packages/compiler/src/frontend/lowering/completion-callbacks.ts @@ -1,7 +1,8 @@ /* Completion callbacks in programs that use AsyncLocalStorage. Node runs * the callback of an operation a callback-style API starts (fs.readFile, * crypto.randomBytes, zlib.gzip, dns.lookup, server.listen, net.connect, - * http.get, ...) with the stores active when the call started it. The runtime invokes these callbacks from + * http.get, stream.finished, stream.pipeline, ...) with the stores active + * when the call started it. The runtime invokes these callbacks from * the event loop, where the context is empty, so in programs whose * scheduling carries contexts (RuntimeFeatures asyncContext) each such * argument is bound to the active context at the call: als.bindCallback @@ -50,6 +51,16 @@ const COMPLETION_CALLBACKS: ReadonlyMap = new Map< ["net.listenPathCb", 2], ["process.stderrWriteBytesCb", 2], ["process.stdoutWriteBytesCb", 2], + // stream.finished and stream.pipeline callbacks, which Node binds to + // the context of the call (eos' and pipeline's AsyncResource.bind). + ["stream.finished", 1], + ["stream.finishedDyn", 1], + ["stream.finishedOpts", "last"], + ["stream.finishedOptsSignal", "last"], + ["stream.pipeline", "last"], + ["stream.pipelineDyn", "last"], + ["stream.pipelineStages", "last"], + ["stream.pipelineStagesDyn", "last"], ["zlib.cbOpts", 3], ["zlib.cbOptsDyn", 3], ["zlib.deflateCb", 1], diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7383-als-stream-events.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7383-als-stream-events.ts.json new file mode 100644 index 000000000..d59808c2f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7383-als-stream-events.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7383-als-stream-events.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7384-als-stream-events.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7384-als-stream-events.mjs.json new file mode 100644 index 000000000..51171c8d5 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7384-als-stream-events.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7384-als-stream-events.mjs" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_async.c b/packages/runtime/src/scr_async.c index e6184ed9b..2bfffa59d 100644 --- a/packages/runtime/src/scr_async.c +++ b/packages/runtime/src/scr_async.c @@ -1235,10 +1235,14 @@ static SCR_TL ScrNtick *scr_nt_tail = NULL; * it, because 'exit' listeners run AFTER the loop's teardown and may * enqueue ticks that must never run (Node) yet must not leak. */ +static ScrClosure *scr_als_bind(ScrClosure *cb /*moves*/); + +/* A tick runs with the AsyncLocalStorage context active when it was + * queued, as Node's process.nextTick does. */ void scr_next_tick(ScrClosure *cb /*moves*/) { ScrNtick *t = calloc(1, sizeof *t); if (!t) scr_oom(); - t->cb = cb; + t->cb = scr_als_bind(cb); if (scr_nt_tail) scr_nt_tail->next = t; else scr_nt_head = t; scr_nt_tail = t; @@ -4022,7 +4026,7 @@ double scr_set_timeout_handle_als(ScrClosure *cb, double ms) { return scr_set_timeout_handle(scr_als_bind(cb), ms); } double scr_set_immediate_als(ScrClosure *cb) { return scr_set_immediate(scr_als_bind(cb)); } -void scr_next_tick_als(ScrClosure *cb) { scr_next_tick(scr_als_bind(cb)); } +void scr_next_tick_als(ScrClosure *cb) { scr_next_tick(cb); } /* scr_next_tick binds */ void scr_queue_microtask_als(ScrClosure *cb) { scr_queue_microtask(scr_als_bind(cb)); } /* A typed callback bound to the context active when it was bound: the diff --git a/packages/runtime/src/scr_fs_stream.c b/packages/runtime/src/scr_fs_stream.c index 70a5ab075..940068a7b 100644 --- a/packages/runtime/src/scr_fs_stream.c +++ b/packages/runtime/src/scr_fs_stream.c @@ -79,6 +79,9 @@ typedef struct ScrFssOp { int64_t result; /* answer or -errno */ int retries; /* WRITE: zero-byte retries */ ScrError *carry; /* FSYNC/CLOSE: the error to pass on (+1) */ + /* The AsyncLocalStorage context the step was started in (+1): its + * continuation runs in it, as Node runs an fs request's callback. */ + ScrAlsCtx *ctx; } ScrFssOp; static void scr_fss_work(void *p) { @@ -101,6 +104,7 @@ static void scr_fss_op_destroy(void *p) { free(op->buf); if (op->chunk) scr_bytes_release(op->chunk); if (op->carry) scr_error_release(op->carry); + scr_als_ctx_release(op->ctx); free(op); } @@ -111,13 +115,20 @@ static ScrFssOp *scr_fss_op_new(ScrStream *s, ScrFssOpKind kind, int fd) { op->kind = kind; op->fd = fd; op->pos = -1; + op->ctx = scr_als_capture(); return op; } static void scr_fss_after(void *p); +static void scr_fss_after_in_context(void *p) { + ScrAlsCtx *prev = scr_als_dispatch_enter(((ScrFssOp *)p)->ctx); + scr_fss_after(p); + scr_als_dispatch_leave(prev); +} + static void scr_fss_submit(ScrFssOp *op) { - scr_work_submit(op, &scr_fss_work, &scr_fss_after, &scr_fss_op_destroy); + scr_work_submit(op, &scr_fss_work, &scr_fss_after_in_context, &scr_fss_op_destroy); } /* ── close (Node's close/_close) ──────────────────────────────────────── */ diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index be5ccb910..835b1db9a 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -515,6 +515,10 @@ typedef struct ScrStreamTick { ScrError *err; /* owned or NULL (SCR_ST_ERROR) */ ScrClosure *cb; /* owned or NULL (SCR_ST_AFTER_WRITE / SCR_ST_WCB_ERR / SCR_ST_WRITE_ERR) */ bool hold; /* SCR_ST_AFTER_WRITE: counted in w.after_holds */ + /* The AsyncLocalStorage context active when the tick was scheduled (+1, + * NULL outside a store): Node schedules these steps with + * process.nextTick, so their events and callbacks run in it. */ + ScrAlsCtx *ctx; struct ScrStreamTick *next; } ScrStreamTick; @@ -531,6 +535,7 @@ static void scr_st_tick(ScrStream *s, ScrStreamTickOp op, ScrError *err /*moves* t->op = op; t->err = err; t->cb = cb; + t->ctx = scr_als_capture(); if (scr_st_tail) scr_st_tail->next = t; else scr_st_head = t; scr_st_tail = t; @@ -6861,6 +6866,21 @@ static void scr_stream_run_tick(ScrStreamTick *t) { static void scr_stream_dispatch(void); +/* A tick runs in the context it was scheduled in. */ +static void scr_stream_run_tick_in_context(ScrStreamTick *t) { + ScrAlsCtx *prev = scr_als_dispatch_enter(t->ctx); + scr_stream_run_tick(t); + scr_als_dispatch_leave(prev); +} + +static void scr_stream_tick_free(ScrStreamTick *t) { + scr_stream_release(t->s); + if (t->err) scr_error_release(t->err); + if (t->cb) scr_closure_release(t->cb); + scr_als_ctx_release(t->ctx); + free(t); +} + /* One tick-marker's dispatch: the queue's FIFO head (markers and entries * are enqueued 1:1 in the same order). After an uncaught throw the * remaining entries drop through the station's exc branch below, keeping @@ -6870,11 +6890,8 @@ static void scr_stream_dispatch_one(void) { if (t == NULL || scr_exc_pending()) return; scr_st_head = t->next; if (scr_st_head == NULL) scr_st_tail = NULL; - scr_stream_run_tick(t); - scr_stream_release(t->s); - if (t->err) scr_error_release(t->err); - if (t->cb) scr_closure_release(t->cb); - free(t); + scr_stream_run_tick_in_context(t); + scr_stream_tick_free(t); if (scr_exc_pending()) scr_stream_dispatch(); /* its exc branch drops the rest */ } @@ -6883,11 +6900,8 @@ static void scr_stream_dispatch(void) { ScrStreamTick *t = scr_st_head; scr_st_head = t->next; if (scr_st_head == NULL) scr_st_tail = NULL; - scr_stream_run_tick(t); - scr_stream_release(t->s); - if (t->err) scr_error_release(t->err); - if (t->cb) scr_closure_release(t->cb); - free(t); + scr_stream_run_tick_in_context(t); + scr_stream_tick_free(t); } /* An uncaught throw ends the process (the loop returns to main): the * undelivered ticks are DROPPED — release their references here so the @@ -6896,10 +6910,7 @@ static void scr_stream_dispatch(void) { while (scr_st_head != NULL) { ScrStreamTick *t = scr_st_head; scr_st_head = t->next; - scr_stream_release(t->s); - if (t->err) scr_error_release(t->err); - if (t->cb) scr_closure_release(t->cb); - free(t); + scr_stream_tick_free(t); } scr_st_tail = NULL; } diff --git a/packages/runtime/src/scr_zlib_stream.c b/packages/runtime/src/scr_zlib_stream.c index 54ffacf02..f1437fc2c 100644 --- a/packages/runtime/src/scr_zlib_stream.c +++ b/packages/runtime/src/scr_zlib_stream.c @@ -51,6 +51,9 @@ typedef struct ScrZlibStream { bool done_pending; /* Transform's kCallback: completion waits for _read */ double rlen_before; /* readableLength when the chunk's transform began */ ScrBytes *flush_bufs[6]; /* zero-length sentinels, one per flush kind */ + /* The AsyncLocalStorage context of the stream's creation (+1, NULL + * outside a store): Node runs the handle's step callbacks in it. */ + ScrAlsCtx *als; } ScrZlibStream; static void scr_zlib_stream_oom(void) { @@ -81,6 +84,7 @@ static void scr_zs_release_v(void *p) { for (size_t i = 0; i < 6; i++) { if (zs->flush_bufs[i]) scr_bytes_release(zs->flush_bufs[i]); } + scr_als_ctx_release(zs->als); free(zs); } @@ -242,6 +246,7 @@ static void scr_zs_step_after(void *payload) { ScrZlibStep *op = payload; ScrZlibStream *zs = op->zs; ScrStream *s = op->s; + ScrAlsCtx *prev = scr_als_dispatch_enter(zs->als); zs->write_in_progress = false; ScrZlibError ze; if (scr_zlib_engine_error(zs->eng, &ze)) { @@ -252,6 +257,7 @@ static void scr_zs_step_after(void *payload) { scr_zs_process(zs, s); } if (zs->pending_close) scr_zs_close_engine(zs); + scr_als_dispatch_leave(prev); } static void scr_zs_step_destroy(void *payload) { @@ -463,6 +469,7 @@ ScrStream *scr_zlib_stream_new(double mode_d, ScrDyn *opts) { ScrZlibStream *zs = calloc(1, sizeof *zs); if (!zs) scr_zlib_stream_oom(); zs->rc = 1; + zs->als = scr_als_capture(); zs->mode = mode; zs->eng = scr_zlib_engine_from_options(mode, &zo); scr_zlib_options_drop(&zo); diff --git a/tests/corpus/7383-als-stream-events.ts b/tests/corpus/7383-als-stream-events.ts new file mode 100644 index 000000000..077e15346 --- /dev/null +++ b/tests/corpus/7383-als-stream-events.ts @@ -0,0 +1,204 @@ +// AsyncLocalStorage contexts of stream events and callbacks. A stream +// step that Node defers with process.nextTick runs in the context of the +// code that scheduled it; other events run in the context of the call that +// emits them: push(), write(), end(), destroy() or pipe(). stream.finished +// and stream.pipeline callbacks run in the context of their call. fs read +// and write streams and zlib streams run their events in the context that +// created them, as Node's handles do. +import { AsyncLocalStorage } from "node:async_hooks"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { finished, pipeline, Readable, Transform, Writable } from "node:stream"; +import { pipeline as pipelinePromise } from "node:stream/promises"; +import * as zlib from "node:zlib"; + +const store = new AsyncLocalStorage(); +const lines: string[] = []; +const log = (what: string): void => { + lines.push(`${what}: ${store.getStore() ?? "none"}`); +}; +const step = (fn: (done: () => void) => void): Promise => + new Promise((resolve) => fn(resolve)); + +// pipe(): the destination's writes and 'finish' follow the pipe() call. +await step((done) => { + const source = new Readable({ read(): void {} }); + const sink = new Writable({ + write(chunk: Buffer, encoding: BufferEncoding, cb: (error?: Error | null) => void): void { + log("pipe write"); + cb(); + }, + }); + sink.on("finish", () => { + log("pipe finish"); + done(); + }); + store.run("pipe", () => source.pipe(sink)); + store.run("push", () => { + source.push("a"); + source.push(null); + }); +}); + +// A Transform's steps follow write() and end(); 'end' follows end(). +await step((done) => { + const upper = new Transform({ + transform(chunk: Buffer, encoding: BufferEncoding, cb: (error?: Error | null, data?: string) => void): void { + log("transform"); + cb(null, chunk.toString().toUpperCase()); + }, + flush(cb: (error?: Error | null, data?: string) => void): void { + log("flush"); + cb(null, "!"); + }, + }); + upper.on("data", () => log("transform data")); + upper.on("end", () => { + log("transform end"); + done(); + }); + store.run("write", () => upper.write("a")); + store.run("end", () => upper.end()); +}); + +// A write that completes later: its callback, _final, 'prefinish', the +// end() callback, 'finish' and 'close' follow the write() call. +await step((done) => { + const slow = new Writable({ + write(chunk: Buffer, encoding: BufferEncoding, cb: (error?: Error | null) => void): void { + log("slow write"); + setImmediate(cb); + }, + final(cb: (error?: Error | null) => void): void { + log("final"); + cb(); + }, + }); + slow.on("prefinish", () => log("prefinish")); + slow.on("finish", () => log("finish")); + slow.on("close", () => { + log("close"); + done(); + }); + store.run("slow-write", () => slow.write("a", () => log("write callback"))); + store.run("slow-end", () => slow.end(() => log("end callback"))); +}); + +// 'readable' and 'end' follow the push() that triggers them. +await step((done) => { + const source = new Readable({ read(): void {} }); + source.on("readable", () => { + log("readable"); + while (source.read() !== null) {} + }); + source.on("end", () => { + log("readable end"); + done(); + }); + store.run("push-x", () => source.push("x")); + setTimeout(() => store.run("push-eof", () => source.push(null)), 5); +}); + +// destroy(): 'error' and 'close' follow the destroy() call. +await step((done) => { + const source = new Readable({ read(): void {} }); + source.on("error", () => log("destroy error")); + source.on("close", () => { + log("destroy close"); + done(); + }); + store.run("destroy", () => source.destroy(new Error("stopped"))); +}); + +// finished() and pipeline() callbacks follow their call, also when another +// context ends or destroys the stream. +await step((done) => { + const source = new Readable({ read(): void {} }); + store.run("finished", () => + finished(source, () => { + log("finished callback"); + done(); + }), + ); + source.resume(); + store.run("finished-eof", () => source.push(null)); +}); +await step((done) => { + const source = new Readable({ read(): void {} }); + const sink = new Writable({ + write(chunk: Buffer, encoding: BufferEncoding, cb: (error?: Error | null) => void): void { + cb(); + }, + }); + store.run("pipeline", () => + pipeline(source, sink, (err: NodeJS.ErrnoException | null) => { + log(`pipeline callback ${err ? err.message : "ok"}`); + done(); + }), + ); + setTimeout(() => store.run("pipeline-destroy", () => source.destroy(new Error("boom"))), 5); +}); + +// stream/promises pipeline: the stages follow the call; after the await +// the caller's context is back. +await store.run("promise-pipeline", () => + pipelinePromise( + Readable.from(["a"]), + new Writable({ + write(chunk: Buffer, encoding: BufferEncoding, cb: (error?: Error | null) => void): void { + log("promise pipeline write"); + cb(); + }, + }), + ), +); +log("after the promise pipeline"); + +// fs streams: the events follow the context that created the stream. +const file = path.join(os.tmpdir(), `scriptc-7383-${process.pid}.txt`); +await step((done) => { + const out = store.run("fs-write-create", () => fs.createWriteStream(file)); + out.on("finish", () => log("fs finish")); + out.on("close", () => { + log("fs write close"); + done(); + }); + store.run("fs-write", () => out.write("hello", () => log("fs write callback"))); + store.run("fs-end", () => out.end()); +}); +await step((done) => { + const input = store.run("fs-read-create", () => fs.createReadStream(file)); + store.run("fs-listen", () => { + input.on("open", () => log("fs open")); + input.on("data", () => log("fs data")); + input.on("end", () => log("fs end")); + input.on("close", () => { + log("fs read close"); + done(); + }); + }); +}); +fs.unlinkSync(file); + +// zlib streams: the events follow the context that created the stream. +await step((done) => { + const gzip = store.run("gzip-create", () => zlib.createGzip()); + const gunzip = store.run("gunzip-create", () => zlib.createGunzip()); + const parts: Buffer[] = []; + const sink = new Writable({ + write(chunk: Buffer, encoding: BufferEncoding, cb: (error?: Error | null) => void): void { + log("gunzip output"); + parts.push(chunk); + cb(); + }, + }); + store.run("zlib-pipeline", () => + pipeline(Readable.from(["hello ", "world"]), gzip, gunzip, sink, () => { + log(`zlib pipeline callback ${Buffer.concat(parts).toString()}`); + done(); + }), + ); +}); + +for (const line of lines) console.log(line); diff --git a/tests/corpus/7384-als-stream-events.mjs b/tests/corpus/7384-als-stream-events.mjs new file mode 100644 index 000000000..6d67bf26f --- /dev/null +++ b/tests/corpus/7384-als-stream-events.mjs @@ -0,0 +1,214 @@ +// AsyncLocalStorage contexts of stream events and callbacks, from +// JavaScript. Stream steps that Node defers with process.nextTick run in +// the context that scheduled them: resume() after a 'data' listener, +// uncork() from a tick, a write after end(), a push() from a user tick. +// Other events follow the call that emits them. stream.finished callbacks +// run in the context of their call, with stream/promises' finished() +// resolving in the context that awaits it. readline lines follow the +// createInterface() call that resumes the input, and zlib streams run +// their events and flush() callbacks in the context that created them. +import { AsyncLocalStorage } from "node:async_hooks"; +import readline from "node:readline"; +import { Duplex, finished, PassThrough, Readable, Writable } from "node:stream"; +import { finished as finishedPromise } from "node:stream/promises"; +import zlib from "node:zlib"; + +const store = new AsyncLocalStorage(); +const lines = []; +const log = (what) => lines.push(`${what}: ${store.getStore() ?? "none"}`); +const step = (fn) => new Promise((resolve) => fn(resolve)); + +// pipe() and unpipe() events, and writes through a PassThrough. +await step((done) => { + const through = new PassThrough(); + const sink = new Writable({ write(chunk, encoding, cb) { log("pipe write"); cb(); } }); + sink.on("pipe", () => log("pipe event")); + sink.on("unpipe", () => log("unpipe event")); + store.run("pipe", () => through.pipe(sink)); + store.run("through-write", () => through.write("q")); + setTimeout(() => { + store.run("unpipe", () => through.unpipe(sink)); + done(); + }, 5); +}); + +// cork() and uncork() from a tick: the writes follow the tick's context. +await step((done) => { + const sink = new Writable({ write(chunk, encoding, cb) { log("corked write"); cb(); } }); + sink.on("finish", () => { + log("corked finish"); + done(); + }); + store.run("cork", () => { + sink.cork(); + sink.write("a"); + sink.write("b"); + }); + store.run("uncork", () => process.nextTick(() => sink.uncork())); + setTimeout(() => store.run("corked-end", () => sink.end()), 5); +}); + +// A write after end(): its callback and 'error' follow the write() call. +await step((done) => { + const sink = new Writable({ write(chunk, encoding, cb) { cb(); } }); + sink.on("error", () => log("write after end error")); + sink.end(); + store.run("late-write", () => + sink.write("x", () => { + log("late write callback"); + done(); + }), + ); +}); + +// A failing _write: the callback, 'error' and 'close' follow write(). +await step((done) => { + const sink = new Writable({ write(chunk, encoding, cb) { cb(new Error("bad")); } }); + sink.on("error", () => log("failed write error")); + sink.on("close", () => { + log("failed write close"); + done(); + }); + store.run("failed-write", () => sink.write("x", () => log("failed write callback"))); +}); + +// A Duplex that pushes from _write: 'data' and 'end' follow the resume() +// tick, which the 'data' listener scheduled with no store. +await step((done) => { + const duplex = new Duplex({ + read() {}, + write(chunk, encoding, cb) { + log("duplex write"); + this.push(chunk); + cb(); + }, + }); + duplex.on("data", () => { + log("duplex data"); + duplex.push(null); + }); + duplex.on("end", () => { + log("duplex end"); + done(); + }); + store.run("duplex-write", () => duplex.write("z")); +}); + +// A push() from a user tick, a 'data' listener registered in a store with +// setEncoding(), and PassThrough's 'end' and 'close' after end(). +await step((done) => { + const source = new Readable({ read() {} }); + source.on("data", () => log("tick data")); + source.on("end", () => { + log("tick end"); + done(); + }); + store.run("user-tick", () => + process.nextTick(() => { + log("user tick"); + source.push("x"); + }), + ); + setTimeout(() => store.run("tick-eof", () => source.push(null)), 5); +}); +await step((done) => { + const source = new Readable({ read() {} }); + source.setEncoding("utf8"); + store.run("data-listener", () => source.on("data", () => log("text data"))); + source.on("end", () => { + log("text end"); + done(); + }); + source.push("hi"); + source.push(null); +}); +await step((done) => { + const through = new PassThrough(); + through.on("end", () => log("through end")); + through.on("close", () => { + log("through close"); + done(); + }); + through.resume(); + store.run("through-end", () => through.end("last")); +}); + +// finished() on a Writable, and stream/promises' finished(). +await step((done) => { + const sink = new Writable({ write(chunk, encoding, cb) { setTimeout(cb, 1); } }); + store.run("finished-writable", () => + finished(sink, () => { + log("finished callback"); + done(); + }), + ); + sink.on("finish", () => log("writable finish")); + store.run("writable-end", () => { + sink.write("a"); + sink.end(); + }); +}); +await step((done) => { + const source = new Readable({ read() {} }); + source.resume(); + store.run("finished-promise", () => finishedPromise(source)).then(() => { + log("finished promise"); + done(); + }); + setTimeout(() => store.run("promise-eof", () => source.push(null)), 5); +}); + +// Readable.from() over an async generator: the reads follow the resume() +// tick, scheduled with no store. +await step((done) => { + async function* letters() { + log("generator start"); + yield "a"; + log("generator end"); + } + const source = store.run("from", () => Readable.from(letters())); + source.on("data", () => log("from data")); + source.on("end", () => { + log("from end"); + done(); + }); +}); + +// readline over a Readable: createInterface() resumes the input. +await step((done) => { + const input = Readable.from(["a\nb\n"]); + const rl = store.run("readline", () => readline.createInterface({ input })); + rl.on("line", (line) => log(`line ${line}`)); + rl.on("close", () => { + log("readline close"); + done(); + }); +}); + +// zlib streams: events and flush() callbacks follow the creation context. +await step((done) => { + const gzip = store.run("gzip-create", () => zlib.createGzip()); + store.run("gzip-listen", () => { + gzip.on("data", () => log("gzip data")); + gzip.on("end", () => { + log("gzip end"); + done(); + }); + }); + store.run("gzip-write", () => { + gzip.write("hello"); + gzip.end(); + }); +}); +await step((done) => { + const deflate = store.run("deflate-create", () => zlib.createDeflate()); + store.run("deflate-flush", () => deflate.flush(() => log("deflate flush callback"))); + deflate.on("data", () => {}); + deflate.on("end", () => { + log("deflate end"); + done(); + }); + store.run("deflate-end", () => deflate.end("x")); +}); + +for (const line of lines) console.log(line); diff --git a/tests/harness/corpus-costs/7383-als-stream-events.ts.json b/tests/harness/corpus-costs/7383-als-stream-events.ts.json new file mode 100644 index 000000000..a806a3ed9 --- /dev/null +++ b/tests/harness/corpus-costs/7383-als-stream-events.ts.json @@ -0,0 +1 @@ +{ "plain": 4500, "san": 9000 } diff --git a/tests/harness/corpus-costs/7384-als-stream-events.mjs.json b/tests/harness/corpus-costs/7384-als-stream-events.mjs.json new file mode 100644 index 000000000..a806a3ed9 --- /dev/null +++ b/tests/harness/corpus-costs/7384-als-stream-events.mjs.json @@ -0,0 +1 @@ +{ "plain": 4500, "san": 9000 } From fd4f14936ff235d6ca9a2b6e587239b5a36b6c5b Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Thu, 22 Oct 2026 15:39:35 -0700 Subject: [PATCH 012/184] Check cycle bindings only where early code can reach the reading function The import-cycle analysis treated every deferred reference (a function body, parameter default or instance field) to a cycle binding as early once any user code could run before the declaration completed. With a static evaluation order it now walks the cycle's top-level code in evaluation order, following stored callables (function and class declarations, functions and methods stored by initializers) across the cycle's modules through import aliases, and counts a deferred reference as early only when that walk reaches it before the declaration ran. Top-level reads, unknown orders, top-level await and default-exported declarations keep the previous rule. This removes the SC1016 refusal on zod 4's `fn[CONSTANT_CATCH] = true` (a computed key folded from a const's literal type inside a function nothing calls during the cycle's evaluation), and drops initialization flags and checks that can never fire. Co-Authored-By: Claude Opus 5.5 --- .../compiler/src/frontend/program.test.ts | 30 +- packages/compiler/src/frontend/program.ts | 355 ++++++++++++++---- .../main.ts.json | 8 + .../main.ts.json | 8 + .../7380-cycle-deferred-folded-key/core.ts | 5 + .../7380-cycle-deferred-folded-key/main.ts | 10 + .../7380-cycle-deferred-folded-key/util.ts | 23 ++ .../7381-cycle-deferred-reached-tdz/alpha.ts | 13 + .../7381-cycle-deferred-reached-tdz/beta.ts | 12 + .../7381-cycle-deferred-reached-tdz/main.ts | 7 + .../main.ts.json | 1 + .../main.ts.json | 1 + 12 files changed, 407 insertions(+), 66 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7380-cycle-deferred-folded-key/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7381-cycle-deferred-reached-tdz/main.ts.json create mode 100644 tests/corpus/7380-cycle-deferred-folded-key/core.ts create mode 100644 tests/corpus/7380-cycle-deferred-folded-key/main.ts create mode 100644 tests/corpus/7380-cycle-deferred-folded-key/util.ts create mode 100644 tests/corpus/7381-cycle-deferred-reached-tdz/alpha.ts create mode 100644 tests/corpus/7381-cycle-deferred-reached-tdz/beta.ts create mode 100644 tests/corpus/7381-cycle-deferred-reached-tdz/main.ts create mode 100644 tests/harness/corpus-costs/7380-cycle-deferred-folded-key/main.ts.json create mode 100644 tests/harness/corpus-costs/7381-cycle-deferred-reached-tdz/main.ts.json diff --git a/packages/compiler/src/frontend/program.test.ts b/packages/compiler/src/frontend/program.test.ts index bdd6260ed..5eb3e363e 100644 --- a/packages/compiler/src/frontend/program.test.ts +++ b/packages/compiler/src/frontend/program.test.ts @@ -371,15 +371,39 @@ test("repeated cycle edges reuse a file's binding index", () => { test.for([ [ - // A shared instance built by a user constructor: the reads of it in - // the partner's function can run early; the class cannot. + // A shared instance built by a user constructor that calls the + // partner's function: its read of the instance can run early. + { + "main.ts": 'import { shared } from "./a.ts"; console.log(shared.n);', + "a.ts": + 'import { f } from "./b.ts"; export class C { n = 1; constructor() { console.log(f()); } } export const shared = new C();', + "b.ts": 'import { shared } from "./a.ts"; export function f(): number { return shared.n; }', + }, + ["shared"], + ], + [ + // User code runs while the instance is built, but nothing it reaches + // calls the partner's function before the declaration completed. { "main.ts": 'import { shared } from "./a.ts"; console.log(shared.n);', "a.ts": 'import { f } from "./b.ts"; export class C { n = 1; } export const shared = new C(); console.log(f());', "b.ts": 'import { shared } from "./a.ts"; export function f(): number { return shared.n; }', }, - ["shared"], + [], + ], + [ + // The partner's top level runs first and calls a hoisted function of + // the importer that reads a const declared later; a second function + // reads another const but nothing calls it early. + { + "main.ts": 'import { late } from "./a.ts"; console.log(late());', + "a.ts": + 'import { probe } from "./b.ts"; export function read(): string { return name; } export function late(): string { return other + probe; } export const name = "n"; export const other = "o";', + "b.ts": + 'import { read } from "./a.ts"; function attempt(): string { try { return read(); } catch { return "tdz"; } } export const probe = attempt();', + }, + ["name"], ], [ // The partner's top level reads the importer's const before it ran. diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index 507d3392d..f42bc426b 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -3760,22 +3760,7 @@ function computeCycleEarlyBindings7( const evaluation = known ? comp.slice().sort((a, b) => position.get(a)! - position.get(b)!) : comp.slice(); - interface Candidate { - symbol: ts.Symbol; - name: ts.Identifier; - kind: CycleEarlyBinding["kind"]; - module: ts.SourceFile; - /** User code can run between the cycle's start and the end of the - * declaration (always true when the order is unknown). */ - exposed: boolean; - rank: number; - start: number; - end: number; - reference: ts.Identifier | null; - folded: ts.Identifier | null; - localNames: Map; - } - const candidates = new Map(); + const candidates = new Map(); const names = new Set(); let dirty = !known; evaluation.forEach((m, rank) => { @@ -3812,6 +3797,7 @@ function computeCycleEarlyBindings7( start: ts.tokenStartOfNode(stmt), end: stmt.end, reference: null, + deferredReference: null, folded: null, localNames: new Map(), }); @@ -3880,15 +3866,27 @@ function computeCycleEarlyBindings7( ref.end <= c.end ) continue; + if (inDeferredPosition7(ref)) { + if (c.exposed && c.deferredReference === null) c.deferredReference = ref; + continue; + } let early: boolean; - if (inDeferredPosition7(ref)) early = c.exposed; - else if (sf === c.module) early = ts.tokenStartOfNode(ref) < c.start; + if (sf === c.module) early = ts.tokenStartOfNode(ref) < c.start; else if (!memberSet.has(sf) || !rankOf.has(sf)) early = false; else early = !known || rankOf.get(sf)! < c.rank; if (early) c.reference = ref; } } + // With a static evaluation order, a deferred reference only runs early + // when the code that runs before the declaration completes can reach + // the function holding it. + const reached = + known && [...candidates.values()].some((c) => c.reference === null && c.deferredReference) + ? cycleReachedDeferredReferences7(program, evaluation, candidates) + : null; for (const c of candidates.values()) { + if (c.reference === null && c.deferredReference !== null) + c.reference = reached === null ? c.deferredReference : (reached.get(c.symbol) ?? null); if (c.reference === null) continue; const settled = new Set(order.filter((sf) => !rankOf.has(sf))); if (known) for (const m of evaluation.slice(c.rank + 1)) settled.add(m); @@ -3907,6 +3905,280 @@ function computeCycleEarlyBindings7( return out; } +/** Collects into `units` the callables an initializer expression stores + * (see moduleEarlyBindingsOf7), recording in `immediate` the parts of a + * stored node that run where it is evaluated. */ +function storedCallables7( + checker: ts.TypeChecker, + e: ts.Expression, + units: ts.Node[], + immediate: Map, +): void { + while ( + ts.isParenthesizedExpression(e) || + ts.isAsExpression(e) || + ts.isSatisfiesExpression(e) || + ts.isNonNullExpression(e) + ) + e = e.expression; + if (ts.isArrowFunction(e) || ts.isFunctionExpression(e)) { + units.push(e); + } else if (ts.isClassExpression(e)) { + units.push(e); + immediate.set(e, classSyncParts7(e)); + } else if (ts.isObjectLiteralExpression(e)) { + for (const p of e.properties.nodes) { + if ( + ts.isMethodDeclaration(p) || + ts.isGetAccessorDeclaration(p) || + ts.isSetAccessorDeclaration(p) + ) { + // A computed key evaluates where the literal does; a plain + // member-chain key (`[Equal.symbol]`) runs no user code. + if (!ts.isComputedPropertyName(p.name)) units.push(p); + else if (memberChainKey7(p.name.expression)) { + units.push(p); + immediate.set(p, [p.name]); + } + } else if (ts.isPropertyAssignment(p)) + storedCallables7(checker, p.initializer, units, immediate); + } + } else if (ts.isArrayLiteralExpression(e)) { + for (const el of e.elements.nodes) + if (!ts.isSpreadElement(el)) storedCallables7(checker, el, units, immediate); + } else if (ts.isCallExpression(e)) { + // `const f = wrap(callback)` where wrap only returns closures over + // the callback: it runs when the stored result is called. + e.arguments.nodes.forEach((arg, index) => { + let fn: ts.Expression = arg; + while (ts.isParenthesizedExpression(fn)) fn = fn.expression; + if ( + (ts.isArrowFunction(fn) || ts.isFunctionExpression(fn)) && + returnsOnlyClosuresOver7(checker, e.expression, index) + ) + units.push(fn); + }); + } +} + +/** A binding of an import cycle and what the reference scan found. */ +interface CycleCandidate7 { + symbol: ts.Symbol; + name: ts.Identifier; + kind: CycleEarlyBinding["kind"]; + module: ts.SourceFile; + /** User code can run between the cycle's start and the end of the + * declaration (always true when the order is unknown). */ + exposed: boolean; + rank: number; + start: number; + end: number; + /** A reference read at top level before the declaration ran. */ + reference: ts.Identifier | null; + /** A deferred reference (function body, parameter default, instance + * field) when user code may run before the declaration completes. */ + deferredReference: ts.Identifier | null; + folded: ts.Identifier | null; + localNames: Map; +} + +/** For one import cycle with a static evaluation order: the first deferred + * reference to each candidate that code running before the candidate's + * declaration completes can reach, or null when the walk cannot be precise + * (top-level await, a default-exported function or class declaration). + * + * The cycle's top-level code runs in evaluation order. A stored callable + * (a function or class declaration, or a function, class or object-literal + * method stored by a variable initializer) becomes reachable once + * reachable code names its binding, through an import alias or a + * namespace member too; every other function-like in reachable code counts + * as reachable, since it may be handed to a callee. A namespace import of + * a cycle member used other than for a member read makes every stored + * callable reachable. moduleEarlyBindingsOf7 is the single-module form. */ +function cycleReachedDeferredReferences7( + program: ts.Program, + evaluation: readonly ts.SourceFile[], + candidates: ReadonlyMap, +): Map | null { + const checker = program.getTypeChecker(); + const members = new Set(evaluation); + for (const sf of evaluation) { + if (hasTopLevelAwait7(sf)) return null; + for (const stmt of sf.statements.nodes) { + if ( + (ts.isFunctionDeclaration(stmt) || ts.isClassDeclaration(stmt)) && + (ts.getCombinedModifierFlags(stmt as unknown as ts.Declaration) & + ts.ModifierFlags.Default) !== + 0 + ) + return null; + } + } + const stored = new Set(); + const immediate = new Map(); + const storedNames: ts.Identifier[] = []; + const unitsOfName = new Map(); + const store = (name: ts.Identifier, units: ts.Node[]): void => { + if (units.length === 0) return; + for (const u of units) stored.add(u); + storedNames.push(name); + unitsOfName.set(name, units); + }; + for (const sf of evaluation) { + for (const stmt of sf.statements.nodes) { + if ( + (ts.getCombinedModifierFlags(stmt as unknown as ts.Declaration) & + ts.ModifierFlags.Ambient) !== + 0 + ) + continue; + if (ts.isFunctionDeclaration(stmt) && stmt.name !== undefined && stmt.body !== undefined) { + store(stmt.name, [stmt]); + } else if (ts.isClassDeclaration(stmt) && stmt.name !== undefined) { + store(stmt.name, [stmt]); + } else if (ts.isVariableStatement(stmt)) { + for (const d of stmt.declarationList.declarations.nodes) { + if (ts.isIdentifier(d.name) && d.initializer !== undefined) { + const units: ts.Node[] = []; + storedCallables7(checker, d.initializer, units, immediate); + store(d.name, units); + } + } + } + } + } + const unitsBySymbol = new Map(); + const allUnits: ts.Node[] = []; + for (const name of storedNames) { + const symbol = checker.getSymbolAtLocation(name); + if (symbol === undefined) continue; + const units = unitsOfName.get(name)!; + unitsBySymbol.set(symbol, [...(unitsBySymbol.get(symbol) ?? []), ...units]); + allUnits.push(...units); + } + // The identifier texts worth resolving: declared names, and the local + // names of imports (named, default and namespace) in cycle members. + const names = new Set(); + for (const c of candidates.values()) names.add(c.name.text); + for (const name of storedNames) names.add(name.text); + for (const sf of evaluation) { + for (const stmt of sf.statements.nodes) { + if (!ts.isImportDeclaration(stmt) || stmt.importClause === undefined) continue; + const clause = stmt.importClause; + if (clause.name !== undefined) names.add(clause.name.text); + const bindings = clause.namedBindings; + if (bindings === undefined) continue; + if (ts.isNamedImports(bindings)) + for (const element of bindings.elements.nodes) names.add(element.name.text); + else names.add(bindings.name.text); + } + } + const idents: ts.Identifier[] = []; + for (const sf of evaluation) { + ts.walkPreorder(sf, (node) => { + if (ts.isImportDeclaration(node)) return "skip"; + if (ts.isIdentifier(node) && names.has(node.text)) idents.push(node); + return undefined; + }); + } + checker.prefetchSymbolNodesExact(idents); + const symbolOf = (id: ts.Identifier): ts.Symbol | undefined => { + const parent = id.parent; + let symbol = checker.getSymbolAtLocation(id); + if (parent !== undefined && ts.isShorthandPropertyAssignment(parent) && parent.name === id) { + symbol = checker.getShorthandAssignmentValueSymbol(parent) ?? symbol; + } + if (symbol !== undefined && symbol.flags & ts.SymbolFlags.Alias) { + symbol = checker.getAliasedSymbol(symbol); + } + return symbol; + }; + const isMemberNamespace = (symbol: ts.Symbol): boolean => + (symbol.flags & ts.SymbolFlags.ValueModule) !== 0 && + symbol.declarations.some((d) => ts.isSourceFile(d) && members.has(d)); + const declared = new Set(); + const reachedUnits = new Set(); + const out = new Map(); + const work: ts.Node[] = []; + const reach = (unit: ts.Node): void => { + if (reachedUnits.has(unit)) return; + reachedUnits.add(unit); + work.push(unit); + }; + const visit = (root: ts.Node): void => { + ts.walkPreorder(root, (node) => { + if (node !== root && stored.has(node)) { + for (const part of immediate.get(node) ?? []) visit(part); + return "skip"; + } + if (ts.isImportDeclaration(node)) return "skip"; + if (!ts.isIdentifier(node) || !names.has(node.text)) return undefined; + if (inTypePosition7(node)) return undefined; + const parent = node.parent; + if (parent !== undefined && ts.isExportSpecifier(parent)) return undefined; + if (parent !== undefined && declaredName7(parent) === node) return undefined; + const symbol = symbolOf(node); + if (symbol === undefined) return undefined; + if (isMemberNamespace(symbol)) { + // `ns.member` resolves through the member's own identifier; any + // other use can reach every export. + if ( + !( + parent !== undefined && + ts.isPropertyAccessExpression(parent) && + parent.expression === node + ) + ) + for (const unit of allUnits) reach(unit); + return undefined; + } + const c = candidates.get(symbol); + if ( + c !== undefined && + node !== c.name && + !declared.has(symbol) && + !out.has(symbol) && + inDeferredPosition7(node) && + !( + c.kind === "class" && + ts.getSourceFileOfNode(node) === c.module && + ts.tokenStartOfNode(node) >= c.start && + node.end <= c.end + ) + ) + out.set(symbol, node); + for (const unit of unitsBySymbol.get(symbol) ?? []) reach(unit); + return undefined; + }); + }; + const drain = (): void => { + while (work.length > 0) visit(work.pop()!); + }; + const declare = (name: ts.BindingName): void => { + for (const id of boundNames7(name)) { + const symbol = checker.getSymbolAtLocation(id); + if (symbol !== undefined) declared.add(symbol); + } + }; + for (const sf of evaluation) { + for (const stmt of sf.statements.nodes) { + if (ts.isClassDeclaration(stmt)) { + for (const part of classSyncParts7(stmt)) visit(part); + drain(); + if (stmt.name !== undefined) declare(stmt.name); + } else if (ts.isVariableStatement(stmt)) { + for (const d of stmt.declarationList.declarations.nodes) { + visit(d); + drain(); + declare(d.name); + } + } else if (!ts.isFunctionDeclaration(stmt)) visit(stmt); + drain(); + } + } + return out; +} + /** Top-level let/const and class bindings, in modules outside the * exactly-initialized cycles, that code running before their declaration * completes can read. @@ -3971,51 +4243,8 @@ function moduleEarlyBindingsOf7( storedBy.set(name.text, list); }; const checker = program.getTypeChecker(); - const storedIn = (e: ts.Expression, units: ts.Node[]): void => { - while ( - ts.isParenthesizedExpression(e) || - ts.isAsExpression(e) || - ts.isSatisfiesExpression(e) || - ts.isNonNullExpression(e) - ) - e = e.expression; - if (ts.isArrowFunction(e) || ts.isFunctionExpression(e)) { - units.push(e); - } else if (ts.isClassExpression(e)) { - units.push(e); - immediate.set(e, classSyncParts7(e)); - } else if (ts.isObjectLiteralExpression(e)) { - for (const p of e.properties.nodes) { - if ( - ts.isMethodDeclaration(p) || - ts.isGetAccessorDeclaration(p) || - ts.isSetAccessorDeclaration(p) - ) { - // A computed key evaluates where the literal does; a plain - // member-chain key (`[Equal.symbol]`) runs no user code. - if (!ts.isComputedPropertyName(p.name)) units.push(p); - else if (memberChainKey7(p.name.expression)) { - units.push(p); - immediate.set(p, [p.name]); - } - } else if (ts.isPropertyAssignment(p)) storedIn(p.initializer, units); - } - } else if (ts.isArrayLiteralExpression(e)) { - for (const el of e.elements.nodes) if (!ts.isSpreadElement(el)) storedIn(el, units); - } else if (ts.isCallExpression(e)) { - // `const f = wrap(callback)` where wrap only returns closures over - // the callback: it runs when the stored result is called. - e.arguments.nodes.forEach((arg, index) => { - let fn: ts.Expression = arg; - while (ts.isParenthesizedExpression(fn)) fn = fn.expression; - if ( - (ts.isArrowFunction(fn) || ts.isFunctionExpression(fn)) && - returnsOnlyClosuresOver7(checker, e.expression, index) - ) - units.push(fn); - }); - } - }; + const storedIn = (e: ts.Expression, units: ts.Node[]): void => + storedCallables7(checker, e, units, immediate); sf.statements.nodes.forEach((stmt, index) => { if (ambient(stmt)) return; if (ts.isFunctionDeclaration(stmt) && stmt.name !== undefined && stmt.body !== undefined) { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7380-cycle-deferred-folded-key/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7380-cycle-deferred-folded-key/main.ts.json new file mode 100644 index 000000000..18ef785c5 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7380-cycle-deferred-folded-key/main.ts.json @@ -0,0 +1,8 @@ +{ + "order": [ + "/tests/corpus/7380-cycle-deferred-folded-key/util.ts", + "/tests/corpus/7380-cycle-deferred-folded-key/core.ts", + "/tests/corpus/7380-cycle-deferred-folded-key/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7381-cycle-deferred-reached-tdz/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7381-cycle-deferred-reached-tdz/main.ts.json new file mode 100644 index 000000000..65d9b19e9 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7381-cycle-deferred-reached-tdz/main.ts.json @@ -0,0 +1,8 @@ +{ + "order": [ + "/tests/corpus/7381-cycle-deferred-reached-tdz/beta.ts", + "/tests/corpus/7381-cycle-deferred-reached-tdz/alpha.ts", + "/tests/corpus/7381-cycle-deferred-reached-tdz/main.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7380-cycle-deferred-folded-key/core.ts b/tests/corpus/7380-cycle-deferred-folded-key/core.ts new file mode 100644 index 000000000..7f4f4ab14 --- /dev/null +++ b/tests/corpus/7380-cycle-deferred-folded-key/core.ts @@ -0,0 +1,5 @@ +import { describe, kind, mark } from "./util.ts"; + +export const version = 3; +export const marked = mark({}); +console.log("core body", kind(), describe()); diff --git a/tests/corpus/7380-cycle-deferred-folded-key/main.ts b/tests/corpus/7380-cycle-deferred-folded-key/main.ts new file mode 100644 index 000000000..92617eba3 --- /dev/null +++ b/tests/corpus/7380-cycle-deferred-folded-key/main.ts @@ -0,0 +1,10 @@ +// A cycle member runs user code before it declares a constant that only +// functions read, one of them as a computed key and one under `typeof` +// (both resolved from the constant's literal type). Nothing calls those +// functions before the declaration ran, so they read the constant +// normally once the other member's top level calls them. +import { marked } from "./core.ts"; +import { describe, kind, mark } from "./util.ts"; + +console.log(JSON.stringify(marked), kind()); +console.log(JSON.stringify(mark({ seen: false })), describe()); diff --git a/tests/corpus/7380-cycle-deferred-folded-key/util.ts b/tests/corpus/7380-cycle-deferred-folded-key/util.ts new file mode 100644 index 000000000..a038d9e63 --- /dev/null +++ b/tests/corpus/7380-cycle-deferred-folded-key/util.ts @@ -0,0 +1,23 @@ +import { version } from "./core.ts"; + +export function makeCounter(): () => number { + let n = 0; + return () => ++n; +} +const next = makeCounter(); +console.log("util body", next(), next()); + +export const TAG = "~tagged"; + +export function mark(box: Record): Record { + box[TAG] = true; + return box; +} + +export function kind(): string { + return typeof TAG; +} + +export function describe(): string { + return `core v${version}`; +} diff --git a/tests/corpus/7381-cycle-deferred-reached-tdz/alpha.ts b/tests/corpus/7381-cycle-deferred-reached-tdz/alpha.ts new file mode 100644 index 000000000..8dc002c27 --- /dev/null +++ b/tests/corpus/7381-cycle-deferred-reached-tdz/alpha.ts @@ -0,0 +1,13 @@ +import { probe } from "./beta.ts"; + +export function report(): string { + return `alpha ${NAME}`; +} + +export function late(): string { + return `${OTHER} after ${probe}`; +} + +export const NAME = "ready"; +export const OTHER = "other"; +console.log("alpha body", report()); diff --git a/tests/corpus/7381-cycle-deferred-reached-tdz/beta.ts b/tests/corpus/7381-cycle-deferred-reached-tdz/beta.ts new file mode 100644 index 000000000..c3d1551ff --- /dev/null +++ b/tests/corpus/7381-cycle-deferred-reached-tdz/beta.ts @@ -0,0 +1,12 @@ +import { report } from "./alpha.ts"; + +function attempt(): string { + try { + return report(); + } catch (error) { + return error instanceof ReferenceError ? `ReferenceError: ${error.message}` : "other error"; + } +} + +export const probe = attempt(); +console.log("beta body", probe); diff --git a/tests/corpus/7381-cycle-deferred-reached-tdz/main.ts b/tests/corpus/7381-cycle-deferred-reached-tdz/main.ts new file mode 100644 index 000000000..6a0b7e3ca --- /dev/null +++ b/tests/corpus/7381-cycle-deferred-reached-tdz/main.ts @@ -0,0 +1,7 @@ +// The first cycle member to run calls a function of the other member +// before that member declared the constant the function reads: Node's +// temporal-dead-zone ReferenceError. A second function of the same member +// reads another constant but runs only after the cycle finished. +import { late, report } from "./alpha.ts"; + +console.log(report(), late()); diff --git a/tests/harness/corpus-costs/7380-cycle-deferred-folded-key/main.ts.json b/tests/harness/corpus-costs/7380-cycle-deferred-folded-key/main.ts.json new file mode 100644 index 000000000..d5d1c5f65 --- /dev/null +++ b/tests/harness/corpus-costs/7380-cycle-deferred-folded-key/main.ts.json @@ -0,0 +1 @@ +{ "plain": 2500, "san": 3500 } diff --git a/tests/harness/corpus-costs/7381-cycle-deferred-reached-tdz/main.ts.json b/tests/harness/corpus-costs/7381-cycle-deferred-reached-tdz/main.ts.json new file mode 100644 index 000000000..b24c0c970 --- /dev/null +++ b/tests/harness/corpus-costs/7381-cycle-deferred-reached-tdz/main.ts.json @@ -0,0 +1 @@ +{ "plain": 1800, "san": 2600 } From 862d5e836d42da00b05dc948d77d9b42c688e5d6 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Thu, 22 Oct 2026 16:12:37 -0700 Subject: [PATCH 013/184] Give the tsgo/tsc-ts IR parity job 60 minutes Its release pass alone takes about 15 minutes, so the dev pass is regularly cancelled at the 30-minute limit on main and on PRs. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a100e374..3b9ef44a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -524,7 +524,7 @@ jobs: if: ${{ needs.validation_scope.outputs.full == 'true' && needs.lanes.outputs.linux == 'true' }} name: test (tsgo and tsc-ts give the same IR) runs-on: ubuntu-24.04 - timeout-minutes: 30 + timeout-minutes: 60 steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 From 43917080c9f8e2af993a632a04178b9b720901c7 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 20:01:47 -0500 Subject: [PATCH 014/184] Reach a class's methods from 'this' in its static initializers The cycle walk resolved only identifiers, so a static field initializer or static block that called a method through 'this' did not reach the method, and its reads of a cycle binding declared later lost their temporal-dead-zone checks. 'this' in static code now reaches the class's stored unit, as naming the class does. --- .../compiler/src/frontend/program.test.ts | 21 ++++++++++ packages/compiler/src/frontend/program.ts | 41 +++++++++++++++++++ .../7382-cycle-static-this-tdz/main.ts.json | 8 ++++ .../7382-cycle-static-this-tdz/alpha.ts | 11 +++++ .../corpus/7382-cycle-static-this-tdz/beta.ts | 26 ++++++++++++ .../corpus/7382-cycle-static-this-tdz/main.ts | 8 ++++ .../7382-cycle-static-this-tdz/main.ts.json | 1 + 7 files changed, 116 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7382-cycle-static-this-tdz/main.ts.json create mode 100644 tests/corpus/7382-cycle-static-this-tdz/alpha.ts create mode 100644 tests/corpus/7382-cycle-static-this-tdz/beta.ts create mode 100644 tests/corpus/7382-cycle-static-this-tdz/main.ts create mode 100644 tests/harness/corpus-costs/7382-cycle-static-this-tdz/main.ts.json diff --git a/packages/compiler/src/frontend/program.test.ts b/packages/compiler/src/frontend/program.test.ts index 5eb3e363e..6a068353a 100644 --- a/packages/compiler/src/frontend/program.test.ts +++ b/packages/compiler/src/frontend/program.test.ts @@ -405,6 +405,27 @@ test.for([ }, ["name"], ], + [ + // The partner's class calls a method through `this` in a static + // initializer; the method reads a const of the importer declared later. + { + "main.ts": 'import { name } from "./a.ts"; console.log(name);', + "a.ts": 'import { Probe } from "./b.ts"; export const name = "n"; console.log(Probe.value);', + "b.ts": + 'import { name } from "./a.ts"; export class Probe { static read(): string { return name; } static value = this.read(); }', + }, + ["name"], + ], + [ + // The same through a static block of a class stored by a const. + { + "main.ts": 'import { name } from "./a.ts"; console.log(name);', + "a.ts": 'import { Box } from "./b.ts"; export const name = "n"; console.log(Box.value);', + "b.ts": + 'import { name } from "./a.ts"; export const Box = class { static value = ""; static read(): string { return name; } static { this.value = this.read(); } };', + }, + ["name"], + ], [ // The partner's top level reads the importer's const before it ran. { diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index f42bc426b..c8384ef65 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -4112,6 +4112,13 @@ function cycleReachedDeferredReferences7( return "skip"; } if (ts.isImportDeclaration(node)) return "skip"; + if (node.kind === ts.SyntaxKind.ThisKeyword) { + // `this` in a static initializer or block names the class, so it + // reaches the class's methods as naming the class does. + const owner = staticThisClass7(node); + if (owner !== undefined && stored.has(owner)) reach(owner); + return undefined; + } if (!ts.isIdentifier(node) || !names.has(node.text)) return undefined; if (inTypePosition7(node)) return undefined; const parent = node.parent; @@ -4497,6 +4504,40 @@ function returnsOnlyClosuresOver7( * expressions, decorators, computed member keys, static field * initializers and static blocks. Methods and instance fields wait for a * call or a construction. */ +/** The class whose static field initializer or static block evaluates `this` + * (through arrow functions, which keep the enclosing `this`), or undefined + * when another function or a class's instance side binds it. */ +function staticThisClass7(node: ts.Node): ts.ClassLikeDeclaration | undefined { + let child = node; + for (let n = node.parent; n !== undefined; child = n, n = n.parent) { + if (ts.isArrowFunction(n)) continue; + if (ts.isClassStaticBlockDeclaration(n)) return n.parent as ts.ClassLikeDeclaration; + if (ts.isPropertyDeclaration(n)) { + if (n.initializer !== child) return undefined; + return (ts.getCombinedModifierFlags(n) & ts.ModifierFlags.Static) !== 0 + ? (n.parent as ts.ClassLikeDeclaration) + : undefined; + } + if (ts.isComputedPropertyName(n)) { + // A computed key evaluates in the scope around the class. + const member = n.parent; + const cls = member?.parent; + if (cls === undefined) return undefined; + child = cls; + n = cls; + continue; + } + if ( + ts.isFunctionLike(n) || + ts.isClassDeclaration(n) || + ts.isClassExpression(n) || + ts.isSourceFile(n) + ) + return undefined; + } + return undefined; +} + function classSyncParts7(c: ts.ClassLikeDeclaration): ts.Node[] { const decorators = (n: ts.Node): ts.Node[] => ((n as { modifiers?: ts.NodeArray }).modifiers?.nodes ?? []).filter( diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7382-cycle-static-this-tdz/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7382-cycle-static-this-tdz/main.ts.json new file mode 100644 index 000000000..986683684 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7382-cycle-static-this-tdz/main.ts.json @@ -0,0 +1,8 @@ +{ + "order": [ + "/tests/corpus/7382-cycle-static-this-tdz/beta.ts", + "/tests/corpus/7382-cycle-static-this-tdz/alpha.ts", + "/tests/corpus/7382-cycle-static-this-tdz/main.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7382-cycle-static-this-tdz/alpha.ts b/tests/corpus/7382-cycle-static-this-tdz/alpha.ts new file mode 100644 index 000000000..ff8eb4a87 --- /dev/null +++ b/tests/corpus/7382-cycle-static-this-tdz/alpha.ts @@ -0,0 +1,11 @@ +import { Probe, Lazy } from "./beta.ts"; + +export { Probe }; +export const label = "ready"; +export const other = "other"; + +export function later(): string { + return `${Lazy.describe()} ${other}`; +} + +console.log("alpha body", Probe.result, Lazy.value); diff --git a/tests/corpus/7382-cycle-static-this-tdz/beta.ts b/tests/corpus/7382-cycle-static-this-tdz/beta.ts new file mode 100644 index 000000000..428907967 --- /dev/null +++ b/tests/corpus/7382-cycle-static-this-tdz/beta.ts @@ -0,0 +1,26 @@ +import { label, other } from "./alpha.ts"; + +function guard(read: () => string): string { + try { + return read(); + } catch (error) { + return error instanceof ReferenceError ? `ReferenceError: ${error.message}` : "other error"; + } +} + +export class Probe { + static helper(): string { + return guard(() => `label ${label}`); + } + static result = this.helper(); +} + +export class Lazy { + static read(): string { + return guard(() => `other ${other}`); + } + static describe(): string { + return `lazy ${other}`; + } + static value = [0].map(() => this.read()).join(""); +} diff --git a/tests/corpus/7382-cycle-static-this-tdz/main.ts b/tests/corpus/7382-cycle-static-this-tdz/main.ts new file mode 100644 index 000000000..e3bbf1581 --- /dev/null +++ b/tests/corpus/7382-cycle-static-this-tdz/main.ts @@ -0,0 +1,8 @@ +// The first cycle member to run builds classes whose static initializers +// call a method through `this`, directly and from an arrow function; both +// methods read constants of the other member before it declared them: +// Node's temporal-dead-zone ReferenceError. A third method reads a constant +// only after the cycle finished. +import { label, Probe, later } from "./alpha.ts"; + +console.log(label, Probe.result, later()); diff --git a/tests/harness/corpus-costs/7382-cycle-static-this-tdz/main.ts.json b/tests/harness/corpus-costs/7382-cycle-static-this-tdz/main.ts.json new file mode 100644 index 000000000..b24c0c970 --- /dev/null +++ b/tests/harness/corpus-costs/7382-cycle-static-this-tdz/main.ts.json @@ -0,0 +1 @@ +{ "plain": 1800, "san": 2600 } From 8d5b3377aa54e2cbfbc89297b3559114cc903808 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 20:33:09 -0500 Subject: [PATCH 015/184] Represent Dates as objects with identity A Date is a refcounted object holding its time value, shared with the checked-dynamic Date handle, so it keeps its identity through unknown. Dates compile in arrays (includes, indexOf, lastIndexOf, join, default sort by toString() text), in Date | undefined and Date | null slots, as Map and Set keys and values, in index signatures, and in records, tuples, class fields, closures, promises and generators. === compares identity, relational operators compare time values, new Date(date) makes a new Date, and JSON.stringify writes toJSON(). Closes #225 Co-authored-by: Max B --- .../content/limitations/builtin-apis/date.mdx | 9 +- .../compiler/src/backend/llvm/debug-info.ts | 7 +- packages/compiler/src/backend/llvm/dyn.ts | 15 ++- .../src/backend/llvm/emit-function.ts | 12 +- .../compiler/src/backend/llvm/emitter.test.ts | 3 +- packages/compiler/src/backend/llvm/emitter.ts | 4 +- .../compiler/src/backend/llvm/expr-async.ts | 24 ++-- .../src/backend/llvm/expr-containers.ts | 16 +++ .../src/backend/llvm/expr-stream-bridges.ts | 7 +- .../compiler/src/backend/llvm/lib-dispatch.ts | 35 +++++- packages/compiler/src/backend/llvm/publish.ts | 6 +- packages/compiler/src/backend/llvm/shapes.ts | 9 +- .../src/backend/llvm/stack-captures.ts | 8 +- packages/compiler/src/backend/llvm/walkers.ts | 15 +++ .../compiler/src/diagnostics/diagnostic.ts | 2 +- .../src/frontend/lowering/builtins/dates.ts | 8 +- .../src/frontend/lowering/call-signatures.ts | 1 - .../lowering/containers/array-callbacks.ts | 13 +- .../lowering/containers/array-methods.ts | 17 ++- .../frontend/lowering/expressions/binary.ts | 45 ++++--- .../src/frontend/lowering/lower-classes.ts | 9 +- .../src/frontend/lowering/lower-generators.ts | 2 +- .../src/frontend/lowering/lower-stmts.ts | 2 +- packages/compiler/src/frontend/type-mapper.ts | 15 +-- packages/compiler/src/ir/ir.test.ts | 3 +- packages/compiler/src/ir/ir.ts | 27 +++- packages/compiler/src/ir/validate.ts | 21 +++- .../compiler/test/collection-keys.test.ts | 4 +- packages/compiler/test/ir.test.ts | 11 +- .../tests/corpus/5965-date-objects.ts.json | 6 + .../diagnostics/date-union-fences.ts.json | 6 - packages/runtime/src/scr_array.c | 47 ++++++- packages/runtime/src/scr_array.h | 4 + packages/runtime/src/scr_dyn.h | 5 + packages/runtime/src/scr_json.h | 4 + packages/runtime/src/scr_lib.c | 110 +++++++++++++---- packages/runtime/src/scr_number.h | 17 +++ tests/corpus/5965-date-objects.ts | 115 ++++++++++++++++++ tests/diagnostics/component-fences.ts | 6 +- tests/diagnostics/date-union-fences.ts | 22 ---- tests/diagnostics/stdlib-fence.ts | 8 -- tests/diagnostics/utility-types.ts | 8 +- .../class-interface-refinements.ts.txt | 2 +- .../__snapshots__/component-fences.ts.txt | 30 ++--- .../__snapshots__/date-union-fences.ts.txt | 56 --------- tests/harness/__snapshots__/generics.ts.txt | 2 +- .../index-signature-values.ts.txt | 4 +- .../harness/__snapshots__/stdlib-fence.ts.txt | 101 ++++++--------- tests/harness/__snapshots__/tuples.ts.txt | 4 +- .../__snapshots__/utility-types.ts.txt | 17 +-- .../corpus-costs/5965-date-objects.ts.json | 1 + 51 files changed, 571 insertions(+), 354 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5965-date-objects.ts.json delete mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/date-union-fences.ts.json create mode 100644 tests/corpus/5965-date-objects.ts delete mode 100644 tests/diagnostics/date-union-fences.ts delete mode 100644 tests/harness/__snapshots__/date-union-fences.ts.txt create mode 100644 tests/harness/corpus-costs/5965-date-objects.ts.json diff --git a/docs/content/limitations/builtin-apis/date.mdx b/docs/content/limitations/builtin-apis/date.mdx index ac8bf9115..89266b745 100644 --- a/docs/content/limitations/builtin-apis/date.mdx +++ b/docs/content/limitations/builtin-apis/date.mdx @@ -1,5 +1,6 @@ -- **Date:** Date unions, spread constructor arguments, setters, statically typed TypeScript identity comparisons, thrown Date values, and locale formatters are unsupported. - Zero-argument, single number/string, and local-time year/month field constructors, storage and passing, `getTime`, `valueOf`, `toISOString`, `toUTCString`, `toJSON` (a `string | null`: null for an invalid date, as in Node), `toString`, `toDateString`, `toTimeString`, `String(date)` and template literal conversion, calendar getters, `getTimezoneOffset`, and single-string `Date.parse` compile. - JavaScript Dates retain identity through checked storage and support Date-copy construction and JSON serialization. - A TypeScript Date converted to `unknown` becomes a Date with the same time value; each conversion creates a new Date, so two `unknown` values converted from one TypeScript Date are not `===`. +- **Date:** Spread constructor arguments, setters, thrown Date values, and locale formatters are unsupported. + Dates are objects: `===` compares identity, `new Date(date)` makes a new Date, and Dates compile in arrays (`includes`, `indexOf`, `lastIndexOf`, `join`, and default `sort` by `toString()` text), in `Date | undefined` and `Date | null` slots, as Map and Set keys and values, and in records, tuples, class fields, closures, promises and generators. + Unions of a Date with other non-nullish types are unsupported. + Zero-argument, single number/string, and local-time year/month field constructors, `getTime`, `valueOf`, `toISOString`, `toUTCString`, `toJSON` (a `string | null`: null for an invalid date, as in Node), `toString`, `toDateString`, `toTimeString`, `String(date)` and template literal conversion, relational comparison, calendar getters, `getTimezoneOffset`, and single-string `Date.parse` compile. + A Date converted to `unknown` is the same Date object. See [Dates and timezones](#dates-and-timezones) for timezone data. diff --git a/packages/compiler/src/backend/llvm/debug-info.ts b/packages/compiler/src/backend/llvm/debug-info.ts index 4e37ee5bc..3a75c6263 100644 --- a/packages/compiler/src/backend/llvm/debug-info.ts +++ b/packages/compiler/src/backend/llvm/debug-info.ts @@ -139,7 +139,6 @@ export class LlvmDebugInfo { let id: string; switch (type.kind) { case "f64": - case "date": case "procStream": id = this.add( `!DIBasicType(name: ${quoted(type.kind === "f64" ? "number" : type.kind)}, size: 64, encoding: DW_ATE_float)`, @@ -190,7 +189,7 @@ export class LlvmDebugInfo { ? [] : [ this.add( - `!DIDerivedType(tag: DW_TAG_member, name: "arm${i}", baseType: ${this.type(arm)}, size: ${arm.kind === "bool" ? 8 : arm.kind === "f64" || arm.kind === "date" ? 64 : this.pointerBits})`, + `!DIDerivedType(tag: DW_TAG_member, name: "arm${i}", baseType: ${this.type(arm)}, size: ${arm.kind === "bool" ? 8 : arm.kind === "f64" ? 64 : this.pointerBits})`, ), ], ); @@ -239,9 +238,7 @@ export class LlvmDebugInfo { // Scalar forward captures use an array-backed TDZ box. Expose that // exceptional representation opaquely rather than reporting a bad value. const opaqueBox = - local.boxed && - local.tdz && - (local.type.kind === "f64" || local.type.kind === "bool" || local.type.kind === "date"); + local.boxed && local.tdz && (local.type.kind === "f64" || local.type.kind === "bool"); const type = opaqueBox ? this.pointer( this.add( diff --git a/packages/compiler/src/backend/llvm/dyn.ts b/packages/compiler/src/backend/llvm/dyn.ts index bb00e4563..a46c81658 100644 --- a/packages/compiler/src/backend/llvm/dyn.ts +++ b/packages/compiler/src/backend/llvm/dyn.ts @@ -421,7 +421,7 @@ export class LlDyn { /** The value LLVM type of a dynCheck result / toDyn operand for `t`. */ private valTy(t: IrType): string { - return t.kind === "f64" || t.kind === "date" || t.kind === "procStream" + return t.kind === "f64" || t.kind === "procStream" ? "double" : t.kind === "bool" ? "i1" @@ -1856,10 +1856,10 @@ export class LlDyn { }; switch (t.kind) { case "date": { - host.declare(`declare double @scr_dyn_native_date_value(ptr)`); + host.declare(`declare ptr @scr_dyn_native_date_check(ptr, ptr)`); const r = B.tmp(); - B.line(`${r} = call double @scr_dyn_native_date_value(ptr %d)`); - B.terminate(`ret double ${r}`); + B.line(`${r} = call ptr @scr_dyn_native_date_check(ptr %d, ptr %path)`); + B.terminate(`ret ptr ${r}`); break; } case "url": @@ -2645,11 +2645,10 @@ export class LlDyn { let sourceAccessor: { name: string; release: string } | null = null; switch (t.kind) { case "date": { - // A Date handle holding the same time value; native Dates are - // immutable values, and so is the handle. - host.declare(`declare ptr @scr_dyn_native_date_from_ms(double)`); + // A Date handle holding the same Date object. + host.declare(`declare ptr @scr_dyn_native_date(ptr)`); const r = B.tmp(); - B.line(`${r} = call ptr @scr_dyn_native_date_from_ms(double %v)`); + B.line(`${r} = call ptr @scr_dyn_native_date(ptr %v)`); B.terminate(`ret ptr ${r}`); break; } diff --git a/packages/compiler/src/backend/llvm/emit-function.ts b/packages/compiler/src/backend/llvm/emit-function.ts index 16e654393..501224369 100644 --- a/packages/compiler/src/backend/llvm/emit-function.ts +++ b/packages/compiler/src/backend/llvm/emit-function.ts @@ -301,11 +301,7 @@ export function emitFunction(emitter: LlEmitter, fn: IrFunction): string { // ScrGen owns the frame), so its OUT slot address is fixed. let out: string | null = null; const yieldKind = fn.generator?.yieldT.kind; - if ( - !emitter.wasi && - fn.async !== true && - (yieldKind === "f64" || yieldKind === "date" || yieldKind === "bool") - ) { + if (!emitter.wasi && fn.async !== true && (yieldKind === "f64" || yieldKind === "bool")) { emitter.declare(`declare ptr @scr_gen_of_fiber(ptr)`); const gen = B.tmp(); B.line(`${gen} = call ptr @scr_gen_of_fiber(ptr ${self})`); @@ -701,7 +697,6 @@ export function emitGenScaffolding(emitter: LlEmitter): string[] { tr.push(` br label %done ; void body: the done value is undefined (NONE)`); break; case "f64": - case "date": emitter.declare(`declare void @scr_gen_out_f64(ptr, double)`); tr.push(` call void @scr_gen_out_f64(ptr %g, double %r)`, ` br label %done`); break; @@ -1050,7 +1045,6 @@ export function emitAsyncAwaitCall( emitter.declare(`declare void @scr_await_void(ptr)`); return ` call void @scr_await_void(ptr ${promise})`; case "f64": - case "date": emitter.declare(`declare double @scr_await_f64(ptr)`); return ` ${dest} = call double @scr_await_f64(ptr ${promise})`; case "bool": @@ -1105,7 +1099,7 @@ export function emitAsyncAwaitCall( const kind = ret.kind === "void" ? "void" - : ret.kind === "f64" || ret.kind === "date" + : ret.kind === "f64" ? "f64" : ret.kind === "bool" ? "bool" @@ -1184,7 +1178,6 @@ export function emitWasiFulfill(emitter: LlEmitter, v: LlValue | null): void { case "void": return; case "f64": - case "date": emitter.declare(`declare void @scr_gen_out_f64(ptr, double)`); emitter.B.line(`call void @scr_gen_out_f64(ptr ${gen}, double ${v!.name})`); return; @@ -1209,7 +1202,6 @@ export function emitWasiFulfill(emitter: LlEmitter, v: LlValue | null): void { emitter.B.line(`call void @scr_promise_fulfill_void(ptr ${pr})`); break; case "f64": - case "date": emitter.declare(`declare void @scr_promise_fulfill_f64(ptr, double)`); emitter.B.line(`call void @scr_promise_fulfill_f64(ptr ${pr}, double ${v!.name})`); break; diff --git a/packages/compiler/src/backend/llvm/emitter.test.ts b/packages/compiler/src/backend/llvm/emitter.test.ts index d1fd62a83..8653dfd4e 100644 --- a/packages/compiler/src/backend/llvm/emitter.test.ts +++ b/packages/compiler/src/backend/llvm/emitter.test.ts @@ -1,7 +1,6 @@ import { expect, test } from "vitest"; import { BOOL, - DATE_T, F64, PROCSTREAM_T, STRING, @@ -287,7 +286,7 @@ test("shared exceptional exits preserve the function's scalar return ABI", () => }); test("exceptional exits use each scalar kind's LLVM return type", () => { - for (const type of [F64, DATE_T, BOOL, PROCSTREAM_T]) { + for (const type of [F64, BOOL, PROCSTREAM_T]) { for (const ownedLocals of [false, true]) { const module = moduleFor([call(), call(), { kind: "throw", value: literal("failed"), loc }]); const work = module.functions[1]!; diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 9c42920b9..8db63daa3 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -1762,7 +1762,6 @@ export class LlEmitter { if (POINTER_KINDS.has(t.kind)) return "ptr"; switch (t.kind) { case "f64": - case "date": return "double"; case "bool": return "i1"; @@ -1895,7 +1894,6 @@ export class LlEmitter { this.declare(`declare void @scr_promise_fulfill_void(ptr)`); return [` call void @scr_promise_fulfill_void(ptr ${promise})`]; case "f64": - case "date": this.declare(`declare void @scr_promise_fulfill_f64(ptr, double)`); return [` call void @scr_promise_fulfill_f64(ptr ${promise}, double %r)`]; case "bool": @@ -2125,7 +2123,7 @@ export class LlEmitter { return { name: "", type }; } const t = B.tmp(); - if (type.kind === "f64" || type.kind === "date") { + if (type.kind === "f64") { this.declare(`declare double @scr_coro_direct_f64(ptr)`); B.line(`${t} = call double @scr_coro_direct_f64(ptr ${record})`); } else if (type.kind === "bool") { diff --git a/packages/compiler/src/backend/llvm/expr-async.ts b/packages/compiler/src/backend/llvm/expr-async.ts index b5d765b16..ee9ff2983 100644 --- a/packages/compiler/src/backend/llvm/expr-async.ts +++ b/packages/compiler/src/backend/llvm/expr-async.ts @@ -184,7 +184,7 @@ export function emitIntrinsicExpr(host: LlvmEmitterContext, e: ExprOf<"intrinsic } const v = host.emitExpr(e.args[0]!); const t = e.args[0]!.type; - if (t.kind === "f64" || t.kind === "date") { + if (t.kind === "f64") { host.declare(`declare void @scr_promise_fulfill_f64(ptr, double)`); B.line(`call void @scr_promise_fulfill_f64(ptr ${p}, double ${v.name})`); } else if (t.kind === "bool") { @@ -500,7 +500,7 @@ export function emitAsyncExpr( host.declare(`declare void @scr_async_gen_hop_done()`); B.line(`call void @scr_async_gen_hop_done()`); } - if (yt.kind === "f64" || yt.kind === "date") { + if (yt.kind === "f64") { if (host.currentCoro !== null) { const coro = host.currentCoro; if (coro.out !== null) host.emitGenSlotStore(coro.out, "f64", v.name); @@ -582,7 +582,7 @@ export function emitAsyncExpr( return { name: "", type: e.type }; } const t = B.tmp(); - if (e.type.kind === "f64" || e.type.kind === "date") { + if (e.type.kind === "f64") { host.declare(`declare double @scr_gen_take_in_f64()`); B.line(`${t} = call double @scr_gen_take_in_f64()`); return { name: t, type: e.type }; @@ -710,12 +710,12 @@ export function emitAsyncExpr( throw new InternalCompilerError(`llvm emitter bug: genResume ${e.mode} on async`); const gb = emitBorrowedInput(host, e.gen); if (e.mode === "value") { - if (!host.wasi && (e.type.kind === "f64" || e.type.kind === "date")) + if (!host.wasi && e.type.kind === "f64") return { name: host.emitGenSlotTake(host.genOutSlot(gb.name), "f64"), type: e.type }; if (!host.wasi && e.type.kind === "bool") return { name: host.emitGenSlotTake(host.genOutSlot(gb.name), "bool"), type: e.type }; const t = B.tmp(); - if (e.type.kind === "f64" || e.type.kind === "date") { + if (e.type.kind === "f64") { host.declare(`declare double @scr_gen_take_out_f64(ptr)`); B.line(`${t} = call double @scr_gen_take_out_f64(ptr ${gb.name})`); return { name: t, type: e.type }; @@ -752,7 +752,7 @@ export function emitAsyncExpr( store(a.name, e.arg!.type); }; const parkIn = (name: string, t: IrType): void => { - if (t.kind === "f64" || t.kind === "date") { + if (t.kind === "f64") { host.declare(`declare void @scr_gen_in_f64(ptr, double)`); B.line(`call void @scr_gen_in_f64(ptr ${g.name}, double ${name})`); } else if (t.kind === "bool") { @@ -787,7 +787,7 @@ export function emitAsyncExpr( const a = host.emitExpr(e.arg); const t = e.arg.type; if (isRefCounted(t)) host.toHeap(a); - if (t.kind === "f64" || t.kind === "date") { + if (t.kind === "f64") { host.declare(`declare ptr @scr_async_gen_next_f64(ptr, double)`); call = `call ptr @scr_async_gen_next_f64(ptr ${g.name}, double ${a.name})`; } else if (t.kind === "bool") { @@ -806,7 +806,7 @@ export function emitAsyncExpr( const a = host.emitExpr(e.arg); const t = e.arg.type; if (isRefCounted(t)) host.toHeap(a); - if (t.kind === "f64" || t.kind === "date") { + if (t.kind === "f64") { host.declare(`declare ptr @scr_async_gen_return_f64(ptr, double)`); call = `call ptr @scr_async_gen_return_f64(ptr ${g.name}, double ${a.name})`; } else if (t.kind === "bool") { @@ -849,7 +849,7 @@ export function emitAsyncExpr( B.line(`call void @scr_gen_ret_none(ptr ${g.name})`); } else { sendArg((name, t) => { - if (t.kind === "f64" || t.kind === "date") { + if (t.kind === "f64") { host.declare(`declare void @scr_gen_ret_f64(ptr, double)`); B.line(`call void @scr_gen_ret_f64(ptr ${g.name}, double ${name})`); } else if (t.kind === "bool") { @@ -960,7 +960,7 @@ export function emitAsyncExpr( return { name: "", type: e.type }; } const t = B.tmp(); - if (e.type.kind === "f64" || e.type.kind === "date") { + if (e.type.kind === "f64") { const f64Await = host.awaitResultFn("f64"); host.declare(`declare double @${f64Await}(ptr)`); B.line(`${t} = call double @${f64Await}(ptr ${pr.name})`); @@ -1050,7 +1050,7 @@ export function emitAsyncExpr( const peek = host.unionPeek(u.name, def.id); if (host.coroAwait) host.emitCoroAwait(peek); let awaited: LlValue; - if (inner.kind === "f64" || inner.kind === "date") { + if (inner.kind === "f64") { const f64Await = host.awaitResultFn("f64"); host.declare(`declare double @${f64Await}(ptr)`); const x = B.tmp(); @@ -1136,7 +1136,6 @@ export function emitAsyncExpr( let resolve: string; const kindNums: Partial> = { f64: 0, - date: 0, bool: 1, string: 2, void: 3, @@ -1191,7 +1190,6 @@ export function emitAsyncExpr( B.line(`${p} = call ptr @scr_promise_new()`); const kindNums: Partial> = { f64: 0, - date: 0, bool: 1, string: 2, void: 3, diff --git a/packages/compiler/src/backend/llvm/expr-containers.ts b/packages/compiler/src/backend/llvm/expr-containers.ts index 2f4cd98e9..b70bfcd23 100644 --- a/packages/compiler/src/backend/llvm/expr-containers.ts +++ b/packages/compiler/src/backend/llvm/expr-containers.ts @@ -863,6 +863,12 @@ export function emitArrIntrinsic( B.line(`${t} = call ptr @${helper}(ptr ${r.name}, ptr ${sep.name})`); return host.own({ name: t, type: e.type }); } + if (elem.kind === "date") { + host.declare(`declare ptr @scr_date_arr_join(ptr, ptr)`); + const t = B.tmp(); + B.line(`${t} = call ptr @scr_date_arr_join(ptr ${r.name}, ptr ${sep.name})`); + return host.own({ name: t, type: e.type }); + } host.declare(`declare ptr @scr_arr_join(ptr, ptr)`); const t = B.tmp(); B.line(`${t} = call ptr @scr_arr_join(ptr ${r.name}, ptr ${sep.name})`); @@ -887,6 +893,16 @@ export function emitArrIntrinsic( case "sortPrimitive": case "toSortedPrimitive": { if (e.method === "sortPrimitive") host.emitPublishedArrayGuard(r.name, e.receiver.type, 7); + if (elem.kind === "date") { + // Dates order by their toString() text. + host.declare(`declare ptr @scr_arr_sort_by_text(ptr, i1 zeroext, ptr)`); + host.declare(`declare ptr @scr_date_object_to_string(ptr)`); + const sorted = B.tmp(); + B.line( + `${sorted} = call ptr @scr_arr_sort_by_text(ptr ${r.name}, i1 ${e.method === "toSortedPrimitive" ? 1 : 0}, ptr @scr_date_object_to_string)`, + ); + return host.own({ name: sorted, type: e.type }); + } host.declare(`declare ptr @scr_arr_sort_primitive(ptr, i1 zeroext)`); const result = B.tmp(); B.line( diff --git a/packages/compiler/src/backend/llvm/expr-stream-bridges.ts b/packages/compiler/src/backend/llvm/expr-stream-bridges.ts index 7cb0fa772..877a01601 100644 --- a/packages/compiler/src/backend/llvm/expr-stream-bridges.ts +++ b/packages/compiler/src/backend/llvm/expr-stream-bridges.ts @@ -118,12 +118,7 @@ export function dynPromiseAdapter(host: LlvmEmitterContext, inner: IrType): stri if (inner.kind === "void") { host.declare(`declare void @scr_promise_fulfill_void(ptr)`); B.line(`call void @scr_promise_fulfill_void(ptr %dst)`); - } else if ( - inner.kind === "f64" || - inner.kind === "date" || - inner.kind === "procStream" || - inner.kind === "bool" - ) { + } else if (inner.kind === "f64" || inner.kind === "procStream" || inner.kind === "bool") { const fn = inner.kind !== "bool" ? "scr_promise_fulfill_f64" : "scr_promise_fulfill_bool"; host.declare( `declare void @${fn}(ptr, ${host.llType(inner)}${inner.kind === "bool" ? " zeroext" : ""})`, diff --git a/packages/compiler/src/backend/llvm/lib-dispatch.ts b/packages/compiler/src/backend/llvm/lib-dispatch.ts index cebe661d4..753f296e6 100644 --- a/packages/compiler/src/backend/llvm/lib-dispatch.ts +++ b/packages/compiler/src/backend/llvm/lib-dispatch.ts @@ -9,7 +9,7 @@ import { emitTestLibCall } from "./lib-test.js"; import { InternalCompilerError } from "../../errors.js"; import { MAY_THROW_LIB_FNS } from "../../ir/builtin-effects.js"; -import type { IrExpr, IrType } from "../../ir/ir.js"; +import { F64, type IrExpr, type IrType } from "../../ir/ir.js"; import { LlvmUnsupportedError } from "./unsupported.js"; import { emitJsLibCall } from "./jsval.js"; import type { LlvmEmitterContext, LibCallExpr, LibCallPrefix, LlValue } from "./expr-context.js"; @@ -569,6 +569,39 @@ export function emitGenericLibCall(host: LlvmEmitterContext, e: LibCallExpr): Ll // The deep comparisons' number test: computed here, with no runtime call. if (e.fn === "assert.sameValue") return { name: emitSameValueF64(host, args[0]!.name, args[1]!.name), type: e.type }; + // The runtime's Date functions take and answer time values: a Date + // operand passes its time value, and a Date result is a new Date object. + const callArgs = args.map((a) => + a.type.kind === "date" ? { name: emitDateTime(host, a.name), type: F64 } : a, + ); + if (e.type.kind === "date") { + const ms = emitSymbolLibCall(host, { ...e, type: F64 }, sym, callArgs); + host.declare(`declare ptr @scr_date_new(double)`); + const date = B.tmp(); + B.line(`${date} = call ptr @scr_date_new(double ${ms.name})`); + return host.own({ name: date, type: e.type }); + } + return emitSymbolLibCall(host, e, sym, callArgs); +} + +/** A Date object's time value (ScrDate.ms). */ +function emitDateTime(host: LlvmEmitterContext, date: string): string { + const B = host.B; + const field = B.tmp(); + const ms = B.tmp(); + B.line(`${field} = getelementptr inbounds i8, ptr ${date}, i64 8 ; ->ms`); + B.line(`${ms} = load double, ptr ${field}`); + return ms; +} + +/** The call of runtime function `sym` with evaluated arguments. */ +function emitSymbolLibCall( + host: LlvmEmitterContext, + e: LibCallExpr, + sym: string, + args: readonly LlValue[], +): LlValue { + const B = host.B; const argDecls = args.map((a) => { const ty = host.llType(a.type); return ty === "i1" ? "i1 zeroext" : ty; diff --git a/packages/compiler/src/backend/llvm/publish.ts b/packages/compiler/src/backend/llvm/publish.ts index 60270adda..62fdd75ec 100644 --- a/packages/compiler/src/backend/llvm/publish.ts +++ b/packages/compiler/src/backend/llvm/publish.ts @@ -596,10 +596,10 @@ export class PublishWalkers { } } -/** Unpublishable types stored as scalars (dates are time values, process - * streams file descriptors): no pointer to push, and always a value. */ +/** Unpublishable types stored as scalars (process streams are file + * descriptors): no pointer to push, and always a value. */ function scalarRefusal(t: IrType): boolean { - return t.kind === "date" || t.kind === "procStream"; + return t.kind === "procStream"; } /** The Node.js implementation's refusal for a value of type `t`. */ diff --git a/packages/compiler/src/backend/llvm/shapes.ts b/packages/compiler/src/backend/llvm/shapes.ts index d13f80a78..e6e375223 100644 --- a/packages/compiler/src/backend/llvm/shapes.ts +++ b/packages/compiler/src/backend/llvm/shapes.ts @@ -1099,6 +1099,7 @@ export function arrNewCall( elem.kind === "stats" || // fs.Stats snapshots: scr_stats_* adapters, no trace elem.kind === "dyn" || // native collection seeds/drains: traced REF elem.kind === "regex" || // RegExp values: scr_regex_* adapters, no trace (no refs inside) + elem.kind === "date" || // Date objects: scr_date_* adapters, no trace (no refs inside) (elem.kind === "array" && traceAdapter(host, elem) !== null); if (!useRef) { if (small) { @@ -1162,7 +1163,6 @@ const SCR_ARR_OPS_RELEASE_NOTHING = 2; export function boxNewCall(host: ShapeHost, t: IrType): string { const plain: Partial> = { f64: 0, - date: 0, procStream: 0, bool: 1, string: 2, @@ -1187,11 +1187,7 @@ export function boxNewCall(host: ShapeHost, t: IrType): string { /** Box accessor suffix (boxAccess): scalars unboxed, ref kinds pointers. */ export function boxAccess(t: IrType): "f64" | "bool" | "ref" { - return t.kind === "f64" || t.kind === "date" || t.kind === "procStream" - ? "f64" - : t.kind === "bool" - ? "bool" - : "ref"; + return t.kind === "f64" || t.kind === "procStream" ? "f64" : t.kind === "bool" ? "bool" : "ref"; } /* ── record shapes ────────────────────────────────────────────────────── */ @@ -1207,7 +1203,6 @@ export function llFieldType( if (POINTER_KINDS.has(t.kind)) return "ptr"; switch (t.kind) { case "f64": - case "date": case "procStream": return "double"; case "bool": diff --git a/packages/compiler/src/backend/llvm/stack-captures.ts b/packages/compiler/src/backend/llvm/stack-captures.ts index f0812b7b0..40e1a0428 100644 --- a/packages/compiler/src/backend/llvm/stack-captures.ts +++ b/packages/compiler/src/backend/llvm/stack-captures.ts @@ -105,13 +105,7 @@ export class StackCaptures { // sites, including duplicated finally bodies and versioned loops. if (!this.locals.has(id) || this.remaining < 48) return null; const ref = isRefCounted(type); - if ( - !ref && - type.kind !== "f64" && - type.kind !== "date" && - type.kind !== "procStream" && - type.kind !== "bool" - ) + if (!ref && type.kind !== "f64" && type.kind !== "procStream" && type.kind !== "bool") return null; this.remaining -= 48; const B = host.B; diff --git a/packages/compiler/src/backend/llvm/walkers.ts b/packages/compiler/src/backend/llvm/walkers.ts index 044cb79ae..7c17a94f4 100644 --- a/packages/compiler/src/backend/llvm/walkers.ts +++ b/packages/compiler/src/backend/llvm/walkers.ts @@ -233,6 +233,10 @@ export class LlWalkers { this.host.declare(`declare void @scr_jb_put_json_str(ptr, ptr)`); B.line(`call void @scr_jb_put_json_str(ptr %b, ptr %v)`); break; + case "date": + this.host.declare(`declare void @scr_jb_put_date(ptr, ptr)`); + B.line(`call void @scr_jb_put_date(ptr %b, ptr %v) ; toJSON(): ISO text, null if invalid`); + break; case "dyn": // A checked-dynamic value serializes itself (runtime walker). // Bare stringify of dyn stays frontend-fenced, and overflow @@ -924,6 +928,17 @@ export class LlWalkers { case "string": this.putScrStr(B, buf, this.unionPeek(B, u, unionId)); break; + case "date": { + host.declare(`declare ptr @scr_date_object_to_string(ptr)`); + host.declare(`declare void @scr_str_release(ptr)`); + const text = B.tmp(); + B.line( + `${text} = call ptr @scr_date_object_to_string(ptr ${this.unionPeek(B, u, unionId)})`, + ); + this.putScrStr(B, buf, text); + B.line(`call void @scr_str_release(ptr ${text})`); + break; + } case "f64": { host.declare(`declare double @scr_union_get_f64(ptr)`); host.declare(`declare void @scr_jb_put_number(ptr, double)`); diff --git a/packages/compiler/src/diagnostics/diagnostic.ts b/packages/compiler/src/diagnostics/diagnostic.ts index 625ee93af..2fd8b6fb0 100644 --- a/packages/compiler/src/diagnostics/diagnostic.ts +++ b/packages/compiler/src/diagnostics/diagnostic.ts @@ -656,7 +656,7 @@ export function genericSignatureTypeDiag(typeText: string, loc: SrcLoc): ScrDiag export function indexSignatureTypeDiag(typeText: string, loc: SrcLoc): ScrDiagnostic { return { code: "SC2006", - message: `values of type '${typeText}' cannot be compiled: this index signature is outside the supported shape (string or number keys; values limited to numbers, strings, booleans, records, classes, arrays, unions, functions, Maps, Sets, RegExps, Promises, or 'unknown')`, + message: `values of type '${typeText}' cannot be compiled: this index signature is outside the supported shape (string or number keys; values limited to numbers, strings, booleans, records, classes, arrays, unions, functions, Maps, Sets, RegExps, Dates, Promises, or 'unknown')`, loc, milestone: "M2", hint: "string- and number-keyed index signatures over the supported value types compile directly; symbol keys and dual signatures with unequal value types have no lowering", diff --git a/packages/compiler/src/frontend/lowering/builtins/dates.ts b/packages/compiler/src/frontend/lowering/builtins/dates.ts index a598f4ee6..0204386ab 100644 --- a/packages/compiler/src/frontend/lowering/builtins/dates.ts +++ b/packages/compiler/src/frontend/lowering/builtins/dates.ts @@ -53,7 +53,13 @@ export function lowerDateCall( call: ts.CallExpression, access: ts.PropertyAccessExpression, ): IrExpr | null { - if (call.questionDotToken || access.questionDotToken) return null; + // An optional chain lowers the call again once it has narrowed the + // receiver (Lowerer.chainHandled). + if ( + (call.questionDotToken && !lowerer.chainHandled.has(call)) || + (access.questionDotToken && !lowerer.chainHandled.has(access)) + ) + return null; const loc = locOf(call); const member = lowerer.stdlibGlobalMember(access, "Date"); if (member === "now" || member === "parse" || member === "UTC") diff --git a/packages/compiler/src/frontend/lowering/call-signatures.ts b/packages/compiler/src/frontend/lowering/call-signatures.ts index dacb4d324..c9fd954b5 100644 --- a/packages/compiler/src/frontend/lowering/call-signatures.ts +++ b/packages/compiler/src/frontend/lowering/call-signatures.ts @@ -611,7 +611,6 @@ export function checkDefaultParamBodyType( if ( bodyType.kind === "void" || bodyType.kind === "regex" || - bodyType.kind === "date" || bodyType.kind === "dyn" || isUnitType(bodyType) ) { diff --git a/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts b/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts index 86f214bc1..edf65ab2b 100644 --- a/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts +++ b/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts @@ -2358,10 +2358,11 @@ function reduceHelper( * the left element first. Undefined values sink without reaching the user * comparator. The callback sequence differs from V8's TimSort, so exact * order and count parity are not claimed; results agree for consistent - * comparators. Without a comparator, string, number and boolean arrays use - * native stable ordering: their String conversions cannot call user code. - * Strings compare UTF-16 units; numbers compare their decimal spellings - * ([10, 9, 1] → [1, 10, 9]). Other element types still need a comparator. */ + * comparators. Without a comparator, string, number, boolean and Date + * arrays use native stable ordering: their String conversions cannot call + * user code. Strings compare UTF-16 units; numbers compare their decimal + * spellings ([10, 9, 1] → [1, 10, 9]); Dates compare their toString() text. + * Other element types still need a comparator. */ export function lowerArraySortCall( lowerer: Lowerer, call: ts.CallExpression, @@ -2374,7 +2375,7 @@ export function lowerArraySortCall( const copyFirst = method === "toSorted"; if ( call.arguments.nodes.length === 0 && - (elem.kind === "string" || elem.kind === "f64" || elem.kind === "bool") + (elem.kind === "string" || elem.kind === "f64" || elem.kind === "bool" || elem.kind === "date") ) { return { kind: "arrIntrinsic", @@ -2389,7 +2390,7 @@ export function lowerArraySortCall( lowerer.noLowering( `.${method} with ${call.arguments.nodes.length} arguments`, call, - "default ordering supports string[], number[] and boolean[]; other element types need a comparator", + "default ordering supports string[], number[], boolean[] and Date[]; other element types need a comparator", ); } const receiver = lowerer.lowerExpr(access.expression); diff --git a/packages/compiler/src/frontend/lowering/containers/array-methods.ts b/packages/compiler/src/frontend/lowering/containers/array-methods.ts index 44182fbda..8a736e75f 100644 --- a/packages/compiler/src/frontend/lowering/containers/array-methods.ts +++ b/packages/compiler/src/frontend/lowering/containers/array-methods.ts @@ -1066,14 +1066,25 @@ export function lowerArrayMethodCall( (lowerer.unions .get(elem.unionId) ?.arms.every( - (a) => a.kind === "f64" || a.kind === "string" || a.kind === "bool" || isUnitType(a), + (a) => + a.kind === "f64" || + a.kind === "string" || + a.kind === "bool" || + a.kind === "date" || + isUnitType(a), ) ?? false); - if (elem.kind !== "f64" && elem.kind !== "string" && elem.kind !== "bool" && !joinableUnion) { + if ( + elem.kind !== "f64" && + elem.kind !== "string" && + elem.kind !== "bool" && + elem.kind !== "date" && + !joinableUnion + ) { lowerer.unsupported( "SC1090", call, - "'.join()' on arrays of this element type (number, string, and boolean arrays join — unions of those with undefined/null arms too, the units printing empty like JS)", + "'.join()' on arrays of this element type (number, string, boolean and Date arrays join — unions of those with undefined/null arms too, the units printing empty like JS)", ); } const receiver = lowerer.lowerExpr(access.expression); diff --git a/packages/compiler/src/frontend/lowering/expressions/binary.ts b/packages/compiler/src/frontend/lowering/expressions/binary.ts index 9702a6dd9..ed4771b05 100644 --- a/packages/compiler/src/frontend/lowering/expressions/binary.ts +++ b/packages/compiler/src/frontend/lowering/expressions/binary.ts @@ -759,10 +759,21 @@ export function ensureString(lowerer: Lowerer, e: IrExpr, node: ts.Node): IrExpr (lowerer.unions .get(elem.unionId) ?.arms.every( - (a) => a.kind === "f64" || a.kind === "string" || a.kind === "bool" || isUnitType(a), + (a) => + a.kind === "f64" || + a.kind === "string" || + a.kind === "bool" || + a.kind === "date" || + isUnitType(a), ) ?? false); - if (elem.kind === "f64" || elem.kind === "string" || elem.kind === "bool" || joinableUnion) { + if ( + elem.kind === "f64" || + elem.kind === "string" || + elem.kind === "bool" || + elem.kind === "date" || + joinableUnion + ) { const sep: IrExpr = { kind: "strLit", value: ",", type: STRING, loc: e.loc }; return { kind: "arrIntrinsic", @@ -2166,14 +2177,18 @@ export function lowerBinary(lowerer: Lowerer, expr: ts.BinaryExpression): IrExpr loc, }; if (bothStr) return { kind: "strCmp", op: cmpOp, left, right, type: BOOL, loc }; - // A Date converts to a new checked-dynamic Date on each crossing, so - // comparisons keep their Date fence instead of comparing copies. - if ( - left.type.kind !== "date" && - right.type.kind !== "date" && - lowerer.dynConvertible(left.type) && - lowerer.dynConvertible(right.type) - ) { + // Dates compare by time value, the number hint of ToPrimitive. + if (left.type.kind === "date" && right.type.kind === "date") { + const time = (d: IrExpr): IrExpr => ({ + kind: "libCall", + fn: "date.getTime", + args: [d], + type: F64, + loc, + }); + return { kind: "bin", op: cmpOp, left: time(left), right: time(right), type: BOOL, loc }; + } + if (lowerer.dynConvertible(left.type) && lowerer.dynConvertible(right.type)) { return { kind: "libCall", fn: "dyn.compare", @@ -2387,6 +2402,7 @@ export function lowerStrictEquality( idLeft.type.kind === "map" || idLeft.type.kind === "set" || idLeft.type.kind === "regex" || + idLeft.type.kind === "date" || idLeft.type.kind === "url" || idLeft.type.kind === "object" || idLeft.type.kind === "record" || @@ -2423,14 +2439,7 @@ export function lowerStrictEquality( loc, }; } - // A Date converts to a new checked-dynamic Date on each crossing; its - // identity has no representation, so equality keeps the fence. - if ( - idLeft.type.kind !== "date" && - idRight.type.kind !== "date" && - lowerer.dynConvertible(idLeft.type) && - lowerer.dynConvertible(idRight.type) - ) { + if (lowerer.dynConvertible(idLeft.type) && lowerer.dynConvertible(idRight.type)) { return { kind: "dynScalarEq", left: lowerer.coerceToExpected(idLeft, DYN), diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index e264e0a3a..5079f3c9e 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -1415,7 +1415,7 @@ function undefArmedFieldType(lowerer: Lowerer, p: ts.Symbol): IrType | null { const arms = mapped.kind === "union" ? (lowerer.unions.get(mapped.unionId)?.arms ?? []) : [mapped]; for (const a of arms) { - if (a.kind === "map" || a.kind === "regex" || a.kind === "date" || a.kind === "generator") { + if (a.kind === "map" || a.kind === "regex" || a.kind === "generator") { return null; } byKey.set(typeKey(a), a); @@ -3971,7 +3971,6 @@ export function collectClassShapeInner( declared.kind !== "union" && declared.kind !== "dyn" && declared.kind !== "map" && - declared.kind !== "date" && declared.kind !== "generator" && declared.kind !== "void" && declared.kind !== "caught"; @@ -10324,8 +10323,10 @@ export function lowerNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr { if (arg.type.kind === "string") { return { kind: "libCall", fn: "date.newString", args: [arg], type: DATE_T, loc }; } + // new Date(date): a new Date object with the same time value. if (arg.type.kind === "date") { - return arg; + const time: IrExpr = { kind: "libCall", fn: "date.getTime", args: [arg], type: F64, loc }; + return { kind: "libCall", fn: "date.newMs", args: [time], type: DATE_T, loc }; } lowerer.noLowering( `new Date of '${lowerer.fmt(arg.type)}' values`, @@ -10528,7 +10529,7 @@ export function lowerNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr { expr, `Map values of type '${lowerer.checker.typeToString(targs[1])}' ` + `(Map values must be number, string, boolean, records, class instances, functions, ` + - `typed arrays, arrays, promises, Maps, Sets, or supported unions and checked values)`, + `typed arrays, arrays, promises, Dates, Maps, Sets, or supported unions and checked values)`, ); } lowerer.badType(expr, tsType); diff --git a/packages/compiler/src/frontend/lowering/lower-generators.ts b/packages/compiler/src/frontend/lowering/lower-generators.ts index 80463fa6c..f837cc0af 100644 --- a/packages/compiler/src/frontend/lowering/lower-generators.ts +++ b/packages/compiler/src/frontend/lowering/lower-generators.ts @@ -333,7 +333,7 @@ export function stepDrivable(genT: GenType): boolean { /** Whether a yield channel's values have an OUT-slot form `value` reads. */ function hasOutSlot(yt: IrType): boolean { - return yt.kind === "f64" || yt.kind === "date" || yt.kind === "bool" || isRefCounted(yt); + return yt.kind === "f64" || yt.kind === "bool" || isRefCounted(yt); } /** `step` and `value` resumes of a generator-typed borrowed read. */ diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index ea5ef72ac..750057bf9 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -6687,7 +6687,7 @@ function lowerDynSwitch( loc: locOf(expression), }; } - if (test.type.kind === "date" || !lowerer.dynConvertible(test.type)) { + if (!lowerer.dynConvertible(test.type)) { lowerer.unsupported( "SC1100", expression, diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index 35514d992..baafd230a 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -3204,10 +3204,9 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { } const collectionView = mapCollectionView(widened, ctx); if (collectionView) return collectionView; - // Date: a TimeClip'd epoch-millisecond scalar in the static runtime. - // This supports stored/passed values and the read-only getter slice; - // identity and mutation stay fenced because the scalar deliberately - // carries no object identity. + // Date: a refcounted object holding its TimeClip'd time value (ScrDate), + // shared with the checked-dynamic Date handle, so it keeps its identity + // through every slot and through `unknown`. Setters stay fenced. if (isStdlibInterface("Date")) return DATE_T; // RegExp: a reference to the lib RegExp interface — provenance, not the // name (a user's own `interface RegExp` maps as a record). Regex values @@ -5025,7 +5024,6 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { (a) => a.kind === "void" || a.kind === "union" || - a.kind === "date" || a.kind === "dyn" || // A generator arm only comes from an Iterable (an optional // one keeps its unit arms beside the iterable union's). @@ -6558,7 +6556,7 @@ export function genResultRecord( for (const a of unions.get(t.unionId)?.arms ?? []) byKey.set(typeKey(a), a); return true; } - if (t.kind === "regex" || t.kind === "date" || t.kind === "generator") { + if (t.kind === "regex" || t.kind === "generator") { return false; // no legal union arm exists for these kinds } byKey.set(typeKey(t), t); @@ -6647,15 +6645,15 @@ export function withUnitArm( if (t.kind === "union") { const def = unions.get(t.unionId); if (!def) return null; - if (def.arms.some((a) => a.kind === "date")) return null; if (def.arms.some((a) => a.kind === kind)) return t; const arms = [...def.arms, unit]; + // Date and Promise arms beside other data arms have no union home. + if (!unionContainerArmsOk(arms)) return null; arms.sort((a, b) => (typeKey(a) < typeKey(b) ? -1 : 1)); return { kind: "union", unionId: unions.transform(def, arms) }; } if ( t.kind === "void" || - t.kind === "date" || t.kind === "dyn" || // A bare unit field type cannot occur (units live only inside unions), // but guard against constructing a single-arm union from one. @@ -8245,7 +8243,6 @@ function armHasUnionHome(arm: IrType): boolean { case "void": case "union": case "regex": - case "date": case "generator": case "dyn": return false; diff --git a/packages/compiler/src/ir/ir.test.ts b/packages/compiler/src/ir/ir.test.ts index d62c9c83c..30fe6f50e 100644 --- a/packages/compiler/src/ir/ir.test.ts +++ b/packages/compiler/src/ir/ir.test.ts @@ -57,7 +57,8 @@ describe("IR kind sets", () => { test("distinguishes pointer values from object-like scalars", () => { expect(POINTER_KINDS.has("record")).toBe(true); - expect(POINTER_KINDS.has("date")).toBe(false); + expect(POINTER_KINDS.has("date")).toBe(true); + expect(POINTER_KINDS.has("procStream")).toBe(false); }); test("distinguishes typed-rest closure ABIs", () => { diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 12400fee9..5afe2b09b 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -440,6 +440,7 @@ const POINTER_KIND_LIST = [ "url", "searchParams", "symbol", + "date", ...POINTER_HANDLE_KINDS, "func", "object", @@ -463,7 +464,7 @@ export const POINTER_KINDS = irKindSet(POINTER_KIND_LIST); export const RUNTIME_RC_STEMS: Record = { f64: "", bigint: "scr_bigint", - date: "", + date: "scr_date", string: "scr_str", bool: "", array: "scr_arr", @@ -645,6 +646,7 @@ export function isSupportedArrayElem(t: IrType): boolean { case "func": case "promise": case "regex": + case "date": case "child": case "netServer": // fs.Stats snapshots (`await Promise.all(paths.map(stat))`): immutable @@ -689,7 +691,8 @@ export function isIdentityCollectionKey(t: IrType): boolean { // functions intern one immortal closure, inner closures allocate once // per evaluation), so pointer identity is JS function identity. // Closures are cycle-headered and traced like records. - t.kind === "func" + t.kind === "func" || + t.kind === "date" ); } @@ -797,6 +800,9 @@ export function isSupportedMapValue(t: IrType): boolean { // the pre-settle closures. case "secureCtx": return true; + // A Date object: immutable, holds no references. + case "date": + return true; // A promise (Map> — the SNI callback's // in-flight dedupe map): refcounted, with trace/cycle support already // in place (scr_promise_trace_v — the race machinery's refcounting). @@ -849,8 +855,9 @@ export function funcOf(params: IrType[], ret: IrType): IrType { * promises nullable-only; declared maybe-promise unions are admitted by * maybePromiseUnionArmsOk. */ export function unionContainerArmsOk(arms: IrType[]): boolean { - return arms.every( - (a, i) => a.kind !== "promise" || arms.every((b, j) => j === i || isUnitType(b)), + return ( + arms.every((a, i) => a.kind !== "promise" || arms.every((b, j) => j === i || isUnitType(b))) && + dateUnionArmsOk(arms) ); } @@ -861,7 +868,13 @@ export function unionContainerArmsOk(arms: IrType[]): boolean { * non-thenable path). Data arms must not be thenable; the type mapper * checks that against the checker types before minting such a union. */ export function maybePromiseUnionArmsOk(arms: IrType[]): boolean { - return arms.filter((a) => a.kind === "promise").length <= 1; + return arms.filter((a) => a.kind === "promise").length <= 1 && dateUnionArmsOk(arms); +} + +/** A Date arm has a union home beside undefined and null arms only: no + * narrowing contract tells it from other data arms yet. */ +function dateUnionArmsOk(arms: IrType[]): boolean { + return arms.every((a, i) => a.kind !== "date" || arms.every((b, j) => j === i || isUnitType(b))); } /** Canonical, injective text form of an IrType — the building block of @@ -3975,6 +3988,9 @@ function isJsonSafeAt( case "string": case "bool": return true; + // JSON.stringify writes a Date's toJSON(); JSON text never parses to one. + case "date": + return stringify; case "array": return isJsonSafeAt( t.elem, @@ -4073,7 +4089,6 @@ function isJsonSafeAt( // Maps are not JSON (JSON.stringify(new Map()) is "{}" in Node — an // empty-object husk nobody wants; stringify/dynCheck reject instead). case "map": - case "date": // URLSearchParams stringifies as the same "{}" husk — rejected; use // sp.toString() instead. case "searchParams": diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index 63aa8ff5c..230d2385d 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -55,6 +55,7 @@ import { typeEquals, typeKey, maybePromiseUnionArmsOk, + unionContainerArmsOk, VOID, } from "./ir.js"; import { DYN_CLASS_PROPERTIES, callSiteReturnType, classValueConstructor } from "./ir.js"; @@ -801,7 +802,6 @@ export function validateModule(mod: IrModule): IrValidationError[] { arm.kind === "void" || arm.kind === "union" || arm.kind === "dyn" || - arm.kind === "date" || (arm.kind === "generator" && iterableArmsElem(u.arms) === null) ) { errors.push({ message: `union ${u.id}: arm ${i} is ${arm.kind}`, loc: noLoc }); @@ -812,6 +812,9 @@ export function validateModule(mod: IrModule): IrValidationError[] { loc: noLoc, }); } + if (arm.kind === "date" && !unionContainerArmsOk(u.arms)) { + errors.push({ message: `union ${u.id}: date arm ${i} beside non-unit arms`, loc: noLoc }); + } if (arm.kind === "record" && !recordsById.has(arm.shapeId)) { errors.push({ message: `union ${u.id}: arm ${i} references undeclared shape "${arm.shapeId}"`, @@ -1505,6 +1508,7 @@ function validateFunction( e.left.type.kind === "map" || e.left.type.kind === "set" || e.left.type.kind === "regex" || + e.left.type.kind === "date" || e.left.type.kind === "url" || e.left.type.kind === "object" || e.left.type.kind === "record" || @@ -2370,9 +2374,13 @@ function validateFunction( (e.method === "sortPrimitive" || e.method === "toSortedPrimitive") && elem.kind !== "string" && elem.kind !== "f64" && - elem.kind !== "bool" + elem.kind !== "bool" && + elem.kind !== "date" ) { - err(`arrIntrinsic ${e.method} requires primitive elements, got ${elem.kind}`, e.loc); + err( + `arrIntrinsic ${e.method} requires primitive or Date elements, got ${elem.kind}`, + e.loc, + ); } if (e.method === "sortValues" || e.method === "sortDense") { const arms = elem.kind === "union" ? unions.get(elem.unionId)?.arms : undefined; @@ -2454,13 +2462,18 @@ function validateFunction( elem.kind !== "f64" && elem.kind !== "string" && elem.kind !== "bool" && + elem.kind !== "date" && !( elem.kind === "union" && (unions .get(elem.unionId) ?.arms.every( (a) => - a.kind === "f64" || a.kind === "string" || a.kind === "bool" || isUnitType(a), + a.kind === "f64" || + a.kind === "string" || + a.kind === "bool" || + a.kind === "date" || + isUnitType(a), ) ?? false) ) diff --git a/packages/compiler/test/collection-keys.test.ts b/packages/compiler/test/collection-keys.test.ts index d73655ef1..ec049f23c 100644 --- a/packages/compiler/test/collection-keys.test.ts +++ b/packages/compiler/test/collection-keys.test.ts @@ -90,7 +90,7 @@ test("mixed-value unions select value equality while unsupported arms remain fen expect(mapKeyKindNum(union, [reference, scalar])).toBe(7); expect(isSupportedSetElem(union, [reference, scalar])).toBe(true); } - for (const unsupported of [{ kind: "date" } as IrType, mapOf(STRING, F64), setOf(STRING)]) { + for (const unsupported of [{ kind: "regex" } as IrType, mapOf(STRING, F64), setOf(STRING)]) { expect(isSupportedMapKey(unsupported)).toBe(false); } }); @@ -171,7 +171,7 @@ test.each(["map", "set"] as const)("validator checks the arms behind a %s key un expect(validateModule(mod)).toEqual([]); mod.unions![0]!.arms = [reference, F64]; expect(validateModule(mod)).toEqual([]); - mod.unions![0]!.arms = [reference, { kind: "date" }]; + mod.unions![0]!.arms = [reference, { kind: "regex" }]; expect( validateModule(mod).some((error) => error.message.includes( diff --git a/packages/compiler/test/ir.test.ts b/packages/compiler/test/ir.test.ts index 7f1a1ef78..7f5e9f3c9 100644 --- a/packages/compiler/test/ir.test.ts +++ b/packages/compiler/test/ir.test.ts @@ -8,10 +8,15 @@ test("hand-built fib module validates", () => { expect(validateModule(fibModule)).toEqual([]); }); -test("validator rejects Date union arms unsupported by the backends", () => { +test("validator rejects Date union arms beside other data arms", () => { const bad = structuredClone(fibModule); - bad.unions = [{ id: "u-date", arms: [{ kind: "date" }, { kind: "undefinedT" }] }]; - expect(validateModule(bad).map((e) => e.message)).toContain("union u-date: arm 0 is date"); + bad.unions = [{ id: "u-date", arms: [{ kind: "date" }, { kind: "string" }] }]; + expect(validateModule(bad).map((e) => e.message)).toContain( + "union u-date: date arm 0 beside non-unit arms", + ); + const nullable = structuredClone(fibModule); + nullable.unions = [{ id: "u-date", arms: [{ kind: "date" }, { kind: "undefinedT" }] }]; + expect(validateModule(nullable)).toEqual([]); }); test("fib module JSON round-trips", () => { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5965-date-objects.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5965-date-objects.ts.json new file mode 100644 index 000000000..c2497d502 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5965-date-objects.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5965-date-objects.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/date-union-fences.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/date-union-fences.ts.json deleted file mode 100644 index cf6ad7398..000000000 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/date-union-fences.ts.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "order": [ - "/tests/diagnostics/date-union-fences.ts" - ], - "diags": [] -} diff --git a/packages/runtime/src/scr_array.c b/packages/runtime/src/scr_array.c index f3e216a36..a3b95146c 100644 --- a/packages/runtime/src/scr_array.c +++ b/packages/runtime/src/scr_array.c @@ -3058,10 +3058,12 @@ typedef struct { typedef struct { ScrElemKind kind; const ScrNumberSortKey *numbers; + ScrStr *const *texts; /* reference elements: each element's String() text */ } ScrPrimitiveOrder; static int scr_arr_primitive_compare(uint64_t left, uint64_t right, const ScrPrimitiveOrder *order) { + if (order->texts) return scr_str_cmp_u16(order->texts[left], order->texts[right]); if (order->kind == SCR_ELEM_STR) return scr_str_cmp_u16(scr_slot_to_ptr(left), scr_slot_to_ptr(right)); if (order->kind == SCR_ELEM_BOOL) return (left > right) - (left < right); @@ -3149,10 +3151,28 @@ static void scr_arr_primitive_order(uint64_t *values, size_t count, if (runs != local_runs) free(runs); } +static ScrArr *scr_arr_sort_default(ScrArr *a, bool copy, ScrStr *(*text)(void *)); + +static ScrArr *scr_arr_new_like(const ScrArr *a, size_t cap) { + return a->elem == SCR_ELEM_REF ? scr_arr_new_ops(scr_arr_ops(a), cap) : scr_arr_new(a->elem, cap); +} + ScrArr *scr_arr_sort_primitive(ScrArr *a, bool copy) { if (a->elem != SCR_ELEM_STR && a->elem != SCR_ELEM_F64 && a->elem != SCR_ELEM_BOOL) scr_trap("scriptc: invalid primitive array ordering\n"); - if (a->len == 0) return copy ? scr_arr_new(a->elem, 0) : scr_arr_retain(a); + return scr_arr_sort_default(a, copy, NULL); +} + +/* The default ordering of an array of reference elements that cannot call + * user code while converting to strings (Dates): `text` answers each + * element's String() text (+1), computed once per element. */ +ScrArr *scr_arr_sort_by_text(ScrArr *a, bool copy, ScrStr *(*text)(void *)) { + if (a->elem != SCR_ELEM_REF) scr_trap("scriptc: invalid text array ordering\n"); + return scr_arr_sort_default(a, copy, text); +} + +static ScrArr *scr_arr_sort_default(ScrArr *a, bool copy, ScrStr *(*text)(void *)) { + if (a->len == 0) return copy ? scr_arr_new_like(a, 0) : scr_arr_retain(a); size_t dense = a->len < a->cap ? a->len : a->cap; size_t count = 0, undefined_count = 0; const uint8_t *states = scr_arr_scan_states(a); @@ -3184,14 +3204,35 @@ ScrArr *scr_arr_sort_primitive(ScrArr *a, bool copy) { values[i] = i; } } - ScrPrimitiveOrder order = {a->elem, numbers}; + ScrStr **texts = NULL; + uint64_t *originals = NULL; + if (text && count > 1) { + if (count > SIZE_MAX / sizeof(*texts)) scr_arr_oom(); + texts = malloc(count * sizeof(*texts)); + originals = malloc(count * sizeof(*originals)); + if (!texts || !originals) scr_arr_oom(); + for (size_t i = 0; i < count; i++) { + originals[i] = values[i]; + texts[i] = text(scr_slot_to_ptr(values[i])); + values[i] = i; + } + } + ScrPrimitiveOrder order = {a->elem, numbers, texts}; scr_arr_primitive_order(values, count, &order); if (numbers) { for (size_t i = 0; i < count; i++) values[i] = numbers[values[i]].value; free(numbers); } + if (texts) { + for (size_t i = 0; i < count; i++) { + values[i] = originals[values[i]]; + scr_str_release(texts[i]); + } + free(texts); + free(originals); + } size_t present = copy ? a->len : count + undefined_count; - ScrArr *out = copy ? scr_arr_new(a->elem, present) : a; + ScrArr *out = copy ? scr_arr_new_like(a, present) : a; if (!copy && !states) { /* Values only, in place. */ if (count) scr_arr_put_slots(a, 0, count, values); diff --git a/packages/runtime/src/scr_array.h b/packages/runtime/src/scr_array.h index e13acb233..2aa1f4c84 100644 --- a/packages/runtime/src/scr_array.h +++ b/packages/runtime/src/scr_array.h @@ -576,6 +576,10 @@ double scr_arr_unshift_spread(ScrArr *a, const ScrArr *src); ScrArr *scr_arr_reverse(ScrArr *a); /* Default primitive String ordering; stable, borrowed receiver, owned result. */ ScrArr *scr_arr_sort_primitive(ScrArr *a, bool copy); +/* Default ordering of reference elements by their String() text, which + * `text` answers (+1) for each element once (Date arrays). Like + * scr_arr_sort_primitive otherwise. */ +ScrArr *scr_arr_sort_by_text(ScrArr *a, bool copy, ScrStr *(*text)(void *)); /* Stable comparator sort of the first `count` slots of a private, compacted * snapshot (every slot a value; f64 or reference elements). `f` is a borrowed * closure of `arity` (0..2) element parameters returning a number. Elements diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 28c2accce..c86d2a71d 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -767,6 +767,8 @@ ScrDyn *scr_dyn_native_date_new_fields(const ScrDyn *arguments); ScrDyn *scr_dyn_native_date_new_value(const ScrDyn *arguments); ScrStr *scr_dyn_object_tag_of(const ScrDyn *d); ScrDyn *scr_dyn_native_date_from_ms(double ms); +typedef struct ScrDate ScrDate; +ScrDyn *scr_dyn_native_date(ScrDate *date); double scr_dyn_native_date_value(const ScrDyn *value); ScrDyn *scr_dyn_native_regex(ScrRegex *value); bool scr_dyn_native_regex_is(const ScrDyn *value); @@ -1406,6 +1408,9 @@ ScrClassObj *scr_dyn_class_check(const ScrDyn *value, const char *type_key, cons * prints as "Uncaught TypeError: ..." and exits 1). This is * scriptc-specific behavior — JS `as` never checks (SEMANTICS.md). */ void scr_dyn_check_fail(const ScrDynPath *path, const char *want, const ScrDyn *got); +/* The Date object a checked-dynamic Date handle holds, +1, or NULL with + * the check failure pending. */ +ScrDate *scr_dyn_native_date_check(const ScrDyn *value, const ScrDynPath *path); /* Matchers and builders of recursive shapes (records and arrays whose type * reaches itself) track the dyn values they visit, so cyclic values match diff --git a/packages/runtime/src/scr_json.h b/packages/runtime/src/scr_json.h index e3077703e..de4510264 100644 --- a/packages/runtime/src/scr_json.h +++ b/packages/runtime/src/scr_json.h @@ -162,6 +162,10 @@ void scr_jb_put_number(ScrJsonBuf *b, double v); * as \u00XX, lone surrogates as \udXXX, everything else (UTF-8 included) * verbatim — exactly the JS JSON.stringify escape set. */ void scr_jb_put_json_str(ScrJsonBuf *b, const ScrStr *s); +/* JSON.stringify of a Date object: toJSON()'s ISO text, null if invalid + * (scr_lib.c). */ +typedef struct ScrDate ScrDate; +void scr_jb_put_date(ScrJsonBuf *b, const ScrDate *date); /* JSON-serialize a dyn value into the buffer — the sc_jw_* walker for the * dyn leaves the compiler cannot type: undefined prints null (an array diff --git a/packages/runtime/src/scr_lib.c b/packages/runtime/src/scr_lib.c index c17f44f9d..18d9c39e5 100644 --- a/packages/runtime/src/scr_lib.c +++ b/packages/runtime/src/scr_lib.c @@ -12549,6 +12549,39 @@ static ScrStr *scr_date_text(double ms, int mode) { } ScrStr *scr_date_to_string(double ms) { return scr_date_text(ms, SCR_DATE_TEXT_FULL); } + +/* JSON.stringify of a Date object: its toJSON(), the quoted + * toISOString() text, or null for an invalid date. */ +void scr_jb_put_date(ScrJsonBuf *b, const ScrDate *date) { + if (isnan(date->ms)) { + scr_jb_puts(b, "null"); + return; + } + ScrStr *iso = scr_date_to_iso(date->ms); + scr_jb_put_json_str(b, iso); + scr_str_release(iso); +} + +/* String(date) of a Date object: Date arrays' default ordering key. +1. */ +ScrStr *scr_date_object_to_string(void *date) { + return scr_date_text(((ScrDate *)date)->ms, SCR_DATE_TEXT_FULL); +} + +/* Array.prototype.join over Date objects: each element's String() text, + * holes empty. Both borrowed; +1. */ +ScrStr *scr_date_arr_join(ScrArr *dates, const ScrStr *sep) { + ScrJsonBuf b; + scr_jb_init(&b); + double len = scr_arr_len(dates); + for (double i = 0; i < len; i++) { + if (i > 0) scr_jb_put_str(&b, sep); + if (scr_arr_state(dates, i) != 1) continue; + ScrStr *text = scr_date_object_to_string(scr_arr_peek_ref(dates, i)); + scr_jb_put_str(&b, text); + scr_str_release(text); + } + return scr_jb_finish(&b); +} ScrStr *scr_date_to_date_string(double ms) { return scr_date_text(ms, SCR_DATE_TEXT_DATE); } ScrStr *scr_date_to_time_string(double ms) { return scr_date_text(ms, SCR_DATE_TEXT_TIME); } @@ -13009,11 +13042,29 @@ ScrArr *scr_set_to_arr_ref(const ScrMap *s, const ScrArrOps *ops) { return out; } -/* JS Date storage owns an identity; the typed read-only Date ABI continues - * to use milliseconds and explicitly extracts them for native getters. */ -typedef struct { size_t rc; double milliseconds; } ScrNativeDate; -static void *scr_native_date_retain(void *ptr) { ((ScrNativeDate *)ptr)->rc++; return ptr; } -static void scr_native_date_release(void *ptr) { if (--((ScrNativeDate *)ptr)->rc == 0) free(ptr); } +/* A Date object: one time value, and its identity. Compiled code and + * checked-dynamic Date handles share these objects, so a Date keeps its + * identity across `unknown` conversions. */ +ScrDate *scr_date_new(double ms) { + ScrDate *date = malloc(sizeof *date); + if (!date) scr_trap("scriptc: out of memory\n"); + date->rc = 1; + date->ms = ms; + return date; +} + +ScrDate *scr_date_retain(ScrDate *date) { + if (date->rc != SIZE_MAX) date->rc++; + return date; +} + +void scr_date_release(ScrDate *date) { + if (!date || date->rc == SIZE_MAX) return; + if (--date->rc == 0) free(date); +} + +void *scr_date_retain_v(void *p) { return scr_date_retain(p); } +void scr_date_release_v(void *p) { scr_date_release(p); } bool scr_dyn_native_date_is(const ScrDyn *value) { return value && value->kind == SCR_DYN_HANDLE && value->v.handle.tag == SCR_DYNH_DATE; @@ -13124,13 +13175,25 @@ static ScrStr *scr_date_class_name(const ScrDyn *value) { } double scr_dyn_native_date_value(const ScrDyn *value) { - if (scr_dyn_native_date_is(value)) return ((ScrNativeDate *)value->v.handle.ptr)->milliseconds; + if (scr_dyn_native_date_is(value)) return ((ScrDate *)value->v.handle.ptr)->ms; double time; if (scr_dyn_date_class_time(value, &time)) return time; scr_dyn_check_fail(NULL, "Date", value); return NAN; } +/* The Date object a checked-dynamic Date value holds, +1: the Date itself, + * or a new Date with the time value of an instance of a class declared + * `extends Date`. NULL with the check failure pending for any other + * value. */ +ScrDate *scr_dyn_native_date_check(const ScrDyn *value, const ScrDynPath *path) { + if (scr_dyn_native_date_is(value)) return scr_date_retain(value->v.handle.ptr); + double time; + if (scr_dyn_date_class_time(value, &time)) return scr_date_new(time); + scr_dyn_check_fail(path, "Date", value); + return NULL; +} + /* Method names without another use in this file, in named arrays (see * scr_tz_posix_ids). */ static const char scr_date_to_date_string_name[] = "toDateString"; @@ -13139,7 +13202,7 @@ static const char scr_date_to_time_string_name[] = "toTimeString"; static ScrDyn *scr_native_date_invoke(void *ptr, ScrDyn *self, const char *method, ScrDyn *const *args, size_t argc, const char *what) { (void)self; (void)what; - double ms = ((ScrNativeDate *)ptr)->milliseconds; + double ms = ((ScrDate *)ptr)->ms; if (!strcmp(method, "getTime") || !strcmp(method, "valueOf")) return scr_dyn_new_num(ms); if (!strcmp(method, "toString") || !strcmp(method, scr_date_to_date_string_name) || !strcmp(method, scr_date_to_time_string_name) || !strcmp(method, "toUTCString")) { @@ -13162,8 +13225,8 @@ static ScrDyn *scr_native_date_invoke(void *ptr, ScrDyn *self, const char *metho if (!strcmp(method, "setTime")) { double time = NAN; if (argc > 0 && !scr_dyn_number_coerce_js(args[0], &time)) return NULL; - ((ScrNativeDate *)ptr)->milliseconds = scr_date_new_ms(time); - return scr_dyn_new_num(((ScrNativeDate *)ptr)->milliseconds); + ((ScrDate *)ptr)->ms = scr_date_new_ms(time); + return scr_dyn_new_num(((ScrDate *)ptr)->ms); } #define DATE_GET(name, fn) if (!strcmp(method, name)) return scr_dyn_new_num(fn(ms)) DATE_GET("getFullYear", scr_date_get_full_year_local); @@ -13209,7 +13272,7 @@ double scr_dyn_date_arguments_time(const ScrDyn *arguments) { ScrDyn *date = arguments->v.arr.len > 1 ? scr_dyn_native_date_new_fields(arguments) : scr_dyn_native_date_new_value(arguments); if (!date) return NAN; - double time = ((ScrNativeDate *)date->v.handle.ptr)->milliseconds; + double time = ((ScrDate *)date->v.handle.ptr)->ms; scr_dyn_release(date); return time; } @@ -13274,10 +13337,10 @@ static ScrDyn *scr_native_date_method_call(ScrClosure *closure, ScrDyn *const *a result = isfinite(time) ? scr_date_class_call(SCR_DATE_CLASS_TO_ISO_STRING, self, NULL) : scr_dyn_new_null(); } else { - ScrNativeDate date = { 1, time }; + ScrDate date = { 1, time }; result = scr_native_date_invoke(&date, self, method, args, argc, method); - if (result && date.milliseconds != time && !(isnan(date.milliseconds) && isnan(time))) { - ScrDyn *stored = scr_dyn_new_num(date.milliseconds); + if (result && date.ms != time && !(isnan(date.ms) && isnan(time))) { + ScrDyn *stored = scr_dyn_new_num(date.ms); ScrDyn *set = scr_date_class_call(SCR_DATE_CLASS_SET_TIME, self, stored); scr_dyn_release(stored); if (!set) { @@ -13330,19 +13393,22 @@ static ScrDyn *scr_native_date_get(void *ptr, const char *key, size_t key_len) { return NULL; } -/* A Date handle holding the time value `ms` (already TimeClip'd). */ -ScrDyn *scr_dyn_native_date_from_ms(double ms) { +/* A checked-dynamic handle for the Date object `date` (borrowed; the + * handle retains it), so the value stays the same object. */ +ScrDyn *scr_dyn_native_date(ScrDate *date) { static const ScrDynHandleOps ops = { - "Date", &scr_native_date_retain, &scr_native_date_release, &scr_native_date_invoke, + "Date", &scr_date_retain_v, &scr_date_release_v, &scr_native_date_invoke, &scr_native_date_get, NULL, NULL, NULL, }; scr_dyn_handle_install(SCR_DYNH_DATE, &ops); - ScrNativeDate *date = malloc(sizeof *date); - if (!date) scr_trap("scriptc: out of memory\n"); - date->rc = 1; - date->milliseconds = ms; - ScrDyn *result = scr_dyn_new_handle(date, SCR_DYNH_DATE); - scr_native_date_release(date); + return scr_dyn_new_handle(date, SCR_DYNH_DATE); +} + +/* A Date handle holding the time value `ms` (already TimeClip'd). */ +ScrDyn *scr_dyn_native_date_from_ms(double ms) { + ScrDate *date = scr_date_new(ms); + ScrDyn *result = scr_dyn_native_date(date); + scr_date_release(date); return result; } diff --git a/packages/runtime/src/scr_number.h b/packages/runtime/src/scr_number.h index 78d19614b..a71b0ccfd 100644 --- a/packages/runtime/src/scr_number.h +++ b/packages/runtime/src/scr_number.h @@ -111,6 +111,18 @@ double scr_date_now(void); /* integer ms since epoch, like Node */ extern SCR_TL const double *scr_date_mock_now; double scr_date_new_ms(double ms); /* TimeClip (NaN when invalid) */ double scr_date_get_time(double ms); +/* A Date object: the TimeClip'd time value in milliseconds (NaN for an + * invalid date). Dates are immutable; their identity is the pointer. + * Retain/release follow the runtime's RC conventions (SIZE_MAX immortal). */ +typedef struct ScrDate { + size_t rc; + double ms; +} ScrDate; +ScrDate *scr_date_new(double ms); /* +1 */ +ScrDate *scr_date_retain(ScrDate *date); +void scr_date_release(ScrDate *date); +void *scr_date_retain_v(void *p); +void scr_date_release_v(void *p); /* Node's exact ISO 8601 UTC format (expanded ±YYYYYY years outside * 0–9999). THROWS Node's "Invalid time value" RangeError on NaN or * |ms| > 8.64e15 and returns NULL; +1 otherwise. */ @@ -150,6 +162,11 @@ double scr_date_get_timezone_offset(double ms); ScrStr *scr_date_to_string(double ms); ScrStr *scr_date_to_date_string(double ms); ScrStr *scr_date_to_time_string(double ms); +/* String() of a Date object (ScrDate *), and Array.prototype.join over an + * array of Date objects (holes empty). +1. */ +ScrStr *scr_date_object_to_string(void *date); +struct ScrArr; +ScrStr *scr_date_arr_join(struct ScrArr *dates, const ScrStr *sep); /* The long time zone name Node prints after the GMT offset for the host * zone at time value `ms`; `isdst` is tm_isdst at `ms`, or -1 when not * known. Borrowed; valid until the next call on the same thread. */ diff --git a/tests/corpus/5965-date-objects.ts b/tests/corpus/5965-date-objects.ts new file mode 100644 index 000000000..c4bb01fb9 --- /dev/null +++ b/tests/corpus/5965-date-objects.ts @@ -0,0 +1,115 @@ +// Dates are objects with identity: arrays of Dates, `Date | undefined` +// and `Date | null` slots, === by identity, includes/indexOf/lastIndexOf, +// at/find/pop answering undefined, Map and Set keys and values, join, +// String() and default sort through toString(), and Dates in records, +// tuples, class fields, closures, async results and generators. +export {}; +process.env.TZ = "America/Chicago"; + +const base = new Date(Date.UTC(2026, 3, 1, 12)); +const same = base; +const copy = new Date(base); +const equalTime = new Date(base.getTime()); +console.log(base === same, base === copy, copy === equalTime, base !== copy); +console.log(copy.getTime() === base.getTime(), base < new Date(base.getTime() + 1), base >= copy); + +// Arrays. +const dates: Date[] = [base, new Date(Date.UTC(2025, 0, 1)), new Date(Date.UTC(2026, 0, 1))]; +console.log(dates.length, dates.includes(base), dates.includes(copy), dates.indexOf(dates[2]!)); +console.log(dates.indexOf(equalTime), dates.lastIndexOf(base), dates.indexOf(base, 1)); +console.log(dates.join(" | ")); +console.log(String(dates)); +console.log(`${dates}`); +console.log([...dates].sort().map((d) => d.toISOString())); +console.log(dates.toSorted((a, b) => a.getTime() - b.getTime()).map((d) => d.getUTCFullYear())); +const first = dates.at(0); +const missing = dates.at(10); +console.log(first === base, missing, missing === undefined, first?.toISOString(), missing?.toISOString()); +const found = dates.find((d) => d.getUTCFullYear() === 2025); +console.log(found === dates[1], dates.find((d) => d.getTime() < 0), dates.findLast((d) => d > base)); +const stack = [...dates]; +console.log(stack.pop() === dates[2], stack.pop()?.getUTCFullYear(), stack.pop() === base, stack.pop()); +const latest = dates.reduce((m, d) => (d > m ? d : m)); +console.log(latest === base, dates.filter((d) => d < base).length, dates.map((d) => d.getUTCMonth())); +const grid: Date[][] = [[base], [copy, base]]; +console.log(grid[1]!.indexOf(base), grid.flat().length, grid.flat().includes(copy)); +const holes: (Date | undefined)[] = [base, undefined, copy]; +console.log(holes.join(","), holes.indexOf(undefined), holes.filter((d) => d !== undefined).length); + +// Nullable slots. +let maybe: Date | null = null; +console.log(maybe === null, maybe); +maybe = base; +console.log(maybe === base, maybe.getUTCHours()); +function stamp(d?: Date): string { + return d === undefined ? "none" : d.toISOString(); +} +function withDefault(d: Date = new Date(0)): number { + return d.getTime(); +} +console.log(stamp(), stamp(base), withDefault(), withDefault(base) === base.getTime()); + +// Collections. +const notes = new Map([[base, "base"]]); +notes.set(copy, "copy"); +console.log(notes.get(base), notes.get(copy), notes.get(equalTime), notes.size); +const byName = new Map([["start", base]]); +console.log(byName.get("start") === base, byName.get("end")); +const seen = new Set([base, same, copy]); +console.log(seen.size, seen.has(base), seen.has(equalTime)); + +// Records, tuples, classes and closures. +interface Event { + name: string; + at: Date; + ends?: Date; +} +const events: Event[] = [ + { name: "a", at: base }, + { name: "b", at: copy, ends: base }, +]; +console.log(events.map((e) => `${e.name}@${e.at.toISOString()} ends=${e.ends?.toISOString() ?? "-"}`)); +console.log(events[1]!.ends === events[0]!.at, events[1]!.at === events[0]!.at); +const pair: [Date, string] = [base, "x"]; +console.log(pair[0] === base, pair[1]); +class Window { + from: Date; + to: Date | null = null; + later!: Date; + constructor(from: Date) { + this.from = from; + } + span(): number { + return this.to === null ? 0 : this.to.getTime() - this.from.getTime(); + } +} +const w = new Window(base); +w.later = copy; +console.log(w.span(), w.later === copy); +w.to = new Date(base.getTime() + 60000); +console.log(w.span(), w.to === w.from); +let captured: Date | undefined; +const remember = (d: Date): void => { + captured = d; +}; +remember(base); +console.log(captured === base, captured?.getUTCDate()); + +// Async results, generators and unknown. +async function later(d: Date): Promise { + await null; + return new Date(d.getTime() + 1); +} +const next = await later(base); +console.log(next.getTime() - base.getTime(), next === base); +function* ticks(start: Date, n: number): Generator { + for (let i = 0; i < n; i++) yield new Date(start.getTime() + i * 1000); +} +console.log([...ticks(base, 3)].map((d) => d.getUTCSeconds())); +const held: unknown = base; +console.log((held as Date) === base, held instanceof Date, String(held) === String(base)); +const bag: unknown = { at: base, list: dates }; +console.log(JSON.stringify(bag)); +console.log(JSON.stringify(events), JSON.stringify([base, new Date(NaN)])); +const byKey: { [k: string]: Date } = { boot: base }; +console.log(byKey["boot"] === base, byKey["missing"], Object.keys(byKey)); diff --git a/tests/diagnostics/component-fences.ts b/tests/diagnostics/component-fences.ts index 6d88f9af5..0d382bf28 100644 --- a/tests/diagnostics/component-fences.ts +++ b/tests/diagnostics/component-fences.ts @@ -5,9 +5,9 @@ const byFlag = new Map(); console.log(byFlag); -// Map values have no slot for Dates. -const stamps = new Map(); -console.log(stamps); +// Map values have no slot for URLSearchParams. +const queries = new Map(); +console.log(queries); // Boolean Set elements remain outside the supported element domain. const flags = new Set(); diff --git a/tests/diagnostics/date-union-fences.ts b/tests/diagnostics/date-union-fences.ts deleted file mode 100644 index 478df74d6..000000000 --- a/tests/diagnostics/date-union-fences.ts +++ /dev/null @@ -1,22 +0,0 @@ -// Compiler-synthesized `T | undefined` slots must observe the same Date -// union fence as source-written unions. Each construct below used to reach -// IR validation as an SC9001 internal compiler error. - -function defaultDate(value: Date = new Date(0)): number { - return value.getTime(); -} -defaultDate(); - -function* dates(): Generator { - yield new Date(0); -} -for (const value of dates()) console.log(value.getTime()); - -class DeferredDate { - value!: Date; - initialize(): void { - this.value = new Date(0); - } -} -const deferred = new DeferredDate(); -deferred.initialize(); diff --git a/tests/diagnostics/stdlib-fence.ts b/tests/diagnostics/stdlib-fence.ts index 1fdc01675..d28333178 100644 --- a/tests/diagnostics/stdlib-fence.ts +++ b/tests/diagnostics/stdlib-fence.ts @@ -23,7 +23,6 @@ const fmt = new Intl.NumberFormat(); const d = new Date(...([2024, 0] as const)); // spread field arguments stay fenced const mutableDate = new Date(0); mutableDate.setUTCFullYear(2024); // mutation is outside the read-only Date slice -const sameDate = mutableDate === mutableDate; // scalar storage must not fake object identity const wr = new WeakRef({}); const px = new Proxy({ a: 1 }, {}); const buf = new ArrayBuffer(8); @@ -74,13 +73,6 @@ function pick(a: string | number, b: number | boolean): boolean { } const picked = pick(1, 2); -// Date values deliberately have no tagged-union payload yet. Nullable -// Date positions must fence in the frontend instead of reaching a backend -// scalar/ref mismatch. -function optionalDate(value: Date | undefined): number { - return value === undefined ? -1 : value.getTime(); -} -optionalDate(undefined); function throwDate(): void { throw new Date(0); // the exception cell cannot preserve Date's object kind } diff --git a/tests/diagnostics/utility-types.ts b/tests/diagnostics/utility-types.ts index c6d10648b..cad1fcca4 100644 --- a/tests/diagnostics/utility-types.ts +++ b/tests/diagnostics/utility-types.ts @@ -5,9 +5,7 @@ // fields + an overflow map; number keys canonicalize to their JS string // spelling; function/Map/Set/nested-record values ride the same store — // differential corpus); what stays rejected: -// - symbol-keyed index signatures (no lowering for symbol keys); -// - index-signature value types outside the overflow-map slice (Dates and -// other lib API objects, even when the value itself has a representation). +// - symbol-keyed index signatures (no lowering for symbol keys). // (Readonly bindings compile, holding the Date as a checked value. // Dot access/writes to index-signature keys compile now — the overflow // path in dot spelling — and self-referential mapped types intern as @@ -15,9 +13,5 @@ const bySymbol: { [s: symbol]: string } = {}; console.log(bySymbol); -const dateBag: { [k: string]: Date } = {}; -const boot = dateBag["boot"]; -console.log(boot); - const stamped: Readonly = new Date(0); console.log(stamped.getTime()); diff --git a/tests/harness/__snapshots__/class-interface-refinements.ts.txt b/tests/harness/__snapshots__/class-interface-refinements.ts.txt index 043cb3996..9d7d7b87f 100644 --- a/tests/harness/__snapshots__/class-interface-refinements.ts.txt +++ b/tests/harness/__snapshots__/class-interface-refinements.ts.txt @@ -39,7 +39,7 @@ class-interface-refinements.ts:17:13 - error SC2004: uses of 'changed' inherit t hint: the declaration of 'changed' did not compile — fix the diagnostic reported there and these sites clear with it -class-interface-refinements.ts:20:18 - error SC2006: values of type 'Indexed' cannot be compiled: this index signature is outside the supported shape (string or number keys; values limited to numbers, strings, booleans, records, classes, arrays, unions, functions, Maps, Sets, RegExps, Promises, or 'unknown') +class-interface-refinements.ts:20:18 - error SC2006: values of type 'Indexed' cannot be compiled: this index signature is outside the supported shape (string or number keys; values limited to numbers, strings, booleans, records, classes, arrays, unions, functions, Maps, Sets, RegExps, Dates, Promises, or 'unknown') 19 | type Indexed = Box & { [name: string]: number }; 20 | function indexed(view: Indexed, key: string): number { return view[key]!; } diff --git a/tests/harness/__snapshots__/component-fences.ts.txt b/tests/harness/__snapshots__/component-fences.ts.txt index 5ed6ad6b9..45d476089 100644 --- a/tests/harness/__snapshots__/component-fences.ts.txt +++ b/tests/harness/__snapshots__/component-fences.ts.txt @@ -1,27 +1,27 @@ -component-fences.ts:9:7 - error SC2009: values of type 'Map' cannot be compiled: the Map shape is supported, but 'Date' values have no Map slot yet +component-fences.ts:9:7 - error SC2009: values of type 'Map' cannot be compiled: the Map shape is supported, but 'URLSearchParams' values have no Map slot yet - 8 | // Map values have no slot for Dates. - 9 | const stamps = new Map(); - | ^~~~~~ - 10 | console.log(stamps); + 8 | // Map values have no slot for URLSearchParams. + 9 | const queries = new Map(); + | ^~~~~~~ + 10 | console.log(queries); hint: the outer shape is supported — the named component is the blocker; restate it inside the compilable set (number, string, boolean, arrays, Maps, Sets, RegExp, functions, classes, records, unions of those, and 'unknown') -component-fences.ts:9:16 - error SC1090: Map values of type 'Date' (Map values must be number, string, boolean, records, class instances, functions, typed arrays, arrays, promises, Maps, Sets, or supported unions and checked values) is not supported yet +component-fences.ts:9:17 - error SC1090: Map values of type 'URLSearchParams' (Map values must be number, string, boolean, records, class instances, functions, typed arrays, arrays, promises, Dates, Maps, Sets, or supported unions and checked values) is not supported yet - 8 | // Map values have no slot for Dates. - 9 | const stamps = new Map(); - | ^~~~~~~~~~~~~~~~~~~~~~~ - 10 | console.log(stamps); + 8 | // Map values have no slot for URLSearchParams. + 9 | const queries = new Map(); + | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + 10 | console.log(queries); -component-fences.ts:10:13 - error SC2004: uses of 'stamps' inherit the blocker on its declaration +component-fences.ts:10:13 - error SC2004: uses of 'queries' inherit the blocker on its declaration - 9 | const stamps = new Map(); - 10 | console.log(stamps); - | ^~~~~~ + 9 | const queries = new Map(); + 10 | console.log(queries); + | ^~~~~~~ 11 | - hint: the declaration of 'stamps' did not compile — fix the diagnostic reported there and these sites clear with it + hint: the declaration of 'queries' did not compile — fix the diagnostic reported there and these sites clear with it component-fences.ts:35:7 - error SC2009: values of type '() => WeakRef' cannot be compiled: the function shape is supported, but its return type 'WeakRef' does not compile diff --git a/tests/harness/__snapshots__/date-union-fences.ts.txt b/tests/harness/__snapshots__/date-union-fences.ts.txt deleted file mode 100644 index 1b504f23a..000000000 --- a/tests/harness/__snapshots__/date-union-fences.ts.txt +++ /dev/null @@ -1,56 +0,0 @@ -date-union-fences.ts:5:22 - error SC1090: parameter default values on 'Date'-typed parameters are not supported yet - - 4 | - 5 | function defaultDate(value: Date = new Date(0)): number { - | ^~~~~~~~~~~~~~~~~~~~~~~~~ - 6 | return value.getTime(); - -date-union-fences.ts:8:1 - error SC2004: uses of 'defaultDate' inherit the blocker on its declaration - - 7 | } - 8 | defaultDate(); - | ^~~~~~~~~~~ - 9 | - - hint: the declaration of 'defaultDate' did not compile — fix the diagnostic reported there and these sites clear with it - -date-union-fences.ts:10:11 - error SC2009: values of type 'Generator' cannot be compiled: the Generator shape is supported, but this instantiation's type arguments are outside the supported set - - 9 | - 10 | function* dates(): Generator { - | ^~~~~ - 11 | yield new Date(0); - - hint: the outer shape is supported — the named component is the blocker; restate it inside the compilable set (number, string, boolean, arrays, Maps, Sets, RegExp, functions, classes, records, unions of those, and 'unknown') - -date-union-fences.ts:13:21 - error SC2004: uses of 'dates' inherit the blocker on its declaration - - 12 | } - 13 | for (const value of dates()) console.log(value.getTime()); - | ^~~~~ - 14 | - - hint: the declaration of 'dates' did not compile — fix the diagnostic reported there and these sites clear with it - -date-union-fences.ts:16:3 - error SC1090: definite assignment assertions on fields not assigned at the constructor's top level ('value!' defers the first assignment past construction — the field would hold garbage, not undefined, until it runs; assign it in the constructor or include undefined in its type) is not supported yet - - 15 | class DeferredDate { - 16 | value!: Date; - | ^~~~~ - 17 | initialize(): void { - -date-union-fences.ts:21:18 - error SC1090: constructing through a class value whose class has no lowering (the class declaration itself was rejected — see its own diagnostic) is not supported yet - - 20 | } - 21 | const deferred = new DeferredDate(); - | ^~~~~~~~~~~~~~~~~~ - 22 | deferred.initialize(); - -date-union-fences.ts:22:1 - error SC2004: uses of 'deferred' inherit the blocker on its declaration - - 21 | const deferred = new DeferredDate(); - 22 | deferred.initialize(); - | ^~~~~~~~ - 23 | - - hint: the declaration of 'deferred' did not compile — fix the diagnostic reported there and these sites clear with it \ No newline at end of file diff --git a/tests/harness/__snapshots__/generics.ts.txt b/tests/harness/__snapshots__/generics.ts.txt index df6e552db..4dfc8019f 100644 --- a/tests/harness/__snapshots__/generics.ts.txt +++ b/tests/harness/__snapshots__/generics.ts.txt @@ -5,7 +5,7 @@ generics.ts:23:3 - error SC1090: generic nested functions (only top-level generi | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 24 | return x; -generics.ts:30:10 - error SC1090: '.join()' on arrays of this element type (number, string, and boolean arrays join — unions of those with undefined/null arms too, the units printing empty like JS) is not supported yet (instantiating 'joinAll' with ) +generics.ts:30:10 - error SC1090: '.join()' on arrays of this element type (number, string, boolean and Date arrays join — unions of those with undefined/null arms too, the units printing empty like JS) is not supported yet (instantiating 'joinAll' with ) 29 | function joinAll(a: T[]): string { 30 | return a.join(","); diff --git a/tests/harness/__snapshots__/index-signature-values.ts.txt b/tests/harness/__snapshots__/index-signature-values.ts.txt index bbbed7fe2..084d6a259 100644 --- a/tests/harness/__snapshots__/index-signature-values.ts.txt +++ b/tests/harness/__snapshots__/index-signature-values.ts.txt @@ -1,4 +1,4 @@ -index-signature-values.ts:8:7 - error SC2006: values of type '{ [s: symbol]: number; }' cannot be compiled: this index signature is outside the supported shape (string or number keys; values limited to numbers, strings, booleans, records, classes, arrays, unions, functions, Maps, Sets, RegExps, Promises, or 'unknown') +index-signature-values.ts:8:7 - error SC2006: values of type '{ [s: symbol]: number; }' cannot be compiled: this index signature is outside the supported shape (string or number keys; values limited to numbers, strings, booleans, records, classes, arrays, unions, functions, Maps, Sets, RegExps, Dates, Promises, or 'unknown') 7 | // A symbol-keyed signature has no lowering. 8 | const bySymbol: { [s: symbol]: number } = {}; @@ -14,7 +14,7 @@ index-signature-values.ts:13:13 - error SC1090: console.log of '{ [key: string]: | ^~~~~~~ 14 | -index-signature-values.ts:18:7 - error SC2006: values of type '{ [k: string]: string | number; [n: number]: number; }' cannot be compiled: this index signature is outside the supported shape (string or number keys; values limited to numbers, strings, booleans, records, classes, arrays, unions, functions, Maps, Sets, RegExps, Promises, or 'unknown') +index-signature-values.ts:18:7 - error SC2006: values of type '{ [k: string]: string | number; [n: number]: number; }' cannot be compiled: this index signature is outside the supported shape (string or number keys; values limited to numbers, strings, booleans, records, classes, arrays, unions, functions, Maps, Sets, RegExps, Dates, Promises, or 'unknown') 17 | // answer per key). 18 | const dual: { [k: string]: string | number; [n: number]: number } = {}; diff --git a/tests/harness/__snapshots__/stdlib-fence.ts.txt b/tests/harness/__snapshots__/stdlib-fence.ts.txt index af1725323..c183f7cdc 100644 --- a/tests/harness/__snapshots__/stdlib-fence.ts.txt +++ b/tests/harness/__snapshots__/stdlib-fence.ts.txt @@ -64,104 +64,79 @@ stdlib-fence.ts:25:1 - error SC2020: 'Date.prototype.setUTCFullYear with argumen 24 | const mutableDate = new Date(0); 25 | mutableDate.setUTCFullYear(2024); // mutation is outside the read-only Date slice | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - 26 | const sameDate = mutableDate === mutableDate; // scalar storage must not fake object identity + 26 | const wr = new WeakRef({}); hint: getTime(), valueOf(), toISOString(), toUTCString(), toJSON(), toString(), toDateString(), toTimeString(), the local/UTC calendar getters, and getTimezoneOffset() are supported; Date setters and locale formatters have no lowering -stdlib-fence.ts:26:18 - error SC1043: comparing non-number, non-string values are not supported yet +stdlib-fence.ts:26:7 - error SC2020: 'WeakRef<{}>' is part of the standard library types but has no scriptc lowering yet 25 | mutableDate.setUTCFullYear(2024); // mutation is outside the read-only Date slice - 26 | const sameDate = mutableDate === mutableDate; // scalar storage must not fake object identity - | ^~~~~~~~~~~~~~~~~~~~~~~~~~~ - 27 | const wr = new WeakRef({}); - -stdlib-fence.ts:27:7 - error SC2020: 'WeakRef<{}>' is part of the standard library types but has no scriptc lowering yet - - 26 | const sameDate = mutableDate === mutableDate; // scalar storage must not fake object identity - 27 | const wr = new WeakRef({}); + 26 | const wr = new WeakRef({}); | ^~ - 28 | const px = new Proxy({ a: 1 }, {}); + 27 | const px = new Proxy({ a: 1 }, {}); hint: the type checker sees the full standard library, but only the supported surface compiles (https://scriptc.dev/limitations) -stdlib-fence.ts:27:12 - error SC2020: 'new WeakRef' is part of the standard library types but has no scriptc lowering yet +stdlib-fence.ts:26:12 - error SC2020: 'new WeakRef' is part of the standard library types but has no scriptc lowering yet - 26 | const sameDate = mutableDate === mutableDate; // scalar storage must not fake object identity - 27 | const wr = new WeakRef({}); + 25 | mutableDate.setUTCFullYear(2024); // mutation is outside the read-only Date slice + 26 | const wr = new WeakRef({}); | ^~~~~~~~~~~~~~~ - 28 | const px = new Proxy({ a: 1 }, {}); + 27 | const px = new Proxy({ a: 1 }, {}); hint: deref()-after-collect exposes GC timing — genuinely dynamic; hold a strong reference instead -stdlib-fence.ts:42:31 - error SC1090: '.split()' on a 'RegExp | string' separator (pass a string, or a regex literal) is not supported yet +stdlib-fence.ts:41:31 - error SC1090: '.split()' on a 'RegExp | string' separator (pass a string, or a regex literal) is not supported yet - 41 | const norm = "abc".normalize(); - 42 | const limited = "a,b,c".split(Math.random() ? "," : /,/, 2); // union separators stay fenced; limits lower + 40 | const norm = "abc".normalize(); + 41 | const limited = "a,b,c".split(Math.random() ? "," : /,/, 2); // union separators stay fenced; limits lower | ^~~~~~~~~~~~~~~~~~~~~~~~~ - 43 | const fixed = (1.5).toFixed(); // digit-free toFixed lowers now (the static ties-up integer) + 42 | const fixed = (1.5).toFixed(); // digit-free toFixed lowers now (the static ties-up integer) -stdlib-fence.ts:44:19 - error SC2020: 'Number.prototype.toLocaleString without a locale' is part of the standard library types but has no scriptc lowering yet +stdlib-fence.ts:43:19 - error SC2020: 'Number.prototype.toLocaleString without a locale' is part of the standard library types but has no scriptc lowering yet - 43 | const fixed = (1.5).toFixed(); // digit-free toFixed lowers now (the static ties-up integer) - 44 | const localized = (1234.5).toLocaleString(); + 42 | const fixed = (1.5).toFixed(); // digit-free toFixed lowers now (the static ties-up integer) + 43 | const localized = (1234.5).toLocaleString(); | ^~~~~~~~~~~~~~~~~~~~~~~~~ - 45 | // Unlowered call FORMS of lowered members. (push is fully variadic now — + 44 | // Unlowered call FORMS of lowered members. (push is fully variadic now — hint: the default locale is the host environment's, which a compiled binary cannot carry — pass it explicitly: x.toLocaleString("en-US") -stdlib-fence.ts:50:1 - error SC2020: '.forEach with 2 arguments' is part of the standard library types but has no scriptc lowering yet +stdlib-fence.ts:49:1 - error SC2020: '.forEach with 2 arguments' is part of the standard library types but has no scriptc lowering yet - 49 | const joined = nums.join(); // the omitted separator lowers - 50 | nums.forEach((x) => console.log(x), { unused: true }); // thisArg stays fenced + 48 | const joined = nums.join(); // the omitted separator lowers + 49 | nums.forEach((x) => console.log(x), { unused: true }); // thisArg stays fenced | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - 51 | const pos = "abc".includes("b", 1); // the position form lowers now (indexOf's clamp) + 50 | const pos = "abc".includes("b", 1); // the position form lowers now (indexOf's clamp) hint: the thisArg parameter has no lowering — use an arrow function -stdlib-fence.ts:80:23 - error SC2009: values of type 'Date | undefined' cannot be compiled: the union shape is supported, but 'Date' arms have no home in a compiled union yet (no runtime narrowing test exists against sibling arms) +stdlib-fence.ts:77:9 - error SC1090: throwing Date values (catch bindings cannot preserve Date's object kind; throw an Error or primitive instead) are not supported yet - 79 | // scalar/ref mismatch. - 80 | function optionalDate(value: Date | undefined): number { - | ^~~~~ - 81 | return value === undefined ? -1 : value.getTime(); + 76 | function throwDate(): void { + 77 | throw new Date(0); // the exception cell cannot preserve Date's object kind + | ^~~~~~~~~~~ + 78 | } - hint: the outer shape is supported — the named component is the blocker; restate it inside the compilable set (number, string, boolean, arrays, Maps, Sets, RegExp, functions, classes, records, unions of those, and 'unknown') - -stdlib-fence.ts:83:1 - error SC2004: uses of 'optionalDate' inherit the blocker on its declaration +stdlib-fence.ts:83:23 - error SC1090: .throw() with a Date value (the exception channel cannot preserve Date's object kind; throw an Error or primitive instead) is not supported yet 82 | } - 83 | optionalDate(undefined); - | ^~~~~~~~~~~~ - 84 | function throwDate(): void { - - hint: the declaration of 'optionalDate' did not compile — fix the diagnostic reported there and these sites clear with it - -stdlib-fence.ts:85:9 - error SC1090: throwing Date values (catch bindings cannot preserve Date's object kind; throw an Error or primitive instead) are not supported yet - - 84 | function throwDate(): void { - 85 | throw new Date(0); // the exception cell cannot preserve Date's object kind - | ^~~~~~~~~~~ - 86 | } - -stdlib-fence.ts:91:23 - error SC1090: .throw() with a Date value (the exception channel cannot preserve Date's object kind; throw an Error or primitive instead) is not supported yet - - 90 | } - 91 | dateGenerator().throw(new Date(0)); + 83 | dateGenerator().throw(new Date(0)); | ^~~~~~~~~~~ - 92 | // Related classes may store one object under different union tags. A + 84 | // Related classes may store one object under different union tags. A -stdlib-fence.ts:102:10 - error SC1090: comparisons of related class values beside structural union members (narrow first: check a discriminant field, or compare with '!== undefined'/'!== null' for unit arms) are not supported yet +stdlib-fence.ts:94:10 - error SC1090: comparisons of related class values beside structural union members (narrow first: check a discriminant field, or compare with '!== undefined'/'!== null' for unit arms) are not supported yet - 101 | function relatedStrict(a: FenceValue, b: FenceValue): boolean { - 102 | return a === b; - | ^~~~~~~ - 103 | } + 93 | function relatedStrict(a: FenceValue, b: FenceValue): boolean { + 94 | return a === b; + | ^~~~~~~ + 95 | } -stdlib-fence.ts:105:10 - error SC2020: 'Object.is over related class values beside structural union members' is part of the standard library types but has no scriptc lowering yet +stdlib-fence.ts:97:10 - error SC2020: 'Object.is over related class values beside structural union members' is part of the standard library types but has no scriptc lowering yet - 104 | function relatedSameValue(a: FenceValue, b: FenceSub): boolean { - 105 | return Object.is(a, b); - | ^~~~~~~~~~~~~~~ - 106 | } + 96 | function relatedSameValue(a: FenceValue, b: FenceSub): boolean { + 97 | return Object.is(a, b); + | ^~~~~~~~~~~~~~~ + 98 | } hint: narrow first: check a discriminant field, or compare with '!== undefined'/'!== null' for unit arms \ No newline at end of file diff --git a/tests/harness/__snapshots__/tuples.ts.txt b/tests/harness/__snapshots__/tuples.ts.txt index 6b43432fe..a6f58710e 100644 --- a/tests/harness/__snapshots__/tuples.ts.txt +++ b/tests/harness/__snapshots__/tuples.ts.txt @@ -40,14 +40,14 @@ tuples.ts:31:13 - error SC1090: '.length' of a computed tuple expression (the ar | ^~~~~~~~~~~ 32 | -tuples.ts:35:13 - error SC1090: '.join()' on arrays of this element type (number, string, and boolean arrays join — unions of those with undefined/null arms too, the units printing empty like JS) is not supported yet +tuples.ts:35:13 - error SC1090: '.join()' on arrays of this element type (number, string, boolean and Date arrays join — unions of those with undefined/null arms too, the units printing empty like JS) is not supported yet 34 | const recs: { id: number }[] = [{ id: 1 }]; 35 | console.log(recs.join(",")); | ^~~~~~~~~~~~~~ 36 | const tuples: [string, string][] = [["k", "v"]]; -tuples.ts:37:13 - error SC1090: '.join()' on arrays of this element type (number, string, and boolean arrays join — unions of those with undefined/null arms too, the units printing empty like JS) is not supported yet +tuples.ts:37:13 - error SC1090: '.join()' on arrays of this element type (number, string, boolean and Date arrays join — unions of those with undefined/null arms too, the units printing empty like JS) is not supported yet 36 | const tuples: [string, string][] = [["k", "v"]]; 37 | console.log(tuples.join(";")); diff --git a/tests/harness/__snapshots__/utility-types.ts.txt b/tests/harness/__snapshots__/utility-types.ts.txt index 02595967b..0c6c98489 100644 --- a/tests/harness/__snapshots__/utility-types.ts.txt +++ b/tests/harness/__snapshots__/utility-types.ts.txt @@ -1,17 +1,8 @@ -utility-types.ts:15:7 - error SC2006: values of type '{ [s: symbol]: string; }' cannot be compiled: this index signature is outside the supported shape (string or number keys; values limited to numbers, strings, booleans, records, classes, arrays, unions, functions, Maps, Sets, RegExps, Promises, or 'unknown') +utility-types.ts:13:7 - error SC2006: values of type '{ [s: symbol]: string; }' cannot be compiled: this index signature is outside the supported shape (string or number keys; values limited to numbers, strings, booleans, records, classes, arrays, unions, functions, Maps, Sets, RegExps, Dates, Promises, or 'unknown') - 14 | // named recursive shapes; differential corpus covers both.) - 15 | const bySymbol: { [s: symbol]: string } = {}; + 12 | // named recursive shapes; differential corpus covers both.) + 13 | const bySymbol: { [s: symbol]: string } = {}; | ^~~~~~~~ - 16 | console.log(bySymbol); - - hint: string- and number-keyed index signatures over the supported value types compile directly; symbol keys and dual signatures with unequal value types have no lowering - -utility-types.ts:18:7 - error SC2006: values of type '{ [k: string]: Date; }' cannot be compiled: this index signature is outside the supported shape (string or number keys; values limited to numbers, strings, booleans, records, classes, arrays, unions, functions, Maps, Sets, RegExps, Promises, or 'unknown') - - 17 | - 18 | const dateBag: { [k: string]: Date } = {}; - | ^~~~~~~ - 19 | const boot = dateBag["boot"]; + 14 | console.log(bySymbol); hint: string- and number-keyed index signatures over the supported value types compile directly; symbol keys and dual signatures with unequal value types have no lowering \ No newline at end of file diff --git a/tests/harness/corpus-costs/5965-date-objects.ts.json b/tests/harness/corpus-costs/5965-date-objects.ts.json new file mode 100644 index 000000000..c59069ff3 --- /dev/null +++ b/tests/harness/corpus-costs/5965-date-objects.ts.json @@ -0,0 +1 @@ +{ "plain": 3455, "san": 5063 } From fa843548a54d0e1ebe0b34a64df2ed4718eadd3f Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 20:36:52 -0500 Subject: [PATCH 016/184] Keep Dates as time values where no use observes their identity --- .../compiler/src/backend/llvm/lib-dispatch.ts | 33 ++- .../compiler/src/backend/llvm/lib-shared.ts | 1 + .../src/frontend/lowering/date-values.ts | 189 ++++++++++++++++++ .../lowering/expressions/identifiers.ts | 7 + .../src/frontend/lowering/lower-stmts.ts | 12 ++ .../compiler/src/frontend/lowering/lowerer.ts | 4 + .../compiler/src/ir/builtin-signatures.ts | 4 + .../compiler/test/date-time-values.test.ts | 100 +++++++++ .../corpus/5966-date-time-values.ts.json | 6 + tests/corpus/5966-date-time-values.ts | 80 ++++++++ .../5966-date-time-values.ts.json | 1 + 11 files changed, 429 insertions(+), 8 deletions(-) create mode 100644 packages/compiler/src/frontend/lowering/date-values.ts create mode 100644 packages/compiler/test/date-time-values.test.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5966-date-time-values.ts.json create mode 100644 tests/corpus/5966-date-time-values.ts create mode 100644 tests/harness/corpus-costs/5966-date-time-values.ts.json diff --git a/packages/compiler/src/backend/llvm/lib-dispatch.ts b/packages/compiler/src/backend/llvm/lib-dispatch.ts index 753f296e6..293d98a91 100644 --- a/packages/compiler/src/backend/llvm/lib-dispatch.ts +++ b/packages/compiler/src/backend/llvm/lib-dispatch.ts @@ -559,21 +559,25 @@ export function emitGenericLibCall(host: LlvmEmitterContext, e: LibCallExpr): Ll ? "scr_stream_dyn_async_iterator_open" : LIB_FN_SYMS[e.fn]); if (sym === undefined) throw new LlvmUnsupportedError(`libCall:${e.fn}`, e.loc); + // The runtime's Date functions take and answer time values: a Date + // operand passes its time value, read at once from the borrowed object, + // and a Date result is a new Date object. const args = READ_ONLY_STRING_LIB_FNS.has(e.fn) ? emitBorrowedInputs(host, e.args) : e.args.map((a) => - borrowsJsonInputs(e.fn) || borrowsIteratorInputs(e.fn) - ? emitBorrowedInput(host, a) - : host.emitExpr(a), + a.type.kind === "date" + ? { name: emitDateTimeArg(host, a), type: F64 } + : borrowsJsonInputs(e.fn) || borrowsIteratorInputs(e.fn) + ? emitBorrowedInput(host, a) + : host.emitExpr(a), ); + // getTime and valueOf answer the time value the argument already is. + if ((e.fn === "date.getTime" || e.fn === "date.valueOf") && e.args[0]?.type.kind === "date") + return { name: args[0]!.name, type: e.type }; // The deep comparisons' number test: computed here, with no runtime call. if (e.fn === "assert.sameValue") return { name: emitSameValueF64(host, args[0]!.name, args[1]!.name), type: e.type }; - // The runtime's Date functions take and answer time values: a Date - // operand passes its time value, and a Date result is a new Date object. - const callArgs = args.map((a) => - a.type.kind === "date" ? { name: emitDateTime(host, a.name), type: F64 } : a, - ); + const callArgs = args; if (e.type.kind === "date") { const ms = emitSymbolLibCall(host, { ...e, type: F64 }, sym, callArgs); host.declare(`declare ptr @scr_date_new(double)`); @@ -584,6 +588,19 @@ export function emitGenericLibCall(host: LlvmEmitterContext, e: LibCallExpr): Ll return emitSymbolLibCall(host, e, sym, callArgs); } +/** The time value of a Date argument. A Date that a runtime Date function + * produces for this argument alone (`new Date(ms).getTime()`, a + * time-value local's read) passes the function's time value and creates + * no Date object. */ +function emitDateTimeArg(host: LlvmEmitterContext, date: IrExpr): string { + // A time-value local's read is its time value itself. + if (date.kind === "libCall" && date.fn === "date.fromTime") + return host.emitExpr(date.args[0]!).name; + if (date.kind === "libCall" && date.fn.startsWith("date.")) + return emitGenericLibCall(host, { ...date, type: F64 }).name; + return emitDateTime(host, emitBorrowedInput(host, date).name); +} + /** A Date object's time value (ScrDate.ms). */ function emitDateTime(host: LlvmEmitterContext, date: string): string { const B = host.B; diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 8e7c74cad..abe43cb5e 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -330,6 +330,7 @@ export const LIB_FN_SYMS: Record = { "date.parse": "scr_date_parse", "date.getTime": "scr_date_get_time", "date.valueOf": "scr_date_get_time", + "date.fromTime": "scr_date_get_time", "date.parseGetTime": "scr_date_parse", "date.utc": "scr_date_utc", "date.newLocal": "scr_date_new_local", diff --git a/packages/compiler/src/frontend/lowering/date-values.ts b/packages/compiler/src/frontend/lowering/date-values.ts new file mode 100644 index 000000000..f40db4e6d --- /dev/null +++ b/packages/compiler/src/frontend/lowering/date-values.ts @@ -0,0 +1,189 @@ +import * as ts from "../ts7/adapter.js"; +import type { Lowerer } from "./lowerer.js"; + +/** Date methods whose receiver is only read for its time value: a Date + * local used as their receiver can store the time value instead of a Date + * object (builtins/dates.ts lowers them). */ +const TIME_VALUE_METHODS = new Set([ + "getTime", + "valueOf", + "toISOString", + "toJSON", + "toString", + "toDateString", + "toTimeString", + "toUTCString", + "getFullYear", + "getUTCFullYear", + "getMonth", + "getUTCMonth", + "getDate", + "getUTCDate", + "getDay", + "getUTCDay", + "getHours", + "getUTCHours", + "getMinutes", + "getUTCMinutes", + "getSeconds", + "getUTCSeconds", + "getMilliseconds", + "getUTCMilliseconds", + "getTimezoneOffset", +]); + +const RELATIONAL = new Set([ + ts.SyntaxKind.LessThanToken, + ts.SyntaxKind.LessThanEqualsToken, + ts.SyntaxKind.GreaterThanToken, + ts.SyntaxKind.GreaterThanEqualsToken, +]); + +/** True when a local `const` (or a `let` that is never assigned again) + * holding a Date can keep the Date's time value instead of a Date object: + * no use observes the Date's identity. Every use reads the time value as + * a Date method receiver, a String() or template operand, a relational or + * string `+` operand, or `new Date(d)`'s argument, inside the declaring + * function, or inside the block or loop that scopes a module-level + * binding (no closure captures it). Top-level module bindings keep the + * object. Other uses box the time value into a new Date at the use. */ +export function isTimeValueDateDeclaration( + lowerer: Lowerer, + decl: ts.VariableDeclaration, +): boolean { + const known = lowerer.timeValueDateDecisions.get(decl); + if (known !== undefined) return known; + const answer = decideTimeValueDate(lowerer, decl); + lowerer.timeValueDateDecisions.set(decl, answer); + return answer; +} + +function decideTimeValueDate(lowerer: Lowerer, decl: ts.VariableDeclaration): boolean { + if (!ts.isIdentifier(decl.name) || decl.initializer === undefined) return false; + const sf = decl.getSourceFile(); + if (sf.isDeclarationFile || /\.[cm]?jsx?$/.test(sf.fileName)) return false; + const list = decl.parent; + if (!ts.isVariableDeclarationList(list)) return false; + const isConst = (list.flags & ts.NodeFlags.Const) !== 0; + const isLet = (list.flags & ts.NodeFlags.Let) !== 0; + if (!isConst && !isLet) return false; + if (!ts.isVariableStatement(list.parent) && !ts.isForStatement(list.parent)) return false; + const fn = enclosingFunction(decl); + // The uses to check: the function body, or for a binding inside a block + // or loop at module level (a local of the module's initializer, not a + // module binding) the block or loop that scopes it. + let scope: ts.Node | undefined; + if (fn !== null) scope = fn.body; + else if (!inClassStaticBlock(decl)) scope = moduleBlockScope(decl); + if (scope === undefined) return false; + // The lib Date interface by its symbol first: mapping every candidate + // binding's type would intern its unions before the lowering does. + const typeSymbol = lowerer.typeOf(decl.name).getSymbol(); + if ( + typeSymbol === undefined || + ts.symbolName(typeSymbol) !== "Date" || + !typeSymbol.declarations.some( + (d) => ts.isInterfaceDeclaration(d) && lowerer.isStdlibFile(d.getSourceFile()), + ) + ) + return false; + if (lowerer.mapTypeOf(lowerer.typeOf(decl.name))?.kind !== "date") return false; + const symbol = lowerer.checker.getSymbolAtLocation(decl.name); + if (!symbol) return false; + let ok = true; + const visit = (node: ts.Node): void => { + if (!ok) return; + if (ts.isIdentifier(node) && node !== decl.name && node.text === decl.name.text) { + if (lowerer.checker.getSymbolAtLocation(node) === symbol) { + if (enclosingFunction(node) !== fn || !readsTimeValue(lowerer, node)) ok = false; + } + return; + } + ts.forEachChild(node, visit); + }; + visit(scope); + return ok; +} + +/** The block or for statement that scopes a module-level binding which is + * not a module binding: a declaration in a block or loop body, or in a + * for statement's initializer. Undefined for a top-level declaration. */ +function moduleBlockScope(decl: ts.VariableDeclaration): ts.Node | undefined { + const list = decl.parent; + if (!list || !ts.isVariableDeclarationList(list)) return undefined; + if (ts.isForStatement(list.parent)) return list.parent; + const statement = list.parent; + if (!ts.isVariableStatement(statement) || ts.isSourceFile(statement.parent)) return undefined; + for (let cur: ts.Node | undefined = statement.parent; cur; cur = cur.parent) { + if (ts.isBlock(cur) || ts.isCaseBlock(cur)) return cur; + if (ts.isSourceFile(cur)) return undefined; + } + return undefined; +} + +function inClassStaticBlock(node: ts.Node): boolean { + for (let cur = node.parent; cur; cur = cur.parent) { + if (ts.isClassStaticBlockDeclaration(cur)) return true; + if (ts.isSourceFile(cur)) return false; + } + return false; +} + +function enclosingFunction(node: ts.Node): ts.FunctionLikeDeclaration | null { + for (let cur = node.parent; cur; cur = cur.parent) { + if ( + ts.isFunctionDeclaration(cur) || + ts.isFunctionExpression(cur) || + ts.isArrowFunction(cur) || + ts.isMethodDeclaration(cur) || + ts.isConstructorDeclaration(cur) || + ts.isGetAccessorDeclaration(cur) || + ts.isSetAccessorDeclaration(cur) + ) + return cur; + if (ts.isClassStaticBlockDeclaration(cur) || ts.isSourceFile(cur)) return null; + } + return null; +} + +/** A use of the binding that reads only its time value. */ +function readsTimeValue(lowerer: Lowerer, id: ts.Identifier): boolean { + let node: ts.Node = id; + while (ts.isParenthesizedExpression(node.parent)) node = node.parent; + const parent = node.parent; + if (ts.isPropertyAccessExpression(parent) && parent.expression === node) { + const call = parent.parent; + return ( + !parent.questionDotToken && + ts.isCallExpression(call) && + call.expression === parent && + !call.questionDotToken && + TIME_VALUE_METHODS.has(parent.name.text) && + lowerer.isStdlibMember(parent) + ); + } + if (ts.isTemplateSpan(parent)) return parent.expression === node; + if (ts.isCallExpression(parent)) { + return ( + parent.arguments.nodes.length === 1 && + parent.arguments.nodes[0] === node && + ts.isIdentifier(parent.expression) && + lowerer.isStdlibGlobal(parent.expression, "String") + ); + } + if (ts.isNewExpression(parent)) { + return ( + parent.arguments?.nodes.length === 1 && + parent.arguments.nodes[0] === node && + ts.isIdentifier(parent.expression) && + lowerer.isStdlibGlobal(parent.expression, "Date") + ); + } + if (ts.isBinaryExpression(parent)) { + const other = parent.left === node ? parent.right : parent.left; + const otherKind = lowerer.mapTypeOf(lowerer.typeOf(other))?.kind; + if (RELATIONAL.has(parent.operatorToken.kind)) return otherKind === "date"; + if (parent.operatorToken.kind === ts.SyntaxKind.PlusToken) return otherKind === "string"; + } + return false; +} diff --git a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts index ff7848930..2c82a3c64 100644 --- a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts +++ b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts @@ -8,6 +8,7 @@ import { type Lowerer, PoisonError } from "../lowerer.js"; import { BOOL, BYTES_ELEMENT_NAME, + DATE_T, DYN, F64, type IrExpr, @@ -170,6 +171,12 @@ export function lowerIdentifierExpr(lowerer: Lowerer, expr: ts.Identifier, loc: // when the checker types this USE as a single arm, maybeNarrow // bridges the tagged representation with a unionNarrow. const local = lowerer.resolveLocal(expr); + // A Date local holding its time value reads as that Date: only uses + // that read the time value reach here (date-values.ts). + if (local && lowerer.timeValueDateLocals.has(local)) { + const time: IrExpr = { kind: "varRef", localId: local.id, type: F64, loc }; + return { kind: "libCall", fn: "date.fromTime", args: [time], type: DATE_T, loc }; + } if (local) { // A switch can dispatch past a lexical declaration. Keep this // previously refused codec form out until switch-scope TDZ boxes diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index 750057bf9..f4d6564ba 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -28,6 +28,7 @@ import * as ts from "../ts7/adapter.js"; import { processDataMemberValue, programReadsExitCode } from "./builtins/process.js"; import { requireInitFlagStmt } from "./require-initialization.js"; import { isJsonRequireDecl } from "./json-require.js"; +import { isTimeValueDateDeclaration } from "./date-values.js"; import { bindingSource } from "../binding-source.js"; import { lowerConstructorReturn } from "./constructor-returns.js"; import type { Lowerer } from "./lowerer.js"; @@ -63,6 +64,7 @@ import { BOOL, BYTES_U8, CAUGHT, + DATE_T, DYN, F64, type IrExpr, @@ -5234,6 +5236,16 @@ export function lowerVarDecl( if (alias) return alias; } + // A Date local whose identity no use observes stores its time value. + if (isTimeValueDateDeclaration(lowerer, decl)) { + const loc = locOf(decl); + const date = lowerer.lowerExprExpecting(decl.initializer!, DATE_T); + const local = lowerer.declareLocal(decl.name, decl.name.text, F64, isLet); + lowerer.timeValueDateLocals.add(local); + const init: IrExpr = { kind: "libCall", fn: "date.getTime", args: [date], type: F64, loc }; + return { kind: "varDecl", localId: local.id, init, loc }; + } + // A NULLISH generic binding (`const i: I = null as any` — the // declared type has no mapping, the value is provably null/undefined // forever): no storage exists; reads know the value (member reads and diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 3ba81dd6b..637dfcaa8 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -2331,6 +2331,10 @@ export class Lowerer { readonly runtimeOptionalFields = new Set(); /** Bindings initialized by a Date's `toJSON()` (analyzeDateJsonResults). */ readonly dateJsonBindings = new Set(); + /** isTimeValueDateDeclaration's answers, and the locals that hold a + * Date's time value instead of a Date object (date-values.ts). */ + readonly timeValueDateDecisions = new Map(); + readonly timeValueDateLocals = new Set(); /** Record bindings whose flow class reaches storage outside the class: * they keep the declared layout, so an initializer from unchecked reads * converts once instead of the binding adopting the widened variant. */ diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index f5da56818..b89ca61c4 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -4738,6 +4738,10 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "date.getMilliseconds": { argTypes: [DATE_T], result: F64 }, "date.getUTCMilliseconds": { argTypes: [DATE_T], result: F64 }, "date.getTimezoneOffset": { argTypes: [DATE_T], result: F64 }, + /** A Date object for a time value a local holds (date-values.ts): a + * lib-call argument passes the time value itself, and any other use + * creates the Date. */ + "date.fromTime": { argTypes: [F64], result: DATE_T }, /** Date.prototype.toString / toDateString / toTimeString and String(date): * V8's local-time formats with the host zone's long name, "Invalid Date" * for an invalid date. Owned (+1) string; never throws. */ diff --git a/packages/compiler/test/date-time-values.test.ts b/packages/compiler/test/date-time-values.test.ts new file mode 100644 index 000000000..4e90ef3e6 --- /dev/null +++ b/packages/compiler/test/date-time-values.test.ts @@ -0,0 +1,100 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile, deserializeModule } from "../src/index.js"; +import { emitLlvmModule } from "../src/backend/llvm/emitter.js"; +import type { IrModule } from "../src/ir/ir.js"; + +async function lowerSource(source: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "scriptc-date-time-values-")); + try { + const entry = join(dir, "main.ts"); + await writeFile(entry, source); + const outPath = join(dir, "main.ir.json"); + const result = await compile(entry, { outPath, outDir: dir, outputKind: "ir" }); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + return deserializeModule(await readFile(outPath, "utf8")); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +/** The IR type of each local named `name` in function `fn`. */ +function localKinds(mod: IrModule, fn: string, name: string): string[] { + const f = mod.functions.find((candidate) => candidate.name === fn); + if (!f) throw new Error(`no function ${fn}`); + return f.locals.filter((local) => local.name === name).map((local) => local.type.kind); +} + +test("a Date local read only for its time value stores the time value", async () => { + const mod = await lowerSource(` +function format(ms: number): string { + const d = new Date(ms); + const later = new Date(ms + 1); + return d.toISOString() + String(d) + \`\${later}\` + (d < later) + d.getUTCHours() + new Date(d).getTime(); +} +console.log(format(0)); +`); + expect(localKinds(mod, "format", "d")).toEqual(["f64"]); + expect(localKinds(mod, "format", "later")).toEqual(["f64"]); + // Reading the time value creates no Date object. + const llvm = emitLlvmModule(mod, { pointerBits: 64 }); + const body = /^define internal [^\n]*@sc_f_format\([^]*?^}/m.exec(llvm)?.[0] ?? ""; + expect(body).not.toContain("@scr_date_new("); +}); + +test("a Date local whose identity a use can observe keeps the Date object", async () => { + const mod = await lowerSource(` +const keep: Date[] = []; +function uses(ms: number): boolean { + const compared = new Date(ms); + const stored = new Date(ms); + const passed = new Date(ms); + const captured = new Date(ms); + const returned = new Date(ms); + let reassigned = new Date(ms); + reassigned = new Date(ms + 1); + keep.push(stored); + const read = (): number => captured.getTime(); + console.log(String(passed), read(), returned.getTime(), reassigned.getTime()); + return compared === compared && pass(passed) === passed && back(returned) === returned; +} +function pass(d: Date): Date { + return d; +} +function back(d: Date): Date { + return d; +} +const top = new Date(0); +console.log(uses(0), top.getTime()); +`); + for (const name of ["compared", "stored", "passed", "captured", "returned", "reassigned"]) { + expect(localKinds(mod, "uses", name), name).toEqual(["date"]); + } +}); + +test("a Date in a block or loop at module level stores the time value unless a use observes it", async () => { + const mod = await lowerSource(` +const seen: Date[] = []; +let total = 0; +for (let i = 0; i < 3; i++) { + const read = new Date(i * 1000); + const kept = new Date(i * 1000); + seen.push(kept); + total += read.getUTCSeconds() + read.toISOString().length; +} +{ + const block = new Date(5); + const captured = new Date(6); + const later = (): number => captured.getTime(); + total += block.getTime() + later(); +} +const top = new Date(7); +console.log(total, seen.length, top.getTime()); +`); + expect(localKinds(mod, "%init.0", "read")).toEqual(["f64"]); + expect(localKinds(mod, "%init.0", "block")).toEqual(["f64"]); + expect(localKinds(mod, "%init.0", "kept")).toEqual(["date"]); + expect(localKinds(mod, "%init.0", "captured")).toEqual(["date"]); +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5966-date-time-values.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5966-date-time-values.ts.json new file mode 100644 index 000000000..e6155bcc4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5966-date-time-values.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5966-date-time-values.ts" + ], + "diags": [] +} diff --git a/tests/corpus/5966-date-time-values.ts b/tests/corpus/5966-date-time-values.ts new file mode 100644 index 000000000..342b4662c --- /dev/null +++ b/tests/corpus/5966-date-time-values.ts @@ -0,0 +1,80 @@ +// Date locals that are only read for their time value (method receivers, +// String() and template operands, relational and string `+` operands, +// `new Date(d)`) print and compare as Dates; locals whose identity a use +// can observe keep it. +export {}; +process.env.TZ = "America/New_York"; + +function describe(ms: number): string { + const d = new Date(ms); + const later = new Date(ms + 3_600_000); + const copy = new Date(d); + return [ + d.toISOString(), + d.toUTCString(), + String(d), + `${later}`, + "at " + d, + d < later, + later >= d, + d.getHours(), + d.getUTCDay(), + d.getTimezoneOffset(), + copy.getTime() === d.getTime(), + d.toJSON(), + ].join(" | "); +} +console.log(describe(Date.UTC(2026, 2, 8, 6, 30))); +console.log(describe(Date.UTC(2026, 10, 1, 5, 30))); +function invalid(): string { + const d = new Date(NaN); + return [String(d), d.getTime(), String(d.toJSON()), d < new Date(0), `${d}`].join(" | "); +} +console.log(invalid()); + +function sum(days: number): number { + let total = 0; + for (let i = 0; i < days; i++) { + const day = new Date(Date.UTC(2026, 0, 1) + i * 86_400_000); + total += day.getUTCDate() + day.getUTCMonth() + day.toISOString().length; + } + return total; +} +console.log(sum(400)); + +// Identity-observing uses keep the Date object. +const kept: Date[] = []; +function identities(ms: number): string { + const a = new Date(ms); + const b = a; + const stored = new Date(ms); + kept.push(stored); + const captured = new Date(ms); + const read = (): number => captured.getTime(); + const passed = new Date(ms); + return [a === b, kept.includes(stored), kept[0] === stored, read(), same(passed, passed)].join(","); +} +function same(x: Date, y: Date): boolean { + return x === y; +} +console.log(identities(5)); +const held: unknown = kept[0]; +console.log(held instanceof Date, (held as Date) === kept[0]); + +// Dates in blocks and loops at module level, read for their time value or +// kept, stored and captured. +const moduleKept: Date[] = []; +let moduleTotal = ""; +for (let i = 0; i < 3; i++) { + const read = new Date(Date.UTC(2026, 2, 8 + i, 6, 30)); + const kept = new Date(read.getTime()); + moduleKept.push(kept); + moduleTotal += `${read.toISOString()} ${read.getHours()} ${String(read)}\n`; +} +{ + const block = new Date(Date.UTC(2026, 10, 1, 5, 59)); + const captured = new Date(block); + const later = (): string => captured.toISOString(); + moduleTotal += `${block.getTimezoneOffset()} ${later()} ${captured === moduleKept[0]}`; +} +console.log(moduleTotal, moduleKept.map((d) => d.getUTCDate()).join(",")); diff --git a/tests/harness/corpus-costs/5966-date-time-values.ts.json b/tests/harness/corpus-costs/5966-date-time-values.ts.json new file mode 100644 index 000000000..a3a32b65e --- /dev/null +++ b/tests/harness/corpus-costs/5966-date-time-values.ts.json @@ -0,0 +1 @@ +{ "plain": 3096, "san": 3319 } From 961aad6ac99b9b3cf8d101c24415cb5b60d9a616 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 20:37:10 -0500 Subject: [PATCH 017/184] Keep the identity of records and class instances converted to dictionaries A record or class instance converted to a string-keyed dictionary type (`Record`, `{ [key: string]: V }`) was copied field by field into a native dictionary record, directly, through a dictionary parameter, or through a `{}`, `object` or `unknown` binding or parameter. Writes through the dictionary missed the original object, and the dictionary was never `===` to it, where under Node it is the same object. ClassProtocols now marks a dictionary value type when a dictionary of that value type receives an existing object (a binding, member, element or `this` whose type is a program class instance, a record with declared properties, or a `{}`, `object` or `unknown` binding or parameter that holds one), and the program writes through, compares or passes to a whole-object observer a dictionary of that value type. usesCheckedIdentity answers true for those dictionaries, so they lower to checked-dynamic storage, which holds the original object. Dictionaries of `unknown` or `any` values are checked-dynamic already and are not marked. The dictionary case of SC6001 is removed, since these conversions now keep the object. --- .../record-identity-and-property-order.mdx | 3 +- .../compiler/src/diagnostics/diagnostic.ts | 10 +- .../src/frontend/lowering/class-protocols.ts | 205 +++++++++++++++++- .../src/frontend/lowering/divergences.ts | 110 +--------- .../compiler/test/dictionary-views.test.ts | 77 +++++++ ...7012-dictionary-views-keep-objects.ts.json | 6 + .../7013-dictionary-view-parameter.ts.json | 6 + .../7012-dictionary-views-keep-objects.ts | 61 ++++++ .../corpus/7013-dictionary-view-parameter.ts | 11 + .../dictionary-view-copies.ts | 29 --- tests/coverage-fixtures/no-divergences.ts | 7 + .../coverage-dictionary-view-copies.txt | 21 -- ...7012-dictionary-views-keep-objects.ts.json | 1 + .../7013-dictionary-view-parameter.ts.json | 1 + tests/harness/coverage.test.ts | 14 -- 15 files changed, 382 insertions(+), 180 deletions(-) create mode 100644 packages/compiler/test/dictionary-views.test.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7012-dictionary-views-keep-objects.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7013-dictionary-view-parameter.ts.json create mode 100644 tests/corpus/7012-dictionary-views-keep-objects.ts create mode 100644 tests/corpus/7013-dictionary-view-parameter.ts delete mode 100644 tests/coverage-fixtures/dictionary-view-copies.ts delete mode 100644 tests/harness/__snapshots__/coverage-dictionary-view-copies.txt create mode 100644 tests/harness/corpus-costs/7012-dictionary-views-keep-objects.ts.json create mode 100644 tests/harness/corpus-costs/7013-dictionary-view-parameter.ts.json diff --git a/docs/content/limitations/record-identity-and-property-order.mdx b/docs/content/limitations/record-identity-and-property-order.mdx index 224757d52..2299122a4 100644 --- a/docs/content/limitations/record-identity-and-property-order.mdx +++ b/docs/content/limitations/record-identity-and-property-order.mdx @@ -1,10 +1,11 @@ A record converted to a record type with a different set of properties, such as a narrower shape or one that adds optional properties, is copied. Mutations through the converted reference do not affect the original, and the copy is not `===` to it. Empty record layouts are also snapshots because they have no property storage. -A class instance or record held as `{}`, `object` or `unknown` and then converted to a dictionary type such as `Record` is copied as well; a dictionary of `unknown` values keeps the original object. `scriptc coverage` and `scriptc build` report `SC6001` where a copied value is then written or compared, whether the record was passed as an argument, stored in a typed binding, asserted with `as`, assigned to a typed binding later, or stored as an element of a typed array literal. Typed records with declared fields, indexed records, and arrays retain identity and mutations through native checked storage. +A record or class instance converted to a string-keyed dictionary type such as `Record`, directly or through a `{}`, `object` or `unknown` binding or parameter, keeps its identity when the program writes through, compares or inspects values of that dictionary type. +Every dictionary type with the same value type then uses checked dynamic storage, which is slower than a native dictionary. Class instances retain identity at their exact type or a base class, and at any interface or object type that a class instance is passed, stored, returned, or asserted as when the program could tell a copy from the instance: values of such a type are checked views of the original object, so writes, identity comparisons, Set and Map keys, `util.inspect`, and `JSON.stringify` through them reach the instance, as in Node.js. These types use checked dynamic storage, which is slower than native record fields, for all of their values, including plain objects. A type whose values are only read field by field, from fields the class does not write after construction and does not implement as accessors, stays a native record and holds a copy. diff --git a/packages/compiler/src/diagnostics/diagnostic.ts b/packages/compiler/src/diagnostics/diagnostic.ts index 47db2193b..17cb2efc0 100644 --- a/packages/compiler/src/diagnostics/diagnostic.ts +++ b/packages/compiler/src/diagnostics/diagnostic.ts @@ -129,7 +129,6 @@ export function widthCopyDivergenceDiag( useText: string, site: "call" | "declaration" | "cast" | "element", loc: SrcLoc, - dictionary = false, ): ScrDiagnostic { const how = site === "call" @@ -143,16 +142,11 @@ export function widthCopyDivergenceDiag( use === "write" ? `the write to '${useText}' updates only the copy, while under Node it updates the original object` : `'${useText}' compares the copy, which is never the original object, while under Node it is the same object`; - const cause = dictionary - ? "converting an object held as '{}', 'object' or 'unknown' to a dictionary type" - : "converting to a record type with other properties"; return { code: "SC6001", - message: `${how} copies the object (${cause} compiles as a field copy): ${effect}`, + message: `${how} copies the object (converting to a record type with other properties compiles as a field copy): ${effect}`, loc, - hint: dictionary - ? `keep the value at its own type ('${sourceType}'), or declare the dictionary's values as 'unknown', which keeps the original object` - : `declare the parameter or binding with the value's full type ('${sourceType}'), or make the shape a class — class instances keep their identity when passed as a base class`, + hint: `declare the parameter or binding with the value's full type ('${sourceType}'), or make the shape a class — class instances keep their identity when passed as a base class`, }; } diff --git a/packages/compiler/src/frontend/lowering/class-protocols.ts b/packages/compiler/src/frontend/lowering/class-protocols.ts index 76091b13e..4edf1ab7e 100644 --- a/packages/compiler/src/frontend/lowering/class-protocols.ts +++ b/packages/compiler/src/frontend/lowering/class-protocols.ts @@ -10,6 +10,9 @@ export class ClassProtocols { private readonly accepted = new Map(); /** What markIdentityFlows learned about the program, while it runs. */ private facts: FlowFacts | null = null; + /** The value types of the string-keyed dictionaries that keep checked + * identity (see markDictionaryViews). */ + private readonly dictionaryViews = new Set(); constructor( private readonly checker: ts.TypeChecker, @@ -323,6 +326,7 @@ export class ClassProtocols { else if (names !== "all") names.add(name); } } + this.markDictionaryViews(sites, sourceNodes, observedNodes, writeNodes); for (let i = 0; i < looseArguments.length; i++) { const type = typeOf(looseNodes[i]!); if (type === undefined) continue; @@ -697,6 +701,163 @@ export class ClassProtocols { return source !== undefined && this.isProgramClassInstance(source); } + /** String-keyed dictionaries (`Record`, `{ [key: string]: V }`) + * that receive an existing record or class instance. Natively a + * dictionary is a key/value record, and converting another object into + * one copies its fields, so writes through the dictionary and identity + * checks would miss the original. When the program writes through, or + * observes, a value of such a dictionary type, every dictionary with that + * value type keeps checked identity instead (records with the same value + * type share one native layout, so values pass between them). A source + * counts when it is an existing object (a binding, member, element or + * `this`, not a fresh literal or call result) whose type is a program + * class instance or a record with declared properties, or a constant + * `{}`, `object` or `unknown` binding initialized with one. */ + private markDictionaryViews( + sites: readonly { + source: ts.Expression; + target: () => ts.Type | undefined; + contextual?: ts.Expression; + }[], + sourceNodes: readonly ts.Expression[], + observedNodes: readonly ts.Expression[], + writeNodes: readonly ts.Expression[], + ): void { + // The calls the sites' arguments belong to, by callee name, built on + // first use. + let calls: Map | null = null; + const callsNamed = (name: string): readonly (ts.CallExpression | ts.NewExpression)[] => { + if (calls === null) { + calls = new Map(); + for (const site of sites) { + const call = site.contextual?.parent; + if (call === undefined || (!ts.isCallExpression(call) && !ts.isNewExpression(call))) + continue; + const callee = call.expression; + const named = ts.isIdentifier(callee) + ? callee.text + : ts.isPropertyAccessExpression(callee) + ? callee.name.text + : undefined; + if (named === undefined) continue; + const list = calls.get(named) ?? []; + if (!list.includes(call)) list.push(call); + calls.set(named, list); + } + } + return calls.get(name) ?? []; + }; + const candidates: { index: number; sourceType: ts.Type }[] = []; + for (let i = 0; i < sites.length; i++) { + const source = sourceNodes[i]!; + if (!isExistingObject(source)) continue; + const sourceType = attempt(() => this.checker.getTypeAtLocation(source)); + if (sourceType !== undefined && this.dictionarySource(source, sourceType, callsNamed)) + candidates.push({ index: i, sourceType }); + } + if (candidates.length === 0) return; + // One batch for the contextual targets of the candidate sites. + attempt(() => + this.checker.prefetchContextualTypes( + candidates.flatMap(({ index }) => { + const contextual = sites[index]!.contextual; + return contextual === undefined ? [] : [contextual]; + }), + ), + ); + const received = new Set(); + for (const { index, sourceType } of candidates) { + const target = attempt(sites[index]!.target); + const parts = + target === undefined ? [] : target.isUnionType() ? ts.constituentTypes(target) : [target]; + for (const part of parts) { + const value = this.dictionaryValueType(part); + // A dictionary of `unknown` or `any` values is checked-dynamic already. + if ( + value !== undefined && + (value.flags & (ts.TypeFlags.Unknown | ts.TypeFlags.Any)) === 0 && + this.dictionaryValueType(sourceType) !== value + ) + received.add(value); + } + } + if (received.size === 0) return; + // The dictionary values the program writes through or observes. + const seen = new Set(); + for (const node of [...writeNodes, ...observedNodes]) { + const type = attempt(() => this.checker.getTypeAtLocation(node)); + const parts = + type === undefined ? [] : type.isUnionType() ? ts.constituentTypes(type) : [type]; + for (const part of parts) { + const value = this.dictionaryValueType(part); + if (value !== undefined) seen.add(value); + } + } + for (const value of received) if (seen.has(value)) this.dictionaryViews.add(value); + } + + /** Whether `source` holds an existing object that a dictionary + * conversion would copy. */ + private dictionarySource( + source: ts.Expression, + type: ts.Type, + callsNamed: (name: string) => readonly (ts.CallExpression | ts.NewExpression)[], + ): boolean { + if (this.isProgramClassInstance(type) || this.isRecordWithFields(type)) return true; + if (!isDynamicView(this.checker, type) || !ts.isIdentifier(source)) return false; + const decl = attempt(() => this.checker.getSymbolAtLocation(source))?.valueDeclaration; + if (decl === undefined) return false; + const holdsObject = (expr: ts.Expression): boolean => { + const held = attempt(() => this.checker.getTypeAtLocation(this.flowSourceNode(expr))); + return ( + held !== undefined && (this.isProgramClassInstance(held) || this.isRecordWithFields(held)) + ); + }; + if (ts.isParameter(decl)) { + // A parameter holds what its callers pass: the calls of the function + // by name whose signature resolves to it. + const fn = decl.parent; + if (fn === undefined || !ts.isFunctionLike(fn)) return false; + const index = fn.parameters.nodes.indexOf(decl); + const name = functionName(fn); + if (name === undefined || index < 0) return false; + return callsNamed(name).some((call) => { + const argument = call.arguments?.nodes[index]; + return ( + argument !== undefined && + !ts.isSpreadElement(argument) && + attempt(() => this.checker.getResolvedSignature(call)?.declaration) === fn && + holdsObject(argument) + ); + }); + } + return ( + ts.isVariableDeclaration(decl) && + decl.initializer !== undefined && + ts.isVariableDeclarationList(decl.parent) && + (decl.parent.flags & ts.NodeFlags.Const) !== 0 && + holdsObject(decl.initializer) + ); + } + + /** A program object type with declared properties that is not a class + * instance, array, tuple, function or library type. */ + private isRecordWithFields(type: ts.Type): boolean { + return this.isStructuralTarget(type) && this.checker.getPropertiesOfType(type).length > 0; + } + + /** The value type of a string-keyed dictionary with no declared + * properties, or undefined for any other type. */ + private dictionaryValueType(type: ts.Type): ts.Type | undefined { + if ((type.flags & ts.TypeFlags.Object) === 0) return undefined; + if (this.checker.isArrayType(type) || this.checker.isTupleType(type)) return undefined; + if (this.checker.getCallSignatures(type).length > 0) return undefined; + if (this.checker.getPropertiesOfType(type).length > 0) return undefined; + return this.checker + .getIndexInfosOfType(type) + .find((index) => (index.keyType.flags & ts.TypeFlags.String) !== 0)?.type; + } + private isProgramClassInstance(type: ts.Type): boolean { const symbol = type.getSymbol(); return ( @@ -822,7 +983,11 @@ export class ClassProtocols { if ((type.flags & ts.TypeFlags.Object) === 0) return false; const previous = this.answers.get(type); if (previous !== undefined) return previous; - const answer = this.protocol(type); + const dictionaryValue = this.dictionaryValueType(type); + const answer = + dictionaryValue !== undefined + ? (this.index(), this.dictionaryViews.has(dictionaryValue)) + : this.protocol(type); this.answers.set(type, answer); return answer; } @@ -922,6 +1087,44 @@ const IDENTITY_METHODS = new Set([ "includes", ]); +/** An expression that denotes an object that already exists, not a fresh + * literal or call result. */ +function isExistingObject(node: ts.Expression): boolean { + return ( + ts.isIdentifier(node) || + ts.isPropertyAccessExpression(node) || + ts.isElementAccessExpression(node) || + node.kind === ts.SyntaxKind.ThisKeyword + ); +} + +/** `{}`, `object` and `unknown`: types whose values are checked-dynamic + * and keep the original object. */ +function isDynamicView(checker: ts.TypeChecker, type: ts.Type): boolean { + if (type.flags & (ts.TypeFlags.Unknown | ts.TypeFlags.NonPrimitive)) return true; + return ( + (type.flags & ts.TypeFlags.Object) !== 0 && + checker.getPropertiesOfType(type).length === 0 && + checker.getIndexInfosOfType(type).length === 0 && + checker.getCallSignatures(type).length === 0 && + checker.getConstructSignatures(type).length === 0 + ); +} + +/** The name calls use for a function, method or function-valued binding. */ +function functionName(fn: ts.Node): string | undefined { + if ((ts.isFunctionDeclaration(fn) || ts.isMethodDeclaration(fn)) && fn.name !== undefined) + return ts.isIdentifier(fn.name) ? fn.name.text : undefined; + if ( + (ts.isArrowFunction(fn) || ts.isFunctionExpression(fn)) && + fn.parent !== undefined && + ts.isVariableDeclaration(fn.parent) && + ts.isIdentifier(fn.parent.name) + ) + return fn.parent.name.text; + return undefined; +} + function isNullishLiteral(node: ts.Expression): boolean { while (ts.isParenthesizedExpression(node)) node = node.expression; return ( diff --git a/packages/compiler/src/frontend/lowering/divergences.ts b/packages/compiler/src/frontend/lowering/divergences.ts index f766aba14..1c362cb09 100644 --- a/packages/compiler/src/frontend/lowering/divergences.ts +++ b/packages/compiler/src/frontend/lowering/divergences.ts @@ -180,15 +180,7 @@ export function scanDivergences( const copy = safely(() => widthCopyBetween(lowerer, arg, param.type!)); if (copy === undefined || copy === null) return; out.push( - widthCopyDivergenceDiag( - copy.source, - copy.target, - use.kind, - use.text, - "call", - locOf(arg), - copy.dictionary, - ), + widthCopyDivergenceDiag(copy.source, copy.target, use.kind, use.text, "call", locOf(arg)), ); }); } @@ -208,7 +200,6 @@ export function scanDivergences( use.text, "declaration", locOf(decl.initializer!), - copy.dictionary, ), ); } @@ -224,15 +215,7 @@ export function scanDivergences( const copy = safely(() => widthCopyBetween(lowerer, source, type)); if (copy === undefined || copy === null) continue; out.push( - widthCopyDivergenceDiag( - copy.source, - copy.target, - use.kind, - use.text, - "cast", - locOf(source), - copy.dictionary, - ), + widthCopyDivergenceDiag(copy.source, copy.target, use.kind, use.text, "cast", locOf(source)), ); } for (const assignment of assignments) { @@ -257,7 +240,6 @@ export function scanDivergences( use.text, "declaration", locOf(assignment.right), - copy.dictionary, ), ); } @@ -272,15 +254,7 @@ export function scanDivergences( const copy = safely(() => widthCopyBetween(lowerer, item, element)); if (copy === undefined || copy === null) continue; out.push( - widthCopyDivergenceDiag( - copy.source, - copy.target, - "write", - use, - "element", - locOf(item), - copy.dictionary, - ), + widthCopyDivergenceDiag(copy.source, copy.target, "write", use, "element", locOf(item)), ); break; } @@ -679,23 +653,14 @@ function enclosingScope(node: ts.Node): ts.Node | undefined { * instance flowing into a plain record type that does not keep class * identity (ClassProtocols marks the targets a class instance visibly * flows into; a value reaching another record type is a field copy). */ -/** The types a width copy converts between, and whether the target is a - * dictionary. */ -interface WidthCopy { - source: string; - target: string; - dictionary?: boolean; -} - function widthCopyBetween( lowerer: Lowerer, source: ts.Expression, targetNode: ts.TypeNode, -): WidthCopy | null { +): { source: string; target: string } | null { const checker = lowerer.checker; const s = checker.getTypeAtLocation(source); const t = checker.getTypeFromTypeNode(targetNode); - if (isDictionaryType(checker, t)) return dictionaryViewCopy(lowerer, source, s, t); if (!isPlainRecordType(checker, t)) return null; const classSource = isProgramClassType(checker, s); if (!classSource && !isPlainRecordType(checker, s)) return null; @@ -715,73 +680,6 @@ function widthCopyBetween( return { source: checker.typeToString(s), target: checker.typeToString(t) }; } -/** A copy made by converting a checked-dynamic view (`{}`, `object`, - * `unknown`) to a dictionary type that compiles to a native record (a - * dictionary of `unknown` values stays checked-dynamic and keeps the - * object). A dictionary of that type comes back out of the view as itself, - * so a copy is reported only when the view is a constant binding - * initialized with a class instance or a record: that object is converted - * field by field into a new dictionary. */ -function dictionaryViewCopy( - lowerer: Lowerer, - source: ts.Expression, - s: ts.Type, - t: ts.Type, -): WidthCopy | null { - const checker = lowerer.checker; - if (!isDynamicViewType(checker, s)) return null; - if (lowerer.mapTypeOf(t)?.kind !== "record") return null; - while (ts.isParenthesizedExpression(source) || ts.isNonNullExpression(source)) - source = source.expression; - if (!ts.isIdentifier(source)) return null; - const decl = checker.getSymbolAtLocation(source)?.valueDeclaration; - if ( - decl === undefined || - !ts.isVariableDeclaration(decl) || - decl.initializer === undefined || - !ts.isVariableDeclarationList(decl.parent) || - !(decl.parent.flags & ts.NodeFlags.Const) - ) - return null; - let origin = decl.initializer; - while ( - ts.isParenthesizedExpression(origin) || - ts.isNonNullExpression(origin) || - ts.isAsExpression(origin) || - ts.isTypeAssertion(origin) - ) - origin = origin.expression; - const o = checker.getTypeAtLocation(origin); - if (!isProgramClassType(checker, o) && !isPlainRecordType(checker, o)) return null; - return { source: checker.typeToString(o), target: checker.typeToString(t), dictionary: true }; -} - -/** `{}`, `object` and `unknown`: types whose values lower to checked - * dynamic storage that keeps the original object. */ -function isDynamicViewType(checker: ts.TypeChecker, type: ts.Type): boolean { - if (type.flags & (ts.TypeFlags.Unknown | ts.TypeFlags.NonPrimitive)) return true; - return ( - (type.flags & ts.TypeFlags.Object) !== 0 && - checker.getPropertiesOfType(type).length === 0 && - checker.getIndexInfosOfType(type).length === 0 && - checker.getCallSignatures(type).length === 0 && - checker.getConstructSignatures(type).length === 0 - ); -} - -/** A string-keyed dictionary with no declared properties - * (`Record`, `{ [key: string]: number }`). */ -function isDictionaryType(checker: ts.TypeChecker, type: ts.Type): boolean { - if ((type.flags & ts.TypeFlags.Object) === 0) return false; - if (checker.isArrayType(type) || checker.isTupleType(type)) return false; - if (checker.getCallSignatures(type).length > 0) return false; - if (checker.getConstructSignatures(type).length > 0) return false; - if (checker.getPropertiesOfType(type).length > 0) return false; - return checker - .getIndexInfosOfType(type) - .some((index) => (index.keyType.flags & ts.TypeFlags.String) !== 0); -} - function isProgramClassType(checker: ts.TypeChecker, type: ts.Type): boolean { if ((type.flags & ts.TypeFlags.Object) === 0) return false; const symbol = type.getSymbol(); diff --git a/packages/compiler/test/dictionary-views.test.ts b/packages/compiler/test/dictionary-views.test.ts new file mode 100644 index 000000000..b841447cd --- /dev/null +++ b/packages/compiler/test/dictionary-views.test.ts @@ -0,0 +1,77 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile, deserializeModule } from "../src/index.js"; + +/** The IR type kind of each named module-level binding. */ +async function globalKinds(source: string): Promise> { + const dir = await mkdtemp(join(tmpdir(), "scriptc-dictionary-views-")); + try { + const entry = join(dir, "main.ts"); + const outPath = join(dir, "main.ir.json"); + await writeFile(entry, source); + const result = await compile(entry, { outDir: dir, outPath, outputKind: "ir" }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const mod = deserializeModule(await readFile(outPath, "utf8")); + return Object.fromEntries((mod.globals ?? []).map((g) => [g.name, g.type.kind])); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +const program = (body: string): string => ` +const key = String(process.argv.length); +const counts: Record = { a: 1 }; +counts[key] = 2; +${body} +console.log(JSON.stringify(counts)); +`; + +test("a record written through a dictionary keeps that dictionary type checked", async () => { + const kinds = await globalKinds( + program(`const point: { a: string } = { a: "x" }; +const table: Record = point; +table[key] = "y"; +console.log(JSON.stringify(point));`), + ); + expect(kinds["table"]).toBe("dyn"); + // Dictionaries of other value types keep their native layout. + expect(kinds["counts"]).toBe("record"); +}); + +test("a record reaching a dictionary through a {} parameter keeps it checked", async () => { + const kinds = await globalKinds( + program(`function stamp(value: {}): void { + const inner: Record = value; + inner[key] = "y"; +} +const point: { a: string } = { a: "x" }; +stamp(point); +const other: Record = { b: "z" }; +console.log(JSON.stringify(point), other["b"]);`), + ); + expect(kinds["other"]).toBe("dyn"); +}); + +test("dictionaries keep their native layout without a record flowing in", async () => { + const kinds = await globalKinds( + program(`const table: Record = { a: "x" }; +table[key] = "y"; +const copy: Record = table; +copy[key] = "z"; +console.log(JSON.stringify(table));`), + ); + expect(kinds["table"]).toBe("record"); + expect(kinds["copy"]).toBe("record"); +}); + +test("a record converted to a dictionary that is only read keeps the native layout", async () => { + const kinds = await globalKinds( + program(`const point: { a: string } = { a: "x" }; +const table: Record = point; +console.log(table["a"]);`), + ); + expect(kinds["table"]).toBe("record"); +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7012-dictionary-views-keep-objects.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7012-dictionary-views-keep-objects.ts.json new file mode 100644 index 000000000..6d7944cf7 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7012-dictionary-views-keep-objects.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7012-dictionary-views-keep-objects.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7013-dictionary-view-parameter.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7013-dictionary-view-parameter.ts.json new file mode 100644 index 000000000..41f80c41e --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7013-dictionary-view-parameter.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7013-dictionary-view-parameter.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7012-dictionary-views-keep-objects.ts b/tests/corpus/7012-dictionary-views-keep-objects.ts new file mode 100644 index 000000000..8ede80926 --- /dev/null +++ b/tests/corpus/7012-dictionary-views-keep-objects.ts @@ -0,0 +1,61 @@ +// A record or class instance converted to a string-keyed dictionary is +// the same object: writes through the dictionary reach it, and the +// dictionary is === to it. +const key = process.argv.length > 99 ? "never" : "added"; +type Point = { a: string }; + +// A type-literal record stored straight in a dictionary binding. +const direct: Point = { a: "x" }; +const directTable: Record = direct; +directTable[key] = "y"; +console.log(JSON.stringify(direct), (directTable as unknown) === direct); + +// The same record through a `{}` view. +const viewed: Point = { a: "x" }; +const view: {} = viewed; +const viewTable: Record = view; +viewTable[key] = "y"; +console.log(JSON.stringify(viewed)); + +// A declared index type, and a parameter. +interface Labels { + [name: string]: string; +} +function label(table: Labels, name: string): void { + table[name] = "set"; +} +const passed: Point = { a: "x" }; +label(passed, key); +console.log(JSON.stringify(passed)); + +// A `{}` parameter converted inside the function, called with a record +// and with a dictionary. +function viaView(value: {}): void { + const table: Record = value; + table[key] = "y"; +} +const record: Point = { a: "x" }; +viaView(record); +const dictionary: Record = { a: "x" }; +viaView(dictionary); +console.log(JSON.stringify(record), JSON.stringify(dictionary)); + +// A class instance through a `{}` view: a computed write replaces a method. +class Greeter { + name = "g"; + greet(): string { + return "hello " + this.name; + } +} +const greeter = new Greeter(); +const greeterView: {} = greeter; +const slots: Record = greeterView; +slots[process.argv.length > 99 ? "never" : "greet"] = "replaced"; +slots["name"] = "renamed"; +console.log(typeof greeter.greet, greeter.name, JSON.stringify(greeter)); + +// Deleting through the dictionary removes the key from the object. +const pruned: { a: string; b?: string } = { a: "x", b: "y" }; +const prunedTable: Record = pruned; +delete prunedTable[process.argv.length > 99 ? "never" : "b"]; +console.log(JSON.stringify(pruned), "b" in pruned); diff --git a/tests/corpus/7013-dictionary-view-parameter.ts b/tests/corpus/7013-dictionary-view-parameter.ts new file mode 100644 index 000000000..859f22c23 --- /dev/null +++ b/tests/corpus/7013-dictionary-view-parameter.ts @@ -0,0 +1,11 @@ +// A record passed to a `{}` parameter and converted to a dictionary inside +// the function is the same object: the write through the dictionary +// reaches the caller's record. +const key = process.argv.length > 99 ? "never" : "added"; +function stamp(value: {}): void { + const table: Record = value; + table[key] = "y"; +} +const record: { a: string } = { a: "x" }; +stamp(record); +console.log(JSON.stringify(record), key in record); diff --git a/tests/coverage-fixtures/dictionary-view-copies.ts b/tests/coverage-fixtures/dictionary-view-copies.ts deleted file mode 100644 index bfcfa61f4..000000000 --- a/tests/coverage-fixtures/dictionary-view-copies.ts +++ /dev/null @@ -1,29 +0,0 @@ -// A class instance or record held as `{}`, `object` or `unknown` keeps the -// original object, and converting it to a dictionary type copies it field -// by field. Each site below then writes through or compares the copy. -class Counter { - label = "c"; - add(): number { - return this.label.length; - } -} -const key = process.argv.length > 5 ? "z" : "b"; - -const counter = new Counter(); -const view: {} = counter; -const table: Record = view; -table[key] = "y"; - -const point = { a: "x" }; -const loose: object = point; -const dict = loose as Record; -dict[key] = "y"; - -const held: unknown = { a: "x" }; -const fromUnknown = held as Record; -console.log(fromUnknown === held); - -const empty: {} = {}; -const filled: Record = empty; -filled[key] = "y"; -console.log(JSON.stringify(counter), JSON.stringify(point), JSON.stringify(empty)); diff --git a/tests/coverage-fixtures/no-divergences.ts b/tests/coverage-fixtures/no-divergences.ts index ca76bcc32..70409d319 100644 --- a/tests/coverage-fixtures/no-divergences.ts +++ b/tests/coverage-fixtures/no-divergences.ts @@ -45,3 +45,10 @@ const opaque: {} = counts; const again: Record = opaque; again[counts["a"] ?? "a"] = "y"; console.log(JSON.stringify(counts)); + +// A record stored in a dictionary binding and written through it stays +// the same object. +const named: { first: string } = { first: "a" }; +const byName: Record = named; +byName[named.first] = "b"; +console.log(JSON.stringify(named)); diff --git a/tests/harness/__snapshots__/coverage-dictionary-view-copies.txt b/tests/harness/__snapshots__/coverage-dictionary-view-copies.txt deleted file mode 100644 index 2359a4ffa..000000000 --- a/tests/harness/__snapshots__/coverage-dictionary-view-copies.txt +++ /dev/null @@ -1,21 +0,0 @@ -scriptc coverage tests/coverage-fixtures/dictionary-view-copies.ts - - statements analyzed 18 - compile statically 18 (100%) - in dynamic tier 3 (operate on values typed only at runtime; --explain-dynamic lists them) - - fully static — all analyzed statements compile to native code. - - differs from Node 4 sites (compiles; the native program can behave differently) - ×1 SC6001 storing a 'Counter' in a 'Record<…>' binding copies the object (converting an object held as '{}', 'object' or 'unknown' to a dictionary type compiles as a field copy): the write to 'table[key]' upda… - at tests/coverage-fixtures/dictionary-view-copies.ts:14:39 - hint: keep the value at its own type ('Counter'), or declare the dictionary's values as 'unknown', which keeps the original object - ×1 SC6001 storing a '{}' in a 'Record<…>' binding copies the object (converting an object held as '{}', 'object' or 'unknown' to a dictionary type compiles as a field copy): the write to 'filled[key]' updates … - at tests/coverage-fixtures/dictionary-view-copies.ts:27:40 - hint: keep the value at its own type ('{}'), or declare the dictionary's values as 'unknown', which keeps the original object - ×1 SC6001 asserting a '{ a: string; }' as 'Record<…>' copies the object (converting an object held as '{}', 'object' or 'unknown' to a dictionary type compiles as a field copy): the write to 'dict[key]' update… - at tests/coverage-fixtures/dictionary-view-copies.ts:19:14 - hint: keep the value at its own type ('{ a: string; }'), or declare the dictionary's values as 'unknown', which keeps the original object - ×1 SC6001 asserting a '{ a: string; }' as 'Record<…>' copies the object (converting an object held as '{}', 'object' or 'unknown' to a dictionary type compiles as a field copy): 'fromUnknown === held' compares… - at tests/coverage-fixtures/dictionary-view-copies.ts:23:21 - hint: keep the value at its own type ('{ a: string; }'), or declare the dictionary's values as 'unknown', which keeps the original object \ No newline at end of file diff --git a/tests/harness/corpus-costs/7012-dictionary-views-keep-objects.ts.json b/tests/harness/corpus-costs/7012-dictionary-views-keep-objects.ts.json new file mode 100644 index 000000000..dcf40a9fa --- /dev/null +++ b/tests/harness/corpus-costs/7012-dictionary-views-keep-objects.ts.json @@ -0,0 +1 @@ +{ "plain": 2368, "san": 3286 } diff --git a/tests/harness/corpus-costs/7013-dictionary-view-parameter.ts.json b/tests/harness/corpus-costs/7013-dictionary-view-parameter.ts.json new file mode 100644 index 000000000..911881768 --- /dev/null +++ b/tests/harness/corpus-costs/7013-dictionary-view-parameter.ts.json @@ -0,0 +1 @@ +{ "plain": 1466, "san": 2392 } diff --git a/tests/harness/coverage.test.ts b/tests/harness/coverage.test.ts index f3958c5fe..86d202b46 100644 --- a/tests/harness/coverage.test.ts +++ b/tests/harness/coverage.test.ts @@ -107,20 +107,6 @@ test("record width copies report at every storing site", async () => { ); }); -test("dictionary conversions of checked-dynamic views report copies", async () => { - const { coverage } = analyze(fixture("dictionary-view-copies.ts")); - expect(coverage.diagnostics).toEqual([]); - expect((coverage.divergences ?? []).map((d) => d.code)).toEqual([ - "SC6001", - "SC6001", - "SC6001", - "SC6001", - ]); - await expect(report(fixture("dictionary-view-copies.ts"))).toMatchFileSnapshot( - "__snapshots__/coverage-dictionary-view-copies.txt", - ); -}); - test("ordinary code reports no divergence", () => { const { coverage } = analyze(fixture("no-divergences.ts")); expect(coverage.diagnostics).toEqual([]); From c009b582e2da398c6e229fab779b9354325fae96 Mon Sep 17 00:00:00 2001 From: Luis Alvarez D Date: Thu, 22 Oct 2026 23:05:28 -0500 Subject: [PATCH 018/184] Support split CommonJS require bindings --- .../frontend/lowering/cjs-inline-require.ts | 323 ++++++++++++++++++ .../lowering/expressions/identifiers.ts | 26 ++ .../src/frontend/lowering/lower-modules.ts | 24 +- .../src/frontend/lowering/lower-stmts.ts | 52 +++ .../compiler/src/frontend/lowering/lowerer.ts | 23 +- .../split-cjs-require-binding/main.cjs.json | 12 + .../split-cjs-require-binding/chain-wrap.cjs | 3 + .../split-cjs-require-binding/chain.cjs | 1 + .../corpus/split-cjs-require-binding/lib.cjs | 5 + .../corpus/split-cjs-require-binding/main.cjs | 55 +++ .../split-cjs-require-binding/package.json | 1 + .../reexport-extra.cjs | 6 + .../split-cjs-require-binding/reexport.cjs | 4 + .../split-cjs-require-binding/whole.cjs | 4 + .../split-cjs-require-binding/main.cjs.json | 1 + 15 files changed, 536 insertions(+), 4 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/split-cjs-require-binding/main.cjs.json create mode 100644 tests/corpus/split-cjs-require-binding/chain-wrap.cjs create mode 100644 tests/corpus/split-cjs-require-binding/chain.cjs create mode 100644 tests/corpus/split-cjs-require-binding/lib.cjs create mode 100644 tests/corpus/split-cjs-require-binding/main.cjs create mode 100644 tests/corpus/split-cjs-require-binding/package.json create mode 100644 tests/corpus/split-cjs-require-binding/reexport-extra.cjs create mode 100644 tests/corpus/split-cjs-require-binding/reexport.cjs create mode 100644 tests/corpus/split-cjs-require-binding/whole.cjs create mode 100644 tests/harness/corpus-costs/split-cjs-require-binding/main.cjs.json diff --git a/packages/compiler/src/frontend/lowering/cjs-inline-require.ts b/packages/compiler/src/frontend/lowering/cjs-inline-require.ts index 2b28c6533..eb05550c0 100644 --- a/packages/compiler/src/frontend/lowering/cjs-inline-require.ts +++ b/packages/compiler/src/frontend/lowering/cjs-inline-require.ts @@ -28,6 +28,7 @@ import { isModuleExportsAccess, locOf, npmStaticDepSf7, + requireBindingEarlyUse7, requireSpecOf, resolveImport, } from "../program.js"; @@ -43,6 +44,328 @@ export interface InlineRequire { dep: ts.SourceFile; } +/** A top-level `var m; m = require("./x")` binding. Node evaluates the + * require at the assignment; the binding is alias plumbing exactly like + * `const m = require("./x")` for every later read. */ +export interface SplitRequireBinding { + decl: ts.VariableDeclaration; + assign: ts.BinaryExpression; + call: ts.CallExpression; + spec: string; + dep: ts.SourceFile; +} + +function symbolOfIdentifierValue(lowerer: Lowerer, ident: ts.Identifier): ts.Symbol | undefined { + const parent = ident.parent; + // A destructuring shorthand target resolves to the property symbol at + // the name, but reads/writes the variable through its value symbol. + if (ts.isShorthandPropertyAssignment(parent) && parent.name === ident) { + return ( + lowerer.checker.getShorthandAssignmentValueSymbol(parent) ?? + lowerer.checker.getSymbolAtLocation(ident) + ); + } + return lowerer.checker.getSymbolAtLocation(ident); +} + +function destructuringContainerOf(node: ts.Node): ts.Node | null { + const parent = node.parent; + if (ts.isShorthandPropertyAssignment(parent) && parent.name === node) return parent; + if (ts.isPropertyAssignment(parent) && parent.initializer === node) return parent; + if (ts.isSpreadElement(parent) && parent.expression === node) return parent; + if (ts.isSpreadAssignment(parent) && parent.expression === node) return parent; + return null; +} + +function writeTargetOf(node: ts.Node): ts.Node | null { + let target = node; + for (;;) { + const parent = target.parent; + if ( + ts.isParenthesizedExpression(parent) || + ts.isArrayLiteralExpression(parent) || + ts.isObjectLiteralExpression(parent) + ) { + target = parent; + continue; + } + const container = destructuringContainerOf(target); + if (container === null) break; + target = container; + } + const parent = target.parent; + if ( + ts.isBinaryExpression(parent) && + parent.left === target && + ts.isAssignmentOperator(parent.operatorToken.kind) + ) + return parent; + if ( + (ts.isPrefixUnaryExpression(parent) || ts.isPostfixUnaryExpression(parent)) && + parent.operand === target && + (parent.operator === ts.SyntaxKind.PlusPlusToken || + parent.operator === ts.SyntaxKind.MinusMinusToken) + ) + return parent; + if ((ts.isForInStatement(parent) || ts.isForOfStatement(parent)) && parent.initializer === target) + return parent; + return null; +} + +/** The split form of a top-level require binding: `var m; m = require("./x")`. + * Exactly one write, at a top-level `m = require("")` statement. + * Null for the ordinary `const m = require("./x")` declaration and for + * every form that needs runtime module selection. */ +export function splitRequireBindingOf( + lowerer: Lowerer, + decl: ts.VariableDeclaration, +): SplitRequireBinding | null { + // Cheap shape gate before the cache: ordinary declarations never enter the + // per-Lowerer map. + if (decl.initializer !== undefined || !ts.isIdentifier(decl.name)) return null; + const list = decl.parent as ts.VariableDeclarationList; + if ((list.flags & ts.NodeFlags.Const) !== 0) return null; + const stmt = list.parent; + if (!ts.isVariableStatement(stmt) || !ts.isSourceFile(stmt.parent)) return null; + const cached = lowerer.cjsSplitRequireBindings.get(decl); + if (cached !== undefined) return cached; + const result = computeSplitRequireBinding(lowerer, decl); + lowerer.cjsSplitRequireBindings.set(decl, result); + return result; +} + +function computeSplitRequireBinding( + lowerer: Lowerer, + decl: ts.VariableDeclaration, +): SplitRequireBinding | null { + const sf = ts.getSourceFileOfNode(decl); + if (!isCjsJsFile(sf, lowerer.program)) return null; + if (!sf.text.includes("require")) return null; + const symbol = lowerer.checker.getSymbolAtLocation(decl.name); + if (symbol === undefined) return null; + // Merged declarations make the binding's initialization order ambiguous; + // keep ordinary storage instead of guessing. + let declarations = 0; + for (const other of symbol.declarations) { + if (!ts.isVariableDeclaration(other) || ++declarations > 1) return null; + } + if (declarations !== 1) return null; + + // Find the one top-level `m = ...` statement first: most uninitialized + // variables never reach that shape, so they avoid the full-file write scan. + let assign: ts.BinaryExpression | null = null; + for (const stmt of sf.statements.nodes) { + if (!ts.isExpressionStatement(stmt)) continue; + const expr = stmt.expression; + if ( + !ts.isBinaryExpression(expr) || + expr.operatorToken.kind !== ts.SyntaxKind.EqualsToken || + !ts.isIdentifier(expr.left) || + expr.left.text !== decl.name.text || + lowerer.checker.getSymbolAtLocation(expr.left) !== symbol + ) + continue; + if (assign !== null) return null; + assign = expr; + } + if (assign === null) return null; + + const req = inlineProgramRequireOf(lowerer, assign.right); + if (req === null) return null; + + // Any other write makes the value depend on runtime control flow. + let writes = 0; + ts.walkPreorder(sf, (node) => { + if (!ts.isIdentifier(node) || node.text !== decl.name.text) return; + if (symbolOfIdentifierValue(lowerer, node) !== symbol) return; + if (writeTargetOf(node) === null) return; + if (++writes > 1) return "stop"; + }); + if (writes !== 1) return null; + + return { decl, assign, call: req.call, spec: req.spec, dep: req.dep }; +} + +/** The split binding an identifier names, or null. */ +export function splitRequireBindingOfIdent( + lowerer: Lowerer, + ident: ts.Identifier, +): SplitRequireBinding | null { + const symbol = symbolOfIdentifierValue(lowerer, ident); + return symbol === undefined ? null : splitBindingOfSymbol(lowerer, symbol); +} + +/** The exact top-level assignment statement that initializes `expr`, or null. */ +export function splitRequireAssignmentOf( + lowerer: Lowerer, + expr: ts.Expression, +): SplitRequireBinding | null { + if ( + !ts.isBinaryExpression(expr) || + expr.operatorToken.kind !== ts.SyntaxKind.EqualsToken || + !ts.isIdentifier(expr.left) + ) + return null; + const bindingOfIdent = splitRequireBindingOfIdent(lowerer, expr.left); + return bindingOfIdent !== null && bindingOfIdent.assign === expr ? bindingOfIdent : null; +} + +/** The whole-export symbol a module's `module.exports = ` names, + * or null when the module is a table. */ +export function wholeExportSymbolOfModule(lowerer: Lowerer, dep: ts.SourceFile): ts.Symbol | null { + const cached = lowerer.cjsWholeExportSymbols.get(dep); + if (cached !== undefined) return cached; + if (cjsNamespaceModule(lowerer, dep)) { + lowerer.cjsWholeExportSymbols.set(dep, null); + return null; + } + const moduleSymbol = lowerer.checker.getSymbolAtLocation(dep); + let symbol = moduleSymbol?.getExports()?.get("export=" as ts.__String); + if (symbol === undefined) { + lowerer.cjsWholeExportSymbols.set(dep, null); + return null; + } + if (symbol.flags & ts.SymbolFlags.Alias) symbol = lowerer.checker.getAliasedSymbol(symbol); + lowerer.cjsWholeExportSymbols.set(dep, symbol); + return symbol; +} + +function splitBindingOfSymbol(lowerer: Lowerer, symbol: ts.Symbol): SplitRequireBinding | null { + const decl = symbol.declarations.find(ts.isVariableDeclaration); + return decl === undefined ? null : splitRequireBindingOf(lowerer, decl); +} + +/** The target whole-export symbol behind a split-binding symbol. */ +function splitWholeExportSymbolOfSymbol(lowerer: Lowerer, symbol: ts.Symbol): ts.Symbol | null { + const binding = splitBindingOfSymbol(lowerer, symbol); + return binding === null ? null : wholeExportSymbolOfModule(lowerer, binding.dep); +} + +/** True when a CommonJS module has a live member export (`exports.x` or + * `module.exports.x`) after all `module.exports =` replacements. Those + * members need a mutable whole-export root. */ +export function cjsHasLiveMemberExports(lowerer: Lowerer, sf: ts.SourceFile): boolean { + const cached = lowerer.cjsLiveMemberExports.get(sf); + if (cached !== undefined) return cached; + const stmts = sf.statements.nodes; + let lastTable = -1; + for (let i = 0; i < stmts.length; i++) { + if (cjsExportAssignmentOf(stmts[i]!)?.kind === "table") lastTable = i; + } + let live = false; + // Only members attached after the last replacement survive in Node. + if (lastTable >= 0) { + for (let i = lastTable + 1; i < stmts.length; i++) { + const stmt = stmts[i]!; + if (cjsExportAssignmentOf(stmt)?.kind === "member" && cjsExportDiscardReason(stmt) === null) { + live = true; + break; + } + } + } + lowerer.cjsLiveMemberExports.set(sf, live); + return live; +} + +/** The wrapper's export= symbol when a split binding is re-exported whole + * (`module.exports = m`) and that assignment owns storage for later member + * writes. Importers must read the wrapper's storage, not the split + * binding's underlying value. */ +function splitWholeExportStorageSymbolOf(lowerer: Lowerer, symbol: ts.Symbol): ts.Symbol | null { + const binding = splitBindingOfSymbol(lowerer, symbol); + if (binding === null) return null; + if (wholeExportSymbolOfModule(lowerer, binding.dep) === null) return null; + const sf = ts.getSourceFileOfNode(binding.decl); + if (!cjsHasLiveMemberExports(lowerer, sf)) return null; + const whole = cjsWholeExportValueOf(sf); + if (whole === null) return null; + let rhs: ts.Expression = whole.rhs; + while (ts.isParenthesizedExpression(rhs)) rhs = rhs.expression; + if (!ts.isIdentifier(rhs) || lowerer.checker.getSymbolAtLocation(rhs) !== symbol) return null; + const moduleSymbol = lowerer.checker.getSymbolAtLocation(sf); + let exportSymbol = moduleSymbol?.getExports()?.get("export=" as ts.__String); + if (exportSymbol === undefined) return null; + if (exportSymbol.flags & ts.SymbolFlags.Alias) + exportSymbol = lowerer.checker.getAliasedSymbol(exportSymbol); + return exportSymbol; +} + +/** The source file whose `module.exports =` assignment declares `symbol`. */ +function moduleExportSourceFileOf(symbol: ts.Symbol): ts.SourceFile | null { + for (const decl of symbol.declarations) { + if ( + ts.isBinaryExpression(decl) && + isModuleExportsAccess(decl.left) && + ts.isExpressionStatement(decl.parent) && + ts.isSourceFile(decl.parent.parent) + ) + return decl.getSourceFile(); + } + return null; +} + +/** One hop from a wrapper's export= symbol to the value it re-exports + * (`module.exports = require(...)` or `= `), or null. A + * wrapper with live members owns storage for them, so it is not followed. */ +function wholeReexportTargetOf(lowerer: Lowerer, symbol: ts.Symbol): ts.Symbol | null { + const sf = moduleExportSourceFileOf(symbol); + if (sf === null) return null; + // A wrapper with live members owns storage for them; importers must keep + // reading that root rather than the inner value. + if (cjsHasLiveMemberExports(lowerer, sf)) return null; + const whole = cjsWholeExportValueOf(sf); + if (whole === null) return null; + const inner = inlineProgramRequireOf(lowerer, whole.rhs); + if (inner !== null) return wholeExportSymbolOfModule(lowerer, inner.dep); + let rhs: ts.Expression = whole.rhs; + while (ts.isParenthesizedExpression(rhs)) rhs = rhs.expression; + if (!ts.isIdentifier(rhs)) return null; + const rhsSymbol = lowerer.checker.getSymbolAtLocation(rhs); + return rhsSymbol === undefined ? null : splitWholeExportSymbolOfSymbol(lowerer, rhsSymbol); +} + +/** Follow split bindings and whole re-exports to the ultimate value symbol. */ +function followSplitWholeTarget(lowerer: Lowerer, candidate: ts.Symbol | null): ts.Symbol | null { + if (candidate === null) return null; + let symbol = candidate; + const seen = new Set(); + for (let depth = 0; depth < 32 && !seen.has(symbol); depth++) { + seen.add(symbol); + const next = + splitWholeExportSymbolOfSymbol(lowerer, symbol) ?? wholeReexportTargetOf(lowerer, symbol); + if (next === null) break; + symbol = next; + } + return symbol; +} + +/** The target whole-export symbol behind `module.exports = `, + * letting importers of that wrapper resolve to the real value symbol. */ +export function splitWholeExportAliasTargetOf( + lowerer: Lowerer, + symbol: ts.Symbol, +): ts.Symbol | null { + const storage = splitWholeExportStorageSymbolOf(lowerer, symbol); + if (storage !== null) return followSplitWholeTarget(lowerer, storage); + const direct = followSplitWholeTarget(lowerer, splitWholeExportSymbolOfSymbol(lowerer, symbol)); + if (direct !== null) return direct; + return followSplitWholeTarget(lowerer, wholeReexportTargetOf(lowerer, symbol)); +} + +/** The first read of a split binding that code before its assignment can + * reach, using the same reachability model as ordinary require bindings. */ +export function splitRequireEarlyReadOf( + lowerer: Lowerer, + binding: SplitRequireBinding, +): ts.Identifier | null { + const sf = ts.getSourceFileOfNode(binding.decl); + const stmt = binding.assign.parent; + // `binding` was recognized only from a top-level `m = require(...)` + // statement, so the statement index is always present. + const index = sf.statements.nodes.indexOf(stmt as ts.ExpressionStatement); + return requireBindingEarlyUse7(lowerer.program, sf, index, binding.decl, []); +} + /** A `require("")` call of the CommonJS global in a CommonJS JS * file whose target is a compiled program module (relative, aliased, or an * opted-in --npm-static package). Builtins, JSON documents and diff --git a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts index b5f99d682..d032cb263 100644 --- a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts +++ b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts @@ -22,6 +22,11 @@ import { } from "../../../ir/ir.js"; import { lowerNodeModuleIdentifier } from "../lower-node-module.js"; import { lowerCjsExportsObjectValue } from "../cjs-exports-object.js"; +import { + splitRequireBindingOfIdent, + wholeExportSymbolOfModule, + wholeExportValue, +} from "../cjs-inline-require.js"; import { captureContextArguments } from "../function-context.js"; import { argumentsOwner, typedArgumentsCandidate } from "../typed-arguments.js"; import { @@ -920,6 +925,27 @@ export function lowerIdentifierExpr(lowerer: Lowerer, expr: ts.Identifier, loc: // Client`): the member holds that binding's value. const memberAlias = cjsMemberAliasValueOf(lowerer, expr); if (memberAlias !== null) return lowerer.lowerExpr(memberAlias); + // `var m; m = require("./x")` as a bare value: a whole-export target is + // the exported value; a table target has no first-class module object. + { + const split = splitRequireBindingOfIdent(lowerer, expr); + if (split !== null) { + if (wholeExportSymbolOfModule(lowerer, split.dep) !== null) { + const value = wholeExportValue(lowerer, split.dep, expr); + if (value !== null) return value; + lowerer.unsupported( + "SC1090", + expr, + `the whole export of '${split.spec}' as a value (its 'module.exports =' value has no static representation)`, + ); + } + lowerer.unsupported( + "SC1090", + expr, + `the exports object of '${split.spec}' as a value (its exports compile to a static table: read require(${JSON.stringify(split.spec)}).member directly)`, + ); + } + } // Preflight guarantees no unresolved identifiers; anything else here // is a blocked declaration's binding (the SC2004 cascade) or a // binding form we don't model yet. diff --git a/packages/compiler/src/frontend/lowering/lower-modules.ts b/packages/compiler/src/frontend/lowering/lower-modules.ts index 6ca21ddaa..a86decf4c 100644 --- a/packages/compiler/src/frontend/lowering/lower-modules.ts +++ b/packages/compiler/src/frontend/lowering/lower-modules.ts @@ -42,10 +42,14 @@ import { } from "./cycle-initialization.js"; import { markRequireEarlyBindings } from "./require-initialization.js"; import { + cjsHasLiveMemberExports, cjsTableMethodEntry, inlineProgramRequireOf, nestedCjsExportAssignmentsOf, requireNestedPatternsOf, + splitRequireBindingOf, + splitRequireBindingOfIdent, + wholeExportSymbolOfModule, } from "./cjs-inline-require.js"; import { isDeadTscHelperDecl, tscInteropBindingOfDecl } from "./cjs-tsc-interop.js"; import { collectJsonRequires, isJsonRequireDecl } from "./json-require.js"; @@ -1588,14 +1592,26 @@ export function collectGlobals( // require() of a program module is not: the exporter re-exports // that module's exports, which requirers resolve through the // checker's alias (the statement lowers to the dep's %init). + // A split binding re-exported whole with later member writes needs + // a mutable root: `module.exports = m; module.exports.extra = 1` + // extends the SAME object Node exposes. Without later members the + // binder is pure alias plumbing (no storage). + const splitWholeBinding = ts.isIdentifier(rhs) + ? splitRequireBindingOfIdent(lowerer, rhs) + : null; + const hasMemberExports = cjsHasLiveMemberExports(lowerer, sf); + const splitWhole = + splitWholeBinding !== null && + wholeExportSymbolOfModule(lowerer, splitWholeBinding.dep) !== null && + hasMemberExports; const factoryValued = - !ts.isIdentifier(rhs) && + (!ts.isIdentifier(rhs) || splitWhole) && !ts.isClassExpression(rhs) && !ts.isObjectLiteralExpression(rhs) && !( requireSpecOf(rhs) !== null && inlineProgramRequireOf(lowerer, rhs) !== null && - !sf.statements.nodes.some((s) => cjsExportAssignmentOf(s)?.kind === "member") + !hasMemberExports ) && cjsExportTargetLiteral(rhs, sf) === null; if (cjsScalarLiteral(rhs) || fnValued || factoryValued) { @@ -1608,6 +1624,9 @@ export function collectGlobals( // A returned object remains the live CommonJS root; later // exports.member writes must update that same open object. if (factoryValued && t?.kind === "record" && lowerer.dynConvertible(t)) t = DYN; + // A split whole-export root is extended by later member writes; + // keep it checked-dynamic so those writes reach the same object. + if (splitWhole && t && t.kind !== "dyn") t = DYN; if (t && t.kind !== "void") { const g: IrGlobal = { id: `%g.${tag}exports`, @@ -2699,6 +2718,7 @@ function noteVarGlobalEntryInit(lowerer: Lowerer, sf: ts.SourceFile, g: IrGlobal * plumbing (lowerVarDeclarator): no storage, the reads resolve through to * the module. JSON documents and native FFI requires keep their storage. */ function requireAliasDecl(lowerer: Lowerer, decl: ts.VariableDeclaration): boolean { + if (splitRequireBindingOf(lowerer, decl) !== null) return true; return ( decl.initializer !== undefined && requireSpecOf(decl.initializer) !== null && diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index b04ba0b09..d10d3b9d2 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -36,6 +36,11 @@ import { cjsTableMethodEntry, inlineProgramRequireOf, requireNestedPatternsOf, + splitRequireAssignmentOf, + splitRequireBindingOf, + splitRequireBindingOfIdent, + splitRequireEarlyReadOf, + wholeExportSymbolOfModule, } from "./cjs-inline-require.js"; import { cjsExportMemberWriteGlobalOf, @@ -1811,6 +1816,23 @@ function lowerVarDeclarator( isConst: boolean, isLet: boolean, ): IrStmt[] { + // `var m; m = require("./x")`: the declaration is alias plumbing; the + // assignment owns the module's run-once %init (cjs-inline-require.ts). + // A read that code before the assignment can reach keeps Node's hoisted + // `undefined` divergence visible at build time, like a var require + // declaration below runnable code. + const split = splitRequireBindingOf(lowerer, decl); + if (split !== null) { + const early = splitRequireEarlyReadOf(lowerer, split); + if (early !== null) { + lowerer.unsupported( + "SC1013", + early, + `var require() binding '${(decl.name as ts.Identifier).text}' below code that can reach it (Node reads undefined before the require runs, which is not modeled — assign the require above the code that can run first)`, + ); + } + return []; + } // The canonical ESM bridge to a compiled program module. Like a // top-level CommonJS require declaration, the binding is alias // plumbing while the dependency evaluates at this statement's @@ -7015,6 +7037,26 @@ function lowerCjsSingleValueExport( return lowerer.requireInitStmt(req.spec, req.call) ?? { kind: "block", body: [], loc }; } if (ts.isIdentifier(rhs)) { + const split = splitRequireBindingOfIdent(lowerer, rhs); + if (split !== null) { + if (wholeExportSymbolOfModule(lowerer, split.dep) === null) { + lowerer.unsupported( + "SC1090", + rhs, + `the exports object of '${split.spec}' as a value (its exports compile to a static table: read require(${JSON.stringify(split.spec)}).member directly)`, + ); + } + const root = lowerer.globalsByDeclNode.get(expr); + if (root !== undefined) { + return { + kind: "assign", + localId: root.id, + value: lowerer.lowerExprExpecting(rhs, root.type), + loc: locOf(rhs), + }; + } + return { kind: "block", body: [], loc }; + } const sym = lowerer.resolveValueSymbol(rhs); if (!sym) return null; if ( @@ -7924,6 +7966,16 @@ export function lowerExprStatement(lowerer: Lowerer, expr: ts.Expression): IrStm if (!ts.isIdentifier(expr.left)) { lowerer.unsupported("SC1090", expr.left, "assignment to non-variables"); } + const splitAssign = splitRequireAssignmentOf(lowerer, expr); + if (splitAssign !== null) { + return ( + lowerer.requireInitStmt(splitAssign.spec, splitAssign.call) ?? { + kind: "block", + body: [], + loc: locOf(expr), + } + ); + } const target = lowerer.resolveWritable(expr.left); if (!target) lowerer.rejectUnresolved( diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index c4d24cef1..f1e7606f2 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -331,10 +331,13 @@ import { } from "./lower-modules.js"; import { prepareCjsModuleGraph, usesCjsTopLevelThis } from "./lower-node-module.js"; import { + type SplitRequireBinding, cjsNamespaceModule, cjsWholeExportValueOf, inlineNamespaceRequireOf, inlineProgramRequireOf, + splitRequireBindingOf, + splitWholeExportAliasTargetOf, } from "./cjs-inline-require.js"; import { tscInteropModuleOf } from "./cjs-tsc-interop.js"; import { usesCjsExportsIdentity } from "./cjs-exports-object.js"; @@ -2176,6 +2179,14 @@ export class Lowerer { /** Per CommonJS file using top-level `this`: the global holding the * object that stands for its original exports (lowerCjsTopLevelThis). */ readonly cjsExportsThis = new Map(); + /** `var m; m = require("./x")` alias bindings (cjs-inline-require.ts). + * Null entries cache declarations that keep ordinary storage. */ + readonly cjsSplitRequireBindings = new Map(); + /** Per-module whole-export symbol (cjs-inline-require.ts). Null means the + * module's exports are a table. */ + readonly cjsWholeExportSymbols = new Map(); + /** Per-module live member-export classification (cjs-inline-require.ts). */ + readonly cjsLiveMemberExports = new Map(); /** Inline program-module require calls whose %init the enclosing member * read or member call is already sequencing (cjs-inline-require.ts). */ readonly inlineRequireInitDone = new Set(); @@ -3711,6 +3722,11 @@ export class Lowerer { // registry (globals, fn signatures, classes) applies unchanged. const cjsValue = this.cjsExportValueSymbol(symbol); if (cjsValue) symbol = cjsValue; + // `var m; m = require("./x"); module.exports = m` over a whole-export + // target: TS types the wrapper's export= as `any`, so reconnect it to + // the target's value symbol explicitly. + const splitAlias = splitWholeExportAliasTargetOf(this, symbol); + if (splitAlias) symbol = splitAlias; this.flushDeferred(symbol); return symbol; } @@ -4104,8 +4120,8 @@ export class Lowerer { const decl = decls.find(ts.isImportClause) ?? decls[0]; if (!decl) return null; if (ts.isImportClause(decl)) return this.cjsDefaultImportDepOf(decl); - if (!ts.isVariableDeclaration(decl) || !ts.isIdentifier(decl.name) || !decl.initializer) - return null; + if (!ts.isVariableDeclaration(decl) || !ts.isIdentifier(decl.name)) return null; + if (decl.initializer === undefined) return splitRequireBindingOf(this, decl)?.dep ?? null; { const interop = tscInteropModuleOf(this, expr); if (interop !== null) return cjsNamespaceModule(this, interop.dep) ? interop.dep : null; @@ -4210,6 +4226,9 @@ export class Lowerer { // a binding (cjs-inline-require.ts sequences the module's %init). if (!ts.isIdentifier(expr)) return this.cjsModuleRefDepOf(expr) !== null; const sym = this.checker.getSymbolAtLocation(expr); + const splitDecl = sym?.declarations.find(ts.isVariableDeclaration); + const split = splitDecl === undefined ? null : splitRequireBindingOf(this, splitDecl); + if (split !== null) return cjsNamespaceModule(this, split.dep); if (this.localModuleBindingDepOf(expr) === null) return false; if (this.createRequireWholeExportSymbolOf(expr) !== null) return false; // SINGLE-VALUE exporters (`module.exports = Countdown` / `= double` / diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/split-cjs-require-binding/main.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/split-cjs-require-binding/main.cjs.json new file mode 100644 index 000000000..c01617f80 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/split-cjs-require-binding/main.cjs.json @@ -0,0 +1,12 @@ +{ + "order": [ + "/tests/corpus/split-cjs-require-binding/whole.cjs", + "/tests/corpus/split-cjs-require-binding/reexport.cjs", + "/tests/corpus/split-cjs-require-binding/reexport-extra.cjs", + "/tests/corpus/split-cjs-require-binding/lib.cjs", + "/tests/corpus/split-cjs-require-binding/chain.cjs", + "/tests/corpus/split-cjs-require-binding/chain-wrap.cjs", + "/tests/corpus/split-cjs-require-binding/main.cjs" + ], + "diags": [] +} diff --git a/tests/corpus/split-cjs-require-binding/chain-wrap.cjs b/tests/corpus/split-cjs-require-binding/chain-wrap.cjs new file mode 100644 index 000000000..435dcaeab --- /dev/null +++ b/tests/corpus/split-cjs-require-binding/chain-wrap.cjs @@ -0,0 +1,3 @@ +var chained; +chained = require("./chain.cjs"); +module.exports = chained; diff --git a/tests/corpus/split-cjs-require-binding/chain.cjs b/tests/corpus/split-cjs-require-binding/chain.cjs new file mode 100644 index 000000000..a279f456e --- /dev/null +++ b/tests/corpus/split-cjs-require-binding/chain.cjs @@ -0,0 +1 @@ +module.exports = require("./whole.cjs"); diff --git a/tests/corpus/split-cjs-require-binding/lib.cjs b/tests/corpus/split-cjs-require-binding/lib.cjs new file mode 100644 index 000000000..b318eb85c --- /dev/null +++ b/tests/corpus/split-cjs-require-binding/lib.cjs @@ -0,0 +1,5 @@ +console.log("lib: init"); +exports.n = 7; +exports.f = function (x) { + return x * 2; +}; diff --git a/tests/corpus/split-cjs-require-binding/main.cjs b/tests/corpus/split-cjs-require-binding/main.cjs new file mode 100644 index 000000000..974421bd3 --- /dev/null +++ b/tests/corpus/split-cjs-require-binding/main.cjs @@ -0,0 +1,55 @@ +console.log("main: start"); +var lib, whole; +lib = require("./lib.cjs"); +whole = require("./whole.cjs"); +console.log("main: after", lib.n, lib.f(3), whole(4)); +exports.fromLib = lib.f; +exports.whole = whole; +console.log("main: exported", exports.fromLib(5), whole(6)); +console.log("main: identity", exports.fromLib === lib.f, exports.whole === whole); +const re = require("./reexport.cjs"); +console.log("main: reexport", re(7)); +let later; +later = require("./lib.cjs"); +console.log("main: let", later.n); +const rx = require("./reexport-extra.cjs"); +console.log("main: reexport extra", rx(8), rx.extra); +const splitObject = { whole }; +console.log("main: shorthand value", splitObject.whole(10)); +var libAgain; +libAgain = require("./lib.cjs"); +const { n: splitN, f: splitF } = libAgain; +console.log("main: destructured split", splitN, splitF(4)); +var chained; +chained = require("./chain-wrap.cjs"); +console.log("main: chained", chained(11)); + +// Any later write, including a destructuring target, keeps ordinary storage. +var shorthand; +shorthand = require("./whole.cjs"); +({ shorthand } = { shorthand: "shorthand" }); +console.log("main: shorthand write", shorthand); +var renamed; +renamed = require("./whole.cjs"); +({ value: renamed } = { value: "renamed" }); +console.log("main: renamed write", renamed); +var nested; +nested = require("./whole.cjs"); +({ a: { nested } } = { a: { nested: "nested" } }); +console.log("main: nested write", nested); +var rest; +rest = require("./whole.cjs"); +({ ...rest } = { a: 1, b: 2 }); +console.log("main: rest write", rest.a, rest.b); +var array; +array = require("./whole.cjs"); +[array] = ["array"]; +console.log("main: array write", array); +var arrayRest; +arrayRest = require("./whole.cjs"); +[...arrayRest] = [1, 2]; +console.log("main: array-rest write", arrayRest[0], arrayRest[1]); +var each; +each = require("./whole.cjs"); +for ({ each } of [{ each: "each" }]) {} +console.log("main: for-of write", each); diff --git a/tests/corpus/split-cjs-require-binding/package.json b/tests/corpus/split-cjs-require-binding/package.json new file mode 100644 index 000000000..a3c15a7a6 --- /dev/null +++ b/tests/corpus/split-cjs-require-binding/package.json @@ -0,0 +1 @@ +{ "type": "commonjs" } diff --git a/tests/corpus/split-cjs-require-binding/reexport-extra.cjs b/tests/corpus/split-cjs-require-binding/reexport-extra.cjs new file mode 100644 index 000000000..584de98fe --- /dev/null +++ b/tests/corpus/split-cjs-require-binding/reexport-extra.cjs @@ -0,0 +1,6 @@ +var whole; +whole = require("./whole.cjs"); +console.log("extra: before", whole.extra, whole(9)); +module.exports = whole; +module.exports.extra = 1; +console.log("extra: after", whole.extra, module.exports.extra); diff --git a/tests/corpus/split-cjs-require-binding/reexport.cjs b/tests/corpus/split-cjs-require-binding/reexport.cjs new file mode 100644 index 000000000..9a931bdef --- /dev/null +++ b/tests/corpus/split-cjs-require-binding/reexport.cjs @@ -0,0 +1,4 @@ +console.log("reexport: init"); +var whole; +whole = require("./whole.cjs"); +module.exports = whole; diff --git a/tests/corpus/split-cjs-require-binding/whole.cjs b/tests/corpus/split-cjs-require-binding/whole.cjs new file mode 100644 index 000000000..31b476c4e --- /dev/null +++ b/tests/corpus/split-cjs-require-binding/whole.cjs @@ -0,0 +1,4 @@ +console.log("whole: init"); +module.exports = function whole(x) { + return x + 1; +}; diff --git a/tests/harness/corpus-costs/split-cjs-require-binding/main.cjs.json b/tests/harness/corpus-costs/split-cjs-require-binding/main.cjs.json new file mode 100644 index 000000000..9d570f700 --- /dev/null +++ b/tests/harness/corpus-costs/split-cjs-require-binding/main.cjs.json @@ -0,0 +1 @@ +{ "plain": 1500, "san": 12000 } From da289bb593ef27c17e45c0121b63c61fdee1dbdb Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 23:13:49 -0500 Subject: [PATCH 019/184] Support writev on Transform and PassThrough, and runtime stream option values --- .../builtin-apis/streams-and-readline.mdx | 4 +- packages/cli/test/native-link-info.test.ts | 4 +- .../src/backend/llvm/expr-stream-callbacks.ts | 11 + .../compiler/src/backend/llvm/lib-shared.ts | 2 + packages/compiler/src/backend/runtime-abi.ts | 4 +- .../src/backend/runtime-pack-targets.test.ts | 2 +- .../compiler/src/backend/runtime-pack.test.ts | 2 +- .../src/frontend/lowering/lower-classes.ts | 6 +- .../src/frontend/lowering/lower-stream.ts | 230 ++++++++++++++++-- packages/compiler/src/ir/builtin-effects.ts | 1 + .../compiler/src/ir/builtin-signatures.ts | 5 + packages/compiler/src/ir/runtime-features.ts | 2 + packages/compiler/src/ir/validate.ts | 2 + .../test/native-codegen-integration.test.ts | 4 +- .../corpus/7460-transform-writev.mjs.json | 6 + .../corpus/7461-transform-writev.cjs.json | 6 + .../corpus/7462-transform-writev.ts.json | 6 + .../corpus/7463-stream-option-values.ts.json | 6 + .../corpus/7464-stream-option-values.mjs.json | 6 + .../7465-transform-callback-values.ts.json | 6 + .../runtime-darwin-arm64/scripts/build.mjs | 2 +- .../runtime-pack-common/scripts/build.mjs | 2 +- packages/runtime/src/scr_console.c | 2 +- packages/runtime/src/scr_core.h | 2 +- packages/runtime/src/scr_dyn_builtin_stream.c | 9 +- packages/runtime/src/scr_stream.c | 77 +++++- packages/runtime/src/scr_stream.h | 8 + tests/corpus/7460-transform-writev.mjs | 33 +++ tests/corpus/7461-transform-writev.cjs | 33 +++ tests/corpus/7462-transform-writev.ts | 109 +++++++++ tests/corpus/7463-stream-option-values.ts | 58 +++++ tests/corpus/7464-stream-option-values.mjs | 28 +++ .../corpus/7465-transform-callback-values.ts | 41 ++++ .../7460-transform-writev.mjs.json | 1 + .../7461-transform-writev.cjs.json | 1 + .../7462-transform-writev.ts.json | 1 + .../7463-stream-option-values.ts.json | 1 + .../7464-stream-option-values.mjs.json | 1 + .../7465-transform-callback-values.ts.json | 1 + .../self-hosting-native-toolchain.test.ts | 2 +- 40 files changed, 670 insertions(+), 57 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7460-transform-writev.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7461-transform-writev.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7462-transform-writev.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7463-stream-option-values.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7464-stream-option-values.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7465-transform-callback-values.ts.json create mode 100644 tests/corpus/7460-transform-writev.mjs create mode 100644 tests/corpus/7461-transform-writev.cjs create mode 100644 tests/corpus/7462-transform-writev.ts create mode 100644 tests/corpus/7463-stream-option-values.ts create mode 100644 tests/corpus/7464-stream-option-values.mjs create mode 100644 tests/corpus/7465-transform-callback-values.ts create mode 100644 tests/harness/corpus-costs/7460-transform-writev.mjs.json create mode 100644 tests/harness/corpus-costs/7461-transform-writev.cjs.json create mode 100644 tests/harness/corpus-costs/7462-transform-writev.ts.json create mode 100644 tests/harness/corpus-costs/7463-stream-option-values.ts.json create mode 100644 tests/harness/corpus-costs/7464-stream-option-values.mjs.json create mode 100644 tests/harness/corpus-costs/7465-transform-callback-values.ts.json diff --git a/docs/content/limitations/builtin-apis/streams-and-readline.mdx b/docs/content/limitations/builtin-apis/streams-and-readline.mdx index fe6bb2dfb..b88ead1f4 100644 --- a/docs/content/limitations/builtin-apis/streams-and-readline.mdx +++ b/docs/content/limitations/builtin-apis/streams-and-readline.mdx @@ -32,9 +32,9 @@ `readable.compose(stage, { signal })` is `stream.compose(readable, stage)` with the signal added; the options are an object literal whose `signal` is a variable or a property read. `pipeline` from `node:stream/promises` accepts a trailing `{ signal, end }` object, with or without function stages, and `finished` accepts `readable`, `writable`, `error`, `cleanup`, and `signal` options, with boolean literals for the flags. Stream constructors accept a `signal` option in an options object literal, and `stream.addAbortSignal` accepts any stream. - The `construct` option and `_construct` method, `decodeStrings`, and `defaultEncoding` work on every stream class, and the `writev` option and `_writev` method on Writable and Duplex; `writev` on a Transform or PassThrough, assigning `_construct` after construction, and a non-literal `decodeStrings` or `defaultEncoding` in an options object literal are unsupported. + The `construct` option and `_construct` method, `decodeStrings`, and `defaultEncoding` work on every stream class, and the `writev` option and `_writev` method on every class with a writable side. + In typed code, a `writev` entry's `encoding` declared as a string receives `""` where Node passes `undefined` or `null` (an object-mode write). `setDefaultEncoding()` takes any encoding Node knows. - A `transform` or `flush` option callback whose completion callback types `data` as `any`, as @types/node's `TransformCallback` does, may pass a Buffer, a string, or nothing; any other value throws a TypeError where the callback is called, instead of Node's `ERR_INVALID_ARG_TYPE` stream error. `node:readline` and `node:readline/promises` (also reached as `readline.promises`) interfaces, from `createInterface(options)` or `createInterface(input, output, completer, terminal)`, read `process.stdin` or a readable stream: `question`, `prompt`, `write` (with a key object literal of `name`, `ctrl`, `meta`, and `shift`), `pause`, `resume`, `'line'`/`'close'`/`'pause'`/`'resume'`/`'error'`/`'history'`/`'SIGINT'`/`'SIGTSTP'`/`'SIGCONT'` listeners, and `for await` compile. `on`, `once`, and `addListener` return the interface, so registrations chain. The `history`, `historySize`, `removeHistoryDuplicates`, `completer`, `tabSize`, and `escapeCodeTimeout` options are validated as in Node. diff --git a/packages/cli/test/native-link-info.test.ts b/packages/cli/test/native-link-info.test.ts index bf5b3be14..8715d9356 100644 --- a/packages/cli/test/native-link-info.test.ts +++ b/packages/cli/test/native-link-info.test.ts @@ -72,7 +72,7 @@ describe.runIf(supported)("macOS arm64 native link info", () => { minimum_os: "14.0", }, program: { object, entry_symbol: "main" }, - runtime_abi: { version: 33, marker: "scr_runtime_abi_v33" }, + runtime_abi: { version: 34, marker: "scr_runtime_abi_v34" }, runtime_pack: { kind: "precompiled", package: "@scriptc/runtime-darwin-arm64", @@ -165,6 +165,6 @@ describe.runIf(supported)("macOS arm64 native link info", () => { () => null, (failure: { stderr?: string }) => failure, ); - expect(error?.stderr).toContain("scr_runtime_abi_v33"); + expect(error?.stderr).toContain("scr_runtime_abi_v34"); }); }); diff --git a/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts b/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts index 14a2e3432..b368c91f4 100644 --- a/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts +++ b/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts @@ -727,6 +727,17 @@ export function streamCbThunkFor( const ty = host.llType(p); host.declare(`declare zeroext i1 @scr_exc_pending()`); if (hasThis) host.declare(`declare void @scr_stream_release_v(ptr)`); + // An entry `encoding` declared as a string (the declarations' + // `encoding: string`) receives "" where Node passes undefined or + // null (an object-mode write), as a write's encoding parameter does. + const entry = + p.kind === "array" && p.elem.kind === "record" + ? host.recordsById.get(p.elem.shapeId) + : undefined; + if (entry?.fields.some((f) => f.name === "encoding" && f.type.kind === "string")) { + host.declare(`declare void @scr_stream_writev_fill_encodings(ptr)`); + d.push(` call void @scr_stream_writev_fill_encodings(ptr %chunks)`); + } d.push( ` %vt${i} = call ${ty === "i1" ? "zeroext i1" : ty} @${helper}(ptr %chunks, ptr null)`, ` %vtbad${i} = call zeroext i1 @scr_exc_pending()`, diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 054d9341b..548e8e597 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1135,6 +1135,8 @@ export const LIB_FN_SYMS: Record = { "readable.pushEncoding": "scr_stream_set_push_encoding", "stream.objectMode": "scr_stream_object_mode", "stream.decodeStrings": "scr_stream_decode_strings", + "stream.optionValues": "scr_stream_option_values", + "stream.checkEncodingOption": "scr_stream_check_encoding_option", "stream.duplexFromGen": "scr_stream_duplex_from_gen", "stream.duplexFromDyn": "scr_stream_duplex_from_dyn", "stream.duplexifyReadable": "scr_stream_duplexify_readable", diff --git a/packages/compiler/src/backend/runtime-abi.ts b/packages/compiler/src/backend/runtime-abi.ts index 7d3dcbc02..173de6408 100644 --- a/packages/compiler/src/backend/runtime-abi.ts +++ b/packages/compiler/src/backend/runtime-abi.ts @@ -2,8 +2,8 @@ * of the link contract: a program object keeps this symbol undefined and the * matching runtime defines it, so incompatible manual links fail before the * program can start. */ -export const RUNTIME_ABI_VERSION = 33 as const; -export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v33" as const; +export const RUNTIME_ABI_VERSION = 34 as const; +export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v34" as const; /** Object consumption is useful today, but the complete scr_* surface may * still change before 1.0. The versioned marker prevents accidental mixing; diff --git a/packages/compiler/src/backend/runtime-pack-targets.test.ts b/packages/compiler/src/backend/runtime-pack-targets.test.ts index 532051b88..923b55c5b 100644 --- a/packages/compiler/src/backend/runtime-pack-targets.test.ts +++ b/packages/compiler/src/backend/runtime-pack-targets.test.ts @@ -52,7 +52,7 @@ async function fixture(target: NativeTargetSpec, workers = false) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 33, marker: "scr_runtime_abi_v33" }, + runtime_abi: { version: 34, marker: "scr_runtime_abi_v34" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: ["SCR_LIB"], sanitizer: "external-toolchain-required" }, flavors: { diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index 18b04b9a6..681f86817 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -127,7 +127,7 @@ async function fixture(target: NativeTargetSpec = MACOS_ARM64_TARGET) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 33, marker: "scr_runtime_abi_v33" }, + runtime_abi: { version: 34, marker: "scr_runtime_abi_v34" }, compiler: { command: "clang", identity: "fixture clang", diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index 6dd0124bd..e3ab29dba 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -2734,8 +2734,8 @@ export function collectClassShapeInner( // state, so an override or shadowing field would split-brain. // Underscore methods are the SUPPORTED override form — but only // the ones the class's own base consumes (a `_read` on a - // Transform, or `_writev`/`_construct` anywhere, would be consumed - // by Node machinery that has no lowering here). + // Transform, or `_writev` on a Readable, would be consumed by Node + // machinery that has no lowering here). if (memberName && ts.isIdentifier(memberName)) { const streamBase = (() => { for (let c = base; c; c = c.base) if (c.builtinStream) return c; @@ -2754,7 +2754,7 @@ export function collectClassShapeInner( lowerer.unsupported( "SC1090", memberName, - `declaring '_writev' on a ${streamBase.def.name.slice(1)} subclass (batched writes are supported on Writable and Duplex subclasses)`, + `declaring '_writev' on a ${streamBase.def.name.slice(1)} subclass (a Readable has no writable side)`, ); } const accepted = streamCtorShape(streamBase.def.name).accepted; diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index c37ae9074..15afb741e 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -179,15 +179,15 @@ export const UNDERSCORE_METHODS: ReadonlyMap = new Map([ /** The option callbacks applied after construction rather than through * the constructor's callback slots (option name → method name): construct - * on every stream class, and writev on the writable classes whose - * machinery hands it batches (Writable and Duplex). */ + * on every stream class, and writev on every class with a writable side + * (a Transform's batches reach writev, not the transform, as in Node). */ export const STREAM_EXTRA_CALLBACKS: ReadonlyMap = new Map([ ["construct", "_construct"], ["writev", "_writev"], ]); export function streamExtraCallbacks(cls: string): readonly string[] { - return cls === "%Writable" || cls === "%Duplex" ? ["construct", "writev"] : ["construct"]; + return cls === "%Readable" ? ["construct"] : ["construct", "writev"]; } /** The completion callback's parameter position in each underscore @@ -442,6 +442,13 @@ interface StreamOptions { * right after construction (Node's constructors call addAbortSignal * once the state exists), or null when absent. */ signal: IrExpr | null; + /** Non-literal decodeStrings and defaultEncoding values: valueDecls + * evaluates them into locals before construction (the defaultEncoding + * validated there, as the state constructors throw before the stream + * exists), and stream.optionValues applies them to the fresh stream. */ + valueDecls: IrStmt[]; + decodeStringsValue: IrExpr | null; + defaultEncodingValue: IrExpr | null; /** The objectMode option's sides (a Duplex's readableObjectMode and * writableObjectMode set one side): applied with stream.objectMode * right after construction. */ @@ -732,6 +739,12 @@ function pinAnyDataCompletion( if (!symbol || lowerer.stageSourceTypes.has(symbol)) return null; const declared = lowerer.paramShape(param).type; if (declared.kind !== "func" || declared.rest || declared.params[1]?.kind !== "dyn") return null; + // Only when every use calls the callback with a Buffer, a string or + // nothing as `data`: another value (or a callback that escapes) keeps + // `data` checked-dynamic, so the runtime pushes it as push(data) does + // and reports ERR_INVALID_ARG_TYPE through 'error', as Node does. + if (fnNode.body === undefined || !completionDataIsChunk(lowerer, fnNode.body, symbol, param.name)) + return null; const data = unionOf(lowerer, [BYTES, STRING, { kind: "undefinedT" }]); lowerer.stageSourceTypes.set( symbol, @@ -740,6 +753,59 @@ function pinAnyDataCompletion( return symbol; } +/** Whether every reference to the completion callback `symbol` in `body` + * calls it with a `data` argument that is a Buffer, a string, null or + * undefined (or with no data). */ +function completionDataIsChunk( + lowerer: Lowerer, + body: ts.Node, + symbol: ts.Symbol, + declaration: ts.Node, +): boolean { + let ok = true; + const visit = (node: ts.Node): void => { + if (!ok) return; + if ( + ts.isIdentifier(node) && + node !== declaration && + lowerer.checker.getSymbolAtLocation(node) === symbol + ) { + const call = node.parent; + if (!call || !ts.isCallExpression(call) || call.expression !== node) { + ok = false; + return; + } + const data = call.arguments.nodes[1]; + if (data !== undefined) { + let t: IrType | null; + try { + t = lowerer.mapTypeOf(lowerer.typeOf(data)); + } catch { + t = null; + } + const arms = + t?.kind === "union" ? (lowerer.unions.get(t.unionId)?.arms ?? null) : t ? [t] : null; + if ( + arms === null || + !arms.every( + (a) => + (a.kind === "bytes" && a.elem === "u8") || + a.kind === "string" || + a.kind === "undefinedT" || + a.kind === "nullT", + ) + ) { + ok = false; + return; + } + } + } + ts.forEachChild(node, visit); + }; + visit(body); + return ok; +} + /** Parses the options object literal of a stream constructor. `which` * names the callbacks this class accepts, in canonical flag order. */ function parseStreamOptions( @@ -767,6 +833,9 @@ function parseStreamOptions( badEncoding: null, pushEncoding: null, signal: null, + valueDecls: [], + decodeStringsValue: null, + defaultEncodingValue: null, readableObject: false, writableObject: false, decodeStringsOff: false, @@ -802,6 +871,21 @@ function parseStreamOptions( optsLit !== undefined && ts.isObjectLiteralExpression(optsLit) && literalWritesValues(optsLit, ctorName !== "Readable" && ctorName !== "Writable"); + // In JavaScript a decodeStrings value known only at run time may let + // string writes through, so the write and transform callbacks' chunk + // parameters read as any (the declarations type it Buffer). + const runtimeDecodeStrings = + optsLit !== undefined && + isJsSourceFile(ts.getSourceFileOfNode(optsLit)) && + ts.isObjectLiteralExpression(optsLit) && + (optsLit.properties?.nodes ?? []).some( + (p) => + ts.isPropertyAssignment(p) && + (ts.isIdentifier(p.name) || ts.isStringLiteral(p.name)) && + p.name.text === "decodeStrings" && + p.initializer.kind !== ts.SyntaxKind.TrueKeyword && + p.initializer.kind !== ts.SyntaxKind.FalseKeyword, + ); if (out.readableObject) out.hwmR = numLit(16, loc); if (out.writableObject) out.hwmW = numLit(16, loc); const tuples = cbTuples(lowerer); @@ -851,6 +935,7 @@ function parseStreamOptions( } const full = tuples[name]!.map((t) => (t === "fn" ? null : t)); const pinned = out.readableObject ? null : pinAnyDataCompletion(lowerer, name, fnNode); + let anyChunk: ts.Symbol | undefined; try { // The completion-callback tail: typed by the checker contextually; // accept the user's declared func type as the tuple's entry so the @@ -867,12 +952,22 @@ function parseStreamOptions( } return declared ?? funcOf([], VOID); }); + const chunkParam = + runtimeDecodeStrings && (name === "write" || name === "transform") + ? fnNode.parameters.nodes[0] + : undefined; + anyChunk = + chunkParam !== undefined && ts.isIdentifier(chunkParam.name) && !chunkParam.type + ? lowerer.checker.getSymbolAtLocation(chunkParam.name) + : undefined; + if (anyChunk) lowerer.anySymbols.add(anyChunk); present.set( name, lowerStreamCallback(lowerer, name, fnNode, thisType, tuple, writesValues), ); } finally { if (pinned) lowerer.stageSourceTypes.delete(pinned); + if (anyChunk) lowerer.anySymbols.delete(anyChunk); } continue; } @@ -925,11 +1020,15 @@ function parseStreamOptions( if (writableClass) out.decodeStringsOff = true; break; } - lowerer.noLowering( - `${ctorName} with a runtime 'decodeStrings' value`, - prop, - "write 'decodeStrings: true' or 'decodeStrings: false'", - ); + // a runtime value: WritableState's `options.decodeStrings !== false` + const local = lowerer.declareHiddenLocal("%decodeStrings", DYN); + out.valueDecls.push({ + kind: "varDecl", + localId: local.id, + init: lowerer.lowerExprExpecting(value, DYN), + loc, + }); + if (writableClass) out.decodeStringsValue = varRef(local.id, DYN, loc); break; } case "encoding": { @@ -973,11 +1072,33 @@ function parseStreamOptions( break; } } - lowerer.noLowering( - `the ${ctorName} option 'defaultEncoding' with a runtime value`, - prop, - "write the encoding as a string literal", - ); + { + // a runtime value: null, undefined and utf8 are the default, + // another name Buffer.isEncoding knows is kept as given, and any + // other value throws ERR_UNKNOWN_ENCODING before construction + const local = lowerer.declareHiddenLocal("%defaultEncoding", DYN); + const ref = varRef(local.id, DYN, loc); + out.valueDecls.push( + { + kind: "varDecl", + localId: local.id, + init: lowerer.lowerExprExpecting(value, DYN), + loc, + }, + { + kind: "exprStmt", + expr: { + kind: "libCall", + fn: "stream.checkEncodingOption", + args: [ref], + type: VOID, + loc, + }, + loc, + }, + ); + out.defaultEncodingValue = ref; + } break; case "signal": { // `if (options.signal) addAbortSignal(options.signal, this)`: @@ -1041,9 +1162,7 @@ function parseStreamOptions( lowerer.noLowering( `the ${ctorName} option 'writev'`, prop, - ctorName === "Readable" - ? "a Readable has no writable side" - : "batched writes on a Transform would bypass its transform — writev is supported on Writable and Duplex", + "a Readable has no writable side", ); break; default: @@ -1083,6 +1202,9 @@ function streamCtorArgs( badEncoding: string | null; pushEncoding: string | null; signal: IrExpr | null; + valueDecls: IrStmt[]; + decodeStringsValue: IrExpr | null; + defaultEncodingValue: IrExpr | null; readableObject: boolean; writableObject: boolean; decodeStringsOff: boolean; @@ -1117,6 +1239,9 @@ function streamCtorArgs( badEncoding: o.badEncoding, pushEncoding: o.pushEncoding, signal: o.signal, + valueDecls: o.valueDecls, + decodeStringsValue: o.decodeStringsValue, + defaultEncodingValue: o.defaultEncodingValue, readableObject: o.readableObject, writableObject: o.writableObject, decodeStringsOff: o.decodeStringsOff, @@ -1237,6 +1362,18 @@ export function lowerStreamNew(lowerer: Lowerer, expr: ts.NewExpression, info: C loc, }; } + const values = optionValuesArgs(parsed, loc); + if (values) { + // runtime decodeStrings and defaultEncoding values (answers the + // receiver +1) + built = { + kind: "libCall", + fn: "stream.optionValues", + args: [built, ...values], + type: thisType, + loc, + }; + } if (parsed.pushEncoding !== null) { // { defaultEncoding }: chain the push-side decode default (the // setEncoding chaining shape — answers the receiver +1). @@ -1269,8 +1406,11 @@ export function lowerStreamNew(lowerer: Lowerer, expr: ts.NewExpression, info: C loc, }; } - if (!parsed.decodeStringsOff && parsed.writeEncoding === null && parsed.extras.length === 0) - return built; + if (!parsed.decodeStringsOff && parsed.writeEncoding === null && parsed.extras.length === 0) { + return parsed.valueDecls.length > 0 + ? { kind: "seqExpr", stmts: parsed.valueDecls, result: built, type: thisType, loc } + : built; + } const fresh = lowerer.declareHiddenLocal("%stream", thisType); const followUps = streamOptionFollowUps( parsed, @@ -1280,13 +1420,30 @@ export function lowerStreamNew(lowerer: Lowerer, expr: ts.NewExpression, info: C ); return { kind: "seqExpr", - stmts: [{ kind: "varDecl", localId: fresh.id, init: built, loc }, ...followUps], + stmts: [ + ...parsed.valueDecls, + { kind: "varDecl", localId: fresh.id, init: built, loc }, + ...followUps, + ], result: varRef(fresh.id, thisType, loc), type: thisType, loc, }; } +/** stream.optionValues' (decodeStrings, defaultEncoding) arguments, or + * null when both options are literals or absent. */ +function optionValuesArgs( + parsed: { decodeStringsValue: IrExpr | null; defaultEncodingValue: IrExpr | null }, + loc: SrcLoc, +): IrExpr[] | null { + if (parsed.decodeStringsValue === null && parsed.defaultEncodingValue === null) return null; + return [ + parsed.decodeStringsValue ?? dynUndefinedExpr(loc), + parsed.defaultEncodingValue ?? dynUndefinedExpr(loc), + ]; +} + /** True when a value of Node's argument type `passed` widens implicitly * into the declared parameter type `declared` (one of its union arms, or a * sub-union of it). */ @@ -1589,6 +1746,7 @@ export function lowerStreamSuperCall( ]; } const out: IrStmt[] = [ + ...parsed.valueDecls, { kind: "exprStmt", expr: { @@ -1614,6 +1772,20 @@ export function lowerStreamSuperCall( loc, }); } + const values = optionValuesArgs(parsed, loc); + if (values) { + out.push({ + kind: "exprStmt", + expr: { + kind: "libCall", + fn: "stream.optionValues", + args: [thisRef(), ...values], + type: thisType, + loc, + }, + loc, + }); + } if (parsed.pushEncoding !== null) { out.push({ kind: "exprStmt", @@ -1699,12 +1871,20 @@ export function lowerStreamUnderscoreAssign( const clsName = cls.slice(1); if (methodName === "_construct") { // Node's constructor reads _construct once; a later assignment has no - // effect there - lowerer.noLowering( - `assigning '${methodName}' on a ${clsName}`, - expr.left, - "declare _construct on the class or pass the construct option", - ); + // effect on the stream, so only the receiver and the value are + // evaluated + const value = lowerer.lowerExpr(expr.right); + return { + kind: "exprStmt", + expr: { + kind: "seqExpr", + stmts: [{ kind: "exprStmt", expr: recv, loc }], + result: value, + type: value.type, + loc, + }, + loc, + }; } const which = methodName === "_writev" ? "writev" : optionByMethod.get(methodName)!; const { accepted } = streamCtorShape(cls); diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index d8e78be08..e786fa9a8 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -362,6 +362,7 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "transform.newDyn", "passthrough.newDyn", "readable.initDyn", + "stream.checkEncodingOption", "writable.initDyn", "duplex.initDyn", "transform.initDyn", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 7cb666ccf..ed135a800 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -4103,6 +4103,11 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ * after construction; answers the receiver (+1, the chaining shape). */ "stream.objectMode": { argTypes: [null, BOOL, BOOL], result: VOID }, "stream.decodeStrings": { argTypes: [null, BOOL], result: VOID }, + /** Runtime decodeStrings and defaultEncoding option values, applied + * after construction (answers the receiver +1); the encoding was + * validated by stream.checkEncodingOption before construction. */ + "stream.optionValues": { argTypes: [null, DYN, DYN], result: VOID }, + "stream.checkEncodingOption": { argTypes: [DYN], result: VOID }, "stream.duplexFromGen": { argTypes: [null, null, null, null, null, null], result: VOID }, "stream.duplexFromDyn": { argTypes: [DYN], result: VOID }, /** Duplex.from over a lone Readable or Writable, or a { readable, diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index 85910369a..bd4727b95 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -219,6 +219,8 @@ const STREAM_LIB_FNS: ReadonlySet = new Set([ "stream.chunkDone", "stream.objectMode", "stream.decodeStrings", + "stream.optionValues", + "stream.checkEncodingOption", "stream.duplexFromGen", "stream.duplexFromDyn", "stream.duplexFromPromise", diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index 770fee6c6..1f3692c9c 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -5495,6 +5495,7 @@ function validateFunction( e.fn === "stream.destroyErr" || e.fn === "stream.objectMode" || e.fn === "stream.decodeStrings" || + e.fn === "stream.optionValues" || e.fn === "stream.duplexFromGen" || e.fn === "stream.duplexFromDyn" || e.fn === "stream.duplexifyReadable" || @@ -5867,6 +5868,7 @@ function validateFunction( e.fn === "readable.pushEncoding" || e.fn === "stream.objectMode" || e.fn === "stream.decodeStrings" || + e.fn === "stream.optionValues" || e.fn === "writable.writeEncoding" || e.fn === "writable.setDefaultEncoding" || e.fn === "stream.destroy" || diff --git a/packages/compiler/test/native-codegen-integration.test.ts b/packages/compiler/test/native-codegen-integration.test.ts index 0fd5249d3..3a1f8fdee 100644 --- a/packages/compiler/test/native-codegen-integration.test.ts +++ b/packages/compiler/test/native-codegen-integration.test.ts @@ -378,7 +378,7 @@ done: expect(await symbols(helperObject, ["-u"])).toEqual(await symbols(clangObject, ["-u"])); expect(await symbols(helperObject, ["-gU"])).toEqual(await symbols(clangObject, ["-gU"])); expect(await symbols(helperObject, ["-gU"])).toEqual(["0000000000000000 T _main"]); - expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v33"); + expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v34"); const clangExe = join(dir, "clang-program"); const linkDriver = join(dir, "clang-driver.c"); @@ -712,7 +712,7 @@ done: .split("\n"); expect(undefinedSymbols).toContain("_sf_scale"); expect(undefinedSymbols).toContain("_sf_callback_mix"); - expect(undefinedSymbols).toContain("_scr_runtime_abi_v33"); + expect(undefinedSymbols).toContain("_scr_runtime_abi_v34"); }); test("helper and clang object paths retain the same outbound FFI C ABI", async () => { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7460-transform-writev.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7460-transform-writev.mjs.json new file mode 100644 index 000000000..1015e1354 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7460-transform-writev.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7460-transform-writev.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7461-transform-writev.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7461-transform-writev.cjs.json new file mode 100644 index 000000000..4d3c86215 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7461-transform-writev.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7461-transform-writev.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7462-transform-writev.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7462-transform-writev.ts.json new file mode 100644 index 000000000..9b42f2d60 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7462-transform-writev.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7462-transform-writev.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7463-stream-option-values.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7463-stream-option-values.ts.json new file mode 100644 index 000000000..5aa543dea --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7463-stream-option-values.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7463-stream-option-values.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7464-stream-option-values.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7464-stream-option-values.mjs.json new file mode 100644 index 000000000..2a438d418 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7464-stream-option-values.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7464-stream-option-values.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7465-transform-callback-values.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7465-transform-callback-values.ts.json new file mode 100644 index 000000000..17633d208 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7465-transform-callback-values.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7465-transform-callback-values.ts" + ], + "diags": [] +} diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index e17312713..9f856b376 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -340,7 +340,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: RUNTIME_PACK_MATRIX.target, - runtime_abi: { version: 33, marker: "scr_runtime_abi_v33" }, + runtime_abi: { version: 34, marker: "scr_runtime_abi_v34" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index 60ef9dc49..50da4a78e 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -437,7 +437,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: matrix.target, - runtime_abi: { version: 33, marker: "scr_runtime_abi_v33" }, + runtime_abi: { version: 34, marker: "scr_runtime_abi_v34" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime/src/scr_console.c b/packages/runtime/src/scr_console.c index f8dd9a0c0..193352424 100644 --- a/packages/runtime/src/scr_console.c +++ b/packages/runtime/src/scr_console.c @@ -148,7 +148,7 @@ static void scr_enable_utf8_console_output(void) { } #endif -void scr_runtime_abi_v33(void) {} +void scr_runtime_abi_v34(void) {} void scr_init(void) { #ifdef SCR_WORKERS diff --git a/packages/runtime/src/scr_core.h b/packages/runtime/src/scr_core.h index d716f0de3..1b5e28545 100644 --- a/packages/runtime/src/scr_core.h +++ b/packages/runtime/src/scr_core.h @@ -81,7 +81,7 @@ void scr_init(void); /* Program objects emitted by the bundled LLVM helper reference this symbol. * Its versioned spelling makes a mismatched manual runtime link fail before * the program can start. */ -void scr_runtime_abi_v33(void); +void scr_runtime_abi_v34(void); void scr_runtime_workers_v8(void); /* ── the trap funnel (scr_console.c; scr_library.c under -DSCR_LIB) ────── diff --git a/packages/runtime/src/scr_dyn_builtin_stream.c b/packages/runtime/src/scr_dyn_builtin_stream.c index 926f85eea..2875cfa06 100644 --- a/packages/runtime/src/scr_dyn_builtin_stream.c +++ b/packages/runtime/src/scr_dyn_builtin_stream.c @@ -34,7 +34,7 @@ * receiver without a reference; the hook closures outlive the record. * * Not supported here (each throws the "not supported yet" Error where it - * runs): the options scr_stream.c's dyn option walk refuses, `_writev` and `_construct`, symbol event names, listeners() / + * runs): the options scr_stream.c's dyn option walk refuses, symbol event names, listeners() / * rawListeners(), `wrap()`, and pipes into values that are neither runtime * streams nor constructor-function streams. */ #include "scr_runtime.h" @@ -898,20 +898,15 @@ static ScrDyn *bcs_construct(int kind, ScrDyn *self, ScrDyn *const *args, size_t } bool failed = false; bool writes = kind != BCS_READABLE; - bool transformish = kind == BCS_TRANSFORM || kind == BCS_PASSTHROUGH; ScrDyn *opts = bcs_arg(args, argc, 0); /* Node's constructors assign options.writev and options.construct to * the instance (this._writev = options.writev), then read _writev and * _construct, so an option shadows the prototype's method. */ if (opts->kind == SCR_DYN_OBJ) { - if (writes && !transformish && !bcs_copy_option(self, opts, "writev", "_writev")) return NULL; + if (writes && !bcs_copy_option(self, opts, "writev", "_writev")) return NULL; if (!bcs_copy_option(self, opts, "construct", "_construct")) return NULL; } bool has_writev = writes && bcs_is_function(self, "_writev", &failed); - if (has_writev && transformish) { - bcs_unsupported("_writev on a constructor-function Transform is not supported yet"); - return NULL; - } bool has_construct = !failed && bcs_is_function(self, "_construct", &failed); if (failed) return NULL; ScrStream *s = kind == BCS_READABLE ? scr_stream_new_readable_dyn(opts) diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index 275c03039..4b962675a 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -2157,6 +2157,24 @@ static ScrDyn *scr_stream_wentry_encoding(const ScrStreamState *st, ScrDyn *enc) return scr_dyn_retain(enc ? enc : scr_dyn_undefined()); } +/* A compiled writev whose entries declare `encoding` a string: an entry + * without one (an object-mode write passes undefined or null) gets "". + * The batch array is the one this call receives. */ +void scr_stream_writev_fill_encodings(ScrDyn *chunks) { + if (!chunks || chunks->kind != SCR_DYN_ARR) return; + double n = scr_dyn_arr_len(chunks); + for (double i = 0; i < n; i++) { + ScrDyn *entry = scr_dyn_arr_at(chunks, i); /* +1 */ + ScrDyn *enc = entry->kind == SCR_DYN_OBJ ? scr_dyn_obj_get(entry, "encoding", 8) : NULL; + if (entry->kind == SCR_DYN_OBJ && (!enc || enc->kind == SCR_DYN_UNDEF || enc->kind == SCR_DYN_NULL)) { + ScrStr *empty = scr_str_new("", 0); + scr_dyn_obj_set(entry, "encoding", 8, scr_dyn_new_str(empty)); + scr_str_release(empty); + } + scr_dyn_release(entry); + } +} + /* A batch entry's callback: the write's callback, or Node's nop. */ static ScrDyn *scr_stream_writev_entry_cb_call(ScrClosure *clo, ScrDyn *const *args, size_t argc) { (void)args; @@ -2675,9 +2693,8 @@ void *scr_stream_native(ScrStream *s, void (*release)(void *)) { * way the compiler-emitted dyn thunks do (chunk as Buffer-flavored bytes, * encoding 'buffer', the error via the boundary encoding, the completion * callback as a callable dyn over the *_done glue). Options that Node's - * base WOULD consume but have no lowering here (objectMode truthy, - * decodeStrings:false, non-utf8 defaultEncoding, signal, construct, - * writev, and read/write/final on the transformish bases) throw the + * base WOULD consume but have no lowering here (signal, and + * read/write/final on the transformish bases) throw the * "not supported yet" Error — the compile-time fence's runtime twin. */ /* The boxed dyn callable (+1 — scr_box_get_ref retains; callers release @@ -2903,11 +2920,8 @@ static bool scr_stream_parse_dyn_opts(const ScrDyn *opts, char base, ScrStreamDy } } out->construct = scr_stream_dynopt_fn(opts, "construct", 9); + /* a Transform's batches reach writev, not the transform, as in Node */ if (base != 'r') out->writev = scr_stream_dynopt_fn(opts, "writev", 6); - if (transformish && out->writev != NULL) { - static const char m[] = "the stream option 'writev' on a Transform is not supported yet (batches would bypass the transform)"; - return scr_stream_dynopt_unsupported(m, sizeof m - 1); - } { /* a known encoding or ERR_UNKNOWN_ENCODING (the states' constructors) */ ScrDyn *v = scr_dyn_obj_get((ScrDyn *)opts, "defaultEncoding", 15); @@ -6296,6 +6310,55 @@ ScrStream *scr_stream_set_default_encoding(ScrStream *s, ScrDyn *enc) { return scr_stream_retain(s); } +/* Whether a defaultEncoding option value is the default: null, undefined, + * 'utf8' or 'utf-8' (the state constructors' kDefaultUTF8Encoding). */ +static bool scr_stream_default_encoding_value(const ScrDyn *v) { + if (!v || v->kind == SCR_DYN_UNDEF || v->kind == SCR_DYN_NULL) return true; + return v->kind == SCR_DYN_STR && ((v->v.str->len == 4 && memcmp(v->v.str->data, "utf8", 4) == 0) || + (v->v.str->len == 5 && memcmp(v->v.str->data, "utf-8", 5) == 0)); +} + +/* A runtime defaultEncoding option value, checked before construction as + * the state constructors check it: the default, a name Buffer.isEncoding + * knows, or ERR_UNKNOWN_ENCODING (thrown). Borrows v. */ +void scr_stream_check_encoding_option(ScrDyn *v) { + if (scr_stream_default_encoding_value(v)) return; + if (v->kind == SCR_DYN_STR) { + ScrStr *canonical = scr_bytes_normalize_encoding(v->v.str); + if (canonical) scr_str_release(canonical); + return; + } + ScrStr *shown = scr_dyn_string_coerce_js(v); + if (!shown) return; + char msg[128]; + int len = snprintf(msg, sizeof msg, "Unknown encoding: %.*s", (int)(shown->len < 64 ? shown->len : 64), + shown->data); + scr_str_release(shown); + scr_throw_error_msg_code(SCR_ERR_TYPE, msg, (size_t)(len < 0 ? 0 : len), "ERR_UNKNOWN_ENCODING"); +} + +/* Runtime decodeStrings and defaultEncoding option values over a fresh + * stream (after its objectMode): decodeStrings === false keeps string + * writes, and a known defaultEncoding is the write side's default as + * given and how a byte-mode readable side decodes push(string). Borrows + * both; answers the receiver (+1). */ +ScrStream *scr_stream_option_values(ScrStream *s, ScrDyn *decode_strings, ScrDyn *default_encoding) { + ScrStreamState *st = s->st; + if (st->has_w && decode_strings && decode_strings->kind == SCR_DYN_BOOL && !decode_strings->v.b) + scr_stream_set_decode_strings(s, false); + if (!scr_stream_default_encoding_value(default_encoding) && default_encoding->kind == SCR_DYN_STR) { + ScrStr *canonical = scr_bytes_normalize_encoding(default_encoding->v.str); + if (canonical) { + scr_stream_set_write_encoding(s, default_encoding->v.str); + if (st->has_r && !st->r.object_entries) scr_stream_release(scr_stream_set_push_encoding(s, canonical)); + scr_str_release(canonical); + } else if (scr_exc_pending()) { + scr_exc_clear(); /* checked before construction */ + } + } + return scr_stream_retain(s); +} + /* stream.objectMode: scr_stream_set_object_mode, answering the receiver * (+1). */ ScrStream *scr_stream_object_mode(ScrStream *s, bool readable, bool writable) { diff --git a/packages/runtime/src/scr_stream.h b/packages/runtime/src/scr_stream.h index d26077b5c..e368ec7ff 100644 --- a/packages/runtime/src/scr_stream.h +++ b/packages/runtime/src/scr_stream.h @@ -563,6 +563,9 @@ void scr_stream_set_writev_concat(ScrStream *s); void scr_stream_set_construct(ScrStream *s, ScrClosure *cb, ScrStreamPlainInv inv); void scr_stream_set_writev(ScrStream *s, ScrClosure *cb, ScrStreamWritevInv inv); void scr_stream_set_writev_default(ScrStream *s, ScrClosure *cb, ScrStreamWritevInv inv); +/* A compiled writev whose entries declare `encoding` a string: entries + * without an encoding (object mode) get "" before the conversion. */ +void scr_stream_writev_fill_encodings(struct ScrDyn *chunks); /* Duplex.from (duplexify.js): over an async generator function (the * compiled stage takes the source generator that the compiled source * makes from next() and ack(); all borrowed, helpers too), and over a @@ -605,6 +608,11 @@ ScrStream *scr_stream_duplex_from_promise(struct ScrPromise *p, ScrClosure *box) * forms answer the stream (+1). */ void scr_stream_set_decode_strings(ScrStream *s, bool decode); ScrStream *scr_stream_decode_strings(ScrStream *s, bool decode); +/* Runtime decodeStrings/defaultEncoding option values: checked before + * construction (ERR_UNKNOWN_ENCODING thrown), then applied to the fresh + * stream, answering it (+1). Values borrowed. */ +void scr_stream_check_encoding_option(struct ScrDyn *v); +ScrStream *scr_stream_option_values(ScrStream *s, struct ScrDyn *decode_strings, struct ScrDyn *default_encoding); void scr_stream_set_write_encoding(ScrStream *s, ScrStr *enc); ScrStream *scr_stream_write_encoding(ScrStream *s, ScrStr *enc); ScrStream *scr_stream_set_default_encoding(ScrStream *s, struct ScrDyn *enc); diff --git a/tests/corpus/7460-transform-writev.mjs b/tests/corpus/7460-transform-writev.mjs new file mode 100644 index 000000000..d78bdfc77 --- /dev/null +++ b/tests/corpus/7460-transform-writev.mjs @@ -0,0 +1,33 @@ +// writev on a Transform and a PassThrough (the writev option): a batch of +// buffered writes reaches writev instead of the transform, a lone write +// still reaches the transform, and the readable side gets what writev +// pushes. +import { Transform, PassThrough } from "node:stream"; +const t = new Transform({ + transform(chunk, enc, cb) { console.log("transform", String(chunk), enc); cb(null, chunk); }, + writev(chunks, cb) { + console.log("writev", chunks.map((c) => `${c.chunk}/${c.encoding}`).join(",")); + for (const { chunk } of chunks) this.push(String(chunk).toUpperCase()); + cb(); + }, +}); +t.on("data", (d) => console.log("data", String(d))); +t.on("finish", () => console.log("finish")); +t.on("end", () => console.log("end")); +t.write("a"); +t.cork(); +t.write("b"); +t.write("c", "latin1"); +process.nextTick(() => { t.uncork(); t.end("d"); }); + +const p = new PassThrough({ + objectMode: true, + writev(chunks, cb) { console.log("pt writev", JSON.stringify(chunks.map((c) => c.chunk))); cb(); }, +}); +p.on("data", (d) => console.log("pt data", JSON.stringify(d))); +p.cork(); +p.write({ x: 1 }); +p.write([2]); +p.uncork(); +p.write("solo"); +p.end(); diff --git a/tests/corpus/7461-transform-writev.cjs b/tests/corpus/7461-transform-writev.cjs new file mode 100644 index 000000000..dd45cf09d --- /dev/null +++ b/tests/corpus/7461-transform-writev.cjs @@ -0,0 +1,33 @@ +// Constructor-function Transform and PassThrough streams with _writev: +// the prototype's _writev takes the batches, and a writev option shadows +// it with the stream as this. +'use strict'; +const { Transform, PassThrough } = require('node:stream'); +function T(opts) { Transform.call(this, opts); } +Object.setPrototypeOf(T.prototype, Transform.prototype); +Object.setPrototypeOf(T, Transform); +T.prototype._transform = function (chunk, enc, cb) { cb(null, 'T(' + chunk + ')'); }; +T.prototype._writev = function (chunks, cb) { + this.push('V(' + chunks.map((c) => String(c.chunk)).join('+') + ')'); + cb(); +}; +const t = Object.create(T.prototype); +T.call(t, {}); +t.on('data', (d) => console.log('t data', String(d))); +t.on('end', () => console.log('t end')); +t.write('a'); +t.cork(); +t.write('b'); +t.write('c'); +process.nextTick(() => { t.uncork(); t.end(); }); +function P(opts) { PassThrough.call(this, opts); } +Object.setPrototypeOf(P.prototype, PassThrough.prototype); +Object.setPrototypeOf(P, PassThrough); +const p = Object.create(P.prototype); +P.call(p, { writev(chunks, cb) { console.log('p writev', chunks.length, this === p); cb(); } }); +p.on('data', (d) => console.log('p data', String(d))); +p.cork(); +p.write('x'); +p.write('y'); +p.uncork(); +p.write('z'); diff --git a/tests/corpus/7462-transform-writev.ts b/tests/corpus/7462-transform-writev.ts new file mode 100644 index 000000000..ca079b3c9 --- /dev/null +++ b/tests/corpus/7462-transform-writev.ts @@ -0,0 +1,109 @@ +// Typed writev on Transform and PassThrough: an options writev with +// decodeStrings: false and flush, a Transform subclass's _writev, a +// batch completing with an error, and object-mode entries whose +// encoding is declared as a string or as Node's string | null | +// undefined. +import { Transform, PassThrough, Writable } from "node:stream"; +type WritevChunks = Array<{ chunk: any; encoding: BufferEncoding }>; +type Cb = (error?: Error | null) => void; +type TransformCallback = (error?: Error | null, data?: any) => void; +const log: string[] = []; +const t = new Transform({ + decodeStrings: false, + transform(chunk: string, enc: BufferEncoding, cb: TransformCallback): void { + log.push(`transform ${chunk} ${enc}`); + cb(null, chunk); + }, + flush(cb: TransformCallback): void { + log.push("flush"); + cb(null, "!"); + }, + writev(chunks: WritevChunks, cb: Cb): void { + log.push(`writev ${chunks.map((c) => `${c.chunk}/${c.encoding}`).join(",")}`); + this.push(chunks.map((c) => String(c.chunk)).join("")); + cb(); + }, +}); +t.on("data", (d: Buffer) => log.push(`data ${d.toString()}`)); +t.on("end", () => log.push("end")); +t.on("finish", () => log.push("finish")); +t.on("close", () => { + console.log(log.join("\n")); + next(); +}); +t.write("one"); +t.cork(); +t.write("two", "latin1"); +t.write("three"); +t.uncork(); +t.end("four"); + +class Batcher extends Transform { + _transform(chunk: Buffer, _enc: BufferEncoding, cb: TransformCallback): void { + console.log("batcher transform", chunk.toString()); + cb(null, chunk); + } + _writev(chunks: Array<{ chunk: any; encoding: BufferEncoding }>, cb: Cb): void { + console.log("batcher writev", chunks.length); + this.push(Buffer.concat(chunks.map((c) => c.chunk as Buffer))); + cb(); + } +} + +function next(): void { + const b = new Batcher(); + b.on("data", (d: Buffer) => console.log("batcher data", d.toString())); + b.write("x"); + b.cork(); + b.write("y"); + b.write("z"); + process.nextTick(() => { + b.uncork(); + b.end(); + }); + b.on("finish", () => { + const p = new PassThrough({ + writev(chunks: WritevChunks, cb: Cb): void { + console.log("pt writev", chunks.length, String(chunks[0]!.chunk)); + cb(new Error("batch failed")); + }, + }); + p.on("error", (e: Error) => { + console.log("pt error", e.message); + objectEntries(); + }); + p.cork(); + p.write("1"); + p.write("2"); + p.uncork(); + }); +} + +function objectEntries(): void { + const declared = new Writable({ + objectMode: true, + write(_chunk: any, _enc: BufferEncoding, cb: Cb): void { cb(); }, + writev(chunks: WritevChunks, cb: Cb): void { + console.log("string encodings", JSON.stringify(chunks.map((c) => c.chunk))); + cb(); + }, + }); + declared.write({ n: 0 }); + declared.cork(); + declared.write({ n: 1 }); + declared.write({ n: 2 }, () => {}); + declared.uncork(); + const nodeTyped = new Writable({ + objectMode: true, + write(_chunk: any, _enc: BufferEncoding, cb: Cb): void { cb(); }, + writev(chunks: Array<{ chunk: any; encoding: string | null | undefined }>, cb: Cb): void { + console.log("node encodings", JSON.stringify(chunks.map((c) => [c.chunk, c.encoding === undefined ? "undefined" : c.encoding]))); + cb(); + }, + }); + nodeTyped.write({ n: 0 }); + nodeTyped.cork(); + nodeTyped.write({ n: 1 }); + nodeTyped.write({ n: 2 }, () => {}); + nodeTyped.uncork(); +} diff --git a/tests/corpus/7463-stream-option-values.ts b/tests/corpus/7463-stream-option-values.ts new file mode 100644 index 000000000..04f497b6b --- /dev/null +++ b/tests/corpus/7463-stream-option-values.ts @@ -0,0 +1,58 @@ +// Typed decodeStrings and defaultEncoding options whose values are known +// only at run time: decodeStrings === false keeps string writes, a known +// defaultEncoding is the write default as given and how push(string) +// decodes, null and utf8 are the default, an unknown name throws +// ERR_UNKNOWN_ENCODING before construction, and a subclass's super() +// options take them the same way. +import { Writable, Readable, Duplex } from "node:stream"; +type Cb = (error?: Error | null) => void; +const flags: boolean[] = JSON.parse("[false, true]"); +const encodings: Array = JSON.parse('["hex", "latin1", null, "UTF8"]'); +for (const decode of flags) { + const w = new Writable({ + decodeStrings: decode, + defaultEncoding: encodings[0] as BufferEncoding, + write(chunk: any, enc: string, cb: Cb): void { + console.log("write", decode, typeof chunk, enc, String(chunk)); + cb(); + }, + }); + w.write("6869"); +} +const r = new Readable({ defaultEncoding: encodings[1] as BufferEncoding, read(): void {} }); +r.push("é"); +r.push(null); +r.on("data", (d: Buffer) => console.log("readable", d.toString("hex"))); +const dflt = new Writable({ + defaultEncoding: encodings[2] as any, + write(chunk: Buffer, enc: string, cb: Cb): void { + console.log("null default", enc, chunk.toString()); + cb(); + }, +}); +dflt.write("ok"); +const upper = new Duplex({ + defaultEncoding: encodings[3] as BufferEncoding, + decodeStrings: flags[0], + read(): void {}, + write(chunk: any, enc: string, cb: Cb): void { + console.log("duplex", typeof chunk, enc); + cb(); + }, +}); +upper.write("x"); +try { + new Writable({ defaultEncoding: JSON.parse('"nope"') }); +} catch (e) { + console.log("bad", (e as Error & { code?: string }).code, (e as Error).message); +} +class Sink extends Writable { + constructor(decode: boolean, encoding: BufferEncoding) { + super({ decodeStrings: decode, defaultEncoding: encoding }); + } + _write(chunk: any, enc: string, cb: Cb): void { + console.log("sink", typeof chunk, enc); + cb(); + } +} +new Sink(flags[0]!, encodings[1] as BufferEncoding).write("y"); diff --git a/tests/corpus/7464-stream-option-values.mjs b/tests/corpus/7464-stream-option-values.mjs new file mode 100644 index 000000000..dfeb7a447 --- /dev/null +++ b/tests/corpus/7464-stream-option-values.mjs @@ -0,0 +1,28 @@ +// Option values known only at run time in JavaScript: decodeStrings and +// defaultEncoding read from an object, a write callback whose chunk is +// then a string, decodeStrings: 0 (only false turns decoding off), a +// defaultEncoding that is not a string, and a _construct assigned after +// construction, which Node's constructor never reads. +import { Writable, Readable } from "node:stream"; +const opts = { decode: false, enc: "base64" }; +const w = new Writable({ + decodeStrings: opts.decode, + defaultEncoding: opts.enc, + write(c, e, cb) { console.log("js", typeof c, e, String(c)); cb(); }, +}); +w.write("aGk="); +try { new Writable({ defaultEncoding: 5 }); } catch (e) { console.log("bad", e.code, e.message); } +const zero = new Writable({ decodeStrings: 0, write(c, e, cb) { console.log("zero", Buffer.isBuffer(c), e); cb(); } }); +zero.write("z"); +const late = new Writable({ write(chunk, enc, cb) { console.log("write", chunk.toString()); cb(); } }); +let ran = false; +late._construct = (cb) => { ran = true; cb(); }; +late.write("a"); +late.end(() => console.log("ended, construct ran:", ran)); +const r = new Readable({ read() {} }); +const order = []; +(order.push("receiver"), r)._construct = (order.push("value"), function (cb) { console.log("never"); cb(); }); +console.log(order.join(",")); +r.push("x"); +r.push(null); +r.on("data", (d) => console.log("data", d.toString())); diff --git a/tests/corpus/7465-transform-callback-values.ts b/tests/corpus/7465-transform-callback-values.ts new file mode 100644 index 000000000..e7003e894 --- /dev/null +++ b/tests/corpus/7465-transform-callback-values.ts @@ -0,0 +1,41 @@ +// A transform or flush callback passing a value that is not a Buffer or a +// string: the push reports Node's ERR_INVALID_ARG_TYPE through 'error' +// and destroys the stream, for option callbacks and _transform methods, +// while Buffer and string values take the typed path. +import { Transform } from "node:stream"; +type TransformCallback = (error?: Error | null, data?: any) => void; +const t = new Transform({ + transform(chunk: Buffer, _enc: BufferEncoding, cb: TransformCallback): void { + console.log("transform", chunk.toString()); + cb(null, 42); + }, +}); +t.on("error", (e: Error & { code?: string }) => console.log("error", e.name, e.code, e.message)); +t.on("data", (d: Buffer) => console.log("data", d.toString())); +t.on("close", () => console.log("close", t.destroyed)); +console.log("write", t.write("a")); +const f = new Transform({ + transform(chunk: Buffer, _enc: BufferEncoding, cb: TransformCallback): void { cb(null, chunk); }, + flush(cb: TransformCallback): void { cb(null, { tail: true }); }, +}); +f.on("error", (e: Error & { code?: string }) => console.log("flush error", e.code, e.message)); +f.on("data", (d: Buffer) => console.log("flush data", d.toString())); +f.on("close", () => console.log("flush close")); +f.end("z"); +class Bad extends Transform { + _transform(chunk: Buffer, _enc: BufferEncoding, cb: TransformCallback): void { + cb(null, chunk.length); + } +} +const bad = new Bad(); +bad.on("error", (e: Error & { code?: string }) => console.log("error", e.code, e.message)); +bad.on("close", () => console.log("close")); +bad.write("abc"); +class Good extends Transform { + _transform(chunk: Buffer, _enc: BufferEncoding, cb: TransformCallback): void { + cb(null, chunk.toString().toUpperCase()); + } +} +const good = new Good(); +good.on("data", (d: Buffer) => console.log("good", d.toString())); +good.end("xyz"); diff --git a/tests/harness/corpus-costs/7460-transform-writev.mjs.json b/tests/harness/corpus-costs/7460-transform-writev.mjs.json new file mode 100644 index 000000000..f6c21fd3d --- /dev/null +++ b/tests/harness/corpus-costs/7460-transform-writev.mjs.json @@ -0,0 +1 @@ +{ "plain": 2266, "san": 2991 } diff --git a/tests/harness/corpus-costs/7461-transform-writev.cjs.json b/tests/harness/corpus-costs/7461-transform-writev.cjs.json new file mode 100644 index 000000000..b5868c90b --- /dev/null +++ b/tests/harness/corpus-costs/7461-transform-writev.cjs.json @@ -0,0 +1 @@ +{ "plain": 1828, "san": 2413 } diff --git a/tests/harness/corpus-costs/7462-transform-writev.ts.json b/tests/harness/corpus-costs/7462-transform-writev.ts.json new file mode 100644 index 000000000..f8fd66c2a --- /dev/null +++ b/tests/harness/corpus-costs/7462-transform-writev.ts.json @@ -0,0 +1 @@ +{ "plain": 2178, "san": 2875 } diff --git a/tests/harness/corpus-costs/7463-stream-option-values.ts.json b/tests/harness/corpus-costs/7463-stream-option-values.ts.json new file mode 100644 index 000000000..0f66a4266 --- /dev/null +++ b/tests/harness/corpus-costs/7463-stream-option-values.ts.json @@ -0,0 +1 @@ +{ "plain": 1786, "san": 4192 } diff --git a/tests/harness/corpus-costs/7464-stream-option-values.mjs.json b/tests/harness/corpus-costs/7464-stream-option-values.mjs.json new file mode 100644 index 000000000..b609b128f --- /dev/null +++ b/tests/harness/corpus-costs/7464-stream-option-values.mjs.json @@ -0,0 +1 @@ +{ "plain": 1678, "san": 4502 } diff --git a/tests/harness/corpus-costs/7465-transform-callback-values.ts.json b/tests/harness/corpus-costs/7465-transform-callback-values.ts.json new file mode 100644 index 000000000..1c67bb43f --- /dev/null +++ b/tests/harness/corpus-costs/7465-transform-callback-values.ts.json @@ -0,0 +1 @@ +{ "plain": 1571, "san": 2074 } diff --git a/tests/harness/self-hosting-native-toolchain.test.ts b/tests/harness/self-hosting-native-toolchain.test.ts index 4d5546ab6..669a3a0d6 100644 --- a/tests/harness/self-hosting-native-toolchain.test.ts +++ b/tests/harness/self-hosting-native-toolchain.test.ts @@ -108,7 +108,7 @@ for (const backend of ["llvm"] as const) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 33, marker: "scr_runtime_abi_v33" }, + runtime_abi: { version: 34, marker: "scr_runtime_abi_v34" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: [], sanitizer: "external-toolchain-required" }, flavors: { From 2ebb89b0ef90150b2f5015732b31fe64c90c9908 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 01:24:41 -0500 Subject: [PATCH 020/184] Store number arrays of proven int32s in 4-byte slots across direct calls An int32 array candidate can now be a module global or a parameter, not only a local. A number[] parameter qualifies when its function is reached by direct calls alone (not as a value, a method or a constructor, and not through a result-slot entry the backend calls in its place), it is never written, and every call passes a candidate at its position; a local, global or parameter may then be passed there as well. A global qualifies when every function reads it only in candidate positions and every write stores a fresh array. moduleInt32ArrayCandidates finds them by dropping, pass after pass, the ones a function or call disqualifies. The candidates one array can reach through calls form a group (int32ArrayGroups): the int32 slot analysis keeps or drops a group as one, since a store through any member writes the same array. A module with a module namespace object keeps its globals in doubles, as member reads can reach any global by name. --- packages/compiler/src/backend/llvm/emitter.ts | 22 +- packages/compiler/src/ir/int32-arrays.ts | 361 ++++++++++++++++-- packages/compiler/src/ir/int32-slots.ts | 102 +++-- .../test/int32-element-array-emission.test.ts | 39 ++ .../7580-int32-arrays-across-calls.ts.json | 6 + .../corpus/7580-int32-arrays-across-calls.ts | 146 +++++++ .../7580-int32-arrays-across-calls.ts.json | 1 + 7 files changed, 608 insertions(+), 69 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7580-int32-arrays-across-calls.ts.json create mode 100644 tests/corpus/7580-int32-arrays-across-calls.ts create mode 100644 tests/harness/corpus-costs/7580-int32-arrays-across-calls.ts.json diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index c0cc9fef2..bfa02dfca 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -68,7 +68,7 @@ import { type StringSplit, } from "./split-loops.js"; import { scalarizeNumericRecords } from "../../ir/scalar-records.js"; -import { addRecordResultEntries } from "./record-results.js"; +import { addRecordResultEntries, resultEntryName } from "./record-results.js"; import { specializeCallbackCalls } from "../../ir/callback-specialization.js"; import { type ChunkResult, @@ -82,6 +82,7 @@ import { versionLoopBoundParameters } from "../../ir/parameter-versions.js"; import { allocatedLiteral, fillAllocationLength, + int32ArrayKey, selectInt32Arrays, } from "../../ir/int32-arrays.js"; import { everyExpr, everyExprChild, everyStmtChild, everyStmtList } from "../../ir/traverse.js"; @@ -509,6 +510,15 @@ function provenInt32Arrays(mod: IrModule): ReadonlySet { return new Set([...slots.int32Arrays, ...slots.checkedInt32Arrays]); } +/** The functions whose calls the backend may emit as calls of their + * result-slot entries (record-results.ts). */ +function resultEntryTargets(mod: IrModule): ReadonlySet { + const names = new Set(mod.functions.map((fn) => fn.name)); + const targets = new Set(); + for (const name of names) if (names.has(resultEntryName(name))) targets.add(name); + return targets; +} + /** The emitter for `input` after the module passes every emission runs. */ function moduleEmitter(input: IrModule, options: LlvmTargetOptions): LlEmitter { // Facts are keyed by node identity: emit the same module whether it was @@ -544,6 +554,7 @@ function moduleEmitter(input: IrModule, options: LlvmTargetOptions): LlEmitter { ), ), provenInt32Arrays, + resultEntryTargets, ) : mod, options, @@ -780,7 +791,7 @@ export class LlEmitter { everyStmtList(fn.body, { stmt: (s) => { if (s.kind !== "varDecl" && s.kind !== "assign") return true; - const key = `${fn.name}\0${s.localId}`; + const key = int32ArrayKey(fn.name, s.localId); const checked = checkedInt32Arrays.has(key); if (!checked && !int32Arrays.has(key)) return true; const value = s.kind === "varDecl" ? s.init : s.value; @@ -798,15 +809,16 @@ export class LlEmitter { }); } - /** Whether `arr` is one of the current function's int32 arrays - * (ir/int32-arrays.ts, kept by the int32 slot analysis). */ + /** Whether `arr` reads one of the int32 arrays (ir/int32-arrays.ts, kept + * by the int32 slot analysis): a local or parameter of the current + * function, or a module global. */ int32Array(arr: IrExpr): boolean { if (arr.kind !== "varRef") return false; if (this.int32KindArray(arr)) return true; if (this.int32Slots.int32Arrays.size === 0) return false; return ( this.int32ArrayTemps.has(arr.localId) || - this.int32Slots.int32Arrays.has(`${this.currentFunctionName}\0${arr.localId}`) + this.int32Slots.int32Arrays.has(int32ArrayKey(this.currentFunctionName, arr.localId)) ); } diff --git a/packages/compiler/src/ir/int32-arrays.ts b/packages/compiler/src/ir/int32-arrays.ts index 02de7268f..0872fcddb 100644 --- a/packages/compiler/src/ir/int32-arrays.ts +++ b/packages/compiler/src/ir/int32-arrays.ts @@ -1,3 +1,4 @@ +import { exprFunctionReference, moduleFunctionReferences } from "./function-references.js"; import { F64, type IrExpr, type IrFunction, type IrModule, type IrStmt } from "./ir.js"; type SwitchCase = Extract["cases"][number]; @@ -12,16 +13,21 @@ import { type IrTransform, } from "./traverse.js"; -/** Local number arrays the backend may store as int32 slots. +/** Number arrays the backend may store as int32 slots. * - * A candidate is a local `number[]` of a synchronous function that no - * other binding can hold and only compiled element operations reach: - * every write of the local stores a fresh array (an array literal, or - * `new Array(n).fill(v)`), and every read of it is the receiver of an - * element read, write, state test, `length`, `push` or `pop` (a discarded - * `pop` included), or the array of an element-reader helper call. The - * array never reaches a runtime function that reads its slots, a library - * call, a closure, a return or untyped code. + * A candidate is a `number[]` binding that only compiled element + * operations reach: a local of a synchronous function, a module global, or + * a parameter. Every write of a local or a global stores a fresh array (an + * array literal, or `new Array(n).fill(v)`); a parameter is never written. + * Every read of a candidate is the receiver of an element read, write, + * state test, `length`, `push` or `pop` (a discarded `pop` included), the + * array of an element-reader helper call, or the argument of a direct call + * whose parameter at that position is a candidate itself. A parameter is a + * candidate only when its function is reached by direct calls alone and + * every call passes a candidate there, so every binding that can hold one + * of these arrays is a candidate: the array never reaches a runtime + * function that reads its slots, a library call, a closure, a return or + * untyped code. * * Candidates are only proposed here. The int32 slot analysis * (int32-slots.ts) keeps the ones whose every stored element is a proven @@ -33,6 +39,18 @@ import { * calls are inlined first (rewriteCandidates), so no shared helper ever * receives an int32 array. */ +/** The candidate key of the binding `localId` as `fnName` reads it: + * `function\0local` for a local or parameter, `\0global` for a module + * global (`%g.` ids), which every function reads alike. */ +export function int32ArrayKey(fnName: string, localId: string): string { + return localId.startsWith("%g.") ? `\0${localId}` : `${fnName}\0${localId}`; +} + +/** The int32 element slot (int32-slots.ts) of a candidate key. */ +export function int32ArraySlot(key: string): string { + return key.startsWith("\0") ? `g:${key.slice(1)}` : `l:${key}`; +} + /** Array methods a candidate may be the receiver of: the runtime paths * behind them read and write int32 slots through the double-valued slot * accessors. */ @@ -202,10 +220,51 @@ function writesLocal(e: IrExpr, id: string): boolean { return !expr(e); } -/** The candidate locals of `fn` (see the module comment). */ -export function int32ArrayCandidates(fn: IrFunction, readers: ReadonlySet): Set { +/** The parameters and globals a module's functions share as candidates, + * still assumed to qualify. */ +export interface SharedCandidates { + /** Candidate parameters, as `function\0local` keys. */ + params: ReadonlySet; + /** Candidate globals, as `%g.` ids. */ + globals: ReadonlySet; + /** Whether the parameter at `index` of `callee` is a candidate. */ + paramAt: (callee: string, index: number) => boolean; +} + +const NO_SHARED: SharedCandidates = { + params: new Set(), + globals: new Set(), + paramAt: () => false, +}; + +/** The candidates of `fn` (see the module comment): its own locals, its + * parameters in `shared.params`, and the globals of `shared.globals` that + * it reads only in candidate positions. `written` receives the globals it + * assigns fresh arrays. */ +export function int32ArrayCandidates( + fn: IrFunction, + readers: ReadonlySet, + shared: SharedCandidates = NO_SHARED, + written: Set = new Set(), +): Set { const result = new Set(); - if (fn.async || fn.generator) return result; + if (fn.async || fn.generator) { + // Globals read here leave the candidates. + for (const id of shared.globals) result.add(id); + everyStmtList(fn.body, { + expr: (e) => { + if ((e.kind === "varRef" || e.kind === "assignExpr") && result.has(e.localId)) + result.delete(e.localId); + return true; + }, + stmt: (s) => { + if ((s.kind === "varDecl" || s.kind === "assign") && result.has(s.localId)) + result.delete(s.localId); + return true; + }, + }); + return result; + } const captured = new Set( [...(fn.captures ?? []), ...(fn.classCaptures ?? [])].map((c) => c.localId), ); @@ -217,11 +276,12 @@ export function int32ArrayCandidates(fn: IrFunction, readers: ReadonlySet(); + const allocated = new Set(); // A receiver position: the candidate is read there without escaping. const receiver = (e: IrExpr): void => { if (e.kind !== "varRef") expr(e); @@ -255,13 +315,22 @@ export function int32ArrayCandidates(fn: IrFunction, readers: ReadonlySet { + if (shared.paramAt(e.callee, index)) receiver(arg); + else expr(arg); + }); return true; } } @@ -272,7 +341,8 @@ export function int32ArrayCandidates(fn: IrFunction, readers: ReadonlySet | undefined, live: ReadonlySet): Set { + const result = new Set(); + if (ids) for (const id of ids) if (live.has(id)) result.add(id); + return result; +} + +/** The candidates of every function of `mod` that has any, only those + * whose keys are in `keep` when given. Parameters and globals are shared: + * starting from every one that may qualify, each pass drops the ones a + * function reads outside candidate positions, the parameters some call + * passes anything but a candidate, and the globals never assigned a fresh + * array, until a pass drops nothing. Functions in `redirected` keep their + * parameters: some of their calls reach another function. */ +export function moduleInt32ArrayCandidates( + mod: IrModule, + readers: ReadonlySet, + keep: ReadonlySet | null = null, + redirected: ReadonlySet = new Set(), +): Map> { + const kept = (key: string): boolean => keep === null || keep.has(key); + const arrayParams = new Map(); + for (const fn of mod.functions) { + // Class methods, constructors and runtime helpers (`%` names) keep + // their parameters. + if (fn.name.startsWith("%") || fn.async || fn.generator || redirected.has(fn.name)) continue; + const indexes: number[] = []; + fn.params.forEach((p, i) => { + if (p.type.kind === "array" && p.type.elem.kind === "f64") indexes.push(i); + }); + if (indexes.length > 0) arrayParams.set(fn.name, indexes); + } + let globals = new Set(); + for (const g of mod.globals ?? []) + if ( + g.type.kind === "array" && + g.type.elem.kind === "f64" && + g.initFlag === undefined && + kept(`\0${g.id}`) + ) + globals.add(g.id); + const result = new Map>(); + if (arrayParams.size === 0 && globals.size === 0) { + for (const fn of mod.functions) { + const candidates = int32ArrayCandidates(fn, readers); + for (const id of candidates) if (!kept(`${fn.name}\0${id}`)) candidates.delete(id); + if (candidates.size > 0) result.set(fn.name, candidates); + } + return result; + } + // One walk: the functions reached other than by direct calls, the direct + // calls of functions with number[] parameters, the globals each function + // reads or writes, and whether a module namespace object exists (its + // member reads can reach any global by name). + const reached = new Set(moduleFunctionReferences(mod)); + const calls: { caller: string; call: IrExpr & { kind: "call" } }[] = []; + const globalsIn = new Map>(); + let namespaces = false; + for (const fn of mod.functions) { + const used = new Set(); + const note = (id: string): void => { + if (globals.has(id)) used.add(id); + }; + everyStmtList(fn.body, { + expr: (e) => { + const name = exprFunctionReference(e); + if (name !== undefined) reached.add(name); + if (e.kind === "moduleNsRef") namespaces = true; + if (e.kind === "call" && arrayParams.has(e.callee)) + calls.push({ caller: fn.name, call: e }); + if (e.kind === "varRef" || e.kind === "assignExpr") note(e.localId); + return true; + }, + stmt: (s) => { + if (s.kind === "varDecl" || s.kind === "assign") note(s.localId); + return true; + }, + }); + if (used.size > 0) globalsIn.set(fn.name, used); + } + if (namespaces) { + globals = new Set(); + globalsIn.clear(); + } + const paramIds = new Map(); + let params = new Set(); + for (const fn of mod.functions) { + const indexes = arrayParams.get(fn.name); + if (!indexes || reached.has(fn.name)) continue; + paramIds.set( + fn.name, + fn.params.map((p) => p.localId), + ); + for (const i of indexes) { + const key = `${fn.name}\0${fn.params[i]!.localId}`; + if (kept(key)) params.add(key); + } + } + for (;;) { + const current = params; + const written = new Set(); + const liveGlobals = new Set(globals); + result.clear(); + for (const fn of mod.functions) { + const shared: SharedCandidates = { + params: current, + globals: liveIn(globalsIn.get(fn.name), globals), + paramAt: (callee, index) => { + const id = paramIds.get(callee)?.[index]; + return id !== undefined && current.has(`${callee}\0${id}`); + }, + }; + const candidates = int32ArrayCandidates(fn, readers, shared, written); + for (const id of shared.globals) if (!candidates.has(id)) liveGlobals.delete(id); + for (const id of candidates) + if (!shared.globals.has(id) && !kept(`${fn.name}\0${id}`)) candidates.delete(id); + if (candidates.size > 0) result.set(fn.name, candidates); + } + for (const id of liveGlobals) if (!written.has(id)) liveGlobals.delete(id); + const live = new Set( + [...current].filter((key) => { + const cut = key.indexOf("\0"); + return result.get(key.slice(0, cut))?.has(key.slice(cut + 1)) === true; + }), + ); + // A parameter needs a candidate at that position of every call; a call + // with fewer arguments passes undefined. + for (const { caller, call } of calls) { + const ids = paramIds.get(call.callee); + if (!ids) continue; + ids.forEach((id, i) => { + const key = `${call.callee}\0${id}`; + if (!live.has(key)) return; + const arg = call.args[i]; + const ok = + arg?.kind === "varRef" && + (arg.localId.startsWith("%g.") + ? liveGlobals.has(arg.localId) + : result.get(caller)?.has(arg.localId) === true); + if (!ok) live.delete(key); + }); + } + if (live.size === current.size && liveGlobals.size === globals.size) break; + params = live; + globals = liveGlobals; + } + return result; +} + +/** The candidates one array can reach together: a direct call's argument + * and the parameter it binds hold the same array, so their element facts + * stand or fall together. `group` maps each of `keys` to its group's keys + * (itself alone for most), and `globalReaders` each candidate global's + * key to the functions that read or write it. */ +export function int32ArrayGroups( + mod: IrModule, + keys: ReadonlySet, +): { group: Map; globalReaders: Map> } { + const parent = new Map(); + const root = (key: string): string => { + let r = key; + while (parent.has(r)) r = parent.get(r)!; + if (r !== key) parent.set(key, r); + return r; + }; + const globalReaders = new Map>(); + const params = new Map(mod.functions.map((fn) => [fn.name, fn.params])); + const paramKeys = new Set( + mod.functions.flatMap((fn) => fn.params.map((p) => `${fn.name}\0${p.localId}`)), + ); + if ([...keys].some((key) => key.startsWith("\0") || paramKeys.has(key))) { + for (const fn of mod.functions) { + const touch = (id: string): void => { + const key = int32ArrayKey(fn.name, id); + if (!key.startsWith("\0") || !keys.has(key)) return; + const set = globalReaders.get(key) ?? new Set(); + set.add(fn.name); + globalReaders.set(key, set); + }; + everyStmtList(fn.body, { + expr: (e) => { + if (e.kind === "varRef" || e.kind === "assignExpr") touch(e.localId); + if (e.kind === "call") + e.args.forEach((arg, i) => { + const param = params.get(e.callee)?.[i]; + if (arg.kind !== "varRef" || !param) return; + const a = int32ArrayKey(fn.name, arg.localId), + p = `${e.callee}\0${param.localId}`; + if (!keys.has(a) || !keys.has(p)) return; + const ra = root(a), + rp = root(p); + if (ra !== rp) parent.set(ra, rp); + }); + return true; + }, + stmt: (s) => { + if (s.kind === "varDecl" || s.kind === "assign") touch(s.localId); + return true; + }, + }); + } + } + const members = new Map(); + for (const key of keys) { + const r = root(key); + const list = members.get(r); + if (list) list.push(key); + else members.set(r, [key]); + } + const group = new Map(); + for (const key of keys) group.set(key, members.get(root(key))!); + return { group, globalReaders }; +} + /** The element values stored into each candidate: element writes, pushed * values, literal elements and fill values. */ export function int32ArrayStores( @@ -471,40 +759,45 @@ function rewriteCandidates( return { ...fn, body, locals: locals.filter((local) => !popLocals.has(local.id)) }; } -/** Propose the module's int32 array candidates (`function\0local` keys in +/** Propose the module's int32 array candidates (int32ArrayKey keys in * `int32ArrayCandidates`), only those in `keep` when given, and rewrite * their functions. */ export function markInt32ArrayCandidates( mod: IrModule, keep: ReadonlySet | null = null, + redirected: ReadonlySet = new Set(), ): IrModule { if (mod.lib !== undefined) return mod; const readers = new Map(); for (const fn of mod.functions) if (isElementReader(fn)) readers.set(fn.name, fn); - const names = new Set(readers.keys()); - const keys: string[] = []; + const all = moduleInt32ArrayCandidates(mod, new Set(readers.keys()), keep, redirected); + if (all.size === 0) return mod; + const keys = new Set(); const functions = mod.functions.map((fn) => { - const candidates = int32ArrayCandidates(fn, names); - for (const id of candidates) - if (keep === null || keep.has(`${fn.name}\0${id}`)) keys.push(`${fn.name}\0${id}`); - else candidates.delete(id); - return candidates.size === 0 ? fn : rewriteCandidates(fn, candidates, readers); + const candidates = all.get(fn.name); + if (!candidates) return fn; + for (const id of candidates) keys.add(int32ArrayKey(fn.name, id)); + return rewriteCandidates(fn, candidates, readers); }); - return keys.length === 0 ? mod : { ...mod, functions, int32ArrayCandidates: keys }; + return { ...mod, functions, int32ArrayCandidates: [...keys] }; } /** The module the backend emits: rewritten for exactly the candidates the * int32 slot analysis (`prove`, returning its int32Arrays) keeps, so a * module without int32 arrays is `mod` itself. A dropped candidate's * rewrite is undone by marking again without it, until every candidate - * marked is kept. */ + * marked is kept. `redirected` names the functions some of whose calls the + * backend emits as calls of another function (a result-slot entry): their + * parameters keep doubles. */ export function selectInt32Arrays( mod: IrModule, prove: (marked: IrModule) => ReadonlySet, + redirected: (mod: IrModule) => ReadonlySet = () => new Set(), ): IrModule { let keep: ReadonlySet | null = null; + const others = redirected(mod); for (let round = 0; round < 4; round++) { - const marked = markInt32ArrayCandidates(mod, keep); + const marked = markInt32ArrayCandidates(mod, keep, others); if (marked === mod) return mod; const kept = prove(marked); const proposed = marked.int32ArrayCandidates ?? []; diff --git a/packages/compiler/src/ir/int32-slots.ts b/packages/compiler/src/ir/int32-slots.ts index 4a2abc5b0..a8e91c86e 100644 --- a/packages/compiler/src/ir/int32-slots.ts +++ b/packages/compiler/src/ir/int32-slots.ts @@ -10,7 +10,13 @@ import { } from "./integer-ranges.js"; import { exprFunctionReference, moduleFunctionReferences } from "./function-references.js"; import { findInitializerBindings, withInitializerBindings } from "./initializer-bindings.js"; -import { checkableInt32Allocations, int32ArrayStores } from "./int32-arrays.js"; +import { + checkableInt32Allocations, + int32ArrayGroups, + int32ArrayKey, + int32ArraySlot, + int32ArrayStores, +} from "./int32-arrays.js"; import { everyExprChild, everyStmtChild, everyStmtList, everyTypeChild } from "./traverse.js"; /** The storage slot an array-valued expression reads, as an @@ -247,9 +253,10 @@ export class Int32Slots { /** Whether the module has int32 array candidates (the narrowedReads * range fact). */ narrowedReads = false; - /** Local number arrays (`function\0local`) stored as int32 slots: the - * module's int32 array candidates whose every stored element the - * fixpoint proved an int32. */ + /** Number arrays stored as int32 slots (int32ArrayKey keys: locals and + * parameters as `function\0local`, globals as `\0global`): the module's + * int32 array candidates whose every stored element, through every + * binding that can hold the same array, the fixpoint proved an int32. */ readonly int32Arrays = new Set(); /** Local number arrays that may hold int32 slots with checked stores: the * candidates whose element stores the fixpoint proved exact integers but @@ -779,17 +786,29 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}): for (const key of contracts?.int32Locals ?? []) slots.addInt32Local(key); for (const key of contracts?.int32Elements ?? []) slots.addInt32Elements(key); // Int32 array candidates (ir/int32-arrays.ts): assumed to hold int32 - // elements until a store the fixpoint cannot prove drops one. + // elements until a store the fixpoint cannot prove drops one, with every + // candidate the same array can reach (its group). const arrayCandidates = new Map>(); - slots.narrowedReads = (mod.int32ArrayCandidates?.length ?? 0) > 0; - for (const key of mod.int32ArrayCandidates ?? []) { + const globalArrays = new Set(); + const arrayKeys = new Set(mod.int32ArrayCandidates ?? []); + slots.narrowedReads = arrayKeys.size > 0; + for (const key of arrayKeys) { const cut = key.indexOf("\0"); - const fnName = key.slice(0, cut); - const set = arrayCandidates.get(fnName) ?? new Set(); - set.add(key.slice(cut + 1)); - arrayCandidates.set(fnName, set); - slots.addInt32Elements(`l:${key}`); + if (cut === 0) globalArrays.add(key.slice(1)); + else { + const fnName = key.slice(0, cut); + const set = arrayCandidates.get(fnName) ?? new Set(); + set.add(key.slice(cut + 1)); + arrayCandidates.set(fnName, set); + } + slots.addInt32Elements(int32ArraySlot(key)); } + const arrayGroups = arrayKeys.size > 0 ? int32ArrayGroups(mod, arrayKeys) : null; + // Functions that read or write a candidate global. + const globalArrayFunctions = new Set(); + if (arrayGroups) + for (const names of arrayGroups.globalReaders.values()) + for (const name of names) globalArrayFunctions.add(name); /* ── candidate fields ── */ const children = new Map(); @@ -1070,6 +1089,7 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}): }; const hasCandidateWrite = (fn: IrFunction): boolean => { if (arrayCandidates.has(fn.name)) return true; + if (globalArrayFunctions.has(fn.name)) return true; const box = (localId: string): boolean => slots.hasBox(slots.boxGroup(fn.name, localId)); const returns = slots.hasReturn(fn.name); return !everyStmtList(fn.body, { @@ -1124,25 +1144,49 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}): // (checkableInt32Allocations), it keeps int32 slots with checked stores // (checkedInt32Arrays), which widen it to doubles in place at the first // store that is not an int32. - const arrays = arrayCandidates.get(fn.name); - if (arrays) { - let dropped = false; + const locals = arrayCandidates.get(fn.name); + let arrays = locals; + if (globalArrayFunctions.has(fn.name)) { + const both = new Set(); + if (locals) for (const id of locals) both.add(id); + for (const id of globalArrays) both.add(id); + arrays = both; + } + if (arrays && arrayGroups) { + const requeue = (name: string): void => { + if (queued.has(name)) return; + queued.add(name); + queue.push(name); + }; + const params = new Set(fn.params.map((param) => param.localId)); for (const [id, values] of int32ArrayStores(fn, arrays)) { - const key = `l:${fn.name}\0${id}`; - if (!slots.hasInt32ElementSlot(key) || values.every(int32)) continue; - slots.dropInt32Elements(key); + const key = int32ArrayKey(fn.name, id); + if (!slots.hasInt32ElementSlot(int32ArraySlot(key)) || values.every(int32)) continue; + // The array's group drops together: every function reading a + // member is analyzed again without its facts. + const group = arrayGroups.group.get(key) ?? [key]; + for (const member of group) { + slots.dropInt32Elements(int32ArraySlot(member)); + if (member.startsWith("\0")) { + const names = arrayGroups.globalReaders.get(member); + if (names) for (const name of names) requeue(name); + } else requeue(member.slice(0, member.indexOf("\0"))); + } + // Checked stores serve a lone local: its allocations are the only + // arrays it holds. const checkable = (value: IrExpr): boolean => { const range = ranges.get(value); return int32(value) || range != null; }; - if (values.every(checkable) && checkableInt32Allocations(fn, id)) - slots.checkedInt32Arrays.add(`${fn.name}\0${id}`); - else reason(`array:${fn.name}\0${id}`, "an element store is not int32"); - dropped = true; - } - if (dropped && !queued.has(fn.name)) { - queued.add(fn.name); - queue.push(fn.name); + if ( + group.length === 1 && + !key.startsWith("\0") && + !params.has(id) && + values.every(checkable) && + checkableInt32Allocations(fn, id) + ) + slots.checkedInt32Arrays.add(key); + else for (const member of group) reason(`array:${member}`, "an element store is not int32"); } } // An int32 written into a slot still assumed non-negative: widen the @@ -1213,10 +1257,8 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}): } for (const proven of provenByFunction.values()) for (const value of proven) slots.provenWrites.add(value); - for (const [fnName, ids] of arrayCandidates) - for (const id of ids) - if (slots.hasInt32ElementSlot(`l:${fnName}\0${id}`)) - slots.int32Arrays.add(`${fnName}\0${id}`); + for (const key of arrayKeys) + if (slots.hasInt32ElementSlot(int32ArraySlot(key))) slots.int32Arrays.add(key); const ranged = rangedParameters(slots, functions, retry, numeric, (callee) => readers.get(`ret:${callee}`), ); diff --git a/packages/compiler/test/int32-element-array-emission.test.ts b/packages/compiler/test/int32-element-array-emission.test.ts index 0b68a5d3e..cd62278db 100644 --- a/packages/compiler/test/int32-element-array-emission.test.ts +++ b/packages/compiler/test/int32-element-array-emission.test.ts @@ -196,3 +196,42 @@ test("only the candidates the proof keeps are rewritten", async () => { for (const fn of selected.functions) if (fn.name !== "compose") expect(fn).toBe(mod.functions.find((f) => f.name === fn.name)); }); + +const sharedSource = ` +const data: number[] = []; +for (let i = 0; i < 64; i++) data.push((i * 7) & 255); +function fnv(values: number[]): number { + let h = 2166136261; + for (let i = 0; i < values.length; i++) h = Math.imul(h ^ values[i]!, 16777619) >>> 0; + return h; +} +function mean(values: number[]): number { + let s = 0; + for (let i = 0; i < values.length; i++) s += values[i]!; + return s / values.length; +} +function first(values: number[]): number { + return values[0]!; +} +const ints = [1, 2, 3]; +const halves = [0.5, 1.5]; +const pick: (values: number[]) => number = first; +console.log(fnv(data), mean(ints), mean(halves), pick([4])); +`; + +test("arrays passed to helpers and held in module globals get 4-byte slots", async () => { + const mod = await lowerIr(sharedSource); + const proposed = markInt32ArrayCandidates(mod).int32ArrayCandidates ?? []; + // A global and the parameter every call passes it to; a function used as + // a value keeps its parameter. + expect(proposed).toEqual(expect.arrayContaining(["\0%g.e.data", "fnv\0values.0"])); + expect(proposed.some((key) => key.startsWith("first\0"))).toBe(false); + const llvm = await emitLlvm(sharedSource); + // The helper reads int32 slots; `mean`, also called with fractions, + // reads doubles. + expect(functions(llvm, "fnv")).toMatch( + /getelementptr inbounds i32, ptr %\w+, i64 %\w+\n\s+%\w+ = load i32/, + ); + expect(functions(llvm, "mean")).not.toMatch(/getelementptr inbounds i32, ptr/); + expect(llvm).toMatch(/@scr_arr_new(_small)?\(i32 7/); +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7580-int32-arrays-across-calls.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7580-int32-arrays-across-calls.ts.json new file mode 100644 index 000000000..33282756c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7580-int32-arrays-across-calls.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7580-int32-arrays-across-calls.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7580-int32-arrays-across-calls.ts b/tests/corpus/7580-int32-arrays-across-calls.ts new file mode 100644 index 000000000..13df54609 --- /dev/null +++ b/tests/corpus/7580-int32-arrays-across-calls.ts @@ -0,0 +1,146 @@ +// Number arrays of proven int32s that reach other functions: passed to +// helpers called directly, along chains of helpers and recursion, and held +// in module globals that several functions read. A helper that also +// receives an array of other numbers or one that is not a fresh array, +// stores a fraction through its parameter, or escapes as a value keeps +// every array it can reach in doubles, as does a global an async function +// reads. Each case uses its own arrays and helpers. + +const data: number[] = []; +let seed = 7; +for (let i = 0; i < 2000; i++) { + seed = (Math.imul(seed, 48271) + 11) & 0x7fffffff; + data.push(seed % 1000); +} + +function fnv(values: number[]): number { + let h = 2166136261; + for (let i = 0; i < values.length; i++) { + h ^= values[i]! & 0xff; + h = Math.imul(h, 16777619) >>> 0; + } + return h; +} + +function histogram(values: number[], buckets: number): number[] { + const counts: number[] = new Array(buckets).fill(0); + for (let i = 0; i < values.length; i++) { + const b = values[i]! % buckets; + counts[b] = counts[b]! + 1; + } + return counts; +} + +// A helper that writes through its parameter and calls another helper. +function scale(values: number[], factor: number): number { + for (let i = 0; i < values.length; i++) values[i] = (values[i]! * factor) & 0xffff; + return sum(values); +} + +function sum(values: number[]): number { + let s = 0; + for (let i = 0; i < values.length; i++) s = (s + values[i]!) | 0; + return s; +} + +// Recursion passes the array on, and the elements index another array. +function chase(next: number[], at: number, steps: number): number { + if (steps === 0) return at; + return chase(next, next[at]!, steps - 1); +} + +function pushAll(target: number[], count: number): void { + for (let i = 0; i < count; i++) target.push((i * 3) & 0xffff); +} + +// `n` is a power of two: the masked elements are indexes of `perm`. +function locals(n: number): string { + const perm: number[] = []; + for (let i = 0; i < n; i++) perm.push((i * 37 + 11) & (n - 1)); + const copy: number[] = []; + pushAll(copy, n); + const s1 = scale(copy, 5); + return `locals ${chase(perm, 0, 1000)} ${s1} ${copy[n - 1]} ${copy.length} ${sum(perm)}`; +} + +// Read only through the element reader (`a[i] ?? 0`) and states. +function readSparse(values: number[], at: number): string { + return `${values[at] ?? -1} ${at in values} ${values.length}`; +} + +// The same helper receives an array of other numbers too: the arrays it +// can reach keep doubles. +function mean(values: number[]): number { + let s = 0; + for (let i = 0; i < values.length; i++) s += values[i]!; + return s / values.length; +} + +// A store through the parameter that is not an int32. +function halve(values: number[]): void { + for (let i = 0; i < values.length; i++) values[i] = values[i]! / 2; +} + +// A function used as a value keeps its parameters. +function first(values: number[]): number { + return values[0]!; +} + +let reassigned: number[] = [1, 2, 3]; +function regrow(n: number): number { + reassigned = []; + for (let i = 0; i < n; i++) reassigned.push(i & 15); + let s = 0; + for (let i = 0; i < reassigned.length; i++) s += reassigned[i]!; + return s; +} + +// Not a fresh array: the result of a call. +function total(values: number[]): number { + let s = 0; + for (let i = 0; i < values.length; i++) s += values[i]!; + return s; +} + +const awaited: number[] = [4, 5, 6]; +async function later(): Promise { + return awaited[0]! + awaited.length; +} + +// A helper whose record result its callers only project: the backend may +// call its result-slot entry instead, so its parameter keeps doubles. +interface Span { + lo: number; + hi: number; +} +function spanAt(values: number[], i: number): Span { + return { lo: values[i]!, hi: values[i + 1]! }; +} +const spans: number[] = [5, 9, 2, 7, 11]; +function spanTotal(): number { + let total = 0; + for (let i = 0; i < 4; i++) { + const span = spanAt(spans, i); + total += span.hi * 10 - span.lo; + } + return total; +} + +const ints: number[] = [3, 4, 5]; +const fractions: number[] = [0.5, 1.5, 2.5]; +const halved: number[] = [8, 6, 4]; +halve(halved); +const pick: (values: number[]) => number = first; +const viaValue: number[] = [9, 8, 7]; + +console.log(`fnv ${fnv(data)} ${fnv(data)}`); +const h = histogram(data, 10); +console.log(`hist ${h.length} ${h[0]} ${h[9]} ${total(h)}`); +console.log(locals(128)); +console.log(readSparse(data, 5), readSparse(data, 5000)); +console.log(`mean ${mean(ints)} ${mean(fractions)}`); +console.log(`halved ${halved[0]} ${halved[2]} ${halved.length}`); +console.log(`value ${pick(viaValue)} ${viaValue.length}`); +console.log(`regrow ${regrow(40)} ${reassigned.length} ${reassigned[39]} ${regrow(5)}`); +console.log(`span ${spanTotal()}`); +void later().then((v) => console.log(`later ${v}`)); diff --git a/tests/harness/corpus-costs/7580-int32-arrays-across-calls.ts.json b/tests/harness/corpus-costs/7580-int32-arrays-across-calls.ts.json new file mode 100644 index 000000000..7c9d20632 --- /dev/null +++ b/tests/harness/corpus-costs/7580-int32-arrays-across-calls.ts.json @@ -0,0 +1 @@ +{ "plain": 3100, "san": 3500 } From b6d6eb60aed3849954d9f69cc0ff4607d63a7504 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 01:34:19 -0500 Subject: [PATCH 021/184] Replace a scheduled _construct hook, and keep runtime option values in order Node's constructNT reads _construct on the construction tick, so an assignment after the constructor and before that tick replaces a scheduled construct hook. The assignment was dropped, so the original hook ran. It now lowers to stream.replaceConstruct, which swaps the hook while its tick has not run and has no effect after it or on a stream without a hook. Runtime decodeStrings and defaultEncoding values are read into locals before the constructor evaluates the other option members, while Node evaluates the members in order. A runtime value after a member whose value has effects (a call or an assignment) now refuses instead of reading a value that member could still change. --- .../builtin-apis/streams-and-readline.mdx | 1 + .../compiler/src/backend/llvm/lib-shared.ts | 1 + .../compiler/src/backend/llvm/lib-stream.ts | 3 + .../src/frontend/lowering/lower-stream.ts | 63 +++++++++++++------ .../compiler/src/ir/builtin-signatures.ts | 1 + packages/compiler/src/ir/validate.ts | 1 + .../7466-stream-construct-assign.mjs.json | 6 ++ .../stream-runtime-option-order.ts.json | 6 ++ packages/runtime/src/scr_stream.c | 13 ++++ packages/runtime/src/scr_stream.h | 4 ++ tests/corpus/7466-stream-construct-assign.mjs | 46 ++++++++++++++ .../stream-runtime-option-order.ts | 28 +++++++++ .../stream-runtime-option-order.ts.txt | 17 +++++ .../7466-stream-construct-assign.mjs.json | 1 + 14 files changed, 173 insertions(+), 18 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7466-stream-construct-assign.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/stream-runtime-option-order.ts.json create mode 100644 tests/corpus/7466-stream-construct-assign.mjs create mode 100644 tests/diagnostics/stream-runtime-option-order.ts create mode 100644 tests/harness/__snapshots__/stream-runtime-option-order.ts.txt create mode 100644 tests/harness/corpus-costs/7466-stream-construct-assign.mjs.json diff --git a/docs/content/limitations/builtin-apis/streams-and-readline.mdx b/docs/content/limitations/builtin-apis/streams-and-readline.mdx index b88ead1f4..166fc3402 100644 --- a/docs/content/limitations/builtin-apis/streams-and-readline.mdx +++ b/docs/content/limitations/builtin-apis/streams-and-readline.mdx @@ -33,6 +33,7 @@ `pipeline` from `node:stream/promises` accepts a trailing `{ signal, end }` object, with or without function stages, and `finished` accepts `readable`, `writable`, `error`, `cleanup`, and `signal` options, with boolean literals for the flags. Stream constructors accept a `signal` option in an options object literal, and `stream.addAbortSignal` accepts any stream. The `construct` option and `_construct` method, `decodeStrings`, and `defaultEncoding` work on every stream class, and the `writev` option and `_writev` method on every class with a writable side. + A `decodeStrings` or `defaultEncoding` value known only at run time is unsupported after an option member whose value has effects (a call or an assignment). In typed code, a `writev` entry's `encoding` declared as a string receives `""` where Node passes `undefined` or `null` (an object-mode write). `setDefaultEncoding()` takes any encoding Node knows. `node:readline` and `node:readline/promises` (also reached as `readline.promises`) interfaces, from `createInterface(options)` or `createInterface(input, output, completer, terminal)`, read `process.stdin` or a readable stream: `question`, `prompt`, `write` (with a key object literal of `name`, `ctrl`, `meta`, and `shift`), `pause`, `resume`, `'line'`/`'close'`/`'pause'`/`'resume'`/`'error'`/`'history'`/`'SIGINT'`/`'SIGTSTP'`/`'SIGCONT'` listeners, and `for await` compile. diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index f31906354..25af12ad6 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1768,6 +1768,7 @@ export const USES_TIMERS_LIB_FNS = new Set([ "stream.setTransform", "stream.setFlush", "stream.setConstruct", + "stream.replaceConstruct", "stream.duplexFromGen", "stream.duplexFromDyn", "stream.duplexifyReadable", diff --git a/packages/compiler/src/backend/llvm/lib-stream.ts b/packages/compiler/src/backend/llvm/lib-stream.ts index 5fd4af1a9..fdbf13d4f 100644 --- a/packages/compiler/src/backend/llvm/lib-stream.ts +++ b/packages/compiler/src/backend/llvm/lib-stream.ts @@ -189,6 +189,7 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV e.fn === "stream.setTransform" || e.fn === "stream.setFlush" || e.fn === "stream.setConstruct" || + e.fn === "stream.replaceConstruct" || e.fn === "stream.setWritev" || e.fn === "stream.setWritevDefault" ) { @@ -203,6 +204,7 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV "stream.setTransform": "t", "stream.setFlush": "l", "stream.setConstruct": "c", + "stream.replaceConstruct": "c", "stream.setWritev": "v", "stream.setWritevDefault": "v", }; @@ -214,6 +216,7 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV "stream.setTransform": "scr_stream_set_transform", "stream.setFlush": "scr_stream_set_flush", "stream.setConstruct": "scr_stream_set_construct", + "stream.replaceConstruct": "scr_stream_replace_construct", "stream.setWritev": "scr_stream_set_writev", "stream.setWritevDefault": "scr_stream_set_writev_default", }; diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index 15afb741e..cf9f19b9d 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -106,6 +106,21 @@ function pureRead(node: ts.Expression): boolean { return false; } +/** Whether every option member before `prop` evaluates without effects. + * Runtime decodeStrings and defaultEncoding values are read into locals + * before the constructor evaluates the other members, while Node evaluates + * the members in order: an earlier member with effects could change them. */ +function earlierOptionsPure(options: ts.ObjectLiteralExpression, prop: ts.Node): boolean { + for (const member of options.properties.nodes) { + if (member === prop) return true; + if (ts.isPropertyAssignment(member)) { + if (!sideEffectFreeOptionValue(member.initializer) && !pureRead(member.initializer)) + return false; + } else if (ts.isSpreadAssignment(member)) return false; + } + return true; +} + /** The stream sides of a receiver class: the nearest stream-class * ancestor's, or null off the stream hierarchy. */ export function streamSidesOf( @@ -1021,6 +1036,16 @@ function parseStreamOptions( break; } // a runtime value: WritableState's `options.decodeStrings !== false` + if ( + optsLit !== undefined && + ts.isObjectLiteralExpression(optsLit) && + !earlierOptionsPure(optsLit, prop) + ) + lowerer.noLowering( + `the ${ctorName} option 'decodeStrings' with a runtime value after an option value with effects`, + prop, + "compute the earlier option values into consts before the options object (the runtime value is read before the other options run)", + ); const local = lowerer.declareHiddenLocal("%decodeStrings", DYN); out.valueDecls.push({ kind: "varDecl", @@ -1076,6 +1101,16 @@ function parseStreamOptions( // a runtime value: null, undefined and utf8 are the default, // another name Buffer.isEncoding knows is kept as given, and any // other value throws ERR_UNKNOWN_ENCODING before construction + if ( + optsLit !== undefined && + ts.isObjectLiteralExpression(optsLit) && + !earlierOptionsPure(optsLit, prop) + ) + lowerer.noLowering( + `the ${ctorName} option 'defaultEncoding' with a runtime value after an option value with effects`, + prop, + "compute the earlier option values into consts before the options object (the runtime value is read before the other options run)", + ); const local = lowerer.declareHiddenLocal("%defaultEncoding", DYN); const ref = varRef(local.id, DYN, loc); out.valueDecls.push( @@ -1869,24 +1904,15 @@ export function lowerStreamUnderscoreAssign( const loc = locOf(expr); const cls = base.def.name; const clsName = cls.slice(1); - if (methodName === "_construct") { - // Node's constructor reads _construct once; a later assignment has no - // effect on the stream, so only the receiver and the value are - // evaluated - const value = lowerer.lowerExpr(expr.right); - return { - kind: "exprStmt", - expr: { - kind: "seqExpr", - stmts: [{ kind: "exprStmt", expr: recv, loc }], - result: value, - type: value.type, - loc, - }, - loc, - }; - } - const which = methodName === "_writev" ? "writev" : optionByMethod.get(methodName)!; + // `_construct`: Node's constructNT reads it on the construction tick, so + // an assignment before that tick replaces a scheduled hook + // (stream.replaceConstruct) and has no effect after it or without one. + const which = + methodName === "_writev" + ? "writev" + : methodName === "_construct" + ? "construct" + : optionByMethod.get(methodName)!; const { accepted } = streamCtorShape(cls); if (!accepted.includes(which) && !streamExtraCallbacks(cls).includes(which)) { lowerer.noLowering( @@ -1925,6 +1951,7 @@ export function lowerStreamUnderscoreAssign( transform: "stream.setTransform", flush: "stream.setFlush", writev: "stream.setWritev", + construct: "stream.replaceConstruct", }; return { kind: "exprStmt", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index aa31105e7..ca91b399c 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -3975,6 +3975,7 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "stream.setTransform": { argTypes: [null, null], result: VOID }, "stream.setFlush": { argTypes: [null, null], result: VOID }, "stream.setConstruct": { argTypes: [null, null], result: VOID }, + "stream.replaceConstruct": { argTypes: [null, null], result: VOID }, "stream.setWritev": { argTypes: [null, null], result: VOID }, "stream.setWritevDefault": { argTypes: [null, null], result: VOID }, /** The dyn-options twins (a checked-dynamic options record — the JS diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index 44c032620..8b3b6c048 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -5508,6 +5508,7 @@ function validateFunction( e.fn === "stream.duplexFromGenSignal" || e.fn === "stream.duplexFromAsyncFnSignal" || e.fn === "stream.setConstruct" || + e.fn === "stream.replaceConstruct" || e.fn === "stream.setWritev" || e.fn === "stream.setWritevDefault" || e.fn === "stream.iteratorClose" || diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7466-stream-construct-assign.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7466-stream-construct-assign.mjs.json new file mode 100644 index 000000000..f5bb52517 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7466-stream-construct-assign.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7466-stream-construct-assign.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/stream-runtime-option-order.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/stream-runtime-option-order.ts.json new file mode 100644 index 000000000..40b64edac --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/stream-runtime-option-order.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/diagnostics/stream-runtime-option-order.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index 4b962675a..cd95e12cd 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -269,6 +269,7 @@ struct ScrStreamState { * destroyed and runs its _destroy once construction completes. */ bool constructing, destroy_deferred; bool construct_finished; /* onConstruct ran: a second callback is ERR_MULTIPLE_CALLBACK */ + bool construct_called; /* the construct tick called the hook */ ScrError *destroy_err; /* owned: the deferred destroy's error */ ScrClosure *construct_cb; ScrStreamPlainInv construct_inv; @@ -7523,6 +7524,17 @@ void scr_stream_set_construct(ScrStream *s, ScrClosure *cb /*moves*/, ScrStreamP scr_stream_construct(s, cb, inv); } +void scr_stream_replace_construct(ScrStream *s, ScrClosure *cb /*moves*/, ScrStreamPlainInv inv) { + ScrStreamState *st = s->st; + if (!st->constructing || st->construct_called || !st->construct_cb) { + scr_closure_release(cb); + return; + } + scr_closure_release(st->construct_cb); + st->construct_cb = cb; + st->construct_inv = inv; +} + /* A class's _writev under options that may carry their own writev (the * option, an instance property in Node, wins). */ void scr_stream_set_writev_default(ScrStream *s, ScrClosure *cb /*moves*/, ScrStreamWritevInv inv) { @@ -7791,6 +7803,7 @@ static void scr_stream_run_tick(ScrStreamTick *t) { scr_stream_error_buffer(s); break; case SCR_ST_CONSTRUCT: { + st->construct_called = true; if (st->construct_cb) { st->construct_inv(st->construct_cb, s); /* a throw inside _construct reaches onConstruct on a tick */ diff --git a/packages/runtime/src/scr_stream.h b/packages/runtime/src/scr_stream.h index e368ec7ff..a4798c629 100644 --- a/packages/runtime/src/scr_stream.h +++ b/packages/runtime/src/scr_stream.h @@ -561,6 +561,10 @@ void scr_stream_set_writev_concat(ScrStream *s); /* The user construct option / _construct method (scheduled now; cb * moves), and the writev option / _writev method (cb moves). */ void scr_stream_set_construct(ScrStream *s, ScrClosure *cb, ScrStreamPlainInv inv); +/* `s._construct = fn` after construction: replaces a construct hook whose + * tick has not run yet (Node's constructNT reads _construct then), and has + * no effect otherwise (cb moves). */ +void scr_stream_replace_construct(ScrStream *s, ScrClosure *cb, ScrStreamPlainInv inv); void scr_stream_set_writev(ScrStream *s, ScrClosure *cb, ScrStreamWritevInv inv); void scr_stream_set_writev_default(ScrStream *s, ScrClosure *cb, ScrStreamWritevInv inv); /* A compiled writev whose entries declare `encoding` a string: entries diff --git a/tests/corpus/7466-stream-construct-assign.mjs b/tests/corpus/7466-stream-construct-assign.mjs new file mode 100644 index 000000000..37684afcd --- /dev/null +++ b/tests/corpus/7466-stream-construct-assign.mjs @@ -0,0 +1,46 @@ +// Assigning _construct after construction: Node reads it on the +// construction tick, so an assignment before that tick replaces the +// scheduled hook, and one after it, or on a stream without a hook, has no +// effect. +import { Readable, Writable } from "node:stream"; + +const w = new Writable({ + construct(cb) { + console.log("original construct"); + cb(); + }, + write(chunk, _enc, cb) { + console.log("write", chunk.toString()); + cb(); + }, +}); +w._construct = function (cb) { + console.log("replacement construct"); + cb(); +}; +w.write("a"); +w.end(); +w.on("finish", () => { + w._construct = function (cb) { + console.log("late construct"); + cb(); + }; + console.log("finish"); + readable(); +}); + +// A stream without a construct hook: the assignment has no effect. +function readable() { + const r = new Readable({ + read() { + this.push("x"); + this.push(null); + }, + }); + r._construct = function (cb) { + console.log("never"); + cb(); + }; + r.on("data", (d) => console.log("data", d.toString())); + r.on("end", () => console.log("end")); +} diff --git a/tests/diagnostics/stream-runtime-option-order.ts b/tests/diagnostics/stream-runtime-option-order.ts new file mode 100644 index 000000000..b3854e328 --- /dev/null +++ b/tests/diagnostics/stream-runtime-option-order.ts @@ -0,0 +1,28 @@ +import { Writable } from "node:stream"; + +// Runtime decodeStrings and defaultEncoding values are read before the +// other option members are evaluated, while Node evaluates the members in +// order: a value after a member with effects refuses. +let encoding = (process.env["STREAM_ENCODING"] ?? "utf8") as BufferEncoding; +let decode = process.env["STREAM_DECODE"] !== "0"; +function bump(): number { + encoding = "hex"; + decode = false; + return 16; +} +const sink = new Writable({ + highWaterMark: bump(), + defaultEncoding: encoding, + write(_chunk, _encoding, done) { + done(); + }, +}); +const other = new Writable({ + highWaterMark: bump(), + decodeStrings: decode, + write(_chunk, _encoding, done) { + done(); + }, +}); +sink.end(); +other.end(); diff --git a/tests/harness/__snapshots__/stream-runtime-option-order.ts.txt b/tests/harness/__snapshots__/stream-runtime-option-order.ts.txt new file mode 100644 index 000000000..8e08dfbf0 --- /dev/null +++ b/tests/harness/__snapshots__/stream-runtime-option-order.ts.txt @@ -0,0 +1,17 @@ +stream-runtime-option-order.ts:15:3 - error SC2020: 'the Writable option 'defaultEncoding' with a runtime value after an option value with effects' is part of the standard library types but has no scriptc lowering yet + + 14 | highWaterMark: bump(), + 15 | defaultEncoding: encoding, + | ^~~~~~~~~~~~~~~~~~~~~~~~~ + 16 | write(_chunk, _encoding, done) { + + hint: compute the earlier option values into consts before the options object (the runtime value is read before the other options run) + +stream-runtime-option-order.ts:22:3 - error SC2020: 'the Writable option 'decodeStrings' with a runtime value after an option value with effects' is part of the standard library types but has no scriptc lowering yet + + 21 | highWaterMark: bump(), + 22 | decodeStrings: decode, + | ^~~~~~~~~~~~~~~~~~~~~ + 23 | write(_chunk, _encoding, done) { + + hint: compute the earlier option values into consts before the options object (the runtime value is read before the other options run) \ No newline at end of file diff --git a/tests/harness/corpus-costs/7466-stream-construct-assign.mjs.json b/tests/harness/corpus-costs/7466-stream-construct-assign.mjs.json new file mode 100644 index 000000000..1c67bb43f --- /dev/null +++ b/tests/harness/corpus-costs/7466-stream-construct-assign.mjs.json @@ -0,0 +1 @@ +{ "plain": 1571, "san": 2074 } From 2f88f352816c22ac570af7720509cf2240efe1ff Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 01:56:34 -0500 Subject: [PATCH 022/184] Freeze records that other references still hold Object.freeze of a record that another reference may still hold stopped at compile time with SC2020 ("possibly-aliased value"), and freezing a record held in an untyped value threw SC2020 at run time. Node freezes the object itself: every later write through any reference throws a TypeError in strict mode code and leaves the object unchanged in sloppy-mode code. - A record a program may freeze keeps a frozen bit in its reference count word. A shape is freezable when a typed Object.freeze names it, or, in a program that freezes untyped values, when a value of it can be held as a live untyped reference. Only those shapes change: their release ignores the bit, and every write to them, keyed writes and deletes included, checks it first. - Object.freeze of a record held in an untyped value sets the bit and freezes the record's checked view, so Object.isFrozen, Object.isSealed and Object.isExtensible through that value answer for the frozen record. A typed Object.isFrozen of a record reads the bit. - Programs that never freeze a record compile to the same IR and LLVM. Corpus 7404 covers typed and untyped freezes, a recursive freeze through unknown, keyed writes and deletes, optional-field deletes, Object.assign, tuples, and frozen records that are released or collected as cycles, in strict mode code. 7405 covers the same writes in sloppy-mode code. Both match Node. Refs #1374 Co-authored-by: blurname <50226652+blurname@users.noreply.github.com> --- .../compiler/src/backend/llvm/lib-dispatch.ts | 13 + .../compiler/src/backend/llvm/lib-shared.ts | 2 + .../compiler/src/backend/llvm/shapes.test.ts | 37 ++ packages/compiler/src/backend/llvm/shapes.ts | 35 +- .../src/backend/llvm/stack-objects.ts | 7 +- .../lowering/expressions/element-access.ts | 2 +- .../src/frontend/lowering/frozen-records.ts | 328 ++++++++++++++++++ .../src/frontend/lowering/lower-calls.ts | 24 +- .../src/frontend/lowering/lower-stmts.ts | 28 +- .../compiler/src/frontend/lowering/lowerer.ts | 10 +- packages/compiler/src/ir/builtin-effects.ts | 1 + .../compiler/src/ir/builtin-signatures.ts | 12 + packages/compiler/src/ir/ir.ts | 4 + .../tests/corpus/7404-frozen-records.ts.json | 6 + .../7405-frozen-records-sloppy.cjs.json | 6 + packages/runtime/src/scr_cycle.c | 22 +- packages/runtime/src/scr_dyn.h | 7 + packages/runtime/src/scr_json.c | 71 +++- packages/runtime/src/scr_json.test.c | 13 + packages/runtime/src/scr_rc.h | 26 +- tests/corpus/7404-frozen-records.ts | 124 +++++++ tests/corpus/7405-frozen-records-sloppy.cjs | 34 ++ .../corpus-costs/7404-frozen-records.ts.json | 1 + .../7405-frozen-records-sloppy.cjs.json | 1 + 24 files changed, 776 insertions(+), 38 deletions(-) create mode 100644 packages/compiler/src/frontend/lowering/frozen-records.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7404-frozen-records.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7405-frozen-records-sloppy.cjs.json create mode 100644 tests/corpus/7404-frozen-records.ts create mode 100644 tests/corpus/7405-frozen-records-sloppy.cjs create mode 100644 tests/harness/corpus-costs/7404-frozen-records.ts.json create mode 100644 tests/harness/corpus-costs/7405-frozen-records-sloppy.cjs.json diff --git a/packages/compiler/src/backend/llvm/lib-dispatch.ts b/packages/compiler/src/backend/llvm/lib-dispatch.ts index cebe661d4..4f1a4fef8 100644 --- a/packages/compiler/src/backend/llvm/lib-dispatch.ts +++ b/packages/compiler/src/backend/llvm/lib-dispatch.ts @@ -770,6 +770,19 @@ export function emitLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue { return { name: t, type: e.type }; } if (e.fn === "bytes.absent") return host.own({ name: "null", type: e.type }); + if (e.fn === "record.isFrozen") { + // The frozen bit of the record's rc word (scr_rc.h SCR_RC_FROZEN) is + // its second-highest bit; an immortal word sets the highest one too. + const value = host.emitReadReceiver(e.args[0]!); + const S = host.sizeType; + const rc = host.B.tmp(), + top = host.B.tmp(), + t = host.B.tmp(); + host.B.line(`${rc} = load ${S}, ptr ${value.name}`); + host.B.line(`${top} = lshr ${S} ${rc}, ${S === "i64" ? 62 : 30}`); + host.B.line(`${t} = icmp eq ${S} ${top}, 1`); + return { name: t, type: e.type }; + } const prefix = e.fn.slice(0, e.fn.indexOf(".")) as LibCallPrefix; // events.once builds typed listener adapters beside the emitter calls. if (e.fn === "events.once" || e.fn === "events.onceSignal") diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 4a9578e5e..6fbad7e11 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1014,6 +1014,8 @@ export const LIB_FN_SYMS: Record = { "class.descriptorHasValue": "scr_class_descriptor_has_value", "class.throwReadOnly": "scr_class_throw_read_only", "class.throwNotExtensible": "scr_class_throw_not_extensible", + "record.freeze": "scr_record_freeze", + "record.throwFrozen": "scr_record_throw_frozen", "dyn.nativeSetNew": "scr_dyn_native_set_new", "dyn.nativeSetIs": "scr_dyn_native_set_is", "dyn.nativeMapIs": "scr_dyn_native_map_is", diff --git a/packages/compiler/src/backend/llvm/shapes.test.ts b/packages/compiler/src/backend/llvm/shapes.test.ts index f7327a688..7af16ab83 100644 --- a/packages/compiler/src/backend/llvm/shapes.test.ts +++ b/packages/compiler/src/backend/llvm/shapes.test.ts @@ -14,6 +14,7 @@ import { type ShapeHost, vAdapters, } from "./shapes.js"; +import { mangleRecordRelease } from "../mangle.js"; import { NullableUnions } from "./nullable-unions.js"; import { InlineUnionFields } from "./inline-unions.js"; @@ -315,6 +316,42 @@ describe("emitted object releases", () => { root.indexOf("call void @scr_cyc_on_release(ptr %o)"), ); }); + + test("freezable records count their references without the frozen bit", () => { + const mod: IrModule = { + irVersion: 17, + sourceFile: "frozen.ts", + entry: "main", + functions: [], + records: [ + { id: "plain", fields: [{ name: "n", type: F64 }] }, + { id: "frozen", fields: [{ name: "n", type: F64 }], freezable: true }, + ], + }; + const body = (defs: string, name: string): string => + new RegExp(`^define internal void @${name}\\([^]*?^}`, "m").exec(defs)?.[0] ?? ""; + for (const [pointerBits, notFrozen, sharedBelow] of [ + [64, "-4611686018427387905", "-4611686018427387907"], + [32, "-1073741825", "-1073741827"], + ] as const) { + const { host } = declarationHost(pointerBits); + const defs = emitRecordShapes({ ...host, objectAudit: false }, mod).defs.join("\n"); + const S = pointerBits === 64 ? "i64" : "i32"; + const plain = body(defs, mangleRecordRelease("plain")); + expect(plain).toContain(`%shared = icmp ult ${S} %rcm2, -3`); + expect(plain).not.toContain("%refs"); + // The immortal word's count is ~FROZEN, which the compare sends to the + // immortal test, like counts 0 and 1. + const fast = body(defs, mangleRecordRelease("frozen")); + expect(fast).toContain(`%refs = and ${S} %rc, ${notFrozen}`); + expect(fast).toContain(`%rcm2 = sub ${S} %refs, 2`); + expect(fast).toContain(`%shared = icmp ult ${S} %rcm2, ${sharedBelow}`); + expect(fast).toContain(`%imm = icmp eq ${S} %rc, -1`); + const slow = body(defs, `${mangleRecordRelease("frozen")}_slow`); + expect(slow).toContain(`%refs = and ${S} %n, ${notFrozen}`); + expect(slow).toContain(`%dead = icmp eq ${S} %refs, 0`); + } + }); }); describe("live-object audit notes", () => { diff --git a/packages/compiler/src/backend/llvm/shapes.ts b/packages/compiler/src/backend/llvm/shapes.ts index d13f80a78..0997d0ace 100644 --- a/packages/compiler/src/backend/llvm/shapes.ts +++ b/packages/compiler/src/backend/llvm/shapes.ts @@ -1527,6 +1527,17 @@ export function destroyDefinitions( ]; } +/** The frozen bit of a freezable record's rc word (scr_rc.h SCR_RC_FROZEN): + * bit 62 on 64-bit targets, bit 30 on 32-bit ones. */ +function recordFrozenBit(sizeType: RcCount): bigint { + return 1n << (sizeType === "i64" ? 62n : 30n); +} + +/** `value` as a signed LLVM integer literal of `sizeType`. */ +function sizeLiteral(sizeType: RcCount, value: bigint): string { + return BigInt.asIntN(sizeType === "i64" ? 64 : 32, value).toString(); +} + /** Common NULL/immortal/decrement skeleton for ordinary object releases. * `freeBody` owns the zero-ref teardown and must leave the current block at * its end; traced objects also get the possible-cycle-root branch. @@ -1543,13 +1554,16 @@ export function releaseBody( bounded = true, immortal = false, chained = false, + frozen = false, ): string[] { const S = host.sizeType; if (bounded) host.declare(`declare void @scr_rc_destroy(ptr, ptr)`); const destroy = `${fnName}_destroy`; + // A freezable record's count is the word without its frozen bit. + const notFrozen = sizeLiteral(S, ~recordFrozenBit(S)); const lines = [ ...(bounded ? destroyDefinitions(destroy, freeBody, chained) : []), - ...releaseFastPath(host, fnName, `${fnName}_slow`, traced, comment, immortal), + ...releaseFastPath(host, fnName, `${fnName}_slow`, traced, comment, immortal, S, frozen), `define internal void @${fnName}_slow(ptr %o) noinline ${FN_ATTRS} {${comment ? ` ; ${comment}` : ""}`, `entry:`, ` %isnull = icmp eq ptr %o, null`, @@ -1561,7 +1575,9 @@ export function releaseBody( `dec:`, ` %n = sub ${S} %rc, 1`, ` store ${S} %n, ptr %o`, - ` %dead = icmp eq ${S} %n, 0`, + ...(frozen + ? [` %refs = and ${S} %n, ${notFrozen}`, ` %dead = icmp eq ${S} %refs, 0`] + : [` %dead = icmp eq ${S} %n, 0`]), ` br i1 %dead, label %free, label %${traced ? "root" : "done"}`, `free:`, ...(traced ? cycleDeadLines(host, "teardown") : []), @@ -1591,9 +1607,14 @@ export function releaseFastPath( comment = "", immortal = false, count: RcCount = host.sizeType, + frozen = false, ): string[] { const S = count; if (immortal) return immortalRcBody(host, fnName, "void", comment); + // A freezable record compares its count without the frozen bit: an + // immortal word's count is then ~FROZEN, which the same compare sends to + // `slow`. + const notFrozen = ~recordFrozenBit(S); return [ `define internal void @${fnName}(ptr %o) ${FN_ATTRS} {${comment ? ` ; ${comment} (fast path)` : ""}`, `entry:`, @@ -1601,8 +1622,13 @@ export function releaseFastPath( ` br i1 %isnull, label %done, label %check`, `check:`, ` %rc = load ${S}, ptr %o`, - ` %rcm2 = sub ${S} %rc, 2`, - ` %shared = icmp ult ${S} %rcm2, -3`, + ...(frozen + ? [ + ` %refs = and ${S} %rc, ${sizeLiteral(S, notFrozen)}`, + ` %rcm2 = sub ${S} %refs, 2`, + ` %shared = icmp ult ${S} %rcm2, ${sizeLiteral(S, notFrozen - 2n)}`, + ] + : [` %rcm2 = sub ${S} %rc, 2`, ` %shared = icmp ult ${S} %rcm2, -3`]), ` br i1 %shared, label %dec, label %slow`, `dec:`, ` %n = sub ${S} %rc, 1`, @@ -1810,6 +1836,7 @@ export function emitRecordShapes( refMembers.some((member) => needsBoundedRelease(member.type)), false, chained, + shape.freezable === true, ), ``, ); diff --git a/packages/compiler/src/backend/llvm/stack-objects.ts b/packages/compiler/src/backend/llvm/stack-objects.ts index c7a58a7ea..ae2b904db 100644 --- a/packages/compiler/src/backend/llvm/stack-objects.ts +++ b/packages/compiler/src/backend/llvm/stack-objects.ts @@ -44,8 +44,9 @@ export type StackObjectFieldOwner = (fields: readonly StackObjectField[]) => voi * standalone scalar class whose constructor never lets `this` escape * (record-results.ts). The caller proves that every use of the value only * projects it. Record fields may own references; the frame releases them - * (stackObjectFields). Published shapes keep heap storage: their field - * guards treat the immortal header as a frozen object. */ + * (stackObjectFields). Published and freezable shapes keep heap storage: + * their field guards treat the immortal header as a frozen object, and a + * frozen record keeps its frozen bit in a counted header. */ export function canStackObject(host: LlvmEmitterContext, e: IrExpr): e is StackObjectSource { // A ThreadSlots key keeps its slot index for its lifetime and has its // values released when it is freed: it lives on the heap. @@ -53,6 +54,8 @@ export function canStackObject(host: LlvmEmitterContext, e: IrExpr): e is StackO return false; if (e.type.kind === "record" && host.publishedTypes?.recordGuarded(e.type.shapeId) === true) return false; + if (e.type.kind === "record" && host.recordsById.get(e.type.shapeId)?.freezable === true) + return false; // A producer with a result-slot entry builds its fresh result in the // caller's storage. if (e.kind === "call") return host.fnByName.has(resultEntryName(e.callee)); diff --git a/packages/compiler/src/frontend/lowering/expressions/element-access.ts b/packages/compiler/src/frontend/lowering/expressions/element-access.ts index 0be9be81a..7a8fd646e 100644 --- a/packages/compiler/src/frontend/lowering/expressions/element-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/element-access.ts @@ -1022,7 +1022,7 @@ function lowerGuardedRecordWrite( const value = stabilized(lowerer, lowerer.lowerExpr(expr.right), "%recordValue", receiver.stmts); const write = lowerer.withExpressionOverride(target.expression, receiver.present, () => lowerer.withExpressionOverride(target.argumentExpression, key, () => - lowerer.withExpressionOverride(expr.right, value, () => lowerElementWrite(lowerer, expr)), + lowerer.withExpressionOverride(expr.right, value, () => lowerer.lowerElementWrite(expr)), ), ); return { diff --git a/packages/compiler/src/frontend/lowering/frozen-records.ts b/packages/compiler/src/frontend/lowering/frozen-records.ts new file mode 100644 index 000000000..f61de39ec --- /dev/null +++ b/packages/compiler/src/frontend/lowering/frozen-records.ts @@ -0,0 +1,328 @@ +import * as ts from "../ts7/adapter.js"; +import { + BOOL, + VOID, + shapeHasAccessorSlots, + type IrExpr, + type IrFunction, + type IrStmt, + type IrType, + type SrcLoc, +} from "../../ir/ir.js"; +import { numLit, strLit, varRef } from "../../ir/build.js"; +import { everyStmtList, transformStmtList } from "../../ir/traverse.js"; +import type { Lowerer } from "./lowerer.js"; +import { recordHasOwnHelper } from "./lower-calls.js"; +import { isStrictCode } from "./strict-mode.js"; + +/* Object.freeze of records that other references can still write. Only the + * shapes a program may freeze pay for it: their values keep a frozen bit in + * the rc word (scr_rc.h SCR_RC_FROZEN), their release ignores the bit, and + * every write to them checks it first (guardFrozenRecordWrites). A shape is + * freezable when a static Object.freeze names it, or, in a program that + * freezes untyped values, when a value of it or of a type reaching it can be + * held as a live untyped reference. */ + +type RecordType = Extract; +type RecordWrite = Extract; + +/** Object.freeze of a record: sets its frozen bit and answers the record. */ +export function lowerRecordFreeze( + lowerer: Lowerer, + target: IrExpr, + type: RecordType, + loc: SrcLoc, +): IrExpr { + const slot = lowerer.declareHiddenLocal("%freezeTarget", type); + const receiver = (): IrExpr => varRef(slot.id, type, loc); + return { + kind: "seqExpr", + stmts: [ + { kind: "varDecl", localId: slot.id, init: target, loc }, + { + kind: "exprStmt", + expr: { kind: "libCall", fn: "record.freeze", args: [receiver()], type: VOID, loc }, + loc, + }, + ], + result: receiver(), + type, + loc, + }; +} + +function isRecordWrite(s: IrStmt): s is RecordWrite { + return s.kind === "recordSet" || s.kind === "recordKeySet" || s.kind === "recordKeyDelete"; +} + +/** The key of a record write's source range: strict mode is lexical, so + * every write lowered from the same range shares its mode. */ +function writeKey(loc: SrcLoc): string { + return `${loc.file}\0${loc.start}\0${loc.end}`; +} + +/** Records the record writes of `stmt`, lowered from `node`, as sloppy-mode + * writes when `node` is sloppy-mode code: a write to a frozen record then + * leaves it unchanged instead of throwing. `deletes` says `stmt` is the + * absent-state write of `delete r.f`, which Node refuses only while the + * field is present. */ +export function markSloppyRecordWrites( + lowerer: Lowerer, + stmt: IrStmt, + node: ts.Node, + deletes = false, +): IrStmt { + if (deletes) lowerer.recordFieldDeletes.add(writeKey(stmt.loc)); + if (isStrictCode(node)) return stmt; + everyStmtList([stmt], { + stmt: (s) => { + if (s.kind === "recordSet" || s.kind === "recordKeySet" || s.kind === "recordKeyDelete") + lowerer.sloppyRecordWrites.add(writeKey(s.loc)); + return true; + }, + expr: () => true, + }); + return stmt; +} + +/** Whether a field write's value can be an optional field's absent state, + * which only a field write itself can hold. */ +function holdsAbsentState(e: IrExpr): boolean { + if (e.kind === "fieldAbsent") return true; + return e.kind === "ternary" && (holdsAbsentState(e.then) || holdsAbsentState(e.else_)); +} + +/** The shapes whose values the program may freeze. */ +function freezableShapes(lowerer: Lowerer, functions: IrFunction[]): Set { + const shapes = new Set(); + const live: IrType[] = []; + let dynFreeze = false; + for (const fn of functions) + everyStmtList(fn.body, { + expr: (e) => { + if (e.kind === "libCall" && e.fn === "record.freeze") { + const target = e.args[0]; + if (target && target.type.kind === "record") shapes.add(target.type.shapeId); + } else if (e.kind === "libCall" && e.fn === "dyn.freeze") dynFreeze = true; + else if (e.kind === "dynFrom") live.push(e.value.type); + return true; + }, + stmt: () => true, + }); + if (!dynFreeze) return shapes; + // An untyped reference to a class instance, or to a record or array + // boxed live, reaches the values its fields and elements hold as further + // live references. Class instances also cross into untyped code through + // compiled callbacks, so every class is a root. + const stack = [...live]; + for (const info of lowerer.classes.values()) + for (const field of info.def.fields) stack.push(field.type); + const seen = new Set(); + while (stack.length > 0) { + const type = stack.pop()!; + switch (type.kind) { + case "record": { + if (seen.has(`record:${type.shapeId}`)) break; + seen.add(`record:${type.shapeId}`); + shapes.add(type.shapeId); + const shape = lowerer.shapes.get(type.shapeId); + for (const field of shape?.fields ?? []) stack.push(field.type); + if (shape?.indexValue) stack.push(shape.indexValue); + break; + } + case "union": + if (seen.has(`union:${type.unionId}`)) break; + seen.add(`union:${type.unionId}`); + for (const arm of lowerer.unions.get(type.unionId)?.arms ?? []) stack.push(arm); + break; + case "array": + case "set": + stack.push(type.elem); + break; + case "map": + stack.push(type.key, type.value); + break; + case "promise": + stack.push(type.inner); + break; + case "generator": + stack.push(type.yieldT, type.retT, type.nextT); + break; + case "func": + for (const param of type.params) stack.push(param); + stack.push(type.ret); + break; + default: + break; + } + } + return shapes; +} + +/** The guarded form of a write to a freezable record: the receiver and the + * operands are evaluated first, in source order, then the frozen bit decides + * between the write and `reject`. */ +function guardedWrite( + receiver: IrExpr, + decls: IrStmt[], + reject: IrStmt[], + store: IrStmt, + loc: SrcLoc, +): IrStmt { + return { + kind: "block", + body: [ + ...decls, + { + kind: "if", + cond: { kind: "libCall", fn: "record.isFrozen", args: [receiver], type: BOOL, loc }, + then: reject, + else_: [store], + loc, + }, + ], + loc, + }; +} + +/** Checks the frozen bit before every write to a record of a freezable + * shape. A write to a frozen record throws Node's TypeError in strict mode + * code; in sloppy-mode code it leaves the record unchanged. Deleting a key + * the record does not have succeeds either way. */ +export function guardFrozenRecordWrites(lowerer: Lowerer, functions: IrFunction[]): void { + const freezable = freezableShapes(lowerer, functions); + for (const id of freezable) { + const shape = lowerer.shapes.get(id); + if (shape) shape.freezable = true; + } + if (freezable.size === 0) return; + const lifted = lowerer.liftedFns.length; + for (const fn of functions) { + const local = (type: IrType, loc: SrcLoc): Extract => { + const id = `%frozenWrite.${fn.locals.length}`; + fn.locals.push({ id, name: id, type, mutable: false }); + return { kind: "varRef", localId: id, type, loc }; + }; + const guarded = new Set(); + fn.body = transformStmtList(fn.body, { + stmt: (statement) => { + if (!isRecordWrite(statement) || guarded.has(statement)) return statement; + if (!freezable.has(statement.shapeId)) return statement; + const sloppy = lowerer.sloppyRecordWrites.has(writeKey(statement.loc)); + const deletes = + statement.kind === "recordSet" && lowerer.recordFieldDeletes.has(writeKey(statement.loc)); + const loc = statement.loc; + const shapeId = statement.shapeId; + const receiver = local(statement.obj.type, loc); + const decls: IrStmt[] = [ + { kind: "varDecl", localId: receiver.localId, init: statement.obj, loc }, + ]; + const owner = strLit( + lowerer.shapes.get(shapeId)?.tuple ? "[object Array]" : "#", + loc, + ); + const thrown = (key: IrExpr, operation: number): IrStmt => ({ + kind: "exprStmt", + expr: { + kind: "libCall", + fn: "record.throwFrozen", + args: [key, owner, numLit(operation, loc)], + type: VOID, + loc, + }, + loc, + }); + if (statement.kind === "recordSet") { + let value = statement.value; + if (!holdsAbsentState(value)) { + const slot = local(value.type, loc); + decls.push({ kind: "varDecl", localId: slot.localId, init: value, loc }); + value = slot; + } + const store: IrStmt = { + kind: "recordSet", + obj: receiver, + shapeId, + field: statement.field, + value, + loc, + }; + guarded.add(store); + const field = strLit(statement.field, loc); + // `delete r.f` of an optional field writes its absent state: Node + // only refuses it when the field is present. + const reject: IrStmt[] = sloppy + ? [] + : deletes + ? [ + { + kind: "if", + cond: lowerer.recordFieldPresent(receiver, shapeId, statement.field, loc), + then: [thrown(field, 2)], + else_: null, + loc, + }, + ] + : [thrown(field, 0)]; + return guardedWrite(receiver, decls, reject, store, loc); + } + const key = local(statement.key.type, loc); + decls.push({ kind: "varDecl", localId: key.localId, init: statement.key, loc }); + const named = key.type.kind === "string"; + const present: IrExpr = named + ? { + kind: "call", + callee: recordHasOwnHelper(lowerer, shapeId, loc), + args: [receiver, key], + type: BOOL, + loc, + } + : { kind: "boolLit", value: true, type: BOOL, loc }; + const text: IrExpr = named ? key : strLit("[key]", loc); + if (statement.kind === "recordKeySet") { + const value = local(statement.value.type, loc); + decls.push({ kind: "varDecl", localId: value.localId, init: statement.value, loc }); + const store: IrStmt = { + kind: "recordKeySet", + obj: receiver, + shapeId, + key, + value, + ...(statement.overflowOnly ? { overflowOnly: true as const } : {}), + loc, + }; + guarded.add(store); + const reject: IrStmt[] = sloppy + ? [] + : [ + { + kind: "if", + cond: present, + then: [thrown(text, 0)], + else_: [thrown(text, 1)], + loc, + }, + ]; + return guardedWrite(receiver, decls, reject, store, loc); + } + const store: IrStmt = { kind: "recordKeyDelete", obj: receiver, shapeId, key, loc }; + guarded.add(store); + const reject: IrStmt[] = sloppy + ? [] + : [{ kind: "if", cond: present, then: [thrown(text, 2)], else_: null, loc }]; + return guardedWrite(receiver, decls, reject, store, loc); + }, + expr: (expression) => expression, + }); + } + // The key-presence helpers the guards call. + for (const helper of lowerer.liftedFns.slice(lifted)) functions.push(helper); +} + +/** Whether a statically typed Object.freeze can freeze a value of `type`: + * a record whose shape has no accessor properties. */ +export function freezableRecordType(lowerer: Lowerer, type: IrType): type is RecordType { + if (type.kind !== "record") return false; + const shape = lowerer.shapes.get(type.shapeId); + return shape !== undefined && !shapeHasAccessorSlots(shape); +} diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index b595aaa43..e2ed5f9fe 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -245,6 +245,7 @@ import { import { fenceSymbolFieldCopy } from "./symbol-fields.js"; import { lowerClassDataDescriptor, lowerClassDescriptorRead } from "./class-descriptors.js"; import { lowerClassFieldDefine, lowerClassIntegrity } from "./class-attributes.js"; +import { freezableRecordType, lowerRecordFreeze } from "./frozen-records.js"; import { classStaticDataFor } from "./class-static-data.js"; import { lowerUrlFactory } from "./lower-url.js"; import { nullishValueUnitOf } from "./binding-analysis.js"; @@ -7206,7 +7207,7 @@ export function recordKeysArrayCall( * (a key is own exactly when Object.keys would list it — the two share * the test), everything else true. Unmatched keys probe the overflow * map when present, so explicit undefined values remain own keys. */ -function recordHasOwnHelper(lowerer: Lowerer, shapeId: string, loc: SrcLoc): string { +export function recordHasOwnHelper(lowerer: Lowerer, shapeId: string, loc: SrcLoc): string { const key = `obj.hasOwn:${shapeId}`; const existing = lowerer.arrHofHelpers.get(key); if (existing) return existing; @@ -8455,10 +8456,23 @@ function lowerObjectStaticCall( // the argument — no alias exists, so the frozen bit is unobservable // (writes through the Readonly result are compile errors, and no // other reference can write). Primitives pass through per ES2015. - // Aliased objects keep a fence: a later write through the original - // reference would need the runtime frozen bit (strict mode throws). + // Aliased records get the frozen bit of their rc word (frozen-records.ts); + // other aliased objects keep a fence: a later write through the original + // reference would need a runtime frozen bit (strict mode throws). if (member === "isFrozen" && call.arguments.nodes.length === 1) { - const value = lowerer.lowerExprExpecting(call.arguments.nodes[0]!, DYN); + const argNode = call.arguments.nodes[0]!; + if (tryLowerExpression(lowerer, argNode)?.type.kind === "record") { + const record = lowerer.lowerExpr(argNode); + if (record.type.kind === "record") + return { + kind: "libCall", + fn: "record.isFrozen", + args: [record], + type: BOOL, + loc: locOf(call), + }; + } + const value = lowerer.lowerExprExpecting(argNode, DYN); return { kind: "libCall", fn: "dyn.isFrozen", args: [value], type: BOOL, loc: locOf(call) }; } if (member === "freeze") { @@ -8508,6 +8522,8 @@ function lowerObjectStaticCall( if (fnIntegrity) return fnIntegrity; const integrity = lowerClassIntegrity(lowerer, call, "freeze", value); if (integrity) return integrity; + if (freezableRecordType(lowerer, value.type)) + return lowerRecordFreeze(lowerer, value, value.type, locOf(call)); lowerer.noLowering( "Object.freeze of a possibly-aliased value", call, diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index 66ecea602..e66d0e855 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -105,6 +105,7 @@ import { requireSpecOf, } from "../program.js"; import { isStrictCode } from "./strict-mode.js"; +import { markSloppyRecordWrites } from "./frozen-records.js"; import { COMPOUND_ASSIGN_OPS, type CompoundOp, @@ -7160,7 +7161,11 @@ function lowerDeleteStatement(lowerer: Lowerer, expr: ts.DeleteExpression): IrSt if (obj.type.kind === "record") { const shape = lowerer.shapes.get(obj.type.shapeId); if (shape?.indexValue && shape.fields.length === 0 && !shape.tuple) { - return { kind: "recordKeyDelete", obj, shapeId: obj.type.shapeId, key: lowerKey(), loc }; + return markSloppyRecordWrites( + lowerer, + { kind: "recordKeyDelete", obj, shapeId: obj.type.shapeId, key: lowerKey(), loc }, + expr, + ); } // `delete r.f` of a declared OPTIONAL field (undefined-armed slot) // writes the field's ABSENT state: a monomorphic shape cannot remove @@ -7181,14 +7186,19 @@ function lowerDeleteStatement(lowerer: Lowerer, expr: ts.DeleteExpression): IrSt if (field) { const absent = lowerer.absentFieldValue(field.type, loc); if (absent) { - return { - kind: "recordSet", - obj, - shapeId: obj.type.shapeId, - field: field.name, - value: absent, - loc, - }; + return markSloppyRecordWrites( + lowerer, + { + kind: "recordSet", + obj, + shapeId: obj.type.shapeId, + field: field.name, + value: absent, + loc, + }, + expr, + true, + ); } } if (shape?.indexValue) { diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index a765c91ee..ba6d1214f 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -150,6 +150,7 @@ import { defaultAfterUndefined, lowerStaticallyUndefinedArgument } from "./optio import { type ProtocolDispatch } from "./class-protocol-dispatch.js"; import { ClassDynamicDispatch, trackClassFieldCreation } from "./class-dynamic-dispatch.js"; import { guardClassAttributeWrites } from "./class-attributes.js"; +import { guardFrozenRecordWrites, markSloppyRecordWrites } from "./frozen-records.js"; import { applyHandleProperties } from "./handle-properties.js"; import { ClassProtocols } from "./class-protocols.js"; import { earlyReadVarSymbols } from "./var-hoisting.js"; @@ -3241,6 +3242,10 @@ export class Lowerer { // Stack of function contexts (bottom = the function being declared at // top level, top = the innermost nested function currently lowering). fnStack: FnCtx[] = []; + /** Source ranges of record writes in sloppy-mode code, and of the + * absent-state writes of `delete r.f` (markSloppyRecordWrites). */ + readonly sloppyRecordWrites = new Set(); + readonly recordFieldDeletes = new Set(); readonly liftedFns: IrFunction[] = []; /** The first lifted function of each name (liftedFnNamed), for the * liftedFns indexed so far. liftedFns only grows. */ @@ -7689,6 +7694,7 @@ export class Lowerer { finalizePrivateBrands(this, functions); trackClassFieldCreation(this, functions); guardClassAttributeWrites(this, functions); + guardFrozenRecordWrites(this, functions); applyHandleProperties(this, functions); // Globals typed by a class that never REGISTERED (a JS class whose @@ -12978,7 +12984,7 @@ export class Lowerer { } lowerElementWrite(expr: ts.BinaryExpression): IrStmt { - return lowerElementWrite(this, expr); + return markSloppyRecordWrites(this, lowerElementWrite(this, expr), expr); } lowerLiveWebValue(node: ts.Expression): IrExpr { @@ -13375,7 +13381,7 @@ export class Lowerer { } fieldSetStmt(target: FieldTarget, value: IrExpr, loc: SrcLoc, blame: ts.Node): IrStmt { - return fieldSetStmt(this, target, value, loc, blame); + return markSloppyRecordWrites(this, fieldSetStmt(this, target, value, loc, blame), blame); } lowerFieldCompound( diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index 9be819dd3..0505129fb 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -687,6 +687,7 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "class.defineField", "class.throwReadOnly", "class.throwNotExtensible", + "record.throwFrozen", "dyn.toString", "dyn.valueOf", "string.fromCodePoint", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index e60a6600b..1b2c1082d 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -271,6 +271,18 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ /** Throws Node's TypeError for a class field initializer on a * non-extensible instance: the field name. */ "class.throwNotExtensible": { argTypes: [STRING], result: VOID }, + /** Object.freeze of a record whose shape is freezable (frozen-records.ts): + * sets the frozen bit in its rc word (scr_record_freeze). The receiver is + * any record type. */ + "record.freeze": { argTypes: [null], result: VOID }, + /** Whether a record is frozen: its rc word's frozen bit, set by + * record.freeze and dyn.freeze of a capsule. Emitted inline. The receiver + * is any record type. */ + "record.isFrozen": { argTypes: [null], result: BOOL }, + /** Throws Node's TypeError for a write to a frozen record: the key, the + * object's text in the message ("#") and the operation (0 assign, + * 1 add, 2 delete). */ + "record.throwFrozen": { argTypes: [STRING, STRING, F64], result: VOID }, "dyn.nativeSetNew": { argTypes: [DYN], result: DYN }, "dyn.nativeSetIs": { argTypes: [DYN], result: BOOL }, "dyn.nativeMapIs": { argTypes: [DYN], result: BOOL }, diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 63286ff54..3211e48f3 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -1685,6 +1685,10 @@ export interface IrRecordShape { * dynCheck), and JSON serialization appends overflow entries after the * declared fields in insertion order. */ indexValue?: IrType; + /** Values of this shape may be frozen (frozen-records.ts): their rc word + * can carry the frozen bit, which their release ignores, and every write + * to them checks it. */ + freezable?: true; /** Field names in FIRST-SEEN declaration order (the checker's property * order of the first ts.Type interned to this shape) — metadata, NOT * part of the interned identity: a later structurally-equal type with a diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7404-frozen-records.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7404-frozen-records.ts.json new file mode 100644 index 000000000..d85f2aab5 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7404-frozen-records.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7404-frozen-records.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7405-frozen-records-sloppy.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7405-frozen-records-sloppy.cjs.json new file mode 100644 index 000000000..3d87d019b --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7405-frozen-records-sloppy.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7405-frozen-records-sloppy.cjs" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_cycle.c b/packages/runtime/src/scr_cycle.c index 6dcd546b0..241164705 100644 --- a/packages/runtime/src/scr_cycle.c +++ b/packages/runtime/src/scr_cycle.c @@ -170,7 +170,8 @@ void scr_weak_dispose(void *object) { #include /* Every heap object's first member is `size_t rc`; the collector works on - * its count (SCR_RC_COUNT). */ + * its count (SCR_RC_COUNT), and tests it without a frozen record's bit + * (SCR_RC_REFS). */ #define SCR_RC(obj) SCR_RC_COUNT(obj) static void scr_cyc_oom(void) { @@ -1308,7 +1309,8 @@ static void scr_mg_visit(void *child, void *ctx) { scr_mg_skip_edge(child); return; } - size_t rc = --SCR_RC(child); + --SCR_RC(child); + size_t rc = SCR_RC_REFS(child); if (scr_cyc_color(h) == SCR_CYC_GRAY) { if (rc == 0) { scr_gray_referenced--; @@ -1336,7 +1338,7 @@ static inline __attribute__((always_inline)) void scr_mark_gray(void *obj, uintp )) h->buf = scr_gray_comp; if (scr_white.n < SCR_CYC_GRAY_LIST_MAX) scr_cyc_push_gray(&scr_white, obj); - if (SCR_RC(obj) > 0) { + if (SCR_RC_REFS(obj) > 0) { scr_gray_referenced++; scr_comp_referenced++; } @@ -1408,7 +1410,7 @@ static void scr_scan_visit(void *child, void *ctx) { static void scr_scan(void *obj, uintptr_t depth) { ScrCycHdr *h = scr_cyc_hdr(obj); if (scr_cyc_color(h) != SCR_CYC_GRAY) return; - if (SCR_RC(obj) > 0) { + if (SCR_RC_REFS(obj) > 0) { scr_restore_from(obj); return; } @@ -1476,7 +1478,7 @@ static void scr_xg_destroy(void *obj) { } static SCR_CYC_COMPACT void scr_xg_release(void *obj) { - if (SCR_RC(obj) > 1) { + if (SCR_RC_REFS(obj) > 1) { SCR_RC(obj) -= 1; scr_cyc_on_release(obj); /* lost a reference: a possible cycle root */ return; @@ -1768,14 +1770,14 @@ static __attribute__((noinline)) size_t scr_yin_collect(void) { if (scr_cyc_gen(h) != SCR_CYC_YOUNG || !scr_cyc_is_buffered(h) || scr_cyc_buf_index(h) != k || scr_cyc_color(h) != SCR_CYC_BLACK) scr_trap("scriptc: corrupt young nursery entry\n"); - if (SCR_RC(v[k]) == 0 || SCR_RC(v[k]) < scr_yin_counts[k]) + if (SCR_RC_REFS(v[k]) == 0 || SCR_RC_REFS(v[k]) < scr_yin_counts[k]) scr_trap("scriptc: young object with uncounted references\n"); } #endif for (size_t k = 0; k < n; k++) { void *obj = v[k]; ScrCycHdr *h = scr_cyc_hdr(obj); - if (scr_cyc_color(h) == SCR_CYC_GRAY || SCR_RC(obj) <= scr_yin_counts[k]) continue; + if (scr_cyc_color(h) == SCR_CYC_GRAY || SCR_RC_REFS(obj) <= scr_yin_counts[k]) continue; scr_cyc_set_color(h, SCR_CYC_GRAY); scr_yin_push(&scr_yin_stack, obj); while (scr_yin_stack.n) { @@ -1799,7 +1801,7 @@ static __attribute__((noinline)) size_t scr_yin_collect(void) { scr_cyc_set_gen(h, SCR_CYC_YIN_PROMOTED); scr_cyc_clear_buffered(h); /* An outside root becomes a candidate (see "candidates" above). */ - if (SCR_RC(obj) > scr_yin_counts[k]) scr_cyc_on_release(obj); + if (SCR_RC_REFS(obj) > scr_yin_counts[k]) scr_cyc_on_release(obj); scr_yin_met(h->id, false); } else { scr_yin_met(h->id, true); @@ -2147,7 +2149,7 @@ static size_t scr_cyc_pass(unsigned gen_limit) { for (size_t j = start; j < end; j++) { void *obj = scr_white.v[j]; if (scr_cyc_color(scr_cyc_hdr(obj)) != SCR_CYC_GRAY) continue; - if (SCR_RC(obj) > 0) scr_restore_from(obj); + if (SCR_RC_REFS(obj) > 0) scr_restore_from(obj); else scr_white.v[kept++] = obj; } from = end; @@ -2176,7 +2178,7 @@ static size_t scr_cyc_pass(unsigned gen_limit) { * whitened only to be restored again. */ for (size_t i = 0; i < scr_cands.n; i++) { void *obj = scr_cands.v[i]; - if (scr_cyc_color(scr_cyc_hdr(obj)) == SCR_CYC_GRAY && SCR_RC(obj) > 0) scr_restore_from(obj); + if (scr_cyc_color(scr_cyc_hdr(obj)) == SCR_CYC_GRAY && SCR_RC_REFS(obj) > 0) scr_restore_from(obj); } for (size_t i = 0; i < scr_cands.n; i++) { void *obj = scr_cands.v[i]; diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index fdffd73cb..8468f8c0f 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -1019,6 +1019,13 @@ void scr_dyn_typed_ref_bind_record(ScrDyn *d, const ScrDynTypedRecordOps *ops); void scr_record_source_note(void *rec, const ScrDyn *src, const char *const *names, const size_t *lens, size_t count); bool scr_record_source_has(const void *rec); +/* Object.freeze of a record a program may freeze: sets SCR_RC_FROZEN in its + * rc word (immortal records are left as they are). */ +void scr_record_freeze(void *record); +/* Node's TypeError for a write to a frozen record: operation 0 assigns the + * existing property `key`, 1 adds it, 2 deletes it. `owner` is the object's + * text in the message ("#"). */ +void scr_record_throw_frozen(ScrStr *key, ScrStr *owner, double operation); /* The commit of a live record view converts the view into a new record * `from`, whose conversion keeps the view's undeclared properties, then * moves its fields into `to`: this moves the kept properties to `to` too, diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 798d53ae9..a542b5c24 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -2293,6 +2293,55 @@ ScrDyn *scr_dyn_set_prototype(ScrDyn *object, ScrDyn *prototype) { return scr_dyn_retain(object); } +/* The native record a record capsule refers to, or NULL for any other + * value. */ +static void *scr_dyn_typed_record(const ScrDyn *d) { + if (d->kind != SCR_DYN_TYPED_REF || d->v.typed_ref.type_key_len < 7 || + memcmp(d->v.typed_ref.type_key, "record:", 7) != 0) + return NULL; + return d->v.typed_ref.ptr; +} + +/* Freezes the checked view of a frozen record: its fields, and every + * property added through the view. */ +static void scr_dyn_record_view_freeze(ScrDyn *view) { + view->non_extensible = true; + if (view->kind == SCR_DYN_OBJ) { + for (size_t i = 0; i < view->v.obj.len; i++) { + ScrDynEntry *entry = &view->v.obj.entries[i]; + entry->configurable = false; + if (!entry->accessor) entry->writable = false; + } + } else if (view->kind == SCR_DYN_ARR) { + view->v.arr.sealed = true; + view->v.arr.frozen = true; + if (view->v.arr.properties) scr_dyn_release(scr_dyn_freeze_data(view->v.arr.properties)); + } + if (view->symbol_properties) scr_dyn_release(scr_dyn_freeze_data(view->symbol_properties)); +} + +void scr_record_freeze(void *record) { + size_t *rc = record; + if (*rc != SIZE_MAX) *rc |= SCR_RC_FROZEN; +} + +void scr_record_throw_frozen(ScrStr *key, ScrStr *owner, double operation) { + ScrJsonBuf buffer; + scr_jb_init(&buffer); + if (operation == 1) { + scr_jb_puts(&buffer, "Cannot add property "); + scr_jb_write(&buffer, key->data, key->len); + scr_jb_puts(&buffer, ", object is not extensible"); + } else { + scr_jb_puts(&buffer, operation == 2 ? "Cannot delete property '" : "Cannot assign to read only property '"); + scr_jb_write(&buffer, key->data, key->len); + scr_jb_puts(&buffer, operation == 2 ? "' of " : "' of object '"); + scr_jb_write(&buffer, owner->data, owner->len); + if (operation != 2) scr_jb_puts(&buffer, "'"); + } + scr_throw_error(SCR_ERR_TYPE, scr_jb_finish(&buffer)); +} + /* A class instance's integrity state lives in its native layout, which only * statically typed calls can reach (the compiler adds the state to the * hierarchies they name). Through an untyped value the operation throws. */ @@ -2301,7 +2350,9 @@ static bool scr_dyn_class_integrity_fence(const ScrDyn *object, const char *oper ScrJsonBuf buffer; scr_jb_init(&buffer); scr_jb_puts(&buffer, operation); - scr_jb_puts(&buffer, " of a class instance held in an untyped value is not supported yet"); + scr_jb_puts(&buffer, scr_dyn_typed_record(object) + ? " of an object held in an untyped value is not supported yet" + : " of a class instance held in an untyped value is not supported yet"); ScrStr *message = scr_jb_finish(&buffer); scr_throw_error_msg_code(SCR_ERR_ERROR, message->data, message->len, "SC2020"); scr_str_release(message); @@ -3628,12 +3679,16 @@ ScrDyn *scr_dyn_typed_ref_materialize(const ScrDyn *d) { fresh->rc = fresh_rc; scr_dyn_release(fresh); } + void *record = scr_dyn_typed_record(capsule); + if (record && scr_rc_frozen(record)) scr_dyn_record_view_freeze(capsule->v.typed_ref.materialized); return scr_dyn_retain(capsule->v.typed_ref.materialized); } void scr_dyn_typed_ref_commit(ScrDyn *d) { + /* A frozen record's view cannot have changed its fields. */ if (d && d->kind == SCR_DYN_TYPED_REF && d->v.typed_ref.commit && - d->v.typed_ref.materialized) { + d->v.typed_ref.materialized && + !(scr_dyn_typed_record(d) && scr_rc_frozen(d->v.typed_ref.ptr))) { d->v.typed_ref.commit(d->v.typed_ref.ptr, d->v.typed_ref.materialized); } @@ -15627,8 +15682,18 @@ ScrStr *scr_dyn_to_string_method(const ScrDyn *d, const ScrStr *enc, const ScrSt return scr_dyn_to_string_method_args(d, enc, what, NULL, 0); } +/* Freezing a record capsule freezes the native record, whose emitted writes + * then refuse to change it, and its checked view. */ +static ScrDyn *scr_dyn_freeze_record(ScrDyn *capsule, void *record) { + scr_record_freeze(record); + if (capsule->v.typed_ref.materialized) scr_dyn_record_view_freeze(capsule->v.typed_ref.materialized); + return scr_dyn_retain(capsule); +} + ScrDyn *scr_dyn_freeze(ScrDyn *value) { if (scr_dyn_class_reflection_fence(value) || scr_dyn_error_prototype_fence(value)) return NULL; + void *record = scr_dyn_typed_record(value); + if (record) return scr_dyn_freeze_record(value, record); if (scr_dyn_class_integrity_fence(value, "Object.freeze")) return NULL; ScrDyn *table = scr_dyn_fn_integrity_table(value); if (!table) return scr_dyn_freeze_data(value); @@ -15640,6 +15705,8 @@ ScrDyn *scr_dyn_freeze(ScrDyn *value) { ScrDyn *scr_dyn_freeze_data(ScrDyn *value) { if (scr_dyn_class_reflection_fence(value) || scr_dyn_error_prototype_fence(value)) return NULL; + void *record = scr_dyn_typed_record(value); + if (record) return scr_dyn_freeze_record(value, record); if (scr_dyn_class_integrity_fence(value, "Object.freeze")) return NULL; if (value->symbol_properties) scr_dyn_release(scr_dyn_freeze_data(value->symbol_properties)); if (value->kind == SCR_DYN_ARR) { diff --git a/packages/runtime/src/scr_json.test.c b/packages/runtime/src/scr_json.test.c index 736eca540..a28759876 100644 --- a/packages/runtime/src/scr_json.test.c +++ b/packages/runtime/src/scr_json.test.c @@ -802,8 +802,21 @@ static void json_schema_union_members(void) { assert(test_opts_live == 0); } +/* Freezing a record sets the frozen bit beside its count; an immortal + * record keeps its word, which does not read as frozen. */ +static void record_frozen_bit(void) { + size_t record[2] = {3, 0}; + assert(!scr_rc_frozen(record)); + scr_record_freeze(record); + assert(scr_rc_frozen(record) && SCR_RC_REFS(record) == 3); + size_t immortal[2] = {SIZE_MAX, 0}; + scr_record_freeze(immortal); + assert(immortal[0] == SIZE_MAX && !scr_rc_frozen(immortal)); +} + int main(void) { json_string_boundaries(); + record_frozen_bit(); json_indent_ownership(); scr_init(); checked_storage(); diff --git a/packages/runtime/src/scr_rc.h b/packages/runtime/src/scr_rc.h index 18df3a237..87eb2536a 100644 --- a/packages/runtime/src/scr_rc.h +++ b/packages/runtime/src/scr_rc.h @@ -45,14 +45,32 @@ typedef size_t ScrRcCount; /* Whether `obj`, an object of any kind, is published (immortal). */ #define SCR_RC_PUBLISHED(obj) (SCR_RC_COUNT(obj) >= SCR_RC_PUBLISHED_MIN) +/* Frozen records (Object.freeze). The rc word of a record a program may + * freeze (llvm/shapes.ts) carries SCR_RC_FROZEN beside its count, and that + * record's emitted release ignores the bit. On 64-bit targets the bit is in + * the high half, outside the count; on 32-bit targets it is the count's bit + * 30, which SCR_RC_REFS masks. An immortal word has the bit set too; it + * does not read as frozen. */ +#if SIZE_MAX > UINT32_MAX +#define SCR_RC_FROZEN ((size_t)1 << 62) +#define SCR_RC_REFS(obj) SCR_RC_COUNT(obj) +#else +#define SCR_RC_FROZEN ((size_t)1 << 30) +#define SCR_RC_REFS(obj) (SCR_RC_COUNT(obj) == SCR_RC_IMMORTAL ? SCR_RC_IMMORTAL : SCR_RC_COUNT(obj) & ~SCR_RC_FROZEN) +#endif +static inline bool scr_rc_frozen(const void *obj) { + size_t word = *(const size_t *)obj; + return word != SIZE_MAX && (word & SCR_RC_FROZEN) != 0; +} + /* The class id in the high half of `obj`'s rc word, or 0 for an object - * without one: the high half of any other object's word is 0, or 0xffffffff - * (immortal) or 0x80000000 (program-lifetime), and a class id is never one of - * these. */ + * without one: the high half of any other object's word is 0, 0xffffffff + * (immortal), 0x80000000 (program-lifetime) or 0x40000000 (a frozen record), + * and a class id is never one of these. */ static inline uint32_t scr_rc_class_id(const void *obj) { #if SIZE_MAX > UINT32_MAX uint32_t high = (uint32_t)(*(const size_t *)obj >> 32); - return high == UINT32_MAX || high == UINT32_C(0x80000000) ? 0 : high; + return high == UINT32_MAX || high == UINT32_C(0x80000000) || high == UINT32_C(0x40000000) ? 0 : high; #else (void)obj; return 0; diff --git a/tests/corpus/7404-frozen-records.ts b/tests/corpus/7404-frozen-records.ts new file mode 100644 index 000000000..acf9b393b --- /dev/null +++ b/tests/corpus/7404-frozen-records.ts @@ -0,0 +1,124 @@ +// Object.freeze of records that other references still hold: writes +// through any reference then throw Node's TypeError (this module is strict +// mode code; 7405 covers sloppy mode). Covers static freezes, freezes +// through untyped values, Object.isFrozen, keyed writes and deletes, +// optional-field deletes, Object.assign, tuples, and frozen records that are +// released or collected as cycles. +export {}; + +type Counter = { count: number; label: string }; +type Link = { name: string; next?: Link }; + +function report(e: unknown): void { + console.log((e as Error).name + ": " + (e as Error).message); +} + +function freezeUnknown(value: unknown): unknown { + return Object.freeze(value); +} + +function isFrozenUnknown(value: unknown): boolean { + return Object.isFrozen(value); +} + +function deepFreeze(value: unknown): unknown { + if (typeof value === "object" && value !== null) { + Object.freeze(value); + for (const key of Object.keys(value)) deepFreeze((value as Record)[key]); + } + return value; +} + +type Config = { name: string; limits: { max: number; min: number } }; + +function bump(c: Counter): void { + c.count += 1; +} + +function writes(): void { + const a: Counter = { count: 1, label: "a" }; + const alias = a; + console.log(Object.isFrozen(a), isFrozenUnknown(a)); + const frozen = Object.freeze(alias); + console.log(frozen === a, Object.isFrozen(a), isFrozenUnknown(alias)); + try { bump(a); } catch (e) { report(e); } + try { a.label = "b"; } catch (e) { report(e); } + console.log(a.count, a.label, JSON.stringify(a)); + + const viaUnknown: Counter = { count: 10, label: "u" }; + const back = freezeUnknown(viaUnknown); + console.log(back === viaUnknown, Object.isFrozen(viaUnknown)); + try { viaUnknown.count = 11; } catch (e) { report(e); } + console.log(viaUnknown.count); + + const dict: Record = { one: 1 }; + const dictAlias = dict; + Object.freeze(dictAlias); + try { dict["one"] = 2; } catch (e) { report(e); } + try { dict["two"] = 2; } catch (e) { report(e); } + try { delete dict["one"]; } catch (e) { report(e); } + delete dict["three"]; + console.log(JSON.stringify(dict), Object.isFrozen(dict)); + + const optional: { id: number; note?: string } = { id: 1, note: "n" }; + const optionalAlias = optional; + Object.freeze(optionalAlias); + try { delete optional.note; } catch (e) { report(e); } + console.log(JSON.stringify(optional)); + const bare: { id: number; note?: string } = { id: 2 }; + Object.freeze(bare); + delete bare.note; + console.log(JSON.stringify(bare)); + + const target: Counter = { count: 0, label: "t" }; + Object.freeze(target); + try { Object.assign(target, { count: 5, label: "x" }); } catch (e) { report(e); } + console.log(JSON.stringify(target)); + + const pair: [string, number] = ["p", 1]; + Object.freeze(pair); + try { pair[1] = 2; } catch (e) { report(e); } + console.log(pair[0], pair[1]); +} + +function deepFreezes(): void { + const config: Config = JSON.parse('{"name":"c","limits":{"max":2,"min":1}}'); + const limits = config.limits; + console.log(Object.isFrozen(config), Object.isFrozen(limits)); + deepFreeze(config); + console.log(Object.isFrozen(config), Object.isFrozen(limits), isFrozenUnknown(config.limits)); + try { config.limits.max = 3; } catch (e) { report(e); } + try { limits.min = 0; } catch (e) { report(e); } + try { config.name = "d"; } catch (e) { report(e); } + console.log(JSON.stringify(config)); +} + +function lifetimes(): void { + const list: Counter[] = []; + for (let i = 0; i < 4; i++) { + const item: Counter = { count: i, label: "x" + i }; + list.push(item); + if (i % 2 === 1) Object.freeze(item); + } + for (const item of list) { + try { item.count = item.count * 10; } catch (e) { console.log(item.label, (e as Error).message); } + } + console.log(JSON.stringify(list)); + const kept = list[1]!; + list.length = 0; + console.log(kept.count, Object.isFrozen(kept)); + + for (let round = 0; round < 3; round++) { + const first: Link = { name: "first" + round }; + const second: Link = { name: "second" + round, next: first }; + first.next = second; + Object.freeze(first); + Object.freeze(second); + try { first.name = "changed"; } catch (e) { report(e); } + console.log(first.next?.name, second.next?.name); + } +} + +writes(); +deepFreezes(); +lifetimes(); diff --git a/tests/corpus/7405-frozen-records-sloppy.cjs b/tests/corpus/7405-frozen-records-sloppy.cjs new file mode 100644 index 000000000..c0656b7b2 --- /dev/null +++ b/tests/corpus/7405-frozen-records-sloppy.cjs @@ -0,0 +1,34 @@ +// Writes to frozen records in sloppy-mode code leave them unchanged, where +// strict mode code throws (7404). The records are typed through JSDoc, so +// the writes are native record writes. + +/** @typedef {{ count: number, label: string }} Counter */ +/** @typedef {{ id: number, note?: string }} Note */ + +/** @param {Counter} c */ +function bump(c) { + c.count += 1; + c.label = c.label + "!"; +} + +/** @param {Note} n */ +function dropNote(n) { + delete n.note; +} + +/** @param {number} count @returns {Counter} */ +function counter(count) { + return { count: count, label: "c" + count }; +} + +const open = counter(1); +const frozen = Object.freeze(counter(2)); +bump(open); +bump(frozen); +console.log(JSON.stringify(open), JSON.stringify(frozen), Object.isFrozen(frozen)); + +/** @type {Note} */ +const note = { id: 1, note: "n" }; +Object.freeze(note); +dropNote(note); +console.log(JSON.stringify(note)); diff --git a/tests/harness/corpus-costs/7404-frozen-records.ts.json b/tests/harness/corpus-costs/7404-frozen-records.ts.json new file mode 100644 index 000000000..765134581 --- /dev/null +++ b/tests/harness/corpus-costs/7404-frozen-records.ts.json @@ -0,0 +1 @@ +{ "plain": 4725, "san": 55833 } diff --git a/tests/harness/corpus-costs/7405-frozen-records-sloppy.cjs.json b/tests/harness/corpus-costs/7405-frozen-records-sloppy.cjs.json new file mode 100644 index 000000000..c7cd5ec8f --- /dev/null +++ b/tests/harness/corpus-costs/7405-frozen-records-sloppy.cjs.json @@ -0,0 +1 @@ +{ "plain": 3175, "san": 37248 } From a00d56e5181bffee04a66ab99246807f680b66f5 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 02:26:57 -0500 Subject: [PATCH 023/184] Fix the int32 array CI failures --- packages/compiler/src/ir/int32-arrays.ts | 57 ++++++++++++++++-------- 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/packages/compiler/src/ir/int32-arrays.ts b/packages/compiler/src/ir/int32-arrays.ts index 0872fcddb..e67f51956 100644 --- a/packages/compiler/src/ir/int32-arrays.ts +++ b/packages/compiler/src/ir/int32-arrays.ts @@ -7,6 +7,8 @@ import { everyExprChild, everyStmtChild, everyStmtList, + mapExprChildren, + mapStmtChildren, transformExpr, transformStmt, transformStmtList, @@ -629,16 +631,34 @@ export function int32ArrayStores( return stores; } +const REPEATED_INDEX_OPS = new Set(["+", "-", "*", "&", "|", "^", "<<", ">>", ">>>"]); + +/** An index the inlined read may evaluate at each of its uses: a plain read, + * a literal, or integer arithmetic over them. Repeating it keeps the forms + * the backend derives from its operands at every use, such as the exact i64 + * of an int32 product, where a bound local holds only the double. */ +function repeatableIndex(e: IrExpr): boolean { + if (e.kind === "varRef" || e.kind === "numLit") return true; + if (e.kind === "unary") return e.op === "-" && repeatableIndex(e.operand); + return ( + e.kind === "bin" && + REPEATED_INDEX_OPS.has(e.op) && + repeatableIndex(e.left) && + repeatableIndex(e.right) + ); +} + /** Rewrite a function's candidates for the backend. A discarded pop uses * the candidate in place of its hidden local. An element-reader helper call * whose array is a candidate is inlined: the helper's returned ternary, with * the candidate for its array and the index for its index, so the read means - * the same. An index that is not a plain read or literal is bound to a fresh - * local first: in its own declaration just before the statement when the - * call is the whole value of a declaration or assignment (the form lowering - * gives `a[i]!`, and the one the backend reads without a box), otherwise in a - * sequence around the ternary. Selection keeps out indexes that write the - * array, so evaluating the index before the array read is unobservable. */ + * the same. An index that is not repeatable (repeatableIndex) is bound to a + * fresh local first: in its own declaration just before the statement when + * the call is the whole value of a declaration or assignment (the form + * lowering gives `a[i]!`, and the one the backend reads without a box), + * otherwise in a sequence around the ternary. Selection keeps out indexes + * that write the array, so evaluating the index before the array read is + * unobservable. */ function rewriteCandidates( fn: IrFunction, candidates: ReadonlySet, @@ -662,14 +682,13 @@ function rewriteCandidates( candidates.has(e.args[0]!.localId) ? e : null; - const simple = (e: IrExpr): boolean => e.kind === "varRef" || e.kind === "numLit"; // The statements of a list, with complex read indexes declared first. const hoist = (list: IrStmt[]): IrStmt[] => list.flatMap((s): IrStmt[] => { if (s.kind !== "varDecl" && s.kind !== "assign") return [s]; const value = s.kind === "varDecl" ? s.init : s.value; const call = readerCall(value); - if (!call || simple(call.args[1]!)) return [s]; + if (!call || repeatableIndex(call.args[1]!)) return [s]; const { decl, ref } = bind(call.args[1]!); const read: IrExpr = { ...call, args: [call.args[0]!, ref] }; if (s.kind === "varDecl") return [decl, { ...s, init: read }]; @@ -732,21 +751,23 @@ function rewriteCandidates( if (ret.kind !== "return" || !ret.value) return e; let index = call.args[1]!; const stmts: IrStmt[] = []; - if (!simple(index)) { + if (!repeatableIndex(index)) { const { decl, ref } = bind(index); stmts.push(decl); index = ref; } const operand = index; - const body = transformExpr(reparseExpr(ret.value), { - expr: (x) => { - if (x.kind !== "varRef") return x; - if (x.localId === arrayParam!.localId) return reparseExpr(array); - if (x.localId === indexParam!.localId) return reparseExpr(operand); - return x; - }, - stmt: (x) => x, - }); + // The arguments' copies are not searched again: an index may read a + // local that has the id of one of the helper's parameters. + const substituteExpr = (x: IrExpr): IrExpr => { + if (x.kind !== "varRef") return mapExprChildren(x, substituteExpr, substituteStmt); + if (x.localId === arrayParam!.localId) return reparseExpr(array); + if (x.localId === indexParam!.localId) return reparseExpr(operand); + return x; + }; + const substituteStmt = (x: IrStmt): IrStmt => + mapStmtChildren(x, substituteExpr, substituteStmt); + const body = substituteExpr(reparseExpr(ret.value)); // The ternary is a fresh copy (reparseExpr): it takes the call's location. const typed = body; typed.loc = e.loc; From 342cf3a063301e46a90e428a848dfd825aff80d8 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 00:29:22 -0700 Subject: [PATCH 024/184] Copy class-view property bags entry by entry Materializing the checked view of a JavaScript class instance copies its property bag through scr_dyn_copy_property_descriptors, which built a key string and a descriptor object per property and defined each through scr_dyn_define_property (about 2k instructions per property). Plain objects on both sides now copy each entry directly with the attributes, value or accessor pair and own-key order the definition gives; properties the target already has, non-extensible targets and other receivers keep the general path. Co-Authored-By: Claude Opus 5.5 --- .../7620-js-class-view-descriptors.js.json | 6 ++ packages/runtime/src/scr_json.c | 73 +++++++++++++-- packages/runtime/test/test_json.c | 86 ++++++++++++++++++ .../corpus/7620-js-class-view-descriptors.js | 91 +++++++++++++++++++ .../7620-js-class-view-descriptors.js.json | 1 + 5 files changed, 248 insertions(+), 9 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7620-js-class-view-descriptors.js.json create mode 100644 tests/corpus/7620-js-class-view-descriptors.js create mode 100644 tests/harness/corpus-costs/7620-js-class-view-descriptors.js.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7620-js-class-view-descriptors.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7620-js-class-view-descriptors.js.json new file mode 100644 index 000000000..2cb1ece22 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7620-js-class-view-descriptors.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7620-js-class-view-descriptors.js" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 907486fea..3676dd059 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -13699,19 +13699,74 @@ ScrDyn *scr_dyn_object_rest(const ScrDyn *source, const ScrDyn *excluded) { /* Compiler-owned native class views retain every property and its flags, * including nonenumerable data, without invoking accessors. */ -ScrDyn *scr_dyn_copy_property_descriptors(ScrDyn *target, const ScrDyn *src) { - if (!src || src->kind == SCR_DYN_UNDEF) return scr_dyn_retain(target); - ScrDyn *keys = scr_dyn_obj_own_keys(src); - if (!keys) return NULL; - for (size_t i = 0; i < keys->v.arr.len; i++) { - ScrDyn *key = keys->v.arr.items[i]; - ScrDyn *descriptor = scr_dyn_own_descriptor(src, key->v.str); +static void scr_dyn_copy_property_descriptor(ScrDyn *target, const ScrDyn *src, + const ScrDynEntry *from) { + /* Exactly scr_dyn_define_property(target, key, own descriptor of key) + * when `target` has no such property and accepts new ones: the entry + * takes the source's attributes, value or accessor pair. */ + ScrDynEntry *prior = scr_dyn_find_entry(target, from->key, from->key_len); + if (prior || target->non_extensible) { + ScrStr *name = scr_str_new(from->key, from->key_len); + ScrDyn *key = scr_dyn_new_str(name); + ScrDyn *descriptor = scr_dyn_own_descriptor(src, name); ScrDyn *result = descriptor ? scr_dyn_define_property(target, key, descriptor) : NULL; scr_dyn_release(descriptor); scr_dyn_release(result); - if (scr_exc_pending()) break; + scr_dyn_release(key); + scr_str_release(name); + return; + } + /* The names whose definition may move into a native Error's slots + * (scr_error_sync_slots). */ + static const char *const slots[] = {"cause", "stack", "name", "message", "code"}; + bool slot = false; + for (size_t i = 0; i < sizeof slots / sizeof *slots && !slot; i++) + slot = from->key_len == strlen(slots[i]) && memcmp(from->key, slots[i], from->key_len) == 0; + bool accessor = from->accessor, writable = !accessor && from->writable; + bool enumerable = from->enumerable, configurable = from->configurable; + ScrDyn *value = scr_dyn_retain(accessor ? scr_dyn_undefined() : from->value); + ScrDyn *getter = accessor && from->getter ? scr_dyn_retain(from->getter) : NULL; + ScrDyn *setter = accessor && from->setter ? scr_dyn_retain(from->setter) : NULL; + ScrStr *name = slot ? scr_str_new(from->key, from->key_len) : NULL; + scr_dyn_obj_set(target, from->key, from->key_len, value); + ScrDynEntry *entry = &target->v.obj.entries[target->v.obj.len - 1]; + entry->accessor = accessor; + entry->getter = getter; + entry->setter = setter; + entry->writable = writable; + entry->enumerable = enumerable; + entry->configurable = configurable; + if (name) { + scr_error_sync_cause(target, name); + scr_str_release(name); + } +} + +ScrDyn *scr_dyn_copy_property_descriptors(ScrDyn *target, const ScrDyn *src) { + if (!src || src->kind == SCR_DYN_UNDEF) return scr_dyn_retain(target); + if (src->kind == SCR_DYN_OBJ && target->kind == SCR_DYN_OBJ && src != target && + src != scr_builtin_object_prototype && target != scr_builtin_error_prototype) { + /* Plain objects on both sides: no fence, proxy trap or accessor runs, + * so each property is copied from its entry without building the key + * and descriptor objects the general path goes through. */ + if (src->v.obj.len) scr_dyn_region_store(target, NULL); + size_t *order = scr_dyn_obj_key_order(src); + for (size_t i = 0; i < src->v.obj.len && !scr_exc_pending(); i++) + scr_dyn_copy_property_descriptor(target, src, &src->v.obj.entries[order ? order[i] : i]); + free(order); + } else { + ScrDyn *keys = scr_dyn_obj_own_keys(src); + if (!keys) return NULL; + for (size_t i = 0; i < keys->v.arr.len; i++) { + ScrDyn *key = keys->v.arr.items[i]; + ScrDyn *descriptor = scr_dyn_own_descriptor(src, key->v.str); + ScrDyn *result = descriptor ? scr_dyn_define_property(target, key, descriptor) : NULL; + scr_dyn_release(descriptor); + scr_dyn_release(result); + if (scr_exc_pending()) break; + } + scr_dyn_release(keys); } - scr_dyn_release(keys); if (!scr_exc_pending()) { ScrDyn *symbols = scr_dyn_get_own_property_symbols(src); if (!symbols) return NULL; diff --git a/packages/runtime/test/test_json.c b/packages/runtime/test/test_json.c index 2c072006f..865f1c757 100644 --- a/packages/runtime/test/test_json.c +++ b/packages/runtime/test/test_json.c @@ -551,6 +551,91 @@ static void test_class_values(void) { scr_str_release(name); } +/* The class-view copy (scr_dyn_copy_property_descriptors) gives each + * property of the bag the attributes, value or accessors, and own-key + * position that defining it from its own descriptor would. */ +static ScrDyn *define_from(ScrDyn *target, const char *key, ScrDyn *value, bool writable, + bool enumerable, bool configurable) { + ScrDyn *descriptor = scr_dyn_new_obj(); + scr_dyn_obj_set(descriptor, "value", 5, value); + scr_dyn_obj_set(descriptor, "writable", 8, scr_dyn_new_bool(writable)); + scr_dyn_obj_set(descriptor, "enumerable", 10, scr_dyn_new_bool(enumerable)); + scr_dyn_obj_set(descriptor, "configurable", 12, scr_dyn_new_bool(configurable)); + ScrStr *name = S(key); + ScrDyn *k = scr_dyn_new_str(name); + ScrDyn *result = scr_dyn_define_property(target, k, descriptor); + scr_dyn_release(k); + scr_str_release(name); + scr_dyn_release(descriptor); + return result; +} + +static const ScrDynEntry *entry_of(const ScrDyn *obj, const char *key) { + for (size_t i = 0; i < obj->v.obj.len; i++) + if (obj->v.obj.entries[i].key_len == strlen(key) && + memcmp(obj->v.obj.entries[i].key, key, strlen(key)) == 0) + return &obj->v.obj.entries[i]; + return NULL; +} + +static void test_copy_property_descriptors(void) { + ScrDyn *bag = scr_dyn_new_obj(); + ScrDyn *getter = scr_dyn_new_obj(); + scr_dyn_release(define_from(bag, "label", scr_dyn_new_num(1), true, true, true)); + scr_dyn_release(define_from(bag, "hidden", scr_dyn_new_num(2), false, false, true)); + scr_dyn_obj_set(bag, "7", 1, scr_dyn_new_num(7)); + scr_dyn_obj_set(bag, "3", 1, scr_dyn_new_num(3)); + /* An accessor entry (the getter object stands in for its function). */ + scr_dyn_obj_set(bag, "computed", 8, scr_dyn_retain(scr_dyn_undefined())); + ScrDynEntry *accessor = (ScrDynEntry *)entry_of(bag, "computed"); + accessor->accessor = true; + accessor->getter = scr_dyn_retain(getter); + accessor->writable = false; + accessor->enumerable = false; + accessor->configurable = false; + + ScrDyn *view = scr_dyn_new_obj(); + scr_dyn_release(scr_dyn_copy_property_descriptors(view, bag)); + check(!scr_exc_pending() && view->v.obj.len == 5, "copy keeps every bag property"); + ScrDyn *keys = scr_dyn_obj_own_keys(view); + const char *order[] = {"3", "7", "label", "hidden", "computed"}; + bool ordered = keys->v.arr.len == 5; + for (size_t i = 0; ordered && i < 5; i++) ordered = str_is(keys->v.arr.items[i]->v.str, order[i]); + check(ordered, "copy keeps the bag's own-key order"); + scr_dyn_release(keys); + const ScrDynEntry *hidden = entry_of(view, "hidden"); + check(hidden && !hidden->accessor && hidden->value->v.num == 2 && !hidden->writable && + !hidden->enumerable && hidden->configurable, + "copy keeps data attributes"); + const ScrDynEntry *copied = entry_of(view, "computed"); + check(copied && copied->accessor && copied->getter == getter && !copied->setter && + !copied->writable && !copied->enumerable && !copied->configurable && + copied->value->kind == SCR_DYN_UNDEF, + "copy keeps an accessor without calling it"); + + /* A property the target already has is redefined like defineProperty. */ + ScrDyn *existing = scr_dyn_new_obj(); + scr_dyn_obj_set(existing, "label", 5, scr_dyn_new_num(9)); + scr_dyn_release(scr_dyn_copy_property_descriptors(existing, bag)); + const ScrDynEntry *label = entry_of(existing, "label"); + check(!scr_exc_pending() && existing->v.obj.len == 5 && existing->v.obj.entries[0].key_len == 5 && + label->value->v.num == 1, + "copy redefines an existing property in place"); + + /* A target that takes no new properties throws as defineProperty does. */ + ScrDyn *closed = scr_dyn_new_obj(); + scr_dyn_release(scr_dyn_prevent_extensions(closed)); + ScrDyn *refused = scr_dyn_copy_property_descriptors(closed, bag); + check(!refused && scr_exc_pending() && closed->v.obj.len == 0, "copy onto a non-extensible object throws"); + scr_exc_clear(); + + scr_dyn_release(closed); + scr_dyn_release(existing); + scr_dyn_release(view); + scr_dyn_release(bag); + scr_dyn_release(getter); +} + static void test_primitive_owners(void) { ScrDyn *values[] = {scr_dyn_new_null(), scr_dyn_new_bool(false), scr_dyn_new_bool(true)}; const char *texts[] = {"null", "false", "true"}; @@ -579,6 +664,7 @@ int main(void) { test_class_values(); scr_init(); test_primitive_owners(); + test_copy_property_descriptors(); /* ── primitives ─────────────────────────────────────────────────── */ ScrDyn *d = parse_ok("null", "parse null"); diff --git a/tests/corpus/7620-js-class-view-descriptors.js b/tests/corpus/7620-js-class-view-descriptors.js new file mode 100644 index 000000000..a0dc42813 --- /dev/null +++ b/tests/corpus/7620-js-class-view-descriptors.js @@ -0,0 +1,91 @@ +// An instance of a JS class with public fields keeps its own properties in +// a property bag, and untyped code that calls a method on it sees a view +// built from that bag: data properties with their attributes, accessors, +// index-like keys (which come first, in ascending order), symbols, and +// properties added or removed after construction. Every refresh of the view +// must copy the bag exactly. +'use strict'; + +class Box { + label = 'box'; + count = 0; + constructor(n) { + this.count = n; + } + describe() { + return `${this.label}:${this.count}`; + } + bump() { + this.count++; + return this; + } +} + +function call(target, method, ...args) { + return target[method](...args); +} + +function put(target, key, value) { + target[key] = value; +} + +function drop(target, key) { + delete target[key]; +} + +function show(title, target) { + const lines = []; + for (const key of Reflect.ownKeys(target)) { + const d = Object.getOwnPropertyDescriptor(target, key); + const kind = 'value' in d ? `value=${String(d.value)} writable=${d.writable}` : `get=${typeof d.get} set=${typeof d.set}`; + lines.push(` ${String(key)}: ${kind} enumerable=${d.enumerable} configurable=${d.configurable}`); + } + console.log(`${title}\n${lines.join('\n')}`); +} + +const box = new Box(2); +Object.defineProperty(box, 'hidden', { value: 42, enumerable: false, writable: false, configurable: true }); +Object.defineProperty(box, 'computed', { + get() { + return this.count * 10; + }, + set(v) { + this.count = v / 10; + }, + enumerable: true, + configurable: true, +}); +Object.defineProperty(box, 'getterOnly', { get: () => 'g', enumerable: false, configurable: false }); +put(box, 2, 'two'); +put(box, 1, 'one'); +put(box, 'extra', 'x'); +const tag = Symbol('tag'); +put(box, tag, 'symbol'); + +console.log(call(box, 'describe')); +show('after construction', box); +console.log('keys', Object.keys(box).join(',')); + +call(box, 'bump'); +put(box, 'computed', 70); +drop(box, 'extra'); +put(box, 'later', true); +console.log(call(box, 'describe')); +show('after changes', box); +console.log('computed', box.computed, 'hidden', box.hidden, 'getterOnly', box.getterOnly); + +// Many views in a row see the same, current properties. +let total = 0; +for (let i = 0; i < 5; i++) { + call(box, 'bump'); + total += Object.keys(box).length; +} +console.log('total keys', total, call(box, 'describe')); + +const other = new Box(7); +console.log(call(other, 'describe'), Object.keys(other).join(',')); +try { + call(box, 'missing'); +} catch (error) { + console.log(error instanceof TypeError, error.message); +} diff --git a/tests/harness/corpus-costs/7620-js-class-view-descriptors.js.json b/tests/harness/corpus-costs/7620-js-class-view-descriptors.js.json new file mode 100644 index 000000000..ef439d096 --- /dev/null +++ b/tests/harness/corpus-costs/7620-js-class-view-descriptors.js.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 4500 } From bcd506fc8b14c50ae7eb8ba96764d7d42091552e Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 00:29:28 -0700 Subject: [PATCH 025/184] Return ASCII header values as their own ByteString Converting a header value to a ByteString read it one UTF-16 code unit at a time through scr_str_char_code_at into a fresh buffer. An all-ASCII value, which nearly every header is, is already its own ByteString, so it is now returned as is. Co-Authored-By: Claude Opus 5.5 --- packages/runtime/src/scr_fetch.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index 96e90e47c..549806f39 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -3287,6 +3287,10 @@ static bool sf_header_value_ok(const ScrStr *value) { } static ScrStr *sf_header_value_bytestring(ScrStr *value) { + /* An ASCII value is its own ByteString: each byte is one UTF-16 unit. */ + size_t ascii = 0; + while (ascii < value->len && (unsigned char)value->data[ascii] < 0x80) ascii++; + if (ascii == value->len) return scr_str_retain(value); size_t len = (size_t)scr_str_utf16_len(value); char *bytes = malloc(len ? len : 1); if (!bytes) sf_oom(); From ac40773362d71f04c5564a7ba85ba4220ce90a3f Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 02:54:19 -0500 Subject: [PATCH 026/184] Excuse date.fromTime in the attestation parity test date.fromTime makes a Date from the time value a Date local holds and reads no ambient state. The local's initializer is the Date expression the fences witness. --- tests/harness/surface-manifest.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/harness/surface-manifest.test.ts b/tests/harness/surface-manifest.test.ts index 2ff3c71c2..58958c102 100644 --- a/tests/harness/surface-manifest.test.ts +++ b/tests/harness/surface-manifest.test.ts @@ -558,6 +558,10 @@ describe("determinism attestation ↔ fence parity", () => { // witnessed Date method; assigning it to getTime would also deny // valueOf/toISOString/etc. under a getTime-only fence. "date.checkedValue", + // Makes a Date from the time value a Date local holds + // (date-values.ts) without reading ambient state. The local's + // initializer is the Date expression the fences witness. + "date.fromTime", ]); const missing = Object.keys(LIB_FN_SIGS).filter( (fn) => From b9275c2693597c3e79287fcbedc80f5d49814d5f Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 03:05:19 -0500 Subject: [PATCH 027/184] buffer: fromArrayBuffer ranges, the deprecated constructor, Buffer as a value --- .../limitations/buffer-construction.mdx | 5 + .../ambient/scriptc-node-fallback.d.ts | 8 + .../compiler/src/backend/llvm/lib-shared.ts | 9 + .../src/frontend/lowering/containers/bytes.ts | 259 ++++++++++++++---- .../lowering/expressions/element-access.ts | 3 + .../lowering/expressions/identifiers.ts | 3 + .../lowering/expressions/property-access.ts | 45 +++ .../src/frontend/lowering/lower-calls.ts | 9 +- .../src/frontend/lowering/lower-classes.ts | 45 +-- .../src/frontend/lowering/lower-inspect.ts | 20 +- .../compiler/src/frontend/lowering/lowerer.ts | 5 + .../src/frontend/lowering/surfaces.ts | 3 + packages/compiler/src/frontend/program.ts | 5 + packages/compiler/src/ir/builtin-effects.ts | 3 + .../compiler/src/ir/builtin-signatures.ts | 9 + packages/compiler/src/ir/int32-slots.ts | 2 + packages/compiler/src/ir/runtime-features.ts | 21 +- .../7527-buffer-from-array-buffer.ts.json | 6 + .../7528-buffer-from-array-buffer.cjs.json | 6 + .../corpus/7529-buffer-constructor.cjs.json | 6 + .../corpus/7530-buffer-constructor.ts.json | 6 + .../main.cjs.json | 18 ++ .../main.cjs.json | 18 ++ .../tests/corpus/7533-buffer-value.cjs.json | 6 + .../7534-buffer-prototype-members.cjs.json | 6 + .../7535-inspect-function-properties.cjs.json | 6 + packages/runtime/src/scr_async.h | 4 + packages/runtime/src/scr_async_dyn.c | 16 ++ packages/runtime/src/scr_builtin_values.c | 214 +++++++++++++++ packages/runtime/src/scr_bytes.c | 20 +- packages/runtime/src/scr_bytes.h | 27 ++ packages/runtime/src/scr_bytes_dyn.c | 203 ++++++++++++++ packages/runtime/src/scr_bytes_io.c | 66 ++++- packages/runtime/src/scr_dyn.h | 4 + packages/runtime/src/scr_inspect.c | 40 ++- packages/runtime/src/scr_json.c | 22 +- tests/corpus/2570-buffer-arg-validation.cjs | 3 +- tests/corpus/7527-buffer-from-array-buffer.ts | 33 +++ .../corpus/7528-buffer-from-array-buffer.cjs | 31 +++ tests/corpus/7529-buffer-constructor.cjs | 80 ++++++ tests/corpus/7530-buffer-constructor.ts | 42 +++ .../main.cjs | 36 +++ .../node_modules/legacy-buf/index.d.ts | 3 + .../node_modules/legacy-buf/index.js | 15 + .../node_modules/legacy-buf/package.json | 5 + .../main.cjs | 10 + .../node_modules/legacy-buf/index.d.ts | 3 + .../node_modules/legacy-buf/index.js | 15 + .../node_modules/legacy-buf/package.json | 5 + tests/corpus/7533-buffer-value.cjs | 101 +++++++ .../corpus/7534-buffer-prototype-members.cjs | 29 ++ .../7535-inspect-function-properties.cjs | 29 ++ .../7527-buffer-from-array-buffer.ts.json | 1 + .../7528-buffer-from-array-buffer.cjs.json | 1 + .../7529-buffer-constructor.cjs.json | 1 + .../7530-buffer-constructor.ts.json | 1 + .../main.cjs.json | 1 + .../main.cjs.json | 1 + .../corpus-costs/7533-buffer-value.cjs.json | 1 + .../7534-buffer-prototype-members.cjs.json | 1 + .../7535-inspect-function-properties.cjs.json | 1 + tests/harness/differential.test.ts | 6 +- tests/harness/errors.test.ts | 49 ++++ 63 files changed, 1535 insertions(+), 117 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7527-buffer-from-array-buffer.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7528-buffer-from-array-buffer.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7529-buffer-constructor.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7530-buffer-constructor.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7531-buffer-constructor-dependency/main.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7532-buffer-constructor-check-limit/main.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7533-buffer-value.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7534-buffer-prototype-members.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7535-inspect-function-properties.cjs.json create mode 100644 tests/corpus/7527-buffer-from-array-buffer.ts create mode 100644 tests/corpus/7528-buffer-from-array-buffer.cjs create mode 100644 tests/corpus/7529-buffer-constructor.cjs create mode 100644 tests/corpus/7530-buffer-constructor.ts create mode 100644 tests/corpus/7531-buffer-constructor-dependency/main.cjs create mode 100644 tests/corpus/7531-buffer-constructor-dependency/node_modules/legacy-buf/index.d.ts create mode 100644 tests/corpus/7531-buffer-constructor-dependency/node_modules/legacy-buf/index.js create mode 100644 tests/corpus/7531-buffer-constructor-dependency/node_modules/legacy-buf/package.json create mode 100644 tests/corpus/7532-buffer-constructor-check-limit/main.cjs create mode 100644 tests/corpus/7532-buffer-constructor-check-limit/node_modules/legacy-buf/index.d.ts create mode 100644 tests/corpus/7532-buffer-constructor-check-limit/node_modules/legacy-buf/index.js create mode 100644 tests/corpus/7532-buffer-constructor-check-limit/node_modules/legacy-buf/package.json create mode 100644 tests/corpus/7533-buffer-value.cjs create mode 100644 tests/corpus/7534-buffer-prototype-members.cjs create mode 100644 tests/corpus/7535-inspect-function-properties.cjs create mode 100644 tests/harness/corpus-costs/7527-buffer-from-array-buffer.ts.json create mode 100644 tests/harness/corpus-costs/7528-buffer-from-array-buffer.cjs.json create mode 100644 tests/harness/corpus-costs/7529-buffer-constructor.cjs.json create mode 100644 tests/harness/corpus-costs/7530-buffer-constructor.ts.json create mode 100644 tests/harness/corpus-costs/7531-buffer-constructor-dependency/main.cjs.json create mode 100644 tests/harness/corpus-costs/7532-buffer-constructor-check-limit/main.cjs.json create mode 100644 tests/harness/corpus-costs/7533-buffer-value.cjs.json create mode 100644 tests/harness/corpus-costs/7534-buffer-prototype-members.cjs.json create mode 100644 tests/harness/corpus-costs/7535-inspect-function-properties.cjs.json diff --git a/docs/content/limitations/buffer-construction.mdx b/docs/content/limitations/buffer-construction.mdx index aca0c74ca..c88e01336 100644 --- a/docs/content/limitations/buffer-construction.mdx +++ b/docs/content/limitations/buffer-construction.mdx @@ -1,3 +1,8 @@ `Buffer.from` accepts untyped strings, byte arrays, arrays, and plain array-like or Buffer JSON objects. Encoding arguments must be supported literals. +`Buffer.from(arrayBuffer, byteOffset, length)` is a view of the ArrayBuffer or SharedArrayBuffer, with Node's offset and length checks. Custom input `valueOf` hooks and opaque class or native-handle inputs throw an unsupported-operation error. +The deprecated `Buffer()` and `new Buffer()` take up to three arguments, without spreads, and follow Node: the first call made outside `node_modules` reports DEP0005 on the next tick, a number allocates zero-filled bytes, and other arguments convert as `Buffer.from` does. +In JavaScript, `Buffer` read as a value (the global or the buffer module's `Buffer`) is one function with Node's statics and `poolSize`. +Calling or constructing it through a stored value, reading its `prototype`, and its `copyBytesFrom` and `of` statics throw an unsupported-operation error. +`Buffer.prototype` members read as values are functions that run on Buffer receivers; inherited and symbol-keyed members throw an unsupported-operation error. diff --git a/packages/compiler/ambient/scriptc-node-fallback.d.ts b/packages/compiler/ambient/scriptc-node-fallback.d.ts index b195c3fbb..e26faf956 100644 --- a/packages/compiler/ambient/scriptc-node-fallback.d.ts +++ b/packages/compiler/ambient/scriptc-node-fallback.d.ts @@ -917,6 +917,14 @@ interface Buffer< readBigUint64LE(offset?: number): bigint; } interface BufferConstructor { + /** @deprecated since v10.0.0 - Use `Buffer.from(string[, encoding])` instead. */ + new (str: string, encoding?: BufferEncoding): Buffer; + /** @deprecated since v10.0.0 - Use `Buffer.alloc()` instead. */ + new (size: number): Buffer; + /** @deprecated since v10.0.0 - Use `Buffer.from(array)` instead. */ + new (array: ArrayLike): Buffer; + /** @deprecated since v10.0.0 - Use `Buffer.from(arrayBuffer[, byteOffset[, length]])` instead. */ + new (arrayBuffer: ArrayBuffer | SharedArrayBuffer): Buffer; from(data: string, encoding?: BufferEncoding): Buffer; /* The ArrayBuffer form is the VIEW construction: x.buffer with an * optional byte offset/length shares x's storage (the subarray rule). */ diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 7dd4370c5..b92047e43 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -454,6 +454,7 @@ export const LIB_FN_SYMS: Record = { "buffer.poolFromDyn": "scr_buffer_from_dyn_pooled", "buffer.poolFromDynRef": "scr_buffer_from_dyn_ref_pooled", "buffer.poolFromDynOffset": "scr_buffer_from_dyn_offset_pooled", + "buffer.poolFromDynRange": "scr_buffer_from_dyn_range_pooled", "buffer.poolSize": "scr_buffer_pool_size", "buffer.setPoolSize": "scr_buffer_set_pool_size", "buffer.byteLenDyn": "scr_buffer_byte_length_dyn", @@ -479,6 +480,12 @@ export const LIB_FN_SYMS: Record = { "bytes.equalsChk": "scr_bytes_equals_chk", "bytes.compareChk": "scr_bytes_compare_chk", "buffer.newStringFail": "scr_buffer_new_string_fail", + "buffer.deprecation": "scr_buffer_deprecation", + "buffer.deprecationCount": "scr_buffer_deprecation_count", + "buffer.constructor": "scr_buffer_constructor_value", + "buffer.installValue": "scr_buffer_install_value", + "buffer.staticValue": "scr_buffer_static_value", + "buffer.prototypeMember": "scr_buffer_prototype_member", "fs.toUnixTimestamp": "scr_fs_to_unix_timestamp", "fs.existsChk": "scr_fs_exists_async", "fs.mkdtempSyncChk": "scr_fs_mkdtemp_sync_chk", @@ -1661,6 +1668,8 @@ export const USES_TIMERS_LIB_FNS = new Set([ "url.legacyParse", "url.legacyFormat", "url.legacyResolve", + // The deprecated Buffer constructor reports DEP0005 on the next tick. + "buffer.deprecation", // The wrappers schedule warnings and callbacks on the tick queue. "util.deprecate", "util.callbackify", diff --git a/packages/compiler/src/frontend/lowering/containers/bytes.ts b/packages/compiler/src/frontend/lowering/containers/bytes.ts index 995cb177a..82c6f3052 100644 --- a/packages/compiler/src/frontend/lowering/containers/bytes.ts +++ b/packages/compiler/src/frontend/lowering/containers/bytes.ts @@ -15,6 +15,7 @@ import { type IrLibFn, type IrLocal, type IrParam, + type IrFunction, type IrStmt, type IrType, STRING, @@ -30,7 +31,8 @@ import { import { isJsSourceFile, locOf } from "../../program.js"; import { stdlibGlobalNameOf } from "../surfaces.js"; import type { Lowerer } from "../lowerer.js"; -import { newFnCtx, own } from "../lowerer.js"; +import { ENTRY_NAME, newFnCtx, own } from "../lowerer.js"; +import { everyStmt } from "../../../ir/traverse.js"; import { buildBytesSortFn, closureKeyOrder } from "../lower-array-sort.js"; import { isSafeToDiscard } from "../expressions/evaluation-safety.js"; import { lowerDynObjectLiteral } from "../expressions/object-literals.js"; @@ -1807,7 +1809,12 @@ function knownBufferProducer( * or empty encoding and throws ERR_UNKNOWN_ENCODING for an unknown name; an * ArrayBuffer takes the second argument as its byte offset; other sources * ignore it. */ -function dynBufferFromParts(source: IrExpr, encoding: IrExpr, loc: SrcLoc): IrExpr { +function dynBufferFromParts( + source: IrExpr, + encoding: IrExpr, + loc: SrcLoc, + length?: IrExpr, +): IrExpr { const utf8: IrExpr = { kind: "strLit", value: "utf8", type: STRING, loc }; const isString = (x: IrExpr): IrExpr => ({ kind: "dynTest", @@ -1848,20 +1855,48 @@ function dynBufferFromParts(source: IrExpr, encoding: IrExpr, loc: SrcLoc): IrEx type: BYTES_U8, loc, }, - // An ArrayBuffer takes the second argument as its byte offset - // (scr_buffer_from_dyn_offset_pooled); other values convert with utf8. - else_: { - kind: "libCall", - fn: "buffer.poolFromDynOffset", - args: [source, encoding], - type: BYTES_U8, - loc, - }, + // An ArrayBuffer takes the second argument as its byte offset and the + // third as its length (scr_buffer_from_dyn_range_pooled); other values + // convert with utf8. + else_: length + ? { + kind: "libCall", + fn: "buffer.poolFromDynRange", + args: [source, encoding, length], + type: BYTES_U8, + loc, + } + : { + kind: "libCall", + fn: "buffer.poolFromDynOffset", + args: [source, encoding], + type: BYTES_U8, + loc, + }, type: BYTES_U8, loc, }; } +/** Buffer.from(arrayBuffer, byteOffset[, length]): each argument in order + * as a checked-dynamic value, read by the runtime as Node's + * fromArrayBuffer reads them. */ +function bufferFromArrayBufferRange( + lowerer: Lowerer, + args: readonly ts.Expression[], + loc: SrcLoc, + first?: IrExpr, +): IrExpr { + const operands = [0, 1, 2].map((i) => + i === 0 && first + ? first + : args[i] + ? lowerer.lowerExprExpecting(args[i]!, DYN) + : dynUndefinedExpr(loc), + ); + return { kind: "libCall", fn: "buffer.poolFromDynRange", args: operands, type: BYTES_U8, loc }; +} + /** Lowers `source` and `encNode` (or undefined) into hidden dyn locals, in * source order, and builds the result from their references. */ function withDynBufferArguments( @@ -1909,52 +1944,81 @@ function dynBufferFromRuntimeEncoding( ); } -/** The deprecated `new Buffer(arg[, encoding])` in a dependency's - * JavaScript, through the global or an alias of it (buffer-crc32 and yazl - * keep it as the fallback for runtimes without Buffer.alloc/Buffer.from). - * Node never emits DEP0005 for calls from node_modules, so the construction - * is exactly Node's: a number allocates zero-filled bytes (and throws - * ERR_INVALID_ARG_TYPE beside a string second argument), anything else - * converts as Buffer.from does. Null elsewhere: other callers owe the - * deprecation warning. */ -export function lowerNodeModulesBufferConstruct( +/** The deprecated `Buffer(arg[, encodingOrOffset[, length]])`, called or + * constructed through the global or an alias of it, as Node's Buffer + * function runs it: the arguments evaluate in order, DEP0005 is reported + * on the next tick (once, and never for a call made from node_modules, + * which only counts toward Node's 10000-call check), then a number + * allocates zero-filled bytes (and throws ERR_INVALID_ARG_TYPE beside a + * string second argument) and anything else converts as Buffer.from does. + * Null for spread arguments and more than three. */ +export function lowerBufferConstruct( lowerer: Lowerer, - expr: ts.NewExpression, + expr: ts.NewExpression | ts.CallExpression, ): IrExpr | null { - const sf = expr.getSourceFile(); - if (!isJsSourceFile(sf) || !/[\\/]node_modules[\\/]/.test(sf.fileName)) return null; + if (ts.isCallExpression(expr) && expr.questionDotToken) return null; const args = expr.arguments?.nodes ?? []; - if (args.length < 1 || args.length > 2 || args.some(ts.isSpreadElement)) return null; + if (args.length > 3 || args.some(ts.isSpreadElement)) return null; if (stdlibGlobalNameOf(lowerer, expr.expression) !== "Buffer") return null; const loc = locOf(expr); - const value = withDynBufferArguments( - lowerer, - lowerer.lowerExpr(args[0]!), - args[1], - loc, - (source, encoding) => ({ + const locals = [0, 1, 2].slice(0, Math.max(args.length, 2)).map((i) => { + const local = lowerer.declareHiddenLocal("%bufferArgument", DYN); + const node = args[i]; + const init = node === undefined ? dynUndefinedExpr(loc) : lowerer.lowerExprExpecting(node, DYN); + return { local, init }; + }); + const [source, second, third] = locals.map(({ local }) => varRef(local.id, DYN, loc)) as [ + IrExpr, + IrExpr, + IrExpr | undefined, + ]; + const inNodeModules = /[\\/]node_modules[\\/]/.test(expr.getSourceFile().fileName); + const value: IrExpr = { + kind: "ternary", + cond: { kind: "dynTest", test: "number", value: source, type: BOOL, loc }, + then: { kind: "ternary", - cond: { kind: "dynTest", test: "number", value: source, type: BOOL, loc }, - then: { - kind: "ternary", - cond: { kind: "dynTest", test: "string", value: encoding, type: BOOL, loc }, - then: { kind: "libCall", fn: "buffer.newStringFail", args: [source], type: BYTES_U8, loc }, - else_: { - kind: "libCall", - fn: "buffer.alloc", - args: [{ kind: "dynCheck", value: source, type: F64, loc }], - type: BYTES_U8, - loc, - }, + cond: { kind: "dynTest", test: "string", value: second, type: BOOL, loc }, + then: { kind: "libCall", fn: "buffer.newStringFail", args: [source], type: BYTES_U8, loc }, + else_: { + kind: "libCall", + fn: "buffer.alloc", + args: [{ kind: "dynCheck", value: source, type: F64, loc }], type: BYTES_U8, loc, }, - else_: dynBufferFromParts(source, encoding, loc), type: BYTES_U8, loc, - }), - ); - return { kind: "libCall", fn: "buffer.brand", args: [value], type: BYTES_U8, loc }; + }, + else_: dynBufferFromParts(source, second, loc, third), + type: BYTES_U8, + loc, + }; + return { + kind: "seqExpr", + stmts: [ + ...locals.map(({ local, init }): IrStmt => ({ + kind: "varDecl", + localId: local.id, + init, + loc, + })), + { + kind: "exprStmt", + expr: { + kind: "libCall", + fn: inNodeModules ? "buffer.deprecationCount" : "buffer.deprecation", + args: [], + type: VOID, + loc, + }, + loc, + }, + ], + result: { kind: "libCall", fn: "buffer.brand", args: [value], type: BYTES_U8, loc }, + type: BYTES_U8, + loc, + }; } /** Buffer.from(array or bytes, extra): Node copies an array-like source and @@ -2018,7 +2082,16 @@ function lowerBufferStaticValue( lowerer.typeOf(args[0]!).getSymbol(), ) ) { - return lowerArrayBufferView(lowerer, args, "u8", loc); + // A byte offset or length follows Node's fromArrayBuffer (its + // conversions and ERR_BUFFER_OUT_OF_BOUNDS), not the typed-array view. + if (args.length === 1) return lowerArrayBufferView(lowerer, args, "u8", loc); + return bufferFromArrayBufferRange(lowerer, args, loc); + } + // Buffer.from(value, byteOffset, length) over an untyped value: an + // ArrayBuffer's range, or the two-argument conversion of other values. + if (args.length === 3 && !args.some(ts.isSpreadElement)) { + const first = lowerer.lowerExpr(args[0]!); + if (first.type.kind === "dyn") return bufferFromArrayBufferRange(lowerer, args, loc, first); } // Buffer.from(x.buffer[, byteOffset[, length]]): a u8 VIEW sharing // x's storage (Node shares the ArrayBuffer; length is in BYTES). @@ -2445,11 +2518,101 @@ function lowerBufferStaticValue( const s = lowerer.lowerExprExpecting(args[0]!, STRING); return { kind: "libCall", fn: "buffer.isEncoding", args: [s], type: BOOL, loc }; } + // Any other single argument: Node answers false for non-strings. + if (args.length === 1 && !ts.isSpreadElement(args[0]!)) { + const value = lowerer.declareHiddenLocal("%bufferEncoding", DYN); + const ref = varRef(value.id, DYN, loc); + return { + kind: "seqExpr", + stmts: [ + { + kind: "varDecl", + localId: value.id, + init: lowerer.lowerExprExpecting(args[0]!, DYN), + loc, + }, + ], + result: { + kind: "logical", + op: "&&", + left: { kind: "dynTest", test: "string", value: ref, type: BOOL, loc }, + right: { + kind: "libCall", + fn: "buffer.isEncoding", + args: [{ kind: "dynCheck", value: ref, type: STRING, loc }], + type: BOOL, + loc, + }, + type: BOOL, + loc, + }, + type: BOOL, + loc, + }; + } lowerer.noLowering( "Buffer.isEncoding with this argument shape", call, - "one string-typed argument is the lowered form", + "one argument is the lowered form", ); } return null; // of, copyBytesFrom, ... → the SC2020 member fence } + +/** `Buffer.prototype.` or `Buffer.prototype[key]` read in JavaScript + * (smart-buffer's `typeof Buffer.prototype[method]` probe): the member as + * scr_buffer_prototype_member answers it. Null for other expressions. */ +export function lowerBufferPrototypeMember( + lowerer: Lowerer, + expr: ts.PropertyAccessExpression | ts.ElementAccessExpression, +): IrExpr | null { + if (expr.questionDotToken || !isJsSourceFile(expr.getSourceFile())) return null; + const receiver = expr.expression; + if ( + !ts.isPropertyAccessExpression(receiver) || + receiver.questionDotToken || + receiver.name.text !== "prototype" || + stdlibGlobalNameOf(lowerer, receiver.expression) !== "Buffer" + ) + return null; + const loc = locOf(expr); + const key: IrExpr = ts.isPropertyAccessExpression(expr) + ? { + kind: "dynFrom", + value: { kind: "strLit", value: expr.name.text, type: STRING, loc }, + type: DYN, + loc, + } + : lowerer.lowerExprExpecting(expr.argumentExpression, DYN); + return { kind: "libCall", fn: "buffer.prototypeMember", args: [key], type: DYN, loc }; +} + +const BUFFER_VALUE_FNS: ReadonlySet = new Set([ + "buffer.constructor", + "buffer.staticValue", + "buffer.prototypeMember", +]); + +/** A program that reads the Buffer function's value installs it in %main, + * so a Buffer's `.constructor` answers the same function. */ +export function installBufferValue(functions: IrFunction[]): void { + let found = false; + const visitor = { + expr: (e: IrExpr): boolean => { + if (e.kind === "libCall" && BUFFER_VALUE_FNS.has(e.fn)) found = true; + return !found; + }, + stmt: (): boolean => !found, + }; + for (const fn of functions) { + for (const s of fn.body) everyStmt(s, visitor); + if (found) break; + } + const main = functions.find((fn) => fn.name === ENTRY_NAME); + if (!found || !main) return; + main.body.unshift({ + kind: "exprStmt", + expr: { kind: "libCall", fn: "buffer.installValue", args: [], type: VOID, loc: main.loc }, + loc: main.loc, + }); +} diff --git a/packages/compiler/src/frontend/lowering/expressions/element-access.ts b/packages/compiler/src/frontend/lowering/expressions/element-access.ts index 0ea94ec7a..24de38630 100644 --- a/packages/compiler/src/frontend/lowering/expressions/element-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/element-access.ts @@ -71,6 +71,7 @@ import { isClassCallback } from "../class-callbacks.js"; import { isKeyedFieldAccess } from "./keyed-field-targets.js"; import { lowerFieldCompoundValue } from "./updates.js"; import { readsHeldEvolvingArray } from "../lower-stmts.js"; +import { lowerBufferPrototypeMember } from "../containers/bytes.js"; /** `a[i]` reads. Only f64 indices into array receivers are modeled; JS * string-key element access (`a["length"]`) and string indexing (`s[0]` @@ -104,6 +105,8 @@ export function lowerElementAccess(lowerer: Lowerer, expr: ts.ElementAccessExpre const en = lowerEnumAccess(lowerer, expr); if (en) return en; } + const bufferMember = lowerBufferPrototypeMember(lowerer, expr); + if (bufferMember) return bufferMember; const cachedModule = lowerRequireCacheElement(lowerer, expr); if (cachedModule) return cachedModule; const namespaceElement = lowerModuleNamespaceElement(lowerer, expr); diff --git a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts index ef0d8a2d1..1570ea709 100644 --- a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts +++ b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts @@ -744,6 +744,9 @@ export function lowerIdentifierExpr(lowerer: Lowerer, expr: ts.Identifier, loc: return { kind: "libCall", fn: "webcrypto.object", args: [], type: DYN, loc }; const nativeAlias = stdlibGlobalNameOf(lowerer, expr); if (nativeAlias === "Buffer" && expr.text !== "Buffer") { + // JavaScript reads the Buffer function's value (builtinGlobalValue). + if (isJsSourceFile(ts.getSourceFileOfNode(expr))) + return { kind: "libCall", fn: "buffer.constructor", args: [], type: DYN, loc }; return { kind: "strLit", value: "[builtin Buffer]", type: STRING, loc }; } if (lowerer.isStdlibGlobal(expr, "fetch")) { diff --git a/packages/compiler/src/frontend/lowering/expressions/property-access.ts b/packages/compiler/src/frontend/lowering/expressions/property-access.ts index 7b12297a8..607ad221b 100644 --- a/packages/compiler/src/frontend/lowering/expressions/property-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/property-access.ts @@ -153,6 +153,7 @@ import { lowerStringCodesValue, } from "../lower-builtin-values.js"; import { lowerArrayFromValue } from "../containers/array-construction.js"; +import { lowerBufferPrototypeMember } from "../containers/bytes.js"; import { lowerNetAddressStaticRead } from "../builtins/net-address.js"; import { STATS_TYPE_TESTS } from "../builtins/filesystem.js"; import { arrayValueRead, arrayValueType } from "../array-values.js"; @@ -310,6 +311,10 @@ export function lowerPropertyAccessExpr( // Cloudflare probe) folds like `navigator.userAgent`. const navigator = lowerNavigatorMember(lowerer, expr, NODE_COMPAT_VERSION); if (navigator) return navigator; + { + const member = lowerBufferPrototypeMember(lowerer, expr); + if (member) return member; + } // A dgram socket's method read as a value in JavaScript (`emit: // socket.emit.bind(socket)`): the socket crosses as its checked-dynamic // handle, whose methods are function values that dispatch on their @@ -1769,6 +1774,28 @@ export function lowerIntrinsicProperty( return lowerNumberParserValue(lowerer, numberMember, locOf(expr)); if (lowerer.stdlibGlobalMember(expr, "Buffer") === "poolSize") return { kind: "libCall", fn: "buffer.poolSize", args: [], type: F64, loc: locOf(expr) }; + { + const member = lowerer.stdlibGlobalMember(expr, "Buffer"); + if (member === "name") + return { kind: "strLit", value: "Buffer", type: STRING, loc: locOf(expr) }; + if (member === "length") return numLit(3, locOf(expr)); + // JavaScript reads the statics from the Buffer function's value, so + // each has one identity and Node's name and length. + if ( + member !== null && + BUFFER_VALUE_STATICS.has(member) && + isJsSourceFile(ts.getSourceFileOfNode(expr)) + ) { + const loc = locOf(expr); + return { + kind: "libCall", + fn: "buffer.staticValue", + args: [{ kind: "strLit", value: member, type: STRING, loc }], + type: DYN, + loc, + }; + } + } const netAddressStatic = lowerNetAddressStaticRead(lowerer, expr); if (netAddressStatic) return netAddressStatic; if (lowerer.stdlibGlobalMember(expr, "Buffer") === "isBuffer") @@ -4173,6 +4200,8 @@ export function builtinFunctionValue(name: string, loc: SrcLoc): IrExpr { * null when it stays an identity token (JavaScript) or fenced. */ export function builtinGlobalValue(canonical: string, loc: SrcLoc, js = false): IrExpr | null { if (js && GLOBAL_FUNCTION_VALUES.has(canonical)) return builtinFunctionValue(canonical, loc); + if (js && canonical === "Buffer") + return { kind: "libCall", fn: "buffer.constructor", args: [], type: DYN, loc }; if (GLOBAL_CONSTRUCTOR_VALUES.has(canonical)) { return { kind: "libCall", @@ -4300,6 +4329,22 @@ function fenceBuiltinConstructorStaticRead( return null; } +/** The Buffer statics the Buffer function's value holds + * (scr_buffer_constructor_value). */ +const BUFFER_VALUE_STATICS: ReadonlySet = new Set([ + "from", + "copyBytesFrom", + "of", + "alloc", + "allocUnsafe", + "allocUnsafeSlow", + "isBuffer", + "compare", + "isEncoding", + "concat", + "byteLength", +]); + /** Object.prototype members a plain record inherits, read as values. */ const OBJECT_PROTOTYPE_VALUE_READS: ReadonlySet = new Set([ "constructor", diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 82f5a27c1..03d19f0de 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -119,7 +119,7 @@ import { } from "./containers/indexed-objects.js"; import { lowerTupleReadMethodCall } from "./containers/array-methods.js"; import { tryFuseIndexRead } from "./array-values.js"; -import { lowerBytesStaticCall } from "./containers/bytes.js"; +import { lowerBufferConstruct, lowerBytesStaticCall } from "./containers/bytes.js"; import { lowerRegexMethodCall, lowerStringIndexCall, @@ -335,6 +335,13 @@ export function lowerCall(lowerer: Lowerer, expr: ts.CallExpression): IrExpr { const codeGeneration = codeGenerationCalleeOf(lowerer, expr.expression); if (codeGeneration !== null) return lowerCodeGenerationCall(lowerer, expr, codeGeneration); } + // The deprecated Buffer(...) call (containers/bytes.ts). The checker + // types it `any`, so the bytes cross as a checked-dynamic value, which + // keeps the Buffer brand for inspection. + { + const buffer = lowerBufferConstruct(lowerer, expr); + if (buffer) return { kind: "dynFrom", value: buffer, type: DYN, loc }; + } // A package shim's runtime intrinsic (a bodiless `declare function` // named scriptcUnixDgram* in a shipped shim): a direct runtime call. if (ts.isIdentifier(expr.expression) && isShimIntrinsicName(expr.expression.text)) { diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index 29afbb4fe..64fd1968d 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -46,7 +46,6 @@ import { DATE_T, DYN, F64, - bytesOf, type IrClassDef, type IrExpr, type IrFunction, @@ -115,8 +114,7 @@ import { isCjsRequireGlobal } from "./lower-node-module.js"; import { lowerArrayConstructor } from "./containers/array-construction.js"; import { lowerMapSeedNew, lowerSetSeedNew } from "./containers/collection-methods.js"; import { lowerObjectConstructor } from "./containers/indexed-objects.js"; -import { knownBufEncoding, lowerNodeModulesBufferConstruct } from "./containers/bytes.js"; -import { isSafeToRepeat } from "./expressions/evaluation-safety.js"; +import { knownBufEncoding, lowerBufferConstruct } from "./containers/bytes.js"; import { lowerSearchParamsNew } from "./builtins/url.js"; import { lowerTextCodecNew } from "./builtins/text-codecs.js"; import { @@ -9776,9 +9774,9 @@ function isCollectionOrErrorClass(ref: BuiltinClassRef | null): boolean { export function lowerNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr { const loc = locOf(expr); - // A dependency's deprecated `new Buffer(...)` fallback (containers/bytes.ts). + // The deprecated `new Buffer(...)` (containers/bytes.ts). { - const buffer = lowerNodeModulesBufferConstruct(lowerer, expr); + const buffer = lowerBufferConstruct(lowerer, expr); if (buffer) return buffer; } // new Function(...): Node's EvalError under @@ -10855,43 +10853,6 @@ export function lowerNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr { // families carry pointed hints: each states WHY no honest static // lowering exists (or what to use instead). if (lowerer.isStdlibSymbol(symbol ?? undefined)) { - // The deprecated `new Buffer(string, encoding?)` ctor's string arm - // with a NON-STRING first argument and a string second: Node - // throws ERR_INVALID_ARG_TYPE synchronously (and DEP0005 never - // fires on this throwing path, so the compiled silence matches). - // Every other new Buffer form keeps the fence below — the - // constructing forms would owe the deprecation warning. - if ( - expr.expression.text === "Buffer" && - expr.arguments?.nodes.length === 2 && - !expr.arguments.nodes.some(ts.isSpreadElement) && - lowerer.mapTypeOf(lowerer.typeOf(expr.arguments.nodes[0]!))?.kind !== "string" && - lowerer.mapTypeOf(lowerer.typeOf(expr.arguments.nodes[1]!))?.kind === "string" - ) { - const first = lowerer.lowerExpr(expr.arguments.nodes[0]!); - if ( - first.kind === "unitLit" || - first.type.kind === "dyn" || - lowerer.dynConvertible(first.type) - ) { - const got: IrExpr = - first.type.kind === "dyn" ? first : { kind: "dynFrom", value: first, type: DYN, loc }; - // The encoding argument still evaluates in Node before the - // throw only via the ctor body's later reads — it does NOT - // observe it before throwing, so dropping it is exact for - // effect-free operands; effectful ones keep the fence. - const enc = lowerer.lowerExpr(expr.arguments.nodes[1]!); - if (enc.kind === "strLit" || isSafeToRepeat(enc)) { - return { - kind: "libCall", - fn: "buffer.newStringFail", - args: [got], - type: bytesOf("u8"), - loc, - }; - } - } - } const ctorHints: Record = { RegExp: "use a regex literal (/pattern/flags) — constructed regexes have no lowering", String: diff --git a/packages/compiler/src/frontend/lowering/lower-inspect.ts b/packages/compiler/src/frontend/lowering/lower-inspect.ts index 5266d5c4e..7f0f3684a 100644 --- a/packages/compiler/src/frontend/lowering/lower-inspect.ts +++ b/packages/compiler/src/frontend/lowering/lower-inspect.ts @@ -14,7 +14,7 @@ import { InternalCompilerError } from "../../errors.js"; import * as ts from "../ts7/adapter.js"; import { unsignedHex } from "../../format-integer.js"; -import type { Lowerer } from "./lowerer.js"; +import { jsFuncNameOf, type Lowerer } from "./lowerer.js"; import { isJsSourceFile } from "../program.js"; import { BOOL, @@ -2416,6 +2416,12 @@ function directCallableInspect(lowerer: Lowerer, node: ts.Expression, loc: SrcLo } const decl = sym.declarations[0]; if (!decl) return null; + // A JavaScript function can hold own properties (`f.x = 1`, computed + // keys), which the runtime walk prints after its name. + if (isJsSourceFile(node.getSourceFile())) { + const type = lowerer.mapTypeOf(lowerer.typeOf(node)); + if (type?.kind === "func" && lowerer.dynConvertible(type)) return null; + } if (ts.isFunctionDeclaration(decl) && decl.name) { return strLit(`[Function: ${decl.name.text}]`, loc); } @@ -2476,7 +2482,17 @@ export function lowerInspectValue( loc, ); } - const value = lowerer.lowerExpr(valueNode); + let value = lowerer.lowerExpr(valueNode); + // A JavaScript function renders through the runtime walk, which prints + // its own properties after its name. + if ( + value.type.kind === "func" && + isJsSourceFile(valueNode.getSourceFile()) && + lowerer.dynConvertible(value.type) + ) { + const name = jsFuncNameOf(valueNode); + value = { kind: "dynFrom", value, type: DYN, ...(name !== null ? { fnName: name } : {}), loc }; + } if (value.type.kind === "bytes") { // One runtime representation serves Buffer AND Uint8Array; their // renderings differ ( vs Uint8Array(1) [ 170 ]), so the diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index e36ffd09f..c4b13c9ab 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -151,6 +151,7 @@ import { type ProtocolDispatch } from "./class-protocol-dispatch.js"; import { ClassDynamicDispatch, trackClassFieldCreation } from "./class-dynamic-dispatch.js"; import { guardClassAttributeWrites } from "./class-attributes.js"; import { applyHandleProperties } from "./handle-properties.js"; +import { installBufferValue } from "./containers/bytes.js"; import { ClassProtocols } from "./class-protocols.js"; import { earlyReadVarSymbols } from "./var-hoisting.js"; import { recordTextCodecClass } from "../../ir/ir.js"; @@ -7698,6 +7699,7 @@ export class Lowerer { trackClassFieldCreation(this, functions); guardClassAttributeWrites(this, functions); applyHandleProperties(this, functions); + installBufferValue(functions); // Globals typed by a class that never REGISTERED (a JS class whose // collection fenced — Symbol-keyed fields, an unsupported base): the @@ -13870,6 +13872,9 @@ export class Lowerer { loc, }; } + // The buffer module's Buffer is the global constructor's value. + if (bi.module === "buffer" && bi.member === "Buffer") + return { kind: "libCall", fn: "buffer.constructor", args: [], type: DYN, loc }; if (bi.module === "fs" && (bi.member === "realpathSync" || bi.member === "realpathSync.native")) return fsRealpathValue(this, bi.member === "realpathSync.native", loc); const fn = builtinModuleFnOf(this, bi.module, bi.member); diff --git a/packages/compiler/src/frontend/lowering/surfaces.ts b/packages/compiler/src/frontend/lowering/surfaces.ts index 8b826304b..84d41e869 100644 --- a/packages/compiler/src/frontend/lowering/surfaces.ts +++ b/packages/compiler/src/frontend/lowering/surfaces.ts @@ -3445,6 +3445,9 @@ export function stdlibGlobalNameOf(lowerer: Lowerer, expr: ts.Expression): strin return ts.symbolName(resolved) === "global" ? "globalThis" : ts.symbolName(resolved); } if (ts.isPropertyAccessExpression(expr) && !expr.questionDotToken) { + // The buffer module's Buffer (`require("buffer").Buffer`, `buffer_1.Buffer`) + // is the global constructor. + if (bufferNamespaceMember(lowerer, expr)) return "Buffer"; if (stdlibGlobalNameOf(lowerer, expr.expression) !== "globalThis") return null; // Through a detection chain the checker types the receiver as the // union of every arm; the selected arm (the global object) owns the diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index 68b649b07..c9c41fceb 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -1378,6 +1378,11 @@ function suppressedJsStrictness7(d: ts.Diagnostic): boolean { if (JS_ANY_OPERATOR_CODES.has(d.code)) { return ts.flattenDiagnosticMessageText(d, "\n").includes("'any'"); } + // The deprecated Buffer(...) call without `new`, which Node's Buffer + // function accepts (containers/bytes.ts lowers it). + if (d.code === 2348) { + return ts.flattenDiagnosticMessageText(d, "\n").includes("'BufferConstructor'"); + } return false; } diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index 0239c1624..aead4b085 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -829,12 +829,15 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "buffer.poolFromDyn", "buffer.poolFromDynRef", "buffer.poolFromDynOffset", + "buffer.poolFromDynRange", // The checked-dynamic compare/equals validators: Node's argument // ladders throw ERR_INVALID_ARG_TYPE / ERR_OUT_OF_RANGE catchably. "buffer.compareChk", "bytes.equalsChk", "bytes.compareChk", "buffer.newStringFail", + // Inherited and symbol-keyed Buffer.prototype members throw SC2020. + "buffer.prototypeMember", "fs.toUnixTimestamp", "fs.existsChk", "fs.mkdtempChk", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 6a645785f..87ddda65d 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -2988,6 +2988,9 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ /** poolFromDyn that also copies an array crossing by reference. */ "buffer.poolFromDynRef": { argTypes: [DYN, STRING], result: BYTES_U8 }, "buffer.poolFromDynOffset": { argTypes: [DYN, DYN], result: BYTES_U8 }, + /** Buffer.from(value, byteOffset, length): an ArrayBuffer's range with + * Node's fromArrayBuffer checks; other values as poolFromDynOffset. */ + "buffer.poolFromDynRange": { argTypes: [DYN, DYN, DYN], result: BYTES_U8 }, "buffer.poolSize": { argTypes: [], result: F64 }, "buffer.setPoolSize": { argTypes: [F64], result: VOID }, /** Buffer.byteLength(string, enc) — enc a NORMALIZED literal like @@ -3022,6 +3025,12 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ * always throws Node's ERR_INVALID_ARG_TYPE ("The \"string\" argument * must be of type string. Received ..."). Borrowed dyn; may-throw. */ "buffer.newStringFail": { argTypes: [DYN], result: BYTES_U8 }, + "buffer.deprecation": { argTypes: [], result: VOID }, + "buffer.deprecationCount": { argTypes: [], result: VOID }, + "buffer.constructor": { argTypes: [], result: DYN }, + "buffer.installValue": { argTypes: [], result: VOID }, + "buffer.staticValue": { argTypes: [STRING], result: DYN }, + "buffer.prototypeMember": { argTypes: [DYN], result: DYN }, /** fs._toUnixTimestamp(time) over a dyn value: numeric strings and * finite numbers coerce (negatives answer now/1000, Node's shape); * everything else throws Node's ERR_INVALID_ARG_TYPE. May-throw. */ diff --git a/packages/compiler/src/ir/int32-slots.ts b/packages/compiler/src/ir/int32-slots.ts index df9e3c0a5..be76cee7c 100644 --- a/packages/compiler/src/ir/int32-slots.ts +++ b/packages/compiler/src/ir/int32-slots.ts @@ -432,6 +432,8 @@ const DYN_READ_ONLY = new Set([ "error.new", "error.newOptions", "fs.utimesSync", + // Buffer.prototype members read as values: a lookup by key. + "buffer.prototypeMember", "insp.dyn", "insp.dynS", // The dynamic-tier operators run their dyn twins (scr_js_value.c); the diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index 819cfbee0..89ed37e21 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -98,6 +98,8 @@ const DYN_ASYNC_LIB_FNS: ReadonlySet = new Set([ "process.setDeprecationFlag", "process.setEmitWarning", "punycode.warn", + // DEP0005 for the deprecated Buffer constructor. + "buffer.deprecation", "timers.immediatePromise", // The performance timeline (scr_perf.c, linked with this unit for its // MaxPerformanceEntryBufferExceededWarning). @@ -358,6 +360,10 @@ const SYMBOL_LIB_FNS: ReadonlySet = new Set([ "dyn.usingDispose", "dyn.usingMethod", "util.isDeepStrictEqual", + // The Buffer function value's Symbol(kIsEncodingSymbol) member + "buffer.constructor", + "buffer.staticValue", + "buffer.prototypeMember", ]); /** Entry points that link the querystring unit. */ @@ -839,6 +845,7 @@ const RUNTIME_FEATURE_RULES = { ...UTIL_CALLABLE_LIB_FNS, ...BUILTIN_CLASS_LIB_FNS, "util.promisifyDyn", + "buffer.prototypeMember", ], // Constant-key reads of Object.prototype member names answer the // inherited member through scr_dyn_invoke.c. @@ -867,10 +874,22 @@ const RUNTIME_FEATURE_RULES = { /** Buffer.prototype methods called on checked-dynamic values * (scr_bytes_dyn.c). */ dynBuffer: rule({ + // Buffer.prototype members read as values dispatch to the same entries. + fns: ["buffer.prototypeMember"], expr: (node, _tables) => node.kind === "dynInvoke" && DYN_BUFFER_METHODS.has(node.method), }), /** Builtin functions read as values (scr_builtin_values.c). */ - builtinValues: rule({ fns: ["buffer.byteLenValue", "buffer.allocValue", "fs.realpathValue"] }), + builtinValues: rule({ + fns: [ + "buffer.byteLenValue", + "buffer.allocValue", + "fs.realpathValue", + "buffer.constructor", + "buffer.installValue", + "buffer.staticValue", + "buffer.prototypeMember", + ], + }), /** The stream unit emits through the emitter unit, also when the * program constructs no stream (a pipeline of functions). */ /** scr_ecdh.c over the mbedTLS archive; DEP0031 reports through the diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7527-buffer-from-array-buffer.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7527-buffer-from-array-buffer.ts.json new file mode 100644 index 000000000..577923b2e --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7527-buffer-from-array-buffer.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7527-buffer-from-array-buffer.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7528-buffer-from-array-buffer.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7528-buffer-from-array-buffer.cjs.json new file mode 100644 index 000000000..6ef4e3917 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7528-buffer-from-array-buffer.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7528-buffer-from-array-buffer.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7529-buffer-constructor.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7529-buffer-constructor.cjs.json new file mode 100644 index 000000000..5b199cd4c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7529-buffer-constructor.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7529-buffer-constructor.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7530-buffer-constructor.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7530-buffer-constructor.ts.json new file mode 100644 index 000000000..1927980cc --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7530-buffer-constructor.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7530-buffer-constructor.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7531-buffer-constructor-dependency/main.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7531-buffer-constructor-dependency/main.cjs.json new file mode 100644 index 000000000..b760b018a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7531-buffer-constructor-dependency/main.cjs.json @@ -0,0 +1,18 @@ +{ + "order": [ + "/tests/corpus/7531-buffer-constructor-dependency/main.cjs" + ], + "diags": [ + { + "code": "SC1010", + "message": "the 'legacy-buf' module is not supported yet", + "loc": { + "file": "/tests/corpus/7531-buffer-constructor-dependency/main.cjs", + "start": 362, + "end": 392 + }, + "milestone": "M4", + "hint": "relative imports (./file, ../dir/file), package.json-mediated project imports (#alias via the imports field, self-name references via exports), statically compiled npm packages, and the built-in fs, fs/promises, path, os, url, crypto, zlib, child_process, net, http, tls, https, http2, dgram, dns, dns/promises, util, util/types, string_decoder, querystring, readline, events, stream, stream/promises, stream/consumers, stream/web, buffer, assert, assert/strict, worker_threads, cluster, tty, async_hooks, timers, timers/promises, diagnostics_channel, perf_hooks, module, v8, and console modules (bare or node:-prefixed) and node:test (node:-prefixed only, like in Node) are supported" + } + ] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7532-buffer-constructor-check-limit/main.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7532-buffer-constructor-check-limit/main.cjs.json new file mode 100644 index 000000000..e2f78ad2e --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7532-buffer-constructor-check-limit/main.cjs.json @@ -0,0 +1,18 @@ +{ + "order": [ + "/tests/corpus/7532-buffer-constructor-check-limit/main.cjs" + ], + "diags": [ + { + "code": "SC1010", + "message": "the 'legacy-buf' module is not supported yet", + "loc": { + "file": "/tests/corpus/7532-buffer-constructor-check-limit/main.cjs", + "start": 279, + "end": 309 + }, + "milestone": "M4", + "hint": "relative imports (./file, ../dir/file), package.json-mediated project imports (#alias via the imports field, self-name references via exports), statically compiled npm packages, and the built-in fs, fs/promises, path, os, url, crypto, zlib, child_process, net, http, tls, https, http2, dgram, dns, dns/promises, util, util/types, string_decoder, querystring, readline, events, stream, stream/promises, stream/consumers, stream/web, buffer, assert, assert/strict, worker_threads, cluster, tty, async_hooks, timers, timers/promises, diagnostics_channel, perf_hooks, module, v8, and console modules (bare or node:-prefixed) and node:test (node:-prefixed only, like in Node) are supported" + } + ] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7533-buffer-value.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7533-buffer-value.cjs.json new file mode 100644 index 000000000..de7123599 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7533-buffer-value.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7533-buffer-value.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7534-buffer-prototype-members.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7534-buffer-prototype-members.cjs.json new file mode 100644 index 000000000..03858cfce --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7534-buffer-prototype-members.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7534-buffer-prototype-members.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7535-inspect-function-properties.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7535-inspect-function-properties.cjs.json new file mode 100644 index 000000000..9831d7def --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7535-inspect-function-properties.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7535-inspect-function-properties.cjs" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_async.h b/packages/runtime/src/scr_async.h index fc0128294..8ea4877c6 100644 --- a/packages/runtime/src/scr_async.h +++ b/packages/runtime/src/scr_async.h @@ -252,6 +252,10 @@ void scr_process_set_emit_warning(ScrDyn *fn); * made from node_modules) — at once for a static ES import, which Node * links before any module evaluates, otherwise on the next tick. */ void scr_punycode_warn(bool immediate, bool report); +/* The deprecated Buffer() and new Buffer() called outside node_modules + * (scr_async_dyn.c): DEP0005 on the next tick, once, within Node's + * 10000-call check (scr_buffer_deprecation_count). */ +void scr_buffer_deprecation(void); /* scr_emit_warning once a 'warning' listener exists (scr_lib.c; set by the * listener registration): units that cannot link the dyn-async unit route * their warnings through it, and print the default report otherwise. */ diff --git a/packages/runtime/src/scr_async_dyn.c b/packages/runtime/src/scr_async_dyn.c index a6cd813de..909d9eb8a 100644 --- a/packages/runtime/src/scr_async_dyn.c +++ b/packages/runtime/src/scr_async_dyn.c @@ -948,6 +948,22 @@ void scr_punycode_warn(bool immediate, bool report) { scr_str_release(message); } +void scr_buffer_deprecation(void) { + /* Node's showFlaggedDeprecation: after 10000 calls the check stops. */ + if (scr_buffer_ctor_warned || ++scr_buffer_ctor_checks > 10000) return; + static const char text[] = + "Buffer() is deprecated due to security and usability issues. Please use the " + "Buffer.alloc(), Buffer.allocUnsafe(), or Buffer.from() methods instead."; + ScrStr *message = scr_str_new(text, sizeof text - 1); + ScrStr *code = scr_str_new("DEP0005", 7); + ScrDyn *w = scr_warning_new(message, "DeprecationWarning", code); + scr_warning_emit_deferred(w); + scr_dyn_release(w); + scr_str_release(code); + scr_str_release(message); + scr_buffer_ctor_warned = true; +} + /* The console members' warnings. Node's emitWarning emits on * process.nextTick, so these queue in order and each rides one raw tick * on the shared tick queue; any left at exit are dropped with it. */ diff --git a/packages/runtime/src/scr_builtin_values.c b/packages/runtime/src/scr_builtin_values.c index ea240359a..f8dd2b79b 100644 --- a/packages/runtime/src/scr_builtin_values.c +++ b/packages/runtime/src/scr_builtin_values.c @@ -259,3 +259,217 @@ ScrDyn *scr_fs_realpath_value(const ScrDyn *path, const ScrDyn *options, bool na scr_str_release(resolved); return out; } + +/* ── Buffer as a value ─────────────────────────────────────────────────── + * The Buffer function read as a value in JavaScript (bn.js assigns + * `require('buffer').Buffer` to a variable and passes it to + * `allocate(ArrayType, size)`): one function per thread, named Buffer + * with length 3, whose own enumerable properties are Node's, in Node's + * order: poolSize, the statics and Symbol(kIsEncodingSymbol). The statics + * apply Node's argument handling. Calling or constructing Buffer through + * the value, reading its prototype, and the copyBytesFrom and of statics + * throw SC2020. poolSize is a data property the pool reads before each + * decision (scr_buffer_pool_refresh_hook). */ +static SCR_TL ScrDyn *scr_bv_buffer; + +#define SCR_BV_ARG(i) ((i) < argc ? args[i] : scr_dyn_undefined()) + +static ScrDyn *scr_bv_unsupported(const char *what) { + char message[160]; + int n = snprintf(message, sizeof message, "%s through a stored Buffer value is not supported yet", what); + scr_throw_error_msg_code(SCR_ERR_ERROR, message, (size_t)n, "SC2020"); + return NULL; +} + +/* A Buffer result as a value: +1, or NULL with the exception pending. */ +static ScrDyn *scr_bv_bytes(ScrBytes *bytes) { + if (!bytes) return NULL; + ScrBytes *buffer = scr_bytes_as_buffer(bytes); + scr_bytes_release(bytes); + ScrDyn *value = scr_dyn_new_bytes(buffer); + scr_bytes_release(buffer); + return value; +} + +static ScrDyn *scr_bv_call(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + (void)closure; (void)args; (void)argc; + return scr_bv_unsupported("Calling or constructing Buffer"); +} + +static ScrDyn *scr_bv_from(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + (void)closure; + const ScrDyn *value = SCR_BV_ARG(0), *second = SCR_BV_ARG(1), *third = SCR_BV_ARG(2); + if (value->kind != SCR_DYN_STR) return scr_bv_bytes(scr_buffer_from_dyn_range_pooled(value, second, third)); + /* fromString: an encoding that is not a non-empty string means utf8. */ + if (second->kind == SCR_DYN_STR && second->v.str->len > 0) + return scr_bv_bytes(scr_buffer_from_str_checked(value->v.str, second->v.str)); + ScrStr *utf8 = scr_str_new("utf8", 4); + ScrBytes *bytes = scr_buffer_from_str_checked(value->v.str, utf8); + scr_str_release(utf8); + return scr_bv_bytes(bytes); +} + +static ScrDyn *scr_bv_copy_bytes_from(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + (void)closure; (void)args; (void)argc; + return scr_bv_unsupported("Buffer.copyBytesFrom"); +} + +static ScrDyn *scr_bv_of(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + (void)closure; (void)args; (void)argc; + return scr_bv_unsupported("Buffer.of"); +} + +static ScrDyn *scr_bv_alloc(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + (void)closure; + return scr_bv_bytes(scr_buffer_alloc_value(SCR_BV_ARG(0), SCR_BV_ARG(1), SCR_BV_ARG(2))); +} + +static ScrDyn *scr_bv_alloc_unsafe(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + (void)closure; + const ScrDyn *size = SCR_BV_ARG(0); + if (size->kind != SCR_DYN_NUM) { + scr_dyn_arg_type_fail("size", "of type number", size); + return NULL; + } + return scr_bv_bytes(scr_buffer_alloc_unsafe(size->v.num)); +} + +static ScrDyn *scr_bv_alloc_unsafe_slow(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + (void)closure; + const ScrDyn *size = SCR_BV_ARG(0); + if (size->kind != SCR_DYN_NUM) { + scr_dyn_arg_type_fail("size", "of type number", size); + return NULL; + } + return scr_bv_bytes(scr_buffer_alloc_unsafe_slow(size->v.num)); +} + +static ScrDyn *scr_bv_is_buffer(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + (void)closure; + const ScrDyn *value = SCR_BV_ARG(0); + return scr_dyn_new_bool(value->kind == SCR_DYN_BYTES && value->buffer); +} + +static ScrDyn *scr_bv_compare(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + (void)closure; + double order = scr_buffer_compare_chk(SCR_BV_ARG(0), SCR_BV_ARG(1)); + return scr_exc_pending() ? NULL : scr_dyn_new_num(order); +} + +static ScrDyn *scr_bv_is_encoding(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + (void)closure; + const ScrDyn *encoding = SCR_BV_ARG(0); + return scr_dyn_new_bool(encoding->kind == SCR_DYN_STR && encoding->v.str->len > 0 && + scr_bytes_is_encoding(encoding->v.str)); +} + +/* Buffer.concat(list, length): Node validates the list, answers an empty + * Buffer for an empty list before it reads the length, and otherwise + * validates the length as an offset. */ +static ScrDyn *scr_bv_concat(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + (void)closure; + const ScrDyn *list = SCR_BV_ARG(0), *length = SCR_BV_ARG(1); + bool empty = list->kind == SCR_DYN_ARR && list->v.arr.len == 0; + if (length->kind == SCR_DYN_UNDEF || empty || list->kind != SCR_DYN_ARR) + return scr_bv_bytes(scr_buffer_concat_dyn(list)); + if (length->kind != SCR_DYN_NUM) { + scr_dyn_arg_type_fail("length", "of type number", length); + return NULL; + } + return scr_bv_bytes(scr_buffer_concat_len_dyn(list, length->v.num)); +} + +static ScrDyn *scr_bv_byte_length(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + (void)closure; + double length = scr_buffer_byte_length_value((ScrDyn *)SCR_BV_ARG(0), (ScrDyn *)SCR_BV_ARG(1)); + return scr_exc_pending() ? NULL : scr_dyn_new_num(length); +} + +static ScrDyn *scr_bv_prototype(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + (void)closure; (void)args; (void)argc; + return scr_bv_unsupported("Buffer.prototype"); +} + +/* The pool's setting is the value's poolSize: a number as it is, another + * primitive through ToNumber, anything else NaN (no pooling). */ +static void scr_bv_pool_refresh(void) { + ScrDyn *table = scr_dyn_function_properties(scr_bv_buffer); + const ScrDyn *size = scr_dyn_obj_get(table, "poolSize", 8); + double setting = NAN; + if (size && size->kind == SCR_DYN_NUM) { + setting = size->v.num; + } else if (size && (size->kind == SCR_DYN_STR || size->kind == SCR_DYN_BOOL || size->kind == SCR_DYN_NULL || + size->kind == SCR_DYN_UNDEF)) { + if (!scr_dyn_number_coerce_js(size, &setting)) setting = NAN; + } + scr_dyn_release(table); + scr_buffer_pool_apply(setting); +} + +/* A static `Buffer.poolSize = n` write updates the value's property. */ +static void scr_bv_pool_publish(double size) { + ScrDyn *table = scr_dyn_function_properties(scr_bv_buffer); + scr_dyn_obj_set(table, "poolSize", 8, scr_dyn_new_num(size)); + scr_dyn_release(table); +} + +static void scr_bv_cleanup(void) { + scr_buffer_pool_refresh_hook = NULL; + scr_buffer_pool_publish_hook = NULL; + scr_dyn_release(scr_bv_buffer); + scr_bv_buffer = NULL; +} + +ScrDyn *scr_buffer_constructor_value(void) { + if (scr_bv_buffer) return scr_dyn_retain(scr_bv_buffer); + static const struct { const char *name; uint32_t length; ScrDynThunk thunk; } statics[] = { + {"from", 3, scr_bv_from}, + {"copyBytesFrom", 3, scr_bv_copy_bytes_from}, + {"of", 0, scr_bv_of}, + {"alloc", 3, scr_bv_alloc}, + {"allocUnsafe", 1, scr_bv_alloc_unsafe}, + {"allocUnsafeSlow", 1, scr_bv_alloc_unsafe_slow}, + {"isBuffer", 1, scr_bv_is_buffer}, + {"compare", 2, scr_bv_compare}, + {"isEncoding", 1, scr_bv_is_encoding}, + {"concat", 2, scr_bv_concat}, + {"byteLength", 2, scr_bv_byte_length}, + }; + double pool_size = scr_buffer_pool_size(); + scr_bv_buffer = scr_dyn_new_func(scr_closure_new(NULL, 0), scr_bv_call, 3, "native:Buffer", "Buffer"); + scr_atexit(scr_bv_cleanup); + ScrDyn *table = scr_dyn_function_properties(scr_bv_buffer); + ScrDyn *getter = scr_dyn_new_func(scr_closure_new(NULL, 0), scr_bv_prototype, 0, "native:function", "get prototype"); + scr_dyn_obj_set_accessor(table, "prototype", 9, getter, NULL); + ScrDynEntry *entry = &table->v.obj.entries[table->v.obj.len - 1]; + entry->enumerable = entry->configurable = false; + scr_dyn_obj_set(table, "poolSize", 8, scr_dyn_new_num(pool_size)); + ScrDyn *is_encoding = NULL; + for (size_t i = 0; i < sizeof statics / sizeof statics[0]; i++) { + ScrDyn *fn = scr_dyn_new_func(scr_closure_new(NULL, 0), statics[i].thunk, statics[i].length, "native:function", + statics[i].name); + if (statics[i].thunk == scr_bv_is_encoding) is_encoding = scr_dyn_retain(fn); + scr_dyn_obj_set(table, statics[i].name, strlen(statics[i].name), fn); + } + scr_dyn_release(table); + ScrStr *description = scr_str_new("kIsEncodingSymbol", 17); + ScrSym *symbol = scr_sym_new(description); + scr_str_release(description); + ScrDyn *key = scr_dyn_new_symbol(symbol); + scr_sym_release(symbol); + scr_dyn_symbol_key_set(scr_bv_buffer, key, is_encoding); + scr_dyn_release(key); + scr_dyn_release(is_encoding); + scr_buffer_pool_refresh_hook = scr_bv_pool_refresh; + scr_buffer_pool_publish_hook = scr_bv_pool_publish; + return scr_dyn_retain(scr_bv_buffer); +} + +void scr_buffer_install_value(void) { scr_bytes_buffer_constructor_fn = scr_buffer_constructor_value; } + +ScrDyn *scr_buffer_static_value(const ScrStr *name) { + ScrDyn *buffer = scr_buffer_constructor_value(); + ScrDyn *member = scr_dyn_fn_get(buffer, name->data, name->len); + scr_dyn_release(buffer); + return member; +} diff --git a/packages/runtime/src/scr_bytes.c b/packages/runtime/src/scr_bytes.c index 7bcfd0cfb..907c177dc 100644 --- a/packages/runtime/src/scr_bytes.c +++ b/packages/runtime/src/scr_bytes.c @@ -2787,16 +2787,30 @@ static SCR_TL bool scr_pool_registered; /* scr_buffer_pool_reset runs at context /* Buffer.poolSize >>> 1, kept with the setting. */ static SCR_TL size_t scr_pool_half = 32768; -double scr_buffer_pool_size(void) { return scr_pool_setting; } +SCR_TL void (*scr_buffer_pool_refresh_hook)(void) = NULL; +SCR_TL void (*scr_buffer_pool_publish_hook)(double size) = NULL; -void scr_buffer_set_pool_size(double size) { +double scr_buffer_pool_size(void) { + if (scr_buffer_pool_refresh_hook) scr_buffer_pool_refresh_hook(); + return scr_pool_setting; +} + +void scr_buffer_pool_apply(double size) { scr_pool_setting = size; double t = isfinite(size) ? fmod(trunc(size), 4294967296.0) : 0; if (t < 0) t += 4294967296.0; scr_pool_half = (size_t)((uint32_t)t >> 1); } -static size_t scr_pool_threshold(void) { return scr_pool_half; } +void scr_buffer_set_pool_size(double size) { + scr_buffer_pool_apply(size); + if (scr_buffer_pool_publish_hook) scr_buffer_pool_publish_hook(size); +} + +static size_t scr_pool_threshold(void) { + if (scr_buffer_pool_refresh_hook) scr_buffer_pool_refresh_hook(); + return scr_pool_half; +} /* Bytes left in the slab. Aligning can move the offset past the end of a * slab whose size is not a multiple of 8; nothing is left then. */ diff --git a/packages/runtime/src/scr_bytes.h b/packages/runtime/src/scr_bytes.h index 9b382dda5..cca8364b1 100644 --- a/packages/runtime/src/scr_bytes.h +++ b/packages/runtime/src/scr_bytes.h @@ -388,6 +388,13 @@ ScrBytes *scr_bytes_from_str(const ScrStr *s, const ScrStr *enc); * slab. A slab lives while any of its views does. All return +1 Buffers. */ double scr_buffer_pool_size(void); void scr_buffer_set_pool_size(double size); +/* The setting without publishing it (scr_buffer_pool_refresh_hook). */ +void scr_buffer_pool_apply(double size); +/* Set while a Buffer function value exists (scr_builtin_values.c): its + * poolSize property is the setting, which the refresh hook reads before + * each pool decision and the publish hook receives from static writes. */ +extern SCR_TL void (*scr_buffer_pool_refresh_hook)(void); +extern SCR_TL void (*scr_buffer_pool_publish_hook)(double size); ScrBytes *scr_buffer_alloc(double size); /* zeroed, never pooled */ ScrBytes *scr_buffer_alloc_unsafe(double size); /* pooled */ ScrBytes *scr_buffer_alloc_unsafe_slow(double size); /* never pooled */ @@ -398,6 +405,15 @@ ScrBytes *scr_buffer_from_arr(const ScrArr *arr); /* number[] */ ScrBytes *scr_buffer_from_dyn_pooled(const ScrDyn *value, const ScrStr *encoding); ScrBytes *scr_buffer_from_dyn_ref_pooled(const ScrDyn *value, const ScrStr *encoding); ScrBytes *scr_buffer_from_dyn_offset_pooled(const ScrDyn *value, const ScrDyn *second); +/* Buffer.from(value, byteOffset, length): an ArrayBuffer's view with + * Node's fromArrayBuffer checks; other values as the two-argument form. */ +ScrBytes *scr_buffer_from_dyn_range_pooled(const ScrDyn *value, const ScrDyn *second, + const ScrDyn *third); +/* The deprecated Buffer constructor's DEP0005 state (scr_bytes_io.c): a + * call from node_modules counts toward Node's 10000-call check. */ +extern SCR_TL bool scr_buffer_ctor_warned; +extern SCR_TL uint64_t scr_buffer_ctor_checks; +void scr_buffer_deprecation_count(void); ScrBytes *scr_buffer_concat(const ScrArr *list); ScrBytes *scr_buffer_concat_len(const ScrArr *list, double total); /* The untyped-list forms: Node's ERR_INVALID_ARG_TYPE checks on the list @@ -406,6 +422,17 @@ ScrBytes *scr_buffer_concat_dyn(const struct ScrDyn *list); ScrBytes *scr_buffer_alloc_value(const struct ScrDyn *size, const struct ScrDyn *fill, const struct ScrDyn *encoding); struct ScrDyn *scr_fs_realpath_value(const struct ScrDyn *path, const struct ScrDyn *options, bool native); +/* The Buffer function as a value (scr_builtin_values.c), one per thread. +1. */ +struct ScrDyn *scr_buffer_constructor_value(void); +/* Installs that function as a Buffer's `.constructor` (the program's + * %main calls it when it reads the value): scr_bytes_buffer_constructor_fn. */ +void scr_buffer_install_value(void); +extern struct ScrDyn *(*scr_bytes_buffer_constructor_fn)(void); +/* A static of that value by name (`Buffer.alloc` read as a value). +1. */ +struct ScrDyn *scr_buffer_static_value(const ScrStr *name); +/* `Buffer.prototype[key]` (scr_bytes_dyn.c). +1, or NULL with the + * exception pending. */ +struct ScrDyn *scr_buffer_prototype_member(const struct ScrDyn *key); ScrBytes *scr_buffer_concat_len_dyn(const struct ScrDyn *list, double total); /* Drops the pool's slab reference and restores Buffer.poolSize (process * exit audits and library session resets). */ diff --git a/packages/runtime/src/scr_bytes_dyn.c b/packages/runtime/src/scr_bytes_dyn.c index c492d43b1..a65cdf239 100644 --- a/packages/runtime/src/scr_bytes_dyn.c +++ b/packages/runtime/src/scr_bytes_dyn.c @@ -592,3 +592,206 @@ ScrDyn *scr_dyn_invoke_buffer(ScrDyn *recv, ScrDyn *callee, const char *method, } return scr_dyn_invoke_prepared_id(recv, callee, method, mid_slot, args, argc, what); } + +/* A fixed-width read or write name (readUInt8, writeDoubleLE, ...). */ +static bool scr_bytes_dyn_num_method_name(const char *method) { + bool read = !strncmp(method, "read", 4); + if (!read && strncmp(method, "write", 5)) return false; + const char *rest = method + (read ? 4 : 5); + for (size_t at = 0; at < sizeof scr_bytes_dyn_names / sizeof scr_bytes_dyn_names[0]; at++) + if (!strcmp(rest, scr_bytes_dyn_names[at].name)) return true; + return false; +} + +/* ── Buffer.prototype members read as values ──────────────────────────── + * `Buffer.prototype[name]` (smart-buffer's typeof probe for the BigInt + * methods): an own member answers one function per name, with the name and + * length Node's function has; the parent and offset getters, read on the + * prototype itself, answer undefined; constructor is the Buffer value. + * Members inherited from Uint8Array.prototype and Object.prototype, symbol + * keys and object keys throw SC2020, and other keys answer undefined. The + * functions run the method on a Buffer receiver (`.call(buf, ...)`); other + * receivers throw SC2020. */ +static const struct { const char *key; const char *name; uint32_t length; } scr_buffer_proto_own[] = { + {"readBigUInt64LE", "readBigUInt64LE", 0}, + {"readBigUInt64BE", "readBigUInt64BE", 0}, + {"readBigUint64LE", "readBigUInt64LE", 0}, + {"readBigUint64BE", "readBigUInt64BE", 0}, + {"readBigInt64LE", "readBigInt64LE", 0}, + {"readBigInt64BE", "readBigInt64BE", 0}, + {"writeBigUInt64LE", "writeBigUInt64LE", 1}, + {"writeBigUInt64BE", "writeBigUInt64BE", 1}, + {"writeBigUint64LE", "writeBigUInt64LE", 1}, + {"writeBigUint64BE", "writeBigUInt64BE", 1}, + {"writeBigInt64LE", "writeBigInt64LE", 1}, + {"writeBigInt64BE", "writeBigInt64BE", 1}, + {"readUIntLE", "readUIntLE", 2}, + {"readUInt32LE", "", 1}, + {"readUInt16LE", "", 1}, + {"readUInt8", "", 1}, + {"readUIntBE", "readUIntBE", 2}, + {"readUInt32BE", "", 1}, + {"readUInt16BE", "", 1}, + {"readUintLE", "readUIntLE", 2}, + {"readUint32LE", "", 1}, + {"readUint16LE", "", 1}, + {"readUint8", "", 1}, + {"readUintBE", "readUIntBE", 2}, + {"readUint32BE", "", 1}, + {"readUint16BE", "", 1}, + {"readIntLE", "readIntLE", 2}, + {"readInt32LE", "", 1}, + {"readInt16LE", "", 1}, + {"readInt8", "", 1}, + {"readIntBE", "readIntBE", 2}, + {"readInt32BE", "", 1}, + {"readInt16BE", "", 1}, + {"writeUIntLE", "writeUIntLE", 3}, + {"writeUInt32LE", "writeUInt32LE", 1}, + {"writeUInt16LE", "writeUInt16LE", 1}, + {"writeUInt8", "writeUInt8", 1}, + {"writeUIntBE", "writeUIntBE", 3}, + {"writeUInt32BE", "writeUInt32BE", 1}, + {"writeUInt16BE", "writeUInt16BE", 1}, + {"writeUintLE", "writeUIntLE", 3}, + {"writeUint32LE", "writeUInt32LE", 1}, + {"writeUint16LE", "writeUInt16LE", 1}, + {"writeUint8", "writeUInt8", 1}, + {"writeUintBE", "writeUIntBE", 3}, + {"writeUint32BE", "writeUInt32BE", 1}, + {"writeUint16BE", "writeUInt16BE", 1}, + {"writeIntLE", "writeIntLE", 3}, + {"writeInt32LE", "writeInt32LE", 1}, + {"writeInt16LE", "writeInt16LE", 1}, + {"writeInt8", "writeInt8", 1}, + {"writeIntBE", "writeIntBE", 3}, + {"writeInt32BE", "writeInt32BE", 1}, + {"writeInt16BE", "writeInt16BE", 1}, + {"readFloatLE", "readFloatForwards", 0}, + {"readFloatBE", "readFloatBackwards", 0}, + {"readDoubleLE", "readDoubleForwards", 0}, + {"readDoubleBE", "readDoubleBackwards", 0}, + {"writeFloatLE", "writeFloatForwards", 1}, + {"writeFloatBE", "writeFloatBackwards", 1}, + {"writeDoubleLE", "writeDoubleForwards", 1}, + {"writeDoubleBE", "writeDoubleBackwards", 1}, + {"asciiSlice", "", 2}, + {"base64Slice", "", 2}, + {"base64urlSlice", "", 2}, + {"latin1Slice", "", 2}, + {"hexSlice", "", 2}, + {"ucs2Slice", "", 2}, + {"utf8Slice", "", 2}, + {"asciiWrite", "", 3}, + {"base64Write", "", 3}, + {"base64urlWrite", "", 3}, + {"latin1Write", "", 3}, + {"hexWrite", "", 3}, + {"ucs2Write", "", 3}, + {"utf8Write", "", 3}, + {"parent", NULL, 0}, + {"offset", NULL, 0}, + {"copy", "copy", 4}, + {"toString", "toString", 3}, + {"equals", "equals", 1}, + {"inspect", "inspect", 2}, + {"compare", "compare", 5}, + {"indexOf", "indexOf", 4}, + {"lastIndexOf", "lastIndexOf", 4}, + {"includes", "includes", 4}, + {"fill", "fill", 4}, + {"write", "write", 4}, + {"toJSON", "toJSON", 0}, + {"subarray", "subarray", 2}, + {"slice", "slice", 2}, + {"swap16", "swap16", 0}, + {"swap32", "swap32", 0}, + {"swap64", "swap64", 0}, + {"toLocaleString", "toString", 3}, +}; + +static const char *const scr_buffer_proto_inherited[] = { + "BYTES_PER_ELEMENT", "buffer", "byteLength", "byteOffset", "length", "entries", "keys", "values", + "at", "copyWithin", "every", "filter", "find", "findIndex", "findLast", "findLastIndex", + "forEach", "join", "map", "reverse", "reduce", "reduceRight", "set", "some", "sort", "toReversed", + "toSorted", "with", "__defineGetter__", "__defineSetter__", "hasOwnProperty", "__lookupGetter__", + "__lookupSetter__", "isPrototypeOf", "propertyIsEnumerable", "valueOf", "__proto__", +}; + +static SCR_TL ScrDyn *scr_buffer_proto_fns; + +static void scr_buffer_proto_cleanup(void) { + scr_dyn_release(scr_buffer_proto_fns); + scr_buffer_proto_fns = NULL; +} + +static ScrDyn *scr_buffer_proto_unsupported(const char *format, const char *name) { + char message[200]; + int n = snprintf(message, sizeof message, format, name); + scr_throw_error_msg_code(SCR_ERR_ERROR, message, (size_t)n, "SC2020"); + return NULL; +} + +static ScrDyn *scr_buffer_proto_run(ScrDyn *recv, const char *method, ScrDyn *const *args, size_t argc) { + if (!scr_bytes_dyn_buffer(recv)) + return scr_buffer_proto_unsupported("Buffer.prototype.%s on a receiver that is not a Buffer is not supported yet", + method); + const char *what = "Buffer.prototype method"; + /* The method-id cache slot of a call site, here a fresh one per call. */ + int32_t slot = -1; + int32_t *mid = &slot; + if (scr_bytes_dyn_num_method_name(method)) return scr_dyn_invoke_buffer_num(recv, NULL, method, mid, args, argc, what); + if (!strcmp(method, "write")) return scr_dyn_invoke_buffer_write(recv, NULL, method, mid, args, argc, what); + if (!strcmp(method, "fill")) return scr_dyn_invoke_buffer_fill(recv, NULL, method, mid, args, argc, what); + if (!strcmp(method, "toString") || !strcmp(method, "toLocaleString")) + return scr_dyn_invoke_buffer_to_string(recv, NULL, "toString", mid, args, argc, what); + if (!strcmp(method, "indexOf") || !strcmp(method, "lastIndexOf") || !strcmp(method, "includes")) + return scr_dyn_invoke_buffer_search(recv, NULL, method, mid, args, argc, what); + if (!strcmp(method, "subarray") || !strcmp(method, "slice")) + return scr_dyn_invoke(recv, method, args, argc, what); + if (!strcmp(method, "inspect")) + return scr_buffer_proto_unsupported("Buffer.prototype.%s through a stored value is not supported yet", method); + return scr_dyn_invoke_buffer(recv, NULL, method, mid, args, argc, what); +} + +static ScrDyn *scr_buffer_proto_call(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + ScrDyn *recv = scr_dyn_this_get(); + ScrDyn *result = scr_buffer_proto_run(recv, closure->fn, args, argc); + scr_dyn_release(recv); + return result; +} + +ScrDyn *scr_buffer_prototype_member(const ScrDyn *key) { + if (key->kind == SCR_DYN_SYMBOL) + return scr_buffer_proto_unsupported("Buffer.prototype members keyed by %s are not supported yet", "symbols"); + if (key->kind != SCR_DYN_STR) { + if (key->kind == SCR_DYN_NUM || key->kind == SCR_DYN_BOOL || key->kind == SCR_DYN_UNDEF || + key->kind == SCR_DYN_NULL || key->kind == SCR_DYN_BIGINT) + return scr_dyn_retain(scr_dyn_undefined()); + return scr_buffer_proto_unsupported("Buffer.prototype members keyed by %s are not supported yet", "objects"); + } + const ScrStr *name = key->v.str; + if (name->len == 11 && memcmp(name->data, "constructor", 11) == 0) return scr_buffer_constructor_value(); + for (size_t i = 0; i < sizeof scr_buffer_proto_inherited / sizeof scr_buffer_proto_inherited[0]; i++) { + const char *inherited = scr_buffer_proto_inherited[i]; + if (strlen(inherited) == name->len && memcmp(inherited, name->data, name->len) == 0) + return scr_buffer_proto_unsupported("Buffer.prototype.%s is not supported yet", inherited); + } + for (size_t i = 0; i < sizeof scr_buffer_proto_own / sizeof scr_buffer_proto_own[0]; i++) { + const char *own = scr_buffer_proto_own[i].key; + if (strlen(own) != name->len || memcmp(own, name->data, name->len) != 0) continue; + if (!scr_buffer_proto_own[i].name) return scr_dyn_retain(scr_dyn_undefined()); + if (!scr_buffer_proto_fns) { + scr_buffer_proto_fns = scr_dyn_new_obj_null_proto(); + scr_atexit(scr_buffer_proto_cleanup); + } + ScrDyn *fn = scr_dyn_obj_get(scr_buffer_proto_fns, own, name->len); + if (!fn) { + fn = scr_dyn_new_func(scr_closure_new((void *)own, 0), scr_buffer_proto_call, scr_buffer_proto_own[i].length, + "native:function", scr_buffer_proto_own[i].name); + scr_dyn_obj_set(scr_buffer_proto_fns, own, name->len, fn); + } + return scr_dyn_retain(fn); + } + return scr_dyn_retain(scr_dyn_undefined()); +} diff --git a/packages/runtime/src/scr_bytes_io.c b/packages/runtime/src/scr_bytes_io.c index 137208938..c93c67cdc 100644 --- a/packages/runtime/src/scr_bytes_io.c +++ b/packages/runtime/src/scr_bytes_io.c @@ -1299,13 +1299,16 @@ ScrBytes *scr_buffer_concat_len_dyn(const ScrDyn *list, double total) { return scr_buffer_concat_dyn_impl(list, total, true); } -/* Buffer.from(arrayBuffer, byteOffset) over a checked ArrayBuffer or - * SharedArrayBuffer, as Node's fromArrayBuffer: a Buffer from the offset to - * the end of the buffer that shares its memory. The offset converts with - * ToNumber, where a missing offset (NULL) or undefined and NaN are 0; an - * offset past the end throws ERR_BUFFER_OUT_OF_BOUNDS, and one at or below - * -1 V8's start-offset RangeError. NULL with the exception pending. */ -static ScrBytes *scr_buffer_from_storage(const ScrDyn *storage, const ScrDyn *offset_value) { +/* Buffer.from(arrayBuffer, byteOffset, length) over a checked ArrayBuffer + * or SharedArrayBuffer, as Node's fromArrayBuffer: a Buffer that shares its + * memory. The offset converts with ToNumber, where a missing offset (NULL) + * or undefined and NaN are 0; an offset past the end throws + * ERR_BUFFER_OUT_OF_BOUNDS, and one at or below -1 V8's start-offset + * RangeError. A missing length runs to the end; a given one converts with + * ToNumber, past the end throws ERR_BUFFER_OUT_OF_BOUNDS for "length", and + * zero, negative and NaN lengths are 0. NULL with the exception pending. */ +static ScrBytes *scr_buffer_from_storage(const ScrDyn *storage, const ScrDyn *offset_value, + const ScrDyn *length_value) { double offset = 0; if (offset_value && offset_value->kind != SCR_DYN_UNDEF) { if (!scr_dyn_number_coerce_js(offset_value, &offset)) return NULL; @@ -1313,11 +1316,26 @@ static ScrBytes *scr_buffer_from_storage(const ScrDyn *storage, const ScrDyn *of } ScrBytes *bytes = storage->v.handle.ptr; ScrBytes *owner = bytes->backing ? bytes->backing : bytes; - if ((double)scr_bytes_byte_len(owner) - offset < 0) { + double max_length = (double)scr_bytes_byte_len(owner) - offset; + if (max_length < 0) { static const char msg[] = "\"offset\" is outside of buffer bounds"; scr_throw_error_msg_code(SCR_ERR_RANGE, msg, sizeof msg - 1, "ERR_BUFFER_OUT_OF_BOUNDS"); return NULL; } + bool has_length = length_value && length_value->kind != SCR_DYN_UNDEF; + double length = max_length; + if (has_length) { + if (!scr_dyn_number_coerce_js(length_value, &length)) return NULL; + if (length > 0) { + if (length > max_length) { + static const char msg[] = "\"length\" is outside of buffer bounds"; + scr_throw_error_msg_code(SCR_ERR_RANGE, msg, sizeof msg - 1, "ERR_BUFFER_OUT_OF_BOUNDS"); + return NULL; + } + } else { + length = 0; + } + } double start = trunc(offset); if (start < 0) { /* The offset as JavaScript prints it (-1.5, -1e+21, -Infinity). */ @@ -1331,7 +1349,7 @@ static ScrBytes *scr_buffer_from_storage(const ScrDyn *storage, const ScrDyn *of scr_throw_error(SCR_ERR_RANGE, message); return NULL; } - ScrBytes *out = scr_bytes_buffer_view(bytes, SCR_BYTES_U8, start, false, 0); + ScrBytes *out = scr_bytes_buffer_view(bytes, SCR_BYTES_U8, start, has_length, trunc(length)); if (out) out->is_buffer = true; return out; } @@ -1340,7 +1358,7 @@ ScrBytes *scr_buffer_from_dyn(const ScrDyn *value, const ScrStr *encoding) { if (value->kind == SCR_DYN_STR) return scr_bytes_from_str(value->v.str, encoding); /* The second argument here is an encoding name, which as a byte offset * converts to NaN: offset 0. */ - if (scr_buffer_storage_is(value)) return scr_buffer_from_storage(value, NULL); + if (scr_buffer_storage_is(value)) return scr_buffer_from_storage(value, NULL, NULL); if (value->kind == SCR_DYN_BYTES) return scr_bytes_convert(SCR_BYTES_U8, value->v.bytes); if (value->kind == SCR_DYN_ARR) { return scr_buffer_from_array_like(value, (double)value->v.arr.len); @@ -1386,7 +1404,7 @@ ScrBytes *scr_buffer_from_dyn_pooled(const ScrDyn *value, const ScrStr *encoding return scr_buffer_from_bytes(value->v.bytes); /* A view of an ArrayBuffer shares its memory, so it never comes from the * pool. The second argument here is an encoding name: offset 0. */ - if (scr_buffer_storage_is(value)) return scr_buffer_from_storage(value, NULL); + if (scr_buffer_storage_is(value)) return scr_buffer_from_storage(value, NULL, NULL); ScrBytes *out = scr_buffer_from_dyn(value, encoding); if (!out) return NULL; out->is_buffer = true; @@ -1422,12 +1440,32 @@ ScrBytes *scr_buffer_from_dyn_offset_pooled(const ScrDyn *value, const ScrDyn *s scr_str_release(utf8); return out; } - return scr_buffer_from_storage(value, second); + return scr_buffer_from_storage(value, second, NULL); +} + +/* Buffer.from(value, second, third): an ArrayBuffer or SharedArrayBuffer + * takes them as its byte offset and length; other values convert as + * scr_buffer_from_dyn_offset_pooled does, without the third argument. */ +ScrBytes *scr_buffer_from_dyn_range_pooled(const ScrDyn *value, const ScrDyn *second, + const ScrDyn *third) { + if (!scr_buffer_storage_is(value)) return scr_buffer_from_dyn_offset_pooled(value, second); + return scr_buffer_from_storage(value, second, third); +} + +/* The deprecated Buffer constructor's DEP0005 state, Node's + * bufferWarningAlreadyEmitted and nodeModulesCheckCounter: a call from + * node_modules only counts toward the 10000-call check, and + * scr_buffer_deprecation (scr_async_dyn.c) reports for the others. */ +SCR_TL bool scr_buffer_ctor_warned = false; +SCR_TL uint64_t scr_buffer_ctor_checks = 0; + +void scr_buffer_deprecation_count(void) { + if (!scr_buffer_ctor_warned) scr_buffer_ctor_checks++; } /* new Buffer(number, encoding) — the deprecated ctor's string arm with a - * non-string first argument: Node's exact ERR_INVALID_ARG_TYPE (the - * throwing path never fires DEP0005, so compiled silence matches). */ + * non-string first argument: Node's exact ERR_INVALID_ARG_TYPE, thrown + * after scr_buffer_deprecation has reported DEP0005. */ ScrBytes *scr_buffer_new_string_fail(const ScrDyn *got) { scr_dyn_arg_type_fail("string", "of type string", got); return NULL; diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 553b897d2..e93a9a0f0 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -1328,6 +1328,10 @@ ScrDyn *scr_dyn_invoke(ScrDyn *recv, const char *method, ScrDyn *const *args, si * for anonymous) and "length" (the boxed arity). Returns +1, or NULL when * the key answers nothing (the caller's undefined). Never throws. */ ScrDyn *scr_dyn_fn_get(const ScrDyn *d, const char *key, size_t key_len); +/* A function value's own property table, created on first use. +1. */ +ScrDyn *scr_dyn_function_properties(const ScrDyn *function); +/* The table a function value already has. +1, or NULL. */ +ScrDyn *scr_dyn_function_properties_existing(const ScrDyn *function); /* Object.defineProperties over dyn values (targets: OBJ and FUNC): each * descriptor's `value` becomes an own data property. OBJ targets preserve * writable/enumerable/configurable; FUNC targets still use plain properties. diff --git a/packages/runtime/src/scr_inspect.c b/packages/runtime/src/scr_inspect.c index 1dce8d3c2..79b2564ff 100644 --- a/packages/runtime/src/scr_inspect.c +++ b/packages/runtime/src/scr_inspect.c @@ -1600,7 +1600,45 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { } else { ib_cstr(&out, "[Function (anonymous)]"); } - return ib_take(&out); + /* A function's own enumerable properties follow its base, as for an + * object (`[Function: f] { x: 1 }`), and past the depth limit it is + * `[Function]` (formatRaw). Classes keep the base. */ + ScrDyn *table = d->v.fn.class_obj ? NULL : scr_dyn_function_properties_existing(d); + bool shown = false; + for (size_t i = 0; table && i < table->v.obj.len && !shown; i++) shown = table->v.obj.entries[i].enumerable; + for (size_t i = 0; table && !shown && table->symbol_keys && i < table->symbol_keys->v.arr.len; i++) + shown = scr_insp_symbol_entry(table, i) != NULL; + if (!shown) { + scr_dyn_release(table); + return ib_take(&out); + } + /* Every value of one function shares its table: the table is the + * function's identity for the circular checks. */ + double circular = scr_insp_circ_check(table); + if (circular != 0) { + scr_dyn_release(table); + free(out.data); + return scr_insp_circular(circular); + } + if (recurse > depth) { + scr_dyn_release(table); + free(out.data); + return scr_str_new("[Function]", 10); + } + ScrStr *base = ib_take(&out); + char *cls = malloc(base->len + 1); + if (!cls) scr_trap("scriptc: out of memory\n"); + memcpy(cls, base->data, base->len); + cls[base->len] = '\0'; + scr_str_release(base); + size_t nseen = g_nseen; + g_pending_seen = table; + ScrStr *rendered = scr_insp_obj(table, recurse, depth, cls); + g_pending_seen = NULL; + if (g_nseen > nseen) g_nseen = nseen; + free(cls); + scr_dyn_release(table); + return rendered; } case SCR_DYN_HANDLE: { if (d->v.handle.tag == SCR_DYNH_ARRAY_BUFFER || diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 615785687..c0154ab57 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -2366,6 +2366,10 @@ static ScrDyn *scr_dyn_fn_existing_table(const ScrDyn *function) { return owner->props ? (ScrDyn *)scr_box_get_ref(owner->props) : NULL; } +ScrDyn *scr_dyn_function_properties_existing(const ScrDyn *function) { + return scr_dyn_fn_existing_table(function); +} + ScrDyn *scr_dyn_prevent_extensions(ScrDyn *object) { if (scr_dyn_class_reflection_fence(object) || scr_dyn_error_prototype_fence(object)) return NULL; if (object->kind == SCR_DYN_PROXY) { scr_dyn_proxy_unsupported("Object.preventExtensions"); return NULL; } @@ -4200,6 +4204,9 @@ bool scr_bytes_instanceof_at(const ScrDyn *value, const ScrDyn *callee, const Sc } if (scr_array_buffer_ctor && callee->v.fn.clo == scr_array_buffer_ctor->v.fn.clo) return scr_array_buffer_is(value); if (scr_array_ctor && callee->v.fn.clo == scr_array_ctor->v.fn.clo) return value->kind == SCR_DYN_ARR || scr_dyn_ref_is_array(value); + /* The Buffer function value: Buffer instances, not other Uint8Arrays. */ + if (callee->v.fn.sig && strcmp(callee->v.fn.sig, "native:Buffer") == 0) + return value->kind == SCR_DYN_BYTES && value->buffer; for (size_t i = 0; i < SCR_BYTES_ELEM_COUNT; i++) { if (scr_bytes_ctors[i] && callee->v.fn.clo == scr_bytes_ctors[i]->v.fn.clo) return scr_dyn_typed_array_is(value, (int)i); @@ -4262,6 +4269,7 @@ bool scr_bytes_instanceof(const ScrDyn *value, const ScrDyn *callee) { return scr_bytes_instanceof_at(value, callee, NULL); } SCR_TL bool (*scr_dyn_global_construct_fn)(const ScrDyn *callee, const ScrDyn *args, ScrDyn **result) = NULL; +ScrDyn *(*scr_bytes_buffer_constructor_fn)(void) = NULL; ScrDyn *scr_bytes_construct(const ScrDyn *callee, const ScrDyn *args, const ScrStr *what) { if (callee->kind == SCR_DYN_FUNC && scr_array_ctor && callee->v.fn.clo == scr_array_ctor->v.fn.clo) @@ -4282,6 +4290,14 @@ ScrDyn *scr_bytes_construct(const ScrDyn *callee, const ScrDyn *args, const ScrS ScrDyn *constructed = NULL; if (scr_dyn_global_construct_fn && scr_dyn_global_construct_fn(callee, args, &constructed)) return constructed; + /* The Buffer function value (scr_builtin_values.c): Node's construction + * reports DEP0005 by the caller's location, which a stored value does + * not know. */ + if (callee->kind == SCR_DYN_FUNC && callee->v.fn.sig && strcmp(callee->v.fn.sig, "native:Buffer") == 0) { + static const char message[] = "Calling or constructing Buffer through a stored Buffer value is not supported yet"; + scr_throw_error_msg_code(SCR_ERR_ERROR, message, sizeof message - 1, "SC2020"); + return NULL; + } ScrJsonBuf message; scr_jb_init(&message); scr_jb_write(&message, what->data, what->len); @@ -4311,7 +4327,9 @@ ScrDyn *scr_dyn_bytes_key_get(const ScrDyn *value, const ScrStr *key) { return scr_dyn_new_num((double)scr_bytes_elem_size(bytes->elem)); if (key->len == 11 && memcmp(key->data, "constructor", 11) == 0) { if (!value->buffer) return scr_bytes_constructor_for(bytes->elem); - /* Buffer retains its separate builtin identity. */ + /* A program that reads the Buffer function's value answers it; + * otherwise Buffer keeps its separate builtin identity token. */ + if (scr_bytes_buffer_constructor_fn) return scr_bytes_buffer_constructor_fn(); char token[64]; int length = snprintf(token, sizeof token, "[builtin %s]", value->buffer ? "Buffer" : scr_bytes_elem_name(bytes->elem)); ScrStr *name = scr_str_new(token, (size_t)length); @@ -5219,6 +5237,8 @@ static ScrDyn *scr_dyn_fn_properties(const ScrDyn *function) { return (ScrDyn *)scr_box_get_ref(owner->props); } +ScrDyn *scr_dyn_function_properties(const ScrDyn *function) { return scr_dyn_fn_properties(function); } + ScrDyn *scr_dyn_class_inherit(ScrDyn *constructor, ScrDyn *base) { if (constructor->kind != SCR_DYN_FUNC || !constructor->v.fn.class_obj || (base->kind != SCR_DYN_FUNC && base->kind != SCR_DYN_UNDEF)) { static const char message[] = "Class inheritance requires native constructors"; diff --git a/tests/corpus/2570-buffer-arg-validation.cjs b/tests/corpus/2570-buffer-arg-validation.cjs index d49593a30..c0980e495 100644 --- a/tests/corpus/2570-buffer-arg-validation.cjs +++ b/tests/corpus/2570-buffer-arg-validation.cjs @@ -1,4 +1,3 @@ -// @no-deprecation // Buffer's invalid-argument ladders, Node-exact: Buffer.compare / // buf.compare / buf.equals over non-buffer values and bad offsets throw // ERR_INVALID_ARG_TYPE / ERR_OUT_OF_RANGE with Node's argument names and @@ -7,6 +6,8 @@ // real answers. Plus the deprecated `new Buffer(number, enc)` string-arm // rejection and the ERR_* codes on the write/swap/fill range ladders. 'use strict'; +// The deprecated constructor's DEP0005 report carries a process id. +process.noDeprecation = true; const show = (fn) => { try { console.log('ret', fn()); diff --git a/tests/corpus/7527-buffer-from-array-buffer.ts b/tests/corpus/7527-buffer-from-array-buffer.ts new file mode 100644 index 000000000..fb650e9f2 --- /dev/null +++ b/tests/corpus/7527-buffer-from-array-buffer.ts @@ -0,0 +1,33 @@ +// Buffer.from(arrayBuffer, byteOffset, length): Node's fromArrayBuffer over +// typed and untyped ArrayBuffers and SharedArrayBuffers: the offset and +// length conversions, the shared memory, zero and negative lengths, and +// ERR_BUFFER_OUT_OF_BOUNDS for an offset or length past the end. +const ab = new ArrayBuffer(6); +new Uint8Array(ab).set([1, 2, 3, 4, 5, 6]); +const offsets: unknown[] = [undefined, NaN, "1", 1.5, -1, 0, 6, 7, null, true]; +const lengths: unknown[] = [undefined, 0, -1, 2.5, "2", 6, 9, NaN, null]; +for (const o of offsets) { + for (const l of lengths) { + try { + const b = Buffer.from(ab, o as number, l as number); + console.log(String(o), String(l), b, b.byteOffset, b.length); + } catch (e) { + const err = e as NodeJS.ErrnoException; + console.log(String(o), String(l), err.name, err.code, err.message); + } + } +} +const typed = Buffer.from(ab, 2, 2); +typed[0] = 99; +console.log(new Uint8Array(ab)[2], Buffer.from(ab, 4).toString("hex"), Buffer.isBuffer(Buffer.from(ab, 1, 1))); +const shared = new SharedArrayBuffer(4); +console.log(Buffer.from(shared, 1, 2).length); +const loose: any = new ArrayBuffer(4); +const pairs: [unknown, unknown][] = [[1, 2], [4, undefined], [5, 1], [0, 5], ["2", "1"]]; +for (const [o, l] of pairs) { + try { + console.log(Buffer.from(loose, o as number, l as number).toString("hex")); + } catch (e) { + console.log((e as NodeJS.ErrnoException).code, (e as Error).message); + } +} diff --git a/tests/corpus/7528-buffer-from-array-buffer.cjs b/tests/corpus/7528-buffer-from-array-buffer.cjs new file mode 100644 index 000000000..e77b5d55b --- /dev/null +++ b/tests/corpus/7528-buffer-from-array-buffer.cjs @@ -0,0 +1,31 @@ +// Buffer.from(arrayBuffer, byteOffset, length) in JavaScript, with offsets +// and lengths of every kind and an ArrayBuffer held in an untyped slot. +const ab = new ArrayBuffer(6); +new Uint8Array(ab).set([1, 2, 3, 4, 5, 6]); +const offsets = [undefined, NaN, "1", 1.5, -1, 0, 6, 7, null, true]; +const lengths = [undefined, 0, -1, 2.5, "2", 6, 9, NaN, null]; +for (const o of offsets) { + for (const l of lengths) { + try { + const b = Buffer.from(ab, o, l); + console.log(String(o), String(l), b, b.byteOffset, b.length); + } catch (e) { + const err = e; + console.log(String(o), String(l), err.name, err.code, err.message); + } + } +} +const typed = Buffer.from(ab, 2, 2); +typed[0] = 99; +console.log(new Uint8Array(ab)[2], Buffer.from(ab, 4).toString("hex"), Buffer.isBuffer(Buffer.from(ab, 1, 1))); +const shared = new SharedArrayBuffer(4); +console.log(Buffer.from(shared, 1, 2).length); +const slots = { buffer: new ArrayBuffer(3) }; +new Uint8Array(slots.buffer).set([9, 8, 7]); +for (const args of [[1], [1, 1], [0, 0], [3, 0], [2, 5], [-0.5, 2]]) { + try { + console.log(args.join(","), Buffer.from(slots.buffer, args[0], args[1])); + } catch (e) { + console.log(args.join(","), e.code, e.message); + } +} diff --git a/tests/corpus/7529-buffer-constructor.cjs b/tests/corpus/7529-buffer-constructor.cjs new file mode 100644 index 000000000..3181cebec --- /dev/null +++ b/tests/corpus/7529-buffer-constructor.cjs @@ -0,0 +1,80 @@ +// The deprecated Buffer constructor, called and constructed, as Node's +// Buffer function runs it: the arguments evaluate in order, the first call +// reports DEP0005 on the next tick (once), a number allocates zero-filled +// bytes, and other values convert as Buffer.from does. +// The program re-runs itself and prints the child's output with process +// ids normalized. +"use strict"; +const { spawnSync } = require("node:child_process"); +const { Buffer: Imported } = require("buffer"); + +function child() { + process.on("warning", (w) => console.log("listener", w.name, w.code, w.message.length)); + const B = Buffer; + + const show = (label, fn) => { + try { + const b = fn(); + console.log(label, b, b.length, Buffer.isBuffer(b)); + } catch (e) { + console.log(label, e.name, e.code, e.message); + } + }; + + // The throwing form still reports the warning, after the catch. + show("number+encoding", () => new Buffer(3, "utf8")); + show("size", () => new Buffer(4)); + show("fraction", () => new Buffer(2.7)); + show("zero", () => Buffer(0)); + show("negative", () => new Buffer(-1)); + show("nan", () => new Buffer(NaN)); + show("string", () => new Buffer("héllo")); + show("hex", () => new Buffer("6869", "hex")); + show("base64", () => Buffer("aGk=", "base64")); + show("empty encoding", () => new Buffer("x", "")); + show("bad encoding", () => new Buffer("x", "nope")); + show("array", () => new Buffer([1, 2, 300, -1])); + show("array-like", () => new Buffer({ length: 2, 0: 7, 1: 8 })); + show("buffer", () => new B(Buffer.from("copy"))); + show("typed", () => Buffer(new Uint16Array([1, 258]))); + const storage = new Uint8Array([10, 20, 30, 40, 50]).buffer; + show("arraybuffer", () => new Buffer(storage)); + show("offset", () => new Buffer(storage, 1)); + show("range", () => Buffer(storage, 1, 3)); + show("range past end", () => new Buffer(storage, 6)); + show("string third", () => new Buffer("abc", "latin1", 1)); + show("undefined", () => new Buffer(undefined)); + show("null", () => Buffer(null)); + show("none", () => new Buffer()); + show("imported", () => new Imported("imp")); + + console.log(Buffer(2), Buffer("ab").length, Buffer("hi").toString("hex"), new Buffer([5])); + console.log(Buffer.isBuffer(Buffer(1)), Buffer(1) instanceof Uint8Array, new Buffer(1) instanceof Uint8Array); + + // The view shares the ArrayBuffer's memory. + const view = new Buffer(storage, 2, 2); + new Uint8Array(storage)[2] = 99; + console.log(view); + + const order = []; + const arg = (v) => (order.push(v), v); + new Buffer(arg("a"), arg("utf8")); + Buffer(arg(2)); + console.log(order); + + console.log("sync end"); + process.nextTick(() => console.log("tick")); + setTimeout(() => { + new Buffer(1); + console.log("timer"); + }, 0); +} + +if (process.argv[2] === "child") { + child(); +} else { + const run = spawnSync(process.execPath, [process.argv[1], "child"], { encoding: "utf8" }); + console.log(`status: ${run.status}`); + console.log(run.stdout); + console.log(run.stderr.replace(/\(node:\d+\)/g, "(node:)")); +} diff --git a/tests/corpus/7530-buffer-constructor.ts b/tests/corpus/7530-buffer-constructor.ts new file mode 100644 index 000000000..14eac61d5 --- /dev/null +++ b/tests/corpus/7530-buffer-constructor.ts @@ -0,0 +1,42 @@ +// The deprecated `new Buffer(...)` in TypeScript, over the forms +// @types/node declares: strings with an encoding, sizes, arrays and +// ArrayBuffers. The first call reports DEP0005 on the next tick. +// The program re-runs itself and prints the child's output with process +// ids normalized. +import { spawnSync } from "node:child_process"; + +function child(): void { + process.on("warning", (w) => console.log("listener", w.name, (w as { code?: string }).code)); + + function sized(n: number): Buffer { + return new Buffer(n); + } + + const encodings: BufferEncoding[] = ["utf8", "hex", "base64", "latin1"]; + for (const encoding of encodings) { + console.log(encoding, new Buffer("6869", encoding)); + } + console.log(sized(3), sized(0).length); + console.log(new Buffer([104, 105]).toString(), new Buffer(new Uint8Array([1, 2]))); + const storage = new ArrayBuffer(4); + const view = new Buffer(storage); + view[1] = 5; + console.log(new Uint8Array(storage), view.buffer === storage); + const text: string = "typed"; + console.log(new Buffer(text).toString("hex")); + try { + sized(-2); + } catch (e) { + console.log((e as Error).name, (e as { code?: string }).code); + } + console.log("sync end"); +} + +if (process.argv[2] === "child") { + child(); +} else { + const run = spawnSync(process.execPath, [process.argv[1]!, "child"], { encoding: "utf8" }); + console.log(`status: ${run.status}`); + console.log(run.stdout); + console.log(run.stderr.replace(/\(node:\d+\)/g, "(node:)")); +} diff --git a/tests/corpus/7531-buffer-constructor-dependency/main.cjs b/tests/corpus/7531-buffer-constructor-dependency/main.cjs new file mode 100644 index 000000000..75b130bc6 --- /dev/null +++ b/tests/corpus/7531-buffer-constructor-dependency/main.cjs @@ -0,0 +1,36 @@ +// The deprecated Buffer constructor called from a dependency under +// node_modules: Node reports no DEP0005 for those calls, and the program's +// own first call reports it once, on the next tick. +// The program re-runs itself and prints the child's output with process +// ids normalized. +"use strict"; +const { spawnSync } = require("node:child_process"); +const legacy = require("legacy-buf"); + +function child() { + process.on("warning", (w) => console.log("listener", w.name, w.code)); + + console.log(legacy.construct("hi", "utf8"), legacy.construct(3), legacy.call([1, 2])); + const storage = new Uint8Array([1, 2, 3, 4]).buffer; + console.log(legacy.construct(storage, 1, 2), legacy.many(5)); + try { + legacy.construct(4, "hex"); + } catch (e) { + console.log(e.code); + } + process.nextTick(() => { + console.log("tick"); + console.log(new Buffer("own")); + console.log(Buffer(2), new Buffer([9])); + setTimeout(() => console.log("timer"), 0); + }); +} + +if (process.argv[2] === "child") { + child(); +} else { + const run = spawnSync(process.execPath, [process.argv[1], "child"], { encoding: "utf8" }); + console.log(`status: ${run.status}`); + console.log(run.stdout); + console.log(run.stderr.replace(/\(node:\d+\)/g, "(node:)")); +} diff --git a/tests/corpus/7531-buffer-constructor-dependency/node_modules/legacy-buf/index.d.ts b/tests/corpus/7531-buffer-constructor-dependency/node_modules/legacy-buf/index.d.ts new file mode 100644 index 000000000..ef314d625 --- /dev/null +++ b/tests/corpus/7531-buffer-constructor-dependency/node_modules/legacy-buf/index.d.ts @@ -0,0 +1,3 @@ +export declare function construct(value: unknown, encodingOrOffset?: unknown, length?: unknown): Buffer; +export declare function call(value: unknown): Buffer; +export declare function many(count: number): number; diff --git a/tests/corpus/7531-buffer-constructor-dependency/node_modules/legacy-buf/index.js b/tests/corpus/7531-buffer-constructor-dependency/node_modules/legacy-buf/index.js new file mode 100644 index 000000000..062e4e41d --- /dev/null +++ b/tests/corpus/7531-buffer-constructor-dependency/node_modules/legacy-buf/index.js @@ -0,0 +1,15 @@ +"use strict"; +// A dependency that keeps the deprecated Buffer constructor. +exports.construct = function (value, encodingOrOffset, length) { + if (length !== undefined) return new Buffer(value, encodingOrOffset, length); + if (encodingOrOffset !== undefined) return new Buffer(value, encodingOrOffset); + return new Buffer(value); +}; +exports.call = function (value) { + return Buffer(value); +}; +exports.many = function (count) { + let total = 0; + for (let i = 0; i < count; i++) total += new Buffer(1).length; + return total; +}; diff --git a/tests/corpus/7531-buffer-constructor-dependency/node_modules/legacy-buf/package.json b/tests/corpus/7531-buffer-constructor-dependency/node_modules/legacy-buf/package.json new file mode 100644 index 000000000..db2277b60 --- /dev/null +++ b/tests/corpus/7531-buffer-constructor-dependency/node_modules/legacy-buf/package.json @@ -0,0 +1,5 @@ +{ + "name": "legacy-buf", + "version": "1.0.0", + "main": "index.js" +} diff --git a/tests/corpus/7532-buffer-constructor-check-limit/main.cjs b/tests/corpus/7532-buffer-constructor-check-limit/main.cjs new file mode 100644 index 000000000..c08690a83 --- /dev/null +++ b/tests/corpus/7532-buffer-constructor-check-limit/main.cjs @@ -0,0 +1,10 @@ +// Node checks the caller of the deprecated Buffer constructor for its first +// 10000 calls only: once a dependency under node_modules has made them, the +// program's own calls report no DEP0005. +"use strict"; +process.on("warning", (w) => console.log("listener", w.code)); +const legacy = require("legacy-buf"); + +console.log(legacy.many(10000)); +console.log(new Buffer("after"), Buffer(1)); +setTimeout(() => console.log("timer"), 0); diff --git a/tests/corpus/7532-buffer-constructor-check-limit/node_modules/legacy-buf/index.d.ts b/tests/corpus/7532-buffer-constructor-check-limit/node_modules/legacy-buf/index.d.ts new file mode 100644 index 000000000..ef314d625 --- /dev/null +++ b/tests/corpus/7532-buffer-constructor-check-limit/node_modules/legacy-buf/index.d.ts @@ -0,0 +1,3 @@ +export declare function construct(value: unknown, encodingOrOffset?: unknown, length?: unknown): Buffer; +export declare function call(value: unknown): Buffer; +export declare function many(count: number): number; diff --git a/tests/corpus/7532-buffer-constructor-check-limit/node_modules/legacy-buf/index.js b/tests/corpus/7532-buffer-constructor-check-limit/node_modules/legacy-buf/index.js new file mode 100644 index 000000000..062e4e41d --- /dev/null +++ b/tests/corpus/7532-buffer-constructor-check-limit/node_modules/legacy-buf/index.js @@ -0,0 +1,15 @@ +"use strict"; +// A dependency that keeps the deprecated Buffer constructor. +exports.construct = function (value, encodingOrOffset, length) { + if (length !== undefined) return new Buffer(value, encodingOrOffset, length); + if (encodingOrOffset !== undefined) return new Buffer(value, encodingOrOffset); + return new Buffer(value); +}; +exports.call = function (value) { + return Buffer(value); +}; +exports.many = function (count) { + let total = 0; + for (let i = 0; i < count; i++) total += new Buffer(1).length; + return total; +}; diff --git a/tests/corpus/7532-buffer-constructor-check-limit/node_modules/legacy-buf/package.json b/tests/corpus/7532-buffer-constructor-check-limit/node_modules/legacy-buf/package.json new file mode 100644 index 000000000..db2277b60 --- /dev/null +++ b/tests/corpus/7532-buffer-constructor-check-limit/node_modules/legacy-buf/package.json @@ -0,0 +1,5 @@ +{ + "name": "legacy-buf", + "version": "1.0.0", + "main": "index.js" +} diff --git a/tests/corpus/7533-buffer-value.cjs b/tests/corpus/7533-buffer-value.cjs new file mode 100644 index 000000000..3fd500ec2 --- /dev/null +++ b/tests/corpus/7533-buffer-value.cjs @@ -0,0 +1,101 @@ +// The Buffer function read as a value: one identity across the global, the +// buffer module and its aliases, Node's own enumerable members in Node's +// order, statics that apply Node's argument handling, instanceof, and a +// poolSize property the pool reads. +"use strict"; +const buffer = require("buffer"); +const nodeBuffer = require("node:buffer"); +const util = require("util"); + +// bn.js keeps the module's Buffer in a variable assigned under try. +var BnBuffer; +try { + BnBuffer = require("buffer").Buffer; +} catch (e) {} +const pass = (value) => value; +const Stored = pass(require("buffer").Buffer); +const Alias = Buffer; +const table = { Buffer: buffer.Buffer }; + +console.log(typeof Stored, Stored.name, Stored.length, Alias.name, Alias.length); +console.log(Stored === Buffer, Stored === globalThis.Buffer, Stored === buffer.Buffer, Stored === nodeBuffer.Buffer); +console.log(Stored === Alias, table.Buffer === Stored, Stored.constructor === Function); +console.log(Stored); +console.log(Object.keys(Stored)); +console.log(util.inspect({ nested: { Buffer: Stored } }, { depth: 1 })); + +// Statics through the value. +const show = (label, run) => { + try { + console.log(label, run()); + } catch (e) { + console.log(label, e.name, e.code, e.message); + } +}; +show("from string", () => Stored.from("héllo")); +show("from hex", () => Stored.from("6869", "hex")); +show("from empty encoding", () => Stored.from("x", "")); +show("from bad encoding", () => Stored.from("x", "nope")); +show("from array", () => Stored.from([1, 2, 300])); +const storage = new Uint8Array([1, 2, 3, 4, 5]).buffer; +show("from range", () => Stored.from(storage, 1, 3)); +show("from number", () => Stored.from(5)); +show("alloc", () => Stored.alloc(3, "ab")); +show("alloc fill", () => Stored.alloc(2, 7)); +show("alloc string size", () => Stored.alloc("3")); +show("allocUnsafe", () => Stored.allocUnsafe(4).fill(9)); +show("allocUnsafe negative", () => Stored.allocUnsafe(-1)); +show("allocUnsafe string", () => Stored.allocUnsafe("4")); +show("allocUnsafeSlow", () => Stored.allocUnsafeSlow(2).fill(1)); +show("isBuffer", () => [Stored.isBuffer(Buffer.alloc(1)), Stored.isBuffer(new Uint8Array(1)), Stored.isBuffer("x")]); +show("isEncoding", () => [Stored.isEncoding("UTF-8"), Stored.isEncoding(""), Stored.isEncoding("nope"), Stored.isEncoding(1)]); +show("compare", () => Stored.compare(Buffer.from("a"), Buffer.from("b"))); +show("compare bad", () => Stored.compare("a", Buffer.from("b"))); +show("concat", () => Stored.concat([Buffer.from("ab"), Buffer.from("cd")])); +show("concat length", () => Stored.concat([Buffer.from("ab"), Buffer.from("cd")], 3)); +show("concat empty", () => Stored.concat([], "x")); +show("concat bad length", () => Stored.concat([Buffer.from("a")], "x")); +show("concat bad list", () => Stored.concat("x")); +show("byteLength", () => [Stored.byteLength("héllo"), Stored.byteLength("aGk=", "base64"), Stored.byteLength(new Uint16Array(3))]); + +// Member values keep one identity, with Node's names and lengths. +const members = [ + [Buffer.from, Stored.from], + [Buffer.alloc, Stored.alloc], + [Buffer.allocUnsafe, Stored.allocUnsafe], + [Buffer.isBuffer, Stored.isBuffer], + [Buffer.compare, Stored.compare], + [Buffer.concat, Stored.concat], + [Buffer.byteLength, Stored.byteLength], + [Buffer.isEncoding, Stored.isEncoding], +]; +console.log(members.map(([a, b]) => `${a.name}/${a.length}/${a === b}`).join(" ")); +const isBuffer = require("buffer").Buffer.isBuffer; +console.log(isBuffer(Buffer.from("x")), isBuffer(null), typeof Buffer.allocUnsafe); + +// A bn.js-style allocator over Buffer or Array. +function allocate(ArrayType, size) { + if (ArrayType.allocUnsafe) return ArrayType.allocUnsafe(size); + return new ArrayType(size); +} +function toArrayLike(ArrayType, size) { + const made = allocate(ArrayType, size); + made.fill(4); + return made; +} +function toBuffer(size) { + return BnBuffer ? toArrayLike(BnBuffer, size) : null; +} +const made = toBuffer(3); +console.log(made, toArrayLike(Array, 2), made instanceof Stored, made instanceof Buffer); +console.log(new Uint8Array(2) instanceof Stored, Buffer.from("x") instanceof Alias, pass("x") instanceof Stored); + +// poolSize: one setting through the value and the static spelling. +console.log(Stored.poolSize, Buffer.poolSize); +Buffer.poolSize = 1024; +console.log(Stored.poolSize); +const small = Buffer.allocUnsafe(600); +console.log(small.buffer.byteLength, Buffer.allocUnsafe(400).buffer.byteLength); +Stored.poolSize = 2048; +console.log(Buffer.poolSize, Buffer.allocUnsafe(600).buffer.byteLength); +Stored.poolSize = 65536; diff --git a/tests/corpus/7534-buffer-prototype-members.cjs b/tests/corpus/7534-buffer-prototype-members.cjs new file mode 100644 index 000000000..2fd347373 --- /dev/null +++ b/tests/corpus/7534-buffer-prototype-members.cjs @@ -0,0 +1,29 @@ +// Buffer.prototype members read as values, as smart-buffer probes and +// calls them: own methods are functions with Node's names and lengths that +// run on a Buffer receiver, the parent and offset getters read on the +// prototype answer undefined, constructor is Buffer, and absent names are +// undefined. +"use strict"; +const buffer_1 = require("buffer"); + +function check(method) { + if (typeof buffer_1.Buffer.prototype[method] === "undefined") return `${method}: missing`; + return `${method}: ${typeof buffer_1.Buffer.prototype[method]}`; +} +console.log(["writeBigInt64BE", "readBigUInt64LE", "readUInt8", "nope", "parent", "offset"].map(check)); +console.log(Buffer.prototype.constructor === Buffer, Buffer.prototype["constructor"] === buffer_1.Buffer); +console.log(Buffer.prototype[0], Buffer.prototype[undefined], Buffer.prototype.toJSON === Buffer.prototype.toJSON); + +const names = ["readUInt8", "readUint8", "readFloatLE", "writeUInt16LE", "readUIntLE", "toString", "toLocaleString", "slice", "hexSlice"]; +console.log(names.map((n) => `${n}=${JSON.stringify(Buffer.prototype[n].name)}/${Buffer.prototype[n].length}`).join(" ")); + +const buf = Buffer.alloc(8); +function write(func, value, offset) { + return func.call(buf, value, offset); +} +console.log(write(Buffer.prototype.writeUInt16LE, 513, 0), write(Buffer.prototype.writeInt8, -2, 2), buf); +console.log(Buffer.prototype.readUInt16LE.call(buf, 0), Buffer.prototype.readInt8.call(buf, 2)); +console.log(Buffer.prototype.toString.call(buf, "hex", 0, 3), Buffer.prototype.toLocaleString.call(Buffer.from("hi"))); +console.log(Buffer.prototype.slice.call(buf, 0, 2), Buffer.prototype.equals.call(buf, Buffer.alloc(8))); +console.log(Buffer.prototype.fill.call(Buffer.alloc(3), 1), Buffer.prototype.indexOf.call(Buffer.from("abc"), "c")); +console.log(Buffer.prototype.write.call(buf, "zz", 4), buf, Buffer.prototype.toJSON.call(Buffer.from([1]))); diff --git a/tests/corpus/7535-inspect-function-properties.cjs b/tests/corpus/7535-inspect-function-properties.cjs new file mode 100644 index 000000000..5e3f8c6a6 --- /dev/null +++ b/tests/corpus/7535-inspect-function-properties.cjs @@ -0,0 +1,29 @@ +// util.inspect of functions with own enumerable properties, as Node's +// formatRaw prints them: the function's base then its keys (symbols +// included), `[Function]` past the depth limit, and the plain base when +// every own property is hidden. +"use strict"; +const util = require("util"); + +function named() {} +named.x = 1; +const arrow = () => {}; +arrow.a = "s"; +arrow[Symbol("k")] = 2; +Object.defineProperty(arrow, "hidden", { value: 3, enumerable: false }); +function hiddenOnly() {} +Object.defineProperty(hiddenOnly, "secret", { value: 1, enumerable: false }); +function big() {} +for (let i = 0; i < 8; i++) big["key" + i] = i; +function self() {} +self.me = self; + +const pass = (value) => value; + +console.log(named, arrow, hiddenOnly); +console.log(util.inspect(named, { depth: 0 })); +console.log(util.inspect(pass({ wrap: named }), { depth: 0 })); +console.log(util.inspect(pass([named, arrow]))); +console.log(util.inspect(big)); +console.log(util.inspect(self)); +console.log(util.inspect(pass({ a: { b: { f: big } } }))); diff --git a/tests/harness/corpus-costs/7527-buffer-from-array-buffer.ts.json b/tests/harness/corpus-costs/7527-buffer-from-array-buffer.ts.json new file mode 100644 index 000000000..593c181c3 --- /dev/null +++ b/tests/harness/corpus-costs/7527-buffer-from-array-buffer.ts.json @@ -0,0 +1 @@ +{ "plain": 4043, "san": 6242 } diff --git a/tests/harness/corpus-costs/7528-buffer-from-array-buffer.cjs.json b/tests/harness/corpus-costs/7528-buffer-from-array-buffer.cjs.json new file mode 100644 index 000000000..759dce51e --- /dev/null +++ b/tests/harness/corpus-costs/7528-buffer-from-array-buffer.cjs.json @@ -0,0 +1 @@ +{ "plain": 3823, "san": 5864 } diff --git a/tests/harness/corpus-costs/7529-buffer-constructor.cjs.json b/tests/harness/corpus-costs/7529-buffer-constructor.cjs.json new file mode 100644 index 000000000..408808da4 --- /dev/null +++ b/tests/harness/corpus-costs/7529-buffer-constructor.cjs.json @@ -0,0 +1 @@ +{ "plain": 5309, "san": 56270 } diff --git a/tests/harness/corpus-costs/7530-buffer-constructor.ts.json b/tests/harness/corpus-costs/7530-buffer-constructor.ts.json new file mode 100644 index 000000000..287d50fdc --- /dev/null +++ b/tests/harness/corpus-costs/7530-buffer-constructor.ts.json @@ -0,0 +1 @@ +{ "plain": 4282, "san": 5688 } diff --git a/tests/harness/corpus-costs/7531-buffer-constructor-dependency/main.cjs.json b/tests/harness/corpus-costs/7531-buffer-constructor-dependency/main.cjs.json new file mode 100644 index 000000000..568483d98 --- /dev/null +++ b/tests/harness/corpus-costs/7531-buffer-constructor-dependency/main.cjs.json @@ -0,0 +1 @@ +{ "plain": 5222, "san": 6614 } diff --git a/tests/harness/corpus-costs/7532-buffer-constructor-check-limit/main.cjs.json b/tests/harness/corpus-costs/7532-buffer-constructor-check-limit/main.cjs.json new file mode 100644 index 000000000..3d262fe41 --- /dev/null +++ b/tests/harness/corpus-costs/7532-buffer-constructor-check-limit/main.cjs.json @@ -0,0 +1 @@ +{ "plain": 4707, "san": 5678 } diff --git a/tests/harness/corpus-costs/7533-buffer-value.cjs.json b/tests/harness/corpus-costs/7533-buffer-value.cjs.json new file mode 100644 index 000000000..126c82265 --- /dev/null +++ b/tests/harness/corpus-costs/7533-buffer-value.cjs.json @@ -0,0 +1 @@ +{ "plain": 4874, "san": 10466 } diff --git a/tests/harness/corpus-costs/7534-buffer-prototype-members.cjs.json b/tests/harness/corpus-costs/7534-buffer-prototype-members.cjs.json new file mode 100644 index 000000000..624b7f12f --- /dev/null +++ b/tests/harness/corpus-costs/7534-buffer-prototype-members.cjs.json @@ -0,0 +1 @@ +{ "plain": 3944, "san": 5992 } diff --git a/tests/harness/corpus-costs/7535-inspect-function-properties.cjs.json b/tests/harness/corpus-costs/7535-inspect-function-properties.cjs.json new file mode 100644 index 000000000..58c48c344 --- /dev/null +++ b/tests/harness/corpus-costs/7535-inspect-function-properties.cjs.json @@ -0,0 +1 @@ +{ "plain": 3458, "san": 4785 } diff --git a/tests/harness/differential.test.ts b/tests/harness/differential.test.ts index 905f648dd..7f86c0dc5 100644 --- a/tests/harness/differential.test.ts +++ b/tests/harness/differential.test.ts @@ -148,9 +148,9 @@ function nodeOracleFile(file: string): string { } /** `// @no-deprecation` in the directive head: the Node oracle runs with - * --no-deprecation — for programs exercising deprecated-API ERROR paths - * (new Buffer's ctor ladder) whose DEP warnings carry a pid and can never - * byte-compare; the compiled runtime emits no process warnings at all. */ + * --no-deprecation, for programs whose DEP warnings carry a pid and cannot + * compare. The compiled program does not read the directive; a program + * whose compiled form reports the warning sets process.noDeprecation. */ function wantsNoDeprecation(file: string): boolean { return directiveHead(file).some((l) => /^\/\/ @no-deprecation\s*$/.test(l)); } diff --git a/tests/harness/errors.test.ts b/tests/harness/errors.test.ts index 319e7329f..39ecd9e79 100644 --- a/tests/harness/errors.test.ts +++ b/tests/harness/errors.test.ts @@ -2050,3 +2050,52 @@ console.log(words.length); expect(r.stdout).toBe(`${fence}${fence}1\n`); }); }); + +// The Buffer function read as a value models Node's statics; the forms it +// cannot answer as Node does throw SC2020: calling or constructing it +// through the value (Node reports DEP0005 by the caller's location), its +// prototype as a whole, copyBytesFrom and of, and Buffer.prototype members +// that are inherited, symbol-keyed or called on other receivers. +describe("Buffer value refusals", () => { + test("unmodeled Buffer value forms throw SC2020", async () => { + const r = await compileAndRun( + "buffer-value-refusals", + `const pass = (value) => value; +const Stored = pass(require("buffer").Buffer); +const runs = [ + () => Stored("x"), + () => new Stored(2), + () => Stored.prototype, + () => Stored.copyBytesFrom(new Uint8Array(2)), + () => Stored.of(1), + () => Buffer.prototype.map, + () => Buffer.prototype[Symbol.iterator], + () => Buffer.prototype.readUInt8.call(new Uint8Array(2), 0), +]; +for (const run of runs) { + try { + run(); + console.log("ran"); + } catch (e) { + console.log(e.name, e.code, e.message); + } +} +`, + "cjs", + ); + expect(r.exitCode).toBe(0); + expect(r.stdout).toBe( + [ + "Error SC2020 Calling or constructing Buffer through a stored Buffer value is not supported yet", + "Error SC2020 Calling or constructing Buffer through a stored Buffer value is not supported yet", + "Error SC2020 Buffer.prototype through a stored Buffer value is not supported yet", + "Error SC2020 Buffer.copyBytesFrom through a stored Buffer value is not supported yet", + "Error SC2020 Buffer.of through a stored Buffer value is not supported yet", + "Error SC2020 Buffer.prototype.map is not supported yet", + "Error SC2020 Buffer.prototype members keyed by symbols are not supported yet", + "Error SC2020 Buffer.prototype.readUInt8 on a receiver that is not a Buffer is not supported yet", + "", + ].join("\n"), + ); + }); +}); From 9c9fb6531a8857425049ffb700069ff008c3e730 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 03:07:29 -0500 Subject: [PATCH 028/184] Replay class objects with their base names, and restore the inspector after a refusal - The emitter journal that forked workers merge (#1919) records each class object entry whole, and the entries carry baseNameSym beside nameSym. The replay cast named only nameSym, which the native compiler refuses (SC1090), so the self-hosted LLVM emitter did not build. - An inspection refusal (the 'not supported yet' Error of insp.dyn) propagates out of the emitted inspect helpers before they close their frames and before util.inspect's layout options are restored. The open frames, their indentation and the custom layout then shaped every later console.log and util.inspect. A top-level render through helpers that can reach a checked-dynamic view, a generator or a class instance now runs in a lifted wrapper that takes insp.mark first and calls insp.unwind in a finally. A render under breakLength, compact or sorted restores the default layout in a finally the same way. - A non-enumerable accessor that %o shows, and any accessor of a checked-dynamic object, renders as [Getter], [Setter] or [Getter/Setter], as Node prints it, instead of its value slot. - Under %o, an array's non-enumerable own properties follow its named ones in brackets (`[ 1, 2, [length]: 2, foo: 'x', [hid]: 3 ]`), as Node prints them; corpus 6357 covers [length] ahead of named properties. --- packages/compiler/src/backend/llvm/emitter.ts | 2 +- .../compiler/src/backend/llvm/lib-shared.ts | 2 + .../src/frontend/lowering/lower-inspect.ts | 174 +++++++++++++++--- .../compiler/src/ir/builtin-signatures.ts | 5 + packages/runtime/src/scr_inspect.c | 73 ++++++-- packages/runtime/src/scr_inspect.h | 4 + tests/corpus/6357-format-o-hidden.ts | 18 +- tests/harness/dyncheck.test.ts | 66 +++++++ 8 files changed, 306 insertions(+), 38 deletions(-) diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 004277a24..7c6fd7a47 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -4859,7 +4859,7 @@ export class LlEmitter { return; } case "classObj": - addEntry(this.classObjs, key, value as { nameSym: string }); + addEntry(this.classObjs, key, value as { nameSym: string; baseNameSym: string | null }); return; case "retainWrapper": addEntry(this.nullableUnions.retainWrappers, key, value as string); diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index dfc5327e2..fc4b1337b 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -663,6 +663,8 @@ export const LIB_FN_SYMS: Record = { "insp.moreItems": "scr_insp_more_items", "insp.end": "scr_insp_end", "insp.nestedBegin": "scr_insp_nested_begin", + "insp.mark": "scr_insp_mark", + "insp.unwind": "scr_insp_unwind", "insp.nestedEnd": "scr_insp_nested_end", // Circular references over cycle-capable composites (Node's // seen/circular machinery — none of these throw; refWrap borrows its diff --git a/packages/compiler/src/frontend/lowering/lower-inspect.ts b/packages/compiler/src/frontend/lowering/lower-inspect.ts index cda4d2972..349fec120 100644 --- a/packages/compiler/src/frontend/lowering/lower-inspect.ts +++ b/packages/compiler/src/frontend/lowering/lower-inspect.ts @@ -900,6 +900,112 @@ export function inspectExpr( recurse: IrExpr, depth: IrExpr, loc: SrcLoc, +): IrExpr { + const render = inspectExprBody(lowerer, t, value, recurse, depth, loc); + // A top-level render through emitted helpers that can reach a refusal + // (the 'not supported yet' Error of insp.dyn) unwinds the inspector to + // its state before the render on every exit: the Error propagates out of + // the helpers before they close their frames. + if ( + render.kind !== "call" || + recurse.kind !== "numLit" || + recurse.value !== 0 || + !inspectMayRefuse(lowerer, t) + ) + return render; + return withInspectUnwind(lowerer, render, t, loc); +} + +/** A top-level helper render (`call helper(v, r, d)`) through a lifted + * wrapper that takes insp.mark first and runs insp.unwind however the + * render ends: `try { return helper(v, r, d); } finally { insp.unwind(); }`. + * A seqExpr cannot hold the try. */ +function withInspectUnwind( + lowerer: Lowerer, + render: IrExpr & { kind: "call" }, + t: IrType, + loc: SrcLoc, +): IrExpr { + const key = `unwind:${render.callee}`; + let name = lowerer.inspectHelpers.get(key); + if (name === undefined) { + name = keyedHelperName(lowerer, "%util.inspUnwound", key); + lowerer.inspectHelpers.set(key, name); + const params = [ + { localId: "v.0", name: "v", type: t }, + { localId: "r.0", name: "r", type: F64 }, + { localId: "d.0", name: "d", type: F64 }, + ]; + const call = (fn: "insp.mark" | "insp.unwind"): IrStmt => ({ + kind: "exprStmt", + expr: { kind: "libCall", fn, args: [], type: VOID, loc }, + loc, + }); + lowerer.liftedFns.push({ + name, + params, + returnType: STRING, + locals: params.map((p) => ({ id: p.localId, name: p.name, type: p.type, mutable: false })), + body: [ + call("insp.mark"), + { + kind: "tryCatch", + tryBody: [ + { + kind: "return", + value: { + kind: "call", + callee: render.callee, + args: params.map((p) => varRef(p.localId, p.type, loc)), + type: STRING, + loc, + }, + loc, + }, + ], + catchBody: null, + catchLocalId: null, + finallyBody: [call("insp.unwind")], + loc, + }, + ], + loc, + }); + } + return { ...render, callee: name }; +} + +/** Whether inspecting a `t` value can end in an inspection refusal: `t` + * or a type reachable from it is a checked-dynamic value, a generator, a + * Dirent row or a class instance, which can render through a + * checked-dynamic view. */ +function inspectMayRefuse(lowerer: Lowerer, t: IrType): boolean { + const seen = new Set(); + const walk = (u: IrType): boolean => { + if (u.kind === "dyn" || u.kind === "generator") return true; + if (u.kind === "record") { + const shape = lowerer.shapes.get(u.shapeId); + if (shape && isDirentShape(shape)) return true; + } + // A class instance can render through its checked-dynamic view, which + // a helper built earlier may only learn at the end of lowering + // (classInspectsAddedProperties). + if (u.kind === "object") return true; + const k = typeKey(u); + if (seen.has(k)) return false; + seen.add(k); + return typeConstituents(lowerer, u).some(walk); + }; + return walk(t); +} + +function inspectExprBody( + lowerer: Lowerer, + t: IrType, + value: IrExpr, + recurse: IrExpr, + depth: IrExpr, + loc: SrcLoc, ): IrExpr { // A Dirent row renders through its dyn view: Node's `Dirent { ... }` // with the Symbol(type) key. @@ -2350,40 +2456,55 @@ function parseInspectOptions( /** A render under non-default breakLength/compact/sorted: the value is evaluated * first, the layout applies to this render only, and the defaults return - * afterwards. */ + * however the render ends, an inspection refusal included. The render runs + * in a lifted function, `(v) => { insp.layout(...); try { return render(v); } + * finally { insp.layout(80, 3, false); } }`, since a seqExpr cannot hold the + * try; `tag` names the render for the function's key. */ function withInspectLayout( lowerer: Lowerer, options: InspectOptions, value: IrExpr, + tag: string, render: (value: IrExpr) => IrExpr, loc: SrcLoc, ): IrExpr { if (options.breakLength === 80 && options.compact === 3 && !options.sorted) return render(value); - const input = lowerer.declareHiddenLocal("%inspectInput", value.type); - const text = lowerer.declareHiddenLocal("%inspectText", STRING); - const layout = (breakLength: number, compact: number, sorted: boolean): IrStmt => ({ - kind: "exprStmt", - expr: { - kind: "libCall", - fn: "insp.layout", - args: [numLit(breakLength, loc), numLit(compact, loc), boolLit(sorted, loc)], - type: VOID, + const key = `layout:${tag}:${typeKey(value.type)}:${options.breakLength}:${options.compact}:${options.sorted}:${options.depth}`; + let name = lowerer.inspectHelpers.get(key); + if (name === undefined) { + name = keyedHelperName(lowerer, "%util.inspLayout", key); + lowerer.inspectHelpers.set(key, name); + const layout = (breakLength: number, compact: number, sorted: boolean): IrStmt => ({ + kind: "exprStmt", + expr: { + kind: "libCall", + fn: "insp.layout", + args: [numLit(breakLength, loc), numLit(compact, loc), boolLit(sorted, loc)], + type: VOID, + loc, + }, loc, - }, - loc, - }); - return { - kind: "seqExpr", - stmts: [ - { kind: "varDecl", localId: input.id, init: value, loc }, - layout(options.breakLength, options.compact, options.sorted), - { kind: "varDecl", localId: text.id, init: render(varRef(input.id, value.type, loc)), loc }, - layout(80, 3, false), - ], - result: varRef(text.id, STRING, loc), - type: STRING, - loc, - }; + }); + lowerer.liftedFns.push({ + name, + params: [{ localId: "v.0", name: "v", type: value.type }], + returnType: STRING, + locals: [{ id: "v.0", name: "v", type: value.type, mutable: false }], + body: [ + layout(options.breakLength, options.compact, options.sorted), + { + kind: "tryCatch", + tryBody: [{ kind: "return", value: render(varRef("v.0", value.type, loc)), loc }], + catchBody: null, + catchLocalId: null, + finallyBody: [layout(80, 3, false)], + loc, + }, + ], + loc, + }); + } + return { kind: "call", callee: name, args: [value], type: STRING, loc }; } /** `util.inspect(fn)` / `util.inspect(Cls)` over a DIRECT identifier @@ -2500,6 +2621,7 @@ export function lowerInspectValue( lowerer, options, value, + "typedArray", (input) => ({ kind: "libCall", fn: "insp.typedArray", @@ -2525,6 +2647,7 @@ export function lowerInspectValue( lowerer, options, value, + "dirDyn", (input) => ({ kind: "libCall", fn: "console.dirDyn", @@ -2539,6 +2662,7 @@ export function lowerInspectValue( lowerer, options, value, + "inspect", (input) => inspectExpr(lowerer, value.type, input, numLit(0, loc), numLit(depth, loc), loc), loc, ); diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index a316b5dd5..6dc8a6826 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -4414,6 +4414,11 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ * `prefix inner` re-indented at the caller's level (+1). */ "insp.nestedBegin": { argTypes: [], result: VOID }, "insp.nestedEnd": { argTypes: [STRING, STRING], result: STRING }, + /** Around a top-level render that an inspection refusal can end: mark + * records the inspector's frame and nesting depth, unwind (in the + * render's finally) closes what the render left open above it. */ + "insp.mark": { argTypes: [], result: VOID }, + "insp.unwind": { argTypes: [], result: VOID }, /** node:string_decoder's utf8 StringDecoder (scr_string.c): the decoder * value is a one-field record whose f64 PACKS the pending partial * sequence (count + up to 3 raw bytes — Node buffers at most 3 for diff --git a/packages/runtime/src/scr_inspect.c b/packages/runtime/src/scr_inspect.c index 9875a7077..e63c1cec1 100644 --- a/packages/runtime/src/scr_inspect.c +++ b/packages/runtime/src/scr_inspect.c @@ -646,6 +646,54 @@ ScrStr *scr_insp_nested_end(ScrStr *prefix, ScrStr *inner) { return ib_take(&out); } +/* The inspector's state at the start of a top-level render. A render that + * an inspection refusal ends leaves its compiled frames open: the Error + * propagates out of the emitted helpers before their scr_insp_end and + * scr_insp_nested_end calls. The emitted render takes a mark first and + * unwinds to it on every exit (lower-inspect.ts), so later renders start + * from the state the refused one found. */ +typedef struct { + size_t nframes; + size_t nsaved; + size_t nseen; + size_t indent; + double cur_depth; +} InspMark; + +static SCR_TL InspMark *g_marks; +static SCR_TL size_t g_nmarks; +static SCR_TL size_t g_marks_cap; + +void scr_insp_mark(void) { + if (g_nmarks == g_marks_cap) { + g_marks_cap = g_marks_cap ? g_marks_cap * 2 : 4; + g_marks = realloc(g_marks, g_marks_cap * sizeof(InspMark)); + if (!g_marks) insp_oom(); + } + g_marks[g_nmarks++] = (InspMark){g_nframes, g_nsaved, g_nseen, g_indent, g_cur_depth}; +} + +void scr_insp_unwind(void) { + InspMark m = g_marks[--g_nmarks]; + while (g_nframes > m.nframes) { + InspFrame *f = &g_frames[--g_nframes]; + for (size_t i = 0; i < f->len; i++) scr_str_release(f->items[i]); + free(f->items); + } + while (g_nsaved > m.nsaved) { + InspSaved *sv = &g_saved[--g_nsaved]; + free(g_seen); + g_seen = sv->seen; + g_seen_cap = sv->seen_cap; + memcpy(g_circ, sv->circ, sizeof g_circ); + g_ncirc = sv->ncirc; + } + if (g_nseen > m.nseen) g_nseen = m.nseen; + g_indent = m.indent; + g_cur_depth = m.cur_depth; + g_pending_seen = NULL; +} + /* ── string values (formatPrimitive's string half) ───────────────────── */ /* Byte offset of UTF-16 unit `units` — when the boundary splits an astral @@ -1422,7 +1470,7 @@ static ScrStr *scr_insp_obj(ScrDyn *d, double recurse, double depth, const char for (size_t i = 0; i < d->v.obj.len; i++) { ScrDynEntry *ent = &d->v.obj.entries[i]; if (!insp_obj_shows(ent)) continue; - ScrStr *val = scr_insp_dyn(ent->value, recurse + 1, depth); + ScrStr *val = ent->accessor ? insp_accessor_tag(ent) : scr_insp_dyn(ent->value, recurse + 1, depth); InspBuf eb = {0}; if (ent->enumerable) { insp_key_into(&eb, ent->key, ent->key_len); @@ -1443,7 +1491,7 @@ static ScrStr *scr_insp_obj(ScrDyn *d, double recurse, double depth, const char ScrDynEntry *ent = scr_insp_symbol_entry(d, i); if (!ent) continue; ScrStr *key = scr_insp_dyn(d->symbol_keys->v.arr.items[i], recurse + 1, depth); - ScrStr *val = scr_insp_dyn(ent->value, recurse + 1, depth); + ScrStr *val = ent->accessor ? insp_accessor_tag(ent) : scr_insp_dyn(ent->value, recurse + 1, depth); InspBuf eb = {0}; /* A hidden symbol key is bracketed like a hidden string key. */ if (!ent->enumerable) ib_char(&eb, '['); @@ -1635,10 +1683,11 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { case SCR_DYN_ARR: { size_t len = d->v.arr.len; /* Named and noncanonical numeric properties follow the elements as - * `key: value` entries (Node's array extras). */ + * `key: value` entries (Node's array extras); under showHidden the + * own `length` comes first, and non-enumerable properties show too. */ const ScrDyn *props = d->v.arr.properties; size_t nprops = 0; - for (size_t i = 0; props && i < props->v.obj.len; i++) nprops += props->v.obj.entries[i].enumerable; + for (size_t i = 0; props && i < props->v.obj.len; i++) nprops += insp_obj_shows(&props->v.obj.entries[i]); if (len == 0 && nprops == 0 && !g_show_hidden) return scr_str_new("[]", 2); if (recurse > depth) return scr_str_new("[Array]", 7); scr_insp_begin(recurse + 1); @@ -1672,9 +1721,9 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { scr_str_release(m); } if (g_show_hidden) { - /* The own non-enumerable `length`, after the items and before the - * other own properties. Node's grouping reads value[k] for every - * output position k, this one included. */ + /* The own non-enumerable `length`, after the items. Node's + * grouping reads value[k] for every output position k, this one + * included. */ char line[48]; int n = snprintf(line, sizeof line, "[length]: %zu", len); ScrStr *s = scr_str_new(line, (size_t)n); @@ -1685,10 +1734,13 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { } for (size_t k = 0; props && k < props->v.obj.len; k++) { const ScrDynEntry *ent = &props->v.obj.entries[k]; - if (!ent->enumerable) continue; + if (!insp_obj_shows(ent)) continue; ScrStr *val = ent->accessor ? insp_accessor_tag(ent) : scr_insp_dyn(ent->value, recurse + 1, depth); InspBuf eb = {0}; + /* formatProperty's hidden key: the key's rendering in brackets. */ + if (!ent->enumerable) ib_char(&eb, '['); insp_key_into(&eb, ent->key, ent->key_len); + if (!ent->enumerable) ib_char(&eb, ']'); ib_cstr(&eb, ": "); ib_bytes(&eb, val->data, val->len); scr_str_release(val); @@ -1702,9 +1754,8 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { entries++; } /* Node's grouping leaves out the last entry when there are more - * entries than maxArrayLength (the `... more items` entry when the - * array has no properties, or under showHidden the `[length]` - * entry). */ + * entries than maxArrayLength (100): the `... more items` entry, + * `[length]` under showHidden, or the last property. */ bool last_out = entries > 100; ScrStr *base = scr_str_new("", 0); ScrStr *b0 = scr_str_new("[", 1); diff --git a/packages/runtime/src/scr_inspect.h b/packages/runtime/src/scr_inspect.h index 9bf18a23c..f36335a3b 100644 --- a/packages/runtime/src/scr_inspect.h +++ b/packages/runtime/src/scr_inspect.h @@ -76,6 +76,10 @@ ScrStr *scr_insp_end(ScrStr *base, ScrStr *b0, ScrStr *b1, double recurse, * caller's level (+1; borrows both). */ void scr_insp_nested_begin(void); ScrStr *scr_insp_nested_end(ScrStr *prefix, ScrStr *inner); +/* Mark the inspector's state before a top-level render, and return to the + * mark when the render ends, normally or by an inspection refusal. */ +void scr_insp_mark(void); +void scr_insp_unwind(void); /* The buf.read* / buf.write* numeric families on u8 bytes, Node-exact: * boundsError's ladder (non-integer offset → 'It must be an integer', a diff --git a/tests/corpus/6357-format-o-hidden.ts b/tests/corpus/6357-format-o-hidden.ts index 9e48ed4ba..44ed1d772 100644 --- a/tests/corpus/6357-format-o-hidden.ts +++ b/tests/corpus/6357-format-o-hidden.ts @@ -1,6 +1,7 @@ // util.format's %o over values reached through `unknown` renders with // showHidden, as Node does: arrays show their hidden `[length]` (with -// Node's grouping), at every depth, inside Maps and records too. +// Node's grouping), at every depth, inside Maps and records too, ahead of +// an array's named properties. import { format } from "node:util"; const cases: [string, unknown][] = [ ["ten", Array.from({ length: 10 }, (_, i) => i)], @@ -17,3 +18,18 @@ const cases: [string, unknown][] = [ ["date", [new Date(0)]], ]; for (const [k, v] of cases) console.log(k, format("%o", v)); +// The hidden `[length]` comes before an array's named properties, and the +// grid's last-entry rule counts both. +type Bag = Record; +const named = [1, 2]; +(named as unknown as Bag).foo = "x"; +const tagged = Array.from({ length: 100 }, (_, i) => i); +(tagged as unknown as Bag).tag = "t"; +const bare: number[] = []; +(bare as unknown as Bag).k = 1; +for (const [k, v] of [ + ["named", named], + ["tagged", tagged], + ["bare", bare], +] as [string, unknown][]) + console.log(k, format("%o", v)); diff --git a/tests/harness/dyncheck.test.ts b/tests/harness/dyncheck.test.ts index dab51a430..21e67d085 100644 --- a/tests/harness/dyncheck.test.ts +++ b/tests/harness/dyncheck.test.ts @@ -1311,6 +1311,72 @@ console.log("after"); expect(r.stderr).toContain(`Uncaught Error: ${refusal}`); }); + test("accessors reached through unknown render as Node's accessor tags", async () => { + const r = await compileAndRun( + "inspect-accessor-tags", + `import { format } from "node:util"; +const o: unknown = JSON.parse('{"x":1}'); +Object.defineProperty(o, "g", { get: () => 2, enumerable: true }); +Object.defineProperty(o, "h", { get: () => 3, enumerable: false }); +Object.defineProperty(o, "gs", { get: () => 4, set: (_v: number) => {}, enumerable: false }); +Object.defineProperty(o, "s", { set: (_v: number) => {}, enumerable: true }); +console.log(o); +console.log(format("%o", o)); +`, + ); + expect(r.exitCode).toBe(0); + expect(r.stdout).toBe( + [ + "{ x: 1, g: [Getter], s: [Setter] }", + "{", + " x: 1,", + " g: [Getter],", + " [h]: [Getter],", + " [gs]: [Getter/Setter],", + " s: [Setter]", + "}", + "", + ].join("\n"), + ); + }); + + test("a refused inspection leaves later renders at the default layout", async () => { + // The refusal's Error skips the call site's layout restore and the + // render's own exits; later renders must not keep the custom layout, + // the indentation, showHidden or open frames of the abandoned one. + const r = await compileAndRun( + "inspect-refusal-state", + `import { format, inspect } from "node:util"; +const p: unknown = Promise.resolve(1); +const rec: unknown = { a: [1], b: p }; +try { + inspect({ a: [1, 2, 3, 4, 5, 6, 7, 8], b: p }, { breakLength: 20 }); +} catch {} +try { + format("%o", rec); +} catch {} +const long: unknown = JSON.parse(JSON.stringify(Array.from({ length: 12 }, (_, i) => "item" + i))); +console.log(long); +console.log(format("%O", [1])); +`, + ); + expect(r.exitCode).toBe(0); + expect(r.stdout).toBe( + [ + "[", + " 'item0', 'item1',", + " 'item2', 'item3',", + " 'item4', 'item5',", + " 'item6', 'item7',", + " 'item8', 'item9',", + " 'item10', 'item11'", + "]", + "[ 1 ]", + "", + ].join("\n"), + ); + }); + test("inspecting a compiled class with static fields through unknown refuses", async () => { // Node prints `[class S] { count: 3 }`; the class object does not // carry compiled static fields, so the inspector refuses rather than From 91e15f34267365c7464c61b4ff42a2e67dce2c77 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 03:40:52 -0500 Subject: [PATCH 029/184] Run child_process and http2 events in Node's AsyncLocalStorage contexts child_process and http2 events ran with no store, or with the context of whichever socket dispatch delivered them. Node runs them in the context of the handle or request that produces them. child_process: a child, its piped stdout, stderr and stdin, and its IPC channel keep the context of the spawn() or fork() call. 'spawn', 'error', 'exit', 'close', stdout's and stderr's 'data' and 'end', and the channel's 'message' and 'disconnect' run in it, in the POSIX and the Windows implementations. stdin's 'drain' and 'error' run in the context of the latest write() call, its 'finish' in the context of its end() call, and a send() callback in the context of its send() call. http2: a session keeps the context it was created in, the http2.connect() call for a client and the server's listen() context for a server session, and the session's socket callbacks and deferred emits run in it. A client stream keeps the context of its request() call, and the frames that drive a stream's events (DATA, WINDOW_UPDATE, RST_STREAM, and a client's HEADERS) and its deferred emits run in the stream's context. A session remembers the stream its last lookup by id found, so the context lookup and the frame's own handling share one search. A server stream and its compatibility request and response run with no store, as Node's do. An accepted h2c socket's own events run with no store, as an accepted net socket's do, so a server.close() that waits for its sessions calls back with no store; a TLS server's sockets keep the listen() context, as before. Corpus 7385 to 7389. --- .../builtin-apis/asynclocalstorage.mdx | 4 +- .../generated/node-v24-internal.json | 61 ++++- internal/compatibility/static-support.json | 2 +- .../7385-als-child-process-events.ts.json | 6 + .../7386-als-child-fork-events/main.mjs.json | 6 + .../corpus/7387-als-http2-events.mjs.json | 6 + .../7388-als-http2-compat-events.mjs.json | 6 + .../7389-als-http2-secure-events.mjs.json | 6 + packages/runtime/src/scr_child.c | 221 +++++++++++++++++- packages/runtime/src/scr_http2.c | 95 +++++++- tests/corpus/7385-als-child-process-events.ts | 104 +++++++++ .../7386-als-child-fork-events/main.mjs | 25 ++ .../7386-als-child-fork-events/worker.mjs | 4 + tests/corpus/7387-als-http2-events.mjs | 91 ++++++++ tests/corpus/7388-als-http2-compat-events.mjs | 49 ++++ tests/corpus/7389-als-http2-secure-events.mjs | 94 ++++++++ .../7385-als-child-process-events.ts.json | 1 + .../7386-als-child-fork-events/main.mjs.json | 1 + .../7387-als-http2-events.mjs.json | 1 + .../7388-als-http2-compat-events.mjs.json | 1 + .../7389-als-http2-secure-events.mjs.json | 1 + 21 files changed, 766 insertions(+), 19 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7385-als-child-process-events.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7386-als-child-fork-events/main.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7387-als-http2-events.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7388-als-http2-compat-events.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7389-als-http2-secure-events.mjs.json create mode 100644 tests/corpus/7385-als-child-process-events.ts create mode 100644 tests/corpus/7386-als-child-fork-events/main.mjs create mode 100644 tests/corpus/7386-als-child-fork-events/worker.mjs create mode 100644 tests/corpus/7387-als-http2-events.mjs create mode 100644 tests/corpus/7388-als-http2-compat-events.mjs create mode 100644 tests/corpus/7389-als-http2-secure-events.mjs create mode 100644 tests/harness/corpus-costs/7385-als-child-process-events.ts.json create mode 100644 tests/harness/corpus-costs/7386-als-child-fork-events/main.mjs.json create mode 100644 tests/harness/corpus-costs/7387-als-http2-events.mjs.json create mode 100644 tests/harness/corpus-costs/7388-als-http2-compat-events.mjs.json create mode 100644 tests/harness/corpus-costs/7389-als-http2-secure-events.mjs.json diff --git a/docs/content/limitations/builtin-apis/asynclocalstorage.mdx b/docs/content/limitations/builtin-apis/asynclocalstorage.mdx index 016b712f6..ce77d57f5 100644 --- a/docs/content/limitations/builtin-apis/asynclocalstorage.mdx +++ b/docs/content/limitations/builtin-apis/asynclocalstorage.mdx @@ -6,5 +6,7 @@ Stream events and callbacks run with the stores of the code that triggers them, as in Node. Stream steps that Node defers with `process.nextTick` see the context that scheduled them, and `stream.finished` and `stream.pipeline` callbacks see the context of their call. `fs` read and write streams and `zlib` streams run their events and callbacks in the context that created them. - Events of `child_process` children and of `http2` sessions and streams do not yet run in the contexts Node gives them. + Events of `child_process` children, their standard streams and their IPC channels see the context of the `spawn()` or `fork()` call; stdin's `'finish'` and `send()` callbacks see the context of their call. + A client `http2` session's events see the context of its `http2.connect()` call, and a client stream's events the context of its `request()` call. + An `http2` server's `'session'` and `'stream'` events and the compatibility API's request handler see the context of its `listen()` call; a server stream's own events, and its compatibility request's and response's, run with no store, as in Node. `AsyncResource` has no lowering yet. diff --git a/internal/compatibility/generated/node-v24-internal.json b/internal/compatibility/generated/node-v24-internal.json index b5542944d..58a707f92 100644 --- a/internal/compatibility/generated/node-v24-internal.json +++ b/internal/compatibility/generated/node-v24-internal.json @@ -1394,7 +1394,12 @@ "tests/corpus/7119-als-net-event-context.mjs", "tests/corpus/7380-als-http-event-context.mjs", "tests/corpus/7383-als-stream-events.ts", - "tests/corpus/7384-als-stream-events.mjs" + "tests/corpus/7384-als-stream-events.mjs", + "tests/corpus/7385-als-child-process-events.ts", + "tests/corpus/7386-als-child-fork-events/main.mjs", + "tests/corpus/7387-als-http2-events.mjs", + "tests/corpus/7388-als-http2-compat-events.mjs", + "tests/corpus/7389-als-http2-secure-events.mjs" ] }, "anchorSource": "chapter" @@ -1451,7 +1456,12 @@ "tests/corpus/7119-als-net-event-context.mjs", "tests/corpus/7380-als-http-event-context.mjs", "tests/corpus/7383-als-stream-events.ts", - "tests/corpus/7384-als-stream-events.mjs" + "tests/corpus/7384-als-stream-events.mjs", + "tests/corpus/7385-als-child-process-events.ts", + "tests/corpus/7386-als-child-fork-events/main.mjs", + "tests/corpus/7387-als-http2-events.mjs", + "tests/corpus/7388-als-http2-compat-events.mjs", + "tests/corpus/7389-als-http2-secure-events.mjs" ] }, "anchorSource": "exact" @@ -1531,7 +1541,12 @@ "tests/corpus/7119-als-net-event-context.mjs", "tests/corpus/7380-als-http-event-context.mjs", "tests/corpus/7383-als-stream-events.ts", - "tests/corpus/7384-als-stream-events.mjs" + "tests/corpus/7384-als-stream-events.mjs", + "tests/corpus/7385-als-child-process-events.ts", + "tests/corpus/7386-als-child-fork-events/main.mjs", + "tests/corpus/7387-als-http2-events.mjs", + "tests/corpus/7388-als-http2-compat-events.mjs", + "tests/corpus/7389-als-http2-secure-events.mjs" ] }, "anchorSource": "exact" @@ -1565,7 +1580,12 @@ "tests/corpus/7119-als-net-event-context.mjs", "tests/corpus/7380-als-http-event-context.mjs", "tests/corpus/7383-als-stream-events.ts", - "tests/corpus/7384-als-stream-events.mjs" + "tests/corpus/7384-als-stream-events.mjs", + "tests/corpus/7385-als-child-process-events.ts", + "tests/corpus/7386-als-child-fork-events/main.mjs", + "tests/corpus/7387-als-http2-events.mjs", + "tests/corpus/7388-als-http2-compat-events.mjs", + "tests/corpus/7389-als-http2-secure-events.mjs" ] }, "anchorSource": "exact" @@ -1599,7 +1619,12 @@ "tests/corpus/7119-als-net-event-context.mjs", "tests/corpus/7380-als-http-event-context.mjs", "tests/corpus/7383-als-stream-events.ts", - "tests/corpus/7384-als-stream-events.mjs" + "tests/corpus/7384-als-stream-events.mjs", + "tests/corpus/7385-als-child-process-events.ts", + "tests/corpus/7386-als-child-fork-events/main.mjs", + "tests/corpus/7387-als-http2-events.mjs", + "tests/corpus/7388-als-http2-compat-events.mjs", + "tests/corpus/7389-als-http2-secure-events.mjs" ] }, "anchorSource": "exact" @@ -1633,7 +1658,12 @@ "tests/corpus/7119-als-net-event-context.mjs", "tests/corpus/7380-als-http-event-context.mjs", "tests/corpus/7383-als-stream-events.ts", - "tests/corpus/7384-als-stream-events.mjs" + "tests/corpus/7384-als-stream-events.mjs", + "tests/corpus/7385-als-child-process-events.ts", + "tests/corpus/7386-als-child-fork-events/main.mjs", + "tests/corpus/7387-als-http2-events.mjs", + "tests/corpus/7388-als-http2-compat-events.mjs", + "tests/corpus/7389-als-http2-secure-events.mjs" ] }, "anchorSource": "exact" @@ -1667,7 +1697,12 @@ "tests/corpus/7119-als-net-event-context.mjs", "tests/corpus/7380-als-http-event-context.mjs", "tests/corpus/7383-als-stream-events.ts", - "tests/corpus/7384-als-stream-events.mjs" + "tests/corpus/7384-als-stream-events.mjs", + "tests/corpus/7385-als-child-process-events.ts", + "tests/corpus/7386-als-child-fork-events/main.mjs", + "tests/corpus/7387-als-http2-events.mjs", + "tests/corpus/7388-als-http2-compat-events.mjs", + "tests/corpus/7389-als-http2-secure-events.mjs" ] }, "anchorSource": "exact" @@ -1978,6 +2013,11 @@ "tests/corpus/7380-als-http-event-context.mjs", "tests/corpus/7383-als-stream-events.ts", "tests/corpus/7384-als-stream-events.mjs", + "tests/corpus/7385-als-child-process-events.ts", + "tests/corpus/7386-als-child-fork-events/main.mjs", + "tests/corpus/7387-als-http2-events.mjs", + "tests/corpus/7388-als-http2-compat-events.mjs", + "tests/corpus/7389-als-http2-secure-events.mjs", "tests/harness/node-test.test.ts" ] }, @@ -2547,7 +2587,12 @@ "tests/corpus/7119-als-net-event-context.mjs", "tests/corpus/7380-als-http-event-context.mjs", "tests/corpus/7383-als-stream-events.ts", - "tests/corpus/7384-als-stream-events.mjs" + "tests/corpus/7384-als-stream-events.mjs", + "tests/corpus/7385-als-child-process-events.ts", + "tests/corpus/7386-als-child-fork-events/main.mjs", + "tests/corpus/7387-als-http2-events.mjs", + "tests/corpus/7388-als-http2-compat-events.mjs", + "tests/corpus/7389-als-http2-secure-events.mjs" ] }, "anchorSource": "exact" diff --git a/internal/compatibility/static-support.json b/internal/compatibility/static-support.json index 640345a02..a711bcf3c 100644 --- a/internal/compatibility/static-support.json +++ b/internal/compatibility/static-support.json @@ -429,7 +429,7 @@ { "symbols": ["crypto.hkdf", "crypto.hkdfSync", "crypto.scrypt", "crypto.scryptSync"], "status": "partial", "evidence": ["tests/corpus/node24-crypto-kdf.ts", "tests/corpus/node24-crypto-kdf-errors.ts", "tests/corpus/node24-crypto-kdf-options.ts", "tests/corpus/5438-crypto-kdf-digests.ts"] }, { "symbols": ["EventEmitter", "emitter.on", "emitter.addListener", "emitter.once", "emitter.prependListener", "emitter.prependOnceListener", "emitter.off", "emitter.removeListener", "emitter.removeAllListeners", "emitter.emit", "emitter.listenerCount", "emitter.listeners", "emitter.rawListeners", "emitter.eventNames", "emitter.setMaxListeners", "emitter.getMaxListeners", "events.once"], "status": "partial", "evidence": ["tests/corpus/1677-emitter-listeners.ts", "tests/corpus/2620-ee-override-once-order.ts"] }, { "symbols": ["events.getEventListeners", "events.getMaxListeners", "events.listenerCount", "events.setMaxListeners", "events.defaultMaxListeners"], "status": "partial", "evidence": ["tests/corpus/5885-events-max-listener-helpers.ts", "tests/corpus/5846-events-esm-default-max.ts", "tests/corpus/5847-events-cjs-default-max.cjs"] }, - { "symbols": ["AsyncLocalStorage", "asyncLocalStorage.run", "asyncLocalStorage.exit", "asyncLocalStorage.getStore", "asyncLocalStorage.enterWith", "asyncLocalStorage.disable", "asyncLocalStorage.name", "AsyncLocalStorage.bind", "AsyncLocalStorage.snapshot"], "status": "partial", "evidence": ["tests/corpus/2213-async-local-storage.cjs", "tests/corpus/5815-als-callback-context.ts", "tests/corpus/7110-als-completion-callbacks.ts", "tests/corpus/7111-als-bind-snapshot-options.ts", "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", "tests/corpus/7119-als-net-event-context.mjs", "tests/corpus/7380-als-http-event-context.mjs", "tests/corpus/7383-als-stream-events.ts", "tests/corpus/7384-als-stream-events.mjs"] }, + { "symbols": ["AsyncLocalStorage", "asyncLocalStorage.run", "asyncLocalStorage.exit", "asyncLocalStorage.getStore", "asyncLocalStorage.enterWith", "asyncLocalStorage.disable", "asyncLocalStorage.name", "AsyncLocalStorage.bind", "AsyncLocalStorage.snapshot"], "status": "partial", "evidence": ["tests/corpus/2213-async-local-storage.cjs", "tests/corpus/5815-als-callback-context.ts", "tests/corpus/7110-als-completion-callbacks.ts", "tests/corpus/7111-als-bind-snapshot-options.ts", "tests/corpus/7112-als-bind-snapshot-dynamic.mjs", "tests/corpus/7119-als-net-event-context.mjs", "tests/corpus/7380-als-http-event-context.mjs", "tests/corpus/7383-als-stream-events.ts", "tests/corpus/7384-als-stream-events.mjs", "tests/corpus/7385-als-child-process-events.ts", "tests/corpus/7386-als-child-fork-events/main.mjs", "tests/corpus/7387-als-http2-events.mjs", "tests/corpus/7388-als-http2-compat-events.mjs", "tests/corpus/7389-als-http2-secure-events.mjs"] }, { "symbols": ["StringDecoder", "stringDecoder.write", "stringDecoder.end"], "status": "partial", "evidence": ["tests/corpus/1662-string-decoder-encodings.ts", "tests/corpus/node24-string-decoder.ts", "tests/corpus/5882-string-decoder-runtime-encoding.ts"] }, { "symbols": ["setImmediate", "setInterval", "setTimeout", "clearImmediate", "clearInterval", "clearTimeout", "timersPromises.setTimeout", "timersPromises.setImmediate", "timersPromises.setInterval", "immediate.hasRef", "immediate.ref", "immediate.unref", "timeout.hasRef", "timeout.ref", "timeout.unref", "timeout.refresh"], "status": "partial", "evidence": ["tests/corpus/1804-timers-module.ts", "tests/corpus/1805-timer-callback-args.ts", "tests/corpus/2093-timers-promises.ts", "tests/corpus/2852-timers-promises-interval.ts", "tests/corpus/5830-timers-promises-signal.ts"] }, { "symbols": ["timersPromises.scheduler.wait", "timersPromises.scheduler.yield"], "status": "partial", "evidence": ["tests/corpus/5839-timers-promises-scheduler.ts", "tests/corpus/5889-timers-scheduler-signal.ts"] }, diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7385-als-child-process-events.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7385-als-child-process-events.ts.json new file mode 100644 index 000000000..aac67defb --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7385-als-child-process-events.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7385-als-child-process-events.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7386-als-child-fork-events/main.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7386-als-child-fork-events/main.mjs.json new file mode 100644 index 000000000..a334cf0aa --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7386-als-child-fork-events/main.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7386-als-child-fork-events/main.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7387-als-http2-events.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7387-als-http2-events.mjs.json new file mode 100644 index 000000000..272024d1b --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7387-als-http2-events.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7387-als-http2-events.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7388-als-http2-compat-events.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7388-als-http2-compat-events.mjs.json new file mode 100644 index 000000000..8a64f395e --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7388-als-http2-compat-events.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7388-als-http2-compat-events.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7389-als-http2-secure-events.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7389-als-http2-secure-events.mjs.json new file mode 100644 index 000000000..aec8e2d2a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7389-als-http2-secure-events.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7389-als-http2-secure-events.mjs" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_child.c b/packages/runtime/src/scr_child.c index ec685656d..19633f8dd 100644 --- a/packages/runtime/src/scr_child.c +++ b/packages/runtime/src/scr_child.c @@ -1404,6 +1404,10 @@ struct ScrChildStream { size_t n_data, cap_data; ScrChildStreamEndL *end_ls; size_t n_end, cap_end; + /* The AsyncLocalStorage context of the spawn() call that created the + * stream (+1, NULL outside a store): its events run in it, as Node's + * pipe handles run theirs. */ + ScrAlsCtx *als; struct ScrChildStream *next; /* the service registry (+1) */ }; @@ -1432,6 +1436,11 @@ struct ScrChildWriter { size_t n_finish, cap_finish; ScrChildErrEntry *err_ls; size_t n_write_err, cap_write_err; + /* AsyncLocalStorage contexts (+1 each, NULL outside a store): the + * latest write() call's, for 'drain' and 'error' (the spawn() call's + * until the first write), and the end() call's, for 'finish'. */ + ScrAlsCtx *als; + ScrAlsCtx *end_als; struct ScrChildWriter *next; /* the service registry (+1) */ }; @@ -1478,6 +1487,9 @@ struct ScrChild { ScrChildWriter *in_stream; ScrDyn *out_dyn, *err_dyn, *in_dyn; ScrIpc *ipc; + /* The AsyncLocalStorage context of the spawn() call (+1, NULL outside + * a store): 'spawn', 'error', 'exit' and 'close' run in it. */ + ScrAlsCtx *als; struct ScrChild *next; /* the pending registry */ }; @@ -1536,6 +1548,7 @@ void scr_child_release(ScrChild *c) { scr_child_stream_release(c->err_stream); scr_child_writer_release(c->in_stream); scr_ipc_release(c->ipc); + scr_als_ctx_release(c->als); if (c->proc != NULL) CloseHandle(c->proc); free(c); } @@ -1568,6 +1581,7 @@ void scr_child_stream_release(ScrChildStream *s) { if (s->h != NULL) CloseHandle(s->h); scr_str_release(s->encoding); free(s->capture_data); + scr_als_ctx_release(s->als); free(s); } } @@ -1587,6 +1601,7 @@ static ScrChildStream *scr_child_stream_new(HANDLE h) { ScrChildStream *s = calloc(1, sizeof *s); if (!s) scr_child_oom(); s->rc = 1; + s->als = scr_als_capture(); s->h = h; if (h == NULL) { s->eof = true; @@ -1633,8 +1648,16 @@ static ScrStr *scr_child_stream_capture_value(ScrChildStream *s) { : scr_str_new((const char *)s->capture_data, s->capture_len); } -static void scr_child_stream_emit(ScrChildStream *s, ScrBytes *bytes, - ScrStr *text) { +static void scr_child_stream_emit_in(ScrChildStream *s, ScrBytes *bytes, ScrStr *text); + +/* 'data' runs in the spawn() call's context. */ +static void scr_child_stream_emit(ScrChildStream *s, ScrBytes *bytes, ScrStr *text) { + ScrAlsCtx *prev = scr_als_dispatch_enter(s->als); + scr_child_stream_emit_in(s, bytes, text); + scr_als_dispatch_leave(prev); +} + +static void scr_child_stream_emit_in(ScrChildStream *s, ScrBytes *bytes, ScrStr *text) { size_t nd = s->n_data; if (nd == 0) return; ScrChildStreamDataL *snap = malloc(nd * sizeof *snap); @@ -1676,7 +1699,16 @@ static void scr_child_stream_emit(ScrChildStream *s, ScrBytes *bytes, free(snap); } +static void scr_child_stream_finish_in(ScrChildStream *s, bool fire_end); + +/* A stream's 'end' runs in the spawn() call's context. */ static void scr_child_stream_finish(ScrChildStream *s, bool fire_end) { + ScrAlsCtx *prev = scr_als_dispatch_enter(s->als); + scr_child_stream_finish_in(s, fire_end); + scr_als_dispatch_leave(prev); +} + +static void scr_child_stream_finish_in(ScrChildStream *s, bool fire_end) { if (fire_end && s->encoding != NULL && s->n_data > 0) { ScrStr *tail = scr_strdec_end(s->encoding, s->dec_pending); s->dec_pending = 0; @@ -1907,6 +1939,8 @@ void scr_child_writer_release(ScrChildWriter *w) { free(w->buf); scr_str_release(w->err_msg); scr_child_writer_drop_listeners(w); + scr_als_ctx_release(w->als); + scr_als_ctx_release(w->end_als); free(w); } } @@ -1922,6 +1956,7 @@ static ScrChildWriter *scr_child_writer_new(HANDLE h) { ScrChildWriter *w = calloc(1, sizeof *w); if (!w) scr_child_oom(); w->rc = 1; + w->als = scr_als_capture(); w->h = h; w->writable = h != NULL; w->destroyed = h == NULL; @@ -2068,6 +2103,8 @@ static void scr_child_writer_pump(ScrChildWriter *w) { static bool scr_child_writer_write(ScrChildWriter *w, const void *data, size_t len) { + scr_als_ctx_release(w->als); + w->als = scr_als_capture(); if (!w->writable) { scr_child_writer_fail(w, "write after end", "ERR_STREAM_WRITE_AFTER_END"); return false; @@ -2098,6 +2135,8 @@ bool scr_child_writer_write_bytes(ScrChildWriter *w, const ScrBytes *data) { void scr_child_writer_end(ScrChildWriter *w) { if (w->ending || w->finish_pending || w->finished || w->destroyed) return; + scr_als_ctx_release(w->end_als); + w->end_als = scr_als_capture(); w->writable = false; w->ending = true; scr_child_writer_pump(w); @@ -2165,7 +2204,16 @@ void scr_child_writer_on_error(ScrChildWriter *w, ScrClosure *cb /*moves*/, w->err_ls[w->n_write_err++] = (ScrChildErrEntry){cb, fn}; } +static void scr_child_writer_fire_drain_in(ScrChildWriter *w); + +/* 'drain' runs in the latest write() call's context. */ static void scr_child_writer_fire_drain(ScrChildWriter *w) { + ScrAlsCtx *prev = scr_als_dispatch_enter(w->als); + scr_child_writer_fire_drain_in(w); + scr_als_dispatch_leave(prev); +} + +static void scr_child_writer_fire_drain_in(ScrChildWriter *w) { w->drain_pending = false; size_t n = w->n_drain; if (n == 0) return; @@ -2192,7 +2240,16 @@ static void scr_child_writer_fire_drain(ScrChildWriter *w) { free(snap); } +static void scr_child_writer_fire_finish_in(ScrChildWriter *w); + +/* 'finish' runs in the end() call's context. */ static void scr_child_writer_fire_finish(ScrChildWriter *w) { + ScrAlsCtx *prev = scr_als_dispatch_enter(w->end_als); + scr_child_writer_fire_finish_in(w); + scr_als_dispatch_leave(prev); +} + +static void scr_child_writer_fire_finish_in(ScrChildWriter *w) { w->finish_pending = false; w->finished = true; w->destroyed = true; @@ -2211,7 +2268,16 @@ static void scr_child_writer_fire_finish(ScrChildWriter *w) { scr_child_writer_drop_zero_listeners(w); } +static void scr_child_writer_fire_error_in(ScrChildWriter *w); + +/* 'error' runs in the latest write() call's context. */ static void scr_child_writer_fire_error(ScrChildWriter *w) { + ScrAlsCtx *prev = scr_als_dispatch_enter(w->als); + scr_child_writer_fire_error_in(w); + scr_als_dispatch_leave(prev); +} + +static void scr_child_writer_fire_error_in(ScrChildWriter *w) { w->error_pending = false; if (w->n_write_err == 0) { fflush(stdout); @@ -2305,6 +2371,7 @@ ScrChild *scr_spawn_opts(ScrStr *cmd, ScrArr *args, double in_mode, ScrChild *c = calloc(1, sizeof(ScrChild)); if (!c) scr_child_oom(); c->rc = 1; + c->als = scr_als_capture(); /* Piped slots: the pipe NOW; a pipe failure degrades the slot to NUL * and the stream husk answers eof (fd exhaustion — nothing real). */ @@ -2703,7 +2770,16 @@ static bool scr_child_close_ready(const ScrChild *c) { scr_ipc_close_ready(c->ipc); } +static void scr_child_fire_close_in(ScrChild *c); + +/* 'close' runs in the spawn() call's context. */ static void scr_child_fire_close(ScrChild *c) { + ScrAlsCtx *prev = scr_als_dispatch_enter(c->als); + scr_child_fire_close_in(c); + scr_als_dispatch_leave(prev); +} + +static void scr_child_fire_close_in(ScrChild *c) { c->close_done = true; bool has_code = c->state == SCR_CHILD_SPAWN_FAILED ? true : c->has_code; double code = c->state == SCR_CHILD_SPAWN_FAILED @@ -2767,7 +2843,16 @@ bool scr_children_wait(double max_wait_ms) { /* Fires one settled child's terminal event — the POSIX arm's settle, * verbatim (streams drain first so 'end' precedes 'exit'; spawn * failures fire 'error' or die unhandled). */ +static void scr_child_settle_in(ScrChild *c); + +/* A child's terminal events run in its spawn() call's context. */ static void scr_child_settle(ScrChild *c) { + ScrAlsCtx *prev = scr_als_dispatch_enter(c->als); + scr_child_settle_in(c); + scr_als_dispatch_leave(prev); +} + +static void scr_child_settle_in(ScrChild *c) { if (c->reffed) { c->reffed = false; scr_children_reffed_n--; @@ -3815,6 +3900,10 @@ struct ScrChildStream { size_t n_data, cap_data; ScrChildStreamEndL *end_ls; size_t n_end, cap_end; + /* The AsyncLocalStorage context of the spawn() call that created the + * stream (+1, NULL outside a store): its events run in it, as Node's + * pipe handles run theirs. */ + ScrAlsCtx *als; struct ScrChildStream *next; /* the service registry (+1) */ }; @@ -3839,6 +3928,11 @@ struct ScrChildWriter { size_t n_finish, cap_finish; ScrChildErrEntry *err_ls; size_t n_write_err, cap_write_err; + /* AsyncLocalStorage contexts (+1 each, NULL outside a store): the + * latest write() call's, for 'drain' and 'error' (the spawn() call's + * until the first write), and the end() call's, for 'finish'. */ + ScrAlsCtx *als; + ScrAlsCtx *end_als; struct ScrChildWriter *next; /* the service registry (+1) */ }; @@ -3889,6 +3983,9 @@ struct ScrChild { ScrChildWriter *in_stream; ScrDyn *out_dyn, *err_dyn, *in_dyn; ScrIpc *ipc; + /* The AsyncLocalStorage context of the spawn() call (+1, NULL outside + * a store): 'spawn', 'error', 'exit' and 'close' run in it. */ + ScrAlsCtx *als; struct ScrChild *next; /* the pending registry */ }; @@ -4085,6 +4182,7 @@ void scr_child_release(ScrChild *c) { scr_child_stream_release(c->err_stream); scr_child_writer_release(c->in_stream); scr_ipc_release(c->ipc); + scr_als_ctx_release(c->als); free(c); } } @@ -4137,6 +4235,7 @@ void scr_child_stream_release(ScrChildStream *s) { if (s->fd >= 0) close(s->fd); scr_str_release(s->encoding); free(s->capture_data); + scr_als_ctx_release(s->als); free(s); } } @@ -4159,6 +4258,7 @@ static ScrChildStream *scr_child_stream_new(int fd) { ScrChildStream *s = calloc(1, sizeof *s); if (!s) scr_child_oom(); s->rc = 1; + s->als = scr_als_capture(); s->fd = fd; if (fd < 0) { s->eof = true; @@ -4213,8 +4313,16 @@ static ScrStr *scr_child_stream_capture_value(ScrChildStream *s) { : scr_str_new((const char *)s->capture_data, s->capture_len); } -static void scr_child_stream_emit_posix(ScrChildStream *s, ScrBytes *bytes, - ScrStr *text) { +static void scr_child_stream_emit_posix_in(ScrChildStream *s, ScrBytes *bytes, ScrStr *text); + +/* 'data' runs in the spawn() call's context. */ +static void scr_child_stream_emit_posix(ScrChildStream *s, ScrBytes *bytes, ScrStr *text) { + ScrAlsCtx *prev = scr_als_dispatch_enter(s->als); + scr_child_stream_emit_posix_in(s, bytes, text); + scr_als_dispatch_leave(prev); +} + +static void scr_child_stream_emit_posix_in(ScrChildStream *s, ScrBytes *bytes, ScrStr *text) { size_t nd = s->n_data; if (nd == 0) return; ScrChildStreamDataL *snap = malloc(nd * sizeof *snap); @@ -4264,7 +4372,16 @@ static void scr_child_stream_emit_posix(ScrChildStream *s, ScrBytes *bytes, /* EOF/error tail: 'end' fires over a snapshot, everything drops, the fd * closes (deleting its kqueue filter), and the liveness/fallback counters * settle. The caller unlinks from the registry and releases its ref. */ +static void scr_child_stream_finish_in(ScrChildStream *s, bool fire_end); + +/* A stream's 'end' runs in the spawn() call's context. */ static void scr_child_stream_finish(ScrChildStream *s, bool fire_end) { + ScrAlsCtx *prev = scr_als_dispatch_enter(s->als); + scr_child_stream_finish_in(s, fire_end); + scr_als_dispatch_leave(prev); +} + +static void scr_child_stream_finish_in(ScrChildStream *s, bool fire_end) { if (fire_end && s->encoding != NULL && s->n_data > 0) { ScrStr *tail = scr_strdec_end(s->encoding, s->dec_pending); s->dec_pending = 0; @@ -4572,6 +4689,8 @@ void scr_child_writer_release(ScrChildWriter *w) { free(w->buf); scr_str_release(w->err_msg); scr_child_writer_drop_listeners(w); + scr_als_ctx_release(w->als); + scr_als_ctx_release(w->end_als); free(w); } } @@ -4587,6 +4706,7 @@ static ScrChildWriter *scr_child_writer_new(int fd) { ScrChildWriter *w = calloc(1, sizeof *w); if (!w) scr_child_oom(); w->rc = 1; + w->als = scr_als_capture(); w->fd = fd; w->writable = fd >= 0; w->destroyed = fd < 0; @@ -4720,6 +4840,8 @@ static void scr_child_writer_pump(ScrChildWriter *w) { static bool scr_child_writer_write(ScrChildWriter *w, const void *data, size_t len) { + scr_als_ctx_release(w->als); + w->als = scr_als_capture(); if (!w->writable) { scr_child_writer_fail(w, "write after end", "ERR_STREAM_WRITE_AFTER_END"); return false; @@ -4756,6 +4878,8 @@ bool scr_child_writer_write_bytes(ScrChildWriter *w, const ScrBytes *data) { void scr_child_writer_end(ScrChildWriter *w) { if (w->ending || w->finish_pending || w->finished || w->destroyed) return; + scr_als_ctx_release(w->end_als); + w->end_als = scr_als_capture(); w->writable = false; w->ending = true; scr_child_writer_pump(w); @@ -4823,7 +4947,16 @@ void scr_child_writer_on_error(ScrChildWriter *w, ScrClosure *cb /*moves*/, w->err_ls[w->n_write_err++] = (ScrChildErrEntry){cb, fn}; } +static void scr_child_writer_fire_drain_in(ScrChildWriter *w); + +/* 'drain' runs in the latest write() call's context. */ static void scr_child_writer_fire_drain(ScrChildWriter *w) { + ScrAlsCtx *prev = scr_als_dispatch_enter(w->als); + scr_child_writer_fire_drain_in(w); + scr_als_dispatch_leave(prev); +} + +static void scr_child_writer_fire_drain_in(ScrChildWriter *w) { w->drain_pending = false; size_t n = w->n_drain; if (n == 0) return; @@ -4850,7 +4983,16 @@ static void scr_child_writer_fire_drain(ScrChildWriter *w) { free(snap); } +static void scr_child_writer_fire_finish_in(ScrChildWriter *w); + +/* 'finish' runs in the end() call's context. */ static void scr_child_writer_fire_finish(ScrChildWriter *w) { + ScrAlsCtx *prev = scr_als_dispatch_enter(w->end_als); + scr_child_writer_fire_finish_in(w); + scr_als_dispatch_leave(prev); +} + +static void scr_child_writer_fire_finish_in(ScrChildWriter *w) { w->finish_pending = false; w->finished = true; w->destroyed = true; @@ -4869,7 +5011,16 @@ static void scr_child_writer_fire_finish(ScrChildWriter *w) { scr_child_writer_drop_zero_listeners(w); } +static void scr_child_writer_fire_error_in(ScrChildWriter *w); + +/* 'error' runs in the latest write() call's context. */ static void scr_child_writer_fire_error(ScrChildWriter *w) { + ScrAlsCtx *prev = scr_als_dispatch_enter(w->als); + scr_child_writer_fire_error_in(w); + scr_als_dispatch_leave(prev); +} + +static void scr_child_writer_fire_error_in(ScrChildWriter *w) { w->error_pending = false; if (w->n_write_err == 0) { fflush(stdout); @@ -4973,6 +5124,7 @@ ScrChild *scr_spawn_opts(ScrStr *cmd, ScrArr *args, double in_mode, ScrChild *c = calloc(1, sizeof(ScrChild)); if (!c) scr_child_oom(); c->rc = 1; + c->als = scr_als_capture(); /* Piped slots: pipe(2) NOW, both ends cloexec (the file action's dup2 * clears the flag on the child's copy — every other descriptor closes @@ -5489,7 +5641,16 @@ static bool scr_child_close_ready(const ScrChild *c) { scr_ipc_close_ready(c->ipc); } +static void scr_child_fire_close_in(ScrChild *c); + +/* 'close' runs in the spawn() call's context. */ static void scr_child_fire_close(ScrChild *c) { + ScrAlsCtx *prev = scr_als_dispatch_enter(c->als); + scr_child_fire_close_in(c); + scr_als_dispatch_leave(prev); +} + +static void scr_child_fire_close_in(ScrChild *c) { c->close_done = true; bool has_code = c->state == SCR_CHILD_SPAWN_FAILED ? true : c->has_code; double code = c->state == SCR_CHILD_SPAWN_FAILED @@ -5527,7 +5688,16 @@ static void scr_children_closing_poll(void) { * ordering runs first: consumer-owning piped streams drain to EOF/EAGAIN * so their 'end' events fire BEFORE 'exit' (Node-exact — verified: data*, * end, then exit for every output shape). */ +static void scr_child_settle_in(ScrChild *c); + +/* A child's terminal events run in its spawn() call's context. */ static void scr_child_settle(ScrChild *c) { + ScrAlsCtx *prev = scr_als_dispatch_enter(c->als); + scr_child_settle_in(c); + scr_als_dispatch_leave(prev); +} + +static void scr_child_settle_in(ScrChild *c) { if (c->reffed) { c->reffed = false; scr_children_reffed_n--; @@ -5695,7 +5865,16 @@ void scr_child_on_spawn(ScrChild *c, ScrClosure *cb) { /* Success is observable on the next loop pass, before any data or exit * event. Detaching the snapshot keeps listeners registered by a callback * from participating in an event that has already started. */ +static void scr_child_fire_spawn_in(ScrChild *c); + +/* 'spawn' runs in the spawn() call's context. */ static void scr_child_fire_spawn(ScrChild *c) { + ScrAlsCtx *prev = scr_als_dispatch_enter(c->als); + scr_child_fire_spawn_in(c); + scr_als_dispatch_leave(prev); +} + +static void scr_child_fire_spawn_in(ScrChild *c) { c->spawn_done = true; ScrClosure **callbacks = c->spawn_cbs; size_t count = c->n_spawn; @@ -5901,6 +6080,7 @@ typedef struct { typedef struct { ScrClosure *cb; ScrIpcSendFn fn; + ScrAlsCtx *ctx; /* the send() call's context (+1): the callback runs in it */ } ScrIpcSendEntry; struct ScrIpc { @@ -5928,6 +6108,9 @@ struct ScrIpc { ScrIpcSendEntry *send_cbs; size_t n_send, cap_send; ScrStr *send_error; + /* The context the channel was created in (+1): 'message' and + * 'disconnect' run in it. */ + ScrAlsCtx *als; struct ScrIpc *next; }; @@ -5963,8 +6146,12 @@ static void scr_ipc_release(ScrIpc *ipc) { for (size_t i = 0; i < ipc->n_pending; i++) scr_dyn_release(ipc->pending[i]); for (size_t i = 0; i < ipc->n_message; i++) scr_closure_release(ipc->message_ls[i].cb); for (size_t i = 0; i < ipc->n_disconnect; i++) scr_closure_release(ipc->disconnect_ls[i].cb); - for (size_t i = 0; i < ipc->n_send; i++) scr_closure_release(ipc->send_cbs[i].cb); + for (size_t i = 0; i < ipc->n_send; i++) { + scr_closure_release(ipc->send_cbs[i].cb); + scr_als_ctx_release(ipc->send_cbs[i].ctx); + } scr_str_release(ipc->send_error); + scr_als_ctx_release(ipc->als); free(ipc->buffer); free(ipc->pending); free(ipc->message_ls); @@ -6084,6 +6271,7 @@ static ScrIpc *scr_ipc_new(ScrChildStream *reader, ScrChildWriter *writer) { ScrIpc *ipc = calloc(1, sizeof(*ipc)); if (!ipc) scr_trap("out of memory"); ipc->rc = 1; + ipc->als = scr_als_capture(); ipc->connected = true; ipc->reader = reader; ipc->writer = writer; @@ -6114,7 +6302,7 @@ static void scr_ipc_add_send(ScrIpc *ipc, ScrClosure *cb, ScrIpcSendFn fn) { ipc->send_cbs = realloc(ipc->send_cbs, ipc->cap_send * sizeof(*ipc->send_cbs)); if (!ipc->send_cbs) scr_trap("out of memory"); } - ipc->send_cbs[ipc->n_send++] = (ScrIpcSendEntry){cb, fn}; + ipc->send_cbs[ipc->n_send++] = (ScrIpcSendEntry){cb, fn, scr_als_capture()}; } static bool scr_ipc_send(ScrIpc *ipc, ScrStr *json, ScrClosure *cb, @@ -6198,7 +6386,16 @@ static void scr_ipc_remove_message_once(ScrIpc *ipc, ScrClosure *cb) { } } +static void scr_ipc_fire_messages_in(ScrIpc *ipc); + +/* 'message' runs in the channel's creation context. */ static void scr_ipc_fire_messages(ScrIpc *ipc) { + ScrAlsCtx *prev = scr_als_dispatch_enter(ipc->als); + scr_ipc_fire_messages_in(ipc); + scr_als_dispatch_leave(prev); +} + +static void scr_ipc_fire_messages_in(ScrIpc *ipc) { while (ipc->n_pending > 0 && ipc->n_message > 0 && !scr_exc_pending()) { ScrDyn *message = ipc->pending[0]; memmove(ipc->pending, ipc->pending + 1, @@ -6235,16 +6432,28 @@ static void scr_ipc_fire_send_callbacks(ScrIpc *ipc) { error = scr_error_new(0, ipc->send_error); scr_error_set_code(error, "ERR_IPC_CHANNEL_CLOSED"); } + ScrAlsCtx *prev = scr_als_dispatch_enter(snapshot[i].ctx); if (!scr_exc_pending()) snapshot[i].fn(snapshot[i].cb, error); else scr_error_release(error); + scr_als_dispatch_leave(prev); scr_closure_release(snapshot[i].cb); + scr_als_ctx_release(snapshot[i].ctx); } free(snapshot); scr_str_release(ipc->send_error); ipc->send_error = NULL; } +static void scr_ipc_fire_disconnect_in(ScrIpc *ipc); + +/* 'disconnect' runs in the channel's creation context. */ static void scr_ipc_fire_disconnect(ScrIpc *ipc) { + ScrAlsCtx *prev = scr_als_dispatch_enter(ipc->als); + scr_ipc_fire_disconnect_in(ipc); + scr_als_dispatch_leave(prev); +} + +static void scr_ipc_fire_disconnect_in(ScrIpc *ipc) { if (!ipc->disconnect_pending || ipc->disconnect_emitted) return; ipc->disconnect_pending = false; ipc->disconnect_emitted = true; diff --git a/packages/runtime/src/scr_http2.c b/packages/runtime/src/scr_http2.c index 4c5267e6a..55d880c01 100644 --- a/packages/runtime/src/scr_http2.c +++ b/packages/runtime/src/scr_http2.c @@ -678,6 +678,10 @@ struct ScrH2Stream { int64_t send_window; ScrNetLs data_ls, end_ls, close_ls, err_ls, resp_ls, aborted_ls, ready_ls, trailers_ls, wanttrailers_ls, push_ls, frameerr_ls; + /* The AsyncLocalStorage context of the request() call that created a + * client stream (+1, NULL outside a store); NULL on server streams, + * whose own events Node runs with no store. */ + ScrAlsCtx *als; }; /* The server-side ctx (the ScrHttpSrvCtx shape; `proto` FIRST — the @@ -757,6 +761,14 @@ struct ScrH2Session { bool error_emitted; ScrNetLs stream_ls, close_ls, err_ls, connect_ls, goaway_ls, remote_settings_ls, local_settings_ls; + /* The stream the last lookup by id found (borrowed from `streams`): + * the frames of one stream arrive together, and a frame's context + * lookup and its handling ask for the same stream. */ + ScrH2Stream *last_found; + /* The AsyncLocalStorage context the session was created in (+1, NULL + * outside a store): the connect() call's for a client session, the + * server's listen() context for a server session. */ + ScrAlsCtx *als; }; /* ── refcounts + the RC audit ────────────────────────────────────────── */ @@ -838,6 +850,7 @@ static void scr_h2_stream_free(ScrH2Stream *st) { } free(st->wbuf.data); scr_http2_session_release(st->sess); + scr_als_ctx_release(st->als); free(st); #ifdef SCR_RC_AUDIT scr_h2_streams_live--; @@ -864,6 +877,7 @@ static void scr_h2_session_free(ScrH2Session *s) { scr_h2_dyntab_free(&s->dec_table); if (s->srv) scr_h2_srv_ctx_release(s->srv); if (s->sock) scr_net_sock_release(s->sock); + scr_als_ctx_release(s->als); free(s); #ifdef SCR_RC_AUDIT scr_h2_sessions_live--; @@ -1011,8 +1025,9 @@ static void scr_h2_send_headers(ScrH2Session *s, int32_t stream_id, const ScrH2B /* ── stream bookkeeping ──────────────────────────────────────────────── */ static ScrH2Stream *scr_h2_find_stream(ScrH2Session *s, int32_t id) { + if (s->last_found != NULL && s->last_found->id == id) return s->last_found; for (size_t i = 0; i < s->nstreams; i++) { - if (s->streams[i]->id == id) return s->streams[i]; + if (s->streams[i]->id == id) return s->last_found = s->streams[i]; } return NULL; } @@ -1023,6 +1038,7 @@ static ScrH2Stream *scr_h2_stream_new(ScrH2Session *sess, int32_t id) { st->rc = 1; st->id = id; st->sess = scr_http2_session_retain(sess); + if (sess->client) st->als = scr_als_capture(); st->send_window = sess->peer_init_window ? (int64_t)sess->peer_init_window : SCR_H2_DEFAULT_WINDOW; #ifdef SCR_RC_AUDIT scr_h2_streams_live++; @@ -1039,6 +1055,7 @@ static ScrH2Stream *scr_h2_stream_new(ScrH2Session *sess, int32_t id) { /* Drop the session's ref on a settled stream (its 'close' is queued or * emitted — the vector entry is what kept it discoverable by id). */ static void scr_h2_session_forget_stream(ScrH2Session *s, ScrH2Stream *st) { + if (s->last_found == st) s->last_found = NULL; for (size_t i = 0; i < s->nstreams; i++) { if (s->streams[i] == st) { memmove(s->streams + i, s->streams + i + 1, (s->nstreams - i - 1) * sizeof *s->streams); @@ -1150,6 +1167,10 @@ static bool scr_h2_proto_pending(void) { return scr_h2_emits_head < scr_h2_emits static void scr_h2_proto_sweep(void) { while (scr_h2_emits_head < scr_h2_emits_len) { ScrH2Emit e = scr_h2_emits[scr_h2_emits_head++]; + ScrAlsCtx *prev = scr_als_dispatch_enter( + e.kind == SCR_H2_EMIT_STREAM_CLOSE || e.kind == SCR_H2_EMIT_STREAM_DRAIN + ? ((ScrH2Stream *)e.h)->als + : ((ScrH2Session *)e.h)->als); switch (e.kind) { case SCR_H2_EMIT_STREAM_CLOSE: { ScrH2Stream *st = (ScrH2Stream *)e.h; @@ -1227,6 +1248,7 @@ static void scr_h2_proto_sweep(void) { break; } } + scr_als_dispatch_leave(prev); scr_h2_emit_release(&e); if (scr_exc_pending()) return; } @@ -1380,8 +1402,29 @@ static void scr_h2_fire_settings(ScrNetLs *ls, ScrH2Session *s, const ScrH2Setti } /* One complete frame. Returns false when the session tore down. */ +static bool scr_h2_on_frame_in(ScrH2Session *s, uint8_t type, uint8_t flags, int32_t stream_id, + const uint8_t *p, size_t n); + +/* A frame that drives an open stream's events (DATA, WINDOW_UPDATE, + * RST_STREAM, and a client's HEADERS, CONTINUATION and PUSH_PROMISE) runs + * in that stream's context; other frames, a server stream's opening + * HEADERS included, in the session's, which the socket callbacks below + * enter. */ static bool scr_h2_on_frame(ScrH2Session *s, uint8_t type, uint8_t flags, int32_t stream_id, const uint8_t *p, size_t n) { + bool stream_frame = type == SCR_H2_F_DATA || type == SCR_H2_F_WINDOW_UPDATE || + type == SCR_H2_F_RST_STREAM || + (s->client && (type == SCR_H2_F_HEADERS || type == SCR_H2_F_CONTINUATION || + type == SCR_H2_F_PUSH_PROMISE)); + ScrH2Stream *st = stream_frame && stream_id != 0 ? scr_h2_find_stream(s, stream_id) : NULL; + ScrAlsCtx *prev = st != NULL ? scr_als_dispatch_enter(st->als) : NULL; + bool ok = scr_h2_on_frame_in(s, type, flags, stream_id, p, n); + if (st != NULL) scr_als_dispatch_leave(prev); + return ok; +} + +static bool scr_h2_on_frame_in(ScrH2Session *s, uint8_t type, uint8_t flags, int32_t stream_id, + const uint8_t *p, size_t n) { /* An open header block admits only its CONTINUATIONs. */ if (s->hblock_active && type != SCR_H2_F_CONTINUATION) { goto proto_err; @@ -1629,7 +1672,16 @@ proto_err: return false; } +static void scr_h2_on_data_in(void *ctx, const char *buf, size_t n); + +/* The socket's callbacks run in the session's context. */ static void scr_h2_on_data(void *ctx, const char *buf, size_t n) { + ScrAlsCtx *prev = scr_als_dispatch_enter(((ScrH2Session *)ctx)->als); + scr_h2_on_data_in(ctx, buf, n); + scr_als_dispatch_leave(prev); +} + +static void scr_h2_on_data_in(void *ctx, const char *buf, size_t n) { ScrH2Session *s = ctx; if (s->destroyed) return; /* append to the reassembly buffer */ @@ -1680,7 +1732,15 @@ done: scr_http2_session_release(s); } +static void scr_h2_on_eof_in(void *ctx); + static void scr_h2_on_eof(void *ctx) { + ScrAlsCtx *prev = scr_als_dispatch_enter(((ScrH2Session *)ctx)->als); + scr_h2_on_eof_in(ctx); + scr_als_dispatch_leave(prev); +} + +static void scr_h2_on_eof_in(void *ctx) { ScrH2Session *s = ctx; if (s->destroyed) return; /* The peer is done sending. After a graceful GOAWAY exchange this is @@ -1688,7 +1748,15 @@ static void scr_h2_on_eof(void *ctx) { scr_h2_session_abort_streams(s); } +static void scr_h2_on_closed_in(void *ctx); + static void scr_h2_on_closed(void *ctx) { + ScrAlsCtx *prev = scr_als_dispatch_enter(((ScrH2Session *)ctx)->als); + scr_h2_on_closed_in(ctx); + scr_als_dispatch_leave(prev); +} + +static void scr_h2_on_closed_in(void *ctx) { ScrH2Session *s = ctx; s->destroyed = true; scr_h2_session_abort_streams(s); @@ -1700,7 +1768,16 @@ static void scr_h2_on_closed(void *ctx) { } } +static bool scr_h2_on_err_in(void *ctx, ScrStr *msg); + static bool scr_h2_on_err(void *ctx, ScrStr *msg) { + ScrAlsCtx *prev = scr_als_dispatch_enter(((ScrH2Session *)ctx)->als); + bool consumed = scr_h2_on_err_in(ctx, msg); + scr_als_dispatch_leave(prev); + return consumed; +} + +static bool scr_h2_on_err_in(void *ctx, ScrStr *msg) { ScrH2Session *s = ctx; if (s->err_ls.n == 0 && (s->srv == NULL || s->srv->session_error_ls.n == 0)) { return false; /* the socket's error story applies */ @@ -1716,6 +1793,7 @@ static ScrH2Session *scr_h2_session_new(bool client) { ScrH2Session *s = calloc(1, sizeof *s); if (!s) scr_h2_oom(); s->rc = 1; + s->als = scr_als_capture(); s->client = client; s->preface_done = client; /* clients never RECEIVE the magic */ s->next_stream_id = 1; @@ -2287,6 +2365,8 @@ static void scr_h2_upgrade_hook(void *ctx, ScrClosure *cb /*moves*/, void *fn, b * 'request' listeners (this = the net server, the http/1 shape). */ static void scr_h2_compat_dispatch(ScrH2Session *s, ScrH2Stream *st, const ScrH2Headers *h, bool end_stream, ScrNetLs *listeners) { + /* The compat pair's own events follow the stream's context. */ + ScrAlsCtx *prev = scr_als_dispatch_enter(st->als); ScrHttpReq *req = scr_http_h2_req_new(s->sock, st); for (size_t i = 0; i < h->n; i++) { ScrStr *name = h->names[i]; @@ -2301,6 +2381,7 @@ static void scr_h2_compat_dispatch(ScrH2Session *s, ScrH2Stream *st, const ScrH2 scr_http_h2_req_header(req, name, value); } ScrHttpRes *res = scr_http_h2_res_new(s->sock, st); + scr_als_dispatch_leave(prev); scr_http_res_set_req(res, req); /* res.req — the compat pair's backref */ st->compat_req = scr_http_req_retain(req); st->compat_res = scr_http_res_retain(res); @@ -2342,7 +2423,7 @@ static void scr_h2_srv_ctx_settle(void *p) { static void scr_h2_on_connection(void *ctx, ScrNetSocket *sock) { ScrH2SrvCtx *srv = (ScrH2SrvCtx *)ctx; - ScrH2Session *sess = scr_h2_session_new(false); + ScrH2Session *sess = scr_h2_session_new(false); /* in the listen() context */ sess->srv = scr_h2_srv_ctx_retain(srv); sess->sock = scr_net_sock_retain(sock); sess->next_stream_id = 2; /* server-initiated ids are even (push, unused) */ @@ -2354,6 +2435,14 @@ static void scr_h2_on_connection(void *ctx, ScrNetSocket *sock) { scr_h2_emit_push(SCR_H2_EMIT_SERVER_SESSION, scr_http2_session_retain(sess)); } +/* An accepted h2c connection: the socket's own events run with no store, + * as an accepted net socket's do (a TLS server's sockets keep the listen() + * context); the session keeps the listen() context the accept runs in. */ +static void scr_h2_on_plain_connection(void *ctx, ScrNetSocket *sock) { + scr_net_sock_set_context(sock, NULL); + scr_h2_on_connection(ctx, sock); +} + ScrNetServer *scr_http2_create_server(void) { scr_http2_install(); ScrNetServer *s = scr_net_create_server(NULL, NULL); @@ -2361,7 +2450,7 @@ ScrNetServer *scr_http2_create_server(void) { if (!ctx) scr_h2_oom(); ctx->proto = SCR_NET_PROTO_H2; ctx->rc = 1; - scr_net_server_set_native_conn(s, &scr_h2_on_connection, ctx, &scr_h2_srv_ctx_free_v); + scr_net_server_set_native_conn(s, &scr_h2_on_plain_connection, ctx, &scr_h2_srv_ctx_free_v); scr_net_server_set_http_ctx(s, ctx); scr_net_server_set_proto_settle(s, &scr_h2_srv_ctx_settle); return s; diff --git a/tests/corpus/7385-als-child-process-events.ts b/tests/corpus/7385-als-child-process-events.ts new file mode 100644 index 000000000..54fd80c3a --- /dev/null +++ b/tests/corpus/7385-als-child-process-events.ts @@ -0,0 +1,104 @@ +// AsyncLocalStorage contexts of node:child_process events. A child's +// 'spawn', 'error', 'exit' and 'close' events and its stdout and stderr +// 'data' and 'end' events run in the context of the spawn() call, +// whatever context registered the listener or killed the child. stdin's +// 'finish' runs in the context of the end() call, and its 'drain' and +// 'error' in the context of the write() call. Lines are sorted at the end: +// the stdout and stderr events of one child may interleave. +import { AsyncLocalStorage } from "node:async_hooks"; +import { spawn } from "node:child_process"; + +const store = new AsyncLocalStorage(); +const lines: string[] = []; +const log = (what: string): void => { + lines.push(`${what}: ${store.getStore() ?? "none"}`); +}; +const step = (fn: (done: () => void) => void): Promise => + new Promise((resolve) => fn(resolve)); + +await step((done) => { + const child = store.run("spawn", () => spawn("/bin/sh", ["-c", "printf out; printf err >&2"])); + store.run("listen", () => { + child.on("spawn", () => log("spawn event")); + child.stdout!.on("data", () => log("stdout data")); + child.stdout!.on("end", () => log("stdout end")); + child.stderr!.on("data", () => log("stderr data")); + child.on("exit", () => log("exit")); + child.on("close", () => { + log("close"); + done(); + }); + }); +}); + +await step((done) => { + const child = store.run("spawn-cat", () => spawn("/bin/cat")); + child.stdout!.on("data", () => log("cat data")); + store.run("stdin-write", () => { + child.stdin!.write("x"); + }); + child.stdin!.on("finish", () => log("stdin finish")); + store.run("stdin-end", () => { + child.stdin!.end(); + }); + child.on("close", () => { + log("cat close"); + done(); + }); +}); + +await step((done) => { + const child = store.run("spawn-missing", () => spawn("/nonexistent/command")); + child.on("error", () => log("spawn error")); + child.on("close", () => { + log("missing close"); + done(); + }); +}); + +await step((done) => { + const child = store.run("spawn-sleep", () => spawn("/bin/sleep", ["5"])); + child.on("exit", () => log("killed exit")); + child.on("close", () => { + log("killed close"); + done(); + }); + setTimeout(() => store.run("kill", () => child.kill()), 20); +}); + +// A write that fills the pipe: 'drain' runs in the write() call's context. +await step((done) => { + const child = store.run("spawn-slow", () => + spawn("/bin/sh", ["-c", "sleep 0.2; cat > /dev/null"]), + ); + child.stdin!.on("drain", () => { + log("stdin drain"); + child.stdin!.end(); + }); + child.on("close", () => { + log("slow close"); + done(); + }); + store.run("large-write", () => { + child.stdin!.write(Buffer.alloc(1 << 22, 97)); + }); +}); + +// A write to a child that closed its stdin: 'error' runs in the write() +// call's context. +await step((done) => { + const child = store.run("spawn-closed", () => spawn("/bin/sh", ["-c", "exec 0<&-; sleep 0.1"])); + child.stdin!.on("error", () => log("stdin error")); + child.on("close", () => { + log("closed close"); + done(); + }); + setTimeout(() => { + store.run("late-write", () => { + child.stdin!.write(Buffer.alloc(1 << 20, 97)); + }); + }, 50); +}); + +lines.sort(); +for (const line of lines) console.log(line); diff --git a/tests/corpus/7386-als-child-fork-events/main.mjs b/tests/corpus/7386-als-child-fork-events/main.mjs new file mode 100644 index 000000000..18b410643 --- /dev/null +++ b/tests/corpus/7386-als-child-fork-events/main.mjs @@ -0,0 +1,25 @@ +// AsyncLocalStorage contexts of a forked child's IPC events. The child's +// 'message', 'disconnect', 'exit' and 'close' events run in the context of +// the fork() call, whatever context registered the listener; a send() +// callback runs in the context of the send() call. +import { AsyncLocalStorage } from "node:async_hooks"; +import childProcess from "node:child_process"; + +const store = new AsyncLocalStorage(); +const lines = []; +const log = (what) => lines.push(`${what}: ${store.getStore() ?? "none"}`); + +const child = store.run("fork", () => + childProcess.fork(new URL("./worker.mjs", import.meta.url), []), +); +store.run("listen", () => { + child.on("message", (message) => log(`message ${JSON.stringify(message)}`)); +}); +child.on("disconnect", () => log("disconnect")); +store.run("send", () => child.send("hi", () => log("send callback"))); +setTimeout(() => store.run("send-bye", () => child.send("bye")), 50); +child.on("exit", () => log("exit")); +child.on("close", () => { + log("close"); + for (const line of lines) console.log(line); +}); diff --git a/tests/corpus/7386-als-child-fork-events/worker.mjs b/tests/corpus/7386-als-child-fork-events/worker.mjs new file mode 100644 index 000000000..79f4bfdc6 --- /dev/null +++ b/tests/corpus/7386-als-child-fork-events/worker.mjs @@ -0,0 +1,4 @@ +process.on("message", (m) => { + process.send({ echo: m }); + if (m === "bye") process.disconnect(); +}); diff --git a/tests/corpus/7387-als-http2-events.mjs b/tests/corpus/7387-als-http2-events.mjs new file mode 100644 index 000000000..4dd17916e --- /dev/null +++ b/tests/corpus/7387-als-http2-events.mjs @@ -0,0 +1,91 @@ +// AsyncLocalStorage contexts of node:http2 events. A client session's +// 'connect' and 'close' run in the context of the http2.connect() call, +// and a client stream's 'response', 'data', 'end' and 'close' in the +// context of the request() call. A server's 'session' and 'stream' events +// run in the context of its listen() call; a server stream's own 'data', +// 'end' and 'close' events run with no store, as does a server.close() +// callback that waits for the sessions. Lines are sorted at the end: +// events on the two sides may interleave. +import { AsyncLocalStorage } from "node:async_hooks"; +import http2 from "node:http2"; + +const store = new AsyncLocalStorage(); +const lines = []; +const log = (what) => lines.push(`${what}: ${store.getStore() ?? "none"}`); + +/** @returns {Promise} */ +function requestScenario() { + return new Promise((resolve) => { + const server = store.run("create", () => http2.createServer()); + server.on("session", () => log("server session")); + server.on("stream", (stream) => { + log("server stream"); + stream.on("data", () => log("server stream data")); + stream.on("end", () => { + log("server stream end"); + store.run("respond", () => { + stream.respond({ ":status": 200 }); + stream.end("ok"); + }); + }); + }); + store.run("listen", () => + server.listen(0, () => { + log("listening"); + const client = store.run("connect", () => + http2.connect(`http://localhost:${server.address().port}`), + ); + client.on("connect", () => log("client connect")); + client.on("close", () => log("client session close")); + const req = store.run("request", () => client.request({ ":method": "POST", ":path": "/" })); + req.on("response", () => log("client response")); + req.on("data", () => log("client data")); + req.on("end", () => { + log("client end"); + store.run("close", () => { + client.close(); + server.close(() => { + log("server close callback"); + resolve(); + }); + }); + }); + store.run("write", () => { + req.write("x"); + req.end(); + }); + }), + ); + }); +} + +/** @returns {Promise} */ +function resetScenario() { + return new Promise((resolve) => { + const server = store.run("reset-create", () => http2.createServer()); + server.on("stream", (stream) => { + log("reset server stream"); + stream.on("close", () => log("reset server stream close")); + store.run("reset", () => stream.close(http2.constants.NGHTTP2_CANCEL)); + }); + store.run("reset-listen", () => + server.listen(0, () => { + const client = store.run("reset-connect", () => + http2.connect(`http://localhost:${server.address().port}`), + ); + const req = store.run("reset-request", () => client.request({ ":path": "/" })); + req.on("close", () => { + log("reset client close"); + client.close(); + server.close(() => resolve()); + }); + req.end(); + }), + ); + }); +} + +await requestScenario(); +await resetScenario(); +lines.sort(); +for (const line of lines) console.log(line); diff --git a/tests/corpus/7388-als-http2-compat-events.mjs b/tests/corpus/7388-als-http2-compat-events.mjs new file mode 100644 index 000000000..da44dc378 --- /dev/null +++ b/tests/corpus/7388-als-http2-compat-events.mjs @@ -0,0 +1,49 @@ +// AsyncLocalStorage contexts of the node:http2 compatibility API. The +// createServer() request handler runs in the context of the server's +// listen() call. The request's 'end' and the response's 'finish' and +// 'close' follow the server stream, which runs with no store, whatever +// context calls res.end(). A client stream's 'response', 'data' and +// 'close' run in the context of the request() call, and the client +// session's 'close' in the context of the http2.connect() call. Lines are +// sorted at the end: events on the two sides may interleave. +import { AsyncLocalStorage } from "node:async_hooks"; +import http2 from "node:http2"; + +const store = new AsyncLocalStorage(); +const lines = []; +const log = (what) => lines.push(`${what}: ${store.getStore() ?? "none"}`); + +const server = store.run("create", () => + http2.createServer((req, res) => { + log("request handler"); + req.resume(); + req.on("end", () => { + log("request end"); + res.on("finish", () => log("response finish")); + res.on("close", () => log("response close")); + store.run("respond", () => res.end("done")); + }); + }), +); +store.run("listen", () => + server.listen(0, () => { + const client = store.run("connect", () => + http2.connect(`http://localhost:${server.address().port}`), + ); + client.on("close", () => { + log("client session close"); + server.close(() => { + lines.sort(); + for (const line of lines) console.log(line); + }); + }); + const req = store.run("request", () => client.request({ ":method": "POST", ":path": "/x" })); + req.on("response", () => log("client response")); + req.on("data", () => log("client data")); + req.on("close", () => { + log("client stream close"); + store.run("close", () => client.close()); + }); + store.run("write", () => req.end("body")); + }), +); diff --git a/tests/corpus/7389-als-http2-secure-events.mjs b/tests/corpus/7389-als-http2-secure-events.mjs new file mode 100644 index 000000000..0772b5267 --- /dev/null +++ b/tests/corpus/7389-als-http2-secure-events.mjs @@ -0,0 +1,94 @@ +// AsyncLocalStorage contexts of node:http2 events over TLS +// (http2.createSecureServer and an https: authority). They follow corpus +// 7387's rules, with one difference that comes from the transport: a TLS +// server's connections run in the context of its listen() call, so a +// server.close() callback that waits for the sessions runs there too. +// Lines are sorted at the end: events on the two sides may interleave. +import { AsyncLocalStorage } from "node:async_hooks"; +import fs from "node:fs"; +import http2 from "node:http2"; + +const key = fs.readFileSync("tests/fixtures/server/certs/localhost-key.pem"); +const cert = fs.readFileSync("tests/fixtures/server/certs/localhost.pem"); +const ca = fs.readFileSync("tests/fixtures/server/certs/ca.pem", "utf8"); + +const store = new AsyncLocalStorage(); +const lines = []; +const log = (what) => lines.push(`${what}: ${store.getStore() ?? "none"}`); + +/** @returns {Promise} */ +function requestScenario() { + return new Promise((resolve) => { + const server = store.run("create", () => http2.createSecureServer({ key, cert })); + server.on("session", () => log("server session")); + server.on("stream", (stream) => { + log("server stream"); + stream.on("data", () => log("server stream data")); + stream.on("end", () => { + log("server stream end"); + store.run("respond", () => { + stream.respond({ ":status": 200 }); + stream.end("ok"); + }); + }); + }); + store.run("listen", () => + server.listen(0, () => { + log("listening"); + const client = store.run("connect", () => + http2.connect(`https://localhost:${server.address().port}`, { ca }), + ); + client.on("connect", () => log("client connect")); + client.on("close", () => log("client session close")); + const req = store.run("request", () => client.request({ ":method": "POST", ":path": "/" })); + req.on("response", () => log("client response")); + req.on("data", () => log("client data")); + req.on("end", () => { + log("client end"); + store.run("close", () => { + client.close(); + server.close(() => { + log("server close callback"); + resolve(); + }); + }); + }); + store.run("write", () => { + req.write("x"); + req.end(); + }); + }), + ); + }); +} + +/** @returns {Promise} */ +function resetScenario() { + return new Promise((resolve) => { + const server = store.run("reset-create", () => http2.createSecureServer({ key, cert })); + server.on("stream", (stream) => { + log("reset server stream"); + stream.on("close", () => log("reset server stream close")); + store.run("reset", () => stream.close(http2.constants.NGHTTP2_CANCEL)); + }); + store.run("reset-listen", () => + server.listen(0, () => { + const client = store.run("reset-connect", () => + http2.connect(`https://localhost:${server.address().port}`, { ca }), + ); + const req = store.run("reset-request", () => client.request({ ":path": "/" })); + req.on("close", () => { + log("reset client close"); + client.close(); + server.close(() => resolve()); + }); + req.end(); + }), + ); + }); +} + +await requestScenario(); +await resetScenario(); +lines.sort(); +for (const line of lines) console.log(line); diff --git a/tests/harness/corpus-costs/7385-als-child-process-events.ts.json b/tests/harness/corpus-costs/7385-als-child-process-events.ts.json new file mode 100644 index 000000000..a806a3ed9 --- /dev/null +++ b/tests/harness/corpus-costs/7385-als-child-process-events.ts.json @@ -0,0 +1 @@ +{ "plain": 4500, "san": 9000 } diff --git a/tests/harness/corpus-costs/7386-als-child-fork-events/main.mjs.json b/tests/harness/corpus-costs/7386-als-child-fork-events/main.mjs.json new file mode 100644 index 000000000..a806a3ed9 --- /dev/null +++ b/tests/harness/corpus-costs/7386-als-child-fork-events/main.mjs.json @@ -0,0 +1 @@ +{ "plain": 4500, "san": 9000 } diff --git a/tests/harness/corpus-costs/7387-als-http2-events.mjs.json b/tests/harness/corpus-costs/7387-als-http2-events.mjs.json new file mode 100644 index 000000000..a806a3ed9 --- /dev/null +++ b/tests/harness/corpus-costs/7387-als-http2-events.mjs.json @@ -0,0 +1 @@ +{ "plain": 4500, "san": 9000 } diff --git a/tests/harness/corpus-costs/7388-als-http2-compat-events.mjs.json b/tests/harness/corpus-costs/7388-als-http2-compat-events.mjs.json new file mode 100644 index 000000000..a806a3ed9 --- /dev/null +++ b/tests/harness/corpus-costs/7388-als-http2-compat-events.mjs.json @@ -0,0 +1 @@ +{ "plain": 4500, "san": 9000 } diff --git a/tests/harness/corpus-costs/7389-als-http2-secure-events.mjs.json b/tests/harness/corpus-costs/7389-als-http2-secure-events.mjs.json new file mode 100644 index 000000000..a806a3ed9 --- /dev/null +++ b/tests/harness/corpus-costs/7389-als-http2-secure-events.mjs.json @@ -0,0 +1 @@ +{ "plain": 4500, "san": 9000 } From 945a519036530065664e07e85754ff3a7dd5cc6f Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 03:49:51 -0500 Subject: [PATCH 030/184] Match Node for typed pipeline callbacks and rl.cursor on non-terminal interfaces --- .../builtin-apis/streams-and-readline.mdx | 4 +- .../frontend/lowering/builtins/readline.ts | 51 +++++++- .../lowering/expressions/narrowing.ts | 3 + .../src/frontend/lowering/lower-stream.ts | 24 +++- .../compiler/src/frontend/lowering/lowerer.ts | 3 +- .../7500-pipeline-callback-undefined.ts.json | 6 + .../7501-pipeline-callback-undefined.mjs.json | 6 + .../7502-readline-cursor-undefined.ts.json | 6 + .../7500-pipeline-callback-undefined.ts | 120 ++++++++++++++++++ .../7501-pipeline-callback-undefined.mjs | 22 ++++ .../corpus/7502-readline-cursor-undefined.ts | 28 ++++ .../7500-pipeline-callback-undefined.ts.json | 1 + .../7501-pipeline-callback-undefined.mjs.json | 1 + .../7502-readline-cursor-undefined.ts.json | 1 + 14 files changed, 268 insertions(+), 8 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7500-pipeline-callback-undefined.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7501-pipeline-callback-undefined.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7502-readline-cursor-undefined.ts.json create mode 100644 tests/corpus/7500-pipeline-callback-undefined.ts create mode 100644 tests/corpus/7501-pipeline-callback-undefined.mjs create mode 100644 tests/corpus/7502-readline-cursor-undefined.ts create mode 100644 tests/harness/corpus-costs/7500-pipeline-callback-undefined.ts.json create mode 100644 tests/harness/corpus-costs/7501-pipeline-callback-undefined.mjs.json create mode 100644 tests/harness/corpus-costs/7502-readline-cursor-undefined.ts.json diff --git a/docs/content/limitations/builtin-apis/streams-and-readline.mdx b/docs/content/limitations/builtin-apis/streams-and-readline.mdx index 166fc3402..45a784742 100644 --- a/docs/content/limitations/builtin-apis/streams-and-readline.mdx +++ b/docs/content/limitations/builtin-apis/streams-and-readline.mdx @@ -18,7 +18,7 @@ Stages may take the `{ signal }` argument. With a function destination, `pipeline()` returns an object-mode PassThrough that receives an async generator destination's values, or an async function destination's value unless it is `null` or `undefined`. `@types/node` types that result as `NodeJS.WritableStream`, so `for await` over it needs the `PassThrough` type (`pipeline(...) as PassThrough`, or a JSDoc `@type`). - In typed code, the callback of a pipeline with an async function destination receives `null` instead of `undefined` when there is no error. + In typed code, a pipeline callback with an async function or async generator destination receives `null` for no error when its error parameter cannot hold `undefined` (@types/node declares it `NodeJS.ErrnoException | null`), and `undefined` when it can, as in Node. `Readable.from` accepts `objectMode` and `highWaterMark` options. The `Readable` operators (`map`, `filter`, `forEach`, `toArray`, `some`, `find`, `every`, `flatMap`, `drop`, `take`, and `reduce`) accept `concurrency`, `highWaterMark`, and `signal` options. Their callbacks may take the `{ signal }` argument, `map` may return any value, and `flatMap` may return any iterable, async iterable, or stream. @@ -43,7 +43,7 @@ The output's `'resize'` event does not redraw the line. The cursor never stops between the two UTF-16 units of a supplementary character: where Node's Up or Down movement through a multiline line would, the cursor moves to the character's start or end. In JavaScript, a `'history'` listener receives a copy of the history; its changes are written back when the listener returns, but the copy is not `rl.history` itself. - `rl.history`, `historySize`, `historyIndex`, `crlfDelay`, `tabSize`, `escapeCodeTimeout`, `line`, `cursor`, and `terminal` read as in Node, except that `rl.cursor` of an interface that is not a terminal throws instead of answering `undefined`. + `rl.history`, `historySize`, `historyIndex`, `crlfDelay`, `tabSize`, `escapeCodeTimeout`, `line`, `cursor`, and `terminal` read as in Node; `rl.cursor` is `undefined` on an interface that is not a terminal. `rl[Symbol.dispose]()` closes the interface, and `instanceof` with `readline.Interface` or `readlinePromises.Interface` answers whether the interface came from that module, as in Node. In JavaScript, the input and output can be any expression that evaluates to a stream, such as a function's result or an object's member; a value that is not a stream is unsupported. In JavaScript, `question()` and `setPrompt()` accept a value that is not a string: when such a prompt is written, an interface that is not a terminal throws `ERR_INVALID_ARG_TYPE` as Node does (the `readline/promises` question rejects with it) and the question stays pending, and a terminal interface writes `String(value)`. diff --git a/packages/compiler/src/frontend/lowering/builtins/readline.ts b/packages/compiler/src/frontend/lowering/builtins/readline.ts index 846b9a926..aca209017 100644 --- a/packages/compiler/src/frontend/lowering/builtins/readline.ts +++ b/packages/compiler/src/frontend/lowering/builtins/readline.ts @@ -6,9 +6,11 @@ import { DYN, F64, type IrExpr, + type IrType, type IrStmt, type SrcLoc, STRING, + UNDEFINED_T, VOID, arrayOf, BYTES_U8, @@ -54,6 +56,21 @@ export function isReadlineTyped(lowerer: Lowerer, node: ts.Expression): boolean }); } +/** `rl.cursor`: @types/node declares a number, but an interface that is + * not a terminal answers undefined, so the read keeps its + * `number | undefined` union. */ +export function isOptionalReadlineCursorRead( + lowerer: Lowerer, + expr: ts.PropertyAccessExpression, +): boolean { + return ( + expr.name.text === "cursor" && + !expr.questionDotToken && + isReadlineTyped(lowerer, expr.expression) && + lowerer.isStdlibMember(expr) + ); +} + /** The runtime's listener event numbers (scr_readline.c). */ const RL_EVENTS: Record = { line: 0, @@ -1341,7 +1358,7 @@ export function lowerReadlineCreateCall( /** Property reads on an Interface: `history` (the history option itself, * or the interface's own empty array), `historySize`, `historyIndex`, * `crlfDelay`, `tabSize`, `escapeCodeTimeout`, `line`, `terminal` and - * `cursor` (a non-terminal interface's undefined cursor throws SC2020). + * `cursor` (undefined on a non-terminal interface). * Null for other receivers. */ export function lowerReadlineProperty( lowerer: Lowerer, @@ -1373,10 +1390,38 @@ export function lowerReadlineProperty( loc, }; } - const reads: Record = { + if (name === "cursor") { + // `number | undefined`: a non-terminal interface has no cursor + const recv = lowerer.lowerExprExpecting(access.expression, F64); + const type: IrType = { kind: "union", unionId: lowerer.unions.intern([F64, UNDEFINED_T]) }; + const local = recv.kind === "varRef" ? null : lowerer.declareHiddenLocal("%rlCursor", F64); + const id: IrExpr = local ? { kind: "varRef", localId: local.id, type: F64, loc } : recv; + const result: IrExpr = { + kind: "ternary", + cond: { kind: "libCall", fn: "rl.isTerminal", args: [id], type: BOOL, loc }, + then: lowerer.coerceToExpected( + { kind: "libCall", fn: "rl.cursor", args: [id], type: F64, loc }, + type, + ), + else_: lowerer.coerceToExpected( + { kind: "unitLit", unit: "undefined", type: UNDEFINED_T, loc }, + type, + ), + type, + loc, + }; + if (!local) return result; + return { + kind: "seqExpr", + stmts: [{ kind: "varDecl", localId: local.id, init: recv, loc }], + result, + type, + loc, + }; + } + const reads: Record = { line: ["rl.line", STRING], terminal: ["rl.isTerminal", BOOL], - cursor: ["rl.cursor", F64], }; const read = reads[name]; if (read !== undefined) { diff --git a/packages/compiler/src/frontend/lowering/expressions/narrowing.ts b/packages/compiler/src/frontend/lowering/expressions/narrowing.ts index fa8ad686f..01cab9901 100644 --- a/packages/compiler/src/frontend/lowering/expressions/narrowing.ts +++ b/packages/compiler/src/frontend/lowering/expressions/narrowing.ts @@ -33,6 +33,7 @@ import { isOptionalProcessVersionsRead, } from "../builtins/process.js"; import { requestGeneralStreamSurface } from "../general-streams.js"; +import { isOptionalReadlineCursorRead } from "../builtins/readline.js"; import { literalValues } from "../../literal-values.js"; import { literalUnionArm } from "../../union-discriminants.js"; import { checkedArmExtraction } from "../coercions/unions.js"; @@ -617,6 +618,8 @@ export function maybeNarrow(lowerer: Lowerer, expr: IrExpr, node: ts.Node): IrEx if (ts.isPropertyAccessExpression(node) && isOptionalProcessStreamProperty(lowerer, node)) return expr; if (isOptionalProcessVersionsRead(lowerer, node)) return expr; + if (ts.isPropertyAccessExpression(node) && isOptionalReadlineCursorRead(lowerer, node)) + return expr; // Represented fields and index values can include undefined even when // checker inference or lib.d.ts declares strings (RegExp captures). diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index cf9f19b9d..687418350 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -2906,6 +2906,22 @@ class StreamArgTypeThrow extends Error { } } +/** Whether a checked-dynamic pipeline callback can take undefined as its + * error: a callback that is already checked-dynamic, one without + * parameters, and one whose first parameter (or rest element) type holds + * undefined. */ +function callbackErrorAdmitsUndefined(lowerer: Lowerer, cb: IrExpr): boolean { + if (cb.kind !== "dynFrom" || cb.value.type.kind !== "func") return true; + const fnT = cb.value.type; + const first = fnT.params[0]; + if (first === undefined) return true; + const p0 = + fnT.rest === true && fnT.params.length === 1 && first.kind === "array" ? first.elem : first; + if (p0.kind === "dyn" || p0.kind === "jsval" || p0.kind === "undefinedT") return true; + if (p0.kind !== "union") return false; + return lowerer.unions.get(p0.unionId)?.arms.some((a) => a.kind === "undefinedT") ?? false; +} + /** The finished/pipeline completion callback: an inline function lowers * through the option-callback machinery (leading `this`, an `Error | * null` prefix — Node binds the stream and passes the error); any other @@ -3367,12 +3383,16 @@ export function lowerStreamModuleCall( // an async generator) reaches the callback through the // checked-dynamic path. const cb = lowerer.lowerExprExpecting(args[args.length - 1]!, DYN); - // SCR_PIPE_TYPED_CB: typed code's callback takes null for no error + // SCR_PIPE_TYPED_CB: a typed callback whose error parameter cannot + // hold undefined takes null for no error; others take undefined. const stagedArgs = [...staged.args]; const flagsArg = stagedArgs[3]!; if (flagsArg.kind !== "numLit") throw new InternalCompilerError("pipeline flags not a literal"); - if (!isJsSourceFile(ts.getSourceFileOfNode(call))) + if ( + !isJsSourceFile(ts.getSourceFileOfNode(call)) && + !callbackErrorAdmitsUndefined(lowerer, cb) + ) stagedArgs[3] = numLit(flagsArg.value | 128, loc); return staged.wrap({ kind: "libCall", diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index a765c91ee..a17522333 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -496,7 +496,7 @@ import { lowerBuiltinExtraProperty } from "./builtins/properties.js"; import { registerPromisifiedBuiltinDecl } from "./builtins/callable-aliases.js"; import { lowerExecFileAsyncCall } from "./builtins/child-process-async.js"; import { lowerStringDecoderMethodCall, lowerTextCodecCall } from "./builtins/text-codecs.js"; -import { lowerReadlineMethodCall } from "./builtins/readline.js"; +import { isOptionalReadlineCursorRead, lowerReadlineMethodCall } from "./builtins/readline.js"; import { lowerDiagnosticsChannelMethodCall, lowerDiagnosticsChannelProperty, @@ -5547,6 +5547,7 @@ export class Lowerer { if (isIndexedRead(e)) return true; if (ts.isPropertyAccessExpression(e) && isRecordIndexPropertyRead(e)) return true; if (ts.isPropertyAccessExpression(e) && isOptionalProcessStreamProperty(this, e)) return true; + if (ts.isPropertyAccessExpression(e) && isOptionalReadlineCursorRead(this, e)) return true; if (isOptionalProcessVersionsRead(this, e)) return true; if (ts.isPropertyAccessExpression(e) && isSettledIteratorValueRead(this, e)) return true; if (ts.isPropertyAccessExpression(e) && isOsConstantsSymbol(symbolOf(e.expression), this)) diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7500-pipeline-callback-undefined.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7500-pipeline-callback-undefined.ts.json new file mode 100644 index 000000000..fe65b63f0 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7500-pipeline-callback-undefined.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7500-pipeline-callback-undefined.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7501-pipeline-callback-undefined.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7501-pipeline-callback-undefined.mjs.json new file mode 100644 index 000000000..8d52980fe --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7501-pipeline-callback-undefined.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7501-pipeline-callback-undefined.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7502-readline-cursor-undefined.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7502-readline-cursor-undefined.ts.json new file mode 100644 index 000000000..6fd23042e --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7502-readline-cursor-undefined.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7502-readline-cursor-undefined.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7500-pipeline-callback-undefined.ts b/tests/corpus/7500-pipeline-callback-undefined.ts new file mode 100644 index 000000000..e1503ff6b --- /dev/null +++ b/tests/corpus/7500-pipeline-callback-undefined.ts @@ -0,0 +1,120 @@ +// pipeline() with an async function or async generator destination: a +// typed callback whose error parameter admits undefined receives +// undefined when the pipeline succeeds, as in Node, and the error when it +// fails. +import { pipeline, Readable } from "node:stream"; + +function source(): Readable { + return Readable.from(["a", "b", "c"]); +} + +async function join(src: AsyncIterable): Promise { + let s = ""; + for await (const c of src) s += c; + return s; +} + +function optional(): Promise { + return new Promise((resolve) => { + pipeline(source(), join, (err?: Error | null, value?: string) => { + console.log("optional", err, err === undefined, value); + resolve(); + }); + }); +} + +function declared(): Promise { + return new Promise((resolve) => { + pipeline( + source(), + async function (src: AsyncIterable) { + let n = 0; + for await (const c of src) n += c.length; + return n; + }, + (err: Error | null | undefined, value?: number) => { + console.log("declared", err, typeof err, value); + resolve(); + }, + ); + }); +} + +function unknownParam(): Promise { + return new Promise((resolve) => { + pipeline(source(), join, (err: unknown, value: unknown) => { + console.log("unknown", err, value); + resolve(); + }); + }); +} + +function rest(): Promise { + return new Promise((resolve) => { + pipeline( + source(), + async function* (src: AsyncIterable) { + for await (const c of src) yield c.toUpperCase(); + }, + async function (src: AsyncIterable) { + for await (const c of src) void c; + }, + (...args: unknown[]) => { + console.log("rest", args.length, args); + resolve(); + }, + ); + }); +} + +function generatorDestination(): Promise { + return new Promise((resolve) => { + pipeline( + source(), + async function* (src: AsyncIterable) { + for await (const c of src) yield c; + }, + (err: Error | null | undefined, value: unknown) => { + console.log("generator", err, value); + resolve(); + }, + ); + }); +} + +function named(): Promise { + return new Promise((resolve) => { + function done(err: Error | null | undefined, value: string | undefined): void { + console.log("named", err, value); + resolve(); + } + pipeline(source(), join, done); + }); +} + +function failing(): Promise { + return new Promise((resolve) => { + pipeline( + source(), + async function (src: AsyncIterable) { + for await (const c of src) if (c === "b") throw new Error("boom at " + c); + return "never"; + }, + (err?: Error | null, value?: string) => { + console.log("failing", err?.message, value); + resolve(); + }, + ); + }); +} + +async function main(): Promise { + await optional(); + await declared(); + await unknownParam(); + await rest(); + await generatorDestination(); + await named(); + await failing(); +} +void main(); diff --git a/tests/corpus/7501-pipeline-callback-undefined.mjs b/tests/corpus/7501-pipeline-callback-undefined.mjs new file mode 100644 index 000000000..c51738e77 --- /dev/null +++ b/tests/corpus/7501-pipeline-callback-undefined.mjs @@ -0,0 +1,22 @@ +// pipeline() with an async function destination in JavaScript: the +// callback receives undefined and the destination's value on success. +import { pipeline, Readable } from "node:stream"; + +pipeline( + Readable.from(["x", "y"]), + async function (src) { + const out = []; + for await (const c of src) out.push(c); + return out.join("-"); + }, + (err, value) => { + console.log("callback", err, typeof err, value); + pipeline( + Readable.from(["z"]), + async function (src) { + for await (const c of src) void c; + }, + (...args) => console.log("no value", args), + ); + }, +); diff --git a/tests/corpus/7502-readline-cursor-undefined.ts b/tests/corpus/7502-readline-cursor-undefined.ts new file mode 100644 index 000000000..abac985cb --- /dev/null +++ b/tests/corpus/7502-readline-cursor-undefined.ts @@ -0,0 +1,28 @@ +// rl.cursor answers undefined on an interface that is not a terminal and +// the cursor position on a terminal interface. +import * as readline from "node:readline"; +import { PassThrough, Writable } from "node:stream"; + +const input = new PassThrough(); +const rl = readline.createInterface({ input }); +const c = rl.cursor; +console.log("cursor", c, typeof rl.cursor, rl.cursor === undefined); +console.log("arithmetic", rl.cursor + 1, Number.isNaN(rl.cursor * 2)); +console.log("fallback", rl.cursor ?? -1, `${rl.cursor}`); + +const interfaces = [rl]; +function first(): readline.Interface { + return interfaces[0]!; +} +console.log("call receiver", first().cursor); +rl.close(); + +const sink = new Writable({ + write(_chunk: Buffer, _enc, cb) { + cb(); + }, +}); +const terminal = readline.createInterface({ input: new PassThrough(), output: sink, terminal: true }); +terminal.write("abc"); +console.log("terminal", terminal.cursor, typeof terminal.cursor, terminal.cursor + 1); +terminal.close(); diff --git a/tests/harness/corpus-costs/7500-pipeline-callback-undefined.ts.json b/tests/harness/corpus-costs/7500-pipeline-callback-undefined.ts.json new file mode 100644 index 000000000..122c66b27 --- /dev/null +++ b/tests/harness/corpus-costs/7500-pipeline-callback-undefined.ts.json @@ -0,0 +1 @@ +{ "plain": 3783, "san": 4994 } diff --git a/tests/harness/corpus-costs/7501-pipeline-callback-undefined.mjs.json b/tests/harness/corpus-costs/7501-pipeline-callback-undefined.mjs.json new file mode 100644 index 000000000..d9c00a7de --- /dev/null +++ b/tests/harness/corpus-costs/7501-pipeline-callback-undefined.mjs.json @@ -0,0 +1 @@ +{ "plain": 2457, "san": 3243 } diff --git a/tests/harness/corpus-costs/7502-readline-cursor-undefined.ts.json b/tests/harness/corpus-costs/7502-readline-cursor-undefined.ts.json new file mode 100644 index 000000000..7a36db8f3 --- /dev/null +++ b/tests/harness/corpus-costs/7502-readline-cursor-undefined.ts.json @@ -0,0 +1 @@ +{ "plain": 2592, "san": 3421 } From fb3806ce42a7717c84e4ce4e285bf62caf86d35a Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 03:54:26 -0500 Subject: [PATCH 031/184] Search TypedArrays, read stdin through 'readable', and assign constructor-function globals - TypedArray indexOf, lastIndexOf and includes on every element kind, typed and checked-dynamic: strict equality (SameValueZero for includes) and fromIndex as ToIntegerOrInfinity. A Buffer reached through a Uint8Array value keeps Buffer's search. - process.stdin 'readable' with read() and read(size), including sizes above the read-ahead mark and input a once('readable') listener leaves for a later 'data' listener. for await over process.stdin yields Buffers. - Module bindings holding `new F()` of a constructor function can be read and reassigned from other functions. --- .../compiler/src/backend/llvm/expr-bytes.ts | 16 +++ .../src/frontend/lowering/containers/bytes.ts | 49 +++++++- .../src/frontend/lowering/lower-modules.ts | 1 - .../frontend/lowering/object-factory-new.ts | 31 +++-- packages/compiler/src/ir/ir.ts | 7 ++ packages/compiler/src/ir/validate.ts | 4 + .../corpus/7680-typed-array-search.ts.json | 6 + .../7681-typed-array-search-dynamic.mjs.json | 6 + .../7682-constructor-function-globals.js.json | 6 + packages/runtime/src/scr_bytes.c | 44 +++++++ packages/runtime/src/scr_bytes.h | 2 + packages/runtime/src/scr_dyn_invoke.c | 15 +++ packages/runtime/src/scr_events.c | 115 ++++++++++++++++-- tests/corpus/7680-typed-array-search.ts | 63 ++++++++++ .../7681-typed-array-search-dynamic.mjs | 28 +++++ .../7682-constructor-function-globals.js | 55 +++++++++ .../event-loop/stdin-for-await-buffers.mjs | 10 ++ .../event-loop/stdio-readable-large-size.mjs | 9 ++ .../event-loop/stdio-readable-once-rest.mjs | 12 ++ .../event-loop/stdio-readable-size.mjs | 18 +++ tests/fixtures/event-loop/stdio-readable.mjs | 19 +++ .../7680-typed-array-search.ts.json | 1 + .../7681-typed-array-search-dynamic.mjs.json | 1 + .../7682-constructor-function-globals.js.json | 1 + tests/harness/event-loop-cases.ts | 49 ++++++++ 25 files changed, 543 insertions(+), 25 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7680-typed-array-search.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7681-typed-array-search-dynamic.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7682-constructor-function-globals.js.json create mode 100644 tests/corpus/7680-typed-array-search.ts create mode 100644 tests/corpus/7681-typed-array-search-dynamic.mjs create mode 100644 tests/corpus/7682-constructor-function-globals.js create mode 100644 tests/fixtures/event-loop/stdin-for-await-buffers.mjs create mode 100644 tests/fixtures/event-loop/stdio-readable-large-size.mjs create mode 100644 tests/fixtures/event-loop/stdio-readable-once-rest.mjs create mode 100644 tests/fixtures/event-loop/stdio-readable-size.mjs create mode 100644 tests/fixtures/event-loop/stdio-readable.mjs create mode 100644 tests/harness/corpus-costs/7680-typed-array-search.ts.json create mode 100644 tests/harness/corpus-costs/7681-typed-array-search-dynamic.mjs.json create mode 100644 tests/harness/corpus-costs/7682-constructor-function-globals.js.json diff --git a/packages/compiler/src/backend/llvm/expr-bytes.ts b/packages/compiler/src/backend/llvm/expr-bytes.ts index 41ddf2b9d..2a368f0e2 100644 --- a/packages/compiler/src/backend/llvm/expr-bytes.ts +++ b/packages/compiler/src/backend/llvm/expr-bytes.ts @@ -936,6 +936,22 @@ export function emitBytesIntrinsic( B.line(`${t} = fcmp one double ${idx.name}, ${f64Lit(-1)}`); return { name: t, type: e.type }; } + case "indexOfElem": + case "lastIndexOfElem": + case "includesElem": { + const includes = method === "includesElem"; + const idx = call( + "scr_bytes_ta_index_of", + "double (ptr, double, double, i1 zeroext, i1 zeroext, i1 zeroext)", + `ptr ${r.name}, double ${args[0]!.name}, double ${args[1]?.name ?? f64Lit(0)}, i1 ${args.length > 1}, i1 ${method !== "lastIndexOfElem"}, i1 ${includes}`, + false, + false, + ); + if (!includes) return idx; + const t = B.tmp(); + B.line(`${t} = fcmp one double ${idx.name}, ${f64Lit(-1)}`); + return { name: t, type: e.type }; + } case "fillElem": // Per-element TypedArray fill (non-u8): slice-style index // defaults, never throws; the receiver comes back +1. diff --git a/packages/compiler/src/frontend/lowering/containers/bytes.ts b/packages/compiler/src/frontend/lowering/containers/bytes.ts index 995cb177a..9cf68da9f 100644 --- a/packages/compiler/src/frontend/lowering/containers/bytes.ts +++ b/packages/compiler/src/frontend/lowering/containers/bytes.ts @@ -907,9 +907,10 @@ function lowerDataViewNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr { * (shared view), toReversed, copyWithin, set(src, * offset?), and the u8-only Buffer surface: toString(enc?) plus the * whole numeric read/write family (fixed widths BE/LE and the - * variable-width read/writeUIntLE quartet — BUF_NUM_METHODS). Everything - * else the lib declares (fill, indexOf, reverse, ...) falls through to - * the SC2020 member fence. Null when this isn't a bytes method call. */ + * variable-width read/writeUIntLE quartet — BUF_NUM_METHODS), and + * TypedArray fill, indexOf, lastIndexOf and includes. Everything else the + * lib declares (reverse, sort, ...) falls through to the SC2020 member + * fence. Null when this isn't a bytes method call. */ export function lowerBytesMethodCall( lowerer: Lowerer, call: ts.CallExpression, @@ -1101,6 +1102,43 @@ export function lowerBytesMethodCall( loc, }; } + if ( + (name === "indexOf" || name === "lastIndexOf" || name === "includes") && + !(() => { + const symbol = lowerer.checker.getSymbolAtLocation(access.name); + return ( + symbol !== undefined && + symbol.declarations.some( + (declaration) => + ts.isInterfaceDeclaration(declaration.parent) && + declaration.parent.name.text === "Buffer", + ) + ); + })() + ) { + // TypedArray search: strict equality (includes: SameValueZero) with a + // relative fromIndex. Buffer's same-named methods keep their byte, + // string and Buffer needles. + if (nArgs < 1 || nArgs > 2 || call.arguments.nodes.some(ts.isSpreadElement)) { + lowerer.noLowering(`.${name} with ${nArgs} arguments on typed arrays`, call); + } + const receiver = lowerer.lowerExprExpecting(access.expression, receiverIr); + const args = call.arguments.nodes.map((a) => lowerer.lowerExprExpecting(a, F64)); + const method = + name === "indexOf" + ? "indexOfElem" + : name === "lastIndexOf" + ? "lastIndexOfElem" + : "includesElem"; + return { + kind: "bytesIntrinsic", + method, + receiver, + args, + type: name === "includes" ? BOOL : F64, + loc, + }; + } if (name === "set") { if (nArgs < 1 || nArgs > 2) { lowerer.noLowering(`.set with ${nArgs} arguments on typed arrays`, call); @@ -1210,9 +1248,8 @@ export function lowerBytesMethodCall( // The Buffer-declared comparison/search/mutation surface. All of // these resolve against the Buffer interface (the checker keeps most // off plain typed arrays; where the lib DOES declare a same-named - // TypedArray member — indexOf, includes, fill — the semantics differ, - // so only Buffer-declared resolutions lower and u8 receivers gate the - // rest). + // TypedArray member — indexOf, includes, fill — the semantics differ: + // those resolutions lowered above, and u8 receivers gate the rest). const declOnBuffer = (() => { const nameSym = lowerer.checker.getSymbolAtLocation(access.name); return ( diff --git a/packages/compiler/src/frontend/lowering/lower-modules.ts b/packages/compiler/src/frontend/lowering/lower-modules.ts index 5238333ad..af2ed4a2c 100644 --- a/packages/compiler/src/frontend/lowering/lower-modules.ts +++ b/packages/compiler/src/frontend/lowering/lower-modules.ts @@ -2361,7 +2361,6 @@ export function collectGlobals( // References from separately-declared functions cascade to // their own per-site runtime fences. const factoryType = - !isLet && isJsSourceFile(sf) && !decl.type && !hasJsTypeAnnotation(decl) && diff --git a/packages/compiler/src/frontend/lowering/object-factory-new.ts b/packages/compiler/src/frontend/lowering/object-factory-new.ts index bcb675692..c672f82e2 100644 --- a/packages/compiler/src/frontend/lowering/object-factory-new.ts +++ b/packages/compiler/src/frontend/lowering/object-factory-new.ts @@ -6,10 +6,14 @@ import { bindingNeverReassigned } from "./binding-analysis.js"; import { implicitDefaultInstance } from "./generic-functions.js"; import { type FnSig } from "./call-signatures.js"; -/** An ordinary constructor's explicit object return replaces its receiver. - * Without observing this or new.target, it can use the function's call ABI. */ -function isObjectFactoryNew(lowerer: Lowerer, expr: ts.NewExpression): boolean { - if (!ts.isIdentifier(expr.expression) || lowerer.peekLocal(expr.expression)) return false; +/** The JavaScript function declaration (or never-reassigned function + * expression binding) `new` calls, when it is an ordinary constructor: not + * a generator or async function. */ +function jsConstructorFunctionOf( + lowerer: Lowerer, + expr: ts.NewExpression, +): ts.FunctionDeclaration | ts.FunctionExpression | null { + if (!ts.isIdentifier(expr.expression) || lowerer.peekLocal(expr.expression)) return null; // This probe also sees ordinary classes during global collection. Leave // their deferred diagnostics for the class declaration's lowering. const wasCollecting = lowerer.collecting; @@ -20,7 +24,7 @@ function isObjectFactoryNew(lowerer: Lowerer, expr: ts.NewExpression): boolean { } finally { lowerer.collecting = wasCollecting; } - if (!symbol) return false; + if (!symbol) return null; const owner = symbol.declarations.find( (node) => (ts.isFunctionDeclaration(node) && node.body !== undefined) || @@ -45,7 +49,15 @@ function isObjectFactoryNew(lowerer: Lowerer, expr: ts.NewExpression): boolean { declaration.asteriskToken || declaration.modifiers?.nodes.some((m) => m.kind === ts.SyntaxKind.AsyncKeyword) ) - return false; + return null; + return declaration; +} + +/** An ordinary constructor's explicit object return replaces its receiver. + * Without observing this or new.target, it can use the function's call ABI. */ +function isObjectFactoryNew(lowerer: Lowerer, expr: ts.NewExpression): boolean { + const declaration = jsConstructorFunctionOf(lowerer, expr); + if (!declaration?.body) return false; const objectReturn = (expression: ts.Expression): boolean => ts.isObjectLiteralExpression(expression) || ts.isArrayLiteralExpression(expression) || @@ -81,8 +93,13 @@ function isObjectFactoryNew(lowerer: Lowerer, expr: ts.NewExpression): boolean { /** Collection must register the shared result before reaching factory * bodies: those bodies can reference later globals in the same module. */ +/** The storage of a JavaScript module binding initialized with `new` of an + * ordinary function: a factory's result and a constructor function's + * receiver (`const head = new Segment()`) are both checked-dynamic + * values, which functions declared elsewhere in the module read through + * the global. */ export function objectFactoryGlobalType(lowerer: Lowerer, expr: ts.NewExpression): IrType | null { - return isObjectFactoryNew(lowerer, expr) ? DYN : null; + return jsConstructorFunctionOf(lowerer, expr) !== null ? DYN : null; } export function objectFactorySignature(lowerer: Lowerer, expr: ts.NewExpression): FnSig | null { diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 6d0760b81..a26cb638f 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -2484,6 +2484,13 @@ export type IrBytesIntrinsicMethod = * (same observable number-fill result; Buffer's throwing offset * validation). */ | "fillElem" + /** TypedArray.prototype.indexOf / lastIndexOf / includes on any + * receiver: [f64 searchElement, f64 fromIndex?] → f64 index (-1 when + * absent) or bool; strict equality (includes: SameValueZero), a present + * fromIndex counts from the end when negative, never throws. */ + | "indexOfElem" + | "lastIndexOfElem" + | "includesElem" | "copy" | "swap16" | "swap32" diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index 02825f0de..77909fb52 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -2196,6 +2196,10 @@ function validateFunction( fillNum: { argTypes: [F64, F64, F64], minArgs: 1, result: BYTES_U8 }, // Per-element TypedArray fill (any elem — the non-u8 fill path). fillElem: { argTypes: [F64, F64, F64], minArgs: 1, result: bytesOf(recv.elem) }, + // TypedArray search (any elem). + indexOfElem: { argTypes: [F64, F64], minArgs: 1, result: F64 }, + lastIndexOfElem: { argTypes: [F64, F64], minArgs: 1, result: F64 }, + includesElem: { argTypes: [F64, F64], minArgs: 1, result: BOOL }, fillStr: { argTypes: [STRING, STRING, F64, F64], minArgs: 2, result: BYTES_U8 }, copy: { argTypes: [BYTES_U8, F64, F64, F64], minArgs: 1, result: F64 }, swap16: { argTypes: [], minArgs: 0, result: BYTES_U8 }, diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7680-typed-array-search.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7680-typed-array-search.ts.json new file mode 100644 index 000000000..d93d9afdc --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7680-typed-array-search.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7680-typed-array-search.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7681-typed-array-search-dynamic.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7681-typed-array-search-dynamic.mjs.json new file mode 100644 index 000000000..ab91940c0 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7681-typed-array-search-dynamic.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7681-typed-array-search-dynamic.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7682-constructor-function-globals.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7682-constructor-function-globals.js.json new file mode 100644 index 000000000..26d05338c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7682-constructor-function-globals.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7682-constructor-function-globals.js" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_bytes.c b/packages/runtime/src/scr_bytes.c index 7bcfd0cfb..26f1a57a5 100644 --- a/packages/runtime/src/scr_bytes.c +++ b/packages/runtime/src/scr_bytes.c @@ -3300,6 +3300,50 @@ double scr_bytes_index_of_num(const ScrBytes *b, double v, double off, bool fwd) return scr_bytes_index_of(b, &needle, off, 1, fwd); } +/* TypedArray.prototype.indexOf and lastIndexOf (strict equality, so NaN + * never matches) and includes (`same_value_zero`: NaN matches NaN). `from` + * is the fromIndex argument when `has_from` (ToIntegerOrInfinity, negative + * counts from the end), else the default: 0, or len - 1 searching + * backwards. Returns the element index, or -1. Never throws. */ +double scr_bytes_ta_index_of(const ScrBytes *b, double v, double from, bool has_from, bool forward, + bool same_value_zero) { + // A Buffer viewed as a Uint8Array keeps Buffer's own search methods, whose + // byte needle and byteOffset rules differ (an absent offset is NaN there). + if (scr_bytes_is_buffer(b)) return scr_bytes_index_of_num(b, v, has_from ? from : NAN, forward); + size_t len = b->len; + if (len == 0) return -1; + double k = has_from ? (from != from ? 0 : trunc(from)) : forward ? 0 : (double)len - 1; + bool nan = v != v; + if (nan && !same_value_zero) return -1; + bool bytes = (b->elem == SCR_BYTES_U8 || b->elem == SCR_BYTES_U8C) && !b->shared; + if (forward) { + if (k >= (double)len) return -1; + if (k < 0) k = k + (double)len < 0 ? 0 : k + (double)len; + size_t start = (size_t)k; + if (bytes) { + if (!(v >= 0 && v <= 255 && v == trunc(v))) return -1; + const uint8_t *hit = memchr(b->data + start, (int)v, len - start); + return hit ? (double)(hit - b->data) : -1; + } + for (size_t i = start; i < len; i++) { + double e = scr_bytes_load(b, i); + if (e == v || (nan && e != e)) return (double)i; + } + return -1; + } + if (k < 0) { + k += (double)len; + if (k < 0) return -1; + } else if (k > (double)len - 1) { + k = (double)len - 1; + } + for (size_t i = (size_t)k + 1; i-- > 0;) { + double e = scr_bytes_load(b, i); + if (e == v || (nan && e != e)) return (double)i; + } + return -1; +} + /* fill's shared core over a normalized byte pattern. nargs counts the * present offset/end args. An EMPTY pattern zero-fills when zero_ok * (Node's '' string) and is the constant TypeError otherwise (an empty diff --git a/packages/runtime/src/scr_bytes.h b/packages/runtime/src/scr_bytes.h index 9b382dda5..13b31b183 100644 --- a/packages/runtime/src/scr_bytes.h +++ b/packages/runtime/src/scr_bytes.h @@ -499,6 +499,8 @@ ScrEmitter *scr_emitter_set_max_chk(ScrEmitter *em, const ScrDyn *n); void scr_emitter_set_default_max_chk(const ScrDyn *n, const ScrStr *name); double scr_bytes_index_of(const ScrBytes *b, const ScrBytes *needle, double off, double align, bool fwd); double scr_bytes_index_of_num(const ScrBytes *b, double v, double off, bool fwd); +double scr_bytes_ta_index_of(const ScrBytes *b, double v, double from, bool has_from, bool forward, + bool same_value_zero); ScrBytes *scr_bytes_fill(ScrBytes *b, const ScrBytes *pattern, double nargs, double offset, double end); ScrBytes *scr_bytes_fill_num(ScrBytes *b, double v, double nargs, double offset, double end); ScrBytes *scr_bytes_fill_str(ScrBytes *b, const ScrStr *s, const ScrStr *enc, diff --git a/packages/runtime/src/scr_dyn_invoke.c b/packages/runtime/src/scr_dyn_invoke.c index 4039fda94..d57ca24ac 100644 --- a/packages/runtime/src/scr_dyn_invoke.c +++ b/packages/runtime/src/scr_dyn_invoke.c @@ -1704,6 +1704,21 @@ static ScrDyn *scr_dyn_invoke_impl( scr_str_release(joined); return out; } + if (!recv->buffer && + ((mid == DYN_M_indexOf) || (mid == DYN_M_lastIndexOf) || (mid == DYN_M_includes))) { + /* TypedArray search: strict equality (includes: SameValueZero), so a + * needle that is not a number matches nothing; fromIndex counts as + * given whenever it is passed, undefined included. */ + double from = 0; + if (argc > 1) { + if (!scr_dyn_number_coerce_js(args[1], &from)) return NULL; + } + bool includes = mid == DYN_M_includes; + double found = argc > 0 && args[0]->kind == SCR_DYN_NUM + ? scr_bytes_ta_index_of(bytes, args[0]->v.num, from, argc > 1, mid != DYN_M_lastIndexOf, includes) + : -1; + return includes ? scr_dyn_new_bool(found >= 0) : scr_dyn_new_num(found); + } if (dyn_bytes_proto_real(mid)) { dyn_throw_unsupported(scr_bytes_elem_name(bytes->elem), method); return NULL; diff --git a/packages/runtime/src/scr_events.c b/packages/runtime/src/scr_events.c index 3391f54b1..bc554a445 100644 --- a/packages/runtime/src/scr_events.c +++ b/packages/runtime/src/scr_events.c @@ -390,6 +390,13 @@ static SCR_TL bool scr_stdin_resumed = false; static SCR_TL bool scr_stdin_read_eof = false; static SCR_TL ScrBytes *scr_stdin_buffer; static SCR_TL size_t scr_stdin_read_demand; +/* 'readable' listeners (paused mode, as Node's once one is added): input + * stays buffered for read(), and the event fires on the next tick when + * input arrives while a read wanted more (`need`), and once at EOF. */ +static SCR_TL ScrStdinEndL *scr_stdin_readable = NULL; +static SCR_TL size_t scr_stdin_nreadable = 0, scr_stdin_readable_cap = 0; +static SCR_TL bool scr_stdin_readable_need = false; +static SCR_TL bool scr_stdin_readable_queued = false; #ifdef SCR_STDIO_EXTENDED /* process.stdin.setEncoding(enc): the canonical encoding name (NULL until @@ -475,6 +482,78 @@ void scr_stdin_on_end(ScrClosure *cb /*moves*/, bool once) { scr_stdin_nend++; } +static void scr_stdin_finish(bool fire_end); + +static void scr_stdin_emit_readable(void) { + scr_stdin_readable_queued = false; + size_t length = scr_stdin_buffer ? scr_stdin_buffer->len : 0; + if (scr_stdin_eof || scr_stdin_destroyed || (!length && !scr_stdin_read_eof)) return; + size_t n = scr_stdin_nreadable; + ScrStdinEndL *snap = malloc((n ? n : 1) * sizeof *snap); + if (!snap) scr_events_oom(); + for (size_t i = 0; i < n; i++) { + snap[i] = scr_stdin_readable[i]; + scr_closure_retain(snap[i].cb); + } + for (size_t i = 0; i < n; i++) { + if (snap[i].once) { + for (size_t j = 0; j < scr_stdin_nreadable; j++) { + if (scr_stdin_readable[j].cb == snap[i].cb) { + scr_closure_release(scr_stdin_readable[j].cb); + memmove(scr_stdin_readable + j, scr_stdin_readable + j + 1, + (scr_stdin_nreadable - j - 1) * sizeof *scr_stdin_readable); + scr_stdin_nreadable--; + break; + } + } + } + if (!scr_exc_pending() && !scr_stdin_eof && !scr_stdin_destroyed) + ((void (*)(ScrClosure *))snap[i].cb->fn)(snap[i].cb); + scr_closure_release(snap[i].cb); + } + free(snap); + length = scr_stdin_buffer ? scr_stdin_buffer->len : 0; + /* Another event is due once the buffer reaches the 64 KiB mark, or the + * larger size a listener's read(size) asked for. */ + scr_stdin_readable_need = + !scr_stdin_read_eof && (length <= 65536 || length < scr_stdin_read_demand); +} + +static void scr_stdin_queue_readable(void) { + if (scr_stdin_readable_queued || !scr_stdin_nreadable) return; + scr_stdin_readable_queued = true; + scr_next_tick_raw(scr_stdin_emit_readable); +} + +/* Adding a 'readable' listener starts reading ahead to the 64 KiB mark; a + * buffer that already holds input fires the event at once (next tick). */ +static void scr_stdin_on_readable(ScrClosure *cb /*moves*/, bool once) { + if (scr_stdin_eof || scr_stdin_destroyed) { + scr_closure_release(cb); + return; + } + if (scr_stdin_nreadable == scr_stdin_readable_cap) { + scr_stdin_readable_cap = scr_stdin_readable_cap ? scr_stdin_readable_cap * 2 : 2; + scr_stdin_readable = realloc(scr_stdin_readable, scr_stdin_readable_cap * sizeof *scr_stdin_readable); + if (!scr_stdin_readable) scr_events_oom(); + } + scr_stdin_readable[scr_stdin_nreadable].cb = cb; + scr_stdin_readable[scr_stdin_nreadable].once = once; + scr_stdin_nreadable++; + scr_stdin_readable_need = true; + if (scr_stdin_read_demand < 65536) scr_stdin_read_demand = 65536; + if (scr_stdin_buffer && scr_stdin_buffer->len) scr_stdin_queue_readable(); +} + +/* The input ended: with 'readable' listeners the event fires once more and + * 'end' waits for the read that finds nothing left; otherwise 'end' fires + * once the buffer is drained. */ +static void scr_stdin_reached_eof(void) { + scr_stdin_read_eof = true; + if (scr_stdin_nreadable) scr_stdin_queue_readable(); + else if (!scr_stdin_buffer) scr_stdin_finish(true); +} + void scr_stdin_on_error(ScrClosure *cb /*moves*/, ScrChildErrFn fn, bool once) { if (scr_stdin_eof || scr_stdin_destroyed) { scr_closure_release(cb); @@ -528,6 +607,10 @@ static void scr_stdin_drop_listeners(void) { for (size_t i = 0; i < scr_stdin_ndata; i++) scr_closure_release(scr_stdin_data[i].cb); for (size_t i = 0; i < scr_stdin_nend; i++) scr_closure_release(scr_stdin_end[i].cb); for (size_t i = 0; i < scr_stdin_nerr; i++) scr_closure_release(scr_stdin_err[i].cb); + for (size_t i = 0; i < scr_stdin_nreadable; i++) scr_closure_release(scr_stdin_readable[i].cb); + free(scr_stdin_readable); + scr_stdin_readable = NULL; + scr_stdin_nreadable = scr_stdin_readable_cap = 0; free(scr_stdin_data); free(scr_stdin_end); free(scr_stdin_err); @@ -627,7 +710,8 @@ static void scr_stdin_deliver(ScrBytes *chunk); /* A chunk that arrived (moves): delivered, or buffered while paused or * without a consumer. */ static void scr_stdin_arrived(ScrBytes *chunk) { - if (scr_stdin_paused || (!scr_stdin_ndata && !scr_stdin_resumed && !scr_stdin_waiter)) { + if (scr_stdin_paused || scr_stdin_nreadable || (!scr_stdin_ndata && !scr_stdin_resumed && !scr_stdin_waiter)) { + if (scr_stdin_nreadable && scr_stdin_readable_need) scr_stdin_queue_readable(); if (scr_stdin_buffer) { size_t before = scr_stdin_buffer->len; ScrBytes *joined = scr_bytes_new(SCR_BYTES_U8, (double)(before + chunk->len)); @@ -645,7 +729,11 @@ static void scr_stdin_arrived(ScrBytes *chunk) { static void scr_stdin_service(void) { if (!scr_stdin_watching()) return; - if (!scr_stdin_paused && scr_stdin_buffer) { + /* Buffered input goes to a consumer: 'data' listeners, a resumed + * stream, or a parked for-await read. After the last 'readable' listener + * it stays buffered until one arrives, as in Node. */ + if (!scr_stdin_paused && !scr_stdin_nreadable && scr_stdin_buffer && + (scr_stdin_ndata > 0 || scr_stdin_resumed || scr_stdin_waiter != NULL)) { ScrBytes *chunk = scr_stdin_buffer; scr_stdin_buffer = NULL; scr_stdin_deliver(chunk); @@ -653,7 +741,7 @@ static void scr_stdin_service(void) { return; } if (scr_stdin_read_eof) { - if (!scr_stdin_buffer) scr_stdin_finish(true); + if (!scr_stdin_buffer && !scr_stdin_nreadable) scr_stdin_finish(true); return; } if (scr_stdin_fed) { @@ -662,10 +750,7 @@ static void scr_stdin_service(void) { bool ended = false; ScrBytes *chunk = scr_stdin_fed_take(&ended); if (chunk) scr_stdin_arrived(chunk); - else if (ended) { - scr_stdin_read_eof = true; - if (!scr_stdin_buffer) scr_stdin_finish(true); - } + else if (ended) scr_stdin_reached_eof(); return; } #ifdef _WIN32 @@ -727,8 +812,7 @@ static void scr_stdin_service(void) { return; } if (n == 0) { - scr_stdin_read_eof = true; - if (!scr_stdin_buffer) scr_stdin_finish(true); + scr_stdin_reached_eof(); return; } ScrBytes *chunk = scr_bytes_new(SCR_BYTES_U8, (double)n); @@ -802,6 +886,8 @@ static void scr_stdin_deliver(ScrBytes *chunk) { #endif ScrPromise *p = scr_stdin_waiter; scr_stdin_waiter = NULL; + /* Node's stdin yields Buffers. */ + chunk->is_buffer = true; scr_promise_fulfill_ref(p, scr_bytes_retain(chunk), scr_bytes_retain_v, scr_bytes_release_v, NULL); scr_promise_release(p); @@ -878,6 +964,7 @@ static void scr_stdio_data_text(ScrClosure *cb, ScrStr *text) { } #endif static void scr_stdio_end(ScrClosure *cb) { scr_stdio_fire(cb, NULL, 0); } +static void scr_stdio_readable(ScrClosure *cb) { scr_stdio_fire(cb, NULL, 0); } static void scr_stdio_error(ScrClosure *cb, ScrStr *message) { ScrError *error = scr_error_new(SCR_ERR_ERROR, message); ScrDyn *arg = scr_dyn_from_error(error); @@ -985,6 +1072,7 @@ static ScrDyn *scr_stdio_read(ScrDyn *size) { if (scr_stdin_read_eof && wanted > available) wanted = available; if (wanted == 0 || wanted > available) { if (!scr_stdin_eof && !scr_stdin_destroyed) { + scr_stdin_readable_need = true; scr_stdin_read_demand = wanted > 65536 ? wanted : 65536; if (scr_stdin_read_eof) scr_next_tick_raw(scr_stdio_finish_read); } @@ -995,6 +1083,7 @@ static ScrDyn *scr_stdio_read(ScrDyn *size) { scr_bytes_release(scr_stdin_buffer); scr_stdin_buffer = rest; scr_stdin_read_demand = 65536; + if (!rest && !scr_stdin_read_eof) scr_stdin_readable_need = true; scr_stdin_deliver(chunk); ScrDyn *result = scr_dyn_new_chunk(chunk); scr_bytes_release(chunk); @@ -1150,12 +1239,14 @@ static ScrDyn *scr_stdio_method(ScrClosure *cb, ScrDyn *const *args, size_t argc bool data = arg->kind == SCR_DYN_STR && arg->v.str->len == 4 && memcmp(event, "data", 4) == 0; bool end = arg->kind == SCR_DYN_STR && arg->v.str->len == 3 && memcmp(event, "end", 3) == 0; bool error = arg->kind == SCR_DYN_STR && arg->v.str->len == 5 && memcmp(event, "error", 5) == 0; + bool readable = stream->fd == 0 && arg->kind == SCR_DYN_STR && arg->v.str->len == 8 && + memcmp(event, "readable", 8) == 0; #ifdef SCR_STDIO_EXTENDED bool resize = stream->fd != 0 && arg->kind == SCR_DYN_STR && arg->v.str->len == 6 && memcmp(event, "resize", 6) == 0; - if ((!data && !end && !error && !resize) || (stream->fd != 0 && !error && !resize)) { + if ((!data && !end && !error && !readable && !resize) || (stream->fd != 0 && !error && !resize)) { #else - if ((!data && !end && !error) || (stream->fd != 0 && !error)) { + if ((!data && !end && !error && !readable) || (stream->fd != 0 && !error)) { #endif result = scr_stdio_refusal("event"); goto done; @@ -1175,6 +1266,7 @@ static ScrDyn *scr_stdio_method(ScrClosure *cb, ScrDyn *const *args, size_t argc if (data) scr_stdin_on_data(scr_stdio_callback(listener, (void *)scr_stdio_data), scr_stdio_data, once); #endif if (end) scr_stdin_on_end(scr_stdio_callback(listener, (void *)scr_stdio_end), once); + if (readable) scr_stdin_on_readable(scr_stdio_callback(listener, (void *)scr_stdio_readable), once); if (error) scr_stdin_on_error(scr_stdio_callback(listener, (void *)scr_stdio_error), scr_stdio_error, once); } else { #define REMOVE_STDIO_LISTENER(list, count, fire) \ @@ -1187,6 +1279,7 @@ static ScrDyn *scr_stdio_method(ScrClosure *cb, ScrDyn *const *args, size_t argc } if (data) { REMOVE_STDIO_LISTENER(scr_stdin_data, scr_stdin_ndata, scr_stdio_data); } if (end) { REMOVE_STDIO_LISTENER(scr_stdin_end, scr_stdin_nend, scr_stdio_end); } + if (readable) { REMOVE_STDIO_LISTENER(scr_stdin_readable, scr_stdin_nreadable, scr_stdio_readable); } if (error) { REMOVE_STDIO_LISTENER(scr_stdin_err, scr_stdin_nerr, scr_stdio_error); } #undef REMOVE_STDIO_LISTENER } diff --git a/tests/corpus/7680-typed-array-search.ts b/tests/corpus/7680-typed-array-search.ts new file mode 100644 index 000000000..353736e09 --- /dev/null +++ b/tests/corpus/7680-typed-array-search.ts @@ -0,0 +1,63 @@ +// TypedArray.prototype.indexOf, lastIndexOf and includes on every numeric +// element kind: strict equality (NaN never found) for indexOf/lastIndexOf, +// SameValueZero (NaN found, -0 equals 0) for includes, and fromIndex as +// ToIntegerOrInfinity, counted from the end when negative. +const bytes = new Uint8Array([1, 2, 10, 3, 10, 255]); +console.log("u8", bytes.indexOf(10), bytes.indexOf(10, 3), bytes.indexOf(10, 5), bytes.lastIndexOf(10)); +console.log("u8 from", bytes.indexOf(10, -2), bytes.indexOf(10, -100), bytes.indexOf(1, Infinity), bytes.indexOf(10, 2.9)); +console.log("u8 last", bytes.lastIndexOf(10, 3), bytes.lastIndexOf(10, -3), bytes.lastIndexOf(1, -100), bytes.lastIndexOf(255, 99)); +console.log("u8 values", bytes.indexOf(256 + 10), bytes.indexOf(-246), bytes.indexOf(10.5), bytes.includes(255), bytes.includes(0)); +console.log("u8 includes from", bytes.includes(1, 1), bytes.includes(255, -1), bytes.includes(2, -5)); + +const clamped = new Uint8ClampedArray([0, 300, -5, 127.5]); +console.log("u8c", clamped.indexOf(255), clamped.indexOf(0, 1), clamped.lastIndexOf(0), clamped.indexOf(128)); + +const signed = new Int8Array([-1, 127, -128, 0]); +console.log("i8", signed.indexOf(-1), signed.indexOf(255), signed.indexOf(-128), signed.lastIndexOf(0)); + +const shorts = new Int16Array([5, -300, 5, 7]); +const ushorts = new Uint16Array([65535, 1, 65535]); +console.log("16", shorts.indexOf(-300), shorts.lastIndexOf(5), ushorts.indexOf(-1), ushorts.lastIndexOf(65535, 1)); + +const ints = new Int32Array([-2147483648, 0, 2147483647, 0]); +const uints = new Uint32Array([4294967295, 3]); +console.log("32", ints.indexOf(2147483647), ints.lastIndexOf(0), uints.indexOf(4294967295), uints.includes(-1)); + +const floats = new Float64Array([1.5, NaN, -0, 0, Infinity]); +console.log("f64", floats.indexOf(NaN), floats.lastIndexOf(NaN), floats.includes(NaN), floats.indexOf(0), floats.indexOf(-0)); +console.log("f64 more", floats.lastIndexOf(0), floats.indexOf(Infinity), floats.includes(-Infinity), floats.includes(NaN, 2)); + +const singles = new Float32Array([1.1, 0.5, NaN]); +console.log("f32", singles.indexOf(1.1), singles.indexOf(Math.fround(1.1)), singles.indexOf(0.5), singles.includes(NaN), singles.indexOf(NaN)); + +const empty = new Float64Array(0); +console.log("empty", empty.indexOf(0), empty.lastIndexOf(0), empty.includes(0), empty.includes(0, -1)); + +// Views search their own window of the buffer. +const buffer = new ArrayBuffer(16); +const whole = new Uint8Array(buffer); +whole.set([9, 9, 1, 2, 9, 3, 9, 9]); +const view = new Uint8Array(buffer, 2, 4); +const sub = whole.subarray(3, 6); +console.log("views", view.indexOf(9), view.lastIndexOf(9), view.indexOf(1), sub.indexOf(9), sub.includes(1), sub.lastIndexOf(3)); +const words = new Uint16Array(buffer, 2, 3); +console.log("word view", words.indexOf(0x0201), words.length); + +// Shared memory searches the same way. +const shared = new Int32Array(new SharedArrayBuffer(16)); +shared[1] = 7; +shared[3] = 7; +console.log("shared", shared.indexOf(7), shared.lastIndexOf(7), shared.includes(0), shared.indexOf(7, 2)); + +// A loop that finds every occurrence, as a line splitter does. +const text = new TextEncoder().encode("one\ntwo\n\nthree\n"); +const lines: number[] = []; +for (let at = text.indexOf(10); at !== -1; at = text.indexOf(10, at + 1)) lines.push(at); +console.log("lines", lines.join(",")); + +// A Buffer passed as a Uint8Array keeps Buffer's search: a number needle +// is taken as a byte, and a NaN byteOffset searches the whole buffer. +function searchView(view: Uint8Array): string { + return [view.indexOf(97 + 256), view.lastIndexOf(98, NaN), view.includes(353), view.indexOf(98, -1)].join(","); +} +console.log(searchView(Buffer.from("abcab")), searchView(new Uint8Array([97, 98, 99, 97, 98]))); diff --git a/tests/corpus/7681-typed-array-search-dynamic.mjs b/tests/corpus/7681-typed-array-search-dynamic.mjs new file mode 100644 index 000000000..643ba0cfa --- /dev/null +++ b/tests/corpus/7681-typed-array-search-dynamic.mjs @@ -0,0 +1,28 @@ +// TypedArray search on untyped values (parameters of untyped functions and +// object members): needles that are not numbers are never found (strict +// equality), fromIndex may be undefined, a string or a fraction, and +// Buffers keep their own indexOf, which wraps number needles and takes +// strings. +function views(bytes, floats, shorts) { + return { bytes, floats, shorts }; +} +const { bytes, floats, shorts } = views( + new Uint8Array([1, 2, 10, 3, 10]), + new Float64Array([0.5, NaN, -0]), + new Int16Array([-1, 300, -1]), +); +console.log(bytes.indexOf(10), bytes.indexOf(10, 3), bytes.lastIndexOf(10), bytes.includes(3), bytes.includes(4)); +console.log(bytes.lastIndexOf(1, undefined), bytes.lastIndexOf(10), bytes.indexOf(10, undefined), bytes.includes(1, undefined)); +console.log(bytes.indexOf(10, '3'), bytes.indexOf(10, 2.9), bytes.indexOf(10, -1), bytes.lastIndexOf(10, -2), bytes.indexOf(10, null)); +console.log(bytes.indexOf('10'), bytes.includes('1'), bytes.indexOf(10n), bytes.indexOf(), bytes.includes(), bytes.indexOf(266)); +console.log(floats.indexOf(NaN), floats.includes(NaN), floats.indexOf(0), floats.lastIndexOf(-0), floats.includes(0.5, 1)); +console.log(shorts.indexOf(-1), shorts.lastIndexOf(-1), shorts.indexOf(300, -2), shorts.includes(65535)); +const holder = { view: bytes.subarray(2) }; +console.log(holder.view.indexOf(10), holder.view.lastIndexOf(10), holder.view.includes(1)); +const buffer = Buffer.from([1, 2, 10, 3, 10]); +const boxed = [buffer]; +console.log(boxed[0].indexOf(10), boxed[0].indexOf(266), boxed[0].indexOf('\n'), boxed[0].includes(Buffer.from([3, 10]))); +console.log(boxed[0].subarray(3).indexOf(10), Buffer.isBuffer(boxed[0].subarray(3)), boxed[0].lastIndexOf(10, -2)); +let count = 0; +for (let at = bytes.indexOf(10); at !== -1; at = bytes.indexOf(10, at + 1)) count++; +console.log('count', count); diff --git a/tests/corpus/7682-constructor-function-globals.js b/tests/corpus/7682-constructor-function-globals.js new file mode 100644 index 000000000..58292e4ac --- /dev/null +++ b/tests/corpus/7682-constructor-function-globals.js @@ -0,0 +1,55 @@ +// Module bindings that hold `new F()` of an ordinary constructor function +// (const, let and var), read and reassigned by functions declared elsewhere +// in the module, such as the head and tail of a linked list. +'use strict'; +function Segment(prev) { + this.prev = prev; + this.next = null; + this.data = new Array(4); + this.pos = 0; +} +const head = new Segment(); +let current = head; +var spare = new Segment(head); + +function append(value) { + if (current.pos === current.data.length) { + current = current.next || (current.next = new Segment(current)); + } + current.data[current.pos++] = value; +} + +function collect() { + const out = []; + for (let node = head; node; node = node.next) out.push(node.data.slice(0, node.pos).join('')); + return out.join('|'); +} + +function reset() { + current = head; + for (let node = head; node; node = node.next) node.pos = 0; +} + +for (const ch of 'abcdefghij') append(ch); +console.log(collect(), current === head, current.prev === head.next ? 'two' : 'more'); +reset(); +append('z'); +console.log(collect(), current === head, head.next !== null); +console.log(spare.prev === head, spare instanceof Segment, typeof spare.data); +spare = new Segment(spare); +console.log(spare.prev.prev === head); + +function Counter(start) { + this.value = start; +} +Counter.prototype.bump = function () { + return ++this.value; +}; +let counter = new Counter(10); +function bumpTwice() { + counter.bump(); + return counter.bump(); +} +console.log(bumpTwice(), counter.value); +counter = new Counter(0); +console.log(bumpTwice()); diff --git a/tests/fixtures/event-loop/stdin-for-await-buffers.mjs b/tests/fixtures/event-loop/stdin-for-await-buffers.mjs new file mode 100644 index 000000000..d64ae460a --- /dev/null +++ b/tests/fixtures/event-loop/stdin-for-await-buffers.mjs @@ -0,0 +1,10 @@ +// for await over process.stdin yields Buffers: Buffer.isBuffer holds and +// indexOf takes a byte and a start offset (only chunking-independent facts +// are printed). +const chunks = []; +for await (const chunk of process.stdin) chunks.push(chunk); +console.log('buffers', chunks.every((chunk) => Buffer.isBuffer(chunk))); +const all = Buffer.concat(chunks); +const newlines = []; +for (let at = all.indexOf(10); at !== -1; at = all.indexOf(10, at + 1)) newlines.push(at); +console.log('newlines', newlines.join(','), all.subarray(newlines[0] + 1).indexOf('#')); diff --git a/tests/fixtures/event-loop/stdio-readable-large-size.mjs b/tests/fixtures/event-loop/stdio-readable-large-size.mjs new file mode 100644 index 000000000..f1f44edce --- /dev/null +++ b/tests/fixtures/event-loop/stdio-readable-large-size.mjs @@ -0,0 +1,9 @@ +// read(size) above the 64 KiB read-ahead mark: 'readable' keeps firing as +// input arrives until the requested size is buffered. +const input = process.stdin; +const sizes = []; +input.on('readable', () => { + let chunk; + while ((chunk = input.read(150000)) !== null) sizes.push(chunk.length); +}); +input.on('end', () => console.log(sizes.join(','))); diff --git a/tests/fixtures/event-loop/stdio-readable-once-rest.mjs b/tests/fixtures/event-loop/stdio-readable-once-rest.mjs new file mode 100644 index 000000000..81eae144b --- /dev/null +++ b/tests/fixtures/event-loop/stdio-readable-once-rest.mjs @@ -0,0 +1,12 @@ +// A once('readable') listener that reads part of the input: the rest stays +// buffered for a 'data' listener added afterwards. +const input = process.stdin; +input.once('readable', () => { + const head = input.read(3); + console.log('head', head === null ? null : head.toString('ascii')); + setTimeout(() => { + const rest = []; + input.on('data', (chunk) => rest.push(chunk.toString('ascii'))); + input.on('end', () => console.log('rest', rest.join(''))); + }, 50); +}); diff --git a/tests/fixtures/event-loop/stdio-readable-size.mjs b/tests/fixtures/event-loop/stdio-readable-size.mjs new file mode 100644 index 000000000..9ff76dafe --- /dev/null +++ b/tests/fixtures/event-loop/stdio-readable-size.mjs @@ -0,0 +1,18 @@ +// Fixed-size reads of stdin: read(n) answers null until n bytes are +// buffered, and the rest once the input ended. +const input = process.stdin; +const chunks = []; +const counts = []; +input.on('readable', () => { + let reads = 0; + let chunk; + while ((chunk = input.read(4)) !== null) { + chunks.push(chunk.toString('ascii')); + reads++; + } + counts.push(reads); +}); +input.on('end', () => { + console.log(chunks.join('|')); + console.log(counts.join(',')); +}); diff --git a/tests/fixtures/event-loop/stdio-readable.mjs b/tests/fixtures/event-loop/stdio-readable.mjs new file mode 100644 index 000000000..324a4b57b --- /dev/null +++ b/tests/fixtures/event-loop/stdio-readable.mjs @@ -0,0 +1,19 @@ +// process.stdin in paused mode through 'readable': each event drains the +// buffer with read(), one more event follows EOF, and 'end' comes after the +// read that finds the buffer empty. +const input = process.stdin; +const events = []; +let text = ''; +input.on('readable', () => { + let reads = 0; + let chunk; + while ((chunk = input.read()) !== null) { + text += String(chunk); + reads++; + } + events.push(`readable ${reads}`); +}); +input.on('end', () => { + console.log(events.join(', ')); + console.log(JSON.stringify(text), input.readableLength); +}); diff --git a/tests/harness/corpus-costs/7680-typed-array-search.ts.json b/tests/harness/corpus-costs/7680-typed-array-search.ts.json new file mode 100644 index 000000000..37f8b6e89 --- /dev/null +++ b/tests/harness/corpus-costs/7680-typed-array-search.ts.json @@ -0,0 +1 @@ +{ "plain": 2204, "san": 3060 } diff --git a/tests/harness/corpus-costs/7681-typed-array-search-dynamic.mjs.json b/tests/harness/corpus-costs/7681-typed-array-search-dynamic.mjs.json new file mode 100644 index 000000000..b6fcd5eff --- /dev/null +++ b/tests/harness/corpus-costs/7681-typed-array-search-dynamic.mjs.json @@ -0,0 +1 @@ +{ "plain": 1649, "san": 958 } diff --git a/tests/harness/corpus-costs/7682-constructor-function-globals.js.json b/tests/harness/corpus-costs/7682-constructor-function-globals.js.json new file mode 100644 index 000000000..90fcede92 --- /dev/null +++ b/tests/harness/corpus-costs/7682-constructor-function-globals.js.json @@ -0,0 +1 @@ +{ "plain": 1623, "san": 962 } diff --git a/tests/harness/event-loop-cases.ts b/tests/harness/event-loop-cases.ts index 40bc7988d..1b75defaa 100644 --- a/tests/harness/event-loop-cases.ts +++ b/tests/harness/event-loop-cases.ts @@ -34,6 +34,55 @@ export const eventLoopCases: EventLoopCase[] = [ fixture: "stdio-paused-exit.mjs", script: { writes: [], end: false }, }, + { + title: "stdio values: 'readable' drains with read(), then fires once more at EOF", + fixture: "stdio-readable.mjs", + script: { + writes: [ + { delayMs: 0, data: "alpha " }, + { delayMs: 60, data: "beta\n" }, + ], + end: true, + }, + }, + { + title: "stdio values: 'readable' with read(size) holds a partial tail until EOF", + fixture: "stdio-readable-size.mjs", + script: { writes: [{ delayMs: 0, data: "abcdefghij" }], end: true }, + }, + { + title: "stdio values: input a once('readable') listener leaves stays buffered for 'data'", + fixture: "stdio-readable-once-rest.mjs", + script: { writes: [{ delayMs: 0, data: "abcdefgh" }], end: true }, + }, + { + title: "stdio values: 'readable' with a read(size) above the read-ahead mark", + fixture: "stdio-readable-large-size.mjs", + script: { + writes: [ + { delayMs: 0, data: "a".repeat(70000) }, + { delayMs: 30, data: "b".repeat(70000) }, + { delayMs: 30, data: "c".repeat(60000) }, + ], + end: true, + }, + }, + { + title: "stdio values: 'readable' on an immediately ended pipe", + fixture: "stdio-readable.mjs", + script: { writes: [], end: true }, + }, + { + title: "for-await: process.stdin yields Buffers", + fixture: "stdin-for-await-buffers.mjs", + script: { + writes: [ + { delayMs: 0, data: "#one\nabcd\n" }, + { delayMs: 40, data: "#two\nefgh\n" }, + ], + end: true, + }, + }, { title: "read-all: one instant write, then EOF", fixture: "stdin-read-all.ts", From 0e0458f29234d104d43c55a753a82703459545d0 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 01:55:25 -0700 Subject: [PATCH 032/184] Keep literal key order through self copies and conversions that build no records A literal that starts by spreading a value of its own type, such as `{ ...current, ...patch, updatedAt }`, made the key-order witness give up on the type: the copy's source order was the type's own order, seeded with the declared table, so it conflicted with the literals that build the values. While no checked conversion builds the type, every value has the one chosen order and the copy adds no witness. Checked conversions also counted as builders of records they cannot create: a class instance's field records, a checked function's record parameters, a boxed primitive (the undefined a dynamic construction passes for a missing argument), and the result of a typed call of a method the program may replace on its instance, which without a replacement is the method's own native value boxed and checked back. Co-Authored-By: Claude Opus 5.5 --- .../src/frontend/lowering/class-callbacks.ts | 13 +- .../lowering/key-order-witness.test.ts | 118 +++++++++++++++++ .../frontend/lowering/key-order-witness.ts | 125 +++++++++++++++--- packages/compiler/src/ir/traverse.ts | 14 +- .../main.ts.json | 7 + .../corpus/7737-key-order-self-spread.ts.json | 6 + .../main.ts | 106 +++++++++++++++ .../registry.js | 21 +++ tests/corpus/7737-key-order-self-spread.ts | 34 +++++ .../main.ts.json | 1 + .../7737-key-order-self-spread.ts.json | 1 + 11 files changed, 421 insertions(+), 25 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7736-key-order-replaceable-method-result/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7737-key-order-self-spread.ts.json create mode 100644 tests/corpus/7736-key-order-replaceable-method-result/main.ts create mode 100644 tests/corpus/7736-key-order-replaceable-method-result/registry.js create mode 100644 tests/corpus/7737-key-order-self-spread.ts create mode 100644 tests/harness/corpus-costs/7736-key-order-replaceable-method-result/main.ts.json create mode 100644 tests/harness/corpus-costs/7737-key-order-self-spread.ts.json diff --git a/packages/compiler/src/frontend/lowering/class-callbacks.ts b/packages/compiler/src/frontend/lowering/class-callbacks.ts index 0bda0068e..f92f42a80 100644 --- a/packages/compiler/src/frontend/lowering/class-callbacks.ts +++ b/packages/compiler/src/frontend/lowering/class-callbacks.ts @@ -156,6 +156,13 @@ export function classCallbackValue( }; } +/** Notes a checked conversion of a replaceable method's result for the + * key-order witness: the result is a value the method or its replacement + * returns, not a record built from a dynamic value. */ +function methodResult(lowerer: Lowerer, checked: IrExpr): IrExpr { + return checked.kind === "dynCheck" ? lowerer.keyOrder.methodResult(checked) : checked; +} + export function classCallbackCall( lowerer: Lowerer, receiver: IrExpr, @@ -197,7 +204,7 @@ export function classCallbackCall( // generator arm does not box) keeps its native type: the unchanged // method's result stays as it is, and a replacement's dynamic result // takes the checked conversion into that type. - const converted = lowerer.coerceToExpected(call, fallback.type); + const converted = methodResult(lowerer, lowerer.coerceToExpected(call, fallback.type)); if (!typeEquals(converted.type, fallback.type)) { lowerer.pushDiag( unsupportedDiag( @@ -234,7 +241,9 @@ export function classCallbackCall( loc, }; answer = - fallback.type.kind === "void" ? result : lowerer.coerceToExpected(result, fallback.type); + fallback.type.kind === "void" + ? result + : methodResult(lowerer, lowerer.coerceToExpected(result, fallback.type)); } // Read after observedSlot built the prototype data and its originals. const originalId = observed ? original?.originalId() : undefined; diff --git a/packages/compiler/src/frontend/lowering/key-order-witness.test.ts b/packages/compiler/src/frontend/lowering/key-order-witness.test.ts index c85902c96..dc9149119 100644 --- a/packages/compiler/src/frontend/lowering/key-order-witness.test.ts +++ b/packages/compiler/src/frontend/lowering/key-order-witness.test.ts @@ -700,3 +700,121 @@ test("a spread reads its source's order right after the literal evaluates the so const snapshot = (prologue[2] as IrStmt & { kind: "varDecl" }).localId; expect(orderIndexOf(lit)).toMatchObject({ kind: "varRef", localId: snapshot }); }); + +/** `{ a, b }` with a declared table `["a", "b"]` and a literal `{ b, a }` + * plus `{ ...value, a }` of the same shape, beside `extra`. */ +function selfCopy(extra: (f: Fixture, ab: IrType & { kind: "record" }) => IrStmt[]): string[] { + const f = fixture(); + const ab = f.record( + [ + ["a", F64], + ["b", F64], + ], + ["a", "b"], + ); + const self: KeyOrderSegment = { kind: "spread", shapeId: ab.shapeId, maybeAbsent: false }; + f.run([ + decl("v", f.literal(ab, ["b", "a"])), + observe(f.literal(ab, ["a", "b"], [self, { kind: "key", name: "a", definite: true }])), + ...extra(f, ab), + ]); + return f.order(ab)!; +} + +test("a literal that starts by spreading its own shape takes the other literals' order", () => { + expect(selfCopy(() => [])).toEqual(["b", "a"]); +}); + +test("a self copy of a shape a checked conversion builds cannot be ordered statically", () => { + const parsed = (ab: IrType): IrExpr => ({ + kind: "dynCheck", + value: ref("j", { kind: "dyn" }), + type: ab, + loc, + }); + expect(selfCopy((_, ab) => [observe(parsed(ab))])).toEqual(["a", "b"]); +}); + +test("a replaceable method's checked result builds no record", () => { + expect( + selfCopy((f, ab) => { + const result: IrExpr = { kind: "dynCheck", value: ref("r", { kind: "dyn" }), type: ab, loc }; + return [observe(f.notes.methodResult(result))]; + }), + ).toEqual(["b", "a"]); +}); + +test("a replaceable method's result keeps its witness through an IR rewrite", () => { + expect( + selfCopy((f, ab) => { + const result = f.notes.methodResult({ + kind: "dynCheck", + value: ref("r", { kind: "dyn" }), + type: ab, + loc, + }); + const identity = { expr: (e: IrExpr): IrExpr => e, stmt: (st: IrStmt): IrStmt => st }; + observeRecordLiteralRebuilds((from, to) => f.notes.follow(from, to)); + try { + const rebuilt = transformExpr(result, identity); + expect(rebuilt).not.toBe(result); + return [observe(rebuilt)]; + } finally { + observeRecordLiteralRebuilds(null); + } + }), + ).toEqual(["b", "a"]); +}); + +test("a checked conversion of a boxed primitive builds no record", () => { + expect( + selfCopy((_, ab) => { + const missing: IrExpr = { + kind: "dynFrom", + value: { kind: "unitLit", unit: "undefined", type: { kind: "undefinedT" }, loc }, + type: { kind: "dyn" }, + loc, + }; + return [observe({ kind: "dynCheck", value: missing, type: ab, loc })]; + }), + ).toEqual(["b", "a"]); +}); + +test("a checked conversion into a class builds none of its fields' records", () => { + expect( + selfCopy((f, ab) => { + (f.lowerer.classes as Map).set("Holder", { + def: { fields: [{ name: "value", type: ab }] }, + }); + const holder: IrType = { kind: "object", className: "Holder" }; + const checked: IrExpr = { + kind: "dynCheck", + value: ref("h", { kind: "dyn" }), + type: holder, + loc, + }; + return [{ kind: "exprStmt", expr: checked, loc }]; + }), + ).toEqual(["b", "a"]); +}); + +test("a checked function passes its record parameters out, and its callbacks' back in", () => { + const callable = (params: IrType[]): IrType => ({ + kind: "func", + params, + ret: { kind: "string" }, + }); + const checkedAs = (type: IrType): IrStmt => ({ + kind: "exprStmt", + expr: { kind: "dynCheck", value: ref("fn", { kind: "dyn" }), type, loc }, + loc, + }); + expect(selfCopy((_, ab) => [checkedAs(callable([ab]))])).toEqual(["b", "a"]); + // A callback parameter receives values the JavaScript function builds. + expect(selfCopy((_, ab) => [checkedAs(callable([callable([ab])]))])).toEqual(["a", "b"]); + // So does a container parameter, which the function can write into. + expect(selfCopy((_, ab) => [checkedAs(callable([{ kind: "array", elem: ab }]))])).toEqual([ + "a", + "b", + ]); +}); diff --git a/packages/compiler/src/frontend/lowering/key-order-witness.ts b/packages/compiler/src/frontend/lowering/key-order-witness.ts index aa84c68c2..2f573e676 100644 --- a/packages/compiler/src/frontend/lowering/key-order-witness.ts +++ b/packages/compiler/src/frontend/lowering/key-order-witness.ts @@ -28,7 +28,19 @@ * build keeps today's table, since another table would make values * parsed in today's order keep a source view; a shape object literals * also build takes the literals' order. Their order is unknown, so a - * copy out of such a shape cannot be ordered statically; + * copy out of such a shape cannot be ordered statically. A conversion + * builds only the records it can create from a JavaScript value: none + * for a boxed primitive or a class instance (a native instance whose + * fields its own writes fill), none for a checked function's record + * parameters, and none for the checked result of a typed call of a + * method the program may replace on its instance. Without a replacement + * that result is the method's own native value, boxed and checked back + * unchanged; a replacement's result keeps its own order in its record + * source, and only copies out of it follow the chosen table, as they + * followed today's; + * - a literal that starts by spreading a value of its own shape copies + * that value's order. While no checked conversion builds the shape, + * every value has the one chosen order, so the copy adds no witness; * - a write to an optional key that may be absent appends it in Node. A * key no construction creates ("late-only") goes after every key a * construction can create; any other moving write goes after every key. @@ -112,6 +124,7 @@ export type KeyOrderSegment = export class KeyOrderNotes { private readonly literals = new Map(); private readonly copies = new Map(); + private readonly methodResults = new Set(); /** Shapes a frontend order surface (keys, values/entries, inspect, * index-signature merges) walks. */ readonly observed = new Set(); @@ -128,13 +141,21 @@ export class KeyOrderNotes { return node; } + /** The checked result of a typed call of a method the program may + * replace on its instance (class-callbacks.ts classCallbackCall). */ + methodResult(node: IrExpr): IrExpr { + this.methodResults.add(node); + return node; + } + /** A pass rebuilt `from` as `to` (traverse.ts mapExprChildren): the - * rebuilt literal is the same construction, so it keeps the witness. */ + * rebuilt node is the same construction, so it keeps the witness. */ follow(from: IrExpr, to: IrExpr): void { const segments = this.literals.get(from); if (segments !== undefined) this.literals.set(to, segments); const source = this.copies.get(from); if (source !== undefined) this.copies.set(to, source); + if (this.methodResults.has(from)) this.methodResults.add(to); } observe(type: IrType): void { @@ -148,6 +169,10 @@ export class KeyOrderNotes { copyOf(node: IrExpr): string | undefined { return this.copies.get(node); } + + isMethodResult(node: IrExpr): boolean { + return this.methodResults.has(node); + } } const isArrayIndex = (name: string): boolean => { @@ -201,6 +226,23 @@ function orderPreservingClone(node: IrExpr & { kind: "dynCheck" }): boolean { ); } +const PRIMITIVE_KINDS = new Set([ + "f64", + "bigint", + "string", + "bool", + "undefinedT", + "nullT", + "symbol", +]); + +/** A checked conversion of a boxed primitive (the `undefined` a dynamic + * construction passes for a missing argument) answers that primitive or + * throws: it builds no record. */ +function primitiveSource(node: IrExpr & { kind: "dynCheck" }): boolean { + return node.value.kind === "dynFrom" && PRIMITIVE_KINDS.has(node.value.value.type.kind); +} + /** Chooses each in-scope record shape's enumeration table from its * witnesses. Runs once retained reachability has settled the historical * tables and derived metadata, before the order-snapshotting helper bodies @@ -239,35 +281,53 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio /** Every record shape a type can hold, through fields, arms, elements, * callables, promises and class fields. */ /** Each walk shares one `seen` set per result set, so every type graph - * is walked once however many roots reach it. */ - const shapesIn = (type: IrType, out: Set, seen: Set): void => { - const visit = (t: IrType): boolean => { + * is walked once however many roots reach it. + * + * `produced` walks only what a checked conversion into the type can + * build. It answers an existing native instance for a class type (a plain + * object cannot acquire a class brand; only %Error rebuilds), so the + * records the instance's fields hold come from the class's own writes and + * its checked field setters, each a witness of its own. A checked + * function's parameters receive native values: a record parameter is + * passed out, never built, though a callback among them receives values + * built from JavaScript again. */ + const shapesIn = (type: IrType, out: Set, seen: Set, produced = false): void => { + const visit = (t: IrType, built: boolean): boolean => { + const mark = built ? "" : "-"; if (t.kind === "record") { - if (seen.has("r:" + t.shapeId)) return true; - seen.add("r:" + t.shapeId); - out.add(t.shapeId); + if (seen.has(mark + "r:" + t.shapeId)) return true; + seen.add(mark + "r:" + t.shapeId); + if (built) out.add(t.shapeId); const shape = shapes.get(t.shapeId); if (shape) { - for (const f of shape.fields) visit(f.type); - if (shape.indexValue) visit(shape.indexValue); + // A record passed out can be written through: its fields take + // values JavaScript builds. + for (const f of shape.fields) visit(f.type, true); + if (shape.indexValue) visit(shape.indexValue, true); } return true; } if (t.kind === "union") { - if (seen.has("u:" + t.unionId)) return true; - seen.add("u:" + t.unionId); - for (const arm of lowerer.unions.get(t.unionId)?.arms ?? []) visit(arm); + if (seen.has(mark + "u:" + t.unionId)) return true; + seen.add(mark + "u:" + t.unionId); + for (const arm of lowerer.unions.get(t.unionId)?.arms ?? []) visit(arm, built); return true; } if (t.kind === "object") { + if (produced && t.className !== "%Error") return true; if (seen.has("c:" + t.className)) return true; seen.add("c:" + t.className); - for (const f of lowerer.classes.get(t.className)?.def.fields ?? []) visit(f.type); + for (const f of lowerer.classes.get(t.className)?.def.fields ?? []) visit(f.type, true); return true; } - return everyTypeChild(t, visit); + if (produced && t.kind === "func") { + for (const param of t.params) visit(param, !built); + return visit(t.ret, built); + } + // Containers passed out stay writable: their elements count as built. + return everyTypeChild(t, (child) => visit(child, true)); }; - visit(type); + visit(type, true); }; const opaqueShapes = new Set(); const opaqueSeen = new Set(); @@ -280,7 +340,7 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio const declaredSeen = new Set(); const producedBy = (type: IrType, opaque: boolean): void => { if (opaque) shapesIn(type, opaqueShapes, opaqueSeen); - else shapesIn(type, runtimeShapes, runtimeSeen); + else shapesIn(type, runtimeShapes, runtimeSeen, true); }; // ── collection ────────────────────────────────────────────────────── @@ -388,7 +448,12 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio return; case "dynCheck": if (e.adopt === true) shapesIn(e.type, declaredShapes, declaredSeen); - else if (!orderPreservingClone(e)) producedBy(e.type, false); + // Conversions that build no record: a boxed primitive, and a + // replaceable method's result, which is the value the method + // returns (boxed and checked back unchanged) or what a replacement + // installed on the instance returns. + else if (!orderPreservingClone(e) && !primitiveSource(e) && !notes.isMethodResult(e)) + producedBy(e.type, false); return; case "dynDestrCheck": case "caughtCheck": @@ -649,6 +714,29 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio for (const c of f.copies) deps.add(c.source); return [...deps]; }; + /** `{ ...value, ...rest }` that starts with a spread of a value of the + * literal's own shape holding every key: it creates its keys in the + * value's order, and the later segments only overwrite them. With no + * checked conversion building the shape, every value enumerates in the + * one table this pass chooses, so the copy agrees with any choice and + * adds no edge. */ + const copiesItself = ( + id: string, + f: ShapeFacts, + table: readonly string[], + segments: readonly KeyOrderSegment[], + ): boolean => { + const first = segments[0]; + return ( + first !== undefined && + first.kind === "spread" && + first.shapeId === id && + !first.maybeAbsent && + !f.runtime && + segments.every((s) => s.kind !== "opaque") && + table.every((k) => !maybeAbsent(id, k)) + ); + }; /** One shape's order given its sources' current orders. */ const resolveOne = (id: string, current: (source: string) => Resolved): Resolved => { const shape = shapes.get(id); @@ -666,6 +754,7 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio const edges: [string, string][] = []; if (f.pinned) chain(table, edges); for (const segments of f.literals) { + if (copiesItself(id, f, table, segments)) continue; const sequences = literalSequences(segments, keySet, current); if (sequences === null) return "unknown"; for (const seq of sequences) chain(seq, edges); diff --git a/packages/compiler/src/ir/traverse.ts b/packages/compiler/src/ir/traverse.ts index 251904351..80425a056 100644 --- a/packages/compiler/src/ir/traverse.ts +++ b/packages/compiler/src/ir/traverse.ts @@ -473,9 +473,10 @@ export function everyStmtChild( throw new Error("unhandled IR node"); } -/** Called with each record literal mapExprChildren rebuilds and its - * replacement, so a side table keyed by node identity (the lowering's - * key-order witness notes) follows the literal into the rewritten tree. */ +/** Called with each record literal and checked conversion mapExprChildren + * rebuilds and its replacement, so a side table keyed by node identity (the + * lowering's key-order witness notes) follows the node into the rewritten + * tree. */ let recordLiteralRebuilt: ((from: IrExpr, to: IrExpr) => void) | null = null; /** Installs (or with null removes) the record-literal rebuild observer. */ @@ -504,8 +505,11 @@ export function mapExprChildren( return node; case "libCall": return { ...node, args: mapOf(node.args, expr) }; - case "dynCheck": - return { ...node, value: expr(node.value) }; + case "dynCheck": { + const rebuilt: IrExpr = { ...node, value: expr(node.value) }; + if (recordLiteralRebuilt !== null) recordLiteralRebuilt(node, rebuilt); + return rebuilt; + } case "strEq": return { ...node, left: expr(node.left), right: expr(node.right) }; case "fieldGet": diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7736-key-order-replaceable-method-result/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7736-key-order-replaceable-method-result/main.ts.json new file mode 100644 index 000000000..006d4a8c7 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7736-key-order-replaceable-method-result/main.ts.json @@ -0,0 +1,7 @@ +{ + "order": [ + "/tests/corpus/7736-key-order-replaceable-method-result/registry.js", + "/tests/corpus/7736-key-order-replaceable-method-result/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7737-key-order-self-spread.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7737-key-order-self-spread.ts.json new file mode 100644 index 000000000..97f2ca7bf --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7737-key-order-self-spread.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7737-key-order-self-spread.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7736-key-order-replaceable-method-result/main.ts b/tests/corpus/7736-key-order-replaceable-method-result/main.ts new file mode 100644 index 000000000..5ebbfa761 --- /dev/null +++ b/tests/corpus/7736-key-order-replaceable-method-result/main.ts @@ -0,0 +1,106 @@ +import { Registry, describe, titleIndex } from "./registry.js"; + +// Records keep the key order their object literals write when values of +// the same type also pass through checked conversions that build none of +// them: +// - a typed method the program may replace on its instance (the +// computed-key write in registry.js makes every method name replaceable) +// returns the records it stores, checked back from the guarded call; +// - an index spec from JavaScript, checked into a type whose function +// field takes a Book, receives Books but never builds one. +// JSON, Object.keys and spreads of those records follow insertion order, +// not the interface's declared order. +interface Book { + id: string; + title: string; + author: string; + createdAt: string; + updatedAt: string; +} + +type Draft = { title: string; author: string }; + +interface IndexSpec { + name: string; + key: (row: T) => string; +} + +class Repository { + readonly #rows = new Map(); + + insert(row: T): T { + this.#rows.set(row.id, row); + return row; + } + + require(id: string): T { + const row = this.#rows.get(id); + if (row === undefined) throw new Error(`missing ${id}`); + return row; + } + + find(id: string): T | undefined { + return this.#rows.get(id); + } + + all(): T[] { + return [...this.#rows.values()]; + } +} + +class Service { + readonly #books = new Repository(); + readonly #index: IndexSpec = titleIndex; + #clock = 0; + + #now(): string { + this.#clock++; + return `t${this.#clock}`; + } + + create(draft: Draft, id: string): Book { + const at = this.#now(); + return this.#books.insert({ ...draft, id, createdAt: at, updatedAt: at }); + } + + update(id: string, patch: Partial): Book { + const current = this.#books.require(id); + return this.#books.insert({ ...current, ...patch, updatedAt: this.#now() }); + } + + get(id: string): Book { + return this.#books.require(id); + } + + maybe(id: string): Book | undefined { + return this.#books.find(id); + } + + list(): Book[] { + return this.#books.all(); + } + + indexed(): string[] { + return this.list().map((book) => `${this.#index.name}=${this.#index.key(book)}`); + } +} + +const registry = new Registry(); +console.log(describe(registry)); + +const service = new Service(); +console.log(JSON.stringify(service.create({ title: "Dune", author: "Herbert" }, "b1"))); +console.log(JSON.stringify(service.create({ title: "Earthsea", author: "Le Guin" }, "b2"))); +console.log(JSON.stringify(service.update("b1", { title: "Dune Messiah" }))); +console.log(Object.keys(service.get("b2")).join(",")); +const found = service.maybe("b1"); +if (found !== undefined) console.log(JSON.stringify(found)); +console.log(JSON.stringify(service.list())); +const copy = { ...service.get("b1") }; +console.log(Object.keys(copy).join(",")); +console.log(service.indexed().join(" ")); +try { + service.get("missing"); +} catch (error) { + console.log((error as Error).message); +} diff --git a/tests/corpus/7736-key-order-replaceable-method-result/registry.js b/tests/corpus/7736-key-order-replaceable-method-result/registry.js new file mode 100644 index 000000000..d6da1edc5 --- /dev/null +++ b/tests/corpus/7736-key-order-replaceable-method-result/registry.js @@ -0,0 +1,21 @@ +// Installs one handler per method name with a computed-key write, the way +// router classes attach `get`, `post`, ... in their constructor. The +// compiler cannot tell which names such a write replaces. +const METHODS = ["get", "post"]; + +export class Registry { + constructor() { + METHODS.forEach((method) => { + this[method] = (...args) => `${method}:${args.length}`; + }); + } +} + +export function describe(registry) { + return `${registry.get(1, 2)} ${registry.post()}`; +} + +export const titleIndex = { + name: "title", + key: (row) => row.title.toLowerCase(), +}; diff --git a/tests/corpus/7737-key-order-self-spread.ts b/tests/corpus/7737-key-order-self-spread.ts new file mode 100644 index 000000000..5577f7531 --- /dev/null +++ b/tests/corpus/7737-key-order-self-spread.ts @@ -0,0 +1,34 @@ +// A literal that starts by spreading a value of its own type copies that +// value's keys in their order, then overwrites some of them. It must not +// stop the type from taking the order its other literals write: the +// records below enumerate as created (title before id), not in the +// interface's declared order. +interface Book { + id: string; + title: string; + tags: string[]; + updatedAt: string; + version: number; +} + +type Draft = { title: string; tags: string[] }; + +function create(draft: Draft, id: string): Book { + return { ...draft, id, updatedAt: "t1", version: 1 }; +} + +function update(current: Book, patch: Partial): Book { + return { ...current, ...patch, updatedAt: "t2", version: current.version + 1 }; +} + +function touch(current: Book): Book { + return { ...current, version: current.version + 1 }; +} + +const book = create({ title: "Dune", tags: ["sf"] }, "b1"); +console.log(JSON.stringify(book)); +const updated = update(book, { title: "Dune Messiah" }); +console.log(JSON.stringify(updated)); +console.log(Object.keys(touch(updated)).join(",")); +console.log(Object.keys(update(updated, { tags: [] })).join(" ")); +for (const key in touch(book)) console.log(key); diff --git a/tests/harness/corpus-costs/7736-key-order-replaceable-method-result/main.ts.json b/tests/harness/corpus-costs/7736-key-order-replaceable-method-result/main.ts.json new file mode 100644 index 000000000..6a79d2c04 --- /dev/null +++ b/tests/harness/corpus-costs/7736-key-order-replaceable-method-result/main.ts.json @@ -0,0 +1 @@ +{ "plain": 10900, "san": 17500 } diff --git a/tests/harness/corpus-costs/7737-key-order-self-spread.ts.json b/tests/harness/corpus-costs/7737-key-order-self-spread.ts.json new file mode 100644 index 000000000..de3f40849 --- /dev/null +++ b/tests/harness/corpus-costs/7737-key-order-self-spread.ts.json @@ -0,0 +1 @@ +{ "plain": 3800, "san": 6100 } From d1f3eda7326b81ca84b9fcdb6a29f574aa14f54a Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 04:50:06 -0500 Subject: [PATCH 033/184] Freeze arrays, Maps and Sets that other references still hold Object.freeze of an array that another reference may still hold stopped at compile time with SC2020, and freezing an array, Map or Set held in an untyped value threw SC2020 at run time. Node freezes the object itself: a frozen array refuses every later write through any reference with the TypeError of the operation's first write, and a frozen Map or Set only stops being extensible, since its entries are not properties. - An array a program may freeze keeps a frozen bit in its storage flags. An array type is freezable when a typed Object.freeze names it, or, in a program that freezes untyped values, when a value of it can be held as a live untyped reference. The writes of those types test the bit with the guard published arrays already use; other array types are unchanged. - The guard's errors follow Node for every operation: splice, fill, copyWithin, sort and reverse throw at their first write, and do not throw when they would write nothing. Published arrays get the same errors, and a statement splice of a published array is now guarded. - A Map or Set records that it is not extensible, for Object.freeze, seal and preventExtensions, typed or untyped. Object.isFrozen, isSealed and isExtensible report it, and its entries stay writable. - Writes to a frozen array through an untyped value are refused with the same errors, as are writes to frozen checked arrays and objects, which now name the property and the object as Node does. - Programs that neither freeze nor publish arrays compile to the same IR and LLVM. Corpus 7406 covers typed and untyped freezes of arrays, Maps and Sets, every array write on frozen arrays of zero, one and three elements, comparator sorts, nested arrays, and frozen arrays that are released, and matches Node. Refs #1374 Co-authored-by: blurname <50226652+blurname@users.noreply.github.com> --- packages/compiler/src/backend/llvm/common.ts | 3 + packages/compiler/src/backend/llvm/emitter.ts | 63 +++++++- .../src/backend/llvm/expr-containers.ts | 39 ++++- .../compiler/src/backend/llvm/lib-dispatch.ts | 1 + .../compiler/src/backend/llvm/lib-shared.ts | 4 + .../compiler/src/backend/publication.test.ts | 31 ++++ packages/compiler/src/backend/publication.ts | 27 +++- .../src/frontend/lowering/frozen-records.ts | 62 +++++--- .../src/frontend/lowering/lower-calls.ts | 57 ++++++- .../compiler/src/frontend/lowering/lowerer.ts | 9 +- .../compiler/src/ir/builtin-signatures.ts | 10 ++ .../src/ir/function-references.test.ts | 1 + packages/compiler/src/ir/ir.ts | 4 + .../corpus/7406-frozen-containers.ts.json | 6 + packages/runtime/src/scr_array.c | 122 +++++++++++++++ packages/runtime/src/scr_array.h | 22 ++- packages/runtime/src/scr_dyn_invoke.c | 35 +++-- packages/runtime/src/scr_json.c | 142 +++++++++++++----- packages/runtime/src/scr_map.c | 6 + packages/runtime/src/scr_map.h | 8 + packages/runtime/src/scr_publish.c | 58 +------ packages/runtime/src/scr_rc.h | 8 +- tests/corpus/7406-frozen-containers.ts | 132 ++++++++++++++++ .../7406-frozen-containers.ts.json | 1 + 24 files changed, 690 insertions(+), 161 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7406-frozen-containers.ts.json create mode 100644 tests/corpus/7406-frozen-containers.ts create mode 100644 tests/harness/corpus-costs/7406-frozen-containers.ts.json diff --git a/packages/compiler/src/backend/llvm/common.ts b/packages/compiler/src/backend/llvm/common.ts index ac4e28708..cca1b603b 100644 --- a/packages/compiler/src/backend/llvm/common.ts +++ b/packages/compiler/src/backend/llvm/common.ts @@ -29,6 +29,9 @@ export const SCR_ARR_FIELD = { * (SCR_ARR_STATES in scr_array.h). */ export const SCR_ARR_STATES = 16; +/** The `spare` flag of a frozen %ScrArr (SCR_ARR_FROZEN in scr_array.h). */ +export const SCR_ARR_FROZEN = 32; + /** Loads the i32 `len` or `cap` of a %ScrArr at `ptr` into `dest` as a * size-type count. */ export function loadArrCount( diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index c0cc9fef2..a66e00736 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -184,7 +184,7 @@ import { } from "./string-slices.js"; import { LazyCaptures } from "./lazy-captures.js"; import { StackCaptures } from "./stack-captures.js"; -import { emitFieldAbsentTest, f64Lit, SCR_ARR_FIELD } from "./common.js"; +import { emitFieldAbsentTest, f64Lit, SCR_ARR_FIELD, SCR_ARR_FROZEN } from "./common.js"; import { ffiExtendsNarrowIntegers } from "../targets.js"; import { emitLiteralExpr, @@ -6264,6 +6264,41 @@ export class LlEmitter { return ok; } + /** Branches on "the array `arr` is frozen" (SCR_ARR_FROZEN in its storage + * flags, scr_array.h), or published too when `published`; like + * emitPublishedTest, leaves the builder in the cold block and answers the + * continuation label. */ + private emitFrozenArrayTest(arr: string, published: boolean): string { + const B = this.B; + const sparep = B.tmp(), + spare = B.tmp(), + bit = B.tmp(); + let frozen = B.tmp(); + B.line( + `${sparep} = getelementptr inbounds %ScrArr, ptr ${arr}, i32 0, i32 ${SCR_ARR_FIELD.spare}`, + ); + B.line(`${spare} = load i16, ptr ${sparep}`); + B.line(`${bit} = and i16 ${spare}, ${SCR_ARR_FROZEN}`); + B.line(`${frozen} = icmp ne i16 ${bit}, 0`); + if (published) { + const rc = B.tmp(), + immortal = B.tmp(), + either = B.tmp(); + B.line(`${rc} = load ${this.sizeType}, ptr ${arr}`); + B.line(`${immortal} = icmp eq ${this.sizeType} ${rc}, -1`); + B.line(`${either} = or i1 ${frozen}, ${immortal}`); + frozen = either; + } + this.declare(`declare i1 @llvm.expect.i1(i1, i1)`); + const expected = B.tmp(); + B.line(`${expected} = call i1 @llvm.expect.i1(i1 ${frozen}, i1 false)`); + const cold = B.newLabel("arr.frozen"), + ok = B.newLabel("arr.ok"); + B.condBr(expected, cold, ok); + B.startBlock(cold); + return ok; + } + private publishedNamesSym: string | null = null; /** The class-name table the hierarchy field guards index by preorder. */ @@ -6328,15 +6363,29 @@ export class LlEmitter { op: number, index = f64Lit(0), count = f64Lit(0), + third: string | null = null, ): void { - if (this.publishedTypes?.containerGuarded(arrType) !== true) return; + const published = this.publishedTypes; + if (published?.containerGuarded(arrType) !== true) return; const B = this.B; - const ok = this.emitPublishedTest(arr); - this.declare(`declare zeroext i1 @scr_throw_published_array(i32, ptr, double, double)`); + const guards = published.arrayGuards(arrType); + const ok = guards.frozen + ? this.emitFrozenArrayTest(arr, guards.published) + : this.emitPublishedTest(arr); const threw = B.tmp(); - B.line( - `${threw} = call zeroext i1 @scr_throw_published_array(i32 ${op}, ptr ${arr}, double ${index}, double ${count})`, - ); + if (third === null) { + this.declare(`declare zeroext i1 @scr_throw_published_array(i32, ptr, double, double)`); + B.line( + `${threw} = call zeroext i1 @scr_throw_published_array(i32 ${op}, ptr ${arr}, double ${index}, double ${count})`, + ); + } else { + this.declare( + `declare zeroext i1 @scr_throw_published_array_range(i32, ptr, double, double, double)`, + ); + B.line( + `${threw} = call zeroext i1 @scr_throw_published_array_range(i32 ${op}, ptr ${arr}, double ${index}, double ${count}, double ${third})`, + ); + } const unwind = B.newLabel("pub.throw"); B.condBr(threw, unwind, ok); B.startBlock(unwind); diff --git a/packages/compiler/src/backend/llvm/expr-containers.ts b/packages/compiler/src/backend/llvm/expr-containers.ts index 1ba5da818..93cc615a4 100644 --- a/packages/compiler/src/backend/llvm/expr-containers.ts +++ b/packages/compiler/src/backend/llvm/expr-containers.ts @@ -257,6 +257,16 @@ function emitDiscarded(host: LlvmEmitterContext, e: IrExpr): void { const start = host.emitExpr(e.args[0]!); const count = e.args[1] ? host.emitExpr(e.args[1]).name : F64_INF; const values = items.map((item) => host.emitExpr(item)); + // Before the values move into the buffer: a throw releases them with the + // frame. + host.emitPublishedArrayGuard( + r.name, + e.receiver.type, + 10, + start.name, + count, + f64Lit(values.length), + ); if (acc === "ref") values.forEach((value) => host.toHeap(value)); let buffer = "null"; if (values.length > 0) { @@ -978,7 +988,25 @@ export function emitArrIntrinsic( case "fill": case "fillUndefined": { const args = e.args.map((arg) => host.emitExpr(arg)); - host.emitPublishedArrayGuard(r.name, e.receiver.type, 7); + // fill's range is its last two arguments; copyWithin's target, + // start and end are its three. + if (e.method === "copyWithin") + host.emitPublishedArrayGuard( + r.name, + e.receiver.type, + 9, + args[0]!.name, + args[1]!.name, + args[2]!.name, + ); + else + host.emitPublishedArrayGuard( + r.name, + e.receiver.type, + 8, + args[args.length - 2]!.name, + args[args.length - 1]!.name, + ); if (host.checkedFills.has(e)) { // A new int32 array with checked stores, filled with a value that // may not be an int32: widen it first then. @@ -1042,7 +1070,7 @@ export function emitArrIntrinsic( // removes to the end (+Infinity, the slice convention). const start = host.emitExpr(e.args[0]!); const cnt = e.args[1] ? host.emitExpr(e.args[1]).name : F64_INF; - host.emitPublishedArrayGuard(r.name, e.receiver.type, 7); + host.emitPublishedArrayGuard(r.name, e.receiver.type, 10, start.name, cnt, f64Lit(0)); host.declare(`declare ptr @scr_arr_splice(ptr, double, double)`); const t = B.tmp(); B.line(`${t} = call ptr @scr_arr_splice(ptr ${r.name}, double ${start.name}, double ${cnt})`); @@ -1052,7 +1080,12 @@ export function emitArrIntrinsic( const start = host.emitExpr(e.args[0]!); const count = host.emitExpr(e.args[1]!); const items = emitBorrowedInput(host, e.args[2]!); - host.emitPublishedArrayGuard(r.name, e.receiver.type, 7); + if (host.publishedTypes?.containerGuarded(e.receiver.type) === true) { + host.declare(`declare double @scr_arr_len(ptr)`); + const n = B.tmp(); + B.line(`${n} = call double @scr_arr_len(ptr ${items.name})`); + host.emitPublishedArrayGuard(r.name, e.receiver.type, 10, start.name, count.name, n); + } host.declare(`declare ptr @scr_arr_splice_insert(ptr, double, double, ptr)`); const t = B.tmp(); B.line( diff --git a/packages/compiler/src/backend/llvm/lib-dispatch.ts b/packages/compiler/src/backend/llvm/lib-dispatch.ts index 4f1a4fef8..cc8953fc4 100644 --- a/packages/compiler/src/backend/llvm/lib-dispatch.ts +++ b/packages/compiler/src/backend/llvm/lib-dispatch.ts @@ -891,6 +891,7 @@ export function emitLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue { case "blob": case "wasm": case "array": + case "collection": case "record": case "disposable": case "punycode": diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 6fbad7e11..f8e59e7b7 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1016,6 +1016,10 @@ export const LIB_FN_SYMS: Record = { "class.throwNotExtensible": "scr_class_throw_not_extensible", "record.freeze": "scr_record_freeze", "record.throwFrozen": "scr_record_throw_frozen", + "array.freeze": "scr_arr_freeze", + "array.isFrozen": "scr_arr_is_frozen", + "collection.preventExtensions": "scr_map_prevent_extensions", + "collection.isExtensible": "scr_map_is_extensible", "dyn.nativeSetNew": "scr_dyn_native_set_new", "dyn.nativeSetIs": "scr_dyn_native_set_is", "dyn.nativeMapIs": "scr_dyn_native_map_is", diff --git a/packages/compiler/src/backend/publication.test.ts b/packages/compiler/src/backend/publication.test.ts index 149ea4bb9..7b3e5f5cc 100644 --- a/packages/compiler/src/backend/publication.test.ts +++ b/packages/compiler/src/backend/publication.test.ts @@ -85,3 +85,34 @@ test("published types close over fields, containers and whole hierarchies", () = computePublishedTypes({ ...withStores, functions: withStores.functions.slice(1) }), ).toBeNull(); }); + +test("freezable array types carry the array write guard without a publish", () => { + const numbers = arrayOf(F64); + const push: IrStmt = { + kind: "exprStmt", + expr: { + kind: "arrIntrinsic", + method: "push", + receiver: ref("a", numbers), + args: [{ kind: "numLit", value: 1, type: F64, loc }], + type: F64, + loc, + }, + loc, + }; + const mod: IrModule = { + irVersion: 17, + sourceFile: loc.file, + entry: "main", + functions: [fn("main", []), fn("pushNumber", [push])], + freezableArrays: ["array"], + }; + const guarded = computePublishedTypes(mod)!; + expect(guarded.containerGuarded(numbers)).toBe(true); + expect(guarded.arrayGuards(numbers)).toEqual({ published: false, frozen: true }); + expect(guarded.containerGuarded(arrayOf(STRING))).toBe(false); + // A frozen array's push throws, so the function may throw. + expect(computeMayThrow(mod).fns.has("pushNumber")).toBe(true); + const { freezableArrays: _frozen, ...plain } = mod; + expect(computePublishedTypes(plain)).toBeNull(); +}); diff --git a/packages/compiler/src/backend/publication.ts b/packages/compiler/src/backend/publication.ts index af65fa39a..7f7918fb1 100644 --- a/packages/compiler/src/backend/publication.ts +++ b/packages/compiler/src/backend/publication.ts @@ -46,6 +46,10 @@ export class PublishedTypes { readonly records = new Set(); /** typeKeys of array, map and set types. */ readonly containers = new Set(); + /** typeKeys of the array types the program may freeze + * (IrModule.freezableArrays): a frozen array refuses writes like a + * published one, so their stores carry the same guard. */ + readonly frozenArrays = new Set(); /** Constructor stores into a `this` nothing else can reach yet * (immutability.ts): never a published target, so never guarded. */ privateStores: ReadonlySet = new Set(); @@ -64,10 +68,16 @@ export class PublishedTypes { } containerGuarded(t: IrType): boolean { - return ( - (t.kind === "array" || t.kind === "map" || t.kind === "set") && - this.containers.has(typeKey(t)) - ); + if (t.kind !== "array" && t.kind !== "map" && t.kind !== "set") return false; + const key = typeKey(t); + return this.containers.has(key) || this.frozenArrays.has(key); + } + + /** Whether an array of type `t` may be published, and whether it may be + * frozen: the two states its write guard tests. */ + arrayGuards(t: IrType): { published: boolean; frozen: boolean } { + const key = typeKey(t); + return { published: this.containers.has(key), frozen: this.frozenArrays.has(key) }; } } @@ -82,7 +92,13 @@ export function computePublishedTypes(mod: IrModule): PublishedTypes | null { }; for (const fn of mod.functions) everyStmtList(fn.body, { expr: visit, stmt: (_s: IrStmt) => true }); - if (roots.length === 0) return null; + const frozenArrays = mod.freezableArrays ?? []; + if (roots.length === 0 && frozenArrays.length === 0) return null; + if (roots.length === 0) { + const frozen = new PublishedTypes(); + for (const key of frozenArrays) frozen.frozenArrays.add(key); + return frozen; + } const classes = new Map((mod.classes ?? []).map((c) => [c.name, c])); const children = new Map(); @@ -95,6 +111,7 @@ export function computePublishedTypes(mod: IrModule): PublishedTypes | null { const records = new Map((mod.records ?? []).map((r) => [r.id, r])); const unions = new Map((mod.unions ?? []).map((u) => [u.id, u])); const out = new PublishedTypes(); + for (const key of frozenArrays) out.frozenArrays.add(key); out.privateStores = computeConstructionFacts(mod).privateStores; const seen = new Set(); const stack = [...roots]; diff --git a/packages/compiler/src/frontend/lowering/frozen-records.ts b/packages/compiler/src/frontend/lowering/frozen-records.ts index f61de39ec..30eb9e455 100644 --- a/packages/compiler/src/frontend/lowering/frozen-records.ts +++ b/packages/compiler/src/frontend/lowering/frozen-records.ts @@ -3,6 +3,7 @@ import { BOOL, VOID, shapeHasAccessorSlots, + typeKey, type IrExpr, type IrFunction, type IrStmt, @@ -15,24 +16,31 @@ import type { Lowerer } from "./lowerer.js"; import { recordHasOwnHelper } from "./lower-calls.js"; import { isStrictCode } from "./strict-mode.js"; -/* Object.freeze of records that other references can still write. Only the - * shapes a program may freeze pay for it: their values keep a frozen bit in - * the rc word (scr_rc.h SCR_RC_FROZEN), their release ignores the bit, and - * every write to them checks it first (guardFrozenRecordWrites). A shape is +/* Object.freeze of records and arrays that other references can still + * write. Only the types a program may freeze pay for it. A record keeps its + * frozen bit in the rc word (scr_rc.h SCR_RC_FROZEN), which its release + * ignores, and every write to it checks the bit first + * (guardFrozenWrites). An array keeps it in its storage flags + * (scr_array.h SCR_ARR_FROZEN), which the backend's array write guards test + * for the array types the module lists (IrModule.freezableArrays). A type is * freezable when a static Object.freeze names it, or, in a program that * freezes untyped values, when a value of it or of a type reaching it can be * held as a live untyped reference. */ type RecordType = Extract; type RecordWrite = Extract; +type FreezeFn = "record.freeze" | "array.freeze" | "collection.preventExtensions"; -/** Object.freeze of a record: sets its frozen bit and answers the record. */ -export function lowerRecordFreeze( +/** Object.freeze of a record or an array, or Object.freeze, seal or + * preventExtensions of a Map or Set: `fn` sets the value's bit, and the + * call answers the value. */ +export function lowerNativeFreeze( lowerer: Lowerer, + fn: FreezeFn, target: IrExpr, - type: RecordType, loc: SrcLoc, ): IrExpr { + const type = target.type; const slot = lowerer.declareHiddenLocal("%freezeTarget", type); const receiver = (): IrExpr => varRef(slot.id, type, loc); return { @@ -41,7 +49,7 @@ export function lowerRecordFreeze( { kind: "varDecl", localId: slot.id, init: target, loc }, { kind: "exprStmt", - expr: { kind: "libCall", fn: "record.freeze", args: [receiver()], type: VOID, loc }, + expr: { kind: "libCall", fn, args: [receiver()], type: VOID, loc }, loc, }, ], @@ -92,9 +100,14 @@ function holdsAbsentState(e: IrExpr): boolean { return e.kind === "ternary" && (holdsAbsentState(e.then) || holdsAbsentState(e.else_)); } -/** The shapes whose values the program may freeze. */ -function freezableShapes(lowerer: Lowerer, functions: IrFunction[]): Set { +/** The record shapes and array types (typeKeys) whose values the program + * may freeze. */ +function freezableTypes( + lowerer: Lowerer, + functions: IrFunction[], +): { shapes: Set; arrays: Set } { const shapes = new Set(); + const arrays = new Set(); const live: IrType[] = []; let dynFreeze = false; for (const fn of functions) @@ -103,13 +116,16 @@ function freezableShapes(lowerer: Lowerer, functions: IrFunction[]): Set if (e.kind === "libCall" && e.fn === "record.freeze") { const target = e.args[0]; if (target && target.type.kind === "record") shapes.add(target.type.shapeId); + } else if (e.kind === "libCall" && e.fn === "array.freeze") { + const target = e.args[0]; + if (target && target.type.kind === "array") arrays.add(typeKey(target.type)); } else if (e.kind === "libCall" && e.fn === "dyn.freeze") dynFreeze = true; else if (e.kind === "dynFrom") live.push(e.value.type); return true; }, stmt: () => true, }); - if (!dynFreeze) return shapes; + if (!dynFreeze) return { shapes, arrays }; // An untyped reference to a class instance, or to a record or array // boxed live, reaches the values its fields and elements hold as further // live references. Class instances also cross into untyped code through @@ -135,7 +151,14 @@ function freezableShapes(lowerer: Lowerer, functions: IrFunction[]): Set seen.add(`union:${type.unionId}`); for (const arm of lowerer.unions.get(type.unionId)?.arms ?? []) stack.push(arm); break; - case "array": + case "array": { + const key = typeKey(type); + if (seen.has(key)) break; + seen.add(key); + arrays.add(key); + stack.push(type.elem); + break; + } case "set": stack.push(type.elem); break; @@ -156,7 +179,7 @@ function freezableShapes(lowerer: Lowerer, functions: IrFunction[]): Set break; } } - return shapes; + return { shapes, arrays }; } /** The guarded form of a write to a freezable record: the receiver and the @@ -186,15 +209,18 @@ function guardedWrite( } /** Checks the frozen bit before every write to a record of a freezable - * shape. A write to a frozen record throws Node's TypeError in strict mode - * code; in sloppy-mode code it leaves the record unchanged. Deleting a key - * the record does not have succeeds either way. */ -export function guardFrozenRecordWrites(lowerer: Lowerer, functions: IrFunction[]): void { - const freezable = freezableShapes(lowerer, functions); + * shape, and lists the freezable array types, whose writes the backend + * guards (IrModule.freezableArrays). A write to a frozen record throws + * Node's TypeError in strict mode code; in sloppy-mode code it leaves the + * record unchanged. Deleting a key the record does not have succeeds either + * way. */ +export function guardFrozenWrites(lowerer: Lowerer, functions: IrFunction[]): void { + const { shapes: freezable, arrays } = freezableTypes(lowerer, functions); for (const id of freezable) { const shape = lowerer.shapes.get(id); if (shape) shape.freezable = true; } + for (const key of arrays) lowerer.freezableArrays.add(key); if (freezable.size === 0) return; const lifted = lowerer.liftedFns.length; for (const fn of functions) { diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index e2ed5f9fe..819f90534 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -245,7 +245,7 @@ import { import { fenceSymbolFieldCopy } from "./symbol-fields.js"; import { lowerClassDataDescriptor, lowerClassDescriptorRead } from "./class-descriptors.js"; import { lowerClassFieldDefine, lowerClassIntegrity } from "./class-attributes.js"; -import { freezableRecordType, lowerRecordFreeze } from "./frozen-records.js"; +import { freezableRecordType, lowerNativeFreeze } from "./frozen-records.js"; import { classStaticDataFor } from "./class-static-data.js"; import { lowerUrlFactory } from "./lower-url.js"; import { nullishValueUnitOf } from "./binding-analysis.js"; @@ -8422,6 +8422,22 @@ function lowerObjectStaticCall( locOf(call), ); if (fnIntegrity) return fnIntegrity; + // A Map or Set has no own properties: freeze, seal and preventExtensions + // all only make it not extensible. + if (value.type.kind === "map" || value.type.kind === "set") { + if (member === "preventExtensions" || member === "seal") + return lowerNativeFreeze(lowerer, "collection.preventExtensions", value, locOf(call)); + const extensible: IrExpr = { + kind: "libCall", + fn: "collection.isExtensible", + args: [value], + type: BOOL, + loc: locOf(call), + }; + return member === "isExtensible" + ? extensible + : { kind: "unary", op: "!", operand: extensible, type: BOOL, loc: locOf(call) }; + } if (member === "preventExtensions" || member === "seal") { const integrity = lowerClassIntegrity(lowerer, call, member, value); if (integrity) return integrity; @@ -8461,15 +8477,36 @@ function lowerObjectStaticCall( // reference would need a runtime frozen bit (strict mode throws). if (member === "isFrozen" && call.arguments.nodes.length === 1) { const argNode = call.arguments.nodes[0]!; - if (tryLowerExpression(lowerer, argNode)?.type.kind === "record") { - const record = lowerer.lowerExpr(argNode); - if (record.type.kind === "record") + const probedKind = tryLowerExpression(lowerer, argNode)?.type.kind; + if ( + probedKind === "record" || + probedKind === "array" || + probedKind === "map" || + probedKind === "set" + ) { + const native = lowerer.lowerExpr(argNode); + const loc = locOf(call); + if (native.type.kind === "record" || native.type.kind === "array") return { kind: "libCall", - fn: "record.isFrozen", - args: [record], + fn: native.type.kind === "record" ? "record.isFrozen" : "array.isFrozen", + args: [native], type: BOOL, - loc: locOf(call), + loc, + }; + if (native.type.kind === "map" || native.type.kind === "set") + return { + kind: "unary", + op: "!", + operand: { + kind: "libCall", + fn: "collection.isExtensible", + args: [native], + type: BOOL, + loc, + }, + type: BOOL, + loc, }; } const value = lowerer.lowerExprExpecting(argNode, DYN); @@ -8523,7 +8560,11 @@ function lowerObjectStaticCall( const integrity = lowerClassIntegrity(lowerer, call, "freeze", value); if (integrity) return integrity; if (freezableRecordType(lowerer, value.type)) - return lowerRecordFreeze(lowerer, value, value.type, locOf(call)); + return lowerNativeFreeze(lowerer, "record.freeze", value, locOf(call)); + if (value.type.kind === "array") + return lowerNativeFreeze(lowerer, "array.freeze", value, locOf(call)); + if (value.type.kind === "map" || value.type.kind === "set") + return lowerNativeFreeze(lowerer, "collection.preventExtensions", value, locOf(call)); lowerer.noLowering( "Object.freeze of a possibly-aliased value", call, diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index ba6d1214f..95deb648b 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -150,7 +150,7 @@ import { defaultAfterUndefined, lowerStaticallyUndefinedArgument } from "./optio import { type ProtocolDispatch } from "./class-protocol-dispatch.js"; import { ClassDynamicDispatch, trackClassFieldCreation } from "./class-dynamic-dispatch.js"; import { guardClassAttributeWrites } from "./class-attributes.js"; -import { guardFrozenRecordWrites, markSloppyRecordWrites } from "./frozen-records.js"; +import { guardFrozenWrites, markSloppyRecordWrites } from "./frozen-records.js"; import { applyHandleProperties } from "./handle-properties.js"; import { ClassProtocols } from "./class-protocols.js"; import { earlyReadVarSymbols } from "./var-hoisting.js"; @@ -3246,6 +3246,8 @@ export class Lowerer { * absent-state writes of `delete r.f` (markSloppyRecordWrites). */ readonly sloppyRecordWrites = new Set(); readonly recordFieldDeletes = new Set(); + /** typeKeys of the array types the program may freeze (IrModule.freezableArrays). */ + readonly freezableArrays = new Set(); readonly liftedFns: IrFunction[] = []; /** The first lifted function of each name (liftedFnNamed), for the * liftedFns indexed so far. liftedFns only grows. */ @@ -7694,7 +7696,7 @@ export class Lowerer { finalizePrivateBrands(this, functions); trackClassFieldCreation(this, functions); guardClassAttributeWrites(this, functions); - guardFrozenRecordWrites(this, functions); + guardFrozenWrites(this, functions); applyHandleProperties(this, functions); // Globals typed by a class that never REGISTERED (a JS class whose @@ -7803,6 +7805,9 @@ export class Lowerer { ...(this.threadSlotKeys.size > 0 ? { threadSlotKeys: [...this.threadSlotKeys].sort() } : {}), + ...(this.freezableArrays.size > 0 + ? { freezableArrays: [...this.freezableArrays].sort() } + : {}), ...(isNodeEsmFile(this.entry, this.program) ? {} : { commonJsEntry: true }), ...(this.ffiImports.length > 0 ? { ffiImports: [...this.ffiImports] } : {}), }; diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 1b2c1082d..717f7ed8a 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -283,6 +283,16 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ * object's text in the message ("#") and the operation (0 assign, * 1 add, 2 delete). */ "record.throwFrozen": { argTypes: [STRING, STRING, F64], result: VOID }, + /** Object.freeze of an array (scr_arr_freeze): sets its frozen bit, which + * the write guards of IrModule.freezableArrays test, and isFrozen reads. + * The receiver is any array type. */ + "array.freeze": { argTypes: [null], result: VOID }, + "array.isFrozen": { argTypes: [null], result: BOOL }, + /** Object.freeze, seal or preventExtensions of a Map or Set + * (scr_map_prevent_extensions), which only makes it not extensible: its + * entries stay writable. The receiver is any map or set type. */ + "collection.preventExtensions": { argTypes: [null], result: VOID }, + "collection.isExtensible": { argTypes: [null], result: BOOL }, "dyn.nativeSetNew": { argTypes: [DYN], result: DYN }, "dyn.nativeSetIs": { argTypes: [DYN], result: BOOL }, "dyn.nativeMapIs": { argTypes: [DYN], result: BOOL }, diff --git a/packages/compiler/src/ir/function-references.test.ts b/packages/compiler/src/ir/function-references.test.ts index 26539b17a..91e1d5f05 100644 --- a/packages/compiler/src/ir/function-references.test.ts +++ b/packages/compiler/src/ir/function-references.test.ts @@ -219,6 +219,7 @@ const MODULE_FIELDS: Record>, Role> = { "wasmUntranslated[].sha256": "other", "wasmUntranslated[].reason": "other", "threadSlotKeys[]": "other", + "freezableArrays[]": "other", }; const loc = { file: "references.ts", start: 0, end: 0 }; diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 3211e48f3..df23f0360 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -1056,6 +1056,10 @@ export interface IrModule { * each one's hierarchy carries a slot word (the slots.* intrinsics; * backend/llvm/thread-slots.ts). */ threadSlotKeys?: string[]; + /** typeKeys of the array types whose values the program may freeze + * (frontend frozen-records.ts): their writes test the array's frozen bit + * (SCR_ARR_FROZEN) and throw Node's TypeError when it is set. */ + freezableArrays?: string[]; /** The entry module is CommonJS under Node's classification. Node runs a * CommonJS main synchronously and drains process.nextTick callbacks * before promise jobs at the first checkpoint; an ES-module main's diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7406-frozen-containers.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7406-frozen-containers.ts.json new file mode 100644 index 000000000..34fb3ba0e --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7406-frozen-containers.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7406-frozen-containers.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_array.c b/packages/runtime/src/scr_array.c index 91328a1e8..af810483f 100644 --- a/packages/runtime/src/scr_array.c +++ b/packages/runtime/src/scr_array.c @@ -1689,6 +1689,126 @@ void scr_arr_release(ScrArr *a) { double scr_arr_len(ScrArr *a) { return SCR_ARR_LEN_F64(a->len); } +/* ── frozen arrays ────────────────────────────────────────────────────── + * Node's TypeErrors for writes to a frozen array, shared by the emitted + * guards of published and frozen arrays and the checked-dynamic tier. */ +static void scr_frozen_throw(const char *before, double index, const char *after) { + char text[64]; + size_t n = scr_f64_to_str(index, text); + ScrJsonBuf buffer; + scr_jb_init(&buffer); + scr_jb_puts(&buffer, before); + scr_jb_write(&buffer, text, n); + scr_jb_puts(&buffer, after); + scr_throw_error(SCR_ERR_TYPE, scr_jb_finish(&buffer)); +} + +/* A relative array index argument (fill, copyWithin, splice): ToIntegerOrInfinity + * of `value`, counted from the end when negative, clamped to [0, len]. */ +static size_t scr_frozen_relative_index(double value, size_t len) { + if (value != value) return 0; + double n = trunc(value); + if (n < 0) return n + (double)len <= 0 ? 0 : (size_t)(n + (double)len); + return n >= (double)len ? len : (size_t)n; +} + +static void scr_frozen_read_only(double index) { + scr_frozen_throw("Cannot assign to read only property '", index, "' of object '[object Array]'"); +} + +static void scr_frozen_not_extensible(double index) { + scr_frozen_throw("Cannot add property ", index, ", object is not extensible"); +} + +static void scr_frozen_undeletable(double index) { + scr_frozen_throw("Cannot delete property '", index, "' of [object Array]"); +} + +/* Each case finds the operation's first write in the order the spec makes + * its writes. */ +bool scr_array_frozen_error(int op, size_t len, double x, double y, double z) { + switch (op) { + case 0: + if (x >= 0 && x < (double)len && x == (double)(size_t)x) scr_frozen_read_only(x); + else scr_frozen_not_extensible(x); + return true; + case 1: + if (y == 0) break; + scr_frozen_not_extensible((double)len); + return true; + case 2: + case 6: + if (len == 0) break; + if (op == 6 && len > 1) scr_frozen_read_only(0); + else scr_frozen_undeletable((double)(len - 1)); + return true; + case 3: break; + case 4: + if (y == 0) break; + scr_frozen_not_extensible(len ? (double)len + y - 1 : 0); + return true; + case 5: + if (!(x >= 0 && x < (double)len && x == (double)(size_t)x)) return false; + scr_frozen_undeletable(x); + return true; + case 7: + if (len < 2) return false; + scr_frozen_read_only(0); + return true; + case 8: { + size_t k = scr_frozen_relative_index(x, len), final = scr_frozen_relative_index(y, len); + if (k >= final) return false; + scr_frozen_read_only((double)k); + return true; + } + case 9: { + size_t to = scr_frozen_relative_index(x, len), from = scr_frozen_relative_index(y, len); + size_t final = scr_frozen_relative_index(z, len); + if (final <= from || len <= to) return false; + size_t count = final - from < len - to ? final - from : len - to; + /* An overlapping copy toward the end writes its last index first. */ + if (from < to && to < from + count) to += count - 1; + scr_frozen_read_only((double)to); + return true; + } + case 10: { + size_t start = scr_frozen_relative_index(x, len); + double count = y != y ? 0 : trunc(y); + size_t removed = count <= 0 ? 0 : count >= (double)(len - start) ? len - start : (size_t)count; + size_t items = (size_t)z; + if (items < removed) { + /* Elements after the removed ones move down, then the tail goes. */ + if (start < len - removed) scr_frozen_read_only((double)(start + items)); + else scr_frozen_undeletable((double)(len - 1)); + return true; + } + if (items > removed) { + /* Elements after the removed ones move up, past the end first. */ + if (len - removed > start) scr_frozen_not_extensible((double)(len - removed + items - 1)); + else if (start < len) scr_frozen_read_only((double)start); + else scr_frozen_not_extensible((double)start); + return true; + } + if (items > 0) { + scr_frozen_read_only((double)start); + return true; + } + break; + } + default: + return false; + } + static const char length[] = "Cannot assign to read only property 'length' of object '[object Array]'"; + scr_throw_error_msg(SCR_ERR_TYPE, length, sizeof length - 1); + return true; +} + +void scr_arr_freeze(ScrArr *a) { + if (a->rc != SIZE_MAX) a->spare |= SCR_ARR_FROZEN; +} + +bool scr_arr_is_frozen(const ScrArr *a) { return scr_arr_frozen(a); } + bool scr_arr_has(const ScrArr *a, double i) { size_t idx; if (!scr_arr_valid_index(i, &idx)) return scr_arr_prop_get(a, i, NULL); @@ -4990,6 +5110,8 @@ bool scr_arr_sort_dense(ScrArr *a, ScrClosure *f, uint32_t arity, bool owned, in if (a->rc == SIZE_MAX || elem == SCR_ELEM_CREF || !scr_arr_dense_values(a)) return false; size_t n = a->len; if (n < 2) return true; + /* A frozen array takes the snapshot path, whose write-back throws. */ + if (scr_arr_frozen(a)) return false; if (n > SIZE_MAX / 2 / sizeof(uint64_t)) scr_arr_oom(); ScrSortCompare c = scr_sort_compare_for(a, f, arity, order, key); /* Nothing else can reach a private copy: its own storage is sorted, and diff --git a/packages/runtime/src/scr_array.h b/packages/runtime/src/scr_array.h index 4666750ea..d0c0ea1a8 100644 --- a/packages/runtime/src/scr_array.h +++ b/packages/runtime/src/scr_array.h @@ -128,7 +128,8 @@ typedef struct ScrArr { * header. Kept beside `elem` so the RC fast paths test it with one load. */ uint8_t cyc; /* The low SCR_ARR_OPS_SHIFT bits: storage flags (SCR_ARR_INLINE_SLOTS, - * SCR_ARR_HEAD_ROOM, SCR_ARR_UNSHIFTED, SCR_ARR_HAS_EXT, SCR_ARR_STATES). The rest: the + * SCR_ARR_HEAD_ROOM, SCR_ARR_UNSHIFTED, SCR_ARR_HAS_EXT, SCR_ARR_STATES) + * and SCR_ARR_FROZEN. The rest: the * registry id of the element type's ops (scr_arr_ops), 0 for every kind * but SCR_ELEM_REF and SCR_ELEM_CREF. */ uint16_t spare; @@ -204,6 +205,10 @@ _Static_assert(sizeof(ScrArr) == sizeof(size_t) + 16 + sizeof(void *), #define SCR_ARR_UNSHIFTED 4u #define SCR_ARR_HAS_EXT 8u #define SCR_ARR_STATES 16u +/* Object.freeze of the array (scr_arr_freeze). Emitted guards test it + * before the writes of array types a program may freeze; the runtime's own + * writes assume it is clear. */ +#define SCR_ARR_FROZEN 32u /* An array starts without state bytes. Without them, every index below * `len` is a present value in dense storage: len <= cap, there are no @@ -214,6 +219,19 @@ _Static_assert(sizeof(ScrArr) == sizeof(size_t) + 16 + sizeof(void *), * states (scr_arr_add_states), which it keeps. */ static inline bool scr_arr_has_states(const ScrArr *a) { return (a->spare & SCR_ARR_STATES) != 0; } +static inline bool scr_arr_frozen(const ScrArr *a) { return (a->spare & SCR_ARR_FROZEN) != 0; } +/* Sets SCR_ARR_FROZEN; immortal arrays are left as they are. */ +void scr_arr_freeze(ScrArr *a); +bool scr_arr_is_frozen(const ScrArr *a); +/* Throws Node's TypeError for the first write that the array operation + * `op` makes to a frozen array of length `len`, or answers false when the + * operation writes nothing. op: 0 index write (x = index), 1 push (y = + * argument count), 2 pop, 3 length write, 4 unshift (y = argument count), + * 5 delete (x = index), 6 shift, 7 reverse or sort, 8 fill (x = start, y = + * end), 9 copyWithin (x = target, y = start, z = end), 10 splice (x = + * start, y = delete count, z = item count). */ +bool scr_array_frozen_error(int op, size_t len, double x, double y, double z); + /* The start of the dense slots' allocation: `data` less any head room. */ static inline uint64_t *scr_arr_block(const ScrArr *a) { return (a->spare & SCR_ARR_HEAD_ROOM) ? a->data - (size_t)a->data[-1] : a->data; @@ -236,7 +254,7 @@ static inline struct ScrArrExt *scr_arr_ext_of(const ScrArr *a) { * Id 0 is no ops; ids are assigned on each descriptor's first array, in * any thread. A program with more element types than the field holds keeps * the ops of the rest in the array's ext (SCR_ARR_OPS_OVERFLOW). */ -#define SCR_ARR_OPS_SHIFT 5 +#define SCR_ARR_OPS_SHIFT 6 #define SCR_ARR_OPS_OVERFLOW (UINT16_MAX >> SCR_ARR_OPS_SHIFT) extern const ScrArrOps *scr_arr_ops_table[SCR_ARR_OPS_OVERFLOW]; /* By registry id: 1 when the descriptor has SCR_ARR_OPS_RELEASE_NOTHING. diff --git a/packages/runtime/src/scr_dyn_invoke.c b/packages/runtime/src/scr_dyn_invoke.c index 4039fda94..727e3e835 100644 --- a/packages/runtime/src/scr_dyn_invoke.c +++ b/packages/runtime/src/scr_dyn_invoke.c @@ -740,6 +740,8 @@ static ScrDyn *dyn_live_array_method(ScrDyn *recv, int mid, ScrDyn *const *args, if (!ops) return NULL; ScrArr *arr = recv->v.typed_ref.ptr; size_t len = arr->len; + /* A frozen array's view refuses the write. */ + if (scr_arr_frozen(arr) && ((mid == DYN_M_push) || (mid == DYN_M_pop))) return NULL; if ((mid == DYN_M_push) && argc == 1 && ops->set) { if (!ops->set(arr, (double)len, args[0])) return NULL; *handled = true; @@ -1098,16 +1100,20 @@ static ScrDyn *scr_dyn_invoke_impl( return NULL; } } - if (recv->v.arr.frozen && ((mid == DYN_M_push) || - (mid == DYN_M_pop) || (mid == DYN_M_shift) || - (mid == DYN_M_unshift) || (mid == DYN_M_splice))) { - static const char message[] = "Cannot assign to read only property 'length'"; - scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); + /* A frozen array throws at the operation's first write, as in Node. */ + int frozen_op = !recv->v.arr.frozen ? -1 + : (mid == DYN_M_push) ? 1 + : (mid == DYN_M_pop) ? 2 + : (mid == DYN_M_shift) ? 6 + : (mid == DYN_M_unshift) ? 4 + : (mid == DYN_M_splice) ? 10 + : ((mid == DYN_M_sort) || (mid == DYN_M_reverse)) ? 7 + : -1; + double items = (mid == DYN_M_splice) ? (argc > 2 ? (double)(argc - 2) : 0) : (double)argc; + if (frozen_op >= 0 && + scr_array_frozen_error(frozen_op, len, (double)splice_start, frozen_op == 10 ? (double)splice_removed : items, items)) return NULL; - } - if ((grows && recv->non_extensible) || (shrinks && recv->v.arr.sealed && !(mid == DYN_M_shift)) || - (recv->v.arr.frozen && (((mid == DYN_M_sort) && len > 1) || - ((mid == DYN_M_reverse) && len > 1)))) { + if ((grows && recv->non_extensible) || (shrinks && recv->v.arr.sealed && !(mid == DYN_M_shift))) { static const char message[] = "Cannot modify a sealed or frozen array"; scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); return NULL; @@ -1376,11 +1382,7 @@ static ScrDyn *scr_dyn_invoke_impl( if (!dyn_arr_length_unchanged(recv, len)) return NULL; size_t start = dyn_rel_index(startD, len); size_t end = dyn_rel_index(endD, len); - if (recv->v.arr.frozen && start < end) { - static const char message[] = "Cannot assign to read only array index"; - scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); - return NULL; - } + if (recv->v.arr.frozen && scr_array_frozen_error(8, len, (double)start, (double)end, 0)) return NULL; ScrDyn *fill = argc ? args[0] : scr_dyn_undefined(); if (start < end) scr_dyn_region_store(recv, fill); for (size_t i = start; i < end; i++) { @@ -1409,11 +1411,8 @@ static ScrDyn *scr_dyn_invoke_impl( size_t end = dyn_rel_index(endD, len); size_t count = end > start ? end - start : 0; if (count > len - target) count = len - target; - if (recv->v.arr.frozen && count > 0) { - static const char message[] = "Cannot assign to read only array index"; - scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); + if (recv->v.arr.frozen && scr_array_frozen_error(9, len, (double)target, (double)start, (double)end)) return NULL; - } if (start < target && target < start + count) { for (size_t i = count; i > 0; i--) { ScrDyn *old = recv->v.arr.items[target + i - 1]; diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index a542b5c24..13bd4d066 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -2302,8 +2302,32 @@ static void *scr_dyn_typed_record(const ScrDyn *d) { return d->v.typed_ref.ptr; } -/* Freezes the checked view of a frozen record: its fields, and every - * property added through the view. */ +/* The native array an array capsule refers to, or NULL for any other + * value. */ +static ScrArr *scr_dyn_typed_array(const ScrDyn *d) { + if (d->kind != SCR_DYN_TYPED_REF || d->v.typed_ref.type_key_len < 6 || + memcmp(d->v.typed_ref.type_key, "array<", 6) != 0) + return NULL; + return d->v.typed_ref.ptr; +} + +/* The native Map or Set a handle holds, or NULL for any other value. */ +static ScrMap *scr_dyn_native_collection(const ScrDyn *d) { + if (d->kind != SCR_DYN_HANDLE || (d->v.handle.tag != SCR_DYNH_MAP && d->v.handle.tag != SCR_DYNH_SET)) + return NULL; + return d->v.handle.ptr; +} + +/* Whether a capsule's native record or array is frozen. */ +static bool scr_dyn_typed_ref_frozen(const ScrDyn *d) { + void *record = scr_dyn_typed_record(d); + if (record) return scr_rc_frozen(record); + ScrArr *array = scr_dyn_typed_array(d); + return array && scr_arr_frozen(array); +} + +/* Freezes the checked view of a frozen record or array: its fields or + * elements, and every property added through the view. */ static void scr_dyn_record_view_freeze(ScrDyn *view) { view->non_extensible = true; if (view->kind == SCR_DYN_OBJ) { @@ -2350,9 +2374,9 @@ static bool scr_dyn_class_integrity_fence(const ScrDyn *object, const char *oper ScrJsonBuf buffer; scr_jb_init(&buffer); scr_jb_puts(&buffer, operation); - scr_jb_puts(&buffer, scr_dyn_typed_record(object) - ? " of an object held in an untyped value is not supported yet" - : " of a class instance held in an untyped value is not supported yet"); + scr_jb_puts(&buffer, scr_dyn_typed_record(object) ? " of an object held in an untyped value is not supported yet" + : scr_dyn_typed_array(object) ? " of an array held in an untyped value is not supported yet" + : " of a class instance held in an untyped value is not supported yet"); ScrStr *message = scr_jb_finish(&buffer); scr_throw_error_msg_code(SCR_ERR_ERROR, message->data, message->len, "SC2020"); scr_str_release(message); @@ -2400,6 +2424,11 @@ static ScrDyn *scr_dyn_fn_existing_table(const ScrDyn *function) { ScrDyn *scr_dyn_prevent_extensions(ScrDyn *object) { if (scr_dyn_class_reflection_fence(object) || scr_dyn_error_prototype_fence(object)) return NULL; if (object->kind == SCR_DYN_PROXY) { scr_dyn_proxy_unsupported("Object.preventExtensions"); return NULL; } + ScrMap *collection = scr_dyn_native_collection(object); + if (collection) { + scr_map_prevent_extensions(collection); + return scr_dyn_retain(object); + } if (scr_dyn_class_integrity_fence(object, "Object.preventExtensions")) return NULL; ScrDyn *table = scr_dyn_fn_integrity_table(object); if (table) { @@ -2419,6 +2448,8 @@ bool scr_dyn_is_extensible(ScrDyn *object) { if (object->kind == SCR_DYN_PROXY) { scr_dyn_proxy_unsupported("Object.isExtensible"); return false; } bool answer; if (scr_dyn_typed_ref_view_answer(object, scr_dyn_is_extensible_view, &answer)) return answer; + ScrMap *collection = scr_dyn_native_collection(object); + if (collection) return scr_map_is_extensible(collection); if (scr_dyn_plain_fn(object)) { ScrDyn *table = scr_dyn_fn_existing_table(object); answer = !table || !table->non_extensible; @@ -2431,6 +2462,11 @@ bool scr_dyn_is_extensible(ScrDyn *object) { ScrDyn *scr_dyn_seal(ScrDyn *object) { if (scr_dyn_class_reflection_fence(object) || scr_dyn_error_prototype_fence(object)) return NULL; if (object->kind == SCR_DYN_PROXY) { scr_dyn_proxy_unsupported("Object.seal"); return NULL; } + ScrMap *collection = scr_dyn_native_collection(object); + if (collection) { + scr_map_prevent_extensions(collection); + return scr_dyn_retain(object); + } if (scr_dyn_class_integrity_fence(object, "Object.seal")) return NULL; if (object->symbol_properties) scr_dyn_release(scr_dyn_seal(object->symbol_properties)); ScrDyn *table = scr_dyn_fn_integrity_table(object); @@ -2455,6 +2491,8 @@ bool scr_dyn_is_sealed(const ScrDyn *object) { if (object->kind == SCR_DYN_PROXY) { scr_dyn_proxy_unsupported("Object.isSealed"); return false; } bool answer; if (scr_dyn_typed_ref_view_answer((ScrDyn *)object, scr_dyn_is_sealed, &answer)) return answer; + ScrMap *collection = scr_dyn_native_collection(object); + if (collection) return !scr_map_is_extensible(collection); if (object->symbol_properties && !scr_dyn_is_sealed(object->symbol_properties)) return false; if (scr_dyn_plain_fn(object)) { ScrDyn *table = scr_dyn_fn_existing_table(object); @@ -3679,16 +3717,14 @@ ScrDyn *scr_dyn_typed_ref_materialize(const ScrDyn *d) { fresh->rc = fresh_rc; scr_dyn_release(fresh); } - void *record = scr_dyn_typed_record(capsule); - if (record && scr_rc_frozen(record)) scr_dyn_record_view_freeze(capsule->v.typed_ref.materialized); + if (scr_dyn_typed_ref_frozen(capsule)) scr_dyn_record_view_freeze(capsule->v.typed_ref.materialized); return scr_dyn_retain(capsule->v.typed_ref.materialized); } void scr_dyn_typed_ref_commit(ScrDyn *d) { - /* A frozen record's view cannot have changed its fields. */ + /* A frozen record's or array's view cannot have changed its contents. */ if (d && d->kind == SCR_DYN_TYPED_REF && d->v.typed_ref.commit && - d->v.typed_ref.materialized && - !(scr_dyn_typed_record(d) && scr_rc_frozen(d->v.typed_ref.ptr))) { + d->v.typed_ref.materialized && !scr_dyn_typed_ref_frozen(d)) { d->v.typed_ref.commit(d->v.typed_ref.ptr, d->v.typed_ref.materialized); } @@ -3927,6 +3963,8 @@ static bool scr_typed_ref_array_has_own(const ScrDyn *d, const ScrStr *key) { static bool scr_typed_ref_array_set(ScrDyn *d, const ScrStr *key, const ScrDyn *value) { const ScrDynTypedArrayOps *ops = scr_dyn_typed_ref_array_ops(d); if (!ops || !ops->set) return false; + /* A frozen array's view refuses the write. */ + if (scr_arr_frozen(d->v.typed_ref.ptr)) return false; /* Numeric properties live in the native array's property table, which * holds values of the element type. */ double number; @@ -9230,6 +9268,26 @@ bool scr_dyn_in(const ScrDyn *v, const ScrStr *key) { return false; } +/* Node's TypeErrors for assigning a read-only property and deleting a + * non-configurable one of a checked object: the object reads as + * #, or [object Object] without a prototype. */ +static void scr_dyn_throw_property(const ScrDyn *object, const char *key, size_t key_len, bool deleting) { + const char *name = scr_dyn_class_name(object); + ScrJsonBuf buffer; + scr_jb_init(&buffer); + scr_jb_puts(&buffer, deleting ? "Cannot delete property '" : "Cannot assign to read only property '"); + scr_jb_write(&buffer, key, key_len); + scr_jb_puts(&buffer, deleting ? "' of " : "' of object '"); + if (object->null_proto) scr_jb_puts(&buffer, "[object Object]"); + else { + scr_jb_puts(&buffer, "#<"); + scr_jb_puts(&buffer, name ? name : "Object"); + scr_jb_puts(&buffer, ">"); + } + if (!deleting) scr_jb_puts(&buffer, "'"); + scr_throw_error(SCR_ERR_TYPE, scr_jb_finish(&buffer)); +} + static void scr_dyn_object_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value, const ScrDyn *receiver) { for (size_t i = 0; i < recv->v.obj.len; i++) { ScrDynEntry *entry = &recv->v.obj.entries[i]; @@ -9245,8 +9303,7 @@ static void scr_dyn_object_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value, con return; } if (entry->accessor || !entry->writable) { - static const char msg[] = "Cannot assign to read only property"; - scr_throw_error_msg(SCR_ERR_TYPE, msg, sizeof msg - 1); + scr_dyn_throw_property(receiver, key->data, key->len, false); return; } } @@ -9273,8 +9330,7 @@ static void scr_dyn_object_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value, con return; } if (inherited->accessor || !inherited->writable) { - static const char message[] = "Cannot assign to read only property"; - scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); + scr_dyn_throw_property(receiver, key->data, key->len, false); scr_dyn_release(table); return; } @@ -9354,8 +9410,7 @@ void scr_dyn_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value) { if (recv->kind == SCR_DYN_ARR) { if (key->len == 6 && memcmp(key->data, "length", 6) == 0) { if (recv->v.arr.frozen) { - static const char message[] = "Cannot assign to read only property"; - scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); + scr_array_frozen_error(3, recv->v.arr.len, 0, 0, 0); return; } double number; @@ -9374,8 +9429,7 @@ void scr_dyn_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value) { while (recv->v.arr.len < length) scr_dyn_arr_push_hole(recv); while (recv->v.arr.len > length) { if (recv->v.arr.sealed && scr_dyn_arr_has_index(recv, recv->v.arr.len - 1)) { - static const char message[] = "Cannot delete non-configurable property"; - scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); + scr_array_frozen_error(5, recv->v.arr.len, (double)(recv->v.arr.len - 1), 0, 0); return; } scr_dyn_release(recv->v.arr.items[--recv->v.arr.len]); @@ -9393,14 +9447,13 @@ void scr_dyn_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value) { } if (is_index && idx < 4294967295ULL) { if (scr_dyn_array_prototype_element_fence(recv)) return; - if (recv->v.arr.frozen && idx < recv->v.arr.len) { - static const char message[] = "Cannot assign to read only property"; - scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); + if (recv->v.arr.frozen && idx < recv->v.arr.len && scr_dyn_arr_has_index(recv, idx)) { + scr_array_frozen_error(0, recv->v.arr.len, (double)idx, 0, 0); return; } if (recv->non_extensible && !scr_dyn_arr_has_index(recv, idx)) { - static const char message[] = "Cannot add property, object is not extensible"; - scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); + /* An absent index, a hole included, is a property to add. */ + scr_array_frozen_error(0, 0, (double)idx, 0, 0); return; } if (idx > recv->v.arr.len + 1000000) { @@ -11229,10 +11282,7 @@ void scr_dyn_key_delete(ScrDyn *recv, const ScrStr *key, bool strict) { if (recv->kind == SCR_DYN_OBJ) { ScrDynEntry *entry = scr_dyn_entry(recv, key); if (entry && !entry->configurable) { - if (strict) { - static const char message[] = "Cannot delete non-configurable property"; - scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); - } + if (strict) scr_dyn_throw_property(recv, key->data, key->len, true); return; } scr_json_delete_member(recv, key); @@ -11244,10 +11294,7 @@ void scr_dyn_key_delete(ScrDyn *recv, const ScrStr *key, bool strict) { for (size_t i = 0; i < key->len; i++) index = index * 10 + (size_t)(key->data[i] - '0'); if (!scr_dyn_arr_has_index(recv, index)) return; if (recv->v.arr.sealed) { - if (strict) { - static const char message[] = "Cannot delete non-configurable property"; - scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); - } + if (strict) scr_array_frozen_error(5, recv->v.arr.len, (double)index, 0, 0); return; } if (!recv->v.arr.presence) { @@ -15682,18 +15729,29 @@ ScrStr *scr_dyn_to_string_method(const ScrDyn *d, const ScrStr *enc, const ScrSt return scr_dyn_to_string_method_args(d, enc, what, NULL, 0); } -/* Freezing a record capsule freezes the native record, whose emitted writes - * then refuse to change it, and its checked view. */ -static ScrDyn *scr_dyn_freeze_record(ScrDyn *capsule, void *record) { - scr_record_freeze(record); - if (capsule->v.typed_ref.materialized) scr_dyn_record_view_freeze(capsule->v.typed_ref.materialized); - return scr_dyn_retain(capsule); +/* Freezing a record or array capsule freezes the native value, whose + * emitted writes then refuse to change it, and its checked view. A Map or + * Set handle only stops being extensible: its entries stay writable. Answers + * NULL for every other value. */ +static ScrDyn *scr_dyn_freeze_native(ScrDyn *value) { + void *record = scr_dyn_typed_record(value); + ScrArr *array = record ? NULL : scr_dyn_typed_array(value); + ScrMap *collection = scr_dyn_native_collection(value); + if (collection) { + scr_map_prevent_extensions(collection); + return scr_dyn_retain(value); + } + if (!record && !array) return NULL; + if (record) scr_record_freeze(record); + else scr_arr_freeze(array); + if (value->v.typed_ref.materialized) scr_dyn_record_view_freeze(value->v.typed_ref.materialized); + return scr_dyn_retain(value); } ScrDyn *scr_dyn_freeze(ScrDyn *value) { if (scr_dyn_class_reflection_fence(value) || scr_dyn_error_prototype_fence(value)) return NULL; - void *record = scr_dyn_typed_record(value); - if (record) return scr_dyn_freeze_record(value, record); + ScrDyn *native = scr_dyn_freeze_native(value); + if (native) return native; if (scr_dyn_class_integrity_fence(value, "Object.freeze")) return NULL; ScrDyn *table = scr_dyn_fn_integrity_table(value); if (!table) return scr_dyn_freeze_data(value); @@ -15705,8 +15763,8 @@ ScrDyn *scr_dyn_freeze(ScrDyn *value) { ScrDyn *scr_dyn_freeze_data(ScrDyn *value) { if (scr_dyn_class_reflection_fence(value) || scr_dyn_error_prototype_fence(value)) return NULL; - void *record = scr_dyn_typed_record(value); - if (record) return scr_dyn_freeze_record(value, record); + ScrDyn *native = scr_dyn_freeze_native(value); + if (native) return native; if (scr_dyn_class_integrity_fence(value, "Object.freeze")) return NULL; if (value->symbol_properties) scr_dyn_release(scr_dyn_freeze_data(value->symbol_properties)); if (value->kind == SCR_DYN_ARR) { @@ -15737,6 +15795,8 @@ bool scr_dyn_is_frozen(const ScrDyn *value) { if (scr_dyn_class_reflection_fence(value)) return false; bool answer; if (scr_dyn_typed_ref_view_answer((ScrDyn *)value, scr_dyn_is_frozen, &answer)) return answer; + ScrMap *collection = scr_dyn_native_collection(value); + if (collection) return !scr_map_is_extensible(collection); if (value->symbol_properties && !scr_dyn_is_frozen(value->symbol_properties)) return false; if (scr_dyn_plain_fn(value)) { ScrDyn *table = scr_dyn_fn_existing_table(value); diff --git a/packages/runtime/src/scr_map.c b/packages/runtime/src/scr_map.c index 1a6a3fc00..1cb77b0f6 100644 --- a/packages/runtime/src/scr_map.c +++ b/packages/runtime/src/scr_map.c @@ -988,6 +988,12 @@ double scr_map_size(const ScrMap *m) { return (double)m->nlive; } * table. Regrowth is amortized against the insertions that need it. * A map without entries has empty tables already: clearing it does no * work and keeps its storage for the next fill. */ +void scr_map_prevent_extensions(ScrMap *m) { + if (m->rc != SIZE_MAX) m->flags |= SCR_MAP_NOT_EXTENSIBLE; +} + +bool scr_map_is_extensible(const ScrMap *m) { return (m->flags & SCR_MAP_NOT_EXTENSIBLE) == 0; } + void scr_map_clear(ScrMap *m) { size_t walked = m->nentries; if (walked == 0) return; diff --git a/packages/runtime/src/scr_map.h b/packages/runtime/src/scr_map.h index d1ca387bf..a581cc1c7 100644 --- a/packages/runtime/src/scr_map.h +++ b/packages/runtime/src/scr_map.h @@ -116,6 +116,10 @@ typedef struct ScrMapOps { * whose object has a null prototype (Object.groupBy results). Compiled * code sets and tests the byte; the runtime never copies it. */ #define SCR_MAP_NULL_PROTO 1u +/* Object.freeze, Object.seal or Object.preventExtensions of a Map or Set: + * it has no own properties, so it is then frozen, sealed and not + * extensible at once. Its entries stay writable, as in Node. */ +#define SCR_MAP_NOT_EXTENSIBLE 2u typedef struct ScrMap { size_t rc; /* SIZE_MAX = immortal (unused for maps; kept per convention) */ @@ -272,6 +276,10 @@ void scr_map_release_v(void *m); void scr_map_trace_v(void *m, ScrTraceVisit visit, void *ctx); double scr_map_size(const ScrMap *m); /* live entries (Map.size) */ +/* Object.freeze, seal and preventExtensions of a Map or Set + * (SCR_MAP_NOT_EXTENSIBLE); immortal maps are left as they are. */ +void scr_map_prevent_extensions(ScrMap *m); +bool scr_map_is_extensible(const ScrMap *m); void scr_map_clear(ScrMap *m); bool scr_map_has_f64(const ScrMap *m, double key); diff --git a/packages/runtime/src/scr_publish.c b/packages/runtime/src/scr_publish.c index 4767122dd..afbe4d6af 100644 --- a/packages/runtime/src/scr_publish.c +++ b/packages/runtime/src/scr_publish.c @@ -527,62 +527,12 @@ void scr_throw_published_field(const char *prop, const char *owner) { scr_pub_throw("Cannot assign to read only property '%s' of object '#<%s>'", prop, owner); } -static void scr_pub_index_text(double index, char *out, size_t size) { - char buf[64]; - size_t n = scr_f64_to_str(index, buf); - if (n > size - 1) n = size - 1; - memcpy(out, buf, n); - out[n] = 0; +bool scr_throw_published_array(int op, const ScrArr *a, double index, double n) { + return scr_array_frozen_error(op, a->len, index, n, 0); } -/* op: 0 index write, 1 push (n = argument count), 2 pop, 3 length write, - * 4 unshift (n = argument count), 5 delete, 6 shift, 7 other in-place - * rewrites (splice, reverse, fill, copyWithin, sort). Returns false when - * Node's operation would not write the frozen array at all (it then is not - * an error: e.g. reversing an empty array). */ -bool scr_throw_published_array(int op, const ScrArr *a, double index, double n) { - static const char length[] = "Cannot assign to read only property 'length' of object '[object Array]'"; - char text[64]; - size_t len = a->len; - switch (op) { - case 0: - scr_pub_index_text(index, text, sizeof text); - if (index >= 0 && index < (double)len && index == (double)(size_t)index) - scr_pub_throw("Cannot assign to read only property '%s' of object '[object Array]'%s", text, ""); - else scr_pub_throw("Cannot add property %s, object is not extensible%s", text, ""); - return true; - case 1: - if (n == 0) break; - scr_pub_index_text((double)len, text, sizeof text); - scr_pub_throw("Cannot add property %s, object is not extensible%s", text, ""); - return true; - case 2: - case 6: - if (len == 0) break; - if (op == 6 && len > 1) { - scr_pub_throw("Cannot assign to read only property '0' of object '[object Array]'%s%s", "", ""); - return true; - } - scr_pub_index_text((double)(len - 1), text, sizeof text); - scr_pub_throw("Cannot delete property '%s' of [object Array]%s", text, ""); - return true; - case 3: break; - case 4: - if (n == 0) break; - scr_pub_index_text(len ? (double)len + n - 1 : 0, text, sizeof text); - scr_pub_throw("Cannot add property %s, object is not extensible%s", text, ""); - return true; - case 5: - scr_pub_index_text(index, text, sizeof text); - scr_pub_throw("Cannot delete property '%s' of [object Array]%s", text, ""); - return true; - default: - if (len == 0) return false; - scr_pub_throw("Cannot assign to read only property '0' of object '[object Array]'%s%s", "", ""); - return true; - } - scr_throw_error_msg(SCR_ERR_TYPE, length, sizeof length - 1); - return true; +bool scr_throw_published_array_range(int op, const ScrArr *a, double x, double y, double z) { + return scr_array_frozen_error(op, a->len, x, y, z); } void scr_throw_published_collection(bool set) { diff --git a/packages/runtime/src/scr_rc.h b/packages/runtime/src/scr_rc.h index 87eb2536a..ccb74514d 100644 --- a/packages/runtime/src/scr_rc.h +++ b/packages/runtime/src/scr_rc.h @@ -743,12 +743,14 @@ void scr_pub_dyn_fn(void *v, ScrPub *ctx); void scr_pub_arr(void *v, ScrPub *ctx, ScrPubFn elem); void scr_pub_map(void *v, ScrPub *ctx, ScrPubFn key, ScrPubFn val); void scr_publish_refuse(const char *message); -/* Writes into published objects (emitted guards): Node's frozen-object - * TypeErrors. scr_throw_published_array answers false (and throws nothing) - * when Node's operation would not write the array. */ +/* Writes into published or frozen objects (emitted guards): Node's + * frozen-object TypeErrors. The array guards answer false (and throw + * nothing) when Node's operation would not write the array; their ops are + * scr_array_frozen_error's (scr_array.h). */ struct ScrArr; SCR_COLD void scr_throw_published_field(const char *prop, const char *owner); bool scr_throw_published_array(int op, const struct ScrArr *a, double index, double n); +bool scr_throw_published_array_range(int op, const struct ScrArr *a, double x, double y, double z); SCR_COLD void scr_throw_published_collection(bool set); /* Young-in nursery (scr_cycle.c built with SCR_YIN, linked by programs diff --git a/tests/corpus/7406-frozen-containers.ts b/tests/corpus/7406-frozen-containers.ts new file mode 100644 index 000000000..450cbf8cb --- /dev/null +++ b/tests/corpus/7406-frozen-containers.ts @@ -0,0 +1,132 @@ +// Object.freeze of arrays, Maps and Sets that other references still hold. +// A frozen array refuses every write through any reference with Node's +// TypeError for the operation's first write (this module is strict mode +// code). A frozen Map or Set only stops being extensible: its entries stay +// writable, and Object.isFrozen, isSealed and isExtensible report it. +export {}; + +function report(e: unknown): void { + console.log((e as Error).name + ": " + (e as Error).message); +} + +function freezeUnknown(value: unknown): unknown { + return Object.freeze(value); +} + +function writeAny(value: any): void { + value[0] = 7; +} + +function pushAny(value: any): void { + value.push(8); +} + +function integrity(value: unknown): string { + return [Object.isFrozen(value), Object.isSealed(value), Object.isExtensible(value)].join(" "); +} + +function arrays(): void { + const a: number[] = [1, 2, 3]; + const alias = a; + console.log(Object.isFrozen(a), integrity(a)); + Object.freeze(alias); + console.log(Object.isFrozen(a), integrity(a)); + try { a[0] = 9; } catch (e) { report(e); } + try { a[5] = 9; } catch (e) { report(e); } + try { a.length = 0; } catch (e) { report(e); } + try { a.push(4); } catch (e) { report(e); } + try { a.pop(); } catch (e) { report(e); } + try { a.shift(); } catch (e) { report(e); } + try { a.unshift(0); } catch (e) { report(e); } + try { a.splice(0, 1); } catch (e) { report(e); } + try { a.splice(1, 0, 7); } catch (e) { report(e); } + try { a.reverse(); } catch (e) { report(e); } + try { a.sort(); } catch (e) { report(e); } + try { a.fill(0, 1); } catch (e) { report(e); } + try { a.copyWithin(1, 0); } catch (e) { report(e); } + console.log(a.join(), a.length); + + const one: string[] = ["x"]; + Object.freeze(one); + console.log(one.reverse().join(), one.sort().join(), one.copyWithin(0, 1).join()); + try { one.splice(0, 1); } catch (e) { report(e); } + try { one.shift(); } catch (e) { report(e); } + const none: string[] = []; + Object.freeze(none); + console.log(none.reverse().length, none.fill("y").length); + try { none.pop(); } catch (e) { report(e); } + try { none.splice(0, 0, "z"); } catch (e) { report(e); } + + let calls = 0; + const byValue = (x: number, y: number): number => { + calls++; + return x - y; + }; + const unsorted: number[] = [3, 1, 2]; + Object.freeze(unsorted); + try { unsorted.sort(byValue); } catch (e) { report(e); } + const single: number[] = [5]; + Object.freeze(single); + console.log(unsorted.join(), calls > 0, single.sort(byValue).join()); + + const open: number[] = [4, 5]; + open.push(6); + open[0] = 1; + console.log(open.join(), Object.isFrozen(open)); +} + +function throughUnknown(): void { + const a: number[] = [1, 2]; + console.log(integrity(a)); + const back = freezeUnknown(a); + console.log(back === a, Object.isFrozen(a), integrity(a)); + try { a.push(3); } catch (e) { report(e); } + try { a[1] = 5; } catch (e) { report(e); } + try { writeAny(a); } catch (e) { report(e); } + try { pushAny(a); } catch (e) { report(e); } + console.log(a.join()); + + const rows: number[][] = [[1], [2]]; + freezeUnknown(rows[1]); + rows[0]!.push(10); + try { rows[1]!.push(20); } catch (e) { report(e); } + console.log(JSON.stringify(rows), Object.isFrozen(rows), Object.isFrozen(rows[1])); +} + +function collections(): void { + const m = new Map([["x", 1]]); + console.log(integrity(m)); + const frozen = Object.freeze(m); + m.set("y", 2); + m.delete("x"); + console.log(frozen === m, Object.isFrozen(m), Object.isSealed(m), Object.isExtensible(m), m.size); + const s = new Set([1]); + freezeUnknown(s); + s.add(2); + console.log(integrity(s), s.size); + const sealed = new Set(); + Object.seal(sealed); + const fresh = new Map(); + Object.preventExtensions(fresh); + console.log(Object.isFrozen(sealed), Object.isExtensible(fresh), integrity(new Map())); +} + +function lifetimes(): void { + const kept: string[][] = []; + for (let round = 0; round < 3; round++) { + const row = ["r" + round, "s" + round]; + if (round !== 1) Object.freeze(row); + kept.push(row); + } + for (const row of kept) { + try { row[0] = "changed"; } catch (e) { console.log(row[1], (e as Error).message); } + } + console.log(JSON.stringify(kept)); + kept.length = 0; + console.log(kept.length); +} + +arrays(); +throughUnknown(); +collections(); +lifetimes(); diff --git a/tests/harness/corpus-costs/7406-frozen-containers.ts.json b/tests/harness/corpus-costs/7406-frozen-containers.ts.json new file mode 100644 index 000000000..30bfa4b73 --- /dev/null +++ b/tests/harness/corpus-costs/7406-frozen-containers.ts.json @@ -0,0 +1 @@ +{ "plain": 3146, "san": 3439 } From 2c44d3e3abe229cab1062e29b23dbe1bfc781948 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 05:15:45 -0500 Subject: [PATCH 034/184] Stop reading stdin at the read-ahead mark while 'readable' listeners hold it - With 'readable' listeners the stream stays paused, as in Node: 'data' listeners, resume() and a parked for-await read no longer keep the loop reading once the held input reaches the 64 KiB mark (or a larger read(size)); reading resumes when a read drains it. - Held input is a list of chunks with a running length, so an arrival appends without copying what is already held, and read() joins only the bytes it returns. - New event-loop case: 'readable', 'data' and resume() with no read() for 400 ms while 1 MiB arrives. --- packages/runtime/src/scr_events.c | 123 +++++++++++++----- .../stdio-readable-backpressure.mjs | 27 ++++ tests/harness/event-loop-cases.ts | 11 ++ 3 files changed, 128 insertions(+), 33 deletions(-) create mode 100644 tests/fixtures/event-loop/stdio-readable-backpressure.mjs diff --git a/packages/runtime/src/scr_events.c b/packages/runtime/src/scr_events.c index bc554a445..6e0bf5146 100644 --- a/packages/runtime/src/scr_events.c +++ b/packages/runtime/src/scr_events.c @@ -388,7 +388,12 @@ static SCR_TL bool scr_stdin_destroyed = false; static SCR_TL bool scr_stdin_paused = false; static SCR_TL bool scr_stdin_resumed = false; static SCR_TL bool scr_stdin_read_eof = false; -static SCR_TL ScrBytes *scr_stdin_buffer; +/* Input held for read() and later consumers, as Node's BufferList: the + * chunks in arrival order and their total length, so an arrival appends + * without copying what is already held. */ +static SCR_TL ScrBytes **scr_stdin_held = NULL; +static SCR_TL size_t scr_stdin_nheld = 0, scr_stdin_held_cap = 0; +static SCR_TL size_t scr_stdin_buffered = 0; static SCR_TL size_t scr_stdin_read_demand; /* 'readable' listeners (paused mode, as Node's once one is added): input * stays buffered for read(), and the event fires on the next tick when @@ -398,6 +403,66 @@ static SCR_TL size_t scr_stdin_nreadable = 0, scr_stdin_readable_cap = 0; static SCR_TL bool scr_stdin_readable_need = false; static SCR_TL bool scr_stdin_readable_queued = false; +/* Appends an arrived chunk (moves) to the held input. */ +static void scr_stdin_hold(ScrBytes *chunk) { + if (chunk->len == 0) { + scr_bytes_release(chunk); + return; + } + if (scr_stdin_nheld == scr_stdin_held_cap) { + scr_stdin_held_cap = scr_stdin_held_cap ? scr_stdin_held_cap * 2 : 4; + scr_stdin_held = realloc(scr_stdin_held, scr_stdin_held_cap * sizeof *scr_stdin_held); + if (!scr_stdin_held) scr_events_oom(); + } + scr_stdin_held[scr_stdin_nheld++] = chunk; + scr_stdin_buffered += chunk->len; +} + +/* The first n held bytes as one chunk (+1), 0 < n <= scr_stdin_buffered. + * A whole first chunk moves out without a copy; otherwise the bytes are + * joined once, and a partly taken chunk keeps its rest. */ +static ScrBytes *scr_stdin_take(size_t n) { + ScrBytes *first = scr_stdin_held[0]; + size_t used = 0; + ScrBytes *out; + if (n == first->len) { + out = first; + used = 1; + } else if (n < first->len) { + out = scr_bytes_slice(first, 0, (double)n); + scr_stdin_held[0] = scr_bytes_slice(first, (double)n, (double)first->len); + scr_bytes_release(first); + } else { + out = scr_bytes_new(SCR_BYTES_U8, (double)n); + size_t at = 0; + while (at < n) { + ScrBytes *chunk = scr_stdin_held[used]; + size_t part = chunk->len <= n - at ? chunk->len : n - at; + memcpy(out->data + at, chunk->data, part); + at += part; + if (part == chunk->len) { + scr_bytes_release(chunk); + used++; + } else { + scr_stdin_held[used] = scr_bytes_slice(chunk, (double)part, (double)chunk->len); + scr_bytes_release(chunk); + } + } + } + memmove(scr_stdin_held, scr_stdin_held + used, (scr_stdin_nheld - used) * sizeof *scr_stdin_held); + scr_stdin_nheld -= used; + scr_stdin_buffered -= n; + return out; +} + +static void scr_stdin_drop_held(void) { + for (size_t i = 0; i < scr_stdin_nheld; i++) scr_bytes_release(scr_stdin_held[i]); + free(scr_stdin_held); + scr_stdin_held = NULL; + scr_stdin_nheld = scr_stdin_held_cap = 0; + scr_stdin_buffered = 0; +} + #ifdef SCR_STDIO_EXTENDED /* process.stdin.setEncoding(enc): the canonical encoding name (NULL until * set) and the StringDecoder's packed pending partial sequence, so a @@ -486,7 +551,7 @@ static void scr_stdin_finish(bool fire_end); static void scr_stdin_emit_readable(void) { scr_stdin_readable_queued = false; - size_t length = scr_stdin_buffer ? scr_stdin_buffer->len : 0; + size_t length = scr_stdin_buffered; if (scr_stdin_eof || scr_stdin_destroyed || (!length && !scr_stdin_read_eof)) return; size_t n = scr_stdin_nreadable; ScrStdinEndL *snap = malloc((n ? n : 1) * sizeof *snap); @@ -512,7 +577,7 @@ static void scr_stdin_emit_readable(void) { scr_closure_release(snap[i].cb); } free(snap); - length = scr_stdin_buffer ? scr_stdin_buffer->len : 0; + length = scr_stdin_buffered; /* Another event is due once the buffer reaches the 64 KiB mark, or the * larger size a listener's read(size) asked for. */ scr_stdin_readable_need = @@ -542,7 +607,7 @@ static void scr_stdin_on_readable(ScrClosure *cb /*moves*/, bool once) { scr_stdin_nreadable++; scr_stdin_readable_need = true; if (scr_stdin_read_demand < 65536) scr_stdin_read_demand = 65536; - if (scr_stdin_buffer && scr_stdin_buffer->len) scr_stdin_queue_readable(); + if (scr_stdin_buffered) scr_stdin_queue_readable(); } /* The input ended: with 'readable' listeners the event fires once more and @@ -551,7 +616,7 @@ static void scr_stdin_on_readable(ScrClosure *cb /*moves*/, bool once) { static void scr_stdin_reached_eof(void) { scr_stdin_read_eof = true; if (scr_stdin_nreadable) scr_stdin_queue_readable(); - else if (!scr_stdin_buffer) scr_stdin_finish(true); + else if (!scr_stdin_buffered) scr_stdin_finish(true); } void scr_stdin_on_error(ScrClosure *cb /*moves*/, ScrChildErrFn fn, bool once) { @@ -595,10 +660,15 @@ bool scr_stdin_ended(void) { return scr_stdin_eof || scr_stdin_destroyed; } * alive (a paused Node stdin with only those never flows — the process * exits with them registered). */ static bool scr_stdin_watching(void) { - return !scr_stdin_eof && !scr_stdin_destroyed && - ((!scr_stdin_paused && (scr_stdin_ndata > 0 || scr_stdin_resumed)) || - scr_stdin_waiter != NULL || - (!scr_stdin_read_eof && scr_stdin_read_demand > (scr_stdin_buffer ? scr_stdin_buffer->len : 0))); + if (scr_stdin_eof || scr_stdin_destroyed) return false; + /* 'readable' listeners keep the stream paused, as in Node, where + * resume() and 'data' listeners do not make it flow: it reads ahead only + * while a read wants more than is held (the 64 KiB mark, or a larger + * read(size)), and stops until something drains it. */ + if (scr_stdin_nreadable) return !scr_stdin_read_eof && scr_stdin_read_demand > scr_stdin_buffered; + return (!scr_stdin_paused && (scr_stdin_ndata > 0 || scr_stdin_resumed)) || + scr_stdin_waiter != NULL || + (!scr_stdin_read_eof && scr_stdin_read_demand > scr_stdin_buffered); } bool scr_stdin_pending(void) { return scr_stdin_watching(); } @@ -676,8 +746,7 @@ static void scr_stdin_finish(bool fire_end) { void scr_stdin_destroy(void) { if (scr_stdin_destroyed) return; scr_stdin_destroyed = true; - scr_bytes_release(scr_stdin_buffer); - scr_stdin_buffer = NULL; + scr_stdin_drop_held(); scr_stdin_read_demand = 0; scr_stdin_settle_done(); scr_stdin_drop_listeners(); @@ -712,15 +781,7 @@ static void scr_stdin_deliver(ScrBytes *chunk); static void scr_stdin_arrived(ScrBytes *chunk) { if (scr_stdin_paused || scr_stdin_nreadable || (!scr_stdin_ndata && !scr_stdin_resumed && !scr_stdin_waiter)) { if (scr_stdin_nreadable && scr_stdin_readable_need) scr_stdin_queue_readable(); - if (scr_stdin_buffer) { - size_t before = scr_stdin_buffer->len; - ScrBytes *joined = scr_bytes_new(SCR_BYTES_U8, (double)(before + chunk->len)); - memcpy(joined->data, scr_stdin_buffer->data, before); - memcpy(joined->data + before, chunk->data, chunk->len); - scr_bytes_release(scr_stdin_buffer); - scr_bytes_release(chunk); - scr_stdin_buffer = joined; - } else scr_stdin_buffer = chunk; + scr_stdin_hold(chunk); return; } scr_stdin_deliver(chunk); @@ -732,16 +793,15 @@ static void scr_stdin_service(void) { /* Buffered input goes to a consumer: 'data' listeners, a resumed * stream, or a parked for-await read. After the last 'readable' listener * it stays buffered until one arrives, as in Node. */ - if (!scr_stdin_paused && !scr_stdin_nreadable && scr_stdin_buffer && + if (!scr_stdin_paused && !scr_stdin_nreadable && scr_stdin_buffered && (scr_stdin_ndata > 0 || scr_stdin_resumed || scr_stdin_waiter != NULL)) { - ScrBytes *chunk = scr_stdin_buffer; - scr_stdin_buffer = NULL; + ScrBytes *chunk = scr_stdin_take(scr_stdin_buffered); scr_stdin_deliver(chunk); scr_bytes_release(chunk); return; } if (scr_stdin_read_eof) { - if (!scr_stdin_buffer && !scr_stdin_nreadable) scr_stdin_finish(true); + if (!scr_stdin_buffered && !scr_stdin_nreadable) scr_stdin_finish(true); return; } if (scr_stdin_fed) { @@ -1058,7 +1118,7 @@ static void scr_stdio_queue_write_error(ScrStdio *stream, ScrDyn *callback, int } static void scr_stdio_finish_read(void) { - if (scr_stdin_read_eof && !scr_stdin_buffer && !scr_stdin_eof && !scr_stdin_destroyed) + if (scr_stdin_read_eof && !scr_stdin_buffered && !scr_stdin_eof && !scr_stdin_destroyed) scr_stdin_finish(true); } @@ -1067,7 +1127,7 @@ static ScrDyn *scr_stdio_read(ScrDyn *size) { if (!all && (size->kind != SCR_DYN_NUM || !isfinite(size->v.num) || size->v.num < 0 || size->v.num > 1073741824 || floor(size->v.num) != size->v.num)) return scr_stdio_refusal("read size"); - size_t available = scr_stdin_buffer ? scr_stdin_buffer->len : 0; + size_t available = scr_stdin_buffered; size_t wanted = all ? available : (size_t)size->v.num; if (scr_stdin_read_eof && wanted > available) wanted = available; if (wanted == 0 || wanted > available) { @@ -1078,10 +1138,8 @@ static ScrDyn *scr_stdio_read(ScrDyn *size) { } return scr_dyn_new_null(); } - ScrBytes *chunk = scr_bytes_slice(scr_stdin_buffer, 0, (double)wanted); - ScrBytes *rest = wanted < available ? scr_bytes_slice(scr_stdin_buffer, (double)wanted, (double)available) : NULL; - scr_bytes_release(scr_stdin_buffer); - scr_stdin_buffer = rest; + ScrBytes *chunk = scr_stdin_take(wanted); + bool rest = scr_stdin_buffered > 0; scr_stdin_read_demand = 65536; if (!rest && !scr_stdin_read_eof) scr_stdin_readable_need = true; scr_stdin_deliver(chunk); @@ -1339,7 +1397,7 @@ static ScrDyn *scr_stdio_get(void *ptr, const char *key, size_t length) { return dimension < 0 ? scr_dyn_retain(scr_dyn_undefined()) : scr_dyn_new_num(dimension); } if (stream->fd == 0 && length == 14 && memcmp(key, "readableLength", 14) == 0) - return scr_dyn_new_num(scr_stdin_buffer ? (double)scr_stdin_buffer->len : 0); + return scr_dyn_new_num((double)scr_stdin_buffered); if (stream->fd == 0 && length == 5 && memcmp(key, "isRaw", 5) == 0) return scr_process_is_tty(0) ? scr_dyn_new_bool(scr_stdio_raw) : scr_dyn_retain(scr_dyn_undefined()); if (stream->fd == 0 && length == 9 && memcmp(key, "destroyed", 9) == 0) return scr_dyn_new_bool(scr_stdin_destroyed); @@ -1624,8 +1682,7 @@ static void scr_events_cleanup_atexit(void) { if (scr_context_is_main()) scr_sig_cleanup(); scr_stdin_settle_done(); scr_stdin_drop_listeners(); - scr_bytes_release(scr_stdin_buffer); - scr_stdin_buffer = NULL; + scr_stdin_drop_held(); if (scr_stdio_initialized) scr_stdio_cleanup(); } diff --git a/tests/fixtures/event-loop/stdio-readable-backpressure.mjs b/tests/fixtures/event-loop/stdio-readable-backpressure.mjs new file mode 100644 index 000000000..df2afb18f --- /dev/null +++ b/tests/fixtures/event-loop/stdio-readable-backpressure.mjs @@ -0,0 +1,27 @@ +// 'readable', 'data' and resume() together, with no read() for a while: +// 'readable' keeps the stream paused, so it stops reading at its read-ahead +// mark until a read drains it, and 'data' sees what read() returns. +const input = process.stdin; +let draining = false; +let peak = 0; +let dataBytes = 0; +let readBytes = 0; +const drain = () => { + let chunk; + while ((chunk = input.read()) !== null) readBytes += chunk.length; +}; +input.on('data', (chunk) => { + dataBytes += chunk.length; +}); +input.on('readable', () => { + peak = Math.max(peak, input.readableLength); + if (draining) drain(); +}); +input.resume(); +setTimeout(() => { + peak = Math.max(peak, input.readableLength); + console.log('held below 128 KiB while not draining', peak <= 131072); + draining = true; + drain(); +}, 400); +input.on('end', () => console.log('read', readBytes, 'data', dataBytes)); diff --git a/tests/harness/event-loop-cases.ts b/tests/harness/event-loop-cases.ts index 1b75defaa..50a7d2b85 100644 --- a/tests/harness/event-loop-cases.ts +++ b/tests/harness/event-loop-cases.ts @@ -67,6 +67,17 @@ export const eventLoopCases: EventLoopCase[] = [ end: true, }, }, + { + title: "stdio values: 'readable' with 'data' and resume() holds input at the read-ahead mark", + fixture: "stdio-readable-backpressure.mjs", + script: { + writes: Array.from({ length: 16 }, (_, i) => ({ + delayMs: i === 0 ? 0 : 5, + data: String.fromCharCode(97 + i).repeat(65536), + })), + end: true, + }, + }, { title: "stdio values: 'readable' on an immediately ended pipe", fixture: "stdio-readable.mjs", From 967f2ec85dcd408bea5d24e2e425aafd3ea248ac Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 05:55:39 -0500 Subject: [PATCH 035/184] Compile minified npm entries and untyped regex arguments Refs #1777 npm packages are no longer refused up front for minified JavaScript. A bundle whose interop helpers lost their names is still refused by the bundle recognizers, and a package whose attempt fails is still refused on its own. A `var` read in a later declarator of the statement that stores it (`var a = {...}, b = {...a}`) runs after the store. A read in a function body is now located at the body's own statement instead of the body, so JavaScript `var` bindings read after their store in a function keep their type instead of gaining undefined. `replace` and `replaceAll` with an untyped search value and a callback call the regex callback replacement for a RegExp and the string-search helper otherwise. That helper now accepts untyped callback parameters, passes undefined to parameters after the subject, and converts number, boolean and untyped results with ToString. In JavaScript, `||` and `&&` over primitives of different types whose result tsc types `any` evaluate in the checked domain. --- docs/content/docs/dependencies.mdx | 2 +- packages/compiler/src/cli/usage.ts | 6 +- .../lowering/containers/string-replacement.ts | 64 +++++++++---- .../frontend/lowering/expressions/binary.ts | 18 ++++ .../src/frontend/lowering/var-hoisting.ts | 43 ++++++--- packages/compiler/src/frontend/npm-static.ts | 26 ++---- .../7543-var-list-declarator-order.js.json | 6 ++ ...545-string-regex-checked-arguments.js.json | 6 ++ .../7546-minified-esm-rules/main.ts.json | 6 ++ .../corpus/7543-var-list-declarator-order.js | 66 ++++++++++++++ .../7545-string-regex-checked-arguments.js | 89 +++++++++++++++++++ .../npm/cases/7546-minified-esm-rules/main.ts | 22 +++++ .../node_modules/gtminimark/dist/index.d.ts | 1 + .../npm/node_modules/gtminimark/dist/index.js | 1 + .../npm/node_modules/gtminimark/package.json | 8 ++ .../7543-var-list-declarator-order.js.json | 1 + ...545-string-regex-checked-arguments.js.json | 1 + tests/harness/library-mode.test.ts | 26 +++--- tests/harness/npm-static.test.ts | 9 ++ tests/library-mode/npm-refuse/lib-minified.ts | 4 +- tests/library-mode/npm-static/lib.ts | 2 +- .../npm-static/node_modules/mathkit/index.js | 2 +- 22 files changed, 340 insertions(+), 69 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7543-var-list-declarator-order.js.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7545-string-regex-checked-arguments.js.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts.json create mode 100644 tests/corpus/7543-var-list-declarator-order.js create mode 100644 tests/corpus/7545-string-regex-checked-arguments.js create mode 100644 tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts create mode 100644 tests/fixtures/npm/node_modules/gtminimark/dist/index.d.ts create mode 100644 tests/fixtures/npm/node_modules/gtminimark/dist/index.js create mode 100644 tests/fixtures/npm/node_modules/gtminimark/package.json create mode 100644 tests/harness/corpus-costs/7543-var-list-declarator-order.js.json create mode 100644 tests/harness/corpus-costs/7545-string-regex-checked-arguments.js.json diff --git a/docs/content/docs/dependencies.mdx b/docs/content/docs/dependencies.mdx index 14130b234..f02dcfbf0 100644 --- a/docs/content/docs/dependencies.mdx +++ b/docs/content/docs/dependencies.mdx @@ -53,7 +53,7 @@ Type-only imports follow Node.js's type stripping. `import type` and `export typ scriptc compiles a package automatically when it: - ships its own type declarations, or is typed by its DefinitelyTyped `@types/` package, or is a dependency of another package that has no declarations at all, and -- ships readable JavaScript: unminified, or minified by a bundler whose helpers keep their names. +- ships JavaScript that isn't a bundle whose bundler-interop helpers, such as `__commonJS`, have lost their names. Minified JavaScript compiles like unminified JavaScript. The declarations check your program; the package's JavaScript compiles with the types scriptc infers from it. A program that doesn't typecheck against the inferred types, for example because an `@types/` package describes another release, fails the build at that use. diff --git a/packages/compiler/src/cli/usage.ts b/packages/compiler/src/cli/usage.ts index 23e57c66f..c4d07656a 100644 --- a/packages/compiler/src/cli/usage.ts +++ b/packages/compiler/src/cli/usage.ts @@ -77,9 +77,9 @@ Options: --npm-static compile the named npm packages' shipped JS statically as program modules (repeatable; "auto" opts in every eligible - direct import: own .d.ts, unminified JS, no build-transform - markers; "pkg..." also names every package pkg depends - on). Packages that fail preflight are refused with a + direct import: own .d.ts, no build-transform markers; + "pkg..." also names every package pkg depends on). + Packages that fail preflight are refused with a diagnostic. Eligible dependencies compile automatically. --provenance-sources EXPERIMENTAL: compile npm dependencies from their diff --git a/packages/compiler/src/frontend/lowering/containers/string-replacement.ts b/packages/compiler/src/frontend/lowering/containers/string-replacement.ts index 780c77d0f..156f74869 100644 --- a/packages/compiler/src/frontend/lowering/containers/string-replacement.ts +++ b/packages/compiler/src/frontend/lowering/containers/string-replacement.ts @@ -2,6 +2,7 @@ import { keyedHelperName } from "../stable-names.js"; import * as ts from "../../ts7/adapter.js"; import { BOOL, + DYN, F64, REGEX, STRING, @@ -14,7 +15,7 @@ import { type IrType, type SrcLoc, } from "../../../ir/ir.js"; -import { numLit, strLit, varRef } from "../../../ir/build.js"; +import { dynUndefinedExpr, numLit, strLit, varRef } from "../../../ir/build.js"; import { InternalCompilerError } from "../../../errors.js"; import { locOf } from "../../program.js"; import type { Lowerer } from "../lowerer.js"; @@ -65,14 +66,14 @@ export function lowerStringReplacement( const search = argument(args[0], false); const replacement = argument(args[1], true); if (search.type.kind === "dyn" || replacement.type.kind === "dyn") { - if (replacement.type.kind === "func") { - lowerer.noLowering(`string .${method} with checked search values and callbacks`, call); - } // Evaluate all inputs once before choosing the search protocol. Regexes // retain their native pattern; plain values use the string-search helper. + // A typed callback receives the regex match's arguments or the plain + // search's (match, offset, subject). const values = [receiver, search, replacement]; const locals = values.map((value) => lowerer.declareHiddenLocal("%replaceArg", value.type)); const refs = locals.map((local) => varRef(local.id, local.type, loc)); + const callback = replacement.type.kind === "func"; const checkedReplacement = replacement.type.kind === "dyn" ? lowerer.coerceInto(args[1] ?? call, refs[2]!, STRING) @@ -91,14 +92,27 @@ export function lowerStringReplacement( ? { kind: "ternary", cond: { kind: "libCall", fn: "dyn.nativeRegexIs", args: [refs[1]!], type: BOOL, loc }, - then: { - kind: "regexIntrinsic", - method, - receiver: refs[0]!, - args: [lowerer.coerceInto(args[0] ?? call, refs[1]!, REGEX), checkedReplacement], - type: STRING, - loc, - }, + then: callback + ? { + kind: "libCall", + fn: "regex.replaceCallback", + args: [ + refs[0]!, + lowerer.coerceInto(args[0] ?? call, refs[1]!, REGEX), + lowerer.coerceToExpected(refs[2]!, DYN), + { kind: "boolLit", value: method === "replaceAll", type: BOOL, loc }, + ], + type: STRING, + loc, + } + : { + kind: "regexIntrinsic", + method, + receiver: refs[0]!, + args: [lowerer.coerceInto(args[0] ?? call, refs[1]!, REGEX), checkedReplacement], + type: STRING, + loc, + }, else_: plain, type: STRING, loc, @@ -148,11 +162,17 @@ function lowerStringReplacementValues( if (callback.kind !== "func") { throw new InternalCompilerError(`string .${method} replacement of type ${callback.kind}`); } + // A checked parameter receives the boxed argument (undefined after the + // subject), and a checked, number or boolean result converts with + // ToString. const full = [STRING, F64, STRING]; + const ret = callback.ret.kind; if ( - callback.ret.kind !== "string" || - callback.params.length > full.length || - !callback.params.every((type, index) => typeEquals(type, full[index]!)) + (ret !== "string" && ret !== "dyn" && ret !== "f64" && ret !== "bool") || + !callback.params.every( + (type, index) => + type.kind === "dyn" || (index < full.length && typeEquals(type, full[index]!)), + ) ) { lowerer.noLowering( `string .${method} callback signature`, @@ -175,13 +195,21 @@ function lowerStringReplacementValues( const chunksType = arrayOf(STRING); const chunks = varRef("chunks.0", chunksType, loc); const builder = stringBuilder(loc, chunks); - const value: IrExpr = { + const result: IrExpr = { kind: "callValue", callee: replace, - args: [needle, match, subject].slice(0, callback.params.length), - type: STRING, + args: callback.params.map((type, index) => { + const arg = [needle, match, subject][index]; + if (arg === undefined) return dynUndefinedExpr(loc); + return type.kind === "dyn" ? lowerer.coerceToExpected(arg, DYN) : arg; + }), + type: callback.ret, loc, }; + const value: IrExpr = + ret === "dyn" + ? { kind: "libCall", fn: "dyn.toStringCoerce", args: [result], type: STRING, loc } + : lowerer.ensureString(result, call); const stop: IrStmt = { kind: "break", loc }; const body: IrStmt[] = [ builder.declare("chunks.0", { kind: "arrayLit", elems: [], type: chunksType, loc }), diff --git a/packages/compiler/src/frontend/lowering/expressions/binary.ts b/packages/compiler/src/frontend/lowering/expressions/binary.ts index 58f6e16b5..17aef7b85 100644 --- a/packages/compiler/src/frontend/lowering/expressions/binary.ts +++ b/packages/compiler/src/frontend/lowering/expressions/binary.ts @@ -2841,6 +2841,24 @@ function lowerLogicalPair(lowerer: Lowerer, expr: ts.BinaryExpression, left: IrE loc, }; } + // A JavaScript default that tsc types `any` (`e.title || null` in an + // instance of a function with untyped parameters, where `e.title` is a + // string): the checked domain, as for union operands above. + if ( + target === null && + isJsSourceFile(ts.getSourceFileOfNode(expr)) && + lowerer.dynConvertible(left.type) && + lowerer.dynConvertible(right.type) + ) { + return { + kind: "logical", + op: op === ts.SyntaxKind.AmpersandAmpersandToken ? "&&" : "||", + left: lowerer.coerceInto(expr.left, left, DYN), + right: lowerer.coerceInto(expr.right, right, DYN), + type: DYN, + loc, + }; + } lowerer.unsupported("SC1042", expr); } return { diff --git a/packages/compiler/src/frontend/lowering/var-hoisting.ts b/packages/compiler/src/frontend/lowering/var-hoisting.ts index 3f950f3ca..1a60e7fda 100644 --- a/packages/compiler/src/frontend/lowering/var-hoisting.ts +++ b/packages/compiler/src/frontend/lowering/var-hoisting.ts @@ -194,6 +194,15 @@ function referencesBySymbol( return out; } +/** Where a binding is first stored unconditionally: the scope statement's + * index, and for a declarator of a `var` statement or a `for` initializer, + * the list and the declarator's end. */ +interface Settle { + index: number; + list: ts.VariableDeclarationList | null; + end: number; +} + class ScopeAnalysis { private readonly js: boolean; /** Whether a function declaration's references all settle, by scope @@ -214,8 +223,10 @@ class ScopeAnalysis { } /** The index of the scope statement that first stores `symbol` - * unconditionally, or Infinity. */ - settleIndex(scope: Scope, symbol: ts.Symbol): number { + * unconditionally (Infinity when none does), and, when that statement + * declares it in a `var` statement or a `for` initializer, the end of the + * storing declarator: later declarators of the list run after it. */ + settleIndex(scope: Scope, symbol: ts.Symbol): Settle { const statements = statementsOf(scope); const checker = this.lowerer.checker; for (let i = 0; i < statements.length; i++) { @@ -234,7 +245,7 @@ class ScopeAnalysis { (d.initializer !== undefined || loop) && boundIdentifiersOf(d.name).some((n) => checker.getSymbolAtLocation(n) === symbol) ) - return i; + return { index: i, list: loop ? null : list, end: d.end }; } } else if ( ts.isExpressionStatement(s) && @@ -243,20 +254,21 @@ class ScopeAnalysis { ts.isIdentifier(s.expression.left) && checker.getSymbolAtLocation(s.expression.left) === symbol ) { - return i; + return { index: i, list: null, end: -1 }; } } - return Infinity; + return { index: Infinity, list: null, end: -1 }; } /** Whether `ref` (inside `scope`, or in another module) runs only after - * the statement at index `settle`. `read`: `ref` reads the binding - * itself, so tsc vouches for one in the scope's own code. */ - settles(scope: Scope, settle: number, ref: ts.Identifier, read = true): boolean { + * the store `settle` describes. `read`: `ref` reads the binding itself, + * so tsc vouches for one in the scope's own code. */ + settles(scope: Scope, settle: Settle, ref: ts.Identifier, read = true): boolean { if (ts.getSourceFileOfNode(ref) !== this.sf) return true; const statements = statementsOf(scope); let deferred: ts.Node | null = null; let top: ts.Node | null = null; + const container: ts.Node | undefined = ts.isSourceFile(scope) ? scope : scope.body; for (let n: ts.Node = ref; n.parent !== undefined && n !== scope; n = n.parent) { if ( ts.isFunctionLike(n) || @@ -265,18 +277,25 @@ class ScopeAnalysis { ts.isClassStaticBlockDeclaration(n) ) deferred = n; - if (n.parent === scope || (!ts.isSourceFile(scope) && n.parent === scope.body)) top = n; + // The scope's own statement, not the function body that holds it. + if (n.parent === container) top = n; } if (top === null) return false; if (deferred === null && read && !this.js) return true; if (deferred !== null && ts.isFunctionDeclaration(deferred) && deferred.name !== undefined) return this.functionSettles(scope, settle, deferred); const index = statements.indexOf(top as ts.Statement); - return index > settle || (index === settle && settlesWithin(top, ref)); + return ( + index > settle.index || + (index === settle.index && + (settlesWithin(top, ref) || + // A later declarator of the storing list (`var a = {}, b = { ...a }`). + (settle.list !== null && isInside(ref, settle.list) && ref.pos >= settle.end))) + ); } - private functionSettles(scope: Scope, settle: number, fn: ts.FunctionDeclaration): boolean { - const key = `${fn.pos}:${settle}`; + private functionSettles(scope: Scope, settle: Settle, fn: ts.FunctionDeclaration): boolean { + const key = `${fn.pos}:${settle.index}:${settle.end}`; const known = this.functionMemo.get(key); if (known !== undefined) return known; this.functionMemo.set(key, true); diff --git a/packages/compiler/src/frontend/npm-static.ts b/packages/compiler/src/frontend/npm-static.ts index 800d02eff..0d6c8ec8e 100644 --- a/packages/compiler/src/frontend/npm-static.ts +++ b/packages/compiler/src/frontend/npm-static.ts @@ -614,27 +614,13 @@ export function clearProgramBuiltinAliasCache(): void { * but the bundler-emitted CJS export rewrite (npm-static-rewrite.ts) * types the getter-table shapes and the offender attribution degrades a * package whose surface still breaks its consumers — so those bundles are - * worth ATTEMPTING: eligible when the .d.ts and unminified criteria hold, - * gracefully per-package-degraded when the attempt fails. */ + * worth ATTEMPTING: eligible when the .d.ts criteria hold, gracefully + * per-package-degraded when the attempt fails. Minified JS is attempted + * too: a whitespace-minified dist is ordinary JavaScript, and an + * identifier-minified bundle whose interop helpers lost their names is + * refused by the bundle recognizers, which name the reason. */ const TRANSFORM_MARKERS = ["__webpack_require__"]; -/** Unminified-JS heuristic over an entry source: at least two lines and - * an average line length under 200 characters (minified dists are one - * enormous line; readable code averages well under 100). */ -function looksUnminified(source: string): boolean { - const lines = source.split("\n"); - if (lines.length < 2) return false; - return source.length / lines.length <= 200; -} - -/** A whitespace/syntax-minified bundler dist whose interop helpers keep - * their names (`var __toESM=…,__toCommonJS=mod=>…`): the recognizers - * verify those shapes and refuse the package precisely when they cannot, - * so the minification heuristic need not refuse it up front. */ -function namedBundlerHelpers(source: string): boolean { - return /\b__(?:toCommonJS|commonJS|toESM)\d*\s*=\s*(?:\(|[A-Za-z_$][\w$]*\s*=>)/.test(source); -} - function hasTransformMarkers(source: string): boolean { return TRANSFORM_MARKERS.some((m) => source.includes(m)); } @@ -676,8 +662,6 @@ export function npmStaticIneligibleReason( if (jsEntry === null) return "no runtime JS entry resolves"; const source = trackedReadFile(jsEntry); if (source === null) return `its runtime entry ${jsEntry} cannot be read`; - if (!looksUnminified(source) && !namedBundlerHelpers(source)) - return "its shipped JS looks minified"; if (hasTransformMarkers(source)) { return "its shipped JS carries build-transform markers (bundled/transpiled dist)"; } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7543-var-list-declarator-order.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7543-var-list-declarator-order.js.json new file mode 100644 index 000000000..d193e84ed --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7543-var-list-declarator-order.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7543-var-list-declarator-order.js" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7545-string-regex-checked-arguments.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7545-string-regex-checked-arguments.js.json new file mode 100644 index 000000000..03b3197c4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7545-string-regex-checked-arguments.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7545-string-regex-checked-arguments.js" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts.json new file mode 100644 index 000000000..d026fb402 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7543-var-list-declarator-order.js b/tests/corpus/7543-var-list-declarator-order.js new file mode 100644 index 000000000..8fae06ff9 --- /dev/null +++ b/tests/corpus/7543-var-list-declarator-order.js @@ -0,0 +1,66 @@ +// A `var` read in a later declarator of the statement that stores it runs +// after the store; a read in the same or an earlier declarator sees undefined. +var base = { a: 1, b: "x" }, + more = { ...base, c: 2 }, + keys = Object.keys({ ...more }).join(","), + first = base.a + 1; +console.log(JSON.stringify(more), keys, first); + +function describeBase() { + return base.b + more.c; +} +var rules = { space: /\s+/, word: /\w+/ }, + merged = { ...rules, digit: /\d/ }, + described = describeBase(); +console.log(Object.keys(merged).join(","), merged.word.test("ab"), described); + +var table = { one: 1, two: 2 }, + lookup = function (key) { + return table[key]; + }, + arrow = (key) => ({ ...table })[key]; +console.log(lookup("one"), arrow("two")); + +var readLater = function () { + return later; + }, + early = readLater(), + later = { k: 1 }, + after = readLater().k; +console.log(early, after); + +var self = self ? 1 : 2; +console.log(self); + +for (var i = 0, j = i + 10; i < 2; i++) console.log(i, j); + +function nested() { + var inner = [1, 2, 3], + copy = [...inner, 4], + total = copy.reduce((sum, n) => sum + n, 0); + return { copy, total }; +} +console.log(JSON.stringify(nested())); + +function earlyInFunction() { + show("before"); + var point = { n: 1 }, + next = { m: point.n + 1 }; + show("after"); + function show(when) { + console.log(when, typeof point, point === undefined ? "none" : point.n, next === undefined ? "none" : next.m); + } + var check = () => console.log("check", items === undefined); + check(); + var items = [1]; + check(); +} +earlyInFunction(); + +function loopLists() { + for (var k = 0, sum = k; k < 3; k++) sum += k; + var doubled = sum * 2, + again = doubled + 1; + return [sum, doubled, again]; +} +console.log(JSON.stringify(loopLists())); diff --git a/tests/corpus/7545-string-regex-checked-arguments.js b/tests/corpus/7545-string-regex-checked-arguments.js new file mode 100644 index 000000000..a7ace9cdb --- /dev/null +++ b/tests/corpus/7545-string-regex-checked-arguments.js @@ -0,0 +1,89 @@ +// String search, match and replace with untyped pattern values: a RegExp +// is used as is, and any other value is converted as Node converts it. +const rules = JSON.parse('{"other":{}}'); +rules.other.space = /\s+/g; +rules.other.word = /o+/; +rules.other.text = "l"; +rules.other.dot = "."; +rules.other.none = undefined; +rules.other.nul = null; +rules.other.one = /(l)(o)/; +rules.other.many = /l+/g; +const t = " hello world"; + +console.log( + t.search(rules.other.word), + t.search(rules.other.text), + t.search(rules.other.none), + t.search(rules.other.nul), + t.search(rules.other.space), + "a.b".search(rules.other.dot), +); +console.log(rules.other.space.lastIndex); + +let conversions = 0; +rules.other.obj = { + toString() { + conversions++; + return "w"; + }, +}; +console.log(t.search(rules.other.obj), t.match(rules.other.obj)[0], conversions); + +const all = t.match(rules.other.many); +console.log(JSON.stringify(all), rules.other.many.lastIndex); +const pair = t.match(rules.other.one); +console.log(pair[0], pair[1], pair[2], pair.index, pair.length, pair.input === t); +const literal = t.match(rules.other.dot); +console.log(literal[0], literal.index, literal.length); +console.log(JSON.stringify(t.match(rules.other.none)), t.match(rules.other.zz)); +rules.other.zz = "zz"; +console.log(t.match(rules.other.zz)); +function first(text, re) { + const m = text.match(re); + return m ? m[0] : "none"; +} +console.log(first("abc", rules.other.dot), first("", rules.other.dot)); + +console.log(t.replace(rules.other.space, (m) => "[" + m.length + "]")); +console.log(t.replace(rules.other.word, (m, offset) => m.toUpperCase() + offset)); +console.log(t.replace(rules.other.text, (m, offset, subject) => "<" + m + offset + subject.length + ">")); +console.log(t.replaceAll(rules.other.text, (m) => m.toUpperCase())); +console.log(t.replaceAll(rules.other.space, () => "_")); +console.log(t.replace(rules.other.one, (m, p1, p2, offset) => p1 + "|" + p2 + "|" + offset)); +console.log(t.replace(rules.other.text, (m, offset, subject, extra) => typeof extra)); +console.log(t.replace(rules.other.text, (m) => m.length)); +try { + t.replaceAll(rules.other.word, (m) => m); +} catch (e) { + console.log(e.constructor.name, e.message); +} + +function tabs(text, width) { + let column = 0; + let out = ""; + for (const ch of text) { + if (ch === "\t") { + const pad = 4 - (column % 4); + out += " ".repeat(pad); + column += pad; + } else { + out += ch; + column++; + } + } + return out + width; +} +class Lexer { + constructor(table) { + this.rules = table; + } + indent(line, width) { + return line.replace(this.rules.leading, (w) => tabs(w, width)); + } + firstText(line) { + return line.search(this.rules.nonSpace); + } +} +const lexer = new Lexer({ leading: /^( {1,4}| {0,3}\t)/, nonSpace: /[^ ]/ }); +console.log(JSON.stringify(lexer.indent("\tabc", 2)), lexer.firstText(" x")); diff --git a/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts b/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts new file mode 100644 index 000000000..0c94acb0d --- /dev/null +++ b/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts @@ -0,0 +1,22 @@ +// A one-line minified ES module. Its module-level var statement spreads +// earlier declarators into later ones, its tokenizer reads regexes through +// an untyped rules table (search, match and replace with callbacks), and +// one function receives both typed and untyped link objects. +import { render } from "gtminimark"; + +const doc = [ + "# Title \\*one\\*", + "## Sub", + '[a](/x "T\\*")', + "[b](/y)", + '[r]: /ref "Ref"', + "[n]: /none", + "[R]", + "[n]", + "[z]", + "\tcode *em* line", + " three spaces *x* and *y*", + "plain words here", +].join("\n"); +console.log(render(doc)); +console.log(render(doc, { pedantic: true })); diff --git a/tests/fixtures/npm/node_modules/gtminimark/dist/index.d.ts b/tests/fixtures/npm/node_modules/gtminimark/dist/index.d.ts new file mode 100644 index 000000000..295430b86 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtminimark/dist/index.d.ts @@ -0,0 +1 @@ +export declare function render(src: string, options?: { pedantic?: boolean }): string; diff --git a/tests/fixtures/npm/node_modules/gtminimark/dist/index.js b/tests/fixtures/npm/node_modules/gtminimark/dist/index.js new file mode 100644 index 000000000..c5e1daa31 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtminimark/dist/index.js @@ -0,0 +1 @@ +var m = { heading: /^(#{1,6}) +(.*)$/, link: /^\[([^\]]*)\]\(([^ )]*)(?: "([^"]*)")?\)/, ref: /^\[([^\]]*)\]$/, lead: /^( {1,4}| {0,3}\t)/, nonSpace: /[^ ]/, punct: /\\([\\*_\[\]])/g, word: /\w+/g, def: /^\[([^\]]*)\]: (\S+)(?: "([^"]*)")?$/ },B = { ...m, emph: /\*([^*]+)\*/g },P = { ...B, word: /[a-z]+/g },K = { normal: { other: m, inline: B }, pedantic: { other: m, inline: P, strict: !0 } };function X(a, e) { let t = 0, n = ""; for (let i of a) if (i === "\t") { let r = 4 - t % 4; n += " ".repeat(r), t += r } else n += i, t++; return n + e; }function Se(a, e, t, n) { let r = e.href, o = e.title || null; n.links++; return { type: "link", raw: t, href: r, title: o, text: a[1] }; }class T {constructor() { this.state = { links: 0 }; }heading(s) { let c = s.match(this.rules.other.heading); return c ? "h" + c[1].length + ":" + c[2].replace(this.rules.other.punct, "$1") : null; }indent(s, w) { return s.replace(this.rules.other.lead, W => X(W, w)); }first(s) { return s.search(this.rules.other.nonSpace); }emph(s) { return s.replace(this.rules.inline.emph, (W, x) => "" + x + ""); }words(s) { let c = s.match(this.rules.inline.word); return c ? c.length : 0; }link(s) { let c = this.rules.other.link.exec(s); if (!c) return null; let r = c[2], o = ""; o = c[3] ? c[3] : ""; return Se(c, { href: r && r.replace(this.rules.other.punct, "$1"), title: o && o.replace(this.rules.other.punct, "$1") }, c[0], this.state); }ref(s, l) { let c = this.rules.other.ref.exec(s); if (!c) return null; let o = l[c[1].toLowerCase()]; return o ? Se(c, o, c[0], this.state) : { type: "text", raw: c[0] }; }}class L {constructor(e) { this.options = e || {}; this.links = {}; this.tokenizer = new T(); this.tokenizer.rules = this.options.pedantic ? K.pedantic : K.normal; }define(s) { let c = s.match(this.tokenizer.rules.other.def); if (c) this.links[c[1].toLowerCase()] = { href: c[2], title: c[3] || "" }; return !!c; }line(s) {let t = this.tokenizer, h = t.heading(s);if (h) return h;let k = t.link(s);if (k) return JSON.stringify(k);let f = t.ref(s, this.links);if (f) return JSON.stringify(f);return t.first(s) + "|" + JSON.stringify(t.indent(s, t.first(s))) + "|" + t.emph(s) + "|" + t.words(s);}}export function render(src, options) {let l = new L(options), out = [];for (let s of src.split("\n")) if (!l.define(s)) out.push(l.line(s));return out.join("\n") + "\nlinks " + l.tokenizer.state.links;} diff --git a/tests/fixtures/npm/node_modules/gtminimark/package.json b/tests/fixtures/npm/node_modules/gtminimark/package.json new file mode 100644 index 000000000..ff188c682 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtminimark/package.json @@ -0,0 +1,8 @@ +{ + "name": "gtminimark", + "version": "1.0.0", + "description": "A one-line minified ES module: a module-level var statement whose later declarators spread earlier ones, regexes read through an untyped rules table, and a function called with typed and untyped link objects", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts" +} diff --git a/tests/harness/corpus-costs/7543-var-list-declarator-order.js.json b/tests/harness/corpus-costs/7543-var-list-declarator-order.js.json new file mode 100644 index 000000000..cc47657bd --- /dev/null +++ b/tests/harness/corpus-costs/7543-var-list-declarator-order.js.json @@ -0,0 +1 @@ +{ "plain": 4300, "san": 5900 } diff --git a/tests/harness/corpus-costs/7545-string-regex-checked-arguments.js.json b/tests/harness/corpus-costs/7545-string-regex-checked-arguments.js.json new file mode 100644 index 000000000..83b82ee18 --- /dev/null +++ b/tests/harness/corpus-costs/7545-string-regex-checked-arguments.js.json @@ -0,0 +1 @@ +{ "plain": 4900, "san": 4600 } diff --git a/tests/harness/library-mode.test.ts b/tests/harness/library-mode.test.ts index 0d4052952..db4fabb3e 100644 --- a/tests/harness/library-mode.test.ts +++ b/tests/harness/library-mode.test.ts @@ -50,8 +50,8 @@ * no-teachings default is pinned by K5/K7 over * the teachings-free traps profile * K13 npm-posture bare npm specifiers are static-or-refuse: an - * eligible package (own .d.ts, unminified JS, - * no transform markers) compiles statically + * eligible package (own .d.ts, no transform + * markers) compiles statically * into the library graph, its own npm deps * judged by the same bar; every ineligible one * refuses SC4020 naming the failed bar — never @@ -746,11 +746,11 @@ describe("K9: library-mode refusals", () => { * synthesized per test like K9's. Emission never engages — SC4020 is a * frontend verdict — so one lane suffices. */ let npmRefusalCounter = 0; -async function npmRefusal( +async function compileNpmFixture( entryFile: string, exports: unknown[], profilePatch: Record = {}, -): Promise<{ code: string; message: string; hint?: string; note?: string; file: string }[]> { +): Promise>> { const outDir = join(cacheDir, `npm-refusal-${npmRefusalCounter++}`); mkdirSync(outDir, { recursive: true }); const profile = { @@ -770,7 +770,15 @@ async function npmRefusal( }; const profilePath = join(outDir, "profile.json"); writeFileSync(profilePath, JSON.stringify(profile)); - const result = await compileLibrary({ profilePath, outDir }); + return compileLibrary({ profilePath, outDir }); +} + +async function npmRefusal( + entryFile: string, + exports: unknown[], + profilePatch: Record = {}, +): Promise<{ code: string; message: string; hint?: string; note?: string; file: string }[]> { + const result = await compileNpmFixture(entryFile, exports, profilePatch); expect(result.ok).toBe(false); if (result.ok) throw new Error("unreachable"); return result.diagnostics.map((d) => ({ @@ -795,13 +803,11 @@ describe("K13: library-mode npm refusals (SC4020, static-or-refuse)", () => { expect(diags[0]!.file.endsWith("lib-markers.ts")).toBe(true); }); - test("minified shipped JS refuses", async () => { - const diags = await npmRefusal("lib-minified.ts", [ + test("minified shipped JS compiles", async () => { + const result = await compileNpmFixture("lib-minified.ts", [ { export: "f", symbol: "kx_f", params: ["f64", "f64"], returns: "f64" }, ]); - expect(diags.map((d) => d.code)).toEqual(["SC4020"]); - expect(diags[0]!.message).toContain("'slimmed'"); - expect(diags[0]!.message).toContain("looks minified"); + expect(result.ok ? [] : result.diagnostics.map((d) => d.message)).toEqual([]); }); test("no own .d.ts refuses by name — never the generic import fence", async () => { diff --git a/tests/harness/npm-static.test.ts b/tests/harness/npm-static.test.ts index 7ee76307f..11e729f60 100644 --- a/tests/harness/npm-static.test.ts +++ b/tests/harness/npm-static.test.ts @@ -1490,6 +1490,15 @@ describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { ]); }, 180_000); + // 7546: a one-line minified ES module: a var statement whose later + // declarators spread earlier ones, regexes read through an untyped rules + // table, and a function instantiated with typed and untyped arguments. + test("the minified module 7546-minified-esm-rules compiles statically and matches Node", async () => { + await expectNodeParity(join(fixturesRoot, "npm/cases/7546-minified-esm-rules/main.ts"), [ + "gtminimark", + ]); + }, 180_000); + // A whitespace/syntax-minified bundle whose helpers keep their names is // judged by the recognizers, not refused up front as minified. test("--npm-static=auto opts a minified bundle with named helpers in", () => { diff --git a/tests/library-mode/npm-refuse/lib-minified.ts b/tests/library-mode/npm-refuse/lib-minified.ts index 9e5c16e75..492cd5369 100644 --- a/tests/library-mode/npm-refuse/lib-minified.ts +++ b/tests/library-mode/npm-refuse/lib-minified.ts @@ -1,5 +1,5 @@ -// SC4020 fixture: slimmed ships a one-long-line dist — minified JS fails -// the eligibility bar. +// slimmed ships a one-long-line dist: minified JS compiles like any other +// eligible package. import { pick } from "slimmed"; export function f(a: number, b: number): number { diff --git a/tests/library-mode/npm-static/lib.ts b/tests/library-mode/npm-static/lib.ts index 27107f29d..15bad17a3 100644 --- a/tests/library-mode/npm-static/lib.ts +++ b/tests/library-mode/npm-static/lib.ts @@ -1,5 +1,5 @@ // Library-mode npm posture fixture (static-or-refuse): a bare specifier -// naming an ELIGIBLE package (own .d.ts, unminified shipped JS, no +// naming an ELIGIBLE package (own .d.ts, no // build-transform markers) compiles statically as part of the library // graph — no flag, no island — and the package's own top-level require // ("mathdep") joins through the same bar. The node:path import alongside diff --git a/tests/library-mode/npm-static/node_modules/mathkit/index.js b/tests/library-mode/npm-static/node_modules/mathkit/index.js index f9f9242c3..eadfefba9 100644 --- a/tests/library-mode/npm-static/node_modules/mathkit/index.js +++ b/tests/library-mode/npm-static/node_modules/mathkit/index.js @@ -1,6 +1,6 @@ "use strict"; // A vendored fake package for the library-mode npm posture: eligible by -// every bar (own .d.ts, unminified CJS, no build-transform markers), with +// every bar (own .d.ts, no build-transform markers), with // its OWN npm dep edge (mathdep) so the transitive fixpoint is exercised. const steps = require("mathdep"); From ba5d73e90b3289a52a31d75095da4b7d50daad50 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 06:20:02 -0500 Subject: [PATCH 036/184] Recognize esbuild's __commonJS helper under a minified name Refs #1777 An identifier-minified esbuild bundle renames the __commonJS helper (var j=(u,t)=>()=>(t||u((t={exports:{}}).exports,t),t.exports)). The recognizer already compared the helper's shape independently of its parameter names, but only used a renamed match to refuse the package. A stable top-level declaration with the helper's shape now splits the bundle's chunks like the named helper, so lru-cache's CommonJS build is no longer refused. --- docs/content/docs/dependencies.mdx | 2 +- .../src/frontend/lowering/lower-calls.ts | 27 +++++++++++++++++++ .../npm-static-bundled-chunks.test.ts | 11 ++++---- .../src/frontend/npm-static-bundled-chunks.ts | 20 ++++++-------- .../src/frontend/npm-static-rewrite-syntax.ts | 14 ++++++++-- packages/compiler/src/frontend/npm-static.ts | 5 ++-- packages/compiler/src/frontend/program.ts | 16 +++++++++-- .../main.cjs.json | 7 +++++ .../main.cjs.json | 8 ++++++ .../7547-minified-cjs-chunks/main.ts.json | 6 +++++ .../lib.cjs | 10 +++++++ .../main.cjs | 19 +++++++++++++ .../7549-cjs-minified-esmodule-stamp/main.cjs | 7 +++++ .../module-stamped.cjs | 3 +++ .../stamped.cjs | 5 ++++ .../cases/7547-minified-cjs-chunks/main.ts | 12 +++++++++ .../node_modules/gtminicache/dist/index.d.ts | 7 +++++ .../node_modules/gtminicache/dist/index.js | 1 + .../npm/node_modules/gtminicache/package.json | 7 +++++ .../main.cjs.json | 1 + .../main.cjs.json | 1 + tests/harness/npm-static.test.ts | 9 +++++++ 22 files changed, 173 insertions(+), 25 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts.json create mode 100644 tests/corpus/7548-cjs-receiverless-builtin-calls/lib.cjs create mode 100644 tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs create mode 100644 tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs create mode 100644 tests/corpus/7549-cjs-minified-esmodule-stamp/module-stamped.cjs create mode 100644 tests/corpus/7549-cjs-minified-esmodule-stamp/stamped.cjs create mode 100644 tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts create mode 100644 tests/fixtures/npm/node_modules/gtminicache/dist/index.d.ts create mode 100644 tests/fixtures/npm/node_modules/gtminicache/dist/index.js create mode 100644 tests/fixtures/npm/node_modules/gtminicache/package.json create mode 100644 tests/harness/corpus-costs/7548-cjs-receiverless-builtin-calls/main.cjs.json create mode 100644 tests/harness/corpus-costs/7549-cjs-minified-esmodule-stamp/main.cjs.json diff --git a/docs/content/docs/dependencies.mdx b/docs/content/docs/dependencies.mdx index f02dcfbf0..a9c5ae138 100644 --- a/docs/content/docs/dependencies.mdx +++ b/docs/content/docs/dependencies.mdx @@ -53,7 +53,7 @@ Type-only imports follow Node.js's type stripping. `import type` and `export typ scriptc compiles a package automatically when it: - ships its own type declarations, or is typed by its DefinitelyTyped `@types/` package, or is a dependency of another package that has no declarations at all, and -- ships JavaScript that isn't a bundle whose bundler-interop helpers, such as `__commonJS`, have lost their names. Minified JavaScript compiles like unminified JavaScript. +- ships JavaScript that isn't a webpack bundle. Minified JavaScript compiles like unminified JavaScript. The declarations check your program; the package's JavaScript compiles with the types scriptc infers from it. A program that doesn't typecheck against the inferred types, for example because an `@types/` package describes another release, fails the build at that use. diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 436d4f1d4..672801e9e 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -306,6 +306,22 @@ function mayHoldError(lowerer: Lowerer, type: ts.Type, errorType: ts.Type | null ); } +/** The `f` of a `(0, f)` callee (a numeric left operand, so dropping it + * changes only the call's receiver), or null. */ +function receiverlessCalleeOf(callee: ts.Expression): ts.Expression | null { + let e = callee; + while (ts.isParenthesizedExpression(e)) e = e.expression; + if ( + !ts.isBinaryExpression(e) || + e.operatorToken.kind !== ts.SyntaxKind.CommaToken || + !ts.isNumericLiteral(e.left) + ) + return null; + let right = e.right; + while (ts.isParenthesizedExpression(right)) right = right.expression; + return right; +} + export function lowerCall(lowerer: Lowerer, expr: ts.CallExpression): IrExpr { const loc = locOf(expr); if (lowerer.errorAccessorReflection === null) noteErrorAccessorReflection(lowerer, expr); @@ -329,6 +345,17 @@ export function lowerCall(lowerer: Lowerer, expr: ts.CallExpression): IrExpr { const json = lowerJsonRequireCall(lowerer, expr); if (json) return json; + // `(0, ns.member)(...)` over a builtin module (tsc's and esbuild's call + // without a receiver): Node's module functions do not read `this`, so it + // is the member call. + { + const member = receiverlessCalleeOf(expr.expression); + if (member !== null && ts.isPropertyAccessExpression(member)) { + const builtin = lowerer.lowerNamespaceBuiltinCall(expr, member); + if (builtin) return builtin; + } + } + // eval(...) and Function(...): Node's EvalError under // --disallow-code-generation-from-strings (code-generation.ts). { diff --git a/packages/compiler/src/frontend/npm-static-bundled-chunks.test.ts b/packages/compiler/src/frontend/npm-static-bundled-chunks.test.ts index 0779db3ce..dc238c622 100644 --- a/packages/compiler/src/frontend/npm-static-bundled-chunks.test.ts +++ b/packages/compiler/src/frontend/npm-static-bundled-chunks.test.ts @@ -276,13 +276,14 @@ export default require_f(); ).toBeNull(); }); -test("an identifier-minified bundle names its lost helper names", () => { - // esbuild --minify output: the helper shapes survive, their names do not. - const result = split( +test("an identifier-minified bundle splits through its helper's shape", () => { + // esbuild --minify output: the helper shape survives, its name does not. + const answer = expectSameBehavior( `var c=(r,e)=>()=>(e||r((e={exports:{}}).exports,e),e.exports);` + - `var x=c((t,m)=>{m.exports={v:1}});console.log(x().v);`, + `var x=c((t,m)=>{m.exports={v:1}});var y=c(t=>{t.w=x().v+1});` + + `console.log(x().v,y().w,x()===x());`, ); - expect(result).toEqual({ degrade: expect.stringContaining("identifier-minified") as string }); + expect(answer.chunks.map((c) => basename(c.path))).toEqual(["index.js.x.cjs", "index.js.y.cjs"]); }); test("an ES-module bundle of CommonJS chunks names the chunk interop", () => { diff --git a/packages/compiler/src/frontend/npm-static-bundled-chunks.ts b/packages/compiler/src/frontend/npm-static-bundled-chunks.ts index 37a74f2be..66c25403b 100644 --- a/packages/compiler/src/frontend/npm-static-bundled-chunks.ts +++ b/packages/compiler/src/frontend/npm-static-bundled-chunks.ts @@ -525,7 +525,7 @@ export function splitBundledChunks( } } } - const helperNames = [...top.keys()].filter((n) => helperBase(n) === "__commonJS"); + let helperNames = [...top.keys()].filter((n) => helperBase(n) === "__commonJS"); // Every name a recognizer relies on is written nowhere and declared // once at the top level; `Object` itself is never rebound. @@ -552,18 +552,14 @@ export function splitBundledChunks( ); }; if (helperNames.length === 0) { - // The helper's shape under another name: an identifier-minified bundle, - // whose other interop helpers lost their names as well. - for (const [name, t] of top) { + // An identifier-minified bundle: the helper's shape under another name. + helperNames = [...top].flatMap(([name, t]) => { const init = ts.isVariableDeclaration(t.node) ? t.node.initializer : undefined; - if (init !== undefined && stable(name) && helperForm(init, sf, ownNames) !== null) - return { - degrade: - "its shipped JS is an identifier-minified bundle whose bundler-interop helpers " + - "(__commonJS and the rest) have lost the names the recognizers verify", - }; - } - return null; + return init !== undefined && stable(name) && helperForm(init, sf, ownNames) !== null + ? [name] + : []; + }); + if (helperNames.length === 0) return null; } const forms = new Map(); for (const name of helperNames) { diff --git a/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts b/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts index c48420625..917041603 100644 --- a/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts +++ b/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts @@ -260,11 +260,21 @@ function definePropertyExportOf( value = getterReturnOf(p); } } - const esModuleStamp = - name === "__esModule" && value !== null && value.kind === ts.SyntaxKind.TrueKeyword; + const esModuleStamp = name === "__esModule" && value !== null && isTrueLiteral(value); return { name, value, esModuleStamp }; } +/** `true`, or `!0`, its minified spelling. */ +function isTrueLiteral(expr: ts.Expression): boolean { + return ( + expr.kind === ts.SyntaxKind.TrueKeyword || + (ts.isPrefixUnaryExpression(expr) && + expr.operator === ts.SyntaxKind.ExclamationToken && + ts.isNumericLiteral(expr.operand) && + expr.operand.text === "0") + ); +} + /** Space-pads [start, end) of `text` preserving newlines — the recognized * statement disappears while every original offset survives. */ function padSpan(text: string, start: number, end: number): string { diff --git a/packages/compiler/src/frontend/npm-static.ts b/packages/compiler/src/frontend/npm-static.ts index 0d6c8ec8e..eed02c95c 100644 --- a/packages/compiler/src/frontend/npm-static.ts +++ b/packages/compiler/src/frontend/npm-static.ts @@ -616,9 +616,8 @@ export function clearProgramBuiltinAliasCache(): void { * package whose surface still breaks its consumers — so those bundles are * worth ATTEMPTING: eligible when the .d.ts criteria hold, gracefully * per-package-degraded when the attempt fails. Minified JS is attempted - * too: a whitespace-minified dist is ordinary JavaScript, and an - * identifier-minified bundle whose interop helpers lost their names is - * refused by the bundle recognizers, which name the reason. */ + * too: a minified dist is ordinary JavaScript, and the bundle recognizers + * know esbuild's __commonJS helper by its shape under any name. */ const TRANSFORM_MARKERS = ["__webpack_require__"]; function hasTransformMarkers(source: string): boolean { diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index 68b649b07..932ac7054 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -1436,8 +1436,20 @@ function requiresOf7( return out; } +/** `true`, or `!0`, its minified spelling. */ +function isTrueLiteral(expr: ts.Expression): boolean { + return ( + expr.kind === ts.SyntaxKind.TrueKeyword || + (ts.isPrefixUnaryExpression(expr) && + expr.operator === ts.SyntaxKind.ExclamationToken && + ts.isNumericLiteral(expr.operand) && + expr.operand.text === "0") + ); +} + /** The exact `Object.defineProperty(exports|module.exports, "__esModule", - * { value: true })` interop stamp (shared by preflight and the no-op lowering). */ + * { value: true })` interop stamp (shared by preflight and the no-op lowering); + * `{ value: !0 }` is the same stamp. */ export function isEsModuleStamp(expr: ts.Expression): boolean { if (!ts.isCallExpression(expr) || expr.questionDotToken !== undefined) return false; const callee = expr.expression; @@ -1471,7 +1483,7 @@ export function isEsModuleStamp(expr: ts.Expression): boolean { ts.isPropertyAssignment(p) && ts.isIdentifier(p.name) && p.name.text === "value" && - p.initializer.kind === ts.SyntaxKind.TrueKeyword + isTrueLiteral(p.initializer) ); } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs.json new file mode 100644 index 000000000..f3b8ba419 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs.json @@ -0,0 +1,7 @@ +{ + "order": [ + "/tests/corpus/7548-cjs-receiverless-builtin-calls/lib.cjs", + "/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs.json new file mode 100644 index 000000000..08adae885 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs.json @@ -0,0 +1,8 @@ +{ + "order": [ + "/tests/corpus/7549-cjs-minified-esmodule-stamp/stamped.cjs", + "/tests/corpus/7549-cjs-minified-esmodule-stamp/module-stamped.cjs", + "/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts.json new file mode 100644 index 000000000..212551ccd --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7548-cjs-receiverless-builtin-calls/lib.cjs b/tests/corpus/7548-cjs-receiverless-builtin-calls/lib.cjs new file mode 100644 index 000000000..248e66270 --- /dev/null +++ b/tests/corpus/7548-cjs-receiverless-builtin-calls/lib.cjs @@ -0,0 +1,10 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: !0 }); +exports.label = exports.describe = void 0; +const node_util_1 = require("node:util"); +const node_path_1 = require("node:path"); +function describe(name, size) { + return (0, node_util_1.format)("%s holds %d (%s)", name, size, (0, node_path_1.basename)("/a/b/c.txt")); +} +exports.describe = describe; +exports.label = "lib"; diff --git a/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs b/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs new file mode 100644 index 000000000..6fd0b78a9 --- /dev/null +++ b/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs @@ -0,0 +1,19 @@ +"use strict"; +// tsc and esbuild output: the __esModule stamp spelled { value: !0 }, and +// calls of builtin module functions without a receiver, (0, ns.member)(...). +const lib = require("./lib.cjs"); +const dc = require("node:diagnostics_channel"); +const util = require("node:util"); +const os = require("node:os"); +const path = require("node:path"); +console.log(lib.label, lib.describe("cache", 3)); +const metrics = (0, dc.channel)("cache:metrics"); +console.log(metrics.name, metrics.hasSubscribers); +const seen = []; +dc.subscribe("cache:metrics", (message) => seen.push(message.key)); +metrics.publish({ key: "a" }); +console.log(seen.join(","), metrics.hasSubscribers); +const tracing = (0, dc.tracingChannel)("cache"); +console.log(tracing.hasSubscribers); +console.log((0, util.inspect)({ a: [1, 2] }), (0, util.isDeepStrictEqual)([1], [1])); +console.log((0, path.join)("a", "b"), typeof (0, os.platform)()); diff --git a/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs b/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs new file mode 100644 index 000000000..c619a847a --- /dev/null +++ b/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs @@ -0,0 +1,7 @@ +"use strict"; +// Minifiers spell `true` as `!0`: Object.defineProperty(exports, +// "__esModule", { value: !0 }) is the same interop stamp as { value: true }. +const stamped = require("./stamped.cjs"); +const moduleStamped = require("./module-stamped.cjs"); +console.log(stamped.answer, stamped.twice(4), moduleStamped.label); +console.log(typeof stamped.twice, moduleStamped.label.length); diff --git a/tests/corpus/7549-cjs-minified-esmodule-stamp/module-stamped.cjs b/tests/corpus/7549-cjs-minified-esmodule-stamp/module-stamped.cjs new file mode 100644 index 000000000..96a2aae7b --- /dev/null +++ b/tests/corpus/7549-cjs-minified-esmodule-stamp/module-stamped.cjs @@ -0,0 +1,3 @@ +"use strict"; +Object.defineProperty(module.exports, "__esModule", { value: !0 }); +module.exports.label = "module"; diff --git a/tests/corpus/7549-cjs-minified-esmodule-stamp/stamped.cjs b/tests/corpus/7549-cjs-minified-esmodule-stamp/stamped.cjs new file mode 100644 index 000000000..cd3b9c400 --- /dev/null +++ b/tests/corpus/7549-cjs-minified-esmodule-stamp/stamped.cjs @@ -0,0 +1,5 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: !0 }); +exports.answer = void 0; +exports.answer = 42; +exports.twice = (n) => n * 2; diff --git a/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts b/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts new file mode 100644 index 000000000..ad2303f47 --- /dev/null +++ b/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts @@ -0,0 +1,12 @@ +// An identifier-minified esbuild CommonJS bundle (lru-cache's shape): its +// __commonJS helper and chunk parameters lost their names, the chunks +// stamp __esModule with { value: !0 }, and they call node:util and +// node:diagnostics_channel functions in the (0, ns.member)(...) form. +import { Cache } from "gtminicache"; + +const cache = new Cache(2); +cache.set("a", 1).set("b", 2); +cache.get("a"); +cache.set("c", 3); +console.log(cache.keys().join(","), cache.get("b"), cache.get("c")); +console.log(cache.describe("cache")); diff --git a/tests/fixtures/npm/node_modules/gtminicache/dist/index.d.ts b/tests/fixtures/npm/node_modules/gtminicache/dist/index.d.ts new file mode 100644 index 000000000..5d12cad7e --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtminicache/dist/index.d.ts @@ -0,0 +1,7 @@ +export declare class Cache { + constructor(max: number); + set(key: K, value: V): this; + get(key: K): V | undefined; + keys(): K[]; + describe(label: string): string; +} diff --git a/tests/fixtures/npm/node_modules/gtminicache/dist/index.js b/tests/fixtures/npm/node_modules/gtminicache/dist/index.js new file mode 100644 index 000000000..31c6deef9 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtminicache/dist/index.js @@ -0,0 +1 @@ +"use strict";var n=(s,e)=>()=>(e||s((e={exports:{}}).exports,e),e.exports);var u=n(i=>{"use strict";Object.defineProperty(i,"__esModule",{value:!0});i.metrics=void 0;var o=require("node:diagnostics_channel");i.metrics=(0,o.channel)("gtminicache:metrics")});var d=n(r=>{"use strict";Object.defineProperty(r,"__esModule",{value:!0});r.describe=a;var l=require("node:util");function a(s,e){return(0,l.format)("%s holds %d",s,e)}});Object.defineProperty(exports,"__esModule",{value:!0});exports.Cache=void 0;var h=u(),_=d(),c=class{#e=new Map;#t;constructor(e){this.#t=e}set(e,t){for(this.#e.delete(e),this.#e.set(e,t);this.#e.size>this.#t;)this.#e.delete(this.#e.keys().next().value);return h.metrics.hasSubscribers&&h.metrics.publish({key:e}),this}get(e){let t=this.#e.get(e);return t!==void 0&&(this.#e.delete(e),this.#e.set(e,t)),t}keys(){return[...this.#e.keys()]}describe(e){return(0,_.describe)(e,this.#e.size)}};exports.Cache=c; diff --git a/tests/fixtures/npm/node_modules/gtminicache/package.json b/tests/fixtures/npm/node_modules/gtminicache/package.json new file mode 100644 index 000000000..3f57e1ae3 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtminicache/package.json @@ -0,0 +1,7 @@ +{ + "name": "gtminicache", + "version": "1.0.0", + "description": "tsc-style CommonJS modules bundled by esbuild 0.27.0 with --bundle --platform=node --format=cjs --minify: the __commonJS helper and the chunk parameters lose their names, the __esModule stamps read { value: !0 }, and calls into node:util and node:diagnostics_channel use the (0, ns.member)(...) form", + "main": "./dist/index.js", + "types": "./dist/index.d.ts" +} diff --git a/tests/harness/corpus-costs/7548-cjs-receiverless-builtin-calls/main.cjs.json b/tests/harness/corpus-costs/7548-cjs-receiverless-builtin-calls/main.cjs.json new file mode 100644 index 000000000..8a603b2f4 --- /dev/null +++ b/tests/harness/corpus-costs/7548-cjs-receiverless-builtin-calls/main.cjs.json @@ -0,0 +1 @@ +{ "plain": 12300, "san": 13600 } diff --git a/tests/harness/corpus-costs/7549-cjs-minified-esmodule-stamp/main.cjs.json b/tests/harness/corpus-costs/7549-cjs-minified-esmodule-stamp/main.cjs.json new file mode 100644 index 000000000..d00a49ec8 --- /dev/null +++ b/tests/harness/corpus-costs/7549-cjs-minified-esmodule-stamp/main.cjs.json @@ -0,0 +1 @@ +{ "plain": 16000, "san": 21200 } diff --git a/tests/harness/npm-static.test.ts b/tests/harness/npm-static.test.ts index 11e729f60..388da2138 100644 --- a/tests/harness/npm-static.test.ts +++ b/tests/harness/npm-static.test.ts @@ -1499,6 +1499,15 @@ describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { ]); }, 180_000); + // 7547: an identifier-minified esbuild CommonJS bundle: the __commonJS + // helper and chunk parameters renamed, { value: !0 } stamps, and + // (0, ns.member)(...) calls of builtin module functions. + test("the identifier-minified bundle 7547-minified-cjs-chunks compiles statically and matches Node", async () => { + await expectNodeParity(join(fixturesRoot, "npm/cases/7547-minified-cjs-chunks/main.ts"), [ + "gtminicache", + ]); + }, 180_000); + // A whitespace/syntax-minified bundle whose helpers keep their names is // judged by the recognizers, not refused up front as minified. test("--npm-static=auto opts a minified bundle with named helpers in", () => { From 6f7d1960677efbe0835b85b20c766cda166d8f7a Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 06:47:08 -0500 Subject: [PATCH 037/184] Count records built for callbacks and replacements as built at run time The key-order witness treats records JavaScript builds as having a run-time order. Two cases this change narrowed are counted again: - A native callback that a checked JavaScript function finds inside a record or array it receives is called with records JavaScript builds. The walk of a checked function's parameters reset to the built direction at the fields and elements of the records it passes out, so the callback's own parameters counted as passed out. Children of a container passed out now count in both directions (written by JavaScript, and read out of it), and class instances keep the direction too. - A replaceable method's checked result builds a record when a replacement returns one. It is left out only while every site the census found for the method name reads it or assigns a function whose return type is primitive, as the handlers a router class installs through a computed key do. The census now records every such site (Lowerer.methodAccessSites). --- .../src/frontend/lowering/class-callbacks.ts | 8 +- .../lowering/class-method-mutations.ts | 18 +-- .../lowering/key-order-witness.test.ts | 34 ++++-- .../frontend/lowering/key-order-witness.ts | 108 ++++++++++++++---- .../src/frontend/lowering/lower-classes.ts | 4 +- .../compiler/src/frontend/lowering/lowerer.ts | 17 +++ .../main.ts.json | 7 ++ .../lib.js | 11 ++ .../main.ts | 50 ++++++++ .../main.ts.json | 1 + 10 files changed, 215 insertions(+), 43 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7623-key-order-javascript-built-records/main.ts.json create mode 100644 tests/corpus/7623-key-order-javascript-built-records/lib.js create mode 100644 tests/corpus/7623-key-order-javascript-built-records/main.ts create mode 100644 tests/harness/corpus-costs/7623-key-order-javascript-built-records/main.ts.json diff --git a/packages/compiler/src/frontend/lowering/class-callbacks.ts b/packages/compiler/src/frontend/lowering/class-callbacks.ts index f92f42a80..3a1c7975d 100644 --- a/packages/compiler/src/frontend/lowering/class-callbacks.ts +++ b/packages/compiler/src/frontend/lowering/class-callbacks.ts @@ -159,8 +159,8 @@ export function classCallbackValue( /** Notes a checked conversion of a replaceable method's result for the * key-order witness: the result is a value the method or its replacement * returns, not a record built from a dynamic value. */ -function methodResult(lowerer: Lowerer, checked: IrExpr): IrExpr { - return checked.kind === "dynCheck" ? lowerer.keyOrder.methodResult(checked) : checked; +function methodResult(lowerer: Lowerer, checked: IrExpr, name: string): IrExpr { + return checked.kind === "dynCheck" ? lowerer.keyOrder.methodResult(checked, name) : checked; } export function classCallbackCall( @@ -204,7 +204,7 @@ export function classCallbackCall( // generator arm does not box) keeps its native type: the unchanged // method's result stays as it is, and a replacement's dynamic result // takes the checked conversion into that type. - const converted = methodResult(lowerer, lowerer.coerceToExpected(call, fallback.type)); + const converted = methodResult(lowerer, lowerer.coerceToExpected(call, fallback.type), name); if (!typeEquals(converted.type, fallback.type)) { lowerer.pushDiag( unsupportedDiag( @@ -243,7 +243,7 @@ export function classCallbackCall( answer = fallback.type.kind === "void" ? result - : methodResult(lowerer, lowerer.coerceToExpected(result, fallback.type)); + : methodResult(lowerer, lowerer.coerceToExpected(result, fallback.type), name); } // Read after observedSlot built the prototype data and its originals. const originalId = observed ? original?.originalId() : undefined; diff --git a/packages/compiler/src/frontend/lowering/class-method-mutations.ts b/packages/compiler/src/frontend/lowering/class-method-mutations.ts index fd23b6db8..035c2c2a5 100644 --- a/packages/compiler/src/frontend/lowering/class-method-mutations.ts +++ b/packages/compiler/src/frontend/lowering/class-method-mutations.ts @@ -84,7 +84,7 @@ export function collectClassMethodMutations( for (const name of literalValues(lowerer.typeOf(key)) ?? []) methods.add(String(name)); const markName = (name: string, site: ts.Node, arrow = false): void => { if (name === "prototype" || name === "__proto__") unknownKey = site; - if (methods.has(name)) lowerer.prototypeMethodAccesses.set(name, site); + if (methods.has(name)) lowerer.noteMethodAccess(name, site); if (!arrow) receiverFields.add(name); }; const unwrap = (value: ts.Expression): ts.Expression => { @@ -395,10 +395,9 @@ export function collectClassMethodMutations( ts.isStringLiteralLike(base.argumentExpression) && base.argumentExpression.text === "prototype"); if (prototype) { - if (ts.isPropertyAccessExpression(node)) - lowerer.prototypeMethodAccesses.set(node.name.text, node); + if (ts.isPropertyAccessExpression(node)) lowerer.noteMethodAccess(node.name.text, node); else if (ts.isStringLiteralLike(node.argumentExpression)) - lowerer.prototypeMethodAccesses.set(node.argumentExpression.text, node); + lowerer.noteMethodAccess(node.argumentExpression.text, node); else markKey(node.argumentExpression, node); } } @@ -435,7 +434,7 @@ export function collectClassMethodMutations( ? lowerer.checker.getSymbolAtLocation(access.name) : lowerer.checker.getPropertyOfType(lowerer.typeOf(access.expression), name); if (symbol && symbol.declarations.some(ts.isMethodDeclaration)) - lowerer.prototypeMethodAccesses.set(name, access); + lowerer.noteMethodAccess(name, access); } lowerer.checker.prefetchSymbolRoots([...globals, ...reflection]); for (const node of globals) @@ -571,9 +570,10 @@ export function collectClassMethodMutations( ); for (const { target, site } of propertyWrites) { const name = target.name.text; - if (lowerer.prototypeMethodAccesses.has(name) || functionFree(lowerer.typeOf(target))) continue; - if (!methodOwnerCanInhabit(target.expression, name)) continue; - lowerer.prototypeMethodAccesses.set(name, site); + if (functionFree(lowerer.typeOf(target)) || !methodOwnerCanInhabit(target.expression, name)) + continue; + if (lowerer.prototypeMethodAccesses.has(name)) lowerer.noteMethodSite(name, site); + else lowerer.noteMethodAccess(name, site); } for (const { key, receiver, site } of computedWrites) { // A class method declared with a well-known symbol key is not a write; @@ -620,7 +620,7 @@ export function collectClassMethodMutations( } } if (unknownKey) { - for (const method of methods) lowerer.prototypeMethodAccesses.set(method, unknownKey); + for (const method of methods) lowerer.noteMethodAccess(method, unknownKey); for (const method of symbolMethods) lowerer.replacedSymbolMethods.add(method); } if (!unknownKey && !unknownOwnField) diff --git a/packages/compiler/src/frontend/lowering/key-order-witness.test.ts b/packages/compiler/src/frontend/lowering/key-order-witness.test.ts index dc9149119..8058d0066 100644 --- a/packages/compiler/src/frontend/lowering/key-order-witness.test.ts +++ b/packages/compiler/src/frontend/lowering/key-order-witness.test.ts @@ -15,6 +15,7 @@ import { type KeyOrderSegment, } from "./key-order-witness.js"; import type { Lowerer } from "./lowerer.js"; +import * as ts from "../ts7/adapter.js"; const loc = { file: "order.ts", start: 0, end: 1 }; /** `number | undefined`: the optional-field slot type. */ @@ -40,6 +41,7 @@ function fixture(): Fixture { keyOrder: notes, unions: new Map([["u0", { id: "u0", arms: [F64, { kind: "undefinedT" }] }]]), classes: new Map(), + methodAccessSites: new Map(), armTag: (unionId: string, arm: IrType): number => unionId === "u0" && arm.kind === "undefinedT" ? 1 : -1, recordFieldPresent: (obj: IrExpr, shapeId: string, field: string): IrExpr => ({ @@ -739,20 +741,38 @@ test("a replaceable method's checked result builds no record", () => { expect( selfCopy((f, ab) => { const result: IrExpr = { kind: "dynCheck", value: ref("r", { kind: "dyn" }), type: ab, loc }; - return [observe(f.notes.methodResult(result))]; + return [observe(f.notes.methodResult(result, "load"))]; }), ).toEqual(["b", "a"]); }); +test("a replaceable method's checked result builds records when a replacement may return one", () => { + expect( + selfCopy((f, ab) => { + // A site the census cannot see through (a computed write of some + // value, a reflection call) may install a replacement returning a + // record built in any order. + (f.lowerer.methodAccessSites as Map).set("load", [ + { kind: ts.SyntaxKind.CallExpression } as unknown as ts.Node, + ]); + const result: IrExpr = { kind: "dynCheck", value: ref("r", { kind: "dyn" }), type: ab, loc }; + return [observe(f.notes.methodResult(result, "load"))]; + }), + ).toEqual(["a", "b"]); +}); + test("a replaceable method's result keeps its witness through an IR rewrite", () => { expect( selfCopy((f, ab) => { - const result = f.notes.methodResult({ - kind: "dynCheck", - value: ref("r", { kind: "dyn" }), - type: ab, - loc, - }); + const result = f.notes.methodResult( + { + kind: "dynCheck", + value: ref("r", { kind: "dyn" }), + type: ab, + loc, + }, + "load", + ); const identity = { expr: (e: IrExpr): IrExpr => e, stmt: (st: IrStmt): IrStmt => st }; observeRecordLiteralRebuilds((from, to) => f.notes.follow(from, to)); try { diff --git a/packages/compiler/src/frontend/lowering/key-order-witness.ts b/packages/compiler/src/frontend/lowering/key-order-witness.ts index 2f573e676..933c9efcd 100644 --- a/packages/compiler/src/frontend/lowering/key-order-witness.ts +++ b/packages/compiler/src/frontend/lowering/key-order-witness.ts @@ -69,6 +69,7 @@ import { transformStmtList, } from "../../ir/traverse.js"; import type { Lowerer } from "./lowerer.js"; +import * as ts from "../ts7/adapter.js"; import { ALWAYS_THROW_LIB_FNS } from "../../ir/builtin-effects.js"; /** The statements a key-order helper runs for record `ref`: `walks[0]`, @@ -124,7 +125,7 @@ export type KeyOrderSegment = export class KeyOrderNotes { private readonly literals = new Map(); private readonly copies = new Map(); - private readonly methodResults = new Set(); + private readonly methodResults = new Map(); /** Shapes a frontend order surface (keys, values/entries, inspect, * index-signature merges) walks. */ readonly observed = new Set(); @@ -141,10 +142,10 @@ export class KeyOrderNotes { return node; } - /** The checked result of a typed call of a method the program may - * replace on its instance (class-callbacks.ts classCallbackCall). */ - methodResult(node: IrExpr): IrExpr { - this.methodResults.add(node); + /** The checked result of a typed call of method `name`, which the program + * may replace on its instance (class-callbacks.ts classCallbackCall). */ + methodResult(node: IrExpr, name: string): IrExpr { + this.methodResults.set(node, name); return node; } @@ -155,7 +156,8 @@ export class KeyOrderNotes { if (segments !== undefined) this.literals.set(to, segments); const source = this.copies.get(from); if (source !== undefined) this.copies.set(to, source); - if (this.methodResults.has(from)) this.methodResults.add(to); + const method = this.methodResults.get(from); + if (method !== undefined) this.methodResults.set(to, method); } observe(type: IrType): void { @@ -170,8 +172,9 @@ export class KeyOrderNotes { return this.copies.get(node); } - isMethodResult(node: IrExpr): boolean { - return this.methodResults.has(node); + /** The method whose checked result `node` is, if it is one. */ + methodResultOf(node: IrExpr): string | undefined { + return this.methodResults.get(node); } } @@ -243,6 +246,54 @@ function primitiveSource(node: IrExpr & { kind: "dynCheck" }): boolean { return node.value.kind === "dynFrom" && PRIMITIVE_KINDS.has(node.value.value.type.kind); } +/** Whether a census site for a method name can only read the method or + * install a replacement whose result holds no object: a read + * (`C.prototype.m`, `obj.m`) outside an assignment target, or an assignment + * of a function expression whose return type is primitive. Other sites (a + * computed write of another value, a reflection call, a property bag) may + * install a replacement that returns a record. */ +function replacesWithoutRecords(lowerer: Lowerer, site: ts.Node): boolean { + const parent = site.parent; + if (ts.isPropertyAccessExpression(site) || ts.isElementAccessExpression(site)) + return !( + parent !== undefined && + ts.isBinaryExpression(parent) && + parent.left === site && + parent.operatorToken.kind >= ts.SyntaxKind.FirstAssignment && + parent.operatorToken.kind <= ts.SyntaxKind.LastAssignment + ); + if (!ts.isBinaryExpression(site) || site.operatorToken.kind !== ts.SyntaxKind.EqualsToken) + return false; + let value = site.right; + while (ts.isParenthesizedExpression(value)) value = value.expression; + if (!ts.isArrowFunction(value) && !ts.isFunctionExpression(value)) return false; + const signatures = lowerer.checker.getCallSignatures(lowerer.typeOf(value)); + return ( + signatures.length > 0 && + signatures.every((signature) => + primitiveOnly(lowerer.checker.getReturnTypeOfSignature(signature)), + ) + ); +} + +/** A type whose values are primitives (or void): it holds no record. */ +function primitiveOnly(type: ts.Type): boolean { + if (type.isUnionType()) return ts.constituentTypes(type).every(primitiveOnly); + return ( + (type.flags & + (ts.TypeFlags.StringLike | + ts.TypeFlags.NumberLike | + ts.TypeFlags.BooleanLike | + ts.TypeFlags.BigIntLike | + ts.TypeFlags.ESSymbolLike | + ts.TypeFlags.Null | + ts.TypeFlags.Undefined | + ts.TypeFlags.Void | + ts.TypeFlags.Never)) !== + 0 + ); +} + /** Chooses each in-scope record shape's enumeration table from its * witnesses. Runs once retained reachability has settled the historical * tables and derived metadata, before the order-snapshotting helper bodies @@ -294,16 +345,24 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio const shapesIn = (type: IrType, out: Set, seen: Set, produced = false): void => { const visit = (t: IrType, built: boolean): boolean => { const mark = built ? "" : "-"; + // A container JavaScript builds converts into fresh native storage, so + // its children are built too. One passed out stays shared: JavaScript + // writes into it (built children) and reads native values out of it, + // such as callbacks whose parameters it then builds (children passed + // out). Those children count in both directions. + const child = (type: IrType): boolean => { + visit(type, true); + if (!built) visit(type, false); + return true; + }; if (t.kind === "record") { if (seen.has(mark + "r:" + t.shapeId)) return true; seen.add(mark + "r:" + t.shapeId); if (built) out.add(t.shapeId); const shape = shapes.get(t.shapeId); if (shape) { - // A record passed out can be written through: its fields take - // values JavaScript builds. - for (const f of shape.fields) visit(f.type, true); - if (shape.indexValue) visit(shape.indexValue, true); + for (const f of shape.fields) child(f.type); + if (shape.indexValue) child(shape.indexValue); } return true; } @@ -315,17 +374,16 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio } if (t.kind === "object") { if (produced && t.className !== "%Error") return true; - if (seen.has("c:" + t.className)) return true; - seen.add("c:" + t.className); - for (const f of lowerer.classes.get(t.className)?.def.fields ?? []) visit(f.type, true); + if (seen.has(mark + "c:" + t.className)) return true; + seen.add(mark + "c:" + t.className); + for (const f of lowerer.classes.get(t.className)?.def.fields ?? []) child(f.type); return true; } if (produced && t.kind === "func") { for (const param of t.params) visit(param, !built); return visit(t.ret, built); } - // Containers passed out stay writable: their elements count as built. - return everyTypeChild(t, (child) => visit(child, true)); + return everyTypeChild(t, child); }; visit(type, true); }; @@ -338,6 +396,14 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio * declaration's order, the order their table holds. */ const declaredShapes = new Set(); const declaredSeen = new Set(); + /** The checked result of a replaceable method call while every site that + * can replace the method installs functions that return no record. */ + const unchangedMethodResult = (node: IrExpr): boolean => { + const name = notes.methodResultOf(node); + if (name === undefined) return false; + const sites = lowerer.methodAccessSites.get(name) ?? []; + return sites.every((site) => replacesWithoutRecords(lowerer, site)); + }; const producedBy = (type: IrType, opaque: boolean): void => { if (opaque) shapesIn(type, opaqueShapes, opaqueSeen); else shapesIn(type, runtimeShapes, runtimeSeen, true); @@ -449,10 +515,10 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio case "dynCheck": if (e.adopt === true) shapesIn(e.type, declaredShapes, declaredSeen); // Conversions that build no record: a boxed primitive, and a - // replaceable method's result, which is the value the method - // returns (boxed and checked back unchanged) or what a replacement - // installed on the instance returns. - else if (!orderPreservingClone(e) && !primitiveSource(e) && !notes.isMethodResult(e)) + // replaceable method's result while no replacement the program can + // install returns a record: then it is the method's own value, + // boxed and checked back unchanged. + else if (!orderPreservingClone(e) && !primitiveSource(e) && !unchangedMethodResult(e)) producedBy(e.type, false); return; case "dynDestrCheck": diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index 29afbb4fe..56c70c5bd 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -4147,7 +4147,7 @@ export function collectClassShapeInner( // An own JS callable shadows its prototype declaration. // Keep the method in the prototype and store the assigned // value in the instance property bag. - lowerer.prototypeMethodAccesses.set(name, assign); + lowerer.noteMethodAccess(name, assign); continue; } const sym = lowerer.checker.getSymbolAtLocation(assign); @@ -4229,7 +4229,7 @@ export function collectClassShapeInner( } if (!key && (fields.has(fieldName) || hasAccessor(fieldName))) continue; if (!key && (methods.has(fieldName) || lowerer.findMethodOn(base, fieldName))) { - lowerer.prototypeMethodAccesses.set(fieldName, lhs); + lowerer.noteMethodAccess(fieldName, lhs); continue; } // A key already declared (own or inherited) makes later diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index e36ffd09f..0922f8928 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -3021,6 +3021,23 @@ export class Lowerer { /** Named method slots observed before class collection need a callable * ABI and shared prototype descriptors instead of call-site specialization. */ readonly prototypeMethodAccesses = new Map(); + /** Every site the census found for a name in prototypeMethodAccesses (which + * keeps one): reads of the method and writes that may replace it. The + * key-order witness asks which replacements can return records. */ + readonly methodAccessSites = new Map(); + + /** Marks `name` as a method the program may read or replace at `site`. */ + noteMethodAccess(name: string, site: ts.Node): void { + this.prototypeMethodAccesses.set(name, site); + this.noteMethodSite(name, site); + } + + /** Records a site for `name` without changing its prototypeMethodAccesses entry. */ + noteMethodSite(name: string, site: ts.Node): void { + const sites = this.methodAccessSites.get(name); + if (sites === undefined) this.methodAccessSites.set(name, [site]); + else sites.push(site); + } /** Whether the program reads `Error.prototype`, the checked-dynamic * object an ES5 constructor function can inherit from, or has a class * whose heritage is evaluated at run time (`extends (Base ?? Error)`): diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7623-key-order-javascript-built-records/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7623-key-order-javascript-built-records/main.ts.json new file mode 100644 index 000000000..c3c2aa5b0 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7623-key-order-javascript-built-records/main.ts.json @@ -0,0 +1,7 @@ +{ + "order": [ + "/tests/corpus/7623-key-order-javascript-built-records/lib.js", + "/tests/corpus/7623-key-order-javascript-built-records/main.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7623-key-order-javascript-built-records/lib.js b/tests/corpus/7623-key-order-javascript-built-records/lib.js new file mode 100644 index 000000000..4ff312846 --- /dev/null +++ b/tests/corpus/7623-key-order-javascript-built-records/lib.js @@ -0,0 +1,11 @@ +// Records built in JavaScript in another key order than the TypeScript +// literals of the same type: a replacement method installed through a +// computed key, and a function stored on the global object, read through a +// typed view, that calls a callback inside its options record. +export function install(target, name) { + target[name] = () => ({ a: 1, b: 2 }); +} + +globalThis.__eachRow = function (options) { + options.visit({ x: 5, y: 6 }); +}; diff --git a/tests/corpus/7623-key-order-javascript-built-records/main.ts b/tests/corpus/7623-key-order-javascript-built-records/main.ts new file mode 100644 index 000000000..ae4876131 --- /dev/null +++ b/tests/corpus/7623-key-order-javascript-built-records/main.ts @@ -0,0 +1,50 @@ +// Copies and enumerations of records JavaScript builds follow the order +// JavaScript wrote, also when the TypeScript literals of the type write +// another order and spread their own type. Each case uses its own type. +import { install } from "./lib.js"; + +interface Pair { + a: number; + b: number; +} + +interface Row { + x: number; + y: number; +} + +interface Options { + visit: (row: Row) => void; +} + +class Store { + get(): Pair { + return { b: 2, a: 1 }; + } +} + +function showPair(label: string, value: Pair): void { + const copy = { ...value }; + console.log(label, Object.keys(value).join(), JSON.stringify(copy), Object.keys(copy).join()); +} + +function showRow(label: string, value: Row): void { + const copy = { ...value }; + console.log(label, Object.keys(value).join(), JSON.stringify(copy), Object.keys(copy).join()); +} + +// A replaceable method whose replacement returns a record JavaScript builds. +const store = new Store(); +showPair("method", store.get()); +showPair("self copy", { ...store.get(), b: 3 }); +install(store, "get"); +showPair("replaced", store.get()); + +// A JavaScript function, checked into a typed slot, calls a native callback +// it receives inside its options record with a record it builds. +const literal: Row = { y: 20, x: 10 }; +showRow("literal", literal); +showRow("self copy", { ...literal, y: 21 }); +const eachRow = (globalThis as typeof globalThis & { __eachRow: (options: Options) => void }) + .__eachRow; +eachRow({ visit: (row: Row) => showRow("checked function", row) }); diff --git a/tests/harness/corpus-costs/7623-key-order-javascript-built-records/main.ts.json b/tests/harness/corpus-costs/7623-key-order-javascript-built-records/main.ts.json new file mode 100644 index 000000000..4ffc07c5a --- /dev/null +++ b/tests/harness/corpus-costs/7623-key-order-javascript-built-records/main.ts.json @@ -0,0 +1 @@ +{ "plain": 6182, "san": 10202 } From b8a6a6a939dbfdd666bbfc48558083dd61b68d48 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 06:59:05 -0500 Subject: [PATCH 038/184] Expect an untyped dependency of an opted-in library package to compile The test refused unchecked-dependency only because its one-line index.js looked minified. A dependency another package's JavaScript imports needs no declarations of its own (npmStaticIneligibleReason's untypedDependency), so with the minified refusal gone it compiles; the test now builds it and checks the library's result. --- tests/harness/library-static-types.test.ts | 23 +++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/harness/library-static-types.test.ts b/tests/harness/library-static-types.test.ts index dfbd70bf9..36e2befac 100644 --- a/tests/harness/library-static-types.test.ts +++ b/tests/harness/library-static-types.test.ts @@ -81,7 +81,8 @@ test.each(["sample", "@scope/sample"])( expect(actual).toBe(reference); expect(actual).toBe("22\n"); } - // Opting in to one runtime package does not admit its untyped dependency. + // A dependency the package's JavaScript imports compiles without + // declarations of its own: no declaration checks that import. const dependency = join(dir, "node_modules/unchecked-dependency"); await mkdir(dependency, { recursive: true }); await writeFile( @@ -96,14 +97,18 @@ test.each(["sample", "@scope/sample"])( join(pkg, "index.js"), "import { adjust } from 'unchecked-dependency'; export function calculate(value) { return adjust(value) * 3 + 1; }\n", ); - const refused = await compileLibrary({ profilePath, outDir: join(dir, "refused") }); - expect(refused.ok).toBe(false); - if (!refused.ok) - expect( - refused.diagnostics.some( - (d) => d.code === "SC4020" && d.message.includes("unchecked-dependency"), - ), - ).toBe(true); + const withDependency = await compileLibrary({ profilePath, outDir: join(dir, "dependency") }); + if (!withDependency.ok) throw new Error(JSON.stringify(withDependency.diagnostics)); + if (process.env["SCRIPTC_PORTABLE_ONLY"] !== "1") { + const source = join(dir, "dependency-host.c"), + binary = join(dir, "dependency-host"); + await writeFile( + source, + '#include \nextern void st_init(void); extern double st_result(void);\nint main(void) { st_init(); printf("%.0f\\n", st_result()); }\n', + ); + execFileSync("clang", [source, withDependency.archivePath, "-lm", "-o", binary]); + expect(execFileSync(binary, { encoding: "utf8" })).toBe("22\n"); + } expect(await readFile(join(types, "index.d.ts"), "utf8")).toBe(declaration); } finally { await rm(dir, { recursive: true, force: true }); From da1dae0625fde0ef8168e07de7092d30213fc5db Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 07:20:08 -0500 Subject: [PATCH 039/184] Run two-counter loops and element copies over number arrays without per-element checks --- .../src/backend/llvm/dense-array-access.ts | 4 + .../backend/llvm/dense-array-loops.test.ts | 96 ++++++- .../src/backend/llvm/dense-array-loops.ts | 235 +++++++++++++++++- .../src/backend/llvm/emit-statements.ts | 17 +- packages/compiler/src/backend/llvm/emitter.ts | 8 +- .../src/backend/llvm/expr-primitives.ts | 4 +- .../src/backend/llvm/local-array-reads.ts | 28 ++- .../src/ir/array-length-stability.test.ts | 31 +++ .../compiler/src/ir/array-length-stability.ts | 32 ++- .../7264-array-dense-paired-loops.ts.json | 6 + tests/corpus/7264-array-dense-paired-loops.ts | 120 +++++++++ .../7264-array-dense-paired-loops.ts.json | 1 + 12 files changed, 561 insertions(+), 21 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7264-array-dense-paired-loops.ts.json create mode 100644 tests/corpus/7264-array-dense-paired-loops.ts create mode 100644 tests/harness/corpus-costs/7264-array-dense-paired-loops.ts.json diff --git a/packages/compiler/src/backend/llvm/dense-array-access.ts b/packages/compiler/src/backend/llvm/dense-array-access.ts index 95612e57c..df37d9c58 100644 --- a/packages/compiler/src/backend/llvm/dense-array-access.ts +++ b/packages/compiler/src/backend/llvm/dense-array-access.ts @@ -505,8 +505,12 @@ export function emitDenseArrayState( indexExpr: IrExpr | undefined, kind: "state" | "has", width: number, + /** The index is a dense array loop counter its entry test proved below + * the hole-free prefix (provenDenseIndex): the slot is a present value. */ + proven = false, ): string { const B = host.B; + if (proven) return kind === "state" ? f64Lit(SCR_ARR_VALUE) : "true"; const fn = kind === "state" ? "scr_arr_state" : "scr_arr_has"; const ty = kind === "state" ? "double" : "i1"; host.declare(`declare ${kind === "state" ? "double" : "zeroext i1"} @${fn}(ptr, double)`); diff --git a/packages/compiler/src/backend/llvm/dense-array-loops.test.ts b/packages/compiler/src/backend/llvm/dense-array-loops.test.ts index f7a4bc892..4a81a9bfe 100644 --- a/packages/compiler/src/backend/llvm/dense-array-loops.test.ts +++ b/packages/compiler/src/backend/llvm/dense-array-loops.test.ts @@ -1,9 +1,10 @@ import { expect, test } from "vitest"; import { BOOL, F64, arrayOf, type IrExpr, type IrLocal, type IrStmt } from "../../ir/ir.js"; -import type { IntegerCountedForLoop } from "../../ir/integer-loops.js"; +import type { IntegerCountedForLoop, IntegerPairedForLoop } from "../../ir/integer-loops.js"; import { countedDenseShape, matchDenseArrayLoop, + pairedDenseShape, type DenseLoopShape, } from "./dense-array-loops.js"; @@ -42,6 +43,8 @@ const locals = new Map([ ["a", { id: "a", name: "a", type: ARR, mutable: false }], ["b", { id: "b", name: "b", type: ARR, mutable: true }], ["c", { id: "c", name: "c", type: ARR, mutable: false }], + ["j", { id: "j", name: "j", type: F64, mutable: true }], + ["t", { id: "t", name: "t", type: F64, mutable: false }], ]); /** for (let i = 0; i < a.length; i++) body */ @@ -193,3 +196,94 @@ test("a loop to another invariant limit covers each array whose length it keeps" expect([...matchLimited(resizing, new Set(["b"]))!.arrays.keys()]).toEqual(["a"]); expect(matchLimited(resizing)).toBeNull(); }); + +test("a two-counter loop covers the arrays indexed at either counter", () => { + // for (let i = 1, j = k - 1; i < j; i++, j--) { t = a[i]; a[i] = a[j]; a[j] = t; } + const paired: IntegerPairedForLoop = { + low: { localId: "i", start: num(1), step: 1, startGuarded: false }, + high: { localId: "j", start: ref("k"), step: -1, startGuarded: true }, + inclusive: false, + guarded: true, + range: { min: -Number.MAX_SAFE_INTEGER, max: Number.MAX_SAFE_INTEGER }, + }; + const pairedShape = pairedDenseShape(paired); + const matchPaired = (body: IrStmt[]) => + matchDenseArrayLoop(loop(body), pairedShape, locals, new Set(), new Set(["a", "b"])); + const swap: IrStmt[] = [ + { kind: "varDecl", localId: "t", init: read("a"), loc }, + write("a", read("a", ref("j"))), + write("a", ref("t"), ref("j")), + ]; + const dense = matchPaired(swap); + expect([...dense!.arrays.keys()]).toEqual(["a"]); + expect(dense!.pair).toEqual({ high: "j", lowStart: num(1), highStart: ref("k") }); + expect(dense!.range).toEqual({ min: 0, max: 2147483646 }); + // Stores at any other index may extend the array. + expect(matchPaired([...swap, write("a", num(0), ref("k"))])).toBeNull(); +}); + +test("a present-undefined store is covered only where it stores a covered read's missing arm", () => { + // const v = b[i] (as number | undefined); + // if (v is undefined) c[i] = undefined; else c[i] = v is number ? v : throw; + const unionType = { kind: "union" as const, unionId: "u" }; + const v: IrExpr = { kind: "varRef", localId: "v", type: unionType, loc }; + const isTag = (tag: number): IrExpr => ({ + kind: "unionIsTag", + unionId: "u", + tag, + negated: false, + value: v, + type: BOOL, + loc, + }); + const reader = (array: string, index = ref("i")): IrExpr => ({ + kind: "call", + callee: "%arr.idxOr.f64", + args: [arr(array), index], + type: unionType, + loc, + }); + const undefinedStore: IrStmt = { kind: "arraySetUndefined", arr: arr("c"), index: ref("i"), loc }; + const guarded = (array: string): IrStmt[] => [ + { kind: "varDecl", localId: "v", init: reader(array), loc }, + { + kind: "if", + cond: isTag(1), + then: [undefinedStore], + else_: [ + write("c", { + kind: "ternary", + cond: isTag(0), + then: { kind: "unionNarrow", unionId: "u", tag: 0, value: v, type: F64, loc }, + else_: { + kind: "libCall", + fn: "error.nodeThrow", + args: [num(1), { kind: "strLit", value: "", type: { kind: "string" }, loc }], + type: F64, + loc, + }, + type: F64, + loc, + }), + ], + loc, + }, + ]; + const withV = new Map([ + ...locals, + ["v", { id: "v", name: "v", type: unionType, mutable: false }], + ]); + const matchStores = (body: IrStmt[], initialized: string[]) => + matchDenseArrayLoop(loop(body), shape, withV, new Set(), new Set(initialized)); + const dense = matchStores(guarded("b"), ["a", "b", "c"]); + expect([...dense!.arrays.keys()].sort()).toEqual(["b", "c"]); + // A read of an array the entry test does not cover may miss in the fast + // copy, and its undefined store would run there. + expect(matchStores(guarded("b"), ["a", "c"])).toBeNull(); + // A present undefined stored on its own leaves its array out. + const alone = matchStores( + [{ kind: "exprStmt", expr: read("b"), loc }, undefinedStore], + ["a", "b", "c"], + ); + expect([...alone!.arrays.keys()]).toEqual(["b"]); +}); diff --git a/packages/compiler/src/backend/llvm/dense-array-loops.ts b/packages/compiler/src/backend/llvm/dense-array-loops.ts index 184ff1353..135d5eaf2 100644 --- a/packages/compiler/src/backend/llvm/dense-array-loops.ts +++ b/packages/compiler/src/backend/llvm/dense-array-loops.ts @@ -20,16 +20,24 @@ * A counted loop with another invariant limit, such as `j < n` after * `const n = a.length` or `j < 5`, is covered the same way: the test bounds * the loop's exclusive limit by 2^31 - 1 and by each indexed array's - * `filled`, and the loop must provably never change any of their lengths. */ + * `filled`, and the loop must provably never change any of their lengths. + * + * A two-counter loop closing in on itself, such as + * `for (let i = 1, j = k - 1; i < j; i++, j--)`, indexes only between its + * low counter's start and its high counter's: the low counter only rises + * and stays below the high one, which only falls. Its test bounds the low + * start below by 0 and the high start by 2^31 - 2 and below each indexed + * array's `filled`, both starts being exact integers. */ import type { IrExpr, IrLocal, IrStmt } from "../../ir/ir.js"; import type { IntegerRange } from "../../ir/integer-ranges.js"; import { preservesArrayLength } from "../../ir/array-length-stability.js"; import { everyExprChild, everyStmtChild } from "../../ir/traverse.js"; -import type { IntegerCountedForLoop } from "../../ir/integer-loops.js"; +import type { IntegerCountedForLoop, IntegerPairedForLoop } from "../../ir/integer-loops.js"; import type { LlvmEmitterContext } from "./expr-context.js"; import { loadArrCount, SCR_ARR_FIELD } from "./common.js"; import { emitCountedLoopLimit } from "./counted-loops.js"; +import { f64Lit } from "./common.js"; export const INT32_MAX = 2147483647; @@ -47,6 +55,17 @@ export interface DenseArrayLoop { arrays: ReadonlyMap; /** The counter's range on every pass of the fast copy. */ range: IntegerRange; + /** A two-counter loop's high counter and both starts, when the counter + * is its low one (`length` and `limit` are then null). */ + pair: DensePair | null; +} + +/** The high counter of a two-counter dense loop, and the starts its entry + * test bounds. */ +export interface DensePair { + high: string; + lowStart: IrExpr; + highStart: IrExpr; } /** The fast copy duplicates the loop; keep the duplicated code small. */ @@ -62,6 +81,8 @@ export interface DenseLoopShape { /** The counter's proven range on every pass of the ordinary loop; the * fast copy keeps its minimum, which must be at least 0. */ range: IntegerRange; + /** A two-counter loop's high counter and starts. */ + pair?: DensePair; } /** `length.length - c` (c an integer literal, 0 when absent). */ @@ -107,6 +128,124 @@ export function countedDenseShape( return { counter: counted.localId, length: start.receiver, range: counted.range }; } +/** The shape of a two-counter loop whose low counter rises and high + * counter falls: both stay between the two starts, which the entry test + * bounds, so the fast copy's counters lie in [0, 2^31 - 2]. */ +export function pairedDenseShape(loop: IntegerPairedForLoop): DenseLoopShape { + return { + counter: loop.low.localId, + length: null, + range: { min: 0, max: INT32_MAX - 1 }, + pair: { high: loop.high.localId, lowStart: loop.low.start, highStart: loop.high.start }, + }; +} + +interface ElementRead { + arr: IrExpr; + index: IrExpr; +} + +/** The array and index of a call of an element-reader helper + * (`%arr.idxOr.*`, ir/int32-arrays.ts isElementReader): `a[i]` as + * `number | undefined`. */ +function elementReaderCall(e: IrExpr): ElementRead | null { + if (e.kind !== "call" || !e.callee.startsWith("%arr.idxOr.") || e.args.length !== 2) return null; + return { arr: e.args[0]!, index: e.args[1]! }; +} + +/** The array and index of an element read as `T | undefined`: an + * element-reader call, or its body inlined for an int32 array + * (`arrayState(a, i) === 1 ? wrap(a[i]) : wrap(undefined)`). */ +function elementRead(e: IrExpr): ElementRead | null { + const call = elementReaderCall(e); + if (call) return call; + if (e.kind !== "ternary") return null; + const cond = e.cond; + if (cond.kind !== "bin" || cond.op !== "===") return null; + const state = cond.left, + present = cond.right, + wrapped = e.then, + missing = e.else_; + if (state.kind !== "arrayState" || present.kind !== "numLit" || present.value !== 1) return null; + if (wrapped.kind !== "unionWrap" || missing.kind !== "unionWrap") return null; + const value = wrapped.value; + if (value.kind !== "arrayGet" || missing.value.kind !== "unitLit") return null; + const same = (x: IrExpr, y: IrExpr): boolean => + x.kind === "varRef" && y.kind === "varRef" && x.localId === y.localId; + return same(state.arr, value.arr) && same(state.index, value.index) + ? { arr: state.arr, index: state.index } + : null; +} + +/** The present-undefined stores of `a[i] = b[j]!` as lowering spells it, + * with the element read each one stores the missing arm of: + * + * const v = %arr.idxOr(b, j); + * if (v is undefined) a[i] = undefined; else a[i] = v is number ? v : throw; + * + * The read's local is declared once in the same list as the `if`, and only + * read after it. */ +function missingElementStores(body: IrStmt[]): Map { + const result = new Map(); + const writes = new Map(); + const count = (id: string): void => void writes.set(id, (writes.get(id) ?? 0) + 1); + const expr = (e: IrExpr): boolean => { + if (e.kind === "assignExpr" || e.kind === "incDec") count(e.localId); + return everyExprChild(e, expr, stmt); + }; + const stmt = (s: IrStmt): boolean => { + if (s.kind === "varDecl" || s.kind === "assign") count(s.localId); + return everyStmtChild(s, expr, stmt); + }; + body.every(stmt); + const list = (stmts: IrStmt[]): void => { + const reads = new Map(); + for (const s of stmts) { + if (s.kind === "varDecl" && s.init && writes.get(s.localId) === 1) { + const read = elementRead(s.init); + if (read) reads.set(s.localId, read); + } + if (s.kind === "if") { + const store = missingArmStore(s, reads); + if (store) result.set(store.stmt, store.read); + } + // Nested loops keep a loop out of the fast copy (matchDenseArrayLoop), + // and other statement lists leave their stores unmatched. + if (s.kind === "block") list(s.body); + if (s.kind === "if") { + list(s.then); + if (s.else_) list(s.else_); + } + } + }; + list(body); + return result; +} + +/** The present-undefined store of `if (v is undefined) a[i] = undefined; + * else a[i] = v is number ? v : throw;`, with the read `v` holds. */ +function missingArmStore( + s: IrStmt & { kind: "if" }, + reads: ReadonlyMap, +): { stmt: IrStmt; read: ElementRead } | null { + const cond = s.cond; + if (cond.kind !== "unionIsTag" || cond.negated || cond.value.kind !== "varRef") return null; + const local = cond.value.localId; + const read = reads.get(local); + if (!read || s.then.length !== 1 || s.else_?.length !== 1) return null; + const store = s.then[0]!, + other = s.else_[0]!; + if (store.kind !== "arraySetUndefined" || other.kind !== "arraySet") return null; + const value = other.value; + if (value.kind !== "ternary") return null; + const test = value.cond, + narrowed = value.then; + if (test.kind !== "unionIsTag" || test.negated || test.tag === cond.tag) return null; + if (test.value.kind !== "varRef" || test.value.localId !== local) return null; + if (narrowed.kind !== "unionNarrow" || narrowed.tag !== test.tag) return null; + return narrowed.type.kind === "f64" ? { stmt: store, read } : null; +} + /** Match the fast copy of a loop of the given shape. `initialized` holds the * bindings definitely assigned when the loop starts; `unaliased` the local * arrays no other binding holds. */ @@ -118,6 +257,8 @@ export function matchDenseArrayLoop( initialized: ReadonlySet, ): DenseArrayLoop | null { const { counter, length } = shape; + const high = shape.pair?.high ?? null; + const isCounter = (id: string): boolean => id === counter || id === high; if (shape.range.min < 0 || !loop.cond) return null; let nodes = 0, nested = false; @@ -126,13 +267,18 @@ export function matchDenseArrayLoop( // element write. const disturbed = new Set(); const access = (arr: IrExpr, index: IrExpr): void => { - if (arr.kind === "varRef" && index.kind === "varRef" && index.localId === counter) + if (arr.kind === "varRef" && index.kind === "varRef" && isCounter(index.localId)) candidates.set(arr.localId, arr); }; + const missing = missingElementStores(loop.body); + // The element reads behind the guarded present-undefined stores. + const guards: ElementRead[] = []; const expr = (e: IrExpr): boolean => { if (++nodes > MAX_LOOP_NODES) return false; if (e.kind === "assignExpr" || e.kind === "incDec") disturbed.add(e.localId); if (e.kind === "arrayGet") access(e.arr, e.index); + const reader = elementReaderCall(e); + if (reader) access(reader.arr, reader.index); if (e.kind === "arrIntrinsic") { if (e.method === "getNumber" && e.args[0]) access(e.receiver, e.args[0]); else if (e.method !== "length" && e.receiver.kind === "varRef") @@ -147,9 +293,21 @@ export function matchDenseArrayLoop( if (s.kind === "assign" || s.kind === "varDecl") disturbed.add(s.localId); if (s.kind === "arraySet") { access(s.arr, s.index); - if (s.arr.kind === "varRef" && (s.index.kind !== "varRef" || s.index.localId !== counter)) + if (s.arr.kind === "varRef" && (s.index.kind !== "varRef" || !isCounter(s.index.localId))) disturbed.add(s.arr.localId); } + if (s.kind === "arraySetUndefined") { + // A present undefined at a counter only where it stores a missing + // element read (`a[i] = b[j]!`): the fast copy reads a present value + // there, so the store never runs in it. + const read = missing.get(s); + if (read === undefined || s.index.kind !== "varRef" || !isCounter(s.index.localId)) { + if (s.arr.kind === "varRef") disturbed.add(s.arr.localId); + } else { + access(s.arr, s.index); + guards.push(read); + } + } return everyStmtChild(s, expr, stmt); }; if (!loop.body.every(stmt) || !expr(loop.cond) || nested) return null; @@ -180,18 +338,33 @@ export function matchDenseArrayLoop( // once: the loop must not change any of their lengths. if ( length === null && - !preservesArrayLength(nodesOfLoop, { receiver: id, inRange: counter }, locals, unaliased) + !preservesArrayLength( + nodesOfLoop, + { receiver: id, inRange: counter, inRangeHigh: high }, + locals, + unaliased, + ) ) continue; arrays.set(id, ref); } if (arrays.size === 0) return null; + // Each guarded store's read must be one the fast copy proves present. + for (const { arr, index } of guards) + if ( + arr.kind !== "varRef" || + !arrays.has(arr.localId) || + index.kind !== "varRef" || + !isCounter(index.localId) + ) + return null; return { counter, length, limit: length === null ? (shape.limit ?? null) : null, arrays, range: { min: shape.range.min, max: Math.min(shape.range.max, INT32_MAX - 1) }, + pair: shape.pair ?? null, }; } @@ -207,6 +380,7 @@ export function emitDenseLoopGuard(host: LlvmEmitterContext, loop: DenseArrayLoo host.markMemoryPointer(ptr, "array:header"); return ptr; }; + if (loop.pair !== null) return emitPairedDenseGuard(host, loop, loop.pair, header); if (loop.length === null) { // A limit at or below the start runs no pass, and passes the test. const limit = emitCountedLoopLimit(host, loop.limit!); @@ -254,3 +428,54 @@ export function emitDenseLoopGuard(host: LlvmEmitterContext, loop: DenseArrayLoo } return valid; } + +/** A two-counter loop's entry test: both starts are exact integers, the + * low one at least 0 and the high one at most 2^31 - 2 and below every + * indexed array's hole-free prefix bound. The starts are stable values + * (matchIntegerPairedForLoop), so evaluating them here again is unobservable. */ +function emitPairedDenseGuard( + host: LlvmEmitterContext, + loop: DenseArrayLoop, + pair: DensePair, + header: (array: string, field: number) => string, +): string { + const B = host.B; + const integer = (expr: IrExpr, min: number, max: number): { value: string; ok: string } => { + const start = host.emitExpr(expr); + const low = B.tmp(), + high = B.tmp(), + raw = B.tmp(), + value = B.tmp(), + back = B.tmp(), + exact = B.tmp(), + inRange = B.tmp(), + ok = B.tmp(); + B.line(`${low} = fcmp oge double ${start.name}, ${f64Lit(min)}`); + B.line(`${high} = fcmp ole double ${start.name}, ${f64Lit(max)}`); + B.line(`${inRange} = and i1 ${low}, ${high}`); + B.line(`${raw} = fptosi double ${start.name} to i64`); + B.line(`${value} = freeze i64 ${raw}`); + B.line(`${back} = sitofp i64 ${value} to double`); + B.line(`${exact} = fcmp oeq double ${back}, ${start.name}`); + B.line(`${ok} = and i1 ${inRange}, ${exact}`); + return { value, ok }; + }; + const low = integer(pair.lowStart, 0, INT32_MAX - 1); + const high = integer(pair.highStart, -1, INT32_MAX - 1); + let valid = B.tmp(); + B.line(`${valid} = and i1 ${low.ok}, ${high.ok}`); + for (const ref of loop.arrays.values()) { + const array = host.emitStableReceiver(ref, []); + const filledPtr = header(array.name, SCR_ARR_FIELD.filled), + filled = B.tmp(), + wide = B.tmp(), + covered = B.tmp(), + both = B.tmp(); + B.line(`${filled} = load i32, ptr ${filledPtr}${host.fieldAliasAttachment(filledPtr)}`); + B.line(`${wide} = zext i32 ${filled} to i64`); + B.line(`${covered} = icmp slt i64 ${high.value}, ${wide}`); + B.line(`${both} = and i1 ${valid}, ${covered}`); + valid = both; + } + return valid; +} diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index c2846a9d4..48f7a1d5f 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -52,7 +52,12 @@ import { matchIntegerPairedForLoop, } from "../../ir/integer-loops.js"; import { emitByteWindowGuard, matchByteWindow } from "./byte-windows.js"; -import { countedDenseShape, emitDenseLoopGuard, matchDenseArrayLoop } from "./dense-array-loops.js"; +import { + countedDenseShape, + emitDenseLoopGuard, + matchDenseArrayLoop, + pairedDenseShape, +} from "./dense-array-loops.js"; import { emitCountedLoopLimit, emitCountedLoopSafety, @@ -966,7 +971,9 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { } : countedLoop && !countedLoop.guarded && countedLoop.followers.length === 0 ? countedDenseShape(s, countedLoop) - : null; + : pairedLoop + ? pairedDenseShape(pairedLoop) + : null; const dense = denseShape && !window ? matchDenseArrayLoop( @@ -1144,11 +1151,15 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { B.line(`${start} = fptosi double ${value.name} to i64`); B.line(`store i64 ${start}, ptr ${slot}`); pairedSlots.push(slot); + const dense = LOOPS.of(emitter).denseLoop; emitter.integerLoopBindings.set(counter.localId, { slot, type: "i64", signed: true, - range: pairedLoop.range, + range: + dense?.pair && dense.counter === pairedLoop.low.localId + ? dense.range + : pairedLoop.range, }); } } else if (s.init) { diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 050c3fb90..3dc21f356 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -974,8 +974,8 @@ export class LlEmitter { } /** Whether an element access at `index` of `arr` lies in the hole-free - * prefix: the counter of a dense array loop's fast copy indexing one of - * the arrays its entry test covered. */ + * prefix: a counter of a dense array loop's fast copy indexing one of the + * arrays its entry test covered. */ provenDenseIndex(arr: IrExpr, index: IrExpr): boolean { const loop = LOOPS.of(this).denseLoop; return ( @@ -983,8 +983,8 @@ export class LlEmitter { arr.kind === "varRef" && loop.arrays.has(arr.localId) && index.kind === "varRef" && - index.localId === loop.counter && - this.integerLoopBindings.has(loop.counter) + (index.localId === loop.counter || index.localId === loop.pair?.high) && + this.integerLoopBindings.has(index.localId) ); } diff --git a/packages/compiler/src/backend/llvm/expr-primitives.ts b/packages/compiler/src/backend/llvm/expr-primitives.ts index 1eec6a965..2c6ce35db 100644 --- a/packages/compiler/src/backend/llvm/expr-primitives.ts +++ b/packages/compiler/src/backend/llvm/expr-primitives.ts @@ -1076,9 +1076,10 @@ export function emitContainerExpr( const idx = host.emitExpr(e.index); if (e.arr.type.kind !== "array") throw new InternalCompilerError("llvm emitter bug: arrayState on non-array"); + const proven = host.provenDenseIndex(e.arr, e.index); const t = host.checkedInt32Array(e.arr) ? host.emitByElementKind(arr.name, "double", (kind) => - emitDenseArrayState(host, arr.name, idx, e.index, "state", slotWidth(kind)), + emitDenseArrayState(host, arr.name, idx, e.index, "state", slotWidth(kind), proven), )! : emitDenseArrayState( host, @@ -1087,6 +1088,7 @@ export function emitContainerExpr( e.index, "state", host.int32Array(e.arr) ? slotWidth("i32") : denseSlotWidth(host, e.arr.type.elem), + proven, ); return { name: t, type: e.type }; } diff --git a/packages/compiler/src/backend/llvm/local-array-reads.ts b/packages/compiler/src/backend/llvm/local-array-reads.ts index 1c540295a..80ed2c440 100644 --- a/packages/compiler/src/backend/llvm/local-array-reads.ts +++ b/packages/compiler/src/backend/llvm/local-array-reads.ts @@ -797,7 +797,33 @@ export function emitLocalArrayRead( ); return null; }); - else if (inline) + else if ( + inline && + loopIndex !== null && + read.element.kind === "f64" && + host.provenDenseIndex(read.array, read.index) + ) { + // In a dense array loop's fast copy, the entry test proved the counter + // below the array's hole-free prefix: the slot holds a present value, + // and the read has no missing arm. + const dataPtr = B.tmp(), + data = B.tmp(), + valuePtr = B.tmp(), + value = B.tmp(), + width = int32 ? "i32" : "i64"; + B.line( + `${dataPtr} = getelementptr inbounds %ScrArr, ptr ${array.name}, i32 0, i32 ${SCR_ARR_FIELD.data}`, + ); + host.markMemoryPointer(dataPtr, "array:header"); + B.line(`${data} = load ptr, ptr ${dataPtr}${host.fieldAliasAttachment(dataPtr)}`); + B.line( + `${valuePtr} = getelementptr inbounds ${width}, ptr ${data}, ${host.sizeType} ${loopIndex}`, + ); + host.markMemoryPointer(valuePtr, "array:elements"); + B.line(`${value} = load ${width}, ptr ${valuePtr}${host.fieldAliasAttachment(valuePtr)}`); + storeValue(value); + B.br(join); + } else if (inline) emitDenseReferenceArrayRead( host, array, diff --git a/packages/compiler/src/ir/array-length-stability.test.ts b/packages/compiler/src/ir/array-length-stability.test.ts index f9b3cc9c2..97270e059 100644 --- a/packages/compiler/src/ir/array-length-stability.test.ts +++ b/packages/compiler/src/ir/array-length-stability.test.ts @@ -193,3 +193,34 @@ test("number and boolean writes to class fields run no user code", () => { // A reference stored into a field releases the old one. expect(preserves([fieldSet(arr("b"))], "i", new Set(), scope)).toBe(false); }); + +test("optional element reads, their assertion errors and present-undefined stores at an in-range index keep the length", () => { + const optional = (receiver: string, index: IrExpr): IrExpr => ({ + kind: "call", + callee: "%arr.idxOr.f64", + args: [arr(receiver), index], + type: F64, + loc, + }); + const undefinedAt = (receiver: string, index: IrExpr): IrStmt => ({ + kind: "arraySetUndefined", + arr: arr(receiver), + index, + loc, + }); + const thrown: IrExpr = { kind: "libCall", fn: "error.nodeThrow", args: [num(1)], type: F64, loc }; + expect(preserves([stmt(optional("a", ref("i"))), stmt(thrown)])).toBe(true); + expect(preserves([undefinedAt("a", ref("i"))])).toBe(true); + // Past the counter, a present undefined extends the array. + expect(preserves([undefinedAt("a", plus(ref("i"), num(1)))])).toBe(false); + // The high counter of a two-counter loop is in range as well. + const paired = (body: IrStmt[]): boolean => + preservesArrayLength( + [cond, ...body], + { receiver: "a", inRange: "i", inRangeHigh: "j" }, + locals, + new Set(), + ); + expect(paired([undefinedAt("a", ref("j")), set("a", ref("j"), num(0))])).toBe(true); + expect(preserves([set("a", ref("j"), num(0))])).toBe(false); +}); diff --git a/packages/compiler/src/ir/array-length-stability.ts b/packages/compiler/src/ir/array-length-stability.ts index ed57972b6..7715ea1e1 100644 --- a/packages/compiler/src/ir/array-length-stability.ts +++ b/packages/compiler/src/ir/array-length-stability.ts @@ -162,6 +162,10 @@ export interface ArrayLengthLoop { * long as the length itself has not changed). An element write at this * index replaces an existing element or sets a non-index property. */ inRange: string | null; + /** A second such binding: the high counter of a two-counter loop + * (`for (let i = lo, j = hi; i < j; i++, j--)`), whose values stay + * between the low counter's start and its own. */ + inRangeHigh?: string | null; } /** Whether running `nodes` (a loop's condition, body and update) can never @@ -191,7 +195,10 @@ export function preservesArrayLength( // escaping. const receiver = (e: IrExpr): boolean => (e.kind === "varRef" && e.localId === loop.receiver) || expr(e); - const written = (id: string): boolean => id === loop.receiver || id === loop.inRange; + const inRange = (id: string): boolean => + (loop.inRange !== null && id === loop.inRange) || + (loop.inRangeHigh != null && id === loop.inRangeHigh); + const written = (id: string): boolean => id === loop.receiver || inRange(id); const expr = (e: IrExpr): boolean => { switch (e.kind) { case "varRef": @@ -224,12 +231,21 @@ export function preservesArrayLength( case "call": // A checked union extraction (`x!`, coercions/unions.ts): the // asserted arm's payload or a TypeError, and no user code. - return e.callee.startsWith("%union.narrow.") && e.args.every(expr); + if (e.callee.startsWith("%union.narrow.")) return e.args.every(expr); + // An element read as `T | undefined` (int32-arrays.ts isElementReader). + return ( + e.callee.startsWith("%arr.idxOr.") && + e.args.length === 2 && + receiver(e.args[0]!) && + expr(e.args[1]!) + ); case "mapIntrinsic": return READING_MAP_METHODS.has(e.method) && expr(e.receiver) && e.args.every(expr); case "setIntrinsic": return READING_SET_METHODS.has(e.method) && expr(e.receiver) && e.args.every(expr); case "libCall": + // The TypeError of an assertion past a union arm leaves the loop. + if (e.fn === "error.nodeThrow") return e.args.every(expr); return ( isBorrowSafeMath(e.fn) && e.args.every((arg) => arg.type.kind === "f64") && @@ -254,14 +270,18 @@ export function preservesArrayLength( break; case "arraySet": return ( - (separate(s.arr) || - (s.index.kind === "varRef" && - loop.inRange !== null && - s.index.localId === loop.inRange)) && + (separate(s.arr) || (s.index.kind === "varRef" && inRange(s.index.localId))) && receiver(s.arr) && expr(s.index) && expr(s.value) ); + case "arraySetUndefined": + // A present undefined replacing an element below the length. + return ( + (separate(s.arr) || (s.index.kind === "varRef" && inRange(s.index.localId))) && + receiver(s.arr) && + expr(s.index) + ); case "return": return s.value === null || expr(s.value); case "fieldSet": diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7264-array-dense-paired-loops.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7264-array-dense-paired-loops.ts.json new file mode 100644 index 000000000..8396d237c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7264-array-dense-paired-loops.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7264-array-dense-paired-loops.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7264-array-dense-paired-loops.ts b/tests/corpus/7264-array-dense-paired-loops.ts new file mode 100644 index 000000000..59cedf60b --- /dev/null +++ b/tests/corpus/7264-array-dense-paired-loops.ts @@ -0,0 +1,120 @@ +// Two-counter loops that index number arrays at both counters +// (`for (let i = lo, j = hi; i < j; i++, j--)`), and element copies through +// `a[i]!` reads, both inside and outside the ranges a single entry test can +// cover: filled prefixes, holes, present undefined, reads past the end, +// fractional and negative starts, int32 arrays and long checked arrays. + +function reversePrefix(a: number[], end: number): void { + for (let i = 0, j = end; i < j; i++, j--) { + const t = a[i]!; + a[i] = a[j]!; + a[j] = t; + } +} + +function show(label: string, a: number[]): void { + console.log(label, a.length, JSON.stringify(a), a); +} + +// The whole prefix is present: the entry test passes. +const plain: number[] = []; +for (let k = 0; k < 8; k++) plain[k] = k * 3; +reversePrefix(plain, 5); +show("plain", plain); +reversePrefix(plain, 7); +show("whole", plain); +reversePrefix(plain, 0); +show("empty range", plain); + +// The high start is past the end: the reads there miss, the stores extend. +const short = [1, 2, 3]; +reversePrefix(short, 5); +show("past the end", short); + +// A hole inside the range. +const holey: number[] = [10, 20]; +holey[3] = 40; +holey[4] = 50; +reversePrefix(holey, 4); +show("hole", holey); + +// A present undefined inside the range, copied by a mirror loop. +const loose: number[] = [7, 8, 9, 10]; +(loose as (number | undefined)[])[2] = undefined; +for (let i = 0, j = 3; i < j; i++, j--) loose[i] = loose[j]!; +show("undefined", loose); + +// Fractional and negative starts index non-element properties. +const fractional = [1, 2, 3, 4]; +reversePrefix(fractional, 2.5); +show("fractional", fractional); +const negative = [5, 6, 7]; +for (let i = -1, j = 2; i < j; i++, j--) negative[i] = negative[j]!; +show("negative", negative); + +// A local int32 array (every stored element is a small integer). +function int32Flips(start: number): void { + const counts = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + let flips = 0; + for (let first = counts[start]!; first !== 0; first = counts[0]!) { + for (let i = 0, j = first; i < j; i++, j--) { + const t = counts[i]!; + counts[i] = counts[j]!; + counts[j] = t; + } + flips++; + if (flips > 20) break; + } + console.log("int32", flips, counts.join(",")); +} +int32Flips(3); +int32Flips(9); + +// A local long array with checked int32 stores, widened by a fraction. +function longChecked(n: number, fraction: number): void { + const long = new Array(n).fill(0); + for (let k = 0; k < long.length; k++) long[k] = k; + for (let i = 0, j = n - 1; i < j; i++, j--) { + const t = long[i]!; + long[i] = long[j]!; + long[j] = t; + } + console.log("long", long[0], long[1], long[n - 2], long[n - 1]); + long[10] = fraction; + for (let i = 0, j = 20; i < j; i++, j--) { + const t = long[i]!; + long[i] = long[j]!; + long[j] = t; + } + console.log("widened", long.slice(0, 12).join(","), long[20]); +} +longChecked(5000, 0.5); + +// Element copies through `a[i]!`: the source is shorter than the range. +function copy(to: number[], from: number[], n: number): void { + for (let i = 0; i < n; i++) to[i] = from[i]!; +} +const into = [0, 0, 0, 0, 0, 0]; +copy(into, [1, 2, 3, 4, 5, 6], 6); +show("copy", into); +copy(into, [9, 8], 4); +show("short source", into); +const gappy: number[] = [1]; +gappy[2] = 3; +gappy[3] = 4; +copy(into, gappy, 4); +show("gappy source", into); + +// Two names for one array. +function aliased(n: number): void { + const same: number[] = []; + for (let k = 0; k < n; k++) same[k] = k + 1; + const alias = same; + for (let i = 0, j = n - 1; i < j; i++, j--) { + const t = alias[i]!; + same[i] = alias[j]!; + same[j] = t; + } + show("alias", same); +} +aliased(5); diff --git a/tests/harness/corpus-costs/7264-array-dense-paired-loops.ts.json b/tests/harness/corpus-costs/7264-array-dense-paired-loops.ts.json new file mode 100644 index 000000000..f591a71f3 --- /dev/null +++ b/tests/harness/corpus-costs/7264-array-dense-paired-loops.ts.json @@ -0,0 +1 @@ +{ "plain": 2900, "san": 4200 } From 5c015be6be05670a87b57d3a63b888f6f24bbae0 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 07:47:26 -0500 Subject: [PATCH 040/184] Try wider record arms first when a dyn value converts to a union Records are width-tolerant, and a union's arms were tried in canonical order, which sorts record arms by shape id. When a `{}` arm, or an arm whose fields are all optional, came first, it took every object, so a value converted to the union lost the fields only a wider arm has: const pairs = [["a", { publicKeyEncoding: 5 }], ["b", {}]]; for (const [, v] of pairs) console.log(Object.keys(v)); printed [] twice, where Node prints [ 'publicKeyEncoding' ] and []. A checked cast to `{ p: number; q: string } | { p?: number }` dropped `q` the same way. The checked conversion now tries record arms with more required fields first, then those with more fields, and keeps the canonical order otherwise. The union's tags are unchanged. --- packages/compiler/src/backend/llvm/dyn.ts | 39 +++++++++++++++++-- packages/compiler/src/ir/ir.ts | 6 ++- .../7447-union-record-arm-width.mjs.json | 6 +++ .../7448-union-record-arm-width.ts.json | 6 +++ tests/corpus/7447-union-record-arm-width.mjs | 37 ++++++++++++++++++ tests/corpus/7448-union-record-arm-width.ts | 21 ++++++++++ .../7447-union-record-arm-width.mjs.json | 1 + .../7448-union-record-arm-width.ts.json | 1 + 8 files changed, 111 insertions(+), 6 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7447-union-record-arm-width.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7448-union-record-arm-width.ts.json create mode 100644 tests/corpus/7447-union-record-arm-width.mjs create mode 100644 tests/corpus/7448-union-record-arm-width.ts create mode 100644 tests/harness/corpus-costs/7447-union-record-arm-width.mjs.json create mode 100644 tests/harness/corpus-costs/7448-union-record-arm-width.ts.json diff --git a/packages/compiler/src/backend/llvm/dyn.ts b/packages/compiler/src/backend/llvm/dyn.ts index 94581cfa0..86626d304 100644 --- a/packages/compiler/src/backend/llvm/dyn.ts +++ b/packages/compiler/src/backend/llvm/dyn.ts @@ -370,6 +370,36 @@ export class LlDyn { B.line(`call void @scr_dyn_build_add(ptr %d, ptr @${name}, ptr ${value})`); } + /** The order a dyn value tries a union's arms in: the canonical order, + * except that among the record arms, those with more required fields go + * first, then those with more fields. Records are width-tolerant, so an + * arm with fewer fields (`{}`, or one whose fields are all optional, + * matches every object) would otherwise take an object that a wider arm + * also matches and drop the fields only the wider arm has. */ + private unionArmOrder(def: IrUnionDef): number[] { + const width = (i: number): [number, number] | null => { + const arm = def.arms[i]!; + if (arm.kind !== "record") return null; + const shape = this.host.recordsById.get(arm.shapeId); + if (!shape || shape.tuple) return null; + const required = shape.fields.filter( + (field) => + field.type.kind !== "union" || + !this.host.unionsById.get(field.type.unionId)?.arms.some((x) => x.kind === "undefinedT"), + ).length; + return [required, shape.fields.length]; + }; + const order = def.arms.map((_, i) => i); + const records = order.filter((i) => width(i) !== null); + const widest = [...records].sort((x, y) => { + const [rx, fx] = width(x)!; + const [ry, fy] = width(y)!; + return ry - rx || fy - fx || x - y; + }); + records.forEach((slot, k) => (order[slot] = widest[k]!)); + return order; + } + /** Branch to a variant only after its literal discriminator and complete * structure match. Use the same path for predicates and builders. */ private unionArmMatch( @@ -2368,9 +2398,10 @@ export class LlDyn { throw new InternalCompilerError( `llvm emitter bug: dynCheck of unknown union ${t.unionId}`, ); - // Arms in CANONICAL order, first FULL match wins. The matched - // arm's builder can no longer fail. - def.arms.forEach((arm, i) => { + // Arms in unionArmOrder, first FULL match wins. The matched arm's + // builder can no longer fail. + for (const i of this.unionArmOrder(def)) { + const arm = def.arms[i]!; const lHit = B.newLabel("dcu.h"); const lNext = B.newLabel("dcu.n"); this.unionArmMatch(B, def, i, lHit, lNext, preserveRefs); @@ -2414,7 +2445,7 @@ export class LlDyn { B.terminate(`ret ptr ${u}`); } B.startBlock(lNext); - }); + } if (preserveRefs) { B.line( `call void @scr_dyn_check_fail(ptr %path, ptr ${host.cstr("reference with its original layout")}, ptr %d)`, diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 6d0760b81..878088d65 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -3898,8 +3898,10 @@ export type IrExpr = * not shape equality; missing or wrong-typed fields throw, EXCEPT that a * missing key for an undefined-armed union field — an optional field — * builds the interned undefined arm instead); arrays check every element; - * unions try arms in canonical order and the first FULL match wins (no - * match → throw; an undefined arm matches no dyn value); JSON null matches + * unions try arms in canonical order, except that record arms with more + * required fields and then more fields go first, and the first FULL match + * wins (no match → throw; an undefined arm matches no dyn value); JSON + * null matches * exactly the nullT arm of a union target (bare null targets cannot * exist). MAY THROW: * backends' may-throw analyses must treat it like a `throw` statement. diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7447-union-record-arm-width.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7447-union-record-arm-width.mjs.json new file mode 100644 index 000000000..68bd750d9 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7447-union-record-arm-width.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7447-union-record-arm-width.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7448-union-record-arm-width.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7448-union-record-arm-width.ts.json new file mode 100644 index 000000000..498ee884a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7448-union-record-arm-width.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7448-union-record-arm-width.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7447-union-record-arm-width.mjs b/tests/corpus/7447-union-record-arm-width.mjs new file mode 100644 index 000000000..dc12f8a59 --- /dev/null +++ b/tests/corpus/7447-union-record-arm-width.mjs @@ -0,0 +1,37 @@ +// Values read back from arrays whose elements mix object literals of +// different widths, an empty object among them, keep every field: the +// record arm with more fields is tried before `{}`, which matches every +// object. +const pairs = [ + ["a", { publicKeyEncoding: 5 }], + ["b", {}], + ["c", { publicKeyEncoding: 6, extra: true }], +]; +for (const [label, value] of pairs) console.log(label, Object.keys(value), JSON.stringify(value)); + +const options = [ + ["rsa", { modulusLength: 1024 }], + ["ec", { namedCurve: "P-256", paramEncoding: "named" }], + ["none", {}], +]; +for (const [type, value] of options) { + const { modulusLength, namedCurve, paramEncoding } = value; + console.log(type, modulusLength, namedCurve, paramEncoding); +} + +const mixed = [{ a: 1 }, {}, "s", { a: 2, b: "two" }]; +for (const value of mixed) { + console.log(typeof value, JSON.stringify(value), typeof value === "object" ? Object.keys(value).length : -1); +} + +function describe(value) { + return Object.entries(value) + .map(([key, field]) => `${key}=${field}`) + .join(";"); +} +const rows = [ + [1, { x: 1, y: 2 }], + [2, {}], + [3, { x: 3 }], +]; +for (const [id, value] of rows) console.log(id, describe(value), value.x, value.y); diff --git a/tests/corpus/7448-union-record-arm-width.ts b/tests/corpus/7448-union-record-arm-width.ts new file mode 100644 index 000000000..1518dd575 --- /dev/null +++ b/tests/corpus/7448-union-record-arm-width.ts @@ -0,0 +1,21 @@ +// A checked cast to a union of record types keeps every field of the +// value: arms with more required fields, then more fields, are tried +// before an arm whose fields are all optional, which matches every object. +type Full = { p: number; q: string }; +type Bare = { p?: number }; +type Other = { r: boolean }; + +const values = JSON.parse('[{"p":5,"q":"x"},{},{"r":true},{"p":7}]') as (Full | Bare | Other)[]; +for (const value of values) { + const q = "q" in value ? value.q : "-"; + const r = "r" in value ? value.r : "-"; + console.log(JSON.stringify(value), "q" in value, "r" in value, q, r); +} + +const one = JSON.parse('{"p":1,"q":"y"}') as Bare | Full; +console.log(JSON.stringify(one), "q" in one ? one.q : "no q"); + +type Point = { x: number; y?: number }; +type Labeled = { x: number; label: string }; +const points = JSON.parse('[{"x":1,"label":"a"},{"x":2,"y":3},{"x":4}]') as (Point | Labeled)[]; +for (const point of points) console.log(point.x, "label" in point ? point.label : "none", "y" in point ? point.y : "none"); diff --git a/tests/harness/corpus-costs/7447-union-record-arm-width.mjs.json b/tests/harness/corpus-costs/7447-union-record-arm-width.mjs.json new file mode 100644 index 000000000..5648bd255 --- /dev/null +++ b/tests/harness/corpus-costs/7447-union-record-arm-width.mjs.json @@ -0,0 +1 @@ +{ "plain": 5785, "san": 5722 } diff --git a/tests/harness/corpus-costs/7448-union-record-arm-width.ts.json b/tests/harness/corpus-costs/7448-union-record-arm-width.ts.json new file mode 100644 index 000000000..c60a813c2 --- /dev/null +++ b/tests/harness/corpus-costs/7448-union-record-arm-width.ts.json @@ -0,0 +1 @@ +{ "plain": 3204, "san": 4052 } From 8eedab9fdeb35a8221dbe36db51bce22e87afa6d Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 08:41:40 -0500 Subject: [PATCH 041/184] Cache untyped property access in programs that declare classes A program that declares a class reads and writes untyped properties through helpers that first select the receiver's property bag. The bag helpers return jsvals, so the helpers read the bag through js.toDyn; keyed helpers test each class's key table between the key's coercion and the read; and a program with subclasses normalizes the receiver (value = %dyn.class.normalize(value)) at the start of each helper. The inline-cache recognizer accepted none of these forms, so every property site in such a program called its helper. It now looks through dyn/jsval conversions of the receiver, accepts capsule branches before a keyed helper's final read, and accepts a rebinding of the receiver to a bag of itself. Plain objects and arrays then read and write through the site's cache, as they do in programs without classes. bench/dynamic-properties (new) exercises these sites. --- benchmarks/runtime/README.md | 1 + .../runtime/workloads/dynamic-properties.json | 6 + .../runtime/workloads/dynamic-properties.mjs | 56 ++++++++ packages/compiler/src/ir/js-ic.test.ts | 134 ++++++++++++++++++ packages/compiler/src/ir/js-ic.ts | 33 ++++- packages/compiler/test/js-ic-emission.test.ts | 44 ++++++ scripts/zero-cost-compare.mjs | 5 +- 7 files changed, 271 insertions(+), 8 deletions(-) create mode 100644 benchmarks/runtime/workloads/dynamic-properties.json create mode 100644 benchmarks/runtime/workloads/dynamic-properties.mjs diff --git a/benchmarks/runtime/README.md b/benchmarks/runtime/README.md index 4f152e4bb..04c90f7bb 100644 --- a/benchmarks/runtime/README.md +++ b/benchmarks/runtime/README.md @@ -13,6 +13,7 @@ Application-shaped workloads for measuring the speed of compiled executables. Ea | `cyclic-graphs` | cpu | object graphs with back references built and dropped: trees with parent and sibling links, rings of records sharing a long-lived registry, closures capturing their owner | | `dictionary-walks` | cpu | `Record` dictionaries walked with for-in and `Object.keys`/`values`/`entries` while entries are added, updated and deleted | | `dynamic-methods` | cpu | untyped JavaScript calling builtin methods on values it does not type: string `slice`/`indexOf`/`toUpperCase`/`padStart`, array `push`/`pop`/`includes`, `Map` and `Set`, and an object's own methods | +| `dynamic-properties` | cpu | untyped JavaScript reading and writing properties of values it does not type: records built by object literals in two key orders, nested records, and instances of a class and its subclass | | `functional-pipeline` | cpu | `filter`/`map`/`reduce`/`flatMap` chains, closures, small records | | `intern-table`, `intern-table-map`, `intern-table-1`, `intern-table-map-1` | cpu | worker threads interning the same type-like objects under string keys and records under number keys, then hits and misses: one shared `InternTable` per kind of key, or a module-level `Map` per thread (the `-map` entries), with 8 threads or 1 (the `-1` entries); under Node.js every thread has its own tables | | `json-records` | cpu | `JSON.stringify`, typed `JSON.parse`, Map aggregation | diff --git a/benchmarks/runtime/workloads/dynamic-properties.json b/benchmarks/runtime/workloads/dynamic-properties.json new file mode 100644 index 000000000..398981ef3 --- /dev/null +++ b/benchmarks/runtime/workloads/dynamic-properties.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/dynamic-properties.mjs", + "args": [ + "1000000" + ] +} diff --git a/benchmarks/runtime/workloads/dynamic-properties.mjs b/benchmarks/runtime/workloads/dynamic-properties.mjs new file mode 100644 index 000000000..d0ee3802c --- /dev/null +++ b/benchmarks/runtime/workloads/dynamic-properties.mjs @@ -0,0 +1,56 @@ +// Untyped JavaScript reading and writing properties of values it never +// declares types for: records built by object literals in two key orders, +// nested records, property updates, and instances of a class and its +// subclass read and written the same way. Argument: the number of +// iterations. +const rounds = Number(process.argv[2] ?? "1000000"); +class Cursor { + constructor(x, y) { + this.x = x; + this.y = y; + } + step(dx, dy) { + this.x += dx; + this.y += dy; + return this; + } +} +class Marker extends Cursor { + constructor(x, y, label) { + super(x, y); + this.label = label; + } +} +function makeItem(i) { + if (i % 3 === 0) + return { id: i, price: i % 17, qty: i % 5, meta: { tag: i % 7, weight: i % 11 } }; + return { id: i, qty: i % 5, price: i % 13, meta: { weight: i % 9, tag: i % 5 } }; +} +function cost(item) { + return item.price * item.qty + item.meta.weight; +} +function restock(item, round) { + item.qty = (item.qty + round) % 7; + item.meta.tag = (item.meta.tag + 1) % 7; +} +function distance(a, b) { + return Math.abs(a.x - b.x) + Math.abs(a.y - b.y); +} +const items = []; +for (let i = 0; i < 1024; i++) items.push(makeItem(i)); +const cursors = []; +for (let i = 0; i < 64; i++) + cursors.push(i % 2 === 0 ? new Cursor(i - 32, 7 - i) : new Marker(i - 32, 7 - i, "m" + i)); +const origin = { x: 0, y: 0 }; +let checksum = 0; +for (let r = 0; r < rounds; r++) { + const item = items[r & 1023]; + checksum += cost(item); + restock(item, r & 3); + checksum += item.meta.tag; + const cursor = cursors[r & 63]; + checksum += distance(cursor.step((r % 3) - 1, (r % 5) - 2), origin); + if (cursor.x > 100 || cursor.x < -100) cursor.x = 0; + if (cursor.y > 100 || cursor.y < -100) cursor.y = 0; +} +console.log(checksum, items[5].qty, items[6].meta.tag, cursors[3].x, cursors[3].y, cursors[3].label); diff --git a/packages/compiler/src/ir/js-ic.test.ts b/packages/compiler/src/ir/js-ic.test.ts index f71a1e9db..026600cc3 100644 --- a/packages/compiler/src/ir/js-ic.test.ts +++ b/packages/compiler/src/ir/js-ic.test.ts @@ -126,6 +126,140 @@ test("property helpers whose class branches only test capsules are cache sites", expect(sites.site("%dyn.class.property.1")).toEqual({ kind: "set", key: "y" }); }); +test("bags converted between dyn and jsval are still the receiver", () => { + // A program that declares a class returns its bags as jsvals, and the + // helpers read them through js.toDyn. + const toDyn = (value: IrExpr): IrExpr => ({ + kind: "libCall", + fn: "js.toDyn", + args: [value], + type: DYN, + loc, + }); + const fromDyn = (value: IrExpr): IrExpr => ({ + kind: "libCall", + fn: "js.fromDyn", + args: [value], + type: { kind: "jsval" }, + loc, + }); + const jsvalBag = helper( + "%dyn.class.readPropertiesOrEmpty.js", + ["v"], + [capsuleBranch("v"), ret(fromDyn(ref("v")))], + ); + const staticBag = helper("%dyn.class.staticProperties", ["v"], [ret(ref("v"))]); + const getter = getHelper( + "%dyn.class.property.0", + "x", + [capsuleBranch("p.0")], + call(staticBag.name, toDyn(call(jsvalBag.name, ref("p.0")))), + ); + const setter = setHelper("%dyn.class.property.1", "y"); + const write = setter.body[0] as Extract; + const keySet = write.expr as Extract; + keySet.args[0] = toDyn(call(jsvalBag.name, ref("p.0"))); + // A converted value that is not the receiver stays a helper call. + const foreign = getHelper( + "%dyn.class.property.2", + "x", + [], + toDyn(fromDyn({ kind: "dynObjLit", type: DYN, loc } as IrExpr)), + ); + const sites = sitesOf(jsvalBag, staticBag, getter, setter, foreign); + expect(sites.site("%dyn.class.property.0")).toEqual({ kind: "get", key: "x" }); + expect(sites.site("%dyn.class.property.1")).toEqual({ kind: "set", key: "y" }); + expect(sites.site("%dyn.class.property.2")).toBeNull(); +}); + +test("a bag helper may first rebind its parameter to a bag of it", () => { + // Programs with subclasses normalize the value before the class branches. + const normalize = helper("%dyn.class.normalize", ["v"], [capsuleBranch("v"), ret(ref("v"))]); + const rebind = (value: IrExpr): IrStmt => ({ kind: "assign", localId: "v", value, loc }); + const normalized = helper( + "%dyn.class.readPropertiesOrEmpty.normalized", + ["v"], + [rebind(call(normalize.name, ref("v"))), capsuleBranch("v"), ret(ref("v"))], + ); + const replaced = helper( + "%dyn.class.readPropertiesOrEmpty.replaced", + ["v"], + [rebind({ kind: "dynObjLit", type: DYN, loc } as IrExpr), ret(ref("v"))], + ); + const sites = sitesOf( + normalize, + normalized, + replaced, + getHelper("%dyn.class.property.0", "x", [], call(normalized.name, ref("p.0"))), + getHelper("%dyn.class.property.1", "x", [], call(replaced.name, ref("p.0"))), + // The property helper itself normalizes its receiver first, too. + getHelper("%dyn.class.property.2", "x", [ + { kind: "assign", localId: "p.0", value: call(normalize.name, ref("p.0")), loc }, + capsuleBranch("p.0"), + ]), + ); + expect(sites.site("%dyn.class.property.0")).toEqual({ kind: "get", key: "x" }); + expect(sites.site("%dyn.class.property.1")).toBeNull(); + expect(sites.site("%dyn.class.property.2")).toEqual({ kind: "get", key: "x" }); +}); + +test("keyed helpers may test capsules between the key's coercion and the read", () => { + const read = (object: IrExpr, key: string): IrExpr => ({ + kind: "libCall", + fn: "js.fromDyn", + args: [{ kind: "dynKeyGet", value: object, key: ref(key), type: DYN, loc }], + type: { kind: "jsval" }, + loc, + }); + const guard = (test: "symbol" | "nullish", tested: string, object: IrExpr): IrStmt => ({ + kind: "if", + cond: { kind: "dynTest", test, value: ref(tested), type: BOOL, loc }, + then: [ret(read(object, "p.key"))], + else_: null, + loc, + }); + const keyed = (name: string, between: IrStmt[]): IrFunction => + helper( + name, + ["p.0", "p.key"], + [ + capsuleBranch("p.0"), + guard("symbol", "p.key", call(identityBag.name, ref("p.0"))), + guard("nullish", "p.0", ref("p.0")), + { + kind: "varDecl", + localId: "key.string", + init: { + kind: "libCall", + fn: "dyn.toStringCoerce", + args: [ref("p.key")], + type: STRING, + loc, + }, + loc, + }, + ...between, + ret(read(call(identityBag.name, ref("p.0")), "key.string")), + ], + ["key.string"], + ); + const errorBranch: IrStmt = { + kind: "if", + cond: { kind: "dynTest", test: "error", value: ref("p.0"), type: BOOL, loc }, + then: [ret(ref("p.0"))], + else_: null, + loc, + }; + const sites = sitesOf( + keyed("%dyn.class.computed.0", []), + keyed("%dyn.class.computed.1", [capsuleBranch("p.0"), capsuleBranch("p.0", "dyn.classIs")]), + keyed("%dyn.class.computed.2", [errorBranch]), + ); + expect(sites.site("%dyn.class.computed.0")).toEqual({ kind: "keyed" }); + expect(sites.site("%dyn.class.computed.1")).toEqual({ kind: "keyed" }); + expect(sites.site("%dyn.class.computed.2")).toBeNull(); +}); + test("any other prefix or tail keeps the helper call", () => { const errorBranch: IrStmt = { kind: "if", diff --git a/packages/compiler/src/ir/js-ic.ts b/packages/compiler/src/ir/js-ic.ts index 752316db3..58c568bbd 100644 --- a/packages/compiler/src/ir/js-ic.ts +++ b/packages/compiler/src/ir/js-ic.ts @@ -62,7 +62,11 @@ export class JsIcSites { const receiver = fn.params[0]?.localId; if (receiver === undefined) return null; let i = 0; - while (i < fn.body.length && this.capsuleBranch(fn.body[i]!, receiver)) i++; + while ( + i < fn.body.length && + (this.capsuleBranch(fn.body[i]!, receiver) || this.rebinding(fn.body[i]!, receiver)) + ) + i++; const tail = fn.body.slice(i); if (fn.params.length === 1) return this.getTail(tail, receiver); if (fn.params.length !== 2) return null; @@ -109,10 +113,14 @@ export class JsIcSites { /** `if (typeof key === "symbol") return dynKeyGet(BAG(receiver), key); * if (receiver == null) return dynKeyGet(receiver, key); * const s = String(key); return dynKeyGet(BAG(receiver), s)`, each read - * possibly under js.fromDyn. */ + * possibly under js.fromDyn. Capsule branches may precede the final read: + * a program that declares classes looks the string key up in each class's + * key table there. */ private keyedTail(tail: IrStmt[], receiver: string, key: string): JsIcSite | null { - const [symbol, nullish, decl, ret] = tail; - if (tail.length !== 4) return null; + const [symbol, nullish, decl] = tail; + const ret = tail[tail.length - 1]; + if (tail.length < 4) return null; + if (!tail.slice(3, -1).every((s) => this.capsuleBranch(s, receiver))) return null; if (!this.guard(symbol, "symbol", key, (read) => this.bagOf(read, receiver), key)) return null; if (!this.guard(nullish, "nullish", receiver, (read) => isVar(read, receiver), key)) return null; @@ -159,9 +167,12 @@ export class JsIcSites { } /** Whether `e` is the receiver itself for every ordinary object or array: - * the receiver, or a chain of property-bag helpers applied to it. */ + * the receiver, or a chain of property-bag helpers applied to it. A + * conversion between the dyn and jsval forms keeps the object it holds. */ private bagOf(e: IrExpr, receiver: string): boolean { if (isVar(e, receiver)) return true; + if (e.kind === "libCall" && (e.fn === "js.toDyn" || e.fn === "js.fromDyn")) + return e.args.length === 1 && this.bagOf(e.args[0]!, receiver); return ( e.kind === "call" && e.args.length === 1 && @@ -182,6 +193,13 @@ export class JsIcSites { return ret.kind === "return" && ret.value !== null && this.bagOf(ret.value, param); } + /** `param = `, such as the subclass normalization + * `value = %dyn.class.normalize(value)`, which leaves an ordinary object + * or array in `param`. */ + private rebinding(s: IrStmt, param: string): boolean { + return s.kind === "assign" && s.localId === param && this.bagOf(s.value, param); + } + /** A property-bag helper that returns its argument unless a capsule * branch is taken. */ private identityHelper(name: string): boolean { @@ -204,7 +222,10 @@ export class JsIcSites { identity = fn.body .slice(0, -1) - .every((s) => this.capsuleBranch(s, param) || this.kindExit(s, param)) && + .every( + (s) => + this.capsuleBranch(s, param) || this.kindExit(s, param) || this.rebinding(s, param), + ) && last?.kind === "return" && last.value !== null && this.bagOf(last.value, param); diff --git a/packages/compiler/test/js-ic-emission.test.ts b/packages/compiler/test/js-ic-emission.test.ts index 47d641bff..54ea13d26 100644 --- a/packages/compiler/test/js-ic-emission.test.ts +++ b/packages/compiler/test/js-ic-emission.test.ts @@ -64,6 +64,50 @@ console.log(read(o), at(dyn([5, 6]), 1), call(o), o.n); } }); +test("property sites keep their caches in programs that declare classes and subclasses", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-js-ic-class-")); + try { + const entry = join(dir, "main.js"); + const outPath = join(dir, "main.ir.json"); + // Declared classes give every property helper class branches and + // property-bag lookups, and a subclass adds a normalization step; plain + // objects still read through the caches. + await writeFile( + entry, + ` +class Point { + constructor(x) { this.x = x; } +} +class Tagged extends Point { + constructor(x, tag) { super(x); this.tag = tag; } +} +function dyn(v) { return v; } +function read(o) { return o.x; } +function write(o, v) { o.x = v; } +function at(a, i) { return a[i]; } +var o = dyn({ x: 1 }); +var p = dyn(new Point(4)); +var t = dyn(new Tagged(6, "t")); +write(o, 3); +write(p, 5); +write(t, 7); +console.log(read(o), read(p), read(t), at(dyn([5, 6]), 1), at(p, "x"), at(t, "tag")); +`, + ); + const result = await compile(entry, { outDir: dir, outPath, outputKind: "ir" }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const mod = deserializeModule(await readFile(outPath, "utf8")); + expect(validateModule(mod)).toEqual([]); + const llvm = emitLlvmModule(mod); + expect(body(llvm, "read")).toContain("@scr_js_load_probe("); + expect(body(llvm, "write")).toContain("@scr_js_store_probe("); + expect(body(llvm, "at")).toContain("@scr_js_keyed_probe("); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + test("typed programs have no cache sites and keep the ordinary runtime", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-js-ic-typed-")); try { diff --git a/scripts/zero-cost-compare.mjs b/scripts/zero-cost-compare.mjs index 160e33438..f202c2ab7 100644 --- a/scripts/zero-cost-compare.mjs +++ b/scripts/zero-cost-compare.mjs @@ -394,8 +394,8 @@ export function dynamicTierSymbols(names) { * not yet traced to its typed root. bench/thread-slots, * bench/channel-messages and bench/worker-messages, added later, link them * through worker messages: - * node:worker_threads values are dyn objects. bench/dynamic-methods is - * untyped JavaScript. Every other bench workload is fully typed and must link none. The gate and + * node:worker_threads values are dyn objects. bench/dynamic-methods and + * bench/dynamic-properties are untyped JavaScript. Every other bench workload is fully typed and must link none. The gate and * tests/harness/bench-dynamic-tier.test.ts enforce that, and the test also * fails when a listed workload stops linking them, so remove a workload here * when a leak fix makes its binary clean. */ @@ -410,6 +410,7 @@ export const DYNAMIC_WORKLOADS = new Set([ "bench/thread-slots", "bench/channel-messages", "bench/dynamic-methods", + "bench/dynamic-properties", "bench/worker-messages", "bench/typed-array-scan-workers", "bench/log-summary", From 17c069667e8508e945639dd1f49415efddd81337 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 09:03:59 -0500 Subject: [PATCH 042/184] Settle a var declared by a for-in or for-of loop only inside the loop body A for-in or for-of loop stores its var binding only when it iterates. The early-read analysis counted the loop as an unconditional store for every later statement, so after a loop that ran zero times a closure read the binding as initialized (typeof answered "string" where Node answers "undefined"). The loop now settles the reads in its body; later reads settle at the next unconditional store. --- .../src/frontend/lowering/var-hoisting.ts | 27 ++++++++++++------- ...7553-var-for-in-of-zero-iterations.js.json | 6 +++++ .../7553-var-for-in-of-zero-iterations.js | 18 +++++++++++++ ...7553-var-for-in-of-zero-iterations.js.json | 1 + 4 files changed, 43 insertions(+), 9 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7553-var-for-in-of-zero-iterations.js.json create mode 100644 tests/corpus/7553-var-for-in-of-zero-iterations.js create mode 100644 tests/harness/corpus-costs/7553-var-for-in-of-zero-iterations.js.json diff --git a/packages/compiler/src/frontend/lowering/var-hoisting.ts b/packages/compiler/src/frontend/lowering/var-hoisting.ts index 1a60e7fda..0b415bb3d 100644 --- a/packages/compiler/src/frontend/lowering/var-hoisting.ts +++ b/packages/compiler/src/frontend/lowering/var-hoisting.ts @@ -6,9 +6,11 @@ * * A binding settles at the first statement of its hoisting scope (the * function body or the source file) that stores it unconditionally: a - * direct `var x = e`, `x = e;`, or a loop declaring it (`for (var i = 0; - * ...)`, `for (var k in o)`), whose own condition, update and body run - * after the store. A read settles when it runs after that statement: + * direct `var x = e`, `x = e;`, or a `for` loop declaring it (`for (var + * i = 0; ...)`), whose own condition, update and body run after the store. + * A `for (var k in o)` or `for (var v of xs)` loop stores the binding only + * when it iterates, so it settles the reads in its body and no others. A + * read settles when it runs after the settling statement: * - a read in the scope's own code, in a TypeScript file: tsc's * definite-assignment analysis rejects any earlier one; * - a read in the scope's own code, in a JavaScript file: its statement @@ -201,6 +203,9 @@ interface Settle { index: number; list: ts.VariableDeclarationList | null; end: number; + /** The first for-in/of loop declaring the binding before `index`, whose + * body runs after its store, or -1. */ + loop: number; } class ScopeAnalysis { @@ -229,6 +234,7 @@ class ScopeAnalysis { settleIndex(scope: Scope, symbol: ts.Symbol): Settle { const statements = statementsOf(scope); const checker = this.lowerer.checker; + let loopIndex = -1; for (let i = 0; i < statements.length; i++) { const s = statements[i]!; const list = ts.isVariableStatement(s) @@ -244,8 +250,10 @@ class ScopeAnalysis { if ( (d.initializer !== undefined || loop) && boundIdentifiersOf(d.name).some((n) => checker.getSymbolAtLocation(n) === symbol) - ) - return { index: i, list: loop ? null : list, end: d.end }; + ) { + if (!loop) return { index: i, list, end: d.end, loop: loopIndex }; + if (loopIndex < 0) loopIndex = i; + } } } else if ( ts.isExpressionStatement(s) && @@ -254,10 +262,10 @@ class ScopeAnalysis { ts.isIdentifier(s.expression.left) && checker.getSymbolAtLocation(s.expression.left) === symbol ) { - return { index: i, list: null, end: -1 }; + return { index: i, list: null, end: -1, loop: loopIndex }; } } - return { index: Infinity, list: null, end: -1 }; + return { index: Infinity, list: null, end: -1, loop: loopIndex }; } /** Whether `ref` (inside `scope`, or in another module) runs only after @@ -290,12 +298,13 @@ class ScopeAnalysis { (index === settle.index && (settlesWithin(top, ref) || // A later declarator of the storing list (`var a = {}, b = { ...a }`). - (settle.list !== null && isInside(ref, settle.list) && ref.pos >= settle.end))) + (settle.list !== null && isInside(ref, settle.list) && ref.pos >= settle.end))) || + (index === settle.loop && settlesWithin(top, ref)) ); } private functionSettles(scope: Scope, settle: Settle, fn: ts.FunctionDeclaration): boolean { - const key = `${fn.pos}:${settle.index}:${settle.end}`; + const key = `${fn.pos}:${settle.index}:${settle.end}:${settle.loop}`; const known = this.functionMemo.get(key); if (known !== undefined) return known; this.functionMemo.set(key, true); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7553-var-for-in-of-zero-iterations.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7553-var-for-in-of-zero-iterations.js.json new file mode 100644 index 000000000..9f470ca37 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7553-var-for-in-of-zero-iterations.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7553-var-for-in-of-zero-iterations.js" + ], + "diags": [] +} diff --git a/tests/corpus/7553-var-for-in-of-zero-iterations.js b/tests/corpus/7553-var-for-in-of-zero-iterations.js new file mode 100644 index 000000000..1a439f715 --- /dev/null +++ b/tests/corpus/7553-var-for-in-of-zero-iterations.js @@ -0,0 +1,18 @@ +// A for-in or for-of loop stores its var binding only when it iterates: +// after a loop that ran zero times, closures read undefined. +function f(o) { + for (var key in o) {} + const read = () => typeof key; + const same = () => key === undefined; + return [read(), same()]; +} +console.log(JSON.stringify(f({})), JSON.stringify(f({ a: 1 }))); +function g(xs) { + for (var x of xs) {} + const show = () => [typeof x, String(x)]; + return show(); +} +console.log(JSON.stringify(g([])), JSON.stringify(g([4]))); +for (var top in {}) {} +const readTop = () => typeof top; +console.log(readTop()); diff --git a/tests/harness/corpus-costs/7553-var-for-in-of-zero-iterations.js.json b/tests/harness/corpus-costs/7553-var-for-in-of-zero-iterations.js.json new file mode 100644 index 000000000..566096d24 --- /dev/null +++ b/tests/harness/corpus-costs/7553-var-for-in-of-zero-iterations.js.json @@ -0,0 +1 @@ +{ "plain": 19400, "san": 25900 } From a10c183b13f1f7b70ea200adf61198b2fb5ff4f7 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 09:05:55 -0500 Subject: [PATCH 043/184] Revert "Cache untyped property access in programs that declare classes (#1995)" --- benchmarks/runtime/README.md | 1 - .../runtime/workloads/dynamic-properties.json | 6 - .../runtime/workloads/dynamic-properties.mjs | 56 -------- packages/compiler/src/ir/js-ic.test.ts | 134 ------------------ packages/compiler/src/ir/js-ic.ts | 33 +---- packages/compiler/test/js-ic-emission.test.ts | 44 ------ scripts/zero-cost-compare.mjs | 5 +- 7 files changed, 8 insertions(+), 271 deletions(-) delete mode 100644 benchmarks/runtime/workloads/dynamic-properties.json delete mode 100644 benchmarks/runtime/workloads/dynamic-properties.mjs diff --git a/benchmarks/runtime/README.md b/benchmarks/runtime/README.md index 04c90f7bb..4f152e4bb 100644 --- a/benchmarks/runtime/README.md +++ b/benchmarks/runtime/README.md @@ -13,7 +13,6 @@ Application-shaped workloads for measuring the speed of compiled executables. Ea | `cyclic-graphs` | cpu | object graphs with back references built and dropped: trees with parent and sibling links, rings of records sharing a long-lived registry, closures capturing their owner | | `dictionary-walks` | cpu | `Record` dictionaries walked with for-in and `Object.keys`/`values`/`entries` while entries are added, updated and deleted | | `dynamic-methods` | cpu | untyped JavaScript calling builtin methods on values it does not type: string `slice`/`indexOf`/`toUpperCase`/`padStart`, array `push`/`pop`/`includes`, `Map` and `Set`, and an object's own methods | -| `dynamic-properties` | cpu | untyped JavaScript reading and writing properties of values it does not type: records built by object literals in two key orders, nested records, and instances of a class and its subclass | | `functional-pipeline` | cpu | `filter`/`map`/`reduce`/`flatMap` chains, closures, small records | | `intern-table`, `intern-table-map`, `intern-table-1`, `intern-table-map-1` | cpu | worker threads interning the same type-like objects under string keys and records under number keys, then hits and misses: one shared `InternTable` per kind of key, or a module-level `Map` per thread (the `-map` entries), with 8 threads or 1 (the `-1` entries); under Node.js every thread has its own tables | | `json-records` | cpu | `JSON.stringify`, typed `JSON.parse`, Map aggregation | diff --git a/benchmarks/runtime/workloads/dynamic-properties.json b/benchmarks/runtime/workloads/dynamic-properties.json deleted file mode 100644 index 398981ef3..000000000 --- a/benchmarks/runtime/workloads/dynamic-properties.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "entry": "workloads/dynamic-properties.mjs", - "args": [ - "1000000" - ] -} diff --git a/benchmarks/runtime/workloads/dynamic-properties.mjs b/benchmarks/runtime/workloads/dynamic-properties.mjs deleted file mode 100644 index d0ee3802c..000000000 --- a/benchmarks/runtime/workloads/dynamic-properties.mjs +++ /dev/null @@ -1,56 +0,0 @@ -// Untyped JavaScript reading and writing properties of values it never -// declares types for: records built by object literals in two key orders, -// nested records, property updates, and instances of a class and its -// subclass read and written the same way. Argument: the number of -// iterations. -const rounds = Number(process.argv[2] ?? "1000000"); -class Cursor { - constructor(x, y) { - this.x = x; - this.y = y; - } - step(dx, dy) { - this.x += dx; - this.y += dy; - return this; - } -} -class Marker extends Cursor { - constructor(x, y, label) { - super(x, y); - this.label = label; - } -} -function makeItem(i) { - if (i % 3 === 0) - return { id: i, price: i % 17, qty: i % 5, meta: { tag: i % 7, weight: i % 11 } }; - return { id: i, qty: i % 5, price: i % 13, meta: { weight: i % 9, tag: i % 5 } }; -} -function cost(item) { - return item.price * item.qty + item.meta.weight; -} -function restock(item, round) { - item.qty = (item.qty + round) % 7; - item.meta.tag = (item.meta.tag + 1) % 7; -} -function distance(a, b) { - return Math.abs(a.x - b.x) + Math.abs(a.y - b.y); -} -const items = []; -for (let i = 0; i < 1024; i++) items.push(makeItem(i)); -const cursors = []; -for (let i = 0; i < 64; i++) - cursors.push(i % 2 === 0 ? new Cursor(i - 32, 7 - i) : new Marker(i - 32, 7 - i, "m" + i)); -const origin = { x: 0, y: 0 }; -let checksum = 0; -for (let r = 0; r < rounds; r++) { - const item = items[r & 1023]; - checksum += cost(item); - restock(item, r & 3); - checksum += item.meta.tag; - const cursor = cursors[r & 63]; - checksum += distance(cursor.step((r % 3) - 1, (r % 5) - 2), origin); - if (cursor.x > 100 || cursor.x < -100) cursor.x = 0; - if (cursor.y > 100 || cursor.y < -100) cursor.y = 0; -} -console.log(checksum, items[5].qty, items[6].meta.tag, cursors[3].x, cursors[3].y, cursors[3].label); diff --git a/packages/compiler/src/ir/js-ic.test.ts b/packages/compiler/src/ir/js-ic.test.ts index 026600cc3..f71a1e9db 100644 --- a/packages/compiler/src/ir/js-ic.test.ts +++ b/packages/compiler/src/ir/js-ic.test.ts @@ -126,140 +126,6 @@ test("property helpers whose class branches only test capsules are cache sites", expect(sites.site("%dyn.class.property.1")).toEqual({ kind: "set", key: "y" }); }); -test("bags converted between dyn and jsval are still the receiver", () => { - // A program that declares a class returns its bags as jsvals, and the - // helpers read them through js.toDyn. - const toDyn = (value: IrExpr): IrExpr => ({ - kind: "libCall", - fn: "js.toDyn", - args: [value], - type: DYN, - loc, - }); - const fromDyn = (value: IrExpr): IrExpr => ({ - kind: "libCall", - fn: "js.fromDyn", - args: [value], - type: { kind: "jsval" }, - loc, - }); - const jsvalBag = helper( - "%dyn.class.readPropertiesOrEmpty.js", - ["v"], - [capsuleBranch("v"), ret(fromDyn(ref("v")))], - ); - const staticBag = helper("%dyn.class.staticProperties", ["v"], [ret(ref("v"))]); - const getter = getHelper( - "%dyn.class.property.0", - "x", - [capsuleBranch("p.0")], - call(staticBag.name, toDyn(call(jsvalBag.name, ref("p.0")))), - ); - const setter = setHelper("%dyn.class.property.1", "y"); - const write = setter.body[0] as Extract; - const keySet = write.expr as Extract; - keySet.args[0] = toDyn(call(jsvalBag.name, ref("p.0"))); - // A converted value that is not the receiver stays a helper call. - const foreign = getHelper( - "%dyn.class.property.2", - "x", - [], - toDyn(fromDyn({ kind: "dynObjLit", type: DYN, loc } as IrExpr)), - ); - const sites = sitesOf(jsvalBag, staticBag, getter, setter, foreign); - expect(sites.site("%dyn.class.property.0")).toEqual({ kind: "get", key: "x" }); - expect(sites.site("%dyn.class.property.1")).toEqual({ kind: "set", key: "y" }); - expect(sites.site("%dyn.class.property.2")).toBeNull(); -}); - -test("a bag helper may first rebind its parameter to a bag of it", () => { - // Programs with subclasses normalize the value before the class branches. - const normalize = helper("%dyn.class.normalize", ["v"], [capsuleBranch("v"), ret(ref("v"))]); - const rebind = (value: IrExpr): IrStmt => ({ kind: "assign", localId: "v", value, loc }); - const normalized = helper( - "%dyn.class.readPropertiesOrEmpty.normalized", - ["v"], - [rebind(call(normalize.name, ref("v"))), capsuleBranch("v"), ret(ref("v"))], - ); - const replaced = helper( - "%dyn.class.readPropertiesOrEmpty.replaced", - ["v"], - [rebind({ kind: "dynObjLit", type: DYN, loc } as IrExpr), ret(ref("v"))], - ); - const sites = sitesOf( - normalize, - normalized, - replaced, - getHelper("%dyn.class.property.0", "x", [], call(normalized.name, ref("p.0"))), - getHelper("%dyn.class.property.1", "x", [], call(replaced.name, ref("p.0"))), - // The property helper itself normalizes its receiver first, too. - getHelper("%dyn.class.property.2", "x", [ - { kind: "assign", localId: "p.0", value: call(normalize.name, ref("p.0")), loc }, - capsuleBranch("p.0"), - ]), - ); - expect(sites.site("%dyn.class.property.0")).toEqual({ kind: "get", key: "x" }); - expect(sites.site("%dyn.class.property.1")).toBeNull(); - expect(sites.site("%dyn.class.property.2")).toEqual({ kind: "get", key: "x" }); -}); - -test("keyed helpers may test capsules between the key's coercion and the read", () => { - const read = (object: IrExpr, key: string): IrExpr => ({ - kind: "libCall", - fn: "js.fromDyn", - args: [{ kind: "dynKeyGet", value: object, key: ref(key), type: DYN, loc }], - type: { kind: "jsval" }, - loc, - }); - const guard = (test: "symbol" | "nullish", tested: string, object: IrExpr): IrStmt => ({ - kind: "if", - cond: { kind: "dynTest", test, value: ref(tested), type: BOOL, loc }, - then: [ret(read(object, "p.key"))], - else_: null, - loc, - }); - const keyed = (name: string, between: IrStmt[]): IrFunction => - helper( - name, - ["p.0", "p.key"], - [ - capsuleBranch("p.0"), - guard("symbol", "p.key", call(identityBag.name, ref("p.0"))), - guard("nullish", "p.0", ref("p.0")), - { - kind: "varDecl", - localId: "key.string", - init: { - kind: "libCall", - fn: "dyn.toStringCoerce", - args: [ref("p.key")], - type: STRING, - loc, - }, - loc, - }, - ...between, - ret(read(call(identityBag.name, ref("p.0")), "key.string")), - ], - ["key.string"], - ); - const errorBranch: IrStmt = { - kind: "if", - cond: { kind: "dynTest", test: "error", value: ref("p.0"), type: BOOL, loc }, - then: [ret(ref("p.0"))], - else_: null, - loc, - }; - const sites = sitesOf( - keyed("%dyn.class.computed.0", []), - keyed("%dyn.class.computed.1", [capsuleBranch("p.0"), capsuleBranch("p.0", "dyn.classIs")]), - keyed("%dyn.class.computed.2", [errorBranch]), - ); - expect(sites.site("%dyn.class.computed.0")).toEqual({ kind: "keyed" }); - expect(sites.site("%dyn.class.computed.1")).toEqual({ kind: "keyed" }); - expect(sites.site("%dyn.class.computed.2")).toBeNull(); -}); - test("any other prefix or tail keeps the helper call", () => { const errorBranch: IrStmt = { kind: "if", diff --git a/packages/compiler/src/ir/js-ic.ts b/packages/compiler/src/ir/js-ic.ts index 58c568bbd..752316db3 100644 --- a/packages/compiler/src/ir/js-ic.ts +++ b/packages/compiler/src/ir/js-ic.ts @@ -62,11 +62,7 @@ export class JsIcSites { const receiver = fn.params[0]?.localId; if (receiver === undefined) return null; let i = 0; - while ( - i < fn.body.length && - (this.capsuleBranch(fn.body[i]!, receiver) || this.rebinding(fn.body[i]!, receiver)) - ) - i++; + while (i < fn.body.length && this.capsuleBranch(fn.body[i]!, receiver)) i++; const tail = fn.body.slice(i); if (fn.params.length === 1) return this.getTail(tail, receiver); if (fn.params.length !== 2) return null; @@ -113,14 +109,10 @@ export class JsIcSites { /** `if (typeof key === "symbol") return dynKeyGet(BAG(receiver), key); * if (receiver == null) return dynKeyGet(receiver, key); * const s = String(key); return dynKeyGet(BAG(receiver), s)`, each read - * possibly under js.fromDyn. Capsule branches may precede the final read: - * a program that declares classes looks the string key up in each class's - * key table there. */ + * possibly under js.fromDyn. */ private keyedTail(tail: IrStmt[], receiver: string, key: string): JsIcSite | null { - const [symbol, nullish, decl] = tail; - const ret = tail[tail.length - 1]; - if (tail.length < 4) return null; - if (!tail.slice(3, -1).every((s) => this.capsuleBranch(s, receiver))) return null; + const [symbol, nullish, decl, ret] = tail; + if (tail.length !== 4) return null; if (!this.guard(symbol, "symbol", key, (read) => this.bagOf(read, receiver), key)) return null; if (!this.guard(nullish, "nullish", receiver, (read) => isVar(read, receiver), key)) return null; @@ -167,12 +159,9 @@ export class JsIcSites { } /** Whether `e` is the receiver itself for every ordinary object or array: - * the receiver, or a chain of property-bag helpers applied to it. A - * conversion between the dyn and jsval forms keeps the object it holds. */ + * the receiver, or a chain of property-bag helpers applied to it. */ private bagOf(e: IrExpr, receiver: string): boolean { if (isVar(e, receiver)) return true; - if (e.kind === "libCall" && (e.fn === "js.toDyn" || e.fn === "js.fromDyn")) - return e.args.length === 1 && this.bagOf(e.args[0]!, receiver); return ( e.kind === "call" && e.args.length === 1 && @@ -193,13 +182,6 @@ export class JsIcSites { return ret.kind === "return" && ret.value !== null && this.bagOf(ret.value, param); } - /** `param = `, such as the subclass normalization - * `value = %dyn.class.normalize(value)`, which leaves an ordinary object - * or array in `param`. */ - private rebinding(s: IrStmt, param: string): boolean { - return s.kind === "assign" && s.localId === param && this.bagOf(s.value, param); - } - /** A property-bag helper that returns its argument unless a capsule * branch is taken. */ private identityHelper(name: string): boolean { @@ -222,10 +204,7 @@ export class JsIcSites { identity = fn.body .slice(0, -1) - .every( - (s) => - this.capsuleBranch(s, param) || this.kindExit(s, param) || this.rebinding(s, param), - ) && + .every((s) => this.capsuleBranch(s, param) || this.kindExit(s, param)) && last?.kind === "return" && last.value !== null && this.bagOf(last.value, param); diff --git a/packages/compiler/test/js-ic-emission.test.ts b/packages/compiler/test/js-ic-emission.test.ts index 54ea13d26..47d641bff 100644 --- a/packages/compiler/test/js-ic-emission.test.ts +++ b/packages/compiler/test/js-ic-emission.test.ts @@ -64,50 +64,6 @@ console.log(read(o), at(dyn([5, 6]), 1), call(o), o.n); } }); -test("property sites keep their caches in programs that declare classes and subclasses", async () => { - const dir = await mkdtemp(join(tmpdir(), "scriptc-js-ic-class-")); - try { - const entry = join(dir, "main.js"); - const outPath = join(dir, "main.ir.json"); - // Declared classes give every property helper class branches and - // property-bag lookups, and a subclass adds a normalization step; plain - // objects still read through the caches. - await writeFile( - entry, - ` -class Point { - constructor(x) { this.x = x; } -} -class Tagged extends Point { - constructor(x, tag) { super(x); this.tag = tag; } -} -function dyn(v) { return v; } -function read(o) { return o.x; } -function write(o, v) { o.x = v; } -function at(a, i) { return a[i]; } -var o = dyn({ x: 1 }); -var p = dyn(new Point(4)); -var t = dyn(new Tagged(6, "t")); -write(o, 3); -write(p, 5); -write(t, 7); -console.log(read(o), read(p), read(t), at(dyn([5, 6]), 1), at(p, "x"), at(t, "tag")); -`, - ); - const result = await compile(entry, { outDir: dir, outPath, outputKind: "ir" }); - if (!result.ok) - throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); - const mod = deserializeModule(await readFile(outPath, "utf8")); - expect(validateModule(mod)).toEqual([]); - const llvm = emitLlvmModule(mod); - expect(body(llvm, "read")).toContain("@scr_js_load_probe("); - expect(body(llvm, "write")).toContain("@scr_js_store_probe("); - expect(body(llvm, "at")).toContain("@scr_js_keyed_probe("); - } finally { - await rm(dir, { recursive: true, force: true }); - } -}); - test("typed programs have no cache sites and keep the ordinary runtime", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-js-ic-typed-")); try { diff --git a/scripts/zero-cost-compare.mjs b/scripts/zero-cost-compare.mjs index f202c2ab7..160e33438 100644 --- a/scripts/zero-cost-compare.mjs +++ b/scripts/zero-cost-compare.mjs @@ -394,8 +394,8 @@ export function dynamicTierSymbols(names) { * not yet traced to its typed root. bench/thread-slots, * bench/channel-messages and bench/worker-messages, added later, link them * through worker messages: - * node:worker_threads values are dyn objects. bench/dynamic-methods and - * bench/dynamic-properties are untyped JavaScript. Every other bench workload is fully typed and must link none. The gate and + * node:worker_threads values are dyn objects. bench/dynamic-methods is + * untyped JavaScript. Every other bench workload is fully typed and must link none. The gate and * tests/harness/bench-dynamic-tier.test.ts enforce that, and the test also * fails when a listed workload stops linking them, so remove a workload here * when a leak fix makes its binary clean. */ @@ -410,7 +410,6 @@ export const DYNAMIC_WORKLOADS = new Set([ "bench/thread-slots", "bench/channel-messages", "bench/dynamic-methods", - "bench/dynamic-properties", "bench/worker-messages", "bench/typed-array-scan-workers", "bench/log-summary", From 0f8dd276d91d11095b73c53393dfcd1251af7a4c Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 09:24:28 -0500 Subject: [PATCH 044/184] Reach a class's methods from its class expression's inner name in static initializers A named class expression's own name (const C = class Impl { static x = Impl.m(); }) names the class inside its body as the binding it is stored in names it outside. The reachability walk followed only stored binding names, so a static initializer calling a method through the inner name did not reach that method, and its early reads of the other cycle member's bindings lost their TDZ checks: the program read the binding before its declaration ran (a crash), where Node throws ReferenceError. The inner name now maps to its class expression. --- packages/compiler/src/frontend/program.ts | 12 +++++++++ .../main.ts.json | 8 ++++++ .../alpha.ts | 11 ++++++++ .../beta.ts | 26 +++++++++++++++++++ .../main.ts | 9 +++++++ .../main.ts.json | 1 + 6 files changed, 67 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7383-cycle-class-expression-name-tdz/main.ts.json create mode 100644 tests/corpus/7383-cycle-class-expression-name-tdz/alpha.ts create mode 100644 tests/corpus/7383-cycle-class-expression-name-tdz/beta.ts create mode 100644 tests/corpus/7383-cycle-class-expression-name-tdz/main.ts create mode 100644 tests/harness/corpus-costs/7383-cycle-class-expression-name-tdz/main.ts.json diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index 34a993c76..ce8f7d822 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -4187,11 +4187,23 @@ function cycleReachedDeferredReferences7( unitsBySymbol.set(symbol, [...(unitsBySymbol.get(symbol) ?? []), ...units]); allUnits.push(...units); } + // A named class expression's inner name (`class Impl { static x = + // Impl.m(); }`) names the class inside its own body, as the binding it + // is stored in names it outside. + const innerNames: ts.Identifier[] = []; + for (const unit of allUnits) { + if (!ts.isClassExpression(unit) || unit.name === undefined) continue; + const symbol = checker.getSymbolAtLocation(unit.name); + if (symbol === undefined) continue; + innerNames.push(unit.name); + unitsBySymbol.set(symbol, [...(unitsBySymbol.get(symbol) ?? []), unit]); + } // The identifier texts worth resolving: declared names, and the local // names of imports (named, default and namespace) in cycle members. const names = new Set(); for (const c of candidates.values()) names.add(c.name.text); for (const name of storedNames) names.add(name.text); + for (const name of innerNames) names.add(name.text); for (const sf of evaluation) { for (const stmt of sf.statements.nodes) { if (!ts.isImportDeclaration(stmt) || stmt.importClause === undefined) continue; diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7383-cycle-class-expression-name-tdz/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7383-cycle-class-expression-name-tdz/main.ts.json new file mode 100644 index 000000000..69b6d63bb --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7383-cycle-class-expression-name-tdz/main.ts.json @@ -0,0 +1,8 @@ +{ + "order": [ + "/tests/corpus/7383-cycle-class-expression-name-tdz/beta.ts", + "/tests/corpus/7383-cycle-class-expression-name-tdz/alpha.ts", + "/tests/corpus/7383-cycle-class-expression-name-tdz/main.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7383-cycle-class-expression-name-tdz/alpha.ts b/tests/corpus/7383-cycle-class-expression-name-tdz/alpha.ts new file mode 100644 index 000000000..ff8eb4a87 --- /dev/null +++ b/tests/corpus/7383-cycle-class-expression-name-tdz/alpha.ts @@ -0,0 +1,11 @@ +import { Probe, Lazy } from "./beta.ts"; + +export { Probe }; +export const label = "ready"; +export const other = "other"; + +export function later(): string { + return `${Lazy.describe()} ${other}`; +} + +console.log("alpha body", Probe.result, Lazy.value); diff --git a/tests/corpus/7383-cycle-class-expression-name-tdz/beta.ts b/tests/corpus/7383-cycle-class-expression-name-tdz/beta.ts new file mode 100644 index 000000000..d1d89ad1a --- /dev/null +++ b/tests/corpus/7383-cycle-class-expression-name-tdz/beta.ts @@ -0,0 +1,26 @@ +import { label, other } from "./alpha.ts"; + +function guard(read: () => string): string { + try { + return read(); + } catch (error) { + return error instanceof ReferenceError ? `ReferenceError: ${error.message}` : "other error"; + } +} + +export const Probe = class ProbeImpl { + static helper(): string { + return guard(() => `label ${label}`); + } + static result = ProbeImpl.helper(); +}; + +export const Lazy = class LazyImpl { + static read(): string { + return guard(() => `other ${other}`); + } + static describe(): string { + return `lazy ${other}`; + } + static value = [0].map(() => LazyImpl.read()).join(""); +}; diff --git a/tests/corpus/7383-cycle-class-expression-name-tdz/main.ts b/tests/corpus/7383-cycle-class-expression-name-tdz/main.ts new file mode 100644 index 000000000..93b7a47ee --- /dev/null +++ b/tests/corpus/7383-cycle-class-expression-name-tdz/main.ts @@ -0,0 +1,9 @@ +// The first cycle member to run builds classes from named class +// expressions whose static initializers call a method through the inner +// name, directly and from an arrow function; both methods read constants +// of the other member before it declared them: Node's temporal-dead-zone +// ReferenceError. A third method reads a constant only after the cycle +// finished. +import { label, Probe, later } from "./alpha.ts"; + +console.log(label, Probe.result, later()); diff --git a/tests/harness/corpus-costs/7383-cycle-class-expression-name-tdz/main.ts.json b/tests/harness/corpus-costs/7383-cycle-class-expression-name-tdz/main.ts.json new file mode 100644 index 000000000..b24c0c970 --- /dev/null +++ b/tests/harness/corpus-costs/7383-cycle-class-expression-name-tdz/main.ts.json @@ -0,0 +1 @@ +{ "plain": 1800, "san": 2600 } From 0e10a01386833d8b9e4d9737c43026bf7500f980 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 07:31:40 -0700 Subject: [PATCH 045/184] Keep record key order through typed iterators and generator resumes Records read out of a typed Map through an `IterableIterator` method (`all() { return this.rows.values(); }`) or out of a `Generator` came back through checked conversions, so the key-order witness treated their type as built at run time and a self copy (`{ ...current, ...patch }`) fell back to the declared order. - A record yield channel keeps its type beside the defaulted `any` return channel, like number, boolean and string already do: for-of and spread read the yielded record unchecked. - `m.values()`/`keys()`/`entries()` on a typed Map or Set flowing into an iterator-typed slot walks the collection with typed reads. It keeps the native iterator's protocol: lazy, live, and parked instead of closed by an early loop exit. - The witness counts a generator resume as building only its IteratorResult record, not the value it carries. Co-Authored-By: Claude Opus 5.5 --- .../frontend/lowering/iterator-adapters.ts | 171 +++++++++++++++++- .../lowering/key-order-witness.test.ts | 35 ++++ .../frontend/lowering/key-order-witness.ts | 17 +- packages/compiler/src/frontend/type-mapper.ts | 9 +- .../7746-key-order-iterable-iterator.ts.json | 6 + .../7746-key-order-iterable-iterator.ts | 58 ++++++ .../7746-key-order-iterable-iterator.ts.json | 1 + 7 files changed, 289 insertions(+), 8 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7746-key-order-iterable-iterator.ts.json create mode 100644 tests/corpus/7746-key-order-iterable-iterator.ts create mode 100644 tests/harness/corpus-costs/7746-key-order-iterable-iterator.ts.json diff --git a/packages/compiler/src/frontend/lowering/iterator-adapters.ts b/packages/compiler/src/frontend/lowering/iterator-adapters.ts index e50652182..a8b4ea496 100644 --- a/packages/compiler/src/frontend/lowering/iterator-adapters.ts +++ b/packages/compiler/src/frontend/lowering/iterator-adapters.ts @@ -28,11 +28,180 @@ export function iteratorSlotAdapter( expr: IrExpr, expected: IrType & { kind: "generator" }, ): IrExpr | null { - if (expr.type.kind === "dyn") return dynIteratorAdapter(lowerer, expr, expected); + if (expr.type.kind === "dyn") + return ( + collectionIteratorSlotValue(lowerer, expr, expected) ?? + dynIteratorAdapter(lowerer, expr, expected) + ); if (expr.type.kind === "generator") return generatorRetypeAdapter(lowerer, expr, expected); return null; } +/** `m.values()` (or keys/entries, or a Set's) on a typed Map or Set, which + * lowers to a dynamic invocation on the boxed collection, flowing into an + * iterator-typed slot (`values(): IterableIterator { return + * this.rows.values(); }`): an interned generator over the collection that + * reads each entry typed, instead of the checked adapter over the boxed + * native iterator (dynIteratorAdapter), which boxes every value and checks + * it back. It keeps that adapter's protocol: the walk is lazy and live, and + * since the native iterator has no return(), an early exit from a + * consumer's loop parks it on the current entry instead of completing it, + * so a later loop continues with the next entry. Null for any other source + * or a slot whose yield type is not the collection's entry type. */ +function collectionIteratorSlotValue( + lowerer: Lowerer, + expr: IrExpr, + expected: IrType & { kind: "generator" }, +): IrExpr | null { + if (expr.kind !== "dynInvoke" || expr.args.length > 0 || expr.calleeNameValue) return null; + const method = expr.method; + if (method !== "values" && method !== "keys" && method !== "entries") return null; + if (expr.recv.kind !== "dynFrom") return null; + const receiver = expr.recv.value; + const collT = receiver.type; + if (collT.kind !== "map" && collT.kind !== "set") return null; + if (expected.async || expected.yieldT.kind === "void") return null; + if (expected.retT.kind !== "void" && expected.retT.kind !== "dyn") return null; + if (expected.nextT.kind !== "undefinedT" && expected.nextT.kind !== "dyn") return null; + const resultType = genResultRecord( + expected.yieldT, + expected.retT, + lowerer.shapes, + lowerer.unions, + ); + if (!resultType) return null; + const loc = expr.loc; + const key = `collIterSlot:${method}:${typeKey(collT)}:${typeKey(expected)}`; + let name = lowerer.valueHelpers.get(key); + if (!name) { + const coll = varRef("coll.0", collT, loc); + const index = varRef("index.0", F64, loc); + const native = varRef("native.0", DYN, loc); + const resumedFlag = varRef("resumed.0", BOOL, loc); + const keyT = collT.kind === "map" ? collT.key : collT.elem; + const valueT = collT.kind === "map" ? collT.value : collT.elem; + const read = (m: "iterKey" | "iterValue", t: IrType): IrExpr => + collT.kind === "map" + ? { kind: "mapIntrinsic", method: m, receiver: coll, args: [index], type: t, loc } + : { kind: "setIntrinsic", method: "iterKey", receiver: coll, args: [index], type: t, loc }; + const entry = (): IrExpr => + method === "keys" || (collT.kind === "set" && method === "values") + ? read("iterKey", keyT) + : method === "values" + ? read("iterValue", valueT) + : { + kind: "recordLit", + fields: [ + { name: "0", value: read("iterKey", keyT) }, + { name: "1", value: read("iterValue", valueT) }, + ], + type: expected.yieldT, + loc, + }; + if (!typeEquals(entry().type, expected.yieldT)) return null; + const suspend = (): IrStmt => ({ + kind: "exprStmt", + expr: { kind: "yieldExpr", value: entry(), type: resumeType(expected.nextT), loc }, + loc, + }); + const finish: IrStmt = { + kind: "return", + value: expected.retT.kind === "dyn" ? dynUndefinedExpr(loc) : null, + loc, + }; + name = keyedHelperName(lowerer, "%coll.iterSlot", key); + lowerer.valueHelpers.set(key, name); + lowerer.liftedFns.push({ + name, + params: [{ localId: "coll.0", name: "coll", type: collT }], + returnType: expected.retT, + locals: [ + { id: "coll.0", name: "coll", type: collT, mutable: false }, + { id: "native.0", name: "native", type: DYN, mutable: false }, + { id: "index.0", name: "index", type: F64, mutable: true }, + { id: "resumed.0", name: "resumed", type: BOOL, mutable: true }, + ], + body: [ + { + kind: "varDecl", + localId: "native.0", + init: { + kind: "dynInvoke", + recv: lowerer.coerceToExpected(coll, DYN), + method: "values", + calleeName: collT.kind === "map" ? "map.values" : "set.values", + args: [], + type: DYN, + loc, + }, + loc, + }, + { + kind: "varDecl", + localId: "resumed.0", + init: { kind: "boolLit", value: false, type: BOOL, loc }, + loc, + }, + { + kind: "while", + cond: { kind: "boolLit", value: true, type: BOOL, loc }, + body: [ + { + kind: "assign", + localId: "index.0", + value: { + kind: "libCall", + fn: "dyn.collectionIteratorIndex", + args: [native], + type: F64, + loc, + }, + loc, + }, + { + kind: "if", + cond: { + kind: "bin", + op: "<", + left: index, + right: { kind: "numLit", value: 0, type: F64, loc }, + type: BOOL, + loc, + }, + then: [finish], + else_: null, + loc, + }, + setResumed(false, loc), + { + kind: "tryCatch", + tryBody: [suspend(), setResumed(true, loc)], + catchBody: null, + catchLocalId: null, + // An early exit parks on the current entry; the next step + // resumes the walk after it. + finallyBody: [ + { + kind: "if", + cond: { kind: "unary", op: "!", operand: resumedFlag, type: BOOL, loc }, + then: [suspend(), { kind: "continue", loc }], + else_: null, + loc, + }, + ], + loc, + }, + ], + loc, + }, + ], + generator: { yieldT: expected.yieldT, nextT: expected.nextT, resultType, tag: "!" }, + loc, + }); + } + return { kind: "call", callee: name, args: [receiver], type: expected, loc }; +} + /** `resumed.0 = value` — the adapters' early-exit marker around a yield. */ function setResumed(value: boolean, loc: SrcLoc): IrStmt { return { diff --git a/packages/compiler/src/frontend/lowering/key-order-witness.test.ts b/packages/compiler/src/frontend/lowering/key-order-witness.test.ts index 8058d0066..733888763 100644 --- a/packages/compiler/src/frontend/lowering/key-order-witness.test.ts +++ b/packages/compiler/src/frontend/lowering/key-order-witness.test.ts @@ -838,3 +838,38 @@ test("a checked function passes its record parameters out, and its callbacks' ba "b", ]); }); + +test("a generator resume builds only its result record, never the value it carries", () => { + const f = fixture(); + const ab = f.record( + [ + ["a", F64], + ["b", F64], + ], + ["a", "b"], + ); + const result = f.record( + [ + ["done", { kind: "bool" }], + ["value", ab], + ], + ["value", "done"], + ); + const gen: IrType = { + kind: "generator", + yieldT: ab, + retT: { kind: "void" }, + nextT: { kind: "undefinedT" }, + }; + const resume = (mode: "next" | "value", type: IrType): IrStmt => + observe({ kind: "genResume", mode, gen: ref("g", gen), arg: null, type, loc }); + f.run([ + observe(f.literal(ab, ["b", "a"])), + observe(f.literal(result, ["done", "value"])), + resume("value", ab), + resume("next", result), + ]); + expect(f.order(ab)).toEqual(["b", "a"]); + // The result record itself is built in its fixed layout: today's table. + expect(f.order(result)).toEqual(["value", "done"]); +}); diff --git a/packages/compiler/src/frontend/lowering/key-order-witness.ts b/packages/compiler/src/frontend/lowering/key-order-witness.ts index 933c9efcd..4a0a1309f 100644 --- a/packages/compiler/src/frontend/lowering/key-order-witness.ts +++ b/packages/compiler/src/frontend/lowering/key-order-witness.ts @@ -506,12 +506,23 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio case "strIntrinsic": case "regexIntrinsic": case "bytesIntrinsic": - // Generator results ({ done, value }) and Promise.withResolvers - // records are built by the backend in a fixed layout. - case "genResume": + // Promise.withResolvers records are built by the backend in a fixed + // layout. case "promiseWithResolvers": producedBy(e.type, true); return; + case "genResume": { + // A resume builds at most its IteratorResult record ({ done, value } + // in a fixed layout; an async generator's settles a promise of it). + // The value it carries is the generator's own yielded or returned + // value, which the generator's body constructs. The record-free + // drives build nothing: `value` moves that value out, and step, + // close and a record-free delegate answer a boolean or nothing. + if (e.mode === "step" || e.mode === "value" || e.mode === "close") return; + const result = e.type.kind === "promise" ? e.type.inner : e.type; + if (result.kind === "record") opaqueShapes.add(result.shapeId); + return; + } case "dynCheck": if (e.adopt === true) shapesIn(e.type, declaredShapes, declaredSeen); // Conversions that build no record: a boxed primitive, and a diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index 8adab0488..24a684078 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -1389,7 +1389,7 @@ function genChannels( : mapType(nextTs, ctx); if (!nextT || nextT.kind === "void" || nextT.kind === "nullT") return null; const ownChannel = (t: IrType): boolean => - t.kind === "f64" || t.kind === "bool" || t.kind === "string"; + t.kind === "f64" || t.kind === "bool" || t.kind === "string" || t.kind === "record"; const dynMix = (yieldT.kind === "dyn" && retT.kind !== "dyn" && retT.kind !== "void" && !ownChannel(retT)) || (retT.kind === "dyn" && yieldT.kind !== "dyn" && yieldT.kind !== "void" && !ownChannel(yieldT)); @@ -3954,9 +3954,10 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { // argument — fenced at the call). // Mixed dyn/concrete value channels share checked storage, so either // branch of the result record has the same representation. A number, - // boolean or string channel keeps its own type beside a dyn one (the - // `Generator` defaults): its values ride OUT typed and box only - // when the result record is built. + // boolean, string or record channel keeps its own type beside a dyn one + // (the `Generator` and `IterableIterator` defaults): its + // values ride OUT typed and box only when the result record is built, so + // for-of and spread read them back unchecked. if ( isStdlibInterface("Generator") || isStdlibInterface("AsyncGenerator") || diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7746-key-order-iterable-iterator.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7746-key-order-iterable-iterator.ts.json new file mode 100644 index 000000000..821fa4f21 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7746-key-order-iterable-iterator.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7746-key-order-iterable-iterator.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7746-key-order-iterable-iterator.ts b/tests/corpus/7746-key-order-iterable-iterator.ts new file mode 100644 index 000000000..13f27ee83 --- /dev/null +++ b/tests/corpus/7746-key-order-iterable-iterator.ts @@ -0,0 +1,58 @@ +// Records that leave a typed collection through an `IterableIterator` +// method or a `Generator` keep the order their literals create (title +// before id), including copies that spread a value of their own type. The +// iterators stay live and hand out the stored objects themselves. +interface Book { + id: string; + title: string; + version: number; +} +type BookDraft = Omit; +type BookPatch = Partial; + +class Repo { + private readonly rows = new Map(); + put(row: T): void { + this.rows.set(row.id, row); + } + get(id: string): T | undefined { + return this.rows.get(id); + } + all(): IterableIterator { + return this.rows.values(); + } + ids(): IterableIterator { + return this.rows.keys(); + } +} + +function* newest(books: readonly Book[]): Generator { + for (let i = books.length - 1; i >= 0; i--) yield books[i]!; +} + +const books = new Repo(); +let n = 0; +function create(draft: BookDraft): Book { + const book: Book = { ...draft, id: "b" + ++n, version: 1 }; + books.put(book); + return book; +} +function update(id: string, patch: BookPatch): Book { + const current = books.get(id)!; + const next: Book = { ...current, ...patch, version: current.version + 1 }; + books.put(next); + return next; +} + +create({ title: "Dune" }); +const live = books.all(); +console.log(JSON.stringify(create({ title: "Emma" }))); +console.log(JSON.stringify(update("b1", { title: "Dune II" }))); +const listed = [...live]; +console.log(JSON.stringify(listed)); +console.log(listed.length, listed[0] === books.get("b1"), [...books.ids()].join()); +for (const book of books.all()) console.log(Object.keys(book).join()); +const ordered = [...newest(listed)]; +console.log(JSON.stringify(ordered), ordered[0] === listed[1]); +const step = newest(listed).next(); +console.log(step.done, JSON.stringify(step.value)); diff --git a/tests/harness/corpus-costs/7746-key-order-iterable-iterator.ts.json b/tests/harness/corpus-costs/7746-key-order-iterable-iterator.ts.json new file mode 100644 index 000000000..de3f40849 --- /dev/null +++ b/tests/harness/corpus-costs/7746-key-order-iterable-iterator.ts.json @@ -0,0 +1 @@ +{ "plain": 3800, "san": 6100 } From 7c33b55435e9684dd14ec8373c4238e6be31e7ca Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 07:39:29 -0700 Subject: [PATCH 046/184] Keep fixed layouts for JavaScript classes next to freezes and unrelated accessors Two rules sent every JavaScript class family to field-creation tracking: - Any Object.freeze, Object.seal or Object.preventExtensions of a value other than a fresh literal. These never create or remove a property, and a class instance keeps its integrity state beside its fields: a typed call sets the CLASS_ATTRIBUTES word that TypeScript classes use with the same fixed layouts, and through an untyped value the runtime refuses the operation on every class instance (SC2020). Only Reflect.deleteProperty stays a removal of any name. - Any getter or setter. An accessor only matters when it shares a name with a field of the instances that see it, or when a subclass accessor can intercept a base constructor's field assignment. Other accessors live on the prototype beside the fields. In zod 3, ParseStatus and ParseInputLazyPath now keep fixed layouts (2 of the 39 tracked classes; the ZodType family stays tracked because its constructor creates own properties before _def). z.array(Order).parse over 500 orders: 270.7M -> 235.8M instructions per round (-12.9%). Co-Authored-By: Claude Opus 5.5 --- .../src/frontend/lowering/js-class-fields.ts | 56 +++++-- .../src/frontend/lowering/lower-classes.ts | 9 +- .../compiler/test/js-class-layouts.test.ts | 45 ++++++ .../7790-js-class-integrity-layouts.js.json | 6 + .../7791-js-class-accessor-layouts.js.json | 6 + .../corpus/7790-js-class-integrity-layouts.js | 139 ++++++++++++++++++ .../corpus/7791-js-class-accessor-layouts.js | 85 +++++++++++ .../7790-js-class-integrity-layouts.js.json | 1 + .../7791-js-class-accessor-layouts.js.json | 1 + 9 files changed, 334 insertions(+), 14 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7790-js-class-integrity-layouts.js.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7791-js-class-accessor-layouts.js.json create mode 100644 tests/corpus/7790-js-class-integrity-layouts.js create mode 100644 tests/corpus/7791-js-class-accessor-layouts.js create mode 100644 tests/harness/corpus-costs/7790-js-class-integrity-layouts.js.json create mode 100644 tests/harness/corpus-costs/7791-js-class-accessor-layouts.js.json diff --git a/packages/compiler/src/frontend/lowering/js-class-fields.ts b/packages/compiler/src/frontend/lowering/js-class-fields.ts index 2f726ce97..28a7ceeda 100644 --- a/packages/compiler/src/frontend/lowering/js-class-fields.ts +++ b/packages/compiler/src/frontend/lowering/js-class-fields.ts @@ -2,11 +2,19 @@ import * as ts from "../ts7/adapter.js"; import { isClassOwnEnumerableFieldName } from "../../ir/ir.js"; import type { ClassInfo } from "./lower-classes.js"; -/** Reflection that can remove a property or change its attributes. */ -const PROPERTY_REFLECTION = new Set(["deleteProperty", "freeze", "seal", "preventExtensions"]); +/* Object.freeze, Object.seal and Object.preventExtensions do not appear + * here: they never create or remove a property, and a class instance keeps + * its integrity state outside the field layout. A statically typed call + * sets the instance's CLASS_ATTRIBUTES word (class-attributes.ts), which + * TypeScript classes use with the same fixed layouts; through an untyped + * value the runtime refuses the operation on any class instance + * (scr_dyn_class_integrity_fence). Lifting that fence must revisit this. */ -/** `Object.freeze({ ... })` and the like apply to a new literal, which is - * not a class instance. */ +/** Reflection that can remove any property. */ +const PROPERTY_REFLECTION = new Set(["deleteProperty"]); + +/** `Reflect.deleteProperty({ ... }, key)` applies to a new literal, which + * is not a class instance. */ function freshLiteralArgument(callee: ts.PropertyAccessExpression): boolean { const call = callee.parent; if (!ts.isCallExpression(call) || call.expression !== callee) return false; @@ -141,12 +149,35 @@ export function constructionPrefix( return { created, sealed }; } -/** Whether a class declares a getter or setter, which can intercept a - * base constructor's field assignment. */ -function declaresAccessor(info: ClassInfo): boolean { - return [...info.methods.keys()].some( - (name) => name.startsWith("get:") || name.startsWith("set:"), - ); +/** The names of the getters and setters among a class's own methods. */ +export function accessorNames(methods: Iterable): Set { + const names = new Set(); + for (const name of methods) + if (name.startsWith("get:") || name.startsWith("set:")) names.add(name.slice(4)); + return names; +} + +/** Whether one of `accessors` can intercept an assignment the base + * constructions make: a base constructor's `this.name = value` runs a + * subclass accessor of that name instead of creating the field. Only field + * names can reach the created prefix; any other assignment already ends + * it. */ +export function accessorInterceptsBase(accessors: ReadonlySet, base: ClassInfo): boolean { + for (const name of accessors) if (base.fields.has(name)) return true; + return false; +} + +/** Whether an accessor that instances of `info` inherit or declare shares + * a name with one of their fields. Such a field and accessor disagree on + * what the instance's property is; any other accessor lives on the + * prototype beside the fields and never touches the layout. */ +function accessorShadowsField(info: ClassInfo): boolean { + for (let current: ClassInfo | null = info; current; current = current.base) { + if (current.def.runtime) break; + for (const name of accessorNames(current.methods.keys())) + if (info.fields.has(name)) return true; + } + return false; } /** `constructionPrefix` for a collected class and its bases. */ @@ -156,7 +187,8 @@ export function construction(info: ClassInfo): Construction | null { if (info.base) { base = construction(info.base); if (!base) return null; - if (declaresAccessor(info)) base = { created: base.created, sealed: false }; + if (accessorInterceptsBase(accessorNames(info.methods.keys()), info.base)) + base = { created: base.created, sealed: false }; } return constructionPrefix(info.decl, base, (name) => info.fields.has(name)); } @@ -169,7 +201,7 @@ export function construction(info: ClassInfo): Construction | null { export function fixedFieldFamily(root: ClassInfo, deleted: Set | null): boolean { if (deleted === null) return false; const fixed = (info: ClassInfo): boolean => { - if (declaresAccessor(info)) return false; + if (accessorShadowsField(info)) return false; const built = construction(info); if (!built) return false; for (const field of info.fieldOrder) { diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index 29afbb4fe..765f24884 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -177,7 +177,12 @@ import { hasLexicalPrivateReference, hasStaticThis, rejectStaticThis } from "./s import { lowerUrlNew } from "./lower-url.js"; import { isNativeProxyInitializer, lowerNativeProxy } from "./expressions/native-proxy.js"; import { classStaticDataFor } from "./class-static-data.js"; -import { construction, constructionPrefix } from "./js-class-fields.js"; +import { + accessorInterceptsBase, + accessorNames, + construction, + constructionPrefix, +} from "./js-class-fields.js"; import { lowerObjectFactoryNew } from "./object-factory-new.js"; import { tryLowerExpression } from "./expressions/try-lower-expression.js"; import { lowerInstanceConstructorNew } from "./class-instance-constructor.js"; @@ -4114,7 +4119,7 @@ export function collectClassShapeInner( // A subclass field created before anything can observe the instance // keeps its type: super() cannot run code that reads it earlier. const baseConstruction = - base && ![...methods.keys()].some((m) => m.startsWith("get:") || m.startsWith("set:")) + base && !accessorInterceptsBase(accessorNames(methods.keys()), base) ? construction(base) : null; const sealedPrefix = baseConstruction diff --git a/packages/compiler/test/js-class-layouts.test.ts b/packages/compiler/test/js-class-layouts.test.ts index c8491f3e8..110d007ee 100644 --- a/packages/compiler/test/js-class-layouts.test.ts +++ b/packages/compiler/test/js-class-layouts.test.ts @@ -101,6 +101,51 @@ delete loose.label; expect(layout(deleted, "Shape").tracks).toBe(true); }); +test("freezing values other than class instances keeps JavaScript class layouts fixed", async () => { + const module = await lower(`${SHAPES} +/** @param {{ value: any }} data */ +function settle(data) { + data.value = Object.freeze(data.value); + Object.seal(data); + return Object.preventExtensions(data.value); +} +settle({ value: { sides: 1 } }); +const square = new Square(); +Object.freeze(square); +`); + expect(layout(module, "Shape").tracks).toBe(false); + expect(layout(module, "Square").tracks).toBe(false); +}); + +test("accessors that share no field's name keep JavaScript class layouts fixed", async () => { + const module = await lower(` +class Path { + constructor(key) { + this.cache = []; + this.key = key; + } + get path() { + if (!this.cache.length) this.cache.push(this.key); + return this.cache; + } +} +class Named extends Path { + constructor(key) { + super(key); + this.label = "named"; + } + /** @param {string} value */ + set label2(value) { + this.label = value; + } +} +console.log(new Named("k").path.length); +`); + expect(layout(module, "Path")).toEqual({ fields: "cache:dyn key:dyn", tracks: false }); + expect(layout(module, "Named").tracks).toBe(false); + expect(layout(module, "Named").fields).toBe("cache:dyn key:dyn label:string"); +}); + test("JavaScript subclass fields keep their types when the base constructor only assigns fields", async () => { const module = await lower(` class Base { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7790-js-class-integrity-layouts.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7790-js-class-integrity-layouts.js.json new file mode 100644 index 000000000..526b86044 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7790-js-class-integrity-layouts.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7790-js-class-integrity-layouts.js" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7791-js-class-accessor-layouts.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7791-js-class-accessor-layouts.js.json new file mode 100644 index 000000000..5350539f0 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7791-js-class-accessor-layouts.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7791-js-class-accessor-layouts.js" + ], + "diags": [] +} diff --git a/tests/corpus/7790-js-class-integrity-layouts.js b/tests/corpus/7790-js-class-integrity-layouts.js new file mode 100644 index 000000000..f889c02f8 --- /dev/null +++ b/tests/corpus/7790-js-class-integrity-layouts.js @@ -0,0 +1,139 @@ +"use strict"; +// Object.freeze, Object.seal and Object.preventExtensions next to JavaScript +// classes whose fields all exist from construction. The integrity state of +// a class instance lives beside its fields, so classes keep fixed layouts +// whether or not a program freezes values; frozen instances still refuse +// writes and additions as in Node. + +class Point { + constructor(x, y) { + this.x = x; + this.y = y; + } + norm() { + return Math.abs(this.x) + Math.abs(this.y); + } +} + +class Point3 extends Point { + constructor(x, y, z) { + super(x, y); + this.z = z; + } + norm() { + return super.norm() + Math.abs(this.z); + } +} + +class Status { + constructor() { + this.value = "valid"; + } + dirty() { + if (this.value === "valid") this.value = "dirty"; + } +} + +/** @param {any} target @param {string} key @param {unknown} value */ +function put(target, key, value) { + target[key] = value; +} + +/** @param {any} value */ +function describe(value) { + const keys = Object.keys(value); + const parts = []; + for (const key in value) parts.push(`${key}=${String(value[key])}`); + return `${keys.join(",")} | ${parts.join(" ")} | ${JSON.stringify(value)}`; +} + +/** @param {() => void} fn */ +function attempt(label, fn) { + try { + fn(); + console.log(label, "ok"); + } catch (e) { + console.log(label, e instanceof TypeError ? "TypeError" : "other", e.message); + } +} + +// The zod shape: freeze whatever value a result carries. Here it is always +// plain data, never a class instance. +/** @param {{ status: string, value: any }} data */ +function freezeResult(data) { + if (data.status === "valid") data.value = Object.freeze(data.value); + return data; +} + +const results = [ + freezeResult({ status: "valid", value: { a: 1, list: [1, 2] } }), + freezeResult({ status: "valid", value: [3, 4] }), + freezeResult({ status: "aborted", value: { b: 2 } }), + freezeResult({ status: "valid", value: "text" }), +]; +for (const result of results) console.log(result.status, Object.isFrozen(result.value), JSON.stringify(result.value)); +/** @param {() => void} fn */ +function throws(fn) { + try { + fn(); + return false; + } catch (e) { + return e instanceof TypeError; + } +} +console.log(throws(() => put(results[0].value, "a", 9)), throws(() => put(results[2].value, "b", 3))); +console.log(JSON.stringify(results[2].value)); + +// Class instances that only ever travel through untyped code. +const status = new Status(); +status.dirty(); +const p = new Point(1, -2); +const q = new Point3(3, 4, -5); +put(p, "tag", "p"); +put(q, "w", 6); +console.log(describe(status), status.value); +console.log(describe(p), p.norm()); +console.log(describe(q), q.norm()); +console.log(Object.isFrozen(p), Object.isExtensible(q), Object.isSealed(status)); + +// Statically typed freeze, seal and preventExtensions of instances. +const frozen = new Point(5, 6); +Object.freeze(frozen); +attempt("frozen field write", () => { + /** @type {Point} */ (frozen).x = 50; +}); +attempt("frozen field write untyped", () => put(frozen, "y", 60)); +attempt("frozen addition", () => put(frozen, "extra", 1)); +console.log(describe(frozen), Object.isFrozen(frozen), Object.isSealed(frozen), Object.isExtensible(frozen)); +console.log(JSON.stringify(Object.getOwnPropertyDescriptor(frozen, "x"))); + +const sealed = new Point3(1, 2, 3); +put(sealed, "before", true); +Object.seal(sealed); +attempt("sealed field write", () => { + sealed.z = 30; +}); +attempt("sealed added write", () => put(sealed, "before", false)); +attempt("sealed addition", () => put(sealed, "after", 1)); +console.log(describe(sealed), Object.isFrozen(sealed), Object.isSealed(sealed), Object.isExtensible(sealed)); +console.log(JSON.stringify(Object.getOwnPropertyDescriptor(sealed, "z"))); + +const closed = new Status(); +Object.preventExtensions(closed); +closed.dirty(); +attempt("closed addition", () => put(closed, "more", 1)); +console.log(describe(closed), Object.isFrozen(closed), Object.isExtensible(closed)); + +// A subclass instance frozen through its base type. +/** @type {Point} */ +const asBase = new Point3(7, 8, 9); +Object.freeze(asBase); +attempt("frozen subclass field", () => put(asBase, "z", 90)); +console.log(describe(asBase), asBase.norm(), Object.isFrozen(asBase)); + +// Fresh instances of the same classes stay writable. +const fresh = new Point3(0, 0, 0); +fresh.x = 1; +put(fresh, "y", 2); +put(fresh, "added", 3); +console.log(describe(fresh), fresh.norm(), Object.isFrozen(fresh)); diff --git a/tests/corpus/7791-js-class-accessor-layouts.js b/tests/corpus/7791-js-class-accessor-layouts.js new file mode 100644 index 000000000..6c87989cb --- /dev/null +++ b/tests/corpus/7791-js-class-accessor-layouts.js @@ -0,0 +1,85 @@ +"use strict"; +// Getters and setters on JavaScript classes. An accessor whose name no +// field uses lives on the prototype beside the fields; an accessor that +// shares a field's name decides what an assignment does. Both behave as in +// Node, for direct use and for instances passed to untyped code. + +class LazyPath { + constructor(parent, value, path, key) { + this._cachedPath = []; + this.parent = parent; + this.data = value; + this._path = path; + this._key = key; + } + get path() { + if (!this._cachedPath.length) { + if (Array.isArray(this._key)) this._cachedPath.push(...this._path, ...this._key); + else this._cachedPath.push(...this._path, this._key); + } + return this._cachedPath; + } +} + +class Temperature { + constructor(celsius) { + this.celsius = celsius; + } + get fahrenheit() { + return this.celsius * 1.8 + 32; + } + set fahrenheit(value) { + this.celsius = (value - 32) / 1.8; + } +} + +class Labeled extends Temperature { + constructor(celsius, label) { + super(celsius); + this.label = label; + } + get description() { + return `${this.label}: ${this.celsius.toFixed(1)}C`; + } +} + +/** @param {any} target @param {string} key @param {unknown} value */ +function put(target, key, value) { + target[key] = value; +} + +/** @param {any} value */ +function describe(value) { + const parts = []; + for (const key in value) parts.push(key); + return `${Object.keys(value).join(",")} | ${parts.join(",")} | ${JSON.stringify(value)}`; +} + +/** @param {any} value @param {string} key */ +function probe(value, key) { + return `${key}: in=${key in value} own=${Object.prototype.hasOwnProperty.call(value, key)} value=${JSON.stringify(value[key])}`; +} + +const root = { common: true }; +const paths = []; +for (let i = 0; i < 3; i++) paths.push(new LazyPath(root, { n: i }, ["items"], i)); +paths.push(new LazyPath(root, null, ["a"], ["b", "c"])); +for (const item of paths) console.log(item.path.join("."), item.data && item.data.n, item.parent === root); +console.log(describe(paths[0])); +console.log(probe(paths[1], "path"), probe(paths[1], "_key")); +put(paths[2], "extra", 1); +console.log(describe(paths[2]), paths[2].path.length); + +const t = new Temperature(100); +console.log(t.fahrenheit, describe(t)); +t.fahrenheit = 32; +console.log(t.celsius, t.fahrenheit); +put(t, "fahrenheit", 212); +console.log(t.celsius, probe(t, "fahrenheit"), probe(t, "celsius")); +put(t, "note", "added"); +console.log(describe(t)); + +const l = new Labeled(20, "room"); +console.log(l.description, l.fahrenheit, describe(l)); +l.fahrenheit = 50; +console.log(l.description, probe(l, "description")); diff --git a/tests/harness/corpus-costs/7790-js-class-integrity-layouts.js.json b/tests/harness/corpus-costs/7790-js-class-integrity-layouts.js.json new file mode 100644 index 000000000..af44b31af --- /dev/null +++ b/tests/harness/corpus-costs/7790-js-class-integrity-layouts.js.json @@ -0,0 +1 @@ +{ "plain": 4400, "san": 5200 } diff --git a/tests/harness/corpus-costs/7791-js-class-accessor-layouts.js.json b/tests/harness/corpus-costs/7791-js-class-accessor-layouts.js.json new file mode 100644 index 000000000..eb36dcefe --- /dev/null +++ b/tests/harness/corpus-costs/7791-js-class-accessor-layouts.js.json @@ -0,0 +1 @@ +{ "plain": 5100, "san": 4700 } From a23ce3043c0ab0d8c8b3ff495c0832fc4bc7a654 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 09:53:05 -0500 Subject: [PATCH 047/184] Keep loop-carried locals written from int32 element reads int32 The int32 array proof assumes its candidates' elements are int32 and asks the integer range analysis whether every stored value is one. A local assigned in a loop from an element read of a candidate array (`const t = next[at]!; next[at] = at; at = t`) never closed under the int32 bound, because the loop-carried closure had no case for element reads, so arrays that store such cursors kept double slots and every index converted from a double. Element reads now take the slot range in that closure, as field reads and calls already did. The index-cursors runtime workload walks an array of indices with such a cursor. --- .../runtime/workloads/index-cursors.json | 6 ++++ benchmarks/runtime/workloads/index-cursors.ts | 25 +++++++++++++++ .../compiler/src/ir/integer-ranges.test.ts | 31 +++++++++++++++++++ packages/compiler/src/ir/integer-ranges.ts | 1 + .../test/int32-element-array-emission.test.ts | 24 ++++++++++++++ 5 files changed, 87 insertions(+) create mode 100644 benchmarks/runtime/workloads/index-cursors.json create mode 100644 benchmarks/runtime/workloads/index-cursors.ts diff --git a/benchmarks/runtime/workloads/index-cursors.json b/benchmarks/runtime/workloads/index-cursors.json new file mode 100644 index 000000000..24f2a263c --- /dev/null +++ b/benchmarks/runtime/workloads/index-cursors.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/index-cursors.ts", + "args": [ + "20000" + ] +} diff --git a/benchmarks/runtime/workloads/index-cursors.ts b/benchmarks/runtime/workloads/index-cursors.ts new file mode 100644 index 000000000..b65ded46b --- /dev/null +++ b/benchmarks/runtime/workloads/index-cursors.ts @@ -0,0 +1,25 @@ +// Arrays of indices walked by loop-carried cursors: each step reads the +// next index from the array and relinks the visited entries, so the +// cursor and every stored value are small integers read back from the +// array itself. +const N = 4096; + +function relink(rounds: number): number { + const next: number[] = []; + for (let i = 0; i < N; i++) next.push(i + 1 < N ? i + 1 : 0); + let at = next[0]!; + let total = 0; + for (let r = 0; r < rounds; r++) { + for (let s = 0; s < N; s++) { + const t = next[at]!; + next[at] = next[t]!; + next[t] = at; + at = t; + total = (total + at) | 0; + } + } + return total; +} + +const rounds = Number(process.argv[2] ?? "1"); +console.log("relink", relink(rounds)); diff --git a/packages/compiler/src/ir/integer-ranges.test.ts b/packages/compiler/src/ir/integer-ranges.test.ts index fcb040275..5ae04d28f 100644 --- a/packages/compiler/src/ir/integer-ranges.test.ts +++ b/packages/compiler/src/ir/integer-ranges.test.ts @@ -440,6 +440,37 @@ test("int32-closed locals keep the int32 range across loops and switches", () => expect(switched.get(afterSwitch)).toEqual(INT32); }); +test("loop-carried locals written from Int32[] element reads keep the int32 range", () => { + const arrType = { kind: "array", elem: F64 } as const; + const loop = (localId: string): { after: IrExpr; f: IrFunction } => { + const after = ref(); + // `x = a[x]!` in a loop: a cursor that follows indices stored in the array. + const read: IrExpr = { + loc, + kind: "arrayGet", + arr: { loc, kind: "varRef", localId, type: arrType }, + index: ref(), + type: F64, + }; + const f = fn([ + assign(num(0)), + { + loc, + kind: "while", + cond: condition("<", ref("unknown"), num(1)), + body: [assign(read)], + }, + observe(after), + ]); + return { after, f }; + }; + const elements = { elements: (arr: IrExpr) => arr.kind === "varRef" && arr.localId === "ints" }; + const ints = loop("ints"); + expect(analyzeIntegerRanges(ints.f, elements).get(ints.after)).toEqual(INT32); + const floats = loop("floats"); + expect(analyzeIntegerRanges(floats.f, elements).get(floats.after) ?? null).toBeNull(); +}); + test("uint32-closed locals keep the uint32 range across loop back edges", () => { // for (let i = 0; i < n; i++) { v = unknown >>> 0; acc = (acc + v) >>> 0; ... } const loop = (start: IrExpr | null, writes: IrExpr[]) => { diff --git a/packages/compiler/src/ir/integer-ranges.ts b/packages/compiler/src/ir/integer-ranges.ts index 0e5ca426d..683ceab75 100644 --- a/packages/compiler/src/ir/integer-ranges.ts +++ b/packages/compiler/src/ir/integer-ranges.ts @@ -480,6 +480,7 @@ export function analyzeIntegerRangeFacts( return boxedRange(e.localId); } case "fieldGet": + case "arrayGet": case "call": return slotRange(e); default: diff --git a/packages/compiler/test/int32-element-array-emission.test.ts b/packages/compiler/test/int32-element-array-emission.test.ts index 0b68a5d3e..bb6623d5f 100644 --- a/packages/compiler/test/int32-element-array-emission.test.ts +++ b/packages/compiler/test/int32-element-array-emission.test.ts @@ -77,6 +77,19 @@ function literalCounter(n: number): number { for (let i = 0; i < n; i++) a[0] = a[0]! + 1; return a[0]!; } +function walk(n: number, steps: number): number { + const next: number[] = []; + for (let i = 0; i < n; i++) next.push(i + 1 < n ? i + 1 : 0); + let at = next[0]!; + let total = 0; + for (let s = 0; s < steps; s++) { + const t = next[at]!; + next[at] = at; + at = t; + total = (total + at) | 0; + } + return total; +} function joined(n: number): string { const a = [n | 0, 2]; return a.join(","); @@ -91,6 +104,7 @@ console.log( shortCounter(), literalCounter(n), joined(n), + walk(n | 0, 5), ); `; @@ -119,6 +133,16 @@ test("arrays storing only int32 elements get 4-byte slots", async () => { expect(counts).not.toContain("popArray"); }); +test("an array that stores a cursor read back from its own elements gets 4-byte slots", async () => { + const llvm = await emitLlvm(source); + const walk = functions(llvm, "walk"); + expect(walk).not.toBe(""); + // `at` is assigned `next[at]!` in the loop and stored back into `next`: + // it holds an int32 whenever the array's elements are int32s. + expect(walk).toMatch(/@scr_arr_new_small\(i32 7\)/); + expect(walk).toMatch(/getelementptr inbounds i32, ptr %\w+, i64 %\w+\n\s+%\w+ = load i32/); +}); + test("arrays with a store that has no exact integer range, or that escape, keep doubles", async () => { const llvm = await emitLlvm(source); for (const name of ["fraction", "negativeZero", "joined"]) { From 056dda3633b6c01e9df77ef9fbca683ef4e4e800 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 07:35:50 -0700 Subject: [PATCH 048/184] Read Web Headers pairs without copying them, and cache their sorted list Headers kept its name/value pairs in a string array and read every element through scr_arr_get_ref (index checks, retain, release). forEach and the iterators re-sorted and rescanned the whole list for every step, and every Response or Headers built from another Headers object re-validated and re-normalized each copied pair. The Hono fixture spends 12-15% of a request there. - Pair loops read borrowed elements of the dense array (sf_pair_at). - The sort-and-combine list is cached on the Headers object and dropped by every mutation, so live iteration keeps Node's semantics. - get() of a single value returns the stored string. - Copying a Headers object whose pairs are canonical (built by the constructor or a Response) into an empty list copies the pairs; wire response headers keep the per-pair path because they may repeat names. Co-Authored-By: Claude Opus 5.5 --- .../corpus/fetch-headers-copy-cache.ts.json | 6 + packages/runtime/src/scr_fetch.c | 268 ++++++++++-------- tests/corpus/fetch-headers-copy-cache.ts | 85 ++++++ .../fetch-headers-copy-cache.ts.json | 1 + 4 files changed, 236 insertions(+), 124 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/fetch-headers-copy-cache.ts.json create mode 100644 tests/corpus/fetch-headers-copy-cache.ts create mode 100644 tests/harness/corpus-costs/fetch-headers-copy-cache.ts.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/fetch-headers-copy-cache.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/fetch-headers-copy-cache.ts.json new file mode 100644 index 000000000..bd69651b1 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/fetch-headers-copy-cache.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/fetch-headers-copy-cache.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index 6049a4936..48b60942e 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -172,7 +172,14 @@ struct SfStream { struct SfHeaders { size_t rc; ScrArr *pairs; + /* Fetch's "sort and combine" list of `pairs` as flat name/value pairs, + * built on the first iteration step and dropped by every mutation. */ + ScrArr *sorted; bool immutable; + /* `pairs` is in the form sf_add_converted_headers builds: lowercase valid + * names, trimmed valid UTF-8 values, one combined pair per name except + * set-cookie. Headers built from wire lines are not (duplicates stay). */ + bool canonical; }; struct SfResponse { @@ -2164,6 +2171,24 @@ static ScrDyn *sf_controller_get(void *ptr, const char *key, size_t len) { static bool sf_header_name_ok(const char *s, size_t len); +/* A borrowed element of a runtime-built header pair array. These arrays are + * only ever appended to or rewritten whole, so they are dense. */ +static inline ScrStr *sf_pair_at(ScrArr *pairs, size_t i) { + if (i < pairs->filled) return (ScrStr *)(uintptr_t)scr_arr_dense_get(pairs, i); + return (ScrStr *)scr_arr_peek_ref(pairs, (double)i); +} + +static ScrStr *sf_header_name_lower(const char *data, size_t len); + +/* `name` in lowercase, +1: the same string when it has no uppercase byte. */ +static ScrStr *sf_header_name_lower_str(ScrStr *name) { + for (size_t i = 0; i < name->len; i++) { + char c = name->data[i]; + if (c >= 'A' && c <= 'Z') return sf_header_name_lower(name->data, name->len); + } + return scr_str_retain(name); +} + static ScrStr *sf_header_name_lower(const char *data, size_t len) { char *lower = malloc(len ? len : 1); if (!lower) sf_oom(); @@ -2232,13 +2257,8 @@ static SfHeaders *sf_headers_new_response(ScrArr *raw_pairs) { size_t n = (size_t)scr_arr_len(raw_pairs); ScrArr *pairs = scr_arr_new(SCR_ELEM_STR, n); for (size_t i = 0; i + 1 < n; i += 2) { - ScrStr *raw_name = scr_arr_get_ref(raw_pairs, (double)i); - scr_arr_push_ref( - pairs, sf_header_name_lower(raw_name->data, raw_name->len)); - scr_str_release(raw_name); - ScrStr *raw = scr_arr_get_ref(raw_pairs, (double)(i + 1)); - scr_arr_push_ref(pairs, sf_latin1_to_utf8(raw)); - scr_str_release(raw); + scr_arr_push_ref(pairs, sf_header_name_lower_str(sf_pair_at(raw_pairs, i))); + scr_arr_push_ref(pairs, sf_latin1_to_utf8(sf_pair_at(raw_pairs, i + 1))); } scr_arr_release(raw_pairs); return sf_headers_new(pairs, true); @@ -2252,9 +2272,17 @@ static SfHeaders *sf_headers_retain(SfHeaders *h) { static void sf_headers_release(SfHeaders *h) { if (!h || --h->rc > 0) return; scr_arr_release(h->pairs); + if (h->sorted) scr_arr_release(h->sorted); free(h); } +/* Every change to h->pairs goes through here first. */ +static void sf_headers_changed(SfHeaders *h) { + if (!h->sorted) return; + scr_arr_release(h->sorted); + h->sorted = NULL; +} + static void *sf_headers_retain_v(void *p) { return sf_headers_retain((SfHeaders *)p); } @@ -2294,7 +2322,7 @@ static ScrStr *sf_headers_name(const ScrDyn *value, const char *method) { scr_str_release(raw); return NULL; } - ScrStr *out = sf_header_name_lower(raw->data, raw->len); + ScrStr *out = sf_header_name_lower_str(raw); scr_str_release(raw); return out; } @@ -2305,40 +2333,36 @@ static bool sf_headers_key_eq(const ScrStr *left, const ScrStr *right) { } static ScrStr *sf_headers_get_value(SfHeaders *h, const ScrStr *name) { - size_t n = (size_t)scr_arr_len(h->pairs); + ScrArr *pairs = h->pairs; + size_t n = (size_t)scr_arr_len(pairs); size_t count = 0; size_t total = 0; + ScrStr *first = NULL; for (size_t i = 0; i + 1 < n; i += 2) { - ScrStr *key = scr_arr_get_ref(h->pairs, (double)i); - ScrStr *value = scr_arr_get_ref(h->pairs, (double)(i + 1)); - if (sf_headers_key_eq(key, name)) { - if (count > 0) total += 2; - total += value->len; - count++; - } - scr_str_release(key); - scr_str_release(value); + if (!sf_headers_key_eq(sf_pair_at(pairs, i), name)) continue; + ScrStr *value = sf_pair_at(pairs, i + 1); + if (count > 0) total += 2; + else first = value; + total += value->len; + count++; } if (count == 0) return NULL; + if (count == 1) return scr_str_retain(first); char *joined = malloc(total > 0 ? total : 1); if (!joined) sf_oom(); size_t at = 0; size_t emitted = 0; for (size_t i = 0; i + 1 < n; i += 2) { - ScrStr *key = scr_arr_get_ref(h->pairs, (double)i); - ScrStr *value = scr_arr_get_ref(h->pairs, (double)(i + 1)); - if (sf_headers_key_eq(key, name)) { - if (emitted > 0) { - joined[at++] = ','; - joined[at++] = ' '; - } - memcpy(joined + at, value->data, value->len); - at += value->len; - emitted++; + if (!sf_headers_key_eq(sf_pair_at(pairs, i), name)) continue; + ScrStr *value = sf_pair_at(pairs, i + 1); + if (emitted > 0) { + joined[at++] = ','; + joined[at++] = ' '; } - scr_str_release(key); - scr_str_release(value); + memcpy(joined + at, value->data, value->len); + at += value->len; + emitted++; } ScrStr *out = scr_str_new(joined, total); free(joined); @@ -2348,17 +2372,17 @@ static ScrStr *sf_headers_get_value(SfHeaders *h, const ScrStr *name) { static void sf_headers_delete_name(SfHeaders *h, const ScrStr *name) { ScrArr *old = h->pairs; size_t n = (size_t)scr_arr_len(old); + bool present = false; + for (size_t i = 0; i + 1 < n && !present; i += 2) + present = sf_headers_key_eq(sf_pair_at(old, i), name); + if (!present) return; + sf_headers_changed(h); ScrArr *next = scr_arr_new(SCR_ELEM_STR, n); for (size_t i = 0; i + 1 < n; i += 2) { - ScrStr *key = scr_arr_get_ref(old, (double)i); - ScrStr *value = scr_arr_get_ref(old, (double)(i + 1)); - if (sf_headers_key_eq(key, name)) { - scr_str_release(key); - scr_str_release(value); - continue; - } - scr_arr_push_ref(next, key); - scr_arr_push_ref(next, value); + ScrStr *key = sf_pair_at(old, i); + if (sf_headers_key_eq(key, name)) continue; + scr_arr_push_ref(next, scr_str_retain(key)); + scr_arr_push_ref(next, scr_str_retain(sf_pair_at(old, i + 1))); } h->pairs = next; scr_arr_release(old); @@ -2399,15 +2423,13 @@ static bool sf_headers_convert_pair( * Fetch-specific semicolon separator for Cookie. */ static void sf_headers_append_pair( SfHeaders *h, ScrStr *name, ScrStr *value) { + sf_headers_changed(h); if (!(name->len == 10 && memcmp(name->data, "set-cookie", 10) == 0)) { size_t n = (size_t)scr_arr_len(h->pairs); for (size_t i = 0; i + 1 < n; i += 2) { - ScrStr *key = scr_arr_get_ref(h->pairs, (double)i); - bool match = sf_headers_key_eq(key, name); - scr_str_release(key); - if (!match) continue; + if (!sf_headers_key_eq(sf_pair_at(h->pairs, i), name)) continue; - ScrStr *previous = scr_arr_get_ref(h->pairs, (double)(i + 1)); + ScrStr *previous = sf_pair_at(h->pairs, i + 1); const char *separator = name->len == 6 && memcmp(name->data, "cookie", 6) == 0 ? "; " @@ -2420,7 +2442,6 @@ static void sf_headers_append_pair( memcpy(joined_data + previous->len + 2, value->data, value->len); ScrStr *joined = scr_str_new(joined_data, joined_len); free(joined_data); - scr_str_release(previous); scr_str_release(name); scr_str_release(value); scr_arr_set_ref(h->pairs, (double)(i + 1), joined); @@ -2447,7 +2468,7 @@ static ScrStr **sf_headers_sorted_names(SfHeaders *h, size_t *count_out) { if (capacity && !names) sf_oom(); size_t count = 0; for (size_t i = 0; i + 1 < n; i += 2) { - ScrStr *key = scr_arr_get_ref(h->pairs, (double)i); + ScrStr *key = sf_pair_at(h->pairs, i); bool seen = false; for (size_t j = 0; j < count; j++) { if (sf_headers_key_eq(names[j], key)) { @@ -2455,8 +2476,7 @@ static ScrStr **sf_headers_sorted_names(SfHeaders *h, size_t *count_out) { break; } } - if (seen) scr_str_release(key); - else names[count++] = key; + if (!seen) names[count++] = key; } if (count > 1) { qsort(names, count, sizeof *names, &sf_headers_name_cmp); @@ -2469,44 +2489,44 @@ static bool sf_headers_is_set_cookie(const ScrStr *name) { return name->len == 10 && memcmp(name->data, "set-cookie", 10) == 0; } -/* The `index`th entry of Fetch's "sort and combine" list: names in - * byte order, each with its combined value, except that every set-cookie - * value is its own entry (in insertion order), as Node's Headers - * iterator yields them. False past the end. Outputs are +1. */ +/* Fetch's "sort and combine" list as flat name/value pairs: names in byte + * order, each with its combined value, except that every set-cookie value + * is its own entry (in insertion order), as Node's Headers iterator yields + * them. Cached until the next mutation; borrowed. */ +static ScrArr *sf_headers_sorted(SfHeaders *h) { + if (h->sorted) return h->sorted; + size_t count = 0; + ScrStr **names = sf_headers_sorted_names(h, &count); /* borrowed names */ + size_t n = (size_t)scr_arr_len(h->pairs); + ScrArr *sorted = scr_arr_new(SCR_ELEM_STR, n); + for (size_t i = 0; i < count; i++) { + if (!sf_headers_is_set_cookie(names[i])) { + scr_arr_push_ref(sorted, scr_str_retain(names[i])); + scr_arr_push_ref(sorted, sf_headers_get_value(h, names[i])); + continue; + } + for (size_t j = 0; j + 1 < n; j += 2) { + if (!sf_headers_is_set_cookie(sf_pair_at(h->pairs, j))) continue; + scr_arr_push_ref(sorted, scr_str_retain(names[i])); + scr_arr_push_ref(sorted, scr_str_retain(sf_pair_at(h->pairs, j + 1))); + } + } + free(names); + h->sorted = sorted; + return sorted; +} + +/* The `index`th entry of the sort-and-combine list (sf_headers_sorted). + * False past the end. Outputs are +1. */ static bool sf_headers_sorted_entry(SfHeaders *h, size_t index, ScrStr **name_out, ScrStr **value_out) { - size_t count = 0; - ScrStr **names = sf_headers_sorted_names(h, &count); - bool found = false; - size_t at = 0; - for (size_t i = 0; i < count && !found; i++) { - if (!sf_headers_is_set_cookie(names[i])) { - if (at == index) { - *name_out = scr_str_retain(names[i]); - *value_out = sf_headers_get_value(h, names[i]); - found = true; - } - at++; - continue; - } - size_t n = (size_t)scr_arr_len(h->pairs); - for (size_t j = 0; j + 1 < n && !found; j += 2) { - ScrStr *key = scr_arr_get_ref(h->pairs, (double)j); - bool match = sf_headers_is_set_cookie(key); - scr_str_release(key); - if (!match) continue; - if (at == index) { - *name_out = scr_str_retain(names[i]); - *value_out = scr_arr_get_ref(h->pairs, (double)(j + 1)); - found = true; - } - at++; - } - } - for (size_t j = 0; j < count; j++) scr_str_release(names[j]); - free(names); - return found; + ScrArr *sorted = sf_headers_sorted(h); + size_t entries = (size_t)scr_arr_len(sorted) / 2; + if (index >= entries) return false; + *name_out = scr_str_retain(sf_pair_at(sorted, 2 * index)); + *value_out = scr_str_retain(sf_pair_at(sorted, 2 * index + 1)); + return true; } /* One step of a live Headers iterator (entries, keys or values): the @@ -2570,14 +2590,8 @@ static ScrDyn *sf_headers_invoke(void *ptr, ScrDyn *self, ScrDyn *out = scr_dyn_new_arr(); size_t n = (size_t)scr_arr_len(h->pairs); for (size_t i = 0; i + 1 < n; i += 2) { - ScrStr *key = scr_arr_get_ref(h->pairs, (double)i); - ScrStr *value = scr_arr_get_ref(h->pairs, (double)(i + 1)); - if (key->len == 10 && - memcmp(key->data, "set-cookie", 10) == 0) { - scr_dyn_arr_push(out, scr_dyn_new_str(value)); - } - scr_str_release(key); - scr_str_release(value); + if (sf_headers_is_set_cookie(sf_pair_at(h->pairs, i))) + scr_dyn_arr_push(out, scr_dyn_new_str(sf_pair_at(h->pairs, i + 1))); } return out; } @@ -3323,14 +3337,13 @@ static bool sf_pairs_have(ScrArr *pairs, const char *name) { size_t n = (size_t)scr_arr_len(pairs); size_t want = strlen(name); for (size_t i = 0; i + 1 < n; i += 2) { - ScrStr *key = (ScrStr *)scr_arr_get_ref(pairs, (double)i); + ScrStr *key = sf_pair_at(pairs, i); bool same = key->len == want; for (size_t j = 0; j < want && same; j++) { char c = key->data[j]; if (c >= 'A' && c <= 'Z') c = (char)(c - 'A' + 'a'); same = c == name[j]; } - scr_str_release(key); if (same) return true; } return false; @@ -3437,11 +3450,14 @@ static ScrStr *sf_trim_header_value(const ScrStr *value) { (value->data[end - 1] == ' ' || value->data[end - 1] == '\t')) { end--; } + if (start == 0 && end == value->len) return scr_str_retain((ScrStr *)value); return scr_str_new(value->data + start, end - start); } -static bool sf_push_header(ScrArr *pairs, const char *name, size_t name_len, +static bool sf_push_header(ScrArr *pairs, ScrStr *name_str, ScrStr *value, bool request_guard) { + const char *name = name_str->data; + size_t name_len = name_str->len; if (!sf_header_name_ok(name, name_len)) { ScrStr *display = scr_str_new(name, name_len); sf_headers_invalid("append", display, "name"); @@ -3467,7 +3483,7 @@ static bool sf_push_header(ScrArr *pairs, const char *name, size_t name_len, if (!sf_header_name_ci(name, name_len, "set-cookie")) { size_t n = (size_t)scr_arr_len(pairs); for (size_t i = 0; i + 1 < n; i += 2) { - ScrStr *key = scr_arr_get_ref(pairs, (double)i); + ScrStr *key = sf_pair_at(pairs, i); bool match = key->len == name_len; for (size_t j = 0; j < name_len && match; j++) { unsigned char a = (unsigned char)key->data[j]; @@ -3476,11 +3492,9 @@ static bool sf_push_header(ScrArr *pairs, const char *name, size_t name_len, if (b >= 'A' && b <= 'Z') b = (unsigned char)(b - 'A' + 'a'); match = a == b; } - scr_str_release(key); if (!match) continue; - ScrStr *previous = - scr_arr_get_ref(pairs, (double)(i + 1)); + ScrStr *previous = sf_pair_at(pairs, i + 1); const char *separator = sf_header_name_ci(name, name_len, "cookie") ? "; " : ", "; size_t separator_len = 2; @@ -3494,7 +3508,6 @@ static bool sf_push_header(ScrArr *pairs, const char *name, size_t name_len, normalized->data, normalized->len); ScrStr *joined = scr_str_new(joined_data, joined_len); free(joined_data); - scr_str_release(previous); scr_str_release(normalized); if (request_guard && !sf_request_header_ok(name, name_len, joined)) { scr_str_release(joined); @@ -3513,8 +3526,8 @@ static bool sf_push_header(ScrArr *pairs, const char *name, size_t name_len, scr_arr_push_ref( pairs, request_guard - ? scr_str_new(name, name_len) - : sf_header_name_lower(name, name_len)); + ? scr_str_retain(name_str) + : sf_header_name_lower_str(name_str)); scr_arr_push_ref(pairs, normalized); return true; } @@ -3541,10 +3554,13 @@ static void sf_headers_pair_length_error(size_t len) { * leaves header-name/value syntax validation for initialization. Response * construction converts this snapshot before status/statusText and only * validates it after body extraction, matching Undici's two phases. */ -static ScrArr *sf_headers_convert_init(const ScrDyn *headers) { +/* *canonical_out reports that the result copies a canonical Headers + * object's pairs (SfHeaders.canonical). */ +static ScrArr *sf_headers_convert_init(const ScrDyn *headers, bool *canonical_out) { + *canonical_out = false; if (headers && headers->kind == SCR_DYN_TYPED_REF) { ScrDyn *materialized = scr_dyn_typed_ref_materialize(headers); - ScrArr *converted = sf_headers_convert_init(materialized); + ScrArr *converted = sf_headers_convert_init(materialized, canonical_out); scr_dyn_release(materialized); return converted; } @@ -3635,13 +3651,9 @@ static ScrArr *sf_headers_convert_init(const ScrDyn *headers) { headers->v.handle.tag == SCR_DYNH_FETCH_HEADERS) { SfHeaders *source = (SfHeaders *)headers->v.handle.ptr; size_t n = (size_t)scr_arr_len(source->pairs); - for (size_t i = 0; i + 1 < n; i += 2) { - ScrStr *name = scr_arr_get_ref(source->pairs, (double)i); - ScrStr *value = - scr_arr_get_ref(source->pairs, (double)(i + 1)); - scr_arr_push_ref(converted, name); - scr_arr_push_ref(converted, value); - } + for (size_t i = 0; i < n - n % 2; i++) + scr_arr_push_ref(converted, scr_str_retain(sf_pair_at(source->pairs, i))); + *canonical_out = source->canonical; return converted; } /* Functions are WebIDL objects and an ordinary function has no @@ -3656,25 +3668,29 @@ fail: } static bool sf_add_converted_headers( - ScrArr *pairs, ScrArr *converted, bool request_guard) { + ScrArr *pairs, ScrArr *converted, bool request_guard, bool canonical) { size_t n = (size_t)scr_arr_len(converted); + /* Pushing canonical pairs into an empty list one by one through + * sf_push_header would reproduce them exactly. */ + if (canonical && !request_guard && scr_arr_len(pairs) == 0) { + for (size_t i = 0; i < n; i++) + scr_arr_push_ref(pairs, scr_str_retain(sf_pair_at(converted, i))); + return true; + } for (size_t i = 0; i + 1 < n; i += 2) { - ScrStr *name = scr_arr_get_ref(converted, (double)i); - ScrStr *value = scr_arr_get_ref(converted, (double)(i + 1)); - bool ok = sf_push_header( - pairs, name->data, name->len, value, request_guard); - scr_str_release(name); - scr_str_release(value); - if (!ok) return false; + if (!sf_push_header(pairs, sf_pair_at(converted, i), + sf_pair_at(converted, i + 1), request_guard)) + return false; } return true; } static bool sf_add_headers( ScrArr *pairs, const ScrDyn *headers, bool request_guard) { - ScrArr *converted = sf_headers_convert_init(headers); + bool canonical = false; + ScrArr *converted = sf_headers_convert_init(headers, &canonical); if (!converted) return false; - bool ok = sf_add_converted_headers(pairs, converted, request_guard); + bool ok = sf_add_converted_headers(pairs, converted, request_guard, canonical); scr_arr_release(converted); return ok; } @@ -4854,6 +4870,7 @@ static ScrDyn *sf_response_construct(ScrDyn *body, ScrDyn *init, SfJsonSerialize serialize) { ScrArr *header_pairs = NULL; ScrArr *converted_headers = NULL; + bool converted_canonical = false; ScrBytes *body_bytes = NULL; ScrStr *status_text = NULL; SfStream *stream = NULL; @@ -4918,7 +4935,7 @@ static ScrDyn *sf_response_construct(ScrDyn *body, ScrDyn *init, * later response-initialization step, after body extraction. */ ScrDyn *init_headers = sf_response_init_get(init, "headers", 7); if (light_ok) light_form = sf_light_header_form(init_headers, json_body, &light_record); - converted_headers = sf_headers_convert_init(init_headers); + converted_headers = sf_headers_convert_init(init_headers, &converted_canonical); scr_dyn_release(init_headers); if (!converted_headers) goto fail; @@ -5013,7 +5030,7 @@ static ScrDyn *sf_response_construct(ScrDyn *body, ScrDyn *init, header_pairs = scr_arr_new(SCR_ELEM_STR, 8); if (!sf_add_converted_headers( - header_pairs, converted_headers, false)) goto fail; + header_pairs, converted_headers, false, converted_canonical)) goto fail; scr_arr_release(converted_headers); converted_headers = NULL; @@ -5044,6 +5061,7 @@ static ScrDyn *sf_response_construct(ScrDyn *body, ScrDyn *init, response->rc = 1; response->body = stream; response->headers = sf_headers_new(header_pairs, false); + response->headers->canonical = true; response->url = scr_str_new("", 0); response->status_text = status_text; response->status = status; @@ -5159,13 +5177,15 @@ static ScrDyn *sf_request_invoke(void *ptr, ScrDyn *self, const char *method, Sc } ScrDyn *scr_fetch_headers_new(ScrDyn *init) { - ScrArr *converted = sf_headers_convert_init(init); + bool canonical = false; + ScrArr *converted = sf_headers_convert_init(init, &canonical); if (!converted) return NULL; ScrArr *pairs = scr_arr_new(SCR_ELEM_STR, 8); - bool ok = sf_add_converted_headers(pairs, converted, false); + bool ok = sf_add_converted_headers(pairs, converted, false, canonical); scr_arr_release(converted); if (!ok) { scr_arr_release(pairs); return NULL; } SfHeaders *headers = sf_headers_new(pairs, false); + headers->canonical = true; ScrDyn *result = scr_dyn_new_handle(headers, SCR_DYNH_FETCH_HEADERS); sf_headers_release(headers); return result; diff --git a/tests/corpus/fetch-headers-copy-cache.ts b/tests/corpus/fetch-headers-copy-cache.ts new file mode 100644 index 000000000..686ecfc61 --- /dev/null +++ b/tests/corpus/fetch-headers-copy-cache.ts @@ -0,0 +1,85 @@ +// Headers copies and the cached sort-and-combine list: copies are +// independent of their source, reads see every mutation, and live +// iterators re-read the current list after a change. +function dump(label: string, h: Headers): void { + const parts: string[] = []; + h.forEach((value, name) => parts.push(`${name}=${value}`)); + console.log(`${label}: ${parts.join(" | ")}`); +} + +// The corpus lib has no DOM iterable typings for Headers, so iterators +// are reached through an untyped view. +const iter = (h: Headers): any => h; + +const source = new Headers({ "X-B": " two ", "x-a": "one", Accept: "text/html" }); +source.append("accept", "application/json"); +source.append("Set-Cookie", "a=1"); +source.append("set-cookie", "b=2"); +dump("source", source); + +// A Headers copy keeps the combined values and stays separate. +const copy = new Headers(source); +source.set("x-a", "changed"); +copy.append("x-c", "three"); +dump("source after set", source); +dump("copy after append", copy); +console.log(JSON.stringify(copy.get("accept")), JSON.stringify(copy.get("set-cookie"))); +console.log(JSON.stringify(copy.getSetCookie())); + +// Copying into an empty Response and a Request, then appending a repeat. +const response = new Response("body", { headers: copy, status: 201 }); +response.headers.append("accept", "*/*"); +response.headers.append("cookie", "c=3"); +response.headers.append("Cookie", "d=4"); +dump("response", response.headers); +dump("copy unchanged", copy); +const clone = new Response(response.body, response); +dump("response clone", clone.headers); +const request = new Request("http://example.test/", { headers: copy }); +console.log("request x-b", JSON.stringify(request.headers.get("X-B"))); + +// Every read after a mutation sees it: get, has, iteration, spread. +const h = new Headers([["x-z", "z"], ["x-m", "m"]]); +console.log([...iter(h).entries()].map(([n, v]) => `${n}:${v}`).join(",")); +h.delete("x-z"); +console.log([...iter(h).keys()].join(","), h.has("x-z")); +h.delete("x-missing"); +console.log([...iter(h).values()].join(",")); +h.append("x-a", "first"); +console.log([...iter(h).entries()].map(([n, v]) => `${n}:${v}`).join(",")); +h.set("X-M", "again"); +console.log(JSON.stringify(h.get("x-m")), [...iter(h).keys()].join(",")); + +// A live iterator: mutations between steps change what comes next. +const it = iter(h).entries(); +console.log(JSON.stringify(it.next().value)); +h.append("x-0", "before"); +console.log(JSON.stringify(it.next().value)); +h.delete("x-m"); +h.append("x-zz", "after"); +console.log(JSON.stringify(it.next()), JSON.stringify(it.next())); + +// forEach that mutates as it walks. +const walk = new Headers([["b", "2"], ["d", "4"]]); +const seen: string[] = []; +walk.forEach((value, name) => { + seen.push(`${name}=${value}`); + if (name === "b") { + walk.append("c", "3"); + walk.append("a", "1"); + } + if (name === "c") walk.set("d", "four"); +}); +console.log(seen.join(" | ")); + +// Invalid input in a copy source is still rejected (plain object source). +try { + new Response("x", { headers: { "bad name": "v" } }); +} catch (error) { + console.log((error as Error).name, (error as Error).message); +} +try { + new Headers(source).append("x", "a\nb"); +} catch (error) { + console.log((error as Error).name, (error as Error).message); +} diff --git a/tests/harness/corpus-costs/fetch-headers-copy-cache.ts.json b/tests/harness/corpus-costs/fetch-headers-copy-cache.ts.json new file mode 100644 index 000000000..6057503d9 --- /dev/null +++ b/tests/harness/corpus-costs/fetch-headers-copy-cache.ts.json @@ -0,0 +1 @@ +{ "plain": 6400, "san": 9200 } From 7b8a07e0f296d88b2c99b13158a1baa93451bb69 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 07:37:41 -0700 Subject: [PATCH 049/184] Search proven-ASCII strings backwards by byte in lastIndexOf lastIndexOf built the receiver's UTF-16 index map and converted positions both ways even when the receiver is proven ASCII, where byte offsets are UTF-16 indexes (indexOf already has this fast path). A short id such as "bk_00042".lastIndexOf("_") cost about 200 instructions; the Hono fixture's list endpoint does it 80 times per request. Co-Authored-By: Claude Opus 5.5 --- .../corpus/string-last-index-of-ascii.ts.json | 6 ++++++ packages/runtime/src/scr_string.c | 16 ++++++++++++++ tests/corpus/string-last-index-of-ascii.ts | 21 +++++++++++++++++++ .../string-last-index-of-ascii.ts.json | 1 + 4 files changed, 44 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/string-last-index-of-ascii.ts.json create mode 100644 tests/corpus/string-last-index-of-ascii.ts create mode 100644 tests/harness/corpus-costs/string-last-index-of-ascii.ts.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/string-last-index-of-ascii.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/string-last-index-of-ascii.ts.json new file mode 100644 index 000000000..1d3107cd5 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/string-last-index-of-ascii.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/string-last-index-of-ascii.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_string.c b/packages/runtime/src/scr_string.c index f94554642..37b92e48c 100644 --- a/packages/runtime/src/scr_string.c +++ b/packages/runtime/src/scr_string.c @@ -1928,6 +1928,22 @@ static __attribute__((noinline)) double scr_str_unit_search(const ScrStr *s, con /* Search backwards from a clamped UTF-16 position. A low-surrogate position * maps to its scalar's first byte, so that scalar remains searchable. */ double scr_str_last_index_of_from(ScrStr *s, ScrStr *needle, double position) { + /* A proven-ASCII receiver: byte offsets are UTF-16 indexes, and a needle + * with other bytes (including lone surrogate halves) is never found. */ + if (s->ascii) { + size_t start = isnan(position) || position >= (double)s->len ? s->len + : position <= 0 ? 0 : (size_t)trunc(position); + if (needle->len == 0) return (double)start; + if (needle->len > s->len) return -1.0; + size_t max_start = s->len - needle->len; + if (start < max_start) max_start = start; + char first = needle->data[0]; + for (size_t i = max_start + 1; i-- > 0;) { + if (s->data[i] == first && memcmp(s->data + i, needle->data, needle->len) == 0) + return (double)i; + } + return -1.0; + } ScrSidx *e = scr_sidx_for(s); size_t len16 = scr_sidx_len(s, e); size_t start16 = isnan(position) || position >= (double)len16 ? len16 diff --git a/tests/corpus/string-last-index-of-ascii.ts b/tests/corpus/string-last-index-of-ascii.ts new file mode 100644 index 000000000..8dcf405f5 --- /dev/null +++ b/tests/corpus/string-last-index-of-ascii.ts @@ -0,0 +1,21 @@ +// lastIndexOf on ASCII receivers: every position form, empty and long +// needles, needles with non-ASCII text, and receivers built at run time. +const ids = ["bk_00042", "mb_00410", "plain", "_lead", "trail_", "a_b_c_d", ""]; +for (const id of ids) console.log(JSON.stringify(id), id.lastIndexOf("_"), Number(id.slice(id.lastIndexOf("_") + 1))); + +const text = "abcabcabc"; +const positions = [Infinity, -Infinity, NaN, -0, 0, 1, 2.9, 3, 5, 8, 9, 100, -5]; +for (const needle of ["abc", "c", "a", "", "abcabcabcabc", "cab", "x", "é", "\ud83d", "😀"]) { + const row = positions.map((position) => text.lastIndexOf(needle, position)); + console.log(JSON.stringify(needle), text.lastIndexOf(needle), row.join(",")); +} + +// Receivers built at run time (concatenation, repeat, join, slices). +const built = ["x", "y"].join("-").repeat(3) + String(42); +console.log(built, built.lastIndexOf("-"), built.lastIndexOf("x-", 3), built.lastIndexOf("42", 6)); +const slice = built.slice(2, 9); +console.log(slice, slice.lastIndexOf("y"), slice.lastIndexOf("y", 0), slice.lastIndexOf("-y-")); + +// A non-ASCII receiver keeps UTF-16 positions. +const mixed = "é_ü_😀_z"; +console.log(mixed.lastIndexOf("_"), mixed.lastIndexOf("_", 3), mixed.lastIndexOf("😀"), mixed.lastIndexOf("\ude00")); diff --git a/tests/harness/corpus-costs/string-last-index-of-ascii.ts.json b/tests/harness/corpus-costs/string-last-index-of-ascii.ts.json new file mode 100644 index 000000000..a970c2f16 --- /dev/null +++ b/tests/harness/corpus-costs/string-last-index-of-ascii.ts.json @@ -0,0 +1 @@ +{ "plain": 6000, "san": 9000 } From a705a3fd09b72b22ae744900fa90e8c88c45cfe0 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 07:35:51 -0700 Subject: [PATCH 050/184] Reject other classes' capsules inline in class-table brand probes A class table (property bags, normalization, computed reads) tests the receiver's capsule key against every compiled class in turn. A capsule boxed by another class has another key pointer, and each such probe called scr_dyn_typed_ref_is_key to compare the names. The probe now rejects it inline when the key length or its last eight bytes differ, and calls the out-of-line comparison only for a likely match, so capsules boxed with an equal key elsewhere still match. Co-Authored-By: Claude Opus 5.5 --- .../compiler/src/backend/llvm/emitter.test.ts | 21 ++++++ .../compiler/src/backend/llvm/lib-dispatch.ts | 35 ++++++++++ .../corpus/class-brand-probe-keys.ts.json | 6 ++ packages/runtime/src/scr_dyn.h | 4 ++ tests/corpus/class-brand-probe-keys.ts | 68 +++++++++++++++++++ .../class-brand-probe-keys.ts.json | 1 + 6 files changed, 135 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/class-brand-probe-keys.ts.json create mode 100644 tests/corpus/class-brand-probe-keys.ts create mode 100644 tests/harness/corpus-costs/class-brand-probe-keys.ts.json diff --git a/packages/compiler/src/backend/llvm/emitter.test.ts b/packages/compiler/src/backend/llvm/emitter.test.ts index 4793663aa..e6334ab4c 100644 --- a/packages/compiler/src/backend/llvm/emitter.test.ts +++ b/packages/compiler/src/backend/llvm/emitter.test.ts @@ -499,6 +499,27 @@ test("class brand probes compare the program's key constant before the key text" const named = work(probe("dyn.typedRefIs"), dyn); expect(named).toContain("; ->v.typed_ref.type_key"); expect(named).toContain("@scr_dyn_typed_ref_is_key"); + // Another key pointer is rejected on its length and last eight bytes + // ("Example" plus the prefix's ':') before the out-of-line comparison. + expect(named).toContain("; ->v.typed_ref.type_key_len"); + expect(named).toContain("and i32 %t"); + expect(named).toMatch(/icmp eq i32 %t\d+, 14\b/); + const tail = Buffer.from("object:Example").readBigInt64LE(6); + expect(named).toContain(`icmp eq i64 %t`); + expect(named).toContain(`, ${tail}`); + const shortKey: IrExpr = { kind: "strLit", value: "o:E", type: STRING, loc: receiverLoc }; + const short = work( + { + kind: "libCall", + fn: "dyn.typedRefIs", + args: [ref(dyn), shortKey], + type: BOOL, + loc: receiverLoc, + }, + dyn, + ); + expect(short).toMatch(/icmp eq i32 %t\d+, 3\b/); + expect(short).not.toContain("load i64"); // The key-pointer test answers false for capsules boxed elsewhere. const exact = work(probe("dyn.typedRefKeyIs"), dyn); expect(exact).toContain("; ->v.typed_ref.type_key"); diff --git a/packages/compiler/src/backend/llvm/lib-dispatch.ts b/packages/compiler/src/backend/llvm/lib-dispatch.ts index eaf8f59f6..3f127e7e9 100644 --- a/packages/compiler/src/backend/llvm/lib-dispatch.ts +++ b/packages/compiler/src/backend/llvm/lib-dispatch.ts @@ -25,6 +25,9 @@ import { rawBytes } from "./lib-abi.js"; import { emitInlineAtomicsOp } from "./expr-bytes.js"; import { regexLiteralCaptures } from "./regex-matchers.js"; +/** ScrDyn.v.typed_ref.type_key_len: the low 27 bits of its bit-field word. */ +const TYPE_KEY_LEN_MASK = (1 << 27) - 1; + export function emitAssertInspectLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue { const B = host.B; if (e.fn === "assert.shapeStr" || e.fn === "assert.shapeRe") { @@ -281,6 +284,38 @@ export function emitGenericLibCall(host: LlvmEmitterContext, e: LibCallExpr): Ll B.line(`store i1 true, ptr ${slot}`); B.condBr(same, done, compare); B.startBlock(compare); + // A capsule with another key pointer usually belongs to another + // class: reject it on its key length and last eight bytes before + // the out-of-line name comparison, so a class table's chain of + // probes costs a few instructions per class it skips. + B.line(`store i1 false, ptr ${slot}`); + const bytes = Buffer.from(literal, "utf8"); + const lenSlot = B.tmp(), + lenWord = B.tmp(), + len = B.tmp(), + lenEq = B.tmp(); + B.line( + `${lenSlot} = getelementptr inbounds i8, ptr ${value.name}, i64 ${host.abiOffset(88, 52)} ; ->v.typed_ref.type_key_len`, + ); + B.line(`${lenWord} = load i32, ptr ${lenSlot}`); + B.line(`${len} = and i32 ${lenWord}, ${TYPE_KEY_LEN_MASK}`); + B.line(`${lenEq} = icmp eq i32 ${len}, ${bytes.length}`); + const full = B.newLabel("brand.full"); + if (bytes.length >= 8) { + const tail = B.newLabel("brand.tail"); + B.condBr(lenEq, tail, done); + B.startBlock(tail); + const tailPtr = B.tmp(), + word = B.tmp(), + wordEq = B.tmp(); + B.line(`${tailPtr} = getelementptr inbounds i8, ptr ${keyPtr}, i64 ${bytes.length - 8}`); + B.line(`${word} = load i64, ptr ${tailPtr}, align 1`); + B.line(`${wordEq} = icmp eq i64 ${word}, ${bytes.readBigInt64LE(bytes.length - 8)}`); + B.condBr(wordEq, full, done); + } else { + B.condBr(lenEq, full, done); + } + B.startBlock(full); } } if (e.fn === "dyn.typedRefIs") { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/class-brand-probe-keys.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/class-brand-probe-keys.ts.json new file mode 100644 index 000000000..fd60e3ff9 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/class-brand-probe-keys.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/class-brand-probe-keys.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 669f4aedb..6f0c28371 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -540,6 +540,10 @@ _Static_assert(offsetof(ScrDyn, null_proto) == sizeof(size_t) + sizeof(ScrDynKin "dyn-to-record conversions read null_proto at byte 13 (9 on wasm32)"); _Static_assert(offsetof(ScrDyn, v.typed_ref.type_key) == 16 + 3 * sizeof(void *), "typed_ref.type_key offset"); +/* Emitted brand probes read type_key_len as the low 27 bits of the 32-bit + * word after `casts` (native 88, wasm32 52). */ +_Static_assert(offsetof(ScrDyn, v.typed_ref.casts) == 16 + 8 * sizeof(void *), + "typed_ref.casts offset"); /* The region links ride in payload space the capsule variant already * reserves, so dyn nodes keep their size (and allocation size class). */ _Static_assert(sizeof(((ScrDyn *)0)->v.arr) <= sizeof(((ScrDyn *)0)->v.typed_ref) && diff --git a/tests/corpus/class-brand-probe-keys.ts b/tests/corpus/class-brand-probe-keys.ts new file mode 100644 index 000000000..77b5ff138 --- /dev/null +++ b/tests/corpus/class-brand-probe-keys.ts @@ -0,0 +1,68 @@ +// Untyped reads of class instances dispatch on the capsule's class key. +// These classes have keys of equal length, keys that share their last +// eight bytes, and short keys, so every stage of the brand probe decides. +class FooHandlerA { + kind = "foo-a"; + label(): string { + return `FooHandlerA:${this.kind}`; + } +} +class BarHandlerA { + kind = "bar-a"; + label(): string { + return `BarHandlerA:${this.kind}`; + } +} +class FooHandlerB extends FooHandlerA { + override kind = "foo-b"; + extra = 2; +} +class Q { + kind = "q"; + label(): string { + return "Q"; + } +} +class R { + kind = "r"; + label(): string { + return "R"; + } +} +class LongerClassNameWithSharedTail { + kind = "long"; + label(): string { + return "long"; + } +} +class OtherClassNameWithSharedTail { + kind = "other"; + label(): string { + return "other"; + } +} + +const values: any[] = [ + new FooHandlerA(), + new BarHandlerA(), + new FooHandlerB(), + new Q(), + new R(), + new LongerClassNameWithSharedTail(), + new OtherClassNameWithSharedTail(), + { kind: "plain", label: () => "plain" }, +]; + +for (const value of values) { + const extra = value.extra; + console.log(value.kind, value.label(), extra === undefined ? "-" : extra, value.constructor.name); +} + +// Writes through the untyped view reach the right instance. +for (const value of values) value.kind = `${value.kind}!`; +console.log(values.map((value) => value.kind).join(",")); + +// Computed reads go through the same tables. +const key: string = ["kind", "label"][values.length % 2]; +console.log(values.map((value) => typeof value[key]).join(",")); +console.log(values.filter((value) => value instanceof FooHandlerA).length); diff --git a/tests/harness/corpus-costs/class-brand-probe-keys.ts.json b/tests/harness/corpus-costs/class-brand-probe-keys.ts.json new file mode 100644 index 000000000..9f464b312 --- /dev/null +++ b/tests/harness/corpus-costs/class-brand-probe-keys.ts.json @@ -0,0 +1 @@ +{ "plain": 8500, "san": 30000 } From 6ebdec28e22eb8111a5de0d82c44c7d250a33b94 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 10:14:37 -0500 Subject: [PATCH 051/184] Update tsc-ts to 0.0.9 tsc-ts 0.0.9 answers getDiagnosticsOfCheckedFiles, which checks only the listed files of a program and answers their semantic diagnostics and the global diagnostics. The release-age exclusions admit it with its platform packages. --- packages/compiler/package.json | 2 +- pnpm-lock.yaml | 66 +++++++++++++++++----------------- pnpm-workspace.yaml | 16 ++++----- 3 files changed, 42 insertions(+), 42 deletions(-) diff --git a/packages/compiler/package.json b/packages/compiler/package.json index a358ae545..a9e3d9616 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -40,7 +40,7 @@ }, "dependencies": { "@scriptc/runtime": "workspace:*", - "tsc-ts": "0.0.8", + "tsc-ts": "0.0.9", "typescript": "7.0.2", "typescript5": "npm:typescript@5.9.3" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 525994ba1..5fc32620a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -110,8 +110,8 @@ importers: specifier: workspace:* version: link:../runtime tsc-ts: - specifier: 0.0.8 - version: 0.0.8 + specifier: 0.0.9 + version: 0.0.9 typescript: specifier: 7.0.2 version: 7.0.2 @@ -1377,42 +1377,42 @@ packages: '@ts-morph/common@0.11.1': resolution: {integrity: sha512-7hWZS0NRpEsNV8vWJzg7FEz6V8MaLNeJOmwmghqUXTpzk16V1LLZhdo+4QvE/+zv4cVci0OviuJFnqhEfoV3+g==} - '@tsc-ts/darwin-arm64@0.0.8': - resolution: {integrity: sha512-+hB9yWk58lwZHGrVEzqEC4D7MQW7Vd2z7blV3TULIKZPTMwdf8aT7XFAmfP48PoGff0H9yuQKyRXJl9I+sQv9w==} + '@tsc-ts/darwin-arm64@0.0.9': + resolution: {integrity: sha512-aIjSulef3DIEX7F3RRWLXyHWFgjXiVffBkrfE83ay/tpbsQaniu16cFxYGYNaqqVGft/oivecpdTDBbpPlsb8Q==} cpu: [arm64] os: [darwin] - '@tsc-ts/darwin-x64@0.0.8': - resolution: {integrity: sha512-yz4UddicqMXe+T7nV7ftpf155lkiK6gIPNwSB8rH9dlwiV6a17CO/vpSk627QLMqnfGNicB6YwLMk/SJ9moPdw==} + '@tsc-ts/darwin-x64@0.0.9': + resolution: {integrity: sha512-LHE74tzIEDtrEN8ENvCH/ejnnZjT0D2Vaz2HmggcvJudgr0W300CcwGf2Kdfw8MVBBk6JN0HPF9+a5AUUtFZGg==} cpu: [x64] os: [darwin] - '@tsc-ts/linux-arm64-gnu@0.0.8': - resolution: {integrity: sha512-LHqfT9mQ4tusPdWmnanCvC8/YBOF5bHM5h5863a3t2HTgFVdps1Qwb0yzrW5DDbojIhQ0Z+GupxtXVUQcV0Q/Q==} + '@tsc-ts/linux-arm64-gnu@0.0.9': + resolution: {integrity: sha512-dNiQIKApLcKIJPzb3pk2gfGrrgmOWa+q3/qjrx5ia/SzIWz0+lENlVvHiIn9ukqDacDE30FW8xHzby8Kyd+10A==} cpu: [arm64] os: [linux] libc: [glibc] - '@tsc-ts/linux-arm64-musl@0.0.8': - resolution: {integrity: sha512-I0CC3xZMwoQM6OiVsubRHb0VI/RrJpneb33kDQhadYHq3taPos71KwP+CbCXNu88rNsDsyn7D1oCVfJb73g4Mw==} + '@tsc-ts/linux-arm64-musl@0.0.9': + resolution: {integrity: sha512-DolAqHtr1GUyG03Ch8+sJzpX1G+xzazAmj18kABzuOPre/6njrSWnmqJmtXQWvhwLBBEjajIkUJ4V+t5qigsrg==} cpu: [arm64] os: [linux] libc: [musl] - '@tsc-ts/linux-x64-gnu@0.0.8': - resolution: {integrity: sha512-PLERuwUuxT92x0BZAMQubiQP5jhlqoLmswaZWqcvfKAZyBp0TbWiVOmuNnDbJozvBaNUrfdGPb9suiQ+Me7M6A==} + '@tsc-ts/linux-x64-gnu@0.0.9': + resolution: {integrity: sha512-leANVZ4tKz3myBSVogb173ttbAurPOOGjy1WeBlRhkVJ7zOPMiyzJi3cPOAK/4O8RcxXOsCiC2AfKPKDYM8bEw==} cpu: [x64] os: [linux] libc: [glibc] - '@tsc-ts/linux-x64-musl@0.0.8': - resolution: {integrity: sha512-zDl36l1uyWc0mQniOAMZmkVD9TrlFExHlDT0fL8JGD4XLgx+bRe+To38yP0KSBehXBzjDNn8xrEoJ+3RP6mYwQ==} + '@tsc-ts/linux-x64-musl@0.0.9': + resolution: {integrity: sha512-a4FW16QXkrwNHAr4Qo/qDA5w9LxCJNrYbpqiz3TJ1sePTqvW/JTJc14tjx6avhW4mIaQp8AB52m7TqNAuDMm4Q==} cpu: [x64] os: [linux] libc: [musl] - '@tsc-ts/win32-x64-msvc@0.0.8': - resolution: {integrity: sha512-9gO+pFoLiIxITlAE2P6JL8ihZwns7z/2Raw5+38RqLzHmnSsRN47+H4p/4EqQsLW5/TTsAo9uSeSkf3EuRg+XA==} + '@tsc-ts/win32-x64-msvc@0.0.9': + resolution: {integrity: sha512-phThYnmOinlvY4dS4DEl1L9t/mFLiHR1zL5plSUfMB5NizA/jESfgMGRu+HW3uwOpCETSxgsuTNAXO7UuEIHIQ==} cpu: [x64] os: [win32] @@ -2577,8 +2577,8 @@ packages: ts-toolbelt@6.15.5: resolution: {integrity: sha512-FZIXf1ksVyLcfr7M317jbB67XFJhOO1YqdTcuGaq9q5jLUoTikukZ+98TPjKiP2jC5CgmYdWWYs0s2nLSU0/1A==} - tsc-ts@0.0.8: - resolution: {integrity: sha512-EiyDziYOryBg/YEvev/mKCn+hD4M6VFjH7VP9nv2WadDKYaPrdb+a3hP+wYr/UsDWNE7AdL7YGnjZppeXlXy6g==} + tsc-ts@0.0.9: + resolution: {integrity: sha512-7fDfC/o1YoITG6sSXj4ZznijQYg8AfoCagmYvwG1BFLR/hz4Lwnq7jqDwcZfoDuPYTkrs8jHUTAVG1turniJHA==} engines: {node: '>=20'} hasBin: true @@ -3454,25 +3454,25 @@ snapshots: mkdirp: 1.0.4 path-browserify: 1.0.1 - '@tsc-ts/darwin-arm64@0.0.8': + '@tsc-ts/darwin-arm64@0.0.9': optional: true - '@tsc-ts/darwin-x64@0.0.8': + '@tsc-ts/darwin-x64@0.0.9': optional: true - '@tsc-ts/linux-arm64-gnu@0.0.8': + '@tsc-ts/linux-arm64-gnu@0.0.9': optional: true - '@tsc-ts/linux-arm64-musl@0.0.8': + '@tsc-ts/linux-arm64-musl@0.0.9': optional: true - '@tsc-ts/linux-x64-gnu@0.0.8': + '@tsc-ts/linux-x64-gnu@0.0.9': optional: true - '@tsc-ts/linux-x64-musl@0.0.8': + '@tsc-ts/linux-x64-musl@0.0.9': optional: true - '@tsc-ts/win32-x64-msvc@0.0.8': + '@tsc-ts/win32-x64-msvc@0.0.9': optional: true '@tybys/wasm-util@0.10.3': @@ -4813,17 +4813,17 @@ snapshots: ts-toolbelt@6.15.5: {} - tsc-ts@0.0.8: + tsc-ts@0.0.9: dependencies: '@scriptc/threads': 0.4.2 optionalDependencies: - '@tsc-ts/darwin-arm64': 0.0.8 - '@tsc-ts/darwin-x64': 0.0.8 - '@tsc-ts/linux-arm64-gnu': 0.0.8 - '@tsc-ts/linux-arm64-musl': 0.0.8 - '@tsc-ts/linux-x64-gnu': 0.0.8 - '@tsc-ts/linux-x64-musl': 0.0.8 - '@tsc-ts/win32-x64-msvc': 0.0.8 + '@tsc-ts/darwin-arm64': 0.0.9 + '@tsc-ts/darwin-x64': 0.0.9 + '@tsc-ts/linux-arm64-gnu': 0.0.9 + '@tsc-ts/linux-arm64-musl': 0.0.9 + '@tsc-ts/linux-x64-gnu': 0.0.9 + '@tsc-ts/linux-x64-musl': 0.0.9 + '@tsc-ts/win32-x64-msvc': 0.0.9 tslib@2.8.1: optional: true diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index e74d40442..c9f873c4e 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -7,14 +7,14 @@ minimumReleaseAgeExclude: - "effect@4.0.0" - "@effect/platform-node@4.0.0" - "@effect/platform-node-shared@4.0.0" - - "@tsc-ts/darwin-arm64@0.0.8" - - "@tsc-ts/darwin-x64@0.0.8" - - "@tsc-ts/linux-arm64-gnu@0.0.8" - - "@tsc-ts/linux-arm64-musl@0.0.8" - - "@tsc-ts/linux-x64-gnu@0.0.8" - - "@tsc-ts/linux-x64-musl@0.0.8" - - "@tsc-ts/win32-x64-msvc@0.0.8" - - "tsc-ts@0.0.8" + - "@tsc-ts/darwin-arm64@0.0.9" + - "@tsc-ts/darwin-x64@0.0.9" + - "@tsc-ts/linux-arm64-gnu@0.0.9" + - "@tsc-ts/linux-arm64-musl@0.0.9" + - "@tsc-ts/linux-x64-gnu@0.0.9" + - "@tsc-ts/linux-x64-musl@0.0.9" + - "@tsc-ts/win32-x64-msvc@0.0.9" + - "tsc-ts@0.0.9" - "@scriptc/threads@0.4.2" allowBuilds: esbuild: true From 779893eef855b16825dabf486c1970360d8c66c5 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 10:15:37 -0500 Subject: [PATCH 052/184] Address review feedback on int32 arrays across calls --- packages/compiler/src/ir/int32-arrays.ts | 29 +++++++++++++++++-- .../corpus/7580-int32-arrays-across-calls.ts | 20 +++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/packages/compiler/src/ir/int32-arrays.ts b/packages/compiler/src/ir/int32-arrays.ts index e67f51956..353755c05 100644 --- a/packages/compiler/src/ir/int32-arrays.ts +++ b/packages/compiler/src/ir/int32-arrays.ts @@ -222,6 +222,25 @@ function writesLocal(e: IrExpr, id: string): boolean { return !expr(e); } +/** Expression kinds an index can be built from without running any code. */ +const PURE_INDEX_KINDS: ReadonlySet = new Set([ + "varRef", + "numLit", + "boolLit", + "bin", + "unary", + "ternary", + "logical", + "arrayGet", + "arrayState", +]); + +/** Whether evaluating an index runs no code (no call, accessor or + * conversion): plain reads, literals and their operators only. */ +function pureIndex(e: IrExpr): boolean { + return PURE_INDEX_KINDS.has(e.kind) && everyExprChild(e, pureIndex, () => false); +} + /** The parameters and globals a module's functions share as candidates, * still assumed to qualify. */ export interface SharedCandidates { @@ -319,8 +338,14 @@ export function int32ArrayCandidates( case "call": { if (readers.has(e.callee) && e.args.length === 2) { const [array, index] = e.args; - // Inlining evaluates the index before the array read. - if (array!.kind === "varRef" && writesLocal(index!, array!.localId)) + // Inlining evaluates the index before the array read: an index + // that rebinds the array keeps it out, and so does one that can run + // code when the array is a module global (a call can reassign it). + if ( + array!.kind === "varRef" && + (writesLocal(index!, array!.localId) || + (array!.localId.startsWith("%g.") && !pureIndex(index!))) + ) result.delete(array!.localId); receiver(array!); expr(index!); diff --git a/tests/corpus/7580-int32-arrays-across-calls.ts b/tests/corpus/7580-int32-arrays-across-calls.ts index 13df54609..df180b264 100644 --- a/tests/corpus/7580-int32-arrays-across-calls.ts +++ b/tests/corpus/7580-int32-arrays-across-calls.ts @@ -133,6 +133,25 @@ halve(halved); const pick: (values: number[]) => number = first; const viaValue: number[] = [9, 8, 7]; +// An index that calls a function reassigning the global: the read uses the +// array the global held before the index ran, as in Node. +let swapped: number[] = [1, 2, 3]; +function swapIn(): number { + swapped = []; + for (let i = 0; i < 4; i++) swapped.push(i + 10); + return 1; +} +function fillSwapped(n: number): number { + swapped = []; + for (let i = 0; i < n; i++) swapped.push(i & 15); + let s = 0; + for (let i = 0; i < swapped.length; i++) s += swapped[i]!; + return s; +} +function readSwapped(): number | undefined { + return swapped[swapIn()]; +} + console.log(`fnv ${fnv(data)} ${fnv(data)}`); const h = histogram(data, 10); console.log(`hist ${h.length} ${h[0]} ${h[9]} ${total(h)}`); @@ -143,4 +162,5 @@ console.log(`halved ${halved[0]} ${halved[2]} ${halved.length}`); console.log(`value ${pick(viaValue)} ${viaValue.length}`); console.log(`regrow ${regrow(40)} ${reassigned.length} ${reassigned[39]} ${regrow(5)}`); console.log(`span ${spanTotal()}`); +console.log(`swap ${fillSwapped(6)} ${readSwapped()} ${swapped[1]}`); void later().then((v) => console.log(`later ${v}`)); From b6e8e149b54250754d6c8651f45a600e83c7f40d Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 10:25:37 -0500 Subject: [PATCH 053/184] Trim combined header values again when copying after an empty append Appending an empty value to an existing header leaves the separator's space at the end of the combined value ("x, "). Node keeps that in the Headers object but trims it when the value is copied into another Headers, Request or Response ("x,"). The copy fast path copied a canonical object's pairs as they were, so the copy kept the space. Such an append now marks the pairs as no longer canonical, and copies normalize them pair by pair. --- packages/runtime/src/scr_fetch.c | 4 ++++ tests/corpus/fetch-headers-copy-cache.ts | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index 48b60942e..d95807b6c 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -2442,6 +2442,10 @@ static void sf_headers_append_pair( memcpy(joined_data + previous->len + 2, value->data, value->len); ScrStr *joined = scr_str_new(joined_data, joined_len); free(joined_data); + /* Appending an empty value leaves the separator's space at the end of + * the combined value, which a copy trims again: the pairs are no + * longer in their canonical, trimmed form. */ + if (value->len == 0) h->canonical = false; scr_str_release(name); scr_str_release(value); scr_arr_set_ref(h->pairs, (double)(i + 1), joined); diff --git a/tests/corpus/fetch-headers-copy-cache.ts b/tests/corpus/fetch-headers-copy-cache.ts index 686ecfc61..24321d818 100644 --- a/tests/corpus/fetch-headers-copy-cache.ts +++ b/tests/corpus/fetch-headers-copy-cache.ts @@ -83,3 +83,26 @@ try { } catch (error) { console.log((error as Error).name, (error as Error).message); } + +// Appending an empty value leaves the separator's space at the end of the +// combined value; a copy trims it again, as Node's does. A leading +// separator has no whitespace to trim. +const spaced = new Headers(); +spaced.append("a", "x"); +spaced.append("a", ""); +spaced.append("b", ""); +spaced.append("b", "y"); +spaced.append("c", ""); +spaced.append("c", ""); +spaced.append("cookie", "k=v"); +spaced.append("cookie", ""); +const spacedList = (headers: Headers): string => { + const out: string[] = []; + headers.forEach((value, name) => out.push(`${name}=${JSON.stringify(value)}`)); + return out.join(" "); +}; +console.log(spacedList(spaced)); +console.log(spacedList(new Headers(spaced))); +console.log(spacedList(new Headers(new Headers(spaced)))); +console.log(spacedList(new Response("", { headers: spaced }).headers)); +console.log(spacedList(new Request("http://example.test/", { headers: spaced }).headers)); From 11426b4fc238a29743161f3836fa3ad31839ad29 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 10:26:22 -0500 Subject: [PATCH 054/184] Read string-keyed record entries as stack unions under projections A keyed read of a string-indexed record whose values are numbers or booleans (`counts[key]` on a `Record`, typed `number | undefined`) allocated a heap union for every read, also when the only consumer was `??`, `!` or a comparison with undefined. The runtime already returns the entry's value and whether it is present; the box came from wrapping that result. Such a read is now a stack union source: under a projection, the read writes the tag and bits of the declared field or index-signature entry the key names, or the undefined arm for a missing key, into a private stack box, and allocates nothing. This applies when every hit is a number or boolean arm of the result union, or an inline field of the same union, and also to an argument a callee only projects (the helper that `x!` lowers to). Other consumers keep the heap union. dictionary-walks allocates no union instead of 363,359 and runs 14.6% faster; no other runtime benchmark's code changes. Corpus 7390. --- .../compiler/src/backend/llvm/expr-calls.ts | 4 + .../compiler/src/backend/llvm/expr-records.ts | 109 ++++++++++++++++++ .../compiler/src/backend/llvm/receivers.ts | 6 +- .../7390-record-key-read-presence.ts.json | 6 + tests/corpus/7390-record-key-read-presence.ts | 49 ++++++++ .../7390-record-key-read-presence.ts.json | 1 + 6 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7390-record-key-read-presence.ts.json create mode 100644 tests/corpus/7390-record-key-read-presence.ts create mode 100644 tests/harness/corpus-costs/7390-record-key-read-presence.ts.json diff --git a/packages/compiler/src/backend/llvm/expr-calls.ts b/packages/compiler/src/backend/llvm/expr-calls.ts index a4e7cde2c..6ff52619e 100644 --- a/packages/compiler/src/backend/llvm/expr-calls.ts +++ b/packages/compiler/src/backend/llvm/expr-calls.ts @@ -44,6 +44,7 @@ import { LlvmUnsupportedError } from "./unsupported.js"; import type { LlvmEmitterContext, ExprOf, LlValue } from "./expr-context.js"; import { f64Lit, ffiNativeTypeLl, ffiNativeParamLl, ffiNativeReturnLl } from "./common.js"; import { canStackUnion, emitStackUnion } from "./stack-unions.js"; +import { emitStackRecordKeyGet, scalarRecordKeyGet } from "./expr-records.js"; import { emitCallArrayRead } from "./local-array-reads.js"; import { emitScalarOptional, isScalarOptionalSource } from "./local-union-storage.js"; import { emitStackMapRead } from "./map-read-lifetimes.js"; @@ -166,6 +167,9 @@ export function emitCallExpr( return { name: host.emitOwnedNullableStack(a).box, type: a.type }; // An inline union field read copies its tag and bits. if (host.inlineFieldGet(a)) return host.emitReadReceiver(a); + // A scalar keyed record read writes its tag and bits. + if (a.kind === "recordKeyGet" && scalarRecordKeyGet(host, a)) + return { name: emitStackRecordKeyGet(host, a), type: a.type }; } if (inputs[index]) return host.emitReadReceiver(a); } diff --git a/packages/compiler/src/backend/llvm/expr-records.ts b/packages/compiler/src/backend/llvm/expr-records.ts index bca728e9d..59ab625e5 100644 --- a/packages/compiler/src/backend/llvm/expr-records.ts +++ b/packages/compiler/src/backend/llvm/expr-records.ts @@ -7,6 +7,7 @@ import type { LlvmEmitterContext, LlValue } from "./expr-context.js"; import { f64Lit, llvmCommentText } from "./common.js"; import { borrowableInputs, emitBorrowedInputs, emitBorrowedInput } from "./borrowed-inputs.js"; import { regexLiteralTemplate } from "./expr-primitives.js"; +import { isScalarUnionDef, loadInlineUnion, presentBits } from "./inline-unions.js"; import { preservesRegexInputs } from "./reference-effects.js"; /** Widen one tagged read. The source is borrowed unless the overflow map @@ -365,6 +366,114 @@ export function emitRecordKeyGet( return host.own({ name: t, type: e.type }); } +/** A keyed read whose result is a scalar union (`r[k]` on a + * `Record`, at `number | undefined`) and whose every hit + * is a number or boolean arm of it, or an inline field of the same union. + * Projections (`??`, `!`, tag tests) read it as a private stack box, so + * the read allocates nothing. */ +export function scalarRecordKeyGet(host: LlvmEmitterContext, e: IrExpr): boolean { + if (e.kind !== "recordKeyGet" || e.missNaN === true || e.type.kind !== "union") return false; + const resultType = e.type; + const def = host.unionsById.get(resultType.unionId); + if (!isScalarUnionDef(def) || undefinedArmTag(resultType, host.unionsById) < 0) return false; + const isArm = (t: IrType): boolean => + (t.kind === "f64" || t.kind === "bool") && def!.arms.some((a) => typeEquals(a, t)); + const shape = host.recordShape(e.shapeId); + if (e.overflowOnly !== true) { + for (const f of shape.fields) { + const inline = host.recordFieldInline(e.shapeId, f.name); + if (!isArm(f.type) && !(inline && typeEquals(f.type, resultType))) return false; + } + } + return shape.indexValue === undefined || isArm(shape.indexValue); +} + +/** A scalarRecordKeyGet read as a private stack box: the tag and bits of + * the declared field or overflow entry the key names, or the undefined arm + * for a miss. The box must never reach an RC entry point. */ +export function emitStackRecordKeyGet( + host: LlvmEmitterContext, + e: IrExpr & { kind: "recordKeyGet" }, +): string { + const B = host.B; + if (e.type.kind !== "union") + throw new InternalCompilerError("llvm emitter bug: stack keyed read of a non-union"); + const resultType = e.type; + const def = host.unionsById.get(resultType.unionId)!; + const armTag = (t: IrType): number => def.arms.findIndex((a) => typeEquals(a, t)); + const obj = host.emitExpr(e.obj); + const key = host.emitExpr(e.key); + const box = B.slot(); + B.entryAllocas.push(`${box} = alloca %ScrUnion`); + const tagPtr = B.tmp(), + bitsPtr = B.tmp(); + B.line(`store ${host.sizeType} -1, ptr ${box}`); + B.line(`${tagPtr} = getelementptr inbounds %ScrUnion, ptr ${box}, i64 0, i32 1`); + B.line(`${bitsPtr} = getelementptr inbounds %ScrUnion, ptr ${box}, i64 0, i32 5`); + const join = B.newLabel("rks.j"); + const storeScalar = (t: IrType, value: string): void => { + const bits = B.tmp(); + if (t.kind === "f64") B.line(`${bits} = bitcast double ${value} to i64`); + else B.line(`${bits} = zext i1 ${value} to i64`); + B.line(`store i32 ${armTag(t)}, ptr ${tagPtr}`); + B.line(`store i64 ${bits}, ptr ${bitsPtr}`); + }; + const shape = host.recordShape(e.shapeId); + for (const f of e.overflowOnly === true ? [] : shape.fields) { + const hit = B.tmp(); + B.line( + `${hit} = call zeroext i1 ${host.literalEquals(f.name)}(ptr ${key.name}) ; ${llvmCommentText(f.name)}`, + ); + const lh = B.newLabel("rks.h"); + const ln = B.newLabel("rks.n"); + B.condBr(hit, lh, ln); + B.startBlock(lh); + const { ptr, type, inline } = host.recordFieldPtr(obj.name, e.shapeId, f.name); + if (inline) { + const { tag, bits } = loadInlineUnion(B, ptr); + const present = presentBits(B, tag, bits, undefinedArmTag(type, host.unionsById)); + B.line(`store i32 ${tag}, ptr ${tagPtr}`); + B.line(`store i64 ${present}, ptr ${bitsPtr}`); + } else { + storeScalar(type, host.loadRecordField(ptr, type, false)); + } + B.br(join); + B.startBlock(ln); + } + const iv = shape.indexValue; + if (iv) { + const ovf = host.recordOvfPtr(obj.name, e.shapeId); + const outTy = iv.kind === "f64" ? "double" : "i8"; + const outSlot = B.slot(); + B.entryAllocas.push(`${outSlot} = alloca ${outTy}`); + B.line(`store ${outTy} ${iv.kind === "f64" ? f64Lit(0) : "0"}, ptr ${outSlot}`); + const getter = `scr_map_get_str_${iv.kind === "f64" ? "f64" : "bool"}`; + host.declare(`declare zeroext i1 @${getter}(ptr, ptr, ptr)`); + const found = B.tmp(); + B.line(`${found} = call zeroext i1 @${getter}(ptr ${ovf}, ptr ${key.name}, ptr ${outSlot})`); + const lh = B.newLabel("rks.h"); + const ln = B.newLabel("rks.n"); + B.condBr(found, lh, ln); + B.startBlock(lh); + const raw = B.tmp(); + B.line(`${raw} = load ${outTy}, ptr ${outSlot}`); + let value = raw; + if (iv.kind === "bool") { + value = B.tmp(); + B.line(`${value} = trunc i8 ${raw} to i1`); + } + storeScalar(iv, value); + B.br(join); + B.startBlock(ln); + } + // A miss is the undefined arm. + B.line(`store i32 ${undefinedArmTag(resultType, host.unionsById)}, ptr ${tagPtr}`); + B.line(`store i64 0, ptr ${bitsPtr}`); + B.br(join); + B.startBlock(join); + return box; +} + export function keyedRecordReadInto( host: LlvmEmitterContext, slot: string, diff --git a/packages/compiler/src/backend/llvm/receivers.ts b/packages/compiler/src/backend/llvm/receivers.ts index 5e1077ee5..2fe9dd66b 100644 --- a/packages/compiler/src/backend/llvm/receivers.ts +++ b/packages/compiler/src/backend/llvm/receivers.ts @@ -18,6 +18,7 @@ import { canStackUnion, emitStackUnion } from "./stack-unions.js"; import { emitDowncastGuard, emitUnionNarrowGuard } from "./narrow-guards.js"; import { emitScalarOptional, isScalarOptionalSource } from "./local-union-storage.js"; import { borrowableInputs } from "./borrowed-inputs.js"; +import { emitStackRecordKeyGet, scalarRecordKeyGet } from "./expr-records.js"; import { type WalkBorrowHost, analyzeWalks } from "./walk-borrows.js"; /** Read a receiver whose value is consumed immediately by a field or tag @@ -95,6 +96,8 @@ export function emitReadReceiver(emitter: LlEmitter, e: IrExpr): LlValue { const narrowed = narrow ? emitCheckedNarrowReceiver(emitter, narrow) : null; if (narrowed) return narrowed; if (canStackUnion(e, emitter.boxedUnionsById)) return emitStackUnion(emitter, e).value; + if (e.kind === "recordKeyGet" && scalarRecordKeyGet(emitter, e)) + return { name: emitStackRecordKeyGet(emitter, e), type: e.type }; if (e.kind === "unionNarrow") { const union = emitter.emitUnionProjection(e.value); emitUnionNarrowGuard(emitter, union.name, e.unionId, e.tag); @@ -410,6 +413,7 @@ export function isStackUnionSource(emitter: LlEmitter, e: IrExpr): boolean { emitter.optionalArrayReads.get(e) !== null || isScalarOptionalSource(e, emitter.unionSources) || emitter.scalarResultCall(e) || - emitter.inlineFieldGet(e) + emitter.inlineFieldGet(e) || + scalarRecordKeyGet(emitter, e) ); } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7390-record-key-read-presence.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7390-record-key-read-presence.ts.json new file mode 100644 index 000000000..74bd7e30d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7390-record-key-read-presence.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7390-record-key-read-presence.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7390-record-key-read-presence.ts b/tests/corpus/7390-record-key-read-presence.ts new file mode 100644 index 000000000..76bb355ca --- /dev/null +++ b/tests/corpus/7390-record-key-read-presence.ts @@ -0,0 +1,49 @@ +// Keyed reads of string-indexed records whose values are numbers or +// booleans, consumed by `??`, `!`, comparisons with undefined and typeof: +// declared fields, an optional declared field, index-signature entries and +// missing keys. A stored NaN is a value, so `??` keeps it; -0 survives the +// read; a missing key is undefined. +interface Scores { + base: number; + bonus?: number; + [name: string]: number | undefined; +} + +const scores: Scores = { base: 10 }; +scores["alice"] = 3; +scores["nan"] = NaN; +scores["negzero"] = -0; + +const keys = ["base", "bonus", "alice", "nan", "negzero", "missing"]; +for (const key of keys) { + const read = scores[key]; + console.log( + key, + scores[key] ?? "default", + scores[key] === undefined, + typeof scores[key], + Object.is(scores[key] ?? 1, -0), + (scores[key] ?? 0) + 1, + read, + ); +} +scores.bonus = 5; +console.log("bonus", scores["bonus"] ?? "default", scores["bonus"]! * 2); + +const counts: Record = {}; +for (const word of "a b a c b a d".split(" ")) counts[word] = (counts[word] ?? 0) + 1; +let total = 0; +for (const word in counts) total += counts[word]!; +console.log(JSON.stringify(counts), total, counts["zzz"] ?? -1); + +const flags: Record = { on: true, off: false }; +for (const key of ["on", "off", "unset"]) { + console.log(key, flags[key] ?? "unset", flags[key] === undefined, typeof flags[key]); +} +let seen = 0; +for (let i = 0; i < 1000; i++) { + const key = "k" + (i % 7); + flags[key] = !(flags[key] ?? false); + if (flags[key]!) seen++; +} +console.log(seen, Object.keys(flags).length); diff --git a/tests/harness/corpus-costs/7390-record-key-read-presence.ts.json b/tests/harness/corpus-costs/7390-record-key-read-presence.ts.json new file mode 100644 index 000000000..3f7384f20 --- /dev/null +++ b/tests/harness/corpus-costs/7390-record-key-read-presence.ts.json @@ -0,0 +1 @@ +{ "plain": 3500, "san": 6000 } From 1f7677d246d92feefcd257456d2c3e4b3a26dde7 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 08:23:28 -0700 Subject: [PATCH 055/184] Answer untyped property reads of plain values without the class tables Every untyped property read or write in a program with dynamic-tier classes calls a generated helper (%dyn.class.property.N). For a value that is not a class instance, the helper still called the normalization, the instance-bag and the static-bag tables, each of which returns the value itself, then compared the bag with the value and finally read the key. When every class branch of a helper tests a capsule key or a class key, its fallthrough is still the access it was created with, and every table that fallthrough calls has its kind exit, a value that is neither a class capsule nor a function now gets the same access on itself at the top of the helper. Helpers for `constructor` and `toString`, which answer primitives and errors specially, keep their full body. zod 3, z.array(Order).parse over 500 orders: 235.8M -> 199.8M instructions per round (-15.3%). Co-Authored-By: Claude Opus 5.5 --- .../lowering/class-dynamic-dispatch.ts | 110 ++++++++++++++++- ...ped-property-reads-mixed-receivers.js.json | 6 + ...-untyped-property-reads-mixed-receivers.js | 111 ++++++++++++++++++ ...ped-property-reads-mixed-receivers.js.json | 1 + 4 files changed, 227 insertions(+), 1 deletion(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7792-untyped-property-reads-mixed-receivers.js.json create mode 100644 tests/corpus/7792-untyped-property-reads-mixed-receivers.js create mode 100644 tests/harness/corpus-costs/7792-untyped-property-reads-mixed-receivers.js.json diff --git a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts index ab3ebb34e..abf03811e 100644 --- a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts +++ b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts @@ -282,6 +282,11 @@ interface PropertyDispatch { fn: IrFunction; classes: Set; reflect?: boolean; + /** The helper's fallthrough statements as created, and the same access + * applied to the receiver itself: what the fallthrough reduces to for a + * value that is neither a class capsule nor a function, whose bag is the + * value. */ + plain?: { tail: readonly IrStmt[]; access: IrStmt[] }; } export function classInstanceOf( @@ -2724,7 +2729,28 @@ export class ClassDynamicDispatch { loc, }); } - dispatch = { name, write: !!write, fn: helper, classes: new Set() }; + const receiverAccess: IrStmt[] = read + ? [{ kind: "return", value: { ...read, value: varRef("p.0", DYN, loc) }, loc }] + : [ + { + kind: "exprStmt", + expr: { + ...write!, + args: [varRef("p.0", DYN, loc), write!.args[1]!, varRef("p.1", DYN, loc)], + }, + loc, + }, + { kind: "return", value: null, loc }, + ]; + dispatch = { + name, + write: !!write, + fn: helper, + classes: new Set(), + ...(name !== "constructor" && name !== "toString" + ? { plain: { tail: [...helper.body], access: receiverAccess } } + : {}), + }; this.properties.set(key, dispatch); this.generated.add(helper); lowerer.liftedFns.push(helper); @@ -4896,6 +4922,7 @@ export class ClassDynamicDispatch { ); } if (lowerer) this.guardClassTables(lowerer); + if (lowerer) this.guardPropertyHelpers(lowerer); } /** Each tracked class's bag initializer over a typed receiver (`this`), @@ -4985,6 +5012,87 @@ export class ClassDynamicDispatch { } } + /** Fast exits for the generated property helpers: when every class + * branch tests a capsule key or a class key and the fallthrough is still + * the bag access the helper was created with, a value of another kind + * (a plain object, array or primitive) normalizes to itself, has itself + * as its bag, and so gets the plain access directly, without the + * normalization, bag and static-bag calls. */ + private guardPropertyHelpers(lowerer: Lowerer): void { + // Every class table the fallthrough or normalization calls must answer + // a plain value with the value itself: its kind exit is in place. + const exits = (name: string): boolean => { + const helper = lowerer.liftedFnNamed(name); + const first = helper?.body[0]; + const param = helper?.params[0]?.localId; + return ( + param !== undefined && + first?.kind === "if" && + first.cond.kind === "dynTest" && + first.cond.negated === true && + first.then.length === 1 && + first.then[0]!.kind === "return" && + first.then[0]!.value !== null && + returnsParam(first.then[0]!.value, param) + ); + }; + const tables = new Set([ + "%dyn.class.properties", + READ_PROPERTIES, + READ_PROPERTIES_OR_EMPTY, + "%dyn.class.normalize", + "%dyn.class.staticProperties", + ]); + const callsOnlyGuardedTables = (statements: IrStmt[]): boolean => + everyStmtList(statements, { + stmt: () => true, + expr: (expression) => + expression.kind !== "call" || (tables.has(expression.callee) && exits(expression.callee)), + }); + for (const dispatch of this.properties.values()) { + const plain = dispatch.plain; + if (!plain) continue; + const fn = dispatch.fn; + const param = fn.params[0]?.localId; + if (param === undefined) continue; + const head = fn.body.length - plain.tail.length; + if (head < 0 || plain.tail.some((stmt, i) => fn.body[head + i] !== stmt)) continue; + let ok = true; + for (const stmt of fn.body.slice(0, head)) { + if ( + stmt.kind === "assign" && + stmt.localId === param && + stmt.value.kind === "call" && + stmt.value.callee === "%dyn.class.normalize" + ) + continue; + if (stmt.kind === "if" && stmt.else_ === null && branchKind(stmt.cond, param) !== null) + continue; + ok = false; + break; + } + // Class branches never run for a plain value; the rest always does. + if (!ok || !callsOnlyGuardedTables(fn.body.filter((stmt) => stmt.kind !== "if"))) continue; + const loc = fn.loc; + const not = (test: "classCapsule" | "funcKind"): IrExpr => ({ + kind: "dynTest", + test, + negated: true, + value: varRef(param, DYN, loc), + type: BOOL, + loc, + }); + fn.body.unshift({ + kind: "if", + cond: not("classCapsule"), + then: [{ kind: "if", cond: not("funcKind"), then: plain.access, else_: null, loc }], + else_: null, + loc, + }); + delete dispatch.plain; + } + } + private normalizedDispatches(): IrFunction[] { return [ ...(this.propertyBag ? [this.propertyBag] : []), diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7792-untyped-property-reads-mixed-receivers.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7792-untyped-property-reads-mixed-receivers.js.json new file mode 100644 index 000000000..1b28b3fbe --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7792-untyped-property-reads-mixed-receivers.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7792-untyped-property-reads-mixed-receivers.js" + ], + "diags": [] +} diff --git a/tests/corpus/7792-untyped-property-reads-mixed-receivers.js b/tests/corpus/7792-untyped-property-reads-mixed-receivers.js new file mode 100644 index 000000000..6a2785541 --- /dev/null +++ b/tests/corpus/7792-untyped-property-reads-mixed-receivers.js @@ -0,0 +1,111 @@ +"use strict"; +// One untyped property read site meets class instances (fixed and tracked +// layouts, getters, own properties added later), plain objects, arrays, +// strings, functions, class constructors with statics, null prototypes and +// optional chains. Each receiver kind answers as in Node. + +class Status { + constructor() { + this.value = "valid"; + } +} + +class Lazy { + constructor(key) { + this.key = key; + } + get value() { + return `lazy:${this.key}`; + } + get length() { + return 7; + } +} + +class Tracked { + constructor(list) { + list.push(this); + this.value = list.length; + } + static value = "static"; +} + +function plainFunction() {} +plainFunction.value = "fn"; + +/** @param {any} target */ +function readValue(target) { + return target.value; +} +/** @param {any} target */ +function readLength(target) { + return target.length; +} +/** @param {any} target */ +function readOptional(target) { + return target?.value; +} +/** @param {any} target @param {unknown} value */ +function writeValue(target, value) { + target.value = value; +} + +const list = []; +const added = new Status(); +writeValue(added, "dirty"); +/** @type {any} */ +const extra = new Lazy("x"); +extra.other = 1; +const nullProto = Object.create(null); +nullProto.value = "bare"; +const inherited = Object.create({ value: "from prototype" }); +const receivers = [ + new Status(), + added, + new Lazy("k"), + extra, + new Tracked(list), + { value: 1, length: 2 }, + { other: true }, + [1, 2, 3], + "text", + 42, + true, + plainFunction, + Tracked, + nullProto, + inherited, +]; +for (const receiver of receivers) { + console.log(typeof receiver, JSON.stringify(readValue(receiver)), JSON.stringify(readLength(receiver)), JSON.stringify(readOptional(receiver))); +} +console.log(readOptional(null), readOptional(undefined)); +for (const bad of [null, undefined]) { + try { + readValue(bad); + } catch (e) { + console.log(e instanceof TypeError, e.message); + } +} + +const plain = { value: 0 }; +const arr = []; +writeValue(plain, 5); +writeValue(arr, "on array"); +writeValue(inherited, "own now"); +writeValue(plainFunction, "fn2"); +const status = new Status(); +writeValue(status, "aborted"); +console.log(plain.value, arr.value, Object.keys(inherited), plainFunction.value, status.value, JSON.stringify(status)); +try { + writeValue("text", 1); +} catch (e) { + console.log(e instanceof TypeError, e.message); +} +const frozen = Object.freeze({ value: "frozen" }); +try { + writeValue(frozen, 2); +} catch (e) { + console.log(e instanceof TypeError); +} +console.log(frozen.value); diff --git a/tests/harness/corpus-costs/7792-untyped-property-reads-mixed-receivers.js.json b/tests/harness/corpus-costs/7792-untyped-property-reads-mixed-receivers.js.json new file mode 100644 index 000000000..d4cd0dbbb --- /dev/null +++ b/tests/harness/corpus-costs/7792-untyped-property-reads-mixed-receivers.js.json @@ -0,0 +1 @@ +{ "plain": 4500, "san": 5000 } From 14678cf4ebc81df929b8009d2c4da6cb53a738e9 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 10:29:35 -0500 Subject: [PATCH 056/184] Keep arrays a present-undefined store may reach out of the dense fast copy A present undefined that the fast copy stores leaves its slot out of the hole-free prefix the entry test read. The matcher left out only the array named by the store, so a store through another name for the same array, such as `const b = a` followed by `(b as (number | undefined)[])[i] = undefined`, kept `a` covered, and the fast copy read the slot as a number. Such a store now leaves out every covered array it may reach: its own, and every other array unless that array or the store's binding is a local array no other binding holds. A store into a field reaches every array except those local ones. --- .../src/backend/llvm/dense-array-loops.test.ts | 16 +++++++++++----- .../src/backend/llvm/dense-array-loops.ts | 15 ++++++++++++++- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/packages/compiler/src/backend/llvm/dense-array-loops.test.ts b/packages/compiler/src/backend/llvm/dense-array-loops.test.ts index 4a81a9bfe..a88c38575 100644 --- a/packages/compiler/src/backend/llvm/dense-array-loops.test.ts +++ b/packages/compiler/src/backend/llvm/dense-array-loops.test.ts @@ -280,10 +280,16 @@ test("a present-undefined store is covered only where it stores a covered read's // A read of an array the entry test does not cover may miss in the fast // copy, and its undefined store would run there. expect(matchStores(guarded("b"), ["a", "c"])).toBeNull(); - // A present undefined stored on its own leaves its array out. - const alone = matchStores( - [{ kind: "exprStmt", expr: read("b"), loc }, undefinedStore], - ["a", "b", "c"], + // A present undefined stored on its own leaves out its array and every + // array the other name may hold. + const alone: IrStmt[] = [{ kind: "exprStmt", expr: read("b"), loc }, undefinedStore]; + expect(matchStores(alone, ["a", "b", "c"])).toBeNull(); + const separate = matchDenseArrayLoop( + loop(alone), + shape, + withV, + new Set(["c"]), + new Set(["a", "b", "c"]), ); - expect([...alone!.arrays.keys()]).toEqual(["b"]); + expect([...separate!.arrays.keys()]).toEqual(["b"]); }); diff --git a/packages/compiler/src/backend/llvm/dense-array-loops.ts b/packages/compiler/src/backend/llvm/dense-array-loops.ts index 135d5eaf2..384c31b0c 100644 --- a/packages/compiler/src/backend/llvm/dense-array-loops.ts +++ b/packages/compiler/src/backend/llvm/dense-array-loops.ts @@ -273,6 +273,9 @@ export function matchDenseArrayLoop( const missing = missingElementStores(loop.body); // The element reads behind the guarded present-undefined stores. const guards: ElementRead[] = []; + // The arrays of the other present-undefined stores. The fast copy runs + // them, and the slot each one writes leaves the hole-free prefix. + const undefinedStores: IrExpr[] = []; const expr = (e: IrExpr): boolean => { if (++nodes > MAX_LOOP_NODES) return false; if (e.kind === "assignExpr" || e.kind === "incDec") disturbed.add(e.localId); @@ -302,7 +305,7 @@ export function matchDenseArrayLoop( // there, so the store never runs in it. const read = missing.get(s); if (read === undefined || s.index.kind !== "varRef" || !isCounter(s.index.localId)) { - if (s.arr.kind === "varRef") disturbed.add(s.arr.localId); + undefinedStores.push(s.arr); } else { access(s.arr, s.index); guards.push(read); @@ -322,6 +325,15 @@ export function matchDenseArrayLoop( ) ) return null; + // A store through another name, such as `const b = a` or a field, lands + // in `a` unless one of the two bindings is a local array no other binding + // holds. + const mayStoreUndefined = (id: string): boolean => + undefinedStores.some((arr) => + arr.kind === "varRef" + ? arr.localId === id || (!unaliased.has(id) && !unaliased.has(arr.localId)) + : !unaliased.has(id), + ); const arrays = new Map(); for (const [id, ref] of candidates) { const local = locals.get(id); @@ -331,6 +343,7 @@ export function matchDenseArrayLoop( local.boxed || local.tdz || disturbed.has(id) || + mayStoreUndefined(id) || (id !== length?.localId && !initialized.has(id)) ) continue; From d3c8155c8595cc394159d2dabad9c904afc3a3b4 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 10:32:15 -0500 Subject: [PATCH 057/184] Reuse the bundled declaration files' diagnostics across builds --- .../src/frontend/ts7/lib-diagnostics.test.ts | 221 ++++++++++++ .../src/frontend/ts7/lib-diagnostics.ts | 324 ++++++++++++++++++ .../compiler/src/frontend/ts7/program-host.ts | 35 +- packages/compiler/src/frontend/ts7/session.ts | 30 ++ packages/compiler/src/index.ts | 31 ++ packages/compiler/src/native/cli.ts | 17 +- packages/compiler/test/source-output.test.ts | 33 ++ 7 files changed, 688 insertions(+), 3 deletions(-) create mode 100644 packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts create mode 100644 packages/compiler/src/frontend/ts7/lib-diagnostics.ts diff --git a/packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts b/packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts new file mode 100644 index 000000000..f041a6cb5 --- /dev/null +++ b/packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts @@ -0,0 +1,221 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, afterEach, expect, test } from "vitest"; +import type { SourceFile } from "./ast-types.js"; +import { + reusedLibDiagnostics, + setLibDiagnosticsStore, + type LibDiagnosticsStore, +} from "./lib-diagnostics.js"; +import type { Ts7DiagnosticData } from "./session-schema.generated.js"; +import type { Ts7CheckedFileDiagnosticData, Ts7SessionProgram } from "./session.js"; +import { closeSourceParser, parseSourceFile } from "./source-parser-node.js"; + +const dir = mkdtempSync(join(tmpdir(), "scriptc-lib-diagnostics-")); +const lib = join(dir, "lib.es5.d.ts"); +const fallback = join(dir, "scriptc-node-fallback.d.ts"); +const libText = "declare var NaN: number;\ninterface Array { length: number }\n"; +const fallbackText = "export {};\ndeclare global {\n var process: { argv: string[] };\n}\n"; +writeFileSync(lib, libText); +writeFileSync(fallback, fallbackText); +const main = join(dir, "main.ts"); +const util = join(dir, "\u{1F600}.ts"); +const other = join(dir, "�.ts"); + +afterEach(() => setLibDiagnosticsStore(null)); +afterAll(() => { + closeSourceParser(); + rmSync(dir, { recursive: true, force: true }); +}); + +function memoryStore(): LibDiagnosticsStore & { entries: Map } { + const entries = new Map(); + return { + entries, + read: (key) => entries.get(key) ?? null, + write: (key, text) => void entries.set(key, text), + }; +} + +function diagnostic(fileName: string | undefined, pos: number, code = 2322): Ts7DiagnosticData { + return { fileName, pos, end: pos + 1, code, category: 1, text: `at ${pos}` }; +} + +/** A session program whose full-check and checked-file answers are given; it + * records which requests it was asked. */ +function sessionProgram(answers: { + semantic: Map; + global?: Ts7DiagnosticData[]; + checked?: boolean; +}): Ts7SessionProgram & { asked: string[] } { + const asked: string[] = []; + const all = [...answers.semantic.values()].flat(); + return { + asked, + getCompilerOptions: () => ({ strict: true }), + diagnosticData: (method: string, file?: string) => { + asked.push(file === undefined ? method : `${method} ${file}`); + if (method === "getGlobalDiagnostics") return answers.global ?? []; + return file === undefined ? all : (answers.semantic.get(file) ?? []); + }, + checkedFileDiagnosticData: (files: readonly string[]): Ts7CheckedFileDiagnosticData | null => { + asked.push(`checked ${files.join(",")}`); + if (answers.checked === false) return null; + return { files: files.map((file) => answers.semantic.get(file) ?? []), global: [] }; + }, + } as unknown as Ts7SessionProgram & { asked: string[] }; +} + +/** The program's files, parsed: the supplied two and the given others. */ +function files(texts: Record): (name: string) => SourceFile | undefined { + const parsed = new Map([ + [lib, parseSourceFile(lib, libText, "ts")], + [fallback, parseSourceFile(fallback, fallbackText, "ts")], + ]); + for (const [name, text] of Object.entries(texts)) + parsed.set(name, parseSourceFile(name, text, "ts")); + return (name) => parsed.get(name); +} + +const names = [lib, fallback, main]; +const supplied = (name: string): boolean => name === lib || name === fallback; +const moduleFile = files({ [main]: "export const n: number = 'x';\n" }); + +test("without a store the full check answers", () => { + const program = sessionProgram({ semantic: new Map() }); + expect(reusedLibDiagnostics(program, names, supplied, moduleFile)).toBeNull(); + expect(program.asked).toEqual([]); +}); + +test("a stored answer stands in for checking the supplied files, merged by file name", () => { + const store = memoryStore(); + setLibDiagnosticsStore(store, "checker"); + const semantic = new Map([ + [lib, [diagnostic(lib, 1)]], + [fallback, []], + [main, [diagnostic(main, 5), diagnostic(main, 9)]], + ]); + const first = sessionProgram({ semantic }); + const full = reusedLibDiagnostics(first, names, supplied, moduleFile); + // The first build runs the full check and stores the supplied files' answers. + expect(first.asked).toEqual([ + "getGlobalDiagnostics", + "getSemanticDiagnostics", + "checked ", + `getSemanticDiagnostics ${lib}`, + `getSemanticDiagnostics ${fallback}`, + ]); + expect(store.entries.size).toBe(1); + const second = sessionProgram({ semantic }); + const reused = reusedLibDiagnostics(second, names, supplied, moduleFile); + expect(second.asked).toEqual([`checked ${main}`]); + // The files' diagnostics in the order of their names (lib.es5.d.ts, main.ts), as the full answer. + expect(reused).toEqual({ + semantic: [diagnostic(lib, 1), diagnostic(main, 5), diagnostic(main, 9)], + global: [], + }); + expect(full?.semantic).toEqual(reused?.semantic); +}); + +test("files merge in code point order, which differs from UTF-16 order", () => { + const store = memoryStore(); + setLibDiagnosticsStore(store, "checker"); + const all = [lib, fallback, util, other]; + const sources = files({ [util]: "export {};\n", [other]: "export {};\n" }); + const semantic = new Map([ + [util, [diagnostic(util, 1)]], + [other, [diagnostic(other, 2)]], + ]); + reusedLibDiagnostics(sessionProgram({ semantic }), all, supplied, sources); + const reused = reusedLibDiagnostics(sessionProgram({ semantic }), all, supplied, sources); + // U+FFFD sorts before U+1F600 by code point (its surrogates sort after it by UTF-16 unit). + expect(reused?.semantic).toEqual([diagnostic(other, 2), diagnostic(util, 1)]); +}); + +test("a full check with global diagnostics is not stored", () => { + const store = memoryStore(); + setLibDiagnosticsStore(store, "checker"); + const program = sessionProgram({ + semantic: new Map([[main, [diagnostic(main, 1)]]]), + global: [diagnostic(undefined, 0, 2318)], + }); + expect(reusedLibDiagnostics(program, names, supplied, moduleFile)?.global).toEqual([ + diagnostic(undefined, 0, 2318), + ]); + expect(store.entries.size).toBe(0); +}); + +test("scripts reuse the answer unless they declare a global the supplied files declare", () => { + const store = memoryStore(); + setLibDiagnosticsStore(store, "checker"); + const semantic = new Map([[main, []]]); + // A script declaring only new names: it stores the answer, and a later one reuses it. + const fresh = files({ + [main]: "const total = 1;\nfunction run() {}\nif (total) { var later = 2; }\n", + }); + reusedLibDiagnostics(sessionProgram({ semantic }), names, supplied, fresh); + expect(store.entries.size).toBe(1); + const again = sessionProgram({ semantic }); + expect(reusedLibDiagnostics(again, names, supplied, fresh)).not.toBeNull(); + expect(again.asked).toEqual([`checked ${main}`]); + // Scripts declaring a supplied global, at the top level, in a block, in a + // `declare global` of a supplied module, or an ambient module: the full check. + for (const text of [ + "interface Array { extra: T }\n", + "if (true) { var NaN = 1; }\n", + "declare var process: unknown;\n", + 'declare module "fs" {}\n', + ]) { + const program = sessionProgram({ semantic }); + const result = reusedLibDiagnostics(program, names, supplied, files({ [main]: text })); + expect(program.asked[0], text).not.toBe(`checked ${main}`); + if (result !== null) expect(program.asked[0]).toBe("getGlobalDiagnostics"); + } + expect(store.entries.size).toBe(1); +}); + +test("a program whose script declares a supplied global does not store its answer", () => { + const store = memoryStore(); + setLibDiagnosticsStore(store, "checker"); + const program = sessionProgram({ semantic: new Map([[main, []]]) }); + reusedLibDiagnostics(program, names, supplied, files({ [main]: "declare var NaN: number;\n" })); + expect(store.entries.size).toBe(0); +}); + +test("modules that can declare something global, and servers without the method, ask as before", () => { + const store = memoryStore(); + setLibDiagnosticsStore(store, "checker"); + const semantic = new Map([[main, []]]); + reusedLibDiagnostics(sessionProgram({ semantic }), names, supplied, moduleFile); + expect(store.entries.size).toBe(1); + for (const text of [ + "export {};\ndeclare global { var extra: number }\n", + 'export {};\ndeclare module "./other" { const extra: number }\n', + "export as namespace Lib;\nexport const n = 1;\n", + ]) { + const program = sessionProgram({ semantic }); + expect(reusedLibDiagnostics(program, names, supplied, files({ [main]: text }))).toBeNull(); + expect(program.asked).toEqual([]); + } + const old = sessionProgram({ semantic, checked: false }); + expect(reusedLibDiagnostics(old, names, supplied, moduleFile)).toBeNull(); + // Nor does a server without the method store an answer it could not use. + const fresh = memoryStore(); + setLibDiagnosticsStore(fresh, "checker"); + reusedLibDiagnostics(sessionProgram({ semantic, checked: false }), names, supplied, moduleFile); + expect(fresh.entries.size).toBe(0); +}); + +test("a supplied file that changes is a different entry", () => { + const store = memoryStore(); + setLibDiagnosticsStore(store, "checker"); + const semantic = new Map([[main, []]]); + reusedLibDiagnostics(sessionProgram({ semantic }), names, supplied, moduleFile); + writeFileSync(fallback, fallbackText + "// edited\n"); + const program = sessionProgram({ semantic }); + reusedLibDiagnostics(program, names, supplied, moduleFile); + expect(program.asked[0]).toBe("getGlobalDiagnostics"); + expect(store.entries.size).toBe(2); + writeFileSync(fallback, fallbackText); +}); diff --git a/packages/compiler/src/frontend/ts7/lib-diagnostics.ts b/packages/compiler/src/frontend/ts7/lib-diagnostics.ts new file mode 100644 index 000000000..827b337bd --- /dev/null +++ b/packages/compiler/src/frontend/ts7/lib-diagnostics.ts @@ -0,0 +1,324 @@ +/* The semantic diagnostics of the declaration files scriptc supplies (the + * TypeScript libs and scriptc's own declaration files), reused across + * builds. Checking them is most of the full check of a small program. When + * no other file of a program can change what checking those files reports + * (every one is a module without global or module augmentations, or a + * script declaring no global name the supplied files declare), the answer + * depends only on the checker, the compiler options and the files + * themselves, so a stored answer for the same three stands in for checking + * them again: the server checks only the program's other files + * (getDiagnosticsOfCheckedFiles), and the semantic diagnostics merge in the + * order the full check sorts them, by file name. An answer is stored only + * when the full check reports no global diagnostics: the global diagnostics + * of checking the other files are then the full check's. */ +import { createHash } from "node:crypto"; +import { statSync } from "node:fs"; +import type { Node, SourceFile } from "./ast-types.js"; +import type { Ts7DiagnosticData } from "./session-schema.generated.js"; +import type { Ts7SessionProgram } from "./session.js"; +import { NodeFlags, SyntaxKind } from "./enums.js"; +import { walkPreorder } from "./ast.js"; + +/** Key-value storage for reused diagnostics (best effort: failures miss). */ +export interface LibDiagnosticsStore { + read(key: string): string | null; + write(key: string, text: string): void; +} + +const SCHEMA = "scriptc.lib-diagnostics.v1"; + +let configured: { store: LibDiagnosticsStore; checker: string } | null = null; + +/** Has later builds of this process reuse the diagnostics of the supplied + * declaration files through `store`, for the checker identified by + * `checker` (its executable's identity), or stop reusing them (null). */ +export function setLibDiagnosticsStore(store: LibDiagnosticsStore | null, checker = ""): void { + configured = store === null ? null : { store, checker }; +} + +/** A checker executable's identity for the stored diagnostics' key: its + * path and file identity, or the path alone when it cannot be read. */ +export function checkerExecutableIdentity(path: string): string { + try { + const info = statSync(path); + return JSON.stringify([path, info.size, info.mtimeMs, info.ino]); + } catch { + return JSON.stringify([path]); + } +} + +/** The program's semantic and global diagnostics as the full check reports + * them, using stored diagnostics for the supplied declaration files; null + * when they cannot be reused for this program (then the caller asks for + * them as before). `supplied` tells whether a program file is one of the + * supplied declaration files. */ +export function reusedLibDiagnostics( + program: Ts7SessionProgram, + fileNames: readonly string[], + supplied: (fileName: string) => boolean, + sourceFile: (fileName: string) => SourceFile | undefined, +): { semantic: Ts7DiagnosticData[]; global: Ts7DiagnosticData[] } | null { + if (configured === null || process.env["SCRIPTC_LIB_DIAGNOSTICS"] === "0") return null; + const { store, checker } = configured; + const libs: string[] = []; + const own: string[] = []; + for (const name of fileNames) (supplied(name) ? libs : own).push(name); + if (libs.length === 0) return null; + // The global names each of the program's scripts declares; a module that + // declares nothing global contributes none. + const scripts: Set[] = []; + for (const name of own) { + const file = sourceFile(name); + if (file === undefined) return null; + if (file.externalModuleIndicator !== undefined) { + if (!declaresNothingGlobal(file)) return null; + continue; + } + const names = isJsFileName(name) ? null : scriptGlobalNames(file); + if (names === null) return null; + scripts.push(names); + } + // A script's globals cannot change what the supplied files report unless + // one of them is also a global the supplied files declare. + const collides = (globals: ReadonlySet): boolean => + scripts.some((names) => [...names].some((name) => globals.has(name))); + // The options without the configuration file's path: it names where the + // project is, which checking the supplied files does not read (the files a + // project's location brings in are part of the key). + const options = { ...program.getCompilerOptions(), configFilePath: undefined }; + const key = entryKey(checker, options, libs); + if (key === null) return null; + const stored = readEntry(store, key, libs); + if (stored !== null && !collides(stored.globals)) { + const checked = program.checkedFileDiagnosticData(own); + if (checked === null) return null; + const byFile = new Map(stored.files); + own.forEach((name, index) => byFile.set(name, checked.files[index] ?? [])); + return { semantic: inFileOrder(byFile), global: checked.global }; + } + // The full check, as before. Its answers for the supplied files are stored + // when it reports no global diagnostics and the program's scripts declare + // none of the supplied files' globals (so nothing of the program reached + // what those files report). + const global = program.diagnosticData("getGlobalDiagnostics"); + const semantic = program.diagnosticData("getSemanticDiagnostics"); + // A server that cannot check only some files could never use the answer. + if (global.length === 0 && stored === null && program.checkedFileDiagnosticData([]) !== null) { + const globals = suppliedGlobalNames(libs, sourceFile); + if (globals !== null && !collides(globals)) { + const files: Record = {}; + for (const name of libs) files[name] = program.diagnosticData("getSemanticDiagnostics", name); + try { + store.write(key, JSON.stringify({ schema: SCHEMA, globals: [...globals], files })); + } catch { + // Best effort: a failed write is a later miss. + } + } + } + return { semantic, global }; +} + +/** Whether a file cannot add to or change the global declarations the + * supplied files declare: a module with no `declare global`, no module + * augmentation and no UMD `export as namespace` declaration. */ +function declaresNothingGlobal(file: SourceFile): boolean { + if (file.externalModuleIndicator === undefined) return false; + if (file.moduleAugmentations.length > 0) return false; + for (const statement of file.statements.nodes) { + if (statement.kind === SyntaxKind.NamespaceExportDeclaration) return false; + // `declare global` (a module declaration named by the identifier `global`). + if ( + statement.kind === SyntaxKind.ModuleDeclaration && + statement.name?.kind === SyntaxKind.Identifier && + statement.name.text === "global" + ) + return false; + } + return true; +} + +function isJsFileName(name: string): boolean { + return /\.[cm]?jsx?$/.test(name); +} + +/** The names a script file declares in the global scope: its top-level + * declarations, and the `var` and function declarations of its top-level + * blocks; null when it declares an ambient module (`declare module "x"`), + * which can merge with one the supplied files declare. Over-approximates: + * a name more only makes the stored answer apply less often. */ +function scriptGlobalNames(file: SourceFile): Set | null { + const names = new Set(); + let ambientModule = false; + walkPreorder(file, (node) => { + if (node === file) return undefined; + if (node.kind === SyntaxKind.ModuleDeclaration && node.name?.kind === SyntaxKind.StringLiteral) + ambientModule = true; + const topLevel = node.parent === file; + if (topLevel || node.kind === SyntaxKind.FunctionDeclaration) addDeclaredNames(node, names); + else if ( + node.kind === SyntaxKind.VariableDeclarationList && + (node.flags & NodeFlags.BlockScoped) === 0 + ) + for (const declaration of node.declarations?.nodes ?? []) + addBindingNames(declaration.name, names); + // Functions, classes and namespaces have scopes of their own. + return GLOBAL_SCOPE_BOUNDARIES.has(node.kind) ? "skip" : undefined; + }); + return ambientModule ? null : names; +} + +/** The global names the supplied files declare: the declarations of their + * scripts (as scriptGlobalNames finds them, with ambient modules as their + * quoted names) and those of the `declare global` blocks of their modules; + * null when one of them is not available. */ +function suppliedGlobalNames( + libs: readonly string[], + sourceFile: (fileName: string) => SourceFile | undefined, +): Set | null { + const names = new Set(); + for (const name of libs) { + const file = sourceFile(name); + if (file === undefined) return null; + if (file.externalModuleIndicator === undefined) { + walkPreorder(file, (node) => { + if (node === file) return undefined; + if ( + node.kind === SyntaxKind.ModuleDeclaration && + node.name?.kind === SyntaxKind.StringLiteral + ) + names.add(JSON.stringify(node.name.text ?? "")); + if (node.parent === file || node.kind === SyntaxKind.FunctionDeclaration) + addDeclaredNames(node, names); + else if ( + node.kind === SyntaxKind.VariableDeclarationList && + (node.flags & NodeFlags.BlockScoped) === 0 + ) + for (const declaration of node.declarations?.nodes ?? []) + addBindingNames(declaration.name, names); + return GLOBAL_SCOPE_BOUNDARIES.has(node.kind) ? "skip" : undefined; + }); + continue; + } + for (const statement of file.statements.nodes) { + if ( + statement.kind !== SyntaxKind.ModuleDeclaration || + statement.name?.kind !== SyntaxKind.Identifier || + statement.name.text !== "global" + ) + continue; + for (const inner of statement.body?.statements?.nodes ?? []) addDeclaredNames(inner, names); + } + } + return names; +} + +const GLOBAL_SCOPE_BOUNDARIES = new Set([ + SyntaxKind.FunctionDeclaration, + SyntaxKind.FunctionExpression, + SyntaxKind.ArrowFunction, + SyntaxKind.MethodDeclaration, + SyntaxKind.Constructor, + SyntaxKind.GetAccessor, + SyntaxKind.SetAccessor, + SyntaxKind.ClassDeclaration, + SyntaxKind.ClassExpression, + SyntaxKind.ModuleDeclaration, +]); + +/** The names a statement declares in the scope it is in. */ +function addDeclaredNames(node: Node, names: Set): void { + switch (node.kind) { + case SyntaxKind.VariableStatement: + for (const declaration of node.declarationList?.declarations?.nodes ?? []) + addBindingNames(declaration.name, names); + return; + case SyntaxKind.FunctionDeclaration: + case SyntaxKind.ClassDeclaration: + case SyntaxKind.InterfaceDeclaration: + case SyntaxKind.TypeAliasDeclaration: + case SyntaxKind.EnumDeclaration: + case SyntaxKind.ModuleDeclaration: + case SyntaxKind.ImportEqualsDeclaration: + if (node.name?.kind === SyntaxKind.Identifier) names.add(node.name.text ?? ""); + return; + } +} + +function addBindingNames(name: Node | undefined, names: Set): void { + if (name === undefined) return; + if (name.kind === SyntaxKind.Identifier) { + names.add(name.text ?? ""); + return; + } + // Object and array binding patterns. + for (const element of name.elements?.nodes ?? []) addBindingNames(element.name, names); +} + +/** The stored diagnostics' key: the checker, the compiler options and the + * supplied files in program order with each one's file identity; null when + * one of them cannot be read. */ +function entryKey(checker: string, options: unknown, libs: readonly string[]): string | null { + const files: unknown[] = []; + for (const name of libs) { + try { + const info = statSync(name); + files.push([name, info.size, info.mtimeMs, info.ino]); + } catch { + return null; + } + } + return createHash("sha256") + .update(JSON.stringify({ schema: SCHEMA, checker, options, files })) + .digest("hex"); +} + +function readEntry( + store: LibDiagnosticsStore, + key: string, + libs: readonly string[], +): { files: Map; globals: Set } | null { + let text: string | null; + try { + text = store.read(key); + } catch { + return null; + } + if (text === null) return null; + let entry: { schema?: unknown; globals?: unknown; files?: Record }; + try { + entry = JSON.parse(text) as typeof entry; + } catch { + return null; + } + if (entry.schema !== SCHEMA || entry.files === undefined || !Array.isArray(entry.globals)) + return null; + const files = new Map(); + for (const name of libs) { + const list = entry.files[name]; + if (!Array.isArray(list)) return null; + files.set(name, list); + } + return { files, globals: new Set(entry.globals.filter((name) => typeof name === "string")) }; +} + +/** Each file's diagnostics (already in the checker's order) in the order the + * checker sorts files: by file name, compared by code point. */ +function inFileOrder(byFile: ReadonlyMap): Ts7DiagnosticData[] { + const names = [...byFile.keys()].sort(compareCodePoints); + const out: Ts7DiagnosticData[] = []; + for (const name of names) for (const d of byFile.get(name)!) out.push(d); + return out; +} + +function compareCodePoints(a: string, b: string): number { + let i = 0; + let j = 0; + while (i < a.length && j < b.length) { + const x = a.codePointAt(i)!; + const y = b.codePointAt(j)!; + if (x !== y) return x < y ? -1 : 1; + i += x > 0xffff ? 2 : 1; + j += y > 0xffff ? 2 : 1; + } + return i < a.length ? 1 : j < b.length ? -1 : 0; +} diff --git a/packages/compiler/src/frontend/ts7/program-host.ts b/packages/compiler/src/frontend/ts7/program-host.ts index 639086e94..ec143e401 100644 --- a/packages/compiler/src/frontend/ts7/program-host.ts +++ b/packages/compiler/src/frontend/ts7/program-host.ts @@ -46,6 +46,10 @@ import { trackedRealpath, } from "../input-tracker.js"; import { packageShimDeclarationText } from "../package-shims.js"; +import { ambientDtsPath, fallbackDtsPath, overridesDtsPath } from "../dts-paths.js"; +import { reusedLibDiagnostics } from "./lib-diagnostics.js"; +import { ts7Diagnostic } from "./diagnostic.js"; +import type { Ts7DiagnosticData } from "./session-schema.generated.js"; /** The compiler options our createProgram accepts: TS7's CompilerOptions * shape (numeric enums for target/module/moduleResolution — the enums module @@ -393,6 +397,22 @@ export class Ts7Program { return this.project.program.getSourceFileMetadata(resolve(fileName)); } + /** The semantic and global diagnostics with the stored diagnostics of the + * declaration files scriptc supplies (lib-diagnostics.ts), or null. */ + reusedLibDiagnostics(): { semantic: Ts7DiagnosticData[]; global: Ts7DiagnosticData[] } | null { + this.snapshot.ensureActive(); + const program = this.project.program; + const declarations = new Set([ambientDtsPath(), overridesDtsPath(), fallbackDtsPath()]); + return reusedLibDiagnostics( + program, + this.getSourceFileNames(), + (name) => + declarations.has(tsgoPath(name)) || + program.getSourceFileMetadataByPath(name)?.isDefaultLibrary === true, + (name) => this.getSourceFile(name), + ); + } + isSourceFileDefaultLibrary(file: SourceFile): boolean { return this.project.program.isSourceFileDefaultLibrary(file); } @@ -423,9 +443,20 @@ export class Ts7Program { * package dropped the aggregate). Order mirrors 5.9.3: config, options/ * program, global, syntactic, semantic. */ export function getPreEmitDiagnostics(program: Ts7Program): readonly Diagnostic[] { + const config = program.getConfigFileParsingDiagnostics(); + const options = program.getProgramDiagnostics(); + const reused = program.reusedLibDiagnostics(); + if (reused !== null) + return [ + ...config, + ...options, + ...reused.global.map(ts7Diagnostic), + ...program.getSyntacticDiagnostics(), + ...reused.semantic.map(ts7Diagnostic), + ]; return [ - ...program.getConfigFileParsingDiagnostics(), - ...program.getProgramDiagnostics(), + ...config, + ...options, ...program.getGlobalDiagnostics(), ...program.getSyntacticDiagnostics(), ...program.getSemanticDiagnostics(), diff --git a/packages/compiler/src/frontend/ts7/session.ts b/packages/compiler/src/frontend/ts7/session.ts index d3f2ae9fe..082bb79a6 100644 --- a/packages/compiler/src/frontend/ts7/session.ts +++ b/packages/compiler/src/frontend/ts7/session.ts @@ -55,6 +55,13 @@ interface ProgramQuery { snapshot?: number; project?: string; file?: SemanticDocument; + files?: string[]; +} + +/** The answer of getDiagnosticsOfCheckedFiles, as the server sends it. */ +export interface Ts7CheckedFileDiagnosticData { + files: Ts7DiagnosticData[][]; + global: Ts7DiagnosticData[]; } /** Platform-independent session over an already connected transport. The @@ -344,6 +351,29 @@ export class Ts7SessionProgram { result.push(ts7Diagnostic(data)); return result; } + /** The diagnostics of `method` for `file`, or for the whole program, as + * the server sends them. */ + diagnosticData(method: string, file?: SemanticDocument): Ts7DiagnosticData[] { + return this.session.request(method, this.query(file)) ?? []; + } + /** The semantic diagnostics of each of `files` and the global diagnostics, + * after the server checks only those files (getDiagnosticsOfCheckedFiles), + * or null when the server does not answer that method. */ + checkedFileDiagnosticData(files: readonly string[]): Ts7CheckedFileDiagnosticData | null { + try { + return this.session.request("getDiagnosticsOfCheckedFiles", { + ...this.query(), + files: [...files], + }); + } catch (error) { + if ( + error instanceof Error && + error.message.includes('unknown API method "getDiagnosticsOfCheckedFiles"') + ) + return null; + throw error; + } + } getSyntacticDiagnostics(file?: SemanticDocument): Ts7Diagnostic[] { return this.diagnostics("getSyntacticDiagnostics", file); } diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 4f7b93cfd..4b988ab8f 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -9,6 +9,7 @@ import { } from "./library/prepare.js"; import { analyzeWithFrontend } from "./frontend/analysis.js"; import { checkerCacheIdentity } from "./frontend/ts7/checker-backend.js"; +import { setLibDiagnosticsStore } from "./frontend/ts7/lib-diagnostics.js"; import { analyzeWithVerdictCache, fileVerdictStore } from "./coverage/verdict-cache.js"; import { coverageCompilerIdentity } from "./coverage/verdict-startup.js"; import { commentOnlyReplay } from "./coverage/verdict-semantic.js"; @@ -450,6 +451,7 @@ export async function compile( ); return withCompileWarnings(result, warnings); } finally { + setLibDiagnosticsStore(null); frontendProcess?.release(); } } @@ -1161,10 +1163,36 @@ async function prepareExecutableInputInChild( * `requestPath` as `compileTracked` would in-process, and writes what it prepared, without the LLVM * text, to `resultPath`. */ +/** Has the build reuse the diagnostics of the declaration files scriptc + * supplies (lib-diagnostics.ts) through the build cache at `root`, when the + * build uses one (executables; source outputs create no cache) and the + * checker is tsc-ts. */ +function configureLibDiagnostics(root: string | null): void { + let checker: string | undefined; + try { + checker = root === null ? undefined : checkerCacheIdentity(); + } catch { + // A missing checker fails where the build starts it. + checker = undefined; + } + setLibDiagnosticsStore( + root === null || checker === undefined + ? null + : fileVerdictStore(join(root, "lib-diagnostics-v1")), + checker ?? "", + ); +} + export async function runFrontendProcess(requestPath: string, resultPath: string): Promise { const request = JSON.parse(await readFile(requestPath, "utf8")) as FrontendProcessRequest; clearCompileSessionCaches(); setProvenanceSources(request.provenance); + // The compile that started this process prepared the cache root. + configureLibDiagnostics( + (request.opts.outputKind ?? "exe") === "exe" && provenanceSources() === null + ? await prepareBuildCacheRoot(buildCacheRoot()) + : null, + ); const frontendInputs = new FrontendInputTracker(); const warnings: CompileWarningSink = { warnings: [], sourceTexts: null }; let result: FrontendProcessResult = { kind: "failed" }; @@ -1212,6 +1240,8 @@ export async function runFrontendProcess(requestPath: string, resultPath: string } } catch { result = { kind: "failed" }; + } finally { + setLibDiagnosticsStore(null); } await writeFile(resultPath, JSON.stringify(result), { mode: 0o600 }); } @@ -1449,6 +1479,7 @@ async function compileTracked( outputKind === "exe" && provenanceSources() === null ? await prepareBuildCacheRoot(buildCacheRoot()) : null; + configureLibDiagnostics(cacheRoot); let earlyCacheOptions: EarlyExecutableCacheOptions | null = null; if (outputKind === "exe") { const helperObjectRoute = diff --git a/packages/compiler/src/native/cli.ts b/packages/compiler/src/native/cli.ts index 21047b929..44d5eb9f0 100644 --- a/packages/compiler/src/native/cli.ts +++ b/packages/compiler/src/native/cli.ts @@ -16,7 +16,11 @@ import { createNativeTs7Api } from "../frontend/ts7/native-api.js"; import { resolveProvenanceSourcesWithParser } from "../frontend/provenance-core.js"; import { setCompilerReleaseVersion } from "../library/sidecar.js"; import { NativeCompiler } from "./compiler.js"; -import { openNativeCache } from "./cache.js"; +import { NATIVE_STRING_BYTES, openNativeCache } from "./cache.js"; +import { + checkerExecutableIdentity, + setLibDiagnosticsStore, +} from "../frontend/ts7/lib-diagnostics.js"; import { loadNativeToolchain } from "./toolchain.js"; import { buildSanitizedRuntime } from "./sanitizer.js"; import { runCompilerTask } from "./task.js"; @@ -53,6 +57,17 @@ async function main(): Promise { if (toolchain.declarationsRoot !== undefined) setDeclarationRoot(toolchain.declarationsRoot); setCompilerReleaseVersion(toolchain.compilerVersion); cache = openNativeCache((path, harden) => compilerNativePrivateDirectory(path, harden)); + if (cache !== null) { + const store = cache; + setLibDiagnosticsStore( + { + read: (key) => + store.read("lib-diagnostics", key, NATIVE_STRING_BYTES)?.toString("utf8") ?? null, + write: (key, text) => store.write("lib-diagnostics", key, text), + }, + checkerExecutableIdentity(toolchain.ts7Executable), + ); + } compiler = new NativeCompiler(toolchain, cache); return compiler; }; diff --git a/packages/compiler/test/source-output.test.ts b/packages/compiler/test/source-output.test.ts index 8b8e3883c..ce411f102 100644 --- a/packages/compiler/test/source-output.test.ts +++ b/packages/compiler/test/source-output.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, expect, test } from "vitest"; import { compile, deserializeModule, validateModule } from "../src/index.js"; +import { setLibDiagnosticsStore } from "../src/frontend/ts7/lib-diagnostics.js"; const dirs: string[] = []; afterEach(async () => { @@ -188,6 +189,38 @@ test("source outputs ignore invalid external compiler selection and create no ex } }); +test("source outputs neither read nor create declaration diagnostics a previous build stored", async () => { + const { dir, entry, outDir } = await fixture(); + const oldCache = process.env["SCRIPTC_CACHE_DIR"]; + const cache = join(dir, "must-not-exist"); + process.env["SCRIPTC_CACHE_DIR"] = cache; + // A store an earlier build of this process left configured. + const reads: string[] = []; + setLibDiagnosticsStore( + { read: (key) => (reads.push(key), null), write: () => {} }, + "earlier-checker", + ); + try { + for (const [kind, name] of [ + ["ir", "main.ir.json"], + ["llvm", "main.ll"], + ] as const) { + const result = await compile(entry, { + outDir, + outPath: join(outDir, name), + outputKind: kind, + }); + expect(result.ok).toBe(true); + } + expect(reads).toEqual([]); + await expect(readdir(cache)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + setLibDiagnosticsStore(null); + if (oldCache === undefined) delete process.env["SCRIPTC_CACHE_DIR"]; + else process.env["SCRIPTC_CACHE_DIR"] = oldCache; + } +}); + test.each(["wasm64-wasi", "totally-invalid"])( "source output rejects unsupported target %s without writing an artifact", async (target) => { From 998a385bde23da4b06051f16b851d20f20674d6d Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 10:45:26 -0500 Subject: [PATCH 058/184] Mark constructed headers canonical only when every value is trimmed The Headers and Response constructors combine duplicate entries (a: x, a: "") into one value with the separator, which can end in its space ("x, "). They marked the pairs canonical anyway, so a copy took them unchanged where Node trims the copied value ("x,"). The constructors now mark the pairs canonical only when every value is already trimmed; otherwise copies normalize them pair by pair. --- packages/runtime/src/scr_fetch.c | 19 +++++++++++++++++-- tests/corpus/fetch-headers-copy-cache.ts | 17 +++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index d95807b6c..d6471040c 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -3443,6 +3443,21 @@ static bool sf_request_header_ok( !sf_header_name_ci(name, name_len, "expect"); } +/* Whether every value in `pairs` is already trimmed (sf_trim_header_value + * would keep it): the form a copy can take unchanged (SfHeaders.canonical). + * Combining duplicate names with an empty value leaves the separator's + * space at the end of the combined value, which a copy trims again. */ +static bool sf_header_values_trimmed(ScrArr *pairs) { + size_t n = (size_t)scr_arr_len(pairs); + for (size_t i = 1; i < n; i += 2) { + const ScrStr *value = sf_pair_at(pairs, i); + if (value->len == 0) continue; + char first = value->data[0], last = value->data[value->len - 1]; + if (first == ' ' || first == '\t' || last == ' ' || last == '\t') return false; + } + return true; +} + static ScrStr *sf_trim_header_value(const ScrStr *value) { size_t start = 0; size_t end = value->len; @@ -5065,7 +5080,7 @@ static ScrDyn *sf_response_construct(ScrDyn *body, ScrDyn *init, response->rc = 1; response->body = stream; response->headers = sf_headers_new(header_pairs, false); - response->headers->canonical = true; + response->headers->canonical = sf_header_values_trimmed(header_pairs); response->url = scr_str_new("", 0); response->status_text = status_text; response->status = status; @@ -5189,7 +5204,7 @@ ScrDyn *scr_fetch_headers_new(ScrDyn *init) { scr_arr_release(converted); if (!ok) { scr_arr_release(pairs); return NULL; } SfHeaders *headers = sf_headers_new(pairs, false); - headers->canonical = true; + headers->canonical = sf_header_values_trimmed(pairs); ScrDyn *result = scr_dyn_new_handle(headers, SCR_DYNH_FETCH_HEADERS); sf_headers_release(headers); return result; diff --git a/tests/corpus/fetch-headers-copy-cache.ts b/tests/corpus/fetch-headers-copy-cache.ts index 24321d818..cbbc7fc6d 100644 --- a/tests/corpus/fetch-headers-copy-cache.ts +++ b/tests/corpus/fetch-headers-copy-cache.ts @@ -106,3 +106,20 @@ console.log(spacedList(new Headers(spaced))); console.log(spacedList(new Headers(new Headers(spaced)))); console.log(spacedList(new Response("", { headers: spaced }).headers)); console.log(spacedList(new Request("http://example.test/", { headers: spaced }).headers)); + +// The same through duplicate constructor entries, in a Headers object and +// in a Response's own headers. +const entries: [string, string][] = [ + ["a", "x"], + ["a", ""], + ["c", ""], + ["c", ""], + ["d", "plain"], +]; +const built = new Headers(entries); +console.log(spacedList(built)); +console.log(spacedList(new Headers(built))); +const builtResponse = new Response("", { headers: entries }); +console.log(spacedList(builtResponse.headers)); +console.log(spacedList(new Headers(builtResponse.headers))); +console.log(spacedList(new Request("http://example.test/", { headers: built }).headers)); From e84f656172aceab97a42205656d410e81daae614 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 07:11:38 -0700 Subject: [PATCH 059/184] Clone Response and Request bodies by teeing their streams Response.clone() and Request.clone() on checked-dynamic handles answered 'res.clone is not a function' (hono's etag middleware hashes res.clone().body). The runtime now implements the streams spec's ReadableStreamDefaultTee over native streams and clones the body with it, so the original and the clone each read every chunk. A used or locked body throws undici's TypeErrors, the clone's headers are a copy with the same guard, and a cloned Request's signal follows the original's. Co-Authored-By: Claude Opus 5.5 --- .../corpus/7742-response-clone-tee.ts.json | 6 + packages/runtime/src/scr_fetch.c | 277 +++++++++++++++++- tests/corpus/7742-response-clone-tee.ts | 154 ++++++++++ .../7742-response-clone-tee.ts.json | 1 + 4 files changed, 437 insertions(+), 1 deletion(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7742-response-clone-tee.ts.json create mode 100644 tests/corpus/7742-response-clone-tee.ts create mode 100644 tests/harness/corpus-costs/7742-response-clone-tee.ts.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7742-response-clone-tee.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7742-response-clone-tee.ts.json new file mode 100644 index 000000000..a62bef2d0 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7742-response-clone-tee.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7742-response-clone-tee.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index 6049a4936..37de698ac 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -1855,6 +1855,210 @@ static ScrPromise *sf_stream_collect(SfStream *s, int mode) { return p; } +/* ── ReadableStreamDefaultTee ────────────────────────────────────── */ + +/* + * The streams spec's ReadableStreamDefaultTee over a native stream: the + * tee locks the source with a reader and feeds two native branches whose + * pull and cancel algorithms are native functions. A pull on either branch + * reads one chunk from the source and enqueues it into every branch that + * was not canceled; canceling both branches cancels the source with both + * reasons. Response.clone and Request.clone use it (undici's cloneBody). + * + * Ownership: the tee owns the source, its reader and both branches; each + * branch's callbacks and every pending read own the tee. Closing, erroring + * or canceling a branch drops its callbacks (sf_stream_drop_source_callbacks), + * and fetch teardown drops the callbacks of branches nobody finished. + */ +typedef struct SfTee { + size_t rc; + SfStream *source; + SfReader *reader; + SfStream *branch[2]; + bool reading; + bool read_again; + bool canceled[2]; + ScrDyn *reason[2]; + ScrPromise *cancel_promise; +} SfTee; + +static SfTee *sf_tee_retain(SfTee *t) { + t->rc++; + return t; +} + +static void sf_tee_release(SfTee *t) { + if (!t || --t->rc > 0) return; + sf_stream_release(t->branch[0]); + sf_stream_release(t->branch[1]); + sf_reader_release(t->reader); + sf_stream_release(t->source); + scr_dyn_release(t->reason[0]); + scr_dyn_release(t->reason[1]); + scr_promise_release(t->cancel_promise); + free(t); +} + +static void *sf_tee_retain_v(void *p) { return sf_tee_retain((SfTee *)p); } +static void sf_tee_release_v(void *p) { sf_tee_release((SfTee *)p); } + +static ScrBox *sf_tee_box(SfTee *t) { + ScrBox *box = scr_box_new_obj(&sf_tee_retain_v, &sf_tee_release_v, NULL); + scr_box_set_ref(box, sf_tee_retain(t)); + return box; +} + +static void sf_tee_pull(SfTee *t); + +/* The read request's chunk steps run in a microtask (spec: "queue a + * microtask"), after the read that produced the chunk settles. */ +static void sf_tee_chunk_task(ScrClosure *cb) { + SfTee *t = (SfTee *)scr_box_get_ref(cb->caps[0]); + ScrDyn *chunk = (ScrDyn *)scr_box_get_ref(cb->caps[1]); + t->read_again = false; + for (int i = 0; i < 2; i++) { + if (t->canceled[i]) continue; + SfStream *b = t->branch[i]; + if (b->close_requested || b->closed || b->error) continue; + sf_stream_enqueue_value(b, chunk); + if (scr_exc_pending()) scr_exc_clear(); + } + t->reading = false; + if (t->read_again) sf_tee_pull(t); + scr_dyn_release(chunk); + sf_tee_release(t); +} + +static void sf_tee_sink_chunk(void *ctx, ScrDyn *value) { + SfTee *t = ctx; + ScrClosure *cb = scr_closure_new((void *)&sf_tee_chunk_task, 2); + cb->caps[0] = sf_tee_box(t); + cb->caps[1] = scr_box_new_obj(&scr_dyn_retain_v, &scr_dyn_release_v, NULL); + scr_box_set_ref(cb->caps[1], scr_dyn_retain(value)); + scr_queue_microtask(cb); + sf_tee_release(t); /* the read request's reference */ +} + +static void sf_tee_sink_close(void *ctx) { + SfTee *t = ctx; + t->reading = false; + for (int i = 0; i < 2; i++) { + if (!t->canceled[i]) sf_stream_close(t->branch[i]); + } + if (!t->canceled[0] || !t->canceled[1]) { + scr_promise_fulfill_void(t->cancel_promise); + } + sf_tee_release(t); +} + +static void sf_tee_sink_error(void *ctx, ScrDyn *reason) { + SfTee *t = ctx; + t->reading = false; + if (reason) { + /* The spec errors both branches when the source reader's closed + * promise rejects; a native read request learns it here. */ + sf_stream_error(t->branch[0], reason); + sf_stream_error(t->branch[1], reason); + if (!t->canceled[0] || !t->canceled[1]) { + scr_promise_fulfill_void(t->cancel_promise); + } + } + sf_tee_release(t); +} + +static const ScrFetchReadSink sf_tee_sink = { + &sf_tee_sink_chunk, &sf_tee_sink_close, &sf_tee_sink_error}; + +static void sf_tee_pull(SfTee *t) { + if (t->reading) { + t->read_again = true; + return; + } + t->reading = true; + sf_tee_retain(t); /* owned by the read request */ + if (!sf_reader_read_request(t->reader, NULL, &sf_tee_sink, t)) { + scr_exc_clear(); + t->reading = false; + sf_tee_release(t); + } +} + +static ScrDyn *sf_tee_pull_thunk(ScrClosure *cb, ScrDyn *const *args, + size_t argc) { + (void)args; + (void)argc; + SfTee *t = (SfTee *)scr_box_get_ref(cb->caps[0]); + sf_tee_pull(t); + sf_tee_release(t); + return scr_dyn_retain(scr_dyn_undefined()); +} + +static ScrDyn *sf_tee_cancel(SfTee *t, int i, ScrDyn *reason) { + t->canceled[i] = true; + scr_dyn_release(t->reason[i]); + t->reason[i] = scr_dyn_retain(reason); + if (t->canceled[0] && t->canceled[1]) { + ScrDyn *composite = scr_dyn_new_arr(); + scr_dyn_arr_push(composite, scr_dyn_retain(t->reason[0])); + scr_dyn_arr_push(composite, scr_dyn_retain(t->reason[1])); + ScrPromise *result = sf_stream_cancel(t->source, composite, true); + scr_dyn_release(composite); + ScrDyn *boxed = scr_dyn_new_promise(result); + scr_promise_release(result); + sf_then_adopt(t->cancel_promise, boxed); + scr_dyn_release(boxed); + } + return scr_dyn_new_promise(t->cancel_promise); +} + +static ScrDyn *sf_tee_cancel_thunk_0(ScrClosure *cb, ScrDyn *const *args, + size_t argc) { + SfTee *t = (SfTee *)scr_box_get_ref(cb->caps[0]); + ScrDyn *out = sf_tee_cancel(t, 0, argc ? args[0] : scr_dyn_undefined()); + sf_tee_release(t); + return out; +} + +static ScrDyn *sf_tee_cancel_thunk_1(ScrClosure *cb, ScrDyn *const *args, + size_t argc) { + SfTee *t = (SfTee *)scr_box_get_ref(cb->caps[0]); + ScrDyn *out = sf_tee_cancel(t, 1, argc ? args[0] : scr_dyn_undefined()); + sf_tee_release(t); + return out; +} + +static ScrDyn *sf_tee_func(SfTee *t, ScrDynThunk thunk, const char *name) { + ScrClosure *cb = scr_closure_new((void *)thunk, 1); + cb->caps[0] = sf_tee_box(t); + return scr_dyn_new_func(cb, thunk, 1, "(controller)", name); +} + +/* Tees an unlocked, undisturbed stream: +1 branches in out[0] and out[1]. + * The source stays locked by the tee. */ +static void sf_stream_tee(SfStream *s, SfStream *out[2]) { + SfTee *t = calloc(1, sizeof *t); + if (!t) sf_oom(); + t->rc = 1; + t->source = sf_stream_retain(s); + t->reader = sf_stream_get_reader(s); + t->cancel_promise = scr_promise_new(); + scr_promise_mark_handled(t->cancel_promise); + for (int i = 0; i < 2; i++) { + SfStream *b = sf_stream_new_native(); + b->pull_cb = sf_tee_func(t, &sf_tee_pull_thunk, "pull"); + b->cancel_cb = sf_tee_func( + t, i == 0 ? &sf_tee_cancel_thunk_0 : &sf_tee_cancel_thunk_1, "cancel"); + b->source_this = scr_dyn_retain(scr_dyn_undefined()); + sf_stream_track_callbacks(b); + t->branch[i] = sf_stream_retain(b); + out[i] = b; + } + /* The branches start at once; a branch with room then pulls. */ + sf_stream_schedule_initial_pull(out[0]); + sf_stream_schedule_initial_pull(out[1]); + sf_tee_release(t); +} + ScrDyn *scr_fetch_stream_new(ScrDyn *source) { ScrDyn *source_view = source; ScrDyn *materialized = NULL; @@ -2729,12 +2933,54 @@ static ScrPromise *sf_response_collect(SfResponse *r, int mode) { return p; } +/* The body and metadata copy of Response.clone and Request.clone (undici + * cloneResponse/cloneBody): the body tees, so the original and the clone + * each keep one branch, and the clone's headers are a copy with the same + * guard. NULL with a TypeError pending when the body is unusable. */ +static SfResponse *sf_response_clone(SfResponse *r, const char *unusable) { + if (!r->null_body && + (r->body->disturbed || r->body->reader || r->body->internal_lock)) { + sf_type_error(unusable); + return NULL; + } + SfResponse *c = calloc(1, sizeof *c); + if (!c) sf_oom(); + c->rc = 1; + if (r->null_body) { + c->body = sf_stream_new_native(); + sf_stream_close(c->body); + } else { + SfStream *branches[2]; + sf_stream_tee(r->body, branches); + sf_stream_release(r->body); + r->body = branches[0]; + c->body = branches[1]; + } + c->headers = sf_headers_new(scr_arr_slice(r->headers->pairs, 0, INFINITY), + r->headers->immutable); + c->url = scr_str_retain(r->url); + c->status_text = scr_str_retain(r->status_text); + c->status = r->status; + c->redirected = r->redirected; + c->null_body = r->null_body; + c->light_length = r->light_length; + return c; +} + static ScrDyn *sf_response_invoke(void *ptr, ScrDyn *self, const char *method, ScrDyn *const *args, size_t argc, const char *what) { SfResponse *r = ptr; sf_response_materialize(r); + if (strcmp(method, "clone") == 0) { + SfResponse *c = sf_response_clone( + r, "Response.clone: Body has already been consumed."); + if (!c) return NULL; + ScrDyn *out = scr_dyn_new_handle(c, SCR_DYNH_FETCH_RESPONSE); + sf_response_release(c); + return out; + } int mode = -1; if (strcmp(method, "json") == 0) mode = SF_COLLECT_JSON; else if (strcmp(method, "text") == 0) mode = SF_COLLECT_TEXT; @@ -5153,8 +5399,37 @@ static ScrDyn *sf_request_get(void *ptr, const char *key, size_t len) { if (sf_name(key, len, "body") || sf_name(key, len, "bodyUsed") || sf_name(key, len, "headers")) return sf_response_get(r->payload, key, len); return NULL; } +/* Request.clone: the payload clones as a Response does, and the clone's + * signal follows the original's (undici: an AbortController aborted with + * the original signal's reason, at once when it already aborted). */ +static ScrDyn *sf_request_clone(SfRequest *r) { + SfResponse *payload = sf_response_clone(r->payload, "unusable"); + if (!payload) return NULL; + ScrDyn *sources = scr_dyn_new_arr(); + scr_dyn_arr_push(sources, + scr_dyn_new_handle(r->signal, SCR_DYNH_ABORT_SIGNAL)); + ScrDyn *follower = scr_fetch_abort_any(sources); + scr_dyn_release(sources); + if (!follower) { + sf_response_release(payload); + return NULL; + } + SfRequest *c = calloc(1, sizeof *c); + if (!c) sf_oom(); + c->rc = 1; + c->payload = payload; + c->url = scr_str_retain(r->url); + c->method = scr_str_retain(r->method); + c->redirect = scr_str_retain(r->redirect); + c->signal = sf_signal_retain((SfSignal *)follower->v.handle.ptr); + scr_dyn_release(follower); + ScrDyn *out = scr_dyn_new_handle(c, SCR_DYNH_FETCH_REQUEST); + sf_request_release_v(c); + return out; +} static ScrDyn *sf_request_invoke(void *ptr, ScrDyn *self, const char *method, ScrDyn *const *args, size_t argc, const char *what) { SfRequest *r = ptr; + if (strcmp(method, "clone") == 0) return sf_request_clone(r); return sf_response_invoke(r->payload, self, method, args, argc, what); } @@ -5668,7 +5943,7 @@ static const ScrDynHandleMethod sf_reader_methods[] = { {NULL, 0, false}}; static const ScrDynHandleMethod sf_body_methods[] = { {"json", 0, true}, {"text", 0, true}, {"bytes", 0, true}, - {"arrayBuffer", 0, true}, {NULL, 0, false}}; + {"arrayBuffer", 0, true}, {"clone", 0, false}, {NULL, 0, false}}; static const ScrDynHandleMethod sf_headers_methods[] = { {"append", 2, false}, {"delete", 1, false}, {"entries", 0, false}, {"forEach", 1, false}, {"get", 1, false}, {"getSetCookie", 0, false}, diff --git a/tests/corpus/7742-response-clone-tee.ts b/tests/corpus/7742-response-clone-tee.ts new file mode 100644 index 000000000..9058c4d70 --- /dev/null +++ b/tests/corpus/7742-response-clone-tee.ts @@ -0,0 +1,154 @@ +// Response.clone and Request.clone on checked-dynamic handles: the body +// tees, so the original and the clone each read every chunk; a clone of a +// used or locked body throws; canceling both branches cancels the source +// with both reasons; the clone's headers are a copy. +const enc = new TextEncoder(); +const dec = new TextDecoder(); + +function describe(e: unknown): string { + const err = e as { name?: string; message?: string }; + return `${String(err.name)}: ${String(err.message)}`; +} + +async function readAll(stream: any): Promise { + const reader = stream.getReader(); + let out = ""; + for (;;) { + const { done, value } = await reader.read(); + if (done) return out; + out += dec.decode(value); + } +} + +async function main(): Promise { + const r: any = new Response("hello world", { status: 201, headers: { "x-a": "1" } }); + const c: any = r.clone(); + console.log(c.status, c.headers.get("x-a"), c.bodyUsed, r.bodyUsed, c.body === r.body); + console.log(await c.text(), "|", await r.text()); + console.log("used", c.bodyUsed, r.bodyUsed); + try { + r.clone(); + } catch (e) { + console.log("clone used:", describe(e), e instanceof TypeError); + } + + const locked: any = new Response("abc"); + locked.body.getReader(); + try { + locked.clone(); + } catch (e) { + console.log("clone locked:", describe(e)); + } + + const empty: any = new Response(null, { status: 204, statusText: "Nothing" }); + const emptyClone: any = empty.clone(); + console.log("null body", emptyClone.body, emptyClone.status, emptyClone.statusText, JSON.stringify(await emptyClone.text())); + + // Headers are copied: changing the clone's leaves the original's alone. + const h: any = new Response("x", { headers: { "x-b": "2" } }); + const hc: any = h.clone(); + hc.headers.set("x-b", "3"); + console.log("headers", h.headers.get("x-b"), hc.headers.get("x-b"), h.headers === hc.headers); + + // A pulled source: both branches see each chunk (the same object). + let n = 0; + const source: any = new ReadableStream({ + start(): void { + console.log("start"); + }, + pull(controller: any): void { + n++; + console.log("pull", n); + if (n <= 3) controller.enqueue(enc.encode(`chunk${n};`)); + else controller.close(); + }, + }); + const streamed: any = new Response(source); + const streamedClone: any = streamed.clone(); + console.log("cloned"); + const r1 = streamed.body.getReader(); + const r2 = streamedClone.body.getReader(); + const a = await r1.read(); + const b = await r2.read(); + console.log("same chunk", a.value === b.value, dec.decode(a.value)); + console.log("rest 1", await readAllFrom(r1)); + console.log("rest 2", await readAllFrom(r2)); + + // One branch read to the end first; the other buffers. + const big: any = new Response(new ReadableStream({ + start(controller: any): void { + controller.enqueue(enc.encode("a")); + controller.enqueue(enc.encode("b")); + controller.close(); + }, + })); + const bigClone: any = big.clone(); + console.log("clone first", await bigClone.text()); + console.log("then original", await readAll(big.body)); + + // Canceling both branches cancels the source with both reasons. + const cancelSource: any = new ReadableStream({ + pull(controller: any): void { + controller.enqueue(enc.encode("z")); + }, + cancel(reason: unknown): void { + console.log("source cancel", JSON.stringify(reason)); + }, + }); + const cr: any = new Response(cancelSource); + const crc: any = cr.clone(); + const p1 = cr.body.cancel("first"); + console.log("one canceled"); + const p2 = crc.body.cancel("second"); + await p1; + await p2; + console.log("both canceled", cr.bodyUsed, crc.bodyUsed); + + // An erroring source errors both branches. + const failing: any = new ReadableStream({ + pull(controller: any): void { + controller.error(new Error("source failed")); + }, + }); + const fr: any = new Response(failing); + const frc: any = fr.clone(); + for (const branch of [fr, frc]) { + try { + await branch.text(); + } catch (e) { + console.log("branch error:", describe(e)); + } + } + + // Request.clone: the body tees and the signal follows the original's. + const q: any = new Request("http://example.test/path", { method: "POST", body: "data" }); + const qc: any = q.clone(); + console.log("request", qc.method, qc.url, await qc.text(), await q.text(), qc.signal === q.signal); + try { + q.clone(); + } catch (e) { + console.log("request clone used:", describe(e)); + } + const ac = new AbortController(); + const q2: any = new Request("http://example.test/", { signal: ac.signal }); + const q2c: any = q2.clone(); + console.log("before abort", q2c.signal.aborted); + ac.abort("why"); + console.log("after abort", q2c.signal.aborted, q2c.signal.reason); + const q3: any = new Request("http://example.test/", { signal: AbortSignal.abort("early") }); + console.log("aborted clone", q3.clone().signal.aborted, q3.clone().signal.reason); + + // clone is a method value like the body readers. + console.log(typeof r.clone, "clone" in r, r.clone === c.clone); +} + +async function readAllFrom(reader: any): Promise { + let out = ""; + for (;;) { + const { done, value } = await reader.read(); + if (done) return out; + out += dec.decode(value); + } +} + +await main(); diff --git a/tests/harness/corpus-costs/7742-response-clone-tee.ts.json b/tests/harness/corpus-costs/7742-response-clone-tee.ts.json new file mode 100644 index 000000000..0c31a05f7 --- /dev/null +++ b/tests/harness/corpus-costs/7742-response-clone-tee.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 4000 } From 3c2911ccad559e53e49339f0862068cb55ad3337 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 07:11:39 -0700 Subject: [PATCH 060/184] Implement the identity TransformStream natively 'new TransformStream()' had no lowering, so hono's stream() and streamSSE() helpers failed to compile their checked-dynamic paths. The constructor without arguments now builds a native handle: its readable side is a native ReadableStream with a zero high-water mark, and its writable side runs the streams spec's WritableStream state machine (getWriter, abort, close, locked; the writer's write, close, abort, releaseLock, ready, closed and desiredSize). A write completes when the readable side reads its chunk, close/abort/cancel propagate across the pair, and deferred reactions take as many microtask turns as in Node. Web ReadableStreams can also be weak collection keys (hono keys a WeakMap by the response stream). A transformer, queuing strategies and the WritableStream constructor keep the SC2020 refusal. Co-Authored-By: Claude Opus 5.5 --- .../builtin-apis/streams-and-readline.mdx | 3 +- .../src/generated/node-v24-compatibility.json | 12 +- .../generated/node-v24-internal.json | 16 +- internal/compatibility/static-support.json | 4 +- .../ambient/scriptc-node-fallback.d.ts | 28 + .../compiler/src/backend/llvm/lib-shared.ts | 2 + packages/compiler/src/compat/fetch-profile.ts | 2 +- .../src/frontend/lowering/lower-stdlib.ts | 13 + .../compiler/src/frontend/lowering/lowerer.ts | 2 + packages/compiler/src/frontend/type-mapper.ts | 5 +- .../compiler/src/ir/builtin-signatures.ts | 1 + .../7743-transform-stream-identity.ts.json | 6 + packages/runtime/src/scr_dyn.h | 3 + packages/runtime/src/scr_fetch.c | 927 ++++++++++++++++++ packages/runtime/src/scr_generator.h | 1 + packages/runtime/src/scr_json.c | 1 + .../corpus/7743-transform-stream-identity.ts | 212 ++++ .../7743-transform-stream-identity.ts.json | 1 + 18 files changed, 1225 insertions(+), 14 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7743-transform-stream-identity.ts.json create mode 100644 tests/corpus/7743-transform-stream-identity.ts create mode 100644 tests/harness/corpus-costs/7743-transform-stream-identity.ts.json diff --git a/docs/content/limitations/builtin-apis/streams-and-readline.mdx b/docs/content/limitations/builtin-apis/streams-and-readline.mdx index fe6bb2dfb..273554a3b 100644 --- a/docs/content/limitations/builtin-apis/streams-and-readline.mdx +++ b/docs/content/limitations/builtin-apis/streams-and-readline.mdx @@ -27,7 +27,8 @@ A `Duplex.from` body function may take the `{ signal }` argument, which is aborted with the error the Duplex is destroyed with. `stream.compose` accepts streams and async generator functions in any position, an async function as the last stage, and an array, a string, or a generator as the first stage; one stage is `Duplex.from` of it. A web `ReadableStream` is read as an async iterable by `Readable.from` and as the first stage of `compose` or `pipeline`, and through its reader by `Readable.fromWeb` (with `objectMode`, `highWaterMark`, `encoding`, and `signal` in an options object literal) and `Duplex.from`. - A web stream in a later compose stage, `Readable.toWeb`, and web `WritableStream` and `TransformStream` interop are unsupported. + `new TransformStream()` without arguments builds the identity transform: its readable side is a web `ReadableStream`, its writable side supports `getWriter`, `abort`, `close`, and `locked`, and the writer supports `write`, `close`, `abort`, `releaseLock`, `ready`, `closed`, and `desiredSize`, with Node's backpressure (a write completes when the readable side reads its chunk). + A web stream in a later compose stage, `Readable.toWeb`, a `TransformStream` with a transformer or queuing strategies, the `WritableStream` constructor, `pipeTo`, and `pipeThrough` are unsupported. The `ERR_INVALID_ARG_VALUE` message for a stream that is no longer readable or writable names the stream's class instead of printing the stream. `readable.compose(stage, { signal })` is `stream.compose(readable, stage)` with the signal added; the options are an object literal whose `signal` is a variable or a property read. `pipeline` from `node:stream/promises` accepts a trailing `{ signal, end }` object, with or without function stages, and `finished` accepts `readable`, `writable`, `error`, `cleanup`, and `signal` options, with boolean literals for the flags. diff --git a/docs/src/generated/node-v24-compatibility.json b/docs/src/generated/node-v24-compatibility.json index c3f1835ab..fc5826666 100644 --- a/docs/src/generated/node-v24-compatibility.json +++ b/docs/src/generated/node-v24-compatibility.json @@ -35971,9 +35971,9 @@ "inherited": true }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented scriptc module-loader subset.", + "verification": "test-backed" } }, { @@ -81657,9 +81657,9 @@ "inherited": true }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { diff --git a/internal/compatibility/generated/node-v24-internal.json b/internal/compatibility/generated/node-v24-internal.json index ba8692b49..6eab15d70 100644 --- a/internal/compatibility/generated/node-v24-internal.json +++ b/internal/compatibility/generated/node-v24-internal.json @@ -52194,6 +52194,7 @@ "tests/corpus/6294-webcrypto-js-values.mjs", "tests/corpus/6295-jose-hs256/main.mjs", "tests/corpus/6331-jose-root-entry/main.ts", + "tests/corpus/7743-transform-stream-identity.ts", "tests/corpus/text-codec-values/main.ts", "tests/corpus/webassembly-api.mjs", "tests/corpus/webassembly-dylink/main.mjs", @@ -53443,8 +53444,11 @@ "inherited": true }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:globals" + "status": "partial", + "evidence": "compiler-feature:globals.TransformStream", + "tests": [ + "tests/corpus/7743-transform-stream-identity.ts" + ] }, "anchorSource": "exact" }, @@ -111483,6 +111487,7 @@ "tests/corpus/2630-stream-consumers-errors.ts", "tests/corpus/2862-web-readable-async-iteration.ts", "tests/corpus/4875-stream-consumers-array-buffer.ts", + "tests/corpus/7743-transform-stream-identity.ts", "tests/corpus/stream-web-readable.ts", "tests/corpus/stream-web-require.cjs", "tests/harness/fetch-conformance.test.ts" @@ -112957,8 +112962,11 @@ "inherited": true }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:webstreams" + "status": "partial", + "evidence": "compiler-feature:webstreams.TransformStream", + "tests": [ + "tests/corpus/7743-transform-stream-identity.ts" + ] }, "anchorSource": "exact" }, diff --git a/internal/compatibility/static-support.json b/internal/compatibility/static-support.json index 100882c1a..4e5e29616 100644 --- a/internal/compatibility/static-support.json +++ b/internal/compatibility/static-support.json @@ -407,7 +407,9 @@ { "chapter": "zlib", "signatures": ["`zlib.constants`"], "status": "partial", "evidence": ["tests/corpus/5376-zlib-constants.cjs", "tests/corpus/5374-zlib-stream-flush-params.ts"] }, { "chapter": "dns", "symbols": ["dns.lookup", "dns.lookupService", "dns.getServers", "dns.getDefaultResultOrder", "dns.setDefaultResultOrder"], "status": "partial", "evidence": ["tests/corpus/5648-dns-lookup-forms.ts", "tests/corpus/5649-dns-service-servers.ts", "tests/corpus/5650-dns-lookup-errors-untyped.cjs"] }, { "chapter": "dns", "symbols": ["dnsPromises.lookup", "dnsPromises.lookupService", "dnsPromises.getDefaultResultOrder"], "status": "partial", "evidence": ["tests/corpus/dns-promises-lookup.ts", "tests/corpus/dns-promises-untyped.cjs"] }, - { "chapter": "webstreams", "symbols": ["ReadableStream"], "status": "partial", "evidence": ["tests/corpus/stream-web-readable.ts", "tests/corpus/stream-web-require.cjs", "tests/corpus/2862-web-readable-async-iteration.ts"] } + { "chapter": "webstreams", "symbols": ["ReadableStream"], "status": "partial", "evidence": ["tests/corpus/stream-web-readable.ts", "tests/corpus/stream-web-require.cjs", "tests/corpus/2862-web-readable-async-iteration.ts"] }, + { "chapter": "webstreams", "symbols": ["TransformStream"], "status": "partial", "evidence": ["tests/corpus/7743-transform-stream-identity.ts"] }, + { "chapter": "globals", "signatures": ["Class: `TransformStream`"], "status": "partial", "evidence": ["tests/corpus/7743-transform-stream-identity.ts"] } ], "dedicated": [ { "symbols": ["assert.doesNotThrow", "assert.doesNotReject"], "status": "partial", "evidence": ["tests/corpus/node24-no-error-assertions.ts", "tests/corpus/node24-no-error-assertions.cjs", "tests/corpus/node24-no-error-assertions-coercion.js", "tests/fixtures/node-types/no-error-assertions.ts", "tests/harness/project-config.test.ts"] }, diff --git a/packages/compiler/ambient/scriptc-node-fallback.d.ts b/packages/compiler/ambient/scriptc-node-fallback.d.ts index d7abe70df..09587aa72 100644 --- a/packages/compiler/ambient/scriptc-node-fallback.d.ts +++ b/packages/compiler/ambient/scriptc-node-fallback.d.ts @@ -1106,6 +1106,34 @@ declare var ReadableStream: { from(iterable: string): ReadableStream; readonly prototype: ReadableStream; }; +/* The writable side of a TransformStream and its writer. The + * WritableStream constructor has no native lowering; writable streams come + * from `new TransformStream()`. */ +interface WritableStreamDefaultWriter { + readonly closed: Promise; + readonly desiredSize: number | null; + readonly ready: Promise; + abort(reason?: unknown): Promise; + close(): Promise; + releaseLock(): void; + write(chunk?: W): Promise; +} +interface WritableStream { + readonly locked: boolean; + abort(reason?: unknown): Promise; + close(): Promise; + getWriter(): WritableStreamDefaultWriter; +} +/* The identity transform: chunks written to `writable` are read from + * `readable`, and a write completes when its chunk is read. */ +interface TransformStream { + readonly readable: ReadableStream; + readonly writable: WritableStream; +} +declare var TransformStream: { + new (): TransformStream; + readonly prototype: TransformStream; +}; /* fetch, typed as the native JSON + readable-stream slice: one URL, an * optional init with the members CLI requests carry, and a Response diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 4f4a6c302..bc95ad98d 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -958,6 +958,7 @@ export const LIB_FN_SYMS: Record = { "fetch.streamIs": "scr_fetch_stream_is", "zlib.streamIs": "scr_zlib_stream_is", "fetch.streamNew": "scr_fetch_stream_new", + "fetch.transformNew": "scr_fetch_transform_new", "fetch.streamFrom": "scr_fetch_stream_from", "fetch.readerRead": "scr_fetch_reader_read", "json.parse": "scr_json_parse", @@ -1689,6 +1690,7 @@ export const USES_TIMERS_LIB_FNS = new Set([ "fetch.abortAny", "fetch.streamNew", "fetch.streamFrom", + "fetch.transformNew", "readable.new", // fs.ReadStream / fs.WriteStream open, read and write on the worker pool. "fs.createReadStream", diff --git a/packages/compiler/src/compat/fetch-profile.ts b/packages/compiler/src/compat/fetch-profile.ts index c4d1e3f50..ff7cedb22 100644 --- a/packages/compiler/src/compat/fetch-profile.ts +++ b/packages/compiler/src/compat/fetch-profile.ts @@ -900,7 +900,7 @@ export const NODE24_FETCH_COMPAT_PROFILE = { { name: "WritableStream/TransformStream", reason: - "native support currently covers readable fetch bodies; general Web Streams graphs are unsupported", + "native support covers readable fetch bodies and the identity `new TransformStream()`; the WritableStream constructor, transformers, and general Web Streams graphs are unsupported", }, { name: "Readable byte/BYOB streams", diff --git a/packages/compiler/src/frontend/lowering/lower-stdlib.ts b/packages/compiler/src/frontend/lowering/lower-stdlib.ts index e671a0a08..dc9d3def8 100644 --- a/packages/compiler/src/frontend/lowering/lower-stdlib.ts +++ b/packages/compiler/src/frontend/lowering/lower-stdlib.ts @@ -1945,6 +1945,19 @@ export function lowerStaticReadableStreamNew( }; } +/** `new TransformStream()`: the identity transform, a native handle whose + * readable side is a native ReadableStream and whose writable side is the + * runtime's WritableStream. A transformer or queuing strategies have no + * native lowering yet and keep the SC2020 refusal. */ +export function lowerStaticTransformStreamNew( + lowerer: Lowerer, + expr: ts.NewExpression, +): IrExpr | null { + if (stdlibGlobalNameOf(lowerer, expr.expression) !== "TransformStream") return null; + if ((expr.arguments?.nodes.length ?? 0) > 0) return null; + return { kind: "libCall", fn: "fetch.transformNew", args: [], type: DYN, loc: locOf(expr) }; +} + /** Native Request and Headers values share the fetch runtime's handles. */ export function lowerFetchWebNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr | null { const name = stdlibGlobalNameOf(lowerer, expr.expression); diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index e36ffd09f..80ff8b4ca 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -558,6 +558,7 @@ import { lowerStaticReadableStreamCancelCall, lowerStaticReadableStreamControllerCall, lowerStaticReadableStreamNew, + lowerStaticTransformStreamNew, lowerStaticReadableStreamReaderCall, lowerStaticResponseCall, lowerStdlibMethodCall, @@ -13423,6 +13424,7 @@ export class Lowerer { lowerBlobNew(this, expr) ?? lowerNetAddressNew(this, expr) ?? lowerStaticReadableStreamNew(this, expr) ?? + lowerStaticTransformStreamNew(this, expr) ?? lowerConsoleNew(this, expr); return ( builtin ?? this.withRuntimeOptionalClassValue(expr.expression, () => lowerNew(this, expr)) diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index f1e86067e..cbb468f98 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -213,6 +213,9 @@ export const FETCH_AMBIENT_TYPES = [ "ReadableStreamReadResult", "ReadableStreamReadValueResult", "ReadableStreamReadDoneResult", + "TransformStream", + "WritableStream", + "WritableStreamDefaultWriter", "Blob", "File", "BlobPropertyBag", @@ -3109,7 +3112,7 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { psym && (FETCH_AMBIENT_TYPES as readonly string[]).includes(ts.symbolName(psym)) && !( - ts.symbolName(psym) === "ReadableStream" && + (ts.symbolName(psym) === "ReadableStream" || ts.symbolName(psym) === "WritableStream") && psym.declarations.some( (d) => ts.isInterfaceDeclaration(d) && isDeclaredInAmbientNamespace(d, "NodeJS"), ) diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 49d281d38..7c7df6adb 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -163,6 +163,7 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "fetch.abortNow": { argTypes: [DYN], result: DYN }, "fetch.abortAny": { argTypes: [DYN], result: DYN }, "fetch.streamNew": { argTypes: [DYN], result: DYN }, + "fetch.transformNew": { argTypes: [], result: DYN }, // Program-dependent iterable: typed arrays/bytes/string stay intact so // the native stream can pull lazily; checked-dynamic values are the // fallback. The libCall validator in validate.ts checks the closed set. diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7743-transform-stream-identity.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7743-transform-stream-identity.ts.json new file mode 100644 index 000000000..0cfff542b --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7743-transform-stream-identity.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7743-transform-stream-identity.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 669f4aedb..d772f5414 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -212,6 +212,9 @@ typedef enum { X(WEB_CONTROLLER, 11) /* native ReadableStreamDefaultController */ \ X(WEB_READER, 10) /* native ReadableStreamDefaultReader */ \ X(WEB_STREAM, 9) /* native WHATWG ReadableStream */ \ + X(WEB_TRANSFORM, 73) /* native identity TransformStream (scr_fetch.c) */ \ + X(WEB_WRITABLE, 74) /* a TransformStream's WritableStream (scr_fetch.c) */ \ + X(WEB_WRITER, 75) /* WritableStreamDefaultWriter (scr_fetch.c) */ \ X(WORKER, 35) typedef enum { diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index 6049a4936..e92f488b2 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -147,6 +147,9 @@ struct SfStream { bool pull_again; bool discarded; bool request_discarding; + /* A zero high-water mark (a TransformStream's readable side): pull only + * while a read is pending. */ + bool hwm_zero; ScrDyn *error; SfReader *reader; /* owned while locked */ SfCollector *collector; /* owned while a body reader is active */ @@ -898,6 +901,7 @@ static void sf_stream_release(SfStream *s) { sf_response_discard_if_unobserved(s); return; } + scr_weak_dispose_inline(s); sf_stream_drop_chunks(s); if (s->reader) { s->reader->stream = NULL; @@ -1625,6 +1629,12 @@ static void sf_stream_pull(SfStream *s) { s->error) { return; } + /* ReadableStreamDefaultControllerShouldCallPull: with no room in the + * queue, only a pending read (or a body collector) asks for a chunk. */ + if (s->hwm_zero && !s->collector && + !(s->reader && s->reader->pending_head)) { + return; + } if (s->pulling) { s->pull_again = true; return; @@ -1852,6 +1862,7 @@ static ScrPromise *sf_stream_collect(SfStream *s, int mode) { c->mode = mode; s->collector = c; sf_stream_drain(s); + if (s->hwm_zero && !s->head) sf_stream_pull(s); return p; } @@ -2664,6 +2675,895 @@ static ScrDyn *sf_headers_get(void *ptr, const char *key, size_t len) { return NULL; } +/* ── TransformStream / WritableStream ────────────────────────────── */ + +/* + * The identity TransformStream (`new TransformStream()`): a native + * readable side (an SfStream with a zero high-water mark, fed by the + * transform) and a writable side implemented here with the streams spec's + * WritableStream state machine (default high-water mark 1, chunk size 1). + * The writable side has no other sink, so the TransformStream object, its + * WritableStream and the sink share one SfTransform; the handles differ + * only by tag. Promise reactions the spec chains on already-settled + * promises run after the same number of microtask turns as in Node's + * implementation (lib/internal/webstreams), so interleavings with user + * code match. + * + * Ownership: the transform owns its readable side and its writer; the + * readable side's pull/cancel callbacks own the transform until that side + * closes or errors (fetch teardown drops the callbacks of transforms + * nobody finished). A writer handle retains the transform while bound, + * like a reader handle retains its stream. + */ +enum { SFW_WRITABLE, SFW_ERRORING, SFW_ERRORED, SFW_CLOSED }; + +typedef struct SfTransform SfTransform; + +typedef struct SfWriteReq { + ScrDyn *chunk; + ScrPromise *promise; + struct SfWriteReq *next; +} SfWriteReq; + +typedef struct SfWriter { + size_t rc; + size_t handle_refs; + SfTransform *stream; /* weak; handle refs retain it while bound */ + ScrPromise *ready; + ScrPromise *closed; +} SfWriter; + +struct SfTransform { + size_t rc; + SfStream *readable; + /* WritableStream */ + bool started; /* the start algorithm's promise settled (one turn) */ + int state; + ScrDyn *stored_error; + SfWriter *writer; /* owned while locked */ + SfWriteReq *head; /* queued writes (chunk + request promise) */ + SfWriteReq *tail; + SfWriteReq *in_flight; + size_t queue_size; /* queueTotalSize: queued chunks + the in-flight one */ + bool close_queued; /* the close sentinel is in the queue */ + ScrPromise *close_request; + ScrPromise *in_flight_close; + bool has_pending_abort; + ScrPromise *pending_abort; + ScrDyn *pending_abort_reason; + bool pending_abort_was_erroring; + bool backpressure; + /* TransformStream */ + bool t_backpressure; + ScrPromise *bp_change; + bool write_waiting; /* the in-flight write waits for bp_change */ + ScrDyn *write_error; /* the in-flight write's failure, if any */ + ScrDyn *cancel_reason; /* the readable side's cancel reason, until it reacts */ + ScrPromise *finish; /* finishPromise of close/abort/cancel */ +}; + +static SfTransform *sf_transform_retain(SfTransform *t) { + t->rc++; + return t; +} + +static void sf_write_req_free(SfWriteReq *w) { + if (!w) return; + scr_dyn_release(w->chunk); + scr_promise_release(w->promise); + free(w); +} + +static void sf_writer_release(SfWriter *w) { + if (!w || --w->rc > 0) return; + scr_promise_release(w->ready); + scr_promise_release(w->closed); + free(w); +} + +static void sf_transform_release(SfTransform *t) { + if (!t || --t->rc > 0) return; + sf_stream_release(t->readable); + scr_dyn_release(t->stored_error); + if (t->writer) { + t->writer->stream = NULL; + sf_writer_release(t->writer); + } + while (t->head) { + SfWriteReq *w = t->head; + t->head = w->next; + sf_write_req_free(w); + } + sf_write_req_free(t->in_flight); + scr_promise_release(t->close_request); + scr_promise_release(t->in_flight_close); + scr_promise_release(t->pending_abort); + scr_dyn_release(t->pending_abort_reason); + scr_promise_release(t->bp_change); + scr_promise_release(t->finish); + scr_dyn_release(t->write_error); + scr_dyn_release(t->cancel_reason); + free(t); +} + +static void *sf_transform_retain_v(void *p) { + return sf_transform_retain((SfTransform *)p); +} +static void sf_transform_release_v(void *p) { + sf_transform_release((SfTransform *)p); +} + +static void *sf_writer_handle_retain_v(void *p) { + SfWriter *w = p; + w->rc++; + w->handle_refs++; + if (w->stream) sf_transform_retain(w->stream); + return w; +} + +static void sf_writer_handle_release_v(void *p) { + SfWriter *w = p; + SfTransform *t = w->stream; + if (w->handle_refs > 0) w->handle_refs--; + sf_writer_release(w); + sf_transform_release(t); +} + +static ScrBox *sf_transform_box(SfTransform *t) { + ScrBox *box = scr_box_new_obj(&sf_transform_retain_v, + &sf_transform_release_v, NULL); + scr_box_set_ref(box, sf_transform_retain(t)); + return box; +} + +static void sf_invalid_state(const char *message) { + char buf[96]; + int n = snprintf(buf, sizeof buf, "Invalid state: %s", message); + scr_throw_error_msg_code(SCR_ERR_TYPE, buf, (size_t)n, "ERR_INVALID_STATE"); +} + +static ScrPromise *sf_promise_invalid_state(const char *message) { + ScrPromise *p = scr_promise_new(); + sf_invalid_state(message); + scr_promise_reject_pending(p); + return p; +} + +static ScrPromise *sf_promise_rejected_with(ScrDyn *reason) { + ScrPromise *p = scr_promise_new(); + sf_reject_promise_reason(p, reason); + return p; +} + +static ScrPromise *sf_promise_resolved(void) { + ScrPromise *p = scr_promise_new(); + scr_promise_fulfill_void(p); + return p; +} + +/* Deferred steps: the reaction `op` after `ticks` microtask turns. */ +enum { + SFT_WRITE_DONE, + SFT_WRITE_CONTINUE, + SFT_CLOSE_DONE, + SFT_ABORT_DONE, + SFT_START, + SFT_CANCEL_DONE, +}; + +static void sf_transform_step(SfTransform *t, int op); + +typedef struct SfTransformTask { + size_t rc; + SfTransform *t; + int op; + int ticks; +} SfTransformTask; + +static void *sf_transform_task_retain_v(void *p) { + ((SfTransformTask *)p)->rc++; + return p; +} + +static void sf_transform_task_release_v(void *p) { + SfTransformTask *task = p; + if (--task->rc > 0) return; + sf_transform_release(task->t); + free(task); +} + +static void sf_transform_tick_task(ScrClosure *cb) { + SfTransformTask *task = (SfTransformTask *)scr_box_get_ref(cb->caps[0]); + if (--task->ticks > 0) { + ScrClosure *next = scr_closure_new((void *)&sf_transform_tick_task, 1); + next->caps[0] = scr_box_new_obj(&sf_transform_task_retain_v, + &sf_transform_task_release_v, NULL); + scr_box_set_ref(next->caps[0], sf_transform_task_retain_v(task)); + scr_queue_microtask(next); + } else { + sf_transform_step(task->t, task->op); + } + sf_transform_task_release_v(task); +} + +static void sf_transform_defer(SfTransform *t, int op, int ticks) { + SfTransformTask *task = calloc(1, sizeof *task); + if (!task) sf_oom(); + task->rc = 1; + task->t = sf_transform_retain(t); + task->op = op; + task->ticks = ticks; + ScrClosure *cb = scr_closure_new((void *)&sf_transform_tick_task, 1); + cb->caps[0] = scr_box_new_obj(&sf_transform_task_retain_v, + &sf_transform_task_release_v, NULL); + scr_box_set_ref(cb->caps[0], task); /* the box takes the reference */ + scr_queue_microtask(cb); +} + +/* ── WritableStream abstract operations ── */ + +static bool sfw_close_queued_or_in_flight(const SfTransform *t) { + return t->close_request || t->in_flight_close; +} + +static bool sfw_has_operation_in_flight(const SfTransform *t) { + return t->in_flight || t->in_flight_close; +} + +static void sfw_writer_ensure_ready_rejected(SfWriter *w, ScrDyn *reason) { + if (!scr_promise_is_pending(w->ready)) { + scr_promise_release(w->ready); + w->ready = scr_promise_new(); + } + sf_reject_promise_reason(w->ready, reason); + scr_promise_mark_handled(w->ready); +} + +static void sfw_writer_ensure_closed_rejected(SfWriter *w, ScrDyn *reason) { + if (!scr_promise_is_pending(w->closed)) { + scr_promise_release(w->closed); + w->closed = scr_promise_new(); + } + sf_reject_promise_reason(w->closed, reason); + scr_promise_mark_handled(w->closed); +} + +static void sfw_update_backpressure(SfTransform *t, bool backpressure) { + SfWriter *w = t->writer; + if (w && backpressure != t->backpressure) { + if (backpressure) { + scr_promise_release(w->ready); + w->ready = scr_promise_new(); + scr_promise_mark_handled(w->ready); + } else { + scr_promise_fulfill_void(w->ready); + } + } + t->backpressure = backpressure; +} + +static void sfw_reject_close_and_closed(SfTransform *t) { + if (t->close_request) { + sf_reject_promise_reason(t->close_request, t->stored_error); + scr_promise_release(t->close_request); + t->close_request = NULL; + } + if (t->writer) { + sf_reject_promise_reason(t->writer->closed, t->stored_error); + scr_promise_mark_handled(t->writer->closed); + } +} + +static void sft_abort_algorithm(SfTransform *t, ScrDyn *reason); + +static void sfw_finish_erroring(SfTransform *t) { + t->state = SFW_ERRORED; + /* The controller's queue resets; every queued write rejects. */ + while (t->head) { + SfWriteReq *w = t->head; + t->head = w->next; + sf_reject_promise_reason(w->promise, t->stored_error); + sf_write_req_free(w); + } + t->tail = NULL; + t->queue_size = 0; + t->close_queued = false; + if (!t->has_pending_abort) { + sfw_reject_close_and_closed(t); + return; + } + if (t->pending_abort_was_erroring) { + t->has_pending_abort = false; + sf_reject_promise_reason(t->pending_abort, t->stored_error); + scr_promise_release(t->pending_abort); + t->pending_abort = NULL; + scr_dyn_release(t->pending_abort_reason); + t->pending_abort_reason = NULL; + sfw_reject_close_and_closed(t); + return; + } + /* The sink's abort runs; its promise settles the abort request. */ + sft_abort_algorithm(t, t->pending_abort_reason); +} + +static void sfw_start_erroring(SfTransform *t, ScrDyn *reason) { + t->state = SFW_ERRORING; + scr_dyn_release(t->stored_error); + t->stored_error = scr_dyn_retain(reason); + if (t->writer) sfw_writer_ensure_ready_rejected(t->writer, reason); + if (!sfw_has_operation_in_flight(t) && t->started) sfw_finish_erroring(t); +} + +static void sfw_deal_with_rejection(SfTransform *t, ScrDyn *error) { + if (t->state == SFW_WRITABLE) { + sfw_start_erroring(t, error); + return; + } + sfw_finish_erroring(t); +} + +static void sfw_error_if_needed(SfTransform *t, ScrDyn *error) { + if (t->state == SFW_WRITABLE) sfw_start_erroring(t, error); +} + +static void sft_set_backpressure(SfTransform *t, bool backpressure); +static void sft_sink_write(SfTransform *t); +static void sft_sink_close(SfTransform *t); + +static void sfw_advance_queue(SfTransform *t) { + if (!t->started || t->in_flight || t->in_flight_close) return; + if (t->state == SFW_ERRORING) { + sfw_finish_erroring(t); + return; + } + if (t->head) { + SfWriteReq *w = t->head; + t->head = w->next; + if (!t->head) t->tail = NULL; + w->next = NULL; + t->in_flight = w; + sft_sink_write(t); + return; + } + if (t->close_queued) { + t->close_queued = false; + t->in_flight_close = t->close_request; + t->close_request = NULL; + sft_sink_close(t); + } +} + +static void sfw_finish_write(SfTransform *t, ScrDyn *error) { + SfWriteReq *w = t->in_flight; + t->in_flight = NULL; + if (error) { + sf_reject_promise_reason(w->promise, error); + sf_write_req_free(w); + sfw_deal_with_rejection(t, error); + return; + } + scr_promise_fulfill_void(w->promise); + sf_write_req_free(w); + if (t->queue_size) t->queue_size--; + if (!sfw_close_queued_or_in_flight(t) && t->state == SFW_WRITABLE) { + sfw_update_backpressure(t, t->queue_size >= 1); + } + sfw_advance_queue(t); +} + +static void sfw_finish_close(SfTransform *t, ScrDyn *error) { + ScrPromise *p = t->in_flight_close; + t->in_flight_close = NULL; + if (error) { + sf_reject_promise_reason(p, error); + scr_promise_release(p); + if (t->has_pending_abort) { + t->has_pending_abort = false; + sf_reject_promise_reason(t->pending_abort, error); + scr_promise_release(t->pending_abort); + t->pending_abort = NULL; + scr_dyn_release(t->pending_abort_reason); + t->pending_abort_reason = NULL; + } + sfw_deal_with_rejection(t, error); + return; + } + scr_promise_fulfill_void(p); + scr_promise_release(p); + if (t->state == SFW_ERRORING) { + scr_dyn_release(t->stored_error); + t->stored_error = NULL; + if (t->has_pending_abort) { + t->has_pending_abort = false; + scr_promise_fulfill_void(t->pending_abort); + scr_promise_release(t->pending_abort); + t->pending_abort = NULL; + scr_dyn_release(t->pending_abort_reason); + t->pending_abort_reason = NULL; + } + } + t->state = SFW_CLOSED; + if (t->writer) scr_promise_fulfill_void(t->writer->closed); +} + +/* WritableStreamAbort: +1 promise. */ +static ScrPromise *sfw_abort(SfTransform *t, ScrDyn *reason) { + if (t->state == SFW_CLOSED || t->state == SFW_ERRORED) { + return sf_promise_resolved(); + } + if (t->has_pending_abort) return scr_promise_retain(t->pending_abort); + bool was_erroring = t->state == SFW_ERRORING; + ScrPromise *p = scr_promise_new(); + t->has_pending_abort = true; + t->pending_abort = scr_promise_retain(p); + t->pending_abort_reason = + scr_dyn_retain(was_erroring ? scr_dyn_undefined() : reason); + t->pending_abort_was_erroring = was_erroring; + if (!was_erroring) sfw_start_erroring(t, reason); + return p; +} + +/* WritableStreamClose: +1 promise. */ +static ScrPromise *sfw_close(SfTransform *t) { + if (t->state == SFW_CLOSED || t->state == SFW_ERRORED) { + char message[64]; + snprintf(message, sizeof message, "WritableStream is %s", + t->state == SFW_CLOSED ? "closed" : "errored"); + return sf_promise_invalid_state(message); + } + ScrPromise *p = scr_promise_new(); + t->close_request = scr_promise_retain(p); + if (t->writer && t->backpressure && t->state == SFW_WRITABLE) { + scr_promise_fulfill_void(t->writer->ready); + } + t->close_queued = true; + sfw_advance_queue(t); + return p; +} + +/* WritableStreamDefaultWriterWrite: +1 promise. */ +static ScrPromise *sfw_writer_write(SfWriter *w, ScrDyn *chunk) { + SfTransform *t = w->stream; + if (!t) return sf_promise_invalid_state("Writer is not bound to a WritableStream"); + if (t->state == SFW_ERRORED || t->state == SFW_ERRORING) { + return sf_promise_rejected_with(t->stored_error); + } + if (sfw_close_queued_or_in_flight(t) || t->state == SFW_CLOSED) { + return sf_promise_invalid_state("WritableStream is closed"); + } + SfWriteReq *req = calloc(1, sizeof *req); + if (!req) sf_oom(); + req->chunk = scr_dyn_retain(chunk); + req->promise = scr_promise_new(); + ScrPromise *p = scr_promise_retain(req->promise); + if (t->tail) t->tail->next = req; + else t->head = req; + t->tail = req; + t->queue_size++; + sfw_update_backpressure(t, t->queue_size >= 1); + sfw_advance_queue(t); + return p; +} + +static SfWriter *sfw_get_writer(SfTransform *t) { + if (t->writer) { + sf_invalid_state("WritableStream is locked"); + return NULL; + } + SfWriter *w = calloc(1, sizeof *w); + if (!w) sf_oom(); + w->rc = 1; + w->stream = t; + w->ready = scr_promise_new(); + w->closed = scr_promise_new(); + scr_promise_mark_handled(w->ready); + scr_promise_mark_handled(w->closed); + switch (t->state) { + case SFW_WRITABLE: + if (sfw_close_queued_or_in_flight(t) || !t->backpressure) { + scr_promise_fulfill_void(w->ready); + } + break; + case SFW_ERRORING: + sf_reject_promise_reason(w->ready, t->stored_error); + break; + case SFW_CLOSED: + scr_promise_fulfill_void(w->ready); + scr_promise_fulfill_void(w->closed); + break; + default: + sf_reject_promise_reason(w->ready, t->stored_error); + sf_reject_promise_reason(w->closed, t->stored_error); + break; + } + t->writer = w; + w->rc++; /* the stream's ownership */ + return w; +} + +static void sfw_writer_release_lock(SfWriter *w) { + SfTransform *t = w->stream; + if (!t) return; + sf_invalid_state("Writer has been released"); + ScrCaught *caught = scr_exc_take(); + ScrDyn *released = scr_caught_to_dyn(caught); + scr_caught_release(caught); + sfw_writer_ensure_ready_rejected(w, released); + sfw_writer_ensure_closed_rejected(w, released); + scr_dyn_release(released); + size_t handle_refs = w->handle_refs; + t->writer = NULL; + w->stream = NULL; + sf_writer_release(w); /* the stream's ownership */ + while (handle_refs-- > 0) sf_transform_release(t); +} + +/* ── TransformStream algorithms (identity transformer) ── */ + +static void sft_set_backpressure(SfTransform *t, bool backpressure) { + if (t->bp_change) { + scr_promise_fulfill_void(t->bp_change); + scr_promise_release(t->bp_change); + } + t->bp_change = scr_promise_new(); + t->t_backpressure = backpressure; + if (t->write_waiting) { + t->write_waiting = false; + sf_transform_defer(t, SFT_WRITE_CONTINUE, 1); + } +} + +static bool sft_readable_wants_chunk(const SfStream *r) { + return r->collector || (r->reader && r->reader->pending_head); +} + +static bool sft_readable_can_enqueue(const SfStream *r) { + return !r->close_requested && !r->closed && !r->error; +} + +/* TransformStreamErrorWritableAndUnblockWrite */ +static void sft_error_writable(SfTransform *t, ScrDyn *error) { + sfw_error_if_needed(t, error); + if (t->t_backpressure) sft_set_backpressure(t, false); +} + +/* The identity transform's enqueue: NULL on success, else +1 error. */ +static ScrDyn *sft_perform_transform(SfTransform *t, ScrDyn *chunk) { + SfStream *r = t->readable; + if (!sft_readable_can_enqueue(r)) { + sf_invalid_state("Unable to enqueue"); + ScrCaught *caught = scr_exc_take(); + ScrDyn *error = scr_caught_to_dyn(caught); + scr_caught_release(caught); + sft_error_writable(t, error); + return error; + } + sf_stream_enqueue_value(r, chunk); + if (scr_exc_pending()) { + ScrCaught *caught = scr_exc_take(); + ScrDyn *error = scr_caught_to_dyn(caught); + scr_caught_release(caught); + sf_stream_error(r, error); + sft_error_writable(t, error); + return error; + } + bool backpressure = !sft_readable_wants_chunk(r); + if (backpressure != t->t_backpressure) sft_set_backpressure(t, true); + return NULL; +} + +static void sft_sink_write(SfTransform *t) { + if (t->t_backpressure) { + t->write_waiting = true; + return; + } + ScrDyn *error = sft_perform_transform(t, t->in_flight->chunk); + if (error) { + scr_dyn_release(t->write_error); + t->write_error = error; + } + sf_transform_defer(t, SFT_WRITE_DONE, 2); +} + +static void sft_sink_close(SfTransform *t) { + if (!t->finish) { + t->finish = scr_promise_new(); + scr_promise_mark_handled(t->finish); + sf_transform_defer(t, SFT_CLOSE_DONE, 1); + return; + } + /* A cancel or abort already finished the transform. */ + sf_transform_defer(t, SFT_CLOSE_DONE, 1); +} + +static void sft_abort_algorithm(SfTransform *t, ScrDyn *reason) { + (void)reason; + if (!t->finish) { + t->finish = scr_promise_new(); + scr_promise_mark_handled(t->finish); + } + sf_transform_defer(t, SFT_ABORT_DONE, 1); +} + +static void sf_transform_step(SfTransform *t, int op) { + switch (op) { + case SFT_WRITE_CONTINUE: { + if (!t->in_flight) return; + if (t->state == SFW_ERRORING) { + ScrDyn *error = scr_dyn_retain(t->stored_error); + scr_dyn_release(t->write_error); + t->write_error = error; + } else { + ScrDyn *error = sft_perform_transform(t, t->in_flight->chunk); + if (error) { + scr_dyn_release(t->write_error); + t->write_error = error; + } + } + sf_transform_defer(t, SFT_WRITE_DONE, 3); + return; + } + case SFT_WRITE_DONE: { + if (!t->in_flight) return; + ScrDyn *error = t->write_error; + t->write_error = NULL; + sfw_finish_write(t, error); + scr_dyn_release(error); + return; + } + case SFT_CLOSE_DONE: { + SfStream *r = t->readable; + ScrDyn *error = NULL; + if (r->error) { + error = scr_dyn_retain(r->error); + } else { + sf_stream_close(r); + } + if (error) sf_reject_promise_reason(t->finish, error); + else scr_promise_fulfill_void(t->finish); + if (t->in_flight_close) sfw_finish_close(t, error); + scr_dyn_release(error); + return; + } + case SFT_ABORT_DONE: { + SfStream *r = t->readable; + ScrDyn *reason = t->pending_abort_reason + ? scr_dyn_retain(t->pending_abort_reason) + : scr_dyn_retain(scr_dyn_undefined()); + bool readable_errored = r->error != NULL; + if (!readable_errored) sf_stream_error(r, reason); + if (t->has_pending_abort) { + t->has_pending_abort = false; + if (readable_errored) { + sf_reject_promise_reason(t->pending_abort, r->error); + } else { + scr_promise_fulfill_void(t->pending_abort); + } + scr_promise_release(t->pending_abort); + t->pending_abort = NULL; + scr_dyn_release(t->pending_abort_reason); + t->pending_abort_reason = NULL; + } + sfw_reject_close_and_closed(t); + scr_dyn_release(reason); + return; + } + case SFT_CANCEL_DONE: { + ScrDyn *reason = t->cancel_reason; + t->cancel_reason = NULL; + if (t->state == SFW_ERRORED) { + sf_reject_promise_reason(t->finish, t->stored_error); + } else { + sfw_error_if_needed(t, reason ? reason : scr_dyn_undefined()); + if (t->t_backpressure) sft_set_backpressure(t, false); + scr_promise_fulfill_void(t->finish); + } + scr_dyn_release(reason); + return; + } + case SFT_START: { + /* Both sides start one turn after construction, the writable side + * first (Node registers its start reaction first). */ + t->started = true; + sfw_advance_queue(t); + SfStream *r = t->readable; + r->started = true; + if (!r->close_requested && !r->closed && !r->error) { + sf_stream_drain(r); + sf_stream_pull(r); + } + return; + } + } +} + +/* The readable side's pull: release backpressure; the pull completes when + * backpressure changes again. */ +static ScrDyn *sft_pull_thunk(ScrClosure *cb, ScrDyn *const *args, + size_t argc) { + (void)args; + (void)argc; + SfTransform *t = (SfTransform *)scr_box_get_ref(cb->caps[0]); + sft_set_backpressure(t, false); + ScrDyn *out = scr_dyn_new_promise(t->bp_change); + sf_transform_release(t); + return out; +} + +/* The readable side's cancel (TransformStreamDefaultSourceCancelAlgorithm): + * errors the writable side with the reason and unblocks a waiting write. */ +static ScrDyn *sft_cancel_thunk(ScrClosure *cb, ScrDyn *const *args, + size_t argc) { + SfTransform *t = (SfTransform *)scr_box_get_ref(cb->caps[0]); + ScrDyn *reason = argc ? args[0] : scr_dyn_undefined(); + if (!t->finish) { + t->finish = scr_promise_new(); + scr_promise_mark_handled(t->finish); + scr_dyn_release(t->cancel_reason); + t->cancel_reason = scr_dyn_retain(reason); + /* The (identity) cancel algorithm's resolved promise reacts a turn + * later. */ + sf_transform_defer(t, SFT_CANCEL_DONE, 1); + } + ScrDyn *out = scr_dyn_new_promise(t->finish); + sf_transform_release(t); + return out; +} + +static ScrDyn *sft_func(SfTransform *t, ScrDynThunk thunk, const char *name) { + ScrClosure *cb = scr_closure_new((void *)thunk, 1); + cb->caps[0] = sf_transform_box(t); + return scr_dyn_new_func(cb, thunk, 1, "(controller)", name); +} + +ScrDyn *scr_fetch_transform_new(void) { + SfTransform *t = calloc(1, sizeof *t); + if (!t) sf_oom(); + t->rc = 1; + t->state = SFW_WRITABLE; + SfStream *r = sf_stream_new_native(); + r->started = false; + r->hwm_zero = true; + r->pull_cb = sft_func(t, &sft_pull_thunk, "pull"); + r->cancel_cb = sft_func(t, &sft_cancel_thunk, "cancel"); + r->source_this = scr_dyn_retain(scr_dyn_undefined()); + sf_stream_track_callbacks(r); + t->readable = r; + /* The writable side's high-water mark of 1 leaves room: no backpressure. + * The readable side's 0 does: writes wait for a read. */ + t->backpressure = false; + sft_set_backpressure(t, true); + sf_transform_defer(t, SFT_START, 1); + ScrDyn *out = scr_dyn_new_handle(t, SCR_DYNH_WEB_TRANSFORM); + sf_transform_release(t); + return out; +} + +static ScrDyn *sf_transform_get(void *ptr, const char *key, size_t len) { + SfTransform *t = ptr; + if (sf_name(key, len, "readable")) { + return scr_dyn_new_handle(t->readable, SCR_DYNH_WEB_STREAM); + } + if (sf_name(key, len, "writable")) { + return scr_dyn_new_handle(t, SCR_DYNH_WEB_WRITABLE); + } + return NULL; +} + +static ScrDyn *sf_transform_invoke(void *ptr, ScrDyn *self, + const char *method, ScrDyn *const *args, + size_t argc, const char *what) { + (void)ptr; + (void)self; + (void)args; + (void)argc; + (void)method; + sf_not_function(what); + return NULL; +} + +static ScrDyn *sf_promise_dyn(ScrPromise *p) { + ScrDyn *out = scr_dyn_new_promise(p); + scr_promise_release(p); + return out; +} + +static ScrDyn *sf_writable_invoke(void *ptr, ScrDyn *self, + const char *method, ScrDyn *const *args, + size_t argc, const char *what) { + SfTransform *t = ptr; + (void)self; + if (strcmp(method, "getWriter") == 0) { + SfWriter *w = sfw_get_writer(t); + if (!w) return NULL; + ScrDyn *out = scr_dyn_new_handle(w, SCR_DYNH_WEB_WRITER); + sf_writer_release(w); + return out; + } + if (strcmp(method, "abort") == 0) { + if (t->writer) { + return sf_promise_dyn( + sf_promise_invalid_state("WritableStream is locked")); + } + return sf_promise_dyn(sfw_abort(t, argc ? args[0] : scr_dyn_undefined())); + } + if (strcmp(method, "close") == 0) { + if (t->writer) { + return sf_promise_dyn( + sf_promise_invalid_state("WritableStream is locked")); + } + if (sfw_close_queued_or_in_flight(t)) { + return sf_promise_dyn( + sf_promise_invalid_state("WritableStream is closed")); + } + return sf_promise_dyn(sfw_close(t)); + } + sf_not_function(what); + return NULL; +} + +static ScrDyn *sf_writable_get(void *ptr, const char *key, size_t len) { + SfTransform *t = ptr; + if (sf_name(key, len, "locked")) return scr_dyn_new_bool(t->writer != NULL); + return NULL; +} + +static ScrDyn *sf_writer_invoke(void *ptr, ScrDyn *self, const char *method, + ScrDyn *const *args, size_t argc, + const char *what) { + SfWriter *w = ptr; + (void)self; + if (strcmp(method, "write") == 0) { + return sf_promise_dyn( + sfw_writer_write(w, argc ? args[0] : scr_dyn_undefined())); + } + if (strcmp(method, "close") == 0) { + SfTransform *t = w->stream; + if (!t) { + return sf_promise_dyn(sf_promise_invalid_state( + "Writer is not bound to a WritableStream")); + } + if (sfw_close_queued_or_in_flight(t)) { + return sf_promise_dyn( + sf_promise_invalid_state("WritableStream is closed")); + } + return sf_promise_dyn(sfw_close(t)); + } + if (strcmp(method, "abort") == 0) { + SfTransform *t = w->stream; + if (!t) { + return sf_promise_dyn(sf_promise_invalid_state( + "Writer is not bound to a WritableStream")); + } + return sf_promise_dyn(sfw_abort(t, argc ? args[0] : scr_dyn_undefined())); + } + if (strcmp(method, "releaseLock") == 0) { + sfw_writer_release_lock(w); + return scr_dyn_retain(scr_dyn_undefined()); + } + sf_not_function(what); + return NULL; +} + +static ScrDyn *sf_writer_get(void *ptr, const char *key, size_t len) { + SfWriter *w = ptr; + if (sf_name(key, len, "ready")) return scr_dyn_new_promise(w->ready); + if (sf_name(key, len, "closed")) return scr_dyn_new_promise(w->closed); + if (sf_name(key, len, "desiredSize")) { + SfTransform *t = w->stream; + if (!t) { + sf_invalid_state("Writer is not bound to a WritableStream"); + return NULL; + } + if (t->state == SFW_ERRORED || t->state == SFW_ERRORING) { + return scr_dyn_new_null(); + } + if (t->state == SFW_CLOSED) return scr_dyn_new_num(0); + return scr_dyn_new_num(1.0 - (double)t->queue_size); + } + return NULL; +} + /* ── Response ────────────────────────────────────────────────────── */ static SfResponse *sf_response_retain(SfResponse *r) { @@ -5675,6 +6575,13 @@ static const ScrDynHandleMethod sf_headers_methods[] = { {"has", 1, false}, {"keys", 0, false}, {"set", 2, false}, {"values", 0, false}, {NULL, 0, false}}; +static const ScrDynHandleMethod sf_writable_methods[] = { + {"abort", 0, true}, {"close", 0, true}, {"getWriter", 0, false}, + {NULL, 0, false}}; +static const ScrDynHandleMethod sf_writer_methods[] = { + {"abort", 0, true}, {"close", 0, true}, {"releaseLock", 0, false}, + {"write", 0, true}, {NULL, 0, false}}; + static ScrDyn *sf_rejected(void) { ScrPromise *p = scr_promise_settled_ref(NULL, &scr_dyn_retain_v, &scr_dyn_release_v, NULL); @@ -5721,6 +6628,23 @@ static const ScrDynHandleOps sf_headers_ops = { &sf_headers_invoke, &sf_headers_get, &sf_no_set, NULL, NULL, &sf_headers_step, .methods = sf_headers_methods}; +static const ScrDynHandleOps sf_transform_ops = { + "TransformStream", &sf_transform_retain_v, &sf_transform_release_v, + &sf_transform_invoke, &sf_transform_get, &sf_no_set, NULL}; +static const ScrDynHandleOps sf_writable_ops = { + "WritableStream", &sf_transform_retain_v, &sf_transform_release_v, + &sf_writable_invoke, &sf_writable_get, &sf_no_set, NULL, + .methods = sf_writable_methods, + .invalid_this = "Value of \"this\" must be of type WritableStream", + .rejected = &sf_rejected}; +static const ScrDynHandleOps sf_writer_ops = { + "WritableStreamDefaultWriter", &sf_writer_handle_retain_v, + &sf_writer_handle_release_v, &sf_writer_invoke, &sf_writer_get, + &sf_no_set, NULL, .methods = sf_writer_methods, + .invalid_this = + "Value of \"this\" must be of type WritableStreamDefaultWriter", + .rejected = &sf_rejected}; + static void sf_teardown(void) { while (sf_live) { SfTransfer *t = sf_live; @@ -5760,6 +6684,9 @@ void scr_fetch_install(void) { scr_dyn_handle_install(SCR_DYNH_FETCH_REQUEST, &sf_request_ops); scr_dyn_handle_install(SCR_DYNH_FETCH_RESPONSE, &sf_response_ops); scr_dyn_handle_install(SCR_DYNH_FETCH_HEADERS, &sf_headers_ops); + scr_dyn_handle_install(SCR_DYNH_WEB_TRANSFORM, &sf_transform_ops); + scr_dyn_handle_install(SCR_DYNH_WEB_WRITABLE, &sf_writable_ops); + scr_dyn_handle_install(SCR_DYNH_WEB_WRITER, &sf_writer_ops); scr_et_signal_core_fn = sf_signal_core_of; scr_atexit(sf_teardown); } diff --git a/packages/runtime/src/scr_generator.h b/packages/runtime/src/scr_generator.h index 828f339d5..c73ceb900 100644 --- a/packages/runtime/src/scr_generator.h +++ b/packages/runtime/src/scr_generator.h @@ -192,6 +192,7 @@ ScrDyn *scr_fetch_abort_timeout(ScrDyn *delay); ScrDyn *scr_fetch_abort_now(ScrDyn *reason); /* borrowed reason; +1 handle */ ScrDyn *scr_fetch_abort_any(ScrDyn *signals); /* borrowed array; +1 handle or NULL pending */ ScrDyn *scr_fetch_stream_new(ScrDyn *source); /* borrowed source; +1 handle or NULL pending */ +ScrDyn *scr_fetch_transform_new(void); /* +1 identity TransformStream handle */ ScrDyn *scr_fetch_stream_from(ScrDyn *iterable); /* borrowed iterable; +1 handle or NULL pending */ ScrDyn *scr_fetch_stream_from_array( ScrArr *iterable, diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 14d0a901f..f23814526 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -15054,6 +15054,7 @@ static int scr_weak_key(const ScrDyn *key, void **ptr, unsigned *kind) { key->v.handle.tag != SCR_DYNH_WEAK_SET && key->v.handle.tag != SCR_DYNH_FINALIZATION_REGISTRY && key->v.handle.tag != SCR_DYNH_STDIO && key->v.handle.tag != SCR_DYNH_FETCH_REQUEST && key->v.handle.tag != SCR_DYNH_FETCH_RESPONSE && + key->v.handle.tag != SCR_DYNH_WEB_STREAM && key->v.handle.tag != SCR_DYNH_MAP && key->v.handle.tag != SCR_DYNH_SET && key->v.handle.tag != SCR_DYNH_NET_SOCKET && key->v.handle.tag != SCR_DYNH_NET_SERVER) return -1; *ptr = key->v.handle.ptr; *kind = 4 + key->v.handle.tag; return 1; diff --git a/tests/corpus/7743-transform-stream-identity.ts b/tests/corpus/7743-transform-stream-identity.ts new file mode 100644 index 000000000..d98303997 --- /dev/null +++ b/tests/corpus/7743-transform-stream-identity.ts @@ -0,0 +1,212 @@ +// The identity TransformStream: writes wait for reads (the readable side's +// high-water mark is 0), the writer's desiredSize/ready track the writable +// side's queue, close/abort/cancel propagate across the pair, and the +// readable side feeds a Response body (the StreamingApi pattern of Hono's +// stream helpers). +const enc = new TextEncoder(); +const dec = new TextDecoder(); + +function describe(e: unknown): string { + if (e instanceof Error) { + const code = (e as NodeJS.ErrnoException).code; + return `${e.name}: ${e.message}${code === undefined ? "" : ` [${code}]`}`; + } + return `value ${String(e)}`; +} + +async function basics(): Promise { + const ts = new TransformStream(); + const writer = ts.writable.getWriter(); + const reader = ts.readable.getReader(); + console.log("desired", writer.desiredSize, ts.writable.locked, ts.readable.locked); + const written = writer.write("a").then(() => console.log("write a done")); + console.log("desired after write", writer.desiredSize); + void writer.ready.then(() => console.log("ready")); + const first = await reader.read(); + console.log("read", first.done, first.value); + await written; + console.log("desired", writer.desiredSize); + const closing = writer.close().then(() => console.log("closed")); + const last = await reader.read(); + console.log("read", last.done, last.value); + await closing; + await writer.closed; + console.log("writer.closed resolved"); + try { + await writer.write("b"); + } catch (e) { + console.log("write after close:", describe(e)); + } + try { + ts.writable.getWriter(); + } catch (e) { + console.log("getWriter locked:", describe(e)); + } + writer.releaseLock(); + try { + await writer.write("c"); + } catch (e) { + console.log("released write:", describe(e)); + } + try { + await writer.closed; + } catch (e) { + console.log("released closed:", describe(e)); + } +} + +async function abortWriter(): Promise { + const ts = new TransformStream(); + const writer = ts.writable.getWriter(); + const reader = ts.readable.getReader(); + const pending = writer.write("z").catch((e: unknown) => console.log("pending write:", describe(e))); + await writer.abort("boom").then(() => console.log("abort resolved")); + await pending; + try { + await reader.read(); + } catch (e) { + console.log("read after abort:", describe(e)); + } + try { + await writer.closed; + } catch (e) { + console.log("closed after abort:", describe(e)); + } + try { + await writer.ready; + } catch (e) { + console.log("ready after abort:", describe(e)); + } + console.log("desired after abort", writer.desiredSize); +} + +async function cancelReadable(): Promise { + const ts = new TransformStream(); + const writer = ts.writable.getWriter(); + const pending = writer.write("q").catch((e: unknown) => console.log("write rejected by cancel:", describe(e))); + await ts.readable.cancel("nope").then(() => console.log("cancel resolved")); + await pending; + try { + await writer.closed; + } catch (e) { + console.log("writer.closed after cancel:", describe(e)); + } +} + +async function intoResponse(): Promise { + const ts = new TransformStream(); + const writer = ts.writable.getWriter(); + void (async (): Promise => { + for (const s of ["x", "y", "z"]) { + await writer.write(enc.encode(s)); + console.log("wrote", s); + } + await writer.close(); + console.log("writer closed"); + })(); + console.log("text", await new Response(ts.readable).text()); +} + +/** Hono's StreamingApi: a writer on the writable side and a pull-based + * ReadableStream over a reader of the readable side. */ +interface StreamingApi { + readonly responseReadable: ReadableStream; + write(input: string): Promise; + close(): Promise; + state(): string; +} + +function streamingApi(writable: WritableStream, readable: ReadableStream): StreamingApi { + const writer = writable.getWriter(); + const reader = readable.getReader(); + let closed = false; + let aborted = false; + const responseReadable = new ReadableStream({ + async pull(controller): Promise { + const { done, value } = await reader.read(); + if (done) controller.close(); + else controller.enqueue(value); + }, + cancel: (): void => { + if (!closed) aborted = true; + }, + }); + return { + responseReadable, + async write(input: string): Promise { + try { + await writer.write(enc.encode(input)); + } catch { + // a closed or aborted stream drops the write + } + }, + async close(): Promise { + closed = true; + try { + await writer.close(); + } catch { + // already closed + } + }, + state: (): string => `closed=${String(closed)} aborted=${String(aborted)}`, + }; +} + +async function streamedResponse(): Promise { + const { readable, writable } = new TransformStream(); + const api = streamingApi(writable, readable); + void (async (): Promise => { + try { + for (let i = 0; i < 3; i++) { + await api.write(`data: ${i}\n\n`); + } + } finally { + await api.close(); + } + })(); + const response = new Response(api.responseReadable, { headers: { "content-type": "text/event-stream" } }); + const reader = response.body!.getReader(); + let chunks = 0; + let text = ""; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + chunks++; + text += dec.decode(value); + } + console.log("streamed", chunks, JSON.stringify(text), api.state()); +} + +async function writableMethods(): Promise { + const a = new TransformStream(); + await a.writable.abort("r").then(() => console.log("writable.abort ok")); + const b = new TransformStream(); + const writer = b.writable.getWriter(); + try { + await b.writable.close(); + } catch (e) { + console.log("close locked:", describe(e)); + } + try { + await b.writable.abort(); + } catch (e) { + console.log("abort locked:", describe(e)); + } + writer.releaseLock(); + const reader = b.readable.getReader(); + const closing = b.writable.close().then(() => console.log("writable.close ok")); + const end = await reader.read(); + console.log("readable done", end.done); + await closing; +} + +async function main(): Promise { + await basics(); + await abortWriter(); + await cancelReadable(); + await intoResponse(); + await streamedResponse(); + await writableMethods(); +} + +await main(); diff --git a/tests/harness/corpus-costs/7743-transform-stream-identity.ts.json b/tests/harness/corpus-costs/7743-transform-stream-identity.ts.json new file mode 100644 index 000000000..0c31a05f7 --- /dev/null +++ b/tests/harness/corpus-costs/7743-transform-stream-identity.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 4000 } From fab4b403e9058092d9dac1b1fb28bd39bba9946d Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 09:07:41 -0700 Subject: [PATCH 061/184] Give corpus scratch paths a per-process name The differential and LLVM differential suites can run the same corpus program's Node oracle at the same time in one shard. Programs whose scratch directory or file was derived only from argv[1] collided, and one oracle removed the paths under the other. Their names now include the pid, as 1403 and 5567 already do. Printed output never carries the name. Co-Authored-By: Claude Sonnet 5.5 --- tests/corpus/1006-json-fs-config.ts | 2 +- tests/corpus/1354-builtin-bare-specifiers.ts | 2 +- tests/corpus/1357-fsp-roundtrip.ts | 2 +- tests/corpus/2844-node-disposable-resources.ts | 2 +- tests/corpus/5566-fs-dirent-dyn-boundaries/main.mjs | 2 +- tests/corpus/957-builtins-namespace.ts | 2 +- tests/corpus/992-fs-roundtrip.ts | 2 +- tests/corpus/993-fs-readdir.ts | 2 +- tests/corpus/994-fs-errors.ts | 2 +- tests/corpus/996-fs-rc-stress.ts | 2 +- tests/corpus/997-fs-modules/store.ts | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/corpus/1006-json-fs-config.ts b/tests/corpus/1006-json-fs-config.ts index f2b1d48c0..1783202c9 100644 --- a/tests/corpus/1006-json-fs-config.ts +++ b/tests/corpus/1006-json-fs-config.ts @@ -11,7 +11,7 @@ function tail(path: string): string { } return path.slice(i + 1); } -const path = "tmp-1006-" + tail(process.argv[1]) + ".json"; +const path = "tmp-1006-" + tail(process.argv[1]) + "-" + process.pid + ".json"; type Config = { name: string; diff --git a/tests/corpus/1354-builtin-bare-specifiers.ts b/tests/corpus/1354-builtin-bare-specifiers.ts index cd0db6984..826c3e7a0 100644 --- a/tests/corpus/1354-builtin-bare-specifiers.ts +++ b/tests/corpus/1354-builtin-bare-specifiers.ts @@ -15,7 +15,7 @@ function tail(path: string): string { return path.slice(i + 1); } -const scratch = "tmp-1354-" + tail(process.argv[1]); +const scratch = "tmp-1354-" + tail(process.argv[1]) + "-" + process.pid; if (existsSync(scratch)) { if (existsSync(j(scratch, "f.txt"))) { rmSync(j(scratch, "f.txt")); diff --git a/tests/corpus/1357-fsp-roundtrip.ts b/tests/corpus/1357-fsp-roundtrip.ts index 4ccae7f3a..7368da360 100644 --- a/tests/corpus/1357-fsp-roundtrip.ts +++ b/tests/corpus/1357-fsp-roundtrip.ts @@ -15,7 +15,7 @@ function tail(path: string): string { } async function main(): Promise { - const scratch = "tmp-1357-" + tail(process.argv[1]); + const scratch = "tmp-1357-" + tail(process.argv[1]) + "-" + process.pid; if (existsSync(scratch)) { if (existsSync(scratch + "/a.txt")) { rmSync(scratch + "/a.txt"); diff --git a/tests/corpus/2844-node-disposable-resources.ts b/tests/corpus/2844-node-disposable-resources.ts index 042f617ce..ebeda87a2 100644 --- a/tests/corpus/2844-node-disposable-resources.ts +++ b/tests/corpus/2844-node-disposable-resources.ts @@ -14,7 +14,7 @@ async function main(): Promise { console.log(timeout.hasRef(), immediate.hasRef()); } - const path = "/tmp/scriptc-disposable-file.txt"; + const path = `/tmp/scriptc-disposable-file-${process.pid}.txt`; await writeFile(path, "resource"); { await using handle = await open(path, "r"); diff --git a/tests/corpus/5566-fs-dirent-dyn-boundaries/main.mjs b/tests/corpus/5566-fs-dirent-dyn-boundaries/main.mjs index 95bcbecee..77bdf8905 100644 --- a/tests/corpus/5566-fs-dirent-dyn-boundaries/main.mjs +++ b/tests/corpus/5566-fs-dirent-dyn-boundaries/main.mjs @@ -8,7 +8,7 @@ function tail(path) { while (i >= 0 && path.charAt(i) !== "/" && path.charAt(i) !== "\\") i = i - 1; return path.slice(i + 1); } -const dir = `/tmp/scr-dirent-dyn-${tail(process.argv[1] === undefined ? "x" : process.argv[1])}`; +const dir = `/tmp/scr-dirent-dyn-${tail(process.argv[1] === undefined ? "x" : process.argv[1])}-${process.pid}`; if (fs.existsSync(dir)) fs.rmSync(dir, { recursive: true, force: true }); fs.mkdirSync(`${dir}/sub`, { recursive: true }); fs.writeFileSync(`${dir}/a.txt`, "x"); diff --git a/tests/corpus/957-builtins-namespace.ts b/tests/corpus/957-builtins-namespace.ts index 4b27bfe79..f6980cda6 100644 --- a/tests/corpus/957-builtins-namespace.ts +++ b/tests/corpus/957-builtins-namespace.ts @@ -17,7 +17,7 @@ function tail(p: string): string { return p.slice(i + 1); } -const scratch = "tmp-957-" + tail(process.argv[1]); +const scratch = "tmp-957-" + tail(process.argv[1]) + "-" + process.pid; if (fs.existsSync(scratch)) { fs.rmSync(scratch + "/a.txt"); fs.rmdirSync(scratch); diff --git a/tests/corpus/992-fs-roundtrip.ts b/tests/corpus/992-fs-roundtrip.ts index a039c936f..92b527fb1 100644 --- a/tests/corpus/992-fs-roundtrip.ts +++ b/tests/corpus/992-fs-roundtrip.ts @@ -32,7 +32,7 @@ function freshDir(dir: string): void { } mkdirSync(dir); } -const scratch = "tmp-992-" + tail(process.argv[1]); +const scratch = "tmp-992-" + tail(process.argv[1]) + "-" + process.pid; freshDir(scratch); const file = scratch + "/notes.txt"; diff --git a/tests/corpus/993-fs-readdir.ts b/tests/corpus/993-fs-readdir.ts index e948d3878..b0409f2ef 100644 --- a/tests/corpus/993-fs-readdir.ts +++ b/tests/corpus/993-fs-readdir.ts @@ -26,7 +26,7 @@ function freshDir(dir: string): void { } mkdirSync(dir); } -const scratch = "tmp-993-" + tail(process.argv[1]); +const scratch = "tmp-993-" + tail(process.argv[1]) + "-" + process.pid; freshDir(scratch); console.log(readdirSync(scratch).length); // fresh dir is empty diff --git a/tests/corpus/994-fs-errors.ts b/tests/corpus/994-fs-errors.ts index 74a1ed74e..d317c1aba 100644 --- a/tests/corpus/994-fs-errors.ts +++ b/tests/corpus/994-fs-errors.ts @@ -30,7 +30,7 @@ function freshDir(dir: string): void { } mkdirSync(dir); } -const scratch = "tmp-994-" + tail(process.argv[1]); +const scratch = "tmp-994-" + tail(process.argv[1]) + "-" + process.pid; freshDir(scratch); let caught = 0; diff --git a/tests/corpus/996-fs-rc-stress.ts b/tests/corpus/996-fs-rc-stress.ts index d6a8f0ba3..063845309 100644 --- a/tests/corpus/996-fs-rc-stress.ts +++ b/tests/corpus/996-fs-rc-stress.ts @@ -28,7 +28,7 @@ function freshDir(dir: string): void { } mkdirSync(dir); } -const scratch = "tmp-996-" + tail(process.argv[1]); +const scratch = "tmp-996-" + tail(process.argv[1]) + "-" + process.pid; freshDir(scratch); // Contents into an array, back through map/filter/join. diff --git a/tests/corpus/997-fs-modules/store.ts b/tests/corpus/997-fs-modules/store.ts index 65e59df1b..1b2a0f1fe 100644 --- a/tests/corpus/997-fs-modules/store.ts +++ b/tests/corpus/997-fs-modules/store.ts @@ -17,7 +17,7 @@ export function scratchDir(tag: string): string { while (i >= 0 && script.charAt(i) !== "/" && script.charAt(i) !== "\\") { i = i - 1; } - const dir = tag + script.slice(i + 1); + const dir = tag + script.slice(i + 1) + "-" + process.pid; if (existsSync(dir)) { for (const name of readdirSync(dir)) { rmSync(dir + "/" + name); From 5a829bd225082bae509c41aede5be68e5a2c9d83 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 11:16:10 -0500 Subject: [PATCH 062/184] Compile unions of typed-array classes --- packages/compiler/src/backend/llvm/emitter.ts | 18 +- .../compiler/src/backend/llvm/union-repr.ts | 21 +- .../lowering/containers/array-construction.ts | 5 + .../src/frontend/lowering/containers/bytes.ts | 60 ++ .../lowering/expressions/array-literals.ts | 5 + .../lowering/expressions/element-access.ts | 103 ++++ .../lowering/expressions/property-access.ts | 8 + .../frontend/lowering/expressions/updates.ts | 38 ++ .../src/frontend/lowering/indexed-unions.ts | 538 ++++++++++++++++++ .../src/frontend/lowering/lower-stmts.ts | 94 +-- packages/compiler/src/ir/integer-loops.ts | 21 +- .../corpus/7745-typed-array-unions.ts.json | 6 + .../7747-typed-array-class-values.ts.json | 6 + ...48-typed-array-number-array-unions.ts.json | 6 + .../test/typed-array-union-emission.test.ts | 56 ++ tests/corpus/7745-typed-array-unions.ts | 71 +++ tests/corpus/7747-typed-array-class-values.ts | 42 ++ .../7748-typed-array-number-array-unions.ts | 78 +++ .../7745-typed-array-unions.ts.json | 1 + .../7747-typed-array-class-values.ts.json | 1 + ...48-typed-array-number-array-unions.ts.json | 1 + 21 files changed, 1131 insertions(+), 48 deletions(-) create mode 100644 packages/compiler/src/frontend/lowering/indexed-unions.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7745-typed-array-unions.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7747-typed-array-class-values.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7748-typed-array-number-array-unions.ts.json create mode 100644 packages/compiler/test/typed-array-union-emission.test.ts create mode 100644 tests/corpus/7745-typed-array-unions.ts create mode 100644 tests/corpus/7747-typed-array-class-values.ts create mode 100644 tests/corpus/7748-typed-array-number-array-unions.ts create mode 100644 tests/harness/corpus-costs/7745-typed-array-unions.ts.json create mode 100644 tests/harness/corpus-costs/7747-typed-array-class-values.ts.json create mode 100644 tests/harness/corpus-costs/7748-typed-array-number-array-unions.ts.json diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 4aa9b65ad..5f51a19ca 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -3803,8 +3803,17 @@ export class LlEmitter { /** A union value's tag (i32): the box's tag word, or the NULL test of a * nullable union's pointer. */ + /** A box's tag and payload words are written only where the box is + * built (runtime constructors and untagged stack stores), so their loads + * carry their own alias tags: element and field stores cannot change + * them, and loops keep them hoisted. */ unionTag(uName: string, unionId: string): string { - return emitUnionTag(this.B, this.nullableUnions.get(unionId), uName); + return emitUnionTag( + this.B, + this.nullableUnions.get(unionId), + uName, + this.memoryAliasAttachment("union.tag"), + ); } /** Guard for helpers that build or read a union's `%ScrUnion` box @@ -3820,7 +3829,12 @@ export class LlEmitter { /** The BORROWED payload pointer of a ref arm (scr_union_peek inlined — * the runtime's is a static inline); a nullable union is its payload. */ unionPeek(uName: string, unionId: string): string { - return emitUnionPeek(this.B, this.nullableUnions.get(unionId), uName); + return emitUnionPeek( + this.B, + this.nullableUnions.get(unionId), + uName, + this.memoryAliasAttachment("union.payload"), + ); } /** Emits `switch` over a union's tag with one block per arm; each arm diff --git a/packages/compiler/src/backend/llvm/union-repr.ts b/packages/compiler/src/backend/llvm/union-repr.ts index 959f9e51e..8c6fb4c85 100644 --- a/packages/compiler/src/backend/llvm/union-repr.ts +++ b/packages/compiler/src/backend/llvm/union-repr.ts @@ -8,8 +8,14 @@ import type { BlockBuilder } from "./blocks.js"; import { NULLABLE_NULL, shapeTagFn, type PointerUnion } from "./nullable-unions.js"; -/** The i32 tag of a union value. */ -export function emitUnionTag(B: BlockBuilder, nullable: PointerUnion | null, u: string): string { +/** The i32 tag of a union value. `alias` is the tag word's alias + * attachment (", !tbaa !N"), when the caller has one. */ +export function emitUnionTag( + B: BlockBuilder, + nullable: PointerUnion | null, + u: string, + alias = "", +): string { const t = B.tmp(); if (nullable?.repr === "shapes") { B.line(`${t} = call i32 ${shapeTagFn(nullable.unionId)}(ptr ${u})`); @@ -31,7 +37,7 @@ export function emitUnionTag(B: BlockBuilder, nullable: PointerUnion | null, u: } const p = B.tmp(); B.line(`${p} = getelementptr inbounds %ScrUnion, ptr ${u}, i64 0, i32 1`); - B.line(`${t} = load i32, ptr ${p}`); + B.line(`${t} = load i32, ptr ${p}${alias}`); return t; } @@ -81,12 +87,17 @@ export function emitNullableIsTag( } /** The BORROWED payload pointer of a reference arm. */ -export function emitUnionPeek(B: BlockBuilder, nullable: PointerUnion | null, u: string): string { +export function emitUnionPeek( + B: BlockBuilder, + nullable: PointerUnion | null, + u: string, + alias = "", +): string { if (nullable) return u; const p = B.tmp(); const t = B.tmp(); B.line(`${p} = getelementptr inbounds %ScrUnion, ptr ${u}, i64 0, i32 5`); - B.line(`${t} = load ptr, ptr ${p}`); + B.line(`${t} = load ptr, ptr ${p}${alias}`); return t; } diff --git a/packages/compiler/src/frontend/lowering/containers/array-construction.ts b/packages/compiler/src/frontend/lowering/containers/array-construction.ts index 1f10640c0..1ca51f526 100644 --- a/packages/compiler/src/frontend/lowering/containers/array-construction.ts +++ b/packages/compiler/src/frontend/lowering/containers/array-construction.ts @@ -1,4 +1,5 @@ import { keyedHelperName } from "../stable-names.js"; +import { indexedUnionArms, lowerIndexedUnionToArray } from "../indexed-unions.js"; import { generatorDrain } from "../iterator-adapters.js"; import { identityPreservingWidening } from "../coercions/identity.js"; import { liftedFreshSpread } from "../coercions/fresh-spread.js"; @@ -1054,6 +1055,10 @@ export function lowerArrayFromCall( loc, }; } + // A union of typed arrays and number arrays (indexed-unions.ts). + const indexedArms = indexedUnionArms(lowerer, src.type); + if (indexedArms && args.length === 1) + return lowerIndexedUnionToArray(lowerer, src, indexedArms, loc, true); if (src.type.kind === "set" && args.length === 1) { return { kind: "setIntrinsic", diff --git a/packages/compiler/src/frontend/lowering/containers/bytes.ts b/packages/compiler/src/frontend/lowering/containers/bytes.ts index 995cb177a..11f56986b 100644 --- a/packages/compiler/src/frontend/lowering/containers/bytes.ts +++ b/packages/compiler/src/frontend/lowering/containers/bytes.ts @@ -1,4 +1,5 @@ import { keyedHelperName } from "../stable-names.js"; +import { allBytesArms, indexedUnionArms, lowerIndexedUnionMethod } from "../indexed-unions.js"; import { boolLit, dynUndefinedExpr, numLit, varRef } from "../../../ir/build.js"; import { BUF_NUM_METHODS } from "./buffer-numeric-methods.js"; import * as ts from "../../ts7/adapter.js"; @@ -910,6 +911,62 @@ function lowerDataViewNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr { * variable-width read/writeUIntLE quartet — BUF_NUM_METHODS). Everything * else the lib declares (fill, indexOf, reverse, ...) falls through to * the SC2020 member fence. Null when this isn't a bytes method call. */ +/** The element-kind-generic typed-array methods on a union of typed + * arrays (indexed-unions.ts): the arguments lower as for one typed array, + * and each arm runs the method on its own receiver. Null for the methods + * that need a narrowing to one kind first. */ +function lowerUnionBytesMethodCall( + lowerer: Lowerer, + call: ts.CallExpression, + access: ts.PropertyAccessExpression, + name: string, +): IrExpr | null { + if (isJsSourceFile(ts.getSourceFileOfNode(call))) return null; + const nodes = call.arguments.nodes; + if (nodes.some(ts.isSpreadElement)) return null; + const loc = locOf(call); + const num = (value: number): IrExpr => ({ kind: "numLit", value, type: F64, loc }); + const optional = (i: number, fallback: IrExpr, type: IrType = F64): IrExpr => + nodes[i] ? lowerOptionalArgument(lowerer, nodes[i]!, type, fallback) : fallback; + let method: IrBytesIntrinsicMethod; + let args: () => IrExpr[]; + let result: "receiver" | IrType; + if ((name === "slice" || name === "subarray") && nodes.length <= 2) { + method = name; + args = () => [optional(0, num(0)), optional(1, num(Infinity))]; + result = "receiver"; + } else if (name === "fill" && nodes.length <= 3) { + method = "fillElem"; + args = () => [optional(0, num(0)), optional(1, num(0)), optional(2, num(Infinity))]; + result = "receiver"; + } else if (name === "copyWithin" && nodes.length <= 3) { + method = "copyWithin"; + args = () => [optional(0, num(0)), optional(1, num(0)), optional(2, num(Infinity))]; + result = "receiver"; + } else if (name === "toReversed" && nodes.length === 0) { + method = "toReversed"; + args = () => []; + result = "receiver"; + } else if (name === "join" && nodes.length <= 1) { + method = "join"; + args = () => [optional(0, { kind: "strLit", value: ",", type: STRING, loc }, STRING)]; + result = STRING; + } else if (name === "set" && (nodes.length === 1 || nodes.length === 2)) { + const source = lowerer.mapTypeOf(lowerer.typeOf(nodes[0]!)); + method = source?.kind === "bytes" ? "setFrom" : "setFromDyn"; + args = () => { + const src = lowerer.lowerExpr(nodes[0]!); + const first = src.type.kind === "bytes" ? src : lowerer.coerceInto(nodes[0]!, src, DYN); + return nodes.length === 2 ? [first, optional(1, num(0))] : [first]; + }; + result = VOID; + } else return null; + const receiver = lowerer.lowerExpr(access.expression); + const arms = indexedUnionArms(lowerer, receiver.type); + if (!arms || !allBytesArms(arms)) return null; + return lowerIndexedUnionMethod(lowerer, receiver, arms, method, args(), result, loc); +} + export function lowerBytesMethodCall( lowerer: Lowerer, call: ts.CallExpression, @@ -918,6 +975,9 @@ export function lowerBytesMethodCall( if (lowerer.chainBlocked(access, call)) return null; const name = access.name.text; const receiverIr = lowerer.mapTypeOf(lowerer.typeOf(access.expression)); + const unionArms = receiverIr ? indexedUnionArms(lowerer, receiverIr) : null; + if (unionArms && allBytesArms(unionArms) && lowerer.isStdlibMember(access)) + return lowerUnionBytesMethodCall(lowerer, call, access, name); if (receiverIr?.kind !== "bytes") return null; if (!lowerer.isStdlibMember(access)) return null; const loc = locOf(call); diff --git a/packages/compiler/src/frontend/lowering/expressions/array-literals.ts b/packages/compiler/src/frontend/lowering/expressions/array-literals.ts index 6c7ce814e..c1b54a417 100644 --- a/packages/compiler/src/frontend/lowering/expressions/array-literals.ts +++ b/packages/compiler/src/frontend/lowering/expressions/array-literals.ts @@ -1,4 +1,5 @@ /* Array literals. */ +import { indexedUnionArms, lowerIndexedUnionToArray } from "../indexed-unions.js"; import * as ts from "../../ts7/adapter.js"; import { type Lowerer, neverTaintedJsType } from "../lowerer.js"; import { notIterableSpelling } from "../iterable-spelling.js"; @@ -504,6 +505,10 @@ export function lowerArrayLiteral( loc: locOf(el), }; } + // A union of typed arrays and number arrays (indexed-unions.ts): + // each arm's elements as a number[]. + const indexedArms = indexedUnionArms(lowerer, src.type); + if (indexedArms) src = lowerIndexedUnionToArray(lowerer, src, indexedArms, locOf(el)); // `[...obj]` over a class whose [Symbol.iterator] is a generator // method spreads the generator that method returns. src = classGeneratorIterable(lowerer, src) ?? src; diff --git a/packages/compiler/src/frontend/lowering/expressions/element-access.ts b/packages/compiler/src/frontend/lowering/expressions/element-access.ts index 0ea94ec7a..e61e7f434 100644 --- a/packages/compiler/src/frontend/lowering/expressions/element-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/element-access.ts @@ -71,6 +71,13 @@ import { isClassCallback } from "../class-callbacks.js"; import { isKeyedFieldAccess } from "./keyed-field-targets.js"; import { lowerFieldCompoundValue } from "./updates.js"; import { readsHeldEvolvingArray } from "../lower-stmts.js"; +import { + type IndexedArm, + indexedUnionArms, + lowerIndexedUnionNumberRead, + lowerIndexedUnionRead, + lowerIndexedUnionWrite, +} from "../indexed-unions.js"; /** `a[i]` reads. Only f64 indices into array receivers are modeled; JS * string-key element access (`a["length"]`) and string indexing (`s[0]` @@ -299,6 +306,31 @@ export function lowerElementAccess(lowerer: Lowerer, expr: ts.ElementAccessExpre } } + // An element read of a union of typed arrays and number arrays: each + // arm's own read, dispatched on the tag (indexed-unions.ts). + const indexedArms = receiverIr ? indexedUnionArms(lowerer, receiverIr) : null; + if (indexedArms && receiverIr && !isJsSourceFile(ts.getSourceFileOfNode(expr))) { + let recv = lowerer.lowerExpr(expr.expression); + if (recv.type.kind === "dyn") recv = lowerer.coerceInto(expr.expression, recv, receiverIr); + const arms = indexedUnionArms(lowerer, recv.type); + if (arms) { + const index = lowerOptionalNumber( + lowerer, + lowerer.lowerExpr(expr.argumentExpression), + locOf(expr.argumentExpression), + expr.argumentExpression, + ); + if (index.type.kind !== "f64") + lowerer.unsupported("SC1090", expr.argumentExpression, "indexing with non-number keys"); + if ( + (ts.isNonNullExpression(expr.parent) && expr.parent.expression === expr) || + lowerer.nanCodedReads?.isNumericRead(expr) === true + ) + return lowerIndexedUnionNumberRead(lowerer, recv, arms, index, locOf(expr)); + return lowerIndexedUnionRead(lowerer, recv, arms, index, locOf(expr)); + } + } + // Typed-array element read `b[i]` — like arrayGet with a scalar // result; any invalid index traps (the array discipline; JS would // read undefined — divergence 4's policy). @@ -1518,6 +1550,9 @@ export function lowerElementCompoundParts( }); return { expr: { kind: "seqExpr", stmts, result: value, type: DYN, loc }, stmts: null }; } + const indexedArms = indexedUnionArms(lowerer, checked.type); + if (indexedArms && !isJsSourceFile(ts.getSourceFileOfNode(expr))) + return lowerIndexedUnionCompound(lowerer, expr, op, checked, indexedArms); const receiverType = checked.type.kind === "array" || checked.type.kind === "bytes" ? checked.type @@ -1702,6 +1737,54 @@ export function lowerElementCompoundParts( }; } +/** `u[i] op= v` on a union of typed arrays and number arrays: the + * receiver and index evaluate once, the old element reads as a number (NaN + * for an invalid index, as in the typed-array form), and the result is + * stored through the union's element write. */ +function lowerIndexedUnionCompound( + lowerer: Lowerer, + expr: ts.BinaryExpression, + op: CompoundOp, + receiver: IrExpr, + arms: readonly IndexedArm[], +): { expr: IrExpr | null; stmts: IrStmt[] | null } { + const target = expr.left as ts.ElementAccessExpression; + const loc = locOf(expr); + const index = lowerOptionalIndex( + lowerer, + lowerer.lowerExpr(target.argumentExpression), + locOf(target.argumentExpression), + target.argumentExpression, + ); + if (index.type.kind !== "f64") + lowerer.unsupported("SC1090", target.argumentExpression, "indexing with non-number keys"); + const receiverLocal = lowerer.declareHiddenLocal("%compoundArray", receiver.type); + const indexLocal = lowerer.declareHiddenLocal("%compoundIndex", F64); + const receiverRef = (): IrExpr => varRef(receiverLocal.id, receiver.type, loc); + const indexRef = (): IrExpr => varRef(indexLocal.id, F64, loc); + const oldValue = lowerIndexedUnionNumberRead(lowerer, receiverRef(), arms, indexRef(), loc); + const rhs = lowerOptionalNumber(lowerer, lowerer.lowerExpr(expr.right), loc); + if (rhs.type.kind !== "f64") compoundUnsupported(lowerer, expr, op, F64, rhs.type); + const computed: IrExpr = { kind: "bin", op, left: oldValue, right: rhs, type: F64, loc }; + const resultLocal = lowerer.declareHiddenLocal("%compoundResult", F64); + const resultRef = (): IrExpr => varRef(resultLocal.id, F64, loc); + return { + expr: { + kind: "seqExpr", + stmts: [ + { kind: "varDecl", localId: receiverLocal.id, init: receiver, loc }, + { kind: "varDecl", localId: indexLocal.id, init: index, loc }, + { kind: "varDecl", localId: resultLocal.id, init: computed, loc }, + lowerIndexedUnionWrite(lowerer, receiverRef(), arms, indexRef(), resultRef(), loc), + ], + result: resultRef(), + type: F64, + loc, + }, + stmts: null, + }; +} + /** ToString of one stabilized `string[]` element read for `+=`: a missing * element concatenates as "undefined". */ function arrayStringOperand(arr: IrExpr, index: IrExpr, loc: SrcLoc): IrExpr { @@ -2243,6 +2326,26 @@ export function lowerElementWrite(lowerer: Lowerer, expr: ts.BinaryExpression): }; } } + // An element write into a union of typed arrays and number arrays + // (indexed-unions.ts). + const indexedArms = receiverIr ? indexedUnionArms(lowerer, receiverIr) : null; + if (indexedArms && receiverIr && !isJsSourceFile(ts.getSourceFileOfNode(expr))) { + let recv = lowerer.lowerExpr(target.expression); + if (recv.type.kind === "dyn") recv = lowerer.coerceInto(target.expression, recv, receiverIr); + const arms = indexedUnionArms(lowerer, recv.type); + if (arms) { + const index = lowerOptionalNumber( + lowerer, + lowerer.lowerExpr(target.argumentExpression), + locOf(target.argumentExpression), + target.argumentExpression, + ); + if (index.type.kind !== "f64") + lowerer.unsupported("SC1090", target.argumentExpression, "indexing with non-number keys"); + const value = lowerer.lowerExprExpecting(expr.right, F64); + return lowerIndexedUnionWrite(lowerer, recv, arms, index, value, locOf(expr)); + } + } if (receiverIr?.kind !== "array") { lowerer.unsupported("SC1090", target, "assignment to non-array elements"); } diff --git a/packages/compiler/src/frontend/lowering/expressions/property-access.ts b/packages/compiler/src/frontend/lowering/expressions/property-access.ts index 7b12297a8..1c544b5c5 100644 --- a/packages/compiler/src/frontend/lowering/expressions/property-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/property-access.ts @@ -1,4 +1,5 @@ /* Property reads (`a.b`): lowerExprInner's property-access branch, intrinsic and union properties, field reads and targets, and match-group reads. */ +import { indexedUnionArms, lowerIndexedUnionMember } from "../indexed-unions.js"; import { keyedHelperName } from "../stable-names.js"; import { staticallySelected } from "../static-typeof.js"; import * as ts from "../../ts7/adapter.js"; @@ -3345,6 +3346,13 @@ export function lowerUnionProperty( loc: locOf(expr), }; } + // A union of typed arrays and number arrays: each arm's own length + // (and, for typed arrays only, byteLength, byteOffset, BYTES_PER_ELEMENT). + const indexedArms = indexedUnionArms(lowerer, value.type); + if (indexedArms) { + const member = lowerIndexedUnionMember(lowerer, value, indexedArms, field, locOf(expr)); + if (member) return member; + } lowerer.unsupported( "SC1090", expr, diff --git a/packages/compiler/src/frontend/lowering/expressions/updates.ts b/packages/compiler/src/frontend/lowering/expressions/updates.ts index 04ca23a96..394dcafea 100644 --- a/packages/compiler/src/frontend/lowering/expressions/updates.ts +++ b/packages/compiler/src/frontend/lowering/expressions/updates.ts @@ -1,4 +1,9 @@ /* Increment, decrement and prefix unary operators, and compound field updates. */ +import { + indexedUnionArms, + lowerIndexedUnionNumberRead, + lowerIndexedUnionWrite, +} from "../indexed-unions.js"; import * as ts from "../../ts7/adapter.js"; import { lowerOptionalIndex, @@ -386,6 +391,39 @@ export function lowerIncDec( loc, })); } + // A union of typed arrays and number arrays (indexed-unions.ts): the + // element read as a number and the union's element write. + const indexedArms = indexedUnionArms(lowerer, raw.type); + if ( + ts.isElementAccessExpression(access) && + indexedArms && + !isJsSourceFile(ts.getSourceFileOfNode(expr)) + ) { + const receiver = save(raw, "%incrementArray"); + const index = save( + lowerOptionalIndex( + lowerer, + lowerer.lowerExpr(access.argumentExpression), + locOf(access.argumentExpression), + access.argumentExpression, + ), + "%incrementIndex", + ); + if (index.type.kind !== "f64") + lowerer.unsupported("SC1090", access, "incrementing non-number array keys"); + return finish( + lowerIndexedUnionNumberRead(lowerer, receiver, indexedArms, index, loc), + (value) => + lowerIndexedUnionWrite( + lowerer, + receiver, + indexedArms, + index, + lowerer.coerceInto(access, value, F64), + loc, + ), + ); + } // A TS member statically typed number on a checked-dynamic receiver (a // class-backed protocol view): the expanded form `o.v = o.v + 1`, its // read checked into number like any other read of the member. diff --git a/packages/compiler/src/frontend/lowering/indexed-unions.ts b/packages/compiler/src/frontend/lowering/indexed-unions.ts new file mode 100644 index 000000000..aa93871c9 --- /dev/null +++ b/packages/compiler/src/frontend/lowering/indexed-unions.ts @@ -0,0 +1,538 @@ +import { keyedHelperName } from "./stable-names.js"; +import { helperLoc, varRef } from "../../ir/build.js"; +import { + BOOL, + BYTES_ELEMENT_SIZE, + F64, + VOID, + type IrBytesIntrinsicMethod, + type IrExpr, + type IrFunction, + type IrStmt, + type IrType, + type SrcLoc, + typeKey, +} from "../../ir/ir.js"; +import { everyExprChild, everyStmtChild } from "../../ir/traverse.js"; +import { arrayNumberRead, lowerSafeBytesRead, lowerSafeIndexRead } from "./array-values.js"; +import type { Lowerer } from "./lowerer.js"; + +/** Unions of number-indexed containers: typed arrays of any element kind + * (`Uint8Array | Uint16Array | Uint32Array`, as lru-cache picks by size) + * and `number[]` arms. + * + * The union keeps its ordinary tagged representation. Each operation on it + * lowers to a lifted helper per union and operation that tests the tag, + * narrows to the arm and runs the arm's own operation: the inline load or + * store of a typed array's element kind, or the dense read or write of a + * number array. The helpers are small, so LLVM inlines them, and a loop + * over one union value tests a loop-invariant tag. */ + +/** One arm: its tag and its type (bytes, or an array of numbers). */ +export interface IndexedArm { + tag: number; + type: IrType & ({ kind: "bytes" } | { kind: "array" }); +} + +/** The arms of a union whose every arm is a typed array or a `number[]`, + * or null. Unions with any other arm (undefined included) are not + * indexed unions: their reads need a narrowing first. */ +export function indexedUnionArms(lowerer: Lowerer, type: IrType): IndexedArm[] | null { + if (type.kind !== "union") return null; + const arms = lowerer.unions.get(type.unionId)?.arms; + if (!arms || arms.length < 2) return null; + const result: IndexedArm[] = []; + for (const [tag, arm] of arms.entries()) { + if (arm.kind === "bytes") result.push({ tag, type: arm }); + else if (arm.kind === "array" && arm.elem.kind === "f64") result.push({ tag, type: arm }); + else return null; + } + return result; +} + +/** Whether every arm is a typed array (so typed-array-only members such + * as `byteLength` apply). */ +export function allBytesArms(arms: readonly IndexedArm[]): boolean { + return arms.every((arm) => arm.type.kind === "bytes"); +} + +type ArmBody = (receiver: IrExpr, arm: IndexedArm, args: readonly IrExpr[]) => IrStmt[]; + +/** The helper `name(u, ...args)`: for each arm but the last, a tag test + * that runs the arm's statements; the last arm's statements run without a + * test (the union holds one of its arms). */ +function dispatchHelper( + lowerer: Lowerer, + key: string, + prefix: string, + union: IrType, + arms: readonly IndexedArm[], + params: readonly { name: string; type: IrType }[], + returnType: IrType, + body: ArmBody, + loc: SrcLoc, +): string { + const existing = lowerer.arrHofHelpers.get(key); + if (existing) return existing; + const name = keyedHelperName(lowerer, prefix, key); + lowerer.arrHofHelpers.set(key, name); + const at = helperLoc(loc); + if (union.kind !== "union") throw new Error("indexed-union helper over a non-union"); + const u = varRef("u.0", union, at); + const args = params.map((p) => varRef(`${p.name}.0`, p.type, at)); + const stmts: IrStmt[] = []; + arms.forEach((arm, i) => { + const receiver: IrExpr = { + kind: "unionNarrow", + unionId: union.unionId, + tag: arm.tag, + value: u, + type: arm.type, + loc: at, + }; + const armStmts = body(receiver, arm, args); + if (i === arms.length - 1) { + stmts.push(...armStmts); + return; + } + stmts.push({ + kind: "if", + cond: { + kind: "unionIsTag", + unionId: union.unionId, + tag: arm.tag, + negated: false, + value: u, + type: BOOL, + loc: at, + }, + then: armStmts, + else_: null, + loc: at, + }); + }); + const fn: IrFunction = { + name, + params: [ + { localId: "u.0", name: "u", type: union }, + ...params.map((p) => ({ localId: `${p.name}.0`, name: p.name, type: p.type })), + ], + returnType, + locals: [ + { id: "u.0", name: "u", type: union, mutable: false }, + ...params.map((p) => ({ id: `${p.name}.0`, name: p.name, type: p.type, mutable: false })), + ], + body: stmts, + loc: at, + }; + lowerer.liftedFns.push(fn); + return name; +} + +const returning = (value: IrExpr, loc: SrcLoc): IrStmt[] => [{ kind: "return", value, loc }]; + +/** Safe-read helpers mapped to their numeric twins, for the fusion of an + * immediate ToNumber (tryLowerNumericIndexRead). */ +const numericTwins = new WeakMap>(); + +/** `u[i]` read as an element or undefined (an invalid index), as the + * typed-array and number-array reads answer. */ +export function lowerIndexedUnionRead( + lowerer: Lowerer, + receiver: IrExpr, + arms: readonly IndexedArm[], + index: IrExpr, + loc: SrcLoc, +): IrExpr { + const resultT = lowerer.withUndefinedArm(F64); + const key = `idxUnion:get:${unionKey(receiver.type)}`; + const name = dispatchHelper( + lowerer, + key, + "%union.idxOr", + receiver.type, + arms, + [{ name: "i", type: F64 }], + resultT, + (arm, info, [i]) => + returning( + info.type.kind === "bytes" + ? lowerSafeBytesRead(lowerer, arm, i!, loc) + : lowerSafeIndexRead(lowerer, arm, i!, loc)!, + loc, + ), + loc, + ); + const numeric = indexedUnionNumberHelper(lowerer, receiver.type, arms, loc); + let twins = numericTwins.get(lowerer); + if (!twins) numericTwins.set(lowerer, (twins = new Map())); + twins.set(name, numeric); + return { kind: "call", callee: name, args: [receiver, index], type: resultT, loc }; +} + +/** `u[i]` read as a number: an invalid index reads as NaN, which is + * ToNumber(undefined). */ +export function lowerIndexedUnionNumberRead( + lowerer: Lowerer, + receiver: IrExpr, + arms: readonly IndexedArm[], + index: IrExpr, + loc: SrcLoc, +): IrExpr { + if (inlineDispatch(receiver, [index])) + return inlineArms(receiver, arms, F64, (arm, info) => numberRead(arm, info, index, loc), loc); + const name = indexedUnionNumberHelper(lowerer, receiver.type, arms, loc); + return { kind: "call", callee: name, args: [receiver, index], type: F64, loc }; +} + +function numberRead(arm: IrExpr, info: IndexedArm, index: IrExpr, loc: SrcLoc): IrExpr { + return info.type.kind === "bytes" + ? { + kind: "bytesIntrinsic", + method: "get", + receiver: arm, + args: [index], + type: F64, + loc, + invalidNaN: true, + } + : arrayNumberRead(arm, index, loc); +} + +function indexedUnionNumberHelper( + lowerer: Lowerer, + union: IrType, + arms: readonly IndexedArm[], + loc: SrcLoc, +): string { + return dispatchHelper( + lowerer, + `idxUnion:num:${unionKey(union)}`, + "%union.getNumber", + union, + arms, + [{ name: "i", type: F64 }], + F64, + (arm, info, [i]) => returning(numberRead(arm, info, i!, loc), loc), + loc, + ); +} + +/** The numeric twin of an indexed-union read helper call, or null. */ +export function indexedUnionNumericTwin(lowerer: Lowerer, call: IrExpr): IrExpr | null { + if (call.kind !== "call") return null; + const twin = numericTwins.get(lowerer)?.get(call.callee); + return twin === undefined ? null : { ...call, callee: twin, type: F64 }; +} + +/** `u[i] = v` for a number `v`: each arm's element store (a typed array + * converts the number to its element kind and ignores an invalid index; a + * number array stores at the index). */ +export function lowerIndexedUnionWrite( + lowerer: Lowerer, + receiver: IrExpr, + arms: readonly IndexedArm[], + index: IrExpr, + value: IrExpr, + loc: SrcLoc, +): IrStmt { + // In place, the value evaluates before the arms read the receiver and + // index: only when it cannot rebind either. + if (inlineDispatch(receiver, [index]) && !rebinds(value, [receiver, index])) { + const union = receiver.type; + const valueLocal = lowerer.declareHiddenLocal("%elementValue", F64); + const v: IrExpr = { kind: "varRef", localId: valueLocal.id, type: F64, loc }; + let chain: IrStmt[] | null = null; + for (let i = arms.length - 1; i >= 0; i--) { + const info = arms[i]!; + const arm: IrExpr = { + kind: "unionNarrow", + unionId: union.kind === "union" ? union.unionId : "", + tag: info.tag, + value: receiver, + type: info.type, + loc, + }; + const store: IrStmt = + info.type.kind === "bytes" + ? { kind: "bytesSet", arr: arm, index, value: v, loc } + : { kind: "arraySet", arr: arm, index, value: v, loc }; + chain = + chain === null + ? [store] + : [ + { + kind: "if", + cond: { + kind: "unionIsTag", + unionId: union.kind === "union" ? union.unionId : "", + tag: info.tag, + negated: false, + value: receiver, + type: BOOL, + loc, + }, + then: [store], + else_: chain, + loc, + }, + ]; + } + return { + kind: "block", + body: [{ kind: "varDecl", localId: valueLocal.id, init: value, loc }, ...chain!], + loc, + }; + } + const name = dispatchHelper( + lowerer, + `idxUnion:set:${unionKey(receiver.type)}`, + "%union.idxSet", + receiver.type, + arms, + [ + { name: "i", type: F64 }, + { name: "v", type: F64 }, + ], + VOID, + (arm, info, [i, v]) => [ + info.type.kind === "bytes" + ? { kind: "bytesSet", arr: arm, index: i!, value: v!, loc } + : { kind: "arraySet", arr: arm, index: i!, value: v!, loc }, + { kind: "return", value: null, loc }, + ], + loc, + ); + return { + kind: "exprStmt", + expr: { kind: "call", callee: name, args: [receiver, index, value], type: VOID, loc }, + loc, + }; +} + +/** The members every arm answers: `length` for all, and `byteLength`, + * `byteOffset` and `BYTES_PER_ELEMENT` when every arm is a typed array. */ +export function lowerIndexedUnionMember( + lowerer: Lowerer, + receiver: IrExpr, + arms: readonly IndexedArm[], + member: string, + loc: SrcLoc, +): IrExpr | null { + const bytesOnly = member !== "length"; + if ( + member !== "length" && + member !== "byteLength" && + member !== "byteOffset" && + member !== "BYTES_PER_ELEMENT" + ) + return null; + if (bytesOnly && !allBytesArms(arms)) return null; + const memberOf = (arm: IrExpr, info: IndexedArm): IrExpr => + info.type.kind === "array" + ? { kind: "arrIntrinsic", method: "length", receiver: arm, args: [], type: F64, loc } + : member === "BYTES_PER_ELEMENT" + ? { kind: "numLit", value: BYTES_ELEMENT_SIZE[info.type.elem], type: F64, loc } + : { kind: "bytesIntrinsic", method: member, receiver: arm, args: [], type: F64, loc }; + if (inlineDispatch(receiver, [])) return inlineArms(receiver, arms, F64, memberOf, loc); + const name = dispatchHelper( + lowerer, + `idxUnion:${member}:${unionKey(receiver.type)}`, + `%union.${member}`, + receiver.type, + arms, + [], + F64, + (arm, info) => returning(memberOf(arm, info), loc), + loc, + ); + return { kind: "call", callee: name, args: [receiver], type: F64, loc }; +} + +function unionKey(type: IrType): string { + return type.kind === "union" ? String(type.unionId) : "?"; +} + +/** A typed-array method on a union whose every arm is a typed array: the + * arguments evaluate once in the caller, and each arm runs the same + * `bytesIntrinsic` on its own receiver. `result` "receiver" is a method + * that answers a typed array of the receiver's kind (subarray, slice, + * fill, ...): each arm wraps its answer back into the union. */ +export function lowerIndexedUnionMethod( + lowerer: Lowerer, + receiver: IrExpr, + arms: readonly IndexedArm[], + method: IrBytesIntrinsicMethod, + args: readonly IrExpr[], + result: "receiver" | IrType, + loc: SrcLoc, +): IrExpr { + const union = receiver.type; + if (union.kind !== "union") throw new Error("indexed-union method over a non-union"); + const resultT = result === "receiver" ? union : result; + const params = args.map((arg, i) => ({ name: `a${i}`, type: arg.type })); + const signature = args.map((arg) => typeKey(arg.type)).join(","); + const name = dispatchHelper( + lowerer, + `idxUnion:${method}(${signature}):${unionKey(union)}`, + `%union.${method}`, + union, + arms, + params, + resultT, + (arm, info, refs) => { + const call: IrExpr = { + kind: "bytesIntrinsic", + method, + receiver: arm, + args: [...refs], + type: result === "receiver" ? info.type : result, + loc, + }; + if (result === "receiver") + return returning( + { + kind: "unionWrap", + unionId: union.unionId, + tag: info.tag, + value: call, + type: union, + loc, + }, + loc, + ); + if (result.kind === "void") + return [ + { kind: "exprStmt", expr: call, loc }, + { kind: "return", value: null, loc }, + ]; + return returning(call, loc); + }, + loc, + ); + return { kind: "call", callee: name, args: [receiver, ...args], type: resultT, loc }; +} + +/** The elements of `u` as a `number[]`: a typed-array arm drains its + * elements into a fresh array. A number-array arm is the array itself for + * consumers that copy it (spreads), or with `fresh` a dense copy whose + * holes read as undefined (Array.from). */ +export function lowerIndexedUnionToArray( + lowerer: Lowerer, + receiver: IrExpr, + arms: readonly IndexedArm[], + loc: SrcLoc, + fresh = false, +): IrExpr { + const arrayT: IrType = { kind: "array", elem: F64 }; + const name = dispatchHelper( + lowerer, + `idxUnion:${fresh ? "from" : "toArray"}:${unionKey(receiver.type)}`, + fresh ? "%union.arrayFrom" : "%union.toArray", + receiver.type, + arms, + [], + arrayT, + (arm, info) => + returning( + info.type.kind === "bytes" + ? { + kind: "bytesIntrinsic", + method: "toArray", + receiver: arm, + args: [], + type: arrayT, + loc, + } + : fresh + ? { + kind: "arrIntrinsic", + method: "reverse", + receiver: { + kind: "arrIntrinsic", + method: "toReversed", + receiver: arm, + args: [], + type: arrayT, + loc, + }, + args: [], + type: arrayT, + loc, + } + : arm, + loc, + ), + loc, + ); + return { kind: "call", callee: name, args: [receiver], type: arrayT, loc }; +} + +/** Whether a dispatch can run in place instead of through a helper call: + * the receiver is a plain read, so testing and narrowing it once per arm + * reads the same value, and the other operands are plain reads or + * literals. The arms' own reads then stay visible to the integer range + * analysis (a loop bounded by `u.length` keeps an integer counter). */ +function inlineDispatch(receiver: IrExpr, operands: readonly IrExpr[]): boolean { + return ( + receiver.kind === "varRef" && operands.every((e) => e.kind === "varRef" || e.kind === "numLit") + ); +} + +/** Whether evaluating `e` can rebind one of the plain reads `reads`. */ +function rebinds(e: IrExpr, reads: readonly IrExpr[]): boolean { + const ids = new Set(reads.flatMap((r) => (r.kind === "varRef" ? [r.localId] : []))); + let found = false; + const expr = (x: IrExpr): boolean => { + if ((x.kind === "assignExpr" || x.kind === "incDec") && ids.has(x.localId)) found = true; + return !found && everyExprChild(x, expr, stmt); + }; + const stmt = (x: IrStmt): boolean => { + if ((x.kind === "assign" || x.kind === "varDecl") && ids.has(x.localId)) found = true; + return !found && everyStmtChild(x, expr, stmt); + }; + expr(e); + return found; +} + +/** The in-place form of a dispatch: a ternary chain over the arms' tags, + * the last arm untested. */ +function inlineArms( + receiver: IrExpr, + arms: readonly IndexedArm[], + type: IrType, + arm: (narrowed: IrExpr, info: IndexedArm) => IrExpr, + loc: SrcLoc, +): IrExpr { + const union = receiver.type; + if (union.kind !== "union") throw new Error("indexed-union dispatch over a non-union"); + const narrowed = (info: IndexedArm): IrExpr => ({ + kind: "unionNarrow", + unionId: union.unionId, + tag: info.tag, + value: receiver, + type: info.type, + loc, + }); + let result = arm(narrowed(arms[arms.length - 1]!), arms[arms.length - 1]!); + for (let i = arms.length - 2; i >= 0; i--) { + const info = arms[i]!; + result = { + kind: "ternary", + cond: { + kind: "unionIsTag", + unionId: union.unionId, + tag: info.tag, + negated: false, + value: receiver, + type: BOOL, + loc, + }, + then: arm(narrowed(info), info), + else_: result, + type, + loc, + }; + } + return result; +} diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index 66ecea602..046c03aa2 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -1,4 +1,11 @@ import { lowerTupleAliasDecl, stripTypeWrappers } from "./typed-arguments.js"; +import { + allBytesArms, + indexedUnionArms, + lowerIndexedUnionMember, + lowerIndexedUnionNumberRead, + lowerIndexedUnionRead, +} from "./indexed-unions.js"; import { declaredMemberOf, isPromisePrototypeRead } from "./expressions/property-access.js"; import { hostsAll } from "./expressions/hosted-narrowing.js"; import { requestGeneralStreamSurface } from "./general-streams.js"; @@ -10132,13 +10139,8 @@ export function lowerForOf(lowerer: Lowerer, stmt: ts.ForOfStatement): IrStmt { loc: locOf(iterSrc), }; } - if (iterable.type.kind === "bytes") { - return lowerForOfBytes( - lowerer, - stmt, - iterable as IrExpr & { type: IrType & { kind: "bytes" } }, - labels, - ); + if (iterable.type.kind === "bytes" || indexedUnionArms(lowerer, iterable.type)) { + return lowerForOfBytes(lowerer, stmt, iterable, labels); } if (iterable.type.kind !== "array") { // The lib types strings as iterable too, so those for-ofs typecheck; @@ -10696,7 +10698,7 @@ export function lowerForOf(lowerer: Lowerer, stmt: ts.ForOfStatement): IrStmt { function lowerForOfBytes( lowerer: Lowerer, stmt: ts.ForOfStatement, - iterable: IrExpr & { type: IrType & { kind: "bytes" } }, + iterable: IrExpr, labels?: string[], ): IrStmt { const loc = locOf(stmt); @@ -10790,7 +10792,16 @@ function lowerForOfStoredNumericIterator( const loc = locOf(stmt); const sourceT = state.sourceType; - if (!((sourceT.kind === "array" && sourceT.elem.kind === "f64") || sourceT.kind === "bytes")) { + // A union of typed arrays and number arrays (indexed-unions.ts) reads + // through its arms' own length and element reads. + const unionArms = indexedUnionArms(lowerer, sourceT); + if ( + !( + (sourceT.kind === "array" && sourceT.elem.kind === "f64") || + sourceT.kind === "bytes" || + unionArms + ) + ) { throw new InternalCompilerError("internal: retained numeric iterator has a non-numeric source"); } @@ -10798,35 +10809,44 @@ function lowerForOfStoredNumericIterator( const doneRef = (): IrExpr => varRef(state.doneLocalId, BOOL, loc); const sourceRef = (): IrExpr => varRef(state.sourceLocalId, sourceT, loc); const sourceLength = (): IrExpr => - sourceT.kind === "array" - ? { - kind: "arrIntrinsic", - method: "length", - receiver: sourceRef(), - args: [], - type: F64, - loc, - } - : { - kind: "bytesIntrinsic", - method: "length", - receiver: sourceRef(), - args: [], - type: F64, - loc, - }; - const valueT = sourceT.kind === "array" ? arrayValueType(lowerer, F64) : F64; + unionArms + ? lowerIndexedUnionMember(lowerer, sourceRef(), unionArms, "length", loc)! + : sourceT.kind === "array" + ? { + kind: "arrIntrinsic", + method: "length", + receiver: sourceRef(), + args: [], + type: F64, + loc, + } + : { + kind: "bytesIntrinsic", + method: "length", + receiver: sourceRef(), + args: [], + type: F64, + loc, + }; + const valueT = + sourceT.kind === "array" || (unionArms && !allBytesArms(unionArms)) + ? arrayValueType(lowerer, F64) + : F64; const sourceValue = (): IrExpr => - sourceT.kind === "array" - ? arrayValueRead(lowerer, sourceRef(), indexRef(), F64, loc) - : { - kind: "bytesIntrinsic", - method: "get", - receiver: sourceRef(), - args: [indexRef()], - type: F64, - loc, - }; + unionArms + ? allBytesArms(unionArms) + ? lowerIndexedUnionNumberRead(lowerer, sourceRef(), unionArms, indexRef(), loc) + : lowerIndexedUnionRead(lowerer, sourceRef(), unionArms, indexRef(), loc) + : sourceT.kind === "array" + ? arrayValueRead(lowerer, sourceRef(), indexRef(), F64, loc) + : { + kind: "bytesIntrinsic", + method: "get", + receiver: sourceRef(), + args: [indexRef()], + type: F64, + loc, + }; const one: IrExpr = { kind: "numLit", value: 1, type: F64, loc }; lowerer.scopes.push(new Map()); diff --git a/packages/compiler/src/ir/integer-loops.ts b/packages/compiler/src/ir/integer-loops.ts index d6942718a..2f7d2bea6 100644 --- a/packages/compiler/src/ir/integer-loops.ts +++ b/packages/compiler/src/ir/integer-loops.ts @@ -119,13 +119,26 @@ function stableLoopValue( stableLoopValue(value.receiver, locals, body, invariant, counter) ); case "bytesIntrinsic": - case "arrIntrinsic": + case "arrIntrinsic": { + // The receiver may be one arm of a union binding (indexed-unions.ts). + const receiver = + value.receiver.kind === "unionNarrow" ? value.receiver.value : value.receiver; return ( (value.method === "length" || (value.kind === "bytesIntrinsic" && value.method === "byteLength")) && - value.receiver.kind === "varRef" && - (value.kind === "bytesIntrinsic" || !invariant || stableLength(value.receiver.localId)) && - stableLoopValue(value.receiver, locals, body, invariant, counter) + receiver.kind === "varRef" && + (value.kind === "bytesIntrinsic" || !invariant || stableLength(receiver.localId)) && + stableLoopValue(receiver, locals, body, invariant, counter) + ); + } + // A union binding dispatched on its tag (indexed-unions.ts): stable + // when the binding is and every arm is. + case "ternary": + return ( + value.cond.kind === "unionIsTag" && + stableLoopValue(value.cond.value, locals, body, invariant, counter) && + stableLoopValue(value.then, locals, body, invariant, counter, stableLength) && + stableLoopValue(value.else_, locals, body, invariant, counter, stableLength) ); default: return false; diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7745-typed-array-unions.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7745-typed-array-unions.ts.json new file mode 100644 index 000000000..c85ce647a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7745-typed-array-unions.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7742-typed-array-unions.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7747-typed-array-class-values.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7747-typed-array-class-values.ts.json new file mode 100644 index 000000000..63b6307ea --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7747-typed-array-class-values.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7744-typed-array-class-values.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7748-typed-array-number-array-unions.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7748-typed-array-number-array-unions.ts.json new file mode 100644 index 000000000..b2274d160 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7748-typed-array-number-array-unions.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7743-typed-array-number-array-unions.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/typed-array-union-emission.test.ts b/packages/compiler/test/typed-array-union-emission.test.ts new file mode 100644 index 000000000..743788af0 --- /dev/null +++ b/packages/compiler/test/typed-array-union-emission.test.ts @@ -0,0 +1,56 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile } from "../src/index.js"; + +async function emitLlvm(source: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "scriptc-typed-array-unions-")); + try { + const entry = join(dir, "main.ts"); + const outPath = join(dir, "main.ll"); + await writeFile(entry, source); + const result = await compile(entry, { outDir: dir, outPath, outputKind: "llvm" }); + expect(result.ok, JSON.stringify(result.diagnostics)).toBe(true); + return await readFile(outPath, "utf8"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +function body(llvm: string, name: string): string { + return new RegExp(`^define [^@]*@sc_b?f_${name}(\\.i32)?\\([^]*?^}`, "m").exec(llvm)?.[0] ?? ""; +} + +const source = ` +type UintArray = Uint8Array | Uint16Array | Uint32Array; +const pick = (max: number, n: number): UintArray => + max <= 256 ? new Uint8Array(n) : max <= 65536 ? new Uint16Array(n) : new Uint32Array(n); +function total(xs: UintArray): number { + let s = 0; + for (let i = 0; i < xs.length; i++) s += xs[i]!; + return s; +} +function first(max: number): number { + return pick(max, 4)[0]!; +} +const xs = pick(Number(process.argv.length) * 100000, 8); +for (let i = 0; i < xs.length; i++) xs[i] = i; +console.log(total(xs), first(10)); +`; + +test("element reads of a union of typed arrays load each arm's own element kind in place", async () => { + const llvm = await emitLlvm(source); + const total = body(llvm, "total"); + // One load per arm, each of its element width, behind the union's tag. + for (const width of ["i8", "i16", "i32"]) expect(total).toMatch(new RegExp(`load ${width}, ptr`)); + expect(total).not.toContain("@sc_bf__x25_union_getNumber"); + // The loop over the union's length keeps an integer counter. + expect(total).toMatch(/alloca i64/); +}); + +test("a typed-array union read through a computed receiver goes through one helper", async () => { + const llvm = await emitLlvm(source); + // The receiver is a call: evaluated once, then dispatched by the helper. + expect(body(llvm, "first")).toContain("@sc_bf__x25_union_getNumber"); +}); diff --git a/tests/corpus/7745-typed-array-unions.ts b/tests/corpus/7745-typed-array-unions.ts new file mode 100644 index 000000000..57c220dcc --- /dev/null +++ b/tests/corpus/7745-typed-array-unions.ts @@ -0,0 +1,71 @@ +// Unions of typed-array classes: every element operation runs on each arm. + +type UintArray = Uint8Array | Uint16Array | Uint32Array; +type Samples = Int16Array | Float32Array | Float64Array; + +function uintArray(max: number, size: number): UintArray { + return max <= 2 ** 8 + ? new Uint8Array(size) + : max <= 2 ** 16 + ? new Uint16Array(size) + : new Uint32Array(size); +} + +function samples(kind: number, size: number): Samples { + return kind === 0 + ? new Int16Array(size) + : kind === 1 + ? new Float32Array(size) + : new Float64Array(size); +} + +function sum(xs: UintArray): number { + let total = 0; + for (let i = 0; i < xs.length; i++) total += xs[i]!; + return total; +} + +function describe(xs: UintArray): string { + if (xs instanceof Uint8Array) return `u8 ${xs.length}`; + if (xs instanceof Uint16Array) return `u16 ${xs[0]}`; + return `u32 ${xs.BYTES_PER_ELEMENT}`; +} + +for (const max of [200, 60000, 5000000]) { + const xs = uintArray(max, 6); + // Stores convert to each arm's element kind (wrapping past its range). + for (let i = 0; i < xs.length; i++) xs[i] = i * 70000 + 255; + console.log(xs.length, xs.byteLength, xs.byteOffset, xs.BYTES_PER_ELEMENT); + console.log(xs[0], xs[5], xs[6], xs[-1], sum(xs)); + xs[1] += 3; + xs[2]++; + --xs[3]; + xs[4] = -1; + xs[5] = 2.75; + console.log(xs[1], xs[2], xs[3], xs[4], xs[5]); + // Writes past the end are ignored, and reads there are undefined. + xs[10] = 9; + console.log(xs[10] === undefined, xs[10] ?? "missing"); + let iterated = 0; + for (const v of xs) iterated += v; + console.log(iterated, [...xs].length, Array.from(xs)[1]); + const part = xs.subarray(1, 4); + part[0] = 42; + console.log(part.length, xs[1], xs.slice(2, 4).length, xs.slice(2, 4)[0] === xs[2]); + xs.fill(7, 4); + xs.copyWithin(0, 4, 5); + console.log(xs.join("-"), xs.toReversed()[0]); + xs.set([1, 2], 2); + console.log(xs[2], xs[3], describe(xs)); +} + +for (const kind of [0, 1, 2]) { + const s = samples(kind, 4); + s[0] = 1.5; + s[1] = -40000; + s[2] = 0.1; + s[3] = s[0]! * 3; + console.log(s.BYTES_PER_ELEMENT, s[0], s[1], s[2], s[3]); + console.log(s); + console.log(s instanceof Float32Array, s instanceof Int16Array); +} diff --git a/tests/corpus/7747-typed-array-class-values.ts b/tests/corpus/7747-typed-array-class-values.ts new file mode 100644 index 000000000..523896c96 --- /dev/null +++ b/tests/corpus/7747-typed-array-class-values.ts @@ -0,0 +1,42 @@ +// Typed-array unions chosen through class values, and held in fields, +// parameters, returns and records. + +type UintArray = Uint8Array | Uint16Array | Uint32Array; + +const pick = (max: number): { new (n: number): UintArray } => + max <= 2 ** 8 ? Uint8Array : max <= 2 ** 16 ? Uint16Array : Uint32Array; + +class Stack { + heap: UintArray; + length = 0; + constructor(max: number) { + const Heap = pick(max); + this.heap = new Heap(max); + } + push(n: number): void { + this.heap[this.length++] = n; + } + pop(): number | undefined { + return this.heap[--this.length]; + } +} + +for (const max of [10, 1000, 100000]) { + const stack = new Stack(max); + for (let i = 1; i <= 4; i++) stack.push(i * 250); + console.log(stack.heap.BYTES_PER_ELEMENT, stack.pop(), stack.pop(), stack.length); +} + +function grow(xs: UintArray, by: number): UintArray { + const Next = pick(xs.length * by); + const out = new Next(xs.length * by); + out.set(xs); + return out; +} + +const record: { data: UintArray; label: string } = { data: new Uint8Array([1, 2, 3]), label: "r" }; +record.data = grow(record.data, 100); +record.data[299] = 65535; +console.log(record.label, record.data.length, record.data[0], record.data[2], record.data[299]); +record.data = grow(record.data, 1000); +console.log(record.data.BYTES_PER_ELEMENT, record.data.length, record.data[1]); diff --git a/tests/corpus/7748-typed-array-number-array-unions.ts b/tests/corpus/7748-typed-array-number-array-unions.ts new file mode 100644 index 000000000..706ed37b5 --- /dev/null +++ b/tests/corpus/7748-typed-array-number-array-unions.ts @@ -0,0 +1,78 @@ +// Unions of typed arrays and number arrays, as a cache picks its index +// arrays by capacity: a linked list over next/prev index arrays. + +type NumberArray = Uint8Array | Uint16Array | Uint32Array | number[]; + +function indexArray(max: number): NumberArray { + if (max <= 2 ** 8) return new Uint8Array(max); + if (max <= 2 ** 16) return new Uint16Array(max); + if (max <= 2 ** 32) return new Uint32Array(max); + return new Array(4).fill(0); +} + +class Order { + next: NumberArray; + prev: NumberArray; + head = 0; + tail = 0; + size = 0; + + constructor(max: number, force = false) { + this.next = force ? new Array(max).fill(0) : indexArray(max); + this.prev = force ? new Array(max).fill(0) : indexArray(max); + } + + push(index: number): void { + if (this.size === 0) { + this.head = this.tail = index; + } else { + this.next[this.tail] = index; + this.prev[index] = this.tail; + this.tail = index; + } + this.size++; + } + + moveToTail(index: number): void { + if (index === this.tail) return; + if (index === this.head) this.head = this.next[index]!; + else this.next[this.prev[index]!] = this.next[index]!; + this.prev[this.next[index]!] = this.prev[index]!; + this.next[this.tail] = index; + this.prev[index] = this.tail; + this.tail = index; + } + + walk(): number[] { + const out: number[] = []; + for (let i = this.head, n = 0; n < this.size; n++, i = this.next[i]!) out.push(i); + return out; + } +} + +for (const [max, force] of [ + [8, false], + [300, false], + [70000, false], + [8, true], +] as const) { + const order = new Order(max, force); + for (let i = 0; i < 6; i++) order.push(i); + order.moveToTail(0); + order.moveToTail(3); + order.moveToTail(5); + console.log(max, force, order.walk().join(","), order.next.length, order.prev[3]); +} + +// The number-array arm reads holes and missing indexes as undefined, +// and writes past its length grow it. +const arms: NumberArray[] = [new Uint8Array(3), [1, 2, 3]]; +for (const a of arms) { + a[1] = 300; + a[2] += 0.5; + a[5] = 8; + console.log(a.length, a[1], a[2], a[4], a[5]); + let total = 0; + for (const v of a) total += v ?? -1; + console.log(total, [...a].length, Array.from(a).length); +} diff --git a/tests/harness/corpus-costs/7745-typed-array-unions.ts.json b/tests/harness/corpus-costs/7745-typed-array-unions.ts.json new file mode 100644 index 000000000..ee546c4fd --- /dev/null +++ b/tests/harness/corpus-costs/7745-typed-array-unions.ts.json @@ -0,0 +1 @@ +{ "plain": 3450, "san": 4000 } diff --git a/tests/harness/corpus-costs/7747-typed-array-class-values.ts.json b/tests/harness/corpus-costs/7747-typed-array-class-values.ts.json new file mode 100644 index 000000000..89e0d92e7 --- /dev/null +++ b/tests/harness/corpus-costs/7747-typed-array-class-values.ts.json @@ -0,0 +1 @@ +{ "plain": 2400, "san": 2350 } diff --git a/tests/harness/corpus-costs/7748-typed-array-number-array-unions.ts.json b/tests/harness/corpus-costs/7748-typed-array-number-array-unions.ts.json new file mode 100644 index 000000000..89fb8ddd2 --- /dev/null +++ b/tests/harness/corpus-costs/7748-typed-array-number-array-unions.ts.json @@ -0,0 +1 @@ +{ "plain": 2550, "san": 2300 } From 6828c23b5051ece0d4fa24e3f8a70e2ba8c4298c Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 11:20:05 -0500 Subject: [PATCH 063/184] Error tee branches when the source errors with no read pending The tee learned a source error only through a pending read's error steps, so a source that errored while both branches held buffered chunks and no read was pending left the branches readable: the next read returned a chunk where Node rejects. The tee now watches its source's error, as the streams spec reacts to the source reader's closed promise, and errors both branches in a microtask (resolving the cancel promise unless both branches were canceled). A read's error steps only end the read, and a clone of an already errored body errors both branches the same way. --- packages/runtime/src/scr_fetch.c | 50 +++++++++++++++++---- tests/corpus/7742-response-clone-tee.ts | 58 +++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 9 deletions(-) diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index 37de698ac..bd1b1ba08 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -167,6 +167,10 @@ struct SfStream { SfTransfer *response_owner; /* weak; transfer owns the stream */ bool callbacks_tracked; SfStream *callbacks_next; + /* Told when the stream errors (a tee watching its source's closed + * promise); ctx is borrowed and the watcher clears it before it goes. */ + void (*error_watch)(void *ctx, ScrDyn *reason); + void *error_watch_ctx; }; struct SfHeaders { @@ -1291,6 +1295,7 @@ static void sf_stream_error(SfStream *s, ScrDyn *reason) { if (s->reader) { sf_reject_promise_reason(s->reader->closed, reason); } + if (s->error_watch) s->error_watch(s->error_watch_ctx, reason); if (s->request_owner) { sf_transfer_stream_error(s->request_owner, reason); } @@ -1889,6 +1894,10 @@ static SfTee *sf_tee_retain(SfTee *t) { static void sf_tee_release(SfTee *t) { if (!t || --t->rc > 0) return; + if (t->source->error_watch_ctx == t) { + t->source->error_watch = NULL; + t->source->error_watch_ctx = NULL; + } sf_stream_release(t->branch[0]); sf_stream_release(t->branch[1]); sf_reader_release(t->reader); @@ -1951,18 +1960,37 @@ static void sf_tee_sink_close(void *ctx) { sf_tee_release(t); } +/* The source reader's closed promise rejected (spec: upon its rejection + * with r, error both branches, and resolve cancelPromise unless both + * branches were canceled), as a promise reaction: in a microtask. */ +static void sf_tee_source_errored_task(ScrClosure *cb) { + SfTee *t = (SfTee *)scr_box_get_ref(cb->caps[0]); + ScrDyn *reason = (ScrDyn *)scr_box_get_ref(cb->caps[1]); + sf_stream_error(t->branch[0], reason); + sf_stream_error(t->branch[1], reason); + if (!t->canceled[0] || !t->canceled[1]) scr_promise_fulfill_void(t->cancel_promise); + scr_dyn_release(reason); + sf_tee_release(t); +} + +/* The source errored, with or without a read of it pending. */ +static void sf_tee_source_errored(void *ctx, ScrDyn *reason) { + SfTee *t = ctx; + t->source->error_watch = NULL; + t->source->error_watch_ctx = NULL; + ScrClosure *cb = scr_closure_new((void *)&sf_tee_source_errored_task, 2); + cb->caps[0] = sf_tee_box(t); + cb->caps[1] = scr_box_new_obj(&scr_dyn_retain_v, &scr_dyn_release_v, NULL); + scr_box_set_ref(cb->caps[1], scr_dyn_retain(reason)); + scr_queue_microtask(cb); +} + +/* A read request's error steps only end the read: the source's error + * reaches the branches through sf_tee_source_errored. */ static void sf_tee_sink_error(void *ctx, ScrDyn *reason) { + (void)reason; SfTee *t = ctx; t->reading = false; - if (reason) { - /* The spec errors both branches when the source reader's closed - * promise rejects; a native read request learns it here. */ - sf_stream_error(t->branch[0], reason); - sf_stream_error(t->branch[1], reason); - if (!t->canceled[0] || !t->canceled[1]) { - scr_promise_fulfill_void(t->cancel_promise); - } - } sf_tee_release(t); } @@ -2041,6 +2069,8 @@ static void sf_stream_tee(SfStream *s, SfStream *out[2]) { t->rc = 1; t->source = sf_stream_retain(s); t->reader = sf_stream_get_reader(s); + s->error_watch = &sf_tee_source_errored; + s->error_watch_ctx = t; t->cancel_promise = scr_promise_new(); scr_promise_mark_handled(t->cancel_promise); for (int i = 0; i < 2; i++) { @@ -2053,6 +2083,8 @@ static void sf_stream_tee(SfStream *s, SfStream *out[2]) { t->branch[i] = sf_stream_retain(b); out[i] = b; } + /* An errored source's closed promise has already rejected. */ + if (s->error) sf_tee_source_errored(t, s->error); /* The branches start at once; a branch with room then pulls. */ sf_stream_schedule_initial_pull(out[0]); sf_stream_schedule_initial_pull(out[1]); diff --git a/tests/corpus/7742-response-clone-tee.ts b/tests/corpus/7742-response-clone-tee.ts index 9058c4d70..56c5e74e6 100644 --- a/tests/corpus/7742-response-clone-tee.ts +++ b/tests/corpus/7742-response-clone-tee.ts @@ -151,4 +151,62 @@ async function readAllFrom(reader: any): Promise { } } + +// Tee branches error when their source errors: after both buffered a +// chunk and with no source read pending, reads reject once the error's +// reaction ran (a read issued at once still takes the buffered chunk, as in +// Node); a clone of an errored body errors both branches. +async function readFirst(label: string, r: any): Promise { + try { + const chunk = await r.body.getReader().read(); + console.log(label, "read", JSON.stringify(chunk.value ? [...chunk.value] : null), chunk.done); + } catch (error) { + console.log(label, "rejects", (error as Error).message); + } +} + +function erroring(): { res: any; error: (reason: Error) => void } { + let ctrl: ReadableStreamDefaultController | undefined; + const source = new ReadableStream({ + start(c) { + ctrl = c; + c.enqueue(new Uint8Array([1, 2])); + }, + }); + return { res: new Response(source), error: (reason) => ctrl!.error(reason) }; +} + +const tick = (): Promise => new Promise((resolve) => setTimeout(resolve, 10)); + +async function sourceErrors(): Promise { + // Error, then read after the reaction: both branches reject. + { + const { res, error } = erroring(); + const clone = res.clone(); + await tick(); + error(new Error("late")); + await tick(); + await readFirst("later original", res); + await readFirst("later clone", clone); + } + // Error, then read at once: the first read takes the buffered chunk. + { + const { res, error } = erroring(); + const clone = res.clone(); + await tick(); + error(new Error("soon")); + await readFirst("at once original", res); + await readFirst("at once clone", clone); + } + // A body that errored before the clone. + { + const { res, error } = erroring(); + error(new Error("before")); + const clone = res.clone(); + await readFirst("errored original", res); + await readFirst("errored clone", clone); + } +} + await main(); +await sourceErrors(); From a77fe8c21c67372b3a1af4ad69fcad34a578ad59 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 09:17:32 -0700 Subject: [PATCH 064/184] Decide an untyped property write with one own-property lookup scr_dyn_object_key_set, behind every untyped property write and every member of an untyped object literal, looked the key up as many as five times: a linear scan for an own accessor or read-only entry, a lookup before the prototype walk, one before the non-extensible check, one to know whether the key was created, and one more inside the store. The own property is now looked up once. An own data property is written in place (an own setter still runs; a read-only or getter-only property still refuses), and an absent key walks the prototype chain as before and is then added by scr_dyn_obj_set, which is unchanged. zod 3, z.array(Order).parse over 500 orders: 199.8M -> 194.1M instructions per round (-2.8%). Co-Authored-By: Claude Opus 5.5 --- .../corpus/7793-untyped-key-writes.js.json | 6 ++ packages/runtime/src/scr_json.c | 21 +++-- tests/corpus/7793-untyped-key-writes.js | 83 +++++++++++++++++++ .../7793-untyped-key-writes.js.json | 1 + 4 files changed, 104 insertions(+), 7 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7793-untyped-key-writes.js.json create mode 100644 tests/corpus/7793-untyped-key-writes.js create mode 100644 tests/harness/corpus-costs/7793-untyped-key-writes.js.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7793-untyped-key-writes.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7793-untyped-key-writes.js.json new file mode 100644 index 000000000..b887e0fdb --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7793-untyped-key-writes.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7793-untyped-key-writes.js" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 14d0a901f..a1ab99a42 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -9369,10 +9369,12 @@ bool scr_dyn_in(const ScrDyn *v, const ScrStr *key) { return false; } +/* One lookup of the own property decides the assignment: an own property + * is written (or runs its setter, or refuses the write); an absent one + * consults the prototype chain and is then added. */ static void scr_dyn_object_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value, const ScrDyn *receiver) { - for (size_t i = 0; i < recv->v.obj.len; i++) { - ScrDynEntry *entry = &recv->v.obj.entries[i]; - if (entry->key_len != key->len || memcmp(entry->key, key->data, key->len) != 0) continue; + ScrDynEntry *entry = scr_dyn_entry(recv, key); + if (entry) { if (entry->accessor && entry->setter) { ScrDyn *args[] = {value}; ScrDyn *setter = scr_dyn_retain(entry->setter); @@ -9388,8 +9390,14 @@ static void scr_dyn_object_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value, con scr_throw_error_msg(SCR_ERR_TYPE, msg, sizeof msg - 1); return; } + scr_dyn_region_store(recv, value); + ScrDyn *old = entry->value; + entry->value = scr_dyn_retain(value); + scr_dyn_release(old); + scr_error_sync_cause(recv, key); + return; } - if (!scr_dyn_entry(recv, key)) { + { for (const ScrDyn *current = scr_dyn_proto_next(recv); current;) { ScrDyn *table = current->kind == SCR_DYN_FUNC ? scr_dyn_fn_properties(current) : NULL; const ScrDyn *object = table ? table : current; @@ -9421,7 +9429,7 @@ static void scr_dyn_object_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value, con break; } } - if (recv->non_extensible && !scr_dyn_entry(recv, key)) { + if (recv->non_extensible) { ScrJsonBuf buffer; scr_jb_init(&buffer); scr_jb_puts(&buffer, "Cannot add property "); @@ -9430,9 +9438,8 @@ static void scr_dyn_object_key_set(ScrDyn *recv, ScrStr *key, ScrDyn *value, con scr_throw_error(SCR_ERR_TYPE, scr_jb_finish(&buffer)); return; } - bool created = scr_dyn_entry(recv, key) == NULL; scr_dyn_obj_set(recv, key->data, key->len, scr_dyn_retain(value)); - if (created && key->len == 5 && memcmp(key->data, "stack", 5) == 0 && + if (key->len == 5 && memcmp(key->data, "stack", 5) == 0 && scr_errdyn_find(recv, false) != SIZE_MAX) { /* An error's own `stack` exists from construction as a non-enumerable * data property, so assigning it adds no enumerable key. */ diff --git a/tests/corpus/7793-untyped-key-writes.js b/tests/corpus/7793-untyped-key-writes.js new file mode 100644 index 000000000..93158145e --- /dev/null +++ b/tests/corpus/7793-untyped-key-writes.js @@ -0,0 +1,83 @@ +"use strict"; +// Untyped property writes through one site: new keys append in order, +// existing keys keep their position, own and inherited setters run, +// getter-only and read-only properties refuse the write, and +// non-extensible objects refuse new keys, as in Node. + +/** @param {any} target @param {string} key @param {unknown} value */ +function put(target, key, value) { + target[key] = value; +} + +/** @param {() => void} fn */ +function attempt(label, fn) { + try { + fn(); + console.log(label, "ok"); + } catch (e) { + console.log(label, e instanceof TypeError ? "TypeError" : String(e)); + } +} + +const order = {}; +for (const key of ["b", "a", "c", "a", "b", "d"]) put(order, key, key.charCodeAt(0)); +console.log(JSON.stringify(order), Object.keys(order).join("")); + +const log = []; +const withSetter = { + _v: 0, + get v() { + return this._v; + }, + set v(value) { + log.push(`own ${value}`); + this._v = value * 2; + }, + get onlyGetter() { + return "g"; + }, +}; +put(withSetter, "v", 4); +console.log(withSetter.v, JSON.stringify(log)); +attempt("getter-only", () => put(withSetter, "onlyGetter", 1)); + +const proto = { + /** @param {number} value */ + set inherited(value) { + log.push(`inherited ${value} on ${this === child}`); + }, +}; +Object.defineProperty(proto, "fixed", { value: 1, writable: false }); +const child = Object.create(proto); +put(child, "inherited", 7); +attempt("inherited read-only", () => put(child, "fixed", 2)); +put(child, "own", 3); +console.log(JSON.stringify(log), Object.keys(child).join(","), child.fixed); + +/** @type {any} */ +const sealed = { x: 1 }; +Object.seal(sealed); +put(sealed, "x", 2); +attempt("sealed add", () => put(sealed, "y", 3)); +/** @type {any} */ +const closed = { x: 1 }; +Object.preventExtensions(closed); +put(closed, "x", 5); +attempt("closed add", () => put(closed, "z", 1)); +const frozen = Object.freeze({ x: 1 }); +attempt("frozen write", () => put(frozen, "x", 9)); +console.log(JSON.stringify(sealed), JSON.stringify(closed), JSON.stringify(frozen)); + +const readOnly = {}; +Object.defineProperty(readOnly, "k", { value: "kept", writable: false, enumerable: true }); +attempt("own read-only", () => put(readOnly, "k", "changed")); +console.log(readOnly.k); + +const error = new Error("boom"); +put(error, "cause", "why"); +put(error, "code", "E_X"); +console.log(error.cause, error.code); + +const wide = {}; +for (let i = 0; i < 40; i++) put(wide, `k${i % 25}`, i); +console.log(Object.keys(wide).length, wide.k0, wide.k24, Object.keys(wide).slice(0, 3).join(",")); diff --git a/tests/harness/corpus-costs/7793-untyped-key-writes.js.json b/tests/harness/corpus-costs/7793-untyped-key-writes.js.json new file mode 100644 index 000000000..eb1b1f127 --- /dev/null +++ b/tests/harness/corpus-costs/7793-untyped-key-writes.js.json @@ -0,0 +1 @@ +{ "plain": 4300, "san": 4800 } From e901526c8b1386f76869734669657b3e6101537b Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 11:35:14 -0500 Subject: [PATCH 065/184] Point the renamed order-parity baselines at their corpus programs --- .../compiler/src/backend/llvm/expr-bytes.ts | 17 +++++ .../compiler/src/backend/llvm/lib-shared.ts | 1 + .../src/backend/llvm/reference-effects.ts | 3 + .../src/frontend/lowering/containers/bytes.ts | 68 ++++++++++++++++++- .../src/frontend/lowering/lower-inspect.ts | 20 +++--- .../compiler/src/ir/builtin-signatures.ts | 3 + packages/compiler/src/ir/ir.ts | 8 +++ packages/compiler/src/ir/validate.ts | 3 + .../corpus/7745-typed-array-unions.ts.json | 2 +- .../7747-typed-array-class-values.ts.json | 2 +- ...48-typed-array-number-array-unions.ts.json | 2 +- .../7749-typed-array-element-searches.ts.json | 6 ++ .../7750-typed-array-union-inspect.ts.json | 6 ++ packages/runtime/src/scr_bytes.c | 45 ++++++++++++ packages/runtime/src/scr_bytes.h | 8 +++ packages/runtime/src/scr_inspect.c | 6 ++ packages/runtime/src/scr_inspect.h | 4 ++ .../7749-typed-array-element-searches.ts | 22 ++++++ .../corpus/7750-typed-array-union-inspect.ts | 14 ++++ .../7749-typed-array-element-searches.ts.json | 1 + .../7750-typed-array-union-inspect.ts.json | 1 + 21 files changed, 228 insertions(+), 14 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7749-typed-array-element-searches.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7750-typed-array-union-inspect.ts.json create mode 100644 tests/corpus/7749-typed-array-element-searches.ts create mode 100644 tests/corpus/7750-typed-array-union-inspect.ts create mode 100644 tests/harness/corpus-costs/7749-typed-array-element-searches.ts.json create mode 100644 tests/harness/corpus-costs/7750-typed-array-union-inspect.ts.json diff --git a/packages/compiler/src/backend/llvm/expr-bytes.ts b/packages/compiler/src/backend/llvm/expr-bytes.ts index 41ddf2b9d..4d27fa647 100644 --- a/packages/compiler/src/backend/llvm/expr-bytes.ts +++ b/packages/compiler/src/backend/llvm/expr-bytes.ts @@ -936,6 +936,23 @@ export function emitBytesIntrinsic( B.line(`${t} = fcmp one double ${idx.name}, ${f64Lit(-1)}`); return { name: t, type: e.type }; } + case "indexOfElem": + case "lastIndexOfElem": + return call( + "scr_bytes_index_of_elem", + "double (ptr, double, double, i1 zeroext)", + `ptr ${r.name}, double ${args[0]!.name}, double ${args[1]!.name}, i1 ${method === "lastIndexOfElem"}`, + false, + false, + ); + case "includesElem": + return call( + "scr_bytes_includes_elem", + "zeroext i1 (ptr, double, double)", + `ptr ${r.name}, double ${args[0]!.name}, double ${args[1]!.name}`, + false, + false, + ); case "fillElem": // Per-element TypedArray fill (non-u8): slice-style index // defaults, never throws; the receiver comes back +1. diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index af32c6936..072e65bba 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -649,6 +649,7 @@ export const LIB_FN_SYMS: Record = { "process.stderrWriteBytes": "scr_process_stderr_write_bytes", "insp.buffer": "scr_insp_buffer", "insp.typedArray": "scr_insp_typed_array", + "insp.bytes": "scr_insp_bytes", // util.inspect (scr_inspect.c): all args borrowed; string results +1; // the begin/entry/key/moreItems/end accumulator drives the emitted // container walks. None of these throw. diff --git a/packages/compiler/src/backend/llvm/reference-effects.ts b/packages/compiler/src/backend/llvm/reference-effects.ts index 6ebe1222b..43a9176d5 100644 --- a/packages/compiler/src/backend/llvm/reference-effects.ts +++ b/packages/compiler/src/backend/llvm/reference-effects.ts @@ -181,6 +181,9 @@ function expressionPreservesEdges( case "bytesIntrinsic": return ( e.method === "get" || + e.method === "indexOfElem" || + e.method === "lastIndexOfElem" || + e.method === "includesElem" || e.method === "length" || e.method === "byteLength" || e.method === "byteOffset" || diff --git a/packages/compiler/src/frontend/lowering/containers/bytes.ts b/packages/compiler/src/frontend/lowering/containers/bytes.ts index 11f56986b..c804d62e1 100644 --- a/packages/compiler/src/frontend/lowering/containers/bytes.ts +++ b/packages/compiler/src/frontend/lowering/containers/bytes.ts @@ -41,7 +41,7 @@ import { lowerStaticallyUndefinedArgument, } from "../optional-arguments.js"; import { lowerArraySpreadItems, lowerCheckedArrayFrom } from "./array-construction.js"; -import { lowerToNumberArgument } from "../lower-exprs.js"; +import { lowerOptionalNumber, lowerToNumberArgument } from "../lower-exprs.js"; import { lowerBytesIteratorValue } from "../iterator-adapters.js"; import { lowerArrayCallback } from "./callback-arguments.js"; import { callbackSite, findPredicateOk, predicateToBoolean } from "./array-callbacks.js"; @@ -911,6 +911,45 @@ function lowerDataViewNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr { * variable-width read/writeUIntLE quartet — BUF_NUM_METHODS). Everything * else the lib declares (fill, indexOf, reverse, ...) falls through to * the SC2020 member fence. Null when this isn't a bytes method call. */ +/** The [needle, fromIndex] arguments of TypedArray indexOf, lastIndexOf + * and includes. A present fromIndex converts even when it is undefined + * (NaN, which ToIntegerOrInfinity makes 0); an absent one is 0, or + * +Infinity for lastIndexOf, which starts at the last element. */ +function elementSearchArgs(lowerer: Lowerer, call: ts.CallExpression, name: string): IrExpr[] { + const nodes = call.arguments.nodes; + const loc = locOf(call); + const needle = lowerer.lowerExprExpecting(nodes[0]!, F64); + const from: IrExpr = nodes[1] + ? lowerOptionalNumber(lowerer, lowerer.lowerExpr(nodes[1]), locOf(nodes[1]), nodes[1]) + : ({ + kind: "numLit", + value: name === "lastIndexOf" ? Infinity : 0, + type: F64, + loc, + } satisfies IrExpr); + // An undefined fromIndex is NaN once converted, which starts at 0. + if (from.kind === "unitLit") return [needle, { kind: "numLit", value: NaN, type: F64, loc }]; + if (from.type.kind === "undefinedT" || from.type.kind === "void") + return [ + needle, + { + kind: "seqExpr", + stmts: [{ kind: "exprStmt", expr: from, loc }], + result: { kind: "numLit", value: NaN, type: F64, loc }, + type: F64, + loc, + }, + ]; + if (from.type.kind !== "f64") lowerer.noLowering(`.${name} with a non-number fromIndex`, call); + return [needle, from]; +} + +const ELEMENT_SEARCHES: Readonly> = { + indexOf: "indexOfElem", + lastIndexOf: "lastIndexOfElem", + includes: "includesElem", +}; + /** The element-kind-generic typed-array methods on a union of typed * arrays (indexed-unions.ts): the arguments lower as for one typed array, * and each arm runs the method on its own receiver. Null for the methods @@ -947,6 +986,10 @@ function lowerUnionBytesMethodCall( method = "toReversed"; args = () => []; result = "receiver"; + } else if (Object.hasOwn(ELEMENT_SEARCHES, name) && (nodes.length === 1 || nodes.length === 2)) { + method = ELEMENT_SEARCHES[name]!; + args = () => elementSearchArgs(lowerer, call, name); + result = name === "includes" ? BOOL : F64; } else if (name === "join" && nodes.length <= 1) { method = "join"; args = () => [optional(0, { kind: "strLit", value: ",", type: STRING, loc }, STRING)]; @@ -1024,6 +1067,29 @@ export function lowerBytesMethodCall( if (value) return value; } } + // TypedArray indexOf / lastIndexOf / includes (the Buffer-declared forms, + // which also take string and Buffer needles, lowered above). + if ( + Object.hasOwn(ELEMENT_SEARCHES, name) && + !lowerer.checker + .getSymbolAtLocation(access.name) + ?.declarations.some( + (d) => ts.isInterfaceDeclaration(d.parent) && d.parent.name.text === "Buffer", + ) + ) { + if (nArgs < 1 || nArgs > 2 || call.arguments.nodes.some(ts.isSpreadElement)) { + lowerer.noLowering(`.${name} with ${nArgs} arguments on typed arrays`, call); + } + const receiver = lowerer.lowerExprExpecting(access.expression, receiverIr); + return { + kind: "bytesIntrinsic", + method: ELEMENT_SEARCHES[name]!, + receiver, + args: elementSearchArgs(lowerer, call, name), + type: name === "includes" ? BOOL : F64, + loc, + }; + } if (name === "toReversed") { if (nArgs !== 0) { lowerer.noLowering(`.toReversed with ${nArgs} arguments on typed arrays`, call); diff --git a/packages/compiler/src/frontend/lowering/lower-inspect.ts b/packages/compiler/src/frontend/lowering/lower-inspect.ts index 5266d5c4e..d914faa5d 100644 --- a/packages/compiler/src/frontend/lowering/lower-inspect.ts +++ b/packages/compiler/src/frontend/lowering/lower-inspect.ts @@ -736,11 +736,7 @@ function inspectSupport( case "array": return inspectSupport(lowerer, t.elem, visiting, out); case "bytes": - // A Uint8Array and a Buffer share the u8 representation and print - // differently; the other element kinds are never Buffers. - return t.elem === "u8" - ? "Uint8Array and Buffer values inside composites have no inspect lowering yet (Buffer's form vs Uint8Array's is not recorded in the static type here)" - : null; + return null; case "record": { if (visiting.has(t.shapeId)) { out.recursive = true; @@ -950,7 +946,9 @@ export function inspectExpr( loc, }; } - return { kind: "libCall", fn: "insp.buffer", args: [value], type: STRING, loc }; + // A Uint8Array, a Buffer and a DataView share the u8 representation + // and print differently: the view's own flags pick. + return { kind: "libCall", fn: "insp.bytes", args: [value, recurse, depth], type: STRING, loc }; case "dyn": return { kind: "libCall", fn: "insp.dyn", args: [value, recurse, depth], type: STRING, loc }; case "generator": @@ -2181,15 +2179,17 @@ export function lowerConsoleInspectArg( ); } if (!isBufferTypeName(tname)) { + // A u8 value the checker types as a Uint8Array may still hold a + // Buffer, which Node prints as one: the view's flags pick. return { kind: "libCall", - fn: "insp.typedArray", + fn: t.elem === "u8" ? "insp.bytes" : "insp.typedArray", args: [value, numLit(0, loc), numLit(2, loc)], type: STRING, loc, }; } - return inspectExpr(lowerer, t, value, numLit(0, loc), numLit(2, loc), loc); + return { kind: "libCall", fn: "insp.buffer", args: [value], type: STRING, loc }; } if (t.kind === "void") { // Node prints "undefined" for a void call's result; a void expression @@ -2510,7 +2510,7 @@ export function lowerInspectValue( value, (input) => ({ kind: "libCall", - fn: "insp.typedArray", + fn: value.type.kind === "bytes" && value.type.elem === "u8" ? "insp.bytes" : "insp.typedArray", args: [input, numLit(0, loc), numLit(options.depth, loc)], type: STRING, loc, @@ -2519,7 +2519,7 @@ export function lowerInspectValue( ); } // Buffers render fully at any depth and on one line (custom inspect). - return inspectExpr(lowerer, value.type, value, numLit(0, loc), numLit(2, loc), loc); + return { kind: "libCall", fn: "insp.buffer", args: [value], type: STRING, loc }; } const walk = { recursive: false }; const reason = inspectSupport(lowerer, value.type, new Set(), walk); diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 1be0f95cd..071e9c59e 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -4408,6 +4408,9 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "insp.buffer": { argTypes: [BYTES_U8], result: STRING }, /** A numeric typed array that is not a Buffer (any element kind). */ "insp.typedArray": { argTypes: [null, F64, F64], result: STRING }, + /** A u8 byte view in a union arm, element or field: a DataView, Buffer + * or Uint8Array by its own flags. */ + "insp.bytes": { argTypes: [null, F64, F64], result: STRING }, "insp.error": { argTypes: [null, F64, F64], result: STRING }, "insp.errorHasCause": { argTypes: [null], result: BOOL }, "insp.errorBagHasProperties": { argTypes: [DYN], result: BOOL }, diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 878088d65..c6842f9e4 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -2474,6 +2474,14 @@ export type IrBytesIntrinsicMethod = | "indexOfNum" | "lastIndexOfNum" | "includesNum" + /** TypedArray.prototype.indexOf / lastIndexOf / includes on any element + * kind: [f64 needle, f64 fromIndex]. indexOf and lastIndexOf compare + * with strict equality (NaN is never found, -0 matches 0), includes with + * SameValueZero (NaN is found). An omitted fromIndex is 0, or +Infinity + * for lastIndexOf (which clamps to the last index). */ + | "indexOfElem" + | "lastIndexOfElem" + | "includesElem" | "fill" | "fillNum" | "fillStr" diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index a404c918a..c319cef15 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -2192,6 +2192,9 @@ function validateFunction( indexOfNum: { argTypes: [F64, F64], minArgs: 1, result: F64 }, lastIndexOfNum: { argTypes: [F64, F64], minArgs: 1, result: F64 }, includesNum: { argTypes: [F64, F64], minArgs: 1, result: BOOL }, + indexOfElem: { argTypes: [F64, F64], minArgs: 2, result: F64 }, + lastIndexOfElem: { argTypes: [F64, F64], minArgs: 2, result: F64 }, + includesElem: { argTypes: [F64, F64], minArgs: 2, result: BOOL }, fill: { argTypes: [BYTES_U8, F64, F64], minArgs: 1, result: BYTES_U8 }, fillNum: { argTypes: [F64, F64, F64], minArgs: 1, result: BYTES_U8 }, // Per-element TypedArray fill (any elem — the non-u8 fill path). diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7745-typed-array-unions.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7745-typed-array-unions.ts.json index c85ce647a..9c770f8dd 100644 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7745-typed-array-unions.ts.json +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7745-typed-array-unions.ts.json @@ -1,6 +1,6 @@ { "order": [ - "/tests/corpus/7742-typed-array-unions.ts" + "/tests/corpus/7745-typed-array-unions.ts" ], "diags": [] } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7747-typed-array-class-values.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7747-typed-array-class-values.ts.json index 63b6307ea..a18e9e7df 100644 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7747-typed-array-class-values.ts.json +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7747-typed-array-class-values.ts.json @@ -1,6 +1,6 @@ { "order": [ - "/tests/corpus/7744-typed-array-class-values.ts" + "/tests/corpus/7747-typed-array-class-values.ts" ], "diags": [] } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7748-typed-array-number-array-unions.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7748-typed-array-number-array-unions.ts.json index b2274d160..783fb1435 100644 --- a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7748-typed-array-number-array-unions.ts.json +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7748-typed-array-number-array-unions.ts.json @@ -1,6 +1,6 @@ { "order": [ - "/tests/corpus/7743-typed-array-number-array-unions.ts" + "/tests/corpus/7748-typed-array-number-array-unions.ts" ], "diags": [] } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7749-typed-array-element-searches.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7749-typed-array-element-searches.ts.json new file mode 100644 index 000000000..8a1372cea --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7749-typed-array-element-searches.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7749-typed-array-element-searches.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7750-typed-array-union-inspect.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7750-typed-array-union-inspect.ts.json new file mode 100644 index 000000000..26cc31982 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7750-typed-array-union-inspect.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7750-typed-array-union-inspect.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_bytes.c b/packages/runtime/src/scr_bytes.c index 7bcfd0cfb..dd6c60783 100644 --- a/packages/runtime/src/scr_bytes.c +++ b/packages/runtime/src/scr_bytes.c @@ -3300,6 +3300,51 @@ double scr_bytes_index_of_num(const ScrBytes *b, double v, double off, bool fwd) return scr_bytes_index_of(b, &needle, off, 1, fwd); } +/* The first index a TypedArray search from `from` starts at + * (ToIntegerOrInfinity, then relative to the end when negative), or false + * when it starts past the last element. */ +static bool scr_bytes_search_start(size_t len, double from, size_t *start) { + double n = from != from ? 0 : trunc(from); + if (n >= (double)len) return false; + if (n < 0) n = n + (double)len < 0 ? 0 : n + (double)len; + *start = (size_t)n; + return true; +} + +double scr_bytes_index_of_elem(const ScrBytes *b, double x, double from, bool last) { + SCR_BYTES_SNAPSHOT(b); + size_t len = b->len; + size_t size = scr_bytes_elem_size(b->elem); + if (len == 0 || x != x) return -1; + if (!last) { + size_t k; + if (!scr_bytes_search_start(len, from, &k)) return -1; + for (; k < len; k++) + if (scr_bytes_decode(b->elem, b->data + k * size) == x) return (double)k; + return -1; + } + double n = from != from ? 0 : trunc(from); + double k = n >= 0 ? (n < (double)len - 1 ? n : (double)len - 1) : (double)len + n; + if (k < 0) return -1; + for (size_t i = (size_t)k + 1; i-- > 0;) + if (scr_bytes_decode(b->elem, b->data + i * size) == x) return (double)i; + return -1; +} + +bool scr_bytes_includes_elem(const ScrBytes *b, double x, double from) { + SCR_BYTES_SNAPSHOT(b); + size_t len = b->len; + size_t size = scr_bytes_elem_size(b->elem); + size_t k; + if (len == 0 || !scr_bytes_search_start(len, from, &k)) return false; + bool nan = x != x; + for (; k < len; k++) { + double v = scr_bytes_decode(b->elem, b->data + k * size); + if (v == x || (nan && v != v)) return true; + } + return false; +} + /* fill's shared core over a normalized byte pattern. nargs counts the * present offset/end args. An EMPTY pattern zero-fills when zero_ok * (Node's '' string) and is the constant TypeError otherwise (an empty diff --git a/packages/runtime/src/scr_bytes.h b/packages/runtime/src/scr_bytes.h index 9b382dda5..1ba7a7c8b 100644 --- a/packages/runtime/src/scr_bytes.h +++ b/packages/runtime/src/scr_bytes.h @@ -499,6 +499,14 @@ ScrEmitter *scr_emitter_set_max_chk(ScrEmitter *em, const ScrDyn *n); void scr_emitter_set_default_max_chk(const ScrDyn *n, const ScrStr *name); double scr_bytes_index_of(const ScrBytes *b, const ScrBytes *needle, double off, double align, bool fwd); double scr_bytes_index_of_num(const ScrBytes *b, double v, double off, bool fwd); +/* TypedArray indexOf (last false) and lastIndexOf (last true) over any + * element kind, with strict equality: NaN is never found and -0 matches 0. + * `from` is the fromIndex argument (ToIntegerOrInfinity applies); the + * caller passes 0, or +Infinity for lastIndexOf, when it is omitted. */ +double scr_bytes_index_of_elem(const ScrBytes *b, double x, double from, bool last); +/* TypedArray includes: indexOf's search from `from` with SameValueZero, + * so NaN is found. */ +bool scr_bytes_includes_elem(const ScrBytes *b, double x, double from); ScrBytes *scr_bytes_fill(ScrBytes *b, const ScrBytes *pattern, double nargs, double offset, double end); ScrBytes *scr_bytes_fill_num(ScrBytes *b, double v, double nargs, double offset, double end); ScrBytes *scr_bytes_fill_str(ScrBytes *b, const ScrStr *s, const ScrStr *enc, diff --git a/packages/runtime/src/scr_inspect.c b/packages/runtime/src/scr_inspect.c index 1dce8d3c2..522a56977 100644 --- a/packages/runtime/src/scr_inspect.c +++ b/packages/runtime/src/scr_inspect.c @@ -1489,6 +1489,12 @@ ScrStr *scr_insp_typed_array(ScrBytes *b, double recurse, double depth) { return out; } +ScrStr *scr_insp_bytes(ScrBytes *b, double recurse, double depth) { + if (b->is_data_view) return scr_insp_data_view(b, recurse, depth); + if (b->is_buffer) return scr_insp_buffer(b); + return scr_insp_typed_array(b, recurse, depth); +} + ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { if (d->kind == SCR_DYN_PROXY) return scr_insp_dyn(d->v.proxy.target, recurse, depth); switch (d->kind) { diff --git a/packages/runtime/src/scr_inspect.h b/packages/runtime/src/scr_inspect.h index 5ac3aa9dd..af146c7d3 100644 --- a/packages/runtime/src/scr_inspect.h +++ b/packages/runtime/src/scr_inspect.h @@ -45,6 +45,10 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth); /* A numeric typed array that is not a Buffer: `Uint8Array(n) [ ... ]`, * `[Uint8Array]` past the depth limit. Borrows b; +1. */ ScrStr *scr_insp_typed_array(ScrBytes *b, double recurse, double depth); +/* A u8 byte view whose static type does not say which one it is (a union + * arm, an element or field): a DataView, a Buffer () or a + * Uint8Array, by the view's own flags. */ +ScrStr *scr_insp_bytes(ScrBytes *b, double recurse, double depth); /* format's %s / rest-args twin: dyn strings pass verbatim. */ ScrStr *scr_insp_dyn_s(ScrDyn *d, double depth); void scr_insp_begin(double recurse); diff --git a/tests/corpus/7749-typed-array-element-searches.ts b/tests/corpus/7749-typed-array-element-searches.ts new file mode 100644 index 000000000..7cae267ec --- /dev/null +++ b/tests/corpus/7749-typed-array-element-searches.ts @@ -0,0 +1,22 @@ +// TypedArray indexOf, lastIndexOf and includes on every element kind, on +// Buffers (their own search), and on unions of typed arrays: strict +// equality for indexOf/lastIndexOf, SameValueZero for includes. +const f = new Float64Array([1, NaN, -0, 0, 2.5, NaN, 1]); +console.log(f.indexOf(NaN), f.includes(NaN), f.lastIndexOf(NaN)); +console.log(f.indexOf(0), f.indexOf(-0), f.lastIndexOf(0), f.lastIndexOf(-0), f.includes(-0)); +console.log(f.indexOf(1, 1), f.indexOf(1, -1), f.indexOf(1, -100), f.indexOf(1, 100), f.indexOf(1, 1.9)); +console.log(f.lastIndexOf(1, 5), f.lastIndexOf(1, -2), f.lastIndexOf(1, -100), f.lastIndexOf(1, 100), f.lastIndexOf(1, undefined)); +console.log(f.includes(2.5, 4), f.includes(2.5, 5), f.includes(2.5, -3), f.includes(NaN, 6)); +const u = new Uint8Array([0, 255, 3, 255]); +console.log(u.indexOf(255), u.indexOf(-1), u.indexOf(511), u.indexOf(3.5), u.includes(NaN), u.lastIndexOf(255)); +const i = new Int16Array([-5, 7, -5]); +console.log(i.indexOf(-5), i.lastIndexOf(-5), i.includes(7), i.indexOf(65531)); +const h = new Float32Array([0.1, 0.5, 16777217]); +console.log(h.indexOf(0.1), h.indexOf(0.5), h.includes(16777216), h.indexOf(16777217)); +const c = new Uint8ClampedArray([300, -4]); +console.log(c.indexOf(255), c.indexOf(0), new Uint32Array(0).indexOf(0), new Uint32Array(0).includes(0)); +const b = Buffer.from([1, 2, 3, 2]); +console.log(b.indexOf(2), b.indexOf(258), b.lastIndexOf(2), b.includes("\u0003")); +type UintArray = Uint8Array | Uint16Array | Uint32Array; +const mk = (n: number): UintArray => (n > 1 ? new Uint16Array([1, 700, 1]) : new Uint8Array([1, 188, 1])); +for (const x of [mk(1), mk(2)]) console.log(x.indexOf(700), x.lastIndexOf(1), x.includes(188), x.indexOf(1, 1)); diff --git a/tests/corpus/7750-typed-array-union-inspect.ts b/tests/corpus/7750-typed-array-union-inspect.ts new file mode 100644 index 000000000..aebb0dbdf --- /dev/null +++ b/tests/corpus/7750-typed-array-union-inspect.ts @@ -0,0 +1,14 @@ +// Printing u8 byte views whose static type does not say what they are: +// union arms, elements and fields print as Node does by the value's kind. +type UintArray = Uint8Array | Uint16Array | Uint32Array; +const mk = (n: number): UintArray => (n > 1 ? new Uint16Array([1, 700]) : new Uint8Array([1, 188])); +const a = mk(1), b = mk(2); +console.log(a, b); +console.log([a, b]); +const bufArm: Uint8Array | Uint16Array = Buffer.from([1, 2, 3]); +console.log(bufArm); +const u8s: Uint8Array[] = [new Uint8Array([9]), Buffer.from([10, 11])]; +console.log(u8s); +const rec: { bytes: Uint8Array; view: DataView } = { bytes: new Uint8Array(2), view: new DataView(new ArrayBuffer(2)) }; +console.log(rec); +console.log(new Uint8Array([5, 6]), Buffer.from([7])); diff --git a/tests/harness/corpus-costs/7749-typed-array-element-searches.ts.json b/tests/harness/corpus-costs/7749-typed-array-element-searches.ts.json new file mode 100644 index 000000000..7d76a0583 --- /dev/null +++ b/tests/harness/corpus-costs/7749-typed-array-element-searches.ts.json @@ -0,0 +1 @@ +{ "plain": 2650, "san": 2850 } diff --git a/tests/harness/corpus-costs/7750-typed-array-union-inspect.ts.json b/tests/harness/corpus-costs/7750-typed-array-union-inspect.ts.json new file mode 100644 index 000000000..c0aef3bb7 --- /dev/null +++ b/tests/harness/corpus-costs/7750-typed-array-union-inspect.ts.json @@ -0,0 +1 @@ +{ "plain": 2700, "san": 3900 } From 70e6e9b59182af4fda8f40a48d4b923c963ba090 Mon Sep 17 00:00:00 2001 From: Luis Alvarez D Date: Fri, 23 Oct 2026 11:40:15 -0500 Subject: [PATCH 066/184] Fence split requires that close a CommonJS cycle --- .../frontend/lowering/cjs-inline-require.ts | 7 ----- .../compiler/src/frontend/lowering/lowerer.ts | 7 +++++ tests/harness/errors.test.ts | 30 +++++++++++++++++++ 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/cjs-inline-require.ts b/packages/compiler/src/frontend/lowering/cjs-inline-require.ts index eb05550c0..3d28dfae3 100644 --- a/packages/compiler/src/frontend/lowering/cjs-inline-require.ts +++ b/packages/compiler/src/frontend/lowering/cjs-inline-require.ts @@ -431,13 +431,6 @@ export function inlineNamespaceRequireOf( /** `value` evaluated after the required module's run-once %init (and its * module-registry link) at the require's own position. */ export function withRequireInit(lowerer: Lowerer, req: InlineRequire, value: IrExpr): IrExpr { - if (lowerer.inlineRequireCycleCalls.has(req.call)) { - lowerer.unsupported( - "SC1090", - req.call, - `require() of '${req.spec}' while it is still evaluating (a require cycle: Node answers its partial exports, which the static export table does not model)`, - ); - } // Every inline program require is a preflight module edge, so its module // has an initializer; never read its exports without evaluating it. if (!lowerer.initNameOf.has(req.dep)) { diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 43303b4c6..2b76080fd 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -4851,6 +4851,13 @@ export class Lowerer { * their external dependencies by name). Null for everything else * (builtins load nothing; the rest kept its preflight fence). */ requireInitStmt(spec: string, node: ts.Node): IrStmt | null { + if (this.inlineRequireCycleCalls.has(node)) { + this.unsupported( + "SC1090", + node, + `require() of '${spec}' while it is still evaluating (a require cycle: Node answers its partial exports, which the static export table does not model)`, + ); + } if (canonicalBuiltinModule(spec) === "punycode") return this.punycodeRequireWarning(node); // Project requires resolve within the program; another bare require can // be a program-module edge when it names an opted-in --npm-static package diff --git a/tests/harness/errors.test.ts b/tests/harness/errors.test.ts index 319e7329f..f379973b3 100644 --- a/tests/harness/errors.test.ts +++ b/tests/harness/errors.test.ts @@ -325,6 +325,36 @@ console.log(cells.join(",")); ); }); + test("a split require closing a CommonJS cycle throws the partial-exports fence", async () => { + // Node answers the closing require with the dependency's partial exports + // ({ fromA: 1 } at this point), which the static export model cannot + // represent. The split assignment must hit the same fence as an inline + // require in expression position instead of reading uninitialized + // export storage. + const r = await compileAndRun( + "split-require-cycle", + `require("./a.cjs"); +`, + "cjs", + { + "a.cjs": `exports.fromA = 1; +const b = require("./b.cjs"); +exports.fromB = b.fromB; +`, + "b.cjs": `exports.fromB = 2; +var a; +a = require("./a.cjs"); +console.log(a.fromA, a.fromB); +`, + }, + ); + expect(r.exitCode).toBe(1); + expect(r.stdout).toBe(""); + expect(r.stderr).toMatch( + /^Uncaught Error: require\(\) of '\.\/a\.cjs' while it is still evaluating \(a require cycle: Node answers its partial exports, which the static export table does not model\) is not supported yet \[SC1090 at .*b\.cjs:3\]\n$/, + ); + }); + test("an EFFECTFUL surplus argument is a named deferred fence (JS lane)", async () => { // Effect-free surplus args drop at compile time, JS-exact (corpus // 1760 — the test-string-decoder writeSequences shape). An effectful From b92fb630435e4850e719729eda255164eb4d185c Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 11:49:45 -0500 Subject: [PATCH 067/184] Mark typed-array header loads invariant A typed array's length, data pointer and shared backing are fixed when the view is constructed. Each element access loads the data pointer beside the length, ahead of its bounds branch. LLVM then sinks the data pointer load into the in-bounds block, and LICM refuses to hoist a load that runs only on some iterations. A loop whose body branches or stores an element therefore reloaded the data pointer for every access. Header loads now carry `!invariant.load` next to their alias tag, so the data pointer and length leave such loops. Debug builds emit neither. --- packages/compiler/src/backend/llvm/bytes-alias.ts | 10 ++++++++-- .../src/backend/llvm/bytes-loop-versions.test.ts | 5 +++++ packages/compiler/src/backend/llvm/emit-statements.ts | 6 ++---- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/packages/compiler/src/backend/llvm/bytes-alias.ts b/packages/compiler/src/backend/llvm/bytes-alias.ts index 9aaeaa6c7..d43620fd9 100644 --- a/packages/compiler/src/backend/llvm/bytes-alias.ts +++ b/packages/compiler/src/backend/llvm/bytes-alias.ts @@ -7,9 +7,15 @@ import type { LlvmEmitterContext } from "./expr-context.js"; * buffer. Neither overlaps the other, a native field, array storage or a * module global's slot, so element stores leave header loads and global * receiver slots loop-invariant. Untagged accesses and calls still alias - * every category. */ + * every category. + * + * Header loads are also `!invariant.load`: no store or call between the + * construction of a view and its release changes them. Without it, a data + * pointer load ahead of a bounds branch is sunk into the in-bounds block, + * and LICM keeps a conditionally executed load inside the loop. */ export function bytesHeaderAlias(host: LlvmEmitterContext): string { - return host.memoryAliasAttachment("bytes:header"); + const tag = host.memoryAliasAttachment("bytes:header"); + return tag === "" ? "" : `${tag}, !invariant.load !{}`; } export function bytesElementAlias(host: LlvmEmitterContext): string { diff --git a/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts b/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts index cee91c66d..34b75861c 100644 --- a/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts +++ b/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts @@ -249,6 +249,11 @@ test("emitted loops version the body and tag typed-array memory", () => { const elements = /store double [^\n]*!tbaa !(\d+)/.exec(body)![1]; const header = /load ptr, ptr [^\n]*!tbaa !(\d+)/.exec(body)![1]; expect(elements).not.toBe(header); + // Header loads are invariant for the view's lifetime; element accesses + // are not. + expect(body).toMatch(/load ptr, ptr [^\n]*, !tbaa !\d+, !invariant\.load !\{\}$/m); + expect(body).toMatch(/load i64, ptr [^\n]*, !tbaa !\d+, !invariant\.load !\{\}$/m); + expect(body).not.toMatch(/(load double|store double)[^\n]*invariant\.load/); }); test("loops to a typed array's length version the other typed arrays they index", () => { diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index 48f7a1d5f..aef5ccf30 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -64,6 +64,7 @@ import { emitPairedLoopSafety, } from "./counted-loops.js"; import { emitBytesLoopVersionGuard, type BytesLoopVersion } from "./bytes-loop-versions.js"; +import { bytesHeaderAlias } from "./bytes-alias.js"; import { mangleLocal } from "../mangle.js"; import { LlvmUnsupportedError } from "./unsupported.js"; import type { LlValue } from "./expr-context.js"; @@ -1229,10 +1230,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { } B.line(`${inBounds} = icmp ult i32 ${narrow}, ${len}`); } else { - emitter.markMemoryPointer(lenPtr, "bytes:header"); - B.line( - `${len} = load ${emitter.sizeType}, ptr ${lenPtr}${emitter.fieldAliasAttachment(lenPtr)}`, - ); + B.line(`${len} = load ${emitter.sizeType}, ptr ${lenPtr}${bytesHeaderAlias(emitter)}`); B.line(`${inBounds} = icmp ult ${emitter.sizeType} ${index}, ${len}`); } B.condBr(inBounds, lb, exit); From bd36b77a8fff8a36550647208de65629b21cb3f0 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 11:57:22 -0500 Subject: [PATCH 068/184] Cache untyped property access in programs that declare classes (reland) --- benchmarks/runtime/README.md | 1 + .../runtime/workloads/dynamic-properties.json | 6 + .../runtime/workloads/dynamic-properties.mjs | 56 ++++++++ packages/compiler/src/ir/js-ic.test.ts | 134 ++++++++++++++++++ packages/compiler/src/ir/js-ic.ts | 33 ++++- packages/compiler/test/js-ic-emission.test.ts | 44 ++++++ packages/runtime/src/scr_js_object.c | 32 ++++- scripts/zero-cost-compare.mjs | 5 +- tests/harness/library-mode.test.ts | 20 +++ tests/library-mode/js-ic-sessions/lib.ts | 6 + tests/library-mode/js-ic-sessions/probe.c | 25 ++++ .../library-mode/js-ic-sessions/profile.json | 14 ++ tests/library-mode/js-ic-sessions/work.mjs | 26 ++++ 13 files changed, 390 insertions(+), 12 deletions(-) create mode 100644 benchmarks/runtime/workloads/dynamic-properties.json create mode 100644 benchmarks/runtime/workloads/dynamic-properties.mjs create mode 100644 tests/library-mode/js-ic-sessions/lib.ts create mode 100644 tests/library-mode/js-ic-sessions/probe.c create mode 100644 tests/library-mode/js-ic-sessions/profile.json create mode 100644 tests/library-mode/js-ic-sessions/work.mjs diff --git a/benchmarks/runtime/README.md b/benchmarks/runtime/README.md index 4f152e4bb..04c90f7bb 100644 --- a/benchmarks/runtime/README.md +++ b/benchmarks/runtime/README.md @@ -13,6 +13,7 @@ Application-shaped workloads for measuring the speed of compiled executables. Ea | `cyclic-graphs` | cpu | object graphs with back references built and dropped: trees with parent and sibling links, rings of records sharing a long-lived registry, closures capturing their owner | | `dictionary-walks` | cpu | `Record` dictionaries walked with for-in and `Object.keys`/`values`/`entries` while entries are added, updated and deleted | | `dynamic-methods` | cpu | untyped JavaScript calling builtin methods on values it does not type: string `slice`/`indexOf`/`toUpperCase`/`padStart`, array `push`/`pop`/`includes`, `Map` and `Set`, and an object's own methods | +| `dynamic-properties` | cpu | untyped JavaScript reading and writing properties of values it does not type: records built by object literals in two key orders, nested records, and instances of a class and its subclass | | `functional-pipeline` | cpu | `filter`/`map`/`reduce`/`flatMap` chains, closures, small records | | `intern-table`, `intern-table-map`, `intern-table-1`, `intern-table-map-1` | cpu | worker threads interning the same type-like objects under string keys and records under number keys, then hits and misses: one shared `InternTable` per kind of key, or a module-level `Map` per thread (the `-map` entries), with 8 threads or 1 (the `-1` entries); under Node.js every thread has its own tables | | `json-records` | cpu | `JSON.stringify`, typed `JSON.parse`, Map aggregation | diff --git a/benchmarks/runtime/workloads/dynamic-properties.json b/benchmarks/runtime/workloads/dynamic-properties.json new file mode 100644 index 000000000..398981ef3 --- /dev/null +++ b/benchmarks/runtime/workloads/dynamic-properties.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/dynamic-properties.mjs", + "args": [ + "1000000" + ] +} diff --git a/benchmarks/runtime/workloads/dynamic-properties.mjs b/benchmarks/runtime/workloads/dynamic-properties.mjs new file mode 100644 index 000000000..d0ee3802c --- /dev/null +++ b/benchmarks/runtime/workloads/dynamic-properties.mjs @@ -0,0 +1,56 @@ +// Untyped JavaScript reading and writing properties of values it never +// declares types for: records built by object literals in two key orders, +// nested records, property updates, and instances of a class and its +// subclass read and written the same way. Argument: the number of +// iterations. +const rounds = Number(process.argv[2] ?? "1000000"); +class Cursor { + constructor(x, y) { + this.x = x; + this.y = y; + } + step(dx, dy) { + this.x += dx; + this.y += dy; + return this; + } +} +class Marker extends Cursor { + constructor(x, y, label) { + super(x, y); + this.label = label; + } +} +function makeItem(i) { + if (i % 3 === 0) + return { id: i, price: i % 17, qty: i % 5, meta: { tag: i % 7, weight: i % 11 } }; + return { id: i, qty: i % 5, price: i % 13, meta: { weight: i % 9, tag: i % 5 } }; +} +function cost(item) { + return item.price * item.qty + item.meta.weight; +} +function restock(item, round) { + item.qty = (item.qty + round) % 7; + item.meta.tag = (item.meta.tag + 1) % 7; +} +function distance(a, b) { + return Math.abs(a.x - b.x) + Math.abs(a.y - b.y); +} +const items = []; +for (let i = 0; i < 1024; i++) items.push(makeItem(i)); +const cursors = []; +for (let i = 0; i < 64; i++) + cursors.push(i % 2 === 0 ? new Cursor(i - 32, 7 - i) : new Marker(i - 32, 7 - i, "m" + i)); +const origin = { x: 0, y: 0 }; +let checksum = 0; +for (let r = 0; r < rounds; r++) { + const item = items[r & 1023]; + checksum += cost(item); + restock(item, r & 3); + checksum += item.meta.tag; + const cursor = cursors[r & 63]; + checksum += distance(cursor.step((r % 3) - 1, (r % 5) - 2), origin); + if (cursor.x > 100 || cursor.x < -100) cursor.x = 0; + if (cursor.y > 100 || cursor.y < -100) cursor.y = 0; +} +console.log(checksum, items[5].qty, items[6].meta.tag, cursors[3].x, cursors[3].y, cursors[3].label); diff --git a/packages/compiler/src/ir/js-ic.test.ts b/packages/compiler/src/ir/js-ic.test.ts index f71a1e9db..026600cc3 100644 --- a/packages/compiler/src/ir/js-ic.test.ts +++ b/packages/compiler/src/ir/js-ic.test.ts @@ -126,6 +126,140 @@ test("property helpers whose class branches only test capsules are cache sites", expect(sites.site("%dyn.class.property.1")).toEqual({ kind: "set", key: "y" }); }); +test("bags converted between dyn and jsval are still the receiver", () => { + // A program that declares a class returns its bags as jsvals, and the + // helpers read them through js.toDyn. + const toDyn = (value: IrExpr): IrExpr => ({ + kind: "libCall", + fn: "js.toDyn", + args: [value], + type: DYN, + loc, + }); + const fromDyn = (value: IrExpr): IrExpr => ({ + kind: "libCall", + fn: "js.fromDyn", + args: [value], + type: { kind: "jsval" }, + loc, + }); + const jsvalBag = helper( + "%dyn.class.readPropertiesOrEmpty.js", + ["v"], + [capsuleBranch("v"), ret(fromDyn(ref("v")))], + ); + const staticBag = helper("%dyn.class.staticProperties", ["v"], [ret(ref("v"))]); + const getter = getHelper( + "%dyn.class.property.0", + "x", + [capsuleBranch("p.0")], + call(staticBag.name, toDyn(call(jsvalBag.name, ref("p.0")))), + ); + const setter = setHelper("%dyn.class.property.1", "y"); + const write = setter.body[0] as Extract; + const keySet = write.expr as Extract; + keySet.args[0] = toDyn(call(jsvalBag.name, ref("p.0"))); + // A converted value that is not the receiver stays a helper call. + const foreign = getHelper( + "%dyn.class.property.2", + "x", + [], + toDyn(fromDyn({ kind: "dynObjLit", type: DYN, loc } as IrExpr)), + ); + const sites = sitesOf(jsvalBag, staticBag, getter, setter, foreign); + expect(sites.site("%dyn.class.property.0")).toEqual({ kind: "get", key: "x" }); + expect(sites.site("%dyn.class.property.1")).toEqual({ kind: "set", key: "y" }); + expect(sites.site("%dyn.class.property.2")).toBeNull(); +}); + +test("a bag helper may first rebind its parameter to a bag of it", () => { + // Programs with subclasses normalize the value before the class branches. + const normalize = helper("%dyn.class.normalize", ["v"], [capsuleBranch("v"), ret(ref("v"))]); + const rebind = (value: IrExpr): IrStmt => ({ kind: "assign", localId: "v", value, loc }); + const normalized = helper( + "%dyn.class.readPropertiesOrEmpty.normalized", + ["v"], + [rebind(call(normalize.name, ref("v"))), capsuleBranch("v"), ret(ref("v"))], + ); + const replaced = helper( + "%dyn.class.readPropertiesOrEmpty.replaced", + ["v"], + [rebind({ kind: "dynObjLit", type: DYN, loc } as IrExpr), ret(ref("v"))], + ); + const sites = sitesOf( + normalize, + normalized, + replaced, + getHelper("%dyn.class.property.0", "x", [], call(normalized.name, ref("p.0"))), + getHelper("%dyn.class.property.1", "x", [], call(replaced.name, ref("p.0"))), + // The property helper itself normalizes its receiver first, too. + getHelper("%dyn.class.property.2", "x", [ + { kind: "assign", localId: "p.0", value: call(normalize.name, ref("p.0")), loc }, + capsuleBranch("p.0"), + ]), + ); + expect(sites.site("%dyn.class.property.0")).toEqual({ kind: "get", key: "x" }); + expect(sites.site("%dyn.class.property.1")).toBeNull(); + expect(sites.site("%dyn.class.property.2")).toEqual({ kind: "get", key: "x" }); +}); + +test("keyed helpers may test capsules between the key's coercion and the read", () => { + const read = (object: IrExpr, key: string): IrExpr => ({ + kind: "libCall", + fn: "js.fromDyn", + args: [{ kind: "dynKeyGet", value: object, key: ref(key), type: DYN, loc }], + type: { kind: "jsval" }, + loc, + }); + const guard = (test: "symbol" | "nullish", tested: string, object: IrExpr): IrStmt => ({ + kind: "if", + cond: { kind: "dynTest", test, value: ref(tested), type: BOOL, loc }, + then: [ret(read(object, "p.key"))], + else_: null, + loc, + }); + const keyed = (name: string, between: IrStmt[]): IrFunction => + helper( + name, + ["p.0", "p.key"], + [ + capsuleBranch("p.0"), + guard("symbol", "p.key", call(identityBag.name, ref("p.0"))), + guard("nullish", "p.0", ref("p.0")), + { + kind: "varDecl", + localId: "key.string", + init: { + kind: "libCall", + fn: "dyn.toStringCoerce", + args: [ref("p.key")], + type: STRING, + loc, + }, + loc, + }, + ...between, + ret(read(call(identityBag.name, ref("p.0")), "key.string")), + ], + ["key.string"], + ); + const errorBranch: IrStmt = { + kind: "if", + cond: { kind: "dynTest", test: "error", value: ref("p.0"), type: BOOL, loc }, + then: [ret(ref("p.0"))], + else_: null, + loc, + }; + const sites = sitesOf( + keyed("%dyn.class.computed.0", []), + keyed("%dyn.class.computed.1", [capsuleBranch("p.0"), capsuleBranch("p.0", "dyn.classIs")]), + keyed("%dyn.class.computed.2", [errorBranch]), + ); + expect(sites.site("%dyn.class.computed.0")).toEqual({ kind: "keyed" }); + expect(sites.site("%dyn.class.computed.1")).toEqual({ kind: "keyed" }); + expect(sites.site("%dyn.class.computed.2")).toBeNull(); +}); + test("any other prefix or tail keeps the helper call", () => { const errorBranch: IrStmt = { kind: "if", diff --git a/packages/compiler/src/ir/js-ic.ts b/packages/compiler/src/ir/js-ic.ts index 752316db3..58c568bbd 100644 --- a/packages/compiler/src/ir/js-ic.ts +++ b/packages/compiler/src/ir/js-ic.ts @@ -62,7 +62,11 @@ export class JsIcSites { const receiver = fn.params[0]?.localId; if (receiver === undefined) return null; let i = 0; - while (i < fn.body.length && this.capsuleBranch(fn.body[i]!, receiver)) i++; + while ( + i < fn.body.length && + (this.capsuleBranch(fn.body[i]!, receiver) || this.rebinding(fn.body[i]!, receiver)) + ) + i++; const tail = fn.body.slice(i); if (fn.params.length === 1) return this.getTail(tail, receiver); if (fn.params.length !== 2) return null; @@ -109,10 +113,14 @@ export class JsIcSites { /** `if (typeof key === "symbol") return dynKeyGet(BAG(receiver), key); * if (receiver == null) return dynKeyGet(receiver, key); * const s = String(key); return dynKeyGet(BAG(receiver), s)`, each read - * possibly under js.fromDyn. */ + * possibly under js.fromDyn. Capsule branches may precede the final read: + * a program that declares classes looks the string key up in each class's + * key table there. */ private keyedTail(tail: IrStmt[], receiver: string, key: string): JsIcSite | null { - const [symbol, nullish, decl, ret] = tail; - if (tail.length !== 4) return null; + const [symbol, nullish, decl] = tail; + const ret = tail[tail.length - 1]; + if (tail.length < 4) return null; + if (!tail.slice(3, -1).every((s) => this.capsuleBranch(s, receiver))) return null; if (!this.guard(symbol, "symbol", key, (read) => this.bagOf(read, receiver), key)) return null; if (!this.guard(nullish, "nullish", receiver, (read) => isVar(read, receiver), key)) return null; @@ -159,9 +167,12 @@ export class JsIcSites { } /** Whether `e` is the receiver itself for every ordinary object or array: - * the receiver, or a chain of property-bag helpers applied to it. */ + * the receiver, or a chain of property-bag helpers applied to it. A + * conversion between the dyn and jsval forms keeps the object it holds. */ private bagOf(e: IrExpr, receiver: string): boolean { if (isVar(e, receiver)) return true; + if (e.kind === "libCall" && (e.fn === "js.toDyn" || e.fn === "js.fromDyn")) + return e.args.length === 1 && this.bagOf(e.args[0]!, receiver); return ( e.kind === "call" && e.args.length === 1 && @@ -182,6 +193,13 @@ export class JsIcSites { return ret.kind === "return" && ret.value !== null && this.bagOf(ret.value, param); } + /** `param = `, such as the subclass normalization + * `value = %dyn.class.normalize(value)`, which leaves an ordinary object + * or array in `param`. */ + private rebinding(s: IrStmt, param: string): boolean { + return s.kind === "assign" && s.localId === param && this.bagOf(s.value, param); + } + /** A property-bag helper that returns its argument unless a capsule * branch is taken. */ private identityHelper(name: string): boolean { @@ -204,7 +222,10 @@ export class JsIcSites { identity = fn.body .slice(0, -1) - .every((s) => this.capsuleBranch(s, param) || this.kindExit(s, param)) && + .every( + (s) => + this.capsuleBranch(s, param) || this.kindExit(s, param) || this.rebinding(s, param), + ) && last?.kind === "return" && last.value !== null && this.bagOf(last.value, param); diff --git a/packages/compiler/test/js-ic-emission.test.ts b/packages/compiler/test/js-ic-emission.test.ts index 47d641bff..54ea13d26 100644 --- a/packages/compiler/test/js-ic-emission.test.ts +++ b/packages/compiler/test/js-ic-emission.test.ts @@ -64,6 +64,50 @@ console.log(read(o), at(dyn([5, 6]), 1), call(o), o.n); } }); +test("property sites keep their caches in programs that declare classes and subclasses", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-js-ic-class-")); + try { + const entry = join(dir, "main.js"); + const outPath = join(dir, "main.ir.json"); + // Declared classes give every property helper class branches and + // property-bag lookups, and a subclass adds a normalization step; plain + // objects still read through the caches. + await writeFile( + entry, + ` +class Point { + constructor(x) { this.x = x; } +} +class Tagged extends Point { + constructor(x, tag) { super(x); this.tag = tag; } +} +function dyn(v) { return v; } +function read(o) { return o.x; } +function write(o, v) { o.x = v; } +function at(a, i) { return a[i]; } +var o = dyn({ x: 1 }); +var p = dyn(new Point(4)); +var t = dyn(new Tagged(6, "t")); +write(o, 3); +write(p, 5); +write(t, 7); +console.log(read(o), read(p), read(t), at(dyn([5, 6]), 1), at(p, "x"), at(t, "tag")); +`, + ); + const result = await compile(entry, { outDir: dir, outPath, outputKind: "ir" }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const mod = deserializeModule(await readFile(outPath, "utf8")); + expect(validateModule(mod)).toEqual([]); + const llvm = emitLlvmModule(mod); + expect(body(llvm, "read")).toContain("@scr_js_load_probe("); + expect(body(llvm, "write")).toContain("@scr_js_store_probe("); + expect(body(llvm, "at")).toContain("@scr_js_keyed_probe("); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + test("typed programs have no cache sites and keep the ordinary runtime", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-js-ic-typed-")); try { diff --git a/packages/runtime/src/scr_js_object.c b/packages/runtime/src/scr_js_object.c index 63929ab57..0e0ec0908 100644 --- a/packages/runtime/src/scr_js_object.c +++ b/packages/runtime/src/scr_js_object.c @@ -18,6 +18,7 @@ #include "scr_key.h" #include +#include /* The compiler's inline cache tests read these in place (64-bit targets; * backend/llvm/js-ic.ts). */ @@ -37,10 +38,10 @@ _Static_assert(sizeof(ScrJsIcEntry) == 40 && sizeof(ScrJsIc) == 168, "cache layo * take the generic path and its hash index. */ #define SCR_JS_SHAPE_MAX 64 -/* Shapes live until the thread exits, so a thread creates at most this - * many, holding at most this many key bytes. Once either budget is spent, - * objects whose key sequence has no shape yet stay unshaped; shapes that - * exist keep serving. */ +/* Shapes live until the thread exits or a library session ends, so a thread + * creates at most this many, holding at most this many key bytes. Once + * either budget is spent, objects whose key sequence has no shape yet stay + * unshaped; shapes that exist keep serving. */ #define SCR_JS_SHAPE_BUDGET ((size_t)1 << 16) #define SCR_JS_SHAPE_KEY_BYTES_BUDGET ((size_t)4 << 20) @@ -63,12 +64,25 @@ static SCR_TL size_t scr_js_transition_count; static SCR_TL size_t scr_js_shape_key_bytes; static SCR_TL bool scr_js_shapes_registered; +/* The cache records that hold entries. Entries name shapes and prototype + * objects, so the reset that frees the shapes also clears these records: a + * library's next session runs the same sites with the same records. */ +static SCR_TL ScrJsIc **scr_js_filled_ics; +static SCR_TL size_t scr_js_filled_count; +static SCR_TL size_t scr_js_filled_capacity; + /* The shape word of an object whose key sequence has no shape within the * budgets. No cache entry names it, so its accesses take the probes, * which answer with a miss until the key set changes. */ static ScrJsShape scr_js_unshaped; static void scr_js_shapes_reset(void) { + for (size_t i = 0; i < scr_js_filled_count; i++) + memset(scr_js_filled_ics[i], 0, sizeof *scr_js_filled_ics[i]); + free(scr_js_filled_ics); + scr_js_filled_ics = NULL; + scr_js_filled_count = 0; + scr_js_filled_capacity = 0; while (scr_js_all_shapes) { ScrJsShape *shape = scr_js_all_shapes; scr_js_all_shapes = shape->all_next; @@ -192,6 +206,16 @@ static size_t scr_js_shape_find(const ScrJsShape *shape, const char *key, size_t /* Fills a cache entry: `stale` (a matching entry whose hit failed) when * there is one, else a free or replaceable one. */ static void scr_js_ic_add(ScrJsIc *ic, uint32_t stale, ScrJsIcEntry entry) { + if (ic->count == 0) { + if (scr_js_filled_count == scr_js_filled_capacity) { + size_t capacity = scr_js_filled_capacity ? scr_js_filled_capacity * 2 : 64; + ScrJsIc **grown = realloc(scr_js_filled_ics, capacity * sizeof *grown); + if (!grown) scr_trap("scriptc: out of memory\n"); + scr_js_filled_ics = grown; + scr_js_filled_capacity = capacity; + } + scr_js_filled_ics[scr_js_filled_count++] = ic; + } uint32_t index; if (stale < ic->count) { index = stale; diff --git a/scripts/zero-cost-compare.mjs b/scripts/zero-cost-compare.mjs index 160e33438..f202c2ab7 100644 --- a/scripts/zero-cost-compare.mjs +++ b/scripts/zero-cost-compare.mjs @@ -394,8 +394,8 @@ export function dynamicTierSymbols(names) { * not yet traced to its typed root. bench/thread-slots, * bench/channel-messages and bench/worker-messages, added later, link them * through worker messages: - * node:worker_threads values are dyn objects. bench/dynamic-methods is - * untyped JavaScript. Every other bench workload is fully typed and must link none. The gate and + * node:worker_threads values are dyn objects. bench/dynamic-methods and + * bench/dynamic-properties are untyped JavaScript. Every other bench workload is fully typed and must link none. The gate and * tests/harness/bench-dynamic-tier.test.ts enforce that, and the test also * fails when a listed workload stops linking them, so remove a workload here * when a leak fix makes its binary clean. */ @@ -410,6 +410,7 @@ export const DYNAMIC_WORKLOADS = new Set([ "bench/thread-slots", "bench/channel-messages", "bench/dynamic-methods", + "bench/dynamic-properties", "bench/worker-messages", "bench/typed-array-scan-workers", "bench/log-summary", diff --git a/tests/harness/library-mode.test.ts b/tests/harness/library-mode.test.ts index db4fabb3e..b60c3ac54 100644 --- a/tests/harness/library-mode.test.ts +++ b/tests/harness/library-mode.test.ts @@ -363,6 +363,17 @@ globals: undefined fenced expect(run.stdout).toBe(SESSION + SESSION + SESSION); }); + test("K4: untyped property caches start empty in every session", async () => { + // The shapes a session's cache entries name are freed when it ends; + // the next session's sites must not find those entries. + const { archive, outDir } = await buildLibrary("js-ic-sessions", emission); + const probe = buildProbe("js-ic-sessions", archive, outDir); + const run = runProbe(probe); + expect(run.signal).toBeNull(); + expect(run.status).toBe(0); + expect(run.stdout).toBe("run: 20000\n".repeat(3)); + }); + /* ── K5/K6/K7: the trap channel ──────────────────────────────────────── */ test("K5: a trap delivers to the sink exactly once, host survives", async () => { @@ -582,6 +593,15 @@ survived, sink_calls=1 expect(run.stdout).toBe(SESSION + SESSION + SESSION); }); + platformTest("K10: property caches across sessions under ASan", async () => { + const { archive, outDir } = await buildLibrary("js-ic-sessions", emission, { sanitize: true }); + const probe = buildProbe("js-ic-sessions", archive, outDir, { sanitize: true }); + const run = runProbe(probe); + expect(run.signal).toBeNull(); + expect(run.status).toBe(0); + expect(run.stdout).toBe("run: 20000\n".repeat(3)); + }); + platformTest("K10: K5/K7 under ASan", async () => { const { archive, outDir } = await buildLibrary("traps", emission, { sanitize: true }); const probe = buildProbe("traps", archive, outDir, { sanitize: true }); diff --git a/tests/library-mode/js-ic-sessions/lib.ts b/tests/library-mode/js-ic-sessions/lib.ts new file mode 100644 index 000000000..ebea59a54 --- /dev/null +++ b/tests/library-mode/js-ic-sessions/lib.ts @@ -0,0 +1,6 @@ +// The library's typed export over untyped code (work.mjs). +import { runAll } from "./work.mjs"; + +export function run(): number { + return runAll() as number; +} diff --git a/tests/library-mode/js-ic-sessions/probe.c b/tests/library-mode/js-ic-sessions/probe.c new file mode 100644 index 000000000..7dbcb56bf --- /dev/null +++ b/tests/library-mode/js-ic-sessions/probe.c @@ -0,0 +1,25 @@ +/* Three sessions of a library whose untyped code reads and writes through + * property caches: each prints the same result, and under the sanitize + * flavor no session reads a cache entry the previous session freed. */ +#include +#include + +extern void kj_init(void); +extern void kj_set_panic_sink(void (*fn)(void *, const uint8_t *, size_t, uint64_t), void *ctx); +extern void kj_collect(void); +extern double kj_run(void); + +static void sink(void *ctx, const uint8_t *msg, size_t len, uint64_t addr) { + (void)ctx; (void)addr; + printf("UNEXPECTED SINK: %.*s", (int)len, (const char *)msg); +} + +int main(void) { + kj_set_panic_sink(sink, NULL); + for (int session = 0; session < 3; session++) { + kj_init(); + printf("run: %.0f\n", kj_run()); + kj_collect(); + } + return 0; +} diff --git a/tests/library-mode/js-ic-sessions/profile.json b/tests/library-mode/js-ic-sessions/profile.json new file mode 100644 index 000000000..fd802b42c --- /dev/null +++ b/tests/library-mode/js-ic-sessions/profile.json @@ -0,0 +1,14 @@ +{ + "profile_format": 1, + "name": "conformance-js-ic-sessions", + "entry": "lib.ts", + "emission": "llvm", + "abi": { + "prefix": "kj_", + "init_symbol": "kj_init", + "sink_register_symbol": "kj_set_panic_sink", + "collect_symbol": "kj_collect", + "result_reset_symbol": null + }, + "exports": [{ "export": "run", "symbol": "kj_run", "params": [], "returns": "f64" }] +} diff --git a/tests/library-mode/js-ic-sessions/work.mjs b/tests/library-mode/js-ic-sessions/work.mjs new file mode 100644 index 000000000..5c1afcd18 --- /dev/null +++ b/tests/library-mode/js-ic-sessions/work.mjs @@ -0,0 +1,26 @@ +// Dynamic-tier property caches across library sessions: objects built empty +// and filled through untyped stores, read back through untyped loads, and a +// class instance at the same kind of sites. Each session must start with +// empty caches, since the shapes their entries named are freed when the +// previous session ends. +class Box { + constructor(v) { + this.v = v; + } +} +function fill(o, n) { + o.a = n; + o.b = n + 1; + return o; +} +function sum(o) { + return o.a + o.b; +} +function unbox(b) { + return b.v; +} +export function runAll() { + let total = 0; + for (let i = 0; i < 100; i++) total += sum(fill({}, i)); + return total + unbox(new Box(total)); +} From d8816b47316dbfb0baefe5a0282f35c1a3823941 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 12:05:34 -0500 Subject: [PATCH 069/184] Evaluate an element assignment's receiver and index before its value An element assignment whose value is an optional element read or call, such as `out[n++] = src[n]!`, held the value in a temporary and then evaluated the receiver and index inside the store, after the value. So `n++` ran after `src[n]` was read, and `src[0]` was stored where Node stores `src[1]`. The receiver and index are now evaluated first whenever the value could change what they read: when it assigns a binding they read, or runs code that can assign their `let` and `var` bindings or fields. A literal, or a binding the value cannot change, is still read at the store. --- .../src/frontend/lowering/array-values.ts | 95 ++++++++++++++++++- .../7642-element-assignment-order.ts.json | 6 ++ tests/corpus/7642-element-assignment-order.ts | 37 ++++++++ .../7642-element-assignment-order.ts.json | 1 + 4 files changed, 137 insertions(+), 2 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7642-element-assignment-order.ts.json create mode 100644 tests/corpus/7642-element-assignment-order.ts create mode 100644 tests/harness/corpus-costs/7642-element-assignment-order.ts.json diff --git a/packages/compiler/src/frontend/lowering/array-values.ts b/packages/compiler/src/frontend/lowering/array-values.ts index 314cdfd24..9f23a860b 100644 --- a/packages/compiler/src/frontend/lowering/array-values.ts +++ b/packages/compiler/src/frontend/lowering/array-values.ts @@ -15,6 +15,7 @@ import { typeKey, unionContainerArmsOk, } from "../../ir/ir.js"; +import { everyExpr } from "../../ir/traverse.js"; import type { Lowerer } from "./lowerer.js"; import type * as ts from "../ts7/adapter.js"; @@ -193,6 +194,81 @@ export function unionArrayValueRead( return result; } +/** Expression kinds that evaluate without running other code or writing + * a binding. */ +const PURE_KINDS = new Set([ + "varRef", + "numLit", + "strLit", + "boolLit", + "unitLit", + "bin", + "unary", + "logical", + "ternary", + "toBool", + "fieldGet", + "recordGet", + "arrayGet", + "arrayState", + "arrayHas", + "unionIsTag", + "unionNarrow", + "unionWrap", +]); + +/** The locals evaluating `value` may assign, and whether it may run other + * code (anything but the pure kinds and the read-only element helpers), + * which can assign locals and fields through closures. */ +function valueEffects(lowerer: Lowerer, value: IrExpr): { writes: Set; opaque: boolean } { + const readers = new Set(); + for (const [key, name] of lowerer.arrHofHelpers) if (key.startsWith("idxOr:")) readers.add(name); + const writes = new Set(); + let opaque = false; + everyExpr(value, { + expr: (e) => { + if (e.kind === "assignExpr" || e.kind === "incDec") writes.add(e.localId); + else if (e.kind === "call" ? !readers.has(e.callee) : !PURE_KINDS.has(e.kind)) opaque = true; + return true; + }, + stmt: (s) => { + if (s.kind === "varDecl" || s.kind === "assign") writes.add(s.localId); + else opaque = true; + return true; + }, + }); + return { writes, opaque }; +} + +/** Whether evaluating `e` after `value` gives what it gives before it: a + * literal, or a local and field read `value` cannot change. */ +function unaffectedBy(e: IrExpr, effects: { writes: Set; opaque: boolean }): boolean { + switch (e.kind) { + case "numLit": + case "strLit": + return true; + case "varRef": + return !effects.writes.has(e.localId); + case "bin": + return unaffectedBy(e.left, effects) && unaffectedBy(e.right, effects); + case "fieldGet": + case "recordGet": + return !effects.opaque && unaffectedBy(e.obj, effects); + default: + return false; + } +} + +/** Whether `e` reads a `let` or `var` binding, which code the value runs + * can assign through a closure. */ +function readsMutableLocal(lowerer: Lowerer, e: IrExpr): boolean { + return !everyExpr(e, { + expr: (x) => + x.kind !== "varRef" || lowerer.ctx.locals.find((l) => l.id === x.localId)?.mutable === false, + stmt: () => true, + }); +} + /** Store a stabilized value without changing the array's element ABI. */ export function arrayValueStore( lowerer: Lowerer, @@ -210,10 +286,25 @@ export function arrayValueStore( // owned union temporary to release on every path. const unionId = value.type.unionId; let stable = value; - let prefix: IrStmt[] = []; + const prefix: IrStmt[] = []; + // The stores below evaluate the receiver and the index after the value. + // JavaScript evaluates them first (`out[n++] = src[n]` reads src[1]), so + // either may need its value held before the value runs. + if (value.kind !== "varRef") { + const effects = valueEffects(lowerer, value); + const hold = (e: IrExpr, name: string): IrExpr => { + if (unaffectedBy(e, effects) && !(effects.opaque && readsMutableLocal(lowerer, e))) + return e; + const temp = lowerer.declareHiddenLocal(name, e.type); + prefix.push({ kind: "varDecl", localId: temp.id, init: e, loc }); + return varRef(temp.id, temp.type, loc); + }; + arr = hold(arr, "%arrayTarget"); + index = hold(index, "%arrayIndex"); + } if (value.kind !== "varRef") { const temp = lowerer.declareHiddenLocal("%arrayValue", value.type); - prefix = [{ kind: "varDecl", localId: temp.id, init: value, loc }]; + prefix.push({ kind: "varDecl", localId: temp.id, init: value, loc }); stable = varRef(temp.id, temp.type, loc); } const write: IrStmt = { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7642-element-assignment-order.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7642-element-assignment-order.ts.json new file mode 100644 index 000000000..17369f496 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7642-element-assignment-order.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7642-element-assignment-order.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7642-element-assignment-order.ts b/tests/corpus/7642-element-assignment-order.ts new file mode 100644 index 000000000..f2f2fae7b --- /dev/null +++ b/tests/corpus/7642-element-assignment-order.ts @@ -0,0 +1,37 @@ +// An element assignment evaluates its receiver and index before its value, +// also when the value is an optional element read or a call that assigns +// the bindings the receiver and index read. +const a = [10, 20, 30, 40]; +const b = [1, 2, 3, 4]; +let n = 0; +a[n++] = b[n]!; +console.log(a.join(), n); +a[n] = b[n++]; +console.log(a.join(), n); + +const optional: (number | undefined)[] = [undefined, 5, undefined]; +let k = 0; +optional[k++] = optional[k]; +console.log(optional, k); + +// A call in the value assigns the receiver and the index through closures. +let target: (number | undefined)[] = [0, 0, 0]; +const first = target; +let at = 0; +function swap(): number | undefined { + target = [9, 9, 9]; + at = 2; + return b[3]; +} +target[at] = swap(); +console.log(first, target, at); + +// The index and receiver are not evaluated twice. +let reads = 0; +function index(): number { + reads++; + return 1; +} +const c: (number | undefined)[] = [0, 0]; +c[index()] = optional[1]; +console.log(c, reads); diff --git a/tests/harness/corpus-costs/7642-element-assignment-order.ts.json b/tests/harness/corpus-costs/7642-element-assignment-order.ts.json new file mode 100644 index 000000000..9f67659a1 --- /dev/null +++ b/tests/harness/corpus-costs/7642-element-assignment-order.ts.json @@ -0,0 +1 @@ +{ "plain": 1500, "san": 2500 } From 849fad0ab6051f3914ba269ad068d07a80011213 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 12:08:08 -0500 Subject: [PATCH 070/184] Prepare v0.4.3 release --- CHANGELOG.md | 24 +++++++++++++++++-- packages/cli-darwin-arm64/package.json | 2 +- packages/cli-darwin-x64/package.json | 2 +- packages/cli-linux-arm64-gnu/package.json | 2 +- packages/cli-linux-arm64-musl/package.json | 2 +- packages/cli-linux-x64-gnu/package.json | 2 +- packages/cli-linux-x64-musl/package.json | 2 +- packages/cli-win32-x64-msvc/package.json | 2 +- packages/cli/package.json | 2 +- packages/compiler/package.json | 2 +- packages/compiler/surface-manifest.json | 2 +- packages/experimental/package.json | 2 +- packages/llvm-darwin-arm64/package.json | 2 +- packages/llvm-darwin-x64/package.json | 2 +- packages/llvm-linux-arm64-gnu/package.json | 2 +- packages/llvm-linux-arm64-musl/package.json | 2 +- packages/llvm-linux-x64-gnu/package.json | 2 +- packages/llvm-linux-x64-musl/package.json | 2 +- packages/llvm-win32-x64-msvc/package.json | 2 +- packages/runtime-android-arm64/package.json | 2 +- packages/runtime-darwin-arm64/package.json | 2 +- packages/runtime-darwin-x64/package.json | 2 +- packages/runtime-ios-arm64/package.json | 2 +- .../runtime-ios-simulator-arm64/package.json | 2 +- packages/runtime-linux-arm64-gnu/package.json | 2 +- .../runtime-linux-arm64-musl/package.json | 2 +- packages/runtime-linux-x64-gnu/package.json | 2 +- packages/runtime-linux-x64-musl/package.json | 2 +- packages/runtime-wasm32-wasi/package.json | 2 +- packages/runtime-win32-x64-msvc/package.json | 2 +- packages/runtime/package.json | 2 +- packages/threads/package.json | 2 +- packages/types/package.json | 2 +- 33 files changed, 54 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec0b27dbf..96f13fa26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,28 @@ All notable changes to scriptc will be documented in this file. +## 0.4.3 + +### Highlights + +- **Faster checking and smaller builds with tsc-ts 0.0.9:** scriptc checks with tsc-ts 0.0.9, and builds reuse the bundled declaration files' diagnostics instead of checking them again. Small IR builds are 10.6 to 16.0% faster and small executable builds 5.5 to 7.6% faster, with identical output. A native build of tsc-ts with this release checks 0.4 to 6.6% faster than with 0.4.2 and peaks 4 to 16% lower. +- **Integer arrays and loops:** + - Number arrays that only hold int32 values use 4-byte slots in locals, across direct calls and in module globals, and loop-carried locals written from their elements stay int32. A workload that walks arrays of cursors runs 54.5% faster. + - Loops with two counters closing in on each other (`for (let i = lo, j = hi; i < j; i++, j--)`) and element copies through `a[i]!` run without per-element checks, and two-counter `while` loops run on integer counters. + - Loops over typed arrays and charAt scans are versioned on their bounds, and ASCII strings are scanned as bytes. +- **Less work for untyped and dynamic code:** + - String-keyed record reads under `??` and `!` no longer allocate: a dictionary-walking workload makes no union boxes instead of 363,359 and runs 12 to 15% faster. + - Checked-dynamic method calls dispatch on method ids, observed class methods call natively while their slot is unchanged, and calls on bag-tracked class instances skip building their views. +- **Node.js parity:** + - `crypto.getCipherInfo`, AES-CCM, AES key wrap, the legacy `createCipheriv` modes, `checkPrime`/`generatePrime`, and Node's argument checks for `generateKeyPair`. + - Streams: `construct`, `writev`, `decodeStrings` and `defaultEncoding`, including `writev` on `Transform` and `PassThrough`. + - `FinalizationRegistry`, `new.target`, `String.prototype.at` on untyped values, `Error.stackTraceLimit`, `spawnSync`'s `env` option, and class constructors called through `call`/`apply` throw Node's TypeError. + - Records enumerate in the key order their object literal wrote, and objects converting to a union with an empty-record arm keep every field. +- **npm packages:** zod 3 compiles natively and zod 4's core runs, minified packages such as marked compile, chalk 5 and hot-shots 8 build from built-in shims, and `@hono/node-server` 2.x is built in. User code types npm package declarations at a checked, identity-preserving boundary. +- **Builds:** the cached build is reused after whitespace and comment edits, function bodies are emitted on forked workers, and the LLVM emitter reads its analyses from one recorded walk of each module. + + + ## 0.4.2 ### Highlights @@ -159,8 +181,6 @@ All notable changes to scriptc will be documented in this file. - **The same program compiles to the same executable wherever it sits on disk** and whatever order its functions lower in, and edits outside a declaration do not rename the helpers and closures inside it. - **Libraries that use untyped byte arrays** link. - - ## 0.4.1 ### Highlights diff --git a/packages/cli-darwin-arm64/package.json b/packages/cli-darwin-arm64/package.json index 23584b60a..eee0ec3c8 100644 --- a/packages/cli-darwin-arm64/package.json +++ b/packages/cli-darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/cli-darwin-arm64", - "version": "0.4.2", + "version": "0.4.3", "description": "Native scriptc compiler for darwin-arm64", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/cli-darwin-x64/package.json b/packages/cli-darwin-x64/package.json index d9e6dafc6..b527a3cd7 100644 --- a/packages/cli-darwin-x64/package.json +++ b/packages/cli-darwin-x64/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/cli-darwin-x64", - "version": "0.4.2", + "version": "0.4.3", "description": "Native scriptc compiler for darwin-x64", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/cli-linux-arm64-gnu/package.json b/packages/cli-linux-arm64-gnu/package.json index 5b8cb4db2..512a7cd13 100644 --- a/packages/cli-linux-arm64-gnu/package.json +++ b/packages/cli-linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/cli-linux-arm64-gnu", - "version": "0.4.2", + "version": "0.4.3", "description": "Native scriptc compiler for linux-arm64-gnu", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/cli-linux-arm64-musl/package.json b/packages/cli-linux-arm64-musl/package.json index 0901602ec..8d0fdfb0f 100644 --- a/packages/cli-linux-arm64-musl/package.json +++ b/packages/cli-linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/cli-linux-arm64-musl", - "version": "0.4.2", + "version": "0.4.3", "description": "Native scriptc compiler for linux-arm64-musl", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/cli-linux-x64-gnu/package.json b/packages/cli-linux-x64-gnu/package.json index f01478aad..b71205614 100644 --- a/packages/cli-linux-x64-gnu/package.json +++ b/packages/cli-linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/cli-linux-x64-gnu", - "version": "0.4.2", + "version": "0.4.3", "description": "Native scriptc compiler for linux-x64-gnu", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/cli-linux-x64-musl/package.json b/packages/cli-linux-x64-musl/package.json index a6986a66a..eaeb6f15a 100644 --- a/packages/cli-linux-x64-musl/package.json +++ b/packages/cli-linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/cli-linux-x64-musl", - "version": "0.4.2", + "version": "0.4.3", "description": "Native scriptc compiler for linux-x64-musl", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/cli-win32-x64-msvc/package.json b/packages/cli-win32-x64-msvc/package.json index 6b03811b8..3e3b8259f 100644 --- a/packages/cli-win32-x64-msvc/package.json +++ b/packages/cli-win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/cli-win32-x64-msvc", - "version": "0.4.2", + "version": "0.4.3", "description": "Native scriptc compiler for win32-x64-msvc", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/cli/package.json b/packages/cli/package.json index 034747c29..264a7f5f1 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "scriptc", - "version": "0.4.2", + "version": "0.4.3", "description": "Compile ordinary TypeScript and JavaScript to small, fast native executables and WebAssembly modules that run without Node.js", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/compiler/package.json b/packages/compiler/package.json index a9e3d9616..60f6bd0f0 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/compiler", - "version": "0.4.2", + "version": "0.4.3", "description": "The scriptc compiler — TypeScript/JavaScript frontend, typed IR, LLVM backend", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/compiler/surface-manifest.json b/packages/compiler/surface-manifest.json index e44f5d31e..3387217b5 100644 --- a/packages/compiler/surface-manifest.json +++ b/packages/compiler/surface-manifest.json @@ -1,6 +1,6 @@ { "schemaVersion": 2, - "compilerVersion": "0.4.2", + "compilerVersion": "0.4.3", "coverage": [ "Entries are projected mechanically from the compiler's own decision tables: the diagnostics registry, the unsupported-syntax dispatch tables, the stdlib and node-builtin lowering tables, and the supported-builtin-module list. Nothing is hand-maintained; the manifest regenerates byte-identically from the source tree at this version.", "Absence from this manifest means 'not projected', never 'unsupported'. Surfaces lowered through dedicated code paths rather than tables are not yet projected: console, JSON, Promise/async and the timer surface, the net/http/tls/https/dgram/dns/assert/test/stream/readline module member surfaces, template literals, the regex slice, global functions (parseInt, parseFloat, isNaN, isFinite), and the process surface outside its ambient slice.", diff --git a/packages/experimental/package.json b/packages/experimental/package.json index cfb4151a4..b3e8a6fb1 100644 --- a/packages/experimental/package.json +++ b/packages/experimental/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/experimental", - "version": "0.4.2", + "version": "0.4.3", "description": "Experimental scriptc APIs, with Node.js implementations: lifetime regions for ProgramLifetime objects, intern tables shared between threads, and per-thread slots on shared objects", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/llvm-darwin-arm64/package.json b/packages/llvm-darwin-arm64/package.json index 615fe9d81..ac99f1661 100644 --- a/packages/llvm-darwin-arm64/package.json +++ b/packages/llvm-darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/llvm-darwin-arm64", - "version": "0.4.2", + "version": "0.4.3", "description": "Pinned LLVM code-generation helper for scriptc on macOS arm64", "license": "Apache-2.0 WITH LLVM-exception", "homepage": "https://scriptc.dev", diff --git a/packages/llvm-darwin-x64/package.json b/packages/llvm-darwin-x64/package.json index fd6caf976..ca632dcd4 100644 --- a/packages/llvm-darwin-x64/package.json +++ b/packages/llvm-darwin-x64/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/llvm-darwin-x64", - "version": "0.4.2", + "version": "0.4.3", "description": "Pinned LLVM code-generation helper for scriptc on macOS x64", "license": "Apache-2.0 WITH LLVM-exception", "homepage": "https://scriptc.dev", diff --git a/packages/llvm-linux-arm64-gnu/package.json b/packages/llvm-linux-arm64-gnu/package.json index 7d66257f4..566b38e40 100644 --- a/packages/llvm-linux-arm64-gnu/package.json +++ b/packages/llvm-linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/llvm-linux-arm64-gnu", - "version": "0.4.2", + "version": "0.4.3", "description": "Pinned LLVM code-generation helper for scriptc on Linux arm64", "license": "Apache-2.0 WITH LLVM-exception", "homepage": "https://scriptc.dev", diff --git a/packages/llvm-linux-arm64-musl/package.json b/packages/llvm-linux-arm64-musl/package.json index b3a01089d..a9288c2ca 100644 --- a/packages/llvm-linux-arm64-musl/package.json +++ b/packages/llvm-linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/llvm-linux-arm64-musl", - "version": "0.4.2", + "version": "0.4.3", "description": "Statically linked LLVM code-generation helper for scriptc on Linux arm64 musl", "license": "Apache-2.0 WITH LLVM-exception", "homepage": "https://scriptc.dev", diff --git a/packages/llvm-linux-x64-gnu/package.json b/packages/llvm-linux-x64-gnu/package.json index 0b3264ec7..c55f51847 100644 --- a/packages/llvm-linux-x64-gnu/package.json +++ b/packages/llvm-linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/llvm-linux-x64-gnu", - "version": "0.4.2", + "version": "0.4.3", "description": "Pinned LLVM code-generation helper for scriptc on Linux x64", "license": "Apache-2.0 WITH LLVM-exception", "homepage": "https://scriptc.dev", diff --git a/packages/llvm-linux-x64-musl/package.json b/packages/llvm-linux-x64-musl/package.json index 6c83d1cb9..b257085bf 100644 --- a/packages/llvm-linux-x64-musl/package.json +++ b/packages/llvm-linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/llvm-linux-x64-musl", - "version": "0.4.2", + "version": "0.4.3", "description": "Statically linked LLVM code-generation helper for scriptc on Linux x64 musl", "license": "Apache-2.0 WITH LLVM-exception", "homepage": "https://scriptc.dev", diff --git a/packages/llvm-win32-x64-msvc/package.json b/packages/llvm-win32-x64-msvc/package.json index 9fb682f0f..0f3f99331 100644 --- a/packages/llvm-win32-x64-msvc/package.json +++ b/packages/llvm-win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/llvm-win32-x64-msvc", - "version": "0.4.2", + "version": "0.4.3", "description": "Pinned LLVM code-generation helper for scriptc on Windows x64", "license": "Apache-2.0 WITH LLVM-exception", "homepage": "https://scriptc.dev", diff --git a/packages/runtime-android-arm64/package.json b/packages/runtime-android-arm64/package.json index 362fbc572..392ef1717 100644 --- a/packages/runtime-android-arm64/package.json +++ b/packages/runtime-android-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime-android-arm64", - "version": "0.4.2", + "version": "0.4.3", "description": "Precompiled scriptc library runtime pack for Android arm64", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/runtime-darwin-arm64/package.json b/packages/runtime-darwin-arm64/package.json index a4176ed60..354526024 100644 --- a/packages/runtime-darwin-arm64/package.json +++ b/packages/runtime-darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime-darwin-arm64", - "version": "0.4.2", + "version": "0.4.3", "description": "Precompiled scriptc runtime pack for macOS arm64", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/runtime-darwin-x64/package.json b/packages/runtime-darwin-x64/package.json index 91bc4182b..58aa358b2 100644 --- a/packages/runtime-darwin-x64/package.json +++ b/packages/runtime-darwin-x64/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime-darwin-x64", - "version": "0.4.2", + "version": "0.4.3", "description": "Precompiled scriptc runtime pack for macOS x64", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/runtime-ios-arm64/package.json b/packages/runtime-ios-arm64/package.json index 2fdd68897..6193c5258 100644 --- a/packages/runtime-ios-arm64/package.json +++ b/packages/runtime-ios-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime-ios-arm64", - "version": "0.4.2", + "version": "0.4.3", "description": "Precompiled scriptc library runtime pack for iOS arm64", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/runtime-ios-simulator-arm64/package.json b/packages/runtime-ios-simulator-arm64/package.json index cbdf96d1d..7d01c45d3 100644 --- a/packages/runtime-ios-simulator-arm64/package.json +++ b/packages/runtime-ios-simulator-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime-ios-simulator-arm64", - "version": "0.4.2", + "version": "0.4.3", "description": "Precompiled scriptc library runtime pack for iOS simulator arm64", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/runtime-linux-arm64-gnu/package.json b/packages/runtime-linux-arm64-gnu/package.json index e0d228925..419080c3a 100644 --- a/packages/runtime-linux-arm64-gnu/package.json +++ b/packages/runtime-linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime-linux-arm64-gnu", - "version": "0.4.2", + "version": "0.4.3", "description": "Precompiled scriptc runtime pack for Linux arm64 glibc", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/runtime-linux-arm64-musl/package.json b/packages/runtime-linux-arm64-musl/package.json index 70eee223f..2bae09884 100644 --- a/packages/runtime-linux-arm64-musl/package.json +++ b/packages/runtime-linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime-linux-arm64-musl", - "version": "0.4.2", + "version": "0.4.3", "description": "Precompiled scriptc runtime pack for Linux arm64 musl", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/runtime-linux-x64-gnu/package.json b/packages/runtime-linux-x64-gnu/package.json index 5822fd21d..6749ddeea 100644 --- a/packages/runtime-linux-x64-gnu/package.json +++ b/packages/runtime-linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime-linux-x64-gnu", - "version": "0.4.2", + "version": "0.4.3", "description": "Precompiled scriptc runtime pack for Linux x64 glibc", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/runtime-linux-x64-musl/package.json b/packages/runtime-linux-x64-musl/package.json index 1db34d4a7..9b4803406 100644 --- a/packages/runtime-linux-x64-musl/package.json +++ b/packages/runtime-linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime-linux-x64-musl", - "version": "0.4.2", + "version": "0.4.3", "description": "Precompiled scriptc runtime pack for Linux x64 musl", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/runtime-wasm32-wasi/package.json b/packages/runtime-wasm32-wasi/package.json index 5bbd23755..63a95601d 100644 --- a/packages/runtime-wasm32-wasi/package.json +++ b/packages/runtime-wasm32-wasi/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime-wasm32-wasi", - "version": "0.4.2", + "version": "0.4.3", "description": "Precompiled scriptc runtime pack for WASI Preview 1", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/runtime-win32-x64-msvc/package.json b/packages/runtime-win32-x64-msvc/package.json index df4a9a109..01bec7c31 100644 --- a/packages/runtime-win32-x64-msvc/package.json +++ b/packages/runtime-win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime-win32-x64-msvc", - "version": "0.4.2", + "version": "0.4.3", "description": "Precompiled scriptc runtime pack for Windows x64 MSVC", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/runtime/package.json b/packages/runtime/package.json index 9232266af..d367ad6bf 100644 --- a/packages/runtime/package.json +++ b/packages/runtime/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/runtime", - "version": "0.4.2", + "version": "0.4.3", "description": "scriptc native runtime sources and vendored dependencies", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/threads/package.json b/packages/threads/package.json index 11b8ef624..a37a3ba82 100644 --- a/packages/threads/package.json +++ b/packages/threads/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/threads", - "version": "0.4.2", + "version": "0.4.3", "description": "Zero-copy publication of immutable object graphs to worker threads (scriptc), with a deep-freezing Node.js implementation", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/types/package.json b/packages/types/package.json index 689c7b72f..906c68dec 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/types", - "version": "0.4.2", + "version": "0.4.3", "description": "Contract types for scriptc: type-only annotations (Int32, NoThrow, Immutable) that scriptc checks and optimizes, with no runtime semantics", "license": "Apache-2.0", "homepage": "https://scriptc.dev", From a36ef0b4ca50392320b92a6d4bb90e04566e359a Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 12:11:48 -0500 Subject: [PATCH 071/184] Settle aborts and pull TransformStream uploads as Node does --- packages/runtime/src/scr_fetch.c | 73 ++++++++++++++----- .../corpus/7743-transform-stream-identity.ts | 33 ++++++++- tests/fixtures/fetch/static-stream/main.mts | 16 ++++ 3 files changed, 102 insertions(+), 20 deletions(-) diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index e92f488b2..9792f8bed 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -1622,6 +1622,14 @@ static void sf_stream_schedule_pull_complete(SfStream *s) { scr_queue_microtask(cb); } +/* A consumer waiting for the stream's next chunk: a pending read, a body + * collector, or a fetch upload (fetch keeps reading the request body, + * including after an aborted transfer stops writing it). */ +static bool sf_stream_has_consumer(const SfStream *s) { + return s->collector || (s->reader && s->reader->pending_head) || s->request_owner || + s->request_discarding; +} + static void sf_stream_pull(SfStream *s) { if (!s->started || (!s->pull_cb && !sf_stream_has_from_source(s)) || @@ -1630,11 +1638,9 @@ static void sf_stream_pull(SfStream *s) { return; } /* ReadableStreamDefaultControllerShouldCallPull: with no room in the - * queue, only a pending read (or a body collector) asks for a chunk. */ - if (s->hwm_zero && !s->collector && - !(s->reader && s->reader->pending_head)) { - return; - } + * queue, only a consumer asks for a chunk: a pending read, a body + * collector, or a fetch upload, whose reader is always waiting. */ + if (s->hwm_zero && !sf_stream_has_consumer(s)) return; if (s->pulling) { s->pull_again = true; return; @@ -2730,6 +2736,12 @@ struct SfTransform { ScrPromise *in_flight_close; bool has_pending_abort; ScrPromise *pending_abort; + /* The sink abort found a finish promise from a cancel or close, which + * settles the abort instead (Node returns that promise). */ + bool abort_joins_finish; + /* The abort request's outcome between SFT_ABORT_DONE and + * SFT_ABORT_SETTLE: a rejection reason (+1), or NULL to fulfill. */ + ScrDyn *abort_settle_error; ScrDyn *pending_abort_reason; bool pending_abort_was_erroring; bool backpressure; @@ -2779,6 +2791,7 @@ static void sf_transform_release(SfTransform *t) { scr_promise_release(t->in_flight_close); scr_promise_release(t->pending_abort); scr_dyn_release(t->pending_abort_reason); + scr_dyn_release(t->abort_settle_error); scr_promise_release(t->bp_change); scr_promise_release(t->finish); scr_dyn_release(t->write_error); @@ -2849,6 +2862,7 @@ enum { SFT_ABORT_DONE, SFT_START, SFT_CANCEL_DONE, + SFT_ABORT_SETTLE, }; static void sf_transform_step(SfTransform *t, int op); @@ -3214,7 +3228,7 @@ static void sft_set_backpressure(SfTransform *t, bool backpressure) { } static bool sft_readable_wants_chunk(const SfStream *r) { - return r->collector || (r->reader && r->reader->pending_head); + return sf_stream_has_consumer(r); } static bool sft_readable_can_enqueue(const SfStream *r) { @@ -3278,6 +3292,7 @@ static void sft_sink_close(SfTransform *t) { static void sft_abort_algorithm(SfTransform *t, ScrDyn *reason) { (void)reason; + t->abort_joins_finish = t->finish != NULL; if (!t->finish) { t->finish = scr_promise_new(); scr_promise_mark_handled(t->finish); @@ -3327,25 +3342,49 @@ static void sf_transform_step(SfTransform *t, int op) { } case SFT_ABORT_DONE: { SfStream *r = t->readable; - ScrDyn *reason = t->pending_abort_reason - ? scr_dyn_retain(t->pending_abort_reason) - : scr_dyn_retain(scr_dyn_undefined()); - bool readable_errored = r->error != NULL; - if (!readable_errored) sf_stream_error(r, reason); + scr_dyn_release(t->abort_settle_error); + t->abort_settle_error = NULL; + if (t->abort_joins_finish && !scr_promise_is_pending(t->finish)) { + /* The cancel or close that created the finish promise settled + * it; the abort request takes its result. */ + if (scr_promise_is_rejected(t->finish)) + t->abort_settle_error = scr_promise_reason_dyn(t->finish); + } else { + ScrDyn *reason = t->pending_abort_reason + ? scr_dyn_retain(t->pending_abort_reason) + : scr_dyn_retain(scr_dyn_undefined()); + bool readable_errored = r->error != NULL; + if (!readable_errored) sf_stream_error(r, reason); + /* The sink abort algorithm's finish promise, which a + * readable.cancel() after the abort answers too: rejected with the + * readable side's stored error when it had already errored, else + * resolved. */ + if (t->finish && scr_promise_is_pending(t->finish)) { + if (readable_errored) sf_reject_promise_reason(t->finish, r->error); + else scr_promise_fulfill_void(t->finish); + } + if (readable_errored) t->abort_settle_error = scr_dyn_retain(r->error); + scr_dyn_release(reason); + } + /* WritableStreamFinishErroring settles the abort request in a + * reaction to the sink abort's promise, two turns after it. */ + sf_transform_defer(t, SFT_ABORT_SETTLE, 2); + return; + } + case SFT_ABORT_SETTLE: { + ScrDyn *error = t->abort_settle_error; + t->abort_settle_error = NULL; if (t->has_pending_abort) { t->has_pending_abort = false; - if (readable_errored) { - sf_reject_promise_reason(t->pending_abort, r->error); - } else { - scr_promise_fulfill_void(t->pending_abort); - } + if (error) sf_reject_promise_reason(t->pending_abort, error); + else scr_promise_fulfill_void(t->pending_abort); scr_promise_release(t->pending_abort); t->pending_abort = NULL; scr_dyn_release(t->pending_abort_reason); t->pending_abort_reason = NULL; } sfw_reject_close_and_closed(t); - scr_dyn_release(reason); + scr_dyn_release(error); return; } case SFT_CANCEL_DONE: { diff --git a/tests/corpus/7743-transform-stream-identity.ts b/tests/corpus/7743-transform-stream-identity.ts index d98303997..3664c8ff1 100644 --- a/tests/corpus/7743-transform-stream-identity.ts +++ b/tests/corpus/7743-transform-stream-identity.ts @@ -1,8 +1,8 @@ // The identity TransformStream: writes wait for reads (the readable side's // high-water mark is 0), the writer's desiredSize/ready track the writable -// side's queue, close/abort/cancel propagate across the pair, and the -// readable side feeds a Response body (the StreamingApi pattern of Hono's -// stream helpers). +// side's queue, close/abort/cancel propagate across the pair (in either +// order when they race), and the readable side feeds a Response body (the +// StreamingApi pattern of Hono's stream helpers). const enc = new TextEncoder(); const dec = new TextDecoder(); @@ -200,6 +200,32 @@ async function writableMethods(): Promise { await closing; } +// writable.abort() and readable.cancel() share the transform's finish +// promise: whichever runs second answers the first one's result. +function settled(label: string, p: Promise): Promise { + return p.then( + () => console.log(label, "fulfilled"), + (e: unknown) => console.log(label, "rejected", describe(e)), + ); +} + +async function abortCancelRaces(): Promise { + const a = new TransformStream(); + const aWriter = a.writable.getWriter(); + await new Promise((r) => setTimeout(r, 0)); + const aborted = settled("abort then cancel: abort", aWriter.abort("abort first")); + const cancelled = settled("abort then cancel: cancel", a.readable.cancel("cancel second")); + await aborted; + await cancelled; + const b = new TransformStream(); + const bWriter = b.writable.getWriter(); + await new Promise((r) => setTimeout(r, 0)); + const cancelledFirst = settled("cancel then abort: cancel", b.readable.cancel("cancel first")); + const abortedSecond = settled("cancel then abort: abort", bWriter.abort("abort second")); + await cancelledFirst; + await abortedSecond; +} + async function main(): Promise { await basics(); await abortWriter(); @@ -207,6 +233,7 @@ async function main(): Promise { await intoResponse(); await streamedResponse(); await writableMethods(); + await abortCancelRaces(); } await main(); diff --git a/tests/fixtures/fetch/static-stream/main.mts b/tests/fixtures/fetch/static-stream/main.mts index 9ff8c050b..0ae400bc3 100644 --- a/tests/fixtures/fetch/static-stream/main.mts +++ b/tests/fixtures/fetch/static-stream/main.mts @@ -259,6 +259,22 @@ try { console.log("consumed request reader:", (error as Error).name); } +// A TransformStream's readable side as the request body: fetch's upload +// reads it, so writes to the writable side do not wait on backpressure. +const uploadTransform = new TransformStream(); +const uploadWriter = uploadTransform.writable.getWriter(); +const transformPosted = fetch(`${process.argv[2]}/post-echo`, { + method: "POST", + body: uploadTransform.readable, + duplex: "half", +}); +for (const part of ["trans", "form ", "body"]) { + await uploadWriter.write(new TextEncoder().encode(part)); +} +await uploadWriter.close(); +const transformResult = (await (await transformPosted).json()) as { body: string }; +console.log("transform request body:", transformResult.body); + const prelockedRequestBody = ReadableStream.from([ Buffer.from("prelocked request"), ]); From 7fb73b7d9da248acc4815d1c03646d83316dc861 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 10:05:54 -0700 Subject: [PATCH 072/184] Keep the key order of records spread from values JavaScript built A literal that starts with `...value` creates value's keys first, in value's own order. When checked conversions build value's shape (records JavaScript hands back through replaceable methods, field setters, generator calls or callbacks), the key-order witness could not order the literal statically and kept the declared table, so `{ ...draft, id }` and `{ ...current, ...patch }` enumerated `id` first natively. Every value of a record shape already enumerates either in its table or in the order its record source holds: a checked conversion keeps the source's order there whenever it differs from the table. Such a literal (an ordered spread) now carries that order over: after it builds the record, record.sourceSpread gives the record a source listing the keys it declares in value's order, and the keys value lacks follow in the table's order, which every literal sequence of the shape agrees with. The literal then resolves as if value were in its table's order, and its own shape counts as built at run time, since its values may carry a source too. It qualifies only when every value of the source has all of the source's keys, both shapes can keep a source, every later spread of a shape whose values may carry a source only overwrites the source's keys, and the source shape resolves (or is the literal's own shape). Literals of shapes that do not resolve are left as they were. Programs without such literals compile to identical IR. Corpus 7750-key-order-javascript-spread-sources. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/backend/llvm/dyn.ts | 13 + .../compiler/src/backend/llvm/lib-dispatch.ts | 9 + .../compiler/src/backend/llvm/lib-shared.ts | 1 + .../lowering/key-order-witness.test.ts | 90 +++++++ .../frontend/lowering/key-order-witness.ts | 234 +++++++++++++++--- .../compiler/src/ir/builtin-signatures.ts | 5 + .../main.ts.json | 7 + packages/runtime/src/scr_dyn.h | 6 + packages/runtime/src/scr_json.c | 32 +++ .../lib.js | 14 ++ .../main.ts | 105 ++++++++ .../main.ts.json | 1 + 12 files changed, 489 insertions(+), 28 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7750-key-order-javascript-spread-sources/main.ts.json create mode 100644 tests/corpus/7750-key-order-javascript-spread-sources/lib.js create mode 100644 tests/corpus/7750-key-order-javascript-spread-sources/main.ts create mode 100644 tests/harness/corpus-costs/7750-key-order-javascript-spread-sources/main.ts.json diff --git a/packages/compiler/src/backend/llvm/dyn.ts b/packages/compiler/src/backend/llvm/dyn.ts index 94581cfa0..2234b8db2 100644 --- a/packages/compiler/src/backend/llvm/dyn.ts +++ b/packages/compiler/src/backend/llvm/dyn.ts @@ -1505,6 +1505,19 @@ export class LlDyn { return descriptor; } + /** `record.sourceSpread(built, value)`: the built record keeps value's + * kept-source order over its own declared names. */ + emitRecordSourceSpread(B: BlockBuilder, built: string, value: string, shapeId: string): void { + const host = this.host; + const shape = host.recordsById.get(shapeId)!; + this.keepsRecordSources = true; + const names = this.recordNameTable(shapeId, shape); + host.declare(`declare void @scr_record_source_spread(ptr, ptr, ptr, ptr, ${host.sizeType})`); + B.line( + `call void @scr_record_source_spread(ptr ${built}, ptr ${value}, ptr ${names.names}, ptr ${names.lens}, ${host.sizeType} ${shape.fields.length})`, + ); + } + /** The declared-name tables scr_record_source_note scans: parallel * constant arrays of field-name bytes and byte lengths in declaration * order (the order the record serializes), interned per shape. Internal diff --git a/packages/compiler/src/backend/llvm/lib-dispatch.ts b/packages/compiler/src/backend/llvm/lib-dispatch.ts index cebe661d4..c8fae3192 100644 --- a/packages/compiler/src/backend/llvm/lib-dispatch.ts +++ b/packages/compiler/src/backend/llvm/lib-dispatch.ts @@ -236,6 +236,15 @@ export function emitGenericLibCall(host: LlvmEmitterContext, e: LibCallExpr): Ll host.emitPendingCheck(); return output; } + if (e.fn === "record.sourceSpread") { + const built = host.emitReadReceiver(e.args[0]!); + const value = host.emitReadReceiver(e.args[1]!); + const type = e.args[0]!.type; + if (type.kind !== "record") + throw new InternalCompilerError("llvm emitter bug: record.sourceSpread of a non-record"); + host.dyn.emitRecordSourceSpread(B, built.name, value.name, type.shapeId); + return { name: "", type: e.type }; + } if (e.fn === "dyn.typedRefIs" || e.fn === "dyn.typedRefKeyIs") { const literal = e.args[1]!.kind === "strLit" ? e.args[1]!.value : null; if (e.fn === "dyn.typedRefKeyIs" && literal === null) diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 7dd4370c5..13f9fcdc5 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -676,6 +676,7 @@ export const LIB_FN_SYMS: Record = { "insp.circular": "scr_insp_circular", // Undeclared source properties kept beside a checked record (scr_json.c). "record.sourceHas": "scr_record_source_has", + "record.sourceSpread": "scr_record_source_spread", // WHATWG URL + URLSearchParams (scr_url.c / scr_url_params.c): // constructions +1; url.new, the fileURLToPath pair, the win32 // pathToFileURL flavor, and sp.fromPairs throw catchably (may-throw). diff --git a/packages/compiler/src/frontend/lowering/key-order-witness.test.ts b/packages/compiler/src/frontend/lowering/key-order-witness.test.ts index 733888763..d68be96a8 100644 --- a/packages/compiler/src/frontend/lowering/key-order-witness.test.ts +++ b/packages/compiler/src/frontend/lowering/key-order-witness.test.ts @@ -42,6 +42,7 @@ function fixture(): Fixture { unions: new Map([["u0", { id: "u0", arms: [F64, { kind: "undefinedT" }] }]]), classes: new Map(), methodAccessSites: new Map(), + dynConvertible: () => true, armTag: (unionId: string, arm: IrType): number => unionId === "u0" && arm.kind === "undefinedT" ? 1 : -1, recordFieldPresent: (obj: IrExpr, shapeId: string, field: string): IrExpr => ({ @@ -873,3 +874,92 @@ test("a generator resume builds only its result record, never the value it carri // The result record itself is built in its fixed layout: today's table. expect(f.order(result)).toEqual(["value", "done"]); }); + +/** `{ a, b }` built by a checked conversion and by the literal `{ b, a }`, + * copied by `{ ...value, a }` (evaluating the value into a binding, as + * spread literals do), observed. Answers the table and the function. */ +function checkedSelfCopy(): { order: string[]; fn: IrFunction; copy: IrExpr } { + const f = fixture(); + const ab = f.record( + [ + ["a", F64], + ["b", F64], + ], + ["a", "b"], + ); + const value = ref("value", ab) as IrExpr & { kind: "varRef" }; + const self: KeyOrderSegment = { kind: "spread", shapeId: ab.shapeId, maybeAbsent: false, value }; + const literal = f.literal(ab, ["a", "b"], [self, { kind: "key", name: "a", definite: true }]); + const copy: IrExpr = { + kind: "seqExpr", + stmts: [decl("value", { kind: "dynCheck", value: ref("j", { kind: "dyn" }), type: ab, loc })], + result: literal, + type: ab, + loc, + }; + const fn = f.run([observe(f.literal(ab, ["b", "a"])), observe(copy)]); + return { order: f.order(ab)!, fn, copy }; +} + +test("a spread of a value a checked conversion built carries the value's order over", () => { + const { order, fn, copy } = checkedSelfCopy(); + // Values the literal builds enumerate in its order; copies of converted + // values keep the order their record source holds. + expect(order).toEqual(["b", "a"]); + expect(copy).toMatchObject({ + kind: "seqExpr", + stmts: [ + { kind: "varDecl", localId: "value" }, + { kind: "varDecl", localId: "%orderedSpread.0", init: { kind: "recordLit" } }, + { + kind: "exprStmt", + expr: { + kind: "libCall", + fn: "record.sourceSpread", + args: [ + { kind: "varRef", localId: "%orderedSpread.0" }, + { kind: "varRef", localId: "value" }, + ], + }, + }, + ], + result: { kind: "varRef", localId: "%orderedSpread.0" }, + }); + expect(fn.locals.map((local) => local.id)).toEqual(["%orderedSpread.0"]); +}); + +test("a spread of a converted value into a wider shape orders the wider shape", () => { + const f = fixture(); + const ab = f.record( + [ + ["a", F64], + ["b", F64], + ], + ["a", "b"], + ); + const abc = f.record( + [ + ["a", F64], + ["b", F64], + ["c", F64], + ], + ["c", "a", "b"], + ); + const value = ref("value", ab) as IrExpr & { kind: "varRef" }; + const spread: KeyOrderSegment = { + kind: "spread", + shapeId: ab.shapeId, + maybeAbsent: false, + value, + }; + const wide: IrExpr = { + kind: "seqExpr", + stmts: [decl("value", { kind: "dynCheck", value: ref("j", { kind: "dyn" }), type: ab, loc })], + result: f.literal(abc, ["a", "b", "c"], [spread, { kind: "key", name: "c", definite: true }]), + type: abc, + loc, + }; + f.run([observe(f.literal(ab, ["b", "a"])), observe(wide)]); + expect(f.order(ab)).toEqual(["b", "a"]); + expect(f.order(abc)).toEqual(["b", "a", "c"]); +}); diff --git a/packages/compiler/src/frontend/lowering/key-order-witness.ts b/packages/compiler/src/frontend/lowering/key-order-witness.ts index 4a0a1309f..6e65d6f6d 100644 --- a/packages/compiler/src/frontend/lowering/key-order-witness.ts +++ b/packages/compiler/src/frontend/lowering/key-order-witness.ts @@ -41,6 +41,12 @@ * - a literal that starts by spreading a value of its own shape copies * that value's order. While no checked conversion builds the shape, * every value has the one chosen order, so the copy adds no witness; + * - a literal that starts by spreading a value of a shape checked + * conversions build (an ordered spread) gives the record it builds the + * value's record source, when the value has one (record.sourceSpread). + * Every value of a shape enumerates in its table or in the order its + * record source holds, so the copy has the value's order either way, + * and the literal resolves as if the value were in its table's order; * - a write to an optional key that may be absent appends it in Node. A * key no construction creates ("late-only") goes after every key a * construction can create; any other moving write goes after every key. @@ -61,6 +67,7 @@ import { shapeHasAccessorSlots, typeEquals, UNDEFINED_T, + VOID, } from "../../ir/ir.js"; import { everyExprChild, @@ -71,6 +78,7 @@ import { import type { Lowerer } from "./lowerer.js"; import * as ts from "../ts7/adapter.js"; import { ALWAYS_THROW_LIB_FNS } from "../../ir/builtin-effects.js"; +import { recordMayKeepSource } from "./record-sources.js"; /** The statements a key-order helper runs for record `ref`: `walks[0]`, * its walk over the shape's table, or, for a shape with orderVariants, a @@ -218,6 +226,14 @@ interface ShapeFacts { type Resolved = { order: string[] } | "unknown"; +/** A literal that starts with `...value`: the sequence that evaluates the + * value into its binding and then builds the literal, and its function. */ +interface LeadingSpread { + wrapper: IrExpr & { kind: "seqExpr" }; + literal: IrExpr & { kind: "recordLit" }; + fn: IrFunction; +} + /** `structuredClone(v)` checked back into v's own type: the copy is built * from v's dyn view, so it creates exactly v's keys in v's order. */ function orderPreservingClone(node: IrExpr & { kind: "dynCheck" }): boolean { @@ -412,6 +428,12 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio // ── collection ────────────────────────────────────────────────────── let pos = 0; let fnName = ""; + let currentFn: IrFunction | null = null; + /** Literals that start by spreading a binding, by their witness: where + * each one is built. A literal's notes are shared by its rebuilt copies, + * so `literalSites` counts every literal of each witness. */ + const leadingSpreads = new Map(); + const literalSites = new Map(); const loops: number[] = []; let loopIds = 0; const loopKey = (): string => loops.join("/"); @@ -463,8 +485,10 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio } const segments = notes.literalOf(e); const copy = notes.copyOf(e); - if (segments !== undefined) f.literals.push(segments); - else if (copy !== undefined) f.copies.push({ source: copy, keys: presentKeys }); + if (segments !== undefined) { + f.literals.push(segments); + literalSites.set(segments, (literalSites.get(segments) ?? 0) + 1); + } else if (copy !== undefined) f.copies.push({ source: copy, keys: presentKeys }); else f.pinned = true; return; } @@ -479,6 +503,18 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio case "assignExpr": noteInit(e.localId, e.value); return; + case "seqExpr": { + // A spread literal evaluates its sources into bindings first. + const literal = e.result; + if (literal.kind !== "recordLit" || currentFn === null) return; + const segments = notes.literalOf(literal); + const first = segments?.[0]; + if (segments === undefined || first?.kind !== "spread" || first.value === undefined) return; + const list = leadingSpreads.get(segments) ?? []; + list.push({ wrapper: e, literal, fn: currentFn }); + leadingSpreads.set(segments, list); + return; + } case "jsonStringify": case "dynFrom": observedRoots.push(e.value.type); @@ -582,6 +618,7 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio }; for (const fn of fns) { fnName = fn.name; + currentFn = fn; for (const st of fn.body) stmt(st); } for (const id of opaqueShapes) { @@ -698,6 +735,61 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio return edges; }; + // ── ordered spreads ───────────────────────────────────────────────── + // A value a checked conversion builds keeps its source's key order in + // its record source when that order is not the table's (record- + // sources.ts), so every value of a shape enumerates in the table's order + // or in the order its record source holds. A literal that starts with + // `...value` creates the value's keys first, in the value's order. When + // the value's shape is one checked conversions build, the literal + // carries that order over: after it builds the record it gives the + // record a source holding the value's order (record.sourceSpread), and + // the keys the value lacks follow in the table's order, which every + // literal sequence of the shape agrees with. Such a literal resolves as + // if its source were in the source's table order, and its own shape + // counts as built at run time too, since its values may now carry a + // source. It qualifies when every value of the source has all of the + // source's keys, both shapes can keep a source, and every later spread + // whose values may carry a source only overwrites the source's keys + // (its own order cannot move them). + const orderedSpreads = new Set(); + const keepsSource = (shapeId: string): boolean => + recordMayKeepSource(lowerer, { kind: "record", shapeId }); + const userKeys = (shapeId: string): string[] => + (shapes.get(shapeId)?.fields ?? []).map((x) => x.name).filter((k) => !k.startsWith("%")); + const ordersSpread = (segments: KeyOrderSegment[], sites: readonly LeadingSpread[]): boolean => { + const first = segments[0]; + if (first?.kind !== "spread" || first.maybeAbsent || first.value === undefined) return false; + const sourceId = first.shapeId; + if (facts.get(sourceId)?.runtime !== true || sites.length !== literalSites.get(segments)) + return false; + const target = sites[0]!.literal.type; + if (target.kind !== "record" || !keepsSource(sourceId) || !keepsSource(target.shapeId)) + return false; + const localId = first.value.localId; + const bound = ({ wrapper, literal }: LeadingSpread): boolean => + typeEquals(literal.type, target) && + wrapper.stmts.some( + (st) => + st.kind === "varDecl" && + st.localId === localId && + st.init !== null && + st.init.type.kind === "record" && + st.init.type.shapeId === sourceId, + ); + if (!sites.every(bound)) return false; + const sourceKeys = new Set(userKeys(sourceId)); + if ([...sourceKeys].some((k) => maybeAbsent(sourceId, k))) return false; + return segments + .slice(1) + .every( + (s) => + s.kind === "key" || + (s.kind === "spread" && + (facts.get(s.shapeId)?.runtime !== true || + userKeys(s.shapeId).every((k) => sourceKeys.has(k)))), + ); + }; // ── witness sequences ─────────────────────────────────────────────── /** The key sequences one literal can create, or null when a spread * source's order is unknown. A key sits at its first occurrence that @@ -707,15 +799,19 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio segments: readonly KeyOrderSegment[], keySet: ReadonlySet, resolved: (id: string) => Resolved, + raw: (id: string) => Resolved, ): string[][] | null => { const occurrences: { key: string; definite: boolean }[] = []; + const ordered = orderedSpreads.has(segments); for (const s of segments) { if (s.kind === "opaque") return null; if (s.kind === "key") { if (keySet.has(s.name)) occurrences.push({ key: s.name, definite: s.definite }); continue; } - const source = resolved(s.shapeId); + // An ordered spread's values without a source enumerate in the + // source's table; the others carry their order over. + const source = ordered && s === segments[0] ? raw(s.shapeId) : resolved(s.shapeId); if (source === "unknown") return null; for (const k of source.order) { if (keySet.has(k)) @@ -809,13 +905,17 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio first.kind === "spread" && first.shapeId === id && !first.maybeAbsent && - !f.runtime && + (!f.runtime || orderedSpreads.has(segments)) && segments.every((s) => s.kind !== "opaque") && table.every((k) => !maybeAbsent(id, k)) ); }; /** One shape's order given its sources' current orders. */ - const resolveOne = (id: string, current: (source: string) => Resolved): Resolved => { + const resolveOne = ( + id: string, + current: (source: string) => Resolved, + raw: (source: string) => Resolved, + ): Resolved => { const shape = shapes.get(id); if (!inScope(shape)) return "unknown"; const table = tableOf(shape); @@ -832,7 +932,7 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio if (f.pinned) chain(table, edges); for (const segments of f.literals) { if (copiesItself(id, f, table, segments)) continue; - const sequences = literalSequences(segments, keySet, current); + const sequences = literalSequences(segments, keySet, current, raw); if (sequences === null) return "unknown"; for (const seq of sequences) chain(seq, edges); } @@ -891,32 +991,57 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio const known = result.get(id); return known === undefined ? "unknown" : asSource(id, known); }; - for (const scc of sccs) { - const members = new Set(scc); - const guess = new Map(); - for (const id of scc) { - const shape = shapes.get(id); - guess.set(id, inScope(shape) ? { order: tableOf(shape) } : "unknown"); - } - const lookup = (id: string): Resolved => - members.has(id) ? asSource(id, guess.get(id)!) : external(id); - let stable = false; - for (let round = 0; round < scc.length + 3 && !stable; round++) { - stable = true; + const resolveAll = (): void => { + result.clear(); + late.clear(); + for (const scc of sccs) { + const members = new Set(scc); + const guess = new Map(); for (const id of scc) { - const next = resolveOne(id, lookup); - const prev = guess.get(id)!; - const same = - next === "unknown" - ? prev === "unknown" - : prev !== "unknown" && prev.order.join("\0") === next.order.join("\0"); - if (!same) { - stable = false; - guess.set(id, next); + const shape = shapes.get(id); + guess.set(id, inScope(shape) ? { order: tableOf(shape) } : "unknown"); + } + const lookup = (id: string): Resolved => + members.has(id) ? asSource(id, guess.get(id)!) : external(id); + const rawLookup = (id: string): Resolved => + members.has(id) ? guess.get(id)! : (result.get(id) ?? "unknown"); + let stable = false; + for (let round = 0; round < scc.length + 3 && !stable; round++) { + stable = true; + for (const id of scc) { + const next = resolveOne(id, lookup, rawLookup); + const prev = guess.get(id)!; + const same = + next === "unknown" + ? prev === "unknown" + : prev !== "unknown" && prev.order.join("\0") === next.order.join("\0"); + if (!same) { + stable = false; + guess.set(id, next); + } } } + for (const id of scc) result.set(id, stable ? guess.get(id)! : "unknown"); } - for (const id of scc) result.set(id, stable ? guess.get(id)! : "unknown"); + }; + // Ordered spreads join once their source resolves; each one can let its + // own shape resolve, which a later spread of it needs. + resolveAll(); + for (let grew = true; grew;) { + grew = false; + for (const [segments, sites] of leadingSpreads) { + const first = segments[0]; + if (first?.kind !== "spread" || orderedSpreads.has(segments)) continue; + // A self copy resolves with its own shape. + const target = sites[0]!.literal.type; + const source = result.get(first.shapeId) ?? "unknown"; + const self = target.kind === "record" && target.shapeId === first.shapeId; + if ((source === "unknown" && !self) || !ordersSpread(segments, sites)) continue; + orderedSpreads.add(segments); + if (target.kind === "record") factsOf(target.shapeId).runtime = true; + grew = true; + } + if (grew) resolveAll(); } // ── observation ───────────────────────────────────────────────────── @@ -925,6 +1050,19 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio for (const id of notes.observed) shapesIn({ kind: "record", shapeId: id }, observed, observedSeen); for (const root of observedRoots) shapesIn(root, observed, observedSeen); + // An ordered spread's copies of values without a source take the source's + // order from its table, which must then hold the resolved order. + const orderedTarget = (sites: readonly LeadingSpread[]): string | null => { + const target = sites[0]!.literal.type; + return target.kind === "record" && (result.get(target.shapeId) ?? "unknown") !== "unknown" + ? target.shapeId + : null; + }; + for (const [segments, sites] of leadingSpreads) { + const first = segments[0]; + if (orderedSpreads.has(segments) && first?.kind === "spread" && orderedTarget(sites) !== null) + observed.add(first.shapeId); + } // ── apply ─────────────────────────────────────────────────────────── for (const [id, resolved] of result) { @@ -937,6 +1075,46 @@ export function applyKeyOrderWitnesses(lowerer: Lowerer, fns: readonly IrFunctio notes.changed.add(id); } + // Ordered spreads carry their source's order over (see above). + let orderedResults = 0; + const rewritten = new Set(); + for (const [segments, sites] of leadingSpreads) { + const first = segments[0]; + if (!orderedSpreads.has(segments) || first?.kind !== "spread" || first.value === undefined) + continue; + // A shape that did not resolve keeps today's table for every value. + if (orderedTarget(sites) === null) continue; + for (const { wrapper, literal, fn } of sites) { + if (rewritten.has(wrapper)) continue; + rewritten.add(wrapper); + const loc = literal.loc; + const localId = `%orderedSpread.${orderedResults++}`; + fn.locals.push({ id: localId, name: "%orderedSpread", type: literal.type, mutable: false }); + const built: IrExpr = { kind: "varRef", localId, type: literal.type, loc }; + const source: IrExpr = { + kind: "varRef", + localId: first.value.localId, + type: { kind: "record", shapeId: first.shapeId }, + loc, + }; + wrapper.stmts.push( + { kind: "varDecl", localId, init: literal, loc }, + { + kind: "exprStmt", + expr: { + kind: "libCall", + fn: "record.sourceSpread", + args: [built, source], + type: VOID, + loc, + }, + loc, + }, + ); + wrapper.result = { kind: "varRef", localId, type: literal.type, loc }; + } + } + // ── per-instance orders ───────────────────────────────────────────── // A shape whose literals write its keys in more than one order has no // one table. Each value then records which of the orders it was written diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 6a645785f..9ea65d47d 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -4424,6 +4424,11 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ * (inspect, Object.keys/values/entries) then answer through its live * checked view. The receiver is any such record type. Never throws. */ "record.sourceHas": { argTypes: [null], result: BOOL }, + /** `record.sourceSpread(built, value)`: the record a literal starting + * with `...value` just built keeps the order of value's kept source + * (key-order-witness.ts ordered spreads). Both are record types. Never + * throws. */ + "record.sourceSpread": { argTypes: [null, null], result: VOID }, "insp.key": { argTypes: [STRING], result: STRING }, "insp.moreItems": { argTypes: [F64], result: STRING }, "insp.end": { argTypes: [STRING, STRING, STRING, F64, BOOL, BOOL], result: STRING }, diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7750-key-order-javascript-spread-sources/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7750-key-order-javascript-spread-sources/main.ts.json new file mode 100644 index 000000000..19f1ba620 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7750-key-order-javascript-spread-sources/main.ts.json @@ -0,0 +1,7 @@ +{ + "order": [ + "/tests/corpus/7750-key-order-javascript-spread-sources/lib.js", + "/tests/corpus/7750-key-order-javascript-spread-sources/main.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 553b897d2..c3a6a8f38 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -1020,6 +1020,12 @@ void scr_dyn_typed_ref_bind_record(ScrDyn *d, const ScrDynTypedRecordOps *ops); void scr_record_source_note(void *rec, const ScrDyn *src, const char *const *names, const size_t *lens, size_t count); bool scr_record_source_has(const void *rec); +/* A literal that starts with `...src` creates src's keys first, in src's + * order: when src kept a source, the fresh record `dst` (whose shape's + * declared names are `names`) keeps one listing the keys it declares in + * that order. Undeclared keys are not copied, as the literal copies none. */ +void scr_record_source_spread(void *dst, const void *src, const char *const *names, + const size_t *lens, size_t count); /* The commit of a live record view converts the view into a new record * `from`, whose conversion keeps the view's undeclared properties, then * moves its fields into `to`: this moves the kept properties to `to` too, diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 615785687..28ce987ff 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -3191,6 +3191,38 @@ void scr_record_source_note(void *rec, const ScrDyn *src, const char *const *nam scr_record_source_put(&probe); } +void scr_record_source_spread(void *dst, const void *src, const char *const *names, + const size_t *lens, size_t count) { + if (!scr_record_source_count || !dst || !src) return; + ScrRecordSource *found = scr_record_source_find(src); + if (!found) return; + ScrRecordSource probe = { dst, NULL, (const char *)names, (const char *)lens, + sizeof *names, sizeof *lens, count }; + /* Nothing to keep when the declared keys come in the table's order and + * the keys src lacks would follow them in it anyway. */ + const ScrDyn *from = found->view; + bool keep = false; + size_t next = 0; + for (size_t i = 0; i < from->v.obj.len && !keep; i++) { + const ScrDynEntry *entry = &from->v.obj.entries[i]; + size_t index = scr_record_source_index(&probe, entry->key, entry->key_len); + if (index == count) continue; + keep = index != next; + next++; + } + if (!keep) return; + ScrDyn *view = scr_dyn_new_obj(); + for (size_t i = 0; i < from->v.obj.len; i++) { + const ScrDynEntry *entry = &from->v.obj.entries[i]; + if (scr_record_source_declared(&probe, entry->key, entry->key_len)) + scr_dyn_obj_set(view, entry->key, entry->key_len, scr_dyn_retain(scr_dyn_undefined())); + } + ScrRecordSource *old = scr_record_source_find(dst); + if (old) scr_record_source_drop(old); + probe.view = view; + scr_record_source_put(&probe); +} + void scr_record_source_move(void *from, void *to) { if (!scr_record_source_count) return; ScrRecordSource *old = scr_record_source_find(to); diff --git a/tests/corpus/7750-key-order-javascript-spread-sources/lib.js b/tests/corpus/7750-key-order-javascript-spread-sources/lib.js new file mode 100644 index 000000000..c8fc0d066 --- /dev/null +++ b/tests/corpus/7750-key-order-javascript-spread-sources/lib.js @@ -0,0 +1,14 @@ +// Records JavaScript builds in its own key order, handed to TypeScript +// through typed slots: a draft from a "validator", a stored row, and a +// replacement method installed through a computed key. +export function validate(input) { + return { year: input.year, title: input.title, tags: input.tags }; +} + +export function storeRow(rows, id) { + rows.set(id, { title: "Stored", id, tags: [], year: 1999, createdAt: "t0", version: 1 }); +} + +export function replace(target, name) { + target[name] = (id) => ({ version: 7, id, year: 2001, title: "Replaced", tags: ["r"], createdAt: "t1" }); +} diff --git a/tests/corpus/7750-key-order-javascript-spread-sources/main.ts b/tests/corpus/7750-key-order-javascript-spread-sources/main.ts new file mode 100644 index 000000000..727f8e367 --- /dev/null +++ b/tests/corpus/7750-key-order-javascript-spread-sources/main.ts @@ -0,0 +1,105 @@ +// Literals that start by spreading a record JavaScript built keep that +// record's key order, as Node does: a draft from JavaScript spread into a +// new record, a self copy of a record JavaScript stored or returned, and +// a wider view of either. Native drafts and copies keep the literal order. +import { replace, storeRow, validate } from "./lib.js"; + +interface Book { + id: string; + title: string; + year: number; + tags: readonly string[]; + createdAt: string; + version: number; +} + +type Draft = Omit; +type Patch = Partial; + +interface Availability { + copies: number; + onLoan: number; +} + +type BookView = Book & { availability: Availability }; + +class Repository { + private readonly rows = new Map(); + private sequence = 0; + + create(draft: Draft): Book { + const book: Book = { + ...draft, + tags: [...draft.tags].sort(), + id: `bk_${++this.sequence}`, + createdAt: "t" + this.sequence, + version: 1, + }; + this.rows.set(book.id, book); + return book; + } + + require(id: string): Book { + const row = this.rows.get(id); + if (row === undefined) throw new Error("missing " + id); + return row; + } + + update(id: string, patch: Patch): Book { + const current = this.require(id); + const next: Book = { ...current, ...patch, version: current.version + 1 }; + this.rows.set(id, next); + return next; + } + + view(book: Book): BookView { + return { ...book, availability: { copies: 2, onLoan: book.version } }; + } + + table(): Map { + return this.rows; + } +} + +function show(label: string, value: Book): void { + console.log(label, JSON.stringify(value)); + console.log(" keys", Object.keys(value).join()); +} + +function showView(label: string, value: BookView): void { + console.log(label, JSON.stringify(value)); + console.log(" keys", Object.keys(value).join()); +} + +const repo = new Repository(); + +// A native draft: the literal's order. +const native = repo.create({ title: "Native", year: 1990, tags: ["b", "a"] }); +show("native", native); +showView("native view", repo.view(native)); +show("native update", repo.update(native.id, { year: 1991 })); + +// A draft JavaScript built in another order. +const draft: Draft = validate({ title: "Checked", year: 2020, tags: ["z", "y"] }); +const checked = repo.create(draft); +show("checked", checked); +showView("checked view", repo.view(checked)); +show("checked update", repo.update(checked.id, { title: "Checked 2" })); +showView("checked update view", repo.view(repo.require(checked.id))); + +// A row JavaScript stored into the typed map. +storeRow(repo.table(), "js_1"); +show("stored", repo.require("js_1")); +show("stored update", repo.update("js_1", { tags: ["s"] })); +showView("stored view", repo.view(repo.require("js_1"))); + +// A replacement method returning rows JavaScript builds. +replace(repo, "require"); +show("replaced", repo.require("x")); +show("replaced update", repo.update("x", { year: 2002 })); +showView("replaced view", repo.view(repo.require("y"))); + +// The copies are independent records. +const again = repo.view(checked); +showView("again", again); +console.log(native.version, checked.version, again.availability.onLoan); diff --git a/tests/harness/corpus-costs/7750-key-order-javascript-spread-sources/main.ts.json b/tests/harness/corpus-costs/7750-key-order-javascript-spread-sources/main.ts.json new file mode 100644 index 000000000..eb6dbd224 --- /dev/null +++ b/tests/harness/corpus-costs/7750-key-order-javascript-spread-sources/main.ts.json @@ -0,0 +1 @@ +{ "plain": 5300, "san": 9200 } From efa797382ef4120f064e55f170788e096299cbce Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 12:20:28 -0500 Subject: [PATCH 073/184] Render the compatibility matrix's tooltips on demand The matrix mounted every badge's tooltip up front, hidden by CSS, and re-rendered every row on each keystroke. Tooltips now mount while their badge is hovered or has keyboard focus, are named by aria-describedby and close on Escape. Badges and rows are memoized, each row's search text is built once, the filter makes one pass for its rows and its count, and the table follows deferred search and status values, with expand all and collapse all as transitions, so input stays responsive. The tree and filter passes move to src/lib/compatibility-tree.ts, with unit tests that also render the badges on the server (pnpm test:unit, part of pnpm check). Closes #601 Co-authored-by: jonbaldie <8376953+jonbaldie@users.noreply.github.com> --- docs/package.json | 3 +- .../components/node-compatibility-matrix.tsx | 348 ++++++++---------- docs/src/lib/compatibility-tree.ts | 157 ++++++++ docs/test/compatibility-matrix.test.mjs | 115 ++++++ 4 files changed, 424 insertions(+), 199 deletions(-) create mode 100644 docs/src/lib/compatibility-tree.ts create mode 100644 docs/test/compatibility-matrix.test.mjs diff --git a/docs/package.json b/docs/package.json index 3c2a950fd..51d74d0a1 100644 --- a/docs/package.json +++ b/docs/package.json @@ -10,9 +10,10 @@ "build": "fumadocs-mdx && next build", "start": "next start", "typecheck": "fumadocs-mdx && next typegen && tsc --noEmit", + "test:unit": "node --test test/compatibility-matrix.test.mjs", "test:routes": "node scripts/test-routes.mjs", "compat:check": "pnpm --dir .. node-compat:check", - "check": "pnpm compat:check && pnpm typecheck && pnpm build && pnpm test:routes" + "check": "pnpm compat:check && pnpm typecheck && pnpm test:unit && pnpm build && pnpm test:routes" }, "dependencies": { "@vercel/geistdocs": "2.7.8", diff --git a/docs/src/components/node-compatibility-matrix.tsx b/docs/src/components/node-compatibility-matrix.tsx index 095bcdaf2..bcc5ece61 100644 --- a/docs/src/components/node-compatibility-matrix.tsx +++ b/docs/src/components/node-compatibility-matrix.tsx @@ -4,52 +4,28 @@ // empty after an edit. // @refresh reset -import { useEffect, useMemo, useState } from "react"; +import { memo, startTransition, useCallback, useDeferredValue, useEffect, useId, useMemo, useState, type FocusEvent, type KeyboardEvent, type ReactNode } from "react"; +import { + buildTree, + expandableIds, + filterTree, + flattenTree, + nodeCount, + rowMatcher, + searchTexts, + statusCounts, + statuses, + type NodeStability, + type Row, + type Snapshot, + type Status, + type Verification, +} from "@/lib/compatibility-tree"; // Set by next.config.mjs from the committed artifact. const artifactVersion = process.env.COMPATIBILITY_ARTIFACT_VERSION; const rowCount = Number(process.env.COMPATIBILITY_ROW_COUNT); -type Status = - | "supported" - | "partial" - | "refused" - | "not-implemented" - | "by-design" - | "unreviewed" - | "not-applicable"; -type VerificationBasis = - | "test-backed" - | "explicit-refusal" - | "verified-absence" - | "declared-gap" - | "registry-gap" - | "architectural-policy" - | "unreviewed" - | "not-applicable" - | "derived"; -type Verification = { label: string; detail: string }; -type Tier = { status: Status; detail: string; verification: VerificationBasis }; -type NodeStability = { index: string; level: number; text: string; inherited: boolean } | null; -type Row = { - id: string; - chapter: string; - kind: string; - name: string; - signature: string; - label: string; - apiSymbol: string; - depth: number; - scope: "api" | "documentation" | "metadata" | "configuration"; - anchor: string; - nodeStability: NodeStability; - static: Tier; -}; -type Chapter = { slug: string; title: string; entries: number; apiEntries: number }; -type Snapshot = { nodeVersion: string; verificationBases: Record; chapters: Chapter[]; rows: Row[] }; -type TreeNode = { row: Row; children: TreeNode[] }; -type VisibleNode = { node: TreeNode; level: number }; - const labels: Record = { supported: "Supported", partial: "Partial", @@ -70,7 +46,6 @@ const styles: Record = { "not-applicable": "border-gray-alpha-400 bg-background-200 text-gray-700", }; -const statuses = ["supported", "partial", "not-implemented", "refused", "by-design", "unreviewed", "not-applicable"] as const; const stabilityStyles: Record = { 0: "border-red-500/40 bg-red-100 text-red-900 dark:bg-red-200", 1: "border-amber-500/50 bg-amber-100 text-amber-900 dark:bg-amber-200", @@ -78,114 +53,76 @@ const stabilityStyles: Record = { 3: "border-gray-alpha-500 bg-gray-alpha-100 text-gray-900", }; -function StatusBadge({ status, detail, verification }: { status: Status; detail?: string; verification?: Verification }) { - const tooltip = verification ? `${detail ?? ""}${detail ? "\n\n" : ""}Verification: ${verification.label}. ${verification.detail}` : detail; +/** A badge's tooltip, mounted only while the badge is hovered or has + * keyboard focus (the :focus-visible rule the CSS-only tooltip used), and + * named by aria-describedby while open. Escape dismisses it. */ +function BadgeTooltip({ tooltipClass, children, badge }: { tooltipClass: string; children: ReactNode; badge: ReactNode }) { + const [open, setOpen] = useState(false); + const id = useId(); + const show = useCallback(() => setOpen(true), []); + const hide = useCallback(() => setOpen(false), []); + const focus = useCallback((event: FocusEvent) => { + if (event.currentTarget.matches(":focus-visible")) setOpen(true); + }, []); + const key = useCallback((event: KeyboardEvent) => { + if (event.key === "Escape") setOpen(false); + }, []); return ( - - - {labels[status]} - - {tooltip ? ( - - {detail ? {detail} : null} - {verification ? Verification: {verification.label}. {verification.detail} : null} + + {badge} + {open ? ( + + {children} ) : null} ); } -function StabilityBadge({ stability }: { stability: NodeStability }) { +export const StatusBadge = memo(function StatusBadge({ status, detail, verification }: { status: Status; detail?: string; verification?: Verification }) { + const tooltip = verification ? `${detail ?? ""}${detail ? "\n\n" : ""}Verification: ${verification.label}. ${verification.detail}` : detail; + const badge = ( + + {labels[status]} + + ); + if (!tooltip) return {badge}; + return ( + + {detail ? {detail} : null} + {verification ? Verification: {verification.label}. {verification.detail} : null} + + ); +}); + +export const StabilityBadge = memo(function StabilityBadge({ stability }: { stability: NodeStability }) { if (stability === null) return —; const detail = `Node.js Stability ${stability.index}: ${stability.text}${stability.inherited ? " (inherited from the nearest parent section)" : ""}`; return ( - - - {stability.index} - - - {detail} - - - ); -} - -function statusCounts(rows: Row[], tier: "static") { - return rows.reduce>( - (counts, row) => { - if (row.scope !== "api" || row.depth === 0) return counts; - counts[row[tier].status] += 1; - return counts; - }, - { - supported: 0, - partial: 0, - refused: 0, - "not-implemented": 0, - "by-design": 0, - unreviewed: 0, - "not-applicable": 0, - }, - ); -} - -/** Node's all.json is preorder with an explicit heading depth. */ -function buildTree(rows: Row[]): TreeNode[] { - const roots: TreeNode[] = []; - const stack: TreeNode[] = []; - for (const row of rows) { - const node: TreeNode = { row, children: [] }; - while (stack.length > row.depth) stack.pop(); - const parent = stack[row.depth - 1]; - if (row.depth === 0 || parent === undefined) roots.push(node); - else parent.children.push(node); - stack[row.depth] = node; - stack.length = row.depth + 1; - } - return roots; -} - -function filterTree(nodes: TreeNode[], matches: (row: Row) => boolean): TreeNode[] { - const filtered: TreeNode[] = []; - for (const node of nodes) { - const children = filterTree(node.children, matches); - if (matches(node.row) || children.length > 0) filtered.push({ row: node.row, children }); - } - return filtered; -} - -function flattenTree(nodes: TreeNode[], expanded: Set, forceExpanded: boolean): VisibleNode[] { - const visible: VisibleNode[] = []; - const visit = (items: TreeNode[], level: number) => { - for (const node of items) { - visible.push({ node, level }); - if (node.children.length > 0 && (forceExpanded || expanded.has(node.row.id))) { - visit(node.children, level + 1); + + {stability.index} + } - } - }; - visit(nodes, 1); - return visible; -} - -function nodeCount(nodes: TreeNode[]): number { - return nodes.reduce((count, node) => count + 1 + nodeCount(node.children), 0); -} - -function expandableIds(nodes: TreeNode[]): Set { - const ids = new Set(); - const visit = (items: TreeNode[]) => { - for (const node of items) { - if (node.children.length > 0) ids.add(node.row.id); - visit(node.children); - } - }; - visit(nodes); - return ids; -} + > + {detail} + + ); +}); function Chevron({ open }: { open: boolean }) { return ( @@ -195,6 +132,47 @@ function Chevron({ open }: { open: boolean }) { ); } +type MatrixRowProps = { + row: Row; + level: number; + hasChildren: boolean; + open: boolean; + filtering: boolean; + displayName: string; + href: string; + verification: Verification | undefined; + onToggle: (id: string) => void; +}; + +/** One table row. Memoized on its own values, so a filter or expansion + * change re-renders only the rows whose values changed. */ +const MatrixRow = memo(function MatrixRow({ row, level, hasChildren, open, filtering, displayName, href, verification, onToggle }: MatrixRowProps) { + const isRoot = level === 1; + return ( + + +
+ {hasChildren ? ( + + ) : } +
+ + {displayName} + +
+ {row.kind}{row.apiSymbol && row.apiSymbol !== row.signature && !isRoot ? ` · ${row.apiSymbol}` : ""} +
+
+
+ + + + + ); +}); + export function NodeCompatibilityMatrix() { const [snapshot, setSnapshot] = useState(null); const [loadError, setLoadError] = useState(false); @@ -202,15 +180,16 @@ export function NodeCompatibilityMatrix() { const [staticFilter, setStaticFilter] = useState<"all" | Status>("all"); const [expanded, setExpanded] = useState>(() => new Set()); - const rows = snapshot?.rows ?? []; - const chapters = snapshot?.chapters ?? []; + const rows = useMemo(() => snapshot?.rows ?? [], [snapshot]); + const chapters = useMemo(() => snapshot?.chapters ?? [], [snapshot]); const verificationBases = snapshot?.verificationBases; const apiRowCount = useMemo(() => rows.filter((row) => row.scope === "api").length, [rows]); const chapterTitles = useMemo(() => new Map(chapters.map((item) => [item.slug, item.title])), [chapters]); const base = `https://nodejs.org/docs/v${snapshot?.nodeVersion ?? "24.22.0"}/api/`; - const staticCounts = useMemo(() => statusCounts(rows, "static"), [rows]); + const staticCounts = useMemo(() => statusCounts(rows), [rows]); const visibleStatuses = useMemo(() => statuses.filter((status) => staticCounts[status] > 0), [staticCounts]); const tree = useMemo(() => buildTree(rows), [rows]); + const texts = useMemo(() => searchTexts(rows, chapterTitles), [rows, chapterTitles]); useEffect(() => { const controller = new AbortController(); @@ -231,36 +210,23 @@ export function NodeCompatibilityMatrix() { return () => controller.abort(); }, []); - const needle = query.trim().toLocaleLowerCase(); - const hasResultFilter = needle !== "" || staticFilter !== "all"; - const rowMatches = (row: Row) => { - if (staticFilter !== "all" && row.static.status !== staticFilter) return false; - if (!needle) return true; - return [chapterTitles.get(row.chapter), row.kind, row.name, row.signature, row.apiSymbol, row.static.detail] - .join(" ") - .toLocaleLowerCase() - .includes(needle); - }; + // The search field and the status select update at once; the table + // follows the deferred values, so input stays responsive while the rows + // re-render. + const deferredQuery = useDeferredValue(query); + const deferredStatic = useDeferredValue(staticFilter); + const needle = deferredQuery.trim().toLocaleLowerCase(); + const hasResultFilter = needle !== "" || deferredStatic !== "all"; - const resultTree = useMemo( - () => hasResultFilter ? filterTree(tree, rowMatches) : tree, - // rowMatches closes over these filter values. - // eslint-disable-next-line react-hooks/exhaustive-deps - [hasResultFilter, needle, staticFilter, tree], + const result = useMemo( + () => hasResultFilter ? filterTree(tree, rowMatcher(texts, needle, deferredStatic)) : { nodes: tree, matched: nodeCount(tree) }, + [hasResultFilter, needle, deferredStatic, texts, tree], ); const visible = useMemo( - () => flattenTree(resultTree, expanded, hasResultFilter), - [expanded, hasResultFilter, resultTree], - ); - const matchedCount = useMemo( - () => hasResultFilter ? tree.reduce((count, node) => { - const visit = (item: TreeNode): number => (rowMatches(item.row) ? 1 : 0) + item.children.reduce((sum, child) => sum + visit(child), 0); - return count + visit(node); - }, 0) : nodeCount(tree), - // rowMatches closes over these filter values. - // eslint-disable-next-line react-hooks/exhaustive-deps - [hasResultFilter, needle, staticFilter, tree], + () => flattenTree(result.nodes, expanded, hasResultFilter), + [expanded, hasResultFilter, result], ); + const matchedCount = result.matched; const changeQuery = (value: string) => setQuery(value); const resetFilters = () => { @@ -268,12 +234,12 @@ export function NodeCompatibilityMatrix() { setStaticFilter("all"); setExpanded(new Set()); }; - const toggle = (id: string) => setExpanded((current) => { + const toggle = useCallback((id: string) => setExpanded((current) => { const next = new Set(current); if (next.has(id)) next.delete(id); else next.add(id); return next; - }); + }), []); useEffect(() => { if (!snapshot) return; @@ -316,8 +282,8 @@ export function NodeCompatibilityMatrix() { {visible.length.toLocaleString()} visible · {matchedCount.toLocaleString()} {hasResultFilter ? "matching" : "API"} rows {!hasResultFilter ? ( - - + + ) : null} @@ -335,33 +301,19 @@ export function NodeCompatibilityMatrix() { {visible.map(({ node, level }) => { const row = node.row; - const hasChildren = node.children.length > 0; - const open = hasResultFilter || expanded.has(row.id); - const href = `${base}${row.chapter}.html${row.anchor ? `#${row.anchor}` : ""}`; - const isRoot = level === 1; - const displayName = isRoot ? chapterTitles.get(row.chapter) ?? row.label : row.label; return ( - - -
- {hasChildren ? ( - - ) : } -
- - {displayName} - -
- {row.kind}{row.apiSymbol && row.apiSymbol !== row.signature && !isRoot ? ` · ${row.apiSymbol}` : ""} -
-
-
- - - - + 0} + open={hasResultFilter || expanded.has(row.id)} + filtering={hasResultFilter} + displayName={level === 1 ? chapterTitles.get(row.chapter) ?? row.label : row.label} + href={`${base}${row.chapter}.html${row.anchor ? `#${row.anchor}` : ""}`} + verification={verificationBases?.[row.static.verification]} + onToggle={toggle} + /> ); })} diff --git a/docs/src/lib/compatibility-tree.ts b/docs/src/lib/compatibility-tree.ts new file mode 100644 index 000000000..4044bbc8d --- /dev/null +++ b/docs/src/lib/compatibility-tree.ts @@ -0,0 +1,157 @@ +// The compatibility matrix's data model: the ledger rows as a tree, and the +// search and filter passes over it. Kept free of React so the passes can be +// tested directly. + +export type Status = + | "supported" + | "partial" + | "refused" + | "not-implemented" + | "by-design" + | "unreviewed" + | "not-applicable"; +export type VerificationBasis = + | "test-backed" + | "explicit-refusal" + | "verified-absence" + | "declared-gap" + | "registry-gap" + | "architectural-policy" + | "unreviewed" + | "not-applicable" + | "derived"; +export type Verification = { label: string; detail: string }; +export type Tier = { status: Status; detail: string; verification: VerificationBasis }; +export type NodeStability = { index: string; level: number; text: string; inherited: boolean } | null; +export type Row = { + id: string; + chapter: string; + kind: string; + name: string; + signature: string; + label: string; + apiSymbol: string; + depth: number; + scope: "api" | "documentation" | "metadata" | "configuration"; + anchor: string; + nodeStability: NodeStability; + static: Tier; +}; +export type Chapter = { slug: string; title: string; entries: number; apiEntries: number }; +export type Snapshot = { + nodeVersion: string; + verificationBases: Record; + chapters: Chapter[]; + rows: Row[]; +}; +export type TreeNode = { row: Row; children: TreeNode[] }; +export type VisibleNode = { node: TreeNode; level: number }; + +export const statuses = ["supported", "partial", "not-implemented", "refused", "by-design", "unreviewed", "not-applicable"] as const; + +export function statusCounts(rows: Row[]): Record { + return rows.reduce>( + (counts, row) => { + if (row.scope !== "api" || row.depth === 0) return counts; + counts[row.static.status] += 1; + return counts; + }, + { + supported: 0, + partial: 0, + refused: 0, + "not-implemented": 0, + "by-design": 0, + unreviewed: 0, + "not-applicable": 0, + }, + ); +} + +/** Node's all.json is preorder with an explicit heading depth. */ +export function buildTree(rows: Row[]): TreeNode[] { + const roots: TreeNode[] = []; + const stack: TreeNode[] = []; + for (const row of rows) { + const node: TreeNode = { row, children: [] }; + while (stack.length > row.depth) stack.pop(); + const parent = stack[row.depth - 1]; + if (row.depth === 0 || parent === undefined) roots.push(node); + else parent.children.push(node); + stack[row.depth] = node; + stack.length = row.depth + 1; + } + return roots; +} + +/** The lowercase text a search matches against, built once per row: the + * chapter title, kind, name, signature, API symbol and support detail. */ +export function searchTexts(rows: Row[], chapterTitles: ReadonlyMap): Map { + return new Map( + rows.map((row) => [ + row.id, + [chapterTitles.get(row.chapter), row.kind, row.name, row.signature, row.apiSymbol, row.static.detail] + .join(" ") + .toLocaleLowerCase(), + ]), + ); +} + +/** The rows a status filter and a lowercase search needle keep. */ +export function rowMatcher( + texts: ReadonlyMap, + needle: string, + status: "all" | Status, +): (row: Row) => boolean { + return (row) => { + if (status !== "all" && row.static.status !== status) return false; + return needle === "" || (texts.get(row.id) ?? "").includes(needle); + }; +} + +/** The matching rows with their ancestors, and the number of matching + * rows, in one pass. */ +export function filterTree(nodes: TreeNode[], matches: (row: Row) => boolean): { nodes: TreeNode[]; matched: number } { + let matched = 0; + const visit = (items: TreeNode[]): TreeNode[] => { + const filtered: TreeNode[] = []; + for (const node of items) { + const children = visit(node.children); + const own = matches(node.row); + if (own) matched += 1; + if (own || children.length > 0) filtered.push({ row: node.row, children }); + } + return filtered; + }; + return { nodes: visit(nodes), matched }; +} + +export function flattenTree(nodes: TreeNode[], expanded: ReadonlySet, forceExpanded: boolean): VisibleNode[] { + const visible: VisibleNode[] = []; + const visit = (items: TreeNode[], level: number) => { + for (const node of items) { + visible.push({ node, level }); + if (node.children.length > 0 && (forceExpanded || expanded.has(node.row.id))) { + visit(node.children, level + 1); + } + } + }; + visit(nodes, 1); + return visible; +} + +export function nodeCount(nodes: TreeNode[]): number { + return nodes.reduce((count, node) => count + 1 + nodeCount(node.children), 0); +} + +export function expandableIds(nodes: TreeNode[]): Set { + const ids = new Set(); + const visit = (items: TreeNode[]) => { + for (const node of items) { + if (node.children.length > 0) ids.add(node.row.id); + visit(node.children); + } + }; + visit(nodes); + return ids; +} diff --git a/docs/test/compatibility-matrix.test.mjs b/docs/test/compatibility-matrix.test.mjs new file mode 100644 index 000000000..abd3df945 --- /dev/null +++ b/docs/test/compatibility-matrix.test.mjs @@ -0,0 +1,115 @@ +import assert from "node:assert/strict"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { test } from "node:test"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import ts from "typescript"; +import { + buildTree, + expandableIds, + filterTree, + flattenTree, + nodeCount, + rowMatcher, + searchTexts, + statusCounts, +} from "../src/lib/compatibility-tree.ts"; + +const row = (id, depth, label, status, detail = "") => ({ + id, + chapter: "zlib", + kind: depth === 0 ? "module" : "method", + name: label, + signature: label, + label, + apiSymbol: label, + depth, + scope: "api", + anchor: "", + nodeStability: null, + static: { status, detail, verification: "test-backed" }, +}); +const rows = [ + row("root", 0, "zlib", "partial"), + row("a", 1, "zlib.gzipSync()", "supported", "Lowered with Node's options"), + row("b", 1, "zlib.Gzip", "partial"), + row("b1", 2, "gzip.flush()", "supported"), + row("c", 1, "zlib.zstdCompress()", "not-implemented"), +]; +const titles = new Map([["zlib", "Zlib"]]); + +test("the tree follows the rows' heading depths", () => { + const tree = buildTree(rows); + assert.equal(tree.length, 1); + assert.deepEqual(tree[0].children.map((node) => node.row.id), ["a", "b", "c"]); + assert.deepEqual(tree[0].children[1].children.map((node) => node.row.id), ["b1"]); + assert.equal(nodeCount(tree), 5); + assert.deepEqual([...expandableIds(tree)].sort(), ["b", "root"]); +}); + +test("filtering keeps matching rows with their ancestors and counts the matches once", () => { + const tree = buildTree(rows); + const texts = searchTexts(rows, titles); + const byText = filterTree(tree, rowMatcher(texts, "flush", "all")); + assert.equal(byText.matched, 1); + assert.deepEqual(flattenTree(byText.nodes, new Set(), true).map(({ node, level }) => `${node.row.id}@${level}`), ["root@1", "b@2", "b1@3"]); + const byDetail = filterTree(tree, rowMatcher(texts, "node's options", "all")); + assert.deepEqual(flattenTree(byDetail.nodes, new Set(), true).map(({ node }) => node.row.id), ["root", "a"]); + const byStatus = filterTree(tree, rowMatcher(texts, "", "supported")); + assert.equal(byStatus.matched, 2); + const both = filterTree(tree, rowMatcher(texts, "gzip", "partial")); + assert.equal(both.matched, 1); + assert.equal(filterTree(tree, rowMatcher(texts, "brotli", "all")).nodes.length, 0); +}); + +test("collapsed rows hide their children until expanded", () => { + const tree = buildTree(rows); + assert.deepEqual(flattenTree(tree, new Set(), false).map(({ node }) => node.row.id), ["root"]); + assert.deepEqual(flattenTree(tree, new Set(["root"]), false).map(({ node }) => node.row.id), ["root", "a", "b", "c"]); + assert.deepEqual(flattenTree(tree, new Set(["root", "b"]), false).map(({ node }) => node.row.id), ["root", "a", "b", "b1", "c"]); +}); + +test("status counts cover API rows below the chapters", () => { + const counts = statusCounts(rows); + assert.equal(counts.supported, 2); + assert.equal(counts.partial, 1); + assert.equal(counts["not-implemented"], 1); +}); + +/** The matrix component, compiled for Node from its TSX source. */ +async function loadMatrix() { + const docs = new URL("../", import.meta.url); + const source = await readFile(new URL("src/components/node-compatibility-matrix.tsx", docs), "utf8"); + const { outputText } = ts.transpileModule(source, { + compilerOptions: { jsx: ts.JsxEmit.ReactJSX, module: ts.ModuleKind.ESNext, target: ts.ScriptTarget.ES2022 }, + }); + const lib = pathToFileURL(fileURLToPath(new URL("src/lib/compatibility-tree.ts", docs))).href; + const dir = new URL("node_modules/.cache/compatibility-matrix-test/", docs); + await mkdir(dir, { recursive: true }); + const file = new URL("node-compatibility-matrix.mjs", dir); + await writeFile(file, outputText.replace('"@/lib/compatibility-tree"', JSON.stringify(lib))); + return import(file.href); +} + +test("badges render without their tooltips until hovered or focused", async () => { + const { StatusBadge, StabilityBadge } = await loadMatrix(); + const status = renderToStaticMarkup( + createElement(StatusBadge, { + status: "partial", + detail: "Lowered with Node's options", + verification: { label: "Test-backed", detail: "Corpus programs cover it." }, + }), + ); + assert.ok(status.includes('tabindex="0"'), status); + assert.ok(status.includes('title="Lowered with Node's options'), status); + assert.ok(!status.includes('role="tooltip"'), status); + assert.ok(!status.includes("aria-describedby"), status); + const stability = renderToStaticMarkup( + createElement(StabilityBadge, { stability: { index: "2", level: 2, text: "Stable", inherited: false } }), + ); + assert.ok(stability.includes('tabindex="0"'), stability); + assert.ok(!stability.includes('role="tooltip"'), stability); + const legend = renderToStaticMarkup(createElement(StatusBadge, { status: "supported" })); + assert.ok(!legend.includes("tabindex"), legend); +}); From 4d706302292d78adc1f31be3e39f9434f6e53ecb Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 12:30:31 -0500 Subject: [PATCH 074/184] Keep a keyboard-focused badge's tooltip open when the pointer leaves Hover and keyboard focus now open the tooltip separately, as the CSS-only tooltip's hover and focus-visible rules did, so moving the pointer off a focused badge no longer unmounts its tooltip or drops aria-describedby. Escape still dismisses it. --- .../components/node-compatibility-matrix.tsx | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/docs/src/components/node-compatibility-matrix.tsx b/docs/src/components/node-compatibility-matrix.tsx index bcc5ece61..af2435668 100644 --- a/docs/src/components/node-compatibility-matrix.tsx +++ b/docs/src/components/node-compatibility-matrix.tsx @@ -55,27 +55,34 @@ const stabilityStyles: Record = { /** A badge's tooltip, mounted only while the badge is hovered or has * keyboard focus (the :focus-visible rule the CSS-only tooltip used), and - * named by aria-describedby while open. Escape dismisses it. */ + * named by aria-describedby while open. Hover and focus are tracked apart, + * so moving the pointer away keeps a keyboard-focused badge's tooltip. + * Escape dismisses it until the next hover or focus. */ function BadgeTooltip({ tooltipClass, children, badge }: { tooltipClass: string; children: ReactNode; badge: ReactNode }) { - const [open, setOpen] = useState(false); + const [hovered, setHovered] = useState(false); + const [focused, setFocused] = useState(false); + const open = hovered || focused; const id = useId(); - const show = useCallback(() => setOpen(true), []); - const hide = useCallback(() => setOpen(false), []); + const enter = useCallback(() => setHovered(true), []); + const leave = useCallback(() => setHovered(false), []); const focus = useCallback((event: FocusEvent) => { - if (event.currentTarget.matches(":focus-visible")) setOpen(true); + if (event.currentTarget.matches(":focus-visible")) setFocused(true); }, []); + const blur = useCallback(() => setFocused(false), []); const key = useCallback((event: KeyboardEvent) => { - if (event.key === "Escape") setOpen(false); + if (event.key !== "Escape") return; + setHovered(false); + setFocused(false); }, []); return ( {badge} From cda584260f660821d30912ea7ff911de8dda65de Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 09:52:26 -0700 Subject: [PATCH 075/184] Run typed arrays that the checker types any on their native bytes A JavaScript binding referenced by its own initializer's inference collapses to `any` in the checker while it still lowers to a Uint8Array: hono's etag digest declares `const nextChunk = new Uint8Array(new ArrayBuffer(f(chunk)))` beside `chunk = nextChunk`. The typed-array lowerings key on the checker type and declined such a receiver, and the dynamic paths required a dyn value, so `nextChunk.set(...)` and `nextChunk.byteLength` threw SC1090/SC2020 at run time and multi-chunk etag digests failed. Method calls and property reads on such a receiver now box it by reference (the box shares the native storage, so writes stay visible) and dispatch on the runtime kind. Co-Authored-By: Claude Opus 5.5 --- .../lowering/expressions/property-access.ts | 19 ++++++ .../src/frontend/lowering/lower-calls.ts | 8 +++ .../7748-npm-any-bytes-receiver/main.ts.json | 6 ++ .../7748-npm-any-bytes-receiver/main.ts | 33 ++++++++++ .../node_modules/chunk-buffers/index.d.ts | 1 + .../node_modules/chunk-buffers/index.js | 18 ++++++ .../node_modules/chunk-buffers/package.json | 1 + .../node_modules/etag-digest/index.d.ts | 1 + .../node_modules/etag-digest/index.js | 60 +++++++++++++++++++ .../node_modules/etag-digest/package.json | 1 + .../7748-npm-any-bytes-receiver/main.ts.json | 1 + 11 files changed, 149 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7748-npm-any-bytes-receiver/main.ts.json create mode 100644 tests/corpus/7748-npm-any-bytes-receiver/main.ts create mode 100644 tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/index.d.ts create mode 100644 tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/index.js create mode 100644 tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/package.json create mode 100644 tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/index.d.ts create mode 100644 tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/index.js create mode 100644 tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/package.json create mode 100644 tests/harness/corpus-costs/7748-npm-any-bytes-receiver/main.ts.json diff --git a/packages/compiler/src/frontend/lowering/expressions/property-access.ts b/packages/compiler/src/frontend/lowering/expressions/property-access.ts index 7b12297a8..b0c7a66bb 100644 --- a/packages/compiler/src/frontend/lowering/expressions/property-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/property-access.ts @@ -1499,6 +1499,25 @@ export function lowerPropertyAccessExpr( ) { return afterReceiver(recvLowered, dynUndefinedExpr(loc), loc); } + // A typed array under a checker `any` (a JS binding its own initializer's + // inference refers to: hono's etag digest `nextChunk`) reads through its + // box, as the method-call twin dispatches. The box shares the native + // storage, so it needs no fresh-result restriction. + if ( + recvLowered.type.kind === "bytes" && + (lowerer.typeOf(expr.expression).flags & ts.TypeFlags.Any) !== 0 + ) { + return lowerer.maybeNarrow( + { + kind: "dynKeyGet", + value: lowerer.coerceToExpected(recvLowered, DYN), + key: { kind: "strLit", value: expr.name.text, type: STRING, loc }, + type: DYN, + loc, + }, + expr, + ); + } lowerer.unsupported( "SC1090", expr, diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 672801e9e..6779cde82 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -3755,6 +3755,14 @@ function lowerDynReceiverMethodCall( // 'unknown' slot does. if (checkerUntyped && DYN_HANDLE_KINDS.has(recv.type.kind)) recv = lowerer.coerceToExpected(recv, DYN); + // A typed array whose checker type collapsed to `any` (a JS binding + // referenced in its own initializer's inference — hono's etag digest + // `const nextChunk = new Uint8Array(new ArrayBuffer(f(chunk)))` beside + // `chunk = nextChunk`) still lowered to its native bytes. The typed + // lowerings key on the checker type and decline it, so it boxes by + // reference (the box shares the native storage, so writes through the + // call stay visible) and the call dispatches on the runtime kind. + if (checkerUntyped && recv.type.kind === "bytes") recv = lowerer.coerceToExpected(recv, DYN); // The last-resort retry (boxTypedTemporary — every other lowering // already declined): a checker-`any` receiver whose value is a FRESH // typed result of an earlier step in the chain (`out.split("\n")` diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7748-npm-any-bytes-receiver/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7748-npm-any-bytes-receiver/main.ts.json new file mode 100644 index 000000000..89db88211 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7748-npm-any-bytes-receiver/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7748-npm-any-bytes-receiver/main.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7748-npm-any-bytes-receiver/main.ts b/tests/corpus/7748-npm-any-bytes-receiver/main.ts new file mode 100644 index 000000000..58301765a --- /dev/null +++ b/tests/corpus/7748-npm-any-bytes-receiver/main.ts @@ -0,0 +1,33 @@ +// Typed arrays the checker types `any` in package JavaScript: hono's etag +// digest (hono/dist/middleware/etag/digest.js) merges multi-chunk bodies +// through `nextChunk.set(...)`, where `nextChunk`'s initializer depends on +// `chunk`, which is assigned `nextChunk`. Method calls and property reads on +// such values run on the native bytes, as in Node. +import { generateDigest } from "etag-digest"; +import { grow } from "chunk-buffers"; + +const enc = new TextEncoder(); +const sha1 = (body: Uint8Array): Promise => crypto.subtle.digest({ name: "SHA-1" }, body); + +function chunked(parts: string[]): ReadableStream { + let i = 0; + return new ReadableStream({ + pull(controller): void { + if (i < parts.length) controller.enqueue(enc.encode(parts[i++]!)); + else controller.close(); + }, + }); +} + +async function main(): Promise { + console.log("single", await generateDigest(chunked([JSON.stringify({ books: 400 })]), sha1)); + console.log("chunked", await generateDigest(chunked(["alpha\n", "beta\n", "gamma\n", "delta\n"]), sha1)); + const large = ["x".repeat(200000), "y".repeat(100000), "z".repeat(5)]; + console.log("large", await generateDigest(chunked(large), sha1)); + console.log("exact", await generateDigest(chunked(["a".repeat(262144), "b"]), sha1)); + console.log("empty", await generateDigest(chunked([]), sha1)); + console.log("null", await generateDigest(null, sha1)); + console.log(grow([new Uint8Array([1, 2, 3]), new Uint8Array([4]), new Uint8Array([5])])); +} + +await main(); diff --git a/tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/index.d.ts b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/index.d.ts new file mode 100644 index 000000000..26fe364a0 --- /dev/null +++ b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/index.d.ts @@ -0,0 +1 @@ +export declare function grow(parts: Uint8Array[]): Uint8Array | undefined; diff --git a/tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/index.js b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/index.js new file mode 100644 index 000000000..a0a1fea79 --- /dev/null +++ b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/index.js @@ -0,0 +1,18 @@ +// `next` is referenced by its own initializer's inference (through +// `chunk = next`), so the checker types it `any` while it is a Uint8Array. +export function grow(parts) { + let chunk; + for (const value of parts) { + if (!chunk) chunk = value.slice(0); + else { + const next = new Uint8Array(new ArrayBuffer(Math.max(8, chunk.byteLength * 2))); + next.set(chunk.subarray(0, 2)); + next.set([9], 2); + next[3] = next.byteLength; + console.log(next.length, next[0], next.subarray(1, 3), next.at(-1), next.byteOffset); + console.log(next.slice(2, 4), next.buffer.byteLength, next.BYTES_PER_ELEMENT); + chunk = next; + } + } + return chunk; +} diff --git a/tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/package.json b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/package.json new file mode 100644 index 000000000..74e354d34 --- /dev/null +++ b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/chunk-buffers/package.json @@ -0,0 +1 @@ +{ "name": "chunk-buffers", "version": "1.0.0", "type": "module", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/index.d.ts b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/index.d.ts new file mode 100644 index 000000000..79d9dc396 --- /dev/null +++ b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/index.d.ts @@ -0,0 +1 @@ +export declare const generateDigest: (stream: ReadableStream | null, generator: (body: Uint8Array) => ArrayBuffer | Promise) => Promise; diff --git a/tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/index.js b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/index.js new file mode 100644 index 000000000..bf438a5ea --- /dev/null +++ b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/index.js @@ -0,0 +1,60 @@ +// hono/dist/middleware/etag/digest.js (hono 4.13.13) +const mergeBuffers = (buffer1, buffer2) => { + if (!buffer1) return buffer2; + const merged = new Uint8Array(new ArrayBuffer(buffer1.byteLength + buffer2.byteLength)); + merged.set(new Uint8Array(buffer1), 0); + merged.set(buffer2, buffer1.byteLength); + return merged; +}; +const CHUNK_SIZE = 262144; +const generateDigest = async (stream, generator) => { + if (!stream) return null; + let result = void 0; + let chunk; + let chunkLength = 0; + const digest = async (body) => { + result = await generator(mergeBuffers(result, body)); + }; + const reader = stream.getReader(); + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + let offset = 0; + while (offset < value.byteLength) { + const remaining = value.byteLength - offset; + if (chunkLength === 0 && remaining >= CHUNK_SIZE) { + await digest(value.subarray(offset, offset + CHUNK_SIZE)); + offset += CHUNK_SIZE; + continue; + } + const requiredLength = chunkLength + remaining; + if (requiredLength < CHUNK_SIZE) { + if (!chunk) chunk = value.slice(offset); + else { + if (chunk.byteLength < requiredLength) { + const nextChunk = new Uint8Array(new ArrayBuffer(Math.min(CHUNK_SIZE, Math.max(requiredLength, chunk.byteLength * 2)))); + nextChunk.set(chunk.subarray(0, chunkLength)); + chunk = nextChunk; + } + chunk.set(value.subarray(offset), chunkLength); + } + chunkLength = requiredLength; + break; + } + const length = CHUNK_SIZE - chunkLength; + if (chunk?.byteLength !== CHUNK_SIZE) { + const nextChunk = new Uint8Array(/* @__PURE__ */ new ArrayBuffer(CHUNK_SIZE)); + if (chunk) nextChunk.set(chunk.subarray(0, chunkLength)); + chunk = nextChunk; + } + chunk.set(value.subarray(offset, offset + length), chunkLength); + await digest(chunk); + chunkLength = 0; + offset += length; + } + } + if (chunk && chunkLength > 0) await digest(chunk.subarray(0, chunkLength)); + if (!result) return null; + return Array.prototype.map.call(new Uint8Array(result), (x) => x.toString(16).padStart(2, "0")).join(""); +}; +export { generateDigest }; diff --git a/tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/package.json b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/package.json new file mode 100644 index 000000000..f15495eed --- /dev/null +++ b/tests/corpus/7748-npm-any-bytes-receiver/node_modules/etag-digest/package.json @@ -0,0 +1 @@ +{ "name": "etag-digest", "version": "1.0.0", "type": "module", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/harness/corpus-costs/7748-npm-any-bytes-receiver/main.ts.json b/tests/harness/corpus-costs/7748-npm-any-bytes-receiver/main.ts.json new file mode 100644 index 000000000..197263cad --- /dev/null +++ b/tests/harness/corpus-costs/7748-npm-any-bytes-receiver/main.ts.json @@ -0,0 +1 @@ +{ "plain": 4600, "san": 7300 } From f14997776b804884e9d71be36c690c772e44ab93 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 09:52:30 -0700 Subject: [PATCH 076/184] Box TextEncoder and TextDecoder records as checked-dynamic handles A codec stored in untyped storage (hono's StreamingApi keeps `this.encoder = new TextEncoder()` in a JavaScript class field) had no dynamic representation and threw SC1101 at run time. The codec records now cross 'unknown' slots as handles of two new tags that hold the record itself, so identity and a decoder's streaming state survive the round trip, and checked casts unbox the same record. The handles answer encode, encodeInto and decode (with options objects), the encoding, fatal and ignoreBOM properties, method values with Node's ERR_INVALID_THIS, instanceof, String() tags and JSON's {}. The unit scr_text_codec_dyn.c links only into programs that convert a codec to or from an untyped value (the textCodecDyn runtime feature, found with the dgram carrier walk, now shared). Co-Authored-By: Claude Opus 5.5 --- .../builtin-apis/text-encoding.mdx | 1 + .../generated/node-v24-internal.json | 20 +- internal/compatibility/static-support.json | 4 +- .../src/backend/executable-features.ts | 1 + packages/compiler/src/backend/library-pack.ts | 1 + .../src/backend/llvm/dyn-handle-tags.test.ts | 8 +- packages/compiler/src/backend/llvm/dyn.ts | 122 ++++++++++ .../compiler/src/backend/llvm/expr-dynamic.ts | 6 +- .../src/backend/llvm/expr-stream-bridges.ts | 14 +- .../compiler/src/backend/llvm/lib-shared.ts | 1 + .../src/backend/native-link-info-core.ts | 3 + .../compiler/src/backend/native/contracts.ts | 4 + .../compiler/src/backend/native/executable.ts | 1 + .../src/backend/runtime-pack-targets.test.ts | 1 + .../compiler/src/backend/runtime-pack.test.ts | 1 + packages/compiler/src/backend/warm-cache.ts | 1 + .../src/executable/executable-cache.test.ts | 1 + .../src/executable/executable-cache.ts | 2 + .../lowering/expressions/relational.ts | 16 ++ packages/compiler/src/index.ts | 1 + packages/compiler/src/ir/analysis.ts | 7 +- .../compiler/src/ir/builtin-signatures.ts | 4 + packages/compiler/src/ir/ir.ts | 16 ++ packages/compiler/src/ir/runtime-features.ts | 110 +++++++-- packages/compiler/src/native/cli.ts | 1 + packages/compiler/src/native/prepare.ts | 1 + .../compiler/src/native/sanitizer.test.ts | 1 + .../7747-text-codec-untyped-slot.js.json | 6 + .../7749-text-codec-unknown-roundtrip.ts.json | 6 + .../runtime-pack-matrix.mjs | 2 + packages/runtime/src/scr_bytes.h | 12 + packages/runtime/src/scr_dyn.h | 2 + packages/runtime/src/scr_text_codec_dyn.c | 215 ++++++++++++++++++ tests/corpus/7747-text-codec-untyped-slot.js | 69 ++++++ .../7749-text-codec-unknown-roundtrip.ts | 22 ++ .../7747-text-codec-untyped-slot.js.json | 1 + .../7749-text-codec-unknown-roundtrip.ts.json | 1 + .../self-hosting-native-toolchain.test.ts | 1 + .../harness/self-hosting-runtime-pack.test.ts | 1 + 39 files changed, 650 insertions(+), 37 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7747-text-codec-untyped-slot.js.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7749-text-codec-unknown-roundtrip.ts.json create mode 100644 packages/runtime/src/scr_text_codec_dyn.c create mode 100644 tests/corpus/7747-text-codec-untyped-slot.js create mode 100644 tests/corpus/7749-text-codec-unknown-roundtrip.ts create mode 100644 tests/harness/corpus-costs/7747-text-codec-untyped-slot.js.json create mode 100644 tests/harness/corpus-costs/7749-text-codec-unknown-roundtrip.ts.json diff --git a/docs/content/limitations/builtin-apis/text-encoding.mdx b/docs/content/limitations/builtin-apis/text-encoding.mdx index bffd24424..e20cfd943 100644 --- a/docs/content/limitations/builtin-apis/text-encoding.mdx +++ b/docs/content/limitations/builtin-apis/text-encoding.mdx @@ -1,6 +1,7 @@ - **Text encoding:** `TextEncoder` and `TextDecoder` can be stored in fields, arguments, return values, arrays, and closures. Encoding supports UTF-8, including `encodeInto` into a Uint8Array or Buffer view. Codec properties expose the canonical encoding and decoder flags. + Codecs held in `any` or `unknown` storage keep their identity and streaming state, and dynamic code can call `encode`, `encodeInto` and `decode`, with options objects, and read their properties. Decoding accepts recognized WHATWG labels, including runtime labels and omitted or undefined encodings. Literal constructor options support `fatal` and `ignoreBOM`; literal decode options support streaming UTF-8, UTF-16, and single-byte encodings. Other legacy streaming encodings, runtime options objects, inspection, JSON conversion, and module-scope declarations using `var` or lacking an initializer are unsupported. diff --git a/internal/compatibility/generated/node-v24-internal.json b/internal/compatibility/generated/node-v24-internal.json index ba8692b49..f2b528d7a 100644 --- a/internal/compatibility/generated/node-v24-internal.json +++ b/internal/compatibility/generated/node-v24-internal.json @@ -52194,6 +52194,7 @@ "tests/corpus/6294-webcrypto-js-values.mjs", "tests/corpus/6295-jose-hs256/main.mjs", "tests/corpus/6331-jose-root-entry/main.ts", + "tests/corpus/7747-text-codec-untyped-slot.js", "tests/corpus/text-codec-values/main.ts", "tests/corpus/webassembly-api.mjs", "tests/corpus/webassembly-dylink/main.mjs", @@ -53346,7 +53347,8 @@ "tests": [ "tests/corpus/1423-text-codec.ts", "tests/corpus/text-codec-values/main.ts", - "tests/corpus/3133-text-decoder-streaming.ts" + "tests/corpus/3133-text-decoder-streaming.ts", + "tests/corpus/7747-text-codec-untyped-slot.js" ] }, "anchorSource": "exact" @@ -53397,7 +53399,8 @@ "tests": [ "tests/corpus/1423-text-codec.ts", "tests/corpus/text-codec-values/main.ts", - "tests/corpus/3133-text-decoder-streaming.ts" + "tests/corpus/3133-text-decoder-streaming.ts", + "tests/corpus/7747-text-codec-untyped-slot.js" ] }, "anchorSource": "exact" @@ -101277,6 +101280,7 @@ "tests/corpus/5835-util-format-integer-specifiers.ts", "tests/corpus/5884-inspect-layout-options.ts", "tests/corpus/6428-promisify-callback-functions.ts", + "tests/corpus/7747-text-codec-untyped-slot.js", "tests/corpus/node24-deep-equality-properties.js", "tests/corpus/node24-deep-equality.cjs", "tests/corpus/node24-deep-equality.ts", @@ -101805,7 +101809,8 @@ "tests/corpus/text-codec-values/main.ts", "tests/fixtures/node-types/text-codecs.mts", "tests/harness/project-config.test.ts", - "tests/corpus/3133-text-decoder-streaming.ts" + "tests/corpus/3133-text-decoder-streaming.ts", + "tests/corpus/7747-text-codec-untyped-slot.js" ] }, "anchorSource": "exact" @@ -101926,7 +101931,8 @@ "tests/corpus/text-codec-values/main.ts", "tests/fixtures/node-types/text-codecs.mts", "tests/harness/project-config.test.ts", - "tests/corpus/3133-text-decoder-streaming.ts" + "tests/corpus/3133-text-decoder-streaming.ts", + "tests/corpus/7747-text-codec-untyped-slot.js" ] }, "anchorSource": "exact" @@ -102036,7 +102042,8 @@ "tests/corpus/text-codec-values/main.ts", "tests/fixtures/node-types/text-codecs.mts", "tests/harness/project-config.test.ts", - "tests/corpus/3133-text-decoder-streaming.ts" + "tests/corpus/3133-text-decoder-streaming.ts", + "tests/corpus/7747-text-codec-untyped-slot.js" ] }, "anchorSource": "exact" @@ -102065,7 +102072,8 @@ "tests/corpus/text-codec-values/main.ts", "tests/fixtures/node-types/text-codecs.mts", "tests/harness/project-config.test.ts", - "tests/corpus/3133-text-decoder-streaming.ts" + "tests/corpus/3133-text-decoder-streaming.ts", + "tests/corpus/7747-text-codec-untyped-slot.js" ] }, "anchorSource": "exact" diff --git a/internal/compatibility/static-support.json b/internal/compatibility/static-support.json index 100882c1a..5950be0a5 100644 --- a/internal/compatibility/static-support.json +++ b/internal/compatibility/static-support.json @@ -332,11 +332,11 @@ { "chapter": "buffer", "signatures": ["`buf.readBigInt64BE([offset])`", "`buf.readBigInt64LE([offset])`", "`buf.readBigUInt64BE([offset])`", "`buf.readBigUInt64LE([offset])`", "`buf.writeBigInt64BE(value[, offset])`", "`buf.writeBigInt64LE(value[, offset])`", "`buf.writeBigUInt64BE(value[, offset])`", "`buf.writeBigUInt64LE(value[, offset])`"], "status": "supported", "evidence": ["tests/corpus/2902-bigint-bytes.ts", "tests/corpus/2904-bigint-errors.ts"] }, { "chapter": "environment_variables", "symbols": ["export"], "status": "not-applicable" }, { "chapter": "globals", "symbols": ["fetch"], "status": "partial", "evidence": ["tests/harness/fetch-conformance.test.ts"] }, - { "chapter": "globals", "signatures": ["Class: `TextEncoder`", "Class: `TextDecoder`"], "status": "partial", "evidence": ["tests/corpus/1423-text-codec.ts", "tests/corpus/text-codec-values/main.ts", "tests/corpus/3133-text-decoder-streaming.ts"] }, + { "chapter": "globals", "signatures": ["Class: `TextEncoder`", "Class: `TextDecoder`"], "status": "partial", "evidence": ["tests/corpus/1423-text-codec.ts", "tests/corpus/text-codec-values/main.ts", "tests/corpus/3133-text-decoder-streaming.ts", "tests/corpus/7747-text-codec-untyped-slot.js"] }, { "chapter": "globals", "signatures": ["Class: `Blob`", "Class: `File`"], "status": "partial", "evidence": ["tests/corpus/5285-blob-construct.ts", "tests/corpus/5286-blob-readers.ts", "tests/corpus/5287-blob-inspect.ts", "tests/corpus/5288-blob-validation.js", "tests/corpus/5290-blob-dynamic.js"] }, { "chapter": "buffer", "signatures": ["Class: `Blob`", "Class: `File`"], "status": "partial", "evidence": ["tests/corpus/5285-blob-construct.ts", "tests/corpus/5288-blob-validation.js", "tests/fixtures/node-types/blob-file.ts"] }, { "chapter": "buffer", "signatures": ["`blob.arrayBuffer()`", "`blob.bytes()`", "`blob.text()`", "`blob.slice([start[, end[, type]]])`", "`blob.size`", "`type` Type: {string}", "`name` Type: {string}", "`lastModified` Type: {number}"], "status": "supported", "evidence": ["tests/corpus/5285-blob-construct.ts", "tests/corpus/5286-blob-readers.ts", "tests/corpus/5288-blob-validation.js", "tests/corpus/5289-blob-read-order.ts", "tests/corpus/5290-blob-dynamic.js", "tests/fixtures/node-types/blob-file.ts"] }, - { "chapter": "util", "signatures": ["Class: `util.TextEncoder`", "Class: `util.TextDecoder`", "`textEncoder.encode([input])`", "`textDecoder.decode([input[, options]])`"], "status": "partial", "evidence": ["tests/corpus/text-codec-values/main.ts", "tests/fixtures/node-types/text-codecs.mts", "tests/harness/project-config.test.ts", "tests/corpus/3133-text-decoder-streaming.ts"] }, + { "chapter": "util", "signatures": ["Class: `util.TextEncoder`", "Class: `util.TextDecoder`", "`textEncoder.encode([input])`", "`textDecoder.decode([input[, options]])`"], "status": "partial", "evidence": ["tests/corpus/text-codec-values/main.ts", "tests/fixtures/node-types/text-codecs.mts", "tests/harness/project-config.test.ts", "tests/corpus/3133-text-decoder-streaming.ts", "tests/corpus/7747-text-codec-untyped-slot.js"] }, { "chapter": "modules", "symbols": ["__dirname", "__filename", "exports", "require", "module.require", "main", "filename"], "status": "partial", "evidence": ["tests/corpus/1612-cjs-module-globals.cjs", "tests/corpus/1596-cjs-modules/main.js", "tests/corpus/1629-require-main-filename.cjs", "tests/corpus/2830-cjs-require-self-name/main.cjs", "tests/corpus/2880-cjs-module-graph/main.cjs", "tests/corpus/2923-cjs-factory-export/main.cjs"] }, { "chapter": "modules", "symbols": ["require.resolve", "require.resolve.paths"], "status": "partial", "evidence": ["tests/corpus/2858-import-meta-resolve/main.mjs", "tests/corpus/2860-require-resolve/main.cjs"] }, { "chapter": "modules", "symbols": ["cache", "children", "id", "isPreloading", "loaded", "parent", "path", "paths", "Module"], "status": "partial", "evidence": ["tests/corpus/2880-cjs-module-graph/main.cjs"] }, diff --git a/packages/compiler/src/backend/executable-features.ts b/packages/compiler/src/backend/executable-features.ts index 32318d53e..a9076af22 100644 --- a/packages/compiler/src/backend/executable-features.ts +++ b/packages/compiler/src/backend/executable-features.ts @@ -50,6 +50,7 @@ export function executableLinkFeatures(mod: IrModule): NativeLinkFeatures { dgram: features.dgram, dnsPromises: features.dnsPromises, dgramDyn: features.dgramDyn, + textCodecDyn: features.textCodecDyn, watch: features.fsWatch, foreignFfi: hasForeignFfiCallback(mod.ffiImports ?? []), nodeTest: features.nodeTest, diff --git a/packages/compiler/src/backend/library-pack.ts b/packages/compiler/src/backend/library-pack.ts index e9f52f7a0..79be316db 100644 --- a/packages/compiler/src/backend/library-pack.ts +++ b/packages/compiler/src/backend/library-pack.ts @@ -54,6 +54,7 @@ export async function compilePackedLibrary( dgram: false, dnsPromises: false, dgramDyn: false, + textCodecDyn: false, watch: false, foreignFfi: false, nodeTest: false, diff --git a/packages/compiler/src/backend/llvm/dyn-handle-tags.test.ts b/packages/compiler/src/backend/llvm/dyn-handle-tags.test.ts index c015b85d0..8c3e5de80 100644 --- a/packages/compiler/src/backend/llvm/dyn-handle-tags.test.ts +++ b/packages/compiler/src/backend/llvm/dyn-handle-tags.test.ts @@ -1,7 +1,7 @@ import { readFileSync } from "node:fs"; import { expect, test } from "vitest"; import { DYN_HANDLE_KINDS } from "../../ir/ir.js"; -import { DYN_HANDLE_TAG_NUM } from "./dyn.js"; +import { DYN_HANDLE_TAG_NUM, TEXT_CODEC_TAG_NUM } from "./dyn.js"; // SCR_DYNH_TAGS in scr_dyn.h lists the ScrDynHandleTag values by name, each // with an explicit value, so changes that add tags edit different lines. The @@ -39,3 +39,9 @@ test("the backend's handle tag numbers match scr_dyn.h", () => { expect(values.get(tag!), `${kind} (${tag})`).toBe(num); } }); + +test("the backend's codec handle tag numbers match scr_dyn.h", () => { + const values = new Map(tags.map((tag) => [tag.name, tag.value])); + expect(values.get("SCR_DYNH_TEXT_DECODER")).toBe(TEXT_CODEC_TAG_NUM.TextDecoder); + expect(values.get("SCR_DYNH_TEXT_ENCODER")).toBe(TEXT_CODEC_TAG_NUM.TextEncoder); +}); diff --git a/packages/compiler/src/backend/llvm/dyn.ts b/packages/compiler/src/backend/llvm/dyn.ts index 86626d304..3987c52c2 100644 --- a/packages/compiler/src/backend/llvm/dyn.ts +++ b/packages/compiler/src/backend/llvm/dyn.ts @@ -43,6 +43,8 @@ import { isDynTypedRefType, isRefCounted, isUnitType, + recordTextCodecClass, + textCodecRecordClass, typeKey, } from "../../ir/ir.js"; import { dynDesc, streamTypedRefEligible } from "../../ir/analysis.js"; @@ -94,6 +96,17 @@ export const DYN_HANDLE_TAG_NUM: Record = { cryptoHmac: 53, }; +/** The handle tags of TextEncoder and TextDecoder records boxed as dyn + * (scr_dyn.h; scr_text_codec_dyn.c boxes and unboxes them). */ +export const TEXT_CODEC_TAG_NUM: Record<"TextEncoder" | "TextDecoder", number> = { + TextDecoder: 76, + TextEncoder: 77, +}; + +/** The decoder record fields scr_text_decoder_to_dyn reads, in its + * parameter order. */ +const TEXT_DECODER_FIELDS = ["%TextDecoder", "%fatal", "%ignoreBOM", "%state"] as const; + /** Handle kinds whose runtime unit boxes them through its own function, * which installs the tag's ops on first use (scr_crypto.h). Programs that * never box one of these values do not link its dynamic members. */ @@ -927,6 +940,21 @@ export class LlDyn { const name = `sc_dm_${this.dynMatchers.size}`; this.dynMatchers.set(matcherKey, name); this.host.journalAllocated?.(name, "dyn.dynMatchHelper", [t, preserveRefs]); + const codec = textCodecRecordClass(t, (id) => this.host.recordsById.get(id)); + if (codec !== null) { + // A codec matches only its own handle box. + const B = new BlockBuilder(); + const matched = this.handleTagMatches(B, TEXT_CODEC_TAG_NUM[codec]); + B.terminate(`ret i1 ${matched}`); + this.defs.push( + `define internal zeroext i1 @${name}(ptr %d) ${FN_ATTRS} { ; matches ${key}`, + B.render(), + `}`, + ``, + ); + this.host.journalBuilt?.(); + return name; + } const recursive = this.isRecursiveShape(t); const wrapped = recursive || this.holdsRecursiveShape(t); const B = new BlockBuilder(); @@ -1643,6 +1671,24 @@ export class LlDyn { this.host.journalBuilt?.(); return name; } + const codec = textCodecRecordClass(t, (id) => host.recordsById.get(id)); + if (codec !== null) { + // The record the codec's handle box holds (+1), or the path TypeError. + host.declare(`declare ptr @scr_text_codec_from_dyn(ptr, i1 zeroext, ptr, ptr)`); + const checked = B.tmp(); + B.line( + `${checked} = call ptr @scr_text_codec_from_dyn(ptr %d, i1 ${codec === "TextDecoder"}, ptr %path, ptr ${want})`, + ); + B.terminate(`ret ptr ${checked}`); + this.defs.push( + `define internal ptr @${name}(ptr %d, ptr %path) ${FN_ATTRS} {`, + B.render(), + `}`, + ``, + ); + this.host.journalBuilt?.(); + return name; + } if (isRefCounted(t) && t.kind !== "dyn") { host.declare(`declare ptr @scr_dyn_typed_ref_unbox(ptr)`); if (t.kind === "object" && t.className === "%Error") { @@ -2671,6 +2717,77 @@ export class LlDyn { return name; } + /** `i1`: %d is a handle box with tag `tag`. */ + private handleTagMatches(B: BlockBuilder, tag: number): string { + const kd = this.kindOf(B, "%d"); + const isHandle = B.tmp(); + B.line(`${isHandle} = icmp eq i32 ${kd}, ${DYN_KIND.HANDLE}`); + const lHandle = B.newLabel("dm.handle"); + const lDone = B.newLabel("dm.handle.done"); + const entry = B.currentLabel(); + B.condBr(isHandle, lHandle, lDone); + B.startBlock(lHandle); + const tagPtr = B.tmp(); + const loaded = B.tmp(); + B.line( + `${tagPtr} = getelementptr inbounds i8, ptr %d, i64 ${this.abiOffset(24, 20)} ; ->v.handle.tag`, + ); + B.line(`${loaded} = load i32, ptr ${tagPtr}`); + const same = B.tmp(); + B.line(`${same} = icmp eq i32 ${loaded}, ${tag}`); + B.br(lDone); + B.startBlock(lDone); + const matched = B.tmp(); + B.line(`${matched} = phi i1 [ false, %${entry} ], [ ${same}, %${lHandle} ]`); + return matched; + } + + /** A TextEncoder or TextDecoder record boxes as its handle (+1), which + * holds the record by reference. A decoder passes the byte offsets of the + * fields its boxed decode reads. */ + private textCodecToDyn( + B: BlockBuilder, + t: IrType & { kind: "record" }, + shape: IrRecordShape, + codec: "TextEncoder" | "TextDecoder", + ): void { + const host = this.host; + const rc = vAdapters(host, t); + const r = B.tmp(); + if (codec === "TextEncoder") { + host.declare(`declare ptr @scr_text_encoder_to_dyn(ptr, ptr, ptr)`); + B.line( + `${r} = call ptr @scr_text_encoder_to_dyn(ptr %v, ptr ${rc.retain}, ptr ${rc.release})`, + ); + } else { + const struct = mangleRecordStruct(t.shapeId); + const base = host.nullableUnions.fieldBase(t.shapeId); + const start = B.tmp(); + B.line(`${start} = ptrtoint ptr %v to ${this.S}`); + const offsets = TEXT_DECODER_FIELDS.map((field) => { + const index = shape.fields.findIndex((f) => f.name === field); + if (index < 0) + throw new InternalCompilerError(`llvm emitter bug: TextDecoder record lacks ${field}`); + const p = B.tmp(); + const at = B.tmp(); + const offset = B.tmp(); + B.line( + `${p} = getelementptr inbounds %${struct}, ptr %v, i64 0, i32 ${index + base} ; .${llvmCommentText(field)}`, + ); + B.line(`${at} = ptrtoint ptr ${p} to ${this.S}`); + B.line(`${offset} = sub ${this.S} ${at}, ${start}`); + return `${this.S} ${offset}`; + }); + host.declare( + `declare ptr @scr_text_decoder_to_dyn(ptr, ptr, ptr, ${this.S}, ${this.S}, ${this.S}, ${this.S})`, + ); + B.line( + `${r} = call ptr @scr_text_decoder_to_dyn(ptr %v, ptr ${rc.retain}, ptr ${rc.release}, ${offsets.join(", ")})`, + ); + } + B.terminate(`ret ptr ${r}`); + } + /* ── toDynHelper (walkers.ts, ported) ─────────────────────────── */ /** `sc_td_( v) -> ptr` — build a fresh dyn value from a @@ -2846,6 +2963,11 @@ export class LlDyn { const shape = host.recordsById.get(t.shapeId); if (!shape) throw new InternalCompilerError(`llvm emitter bug: to-dyn of unknown shape ${t.shapeId}`); + const codec = recordTextCodecClass(shape); + if (codec !== null) { + this.textCodecToDyn(B, t, shape, codec); + break; + } const struct = mangleRecordStruct(t.shapeId); const base = host.nullableUnions.fieldBase(t.shapeId); const fieldIndex = new Map(shape.fields.map((f, i) => [f.name, i + base])); diff --git a/packages/compiler/src/backend/llvm/expr-dynamic.ts b/packages/compiler/src/backend/llvm/expr-dynamic.ts index 07a538184..26905a7ef 100644 --- a/packages/compiler/src/backend/llvm/expr-dynamic.ts +++ b/packages/compiler/src/backend/llvm/expr-dynamic.ts @@ -18,6 +18,7 @@ import { type IrExpr, type IrType, isDynTypedRefType, + textCodecRecordClass, isRefCounted, isUnitType, typeEquals, @@ -86,7 +87,10 @@ export function emitDynamicExpr( // Bytes already box their shared mutable storage directly. A second // capsule would split identity between checked and native views. const errorRef = v.type.kind === "object" && host.shapeHost.isErrorClass(v.type.className); - if (!errorRef && ((e.liveRef && v.type.kind !== "bytes") || identityRef)) { + // A TextEncoder or TextDecoder record boxes as its own handle, which + // keeps the record's identity (toDynHelper). + const codecRef = textCodecRecordClass(v.type, (id) => host.recordsById.get(id)) !== null; + if (!errorRef && !codecRef && ((e.liveRef && v.type.kind !== "bytes") || identityRef)) { if (v.type.kind === "union") { const adapter = host.liveDynUnionRefAdapter(v.type); const boxed = B.tmp(); diff --git a/packages/compiler/src/backend/llvm/expr-stream-bridges.ts b/packages/compiler/src/backend/llvm/expr-stream-bridges.ts index aee7be180..a2f7208ec 100644 --- a/packages/compiler/src/backend/llvm/expr-stream-bridges.ts +++ b/packages/compiler/src/backend/llvm/expr-stream-bridges.ts @@ -16,6 +16,7 @@ import { KEY_ORDER_FIELD, shapeKeyOrders, isDynTypedRefType, + textCodecRecordClass, isRefCounted, shapeHasAccessorSlots, typeKey, @@ -597,12 +598,21 @@ export function liveDynUnionRefAdapter( if (!union) { throw new InternalCompilerError(`llvm emitter bug: live dyn ref of unknown union ${t.unionId}`); } + // Codec records box as their own handles, by reference, through toDynHelper. + const codecArm = (arm: IrType): boolean => + textCodecRecordClass(arm, (id) => host.recordsById.get(id)) !== null; const mutableArms = union.arms .map((arm, tag) => ({ arm, tag })) .filter( - ({ arm }) => arm.kind !== "bytes" && (streamTypedRefEligible(arm) || isDynTypedRefType(arm)), + ({ arm }) => + arm.kind !== "bytes" && + !codecArm(arm) && + (streamTypedRefEligible(arm) || isDynTypedRefType(arm)), ); - if (mutableArms.length === 0 && !union.arms.some((arm) => arm.kind === "bytes")) { + if ( + mutableArms.length === 0 && + !union.arms.some((arm) => arm.kind === "bytes" || codecArm(arm)) + ) { throw new InternalCompilerError(`llvm emitter bug: live dyn ref of immutable union ${key}`); } diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index af32c6936..7b411f2bb 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -560,6 +560,7 @@ export const LIB_FN_SYMS: Record = { "text.decoderEncoding": "scr_text_decoder_encoding", "text.decoderName": "scr_text_decoder_name", "text.encodeInto": "scr_text_encode_into", + "text.codecIs": "scr_text_codec_is", "buffer.isAscii": "scr_buffer_is_ascii", "buffer.isUtf8": "scr_buffer_is_utf8", "buffer.transcode": "scr_buffer_transcode", diff --git a/packages/compiler/src/backend/native-link-info-core.ts b/packages/compiler/src/backend/native-link-info-core.ts index e018037fd..c073bb9a6 100644 --- a/packages/compiler/src/backend/native-link-info-core.ts +++ b/packages/compiler/src/backend/native-link-info-core.ts @@ -81,6 +81,9 @@ export interface NativeLinkFeatures { dnsPromises: boolean; /** dgram sockets as untyped values: scr_dgram_dyn.c. */ dgramDyn: boolean; + /** TextEncoder and TextDecoder records as untyped values: + * scr_text_codec_dyn.c. */ + textCodecDyn: boolean; watch: boolean; foreignFfi: boolean; nodeTest: boolean; diff --git a/packages/compiler/src/backend/native/contracts.ts b/packages/compiler/src/backend/native/contracts.ts index 61de82e72..53e160984 100644 --- a/packages/compiler/src/backend/native/contracts.ts +++ b/packages/compiler/src/backend/native/contracts.ts @@ -220,6 +220,10 @@ export interface CcOptions { * scr_dgram_dyn.c over the dgram unit, so typed dgram binaries keep * their exact link line. */ dgramDyn?: boolean; + /** A TextEncoder or TextDecoder record converts to or from an untyped + * value: compiles scr_text_codec_dyn.c, so typed codec binaries keep + * their exact link line. */ + textCodecDyn?: boolean; /** The program uses fs.watch (moduleUsesFsWatch on the IR): compiles * scr_watch.c into the binary — the net gating precedent, so watch-free diff --git a/packages/compiler/src/backend/native/executable.ts b/packages/compiler/src/backend/native/executable.ts index 233b9cac2..1578a0c80 100644 --- a/packages/compiler/src/backend/native/executable.ts +++ b/packages/compiler/src/backend/native/executable.ts @@ -302,6 +302,7 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [ unit((u) => u.opts.dgram, "scr_dgram.c"), unit((u) => u.opts.dgram && u.opts.dnsPromises, "scr_dns_promises.c"), unit((u) => u.opts.dgram && u.opts.dgramDyn, "scr_dgram_dyn.c"), + unit((u) => u.opts.textCodecDyn, "scr_text_codec_dyn.c"), unit((u) => u.opts.watch, "scr_watch.c"), unit((u) => u.opts.foreignFfi, "scr_ffi_queue.c"), unit((u) => u.opts.nodeTest, "scr_test.c"), diff --git a/packages/compiler/src/backend/runtime-pack-targets.test.ts b/packages/compiler/src/backend/runtime-pack-targets.test.ts index 44c3d6290..9846e8d89 100644 --- a/packages/compiler/src/backend/runtime-pack-targets.test.ts +++ b/packages/compiler/src/backend/runtime-pack-targets.test.ts @@ -118,6 +118,7 @@ const features = { dgram: false, dnsPromises: false, dgramDyn: false, + textCodecDyn: false, watch: false, foreignFfi: false, nodeTest: false, diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index e3bc999a7..e2b24f8ca 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -54,6 +54,7 @@ const BASE: NativeLinkFeatures = { dgram: false, dnsPromises: false, dgramDyn: false, + textCodecDyn: false, watch: false, foreignFfi: false, nodeTest: false, diff --git a/packages/compiler/src/backend/warm-cache.ts b/packages/compiler/src/backend/warm-cache.ts index b48cf6739..02acb5462 100644 --- a/packages/compiler/src/backend/warm-cache.ts +++ b/packages/compiler/src/backend/warm-cache.ts @@ -61,6 +61,7 @@ export async function warmNativeCaches( dgram: false, dnsPromises: false, dgramDyn: false, + textCodecDyn: false, watch: false, foreignFfi: false, nodeTest: false, diff --git a/packages/compiler/src/executable/executable-cache.test.ts b/packages/compiler/src/executable/executable-cache.test.ts index 5cf8c6588..e066c0412 100644 --- a/packages/compiler/src/executable/executable-cache.test.ts +++ b/packages/compiler/src/executable/executable-cache.test.ts @@ -58,6 +58,7 @@ const native: EarlyExecutableNativeFeatures = { dgram: false, dnsPromises: false, dgramDyn: false, + textCodecDyn: false, watch: false, foreignFfi: false, nodeTest: false, diff --git a/packages/compiler/src/executable/executable-cache.ts b/packages/compiler/src/executable/executable-cache.ts index 7b7febc06..96922ca13 100644 --- a/packages/compiler/src/executable/executable-cache.ts +++ b/packages/compiler/src/executable/executable-cache.ts @@ -92,6 +92,7 @@ export interface EarlyExecutableNativeFeatures { dgram: boolean; dnsPromises: boolean; dgramDyn: boolean; + textCodecDyn: boolean; watch: boolean; foreignFfi: boolean; nodeTest: boolean; @@ -235,6 +236,7 @@ const BOOLEAN_NATIVE_KEYS = [ "dgram", "dnsPromises", "dgramDyn", + "textCodecDyn", "watch", "foreignFfi", "nodeTest", diff --git a/packages/compiler/src/frontend/lowering/expressions/relational.ts b/packages/compiler/src/frontend/lowering/expressions/relational.ts index f1590e9cc..1651bf0d9 100644 --- a/packages/compiler/src/frontend/lowering/expressions/relational.ts +++ b/packages/compiler/src/frontend/lowering/expressions/relational.ts @@ -338,6 +338,22 @@ export function lowerInstanceOf(lowerer: Lowerer, expr: ts.BinaryExpression, loc type: BOOL, loc, }; + // A codec in untyped storage is its record's handle box. + const codec = (["TextEncoder", "TextDecoder"] as const).find((name) => + lowerer.isStdlibGlobal(expr.right, name), + ); + if (codec && lowerer.mapTypeOf(lowerer.typeOf(expr.left))?.kind === "dyn") { + return { + kind: "libCall", + fn: "text.codecIs", + args: [ + lowerer.lowerExprExpecting(expr.left, DYN), + { kind: "boolLit", value: codec === "TextDecoder", type: BOOL, loc }, + ], + type: BOOL, + loc, + }; + } if (lowerer.isStdlibGlobal(expr.right, "ReadableStream")) { const value = lowerer.lowerExprExpecting(expr.left, DYN); return { kind: "libCall", fn: "fetch.streamIs", args: [value], type: BOOL, loc }; diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 4f7b93cfd..ae93598d6 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -623,6 +623,7 @@ async function compileExecutableNative( dgram: features.dgram, dnsPromises: features.dnsPromises, dgramDyn: features.dgramDyn, + textCodecDyn: features.textCodecDyn, watch: features.watch, foreignFfi: features.foreignFfi, ...(features.workers ? { workers: true } : {}), diff --git a/packages/compiler/src/ir/analysis.ts b/packages/compiler/src/ir/analysis.ts index 70ec381a1..ee1cee6be 100644 --- a/packages/compiler/src/ir/analysis.ts +++ b/packages/compiler/src/ir/analysis.ts @@ -4,6 +4,7 @@ import { BYTES_ELEMENT_NAME, DYN_HANDLE_KINDS, RUNTIME_STREAM_CLASSES, + recordTextCodecClass, typeEquals, type IrExpr, type IrLibFn, @@ -141,8 +142,10 @@ export function dynDesc( return "string"; case "bool": return "boolean"; - case "record": - return recordsById.get(t.shapeId)?.tuple ? "array" : "object"; + case "record": { + const shape = recordsById.get(t.shapeId); + return shape?.tuple ? "array" : ((shape && recordTextCodecClass(shape)) ?? "object"); + } case "array": return "array"; case "nullT": diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 1be0f95cd..9f5b3334f 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -4852,6 +4852,10 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "text.decoderEncoding": { argTypes: [DYN], result: F64 }, "text.decoderName": { argTypes: [F64], result: STRING }, "text.encodeInto": { argTypes: [DYN, DYN], result: DYN }, + /** `value instanceof TextDecoder` (true) or `TextEncoder` (false) on a + * checked-dynamic value: a codec record's handle box (scr_text_codec_dyn.c). + * Borrowed; never throws. */ + "text.codecIs": { argTypes: [DYN, BOOL], result: BOOL }, "buffer.isAscii": { argTypes: [DYN], result: BOOL }, "buffer.isUtf8": { argTypes: [DYN], result: BOOL }, "buffer.transcode": { argTypes: [DYN, DYN, DYN], result: BYTES_U8 }, diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 878088d65..29dda102d 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -1745,6 +1745,18 @@ export function recordTextCodecClass(shape: IrRecordShape): "TextEncoder" | "Tex return null; } +/** The codec class of a TextEncoder or TextDecoder record type, else null. + * Such a record crosses an `unknown` slot as a native handle that holds the + * record itself (scr_text_codec_dyn.c), never as a copy or a field view. */ +export function textCodecRecordClass( + t: IrType, + getRecord: (shapeId: string) => IrRecordShape | undefined, +): "TextEncoder" | "TextDecoder" | null { + if (t.kind !== "record") return null; + const shape = getRecord(t.shapeId); + return shape ? recordTextCodecClass(shape) : null; +} + /** Object-literal ACCESSOR properties (`{ get x() {...}, set x(v) {...} }`) * live on the shape as reserved '%'-fields holding closures: `%get:x` a * `() => T` invoked once per property READ (side effects and all — JS's @@ -4273,6 +4285,7 @@ export function canConvertToDyn( visiting: Set = new Set(), ): boolean { if (isJsonSafeType(t, getRecord, getUnion)) return true; + if (textCodecRecordClass(t, getRecord) !== null) return true; // numeric typed arrays and boxable functions are dyn kinds the walker boxes // ANYWHERE (bytes and functions held by identity), including nested // in records/arrays/unions. isJsonSafeType rejects them, but dynFrom @@ -4319,6 +4332,7 @@ export function canConvertToDyn( // either its result or a promise of it. canBoxDynComposite(a, getRecord, getUnion, visiting) || isDynTypedRefType(a) || + textCodecRecordClass(a, getRecord) !== null || a.kind === "classval" || DYN_HANDLE_KINDS.has(a.kind) || (a.kind === "func" && canBoxFuncIntoDyn(a, getRecord, getUnion, visiting)) || @@ -4499,6 +4513,8 @@ function canDynCheckToAt( visiting.delete(key); return result; } + // A codec handle unboxes to the record it holds. + if (textCodecRecordClass(t, getRecord) !== null) return true; if (t.kind === "record") { const key = typeKey(t); if (visiting.has(key)) return false; diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index 4ad3dcc5c..e1e7e28be 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -2,6 +2,7 @@ import type { IrClassDef, IrExpr, IrModule, IrRecordShape, IrType, IrUnionDef } import type { IrLibFn } from "./builtin-signatures.js"; import { isDirentShape, + recordTextCodecClass, RUNTIME_EMITTER_CLASS, RUNTIME_EVENT_CLASS, RUNTIME_EVENT_TARGET_CLASS, @@ -415,7 +416,10 @@ interface TypeTables { unions: Map; classes: Map; /** mentionsDgramSocket's carriers, computed on first use. */ - dgramCarriers?: DgramCarriers; + dgramCarriers?: TypeCarriers; + /** mentionsTextCodec's and mentionsTextDecoder's carriers. */ + textCodecCarriers?: TypeCarriers; + textDecoderCarriers?: TypeCarriers; } /** Whether boxing a value of type `t` as dyn boxes a promise, which goes @@ -479,30 +483,42 @@ function boxesPromise(t: IrType, tables: TypeTables, seen: Set): boolean } } -/** The record shapes, unions and classes whose values carry a dgram socket - * (directly or through other members), computed once per module. */ -interface DgramCarriers { +/** The record shapes, unions and classes whose values carry a leaf type + * (directly or through other members), computed once per module and leaf. */ +interface TypeCarriers { records: Set; unions: Set; anyClass: boolean; } -/** Whether `t` carries a dgram socket, given the carrier sets so far: - * structural through functions (parameters and result), promises and - * collections; named records, unions and class instances by the sets. */ -function carriesDgram(t: IrType, carriers: DgramCarriers): boolean { +/** The leaf a carrier set tracks: a dgram socket, a codec record. */ +type CarrierLeaf = (t: IrType, tables: TypeTables) => boolean; + +/** Whether `t` carries a leaf, given the carrier sets so far: structural + * through functions (parameters and result), promises and collections; + * named records, unions and class instances by the sets. */ +function carriesLeaf( + t: IrType, + carriers: TypeCarriers, + leaf: CarrierLeaf, + tables: TypeTables, +): boolean { + if (leaf(t, tables)) return true; switch (t.kind) { - case "dgramSocket": - return true; case "func": - return t.params.some((p) => carriesDgram(p, carriers)) || carriesDgram(t.ret, carriers); + return ( + t.params.some((p) => carriesLeaf(p, carriers, leaf, tables)) || + carriesLeaf(t.ret, carriers, leaf, tables) + ); case "promise": - return carriesDgram(t.inner, carriers); + return carriesLeaf(t.inner, carriers, leaf, tables); case "array": case "set": - return carriesDgram(t.elem, carriers); + return carriesLeaf(t.elem, carriers, leaf, tables); case "map": - return carriesDgram(t.key, carriers) || carriesDgram(t.value, carriers); + return ( + carriesLeaf(t.key, carriers, leaf, tables) || carriesLeaf(t.value, carriers, leaf, tables) + ); case "record": return carriers.records.has(t.shapeId); case "union": @@ -515,25 +531,29 @@ function carriesDgram(t: IrType, carriers: DgramCarriers): boolean { } } -/** The module's dgram carriers: a fixpoint over its records, unions and +/** The module's carriers of `leaf`: a fixpoint over its records, unions and * class fields. */ -function dgramCarriers(tables: TypeTables): DgramCarriers { - const carriers: DgramCarriers = { records: new Set(), unions: new Set(), anyClass: false }; +function leafCarriers(tables: TypeTables, leaf: CarrierLeaf): TypeCarriers { + const carriers: TypeCarriers = { records: new Set(), unions: new Set(), anyClass: false }; + const carries = (t: IrType): boolean => carriesLeaf(t, carriers, leaf, tables); for (let changed = true; changed;) { changed = false; for (const [id, shape] of tables.records) - if (!carriers.records.has(id) && shape.fields.some((f) => carriesDgram(f.type, carriers))) { + if ( + !carriers.records.has(id) && + (leaf({ kind: "record", shapeId: id }, tables) || shape.fields.some((f) => carries(f.type))) + ) { carriers.records.add(id); changed = true; } for (const [id, def] of tables.unions) - if (!carriers.unions.has(id) && def.arms.some((a) => carriesDgram(a, carriers))) { + if (!carriers.unions.has(id) && def.arms.some(carries)) { carriers.unions.add(id); changed = true; } if ( !carriers.anyClass && - [...tables.classes.values()].some((c) => c.fields.some((f) => carriesDgram(f.type, carriers))) + [...tables.classes.values()].some((c) => c.fields.some((f) => carries(f.type))) ) { carriers.anyClass = true; changed = true; @@ -542,13 +562,42 @@ function dgramCarriers(tables: TypeTables): DgramCarriers { return carriers; } +const dgramLeaf: CarrierLeaf = (t) => t.kind === "dgramSocket"; + /** Whether converting a value of type `t` to or from dyn converts a dgram * socket, which goes through scr_dgram_dyn.c: the socket itself, or a * function, collection, record, union, promise or class instance that * carries one (boxed functions convert their parameters too). */ function mentionsDgramSocket(t: IrType, tables: TypeTables): boolean { - tables.dgramCarriers ??= dgramCarriers(tables); - return carriesDgram(t, tables.dgramCarriers); + tables.dgramCarriers ??= leafCarriers(tables, dgramLeaf); + return carriesLeaf(t, tables.dgramCarriers, dgramLeaf, tables); +} + +const textCodecLeaf: CarrierLeaf = (t, tables) => { + if (t.kind !== "record") return false; + const shape = tables.records.get(t.shapeId); + return shape !== undefined && recordTextCodecClass(shape) !== null; +}; + +const textDecoderLeaf: CarrierLeaf = (t, tables) => { + if (t.kind !== "record") return false; + const shape = tables.records.get(t.shapeId); + return shape !== undefined && recordTextCodecClass(shape) === "TextDecoder"; +}; + +/** Whether converting a value of type `t` to or from dyn converts a + * TextEncoder or TextDecoder record, which boxes it as a handle of + * scr_text_codec_dyn.c (the same carrier walk as dgram sockets). */ +function mentionsTextCodec(t: IrType, tables: TypeTables): boolean { + tables.textCodecCarriers ??= leafCarriers(tables, textCodecLeaf); + return carriesLeaf(t, tables.textCodecCarriers, textCodecLeaf, tables); +} + +/** The TextDecoder half: its boxed decode runs scr_text_decode_stream, + * which reaches the legacy decoders. */ +function mentionsTextDecoder(t: IrType, tables: TypeTables): boolean { + tables.textDecoderCarriers ??= leafCarriers(tables, textDecoderLeaf); + return carriesLeaf(t, tables.textDecoderCarriers, textDecoderLeaf, tables); } /** The libCall fns served by the OPTIONAL events unit (scr_events.c): @@ -794,6 +843,15 @@ const RUNTIME_FEATURE_RULES = { (node.kind === "dynFrom" && mentionsDgramSocket(node.value.type, tables())) || (node.kind === "dynCheck" && mentionsDgramSocket(node.type, tables())), }), + /** A TextEncoder or TextDecoder record converts to or from an untyped + * value (its handle box, or a boxed function, record or collection that + * carries one): scr_text_codec_dyn.c. */ + textCodecDyn: rule({ + fns: ["text.codecIs"], + expr: (node, tables) => + (node.kind === "dynFrom" && mentionsTextCodec(node.value.type, tables())) || + (node.kind === "dynCheck" && mentionsTextCodec(node.type, tables())), + }), /** node:dns/promises: scr_dns_promises.c over the dgram unit. */ dnsPromises: rule({ fns: ["dns.promisesLookup", "dns.promisesLookupService"] }), dynAsync: rule({ @@ -942,7 +1000,12 @@ const RUNTIME_FEATURE_RULES = { prefixes: ["insp.", "console.", "test."], libCall: (node) => node.fn === "rl.terminal" && !isDynUndefined(node.args[3]!), }), - legacyTextDecoder: rule({ fns: [...LEGACY_TEXT_DECODER_LIB_FNS] }), + legacyTextDecoder: rule({ + fns: [...LEGACY_TEXT_DECODER_LIB_FNS], + expr: (node, tables) => + (node.kind === "dynFrom" && mentionsTextDecoder(node.value.type, tables())) || + (node.kind === "dynCheck" && mentionsTextDecoder(node.type, tables())), + }), /** The program calls mock.timers members: its timer calls go through * scr_test.c's mock-aware entry points. */ mockTimers: rule({ prefixes: ["test.mockTimers", "test.ctxMockTimers"] }), @@ -1056,6 +1119,7 @@ function noRuntimeFeatures(): RuntimeFeatures { dc: false, dgram: false, dgramDyn: false, + textCodecDyn: false, dnsPromises: false, dynAsync: false, dynBuiltinClass: false, diff --git a/packages/compiler/src/native/cli.ts b/packages/compiler/src/native/cli.ts index 21047b929..a35ce811f 100644 --- a/packages/compiler/src/native/cli.ts +++ b/packages/compiler/src/native/cli.ts @@ -122,6 +122,7 @@ async function main(): Promise { dgram: false, dnsPromises: false, dgramDyn: false, + textCodecDyn: false, watch: false, foreignFfi: false, nodeTest: false, diff --git a/packages/compiler/src/native/prepare.ts b/packages/compiler/src/native/prepare.ts index f7f399f80..a238029dc 100644 --- a/packages/compiler/src/native/prepare.ts +++ b/packages/compiler/src/native/prepare.ts @@ -239,6 +239,7 @@ function validFeatures(value: unknown): value is NativeLinkFeatures { "dgram", "dnsPromises", "dgramDyn", + "textCodecDyn", "watch", "foreignFfi", "nodeTest", diff --git a/packages/compiler/src/native/sanitizer.test.ts b/packages/compiler/src/native/sanitizer.test.ts index 4c031a8e7..33a4c19b9 100644 --- a/packages/compiler/src/native/sanitizer.test.ts +++ b/packages/compiler/src/native/sanitizer.test.ts @@ -114,6 +114,7 @@ test.skipIf(process.platform !== "linux" && process.platform !== "darwin")( dgram: false, dnsPromises: false, dgramDyn: false, + textCodecDyn: false, watch: false, foreignFfi: false, nodeTest: false, diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7747-text-codec-untyped-slot.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7747-text-codec-untyped-slot.js.json new file mode 100644 index 000000000..23efb7db6 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7747-text-codec-untyped-slot.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7747-text-codec-untyped-slot.js" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7749-text-codec-unknown-roundtrip.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7749-text-codec-unknown-roundtrip.ts.json new file mode 100644 index 000000000..dd9f068f3 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7749-text-codec-unknown-roundtrip.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7749-text-codec-unknown-roundtrip.ts" + ], + "diags": [] +} diff --git a/packages/runtime-pack-common/runtime-pack-matrix.mjs b/packages/runtime-pack-common/runtime-pack-matrix.mjs index 89b6eaf19..0ab467236 100644 --- a/packages/runtime-pack-common/runtime-pack-matrix.mjs +++ b/packages/runtime-pack-common/runtime-pack-matrix.mjs @@ -148,6 +148,8 @@ export const OPTIONAL_RUNTIME_SOURCES = [ ["scr_dns_promises.c", all("dgram", "dnsPromises")], // Sockets as untyped values run over the dgram unit. ["scr_dgram_dyn.c", all("dgram", "dgramDyn")], + // TextEncoder and TextDecoder records boxed as untyped values. + ["scr_text_codec_dyn.c", "textCodecDyn"], ["scr_watch.c", "watch"], ["scr_ffi_queue.c", "foreignFfi"], ["scr_test.c", "nodeTest"], diff --git a/packages/runtime/src/scr_bytes.h b/packages/runtime/src/scr_bytes.h index 9b382dda5..2e4b44409 100644 --- a/packages/runtime/src/scr_bytes.h +++ b/packages/runtime/src/scr_bytes.h @@ -357,6 +357,18 @@ ScrStr *scr_text_decode(const ScrBytes *b); double scr_text_decoder_encoding(const ScrDyn *label); ScrStr *scr_text_decoder_name(double encoding); ScrDyn *scr_text_encode_into(const ScrDyn *source, const ScrDyn *destination); +/* TextEncoder and TextDecoder records as checked-dynamic handles + * (scr_text_codec_dyn.c, linked by the textCodecDyn runtime feature). The + * boxes borrow the record and retain it through `retain`; a decoder also + * names the byte offsets of its encoding (f64), fatal and ignoreBOM (i8) + * and state (dyn) fields. from_dyn answers the retained record or throws + * the path-annotated TypeError and answers NULL. */ +ScrDyn *scr_text_encoder_to_dyn(void *rec, void *(*retain)(void *), void (*release)(void *)); +ScrDyn *scr_text_decoder_to_dyn(void *rec, void *(*retain)(void *), void (*release)(void *), + size_t encoding, size_t fatal, size_t ignore_bom, size_t state); +void *scr_text_codec_from_dyn(const ScrDyn *d, bool decoder, const struct ScrDynPath *path, + const char *want); +bool scr_text_codec_is(const ScrDyn *d, bool decoder); bool scr_buffer_is_ascii(const ScrDyn *input); bool scr_buffer_is_utf8(const ScrDyn *input); ScrBytes *scr_buffer_transcode(const ScrDyn *source, const ScrDyn *from, const ScrDyn *to); diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 6f0c28371..740e33a09 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -200,6 +200,8 @@ typedef enum { X(SHARED_ARRAY_BUFFER, 37) \ X(SIGN, 67) /* node:crypto Sign (scr_pki.c) */ \ X(STDIO, 27) /* stable process stdin/stdout/stderr values */ \ + X(TEXT_DECODER, 76) /* compiled TextDecoder record (scr_text_codec_dyn.c) */ \ + X(TEXT_ENCODER, 77) /* compiled TextEncoder record (scr_text_codec_dyn.c) */ \ X(URL, 28) /* native WHATWG URL */ \ X(VERIFY, 68) /* node:crypto Verify (scr_pki.c) */ \ X(WASM_GLOBAL, 61) /* WebAssembly.Global */ \ diff --git a/packages/runtime/src/scr_text_codec_dyn.c b/packages/runtime/src/scr_text_codec_dyn.c new file mode 100644 index 000000000..5102bc558 --- /dev/null +++ b/packages/runtime/src/scr_text_codec_dyn.c @@ -0,0 +1,215 @@ +/* WHATWG TextEncoder and TextDecoder instances as checked-dynamic values + * (SCR_DYNH_TEXT_ENCODER, SCR_DYNH_TEXT_DECODER). A codec is a compiled + * record (type-mapper.ts): the handle holds that record by reference, so + * a codec stored in an untyped slot and read back keeps its identity and + * a decoder keeps its streaming state. The compiler passes the record's + * reference-count entry points and, for decoders, the byte offsets of the + * encoding, fatal, ignoreBOM and state fields when it boxes one. A unit + * of its own so programs that never box a codec link none of it + * (native-toolchain gating: the textCodecDyn runtime feature). */ +#include "scr_platform.h" +#include "scr_runtime.h" + +#include + +typedef struct { + void *(*retain)(void *); + void (*release)(void *); + size_t encoding, fatal, ignore_bom, state; +} ScrTextCodecLayout; + +/* A program has one record shape per codec class, so one layout each. */ +static SCR_TL ScrTextCodecLayout scr_tcd_encoder_layout, scr_tcd_decoder_layout; + +static void *scr_tcd_encoder_retain(void *rec) { return scr_tcd_encoder_layout.retain(rec); } +static void scr_tcd_encoder_release(void *rec) { scr_tcd_encoder_layout.release(rec); } +static void *scr_tcd_decoder_retain(void *rec) { return scr_tcd_decoder_layout.retain(rec); } +static void scr_tcd_decoder_release(void *rec) { scr_tcd_decoder_layout.release(rec); } + +static void scr_tcd_codec_not_a_function(const char *what) { + ScrJsonBuf b; + scr_jb_init(&b); + scr_jb_puts(&b, what); + scr_jb_puts(&b, " is not a function"); + scr_throw_error(SCR_ERR_TYPE, scr_jb_finish(&b)); +} + +/* Boxes an owned string (+1 box; the string's reference moves in). */ +static ScrDyn *scr_tcd_str(ScrStr *text) { + ScrDyn *boxed = scr_dyn_new_str(text); + scr_str_release(text); + return boxed; +} + +static ScrDyn *scr_tcd_codec_arg(ScrDyn *const *args, size_t argc, size_t index) { + return index < argc ? args[index] : scr_dyn_undefined(); +} + +static double scr_tcd_decoder_encoding(const void *rec) { + double encoding; + memcpy(&encoding, (const char *)rec + scr_tcd_decoder_layout.encoding, sizeof encoding); + return encoding; +} + +static bool scr_tcd_decoder_flag(const void *rec, size_t offset) { + return ((const unsigned char *)rec)[offset] != 0; +} + +static ScrDyn *scr_tcd_encoder_invoke(void *rec, ScrDyn *self, const char *method, + ScrDyn *const *args, size_t argc, const char *what) { + (void)rec; + (void)self; + if (strcmp(method, "encode") == 0) { + /* encode(input = ""): any other input converts with ToString. */ + ScrDyn *input = scr_tcd_codec_arg(args, argc, 0); + ScrStr *text = + input->kind == SCR_DYN_UNDEF ? scr_str_new("", 0) : scr_dyn_string_coerce_js(input); + if (!text) return NULL; + ScrStr *utf8 = scr_str_new("utf8", 4); + ScrBytes *bytes = scr_bytes_from_str(text, utf8); + scr_str_release(utf8); + scr_str_release(text); + /* A fresh Uint8Array, not a Buffer. */ + bytes->is_buffer = false; + ScrDyn *result = scr_dyn_new_bytes(bytes); + scr_bytes_release(bytes); + return result; + } + if (strcmp(method, "encodeInto") == 0) + return scr_text_encode_into(scr_tcd_codec_arg(args, argc, 0), + scr_tcd_codec_arg(args, argc, 1)); + scr_tcd_codec_not_a_function(what); + return NULL; +} + +static ScrDyn *scr_tcd_encoder_get(void *rec, const char *key, size_t len) { + (void)rec; + if (len == 8 && memcmp(key, "encoding", 8) == 0) return scr_tcd_str(scr_str_new("utf-8", 5)); + return NULL; +} + +/* decode(input, options): the options object's `stream` member, read + * with JS truthiness; undefined and null mean no options. */ +static bool scr_tcd_decoder_stream_option(const ScrDyn *options, bool *stream) { + *stream = false; + if (options->kind == SCR_DYN_UNDEF || options->kind == SCR_DYN_NULL) return true; + if (options->kind != SCR_DYN_OBJ && options->kind != SCR_DYN_ARR && + options->kind != SCR_DYN_FUNC && options->kind != SCR_DYN_TYPED_REF && + options->kind != SCR_DYN_HANDLE) { + scr_dyn_arg_type_fail("options", "of type object", options); + return false; + } + ScrDyn *value = scr_dyn_obj_read(options, "stream", 6); + if (!value) return false; + *stream = scr_dyn_truthy(value); + scr_dyn_release(value); + return !scr_exc_pending(); +} + +static ScrDyn *scr_tcd_decoder_invoke(void *rec, ScrDyn *self, const char *method, + ScrDyn *const *args, size_t argc, const char *what) { + (void)self; + if (strcmp(method, "decode") != 0) { + scr_tcd_codec_not_a_function(what); + return NULL; + } + ScrBytes *input = scr_bytes_buffer_source(scr_tcd_codec_arg(args, argc, 0)); + if (!input) return NULL; + bool stream; + if (!scr_tcd_decoder_stream_option(scr_tcd_codec_arg(args, argc, 1), &stream)) { + scr_bytes_release(input); + return NULL; + } + const ScrTextCodecLayout *layout = &scr_tcd_decoder_layout; + ScrDyn *state; + memcpy(&state, (const char *)rec + layout->state, sizeof state); + ScrStr *text = scr_text_decode_stream(state, input, scr_tcd_decoder_encoding(rec), + scr_tcd_decoder_flag(rec, layout->fatal), + scr_tcd_decoder_flag(rec, layout->ignore_bom), stream); + scr_bytes_release(input); + return text ? scr_tcd_str(text) : NULL; +} + +static ScrDyn *scr_tcd_decoder_get(void *rec, const char *key, size_t len) { + if (len == 8 && memcmp(key, "encoding", 8) == 0) + return scr_tcd_str(scr_text_decoder_name(scr_tcd_decoder_encoding(rec))); + if (len == 5 && memcmp(key, "fatal", 5) == 0) + return scr_dyn_new_bool(scr_tcd_decoder_flag(rec, scr_tcd_decoder_layout.fatal)); + if (len == 9 && memcmp(key, "ignoreBOM", 9) == 0) + return scr_dyn_new_bool(scr_tcd_decoder_flag(rec, scr_tcd_decoder_layout.ignore_bom)); + return NULL; +} + +static bool scr_tcd_codec_set(void *rec, const char *key, size_t len, const ScrDyn *value) { + (void)rec; + (void)key; + (void)len; + (void)value; + return false; +} + +static const ScrDynHandleMethod scr_tcd_encoder_methods[] = { + {.name = "encode", .length = 0}, + {.name = "encodeInto", .length = 2}, + {.name = NULL}, +}; + +static const ScrDynHandleMethod scr_tcd_decoder_methods[] = { + {.name = "decode", .length = 0}, + {.name = NULL}, +}; + +static const ScrDynHandleOps scr_tcd_encoder_ops = { + .cls = "TextEncoder", + .to_string_tag = "TextEncoder", + .retain = scr_tcd_encoder_retain, + .release = scr_tcd_encoder_release, + .invoke = scr_tcd_encoder_invoke, + .get = scr_tcd_encoder_get, + .set = scr_tcd_codec_set, + .json_empty_object = true, + .methods = scr_tcd_encoder_methods, + .invalid_this = "Value of \"this\" must be of type TextEncoder", +}; + +static const ScrDynHandleOps scr_tcd_decoder_ops = { + .cls = "TextDecoder", + .to_string_tag = "TextDecoder", + .retain = scr_tcd_decoder_retain, + .release = scr_tcd_decoder_release, + .invoke = scr_tcd_decoder_invoke, + .get = scr_tcd_decoder_get, + .set = scr_tcd_codec_set, + .json_empty_object = true, + .methods = scr_tcd_decoder_methods, + .invalid_this = "Value of \"this\" must be of type TextDecoder", +}; + +ScrDyn *scr_text_encoder_to_dyn(void *rec, void *(*retain)(void *), void (*release)(void *)) { + if (!scr_tcd_encoder_layout.retain) { + scr_tcd_encoder_layout.retain = retain; + scr_tcd_encoder_layout.release = release; + scr_dyn_handle_install(SCR_DYNH_TEXT_ENCODER, &scr_tcd_encoder_ops); + } + return scr_dyn_new_handle(rec, SCR_DYNH_TEXT_ENCODER); +} + +ScrDyn *scr_text_decoder_to_dyn(void *rec, void *(*retain)(void *), void (*release)(void *), + size_t encoding, size_t fatal, size_t ignore_bom, size_t state) { + if (!scr_tcd_decoder_layout.retain) { + scr_tcd_decoder_layout = (ScrTextCodecLayout){retain, release, encoding, fatal, ignore_bom, state}; + scr_dyn_handle_install(SCR_DYNH_TEXT_DECODER, &scr_tcd_decoder_ops); + } + return scr_dyn_new_handle(rec, SCR_DYNH_TEXT_DECODER); +} + +void *scr_text_codec_from_dyn(const ScrDyn *d, bool decoder, const ScrDynPath *path, + const char *want) { + return scr_dyn_handle_unbox(d, decoder ? SCR_DYNH_TEXT_DECODER : SCR_DYNH_TEXT_ENCODER, path, + want); +} + +bool scr_text_codec_is(const ScrDyn *d, bool decoder) { + return d->kind == SCR_DYN_HANDLE && + d->v.handle.tag == (decoder ? SCR_DYNH_TEXT_DECODER : SCR_DYNH_TEXT_ENCODER); +} diff --git a/tests/corpus/7747-text-codec-untyped-slot.js b/tests/corpus/7747-text-codec-untyped-slot.js new file mode 100644 index 000000000..cdf8345da --- /dev/null +++ b/tests/corpus/7747-text-codec-untyped-slot.js @@ -0,0 +1,69 @@ +// TextEncoder and TextDecoder instances held in untyped storage (hono's +// StreamingApi keeps `this.encoder = new TextEncoder()` in a JavaScript +// class field) and called from dynamic code, with their identity, streaming +// state, properties, method values and argument errors. +class StreamingApi { + /** @type {any} */ encoder; + /** @type {any} */ decoder; + /** @type {any} */ latin; + /** @type {any} */ strict; + constructor() { + this.encoder = new TextEncoder(); + this.decoder = new TextDecoder(); + this.latin = new TextDecoder("latin1"); + this.strict = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + } + write(input) { + if (typeof input === "string") input = this.encoder.encode(input); + return input; + } +} + +const show = (error) => console.log(error.name, error.code); +const api = new StreamingApi(); +console.log(api.write("héllo 😀")); +console.log(api.encoder.encoding, typeof api.encoder, api.encoder === api.encoder); +console.log(String(api.encoder), JSON.stringify(api.encoder), JSON.stringify({ d: api.decoder })); +console.log(api.encoder.encode(), api.encoder.encode(42), api.encoder.encode(undefined)); +const dest = new Uint8Array(4); +console.log(api.encoder.encodeInto("héllo", dest), dest); +try { + api.encoder.encodeInto(1, dest); +} catch (error) { + show(error); +} + +console.log(api.decoder.encoding, api.decoder.fatal, api.decoder.ignoreBOM); +console.log(api.latin.encoding, api.strict.fatal, api.strict.ignoreBOM); +const euro = api.encoder.encode("€uro"); +const head = api.decoder.decode(euro.subarray(0, 2), { stream: true }); +console.log(JSON.stringify(head), api.decoder.decode(euro.subarray(2))); +console.log(api.latin.decode(new Uint8Array([0xe9, 0x41]))); +console.log(JSON.stringify(api.strict.decode(new Uint8Array([0xef, 0xbb, 0xbf, 0x41])))); +try { + api.strict.decode(new Uint8Array([0xff])); +} catch (error) { + show(error); +} +try { + api.decoder.decode("text"); +} catch (error) { + show(error); +} +console.log(JSON.stringify(api.decoder.decode()), JSON.stringify(api.decoder.decode(undefined, null))); + +const encode = api.encoder.encode; +try { + encode("x"); +} catch (error) { + console.log(error.name, error.code, error.message); +} +console.log(typeof api.encoder.encode, api.encoder.encode.length, api.encoder.encodeInto.length); +console.log(api.decoder.decode.length, encode === api.encoder.encode); +console.log(api.encoder.encode.call(api.encoder, "ab")); +console.log("encode" in api.encoder, "decode" in api.decoder, api.encoder.missing); + +console.log(api.encoder instanceof TextEncoder, api.decoder instanceof TextDecoder); +console.log(api.encoder instanceof TextDecoder, JSON.parse("{}") instanceof TextEncoder); +const codecs = [api.encoder, api.decoder]; +console.log(codecs.indexOf(api.decoder), codecs.includes(api.encoder)); diff --git a/tests/corpus/7749-text-codec-unknown-roundtrip.ts b/tests/corpus/7749-text-codec-unknown-roundtrip.ts new file mode 100644 index 000000000..14c35d2bb --- /dev/null +++ b/tests/corpus/7749-text-codec-unknown-roundtrip.ts @@ -0,0 +1,22 @@ +// TypeScript codecs crossing 'unknown' slots: boxed by reference and +// checked back, keeping identity and a decoder's streaming state. +const holder: { codec: unknown; decoder: unknown } = { + codec: new TextEncoder(), + decoder: new TextDecoder("utf-16le"), +}; + +const encoder = holder.codec as TextEncoder; +console.log(encoder.encode("typed"), encoder === (holder.codec as TextEncoder)); +const decoder = holder.decoder as TextDecoder; +const bytes = new Uint8Array([0x68, 0x00, 0x69]); +console.log(JSON.stringify(decoder.decode(bytes, { stream: true }))); +console.log(JSON.stringify(decoder.decode(new Uint8Array([0x00]))), decoder.encoding); + +function store(value: unknown): unknown[] { + return [value, value]; +} +const pair = store(encoder); +console.log(pair[0] === pair[1], pair[0] === holder.codec); + +const maybe: unknown = Math.random() > 2 ? 1 : new TextDecoder(); +if (typeof maybe === "object" && maybe !== null) console.log((maybe as TextDecoder).encoding); diff --git a/tests/harness/corpus-costs/7747-text-codec-untyped-slot.js.json b/tests/harness/corpus-costs/7747-text-codec-untyped-slot.js.json new file mode 100644 index 000000000..a537f9ede --- /dev/null +++ b/tests/harness/corpus-costs/7747-text-codec-untyped-slot.js.json @@ -0,0 +1 @@ +{ "plain": 2400, "san": 6500 } diff --git a/tests/harness/corpus-costs/7749-text-codec-unknown-roundtrip.ts.json b/tests/harness/corpus-costs/7749-text-codec-unknown-roundtrip.ts.json new file mode 100644 index 000000000..aa8658007 --- /dev/null +++ b/tests/harness/corpus-costs/7749-text-codec-unknown-roundtrip.ts.json @@ -0,0 +1 @@ +{ "plain": 2200, "san": 3600 } diff --git a/tests/harness/self-hosting-native-toolchain.test.ts b/tests/harness/self-hosting-native-toolchain.test.ts index fb15e80a4..c5476328e 100644 --- a/tests/harness/self-hosting-native-toolchain.test.ts +++ b/tests/harness/self-hosting-native-toolchain.test.ts @@ -41,6 +41,7 @@ const features: NativeLinkFeatures = { dgram: false, dnsPromises: false, dgramDyn: false, + textCodecDyn: false, watch: false, foreignFfi: false, nodeTest: false, diff --git a/tests/harness/self-hosting-runtime-pack.test.ts b/tests/harness/self-hosting-runtime-pack.test.ts index 57857d642..64ddd0213 100644 --- a/tests/harness/self-hosting-runtime-pack.test.ts +++ b/tests/harness/self-hosting-runtime-pack.test.ts @@ -42,6 +42,7 @@ const base: NativeLinkFeatures = { dgram: false, dnsPromises: false, dgramDyn: false, + textCodecDyn: false, watch: false, foreignFfi: false, nodeTest: false, From 934d254a417f48df2264f26c2409104e6ce398d2 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 12:44:35 -0500 Subject: [PATCH 077/184] Keep the inline-cache record list within the shape budget --- packages/runtime/src/scr_js.h | 11 ++++++---- packages/runtime/src/scr_js_object.test.c | 26 +++++++++++++---------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/packages/runtime/src/scr_js.h b/packages/runtime/src/scr_js.h index 6fa20fdb8..1d874b31e 100644 --- a/packages/runtime/src/scr_js.h +++ b/packages/runtime/src/scr_js.h @@ -195,10 +195,13 @@ bool scr_js_truthy(ScrJsVal value); * index and which keys an object lacks. * * Each property site of compiled dynamic-tier code owns one zeroed, - * thread-local ScrJsIc. The code at the site is fixed: it tests entry 0 - * inline and calls a probe for everything else. Probes never run user - * code: they answer from plain data properties or return a miss (NULL / - * false) with nothing done, and the site then runs its generic path. + * thread-local ScrJsIc with static storage: the reset that frees a + * thread's shapes (at thread exit, and when a library session ends) clears + * every record that holds entries. The code at the site is fixed: it tests + * entry 0 inline and calls a probe for everything else. Probes never run + * user code: they answer from plain data properties or return a miss + * (NULL / false) with nothing done, and the site then runs its generic + * path. * * load entry: shape = receiver shape; target = NULL for an own property * (slot indexes the receiver's entries), else the shape of diff --git a/packages/runtime/src/scr_js_object.test.c b/packages/runtime/src/scr_js_object.test.c index a14aa9ff6..6b6934dea 100644 --- a/packages/runtime/src/scr_js_object.test.c +++ b/packages/runtime/src/scr_js_object.test.c @@ -24,11 +24,17 @@ static ScrDyn *object_with(const ScrStr *key, double n) { return object; } +/* Cache records have static storage, as the compiled sites' do: the reset + * at exit clears every record that holds entries. Each check starts from + * an empty record. */ +static ScrJsIc load_ic; +static ScrJsIc store_ic; + /* Whether a load probe answers `object.key` and returns `n`. */ static bool probe_reads(ScrDyn *object, const ScrStr *key, double n) { - ScrJsIc ic; - memset(&ic, 0, sizeof ic); - ScrDyn *value = scr_js_load_probe(object, key, &ic); + ScrJsIc *ic = &load_ic; + memset(ic, 0, sizeof *ic); + ScrDyn *value = scr_js_load_probe(object, key, ic); if (!value) return false; assert(value->kind == SCR_DYN_NUM && value->v.num == n); scr_dyn_release(value); @@ -66,14 +72,13 @@ static void budget_bounds_shapes(size_t key_len, size_t budget) { /* A store probe that would add a new shape leaves the add to the generic * store; one that replaces through an existing shape still runs. */ - ScrJsIc ic; - memset(&ic, 0, sizeof ic); + memset(&store_ic, 0, sizeof store_ic); ScrStr *added = scr_str_new("added", 5); ScrDyn *value = scr_dyn_new_num(8); - assert(!scr_js_store_probe(object, added, value, &ic)); + assert(!scr_js_store_probe(object, added, value, &store_ic)); assert(object->v.obj.len == 1); - memset(&ic, 0, sizeof ic); - assert(scr_js_store_probe(object, first, value, &ic)); + memset(&store_ic, 0, sizeof store_ic); + assert(scr_js_store_probe(object, first, value, &store_ic)); assert(probe_reads(object, first, 8)); scr_dyn_release(value); scr_str_release(added); @@ -90,10 +95,9 @@ static void immortal_objects_stay_unshaped(void) { object->rc = SIZE_MAX; assert(!probe_reads(object, key, 3)); assert(object->v.obj.shape == NULL); - ScrJsIc ic; - memset(&ic, 0, sizeof ic); + memset(&store_ic, 0, sizeof store_ic); ScrDyn *value = scr_dyn_new_num(4); - assert(!scr_js_store_probe(object, key, value, &ic)); + assert(!scr_js_store_probe(object, key, value, &store_ic)); assert(object->v.obj.shape == NULL); scr_dyn_release(value); scr_str_release(key); From c0e24cdb3f6df5a56cf84a2331ab83492f4b1892 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 12:52:57 -0500 Subject: [PATCH 078/184] Embed island modules, bound V8's stack scan in-thread, and keep Node's environment out --- .github/workflows/ci.yml | 6 +- packages/cli/src/bootstrap.ts | 5 +- .../compiler/src/backend/llvm/emit-module.ts | 38 +- packages/compiler/src/backend/llvm/emitter.ts | 5 + .../compiler/src/diagnostics/diagnostic.ts | 15 +- .../src/executable/executable-cache.ts | 17 +- .../compiler/src/frontend/input-tracker.ts | 27 + .../frontend/lowering/lower-node-island.ts | 39 +- .../compiler/src/frontend/lowering/lowerer.ts | 17 +- packages/compiler/src/index.ts | 57 +- .../src/ir/function-references.test.ts | 6 + packages/compiler/src/ir/ir.ts | 20 +- packages/compiler/src/node-islands/libnode.ts | 14 +- .../compiler/src/node-islands/modules.test.ts | 188 ++++++ packages/compiler/src/node-islands/modules.ts | 627 ++++++++++++++++++ packages/compiler/src/startup-cache.ts | 1 + .../7661-node-call-modules/main.ts.json | 6 + .../7662-node-call-exit-events/main.ts.json | 6 + .../libnode-common/bridge/scr_node_bridge.cc | 71 +- packages/libnode-common/test/bridge.test.ts | 2 +- packages/node/README.md | 2 + packages/node/index.d.ts | 2 + packages/runtime/src/scr_node_bridge.h | 12 +- packages/runtime/src/scr_node_island.c | 621 ++++++++++++++++- packages/runtime/src/scr_node_island.h | 9 + packages/runtime/src/scr_node_island_host.cjs | 54 +- packages/runtime/src/scr_node_island_host.h | 54 +- .../7661-node-call-modules/lib/entry.cjs | 14 + .../corpus/7661-node-call-modules/lib/esm.mjs | 5 + .../7661-node-call-modules/lib/factor.mjs | 1 + .../corpus/7661-node-call-modules/lib/util.js | 5 + tests/corpus/7661-node-call-modules/main.ts | 12 + .../node_modules/greeter/data/message.txt | 1 + .../node_modules/greeter/index.js | 8 + .../node_modules/greeter/package.json | 1 + .../node_modules/shout/index.js | 2 + .../node_modules/shout/package.json | 1 + .../7661-node-call-modules/settings.json | 1 + .../7662-node-call-exit-events/listeners.cjs | 13 + .../corpus/7662-node-call-exit-events/main.ts | 8 + tests/fixtures/node-island/gc-stress.ts | 24 + tests/fixtures/node-island/gc.cjs | 41 ++ tests/fixtures/node-island/helper.cjs | 10 + tests/fixtures/node-island/lifecycle.ts | 1 + tests/harness/ci-stack-scope.test.ts | 16 + .../7661-node-call-modules/main.ts.json | 1 + .../7662-node-call-exit-events/main.ts.json | 1 + tests/harness/node-island.test.ts | 124 +++- 48 files changed, 2138 insertions(+), 73 deletions(-) create mode 100644 packages/compiler/src/node-islands/modules.test.ts create mode 100644 packages/compiler/src/node-islands/modules.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7661-node-call-modules/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7662-node-call-exit-events/main.ts.json create mode 100644 tests/corpus/7661-node-call-modules/lib/entry.cjs create mode 100644 tests/corpus/7661-node-call-modules/lib/esm.mjs create mode 100644 tests/corpus/7661-node-call-modules/lib/factor.mjs create mode 100644 tests/corpus/7661-node-call-modules/lib/util.js create mode 100644 tests/corpus/7661-node-call-modules/main.ts create mode 100644 tests/corpus/7661-node-call-modules/node_modules/greeter/data/message.txt create mode 100644 tests/corpus/7661-node-call-modules/node_modules/greeter/index.js create mode 100644 tests/corpus/7661-node-call-modules/node_modules/greeter/package.json create mode 100644 tests/corpus/7661-node-call-modules/node_modules/shout/index.js create mode 100644 tests/corpus/7661-node-call-modules/node_modules/shout/package.json create mode 100644 tests/corpus/7661-node-call-modules/settings.json create mode 100644 tests/corpus/7662-node-call-exit-events/listeners.cjs create mode 100644 tests/corpus/7662-node-call-exit-events/main.ts create mode 100644 tests/fixtures/node-island/gc-stress.ts create mode 100644 tests/fixtures/node-island/gc.cjs create mode 100644 tests/harness/corpus-costs/7661-node-call-modules/main.ts.json create mode 100644 tests/harness/corpus-costs/7662-node-call-exit-events/main.ts.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd2e7aec5..098af5326 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,9 +52,13 @@ jobs: const stackBase = pr?.stack?.base?.ref; const prBase = pr?.base?.ref; // Position 1 can already be merged; compare the current base branches. + // A PR into the default branch is the lowest unmerged layer of any + // stack, whatever the stack metadata names as its base. + const defaultBranch = event.repository?.default_branch; const upperLayer = process.env.GITHUB_EVENT_NAME === 'pull_request' && sameRepository && trustedAuthor && typeof stackBase === 'string' && stackBase.length > 0 && - typeof prBase === 'string' && prBase.length > 0 && stackBase !== prBase; + typeof prBase === 'string' && prBase.length > 0 && stackBase !== prBase && + prBase !== defaultBranch; const full = !upperLayer; appendFileSync(process.env.GITHUB_OUTPUT, `full=${full}\n`); console.log(full ? 'Run full CI' : 'Defer full CI until this PR becomes the lowest unmerged layer'); diff --git a/packages/cli/src/bootstrap.ts b/packages/cli/src/bootstrap.ts index 9eb49f823..dc5a155d5 100644 --- a/packages/cli/src/bootstrap.ts +++ b/packages/cli/src/bootstrap.ts @@ -70,9 +70,6 @@ async function tryFastPath(): Promise { values["profile-generate"] !== undefined || values["profile-use"] !== undefined || values.sparse || - // The full compiler keys these builds by the recorded libnode pack. - values.libnode !== undefined || - process.env["SCRIPTC_LIBNODE"] !== undefined || (values["external-types"] ?? []).length > 0 ) return null; @@ -159,6 +156,8 @@ async function tryFastPath(): Promise { ) : startup.executableNativeEnvironmentFingerprint(), nodeVersion: process.version, + // An entry of a program that calls into Node.js serves only the same pack. + libnodeIdentity: () => startup.libnodeCacheKey(values.libnode), }); if (hit === null) return null; if (!values["keep-llvm"]) await rm(hit.llvmPath, { force: true }); diff --git a/packages/compiler/src/backend/llvm/emit-module.ts b/packages/compiler/src/backend/llvm/emit-module.ts index c16834910..f4f3bd0f1 100644 --- a/packages/compiler/src/backend/llvm/emit-module.ts +++ b/packages/compiler/src/backend/llvm/emit-module.ts @@ -17,6 +17,7 @@ import { } from "../mangle.js"; import { type IrGlobal, + type IrNodeIslandModules, type IrType, RUNTIME_ERROR_CLASSES, RUNTIME_EVENT_CLASS, @@ -57,7 +58,8 @@ import { ffiNativeTypeLl, } from "./common.js"; import { literalCapWord, literalCapWord32 } from "./string-key-hash.js"; -import { hasLoneSurrogate, wtf8Bytes } from "../literals.js"; +import { hasLoneSurrogate, llvmBytes, wtf8Bytes } from "../literals.js"; +import { nodeIslandManifest } from "../../node-islands/modules.js"; import { undefinedArmTag } from "../../ir/analysis.js"; import { NULLABLE_ABSENT, NULLABLE_NULL } from "./nullable-unions.js"; import { literalEqualityDefinition } from "./string-literal-equality.js"; @@ -381,6 +383,11 @@ export function emitModuleParts(emitter: LlEmitter): string[] { emitter.cstr(emitter.mod.nodeIsland?.nodeVersion ?? ""), ] : null; + // The island module tree: its manifest, and its files unless they ship + // beside the executable (node-islands/modules.ts). + const nodeIslandModules = usesNodeIsland ? emitter.mod.nodeIsland?.modules : undefined; + const nodeIslandModulesCall = + nodeIslandModules === undefined ? null : nodeIslandModulesInstall(emitter, nodeIslandModules); if (emitter.ffiHasForeignCallback) emitter.declare(`declare void @scr_ffi_install()`); if (usesStream) emitter.declare(`declare void @scr_stream_install()`); if (usesNetLookup) emitter.declare(`declare void @scr_net_lookup_install()`); @@ -732,6 +739,7 @@ export function emitModuleParts(emitter: LlEmitter): string[] { for (let site = 0; site < emitter.jsIcSiteCount; site++) out.push(`@sc_js_ic_${site} = internal ${tl}global [${JS_IC_WORDS} x i64] zeroinitializer`); } + if (nodeIslandModulesCall !== null) out.push(...nodeIslandModulesCall.globals); for (const table of emitter.constantNumericTables.values()) { const n = table.values.length; let tableTmp = 0; @@ -1027,6 +1035,7 @@ export function emitModuleParts(emitter: LlEmitter): string[] { ` call void @scr_node_island_install(ptr ${nodeIslandConfig[0]}, ptr ${nodeIslandConfig[1]})`, ] : []), + ...(nodeIslandModulesCall === null ? [] : [nodeIslandModulesCall.call]), ...(emitter.ffiHasForeignCallback ? [` call void @scr_ffi_install()`] : []), ...(snapshotsTlsCa ? [` call void @scr_tls_ca_install()`] : []), ...(usesFetch ? [` call void @scr_fetch_install()`] : []), @@ -1171,6 +1180,33 @@ export function emitModuleParts(emitter: LlEmitter): string[] { /** The shared abort helpers (emitted only when referenced): the OOM * abort of untraced shape allocation and the invalid-union-tag abort — * both print the runtime ABI’s exact message on fd 2 and abort. */ +/** The island module tree's install call before %main, and the constant + * that carries its files when the executable ships them itself + * (scr_node_island.c: scr_node_island_install_modules). */ +function nodeIslandModulesInstall( + emitter: LlEmitter, + modules: IrNodeIslandModules, +): { call: string; globals: string[] } { + emitter.declare( + `declare void @scr_node_island_install_modules(ptr, ${emitter.sizeType}, ptr, ${emitter.sizeType}, ptr)`, + ); + const manifest = nodeIslandManifest(modules); + const manifestPtr = emitter.cstr(manifest); + const location = emitter.cstr(modules.location ?? ""); + const data = modules.location === undefined ? emitter.nodeIslandModuleData : null; + const globals = + data === null || data.length === 0 + ? [] + : [ + `@sc_node_island_modules = private unnamed_addr constant [${data.length} x i8] c"${llvmBytes(data, false)}", align 16`, + ]; + const st = emitter.sizeType; + return { + call: ` call void @scr_node_island_install_modules(ptr ${manifestPtr}, ${st} ${Buffer.byteLength(manifest, "utf8")}, ptr ${globals.length > 0 ? "@sc_node_island_modules" : "null"}, ${st} ${globals.length > 0 ? data!.length : 0}, ptr ${location})`, + globals, + }; +} + export function helperDefs(emitter: LlEmitter): string[] { const defs: string[] = []; const msgHelper = (fnName: string, msgSym: string, msg: string): void => { diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 050c3fb90..545d146a4 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -473,6 +473,9 @@ export interface LlvmTargetOptions { * teardown after `process.exit` to a helper process (`--detached-exit`). * Executables only; the runtime's weak default keeps it off. */ detachedExit?: boolean; + /** The files of the program's Node island module tree, in manifest order + * (node-islands/modules.ts), when the executable carries the tree. */ + nodeIslandModuleData?: Uint8Array; /** Translated WebAssembly memories check bounds explicitly instead of * relying on guard pages (wasm-modules.ts wasmBoundsChecks). */ wasmBoundsChecks?: boolean; @@ -1036,6 +1039,7 @@ export class LlEmitter { readonly emitLibraryIdentity: boolean; readonly runtimeAbiMarker: boolean; readonly detachedExit: boolean; + readonly nodeIslandModuleData: Uint8Array | null; readonly wasmBoundsChecks: boolean; /** Interned string literals: UTF-8 text → { symbol, byte length } — * first-use order, the runtime ABI’s determinism discipline. */ @@ -1618,6 +1622,7 @@ export class LlEmitter { this.emitLibraryIdentity = options.emitLibraryIdentity !== false; this.runtimeAbiMarker = options.runtimeAbiMarker === true; this.detachedExit = options.detachedExit === true; + this.nodeIslandModuleData = options.nodeIslandModuleData ?? null; this.wasmBoundsChecks = options.wasmBoundsChecks === true || wasmBoundsChecks({ diff --git a/packages/compiler/src/diagnostics/diagnostic.ts b/packages/compiler/src/diagnostics/diagnostic.ts index 6d6e81516..96b639b96 100644 --- a/packages/compiler/src/diagnostics/diagnostic.ts +++ b/packages/compiler/src/diagnostics/diagnostic.ts @@ -66,7 +66,8 @@ * likely-hot functions that use untyped values (SC6101, * diagnostics/dynamic-tier.ts) * SC8xxx Node islands: the program calls into Node.js, and the build - * finds no usable libnode pack (SC8005) + * finds no usable libnode pack (SC8005), or cannot collect the + * modules a call loads (SC8006) * SC9xxx internal compiler errors (still source-anchored) */ import type { SrcLoc } from "../ir/ir.js"; @@ -133,6 +134,18 @@ export function nodeIslandNodeDiag(message: string, file: string): ScrDiagnostic }; } +/** SC8006 — a Node island call whose modules the build cannot collect into + * the executable: a computed specifier, or a module of the program that + * loads others through computed specifiers. */ +export function nodeIslandModuleDiag(message: string, loc: SrcLoc): ScrDiagnostic { + return { + code: "SC8006", + message, + loc, + hint: "name the module with a string literal (a relative path or a package name); the build ships the modules it can see with the executable", + }; +} + /* ── SC6xxx: divergences from Node (warnings) ─────────────────────────── */ /** SC6001 — a value passed to (or declared as) a narrower record type is diff --git a/packages/compiler/src/executable/executable-cache.ts b/packages/compiler/src/executable/executable-cache.ts index b9be0724d..664b23f65 100644 --- a/packages/compiler/src/executable/executable-cache.ts +++ b/packages/compiler/src/executable/executable-cache.ts @@ -121,6 +121,9 @@ interface EarlyExecutableCacheStamp { native: EarlyExecutableNativeFeatures; /** The build's performance warnings (SC61xx), which a cache hit repeats. */ performance?: ScrDiagnostic[]; + /** Programs that call into Node.js: the libnode pack identity the build + * recorded (EarlyExecutableCacheOptions.libnodeIdentity). */ + libnode?: string; integrity: string; } @@ -139,9 +142,12 @@ export interface EarlyExecutableCacheOptions { windowsSubsystem?: "gui"; /** Omitted unless --detached-exit is set. */ detachedExit?: true; - /** Omitted unless --libnode or SCRIPTC_LIBNODE is set (calls into Node.js - * record the pack). */ - libnode?: string; + /** The identity of the libnode pack a program that calls into Node.js + * records (node-islands/libnode.ts libnodeCacheKey), or undefined when the + * build names none. Not part of the key: an entry of a program that calls + * into Node records it, and a hit requires the same identity, so the pack + * settings never change the key of a program that does not. */ + libnodeIdentity?: () => string | undefined; /** Omitted without --profile-generate/--profile-use; otherwise the * pattern, or the digest of the profile's bytes. */ profile?: string; @@ -283,7 +289,6 @@ function cacheKey(options: EarlyExecutableCacheOptions): string { ...(options.strip ? ["strip"] : []), ...(options.windowsSubsystem === "gui" ? ["windows-subsystem-gui"] : []), ...(options.detachedExit ? ["detached-exit"] : []), - ...(options.libnode !== undefined ? [`libnode:${options.libnode}`] : []), ...(options.profile !== undefined ? [options.profile] : []), ...(options.checker !== undefined ? [`checker:${options.checker}`] : []), ...(options.youngNursery ? ["young-nursery"] : []), @@ -326,7 +331,6 @@ function routeKey(options: Omit { const value = lowerer.lowerExpr(arg); @@ -109,15 +127,18 @@ export function lowerNodeApiCall( lowerer.noLowering(`'${lowerer.fmt(bad)}' results of ${surface}`, call, JSON_HINT); } lowerer.usesNodeIsland = true; + // The caller's tree path is known once every call is collected (Lowerer). + const callerLiteral: IrExpr & { kind: "strLit" } = { + kind: "strLit", + value: caller, + type: STRING, + loc, + }; + lowerer.nodeIslandCallers.push({ literal: callerLiteral, caller }); const raw: IrExpr = { kind: "libCall", fn: member === "call" ? "nodeIsland.call" : "nodeIsland.callSync", - args: [ - { kind: "strLit", value: callerPath(call), type: STRING, loc }, - specifier, - name, - { kind: "dynArrLit", elems: values, type: DYN, loc }, - ], + args: [callerLiteral, specifier, name, { kind: "dynArrLit", elems: values, type: DYN, loc }], type: member === "call" ? { kind: "promise", inner: DYN } : DYN, loc, }; diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 43297e77c..4b6b0acdf 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -675,6 +675,7 @@ import { maybePromiseUnionOf, promiseArmedUnionOf, } from "./maybe-promise.js"; +import { NodeIslandModuleCollector } from "../../node-islands/modules.js"; /** Entry function name. '%' cannot appear in a TS identifier, so a user * function can never collide with it (mangling is injective per prefix). */ @@ -3140,6 +3141,10 @@ export class Lowerer { /** The program calls into Node.js (@scriptc/node): the module carries * the nodeIsland record, which the executable build fills in. */ usesNodeIsland = false; + /** The modules Node island calls load (lower-node-island.ts). */ + readonly nodeIslandModules = new NodeIslandModuleCollector(); + /** Each call's caller literal, set to its tree path when lowering ends. */ + readonly nodeIslandCallers: { literal: IrExpr & { kind: "strLit" }; caller: string }[] = []; /** Per-directory answers of experimental-import.ts's declaration-file test. */ readonly experimentalFiles = new Map(); /** Interned LifetimeRegion.run helpers, by callback type key. */ @@ -7805,7 +7810,7 @@ export class Lowerer { entry: ENTRY_NAME, ...(this.usesWorkers ? { workers: true } : {}), ...(this.usesLifetimeRegions ? { lifetimeRegions: true as const } : {}), - ...(this.usesNodeIsland ? { nodeIsland: { libnode: "", nodeVersion: "" } } : {}), + ...(this.usesNodeIsland ? { nodeIsland: this.nodeIslandRecord() } : {}), ...(this.declBoundary ? { declBoundary: true as const } : {}), ...(this.threadSlotKeys.size > 0 ? { threadSlotKeys: [...this.threadSlotKeys].sort() } @@ -8644,6 +8649,16 @@ export class Lowerer { this.diags.push(d); } + /** The module's Node island record: the collected module tree, with each + * call's caller literal set to its path in that tree. The executable + * build fills in the libnode pack (index.ts). */ + private nodeIslandRecord(): NonNullable { + const modules = this.nodeIslandModules.tree(); + for (const { literal, caller } of this.nodeIslandCallers) + literal.value = NodeIslandModuleCollector.callerPath(modules, caller); + return { libnode: "", nodeVersion: "", ...(modules === null ? {} : { modules }) }; + } + unsupported( code: keyof typeof UNSUPPORTED & `SC${number}`, node: ts.Node, diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index bcaf5a74f..415e12531 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -111,9 +111,11 @@ import { iceDiag, nativeCodegenDiag, nodeIslandNodeDiag, + nodeIslandModuleDiag, type ScrDiagnostic, } from "./diagnostics/diagnostic.js"; import { libnodeCacheKey, libnodeRecord } from "./node-islands/libnode.js"; +import { placeNodeIslandModules, readNodeIslandModules } from "./node-islands/modules.js"; import { loadLibraryProfile, type LibraryProfile } from "./library/library-profile.js"; import { clearFenceEvalCaches, decorateLibraryRefusals } from "./library/fence-eval.js"; import { @@ -785,7 +787,32 @@ async function prepareExecutableInput( const recorded = libnodeRecord(opts.libnode, opts.outPath); if (typeof recorded === "string") return { ok: false, diagnostics: [nodeIslandNodeDiag(recorded, entryPath)], sourceTexts }; - mod.nodeIsland = recorded; + mod.nodeIsland = { ...mod.nodeIsland, ...recorded }; + } + } + // The island module tree ships with the program: beside the executable + // with --libnode=beside, else inside it (node-islands/modules.ts). + let nodeIslandModuleData: Uint8Array | undefined; + const islandModules = mod.nodeIsland?.modules; + if (islandModules !== undefined && outputKind !== "ir") { + if (outputKind === "exe" && opts.libnode === "beside") { + const problem = placeNodeIslandModules(islandModules, `${opts.outPath}.libnode/modules`); + if (problem !== null) + return { + ok: false, + diagnostics: [nodeIslandModuleDiag(problem, { file: entryPath, start: 0, end: 0 })], + sourceTexts, + }; + islandModules.location = `$ORIGIN/${basename(opts.outPath)}.libnode/modules`; + } else { + const data = readNodeIslandModules(islandModules); + if (typeof data === "string") + return { + ok: false, + diagnostics: [nodeIslandModuleDiag(data, { file: entryPath, start: 0, end: 0 })], + sourceTexts, + }; + nodeIslandModuleData = data; } } if (warningSink !== undefined && prepared.warnings.length > 0) { @@ -820,6 +847,7 @@ async function prepareExecutableInput( lowRegion: lowRegionTarget(buildPlatform, opts.sanitize === true), ...(opts.sanitize === true ? { wasmBoundsChecks: true } : {}), ...inlineRcOption(opts.optimization, opts.sanitize === true), + ...(nodeIslandModuleData === undefined ? {} : { nodeIslandModuleData }), }); } catch (err) { if (!(err instanceof LlvmUnsupportedError)) throw err; @@ -890,6 +918,7 @@ async function prepareExecutableInput( ...(opts.sanitize === true ? { wasmBoundsChecks: true } : {}), ...inlineRcOption(opts.optimization, opts.sanitize === true), ...(opts.detachedExit === true ? { detachedExit: true } : {}), + ...(nodeIslandModuleData === undefined ? {} : { nodeIslandModuleData }), }); } catch (err) { if (!(err instanceof LlvmUnsupportedError)) throw err; @@ -1469,8 +1498,6 @@ async function compileTracked( if (outputKind === "exe") { const helperObjectRoute = opts.nativeProgramObject === true || usesPrecompiledRuntimePack(opts, "llvm"); - // Calls into Node record the libnode pack, keyed by its manifest (node-islands/libnode.ts). - const libnodeKey = libnodeCacheKey(opts.libnode); const routeOptions: Omit< EarlyExecutableCacheOptions, "implementation" | "implementationDependencies" | "nativeEnvironment" @@ -1485,7 +1512,9 @@ async function compileTracked( ...(opts.strip ? { strip: true as const } : {}), ...(opts.windowsSubsystem === "gui" ? { windowsSubsystem: "gui" as const } : {}), ...(opts.detachedExit === true ? { detachedExit: true as const } : {}), - ...(libnodeKey === undefined ? {} : { libnode: libnodeKey }), + // Calls into Node record the libnode pack; their entries record its + // identity (node-islands/libnode.ts) and a hit requires it unchanged. + libnodeIdentity: () => libnodeCacheKey(opts.libnode), ...(profile === undefined ? {} : { profile: nativeProfileKey(profile) }), ...checkerField(), ...(youngNurseryRequested() ? { youngNursery: true as const } : {}), @@ -1531,26 +1560,22 @@ async function compileTracked( implementationDependencies: implementation.dependencies, }; } - const earlyHit = + const cachedHit = earlyCacheOptions === null ? null : await readEarlyExecutableCache(cacheRoot, earlyCacheOptions); + // A --libnode=beside build that calls into Node places its island modules + // next to the executable from the lowered module, which a cache hit does + // not have: such builds take the full path. + const earlyHit = + cachedHit !== null && cachedHit.native.nodeIsland === true && opts.libnode === "beside" + ? null + : cachedHit; if (earlyHit !== null) { timing("executable-cache-hit"); if (earlyCacheOptions === null) { throw new InternalCompilerError("executable cache hit without executable cache options"); } - // The pack a --libnode=beside build places next to the executable is an output of the - // build, like the executable: a restored executable that calls into Node gets it again. - if (earlyHit.native.nodeIsland === true && opts.libnode === "beside") { - const recorded = libnodeRecord(opts.libnode, opts.outPath); - if (typeof recorded === "string") - return { - ok: false, - diagnostics: [nodeIslandNodeDiag(recorded, entryPath)], - sourceTexts: new Map(), - }; - } // A cached build repeats its performance warning (SC6101): the code it // names is just as untyped as when it was built. if (warningSink !== undefined && earlyHit.performance !== undefined) { diff --git a/packages/compiler/src/ir/function-references.test.ts b/packages/compiler/src/ir/function-references.test.ts index 0e09a7567..9541cb723 100644 --- a/packages/compiler/src/ir/function-references.test.ts +++ b/packages/compiler/src/ir/function-references.test.ts @@ -221,6 +221,12 @@ const MODULE_FIELDS: Record>, Role> = { "threadSlotKeys[]": "other", "nodeIsland.libnode": "other", "nodeIsland.nodeVersion": "other", + "nodeIsland.modules.root": "other", + "nodeIsland.modules.location": "other", + "nodeIsland.modules.files[].path": "other", + "nodeIsland.modules.files[].sha256": "other", + "nodeIsland.modules.links[].path": "other", + "nodeIsland.modules.links[].target": "other", }; const loc = { file: "references.ts", start: 0, end: 0 }; diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 6ed304d73..e458c9045 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -1009,6 +1009,18 @@ export function isRefCounted(t: IrType): boolean { /* ── module ────────────────────────────────────────────────────────────── */ +/** The island module tree of a program (node-islands/modules.ts): paths + * relative to `root`, the directory on the build machine that holds them. + * Only the relative paths enter an executable. */ +export interface IrNodeIslandModules { + root: string; + files: { path: string; size: number; sha256: string }[]; + links: { path: string; target: string }[]; + /** Where the executable finds the tree: absent when it carries the tree + * itself, else a `$ORIGIN/` path (`--libnode=beside`). */ + location?: string; +} + export interface IrModule { /** Bumped on any breaking IR change; serialize.ts refuses mismatches. */ irVersion: 17; @@ -1067,7 +1079,13 @@ export interface IrModule { * executable) and the exact Node version its bridge links. Set by the * executable build (node-islands/libnode.ts); the emitted main records * both before %main. */ - nodeIsland?: { libnode: string; nodeVersion: string }; + nodeIsland?: { + libnode: string; + nodeVersion: string; + /** The modules the calls load (node-islands/modules.ts), when any call + * loads a file. */ + modules?: IrNodeIslandModules; + }; /** Outbound native FFI declarations used by `ffiCall` expressions. * These are link-time C ABI imports, not runtime dynamic-library handles: * executable builds resolve their symbols from the manifest's archive diff --git a/packages/compiler/src/node-islands/libnode.ts b/packages/compiler/src/node-islands/libnode.ts index 8bf85a548..a0ae29d2f 100644 --- a/packages/compiler/src/node-islands/libnode.ts +++ b/packages/compiler/src/node-islands/libnode.ts @@ -30,7 +30,7 @@ import { basename, isAbsolute, join, resolve, sep } from "node:path"; export const NODE_ISLAND_MIN_NODE = "24.13.1"; /** The bridge ABI this compiler's runtime speaks (scr_node_bridge.h). */ -export const NODE_BRIDGE_ABI = 2; +export const NODE_BRIDGE_ABI = 3; /** The prefix of a recorded path relative to the executable's directory. */ export const ORIGIN = "$ORIGIN/"; @@ -142,11 +142,13 @@ export function libnodeRecord( return { libnode: `${ORIGIN}${name}`, nodeVersion: manifest.nodeVersion }; } -/** The part of an executable's cache key that names its libnode pack, or - * undefined when neither --libnode nor SCRIPTC_LIBNODE is set. It holds the - * option, the pack directory and a digest of the pack's manifest.json, so - * that a pack upgraded in place (another Node version, bridge ABI or file - * list) gives cached executables a new key instead of their old record. */ +/** The identity of the libnode pack a build that calls into Node.js + * records, which the program's executable cache entry keeps + * (EarlyExecutableCacheOptions.libnodeIdentity), or undefined when neither + * --libnode nor SCRIPTC_LIBNODE is set. It holds the option, the pack + * directory and a digest of the pack's manifest.json, so that a pack + * upgraded in place (another Node version, bridge ABI or file list) + * invalidates the entry instead of restoring its old record. */ export function libnodeCacheKey( option: string | undefined, env: NodeJS.ProcessEnv = process.env, diff --git a/packages/compiler/src/node-islands/modules.test.ts b/packages/compiler/src/node-islands/modules.test.ts new file mode 100644 index 000000000..6badf376a --- /dev/null +++ b/packages/compiler/src/node-islands/modules.test.ts @@ -0,0 +1,188 @@ +import { mkdirSync, mkdtempSync, realpathSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { describe, expect, test } from "vitest"; +import { + NodeIslandModuleCollector, + nodeIslandManifest, + readNodeIslandModules, + scanModuleSpecifiers, +} from "./modules.js"; + +function project(files: Record): string { + const root = realpathSync(mkdtempSync(join(tmpdir(), "scriptc-island-modules-"))); + for (const [path, text] of Object.entries(files)) { + mkdirSync(dirname(join(root, path)), { recursive: true }); + writeFileSync(join(root, path), text); + } + return root; +} + +describe("module specifiers in JavaScript", () => { + test("string-literal loads in every form", () => { + const scan = scanModuleSpecifiers(` + const a = require("./a"); + const b = require.resolve('./b.json'); + import c from "c"; + import * as d from "d/sub"; + import { e } from "./e.mjs"; + import "./side"; + export * from "./f"; + export { g } from "./g"; + const h = await import("./h.js"); + import.meta.url; + obj.require("x"); + const t = \`\${require("./t")}\`; + `); + expect(scan.specifiers).toEqual([ + "./a", + "./b.json", + "c", + "d/sub", + "./e.mjs", + "./side", + "./f", + "./g", + "./h.js", + "./t", + ]); + expect(scan.computed).toEqual([]); + }); + + test("comments, strings and regular expressions hide nothing and invent nothing", () => { + const scan = scanModuleSpecifiers(` + // require("./commented") + /* import x from "./block" */ + const s = "require('./in-string')"; + const r = /require\\("\\.\\/in-regex"\\)/g; + const q = a / b / c; + const real = require("./real"); + `); + expect(scan.specifiers).toEqual(["./real"]); + expect(scan.computed).toEqual([]); + }); + + test("computed loads are reported", () => { + const forms = (text: string) => scanModuleSpecifiers(text).computed.map((c) => c.form); + expect(forms("require(name)")).toEqual(["require() with a computed specifier"]); + expect(forms("require('./a' + b)")).toEqual(["require() with a computed specifier"]); + expect(forms("import(`./x/${y}`)")).toEqual(["import() with a computed specifier"]); + expect(forms("const r = require; r('./a')")).toEqual(["require as a value"]); + expect(forms("createRequire(import.meta.url)('./a')")).toEqual([ + "a require() from createRequire()", + ]); + expect(forms("function require(x) {} typeof require")).toEqual([]); + }); +}); + +describe("the collected module tree", () => { + test("a relative module and what it loads; missing modules are not shipped", () => { + const root = project({ + "src/main.ts": "", + "src/math.cjs": + "module.exports = require('./util');\ntry { require('./optional'); } catch {}", + "src/util.js": "exports.x = require('../data.json');", + "data.json": "{}", + "src/unused.js": "", + }); + const collector = new NodeIslandModuleCollector(); + expect(collector.add(join(root, "src/main.ts"), "./math.cjs")).toBeNull(); + expect(collector.add(join(root, "src/main.ts"), "./does-not-exist.cjs")).toBeNull(); + expect(collector.add(join(root, "src/main.ts"), "node:fs")).toBeNull(); + const tree = collector.tree()!; + expect(tree.root).toBe(root); + expect(tree.files.map((f) => f.path)).toEqual(["data.json", "src/math.cjs", "src/util.js"]); + expect(NodeIslandModuleCollector.callerPath(tree, join(root, "src/main.ts"))).toBe( + "src/main.ts", + ); + }); + + test("a module of the program ships with the package.json that scopes it", () => { + const root = project({ + "package.json": JSON.stringify({ type: "module" }), + "src/main.ts": "", + "src/esm.js": "export const x = 1;", + }); + const collector = new NodeIslandModuleCollector(); + expect(collector.add(join(root, "src/main.ts"), "./esm.js")).toBeNull(); + expect(collector.tree()!.files.map((f) => f.path)).toEqual(["package.json", "src/esm.js"]); + }); + + test("a computed load in a module of the program is refused", () => { + const root = project({ "main.ts": "", "dyn.cjs": "module.exports = require(process.env.X);" }); + const problem = new NodeIslandModuleCollector().add(join(root, "main.ts"), "./dyn.cjs"); + expect(problem).toBe( + `${join(root, "dyn.cjs")}:1 loads a module through require() with a computed specifier; a module that runs in a Node island loads modules by string-literal specifiers only`, + ); + }); + + test("a package ships whole, with its dependencies, but not its nested node_modules", () => { + const root = project({ + "main.ts": "", + "node_modules/pkg/package.json": JSON.stringify({ dependencies: { dep: "1", absent: "1" } }), + "node_modules/pkg/index.js": "module.exports = require(process.env.ANYTHING_GOES);", + "node_modules/pkg/lib/extra.json": "{}", + "node_modules/pkg/node_modules/.bin/x": "", + "node_modules/dep/package.json": "{}", + "node_modules/dep/index.js": "", + "node_modules/other/index.js": "", + }); + const collector = new NodeIslandModuleCollector(); + expect(collector.add(join(root, "main.ts"), "pkg/lib/extra.json")).toBeNull(); + expect(collector.tree()!.files.map((f) => f.path)).toEqual([ + "node_modules/dep/index.js", + "node_modules/dep/package.json", + "node_modules/pkg/index.js", + "node_modules/pkg/lib/extra.json", + "node_modules/pkg/package.json", + ]); + }); + + test("pnpm's links between package directories stay links inside the tree", () => { + const root = project({ + "app/main.ts": "", + "app/node_modules/.pnpm/a@1/node_modules/a/package.json": JSON.stringify({ + dependencies: { b: "1" }, + }), + "app/node_modules/.pnpm/a@1/node_modules/a/index.js": "", + "app/node_modules/.pnpm/b@1/node_modules/b/index.js": "", + }); + symlinkSync(".pnpm/a@1/node_modules/a", join(root, "app/node_modules/a")); + symlinkSync( + "../../b@1/node_modules/b", + join(root, "app/node_modules/.pnpm/a@1/node_modules/b"), + ); + const collector = new NodeIslandModuleCollector(); + expect(collector.add(join(root, "app/main.ts"), "a")).toBeNull(); + const tree = collector.tree()!; + expect(tree.root).toBe(join(root, "app")); + expect(tree.links).toEqual([ + { path: "node_modules/.pnpm/a@1/node_modules/b", target: "../../b@1/node_modules/b" }, + { path: "node_modules/a", target: ".pnpm/a@1/node_modules/a" }, + ]); + expect(tree.files.map((f) => f.path)).toEqual([ + "node_modules/.pnpm/a@1/node_modules/a/index.js", + "node_modules/.pnpm/a@1/node_modules/a/package.json", + "node_modules/.pnpm/b@1/node_modules/b/index.js", + ]); + }); + + test("built-in modules alone ship no tree", () => { + const root = project({ "main.ts": "" }); + const collector = new NodeIslandModuleCollector(); + expect(collector.add(join(root, "main.ts"), "os")).toBeNull(); + expect(collector.tree()).toBeNull(); + expect(NodeIslandModuleCollector.callerPath(null, join(root, "main.ts"))).toBe("main.ts"); + }); + + test("the manifest names relative paths only, and the data is checked against it", () => { + const root = project({ "main.ts": "", "m.cjs": "exports.f = () => 1;" }); + const collector = new NodeIslandModuleCollector(); + collector.add(join(root, "main.ts"), "./m.cjs"); + const tree = collector.tree()!; + expect(nodeIslandManifest(tree)).not.toContain(root); + expect(readNodeIslandModules(tree)).toEqual(new TextEncoder().encode("exports.f = () => 1;")); + writeFileSync(join(root, "m.cjs"), "exports.f = () => 2;"); + expect(readNodeIslandModules(tree)).toBe(`${join(root, "m.cjs")} changed during the build`); + }); +}); diff --git a/packages/compiler/src/node-islands/modules.ts b/packages/compiler/src/node-islands/modules.ts new file mode 100644 index 000000000..0d63dc333 --- /dev/null +++ b/packages/compiler/src/node-islands/modules.ts @@ -0,0 +1,627 @@ +/* The island module tree: the JavaScript a program's Node island calls load, + * collected at build time and shipped with the executable, so a deployed + * binary never reads modules from the machine that built it. + * + * Each `call`/`callSync` names its module with a string literal. The build + * resolves it from the calling module as Node's require() does: + * - a relative or absolute specifier names a module of the program: the + * module and every module its string-literal `require()`, `import` and + * `export ... from` reach are collected; a computed specifier in such a + * module is refused, since the build cannot know what it loads; + * - a package specifier names a package: its whole directory is collected + * (computed requires inside packages keep working), and so are its + * `dependencies`, `optionalDependencies` and `peerDependencies`, found as + * Node finds them; + * - a Node built-in module needs nothing, and neither does a module the + * build cannot find: loading it throws MODULE_NOT_FOUND when the call + * runs, as under Node. + * Paths keep their layout relative to one root directory (the deepest + * directory that holds every collected path), with symbolic links between + * package directories (pnpm's layout) kept as links. Only these relative + * paths enter the executable. Every read goes through the frontend input + * tracker, so an edit to an island module invalidates cached builds. */ +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { builtinModules } from "node:module"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { + trackedAccessibleEntries, + trackedDirectoryExists, + trackedFileExists, + trackedReadBytes, + trackedReadFile, + trackedRealpath, +} from "../frontend/input-tracker.js"; +import type { IrNodeIslandModules } from "../ir/ir.js"; + +export type NodeIslandModuleTree = IrNodeIslandModules; +type NodeIslandModuleFile = IrNodeIslandModules["files"][number]; +type NodeIslandModuleLink = IrNodeIslandModules["links"][number]; + +const BUILTINS: ReadonlySet = new Set(builtinModules); + +/** Extensions require() tries for an extensionless file, in its order. */ +const FILE_EXTENSIONS = [".js", ".json", ".node"]; + +/** Module files whose imports are traced. */ +const TRACED = /\.(?:[cm]?js|[cm]?ts)$/; + +function isBuiltin(specifier: string): boolean { + return specifier.startsWith("node:") || BUILTINS.has(specifier); +} + +function isPathSpecifier(specifier: string): boolean { + return ( + specifier === "." || + specifier === ".." || + specifier.startsWith("./") || + specifier.startsWith("../") || + isAbsolute(specifier) + ); +} + +function packageName(specifier: string): string { + const parts = specifier.split("/"); + return specifier.startsWith("@") ? parts.slice(0, 2).join("/") : parts[0]!; +} + +/** The file require() loads for `base` (a file, an extension away, or a + * directory with a package.json main or an index file), or null. */ +function resolveFile(base: string): string | null { + if (trackedFileExists(base)) return base; + for (const extension of FILE_EXTENSIONS) + if (trackedFileExists(base + extension)) return base + extension; + if (!trackedDirectoryExists(base)) return null; + const manifest = trackedReadFile(join(base, "package.json")); + if (manifest !== null) { + let main: unknown; + try { + main = (JSON.parse(manifest) as { main?: unknown }).main; + } catch { + main = undefined; + } + if (typeof main === "string" && main !== "") { + const target = join(base, main); + const found = resolveFile(target) ?? resolveFile(join(target, "index")); + if (found !== null) return found; + } + } + for (const extension of FILE_EXTENSIONS) { + const index = join(base, `index${extension}`); + if (trackedFileExists(index)) return index; + } + return null; +} + +/** The directory of package `name` as require() finds it from `fromDir`: + * `node_modules/` in `fromDir` and each ancestor not itself named + * node_modules. */ +function packageDirectory(fromDir: string, name: string): string | null { + for (let dir = fromDir; ;) { + if (basename(dir) !== "node_modules") { + const candidate = join(dir, "node_modules", name); + if (trackedDirectoryExists(candidate)) return candidate; + } + const parent = dirname(dir); + if (parent === dir) return null; + dir = parent; + } +} + +/** The package.json nearest to `dir`, as Node finds a module's package + * scope, or null. */ +function packageScope(dir: string): string | null { + for (let at = dir; ;) { + if (basename(at) === "node_modules") return null; + const candidate = join(at, "package.json"); + if (trackedFileExists(candidate)) return candidate; + const parent = dirname(at); + if (parent === at) return null; + at = parent; + } +} + +function sha256(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +/** The deepest directory that contains every path. */ +function commonDirectory(paths: readonly string[]): string { + let common = paths[0]!.split(sep); + for (const path of paths.slice(1)) { + const parts = path.split(sep); + let n = 0; + while (n < common.length && n < parts.length && common[n] === parts[n]) n++; + common = common.slice(0, n); + } + const joined = common.join(sep); + return joined === "" ? sep : joined; +} + +function portable(path: string): string { + return sep === "/" ? path : path.split(sep).join("/"); +} + +export class NodeIslandModuleCollector { + /** Real paths of collected files. */ + private readonly files = new Set(); + /** Link path (its directory real) → real target. */ + private readonly links = new Map(); + private readonly packages = new Set(); + private readonly traced = new Set(); + /** Real paths of the calling modules. */ + private readonly callers = new Set(); + + /** Collects the module `specifier` names from `caller` (a real path). + * Returns null, or the reason the build cannot ship it. */ + add(caller: string, specifier: string): string | null { + this.callers.add(caller); + return this.resolveFrom(caller, specifier); + } + + /** The collected tree, or null when no call loads a file. */ + tree(): NodeIslandModuleTree | null { + if (this.files.size === 0 && this.links.size === 0) return null; + const anchors = [ + ...this.files, + ...this.links.keys(), + ...this.links.values(), + ...[...this.callers].map((caller) => dirname(caller)), + ]; + const root = commonDirectory(anchors); + const files: NodeIslandModuleFile[] = []; + for (const file of [...this.files].sort()) { + const bytes = trackedReadBytes(file); + if (bytes === null) continue; + files.push({ + path: portable(relative(root, file)), + size: bytes.length, + sha256: sha256(bytes), + }); + } + const links: NodeIslandModuleLink[] = [...this.links] + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([path, target]) => ({ + path: portable(relative(root, path)), + target: portable(relative(dirname(path), target)), + })); + return { root, files, links }; + } + + /** The tree path a caller's module has: relative to the tree root. */ + static callerPath(tree: NodeIslandModuleTree | null, caller: string): string { + return tree === null ? basename(caller) : portable(relative(tree.root, caller)); + } + + private resolveFrom(fromFile: string, specifier: string): string | null { + if (isBuiltin(specifier)) return null; + if (specifier.startsWith("#")) + return `'${specifier}' is a package import ("imports" in package.json), which Node island modules of the program cannot use`; + if (specifier.startsWith("file:") || /^[a-z][a-z0-9+.-]*:/i.test(specifier)) + return `'${specifier}' is a URL; name the module by its relative path or package name`; + // A module the build cannot find is not shipped: loading it throws + // MODULE_NOT_FOUND when the call runs, as it does under Node (an + // optional require in a try block keeps working). + if (isPathSpecifier(specifier)) { + const target = resolveFile(resolve(dirname(fromFile), specifier)); + return target === null ? null : this.addModule(this.placed(target)); + } + const dir = packageDirectory(dirname(fromFile), packageName(specifier)); + if (dir !== null) this.addPackage(dir); + return null; + } + + /** The real path of `path`, recording a link when `path` itself (not an + * ancestor) is a symbolic link. */ + private placed(path: string): string { + const target = trackedRealpath(path) ?? path; + const parent = trackedRealpath(dirname(path)) ?? dirname(path); + const link = join(parent, basename(path)); + if (link !== target) this.links.set(link, target); + return target; + } + + /** A module of the program: collected with the package.json that scopes + * it (Node reads its "type"), and its imports traced. */ + private addModule(file: string): string | null { + if (this.traced.has(file)) return null; + this.traced.add(file); + this.files.add(file); + const scope = packageScope(dirname(file)); + if (scope !== null) this.files.add(scope); + if (!TRACED.test(file)) return null; + const text = trackedReadFile(file); + if (text === null) return `cannot read ${file}`; + const scan = scanModuleSpecifiers(text); + if (scan.computed.length > 0) { + const first = scan.computed[0]!; + const line = text.slice(0, first.at).split("\n").length; + return `${file}:${line} loads a module through ${first.form}; a module that runs in a Node island loads modules by string-literal specifiers only`; + } + for (const specifier of scan.specifiers) { + const problem = this.resolveFrom(file, specifier); + if (problem !== null) return problem; + } + return null; + } + + /** A package: its whole directory, then its dependencies. */ + private addPackage(found: string): void { + const dir = this.placed(found); + if (this.packages.has(dir)) return; + this.packages.add(dir); + this.addDirectory(dir, new Set()); + const manifest = trackedReadFile(join(dir, "package.json")); + if (manifest === null) return; + let parsed: Record; + try { + parsed = JSON.parse(manifest) as Record; + } catch { + return; + } + const names = new Set(); + for (const field of ["dependencies", "optionalDependencies", "peerDependencies"]) { + const deps = parsed[field]; + if (deps !== null && typeof deps === "object") + for (const name of Object.keys(deps)) names.add(name); + } + for (const name of [...names].sort()) { + // A dependency that is not installed (optional, or a peer the program + // does not use) is not loaded either. + const depDir = packageDirectory(dir, name); + if (depDir !== null) this.addPackage(depDir); + } + } + + /** Every file below `dir`, except nested node_modules (dependencies come + * through package.json) and .git. */ + private addDirectory(dir: string, visited: Set): void { + const real = trackedRealpath(dir) ?? dir; + if (visited.has(real)) return; + visited.add(real); + const entries = trackedAccessibleEntries(dir); + if (entries === null) return; + for (const file of entries.files) this.files.add(join(dir, file)); + for (const sub of entries.directories) { + if (sub === "node_modules" || sub === ".git") continue; + this.addDirectory(join(dir, sub), visited); + } + } +} + +/* ── module specifiers in JavaScript source ───────────────────────────── */ + +interface Token { + kind: "name" | "string" | "punct" | "other"; + value: string; + at: number; +} + +/** Keywords after which a `/` starts a regular expression. */ +const REGEX_AFTER = new Set([ + "return", + "typeof", + "instanceof", + "in", + "of", + "new", + "delete", + "void", + "throw", + "case", + "do", + "else", + "yield", + "await", +]); + +function isNameStart(c: string): boolean { + return /[A-Za-z_$\u0080-￿]/.test(c); +} + +function isNamePart(c: string): boolean { + return /[A-Za-z0-9_$\u0080-￿]/.test(c); +} + +/** The significant tokens of a JavaScript or TypeScript source: names, + * string literals (cooked) and punctuators, without comments, template + * text or regular expressions. */ +export function moduleTokens(text: string): Token[] { + const tokens: Token[] = []; + /** Brace depths at which a template literal's `${` was opened. */ + const templates: number[] = []; + let depth = 0; + let i = 0; + const n = text.length; + const regexAllowed = (): boolean => { + const last = tokens[tokens.length - 1]; + if (last === undefined) return true; + if (last.kind === "name") return REGEX_AFTER.has(last.value); + if (last.kind === "punct") + return last.value !== ")" && last.value !== "]" && last.value !== "}"; + return false; + }; + const template = (start: number): void => { + // From just after a backtick or a closing `}` to the end of the + // template or the next `${`. + while (i < n) { + const c = text[i]!; + if (c === "\\") { + i += 2; + continue; + } + if (c === "`") { + i++; + tokens.push({ kind: "other", value: "`", at: start }); + return; + } + if (c === "$" && text[i + 1] === "{") { + i += 2; + templates.push(depth); + depth++; + tokens.push({ kind: "punct", value: "${", at: start }); + return; + } + i++; + } + }; + while (i < n) { + const c = text[i]!; + if (c === " " || c === "\t" || c === "\n" || c === "\r" || c === "\f" || c === "\v") { + i++; + continue; + } + if (c === "/" && text[i + 1] === "/") { + while (i < n && text[i] !== "\n") i++; + continue; + } + if (c === "/" && text[i + 1] === "*") { + const end = text.indexOf("*/", i + 2); + i = end < 0 ? n : end + 2; + continue; + } + if (c === "#" && i === 0 && text[1] === "!") { + while (i < n && text[i] !== "\n") i++; + continue; + } + if (c === "'" || c === '"') { + const start = i; + let value = ""; + i++; + while (i < n && text[i] !== c && text[i] !== "\n") { + if (text[i] === "\\") { + const e = text[i + 1] ?? ""; + value += e === "n" ? "\n" : e === "t" ? "\t" : e === "r" ? "\r" : e === "\n" ? "" : e; + i += 2; + continue; + } + value += text[i]; + i++; + } + i++; + tokens.push({ kind: "string", value, at: start }); + continue; + } + if (c === "`") { + const start = i; + i++; + template(start); + continue; + } + if (c === "/" && regexAllowed()) { + const start = i; + i++; + let inClass = false; + while (i < n && text[i] !== "\n") { + const r = text[i]!; + if (r === "\\") { + i += 2; + continue; + } + if (r === "[") inClass = true; + else if (r === "]") inClass = false; + else if (r === "/" && !inClass) break; + i++; + } + i++; + while (i < n && isNamePart(text[i]!)) i++; + tokens.push({ kind: "other", value: "/re/", at: start }); + continue; + } + if (isNameStart(c)) { + const start = i; + while (i < n && isNamePart(text[i]!)) i++; + tokens.push({ kind: "name", value: text.slice(start, i), at: start }); + continue; + } + if (/[0-9]/.test(c) || (c === "." && /[0-9]/.test(text[i + 1] ?? ""))) { + const start = i; + while (i < n && /[0-9A-Za-z_.]/.test(text[i]!)) i++; + tokens.push({ kind: "other", value: text.slice(start, i), at: start }); + continue; + } + if (c === "{") depth++; + if (c === "}") { + if (templates.length > 0 && templates[templates.length - 1] === depth - 1) { + templates.pop(); + depth--; + const start = i; + i++; + tokens.push({ kind: "punct", value: "}", at: start }); + template(start); + continue; + } + depth--; + } + if (c === "." && text[i + 1] === "." && text[i + 2] === ".") { + tokens.push({ kind: "punct", value: "...", at: i }); + i += 3; + continue; + } + if (c === "?" && text[i + 1] === ".") { + tokens.push({ kind: "punct", value: "?.", at: i }); + i += 2; + continue; + } + tokens.push({ kind: "punct", value: c, at: i }); + i++; + } + return tokens; +} + +export interface ModuleSpecifierScan { + /** String-literal specifiers of require(), require.resolve(), import() + * and import/export declarations, in source order. */ + specifiers: string[]; + /** Loads whose module the source does not name with a string literal. */ + computed: { at: number; form: string }[]; +} + +export function scanModuleSpecifiers(text: string): ModuleSpecifierScan { + const tokens = moduleTokens(text); + const specifiers: string[] = []; + const computed: { at: number; form: string }[] = []; + const at = (k: number): Token | undefined => tokens[k]; + const is = (k: number, kind: Token["kind"], value?: string): boolean => { + const t = tokens[k]; + return t !== undefined && t.kind === kind && (value === undefined || t.value === value); + }; + /** A call `(` at k whose single argument is a string literal. */ + const literalCall = (k: number): string | null => + is(k, "punct", "(") && + is(k + 1, "string") && + (is(k + 2, "punct", ")") || is(k + 2, "punct", ",")) + ? at(k + 1)!.value + : null; + for (let k = 0; k < tokens.length; k++) { + const t = tokens[k]!; + if (t.kind !== "name") continue; + const member = is(k - 1, "punct", ".") || is(k - 1, "punct", "?."); + if (t.value === "require" && !member && !is(k + 1, "punct", ":")) { + if (is(k - 1, "name", "function") || is(k - 1, "name", "typeof")) continue; + let call = k + 1; + let form = "require()"; + if (is(k + 1, "punct", ".") && is(k + 2, "name", "resolve")) { + call = k + 3; + form = "require.resolve()"; + } else if (is(k + 1, "punct", ".")) { + continue; + } + if (!is(call, "punct", "(")) { + computed.push({ at: t.at, form: "require as a value" }); + continue; + } + const literal = literalCall(call); + if (literal === null) computed.push({ at: t.at, form: `${form} with a computed specifier` }); + else specifiers.push(literal); + continue; + } + if (t.value === "createRequire" && !member && is(k + 1, "punct", "(")) { + computed.push({ at: t.at, form: "a require() from createRequire()" }); + continue; + } + if (t.value === "import" && !member) { + if (is(k + 1, "punct", ".")) continue; // import.meta + if (is(k + 1, "punct", "(")) { + const literal = literalCall(k + 1); + if (literal === null) + computed.push({ at: t.at, form: "import() with a computed specifier" }); + else specifiers.push(literal); + continue; + } + if (is(k + 1, "string")) { + specifiers.push(at(k + 1)!.value); + continue; + } + for (let j = k + 1; j < tokens.length && j < k + 400; j++) { + if (is(j, "punct", ";")) break; + if (is(j, "name", "from") && is(j + 1, "string")) { + specifiers.push(at(j + 1)!.value); + break; + } + } + continue; + } + if (t.value === "export" && !member) { + let j = k + 1; + if (is(j, "name", "type")) j++; + if (is(j, "punct", "*")) { + for (; j < tokens.length && j < k + 400; j++) { + if (is(j, "punct", ";")) break; + if (is(j, "name", "from") && is(j + 1, "string")) { + specifiers.push(at(j + 1)!.value); + break; + } + } + } else if (is(j, "punct", "{")) { + while (j < tokens.length && !is(j, "punct", "}")) j++; + if (is(j + 1, "name", "from") && is(j + 2, "string")) specifiers.push(at(j + 2)!.value); + } + } + } + return { specifiers, computed }; +} + +/* ── shipping the tree ────────────────────────────────────────────────── */ + +/** The manifest the executable records (scr_node_island.c reads it): + * `{"files":[[path,size,sha256],...],"links":[[path,target],...]}`. Its + * SHA-256 names the tree. */ +export function nodeIslandManifest(modules: IrNodeIslandModules): string { + return JSON.stringify({ + files: modules.files.map((file) => [file.path, file.size, file.sha256]), + links: modules.links.map((link) => [link.path, link.target]), + }); +} + +/** The tree's files concatenated in manifest order, each checked against + * the digest the build recorded, or the reason one no longer matches. */ +export function readNodeIslandModules(modules: IrNodeIslandModules): Uint8Array | string { + const parts: Uint8Array[] = []; + let total = 0; + for (const file of modules.files) { + const path = join(modules.root, ...file.path.split("/")); + let bytes: Uint8Array; + try { + bytes = new Uint8Array(readFileSync(path)); + } catch { + return `${path} was removed during the build`; + } + if (bytes.length !== file.size || sha256(bytes) !== file.sha256) + return `${path} changed during the build`; + parts.push(bytes); + total += bytes.length; + } + const data = new Uint8Array(total); + let offset = 0; + for (const part of parts) { + data.set(part, offset); + offset += part.length; + } + return data; +} + +/** Writes the tree to `target` (replacing it), for `--libnode=beside`, or + * returns the reason it cannot. */ +export function placeNodeIslandModules( + modules: IrNodeIslandModules, + target: string, +): string | null { + const data = readNodeIslandModules(modules); + if (typeof data === "string") return data; + rmSync(target, { recursive: true, force: true }); + mkdirSync(target, { recursive: true }); + let offset = 0; + for (const file of modules.files) { + const path = join(target, ...file.path.split("/")); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, data.subarray(offset, offset + file.size)); + offset += file.size; + } + for (const link of modules.links) { + const path = join(target, ...link.path.split("/")); + mkdirSync(dirname(path), { recursive: true }); + symlinkSync(link.target, path); + } + return null; +} diff --git a/packages/compiler/src/startup-cache.ts b/packages/compiler/src/startup-cache.ts index 1bc74bef0..950a96d08 100644 --- a/packages/compiler/src/startup-cache.ts +++ b/packages/compiler/src/startup-cache.ts @@ -2,6 +2,7 @@ * full lowering/emitter graph; exact builds need only these cache/native * identity primitives before deciding whether that graph is necessary. */ export { readRoutedExecutableCache } from "./executable/executable-cache.js"; +export { libnodeCacheKey } from "./node-islands/libnode.js"; export type { EarlyExecutableRouteOptions } from "./executable/executable-cache.js"; export { executableNativeEnvironmentFingerprint, diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7661-node-call-modules/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7661-node-call-modules/main.ts.json new file mode 100644 index 000000000..85440ddc4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7661-node-call-modules/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7661-node-call-modules/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7662-node-call-exit-events/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7662-node-call-exit-events/main.ts.json new file mode 100644 index 000000000..455735463 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7662-node-call-exit-events/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7662-node-call-exit-events/main.ts" + ], + "diags": [] +} diff --git a/packages/libnode-common/bridge/scr_node_bridge.cc b/packages/libnode-common/bridge/scr_node_bridge.cc index e770d140f..ef60851d6 100644 --- a/packages/libnode-common/bridge/scr_node_bridge.cc +++ b/packages/libnode-common/bridge/scr_node_bridge.cc @@ -49,6 +49,7 @@ node::EmbedderSnapshotData::Pointer g_snapshot; std::unique_ptr g_setup; Locker *g_locker = nullptr; Global g_call; +Global g_finish; char *Copy(const std::string &text, size_t *len) { char *out = static_cast(std::malloc(text.size() + 1)); @@ -59,6 +60,12 @@ char *Copy(const std::string &text, size_t *len) { return out; } +// The start of the runtime's stack when the caller runs on it (see the +// v8::base::Stack definitions at the end of this file), else 0. +uintptr_t HostStackStart() { + return g_host != nullptr && g_host->stack_start != nullptr ? g_host->stack_start() : 0; +} + Local Utf8(Isolate *isolate, const char *text, size_t len) { return String::NewFromUtf8(isolate, text, NewStringType::kNormal, static_cast(len)) .ToLocalChecked(); @@ -117,12 +124,18 @@ char *Start(const ScrNodeBridgeHost *host, const char *bootstrap, const char *ar if (dladdr(reinterpret_cast(&napi_create_function), &info) && info.dli_fname) dlopen(info.dli_fname, RTLD_NOW | RTLD_NOLOAD | RTLD_GLOBAL); std::vector args = {argv0}; + // The program is not a node process: NODE_OPTIONS from its environment + // does not apply, and Node leaves the process's resource limits and code + // pages as they are. auto result = node::InitializeOncePerProcess( args, {node::ProcessInitializationFlags::kNoInitializeV8, node::ProcessInitializationFlags::kNoInitializeNodeV8Platform, node::ProcessInitializationFlags::kNoStdioInitialization, node::ProcessInitializationFlags::kNoDefaultSignalHandling, - node::ProcessInitializationFlags::kNoPrintHelpOrVersionOutput}); + node::ProcessInitializationFlags::kNoPrintHelpOrVersionOutput, + node::ProcessInitializationFlags::kDisableNodeOptionsEnv, + node::ProcessInitializationFlags::kNoAdjustResourceLimits, + node::ProcessInitializationFlags::kNoUseLargePages}); if (result->early_return() || result->exit_code() != 0) { std::string message = "Node initialization failed"; for (const std::string &error : result->errors()) message += ": " + error; @@ -132,9 +145,12 @@ char *Start(const ScrNodeBridgeHost *host, const char *bootstrap, const char *ar v8::V8::InitializePlatform(g_platform.get()); v8::V8::Initialize(); g_snapshot = node::EmbedderSnapshotData::BuiltinSnapshotData(); + // Modules resolve inside the program's own module tree, never from + // NODE_PATH or the user's global module directories. auto flags = static_cast( node::EnvironmentFlags::kDefaultFlags | node::EnvironmentFlags::kNoCreateInspector | - node::EnvironmentFlags::kNoStartDebugSignalHandler); + node::EnvironmentFlags::kNoStartDebugSignalHandler | + node::EnvironmentFlags::kNoGlobalSearchPaths); std::vector errors; g_setup = g_snapshot ? node::CommonEnvironmentSetup::CreateFromSnapshot( g_platform.get(), &errors, g_snapshot.get(), result->args(), @@ -182,12 +198,16 @@ char *Start(const ScrNodeBridgeHost *host, const char *bootstrap, const char *ar } Local dispatcher; Local call; + Local finish; if (!global->Get(context, Utf8(isolate, "__scriptcNodeIsland")).ToLocal(&dispatcher) || !dispatcher->IsObject() || !dispatcher.As()->Get(context, Utf8(isolate, "call")).ToLocal(&call) || - !call->IsFunction()) + !call->IsFunction() || + !dispatcher.As()->Get(context, Utf8(isolate, "finish")).ToLocal(&finish) || + !finish->IsFunction()) return Copy("the Node island bootstrap left no dispatcher", nullptr); g_call.Reset(isolate, call.As()); + g_finish.Reset(isolate, finish.As()); global->Delete(context, Utf8(isolate, "__scriptcNodeIsland")).FromMaybe(false); global->Delete(context, Utf8(isolate, "__scriptcNodeIslandHost")).FromMaybe(false); return nullptr; @@ -254,11 +274,29 @@ void Pump() { g_platform->DrainTasks(isolate); } +// The program ends with `code`: Node's process ends with it too. Node's +// 'beforeExit' runs (and the work it schedules), then 'exit'; the result +// is the exit code Node's process ends with (process.exitCode). +int Finish(int code) { + if (!g_setup) return code; + Isolate *isolate = g_setup->isolate(); + { + HandleScope handles(isolate); + Local context = g_setup->context(); + Context::Scope context_scope(context); + TryCatch try_catch(isolate); + Local argv[] = {Integer::New(isolate, code)}; + (void)g_finish.Get(isolate)->Call(context, v8::Undefined(isolate), 1, argv); + } + return node::SpinEventLoop(g_setup->env()).FromMaybe(code); +} + void Stop() { if (!g_setup) return; Isolate *isolate = g_setup->isolate(); node::Stop(g_setup->env()); g_call.Reset(); + g_finish.Reset(); isolate->Exit(); delete g_locker; g_locker = nullptr; @@ -271,10 +309,37 @@ void Stop() { const ScrNodeBridge kBridge = { SCR_NODE_BRIDGE_ABI, NODE_VERSION_STRING, NODE_MODULE_VERSION, Start, Call, Release, BackendFd, BackendTimeout, Alive, Pump, Stop, + Finish, }; } // namespace +#if defined(__linux__) +// V8 asks v8::base::Stack for the start of the current thread's stack when +// it records the bound of its conservative stack scan (isolate creation, +// Locker). libnode calls these exported functions through its PLT, and this +// library precedes libnode in the lookup scope, so the definitions below +// answer for it: the runtime's stack while the caller runs on it, else +// libnode's own answer (RTLD_NEXT). Without them a garbage collection that +// scans the stack while JavaScript runs on the runtime's stack walks from +// that stack up to the main thread's and faults in the unmapped gap. +// (macOS binds libnode's calls inside libnode; its packs need another way.) +#define SCR_V8_STACK_START(fn, symbol) \ + extern "C" __attribute__((visibility("default"))) void *fn() __asm__(symbol); \ + extern "C" void *fn() { \ + static void *(*next)() = nullptr; \ + uintptr_t start = HostStackStart(); \ + if (start != 0) return reinterpret_cast(start); \ + if (next == nullptr) next = reinterpret_cast(dlsym(RTLD_NEXT, symbol)); \ + return next != nullptr ? next() : nullptr; \ + } +SCR_V8_STACK_START(ScrV8GetStackStart, "_ZN2v84base5Stack13GetStackStartEv") +SCR_V8_STACK_START(ScrV8GetStackStartUnchecked, "_ZN2v84base5Stack22GetStackStartUncheckedEv") +SCR_V8_STACK_START(ScrV8ObtainCurrentThreadStackStart, + "_ZN2v84base5Stack29ObtainCurrentThreadStackStartEv") +#undef SCR_V8_STACK_START +#endif + extern "C" __attribute__((visibility("default"))) const ScrNodeBridge *scr_node_bridge_v1() { return &kBridge; } diff --git a/packages/libnode-common/test/bridge.test.ts b/packages/libnode-common/test/bridge.test.ts index 1ab6ab95b..4c37b31a3 100644 --- a/packages/libnode-common/test/bridge.test.ts +++ b/packages/libnode-common/test/bridge.test.ts @@ -45,7 +45,7 @@ exports.quit = (code) => process.exit(code); expect(result.stderr).toBe(""); expect(result.stdout).toBe( [ - "abi 2 ok 1", + "abi 3 ok 1", "sync add -> R5", 'sync fails -> T{"error":{"kind":2,"message":"out of range","code":"ERR_X"}}', `sync missing -> T{"error":{"kind":1,"message":"'missing' exported by './smoke.cjs' is not a function"}}`, diff --git a/packages/node/README.md b/packages/node/README.md index 77002db49..7b1c3656d 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -10,6 +10,8 @@ const config = await call<{ regions: string[] }>("config-parser", "parse", sourc ``` - The module loads with `require()` from the calling module: relative specifiers resolve from its directory, bare ones from its `node_modules`. +- In scriptc builds the specifier is a string literal. The build collects the module into the executable: a module of the program with the modules its string-literal `require()` and `import` reach, or a whole package with its dependencies. The executable loads them from its own copy, never from the build machine's files. A module of the program that loads others through computed specifiers is refused (SC8006); a module the build cannot find throws `MODULE_NOT_FOUND` when the call runs. +- The executable extracts its modules into `$XDG_CACHE_HOME/scriptc/node-islands/` (or `~/.cache/scriptc/node-islands/`) and checks every file against the build's digests on each start. With `--libnode=beside` they are written next to the executable instead, in `.libnode/modules/`. - Arguments and results cross as JSON copies; scriptc checks the result against the declared type. - A thrown `Error` crosses as its nearest intrinsic error class with its `message` and string `code`. - The executable loads Node from the libnode pack recorded at build time (`SCRIPTC_LIBNODE` or `scriptc build --libnode=`; `--libnode=beside` copies the pack next to the executable). `SCRIPTC_LIBNODE` overrides it at run time. A program that never calls into Node never loads it. diff --git a/packages/node/index.d.ts b/packages/node/index.d.ts index 10642b098..87a8eb541 100644 --- a/packages/node/index.d.ts +++ b/packages/node/index.d.ts @@ -9,6 +9,8 @@ * - the module loads with require() from the calling module, so a relative * specifier resolves from the caller's directory and a bare one from its * node_modules (ES modules load through require(esm)); + * - in scriptc builds `specifier` is a string literal: the build ships the + * module, and what it loads, inside the executable; * - the arguments cross as a copy, `JSON.parse(JSON.stringify(args))`, and * the function runs with the module namespace as `this`; * - the result crosses as its JSON copy (undefined stays undefined), and diff --git a/packages/runtime/src/scr_node_bridge.h b/packages/runtime/src/scr_node_bridge.h index 8ccdeaf8d..f88764691 100644 --- a/packages/runtime/src/scr_node_bridge.h +++ b/packages/runtime/src/scr_node_bridge.h @@ -16,7 +16,7 @@ #include #include -#define SCR_NODE_BRIDGE_ABI 2u +#define SCR_NODE_BRIDGE_ABI 3u /* The symbol a bridge library exports: `const ScrNodeBridge *(void)`. */ #define SCR_NODE_BRIDGE_ENTRY "scr_node_bridge_v1" @@ -27,6 +27,12 @@ typedef struct ScrNodeBridgeHost { /* Node ends the process (process.exit, an uncaught exception): the * runtime runs its exit sequence with this code and never returns. */ void (*exit)(int code); + /* The start (highest address) of the stack the caller runs on when the + * runtime provides it (Node's own stack), 0 for a thread's own stack. + * V8 scans the stack conservatively from the current position up to the + * start it records for the thread, so it must not record the thread's + * when JavaScript runs on the runtime's stack. */ + uintptr_t (*stack_start)(void); } ScrNodeBridgeHost; typedef struct ScrNodeBridge { @@ -58,6 +64,10 @@ typedef struct ScrNodeBridge { void (*pump)(void); /* Tears Node down at process exit. */ void (*stop)(void); + /* The program ends with `code`, before stop: Node's 'beforeExit' and + * 'exit' run, and the result is the code Node's process ends with + * (process.exitCode in Node, else `code`). */ + int (*finish)(int code); } ScrNodeBridge; #endif /* SCR_NODE_BRIDGE_H */ diff --git a/packages/runtime/src/scr_node_island.c b/packages/runtime/src/scr_node_island.c index 55ef2ff9d..e32f8da41 100644 --- a/packages/runtime/src/scr_node_island.c +++ b/packages/runtime/src/scr_node_island.c @@ -45,8 +45,12 @@ #if !defined(_WIN32) && !defined(__wasi__) #include "scr_node_island_host.h" +#include #include +#include +#include #include +#include #include #include #if defined(__linux__) @@ -92,6 +96,15 @@ void scr_node_island_install(const char *libnode, const char *node_version) { SCR_COLD static void ni_oom(void) { scr_trap("scriptc: out of memory\n"); } +/* `len` bytes as a NUL-terminated malloc'd copy. */ +static char *ni_cstr_of(const char *bytes, size_t len) { + char *copy = malloc(len + 1); + if (copy == NULL) ni_oom(); + memcpy(copy, bytes, len); + copy[len] = '\0'; + return copy; +} + SCR_COLD static void ni_fatal(const char *message) { fflush(stdout); fputs(message, stderr); @@ -149,6 +162,550 @@ static const char *ni_pack_dir(void) { return ni_dir; } +/* ── the island module tree ───────────────────────────────────────────── */ + +/* The JavaScript the program's calls load (node-islands/modules.ts), named + * by a manifest of relative paths with their sizes and SHA-256 digests, plus + * the links between package directories. Before Node loads anything, the + * tree is made available at one directory, and every file is checked against + * the manifest: + * - beside the executable (`--libnode=beside`), at the recorded `$ORIGIN/` + * path, checked on every start; + * - otherwise from the executable itself, extracted once into + * $XDG_CACHE_HOME/scriptc/node-islands// (or + * ~/.cache/...), checked on every start; when that directory cannot be + * used, into a private directory under $TMPDIR removed at exit. + * Directories are created 0700 and owned by the process's user; no link is + * followed below the cache directory. A process with elevated privileges + * refuses the tree, as it refuses SCRIPTC_LIBNODE. A call names its module + * by its path relative to the tree. */ +static const char *ni_manifest = NULL; +static size_t ni_manifest_len = 0; +static const unsigned char *ni_modules = NULL; +static size_t ni_modules_len = 0; +static const char *ni_modules_location = NULL; +/* The tree's directory at run time; "/" when the calls load only Node's + * built-in modules. */ +static char ni_root[4096] = "/"; + +void scr_node_island_install_modules(const char *manifest, size_t manifest_len, + const unsigned char *data, size_t data_len, + const char *location) { + ni_manifest = manifest; + ni_manifest_len = manifest_len; + ni_modules = data; + ni_modules_len = data_len; + ni_modules_location = location; +} + +/* SHA-256 (FIPS 180-4), for the manifest's digests. */ +typedef struct { + uint32_t h[8]; + uint64_t bytes; + unsigned char tail[64]; + size_t tail_len; +} NiSha256; + +static const uint32_t ni_sha256_k[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + +static uint32_t ni_rotr(uint32_t x, unsigned n) { return (x >> n) | (x << (32 - n)); } + +static void ni_sha256_block(uint32_t h[8], const unsigned char *p) { + uint32_t w[64]; + for (int i = 0; i < 16; i++) + w[i] = ((uint32_t)p[i * 4] << 24) | ((uint32_t)p[i * 4 + 1] << 16) | + ((uint32_t)p[i * 4 + 2] << 8) | (uint32_t)p[i * 4 + 3]; + for (int i = 16; i < 64; i++) { + uint32_t s0 = ni_rotr(w[i - 15], 7) ^ ni_rotr(w[i - 15], 18) ^ (w[i - 15] >> 3); + uint32_t s1 = ni_rotr(w[i - 2], 17) ^ ni_rotr(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + uint32_t a = h[0], b = h[1], c = h[2], d = h[3], e = h[4], f = h[5], g = h[6], k = h[7]; + for (int i = 0; i < 64; i++) { + uint32_t t1 = k + (ni_rotr(e, 6) ^ ni_rotr(e, 11) ^ ni_rotr(e, 25)) + ((e & f) ^ (~e & g)) + + ni_sha256_k[i] + w[i]; + uint32_t t2 = (ni_rotr(a, 2) ^ ni_rotr(a, 13) ^ ni_rotr(a, 22)) + ((a & b) ^ (a & c) ^ (b & c)); + k = g; + g = f; + f = e; + e = d + t1; + d = c; + c = b; + b = a; + a = t1 + t2; + } + h[0] += a; + h[1] += b; + h[2] += c; + h[3] += d; + h[4] += e; + h[5] += f; + h[6] += g; + h[7] += k; +} + +static void ni_sha256_init(NiSha256 *s) { + static const uint32_t initial[8] = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + memcpy(s->h, initial, sizeof initial); + s->bytes = 0; + s->tail_len = 0; +} + +static void ni_sha256_update(NiSha256 *s, const unsigned char *data, size_t len) { + s->bytes += len; + while (len > 0) { + if (s->tail_len == 0 && len >= 64) { + ni_sha256_block(s->h, data); + data += 64; + len -= 64; + continue; + } + size_t take = 64 - s->tail_len < len ? 64 - s->tail_len : len; + memcpy(s->tail + s->tail_len, data, take); + s->tail_len += take; + data += take; + len -= take; + if (s->tail_len == 64) { + ni_sha256_block(s->h, s->tail); + s->tail_len = 0; + } + } +} + +static void ni_sha256_final(NiSha256 *s, unsigned char out[32]) { + uint64_t bits = s->bytes * 8; + unsigned char pad[72] = {0x80}; + size_t padding = (s->tail_len < 56 ? 56 : 120) - s->tail_len; + for (int i = 0; i < 8; i++) pad[padding + i] = (unsigned char)(bits >> (56 - 8 * i)); + ni_sha256_update(s, pad, padding + 8); + for (int i = 0; i < 8; i++) + for (int b = 0; b < 4; b++) out[i * 4 + b] = (unsigned char)(s->h[i] >> (24 - 8 * b)); +} + +typedef struct { + char *path; + size_t size; + unsigned char sha256[32]; + size_t offset; /* into ni_modules */ +} NiFile; + +typedef struct { + char *path, *target; +} NiLink; + +static NiFile *ni_files = NULL; +static size_t ni_nfiles = 0; +static NiLink *ni_links = NULL; +static size_t ni_nlinks = 0; + +SCR_COLD static void ni_bad_manifest(void) { + ni_fatal("scriptc: this program's Node island module manifest is malformed\n"); +} + +static char *ni_dup_str(const ScrDyn *d) { + if (d == NULL || d->kind != SCR_DYN_STR) ni_bad_manifest(); + if (memchr(d->v.str->data, '\0', d->v.str->len) != NULL) ni_bad_manifest(); + return ni_cstr_of(d->v.str->data, d->v.str->len); +} + +/* A relative path inside the tree: no leading `/`, no empty, `.` or `..` + * component. */ +static bool ni_relative_inside(const char *path) { + if (path[0] == '\0' || path[0] == '/') return false; + for (const char *p = path;;) { + const char *slash = strchr(p, '/'); + size_t n = slash != NULL ? (size_t)(slash - p) : strlen(p); + if (n == 0 || (n == 1 && p[0] == '.') || (n == 2 && p[0] == '.' && p[1] == '.')) return false; + if (slash == NULL) return true; + p = slash + 1; + } +} + +/* A link target that, read from the link's directory, stays inside the + * tree. */ +static bool ni_link_inside(const char *link, const char *target) { + if (target[0] == '\0' || target[0] == '/') return false; + long depth = 0; + for (const char *p = link; (p = strchr(p, '/')) != NULL; p++) depth++; + for (const char *p = target;;) { + const char *slash = strchr(p, '/'); + size_t n = slash != NULL ? (size_t)(slash - p) : strlen(p); + if (n == 2 && p[0] == '.' && p[1] == '.') { + if (--depth < 0) return false; + } else if (n == 0 || !(n == 1 && p[0] == '.')) { + if (n == 0) return false; + depth++; + } + if (slash == NULL) return true; + p = slash + 1; + } +} + +static int ni_hex(char c) { + return c >= '0' && c <= '9' ? c - '0' : c >= 'a' && c <= 'f' ? c - 'a' + 10 : -1; +} + +static void ni_parse_manifest(void) { + ScrStr *text = scr_str_new(ni_manifest, ni_manifest_len); + ScrDyn *body = scr_json_parse(text); + scr_str_release(text); + if (body == NULL || body->kind != SCR_DYN_OBJ) ni_bad_manifest(); + ScrDyn *files = scr_dyn_obj_get(body, "files", 5); + ScrDyn *links = scr_dyn_obj_get(body, "links", 5); + if (files == NULL || files->kind != SCR_DYN_ARR || links == NULL || links->kind != SCR_DYN_ARR) + ni_bad_manifest(); + ni_nfiles = (size_t)scr_dyn_arr_len(files); + ni_nlinks = (size_t)scr_dyn_arr_len(links); + ni_files = calloc(ni_nfiles + 1, sizeof *ni_files); + ni_links = calloc(ni_nlinks + 1, sizeof *ni_links); + if (ni_files == NULL || ni_links == NULL) ni_oom(); + size_t offset = 0; + for (size_t i = 0; i < ni_nfiles; i++) { + ScrDyn *entry = scr_dyn_arr_at(files, (double)i); + if (entry->kind != SCR_DYN_ARR || scr_dyn_arr_len(entry) != 3) ni_bad_manifest(); + ScrDyn *path = scr_dyn_arr_at(entry, 0), *size = scr_dyn_arr_at(entry, 1), + *digest = scr_dyn_arr_at(entry, 2); + NiFile *f = &ni_files[i]; + f->path = ni_dup_str(path); + if (!ni_relative_inside(f->path) || size->kind != SCR_DYN_NUM || size->v.num < 0 || + digest->kind != SCR_DYN_STR || digest->v.str->len != 64) + ni_bad_manifest(); + f->size = (size_t)size->v.num; + for (int b = 0; b < 32; b++) { + int hi = ni_hex(digest->v.str->data[b * 2]), lo = ni_hex(digest->v.str->data[b * 2 + 1]); + if (hi < 0 || lo < 0) ni_bad_manifest(); + f->sha256[b] = (unsigned char)(hi * 16 + lo); + } + f->offset = offset; + offset += f->size; + scr_dyn_release(path); + scr_dyn_release(size); + scr_dyn_release(digest); + scr_dyn_release(entry); + } + if (ni_modules != NULL && offset != ni_modules_len) ni_bad_manifest(); + for (size_t i = 0; i < ni_nlinks; i++) { + ScrDyn *entry = scr_dyn_arr_at(links, (double)i); + if (entry->kind != SCR_DYN_ARR || scr_dyn_arr_len(entry) != 2) ni_bad_manifest(); + ScrDyn *path = scr_dyn_arr_at(entry, 0), *target = scr_dyn_arr_at(entry, 1); + ni_links[i].path = ni_dup_str(path); + ni_links[i].target = ni_dup_str(target); + if (!ni_relative_inside(ni_links[i].path) || + !ni_link_inside(ni_links[i].path, ni_links[i].target)) + ni_bad_manifest(); + scr_dyn_release(path); + scr_dyn_release(target); + scr_dyn_release(entry); + } + scr_dyn_release(body); +} + +/* The directory `name` in `dirfd`, never through a link; created 0700 when + * `create` and absent. -1 when it cannot be opened. */ +static int ni_dir_at(int dirfd, const char *name, bool create) { + if (create && mkdirat(dirfd, name, 0700) != 0 && errno != EEXIST) return -1; + return openat(dirfd, name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); +} + +/* The parent directory of the relative `path` below `rootfd` (its last + * component is left in *base), opened component by component without + * following links; created when `create`. -1 when it cannot be opened. */ +static int ni_parent_at(int rootfd, const char *path, bool create, const char **base) { + int fd = dup(rootfd); + const char *p = path; + for (const char *slash; fd >= 0 && (slash = strchr(p, '/')) != NULL; p = slash + 1) { + char component[256]; + size_t n = (size_t)(slash - p); + if (n >= sizeof component) { + close(fd); + return -1; + } + memcpy(component, p, n); + component[n] = '\0'; + int next = ni_dir_at(fd, component, create); + close(fd); + fd = next; + } + *base = p; + return fd; +} + +/* A directory this process may trust: owned by its user, writable by no + * one else. */ +static bool ni_private_dir(int fd) { + struct stat st; + return fd >= 0 && fstat(fd, &st) == 0 && S_ISDIR(st.st_mode) && st.st_uid == geteuid() && + (st.st_mode & 022) == 0; +} + +static bool ni_digest_matches(const unsigned char *data, size_t len, + const unsigned char expected[32]) { + NiSha256 s; + unsigned char got[32]; + ni_sha256_init(&s); + ni_sha256_update(&s, data, len); + ni_sha256_final(&s, got); + return memcmp(got, expected, 32) == 0; +} + +/* Whether the tree below `rootfd` holds exactly the manifest's files (by + * size and digest) and links. */ +static bool ni_tree_matches(int rootfd) { + for (size_t i = 0; i < ni_nfiles; i++) { + const char *base; + int dirfd = ni_parent_at(rootfd, ni_files[i].path, false, &base); + if (dirfd < 0) return false; + int fd = openat(dirfd, base, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + close(dirfd); + struct stat st; + if (fd < 0 || fstat(fd, &st) != 0 || !S_ISREG(st.st_mode) || + (size_t)st.st_size != ni_files[i].size) { + if (fd >= 0) close(fd); + return false; + } + NiSha256 s; + ni_sha256_init(&s); + unsigned char buffer[65536]; + size_t total = 0; + for (;;) { + ssize_t n = read(fd, buffer, sizeof buffer); + if (n < 0 && errno == EINTR) continue; + if (n <= 0) break; + ni_sha256_update(&s, buffer, (size_t)n); + total += (size_t)n; + } + close(fd); + unsigned char got[32]; + ni_sha256_final(&s, got); + if (total != ni_files[i].size || memcmp(got, ni_files[i].sha256, 32) != 0) return false; + } + for (size_t i = 0; i < ni_nlinks; i++) { + const char *base; + int dirfd = ni_parent_at(rootfd, ni_links[i].path, false, &base); + if (dirfd < 0) return false; + char target[4096]; + ssize_t n = readlinkat(dirfd, base, target, sizeof target - 1); + close(dirfd); + if (n < 0) return false; + target[n] = '\0'; + if (strcmp(target, ni_links[i].target) != 0) return false; + } + return true; +} + +/* Writes the executable's tree below `rootfd` (an empty directory). */ +static bool ni_extract(int rootfd) { + for (size_t i = 0; i < ni_nfiles; i++) { + const unsigned char *data = ni_modules + ni_files[i].offset; + if (!ni_digest_matches(data, ni_files[i].size, ni_files[i].sha256)) return false; + const char *base; + int dirfd = ni_parent_at(rootfd, ni_files[i].path, true, &base); + if (dirfd < 0) return false; + int fd = openat(dirfd, base, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC, 0600); + close(dirfd); + if (fd < 0) return false; + size_t done = 0; + while (done < ni_files[i].size) { + ssize_t n = write(fd, data + done, ni_files[i].size - done); + if (n < 0 && errno == EINTR) continue; + if (n <= 0) break; + done += (size_t)n; + } + if (close(fd) != 0 || done != ni_files[i].size) return false; + } + for (size_t i = 0; i < ni_nlinks; i++) { + const char *base; + int dirfd = ni_parent_at(rootfd, ni_links[i].path, true, &base); + if (dirfd < 0) return false; + int status = symlinkat(ni_links[i].target, dirfd, base); + close(dirfd); + if (status != 0) return false; + } + return true; +} + +/* Removes everything below `dirfd` and closes it, never following links. */ +static void ni_remove_below(int dirfd) { + DIR *dir = fdopendir(dirfd); + if (dir == NULL) { + close(dirfd); + return; + } + for (struct dirent *e; (e = readdir(dir)) != NULL;) { + if (strcmp(e->d_name, ".") == 0 || strcmp(e->d_name, "..") == 0) continue; + int sub = openat(dirfd, e->d_name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + if (sub >= 0) { + ni_remove_below(sub); + (void)unlinkat(dirfd, e->d_name, AT_REMOVEDIR); + } else { + (void)unlinkat(dirfd, e->d_name, 0); + } + } + closedir(dir); +} + + +static char ni_temp_root[4096] = ""; + +static void ni_remove_temp_root(void) { + int fd = open(ni_temp_root, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + if (fd >= 0) ni_remove_below(fd); + (void)rmdir(ni_temp_root); +} + +static bool ni_set_root(const char *a, const char *b, const char *c) { + int n = snprintf(ni_root, sizeof ni_root, "%s%s%s", a, b, c); + return n > 0 && (size_t)n < sizeof ni_root; +} + +/* The tree in the cache directory, extracted there when absent. */ +static bool ni_cached_root(const char *digest) { + const char *xdg = getenv("XDG_CACHE_HOME"); + const char *home = getenv("HOME"); + char base[4096]; + int n; + if (xdg != NULL && xdg[0] == '/') + n = snprintf(base, sizeof base, "%s", xdg); + else if (home != NULL && home[0] == '/') + n = snprintf(base, sizeof base, "%s/.cache", home); + else + return false; + if (n <= 0 || (size_t)n >= sizeof base) return false; + (void)mkdir(base, 0700); + int basefd = open(base, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + if (basefd < 0) return false; + int scriptc = ni_dir_at(basefd, "scriptc", true); + close(basefd); + int islands = ni_private_dir(scriptc) ? ni_dir_at(scriptc, "node-islands", true) : -1; + if (scriptc >= 0) close(scriptc); + if (!ni_private_dir(islands)) { + if (islands >= 0) close(islands); + return false; + } + bool ok = false; + int tree = ni_dir_at(islands, digest, false); + if (tree < 0 && errno == ENOENT) { + /* Extract into a fresh sibling, then move it into place. */ + char staging[128]; + snprintf(staging, sizeof staging, "%s.%ld.tmp", digest, (long)getpid()); + int fd = ni_dir_at(islands, staging, true); + if (ni_private_dir(fd) && ni_extract(fd) && ni_tree_matches(fd) && + renameat(islands, staging, islands, digest) == 0) { + close(fd); + tree = ni_dir_at(islands, digest, false); + } else { + if (fd >= 0) ni_remove_below(fd); + (void)unlinkat(islands, staging, AT_REMOVEDIR); + tree = ni_dir_at(islands, digest, false); /* another process may have placed it */ + } + } + if (ni_private_dir(tree) && ni_tree_matches(tree)) + ok = ni_set_root(base, "/scriptc/node-islands/", digest); + if (tree >= 0) close(tree); + close(islands); + return ok; +} + +/* The tree in a private directory of this run, removed at exit. */ +static bool ni_temporary_root(void) { + const char *tmp = getenv("TMPDIR"); + if (tmp == NULL || tmp[0] != '/') tmp = "/tmp"; + int n = snprintf(ni_temp_root, sizeof ni_temp_root, "%s/scriptc-node-islands-XXXXXX", tmp); + if (n <= 0 || (size_t)n >= sizeof ni_temp_root || mkdtemp(ni_temp_root) == NULL) return false; + scr_atexit(ni_remove_temp_root); + int fd = open(ni_temp_root, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + bool ok = ni_private_dir(fd) && ni_extract(fd) && ni_tree_matches(fd); + if (fd >= 0) close(fd); + return ok && ni_set_root(ni_temp_root, "", ""); +} + +/* Makes the tree available and sets ni_root; fatal when it cannot. */ +static void ni_open_modules(void) { + if (ni_manifest == NULL) return; /* only built-in modules */ + if (ni_secure()) + ni_fatal("scriptc: a program running with elevated privileges cannot load Node island " + "modules\n"); + ni_parse_manifest(); + if (ni_modules_location != NULL && ni_modules_location[0] != '\0') { + static const char origin[] = "$ORIGIN/"; + if (strncmp(ni_modules_location, origin, sizeof origin - 1) != 0) ni_bad_manifest(); + ScrStr *self = scr_process_exec_path(); + size_t len = self->len; + while (len > 0 && self->data[len - 1] != '/') len--; + bool absolute = len > 0 && self->data[0] == '/'; + char dir[4096]; + int n = snprintf(dir, sizeof dir, "%.*s", (int)len, self->data); + scr_str_release(self); + if (!absolute || n <= 0 || (size_t)n >= sizeof dir || + !ni_set_root(dir, ni_modules_location + sizeof origin - 1, "")) + ni_fatal("scriptc: cannot locate this program's Node island modules\n"); + int fd = open(ni_root, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + bool ok = fd >= 0 && ni_tree_matches(fd); + if (fd >= 0) close(fd); + if (!ok) { + char message[8192]; + snprintf(message, sizeof message, + "scriptc: the Node island modules at %s do not match this program\n", ni_root); + ni_fatal(message); + } + return; + } + NiSha256 s; + unsigned char digest[32]; + ni_sha256_init(&s); + ni_sha256_update(&s, (const unsigned char *)ni_manifest, ni_manifest_len); + ni_sha256_final(&s, digest); + char hex[65]; + for (int i = 0; i < 32; i++) snprintf(hex + i * 2, 3, "%02x", digest[i]); + if (ni_modules == NULL && ni_nfiles > 0) ni_bad_manifest(); + if (!ni_cached_root(hex) && !ni_temporary_root()) + ni_fatal("scriptc: cannot write this program's Node island modules to a private directory\n"); +} + +/* The calling module's path inside the tree: `/`. */ +static char *ni_caller_path(const ScrStr *relative) { + size_t root = strlen(ni_root); + bool slash = root > 0 && ni_root[root - 1] == '/'; + char *path = malloc(root + 1 + relative->len + 1); + if (path == NULL) ni_oom(); + memcpy(path, ni_root, root); + if (!slash) path[root++] = '/'; + memcpy(path + root, relative->data, relative->len); + path[root + relative->len] = '\0'; + return path; +} + +/* The bootstrap: the tree root for the host's resolution guard, then the + * host script. +1 malloc'd. */ +static char *ni_bootstrap(void) { + size_t host = strlen(scr_node_island_host_js); + size_t cap = 64 + strlen(ni_root) * 6 + host; + char *out = malloc(cap); + if (out == NULL) ni_oom(); + size_t n = (size_t)snprintf(out, cap, "globalThis.__scriptcNodeIslandRoot = \""); + for (const char *p = ni_root; *p != '\0'; p++) { + unsigned char c = (unsigned char)*p; + if (c == '"' || c == '\\') + n += (size_t)snprintf(out + n, cap - n, "\\%c", c); + else if (c < 0x20) + n += (size_t)snprintf(out + n, cap - n, "\\u%04x", c); + else + out[n++] = (char)c; + } + n += (size_t)snprintf(out + n, cap - n, "\";\n"); + memcpy(out + n, scr_node_island_host_js, host + 1); + return out; +} + /* ── outcomes ─────────────────────────────────────────────────────────── */ /* The result of an "R" outcome: its JSON copy, or undefined. +1, or NULL @@ -215,7 +772,10 @@ static void ni_exit(int code) { ni_fatal(""); } -static const ScrNodeBridgeHost ni_host = {&ni_settle, &ni_exit}; +/* The start of Node's stack for callers on it (V8's stack-scan bound). */ +static uintptr_t ni_stack_start(void); + +static const ScrNodeBridgeHost ni_host = {&ni_settle, &ni_exit, &ni_stack_start}; /* ── the loop slot ────────────────────────────────────────────────────── */ @@ -226,7 +786,27 @@ static void ni_on_node_stack(void (*job)(void *), void *arg); static void ni_pump(void) { ni_on_node_stack(ni_pump_job, NULL); } static int ni_backend_fd(void) { return ni_bridge->backend_fd(); } static int ni_backend_timeout(void) { return ni_bridge->backend_timeout(); } -static void ni_stop_atexit(void) { ni_on_node_stack(ni_stop_job, NULL); } +typedef struct { + int code; +} NiFinish; +static void ni_finish_job(void *p); + +/* At exit, after the program's own exit work: Node's process ends with + * the program's code ('beforeExit', then 'exit'), and Node stops. A code + * Node's process changed (process.exitCode in Node) is the program's. */ +static void ni_stop_atexit(void) { + int code = (int)scr_process_exit_code_or_zero(); + NiFinish finish = {code}; + /* Node writes to the same stdout. */ + fflush(stdout); + ni_on_node_stack(ni_finish_job, &finish); + ni_on_node_stack(ni_stop_job, NULL); + if (finish.code != code) { + fflush(NULL); + if (ni_temp_root[0] != '\0') ni_remove_temp_root(); + _Exit(finish.code & 0xff); + } +} /* ── Node's stack ─────────────────────────────────────────────────────── */ @@ -254,6 +834,13 @@ static void ni_trampoline(void) { } } +static uintptr_t ni_stack_start(void) { + uintptr_t sp = (uintptr_t)__builtin_frame_address(0); + return ni_stack != NULL && sp >= (uintptr_t)ni_stack && sp < (uintptr_t)ni_stack + NI_STACK + ? (uintptr_t)ni_stack + NI_STACK + : 0; +} + /* Runs job(arg) on Node's stack (directly when already there). */ static void ni_on_node_stack(void (*job)(void *), void *arg) { if (ni_on_stack) { @@ -329,11 +916,17 @@ static void ni_stop_job(void *p) { ni_bridge->stop(); } +static void ni_finish_job(void *p) { + NiFinish *a = p; + a->code = ni_bridge->finish(a->code); +} + /* ── loading Node ─────────────────────────────────────────────────────── */ static void ni_start(void) { if (ni_bridge != NULL) return; const char *dir = ni_pack_dir(); + ni_open_modules(); char path[4352]; int written = snprintf(path, sizeof path, "%s/%s", dir, NI_BRIDGE_FILE); if (written < 0 || (size_t)written >= sizeof path) @@ -374,8 +967,10 @@ static void ni_start(void) { /* Flush first: Node writes to the same stdout. */ fflush(stdout); ni_bridge = bridge; - NiStart start = {scr_node_island_host_js, argv0, NULL}; + char *bootstrap = ni_bootstrap(); + NiStart start = {bootstrap, argv0, NULL}; ni_on_node_stack(ni_start_job, &start); + free(bootstrap); char *error = start.error; if (error != NULL) { char message[8192]; @@ -401,17 +996,11 @@ static ScrDyn *ni_args_json(ScrDyn *args) { } /* A borrowed ScrStr as a NUL-terminated copy for the bridge. */ -static char *ni_cstr(const ScrStr *s) { - char *copy = malloc(s->len + 1); - if (copy == NULL) ni_oom(); - memcpy(copy, s->data, s->len); - copy[s->len] = '\0'; - return copy; -} +static char *ni_cstr(const ScrStr *s) { return ni_cstr_of(s->data, s->len); } static char *ni_call(uint32_t id, bool sync, ScrStr *file, ScrStr *specifier, ScrStr *name, ScrDyn *json, size_t *len) { - char *f = ni_cstr(file), *s = ni_cstr(specifier), *n = ni_cstr(name); + char *f = ni_caller_path(file), *s = ni_cstr(specifier), *n = ni_cstr(name); NiCall call = {id, sync ? 1 : 0, f, s, n, json->v.str->data, json->v.str->len, 0, NULL}; ni_on_node_stack(ni_call_job, &call); char *outcome = call.outcome; @@ -491,6 +1080,16 @@ void scr_node_island_install(const char *libnode, const char *node_version) { (void)node_version; } +void scr_node_island_install_modules(const char *manifest, size_t manifest_len, + const unsigned char *data, size_t data_len, + const char *location) { + (void)manifest; + (void)manifest_len; + (void)data; + (void)data_len; + (void)location; +} + SCR_COLD static void ni_unsupported(void) { fputs("scriptc: Node islands are not supported on this target\n", stderr); fflush(stderr); diff --git a/packages/runtime/src/scr_node_island.h b/packages/runtime/src/scr_node_island.h index 9904047f5..33d1d17b4 100644 --- a/packages/runtime/src/scr_node_island.h +++ b/packages/runtime/src/scr_node_island.h @@ -14,6 +14,15 @@ * before %main. */ void scr_node_island_install(const char *libnode, const char *node_version); +/* Records the island module tree (node-islands/modules.ts): its manifest, + * and its files concatenated in manifest order when the executable carries + * them (NULL when they ship beside it at `location`, a `$ORIGIN/` path; "" + * otherwise). All static. The emitted main calls it after + * scr_node_island_install. */ +void scr_node_island_install_modules(const char *manifest, size_t manifest_len, + const unsigned char *data, size_t data_len, + const char *location); + /* @scriptc/node callSync(specifier, name, ...args): `file` is the calling * module's path (relative specifiers resolve from it), `args` the boxed * argument array, copied as JSON. Returns the JSON copy of the result, +1, diff --git a/packages/runtime/src/scr_node_island_host.cjs b/packages/runtime/src/scr_node_island_host.cjs index d98b1c3ad..90c6187ab 100644 --- a/packages/runtime/src/scr_node_island_host.cjs +++ b/packages/runtime/src/scr_node_island_host.cjs @@ -13,10 +13,36 @@ * them in-process under Node: both must produce the same results, errors * and messages. Outcomes are "R" (returned; "R" alone is undefined) * or "T" (threw), the encoding of scr_node_bridge.h. */ -const { createRequire } = require("node:module"); +const { createRequire, registerHooks } = require("node:module"); const nativeHost = globalThis.__scriptcNodeIslandHost; +// The program's island module tree (scr_node_island.c prefixes this script +// with its root, then the bridge removes the global): modules resolve only +// to files inside it, never to a file elsewhere on the machine. +const islandRoot = globalThis.__scriptcNodeIslandRoot; +delete globalThis.__scriptcNodeIslandRoot; +if (typeof islandRoot === "string" && islandRoot !== "/") { + const { realpathSync } = require("node:fs"); + const { sep } = require("node:path"); + const { fileURLToPath } = require("node:url"); + const root = realpathSync(islandRoot); + const inside = root.endsWith(sep) ? root : root + sep; + registerHooks({ + resolve(specifier, context, nextResolve) { + const result = nextResolve(specifier, context); + if (result.url.startsWith("file:") && !fileURLToPath(result.url).startsWith(inside)) { + const error = new Error( + `Cannot find module '${specifier}': it is not one of the modules this program ships`, + ); + error.code = "MODULE_NOT_FOUND"; + throw error; + } + return result; + }, + }); +} + // Error classes that cross as copies, at their scr_error.h kind index. const ERROR_KINDS = [ [TypeError, 1], @@ -135,4 +161,28 @@ for (const stream of [process.stdout, process.stderr]) { stream._handle.setBlocking(true); } -globalThis.__scriptcNodeIsland = { call }; +// The program ends with `code` (the bridge then runs Node's loop to its +// end, which emits 'beforeExit' and 'exit'): Node's process shares it, as +// it shares one process under Node. +function finish(code) { + if (code !== 0) process.exitCode = code; +} + +// process.execPath is this program, not a node executable: a fork without +// an execPath of its own would start the program again. +const childProcess = require("node:child_process"); +const nodeFork = childProcess.fork; +childProcess.fork = function fork(modulePath, args, options) { + const forkOptions = Array.isArray(args) ? options : args; + const execPath = + forkOptions === null || typeof forkOptions !== "object" ? undefined : forkOptions.execPath; + if (execPath === undefined || execPath === process.execPath) { + throw new Error( + "child_process.fork() in a Node island needs options.execPath (a node executable): process.execPath is this program", + ); + } + return Reflect.apply(nodeFork, this, arguments); +}; +require("node:module").syncBuiltinESMExports(); + +globalThis.__scriptcNodeIsland = { call, finish }; diff --git a/packages/runtime/src/scr_node_island_host.h b/packages/runtime/src/scr_node_island_host.h index 5d2286450..ed78c8951 100644 --- a/packages/runtime/src/scr_node_island_host.h +++ b/packages/runtime/src/scr_node_island_host.h @@ -19,10 +19,36 @@ static const char scr_node_island_host_js[] = " * them in-process under Node: both must produce the same results, errors\012" " * and messages. Outcomes are \"R\" (returned; \"R\" alone is undefined)\012" " * or \"T\" (threw), the encoding of scr_node_bridge.h. */\012" - "const { createRequire } = require(\"node:module\");\012" + "const { createRequire, registerHooks } = require(\"node:module\");\012" "\012" "const nativeHost = globalThis.__scriptcNodeIslandHost;\012" "\012" + "// The program's island module tree (scr_node_island.c prefixes this script\012" + "// with its root, then the bridge removes the global): modules resolve only\012" + "// to files inside it, never to a file elsewhere on the machine.\012" + "const islandRoot = globalThis.__scriptcNodeIslandRoot;\012" + "delete globalThis.__scriptcNodeIslandRoot;\012" + "if (typeof islandRoot === \"string\" && islandRoot !== \"/\") {\012" + " const { realpathSync } = require(\"node:fs\");\012" + " const { sep } = require(\"node:path\");\012" + " const { fileURLToPath } = require(\"node:url\");\012" + " const root = realpathSync(islandRoot);\012" + " const inside = root.endsWith(sep) \077 root : root + sep;\012" + " registerHooks({\012" + " resolve(specifier, context, nextResolve) {\012" + " const result = nextResolve(specifier, context);\012" + " if (result.url.startsWith(\"file:\") && !fileURLToPath(result.url).startsWith(inside)) {\012" + " const error = new Error(\012" + " `Cannot find module '${specifier}': it is not one of the modules this program ships`,\012" + " );\012" + " error.code = \"MODULE_NOT_FOUND\";\012" + " throw error;\012" + " }\012" + " return result;\012" + " },\012" + " });\012" + "}\012" + "\012" "// Error classes that cross as copies, at their scr_error.h kind index.\012" "const ERROR_KINDS = [\012" " [TypeError, 1],\012" @@ -141,6 +167,30 @@ static const char scr_node_island_host_js[] = " stream._handle.setBlocking(true);\012" "}\012" "\012" - "globalThis.__scriptcNodeIsland = { call };\012"; + "// The program ends with `code` (the bridge then runs Node's loop to its\012" + "// end, which emits 'beforeExit' and 'exit'): Node's process shares it, as\012" + "// it shares one process under Node.\012" + "function finish(code) {\012" + " if (code !== 0) process.exitCode = code;\012" + "}\012" + "\012" + "// process.execPath is this program, not a node executable: a fork without\012" + "// an execPath of its own would start the program again.\012" + "const childProcess = require(\"node:child_process\");\012" + "const nodeFork = childProcess.fork;\012" + "childProcess.fork = function fork(modulePath, args, options) {\012" + " const forkOptions = Array.isArray(args) \077 options : args;\012" + " const execPath =\012" + " forkOptions === null || typeof forkOptions !== \"object\" \077 undefined : forkOptions.execPath;\012" + " if (execPath === undefined || execPath === process.execPath) {\012" + " throw new Error(\012" + " \"child_process.fork() in a Node island needs options.execPath (a node executable): process.execPath is this program\",\012" + " );\012" + " }\012" + " return Reflect.apply(nodeFork, this, arguments);\012" + "};\012" + "require(\"node:module\").syncBuiltinESMExports();\012" + "\012" + "globalThis.__scriptcNodeIsland = { call, finish };\012"; #endif /* SCR_NODE_ISLAND_HOST_H */ diff --git a/tests/corpus/7661-node-call-modules/lib/entry.cjs b/tests/corpus/7661-node-call-modules/lib/entry.cjs new file mode 100644 index 000000000..3fa55af87 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/lib/entry.cjs @@ -0,0 +1,14 @@ +"use strict"; +const { label } = require("./util"); +const settings = require("../settings.json"); + +exports.describe = (name) => `${label(name)} (${settings.mode})`; + +exports.hasOptional = () => { + try { + require("./not-shipped.cjs"); + return true; + } catch (error) { + return error.code !== "MODULE_NOT_FOUND"; + } +}; diff --git a/tests/corpus/7661-node-call-modules/lib/esm.mjs b/tests/corpus/7661-node-call-modules/lib/esm.mjs new file mode 100644 index 000000000..467987471 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/lib/esm.mjs @@ -0,0 +1,5 @@ +import { factor } from "./factor.mjs"; + +export function twice(n) { + return n * factor; +} diff --git a/tests/corpus/7661-node-call-modules/lib/factor.mjs b/tests/corpus/7661-node-call-modules/lib/factor.mjs new file mode 100644 index 000000000..adb2e40cb --- /dev/null +++ b/tests/corpus/7661-node-call-modules/lib/factor.mjs @@ -0,0 +1 @@ +export const factor = 2; diff --git a/tests/corpus/7661-node-call-modules/lib/util.js b/tests/corpus/7661-node-call-modules/lib/util.js new file mode 100644 index 000000000..ea7711f79 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/lib/util.js @@ -0,0 +1,5 @@ +// An ES module by the nearest package.json ("type": "module"), loaded with +// require() from a CommonJS module. +export function label(name) { + return `module of ${name}`; +} diff --git a/tests/corpus/7661-node-call-modules/main.ts b/tests/corpus/7661-node-call-modules/main.ts new file mode 100644 index 000000000..7f6f4ac76 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/main.ts @@ -0,0 +1,12 @@ +// @requires-libnode +// The modules island calls load ship with the program: a module of the +// program and the modules it loads, and whole packages with their +// dependencies, including files a package reads or loads by computed names. +import { callSync } from "@scriptc/node"; + +console.log(callSync("./lib/entry.cjs", "describe", "scriptc")); +console.log(callSync("greeter", "greet", "world")); +console.log(callSync("greeter", "fromData")); +console.log(callSync("greeter", "byName", "shout", "computed")); +console.log(callSync("./lib/entry.cjs", "hasOptional")); +console.log(callSync("./lib/esm.mjs", "twice", 21)); diff --git a/tests/corpus/7661-node-call-modules/node_modules/greeter/data/message.txt b/tests/corpus/7661-node-call-modules/node_modules/greeter/data/message.txt new file mode 100644 index 000000000..8e4d1b0c2 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/node_modules/greeter/data/message.txt @@ -0,0 +1 @@ +read from a file the package ships diff --git a/tests/corpus/7661-node-call-modules/node_modules/greeter/index.js b/tests/corpus/7661-node-call-modules/node_modules/greeter/index.js new file mode 100644 index 000000000..4e3f7bc55 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/node_modules/greeter/index.js @@ -0,0 +1,8 @@ +"use strict"; +const fs = require("node:fs"); +const path = require("node:path"); +const shout = require("shout"); + +exports.greet = (who) => shout(`hello, ${who}`); +exports.fromData = () => fs.readFileSync(path.join(__dirname, "data", "message.txt"), "utf8").trim(); +exports.byName = (name, text) => require(name)(text); diff --git a/tests/corpus/7661-node-call-modules/node_modules/greeter/package.json b/tests/corpus/7661-node-call-modules/node_modules/greeter/package.json new file mode 100644 index 000000000..07b35f1af --- /dev/null +++ b/tests/corpus/7661-node-call-modules/node_modules/greeter/package.json @@ -0,0 +1 @@ +{ "name": "greeter", "version": "1.0.0", "main": "index.js", "dependencies": { "shout": "1.0.0" } } diff --git a/tests/corpus/7661-node-call-modules/node_modules/shout/index.js b/tests/corpus/7661-node-call-modules/node_modules/shout/index.js new file mode 100644 index 000000000..6401aa6da --- /dev/null +++ b/tests/corpus/7661-node-call-modules/node_modules/shout/index.js @@ -0,0 +1,2 @@ +"use strict"; +module.exports = (text) => `${text.toUpperCase()}!`; diff --git a/tests/corpus/7661-node-call-modules/node_modules/shout/package.json b/tests/corpus/7661-node-call-modules/node_modules/shout/package.json new file mode 100644 index 000000000..d32731e21 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/node_modules/shout/package.json @@ -0,0 +1 @@ +{ "name": "shout", "version": "1.0.0", "main": "index.js" } diff --git a/tests/corpus/7661-node-call-modules/settings.json b/tests/corpus/7661-node-call-modules/settings.json new file mode 100644 index 000000000..42201c528 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/settings.json @@ -0,0 +1 @@ +{ "mode": "shipped" } diff --git a/tests/corpus/7662-node-call-exit-events/listeners.cjs b/tests/corpus/7662-node-call-exit-events/listeners.cjs new file mode 100644 index 000000000..c21ad324f --- /dev/null +++ b/tests/corpus/7662-node-call-exit-events/listeners.cjs @@ -0,0 +1,13 @@ +"use strict"; +let beforeExit = 0; + +exports.register = () => { + process.on("beforeExit", (code) => { + beforeExit++; + console.log("beforeExit", code, beforeExit); + if (beforeExit === 1) setTimeout(() => console.log("work from beforeExit"), 1); + }); + process.on("exit", (code) => console.log("exit", code)); + process.exitCode = 4; + return true; +}; diff --git a/tests/corpus/7662-node-call-exit-events/main.ts b/tests/corpus/7662-node-call-exit-events/main.ts new file mode 100644 index 000000000..909550588 --- /dev/null +++ b/tests/corpus/7662-node-call-exit-events/main.ts @@ -0,0 +1,8 @@ +// @requires-libnode +// The program and the Node.js it loads end as one process: Node's +// 'beforeExit' and 'exit' run when the program ends, work that 'beforeExit' +// schedules runs first, and process.exitCode set in Node is the program's. +import { callSync } from "@scriptc/node"; + +console.log("registered", callSync("./listeners.cjs", "register")); +console.log("main done"); diff --git a/tests/fixtures/node-island/gc-stress.ts b/tests/fixtures/node-island/gc-stress.ts new file mode 100644 index 000000000..8c71099d1 --- /dev/null +++ b/tests/fixtures/node-island/gc-stress.ts @@ -0,0 +1,24 @@ +// Full garbage collections in Node during island calls +// (tests/harness/node-island.test.ts): from the main stack, from deep native +// recursion, from async functions and from timers, with deep JavaScript +// recursion on Node's side. +import { call, callSync } from "@scriptc/node"; + +function nested(depth: number): number { + if (depth === 0) return callSync("./gc.cjs", "deep", 2000); + return nested(depth - 1) + 1; +} + +async function inFiber(n: number): Promise { + const later = await call("./gc.cjs", "later", n); + return later + callSync("./gc.cjs", "churn", 1); +} + +console.log("churn", callSync("./gc.cjs", "churn", 20)); +console.log("nested", nested(3000)); +console.log("deep", callSync("./gc.cjs", "deep", 8000)); +const results = await Promise.all([1, 2, 3, 4].map((n) => inFiber(n))); +console.log("fibers", results.join(",")); +setTimeout(() => { + console.log("timer", callSync("./gc.cjs", "churn", 5)); +}, 1); diff --git a/tests/fixtures/node-island/gc.cjs b/tests/fixtures/node-island/gc.cjs new file mode 100644 index 000000000..f8f5a32fa --- /dev/null +++ b/tests/fixtures/node-island/gc.cjs @@ -0,0 +1,41 @@ +"use strict"; +// Garbage-collection pressure inside Node island calls +// (tests/harness/node-island.test.ts): full collections while the call +// runs on Node's stack, at depth on both sides. +const v8 = require("node:v8"); +const vm = require("node:vm"); +v8.setFlagsFromString("--expose-gc"); +const gc = vm.runInNewContext("gc"); + +const kept = []; + +// Allocates `rounds` batches of objects that survive until the next batch, +// with a full collection after each. +exports.churn = (rounds) => { + let total = 0; + for (let r = 0; r < rounds; r++) { + const batch = []; + for (let i = 0; i < 100000; i++) batch.push({ i, s: `item ${i}`, tags: [i, i + 1] }); + kept[r % 4] = batch; + total += batch.length; + gc(); + } + return total; +}; + +// Recurses `depth` frames in JavaScript and collects at the bottom. +exports.deep = (depth) => { + if (depth === 0) { + gc(); + return 0; + } + const local = { depth, text: String(depth) }; + return 1 + exports.deep(depth - 1) + (local.depth > 0 ? 0 : 1); +}; + +// Collects after a timer, from Node's loop. +exports.later = async (n) => { + await new Promise((resolve) => setTimeout(resolve, 1)); + exports.churn(1); + return n; +}; diff --git a/tests/fixtures/node-island/helper.cjs b/tests/fixtures/node-island/helper.cjs index 2268d324b..92c4ac77b 100644 --- a/tests/fixtures/node-island/helper.cjs +++ b/tests/fixtures/node-island/helper.cjs @@ -11,3 +11,13 @@ exports.unreadable = () => { }); throw error; }; +// A fork without an execPath of its own: in a program it would start the +// program again. +exports.forkSelf = () => { + try { + require("node:child_process").fork(__filename); + return "forked"; + } catch (error) { + return error.message; + } +}; diff --git a/tests/fixtures/node-island/lifecycle.ts b/tests/fixtures/node-island/lifecycle.ts index 7bac91a40..262ab7a6b 100644 --- a/tests/fixtures/node-island/lifecycle.ts +++ b/tests/fixtures/node-island/lifecycle.ts @@ -16,6 +16,7 @@ if (mode !== "never") { console.log("mapped after", mapped()); } if (mode === "exit") process.exit(5); +if (mode === "fork") console.log(callSync("./helper.cjs", "forkSelf")); if (mode === "unreadable") { try { callSync("./helper.cjs", "unreadable"); diff --git a/tests/harness/ci-stack-scope.test.ts b/tests/harness/ci-stack-scope.test.ts index 4abd321d0..9408433c4 100644 --- a/tests/harness/ci-stack-scope.test.ts +++ b/tests/harness/ci-stack-scope.test.ts @@ -62,6 +62,22 @@ test.each([ ["bottom layer", "pull_request", stacked("main", 1), true], ["new bottom after a merge", "pull_request", stacked("main", 2), true], ["upper layer", "pull_request", upper, false], + [ + "a PR into the default branch whose stack names another base", + "pull_request", + { + ...stacked("main"), + repository: { full_name: repository, default_branch: "main" }, + pull_request: { ...stacked("main").pull_request, stack: { base: { ref: "ni/base" } } }, + }, + true, + ], + [ + "upper layer with the default branch known", + "pull_request", + { ...upper, repository: { full_name: repository, default_branch: "main" } }, + false, + ], ["main push", "push", {}, true], ["push with stack metadata", "push", upper, true], ["merge queue group", "merge_group", upper, true], diff --git a/tests/harness/corpus-costs/7661-node-call-modules/main.ts.json b/tests/harness/corpus-costs/7661-node-call-modules/main.ts.json new file mode 100644 index 000000000..a970c2f16 --- /dev/null +++ b/tests/harness/corpus-costs/7661-node-call-modules/main.ts.json @@ -0,0 +1 @@ +{ "plain": 6000, "san": 9000 } diff --git a/tests/harness/corpus-costs/7662-node-call-exit-events/main.ts.json b/tests/harness/corpus-costs/7662-node-call-exit-events/main.ts.json new file mode 100644 index 000000000..a970c2f16 --- /dev/null +++ b/tests/harness/corpus-costs/7662-node-call-exit-events/main.ts.json @@ -0,0 +1 @@ +{ "plain": 6000, "san": 9000 } diff --git a/tests/harness/node-island.test.ts b/tests/harness/node-island.test.ts index b67f0c84b..132af41f0 100644 --- a/tests/harness/node-island.test.ts +++ b/tests/harness/node-island.test.ts @@ -6,7 +6,18 @@ * (6800-6803) pins the call semantics against Node. Needs a libnode pack * in SCRIPTC_LIBNODE; required in the Linux x64 CI lanes (libnode.ts). */ import { execFile } from "node:child_process"; -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { + chmodSync, + copyFileSync, + cpSync, + existsSync, + mkdirSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; import { basename, join } from "node:path"; import { promisify } from "node:util"; import { beforeAll, describe, expect, test } from "vitest"; @@ -36,10 +47,13 @@ let binary = ""; const before = `mapped before ${process.platform === "linux" ? "false" : "unknown"}`; const after = `mapped after ${process.platform === "linux" ? "true" : "unknown"}`; +// The island module cache of these runs (scr_node_island.c). +const cacheHome = join(outDir, "cache"); + async function run( path: string, args: string[], - env: NodeJS.ProcessEnv = process.env, + env: NodeJS.ProcessEnv = { ...process.env, XDG_CACHE_HOME: cacheHome }, ): Promise { try { const { stdout, stderr } = await execFileAsync(path, args, { env, encoding: "utf8" }); @@ -125,18 +139,57 @@ describe.skipIf(skip)("Node islands in-process", () => { expect(result.stdout).toBe(`${before}\n42\n${after}\n`); }); - test("beside packaging carries its own pack", async () => { + test("beside packaging carries its own pack and island modules", async () => { const beside = await build("beside", "beside"); expect(existsSync(join(outDir, "beside/lifecycle.libnode/manifest.json"))).toBe(true); - const env = { ...process.env }; + const helper = join(outDir, "beside/lifecycle.libnode/modules/helper.cjs"); + expect(readFileSync(helper, "utf8")).toBe( + readFileSync(join(repoRoot, "tests/fixtures/node-island/helper.cjs"), "utf8"), + ); + const env = { ...process.env, XDG_CACHE_HOME: cacheHome }; delete env["SCRIPTC_LIBNODE"]; const result = await run(beside, [], env); expect(result.stdout).toBe(`${before}\n42\n${after}\ndone\n`); expect(result.code).toBe(0); + // Modules that no longer match the executable are refused. + writeFileSync(helper, `${readFileSync(helper, "utf8")}\n// edited\n`); + const edited = await run(beside, [], env); + expect(edited.code).toBe(1); + expect(edited.stderr).toBe( + `scriptc: the Node island modules at ${join(outDir, "beside/lifecycle.libnode/modules")} do not match this program\n`, + ); }, 300_000); + test("the executable carries its island modules and no path of the build machine", async () => { + const bytes = readFileSync(binary); + expect(bytes.includes(Buffer.from(repoRoot))).toBe(false); + // A copy elsewhere runs with the modules it carries, extracted into a + // private cache directory named by the manifest's digest. + const moved = join(outDir, "moved"); + mkdirSync(moved, { recursive: true }); + copyFileSync(binary, join(moved, "app")); + chmodSync(join(moved, "app"), 0o755); + const cache = join(outDir, "moved-cache"); + rmSync(cache, { recursive: true, force: true }); + const env = { ...process.env, XDG_CACHE_HOME: cache }; + expect((await run(join(moved, "app"), [], env)).stdout).toBe(`${before}\n42\n${after}\ndone\n`); + const islands = join(cache, "scriptc/node-islands"); + const [digest] = readdirSync(islands); + expect(digest).toMatch(/^[0-9a-f]{64}$/); + expect(statSync(islands).mode & 0o777).toBe(0o700); + expect(statSync(join(islands, digest!)).mode & 0o777).toBe(0o700); + // An entry that no longer matches is not used: the run extracts the + // modules into a directory of its own instead. + writeFileSync(join(islands, digest!, "helper.cjs"), "exports.answer = () => 0;\n"); + expect((await run(join(moved, "app"), [], env)).stdout).toBe(`${before}\n42\n${after}\ndone\n`); + }); + test("a missing pack fails the first call", async () => { - const result = await run(binary, [], { ...process.env, SCRIPTC_LIBNODE: "/nonexistent/pack" }); + const result = await run(binary, [], { + ...process.env, + XDG_CACHE_HOME: cacheHome, + SCRIPTC_LIBNODE: "/nonexistent/pack", + }); expect(result.code).toBe(1); expect(result.stdout).toBe(`${before}\n`); expect(result.stderr).toMatch( @@ -145,7 +198,11 @@ describe.skipIf(skip)("Node islands in-process", () => { }); test("SCRIPTC_LIBNODE must be an absolute path", async () => { - const result = await run(binary, [], { ...process.env, SCRIPTC_LIBNODE: "relative/pack" }); + const result = await run(binary, [], { + ...process.env, + XDG_CACHE_HOME: cacheHome, + SCRIPTC_LIBNODE: "relative/pack", + }); expect(result.code).toBe(1); expect(result.stdout).toBe(`${before}\n`); expect(result.stderr).toBe( @@ -175,9 +232,54 @@ describe.skipIf(skip)("Node islands in-process", () => { }); }, 300_000); + test("full garbage collections during calls, at depth on both sides", async () => { + const stress = await build( + "gc-stress", + undefined, + join(repoRoot, "tests/fixtures/node-island/gc-stress.ts"), + ); + expect(await run(stress, [])).toEqual({ + stdout: + "churn 2000000\nnested 5000\ndeep 8000\nfibers 100001,100002,100003,100004\ntimer 500000\n", + stderr: "", + code: 0, + }); + }, 300_000); + + test("Node ignores NODE_OPTIONS from the program's environment", async () => { + const result = await run(binary, [], { + ...process.env, + XDG_CACHE_HOME: cacheHome, + NODE_OPTIONS: "--require /nonexistent/preload.cjs", + }); + expect(result).toEqual({ stdout: `${before}\n42\n${after}\ndone\n`, stderr: "", code: 0 }); + }); + + test("child_process.fork() needs an execPath of its own", async () => { + expect((await run(binary, ["fork"])).stdout).toBe( + `${before}\n42\n${after}\nchild_process.fork() in a Node island needs options.execPath (a node executable): process.execPath is this program\ndone\n`, + ); + }); + + test("a cached build follows a change of the recorded libnode pack", async () => { + // The same program built against a copy of the pack elsewhere records + // the copy: the executable cache does not serve the first build's entry. + const copy = join(outDir, "pack-copy"); + rmSync(copy, { recursive: true, force: true }); + cpSync(pack!, copy, { recursive: true }); + const first = readFileSync(await build("cached-pack", pack!)); + const second = readFileSync(await build("cached-pack", copy)); + expect(first.includes(Buffer.from(copy))).toBe(false); + expect(second.includes(Buffer.from(copy))).toBe(true); + }, 300_000); + test("a pack of another Node version fails the first call", async () => { const dir = await fakePack("old-node", "SCR_NODE_BRIDGE_ABI", "22.11.0"); - const result = await run(binary, [], { ...process.env, SCRIPTC_LIBNODE: dir }); + const result = await run(binary, [], { + ...process.env, + XDG_CACHE_HOME: cacheHome, + SCRIPTC_LIBNODE: dir, + }); expect(result.code).toBe(1); expect(result.stderr).toMatch( new RegExp( @@ -188,10 +290,14 @@ describe.skipIf(skip)("Node islands in-process", () => { test("a bridge of another ABI fails the first call", async () => { const dir = await fakePack("other-abi", "999u", "24.22.0"); - const result = await run(binary, [], { ...process.env, SCRIPTC_LIBNODE: dir }); + const result = await run(binary, [], { + ...process.env, + XDG_CACHE_HOME: cacheHome, + SCRIPTC_LIBNODE: dir, + }); expect(result.code).toBe(1); expect(result.stderr).toBe( - `scriptc: the libnode pack at ${dir} has an incompatible bridge (ABI 999; this program needs 2)\n`, + `scriptc: the libnode pack at ${dir} has an incompatible bridge (ABI 999; this program needs 3)\n`, ); }); From c016b97acb4a67cabd18527c2635de773834c144 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 12:54:17 -0500 Subject: [PATCH 079/184] Fix CI on typed-array unions --- .../src/frontend/lowering/containers/bytes.ts | 5 ++ .../src/frontend/lowering/indexed-unions.ts | 49 +++++++++++++---- .../src/frontend/lowering/lower-inspect.ts | 13 ++++- .../compiler/src/frontend/lowering/lowerer.ts | 3 ++ .../test/typed-array-union-emission.test.ts | 52 +++++++++++++++++++ tests/diagnostics/index-signature-values.ts | 4 -- .../index-signature-values.ts.txt | 15 ++---- 7 files changed, 115 insertions(+), 26 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/containers/bytes.ts b/packages/compiler/src/frontend/lowering/containers/bytes.ts index c804d62e1..1c5513506 100644 --- a/packages/compiler/src/frontend/lowering/containers/bytes.ts +++ b/packages/compiler/src/frontend/lowering/containers/bytes.ts @@ -961,6 +961,11 @@ function lowerUnionBytesMethodCall( name: string, ): IrExpr | null { if (isJsSourceFile(ts.getSourceFileOfNode(call))) return null; + // A Buffer arm's methods differ from a typed array's (slice aliases, fill + // checks its range, searches take strings and wrap numbers), so such a + // union narrows to its arms first. + const receiverName = lowerer.checker.typeToString(lowerer.typeOf(access.expression)); + if (/(^|\| )(NonShared)?Buffer\b/.test(receiverName)) return null; const nodes = call.arguments.nodes; if (nodes.some(ts.isSpreadElement)) return null; const loc = locOf(call); diff --git a/packages/compiler/src/frontend/lowering/indexed-unions.ts b/packages/compiler/src/frontend/lowering/indexed-unions.ts index aa93871c9..774f3aef3 100644 --- a/packages/compiler/src/frontend/lowering/indexed-unions.ts +++ b/packages/compiler/src/frontend/lowering/indexed-unions.ts @@ -131,10 +131,6 @@ function dispatchHelper( const returning = (value: IrExpr, loc: SrcLoc): IrStmt[] => [{ kind: "return", value, loc }]; -/** Safe-read helpers mapped to their numeric twins, for the fusion of an - * immediate ToNumber (tryLowerNumericIndexRead). */ -const numericTwins = new WeakMap>(); - /** `u[i]` read as an element or undefined (an invalid index), as the * typed-array and number-array reads answer. */ export function lowerIndexedUnionRead( @@ -164,9 +160,7 @@ export function lowerIndexedUnionRead( loc, ); const numeric = indexedUnionNumberHelper(lowerer, receiver.type, arms, loc); - let twins = numericTwins.get(lowerer); - if (!twins) numericTwins.set(lowerer, (twins = new Map())); - twins.set(name, numeric); + lowerer.indexedUnionNumericTwins.set(name, numeric); return { kind: "call", callee: name, args: [receiver, index], type: resultT, loc }; } @@ -221,7 +215,7 @@ function indexedUnionNumberHelper( /** The numeric twin of an indexed-union read helper call, or null. */ export function indexedUnionNumericTwin(lowerer: Lowerer, call: IrExpr): IrExpr | null { if (call.kind !== "call") return null; - const twin = numericTwins.get(lowerer)?.get(call.callee); + const twin = lowerer.indexedUnionNumericTwins.get(call.callee); return twin === undefined ? null : { ...call, callee: twin, type: F64 }; } @@ -479,12 +473,49 @@ function inlineDispatch(receiver: IrExpr, operands: readonly IrExpr[]): boolean ); } -/** Whether evaluating `e` can rebind one of the plain reads `reads`. */ +/** Expressions that run no program code: reads, literals and operators + * over primitive values. */ +const CODE_FREE_EXPRS = new Set([ + "numLit", + "boolLit", + "strLit", + "unitLit", + "varRef", + "unary", + "logical", + "ternary", + "toBool", + "strEq", + "strCmp", + "arrayGet", + "arrayHas", + "arrayState", + "bytesIntrinsic", + "fieldGet", + "recordGet", + "unionNarrow", + "unionIsTag", + "unionDisc", + "upcast", + "nullish", +]); +const PRIMITIVE_OPERANDS = new Set(["f64", "bool", "string", "bigint"]); + +/** Whether evaluating `e` can rebind one of the plain reads `reads`: by + * assigning it, or by running program code (a call, a getter, a + * conversion) that could assign a captured binding. */ function rebinds(e: IrExpr, reads: readonly IrExpr[]): boolean { const ids = new Set(reads.flatMap((r) => (r.kind === "varRef" ? [r.localId] : []))); let found = false; const expr = (x: IrExpr): boolean => { if ((x.kind === "assignExpr" || x.kind === "incDec") && ids.has(x.localId)) found = true; + else if (x.kind === "bin") { + if (!PRIMITIVE_OPERANDS.has(x.left.type.kind) || !PRIMITIVE_OPERANDS.has(x.right.type.kind)) + found = true; + } else if (x.kind === "call") { + // The union dispatch helpers only read and store elements. + if (!x.callee.startsWith("%union.")) found = true; + } else if (!CODE_FREE_EXPRS.has(x.kind)) found = true; return !found && everyExprChild(x, expr, stmt); }; const stmt = (x: IrStmt): boolean => { diff --git a/packages/compiler/src/frontend/lowering/lower-inspect.ts b/packages/compiler/src/frontend/lowering/lower-inspect.ts index d914faa5d..fc52ed555 100644 --- a/packages/compiler/src/frontend/lowering/lower-inspect.ts +++ b/packages/compiler/src/frontend/lowering/lower-inspect.ts @@ -948,7 +948,13 @@ export function inspectExpr( } // A Uint8Array, a Buffer and a DataView share the u8 representation // and print differently: the view's own flags pick. - return { kind: "libCall", fn: "insp.bytes", args: [value, recurse, depth], type: STRING, loc }; + return { + kind: "libCall", + fn: "insp.bytes", + args: [value, recurse, depth], + type: STRING, + loc, + }; case "dyn": return { kind: "libCall", fn: "insp.dyn", args: [value, recurse, depth], type: STRING, loc }; case "generator": @@ -2510,7 +2516,10 @@ export function lowerInspectValue( value, (input) => ({ kind: "libCall", - fn: value.type.kind === "bytes" && value.type.elem === "u8" ? "insp.bytes" : "insp.typedArray", + fn: + value.type.kind === "bytes" && value.type.elem === "u8" + ? "insp.bytes" + : "insp.typedArray", args: [input, numLit(0, loc), numLit(options.depth, loc)], type: STRING, loc, diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 3c74cfbad..090c957ad 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -2896,6 +2896,9 @@ export class Lowerer { /** util.inspect's per-type traversal helpers (%util.insp.N), interned * by typeKey — the assertHelpers pattern with its own namespace. */ readonly inspectHelpers = new Map(); + /** Indexed-union safe-read helpers mapped to their numeric twins, for the + * fusion of an immediate ToNumber (tryLowerNumericIndexRead). */ + readonly indexedUnionNumericTwins = new Map(); private readonly unionLiteralOwners = new Map< string, { owners: Map | null; shapes: number; unions: number } diff --git a/packages/compiler/test/typed-array-union-emission.test.ts b/packages/compiler/test/typed-array-union-emission.test.ts index 743788af0..5aaf846ed 100644 --- a/packages/compiler/test/typed-array-union-emission.test.ts +++ b/packages/compiler/test/typed-array-union-emission.test.ts @@ -54,3 +54,55 @@ test("a typed-array union read through a computed receiver goes through one help // The receiver is a call: evaluated once, then dispatched by the helper. expect(body(llvm, "first")).toContain("@sc_bf__x25_union_getNumber"); }); + +const stores = ` +type Wide = Uint8Array | Uint16Array; +const pick = (wide: boolean): Wide => (wide ? new Uint16Array(4) : new Uint8Array(4)); +function fill(xs: Wide): void { + for (let i = 0; i < xs.length; i++) xs[i] = i + 1; +} +function rebind(): void { + let u = pick(false); + let i = 0; + const swap = (): number => { + u = pick(true); + i = 2; + return 3; + }; + u[i] = swap(); + console.log(u, i); +} +fill(pick(process.argv.length > 9)); +rebind(); +`; + +test("a typed-array union store runs in place unless its value can rebind the receiver or index", async () => { + const llvm = await emitLlvm(stores); + expect(body(llvm, "fill")).not.toContain("@sc_bf__x25_union_idxSet"); + // The call may assign the captured receiver and index, which the store + // read before it: the helper evaluates them first. + expect(body(llvm, "rebind")).toContain("@sc_bf__x25_union_idxSet"); +}); + +test("typed-array methods on a union with a Buffer arm need a narrowing first", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-typed-array-unions-")); + try { + const entry = join(dir, "main.ts"); + await writeFile( + entry, + `import { Buffer } from "node:buffer"; +const u: Buffer | Uint16Array = process.argv.length > 9 ? new Uint16Array(2) : Buffer.alloc(2); +console.log(u.slice(1)); +`, + ); + const result = await compile(entry, { + outDir: dir, + outPath: join(dir, "main.ll"), + outputKind: "llvm", + }); + expect(result.ok).toBe(false); + expect(JSON.stringify(result.diagnostics)).toContain("slice"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/tests/diagnostics/index-signature-values.ts b/tests/diagnostics/index-signature-values.ts index 6c53b141c..1f3a01ba5 100644 --- a/tests/diagnostics/index-signature-values.ts +++ b/tests/diagnostics/index-signature-values.ts @@ -8,10 +8,6 @@ const bySymbol: { [s: symbol]: number } = {}; console.log(bySymbol); -// Storage is supported, but composite typed-array inspection is not. -const buffers: Record = {}; -console.log(buffers); - // Dual signatures whose value types intern DIFFERENTLY cannot share the // one store (tsc requires only assignability, but the store gives one // answer per key). diff --git a/tests/harness/__snapshots__/index-signature-values.ts.txt b/tests/harness/__snapshots__/index-signature-values.ts.txt index bbbed7fe2..ea36a9c76 100644 --- a/tests/harness/__snapshots__/index-signature-values.ts.txt +++ b/tests/harness/__snapshots__/index-signature-values.ts.txt @@ -7,18 +7,11 @@ index-signature-values.ts:8:7 - error SC2006: values of type '{ [s: symbol]: num hint: string- and number-keyed index signatures over the supported value types compile directly; symbol keys and dual signatures with unequal value types have no lowering -index-signature-values.ts:13:13 - error SC1090: console.log of '{ [key: string]: Uint8Array }' values (Uint8Array and Buffer values inside composites have no inspect lowering yet (Buffer's form vs Uint8Array's is not recorded in the static type here)) are not supported yet +index-signature-values.ts:14:7 - error SC2006: values of type '{ [k: string]: string | number; [n: number]: number; }' cannot be compiled: this index signature is outside the supported shape (string or number keys; values limited to numbers, strings, booleans, records, classes, arrays, unions, functions, Maps, Sets, RegExps, Promises, or 'unknown') - 12 | const buffers: Record = {}; - 13 | console.log(buffers); - | ^~~~~~~ - 14 | - -index-signature-values.ts:18:7 - error SC2006: values of type '{ [k: string]: string | number; [n: number]: number; }' cannot be compiled: this index signature is outside the supported shape (string or number keys; values limited to numbers, strings, booleans, records, classes, arrays, unions, functions, Maps, Sets, RegExps, Promises, or 'unknown') - - 17 | // answer per key). - 18 | const dual: { [k: string]: string | number; [n: number]: number } = {}; + 13 | // answer per key). + 14 | const dual: { [k: string]: string | number; [n: number]: number } = {}; | ^~~~ - 19 | console.log(dual); + 15 | console.log(dual); hint: string- and number-keyed index signatures over the supported value types compile directly; symbol keys and dual signatures with unequal value types have no lowering \ No newline at end of file From 0af86e2e3ef7862ca6d91b8917ff6ccf36735fd1 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 12:58:05 -0500 Subject: [PATCH 080/184] Version two-counter loops over typed arrays A loop with two counters closing in on each other, such as `for (let i = lo, j = hi; i < j; i++, j--)`, checked the bounds of every typed-array element it read or wrote at either counter. Within the body the low counter only rises and stays below the high one, which only falls, so every such index lies between the two starts. Typed-array loop versions now cover these loops. The guard tests that both starts are exact integers, that the low start is at least 0, and that the high start is below the length of each view the loop indexes at a counter. The fast copy then reads and writes those elements without checks, and the checked copy runs whenever the guard fails. In the fast copy the guard also stands for the loop's own start test. The exact-integer test of a start is now one function, shared by the typed-array version guards and the dense array two-counter guard. The dense guard, which compared the round trip with `fcmp oeq`, now also leaves a -0 start to the checked loop. --- .../backend/llvm/bytes-loop-versions.test.ts | 77 ++++++++++++++- .../src/backend/llvm/bytes-loop-versions.ts | 97 ++++++++++++------- .../src/backend/llvm/counted-loops.ts | 33 +++++++ .../src/backend/llvm/dense-array-loops.ts | 31 ++---- .../src/backend/llvm/emit-statements.ts | 11 ++- packages/compiler/src/backend/llvm/emitter.ts | 30 +++--- .../7981-typed-array-paired-loops.ts.json | 6 ++ tests/corpus/7981-typed-array-paired-loops.ts | 57 +++++++++++ .../7981-typed-array-paired-loops.ts.json | 1 + 9 files changed, 265 insertions(+), 78 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7981-typed-array-paired-loops.ts.json create mode 100644 tests/corpus/7981-typed-array-paired-loops.ts create mode 100644 tests/harness/corpus-costs/7981-typed-array-paired-loops.ts.json diff --git a/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts b/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts index 34b75861c..de71a6ef0 100644 --- a/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts +++ b/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts @@ -11,7 +11,7 @@ import { type IrType, } from "../../ir/ir.js"; import { analyzeIntegerRanges } from "../../ir/integer-ranges.js"; -import { matchIntegerCountedForLoop } from "../../ir/integer-loops.js"; +import { matchIntegerCountedForLoop, matchIntegerPairedForLoop } from "../../ir/integer-loops.js"; import { findInitializedByteLoopBindings } from "./byte-windows.js"; import { matchBytesLoopVersion, type BytesLoopVersion } from "./bytes-loop-versions.js"; import { emitLlvmModule } from "./emitter.js"; @@ -623,3 +623,78 @@ test("while loop counters below a string's length version its reads", () => { expect(version.bounded.map((c) => c.slack)).toEqual([0]); expect(version.fixed).toEqual([]); }); + +test("a two-counter loop versions the typed arrays it indexes at either counter", () => { + // for (let i = 0, j = k; i < j; i++, j--) { const t = out[i]; out[i] = out[j]; out[j] = t; } + const loop: IrStmt & { kind: "for" } = { + kind: "for", + init: { + kind: "block", + body: [ + { kind: "varDecl", localId: "i", init: num(0), loc }, + { kind: "varDecl", localId: "j", init: ref("k"), loc }, + ], + loc, + }, + cond: { kind: "bin", op: "<", left: ref("i"), right: ref("j"), type: BOOL, loc }, + update: { + kind: "block", + body: [ + { kind: "assign", localId: "i", value: bin("+", ref("i"), num(1)), loc }, + { kind: "assign", localId: "j", value: bin("-", ref("j"), num(1)), loc }, + ], + loc, + }, + body: [ + { kind: "varDecl", localId: "t", init: get(ref("out", f64s), ref("i")), loc }, + set(ref("out", f64s), ref("i"), get(ref("out", f64s), ref("j"))), + set(ref("out", f64s), ref("j"), ref("t")), + // Another index stays checked. + set(ref("src", f64s), bin("+", ref("i"), num(1)), num(0)), + ], + loc, + }; + const kernel = fn([loop], [local("t", F64)]); + const ranges = analyzeIntegerRanges(kernel); + const locals = new Map(kernel.locals.map((l) => [l.id, l])); + const paired = matchIntegerPairedForLoop(loop, locals, ranges)!; + expect(paired).not.toBeNull(); + const version = matchBytesLoopVersion( + loop, + null, + { + locals, + invariantGlobals: new Set(), + captured: new Set(), + ranges, + initialized: findInitializedByteLoopBindings(kernel).get(loop) ?? new Set(), + proven: new Set(), + workers: false, + owned: new Set(), + scalarStrings: new Set(), + }, + paired, + )!; + expect(version.paired?.loop).toBe(paired); + expect(version.paired?.receivers.map((r) => version.receivers[r]!.ref)).toEqual([ + ref("out", f64s), + ]); + expect(version.bounds.size).toBe(4); + const mod: IrModule = { + irVersion: 17, + sourceFile: loc.file, + entry: "__main", + functions: [ + { name: "__main", params: [], returnType: VOID, locals: [], body: [], loc }, + kernel, + ], + }; + const body = /^define internal [^\n]*@sc_bf_kernel[^]*?^}/m.exec(emitLlvmModule(mod))![0]; + const guard = body.slice(0, body.search(/^bytes\.version\.(fast|slow)\d+:$/m)); + expect(guard).toMatch(/icmp sge i64 %\S+, 0/); + expect(guard).toMatch(/icmp slt i64 %\S+, %\S+/); + const fast = /^bytes\.version\.fast\d+:$[^]*?^bytes\.version\.done\d+:$/m.exec(body)![0]; + // The fast copy keeps only the `src[i + 1]` check, and no second guard. + expect(fast.match(/bytes\.index\.invalid\d+:/g)).toHaveLength(1); + expect(fast).not.toContain("counted.fast"); +}); diff --git a/packages/compiler/src/backend/llvm/bytes-loop-versions.ts b/packages/compiler/src/backend/llvm/bytes-loop-versions.ts index 704b6723a..2013d03e4 100644 --- a/packages/compiler/src/backend/llvm/bytes-loop-versions.ts +++ b/packages/compiler/src/backend/llvm/bytes-loop-versions.ts @@ -1,11 +1,15 @@ import { F64, type IrExpr, type IrLocal, type IrStmt } from "../../ir/ir.js"; import type { IntegerRange, IntegerRanges } from "../../ir/integer-ranges.js"; -import type { IntegerCountedForLoop } from "../../ir/integer-loops.js"; +import type { IntegerCountedForLoop, IntegerPairedForLoop } from "../../ir/integer-loops.js"; import type { LlvmEmitterContext } from "./expr-context.js"; import { byteNumberAccess } from "../../ir/byte-numbers.js"; import { everyExprChild, everyStmtChild } from "../../ir/traverse.js"; import { exactInteger, widenInteger } from "./integer-values.js"; -import { emitCountedLoopLimit, emitCountedLoopSafety } from "./counted-loops.js"; +import { + emitCountedLoopLimit, + emitCountedLoopSafety, + emitExactIntegerTest, +} from "./counted-loops.js"; import { emitBytesHeaderLength, emitBytesSharedStorage } from "./expr-bytes.js"; import { f64Lit } from "./common.js"; import { emitStringAsciiFlag, emitStringByteLength } from "./string-reads.js"; @@ -79,6 +83,10 @@ export interface BytesLoopVersion { guarded: ReadonlySet; /** The counted loop whose iteration space the affine checks cover. */ counted: IntegerCountedForLoop | null; + /** A two-counter loop (`for (let i = lo, j = hi; i < j; i++, j--)`) and + * the receivers it indexes at either counter. Both counters stay between + * the two starts, so the guard requires `lo >= 0` and `hi < length`. */ + paired: { loop: IntegerPairedForLoop; receivers: number[] } | null; /** The guard also requires every counter value within [-bound, bound]. */ counterBound: number | null; /** Integer ranges of affine index operands that hold in the unchecked @@ -153,6 +161,7 @@ export function matchBytesLoopVersion( loop: IrStmt & { kind: "for" | "while" }, counted: IntegerCountedForLoop | null, facts: BytesLoopFacts, + paired: IntegerPairedForLoop | null = null, ): BytesLoopVersion | null { const written = new Set(); const temps = new Map(); @@ -438,6 +447,9 @@ export function matchBytesLoopVersion( string, { check: BoundedCheck; nodes: { node: Node; access: WhileAccess }[] } >(); + const pairedCounters = + paired !== null && counted === null ? new Set([paired.low.localId, paired.high.localId]) : null; + const pairedNodes = new Map(); const access = (node: Node, array: IrExpr, index: IrExpr, width: number): void => { const ref = root(array); if (!ref) return; @@ -448,6 +460,20 @@ export function matchBytesLoopVersion( else owned.set(receiver, [node]); } if (facts.proven.has(node)) return; + // An element at either counter of a two-counter loop. + if ( + pairedCounters !== null && + width === 1 && + ref.type.kind === "bytes" && + index.kind === "varRef" && + pairedCounters.has(index.localId) + ) { + const receiver = receiverOf(ref); + const nodes = pairedNodes.get(receiver); + if (nodes) nodes.push(node); + else pairedNodes.set(receiver, [node]); + return; + } const shape = counter !== null ? affine(index) : null; if (shape && shape.counter !== 0) { const operands = new Set(); @@ -543,11 +569,16 @@ export function matchBytesLoopVersion( Math.max(0, MAX_CHECKS - fixedKept.length - affineKept.length), ); const ownedKept = [...owned.entries()].slice(0, MAX_CHECKS); + const pairedKept = [...pairedNodes.entries()].slice( + 0, + Math.max(0, MAX_CHECKS - fixedKept.length - affineKept.length - boundedKept.length), + ); if ( fixedKept.length === 0 && affineKept.length === 0 && boundedKept.length === 0 && - ownedKept.length === 0 + ownedKept.length === 0 && + pairedKept.length === 0 ) return null; const bounds = new Set(); @@ -616,6 +647,10 @@ export function matchBytesLoopVersion( if (access.index !== access.ref) narrow(access.index, { min: 0, max: SAFE }); } } + for (const [receiver, nodes] of pairedKept) { + used.add(receiver); + for (const node of nodes) bounds.add(node); + } const counterChecks: CounterCheck[] = []; for (const [id, min] of counterMins) { const info = whileAccesses!.counters.get(id)!; @@ -658,6 +693,10 @@ export function matchBytesLoopVersion( counters: counterChecks, guarded, counted: affineKept.length > 0 ? counted : null, + paired: + paired !== null && pairedKept.length > 0 + ? { loop: paired, receivers: pairedKept.map(([receiver]) => renumber.get(receiver)!) } + : null, counterBound: needsBound ? bound : null, ranges, }; @@ -1143,6 +1182,25 @@ export function emitBytesLoopVersionGuard( conditions.push(both); } } + if (version.paired !== null) { + // Both counters stay within [lo, hi] (exact integers below 2^53): every + // access at either one is in bounds when `lo >= 0` and `hi < length`. + const { loop } = version.paired; + const [low, high] = [loop.low.start, loop.high.start].map((expr) => { + const tested = emitExactIntegerTest(host, host.emitExpr(expr).name, -SAFE, SAFE); + conditions.push(tested.ok); + return tested.integer; + }); + const nonNegative = B.tmp(); + B.line(`${nonNegative} = icmp sge i64 ${low}, 0`); + conditions.push(nonNegative); + for (const receiver of version.paired.receivers) { + // Lengths are at most 2^53 elements: a signed comparison is exact. + const below = B.tmp(); + B.line(`${below} = icmp slt i64 ${high}, ${lengths[receiver]}`); + conditions.push(below); + } + } let valid = "true"; for (const cond of conditions) { if (valid === "true") valid = cond; @@ -1156,39 +1214,6 @@ export function emitBytesLoopVersionGuard( else B.condBr(valid, fast, slow); } -/** Tests that a double is an exact integer within [min, max] (not -0; NaN - * fails the ordered comparisons) and gives its i64. The integer is frozen, - * so it is a fixed value even when the test fails. */ -function emitExactIntegerTest( - host: LlvmEmitterContext, - value: string, - min: number, - max: number, -): { ok: string; integer: string } { - const B = host.B; - const low = B.tmp(), - high = B.tmp(), - inRange = B.tmp(), - raw = B.tmp(), - integer = B.tmp(), - back = B.tmp(), - bits = B.tmp(), - backBits = B.tmp(), - exact = B.tmp(), - ok = B.tmp(); - B.line(`${low} = fcmp oge double ${value}, ${f64Lit(min)}`); - B.line(`${high} = fcmp ole double ${value}, ${f64Lit(max)}`); - B.line(`${inRange} = and i1 ${low}, ${high}`); - B.line(`${raw} = fptosi double ${value} to i64`); - B.line(`${integer} = freeze i64 ${raw}`); - B.line(`${back} = sitofp i64 ${integer} to double`); - B.line(`${bits} = bitcast double ${value} to i64`); - B.line(`${backBits} = bitcast double ${back} to i64`); - B.line(`${exact} = icmp eq i64 ${bits}, ${backBits}`); - B.line(`${ok} = and i1 ${inRange}, ${exact}`); - return { ok, integer }; -} - function scaleCounter(host: LlvmEmitterContext, value: string, scale: number): string { if (scale === 1) return value; const scaled = host.B.tmp(); diff --git a/packages/compiler/src/backend/llvm/counted-loops.ts b/packages/compiler/src/backend/llvm/counted-loops.ts index ca2dd6b92..1ff4c6580 100644 --- a/packages/compiler/src/backend/llvm/counted-loops.ts +++ b/packages/compiler/src/backend/llvm/counted-loops.ts @@ -151,3 +151,36 @@ function emitExactStart(host: LlvmEmitterContext, expr: IrExpr): string { B.line(`${ok} = and i1 ${inRange}, ${exact}`); return ok; } + +/** Tests that a double is an exact integer within [min, max] (not -0; NaN + * fails the ordered comparisons) and gives its i64. The integer is frozen, + * so it is a fixed value even when the test fails. */ +export function emitExactIntegerTest( + host: LlvmEmitterContext, + value: string, + min: number, + max: number, +): { ok: string; integer: string } { + const B = host.B; + const low = B.tmp(), + high = B.tmp(), + inRange = B.tmp(), + raw = B.tmp(), + integer = B.tmp(), + back = B.tmp(), + bits = B.tmp(), + backBits = B.tmp(), + exact = B.tmp(), + ok = B.tmp(); + B.line(`${low} = fcmp oge double ${value}, ${f64Lit(min)}`); + B.line(`${high} = fcmp ole double ${value}, ${f64Lit(max)}`); + B.line(`${inRange} = and i1 ${low}, ${high}`); + B.line(`${raw} = fptosi double ${value} to i64`); + B.line(`${integer} = freeze i64 ${raw}`); + B.line(`${back} = sitofp i64 ${integer} to double`); + B.line(`${bits} = bitcast double ${value} to i64`); + B.line(`${backBits} = bitcast double ${back} to i64`); + B.line(`${exact} = icmp eq i64 ${bits}, ${backBits}`); + B.line(`${ok} = and i1 ${inRange}, ${exact}`); + return { ok, integer }; +} diff --git a/packages/compiler/src/backend/llvm/dense-array-loops.ts b/packages/compiler/src/backend/llvm/dense-array-loops.ts index 384c31b0c..b806939b5 100644 --- a/packages/compiler/src/backend/llvm/dense-array-loops.ts +++ b/packages/compiler/src/backend/llvm/dense-array-loops.ts @@ -36,8 +36,7 @@ import { everyExprChild, everyStmtChild } from "../../ir/traverse.js"; import type { IntegerCountedForLoop, IntegerPairedForLoop } from "../../ir/integer-loops.js"; import type { LlvmEmitterContext } from "./expr-context.js"; import { loadArrCount, SCR_ARR_FIELD } from "./common.js"; -import { emitCountedLoopLimit } from "./counted-loops.js"; -import { f64Lit } from "./common.js"; +import { emitCountedLoopLimit, emitExactIntegerTest } from "./counted-loops.js"; export const INT32_MAX = 2147483647; @@ -453,28 +452,10 @@ function emitPairedDenseGuard( header: (array: string, field: number) => string, ): string { const B = host.B; - const integer = (expr: IrExpr, min: number, max: number): { value: string; ok: string } => { - const start = host.emitExpr(expr); - const low = B.tmp(), - high = B.tmp(), - raw = B.tmp(), - value = B.tmp(), - back = B.tmp(), - exact = B.tmp(), - inRange = B.tmp(), - ok = B.tmp(); - B.line(`${low} = fcmp oge double ${start.name}, ${f64Lit(min)}`); - B.line(`${high} = fcmp ole double ${start.name}, ${f64Lit(max)}`); - B.line(`${inRange} = and i1 ${low}, ${high}`); - B.line(`${raw} = fptosi double ${start.name} to i64`); - B.line(`${value} = freeze i64 ${raw}`); - B.line(`${back} = sitofp i64 ${value} to double`); - B.line(`${exact} = fcmp oeq double ${back}, ${start.name}`); - B.line(`${ok} = and i1 ${inRange}, ${exact}`); - return { value, ok }; - }; - const low = integer(pair.lowStart, 0, INT32_MAX - 1); - const high = integer(pair.highStart, -1, INT32_MAX - 1); + const start = (expr: IrExpr, min: number, max: number) => + emitExactIntegerTest(host, host.emitExpr(expr).name, min, max); + const low = start(pair.lowStart, 0, INT32_MAX - 1); + const high = start(pair.highStart, -1, INT32_MAX - 1); let valid = B.tmp(); B.line(`${valid} = and i1 ${low.ok}, ${high.ok}`); for (const ref of loop.arrays.values()) { @@ -486,7 +467,7 @@ function emitPairedDenseGuard( both = B.tmp(); B.line(`${filled} = load i32, ptr ${filledPtr}${host.fieldAliasAttachment(filledPtr)}`); B.line(`${wide} = zext i32 ${filled} to i64`); - B.line(`${covered} = icmp slt i64 ${high.value}, ${wide}`); + B.line(`${covered} = icmp slt i64 ${high.integer}, ${wide}`); B.line(`${both} = and i1 ${valid}, ${covered}`); valid = both; } diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index aef5ccf30..55e31219c 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -1051,15 +1051,17 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { LOOPS.of(emitter).unaliasedArrays, ) : countedLoop; - const version = versionable ? emitter.matchBytesLoopVersion(s, versionCounted) : null; + const version = versionable + ? emitter.matchBytesLoopVersion(s, versionCounted, versionCounted ? null : pairedLoop) + : null; if (version) { emitVersionedLoop(emitter, s, version); break; } let countedJoin: string | null = null; if ( - (countedLoop?.guarded && !LOOPS.of(emitter).provenCountedGuards.has(s)) || - pairedLoop?.guarded + (countedLoop?.guarded || pairedLoop?.guarded) && + !LOOPS.of(emitter).provenCountedGuards.has(s) ) { const safe = countedLoop ? emitCountedLoopSafety(emitter, countedLoop) @@ -2094,7 +2096,8 @@ function emitVersionedLoop( if (version.owned.size > 0) emitter.bytesOwned = new Set([...originalOwned, ...version.owned]); if (version.ranges.size > 0) emitter.integerRanges = new Map([...originalRanges, ...version.ranges]); - if (version.counted?.guarded) LOOPS.of(emitter).provenCountedGuards.add(s); + if (version.counted?.guarded || version.paired?.loop.guarded) + LOOPS.of(emitter).provenCountedGuards.add(s); emitter.emitStmt(s); B.br(done); } finally { diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 4aa9b65ad..ad59c3b1b 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -60,7 +60,7 @@ import { captureFreeFunctions, observesStacks } from "../stack-observations.js"; import { findInitializedByteLoopBindings } from "./byte-windows.js"; import { matchBytesLoopVersion } from "./bytes-loop-versions.js"; import { globalSlotAlias } from "./bytes-alias.js"; -import type { IntegerCountedForLoop } from "../../ir/integer-loops.js"; +import type { IntegerCountedForLoop, IntegerPairedForLoop } from "../../ir/integer-loops.js"; import { materializeSplitPiece, type SplitSpan, @@ -4263,6 +4263,7 @@ export class LlEmitter { matchBytesLoopVersion( s: IrStmt & { kind: "for" | "while" }, counted: IntegerCountedForLoop | null, + paired: IntegerPairedForLoop | null = null, ): ReturnType { const fn = LOOPS.of(this).bytesLoopFunction; if (!fn) return null; @@ -4272,17 +4273,22 @@ export class LlEmitter { const invariantGlobals = new Set(this.stableGlobals); for (const global of this.initializerBindings.get(fn.name) ?? []) invariantGlobals.add(global.id); - return matchBytesLoopVersion(s, counted, { - locals: this.currentLocals, - invariantGlobals, - captured: this.captureIds, - ranges: this.integerRanges, - initialized, - proven: this.bytesBounds, - workers: this.mod.workers === true, - owned: this.bytesOwned, - scalarStrings: new Set(this.stringSlices.keys()), - }); + return matchBytesLoopVersion( + s, + counted, + { + locals: this.currentLocals, + invariantGlobals, + captured: this.captureIds, + ranges: this.integerRanges, + initialized, + proven: this.bytesBounds, + workers: this.mod.workers === true, + owned: this.bytesOwned, + scalarStrings: new Set(this.stringSlices.keys()), + }, + paired, + ); } /** The alias tag of a whole-slot load of a binding (module globals only). */ diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7981-typed-array-paired-loops.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7981-typed-array-paired-loops.ts.json new file mode 100644 index 000000000..71fa483a8 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7981-typed-array-paired-loops.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7981-typed-array-paired-loops.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7981-typed-array-paired-loops.ts b/tests/corpus/7981-typed-array-paired-loops.ts new file mode 100644 index 000000000..d13eecbd6 --- /dev/null +++ b/tests/corpus/7981-typed-array-paired-loops.ts @@ -0,0 +1,57 @@ +// Two-counter loops over typed arrays (`for (let i = lo, j = hi; i < j; +// i++, j--)`), with starts inside the arrays and outside them: past the +// end, negative, fractional and NaN. Views over one buffer, other element +// types, inclusive conditions and steps other than one. + +function reverse(a: Int32Array, lo: number, hi: number): string { + for (let i = lo, j = hi; i < j; i++, j--) { + const t = a[i]!; + a[i] = a[j]!; + a[j] = t; + } + return Array.from(a).join(); +} + +function mirror(a: Float64Array, b: Float64Array, hi: number): number { + let sum = 0; + for (let i = 0, j = hi; i <= j; i++, j--) { + b[i] = a[j]! * 2; + sum += a[i]! - b[j]!; + } + return sum; +} + +function skipping(a: Uint8ClampedArray, hi: number): string { + for (let i = 0, j = hi; i < j; i += 2, j -= 3) { + a[i] = a[j]! + 200; + a[j] = i - 10; + } + return Array.from(a).join(); +} + +function palindrome(text: Uint16Array, n: number): boolean { + for (let i = 0, j = n - 1; i < j; i++, j--) if (text[i] !== text[j]) return false; + return true; +} + +console.log(reverse(new Int32Array([1, 2, 3, 4, 5, 6]), 0, 5)); +console.log(reverse(new Int32Array([1, 2, 3, 4, 5, 6]), 1, 3)); +console.log(reverse(new Int32Array([1, 2, 3, 4]), 0, 6)); +console.log(reverse(new Int32Array([1, 2, 3, 4]), -2, 2)); +console.log(reverse(new Int32Array([1, 2, 3, 4]), 0.5, 3.5)); +console.log(reverse(new Int32Array([1, 2, 3, 4]), 0, NaN)); +console.log(reverse(new Int32Array([1, 2, 3, 4]), 3, 0)); + +const buffer = new ArrayBuffer(48); +const whole = new Float64Array(buffer); +for (let k = 0; k < whole.length; k++) whole[k] = k + 0.5; +const tail = new Float64Array(buffer, 16, 3); +console.log(mirror(whole, tail, 4), Array.from(whole).join()); +console.log(mirror(tail, tail, 2), Array.from(whole).join()); +console.log(mirror(tail, whole, 7), Array.from(whole).join()); + +console.log(skipping(new Uint8ClampedArray([10, 20, 30, 40, 50, 60, 70, 80]), 7)); +console.log(skipping(new Uint8ClampedArray([10, 20, 30]), 9)); + +const word = new Uint16Array([114, 97, 99, 101, 99, 97, 114]); +console.log(palindrome(word, 7), palindrome(word, 6), palindrome(word, 9), palindrome(word, 0)); diff --git a/tests/harness/corpus-costs/7981-typed-array-paired-loops.ts.json b/tests/harness/corpus-costs/7981-typed-array-paired-loops.ts.json new file mode 100644 index 000000000..c93369880 --- /dev/null +++ b/tests/harness/corpus-costs/7981-typed-array-paired-loops.ts.json @@ -0,0 +1 @@ +{ "plain": 3500, "san": 4500 } From 8b6d3329f4666d641fcbebaf8be501ee4b6d0131 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 13:00:50 -0500 Subject: [PATCH 081/184] Hold an element assignment's receiver against its index's effects When the index assigns the receiver's binding (`a[(a = b, 0)] = c[0]!`), holding the index alone left the receiver read at the store, after the index ran, so the value landed in the replacement array. The receiver is now held whenever the index or the value could change what it reads. --- .../compiler/src/frontend/lowering/array-values.ts | 14 ++++++++++---- tests/corpus/7642-element-assignment-order.ts | 8 ++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/array-values.ts b/packages/compiler/src/frontend/lowering/array-values.ts index 9f23a860b..f191ad942 100644 --- a/packages/compiler/src/frontend/lowering/array-values.ts +++ b/packages/compiler/src/frontend/lowering/array-values.ts @@ -291,16 +291,22 @@ export function arrayValueStore( // JavaScript evaluates them first (`out[n++] = src[n]` reads src[1]), so // either may need its value held before the value runs. if (value.kind !== "varRef") { - const effects = valueEffects(lowerer, value); - const hold = (e: IrExpr, name: string): IrExpr => { + const valueFx = valueEffects(lowerer, value); + const hold = (e: IrExpr, name: string, effects: typeof valueFx): IrExpr => { if (unaffectedBy(e, effects) && !(effects.opaque && readsMutableLocal(lowerer, e))) return e; const temp = lowerer.declareHiddenLocal(name, e.type); prefix.push({ kind: "varDecl", localId: temp.id, init: e, loc }); return varRef(temp.id, temp.type, loc); }; - arr = hold(arr, "%arrayTarget"); - index = hold(index, "%arrayIndex"); + // The receiver runs before the index and the value, so the index's + // effects can change it too (`a[(a = b, 0)] = c[0]!`). + const indexFx = valueEffects(lowerer, index); + arr = hold(arr, "%arrayTarget", { + writes: new Set([...valueFx.writes, ...indexFx.writes]), + opaque: valueFx.opaque || indexFx.opaque, + }); + index = hold(index, "%arrayIndex", valueFx); } if (value.kind !== "varRef") { const temp = lowerer.declareHiddenLocal("%arrayValue", value.type); diff --git a/tests/corpus/7642-element-assignment-order.ts b/tests/corpus/7642-element-assignment-order.ts index f2f2fae7b..e020d05fc 100644 --- a/tests/corpus/7642-element-assignment-order.ts +++ b/tests/corpus/7642-element-assignment-order.ts @@ -35,3 +35,11 @@ function index(): number { const c: (number | undefined)[] = [0, 0]; c[index()] = optional[1]; console.log(c, reads); + +// The index assigns the receiver's binding: the store uses the receiver +// evaluated before it. +let receiver: (number | undefined)[] = [0, 0]; +const original = receiver; +const replacement: (number | undefined)[] = [7, 7]; +receiver[((receiver = replacement), 1)] = optional[1]; +console.log(original, replacement, receiver === replacement); From e3b42a311d988a04ad38cd805045adc616eec8b3 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 13:10:16 -0500 Subject: [PATCH 082/184] Add --target-cpu to compile x86-64 programs for a microarchitecture level Release builds compile for the x86-64 baseline, so LLVM vectorizes with SSE2 only. `--target-cpu=` compiles the program's code for a higher level: every emitted function carries the level as its `target-cpu`, so LLVM can use AVX2, BMI2, FMA or AVX-512. The executable then runs only on CPUs of that level. - `native` resolves to the highest level the build machine supports, read from /proc/cpuinfo on Linux and `sysctl machdep.cpu` on macOS. - The level is part of the executable cache key; `baseline` and builds without the option produce the same output and keys as before. - The runtime pack keeps its baseline code. - Targets other than x86-64 refuse a level (SC3002). The CLI's cached bootstrap route leaves builds with the option to the full compiler. - `bench:runtime` takes `--baseline-target-cpu` and `--candidate-target-cpu`. --- docs/content/docs/cli.mdx | 40 ++++++ packages/cli/src/bootstrap.ts | 3 + packages/cli/test/target-cpu.test.ts | 66 ++++++++++ .../compiler/src/backend/llvm/emit-module.ts | 23 +++- .../compiler/src/backend/llvm/emitter.test.ts | 10 ++ packages/compiler/src/backend/llvm/emitter.ts | 7 ++ .../compiler/src/backend/target-cpu.test.ts | 51 ++++++++ packages/compiler/src/backend/target-cpu.ts | 119 ++++++++++++++++++ packages/compiler/src/cli/command.ts | 29 ++++- packages/compiler/src/cli/usage.ts | 7 ++ packages/compiler/src/compile-types.ts | 7 ++ .../src/executable/executable-cache.test.ts | 2 + .../src/executable/executable-cache.ts | 5 + packages/compiler/src/index.ts | 11 ++ packages/compiler/src/native/prepare.ts | 2 + scripts/bench-runtime.mjs | 24 +++- 16 files changed, 395 insertions(+), 11 deletions(-) create mode 100644 packages/cli/test/target-cpu.test.ts create mode 100644 packages/compiler/src/backend/target-cpu.test.ts create mode 100644 packages/compiler/src/backend/target-cpu.ts diff --git a/docs/content/docs/cli.mdx b/docs/content/docs/cli.mdx index 8b0353620..9fa5e9447 100644 --- a/docs/content/docs/cli.mdx +++ b/docs/content/docs/cli.mdx @@ -135,6 +135,9 @@ With `--sparse`, the profile leaves out the functions the runs never reached. Th
--detached-exit
On Linux, when the program calls process.exit(), hand the release of its memory to a helper process so the parent sees the exit sooner. The kernel otherwise frees the whole address space before the exit is reported, which takes tens of milliseconds per gigabyte. Opt in only where the trade-offs are acceptable: for up to about 100 ms after the exit, the helper keeps the executable mapped (overwriting the binary in place fails with "text file busy"), its memory stays charged to the cgroup, and the teardown CPU time no longer appears in the parent's resource usage. Valid only for executable builds; it has no effect on other platforms.
+
--target-cpu <baseline|x86-64-v2|x86-64-v3|x86-64-v4|native>
+
On x86-64 targets, compile the program's code for an x86-64 microarchitecture level. The default, baseline, runs on any x86-64 CPU. A higher level lets LLVM use more instructions, and the executable then runs only on CPUs that support that level. native uses the highest level the build machine supports. Valid for executable, assembly, and object output on x86-64 targets. See Target CPU.
+
--profile-generate <pattern>
On Linux, build an instrumented executable for profile-guided optimization. Each run writes a raw profile to the path pattern, where %p expands to the process ID; the LLVM_PROFILE_FILE environment variable overrides the pattern at run time. A relative pattern resolves from the current working directory. Instrumented programs run slower. Valid only for executable builds with --optimization=release or speed.
@@ -228,6 +231,43 @@ $ scriptc build app.ts --optimization=speed --profile-use app.profdata Train on workloads like the ones the program will run: code the training never reached is optimized as without a profile. The profile can be committed and reused as the program changes. Functions whose control flow changed since the profile was collected lose their profile, and the rest keep theirs, so regenerate it when much of the program has changed. [Build parallelism](/docs/how-it-works#build-parallelism) describes what a profile adds to build time. +## Target CPU + +By default, x86-64 executables run on any x86-64 CPU, so the compiler uses only the baseline instruction set (SSE2). `--target-cpu` compiles the program's code for a higher x86-64 microarchitecture level. Each level includes the ones below it: + + + + + + + + + + + + + + + + + + + + + + +
LevelAdds
x86-64-v2SSE3, SSSE3, SSE4.1, SSE4.2, POPCNT
x86-64-v3AVX, AVX2, BMI1, BMI2, F16C, FMA, LZCNT, MOVBE
x86-64-v4AVX-512 (F, BW, CD, DQ, VL)
+ +```console +$ scriptc build app.ts --target-cpu=x86-64-v3 -o app +``` + +Loops over typed arrays and numbers gain the most, because LLVM vectorizes them with wider registers. Some loops run slower at a higher level, so measure the program on the machines it will run on. + +An executable built for a level runs only on CPUs that support that level. On an older CPU, it crashes with an illegal-instruction signal (`SIGILL`) when it reaches an instruction the CPU lacks. A named level is portable to every machine of that level. `native` selects the highest level the build machine supports, so use it only for executables that run on the machine that built them or on identical machines. + +The level applies to the program's own code. The runtime pack keeps its baseline code. Builds for different levels never share cached artifacts, and `baseline` builds are the same as builds without the option. + ## LLVM backend scriptc generates LLVM IR and compiles it to native code. Executable builds link the result with the matching precompiled runtime pack. Use --emit=llvm to inspect the generated code. diff --git a/packages/cli/src/bootstrap.ts b/packages/cli/src/bootstrap.ts index 242a7c6f7..f14866a9c 100644 --- a/packages/cli/src/bootstrap.ts +++ b/packages/cli/src/bootstrap.ts @@ -70,6 +70,8 @@ async function tryFastPath(): Promise { values["profile-generate"] !== undefined || values["profile-use"] !== undefined || values.sparse || + // The full compiler resolves `native` against the build host. + values["target-cpu"] !== undefined || (values["external-types"] ?? []).length > 0 ) return null; @@ -200,6 +202,7 @@ async function coverageFastPath( values.emit !== undefined || values.strip || values["detached-exit"] || + values["target-cpu"] !== undefined || values.lib || values.profile !== undefined || values.ffi !== undefined || diff --git a/packages/cli/test/target-cpu.test.ts b/packages/cli/test/target-cpu.test.ts new file mode 100644 index 000000000..c5852fb40 --- /dev/null +++ b/packages/cli/test/target-cpu.test.ts @@ -0,0 +1,66 @@ +import { execFile } from "node:child_process"; +import { createRequire } from "node:module"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; +import { hostTargetCpu } from "../../compiler/src/backend/target-cpu.js"; + +const execFileAsync = promisify(execFile); +const require = createRequire(import.meta.url); +const repoRoot = join(import.meta.dirname, "../../.."); +const cliEntry = join(repoRoot, "packages/cli/src/main.ts"); +const bootstrapEntry = join(repoRoot, "packages/cli/dist/bootstrap.js"); +const tsxLoader = join(dirname(require.resolve("tsx/package.json")), "dist/loader.mjs"); +const host = hostTargetCpu(); +const runsV3 = host === "x86-64-v3" || host === "x86-64-v4"; + +const program = `const a = new Int32Array(4096); +for (let i = 0; i < a.length; i++) a[i] = (i * 2654435761) | 0; +let s = 0; +for (let r = 0; r < 64; r++) for (let i = 0; i < a.length; i++) s = (s + a[i]!) | 0; +console.log(s); +`; + +test.runIf(process.platform === "linux" && process.arch === "x64")( + "--target-cpu compiles the program for the level without mixing cache variants", + async () => { + const dir = await mkdtemp(join("/tmp", "scriptc-target-cpu-")); + const entry = join(dir, "main.ts"); + const outPath = join(dir, "program"); + const llvmPath = join(dir, "main.ll"); + const env = { ...process.env, SCRIPTC_CACHE_DIR: join(dir, "cache") }; + const build = async (args: string[], bootstrap = false): Promise => { + await execFileAsync( + process.execPath, + [ + ...(bootstrap ? [bootstrapEntry] : ["--import", tsxLoader, cliEntry]), + "build", + entry, + "-o", + outPath, + ...args, + ], + { env, maxBuffer: 4 * 1024 * 1024 }, + ); + return readFile(llvmPath, "utf8"); + }; + try { + await writeFile(entry, program); + const expected = `${(await execFileAsync(process.execPath, [entry])).stdout}`; + const baseline = await build([]); + expect(baseline).not.toContain('"target-cpu"'); + expect(await build(["--target-cpu=baseline"])).toBe(baseline); + const v3 = await build(["--target-cpu=x86-64-v3"]); + expect(v3).toMatch(/^attributes #0 = \{ sanitize_address "target-cpu"="x86-64-v3" \}$/m); + if (runsV3) expect((await execFileAsync(outPath)).stdout).toBe(expected); + // The bootstrap's cached route builds without a level. + expect(await build([], true)).toBe(baseline); + expect((await execFileAsync(outPath)).stdout).toBe(expected); + await expect(build(["--target-cpu=haswell"])).rejects.toThrow(/unknown target CPU "haswell"/); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, + 240_000, +); diff --git a/packages/compiler/src/backend/llvm/emit-module.ts b/packages/compiler/src/backend/llvm/emit-module.ts index 16a1b1601..e5258e583 100644 --- a/packages/compiler/src/backend/llvm/emit-module.ts +++ b/packages/compiler/src/backend/llvm/emit-module.ts @@ -64,6 +64,7 @@ import { literalEqualityDefinition } from "./string-literal-equality.js"; import { STRING_READ_HELPERS, stringScanDecl } from "./string-reads.js"; import { emitLibraryIdentityLines } from "../library-identity-markers.js"; import { InternalCompilerError } from "../../errors.js"; +import { targetCpuAttribute } from "../target-cpu.js"; /** The function bodies emitted in `chunks` contiguous ranges, each by its * own emitter (LlEmitter.chunkEmitter), then merged in function order: the @@ -191,6 +192,20 @@ function readRange(dir: string, range: number): { result: ChunkResult; texts: st } } +/** The attribute groups of the emitted functions: #0 for every function, + * #1 for coroutine bodies and #2 for out-of-line record clones. Each carries + * the build's `--target-cpu` level. */ +function functionAttributes(emitter: LlEmitter, hasNoInlineRecordClone: boolean): string[] { + const cpu = targetCpuAttribute(emitter.targetCpu); + return [ + `attributes #0 = { sanitize_address${cpu} }`, + ...(emitter.usesCoroutines + ? [`attributes #1 = { sanitize_address presplitcoroutine${cpu} }`] + : []), + ...(hasNoInlineRecordClone ? [`attributes #2 = { noinline sanitize_address${cpu} }`] : []), + ]; +} + export function emitModuleParts(emitter: LlEmitter): string[] { // Function bodies first (the literal/unit/fn-value tables fill as they // emit), then the file assembles around them — the runtime ABI’s order. @@ -972,9 +987,7 @@ export function emitModuleParts(emitter: LlEmitter): string[] { // symbols specified by the library IR instead. for (const line of emitter.emitLibDefs(globals, globalReleaseLines, stamps)) out.push(line); out.push(...emitter.inlineRcTail(flushedDecls)); - out.push(`attributes #0 = { sanitize_address }`); - if (emitter.usesCoroutines) out.push(`attributes #1 = { sanitize_address presplitcoroutine }`); - if (hasNoInlineRecordClone) out.push(`attributes #2 = { noinline sanitize_address }`); + out.push(...functionAttributes(emitter, hasNoInlineRecordClone)); out.push(``); out.push(...emitter.fieldAliasMetadata()); if (emitter.debug !== null) out.push(emitter.debug.render()); @@ -1122,9 +1135,7 @@ export function emitModuleParts(emitter: LlEmitter): string[] { // sanitize_address is inert under the plain pipeline; the sanitized // lane's -fsanitize=address link activates instrumentation over the // emitted functions too (the runtime TUs get theirs from clang). - `attributes #0 = { sanitize_address }`, - ...(emitter.usesCoroutines ? [`attributes #1 = { sanitize_address presplitcoroutine }`] : []), - ...(hasNoInlineRecordClone ? [`attributes #2 = { noinline sanitize_address }`] : []), + ...functionAttributes(emitter, hasNoInlineRecordClone), ``, ); if (emitter.mod.workers) { diff --git a/packages/compiler/src/backend/llvm/emitter.test.ts b/packages/compiler/src/backend/llvm/emitter.test.ts index e6334ab4c..7dffc9183 100644 --- a/packages/compiler/src/backend/llvm/emitter.test.ts +++ b/packages/compiler/src/backend/llvm/emitter.test.ts @@ -202,6 +202,16 @@ test("detached exit defines the runtime opt-in symbol only when requested", () = expect(detached.replace("@scr_detached_exit = constant i8 1\n", "")).toBe(plain); }); +test("a target CPU level reaches every function attribute group and nothing else", () => { + const module = moduleFor([call()]); + const plain = emitLlvmModule(module); + expect(plain).not.toContain("target-cpu"); + const v3 = emitLlvmModule(module, { targetCpu: "x86-64-v3" }); + expect(v3).toMatch(/^attributes #0 = \{ sanitize_address "target-cpu"="x86-64-v3" \}$/m); + expect(v3.replaceAll(' "target-cpu"="x86-64-v3"', "")).toBe(plain); + expect(emitLlvmModule(module, { targetCpu: "baseline" })).toBe(plain); +}); + test("inline RC inlines every fast path at speed and keeps box releases as calls in release", () => { for (const boxed of [false, true]) { const module = moduleFor(Array.from({ length: 4 }, call), { boxed }); diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 4aa9b65ad..2e46f60b9 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -1,3 +1,4 @@ +import type { TargetCpu } from "../target-cpu.js"; import { relieveHeapPressure } from "../../heap-pressure.js"; import { wasmBoundsChecks } from "./wasm-modules.js"; import { jsTruthy } from "./jsval.js"; @@ -482,6 +483,10 @@ export interface LlvmTargetOptions { * store the instance's address over 8 in 32 bits (classes.ts * compactRefLoadLines). Elsewhere they keep 8-byte pointers. */ lowRegion?: boolean; + /** The x86-64 level the program's code may require (`--target-cpu`): + * every emitted function carries it as its `target-cpu`. Unset, code + * runs on any x86-64 CPU. */ + targetCpu?: TargetCpu; } /** LlvmTargetOptions.lowRegion for an executable: Linux targets without @@ -1048,6 +1053,7 @@ export class LlEmitter { readonly emitLibraryIdentity: boolean; readonly runtimeAbiMarker: boolean; readonly detachedExit: boolean; + readonly targetCpu: TargetCpu | undefined; readonly wasmBoundsChecks: boolean; /** Interned string literals: UTF-8 text → { symbol, byte length } — * first-use order, the runtime ABI’s determinism discipline. */ @@ -1630,6 +1636,7 @@ export class LlEmitter { this.emitLibraryIdentity = options.emitLibraryIdentity !== false; this.runtimeAbiMarker = options.runtimeAbiMarker === true; this.detachedExit = options.detachedExit === true; + this.targetCpu = options.targetCpu; this.wasmBoundsChecks = options.wasmBoundsChecks === true || wasmBoundsChecks({ diff --git a/packages/compiler/src/backend/target-cpu.test.ts b/packages/compiler/src/backend/target-cpu.test.ts new file mode 100644 index 000000000..428caee32 --- /dev/null +++ b/packages/compiler/src/backend/target-cpu.test.ts @@ -0,0 +1,51 @@ +import { expect, test } from "vitest"; +import { + isTargetCpuOption, + targetCpuAttribute, + targetCpuField, + targetCpuKeyParts, + targetCpuTargetRefusal, + x86Level, +} from "./target-cpu.js"; + +const V2 = ["cx16", "lahf_lm", "popcnt", "pni", "sse4_1", "sse4_2", "ssse3"]; +const V3 = ["avx", "avx2", "bmi1", "bmi2", "f16c", "fma", "abm", "movbe", "xsave"]; +const V4 = ["avx512f", "avx512bw", "avx512cd", "avx512dq", "avx512vl"]; +const level = (flags: string[]) => x86Level((flag) => flags.includes(flag)); + +test("a host's level is the highest one whose features and lower levels' features it has", () => { + expect(level([])).toBe("baseline"); + expect(level(V2)).toBe("x86-64-v2"); + expect(level([...V2, ...V3])).toBe("x86-64-v3"); + expect(level([...V2, ...V3, ...V4])).toBe("x86-64-v4"); + // AVX-512 without BMI2 is still v2. + expect(level([...V2, ...V3.filter((f) => f !== "bmi2"), ...V4])).toBe("x86-64-v2"); + expect(level([...V3, ...V4])).toBe("baseline"); +}); + +test("the baseline adds no key part, attribute or option field", () => { + for (const cpu of [undefined, "baseline"] as const) { + expect(targetCpuKeyParts(cpu)).toEqual([]); + expect(targetCpuAttribute(cpu)).toBe(""); + expect(targetCpuField(cpu)).toEqual({}); + expect(targetCpuTargetRefusal(cpu, "linux", "arm64")).toBeNull(); + } + expect(targetCpuKeyParts("x86-64-v3")).toEqual(["target-cpu-x86-64-v3"]); + expect(targetCpuAttribute("x86-64-v4")).toBe(' "target-cpu"="x86-64-v4"'); + expect(targetCpuField("x86-64-v2")).toEqual({ targetCpu: "x86-64-v2" }); +}); + +test("a level is refused for targets that are not x86-64", () => { + expect(targetCpuTargetRefusal("x86-64-v3", "linux", "x64")).toBeNull(); + expect(targetCpuTargetRefusal("x86-64-v3", "darwin", "arm64")).toBe( + "--target-cpu=x86-64-v3 requires an x86-64 target", + ); + expect(targetCpuTargetRefusal("x86-64-v3", "wasi", "x64")).not.toBeNull(); +}); + +test("the option accepts the levels and native", () => { + for (const value of ["baseline", "x86-64-v2", "x86-64-v3", "x86-64-v4", "native"]) + expect(isTargetCpuOption(value)).toBe(true); + for (const value of ["x86-64", "haswell", "", undefined]) + expect(isTargetCpuOption(value)).toBe(false); +}); diff --git a/packages/compiler/src/backend/target-cpu.ts b/packages/compiler/src/backend/target-cpu.ts new file mode 100644 index 000000000..228652ab3 --- /dev/null +++ b/packages/compiler/src/backend/target-cpu.ts @@ -0,0 +1,119 @@ +import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { nativeCodegenTarget } from "./targets.js"; + +/** The x86-64 microarchitecture levels an executable may require + * (`--target-cpu`). `baseline` is the x86-64 baseline every release build + * targets; the other levels are LLVM's `x86-64-v2` to `x86-64-v4`. */ +export type TargetCpu = "baseline" | "x86-64-v2" | "x86-64-v3" | "x86-64-v4"; + +/** The values `--target-cpu` accepts: a level, or `native` for the highest + * level the build host supports. */ +export const TARGET_CPU_OPTIONS = ["baseline", "x86-64-v2", "x86-64-v3", "x86-64-v4", "native"]; + +export function isTargetCpuOption(value: unknown): value is TargetCpu | "native" { + return typeof value === "string" && TARGET_CPU_OPTIONS.includes(value); +} + +/** The CPU features each level adds to the one below it, as Linux names + * them in /proc/cpuinfo (`pni` is SSE3, `abm` includes LZCNT). */ +const LEVEL_FLAGS: readonly [TargetCpu, readonly string[]][] = [ + ["x86-64-v2", ["cx16", "lahf_lm", "popcnt", "pni", "sse4_1", "sse4_2", "ssse3"]], + ["x86-64-v3", ["avx", "avx2", "bmi1", "bmi2", "f16c", "fma", "abm", "movbe", "xsave"]], + ["x86-64-v4", ["avx512f", "avx512bw", "avx512cd", "avx512dq", "avx512vl"]], +]; + +/** The macOS `machdep.cpu` spellings of the same features. */ +const DARWIN_FLAGS: Readonly> = { + cx16: "CX16", + lahf_lm: "LAHF", + popcnt: "POPCNT", + pni: "SSE3", + sse4_1: "SSE4.1", + sse4_2: "SSE4.2", + ssse3: "SSSE3", + avx: "AVX1.0", + avx2: "AVX2", + bmi1: "BMI1", + bmi2: "BMI2", + f16c: "F16C", + fma: "FMA", + abm: "LZCNT", + movbe: "MOVBE", + xsave: "XSAVE", + avx512f: "AVX512F", + avx512bw: "AVX512BW", + avx512cd: "AVX512CD", + avx512dq: "AVX512DQ", + avx512vl: "AVX512VL", +}; + +/** The highest level whose features (and every lower level's) are all in + * `has`. */ +export function x86Level(has: (flag: string) => boolean): TargetCpu { + let level: TargetCpu = "baseline"; + for (const [next, flags] of LEVEL_FLAGS) { + if (!flags.every(has)) break; + level = next; + } + return level; +} + +/** The highest level the build host supports, or null when the host is not + * an x86-64 Linux or macOS machine. */ +export function hostTargetCpu( + platform: string = process.platform, + arch: string = process.arch, +): TargetCpu | null { + if (arch !== "x64") return null; + try { + if (platform === "linux") { + const info = readFileSync("/proc/cpuinfo", "utf8"); + const line = /^flags\s*:(.*)$/m.exec(info); + if (!line) return null; + const flags = new Set(line[1]!.trim().split(/\s+/)); + return x86Level((flag) => flags.has(flag)); + } + if (platform === "darwin") { + const text = execFileSync( + "sysctl", + ["-n", "machdep.cpu.features", "machdep.cpu.leaf7_features", "machdep.cpu.extfeatures"], + { encoding: "utf8" }, + ); + const flags = new Set(text.trim().split(/\s+/)); + return x86Level((flag) => flags.has(DARWIN_FLAGS[flag]!)); + } + } catch { + return null; + } + return null; +} + +/** Cache-key parts for a target level; empty for the baseline so existing + * keys are unchanged. */ +export function targetCpuKeyParts(cpu: TargetCpu | undefined): string[] { + return cpu === undefined || cpu === "baseline" ? [] : [`target-cpu-${cpu}`]; +} + +/** The function attribute that makes LLVM generate code for the level, or + * "" for the baseline. */ +export function targetCpuAttribute(cpu: TargetCpu | undefined): string { + return cpu === undefined || cpu === "baseline" ? "" : ` "target-cpu"="${cpu}"`; +} + +/** The options field for a level: omitted for the baseline. */ +export function targetCpuField(cpu: TargetCpu | undefined): { targetCpu?: TargetCpu } { + return cpu === undefined || cpu === "baseline" ? {} : { targetCpu: cpu }; +} + +/** The error for a level on a target that is not x86-64, or null. */ +export function targetCpuTargetRefusal( + cpu: TargetCpu | undefined, + platform: string, + architecture: string = nativeCodegenTarget()?.architecture ?? process.arch, +): string | null { + if (cpu === undefined || cpu === "baseline") return null; + if (architecture !== "x64" || platform === "wasi") + return `--target-cpu=${cpu} requires an x86-64 target`; + return null; +} diff --git a/packages/compiler/src/cli/command.ts b/packages/compiler/src/cli/command.ts index 3285aa7fa..57b15349e 100644 --- a/packages/compiler/src/cli/command.ts +++ b/packages/compiler/src/cli/command.ts @@ -15,6 +15,13 @@ import { buildEnvelope } from "../diagnostics/envelope.js"; import { renderWarnings } from "../diagnostics/render.js"; import { isPerformanceCode, renderPerformanceWarnings } from "../diagnostics/dynamic-tier.js"; import { isNativeOptimization } from "../backend/optimization.js"; +import { + hostTargetCpu, + isTargetCpuOption, + TARGET_CPU_OPTIONS, + targetCpuField, + type TargetCpu, +} from "../backend/target-cpu.js"; import { setProvenanceSources } from "../frontend/provenance-registry.js"; import type { CliHost, NativeCacheWarmProfile } from "./host.js"; import { resolveOutputOptions } from "./output-options.js"; @@ -96,6 +103,7 @@ async function main(args: string[], host: CliHost): Promise { values.strip || values["windows-subsystem"] !== undefined || values["detached-exit"] || + values["target-cpu"] !== undefined || values["profile-generate"] !== undefined || values["profile-use"] !== undefined || values.sparse || @@ -154,6 +162,7 @@ async function main(args: string[], host: CliHost): Promise { values.sanitize || values["windows-subsystem"] !== undefined || values["detached-exit"] || + values["target-cpu"] !== undefined || values["profile-generate"] !== undefined || values["profile-use"] !== undefined || (values["npm-static"] ?? []).length > 0 || @@ -210,6 +219,7 @@ async function main(args: string[], host: CliHost): Promise { values.ffi !== undefined || values["windows-subsystem"] !== undefined || values["detached-exit"] || + values["target-cpu"] !== undefined || values["profile-generate"] !== undefined || values["profile-use"] !== undefined || (values["npm-static"] ?? []).length > 0 || @@ -218,7 +228,7 @@ async function main(args: string[], host: CliHost): Promise { values["explain-dynamic"] !== undefined ) { fail( - "scriptc build --lib takes no --backend/--emit/--print/--optimization/--strip/--windows-subsystem/--detached-exit/--profile-generate/--profile-use/--npm-static/--ffi/--external-types/--tier/--explain-dynamic: the profile pins the emission and optimization, npm imports are judged automatically, outbound FFI belongs to executable builds, external type mappings belong to coverage, and library artifacts are static-tier only", + "scriptc build --lib takes no --backend/--emit/--print/--optimization/--strip/--windows-subsystem/--detached-exit/--target-cpu/--profile-generate/--profile-use/--npm-static/--ffi/--external-types/--tier/--explain-dynamic: the profile pins the emission and optimization, npm imports are judged automatically, outbound FFI belongs to executable builds, external type mappings belong to coverage, and library artifacts are static-tier only", ); } const profilePath = resolve(profileArg); @@ -261,6 +271,9 @@ async function main(args: string[], host: CliHost): Promise { if (command === "coverage" && values.strip) { fail(`--strip is a build/run option\n\n${USAGE}`); } + if (command === "coverage" && values["target-cpu"] !== undefined) { + fail(`--target-cpu is a build/run option\n\n${USAGE}`); + } if (command === "coverage" && values["detached-exit"]) { fail(`--detached-exit is a build/run option\n\n${USAGE}`); } @@ -377,6 +390,19 @@ async function main(args: string[], host: CliHost): Promise { if (windowsSubsystem !== undefined && command === "coverage") { fail(`--windows-subsystem is only supported for executable builds\n\n${USAGE}`); } + const targetCpuOption = values["target-cpu"]; + if (targetCpuOption !== undefined && !isTargetCpuOption(targetCpuOption)) { + fail( + `unknown target CPU "${targetCpuOption}" (supported: ${TARGET_CPU_OPTIONS.join(", ")})\n\n${USAGE}`, + ); + } + let targetCpu: TargetCpu | undefined = targetCpuOption === "native" ? undefined : targetCpuOption; + if (targetCpuOption === "native") { + const level = hostTargetCpu(); + if (level === null) + fail(`--target-cpu=native needs an x86-64 Linux or macOS build host; name a level instead`); + targetCpu = level; + } const output = command === "coverage" ? null @@ -498,6 +524,7 @@ async function main(args: string[], host: CliHost): Promise { ...(values.strip ? { strip: true } : {}), ...(windowsSubsystem !== undefined ? { windowsSubsystem } : {}), ...(values["detached-exit"] ? { detachedExit: true } : {}), + ...targetCpuField(targetCpu), // The instrumented program resolves a relative pattern against its own // working directory; resolve it here, where the user wrote it. ...(values["profile-generate"] === undefined diff --git a/packages/compiler/src/cli/usage.ts b/packages/compiler/src/cli/usage.ts index c4d07656a..01afbdd41 100644 --- a/packages/compiler/src/cli/usage.ts +++ b/packages/compiler/src/cli/usage.ts @@ -54,6 +54,12 @@ Options: a helper process so the parent sees the exit sooner. The helper keeps the executable mapped and its memory charged for up to ~100 ms after exit (opt in; --emit=exe only) + --target-cpu + x86-64: compile the program's code for a CPU level + (default: baseline, which runs on any x86-64 CPU). The + executable then requires that level: v3 adds AVX2, BMI2 + and FMA, v4 adds AVX-512. native uses the build host's + level. The runtime keeps its baseline code --profile-generate Linux: instrument the executable for profile-guided optimization. Each run writes a raw profile to the @@ -123,6 +129,7 @@ export const CLI_OPTIONS = { strip: { type: "boolean", default: false }, "windows-subsystem": { type: "string" }, "detached-exit": { type: "boolean", default: false }, + "target-cpu": { type: "string" }, "profile-generate": { type: "string" }, "profile-use": { type: "string" }, sparse: { type: "boolean", default: false }, diff --git a/packages/compiler/src/compile-types.ts b/packages/compiler/src/compile-types.ts index 9ecd75974..c50931c6a 100644 --- a/packages/compiler/src/compile-types.ts +++ b/packages/compiler/src/compile-types.ts @@ -1,3 +1,4 @@ +import type { TargetCpu } from "./backend/target-cpu.js"; import type { WindowsSubsystem } from "./backend/targets.js"; import type { NativeLinkInfo } from "./backend/native-link-info.js"; import type { NativeOptimization } from "./backend/optimization.js"; @@ -36,6 +37,12 @@ export interface CompileBaseOptions { * executable mapped and the memory charged for up to ~100 ms after exit. * Executables only; no effect on other platforms. */ detachedExit?: boolean; + /** The x86-64 microarchitecture level the executable's code may require + * (`x86-64-v2` to `x86-64-v4`). The program's functions are compiled for + * that level, and the executable then runs only on CPUs that support it. + * Unset or `baseline`, the code runs on any x86-64 CPU. x86-64 targets + * only; the runtime pack keeps its baseline code. */ + targetCpu?: TargetCpu; /** Instrument the executable for profile-guided optimization. Each run * writes a raw profile to this path pattern (`%p` expands to the process * id; `LLVM_PROFILE_FILE` overrides the pattern at run time). Linux diff --git a/packages/compiler/src/executable/executable-cache.test.ts b/packages/compiler/src/executable/executable-cache.test.ts index 5cf8c6588..923730e17 100644 --- a/packages/compiler/src/executable/executable-cache.test.ts +++ b/packages/compiler/src/executable/executable-cache.test.ts @@ -235,6 +235,8 @@ test("early executable keys isolate compile modes, paths, implementation, checke { ...f.options, optimization: "dev" }, { ...f.options, optimization: "speed" }, { ...f.options, detachedExit: true }, + { ...f.options, targetCpu: "x86-64-v3" }, + { ...f.options, targetCpu: "x86-64-v4" }, { ...f.options, profile: "profile-generate:/tmp/%p.profraw" }, { ...f.options, profile: "profile-use:digest" }, { ...f.options, checker: "tsc-ts-executable-identity" }, diff --git a/packages/compiler/src/executable/executable-cache.ts b/packages/compiler/src/executable/executable-cache.ts index 7b7febc06..343b03430 100644 --- a/packages/compiler/src/executable/executable-cache.ts +++ b/packages/compiler/src/executable/executable-cache.ts @@ -18,6 +18,7 @@ import { compilerReleaseVersion } from "../library/sidecar.js"; import { nativeArtifactDependenciesStillMatch } from "../backend/native/artifact-stamps.js"; import { type NativeArtifactDependency } from "../backend/native/contracts.js"; import { optimizationKeyParts } from "../backend/optimization.js"; +import { type TargetCpu, targetCpuKeyParts } from "../backend/target-cpu.js"; import { privateSiblingPath } from "../backend/build-cache.js"; import type { CompilerImplementationDependency } from "../library/compiler-self-identity.js"; import { @@ -138,6 +139,8 @@ export interface EarlyExecutableCacheOptions { windowsSubsystem?: "gui"; /** Omitted unless --detached-exit is set. */ detachedExit?: true; + /** Omitted for the x86-64 baseline (`--target-cpu`). */ + targetCpu?: TargetCpu; /** Omitted without --profile-generate/--profile-use; otherwise the * pattern, or the digest of the profile's bytes. */ profile?: string; @@ -279,6 +282,7 @@ function cacheKey(options: EarlyExecutableCacheOptions): string { ...(options.strip ? ["strip"] : []), ...(options.windowsSubsystem === "gui" ? ["windows-subsystem-gui"] : []), ...(options.detachedExit ? ["detached-exit"] : []), + ...targetCpuKeyParts(options.targetCpu), ...(options.profile !== undefined ? [options.profile] : []), ...(options.checker !== undefined ? [`checker:${options.checker}`] : []), ...(options.youngNursery ? ["young-nursery"] : []), @@ -321,6 +325,7 @@ function routeKey(options: Omit] [--baseline=] [--workloads=a,b] [--runs=15] [--warmup=2] [--timeout=120] [--layouts=1] [--json=] [--baseline-optimization=release] [--candidate-optimization=release] + [--baseline-target-cpu=] [--candidate-target-cpu=] [--no-lock] [--no-node] [--keep] Each checkout must have a built CLI (packages/cli/dist/bootstrap.js) and native @@ -69,7 +72,10 @@ Workload names are the file names in benchmarks/runtime/workloads/*.json; for ex --baseline-optimization/--candidate-optimization pick each contender's --optimization posture (default release). The same checkout may be both contenders, e.g. --baseline=. --candidate-optimization=speed compares speed with -release. cold_build_ms records each contender's first, uncached build.`); +release. --baseline-target-cpu/--candidate-target-cpu pass --target-cpu to each +contender's builds, e.g. --baseline=. --candidate-target-cpu=x86-64-v3 compares +x86-64-v3 code with the baseline. cold_build_ms records each contender's first, +uncached build.`); process.exit(0); } const runs = Number(values.runs); @@ -428,7 +434,7 @@ function revision(root) { }); return r.stdout.trim() + (dirty.stdout.trim() ? "+dirty" : ""); } -function build(label, root, workload, layout, optimization) { +function build(label, root, workload, layout, optimization, targetCpu) { const cli = cliFor(root); const out = join(work, label, layout === 0 ? workload.name : `${workload.name}.layout${layout}`); mkdirSync(dirname(out), { recursive: true }); @@ -444,7 +450,15 @@ function build(label, root, workload, layout, optimization) { const entry = join(suiteRoot, workload.entry); const result = run( process.execPath, - [cli, "build", entry, `--optimization=${optimization}`, "-o", out], + [ + cli, + "build", + entry, + `--optimization=${optimization}`, + ...(targetCpu === undefined ? [] : [`--target-cpu=${targetCpu}`]), + "-o", + out, + ], { env, timeout: 900_000, @@ -467,6 +481,7 @@ const contenders = [ label: "baseline", root: resolve(values.baseline), optimization: values["baseline-optimization"], + targetCpu: values["baseline-target-cpu"], }, ] : []), @@ -474,6 +489,7 @@ const contenders = [ label: "candidate", root: resolve(values.candidate), optimization: values["candidate-optimization"], + targetCpu: values["candidate-target-cpu"], }, ]; const report = { @@ -580,7 +596,7 @@ try { entry.results[c.label] = result; result.binaries = []; for (let layout = 0; layout < layouts && result.status === undefined; layout++) { - const built = build(c.label, c.root, w, layout, c.optimization); + const built = build(c.label, c.root, w, layout, c.optimization, c.targetCpu); result.build_ms += Math.round(built.buildMs); if (layout === 0) result.cold_build_ms = Math.round(built.buildMs); if (!built.ok) { From 26678e3c72385b44fe784ffea31c6a3f7fe2318c Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 13:20:59 -0500 Subject: [PATCH 083/184] Keep class methods native when element writes store data under their names --- benchmarks/runtime/README.md | 1 + .../workloads/keyed-option-writes.json | 6 ++ .../runtime/workloads/keyed-option-writes.ts | 76 ++++++++++++++++++ .../compiler/src/contracts/int32-elements.ts | 3 +- .../frontend/lowering/builtins/webassembly.ts | 5 +- .../lowering/class-method-mutations.ts | 53 +++++++++++-- .../lowering/containers/array-pipelines.ts | 4 +- .../lowering/containers/iterator-helpers.ts | 18 +++-- .../frontend/lowering/generic-functions.ts | 6 +- .../lowering/instantiation-path.test.ts | 12 ++- .../frontend/lowering/instantiation-path.ts | 6 +- .../compiler/src/frontend/lowering/lowerer.ts | 58 +++++++++----- .../lowering/site-dependencies.test.ts | 14 ++-- .../frontend/lowering/site-dependencies.ts | 15 +++- .../frontend/lowering/string-index-bounds.ts | 23 +++++- packages/compiler/src/frontend/npm-static.ts | 59 +++++++------- packages/compiler/src/frontend/services.ts | 9 ++- .../src/frontend/ts7/semantic-model.ts | 4 +- .../src/frontend/ts7/source-parser.ts | 4 +- .../compiler/test/keyed-method-writes.test.ts | 79 +++++++++++++++++++ 20 files changed, 361 insertions(+), 94 deletions(-) create mode 100644 benchmarks/runtime/workloads/keyed-option-writes.json create mode 100644 benchmarks/runtime/workloads/keyed-option-writes.ts create mode 100644 packages/compiler/test/keyed-method-writes.test.ts diff --git a/benchmarks/runtime/README.md b/benchmarks/runtime/README.md index 4f152e4bb..f7c21571a 100644 --- a/benchmarks/runtime/README.md +++ b/benchmarks/runtime/README.md @@ -16,6 +16,7 @@ Application-shaped workloads for measuring the speed of compiled executables. Ea | `functional-pipeline` | cpu | `filter`/`map`/`reduce`/`flatMap` chains, closures, small records | | `intern-table`, `intern-table-map`, `intern-table-1`, `intern-table-map-1` | cpu | worker threads interning the same type-like objects under string keys and records under number keys, then hits and misses: one shared `InternTable` per kind of key, or a module-level `Map` per thread (the `-map` entries), with 8 threads or 1 (the `-1` entries); under Node.js every thread has its own tables | | `json-records` | cpu | `JSON.stringify`, typed `JSON.parse`, Map aggregation | +| `keyed-option-writes` | cpu | class methods whose names are also keys of option records written through element accesses: `enabled[name] = true` over a union of literal names and `totals["count"] = n` | | `log-lines` | cpu | 200k immediate `console.log`/`process.stdout.write` lines into the runner's pipe | | `log-summary`, `inventory-report` | cpu | the multi-module applications from `benchmarks/builds`, on 300k-line inputs | | `number-maps` | cpu | number-keyed Map/Set: a dense-id table, sparse ids in many small maps, get/has/set mixes | diff --git a/benchmarks/runtime/workloads/keyed-option-writes.json b/benchmarks/runtime/workloads/keyed-option-writes.json new file mode 100644 index 000000000..802911e7d --- /dev/null +++ b/benchmarks/runtime/workloads/keyed-option-writes.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/keyed-option-writes.ts", + "args": [ + "4" + ] +} diff --git a/benchmarks/runtime/workloads/keyed-option-writes.ts b/benchmarks/runtime/workloads/keyed-option-writes.ts new file mode 100644 index 000000000..95cdbb86b --- /dev/null +++ b/benchmarks/runtime/workloads/keyed-option-writes.ts @@ -0,0 +1,76 @@ +// Class methods whose names are also keys of option records written through +// element accesses: `enabled[name] = true` over a union of literal names and +// `totals["count"] = n`. The writes store booleans and numbers into plain +// records, so the methods keep their native calls. +type Step = "scale" | "offset" | "clamp"; + +const enabled: Record = { scale: false, offset: false, clamp: false }; +function enable(name: Step): void { + enabled[name] = true; +} + +const totals: Record = {}; + +class Transform { + private readonly factor: number; + private readonly shift: number; + constructor(factor: number, shift: number) { + this.factor = factor; + this.shift = shift; + } + scale(x: number): number { + return enabled.scale ? x * this.factor : x; + } + offset(x: number): number { + return enabled.offset ? x + this.shift : x; + } + clamp(x: number): number { + if (!enabled.clamp) return x; + return x < 0 ? 0 : x > 1000 ? 1000 : x; + } +} + +class Counter { + private seen = 0; + private sum = 0; + add(value: number): void { + this.seen++; + this.sum += value; + } + count(): number { + return this.seen; + } + mean(): number { + return this.seen === 0 ? 0 : this.sum / this.seen; + } +} + +const scale = Number(process.argv[2] ?? "1"); +const steps: Step[] = ["scale", "offset", "clamp"]; +for (const step of steps) enable(step); + +const transforms: Transform[] = []; +for (let i = 0; i < 16; i++) transforms.push(new Transform(1 + (i % 5) * 0.25, (i % 7) - 3)); +const counters: Counter[] = []; +for (let i = 0; i < 8; i++) counters.push(new Counter()); + +let seed = 12345; +let checksum = 0; +const rounds = Math.floor(2_000_000 * scale); +for (let i = 0; i < rounds; i++) { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + const t = transforms[seed & 15]!; + const value = t.clamp(t.offset(t.scale((seed >>> 8) % 1200))); + const counter = counters[(seed >>> 4) & 7]!; + counter.add(value); + checksum += value; + if ((i & 65535) === 0) totals["count"] = counter.count(); +} + +let mean = 0; +let seen = 0; +for (const counter of counters) { + mean += counter.mean(); + seen += counter.count(); +} +console.log("seen", seen, "count", totals["count"], "mean", (mean / counters.length).toFixed(4), "checksum", checksum); diff --git a/packages/compiler/src/contracts/int32-elements.ts b/packages/compiler/src/contracts/int32-elements.ts index ca61a8e49..67446f2d1 100644 --- a/packages/compiler/src/contracts/int32-elements.ts +++ b/packages/compiler/src/contracts/int32-elements.ts @@ -709,7 +709,8 @@ export class Int32ElementContracts { const findChainsIn = (st: IrStmt): boolean => everyStmtChild(st, findChains, findChainsIn); for (const st of fn.body) findChainsIn(st); // Parameters whose callers are not all visible hold opaque arrays. - if (this.valueEscaped.has(fn.name) || this.runtimeVirtual(fn.name)) + const runtimeVirtual = this.runtimeVirtual; + if (this.valueEscaped.has(fn.name) || runtimeVirtual(fn.name)) for (const p of fn.params) if (this.holds(p.type)) { const why = this.valueEscaped.has(fn.name) diff --git a/packages/compiler/src/frontend/lowering/builtins/webassembly.ts b/packages/compiler/src/frontend/lowering/builtins/webassembly.ts index 8e5c3a048..f33ebe9fe 100644 --- a/packages/compiler/src/frontend/lowering/builtins/webassembly.ts +++ b/packages/compiler/src/frontend/lowering/builtins/webassembly.ts @@ -403,7 +403,10 @@ export function lowerWebAssemblyValueOf( recv: IrExpr, loc: IrExpr["loc"], ): IrExpr | null { - const ctx = { checker: lowerer.checker, isStdlibFile: lowerer.isStdlibFile }; + const ctx = { + checker: lowerer.checker, + isStdlibFile: (sf: ts.SourceFile) => lowerer.isStdlibFile(sf), + }; if (!isWebAssemblyType(lowerer.typeOf(receiver), ctx)) return null; lowerer.usesWebAssembly = true; return { kind: "libCall", fn: "wasm.valueOf", args: [recv], type: DYN, loc }; diff --git a/packages/compiler/src/frontend/lowering/class-method-mutations.ts b/packages/compiler/src/frontend/lowering/class-method-mutations.ts index fd23b6db8..d75f1d499 100644 --- a/packages/compiler/src/frontend/lowering/class-method-mutations.ts +++ b/packages/compiler/src/frontend/lowering/class-method-mutations.ts @@ -36,9 +36,20 @@ export function collectClassMethodMutations( const computedWrites: { key: ts.Expression | undefined; receiver: ts.Expression | undefined; + /** The written element access, when the write has one. */ + target?: ts.ElementAccessExpression; site: ts.Node; }[] = []; const propertyWrites: { target: ts.PropertyAccessExpression; site: ts.Node }[] = []; + /** Writes and deletes of a method name through an element access with a + * known key (`o["m"] = f`, `o[k] = f` with `k: "m" | "n"`), judged like + * the named writes in propertyWrites. */ + const keyedWrites: { + name: string; + receiver: ts.Expression; + target: ts.ElementAccessExpression; + site: ts.Node; + }[] = []; const reflection: ts.PropertyAccessExpression[] = []; const globals: ts.Identifier[] = []; const jsAssignments: (ts.PropertyAccessExpression | ts.ElementAccessExpression)[] = []; @@ -242,13 +253,32 @@ export function collectClassMethodMutations( sameBinding(receiver, keys.arguments.nodes[0]!) ); }; - const markKey = (key: ts.Expression, site: ts.Node, receiver?: ts.Expression): void => { + /** A write of the known key `name`: through an element access, a write + * to a method name is judged like a named write (keyedWrites). */ + const markKeyed = ( + name: string, + site: ts.Node, + receiver?: ts.Expression, + target?: ts.ElementAccessExpression, + ): void => { + if (receiver && target && methods.has(name) && name !== "prototype" && name !== "__proto__") { + receiverFields.add(name); + keyedWrites.push({ name, receiver, target, site }); + } else markName(name, site); + }; + const markKey = ( + key: ts.Expression, + site: ts.Node, + receiver?: ts.Expression, + target?: ts.ElementAccessExpression, + ): void => { const value = unwrap(key); - if (ts.isStringLiteralLike(value)) markName(value.text, site); - else if (ts.isNumericLiteral(value)) markName(String(Number(value.text)), site); + if (ts.isStringLiteralLike(value)) markKeyed(value.text, site, receiver, target); + else if (ts.isNumericLiteral(value)) + markKeyed(String(Number(value.text)), site, receiver, target); // The key keeps its assertions: `slots[at as number]` and `slots[at!]` // write a number key whatever the type of `at`. - else computedWrites.push({ key, receiver, site }); + else computedWrites.push(target ? { key, receiver, target, site } : { key, receiver, site }); }; const markFields = (value: ts.Expression, site: ts.Node, receiver?: ts.Expression): void => { value = unwrap(value); @@ -275,7 +305,7 @@ export function collectClassMethodMutations( propertyWrites.push({ target, site }); } else markName(name, site, arrow); } else if (ts.isElementAccessExpression(target)) - markKey(target.argumentExpression, site, target.expression); + markKey(target.argumentExpression, site, target.expression, target); else if (ts.isArrayLiteralExpression(target)) { for (const element of target.elements.nodes) { if (ts.isSpreadElement(element)) markTarget(element.expression, site); @@ -575,7 +605,7 @@ export function collectClassMethodMutations( if (!methodOwnerCanInhabit(target.expression, name)) continue; lowerer.prototypeMethodAccesses.set(name, site); } - for (const { key, receiver, site } of computedWrites) { + for (const { key, receiver, target, site } of computedWrites) { // A class method declared with a well-known symbol key is not a write; // any other write with one can replace that method. const symbolName = key ? wellKnownMethodName(unwrap(key)) : null; @@ -592,7 +622,7 @@ export function collectClassMethodMutations( const type = key ? lowerer.typeOf(key) : undefined; const names = type && (literalValues(type) ?? (key ? literalLoopKeys(unwrap(key)) : null)); if (names) { - for (const name of names) markName(String(name), site); + for (const name of names) markKeyed(String(name), site, receiver, target); continue; } if (key && receiver && preservesExistingMethod(key, receiver, site)) continue; @@ -614,6 +644,15 @@ export function collectClassMethodMutations( unknownKey = site; } } + lowerer.checker.prefetchClassCollection( + keyedWrites.flatMap((write) => [write.target, write.receiver]), + [], + ); + for (const { name, receiver, target, site } of keyedWrites) { + if (lowerer.prototypeMethodAccesses.has(name) || functionFree(lowerer.typeOf(target))) continue; + if (!methodOwnerCanInhabit(receiver, name)) continue; + lowerer.prototypeMethodAccesses.set(name, site); + } if (numericKey) { for (const name of [...methods, ...arrowFields]) { if (String(Number(name)) === name) markName(name, numericKey); diff --git a/packages/compiler/src/frontend/lowering/containers/array-pipelines.ts b/packages/compiler/src/frontend/lowering/containers/array-pipelines.ts index 2f839a20b..ff7da5fd7 100644 --- a/packages/compiler/src/frontend/lowering/containers/array-pipelines.ts +++ b/packages/compiler/src/frontend/lowering/containers/array-pipelines.ts @@ -1019,7 +1019,9 @@ class FusedLoop { private callee(arg: IrExpr & { type: FuncType }): Callee { const known = this.callees.find((c) => c.arg === arg); if (known) return known; - const target = this.inlineTarget(arg); + // Called as a plain function: the loop is not the target's receiver. + const inlineTarget = this.inlineTarget; + const target = inlineTarget(arg); let callee: Callee; if (target && arg.kind === "closure") { const captures = new Map(); diff --git a/packages/compiler/src/frontend/lowering/containers/iterator-helpers.ts b/packages/compiler/src/frontend/lowering/containers/iterator-helpers.ts index ba5d0684d..f6580c70e 100644 --- a/packages/compiler/src/frontend/lowering/containers/iterator-helpers.ts +++ b/packages/compiler/src/frontend/lowering/containers/iterator-helpers.ts @@ -1182,10 +1182,10 @@ function bindSource( /** Where chain code allocates its locals: a lifted helper function's own * frame, or hidden locals of the function being lowered (a fused for-of). */ -interface ChainFrame { - local(base: string, type: IrType, mutable: boolean): string; +abstract class ChainFrame { + abstract local(base: string, type: IrType, mutable: boolean): string; /** `id = id + 1` over a number local. */ - bump(id: string): IrStmt; + abstract bump(id: string): IrStmt; } /** `id = id + 1` over a number local. */ @@ -1206,11 +1206,13 @@ function bumpLocal(id: string, loc: SrcLoc): IrStmt { } /** Hidden locals of the function being lowered. */ -class HostFrame implements ChainFrame { +class HostFrame extends ChainFrame { constructor( private readonly lowerer: Lowerer, private readonly loc: SrcLoc, - ) {} + ) { + super(); + } local(base: string, type: IrType, mutable: boolean): string { const local = this.lowerer.declareHiddenLocal(`%${base}`, type); local.mutable = mutable; @@ -1223,11 +1225,13 @@ class HostFrame implements ChainFrame { /** The locals and parameters of a lifted helper function under * construction. */ -export class LiftedFrame implements ChainFrame { +export class LiftedFrame extends ChainFrame { readonly locals: IrLocal[] = []; readonly params: IrParam[] = []; private n = 0; - constructor(private readonly loc: SrcLoc) {} + constructor(private readonly loc: SrcLoc) { + super(); + } local(base: string, type: IrType, mutable: boolean): string { const id = `${base}.${this.n++}`; this.locals.push({ id, name: base, type, mutable }); diff --git a/packages/compiler/src/frontend/lowering/generic-functions.ts b/packages/compiler/src/frontend/lowering/generic-functions.ts index 222cb89f9..7896da578 100644 --- a/packages/compiler/src/frontend/lowering/generic-functions.ts +++ b/packages/compiler/src/frontend/lowering/generic-functions.ts @@ -645,8 +645,7 @@ function extendsParameter( info: GenericFnInfo, ): boolean { const params = new Set(); - const fn = info.decl as { parameters?: { nodes: readonly ts.ParameterDeclaration[] } }; - for (const param of fn.parameters?.nodes ?? []) { + for (const param of info.decl.parameters.nodes) { const symbol = ts.isIdentifier(param.name) ? lowerer.checker.getSymbolAtLocation(param.name) : undefined; @@ -672,8 +671,7 @@ function declaredInside(node: ts.Node, owner: ts.Node): boolean { * return gives anything else (or nothing). Nested functions and classes keep * their own returns. */ function returnedLocalClass(lowerer: Lowerer, info: GenericFnInfo): ts.ClassLikeDeclaration | null { - const fn = info.decl as ts.Node & { body?: ts.Node }; - const body = fn.body; + const body = info.decl.body; if (!body) return null; const classOf = (node: ts.Expression): ts.ClassLikeDeclaration | null => { let expr = node; diff --git a/packages/compiler/src/frontend/lowering/instantiation-path.test.ts b/packages/compiler/src/frontend/lowering/instantiation-path.test.ts index a31617106..267b402bc 100644 --- a/packages/compiler/src/frontend/lowering/instantiation-path.test.ts +++ b/packages/compiler/src/frontend/lowering/instantiation-path.test.ts @@ -1,19 +1,23 @@ import { expect, test } from "vitest"; +import type * as ts from "../ts7/adapter.js"; import { extendInstantiationPath, MAX_INSTANTIATION_RECURSION } from "./instantiation-path.js"; +/** A stand-in declaration: paths compare declarations by identity only. */ +const node = (): ts.Node => ({}) as ts.Node; + test("independent declarations and roots do not consume a recursive budget", () => { - const declaration = {}; + const declaration = node(); for (let i = 0; i < MAX_INSTANTIATION_RECURSION * 2; i++) expect(extendInstantiationPath(null, declaration)).not.toBeNull(); let path = extendInstantiationPath(null, declaration)!; for (let i = 0; i < MAX_INSTANTIATION_RECURSION * 2; i++) - path = extendInstantiationPath(path, {})!; + path = extendInstantiationPath(path, node())!; expect(extendInstantiationPath(path, declaration)).not.toBeNull(); }); test("bounds fresh recursive demands across intervening declarations", () => { - const first = {}; - const second = {}; + const first = node(); + const second = node(); let path = extendInstantiationPath(null, first)!; for (let i = 1; i < MAX_INSTANTIATION_RECURSION; i++) { path = extendInstantiationPath(path, second)!; diff --git a/packages/compiler/src/frontend/lowering/instantiation-path.ts b/packages/compiler/src/frontend/lowering/instantiation-path.ts index d79760988..b4a8375f6 100644 --- a/packages/compiler/src/frontend/lowering/instantiation-path.ts +++ b/packages/compiler/src/frontend/lowering/instantiation-path.ts @@ -1,7 +1,9 @@ +import type * as ts from "../ts7/adapter.js"; + /** Queued bodies preserve their demand ancestry. Independent concrete uses * do not consume a recursion budget; only fresh instances on one path do. */ export interface InstantiationPath { - declaration: object; + declaration: ts.Node; parent: InstantiationPath | null; } @@ -9,7 +11,7 @@ export const MAX_INSTANTIATION_RECURSION = 100; export function extendInstantiationPath( parent: InstantiationPath | null, - declaration: object, + declaration: ts.Node, ): InstantiationPath | null { let repetitions = 0; for (let path = parent; path; path = path.parent) { diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 3c74cfbad..ae2734ad4 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -3694,7 +3694,7 @@ export class Lowerer { positionOf: (node) => this.positions.of(node), shapes: this.shapes, unions: this.unions, - classNamer: this.classNamer, + classNamer: (decl) => this.classNamer(decl), resolveTypeParam: this.typeParamResolver, resolveTypeParamTs: this.typeParamTsResolver, isErasing: () => this.erasingDepth > 0, @@ -3775,8 +3775,8 @@ export class Lowerer { }, mixinIntersectionInstance: (widened) => mixinIntersectionInstanceType(this, widened), classImplementedProtocol: (type) => this.classImplementedProtocol(type), - isStdlibFile: this.isStdlibFile, - isNpmFile: this.isNpmFile, + isStdlibFile: (sf) => this.isStdlibFile(sf), + isNpmFile: (sf) => this.isNpmFile(sf), // The declaration-typed boundary's generated types (decl-projection.ts) // describe data, like an explicitly mapped external type surface. isExternalTypeFile: (sf) => @@ -8002,16 +8002,24 @@ export class Lowerer { lower: () => IrFunction | null; } >(); - const bodyRoots = (...roots: (ts.Node | undefined | null)[]): ts.Node[] => - roots.filter((root) => root !== undefined && root !== null); - const functionRoots = (decl: ts.FunctionLikeDeclaration): ts.Node[] => - bodyRoots(...decl.parameters.nodes.map((param) => param.initializer), decl.body); - const classCtorRoots = (info: ClassInfo): ts.Node[] => - bodyRoots( - ...(info.ctor?.parameters?.nodes ?? []).map((param) => param.initializer), - info.ctor?.body, - ...info.fieldOrder.map((field) => field.initializer), - ); + // Roots are collected with plain pushes: spread arguments into a rest + // parameter pass the nodes as dynamic values. + const addRoot = (roots: ts.Node[], root: ts.Node | undefined | null): void => { + if (root !== undefined && root !== null) roots.push(root); + }; + const functionRoots = (decl: ts.FunctionLikeDeclaration): ts.Node[] => { + const roots: ts.Node[] = []; + for (const param of decl.parameters.nodes) addRoot(roots, param.initializer); + addRoot(roots, decl.body); + return roots; + }; + const classCtorRoots = (info: ClassInfo): ts.Node[] => { + const roots: ts.Node[] = []; + for (const param of info.ctor?.parameters?.nodes ?? []) addRoot(roots, param.initializer); + addRoot(roots, info.ctor?.body); + for (const field of info.fieldOrder) addRoot(roots, field.initializer); + return roots; + }; const classMemberRoots = (info: ClassInfo): ts.Node[] => { const roots = classCtorRoots(info); for (const { member } of this.classMethodMembers(info)) roots.push(...functionRoots(member)); @@ -8188,7 +8196,8 @@ export class Lowerer { register(`%${cName}.set:${prop}`, [], () => this.throwingSetterFn(info, prop)); } }; - for (const name of collectionEdges) this.onEdge(name); + const onEdge = this.onEdge; + for (const name of collectionEdges) onEdge(name); // Generic instances queued by the bodies above lower here (an instance // body fires edges of its own and can queue further instances of // either kind — function instances and class instantiations drain to @@ -8323,7 +8332,8 @@ export class Lowerer { // profile-mapped exports are called from outside the graph, so they // seed the worklist beside the init bodies. Unknown names are inert // (the export-map resolution reports them as SC4002 later). - for (const root of extraRoots ?? []) this.onEdge?.(root); + const rootEdge = this.onEdge; + for (const root of extraRoots ?? []) rootEdge?.(root); const shuffle = lowerShuffle(); const drainUnits = (): void => { while (queue.length > 0) { @@ -8441,7 +8451,9 @@ export class Lowerer { /** Reachability hook (see emitReachable): fires when lowering resolves a * reference to a lowerable body. */ noteEdge(name: string): void { - if (this.onEdge) this.onEdge(name); + // Called as a plain function: the instance is not its receiver. + const onEdge = this.onEdge; + if (onEdge) onEdge(name); } /** A subclass constructor's super() edge: the base constructor runs, but @@ -11827,12 +11839,15 @@ export class Lowerer { /** Method-like members with their collected method-map names. Body * lowering omits abstract declarations; ABI analysis must include them - * because an abstract-typed call selects the same virtual slot. */ - *classMethodMembers( + * because an abstract-typed call selects the same virtual slot. An array, + * not a generator: a generator's yielded values are dynamic values, which + * would make every syntax node one. */ + classMethodMembers( info: ClassInfo, includeAbstract = false, - ): Generator<{ mName: string; member: ts.MethodDeclaration | ts.AccessorDeclaration }> { - if (!info.decl) return; // builtin error classes: runtime-provided bodies + ): { mName: string; member: ts.MethodDeclaration | ts.AccessorDeclaration }[] { + const members: { mName: string; member: ts.MethodDeclaration | ts.AccessorDeclaration }[] = []; + if (!info.decl) return members; // builtin error classes: runtime-provided bodies for (const member of info.decl.members.nodes) { const fnLike = ts.isMethodDeclaration(member) || ts.isGetAccessor(member) || ts.isSetAccessor(member) @@ -11856,8 +11871,9 @@ export class Lowerer { : `${ts.isGetAccessor(fnLike) ? "get" : "set"}:${baseName}`; const signature = info.methods.get(mName); if (!signature || (!fnLike.body && !(includeAbstract && signature.abstract))) continue; - yield { mName, member: fnLike }; + members.push({ mName, member: fnLike }); } + return members; } lowerClassCtor(info: ClassInfo): IrFunction { diff --git a/packages/compiler/src/frontend/lowering/site-dependencies.test.ts b/packages/compiler/src/frontend/lowering/site-dependencies.test.ts index ca5affcbe..0580a52ee 100644 --- a/packages/compiler/src/frontend/lowering/site-dependencies.test.ts +++ b/packages/compiler/src/frontend/lowering/site-dependencies.test.ts @@ -1,9 +1,13 @@ import { expect, test } from "vitest"; +import type * as ts from "../ts7/adapter.js"; import { SiteDependencies } from "./site-dependencies.js"; +/** A stand-in node fact: facts are compared by identity. */ +const node = (): ts.Node => ({}) as ts.Node; + test("a changed fact makes the sites that read it stale", () => { const dependencies = new SiteDependencies(3); - const a = {}; + const a = node(); const b = "key"; dependencies.begin(0); dependencies.read(a); @@ -20,8 +24,8 @@ test("a changed fact makes the sites that read it stale", () => { test("a visit that changes a fact makes every reader of its facts stale", () => { const dependencies = new SiteDependencies(3); - const shared = {}; - const own = {}; + const shared = node(); + const own = node(); dependencies.begin(0); dependencies.read(shared); dependencies.end(false); @@ -37,7 +41,7 @@ test("a visit that changes a fact makes every reader of its facts stale", () => test("reads outside a visit are not recorded", () => { const dependencies = new SiteDependencies(1); - const fact = {}; + const fact = node(); dependencies.read(fact); dependencies.changed(fact); expect(dependencies.isStale(0)).toBe(false); @@ -45,7 +49,7 @@ test("reads outside a visit are not recorded", () => { test("a site visited again is listed once per fact", () => { const dependencies = new SiteDependencies(3); - const fact = {}; + const fact = node(); for (const site of [0, 2, 1, 2, 0]) { dependencies.begin(site); dependencies.read(fact); diff --git a/packages/compiler/src/frontend/lowering/site-dependencies.ts b/packages/compiler/src/frontend/lowering/site-dependencies.ts index da8531e48..868531353 100644 --- a/packages/compiler/src/frontend/lowering/site-dependencies.ts +++ b/packages/compiler/src/frontend/lowering/site-dependencies.ts @@ -1,3 +1,5 @@ +import type * as ts from "../ts7/adapter.js"; + /** The facts each site of a fixed-point scan read on its last visit, so that * a pass visits again only the sites whose facts changed since. * @@ -13,14 +15,19 @@ * that with a pass over every site once a pass changes nothing, and resumes * when that pass changes a fact: a fact this tracking misses costs passes, * not a different result. */ +/** A fact a site reads: a symbol, a syntax node, or the name of a derived + * fact. A typed union keeps nodes and symbols native values; as `unknown` + * every syntax node would become a dynamic value. */ +export type SiteFact = ts.Node | ts.Symbol | string; + export class SiteDependencies { /** Each fact's entry: the visit that last read it, so that a visit lists a * fact once, followed by the sites that read it, ascending. */ - private readonly readers = new Map(); + private readonly readers = new Map(); private readonly stale: Uint8Array; private site = -1; private visit = 0; - private facts: unknown[] = []; + private facts: SiteFact[] = []; constructor(siteCount: number) { this.stale = new Uint8Array(siteCount); @@ -46,7 +53,7 @@ export class SiteDependencies { } /** The site being visited read `fact`. */ - read(fact: unknown): void { + read(fact: SiteFact): void { if (this.site < 0) return; const entry = this.readers.get(fact); if (entry === undefined) { @@ -72,7 +79,7 @@ export class SiteDependencies { } /** `fact` changed: every site that read it is visited again. */ - changed(fact: unknown): void { + changed(fact: SiteFact): void { const entry = this.readers.get(fact); if (entry !== undefined) for (let i = 1; i < entry.length; i++) this.stale[entry[i]!] = 1; } diff --git a/packages/compiler/src/frontend/lowering/string-index-bounds.ts b/packages/compiler/src/frontend/lowering/string-index-bounds.ts index af1b00ec6..6418aed76 100644 --- a/packages/compiler/src/frontend/lowering/string-index-bounds.ts +++ b/packages/compiler/src/frontend/lowering/string-index-bounds.ts @@ -182,11 +182,28 @@ export class StringIndexBounds { private readonly identifiers = new Map>(); constructor( - private readonly symbolAt: (node: ts.Node) => ts.Symbol | null, - private readonly declarationOf: (symbol: ts.Symbol) => ts.Node | undefined, - private readonly isString: (node: ts.Expression) => boolean, + private readonly lookupSymbol: (node: ts.Node) => ts.Symbol | null, + private readonly lookupDeclaration: (symbol: ts.Symbol) => ts.Node | undefined, + private readonly stringTyped: (node: ts.Expression) => boolean, ) {} + // The lookups are called as plain functions: the instance is not their + // receiver. + private symbolAt(node: ts.Node): ts.Symbol | null { + const lookup = this.lookupSymbol; + return lookup(node); + } + + private declarationOf(symbol: ts.Symbol): ts.Node | undefined { + const lookup = this.lookupDeclaration; + return lookup(symbol); + } + + private isString(node: ts.Expression): boolean { + const test = this.stringTyped; + return test(node); + } + /** True when `read` (an element access on a string receiver) is proven * to name an existing code unit. */ inBounds(read: ts.ElementAccessExpression): boolean { diff --git a/packages/compiler/src/frontend/npm-static.ts b/packages/compiler/src/frontend/npm-static.ts index eed02c95c..b00e14aa0 100644 --- a/packages/compiler/src/frontend/npm-static.ts +++ b/packages/compiler/src/frontend/npm-static.ts @@ -254,13 +254,18 @@ function aliasedPackageName(pkgDir: string): string | null { export function npmStaticTransformPkgJson(pkg: Record): void { delete pkg["types"]; delete pkg["typings"]; - const transform = (v: unknown): void => { - if (v === null || typeof v !== "object") return; - if (Array.isArray(v)) { - for (const item of v) transform(item); - return; - } - const obj = v as Record; + const transform = (v: unknown): unknown => { + if (v === null || typeof v !== "object") return v; + if (Array.isArray(v)) return v.map(transform); + // The result is a new object, built in key order; the first entry of a + // key wins. + const entries: [string, unknown][] = []; + const seen = new Set(); + const add = (key: string, value: unknown): void => { + if (seen.has(key)) return; + seen.add(key); + entries.push([key, value]); + }; // 0. The "node" condition HOISTS: Node's runtime resolution matches it // (yaml's `{ types, node: "./dist/index.js", default: "./browser/ // index.js" }` runs the node dist; the browser dist is a DIFFERENT @@ -270,24 +275,18 @@ export function npmStaticTransformPkgJson(pkg: Record): void { // under import+require (either matcher finds it there), an object's // entries splice inline — earlier real keys keep winning, exactly // Node's first-match-in-key-order rule. - if (obj["node"] !== undefined) { - const entries = Object.entries(obj); - for (const k of Object.keys(obj)) delete obj[k]; - for (const [k, val] of entries) { - if (k !== "node") { - if (!(k in obj)) obj[k] = val; - continue; - } - const inner: Record = - typeof val === "string" - ? { import: val, require: val } - : val !== null && typeof val === "object" && !Array.isArray(val) - ? (val as Record) - : {}; - for (const [ik, iv] of Object.entries(inner)) { - if (!(ik in obj)) obj[ik] = iv; - } + for (const [key, value] of Object.entries(v as Record)) { + if (key !== "node") { + add(key, value); + continue; } + const inner: Record = + typeof value === "string" + ? { import: value, require: value } + : value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; + for (const [innerKey, innerValue] of Object.entries(inner)) add(innerKey, innerValue); } // 1. The types claims go AFTER the hoist: a node object's own // "types" key (uuid's `node: { types: "./dist/index.d.ts", default: @@ -295,15 +294,17 @@ export function npmStaticTransformPkgJson(pkg: Record): void { // in place tsgo follows it, maps the hidden declaration to its // same-stem JS (dist/index.js, the browser build) and disagrees with // this resolver's runtime answer. - delete obj["types"]; - delete obj["typings"]; - for (const value of Object.values(obj)) transform(value); + return Object.fromEntries( + entries + .filter(([key]) => key !== "types" && key !== "typings") + .map(([key, value]): [string, unknown] => [key, transform(value)]), + ); }; - if (pkg["exports"] !== undefined) transform(pkg["exports"]); + if (pkg["exports"] !== undefined) pkg["exports"] = transform(pkg["exports"]); // The "imports" field maps "#" specifiers with the same condition objects // (chalk 5's `"#supports-color": { node, default }`), so Node's "node" // target hoists there too. - if (pkg["imports"] !== undefined) transform(pkg["imports"]); + if (pkg["imports"] !== undefined) pkg["imports"] = transform(pkg["imports"]); } /** npmStaticTransformPkgJson over a package.json TEXT (the fs shadow's diff --git a/packages/compiler/src/frontend/services.ts b/packages/compiler/src/frontend/services.ts index 19728d6ce..14b366bfa 100644 --- a/packages/compiler/src/frontend/services.ts +++ b/packages/compiler/src/frontend/services.ts @@ -52,8 +52,10 @@ export class FrontendServices { } private sourceParser(): Ts7SourceParser { this.ensureOpen(); + // Called as a plain function: the instance is not its receiver. + const createApi = this.createApi; return (this.parser ??= new Ts7SourceParser( - (options) => this.createApi({ ...options, collectTiming: false }), + (options) => createApi({ ...options, collectTiming: false }), this.cwd, )); } @@ -62,9 +64,10 @@ export class FrontendServices { } evaluateComptime(source: string, timeoutMs: number): unknown { this.ensureOpen(); - if (this.comptimeEvaluator === undefined) + const evaluate = this.comptimeEvaluator; + if (evaluate === undefined) throw new Error("this compiler host does not provide compile-time evaluation"); - return this.comptimeEvaluator(source, timeoutMs); + return evaluate(source, timeoutMs); } createProgramHost(options?: Ts7HostOptions): Ts7Host { this.ensureOpen(); diff --git a/packages/compiler/src/frontend/ts7/semantic-model.ts b/packages/compiler/src/frontend/ts7/semantic-model.ts index d3d5fe61a..1a6560348 100644 --- a/packages/compiler/src/frontend/ts7/semantic-model.ts +++ b/packages/compiler/src/frontend/ts7/semantic-model.ts @@ -64,7 +64,9 @@ export interface SemanticQuery { export class SemanticProgram { constructor(private readonly sourceFile: (path: string) => AstNode | undefined) {} getSourceFile(path: string): AstNode | undefined { - return this.sourceFile(path); + // Called as a plain function: the instance is not its receiver. + const sourceFile = this.sourceFile; + return sourceFile(path); } } diff --git a/packages/compiler/src/frontend/ts7/source-parser.ts b/packages/compiler/src/frontend/ts7/source-parser.ts index f9029774c..bbb6cae7a 100644 --- a/packages/compiler/src/frontend/ts7/source-parser.ts +++ b/packages/compiler/src/frontend/ts7/source-parser.ts @@ -42,7 +42,9 @@ export class Ts7SourceParser { } private connect(): Ts7Api { - return this.createApi({ + // Called as a plain function: the instance is not its receiver. + const createApi = this.createApi; + return createApi({ cwd: this.cwd, fs: { readFile: (path) => this.files.get(tsgoPath(path)) ?? null, diff --git a/packages/compiler/test/keyed-method-writes.test.ts b/packages/compiler/test/keyed-method-writes.test.ts new file mode 100644 index 000000000..407530945 --- /dev/null +++ b/packages/compiler/test/keyed-method-writes.test.ts @@ -0,0 +1,79 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile, deserializeModule } from "../src/index.js"; +import type { IrModule } from "../src/ir/ir.js"; +import { everyStmtList } from "../src/ir/traverse.js"; + +async function lowerSource(source: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "scriptc-keyed-writes-")); + try { + const entry = join(dir, "main.ts"); + await writeFile(entry, source); + const outPath = join(dir, "main.ir.json"); + const result = await compile(entry, { outPath, outDir: dir, outputKind: "ir" }); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + return deserializeModule(await readFile(outPath, "utf8")); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +/** Calls in `run` that go through a dynamic value. */ +function dynamicCalls(mod: IrModule): number { + const fn = mod.functions.find((f) => f.name === "run"); + if (!fn) throw new Error("no run function"); + let n = 0; + everyStmtList(fn.body, { + expr: (e) => { + if (e.kind === "dynCall" || e.kind === "dynInvoke") n++; + return true; + }, + stmt: () => true, + }); + return n; +} + +const shape = ` +type Step = "scale" | "clamp"; +class Transform { + scale(x: number): number { + return x * 2; + } + clamp(x: number): number { + return x < 0 ? 0 : x; + } +} +`; + +test("element writes of data under method names keep the methods native", async () => { + const mod = await lowerSource(`${shape} +const enabled: Record = { scale: false, clamp: false }; +function enable(name: Step): void { + enabled[name] = true; +} +const totals: Record = {}; +function run(t: Transform): number { + enable("scale"); + totals["clamp"] = 1; + return t.clamp(t.scale(3)); +} +console.log(run(new Transform())); +`); + expect(dynamicCalls(mod)).toBe(0); +}); + +test("element writes through untyped receivers can still replace a method", async () => { + const mod = await lowerSource(`${shape} +function replace(target: object, name: Step): void { + (target as any)[name] = (x: number) => x + 100; +} +function run(t: Transform): number { + replace(t, "scale"); + return t.clamp(t.scale(3)); +} +console.log(run(new Transform())); +`); + expect(dynamicCalls(mod)).toBeGreaterThan(0); +}); From 285a92354e998dcb5a329f43f80bb158b4847211 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 13:21:29 -0500 Subject: [PATCH 084/184] Wake TransformStream pulls waiting on a static fetch body --- packages/runtime/src/scr_fetch.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index 9792f8bed..03de9eed2 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -3227,8 +3227,17 @@ static void sft_set_backpressure(SfTransform *t, bool backpressure) { } } +/* After an enqueue: whether a read is still waiting. A fetch upload's read + * was just satisfied by the chunk, and it reads again through the next + * pull, as a reader's next read() does. */ static bool sft_readable_wants_chunk(const SfStream *r) { - return sf_stream_has_consumer(r); + return r->collector || (r->reader && r->reader->pending_head); +} + +/* The readable side is closed or errored: a pull still waiting on + * bp_change can no longer receive a chunk, so complete it. */ +static void sft_finish_pull(SfTransform *t) { + if (t->bp_change && scr_promise_is_pending(t->bp_change)) scr_promise_fulfill_void(t->bp_change); } static bool sft_readable_can_enqueue(const SfStream *r) { @@ -3334,6 +3343,7 @@ static void sf_transform_step(SfTransform *t, int op) { } else { sf_stream_close(r); } + sft_finish_pull(t); if (error) sf_reject_promise_reason(t->finish, error); else scr_promise_fulfill_void(t->finish); if (t->in_flight_close) sfw_finish_close(t, error); @@ -3355,6 +3365,7 @@ static void sf_transform_step(SfTransform *t, int op) { : scr_dyn_retain(scr_dyn_undefined()); bool readable_errored = r->error != NULL; if (!readable_errored) sf_stream_error(r, reason); + sft_finish_pull(t); /* The sink abort algorithm's finish promise, which a * readable.cancel() after the abort answers too: rejected with the * readable side's stored error when it had already errored, else @@ -3395,6 +3406,7 @@ static void sf_transform_step(SfTransform *t, int op) { } else { sfw_error_if_needed(t, reason ? reason : scr_dyn_undefined()); if (t->t_backpressure) sft_set_backpressure(t, false); + sft_finish_pull(t); scr_promise_fulfill_void(t->finish); } scr_dyn_release(reason); From 26d2c8bae194b6f6c0bf84907341c2cd9c2cc1e6 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 10:33:57 -0700 Subject: [PATCH 085/184] Iterate untyped arrays without the generic iterator protocol for...of, spread and destructuring over an untyped array opened the iterator by calling the builtin values() function through scr_dyn_call with the array as `this`, and stepped it through the general element path: retain the source, compute the live length, read the element with hole handling, release the source. An array whose iterator method is the builtin values() now gets the native iterator that call returns directly. A step over a plain array whose index is an own element, or past the end, answers without the general path; no getter can run there. Holes, keys(), entries(), typed array capsules and every other source keep the general path. zod 3, z.array(Order).parse over 500 orders: 187.3M -> 182.2M instructions per round (-2.7%). Co-Authored-By: Claude Opus 5.5 --- .../7794-untyped-array-iteration.js.json | 6 ++ packages/runtime/src/scr_json.c | 23 +++++ tests/corpus/7794-untyped-array-iteration.js | 86 +++++++++++++++++++ .../7794-untyped-array-iteration.js.json | 1 + 4 files changed, 116 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7794-untyped-array-iteration.js.json create mode 100644 tests/corpus/7794-untyped-array-iteration.js create mode 100644 tests/harness/corpus-costs/7794-untyped-array-iteration.js.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7794-untyped-array-iteration.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7794-untyped-array-iteration.js.json new file mode 100644 index 000000000..5ffe60f53 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7794-untyped-array-iteration.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7794-untyped-array-iteration.js" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index eed8a9da2..a00a18d82 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -5719,6 +5719,23 @@ static void scr_native_iterator_release(void *ptr) { * next() calls still receive a fresh, independently mutable result object. */ static ScrDyn *scr_native_iterator_value(ScrNativeIterator *iterator, bool *done_out) { ScrDyn *source = iterator->source; + /* An own element of an untyped array, or its end: no getter can run, so + * the general path below is not needed. */ + if (source && source->kind == SCR_DYN_ARR && iterator->kind == 0 && !iterator->array_like && + iterator->selection == 0) { + if (iterator->index >= (double)source->v.arr.len) { + iterator->source = NULL; + scr_dyn_release(source); + *done_out = true; + return scr_dyn_retain(scr_dyn_undefined()); + } + if (scr_dyn_arr_has_index(source, (size_t)iterator->index)) { + size_t index = (size_t)iterator->index; + iterator->index++; + *done_out = false; + return scr_dyn_retain(source->v.arr.items[index]); + } + } /* An inherited indexed getter can advance this same iterator recursively. * Keep the source alive even if that nested call exhausts the iterator. */ if (source) scr_dyn_retain(source); @@ -6863,6 +6880,12 @@ ScrDyn *scr_dyn_iterator(const ScrDyn *source, const ScrStr *spell) { scr_dyn_release(method); return scr_dyn_not_iterable(source, spell); } + /* An array whose iterator method is the builtin values(): what calling + * it with the array as `this` returns, without the call. */ + if (source->kind == SCR_DYN_ARR && method == scr_iterator_functions[1]) { + scr_dyn_release(method); + return scr_native_iterator_new((ScrDyn *)source); + } ScrDyn *iterator = scr_iterator_call(method, source, "value[Symbol.iterator]"); scr_dyn_release(method); if (!iterator) return NULL; diff --git a/tests/corpus/7794-untyped-array-iteration.js b/tests/corpus/7794-untyped-array-iteration.js new file mode 100644 index 000000000..ba7fe0810 --- /dev/null +++ b/tests/corpus/7794-untyped-array-iteration.js @@ -0,0 +1,86 @@ +"use strict"; +// for...of, spread and destructuring over untyped arrays: elements are read +// live, so pushes and truncation during the loop show, holes read as +// undefined, an own Symbol.iterator replaces the builtin one, and keys() +// and entries() keep their own shapes, as in Node. + +/** @param {any} list */ +function collect(list) { + const out = []; + for (const item of list) out.push(item); + return out; +} + +/** @param {any} list */ +function growWhileIterating(list) { + const seen = []; + for (const item of list) { + seen.push(item); + if (item === 2) list.push(99); + if (seen.length > 10) break; + } + return seen; +} + +/** @param {any} list */ +function shrinkWhileIterating(list) { + const seen = []; + for (const item of list) { + seen.push(item); + list.length = 2; + } + return seen; +} + +const parsed = JSON.parse('[1, "two", {"three": 3}, [4], null, true]'); +console.log(JSON.stringify(collect(parsed))); +console.log(JSON.stringify(collect([]))); +console.log(JSON.stringify(growWhileIterating(JSON.parse("[1, 2, 3]")))); +console.log(JSON.stringify(shrinkWhileIterating(JSON.parse("[1, 2, 3, 4, 5]")))); + +/** @type {any} */ +const holes = [1, , 3]; +holes.length = 5; +console.log(collect(holes).map((x) => String(x)).join(",")); + +/** @type {any} */ +const custom = JSON.parse("[1, 2, 3]"); +custom[Symbol.iterator] = function* () { + yield "custom"; + yield this.length; +}; +console.log(JSON.stringify(collect(custom))); + +/** @type {any} */ +const data = JSON.parse('[10, 20, 30]'); +console.log(JSON.stringify([...data]), JSON.stringify([...data.keys()]), JSON.stringify([...data.entries()])); +const [first, , third = "default", fourth = "missing"] = data; +console.log(first, third, fourth); + +let total = 0; +for (const value of data) { + if (value === 20) continue; + total += value; +} +console.log(total); + +/** @param {any} list */ +function nested(list) { + const pairs = []; + for (const a of list) for (const b of list) pairs.push(`${a}${b}`); + return pairs.join(" "); +} +console.log(nested(JSON.parse('["x", "y"]'))); + +/** @param {any} list */ +function earlyReturn(list) { + for (const item of list) if (item > 1) return item; + return -1; +} +console.log(earlyReturn(JSON.parse("[0, 1, 2, 3]")), earlyReturn(JSON.parse("[0]"))); + +try { + console.log(collect(null)); +} catch (e) { + console.log(e instanceof TypeError, e.message); +} diff --git a/tests/harness/corpus-costs/7794-untyped-array-iteration.js.json b/tests/harness/corpus-costs/7794-untyped-array-iteration.js.json new file mode 100644 index 000000000..6c25b05fd --- /dev/null +++ b/tests/harness/corpus-costs/7794-untyped-array-iteration.js.json @@ -0,0 +1 @@ +{ "plain": 3100, "san": 3700 } From 0a98779dd979c1bc2a0d2a18dbda6fd4d1e8944b Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 13:49:22 -0500 Subject: [PATCH 086/184] Convert arrays and records that hold text codecs to unknown The recursive boxability check rejected TextEncoder and TextDecoder records, so an array or record holding a codec could not convert to 'unknown' (SC1101), although the converter boxes a codec as its own handle at any depth. Codec records are now boxable there, and a live view of a record or array boxes a codec field or element as its handle too, instead of a view of the codec's internal fields. --- .../src/backend/llvm/expr-stream-bridges.ts | 8 +++++++- packages/compiler/src/ir/ir.ts | 3 ++- tests/corpus/7749-text-codec-unknown-roundtrip.ts | 14 ++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/packages/compiler/src/backend/llvm/expr-stream-bridges.ts b/packages/compiler/src/backend/llvm/expr-stream-bridges.ts index a2f7208ec..70a8dcdee 100644 --- a/packages/compiler/src/backend/llvm/expr-stream-bridges.ts +++ b/packages/compiler/src/backend/llvm/expr-stream-bridges.ts @@ -722,7 +722,13 @@ export function streamTypedRefBoxValue( B.line(`${boxed} = call ptr @${host.liveDynUnionRefAdapter(t)}(ptr ${value})`); return boxed; } - if (t.kind === "bytes" || (!streamTypedRefEligible(t) && !isDynTypedRefType(t))) { + // A codec record boxes as its own handle, by reference (toDynHelper), as + // it does outside a live view. + if ( + t.kind === "bytes" || + textCodecRecordClass(t, (id) => host.recordsById.get(id)) !== null || + (!streamTypedRefEligible(t) && !isDynTypedRefType(t)) + ) { const valueTy = host.llType(t); B.line(`${boxed} = call ptr @${host.dyn.toDynHelper(t)}(${valueTy} ${value})`); return boxed; diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 29dda102d..03b323cd5 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -4412,7 +4412,8 @@ function canBoxDynCompositeAt( case "record": { const shape = getRecord(t.shapeId); if (!shape) return false; - if (recordTextCodecClass(shape) !== null) return false; + // A codec boxes as its own handle wherever it sits (toDynHelper). + if (recordTextCodecClass(shape) !== null) return true; // Recursive shapes answer coinductively, like isJsonSafeType. if (visiting.has(t.shapeId)) return true; visiting.add(t.shapeId); diff --git a/tests/corpus/7749-text-codec-unknown-roundtrip.ts b/tests/corpus/7749-text-codec-unknown-roundtrip.ts index 14c35d2bb..b7c18b3ef 100644 --- a/tests/corpus/7749-text-codec-unknown-roundtrip.ts +++ b/tests/corpus/7749-text-codec-unknown-roundtrip.ts @@ -20,3 +20,17 @@ console.log(pair[0] === pair[1], pair[0] === holder.codec); const maybe: unknown = Math.random() > 2 ? 1 : new TextDecoder(); if (typeof maybe === "object" && maybe !== null) console.log((maybe as TextDecoder).encoding); + +// Codecs inside arrays and records cross as their handles too. +const codecs: TextEncoder[] = [encoder, new TextEncoder()]; +const asUnknown: unknown = codecs; +const back = asUnknown as TextEncoder[]; +console.log(back.length, back[0] === encoder, back[1]!.encode("x")); +const streaming = new TextDecoder(); +const nested: { name: string; decoder: TextDecoder } = { name: "nested", decoder: streaming }; +const boxed: unknown = nested; +const unboxed = boxed as { name: string; decoder: TextDecoder }; +console.log(unboxed.name, unboxed.decoder === streaming); +console.log(JSON.stringify(unboxed.decoder.decode(new Uint8Array([0xe2, 0x82]), { stream: true }))); +console.log(JSON.stringify(streaming.decode(new Uint8Array([0xac])))); +console.log(JSON.stringify(boxed)); From c81ee69da78cb380b55a71e86a7633721b5221fa Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 10:34:07 -0700 Subject: [PATCH 087/184] Rule out global constructor names by their first byte A global constructor read as a value in untyped code (zod's `data instanceof Map`, `Set`, `Date` in getParsedType) resolves its name at run time: Function, then the Error constructors, then the other globals, each by strcmp. Every miss in the Error and global tables cost a call, about a dozen per read. Each table entry is now skipped unless its first byte matches the name's, before the strcmp. The tables, their order and the values they return are unchanged. zod 3, z.array(Order).parse over 500 orders: 187.3M -> 183.2M instructions per round (-2.2%). Co-Authored-By: Claude Opus 5.5 --- .../7795-untyped-global-constructors.js.json | 6 ++++ packages/runtime/src/scr_dyn_invoke.c | 2 +- packages/runtime/src/scr_json.c | 5 +++- .../7795-untyped-global-constructors.js | 29 +++++++++++++++++++ .../7795-untyped-global-constructors.js.json | 1 + 5 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7795-untyped-global-constructors.js.json create mode 100644 tests/corpus/7795-untyped-global-constructors.js create mode 100644 tests/harness/corpus-costs/7795-untyped-global-constructors.js.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7795-untyped-global-constructors.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7795-untyped-global-constructors.js.json new file mode 100644 index 000000000..48eb29e33 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7795-untyped-global-constructors.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7795-untyped-global-constructors.js" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_dyn_invoke.c b/packages/runtime/src/scr_dyn_invoke.c index 364017e05..2a797b626 100644 --- a/packages/runtime/src/scr_dyn_invoke.c +++ b/packages/runtime/src/scr_dyn_invoke.c @@ -1972,7 +1972,7 @@ static bool dyn_global_construct(const ScrDyn *callee, const ScrDyn *args, ScrDy static ScrDyn *dyn_global_ctor(const char *name) { for (size_t i = 0; i < DYN_GLOBAL_CTORS; i++) { - if (strcmp(dyn_global_ctor_names[i], name) != 0) continue; + if (dyn_global_ctor_names[i][0] != name[0] || strcmp(dyn_global_ctor_names[i], name) != 0) continue; if (!dyn_global_ctors_registered) { dyn_global_ctors_registered = true; scr_atexit(dyn_global_ctors_cleanup); diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index eed8a9da2..c63b97145 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -1911,7 +1911,10 @@ static ScrDyn *scr_error_ctor_call(ScrClosure *closure, ScrDyn *const *args, siz ScrDyn *scr_dyn_error_constructor(const char *name) { for (size_t i = 0; i < SCR_ERROR_CTORS; i++) { - if (!scr_error_ctor_names[i] || strcmp(scr_error_ctor_names[i], name) != 0) continue; + /* The first byte rules out most names without a call. */ + if (!scr_error_ctor_names[i] || scr_error_ctor_names[i][0] != name[0] || + strcmp(scr_error_ctor_names[i], name) != 0) + continue; if (!scr_error_ctors_registered) { scr_error_ctors_registered = true; scr_atexit(scr_error_ctors_cleanup); diff --git a/tests/corpus/7795-untyped-global-constructors.js b/tests/corpus/7795-untyped-global-constructors.js new file mode 100644 index 000000000..0b464b7b2 --- /dev/null +++ b/tests/corpus/7795-untyped-global-constructors.js @@ -0,0 +1,29 @@ +"use strict"; +// Global constructors read as values in untyped code: instanceof tests, +// identity comparisons and constructor names, as zod's type sniffing uses +// them, answer as in Node for every global the lookup knows. + +/** @param {unknown} data */ +function kind(data) { + if (typeof Map !== "undefined" && data instanceof Map) return "map"; + if (typeof Set !== "undefined" && data instanceof Set) return "set"; + if (typeof Date !== "undefined" && data instanceof Date) return "date"; + if (data instanceof TypeError) return "type-error"; + if (data instanceof RangeError) return "range-error"; + if (data instanceof Error) return "error"; + if (data instanceof Promise) return "promise"; + if (Array.isArray(data)) return "array"; + return typeof data; +} + +const values = [new Map(), new Set([1]), new Date(0), new TypeError("t"), new RangeError("r"), new SyntaxError("s"), new Error("e"), Promise.resolve(1), [1], { a: 1 }, "s", 1, null]; +for (const value of values) console.log(kind(value)); + +const ctors = [Map, Set, Date, Error, TypeError, RangeError, SyntaxError, ReferenceError, EvalError, URIError, Object, Promise, WeakMap, WeakSet, RegExp]; +console.log(ctors.map((ctor) => ctor.name).join(",")); +console.log(ctors.every((ctor, i) => ctors.indexOf(ctor) === i)); +const again = [Map, Set, Date, Error, TypeError]; +console.log(again.every((ctor, i) => ctor === ctors[i])); +/** @type {any} */ +const err = new TypeError("x"); +console.log(err.constructor === TypeError, err.constructor === Error, err instanceof Error); diff --git a/tests/harness/corpus-costs/7795-untyped-global-constructors.js.json b/tests/harness/corpus-costs/7795-untyped-global-constructors.js.json new file mode 100644 index 000000000..05dfbf1a6 --- /dev/null +++ b/tests/harness/corpus-costs/7795-untyped-global-constructors.js.json @@ -0,0 +1 @@ +{ "plain": 1200, "san": 1200 } From 693e33742ac334e3778872f3478d4f70aafd18b1 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 13:59:28 -0500 Subject: [PATCH 088/184] Support the raw-public and raw-private key formats EC and Ed25519 keys read and write Node 24's raw formats: - generateKeyPair(Sync) encodings: raw-public gives the EC point, uncompressed or, with type "compressed", compressed, or the Ed25519 public key; raw-private gives the EC scalar at the curve's size or the Ed25519 seed. - KeyObject.export({ format: "raw-public" | "raw-private" | "raw-seed" }), with Node's check of the point type, and Node's error for a passphrase with any format other than PEM and DER, which now also covers JWK. - createPublicKey and createPrivateKey of { key, format, asymmetricKeyType, namedCurve }, in Node's argument order, with compressed EC points, and createPublicKey of a raw private key. As in Node, raw-seed and every raw format of an RSA, RSA-PSS, DSA or DH key throw ERR_CRYPTO_INCOMPATIBLE_KEY_OPTIONS, which generateKeyPair passes to its callback. Invalid key data throws ERR_INVALID_ARG_VALUE. Raw X25519, X448, Ed448 and post-quantum keys, EC scalars of 0 or at least the curve order, and the EC point at infinity, which Node accepts, throw a coded SC2020 error. --- .../limitations/builtin-apis/crypto.mdx | 4 +- .../corpus/7449-crypto-raw-keys.mjs.json | 6 + packages/runtime/src/scr_pki.c | 252 +++++++++++++++--- tests/corpus/7449-crypto-raw-keys.mjs | 120 +++++++++ .../7449-crypto-raw-keys.mjs.json | 1 + 5 files changed, 352 insertions(+), 31 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7449-crypto-raw-keys.mjs.json create mode 100644 tests/corpus/7449-crypto-raw-keys.mjs create mode 100644 tests/harness/corpus-costs/7449-crypto-raw-keys.mjs.json diff --git a/docs/content/limitations/builtin-apis/crypto.mdx b/docs/content/limitations/builtin-apis/crypto.mdx index e4b555806..964ca4ec9 100644 --- a/docs/content/limitations/builtin-apis/crypto.mdx +++ b/docs/content/limitations/builtin-apis/crypto.mdx @@ -33,9 +33,11 @@ Node calls their callbacks with `undefined` as the error, which a callback whose error parameter is typed `Error | null`, as @types/node declares it, receives as `null`. RSA, EC, and Ed25519 private and public KeyObjects come from `generateKeyPairSync`, `generateKeyPair`, `createPrivateKey`, and `createPublicKey`, on native targets. `generateKeyPairSync` and `generateKeyPair` check their arguments in Node's order, with Node's errors, including the OpenSSL errors for RSA modulus lengths below 512 or above 2147483647 bits and for public exponents of 0, 1, or an even number, which `generateKeyPair` passes to its callback. - RSA keys generate with even modulus lengths from 1024 to 8192 bits and public exponents up to 2147483647; other lengths and exponents that Node accepts, the `raw-public`, `raw-private`, and `raw-seed` encodings, and `paramEncoding: 'explicit'` throw a coded SC2020 error. + RSA keys generate with even modulus lengths from 1024 to 8192 bits and public exponents up to 2147483647; other lengths and exponents that Node accepts, and `paramEncoding: 'explicit'`, throw a coded SC2020 error. The EC curves are prime256v1 (P-256), secp384r1 (P-384), secp521r1 (P-521), and secp256k1. Keys read and write PEM and DER as PKCS#1, SEC1, PKCS#8, and SubjectPublicKeyInfo, and JWK (Ed25519 keys use PKCS#8, SubjectPublicKeyInfo, and JWK); `createPublicKey` also takes a private key or an X.509 certificate. + EC and Ed25519 keys also read and write the `raw-public` and `raw-private` formats, including compressed EC points. + Raw X25519, X448, Ed448, and post-quantum keys, EC scalars of 0 or at least the curve order, and the EC point at infinity, which Node accepts, throw a coded SC2020 error. They support `type`, `asymmetricKeyType`, `asymmetricKeyDetails`, `export()`, `equals()`, and `instanceof KeyObject`. `crypto.sign`, `crypto.verify`, `createSign`, and `createVerify` support RSA PKCS#1 v1.5 and PSS padding and ECDSA with DER and IEEE P1363 encodings over SHA-1 and SHA-2 digests, and Ed25519 through `crypto.sign` and `crypto.verify` with a `null` or `undefined` algorithm. Ed25519 signing, verification, and private keys link when a string literal in the program names `ed25519` (in any case); without one they throw a coded SC2020 error, while Ed25519 public keys still import and export. diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7449-crypto-raw-keys.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7449-crypto-raw-keys.mjs.json new file mode 100644 index 000000000..48bf2dfcc --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7449-crypto-raw-keys.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7449-crypto-raw-keys.mjs" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_pki.c b/packages/runtime/src/scr_pki.c index a1a5e97c2..94a0000e7 100644 --- a/packages/runtime/src/scr_pki.c +++ b/packages/runtime/src/scr_pki.c @@ -5,8 +5,9 @@ * asymmetricKeyDetails, export(), equals()), crypto.sign and crypto.verify, * and the Sign and Verify objects of createSign and createVerify. Keys read * and write PEM and DER as PKCS#1, SEC1, PKCS#8 and SubjectPublicKeyInfo, - * and JWK; the EC curves are prime256v1, secp384r1, secp521r1 and - * secp256k1, and the Ed25519 arithmetic is scr_ed25519.c. Argument checks + * JWK, and the raw-public and raw-private encodings; the EC curves are + * prime256v1, secp384r1, secp521r1 and secp256k1, and the Ed25519 + * arithmetic is scr_ed25519.c. Argument checks * and errors follow Node 24's lib/internal/crypto/keys.js, keygen.js and * sig.js. Encrypted keys and other key types throw a coded SC2020 error. * All arguments are borrowed. */ @@ -878,6 +879,8 @@ static ScrBytes *sp_source_bytes(const ScrDyn *v, const ScrDyn *encoding) { return NULL; } +static SpKey *sp_key_from_raw(const ScrDyn *input, const ScrDyn *data, const ScrDyn *format, bool private_input); + static const char sp_key_input_types[] = "of type string or an instance of ArrayBuffer, Buffer, TypedArray, DataView, or URL"; @@ -929,6 +932,14 @@ static ScrDyn *sp_create_key(ScrDyn *input, bool private_key) { SpKey *k = sp_key_from_jwk(data, private_key); return k ? sp_key_box(k) : NULL; } + if (sp_is_str(format, "raw-public") || sp_is_str(format, "raw-private") || sp_is_str(format, "raw-seed")) { + SpKey *k = sp_key_from_raw(input, data, format, private_key); + if (!k || private_key || !k->private_key) return k ? sp_key_box(k) : NULL; + SpKey *pub = sp_public_of(k); + sp_key_release(k); + if (!pub) sp_unsupported_input(); + return pub ? sp_key_box(pub) : NULL; + } if (!sp_absent(format) && !sp_is_str(format, "pem") && !sp_is_str(format, "der")) { sp_property_invalid("key.format", format); return NULL; @@ -959,6 +970,183 @@ ScrDyn *scr_crypto_create_private_key(ScrDyn *key) { return sp_create_key(key, t ScrDyn *scr_crypto_create_public_key(ScrDyn *key) { return sp_create_key(key, false); } +/* ── raw keys ─────────────────────────────────────────────────────────── */ + +/* A namedCurve as Node reads it: the OpenSSL short name or the NIST name, + * matched exactly. NULL for other names. */ +static const SpCurve *sp_curve_named(const ScrDyn *name) { + for (size_t i = 0; i < sizeof sp_curves / sizeof sp_curves[0]; i++) + if (sp_is_str(name, sp_curves[i].name) || (sp_curves[i].nist && sp_is_str(name, sp_curves[i].nist))) + return &sp_curves[i]; + return NULL; +} + +static void sp_incompatible_key_type(void) { + sp_throw(SCR_ERR_ERROR, "ERR_CRYPTO_INCOMPATIBLE_KEY_OPTIONS", + "The selected key encoding is incompatible with the key type"); +} + +static void sp_invalid_key_data(void) { sp_throw(SCR_ERR_TYPE, "ERR_INVALID_ARG_VALUE", "Invalid key data"); } + +/* A key's raw encoding: an EC public point, compressed when asked, or the + * private scalar at the curve's size, or an Ed25519 public key or seed. + * raw-seed, and every encoding of an RSA key, throw + * ERR_CRYPTO_INCOMPATIBLE_KEY_OPTIONS. NULL with the error pending. */ +static ScrDyn *sp_export_raw(SpKey *k, bool private_part, bool seed, bool compressed) { + if (seed || sp_is_rsa(k)) { + sp_incompatible_key_type(); + return NULL; + } + if (sp_is_ed(k)) return sp_dyn_buffer(private_part ? k->ed_seed : k->ed_pub, 32); + mbedtls_ecp_keypair *ec = mbedtls_pk_ec(k->pk); + const mbedtls_ecp_group *grp = &ec->MBEDTLS_PRIVATE(grp); + unsigned char buf[1 + 2 * 66]; + size_t n = (grp->nbits + 7) / 8; + int rc = private_part ? mbedtls_mpi_write_binary(&ec->MBEDTLS_PRIVATE(d), buf, n) + : mbedtls_ecp_point_write_binary( + grp, &ec->MBEDTLS_PRIVATE(Q), + compressed ? MBEDTLS_ECP_PF_COMPRESSED : MBEDTLS_ECP_PF_UNCOMPRESSED, &n, buf, sizeof buf); + ScrDyn *out = rc == 0 ? sp_dyn_buffer(buf, n) : NULL; + mbedtls_platform_zeroize(buf, sizeof buf); + if (!out) sp_unsupported_input(); + return out; +} + +/* Node's named key types that this runtime does not generate or read: + * the other OKP curves and the post-quantum types. */ +static const char *const sp_other_named_types[] = { + "ed448", + "x25519", + "x448", + "ml-dsa-44", + "ml-dsa-65", + "ml-dsa-87", + "ml-kem-512", + "ml-kem-768", + "ml-kem-1024", + "slh-dsa-sha2-128f", + "slh-dsa-sha2-128s", + "slh-dsa-sha2-192f", + "slh-dsa-sha2-192s", + "slh-dsa-sha2-256f", + "slh-dsa-sha2-256s", + "slh-dsa-shake-128f", + "slh-dsa-shake-128s", + "slh-dsa-shake-192f", + "slh-dsa-shake-192s", + "slh-dsa-shake-256f", + "slh-dsa-shake-256s", +}; + +/* createPrivateKey / createPublicKey of { key, format: "raw-public" | + * "raw-private" | "raw-seed", asymmetricKeyType, namedCurve }: an EC point + * (uncompressed or compressed) or scalar, or an Ed25519 public key or + * seed. The argument checks are Node's prepareAsymmetricKey order, then + * the key type, the seed format, the curve and the data. NULL with the + * error pending. */ +static SpKey *sp_key_from_raw(const ScrDyn *input, const ScrDyn *data, const ScrDyn *format, bool private_input) { + bool raw_public = sp_is_str(format, "raw-public"); + bool seed = sp_is_str(format, "raw-seed"); + if (private_input && raw_public) { + sp_property_invalid("key.format", format); + return NULL; + } + if (data->kind != SCR_DYN_BYTES && !scr_buffer_storage_is(data)) { + scr_dyn_arg_type_fail("key.key", "an instance of ArrayBuffer, Buffer, TypedArray, or DataView", data); + return NULL; + } + const ScrDyn *type = sp_opt(input, "asymmetricKeyType"); + if (type->kind != SCR_DYN_STR) { + scr_dyn_prop_type_fail("key.asymmetricKeyType", "of type string", type); + return NULL; + } + bool ec = sp_is_str(type, "ec"); + bool ed = sp_is_str(type, "ed25519"); + const ScrDyn *curve_name = sp_opt(input, "namedCurve"); + if (ec && curve_name->kind != SCR_DYN_STR) { + scr_dyn_prop_type_fail("key.namedCurve", "of type string", curve_name); + return NULL; + } + if (sp_is_str(type, "rsa") || sp_is_str(type, "rsa-pss") || sp_is_str(type, "dsa") || sp_is_str(type, "dh")) { + sp_incompatible_key_type(); + return NULL; + } + bool other = false; + for (size_t i = 0; i < sizeof sp_other_named_types / sizeof sp_other_named_types[0]; i++) + other = other || sp_is_str(type, sp_other_named_types[i]); + if (!ec && !ed && !other) { + ScrStr *s = type->v.str; + scr_pki_throw_fmt(SCR_ERR_TYPE, "ERR_INVALID_ARG_VALUE", "Invalid asymmetricKeyType: %.*s", (int)s->len, s->data); + return NULL; + } + bool ml = !ec && !ed && type->v.str->len > 3 && memcmp(type->v.str->data, "ml-", 3) == 0; + if (seed && !ml) { + sp_incompatible_key_type(); + return NULL; + } + if (other) { + sp_not_supported("raw keys of this asymmetricKeyType"); + return NULL; + } + const SpCurve *curve = NULL; + if (ec && !(curve = sp_curve_named(curve_name))) { + sp_throw(SCR_ERR_TYPE, "ERR_CRYPTO_INVALID_CURVE", "Invalid EC curve name"); + return NULL; + } + ScrBytes *bytes = scr_bytes_buffer_source(data); + if (!bytes) return NULL; + const unsigned char *p = bytes->data; + size_t len = (size_t)scr_bytes_byte_len(bytes); + SpKey *k = sp_key_alloc(!raw_public); + bool ok = false; + if (ed) { + k->ed25519 = true; + if (len == 32 && raw_public) { + memcpy(k->ed_pub, p, 32); + ok = true; + } else if (len == 32) { + const ScrPkiEd25519 *eds = sp_ed(); + if (!eds) { + sp_key_release(k); + scr_bytes_release(bytes); + return NULL; + } + memcpy(k->ed_seed, p, 32); + eds->public_key(k->ed_pub, k->ed_seed); + ok = true; + } + } else if (raw_public && len == 1 && p[0] == 0) { + sp_key_release(k); + scr_bytes_release(bytes); + sp_not_supported("EC public keys at the point at infinity"); + return NULL; + } else if (mbedtls_pk_setup(&k->pk, mbedtls_pk_info_from_type(MBEDTLS_PK_ECKEY)) == 0) { + mbedtls_ecp_keypair *kp = mbedtls_pk_ec(k->pk); + mbedtls_ecp_group *grp = &kp->MBEDTLS_PRIVATE(grp); + ok = mbedtls_ecp_group_load(grp, curve->id) == 0; + if (ok && raw_public) { + ok = mbedtls_ecp_point_read_binary(grp, &kp->MBEDTLS_PRIVATE(Q), p, len) == 0 && + mbedtls_ecp_check_pubkey(grp, &kp->MBEDTLS_PRIVATE(Q)) == 0; + } else if (ok) { + ok = len == (grp->nbits + 7) / 8 && mbedtls_mpi_read_binary(&kp->MBEDTLS_PRIVATE(d), p, len) == 0; + if (ok && mbedtls_ecp_check_privkey(grp, &kp->MBEDTLS_PRIVATE(d)) != 0) { + sp_key_release(k); + scr_bytes_release(bytes); + sp_not_supported("EC private keys outside 1 to n - 1"); + return NULL; + } + ok = ok && mbedtls_ecp_mul(grp, &kp->MBEDTLS_PRIVATE(Q), &kp->MBEDTLS_PRIVATE(d), &grp->G, sp_random, NULL) == 0; + } + } + scr_bytes_release(bytes); + if (!ok) { + sp_key_release(k); + sp_invalid_key_data(); + return NULL; + } + return k; +} + /* ── export ───────────────────────────────────────────────────────────── */ /* export({ format, type }) per Node's parseKeyEncoding: format "pem", @@ -975,11 +1163,37 @@ static ScrDyn *sp_export(SpKey *k, const ScrDyn *options) { } static ScrDyn *sp_export_plain(SpKey *k, const ScrDyn *options) { + const ScrDyn *format = sp_opt(options, "format"); + /* A private key's passphrase applies only to PEM and DER. */ + if (k->private_key && !sp_absent(sp_opt(options, "passphrase")) && !sp_is_str(format, "pem") && + !sp_is_str(format, "der")) { + ScrStr *text = scr_dyn_string_coerce_js(format); + if (!text) return NULL; + scr_pki_throw_fmt(SCR_ERR_ERROR, "ERR_CRYPTO_INCOMPATIBLE_KEY_OPTIONS", + "The selected key encoding %.*s does not support encryption.", (int)text->len, text->data); + scr_str_release(text); + return NULL; + } + if (!k->private_key && sp_is_str(format, "raw-public")) { + const ScrDyn *type = sp_opt(options, "type"); + bool compressed = sp_is_str(type, "compressed"); + /* Only an EC point has a compressed form. */ + if (sp_is_ec(k) && !sp_absent(type) && !compressed && !sp_is_str(type, "uncompressed")) { + ScrStr *received = sp_received(type, false); + scr_pki_throw_fmt(SCR_ERR_TYPE, "ERR_INVALID_ARG_VALUE", + "The property 'options.type' must be one of: 'compressed', 'uncompressed'. Received %.*s", + (int)received->len, received->data); + scr_str_release(received); + return NULL; + } + return sp_export_raw(k, false, false, sp_is_ec(k) && compressed); + } + if (k->private_key && (sp_is_str(format, "raw-private") || sp_is_str(format, "raw-seed"))) + return sp_export_raw(k, true, sp_is_str(format, "raw-seed"), false); if (options == NULL || !sp_is_object(options)) { scr_dyn_arg_type_fail("options", "of type object", options ? options : scr_dyn_undefined()); return NULL; } - const ScrDyn *format = sp_opt(options, "format"); if (sp_is_str(format, "jwk")) return sp_export_jwk(k); bool pem = sp_is_str(format, "pem"); if (!pem && !sp_is_str(format, "der")) { @@ -1227,7 +1441,7 @@ static bool sp_check_encoding_plain(const ScrDyn *enc, const char *name, const S /* Node's parseKeyEncoding for options.publicKeyEncoding or * options.privateKeyEncoding of a `type` key: true when the encoding is - * undefined, null or valid, false with the error pending. A valid raw or + * undefined, null or valid, false with the error pending. A valid * encrypted encoding, which this runtime does not write, sets *refusal * when it is unset. */ static bool sp_check_encoding(const ScrDyn *options, const char *key, const ScrDyn *type, bool private_key, @@ -1323,7 +1537,6 @@ static bool sp_check_encoding_plain(const ScrDyn *enc, const char *name, const S return false; } } - if (!*refusal && (raw_public || raw_private)) *refusal = "generateKeyPair with a raw key encoding"; if (!*refusal && encrypted) *refusal = "encrypted private key export"; return true; } @@ -1335,31 +1548,6 @@ typedef struct { const SpCurve *curve; /* EC */ } SpKeyGen; -/* The key types Node generates that this runtime does not, apart from - * rsa-pss, dsa and dh: Node's named key pairs. */ -static const char *const sp_other_named_types[] = { - "ed448", - "x25519", - "x448", - "ml-dsa-44", - "ml-dsa-65", - "ml-dsa-87", - "ml-kem-512", - "ml-kem-768", - "ml-kem-1024", - "slh-dsa-sha2-128f", - "slh-dsa-sha2-128s", - "slh-dsa-sha2-192f", - "slh-dsa-sha2-192s", - "slh-dsa-sha2-256f", - "slh-dsa-sha2-256s", - "slh-dsa-shake-128f", - "slh-dsa-shake-128s", - "slh-dsa-shake-192f", - "slh-dsa-shake-192s", - "slh-dsa-shake-256f", - "slh-dsa-shake-256s", -}; /* createJob's checks for generateKeyPair(Sync)(type, options) in Node's * order: the type, the options object, the public and private key @@ -1502,14 +1690,18 @@ static ScrDyn *sp_key_pair(const ScrDyn *type, const ScrDyn *options, bool *job_ sp_unsupported_input(); return NULL; } + /* An encoding the key type has no form for (a raw encoding of an RSA + * key, raw-seed) fails the job, as in Node. */ ScrDyn *public_value = sp_encoded(pub, sp_opt(options, "publicKeyEncoding")); if (!public_value) { sp_key_release(k); + *job_error = true; return NULL; } ScrDyn *private_value = sp_encoded(k, sp_opt(options, "privateKeyEncoding")); if (!private_value) { scr_dyn_release(public_value); + *job_error = true; return NULL; } ScrDyn *pair = scr_dyn_new_obj(); diff --git a/tests/corpus/7449-crypto-raw-keys.mjs b/tests/corpus/7449-crypto-raw-keys.mjs new file mode 100644 index 000000000..0795dbb0b --- /dev/null +++ b/tests/corpus/7449-crypto-raw-keys.mjs @@ -0,0 +1,120 @@ +// The raw-public, raw-private and raw-seed key formats: generateKeyPair +// encodings, KeyObject.export and createPublicKey/createPrivateKey with +// asymmetricKeyType and namedCurve, for EC keys (uncompressed and +// compressed points, scalars at the curve's size) and Ed25519 keys, and +// Node's errors for the other key types and formats. +import * as crypto from "node:crypto"; + +function describe(value) { + if (Buffer.isBuffer(value)) return `Buffer(${value.length})`; + if (value instanceof crypto.KeyObject) return `${value.type} ${value.asymmetricKeyType} ${JSON.stringify(value.asymmetricKeyDetails)}`; + if (value && typeof value === "object") return Object.entries(value).map(([key, field]) => `${key}=${describe(field)}`).join(" "); + return String(value); +} + +function attempt(label, run) { + try { + console.log(label, describe(run())); + } catch (e) { + console.log(label, "throws", e.name, e.code, e.message); + } +} + +// generateKeyPair encodings +for (const namedCurve of ["P-256", "P-384", "P-521", "secp256k1"]) { + attempt(`ec ${namedCurve} raw-public`, () => crypto.generateKeyPairSync("ec", { namedCurve, publicKeyEncoding: { format: "raw-public" } })); + attempt(`ec ${namedCurve} compressed`, () => crypto.generateKeyPairSync("ec", { namedCurve, publicKeyEncoding: { format: "raw-public", type: "compressed" } })); + attempt(`ec ${namedCurve} raw-private`, () => crypto.generateKeyPairSync("ec", { namedCurve, privateKeyEncoding: { format: "raw-private" } })); +} +attempt("ec raw-seed", () => crypto.generateKeyPairSync("ec", { namedCurve: "P-256", privateKeyEncoding: { format: "raw-seed" } })); +attempt("ed25519 raw", () => + crypto.generateKeyPairSync("ed25519", { publicKeyEncoding: { format: "raw-public" }, privateKeyEncoding: { format: "raw-private" } }), +); +attempt("ed25519 raw-seed", () => crypto.generateKeyPairSync("ed25519", { privateKeyEncoding: { format: "raw-seed" } })); +attempt("rsa raw-public", () => crypto.generateKeyPairSync("rsa", { modulusLength: 1024, publicKeyEncoding: { format: "raw-public" } })); + +// export +const ec = crypto.generateKeyPairSync("ec", { namedCurve: "P-256" }); +const ed = crypto.generateKeyPairSync("ed25519"); +const rsa = crypto.generateKeyPairSync("rsa", { modulusLength: 1024 }); +const point = Buffer.from(ec.publicKey.export({ format: "raw-public" })); +const compressed = Buffer.from(ec.publicKey.export({ format: "raw-public", type: "compressed" })); +const scalar = Buffer.from(ec.privateKey.export({ format: "raw-private" })); +const jwk = ec.privateKey.export({ format: "jwk" }); +console.log("ec point", point.length, point[0], point.subarray(1, 33).toString("base64url") === jwk.x, point.subarray(33).toString("base64url") === jwk.y); +console.log("ec compressed", compressed.length, compressed[0] === 2 + (point[64] & 1), compressed.subarray(1).equals(point.subarray(1, 33))); +console.log("ec scalar", scalar.length, scalar.toString("base64url") === jwk.d); +console.log("ec uncompressed", Buffer.from(ec.publicKey.export({ format: "raw-public", type: "uncompressed" })).equals(point)); +const edPublic = Buffer.from(ed.publicKey.export({ format: "raw-public" })); +const edSeed = Buffer.from(ed.privateKey.export({ format: "raw-private" })); +const edJwk = ed.privateKey.export({ format: "jwk" }); +console.log("ed25519", edPublic.toString("base64url") === edJwk.x, edSeed.toString("base64url") === edJwk.d); +attempt("ed25519 raw-public with a type", () => ed.publicKey.export({ format: "raw-public", type: "x" })); +attempt("ec bad point type", () => ec.publicKey.export({ format: "raw-public", type: "x" })); +attempt("ec null point type", () => ec.publicKey.export({ format: "raw-public", type: null })); +attempt("private raw-public", () => ec.privateKey.export({ format: "raw-public" })); +attempt("public raw-private", () => ec.publicKey.export({ format: "raw-private" })); +attempt("ec raw-seed", () => ec.privateKey.export({ format: "raw-seed" })); +attempt("ed25519 raw-seed", () => ed.privateKey.export({ format: "raw-seed" })); +attempt("rsa raw-public", () => rsa.publicKey.export({ format: "raw-public" })); +attempt("rsa raw-private", () => rsa.privateKey.export({ format: "raw-private" })); +for (const format of [undefined, 5, "jwk", "raw-private", "raw-seed"]) + attempt(`passphrase with ${format}`, () => ec.privateKey.export({ format, passphrase: "x" })); + +// import +const pub = (key) => () => crypto.createPublicKey(key); +const priv = (key) => () => crypto.createPrivateKey(key); +attempt("ec point", pub({ key: point, format: "raw-public", asymmetricKeyType: "ec", namedCurve: "P-256" })); +attempt("ec compressed point", pub({ key: compressed, format: "raw-public", asymmetricKeyType: "ec", namedCurve: "prime256v1" })); +attempt("ec scalar", priv({ key: scalar, format: "raw-private", asymmetricKeyType: "ec", namedCurve: "P-256" })); +attempt("ec scalar as public", pub({ key: scalar, format: "raw-private", asymmetricKeyType: "ec", namedCurve: "P-256" })); +const reread = crypto.createPublicKey({ key: compressed, format: "raw-public", asymmetricKeyType: "ec", namedCurve: "P-256" }); +console.log("ec reread", reread.equals(ec.publicKey), Buffer.from(reread.export({ format: "raw-public" })).equals(point)); +const rereadPrivate = crypto.createPrivateKey({ key: scalar, format: "raw-private", asymmetricKeyType: "ec", namedCurve: "P-256" }); +console.log("ec private reread", crypto.createPublicKey(rereadPrivate).equals(ec.publicKey)); +attempt("ed25519 public", pub({ key: edPublic, format: "raw-public", asymmetricKeyType: "ed25519" })); +attempt("ed25519 seed", priv({ key: edSeed, format: "raw-private", asymmetricKeyType: "ed25519" })); +console.log("ed25519 reread", crypto.createPublicKey({ key: edSeed, format: "raw-private", asymmetricKeyType: "ed25519" }).equals(ed.publicKey)); + +// fixed keys: the EC scalar 1 is the generator, and an Ed25519 seed of +// zeros has a known public key +const one = Buffer.concat([Buffer.alloc(31), Buffer.from([1])]); +const generator = crypto.createPrivateKey({ key: one, format: "raw-private", asymmetricKeyType: "ec", namedCurve: "P-256" }); +console.log("generator", Buffer.from(crypto.createPublicKey(generator).export({ format: "raw-public", type: "compressed" })).toString("hex")); +const zeros = crypto.createPrivateKey({ key: Buffer.alloc(32), format: "raw-private", asymmetricKeyType: "ed25519" }); +console.log("zero seed", Buffer.from(crypto.createPublicKey(zeros).export({ format: "raw-public" })).toString("hex")); +const message = Buffer.from("raw keys"); +console.log("zero seed signature", crypto.sign(null, message, zeros).toString("hex").slice(0, 32)); + +// import errors +attempt("raw-public to createPrivateKey", priv({ key: point, format: "raw-public", asymmetricKeyType: "ec", namedCurve: "P-256" })); +attempt("string data", pub({ key: "abc", format: "raw-public", asymmetricKeyType: "ec", namedCurve: "P-256" })); +attempt("no asymmetricKeyType", pub({ key: point, format: "raw-public" })); +attempt("number asymmetricKeyType", pub({ key: point, format: "raw-public", asymmetricKeyType: 5 })); +attempt("no namedCurve", pub({ key: point, format: "raw-public", asymmetricKeyType: "ec" })); +attempt("unknown asymmetricKeyType", pub({ key: point, format: "raw-public", asymmetricKeyType: "nope" })); +attempt("Ed25519 spelling", pub({ key: edPublic, format: "raw-public", asymmetricKeyType: "Ed25519" })); +attempt("unknown curve", pub({ key: point, format: "raw-public", asymmetricKeyType: "ec", namedCurve: "nope" })); +attempt("lower-case NIST curve", pub({ key: point, format: "raw-public", asymmetricKeyType: "ec", namedCurve: "p-256" })); +attempt("other curve", pub({ key: point, format: "raw-public", asymmetricKeyType: "ec", namedCurve: "P-384" })); +attempt("short point", pub({ key: point.subarray(0, 64), format: "raw-public", asymmetricKeyType: "ec", namedCurve: "P-256" })); +attempt("off-curve point", pub({ key: Buffer.concat([Buffer.from([4]), Buffer.alloc(64, 1)]), format: "raw-public", asymmetricKeyType: "ec", namedCurve: "P-256" })); +attempt("bad compressed x", pub({ key: Buffer.concat([Buffer.from([2]), Buffer.alloc(32, 0xff)]), format: "raw-public", asymmetricKeyType: "ec", namedCurve: "P-256" })); +attempt("short scalar", priv({ key: scalar.subarray(0, 31), format: "raw-private", asymmetricKeyType: "ec", namedCurve: "P-256" })); +attempt("long scalar", priv({ key: Buffer.concat([Buffer.alloc(1), scalar]), format: "raw-private", asymmetricKeyType: "ec", namedCurve: "P-256" })); +attempt("ec raw-seed", priv({ key: scalar, format: "raw-seed", asymmetricKeyType: "ec", namedCurve: "nope" })); +attempt("ed25519 raw-seed", priv({ key: edSeed, format: "raw-seed", asymmetricKeyType: "ed25519" })); +attempt("short ed25519 key", pub({ key: edPublic.subarray(0, 31), format: "raw-public", asymmetricKeyType: "ed25519" })); +attempt("short ed25519 seed", priv({ key: edSeed.subarray(0, 31), format: "raw-private", asymmetricKeyType: "ed25519" })); +for (const asymmetricKeyType of ["rsa", "rsa-pss", "dsa", "dh"]) + attempt(`${asymmetricKeyType} raw`, pub({ key: edPublic, format: "raw-public", asymmetricKeyType })); + +// generateKeyPair passes an encoding the key type has no form for to its +// callback +crypto.generateKeyPair("ed25519", { privateKeyEncoding: { format: "raw-seed" } }, (...args) => { + console.log("async raw-seed", args.length, args[0].code); + crypto.generateKeyPair("ec", { namedCurve: "P-384", publicKeyEncoding: { format: "raw-public", type: "compressed" } }, (err, publicKey, privateKey) => { + console.log("async raw-public", err, describe(publicKey), describe(privateKey)); + }); +}); +console.log("returned"); diff --git a/tests/harness/corpus-costs/7449-crypto-raw-keys.mjs.json b/tests/harness/corpus-costs/7449-crypto-raw-keys.mjs.json new file mode 100644 index 000000000..c8b39ec8a --- /dev/null +++ b/tests/harness/corpus-costs/7449-crypto-raw-keys.mjs.json @@ -0,0 +1 @@ +{ "plain": 7592, "san": 12217 } From 037b0b1dcc42435aa1c56a96efebe22244dc1da7 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 14:02:28 -0500 Subject: [PATCH 089/184] Keep libnode pack manifest paths inside the bundle --- .../frontend/lowering/lower-node-island.ts | 7 +++ .../compiler/src/frontend/lowering/lowerer.ts | 8 +++ .../compiler/src/node-islands/libnode.test.ts | 43 ++++++++++++++- packages/compiler/src/node-islands/libnode.ts | 35 +++++++++---- .../compiler/src/node-islands/modules.test.ts | 27 ++++++++++ packages/compiler/src/node-islands/modules.ts | 52 +++++++++++++++---- .../corpus/7662-node-call-exit-events/main.ts | 1 + 7 files changed, 152 insertions(+), 21 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/lower-node-island.ts b/packages/compiler/src/frontend/lowering/lower-node-island.ts index bab5f15a1..fae8aa219 100644 --- a/packages/compiler/src/frontend/lowering/lower-node-island.ts +++ b/packages/compiler/src/frontend/lowering/lower-node-island.ts @@ -17,6 +17,7 @@ * thrown Error crosses as its nearest intrinsic class with its message and * string `code`. */ import { realpathSync } from "node:fs"; +import { isAbsolute } from "node:path"; import * as ts from "../ts7/adapter.js"; import { locOf } from "../program.js"; import type { NodeApiMember } from "../node-api-import.js"; @@ -108,6 +109,12 @@ export function lowerNodeApiCall( throw new PoisonError(); } const specifier = lowerer.lowerExprExpecting(specifierNode, STRING); + // An absolute specifier names a file of the build machine: the call names + // the shipped copy, relative to the caller inside the tree. + if (isAbsolute(specifierNode.text) && specifier.kind === "strLit") { + const target = lowerer.nodeIslandModules.resolvedPath(caller, specifierNode.text); + if (target !== null) lowerer.nodeIslandSpecifiers.push({ literal: specifier, caller, target }); + } const name = lowerer.lowerExprExpecting(argNodes[1]!, STRING); const values = argNodes.slice(2).map((arg) => { const value = lowerer.lowerExpr(arg); diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 4b6b0acdf..6594d8175 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -3145,6 +3145,12 @@ export class Lowerer { readonly nodeIslandModules = new NodeIslandModuleCollector(); /** Each call's caller literal, set to its tree path when lowering ends. */ readonly nodeIslandCallers: { literal: IrExpr & { kind: "strLit" }; caller: string }[] = []; + /** Absolute specifiers, rewritten to the shipped file relative to their caller. */ + readonly nodeIslandSpecifiers: { + literal: IrExpr & { kind: "strLit" }; + caller: string; + target: string; + }[] = []; /** Per-directory answers of experimental-import.ts's declaration-file test. */ readonly experimentalFiles = new Map(); /** Interned LifetimeRegion.run helpers, by callback type key. */ @@ -8656,6 +8662,8 @@ export class Lowerer { const modules = this.nodeIslandModules.tree(); for (const { literal, caller } of this.nodeIslandCallers) literal.value = NodeIslandModuleCollector.callerPath(modules, caller); + for (const { literal, caller, target } of this.nodeIslandSpecifiers) + literal.value = NodeIslandModuleCollector.relativeSpecifier(caller, target); return { libnode: "", nodeVersion: "", ...(modules === null ? {} : { modules }) }; } diff --git a/packages/compiler/src/node-islands/libnode.test.ts b/packages/compiler/src/node-islands/libnode.test.ts index cd310e0e1..94add4f1c 100644 --- a/packages/compiler/src/node-islands/libnode.test.ts +++ b/packages/compiler/src/node-islands/libnode.test.ts @@ -1,4 +1,12 @@ -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, test } from "vitest"; @@ -94,6 +102,39 @@ describe("the recorded libnode pack", () => { expect(existsSync(join(out, "app.libnode"))).toBe(false); }); + test("beside refuses manifest file names outside the pack", () => { + for (const name of ["..", ".", "../escaped", "/etc/passwd", "a/b", "a\\b", "x\u0000y"]) { + const dir = pack("24.22.0"); + writeFileSync( + join(dir, "manifest.json"), + JSON.stringify({ + nodeVersion: "24.22.0", + bridgeAbi: NODE_BRIDGE_ABI, + files: { [name]: {} }, + }), + ); + const out = mkdtempSync(join(tmpdir(), "scriptc-libnode-out-")); + expect(libnodeRecord("beside", join(out, "app"), { SCRIPTC_LIBNODE: dir })).toBe( + `the libnode pack at ${dir} lists a file outside the pack ('${name}')`, + ); + expect(existsSync(join(out, "app.libnode"))).toBe(false); + expect(existsSync(join(out, "escaped"))).toBe(false); + } + }); + + test("beside refuses pack files that are symbolic links", () => { + const dir = pack("24.22.0"); + const elsewhere = join(mkdtempSync(join(tmpdir(), "scriptc-libnode-elsewhere-")), "secret"); + writeFileSync(elsewhere, "secret"); + rmSync(join(dir, "libnode.so.137")); + symlinkSync(elsewhere, join(dir, "libnode.so.137")); + const out = mkdtempSync(join(tmpdir(), "scriptc-libnode-out-")); + expect(libnodeRecord("beside", join(out, "app"), { SCRIPTC_LIBNODE: dir })).toBe( + `the libnode pack at ${dir} holds libnode.so.137 as a symbolic link; pack files must be regular files`, + ); + expect(existsSync(join(out, "app.libnode"))).toBe(false); + }); + test("a pack the deployment places beside the executable needs no manifest at build time", () => { expect(libnodeRecord("$ORIGIN/node", "/out/app", {})).toEqual({ libnode: "$ORIGIN/node", diff --git a/packages/compiler/src/node-islands/libnode.ts b/packages/compiler/src/node-islands/libnode.ts index a0ae29d2f..dc6498924 100644 --- a/packages/compiler/src/node-islands/libnode.ts +++ b/packages/compiler/src/node-islands/libnode.ts @@ -14,13 +14,14 @@ * also the hook for bundling Node into the executable itself later. */ import { createHash } from "node:crypto"; import { + constants, copyFileSync, linkSync, + lstatSync, mkdirSync, readFileSync, realpathSync, rmSync, - statSync, } from "node:fs"; import { basename, isAbsolute, join, resolve, sep } from "node:path"; @@ -76,12 +77,15 @@ function readManifest(dir: string): PackManifest | string { } /** Copies the pack next to the executable (hard links where possible), or - * returns why it cannot: the target would replace the pack itself, or the - * manifest names a file outside the pack. */ + * returns why it cannot: the target would replace the pack itself, the + * manifest names a file outside the pack, or a pack file is not a regular + * file (a symbolic link could point anywhere). Every destination is checked + * to resolve inside the target directory. */ function placeBeside(source: string, manifest: PackManifest, target: string): string | undefined { const names = ["manifest.json", "LICENSE", ...Object.keys(manifest.files)]; const unsafe = names.find( - (name) => name === "" || name === "." || name === ".." || /[/\\]/.test(name), + (name) => + name === "" || name === "." || name === ".." || /[/\\\0]/.test(name) || isAbsolute(name), ); if (unsafe !== undefined) return `the libnode pack at ${source} lists a file outside the pack ('${unsafe}')`; @@ -89,19 +93,30 @@ function placeBeside(source: string, manifest: PackManifest, target: string): st const to = realpathOr(target); if (from === to || from.startsWith(to + sep) || to.startsWith(from + sep)) return `--libnode=beside would replace the libnode pack it copies (${source}); point SCRIPTC_LIBNODE at a pack outside ${target}`; - rmSync(target, { recursive: true, force: true }); - mkdirSync(target, { recursive: true }); + const present: string[] = []; for (const name of names) { - const file = join(source, name); + let info; try { - statSync(file); + info = lstatSync(join(source, name)); } catch { continue; } + if (!info.isFile()) + return `the libnode pack at ${source} holds ${name} as a ${info.isSymbolicLink() ? "symbolic link" : "non-regular file"}; pack files must be regular files`; + present.push(name); + } + rmSync(target, { recursive: true, force: true }); + mkdirSync(target, { recursive: true }); + const inside = resolve(target) + sep; + for (const name of present) { + const destination = resolve(target, name); + if (!destination.startsWith(inside)) + return `the libnode pack at ${source} lists a file outside the pack ('${name}')`; + const file = join(source, name); try { - linkSync(file, join(target, name)); + linkSync(file, destination); } catch { - copyFileSync(file, join(target, name)); + copyFileSync(file, destination, constants.COPYFILE_EXCL); } } return undefined; diff --git a/packages/compiler/src/node-islands/modules.test.ts b/packages/compiler/src/node-islands/modules.test.ts index 6badf376a..63201e1e0 100644 --- a/packages/compiler/src/node-islands/modules.test.ts +++ b/packages/compiler/src/node-islands/modules.test.ts @@ -167,6 +167,33 @@ describe("the collected module tree", () => { ]); }); + test("a require through a linked directory keeps the link in the tree", () => { + const root = project({ + "src/main.ts": "", + "src/entry.cjs": "module.exports = require('./lib/x.cjs');", + "shared/x.cjs": "exports.x = 1;", + }); + symlinkSync("../shared", join(root, "src/lib")); + const collector = new NodeIslandModuleCollector(); + expect(collector.add(join(root, "src/main.ts"), "./entry.cjs")).toBeNull(); + const tree = collector.tree()!; + expect(tree.links).toEqual([{ path: "src/lib", target: "../shared" }]); + expect(tree.files.map((f) => f.path)).toEqual(["shared/x.cjs", "src/entry.cjs"]); + }); + + test("an absolute specifier names the shipped file relative to its caller", () => { + const root = project({ "src/main.ts": "", "lib/m.cjs": "" }); + const collector = new NodeIslandModuleCollector(); + const target = collector.resolvedPath(join(root, "src/main.ts"), join(root, "lib/m.cjs"))!; + expect(target).toBe(join(root, "lib/m.cjs")); + expect(NodeIslandModuleCollector.relativeSpecifier(join(root, "src/main.ts"), target)).toBe( + "../lib/m.cjs", + ); + expect( + NodeIslandModuleCollector.relativeSpecifier(join(root, "main.ts"), join(root, "m.cjs")), + ).toBe("./m.cjs"); + }); + test("built-in modules alone ship no tree", () => { const root = project({ "main.ts": "" }); const collector = new NodeIslandModuleCollector(); diff --git a/packages/compiler/src/node-islands/modules.ts b/packages/compiler/src/node-islands/modules.ts index 0d63dc333..e1551c070 100644 --- a/packages/compiler/src/node-islands/modules.ts +++ b/packages/compiler/src/node-islands/modules.ts @@ -211,14 +211,34 @@ export class NodeIslandModuleCollector { return null; } - /** The real path of `path`, recording a link when `path` itself (not an - * ancestor) is a symbolic link. */ + /** The real path of `path`, recording a link for each component of it + * that is a symbolic link (a linked directory on the way, or the file + * itself), so that the path resolves inside the tree as it does here. */ private placed(path: string): string { - const target = trackedRealpath(path) ?? path; - const parent = trackedRealpath(dirname(path)) ?? dirname(path); - const link = join(parent, basename(path)); - if (link !== target) this.links.set(link, target); - return target; + let real: string = sep; + for (const part of resolve(path).split(sep)) { + if (part === "") continue; + const lexical = join(real, part); + const target = trackedRealpath(lexical) ?? lexical; + if (target !== lexical) this.links.set(lexical, target); + real = target; + } + return real; + } + + /** The real path of the file an absolute or relative `specifier` names + * from `fromFile`, or null when there is none. */ + resolvedPath(fromFile: string, specifier: string): string | null { + if (!isPathSpecifier(specifier)) return null; + const target = resolveFile(resolve(dirname(fromFile), specifier)); + return target === null ? null : (trackedRealpath(target) ?? target); + } + + /** A specifier that names `target` from `caller` inside the tree: the + * relative path between them, with `/` separators. */ + static relativeSpecifier(caller: string, target: string): string { + const path = portable(relative(dirname(caller), target)); + return path.startsWith("../") ? path : `./${path}`; } /** A module of the program: collected with the package.json that scopes @@ -609,17 +629,29 @@ export function placeNodeIslandModules( ): string | null { const data = readNodeIslandModules(modules); if (typeof data === "string") return data; + // Every path, and every link's target, stays inside the tree. + const inside = resolve(target) + sep; + const placed = (relativePath: string): string | null => { + const path = resolve(target, ...relativePath.split("/")); + return path.startsWith(inside) ? path : null; + }; + for (const entry of [...modules.files, ...modules.links]) { + const path = placed(entry.path); + if (path === null) return `the island module tree names a path outside it ('${entry.path}')`; + if ("target" in entry && !resolve(dirname(path), entry.target).startsWith(inside)) + return `the island module tree links outside it ('${entry.path}' to '${entry.target}')`; + } rmSync(target, { recursive: true, force: true }); mkdirSync(target, { recursive: true }); let offset = 0; for (const file of modules.files) { - const path = join(target, ...file.path.split("/")); + const path = placed(file.path)!; mkdirSync(dirname(path), { recursive: true }); - writeFileSync(path, data.subarray(offset, offset + file.size)); + writeFileSync(path, data.subarray(offset, offset + file.size), { flag: "wx" }); offset += file.size; } for (const link of modules.links) { - const path = join(target, ...link.path.split("/")); + const path = placed(link.path)!; mkdirSync(dirname(path), { recursive: true }); symlinkSync(link.target, path); } diff --git a/tests/corpus/7662-node-call-exit-events/main.ts b/tests/corpus/7662-node-call-exit-events/main.ts index 909550588..15bb6ad5d 100644 --- a/tests/corpus/7662-node-call-exit-events/main.ts +++ b/tests/corpus/7662-node-call-exit-events/main.ts @@ -1,3 +1,4 @@ +// @exit: 4 // @requires-libnode // The program and the Node.js it loads end as one process: Node's // 'beforeExit' and 'exit' run when the program ends, work that 'beforeExit' From c24ae2d7e921eb70d92d435f287ddde12d1123f4 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 14:11:21 -0500 Subject: [PATCH 090/184] Keep caches for property helpers that answer plain values first Property helpers now start with an exit for values no class table matches: if the receiver is neither a class capsule nor a function, they run the generic read or write on it directly. The inline-cache recognizer accepted only capsule branches and the subclass normalization before a helper's generic tail, so these helpers stopped being cache sites. It now also accepts that exit when its operation is the one the site caches: a read or write of the same key on the receiver. --- packages/compiler/src/ir/js-ic.test.ts | 54 ++++++++++++++++++++ packages/compiler/src/ir/js-ic.ts | 69 ++++++++++++++++++++++---- 2 files changed, 114 insertions(+), 9 deletions(-) diff --git a/packages/compiler/src/ir/js-ic.test.ts b/packages/compiler/src/ir/js-ic.test.ts index 026600cc3..f5391b83e 100644 --- a/packages/compiler/src/ir/js-ic.test.ts +++ b/packages/compiler/src/ir/js-ic.test.ts @@ -371,3 +371,57 @@ test("bag helpers that exit early for values no class branch matches are still t expect(sites.site("%dyn.class.property.2")).toBeNull(); expect(sites.site("%dyn.class.property.3")).toBeNull(); }); + +test("helpers that answer plain values first with the same operation are cache sites", () => { + const negated = (test: "classCapsule" | "funcKind"): IrExpr => ({ + kind: "dynTest", + test, + negated: true, + value: ref("p.0"), + type: BOOL, + loc, + }); + const exit = (body: IrStmt[], test: IrExpr = negated("classCapsule")): IrStmt => ({ + kind: "if", + cond: test, + then: [{ kind: "if", cond: negated("funcKind"), then: body, else_: null, loc }], + else_: null, + loc, + }); + const store = (key: string): IrStmt[] => [ + { + kind: "exprStmt", + expr: { + kind: "libCall", + fn: "dyn.keySet", + args: [ref("p.0"), str(key), ref("p.1")], + type: VOID, + loc, + }, + loc, + }, + ret(null), + ]; + const load = (key: string): IrStmt[] => [ + ret({ kind: "dynKeyGet", value: ref("p.0"), key: str(key), type: DYN, loc }), + ]; + const sites = sitesOf( + setHelper("%dyn.class.property.0", "x", [exit(store("x")), capsuleBranch("p.0")]), + getHelper("%dyn.class.property.1", "x", [exit(load("x")), capsuleBranch("p.0")]), + // An exit for another key, or one plain values do not take. + setHelper("%dyn.class.property.2", "x", [exit(store("y"))]), + setHelper("%dyn.class.property.3", "x", [ + exit(store("x"), { + kind: "dynTest", + test: "classCapsule", + value: ref("p.0"), + type: BOOL, + loc, + }), + ]), + ); + expect(sites.site("%dyn.class.property.0")).toEqual({ kind: "set", key: "x" }); + expect(sites.site("%dyn.class.property.1")).toEqual({ kind: "get", key: "x" }); + expect(sites.site("%dyn.class.property.2")).toBeNull(); + expect(sites.site("%dyn.class.property.3")).toBeNull(); +}); diff --git a/packages/compiler/src/ir/js-ic.ts b/packages/compiler/src/ir/js-ic.ts index 58c568bbd..48019636a 100644 --- a/packages/compiler/src/ir/js-ic.ts +++ b/packages/compiler/src/ir/js-ic.ts @@ -61,17 +61,68 @@ export class JsIcSites { if (fn.params.some((p) => p.type.kind !== "dyn")) return null; const receiver = fn.params[0]?.localId; if (receiver === undefined) return null; + const second = fn.params[1]?.localId; + const exits: JsIcSite[] = []; let i = 0; - while ( - i < fn.body.length && - (this.capsuleBranch(fn.body[i]!, receiver) || this.rebinding(fn.body[i]!, receiver)) - ) - i++; + for (; i < fn.body.length; i++) { + const s = fn.body[i]!; + if (this.capsuleBranch(s, receiver) || this.rebinding(s, receiver)) continue; + const exit = this.plainExit(s, receiver, second); + if (exit === null) break; + exits.push(exit); + } const tail = fn.body.slice(i); - if (fn.params.length === 1) return this.getTail(tail, receiver); - if (fn.params.length !== 2) return null; - const second = fn.params[1]!.localId; - return this.setTail(tail, receiver, second) ?? this.keyedTail(tail, receiver, second); + let site: JsIcSite | null; + if (fn.params.length === 1) site = this.getTail(tail, receiver); + else if (fn.params.length !== 2 || second === undefined) return null; + else site = this.setTail(tail, receiver, second) ?? this.keyedTail(tail, receiver, second); + if (site === null || site.kind === "keyed") return exits.length === 0 ? site : null; + // An exit answers plain values with the operation the site caches. + return exits.every((exit) => exit.kind === site.kind && exit.key === site.key) ? site : null; + } + + /** `if (!classCapsule(receiver)) { [if (!funcKind(receiver))] OP }`, no + * else: an exit that runs the helper's generic operation directly on a + * value no class table matches. OP is `return [js.fromDyn](dynKeyGet(R, + * K))` or `dyn.keySet(R, K, value); return`, with R the receiver (or a + * bag of it). */ + private plainExit(s: IrStmt, receiver: string, value: string | undefined): JsIcSite | null { + const negated = (e: IrExpr, test: "classCapsule" | "funcKind"): boolean => + e.kind === "dynTest" && e.test === test && e.negated === true && isVar(e.value, receiver); + if (s.kind !== "if" || s.else_ !== null || !negated(s.cond, "classCapsule")) return null; + let body = s.then; + const inner = body[0]; + if ( + body.length === 1 && + inner?.kind === "if" && + inner.else_ === null && + negated(inner.cond, "funcKind") + ) + body = inner.then; + const [first, ret] = body; + if (body.length === 1 && first?.kind === "return") { + const read = unwrapFromDyn(first.value); + if ( + read?.kind !== "dynKeyGet" || + read.key.kind !== "strLit" || + !this.bagOf(read.value, receiver) + ) + return null; + return { kind: "get", key: read.key.value }; + } + if ( + body.length !== 2 || + first?.kind !== "exprStmt" || + ret?.kind !== "return" || + ret.value !== null + ) + return null; + const call = first.expr; + if (call.kind !== "libCall" || call.fn !== "dyn.keySet" || call.args.length !== 3) return null; + const [bag, key, stored] = call.args; + if (!bag || !this.bagOf(bag, receiver) || key?.kind !== "strLit") return null; + if (value === undefined || !isVar(stored, value)) return null; + return { kind: "set", key: key.value }; } /** `varDecl bag = BAG(receiver); return [js.fromDyn](bag === receiver ? From c2963628343abb47685249cedaa2613bb612c888 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 14:21:21 -0500 Subject: [PATCH 091/184] Park typed collection iterators on the entry they read An early exit from a loop over an iterator-typed Map or Set method parks the iterator with a second yield of the current entry. That yield read the entry again by its index, which aborted with a map iteration error when the loop had deleted the entry. Each step now reads the entry once into a local that both yields answer. `entries()` takes the typed walk only for a slot whose yield type is the key and value pair record. Other yield types, such as `IterableIterator` or ``, keep the checked adapter instead of failing to compile. --- .../frontend/lowering/iterator-adapters.ts | 28 +++++++++- .../7746-key-order-iterable-iterator.ts | 51 +++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/packages/compiler/src/frontend/lowering/iterator-adapters.ts b/packages/compiler/src/frontend/lowering/iterator-adapters.ts index a8b4ea496..9f2b1e31f 100644 --- a/packages/compiler/src/frontend/lowering/iterator-adapters.ts +++ b/packages/compiler/src/frontend/lowering/iterator-adapters.ts @@ -84,6 +84,26 @@ function collectionIteratorSlotValue( collT.kind === "map" ? { kind: "mapIntrinsic", method: m, receiver: coll, args: [index], type: t, loc } : { kind: "setIntrinsic", method: "iterKey", receiver: coll, args: [index], type: t, loc }; + // entries() answers [key, value] pairs: only a slot whose yield type is + // that pair as a record of the two fields takes the typed walk; a tuple + // or other pair type keeps the checked adapter. + if (method === "entries") { + const pair = + expected.yieldT.kind === "record" ? lowerer.shapes.get(expected.yieldT.shapeId) : undefined; + const field = (name: string): IrType | undefined => + pair?.fields.find((f) => f.name === name)?.type; + const first = field("0"); + const second = field("1"); + if ( + pair === undefined || + pair.fields.length !== 2 || + first === undefined || + second === undefined || + !typeEquals(first, keyT) || + !typeEquals(second, valueT) + ) + return null; + } const entry = (): IrExpr => method === "keys" || (collT.kind === "set" && method === "values") ? read("iterKey", keyT) @@ -99,9 +119,13 @@ function collectionIteratorSlotValue( loc, }; if (!typeEquals(entry().type, expected.yieldT)) return null; + // The entry is read once per step and kept: the parking yield of an + // early exit answers it again rather than rereading a slot the loop may + // have deleted. + const current = varRef("current.0", expected.yieldT, loc); const suspend = (): IrStmt => ({ kind: "exprStmt", - expr: { kind: "yieldExpr", value: entry(), type: resumeType(expected.nextT), loc }, + expr: { kind: "yieldExpr", value: current, type: resumeType(expected.nextT), loc }, loc, }); const finish: IrStmt = { @@ -120,6 +144,7 @@ function collectionIteratorSlotValue( { id: "native.0", name: "native", type: DYN, mutable: false }, { id: "index.0", name: "index", type: F64, mutable: true }, { id: "resumed.0", name: "resumed", type: BOOL, mutable: true }, + { id: "current.0", name: "current", type: expected.yieldT, mutable: false }, ], body: [ { @@ -173,6 +198,7 @@ function collectionIteratorSlotValue( loc, }, setResumed(false, loc), + { kind: "varDecl", localId: "current.0", init: entry(), loc }, { kind: "tryCatch", tryBody: [suspend(), setResumed(true, loc)], diff --git a/tests/corpus/7746-key-order-iterable-iterator.ts b/tests/corpus/7746-key-order-iterable-iterator.ts index 13f27ee83..d9d4ea15a 100644 --- a/tests/corpus/7746-key-order-iterable-iterator.ts +++ b/tests/corpus/7746-key-order-iterable-iterator.ts @@ -56,3 +56,54 @@ const ordered = [...newest(listed)]; console.log(JSON.stringify(ordered), ordered[0] === listed[1]); const step = newest(listed).next(); console.log(step.done, JSON.stringify(step.value)); + +// Deleting the current entry before leaving a loop over an iterator-typed +// method: the iterator parks after that entry and continues with the next. +class Store { + readonly rows = new Map(); + readonly tags = new Set(); + values(): IterableIterator { + return this.rows.values(); + } + keys(): IterableIterator { + return this.rows.keys(); + } + tagValues(): IterableIterator { + return this.tags.values(); + } + anyPairs(): IterableIterator { + return this.rows.entries(); + } + unknownPairs(): IterableIterator { + return this.rows.entries(); + } + tagAny(): IterableIterator { + return this.tags.entries(); + } +} +const store = new Store(); +for (const [k, v] of [["a", 1], ["b", 2], ["c", 3]] as const) store.rows.set(k, v); +for (const t of ["x", "y", "z"]) store.tags.add(t); +console.log(JSON.stringify([...store.anyPairs()]), JSON.stringify([...store.unknownPairs()])); +console.log(JSON.stringify([...store.tagAny()])); +const values = store.values(); +for (const v of values) { + if (v === 2) { + store.rows.delete("b"); + break; + } +} +console.log([...values].join(), [...store.rows.keys()].join()); +for (const k of store.keys()) { + store.rows.delete(k); + break; +} +console.log([...store.rows.keys()].join()); +const tags = store.tagValues(); +for (const t of tags) { + if (t === "y") { + store.tags.delete("y"); + break; + } +} +console.log([...tags].join(), [...store.tags].join()); From 7cdfe16c9b3849f3bd75efcb1e352ecae0561eac Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 14:24:38 -0500 Subject: [PATCH 092/184] Cap the Linux test step's memory and spread memory-heavy side files The Linux test shards run their suites in a transient systemd scope with MemoryMax=13G and no swap, below the runner's 16 GB. A process tree that grows past the cap loses its largest process to the kernel, and the job log names the failing test, where the runner itself ran out of memory and left no log. Hosts without systemd run the suites directly. The side-queue balancer also takes a table of memory-heavy files, with their measured peak. They run whole in the cost-balanced task, and each goes to the shard with the least memory-heavy work so far, after the time-balanced files. decl-boundary.test.ts, which peaks at about 1.8 GB alone and had no time measurement, so vitest's file sharding put it in shard 11, is the first entry. It now runs in shard 1 (plain) and 6 (san). --- .github/workflows/ci.yml | 7 +++++ scripts/ci-test-plan.mjs | 36 ++++++++++++++++++++++--- tests/harness/ci-test-plan.test.ts | 43 +++++++++++++++++++++++++++--- 3 files changed, 78 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3bf0e770..5ab1d9331 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -216,9 +216,16 @@ jobs: # Start the corpus immediately beside a serial side queue. One shared # worker budget covers both processes; file and case shard axes stay # separate so every fixture still runs exactly once in each lane. + # The tests run in a transient scope capped below the runner's 16 GB: + # past 13 GB the kernel kills the scope's largest process, so the job + # log names the test instead of the runner itself running out of memory. - name: Tests (corpus and integration suites) run: | ulimit -c 0 + if [ -d /run/systemd/system ]; then + exec sudo --preserve-env systemd-run --scope --quiet -p MemoryMax=13G -p MemorySwapMax=0 \ + --uid="$(id -u)" --gid="$(id -g)" -- env PATH="$PATH" HOME="$HOME" node scripts/ci-test.mjs + fi node scripts/ci-test.mjs tests_macos: diff --git a/scripts/ci-test-plan.mjs b/scripts/ci-test-plan.mjs index 2b76b52d6..184f04d1c 100644 --- a/scripts/ci-test-plan.mjs +++ b/scripts/ci-test-plan.mjs @@ -42,22 +42,38 @@ export const testCosts = JSON.parse( readFileSync(new URL("./ci-test-costs.json", import.meta.url), "utf8"), ); +// Side-queue files whose process tree, run alone, peaks far above a plain +// vitest run (about 1 GB): megabytes at the peak, measured with CI's settings. +// They run beside the corpus task's three workers, so no two of them share a +// shard while there are shards without one. +export const memoryHeavyFiles = { + "tests/harness/decl-boundary.test.ts": 1838, +}; + // The shard (1-based) of each measured side-queue file and isolated suite: // longest first to the shard with the least work so far, counting about a // second per file for vitest to import it. Ties go to the highest shard, so the // longest files start from the last shard while the cost-balanced corpus and // three.js cases (tests/harness/shard.ts) place their longest from the first. -// An isolated suite without a measurement weighs as much as the median -// measured file. Files without a measurement are left to vitest's --shard, -// which splits them by count. +// An isolated suite or memory-heavy file without a measurement weighs as much +// as the median measured file. Memory-heavy files are placed after the others, +// each on the shard with the least memory-heavy work so far and, among those, +// the least work. Files without a +// measurement are left to vitest's --shard, which splits them by count. export function sideOwners(flavor, count) { const costs = testCosts[flavor]; const known = Object.values(costs).sort((a, b) => a - b); const median = known.length === 0 ? 0 : known[Math.floor(known.length / 2)]; const weight = (file) => (costs[file] ?? median) + 1000; - const files = [...new Set([...Object.keys(costs), ...isolatedFiles.map(([, file]) => file)])]; + const heavy = Object.keys(memoryHeavyFiles).sort( + (a, b) => memoryHeavyFiles[b] - memoryHeavyFiles[a] || (a < b ? -1 : a > b ? 1 : 0), + ); + const files = [ + ...new Set([...Object.keys(costs), ...isolatedFiles.map(([, file]) => file)]), + ].filter((file) => !(file in memoryHeavyFiles)); files.sort((a, b) => weight(b) - weight(a) || (a < b ? -1 : a > b ? 1 : 0)); const loads = new Array(count).fill(0); + const memory = new Array(count).fill(0); const owners = new Map(); for (const file of files) { let lightest = count - 1; @@ -66,6 +82,18 @@ export function sideOwners(flavor, count) { loads[lightest] += weight(file); owners.set(file, lightest + 1); } + for (const file of heavy) { + let lightest = count - 1; + for (let shard = count - 2; shard >= 0; shard--) + if ( + memory[shard] < memory[lightest] || + (memory[shard] === memory[lightest] && loads[shard] < loads[lightest]) + ) + lightest = shard; + memory[lightest] += memoryHeavyFiles[file]; + loads[lightest] += weight(file); + owners.set(file, lightest + 1); + } return owners; } diff --git a/tests/harness/ci-test-plan.test.ts b/tests/harness/ci-test-plan.test.ts index 34d61346f..9ff730ddf 100644 --- a/tests/harness/ci-test-plan.test.ts +++ b/tests/harness/ci-test-plan.test.ts @@ -11,6 +11,8 @@ import { caseShardedSideFiles, isolatedFiles, leakSuppressions, + memoryHeavyFiles, + sideOwners, testCosts, } from "../../scripts/ci-test-plan.mjs"; import { runCiTestPlan } from "../../scripts/ci-test.mjs"; @@ -40,20 +42,27 @@ test.for([1, 3, 5, 10, 20])( file, ).toHaveLength(everyShard ? count : 1); } - // Each measured file runs on exactly one shard, outside vitest's file shard. + // Each measured or memory-heavy file runs on exactly one shard, outside + // vitest's file shard. const isolated = new Set(isolatedFiles.map(([, file]) => file)); const balanced = plans.flatMap((plan) => plan.side .filter((task) => task.name === "cost-balanced tests") .flatMap((task) => task.args.slice(2)), ); - const measured = Object.keys(testCosts[flavor]).filter((file) => !isolated.has(file)); + const measured = [ + ...new Set([...Object.keys(testCosts[flavor]), ...Object.keys(memoryHeavyFiles)]), + ].filter((file) => !isolated.has(file)); expect(measured.length).toBeGreaterThan(0); expect(balanced.toSorted()).toEqual(measured.toSorted()); for (const plan of plans) { expect(plan.corpus.args).toEqual(["test", ...corpusFiles]); expect(plan.side[0]!.args).toContain(`--shard=${plan.corpus.env.SCRIPTC_TEST_SHARD}`); - for (const file of [...fileExclusions, ...Object.keys(testCosts[flavor])]) + for (const file of [ + ...fileExclusions, + ...Object.keys(testCosts[flavor]), + ...Object.keys(memoryHeavyFiles), + ]) expect(plan.side[0]!.args).toContain(`--exclude=${file}`); const cache = plan.side.find((task) => task.name === "native-cache correctness"); expect(Boolean(cache)).toBe(flavor === "plain"); @@ -103,6 +112,26 @@ test("the side queue balances measured files across the twenty Linux shards", () } }); +test("memory-heavy side files run whole in the cost-balanced task, on shards of their own", () => { + for (const flavor of ["plain", "san"] as const) { + const owners = sideOwners(flavor, 20); + const heavy = Object.keys(memoryHeavyFiles); + const shards = heavy.map((file) => owners.get(file)); + expect(shards.every((shard) => shard !== undefined)).toBe(true); + expect(new Set(shards).size).toBe(Math.min(heavy.length, 20)); + for (const file of heavy) { + const plan = ciTestPlan({ shard: `${owners.get(file)}/20`, flavor }); + const balanced = plan.side.find((task) => task.name === "cost-balanced tests")!; + expect(balanced.args).toContain(file); + // The file-sharded task leaves it out on every shard. + for (let shard = 1; shard <= 20; shard++) + expect(ciTestPlan({ shard: `${shard}/20`, flavor }).side[0]!.args).toContain( + `--exclude=${file}`, + ); + } + } +}); + test.for([1, 2, 4, 8])( "CI overlaps reserved lanes within a budget of %i workers", async (workers) => { @@ -265,7 +294,13 @@ test("pull requests run the full suite on Linux and the darwin suite on macOS", ); expect(coreStep).toBeGreaterThan(-1); expect(coreStep).toBeLessThan(linux.steps.length - 1); - expect(linux.steps.at(-1)!.run).toBe("ulimit -c 0\nnode scripts/ci-test.mjs\n"); + // Linux runs the suites in a memory-capped scope where systemd manages the + // runner, and directly elsewhere. + const linuxRun = linux.steps.at(-1)!.run as string; + expect(linuxRun.startsWith("ulimit -c 0\n")).toBe(true); + expect(linuxRun).toContain("systemd-run --scope --quiet -p MemoryMax=13G -p MemorySwapMax=0"); + expect(linuxRun).toMatch(/-- env PATH="\$PATH" HOME="\$HOME" node scripts\/ci-test\.mjs\n/); + expect(linuxRun.endsWith("fi\nnode scripts/ci-test.mjs\n")).toBe(true); expect(macos.steps.at(-1)!.run).toBe("node scripts/ci-test.mjs"); for (const job of [linux, macos]) expect(job.strategy.matrix.flavor).toEqual(["plain", "san"]); }); From 89d122574f708d0bede331b8b21a9fd3505fbd35 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 14:27:20 -0500 Subject: [PATCH 093/184] Run Test262 tests that include testTypedArray.js --- tests/harness/test262-runner.test.ts | 20 + tests/harness/test262.test.ts | 69 +++ tests/test262/README.md | 4 +- tests/test262/run.ts | 6 +- tests/test262/support.mjs | 12 +- tests/test262/test-typed-array.js | 313 +++++++++++++ tests/test262/upstream.json | 13 + .../test262/vendor/harness/testTypedArray.js | 431 ++++++++++++++++++ .../prototype/byteOffset/return-byteoffset.js | 31 ++ .../return-abrupt-from-begin-symbol.js | 24 + .../byteoffset-is-negative-zero-sab.js | 23 + .../is-infinity-throws-rangeerror.js | 27 ++ .../undefined-newtarget-throws.js | 27 ++ ...mericindex-desc-not-configurable-throws.js | 39 ++ 14 files changed, 1032 insertions(+), 7 deletions(-) create mode 100644 tests/test262/test-typed-array.js create mode 100644 tests/test262/vendor/harness/testTypedArray.js create mode 100644 tests/test262/vendor/test/built-ins/TypedArray/prototype/byteOffset/return-byteoffset.js create mode 100644 tests/test262/vendor/test/built-ins/TypedArray/prototype/subarray/return-abrupt-from-begin-symbol.js create mode 100644 tests/test262/vendor/test/built-ins/TypedArrayConstructors/ctors/buffer-arg/byteoffset-is-negative-zero-sab.js create mode 100644 tests/test262/vendor/test/built-ins/TypedArrayConstructors/ctors/length-arg/is-infinity-throws-rangeerror.js create mode 100644 tests/test262/vendor/test/built-ins/TypedArrayConstructors/ctors/typedarray-arg/undefined-newtarget-throws.js create mode 100644 tests/test262/vendor/test/built-ins/TypedArrayConstructors/internals/DefineOwnProperty/key-is-numericindex-desc-not-configurable-throws.js diff --git a/tests/harness/test262-runner.test.ts b/tests/harness/test262-runner.test.ts index 6f6df35d4..3ddf3ca84 100644 --- a/tests/harness/test262-runner.test.ts +++ b/tests/harness/test262-runner.test.ts @@ -101,6 +101,26 @@ test("the property helper only admits direct inline descriptor checks", () => { } }); +test("the typed array helper is admitted without the %TypedArray% intrinsic", () => { + const text = source( + "includes: [testTypedArray.js]", + "testWithTypedArrayConstructors(function(TA) { assert.sameValue(new TA(1).length, 1); });", + ); + expect(exclusion(text, metadata(text), "strict")).toBeUndefined(); + const prepared = prepare(text, false, undefined, "strict", ["testTypedArray.js"]); + expect(prepared).toContain("function testWithTypedArrayConstructors("); + expect(prepared).not.toContain("Object.getPrototypeOf(Int8Array)"); + for (const body of [ + "assert.sameValue(typeof TypedArray, 'function');", + "testWithTypedArrayConstructors(function(TA) { assert.sameValue(Object.getPrototypeOf(TA), TypedArray); });", + ]) { + const named = source("includes: [testTypedArray.js]", body); + expect(exclusion(named, metadata(named), "strict")).toBe("harness:TypedArray-intrinsic"); + } + const other = source("includes: [testTypedArray.js, detachArrayBuffer.js]"); + expect(exclusion(other, metadata(other), "strict")).toBe("harness-includes:detachArrayBuffer.js"); +}); + test("receiver-bound this is admitted without adapting script-level this", () => { for (const [body, expected] of [ ["class C { read() { return this.value; } }", undefined], diff --git a/tests/harness/test262.test.ts b/tests/harness/test262.test.ts index a36fa2cdf..f1b616578 100644 --- a/tests/harness/test262.test.ts +++ b/tests/harness/test262.test.ts @@ -19,6 +19,10 @@ const upstreamHarness = ["assert.js", "sta.js"] .map((name) => readFileSync(join(vendorRoot, "harness", name), "utf8")) .join("\n"); const upstreamPropertyHelper = readFileSync(join(vendorRoot, "harness/propertyHelper.js"), "utf8"); +const upstreamTypedArrayHelper = readFileSync( + join(vendorRoot, "harness/testTypedArray.js"), + "utf8", +); function runUpstream( source: string, @@ -29,6 +33,8 @@ function runUpstream( runInContext(upstreamHarness, context, { timeout: 5000 }); if (includes.includes("propertyHelper.js")) runInContext(upstreamPropertyHelper, context, { timeout: 5000 }); + if (includes.includes("testTypedArray.js")) + runInContext(upstreamTypedArrayHelper, context, { timeout: 5000 }); runInContext(variant === "strict" ? `"use strict";\n${source}` : source, context, { timeout: 5000, }); @@ -52,6 +58,8 @@ async function runUpstreamAsync( runInContext(upstreamHarness, context, { timeout: 5000 }); if (includes.includes("propertyHelper.js")) runInContext(upstreamPropertyHelper, context, { timeout: 5000 }); + if (includes.includes("testTypedArray.js")) + runInContext(upstreamTypedArrayHelper, context, { timeout: 5000 }); runInContext(variant === "strict" ? `"use strict";\n${source}` : source, context, { timeout: 5000, }); @@ -375,6 +383,67 @@ if (propertyControls.length > 0) } }); +const typedArrayControls = shardSelect( + [ + { + name: "every constructor with array, array-like and buffer arguments", + status: "pass", + source: + "testWithTypedArrayConstructors(function(TA, makeCtorArg) { var sample = new TA(makeCtorArg([40, 41, 42])); assert.sameValue(sample.length, 3); assert.sameValue(sample[2], 42); }, null, ['passthrough', 'arraylike', 'arraybuffer']);", + }, + { + name: "failing element assertion", + status: "fail", + source: + "testWithTypedArrayConstructors(function(TA, makeCtorArg) { var sample = new TA(makeCtorArg([40, 41, 42])); assert.sameValue(sample[2], 43); });", + }, + { + name: "failure in one constructor", + status: "fail", + source: + "testWithTypedArrayConstructors(function(TA) { assert(TA !== Int16Array, 'Int16Array'); });", + }, + { + name: "passthrough factories visit each constructor once", + status: "pass", + source: + "var count = 0; testWithTypedArrayConstructors(function(TA, makeCtorArg) { assert.sameValue(new TA(makeCtorArg(4)).length, 4); count++; }, null, ['passthrough']); assert.sameValue(count, typedArrayConstructors.length);", + }, + { + name: "explicit constructor list", + status: "pass", + source: + "var names = []; testWithTypedArrayConstructors(function(TA) { names.push(TA.name); }, [Uint8Array, Float64Array], ['passthrough']); assert.compareArray(names, ['Uint8Array', 'Float64Array']);", + }, + { + name: "float constructor classification", + status: "pass", + source: + "assert(isFloatTypedArrayConstructor(Float32Array)); assert(!isFloatTypedArrayConstructor(Int8Array)); assert.sameValue(floatTypedArrayConstructorPrecision(Float64Array), 'double');", + }, + { + name: "unknown argument factory feature", + status: "fail", + source: "testWithTypedArrayConstructors(function() {}, null, ['unknown']);", + }, + ], + (item) => `typed-array:${item.name}`, +); + +if (typedArrayControls.length > 0) + describe(`Test262 typed array helper contract${shardSuffix()}`, () => { + for (const control of typedArrayControls) { + test(control.name, async () => { + const includes = ["compareArray.js", "testTypedArray.js"]; + if (control.status === "fail") + expect(() => runUpstream(control.source, "strict", includes)).toThrow(); + else expect(() => runUpstream(control.source, "strict", includes)).not.toThrow(); + const result = await runSource(control.source, { sanitize, includes }); + expect(result, JSON.stringify(result, null, 2)).toMatchObject({ status: control.status }); + }); + } + }); + const asyncControls = shardSelect( [ { diff --git a/tests/test262/README.md b/tests/test262/README.md index 46465645f..2f8a0ed3e 100644 --- a/tests/test262/README.md +++ b/tests/test262/README.md @@ -28,11 +28,11 @@ Pass the resulting directory as `--root` and use `--filter` to survey an API fam ## Execution profile and limits -The current profile is `static-adapted-v5`. It adapts strict scripts to standalone modules and sloppy scripts to CommonJS entries. Preparation supplies host assertions and a completion marker while leaving the upstream test body unchanged. Each program executes in a fresh native process. Conservative syntax checks reject known global-script dependencies, but cannot establish that every adaptation preserves the original script's semantics. Reports are local investigation artifacts, not published compatibility assessments. +The current profile is `static-adapted-v6`. It adapts strict scripts to standalone modules and sloppy scripts to CommonJS entries. Preparation supplies host assertions and a completion marker while leaving the upstream test body unchanged. Each program executes in a fresh native process. Conservative syntax checks reject known global-script dependencies, but cannot establish that every adaptation preserves the original script's semantics. Reports are local investigation artifacts, not published compatibility assessments. The unmodified upstream assertion helper currently reaches scriptc's JavaScript function-expando refusal. Test262 explicitly permits host implementations of harness functions in its [interpretation rules](https://github.com/tc39/test262/blob/7ab7fafa0003f73fc85c1b95d88094d33f7eb8bd/INTERPRETING.md#host-defined-functions). `harness.ts` supplies `assert`, `assert.sameValue`, `assert.notSameValue`, `assert.compareArray`, `assert.throws` for selected Error constructors, `$DONE`, and `Test262Error`. Scalar SameValue and scalar array-element comparisons use the statically compiled `node:assert/strict` implementation. Contract tests compare successful and failing assertions with the original Test262 helpers under Node, including NaN, signed zero, null/undefined, differing primitive types, and premature termination. -Reference-equality assertions terminate with a distinct harness-refusal result: conversion at a native function boundary can copy references, so this adapter cannot faithfully compare every object, array, or function identity. Tests cannot catch this refusal and accidentally pass. Sparse array comparisons are not yet handled faithfully across that boundary. The adapted `propertyHelper.js` admits direct `verifyProperty` calls with an inline descriptor literal or `undefined`; reference-valued comparisons still refuse, while other property helpers and descriptor expressions remain excluded. Assertion aliases, mutation, reflection, additional assertion methods, and other `includes` helpers remain excluded. +Reference-equality assertions terminate with a distinct harness-refusal result: conversion at a native function boundary can copy references, so this adapter cannot faithfully compare every object, array, or function identity. Tests cannot catch this refusal and accidentally pass. Sparse array comparisons are not yet handled faithfully across that boundary. The adapted `propertyHelper.js` admits direct `verifyProperty` calls with an inline descriptor literal or `undefined`; reference-valued comparisons still refuse, while other property helpers and descriptor expressions remain excluded. The adapted `testTypedArray.js` keeps the upstream constructor lists, argument factories and `testWith*` helpers. It omits the `TypedArray` (%TypedArray%) binding, so tests that name it are excluded. Its resizable and immutable buffer factories run only where `ArrayBuffer.prototype` provides `resize` and `transferToImmutable`, as the upstream feature checks intend. Contract tests compare its successful and failing uses with the upstream helper under Node. Assertion aliases, mutation, reflection, additional assertion methods, and other `includes` helpers remain excluded. Raw tests, module variants, agents, resolution negatives, and runtime negatives remain excluded. Parse negatives pass only when a source-matched syntax or vetted JavaScript early-error diagnostic appears; TypeScript type errors and harness-only failures never satisfy a negative expectation. `$262`, dynamic evaluation, observable script-global state, and other unsupported host requirements are excluded as well. The runtime and compiler continue to enforce their existing refusal boundaries for admitted source. diff --git a/tests/test262/run.ts b/tests/test262/run.ts index b798b3f21..2ad02100b 100644 --- a/tests/test262/run.ts +++ b/tests/test262/run.ts @@ -4,7 +4,7 @@ import { parseArgs } from "node:util"; import { shardSelect } from "../harness/shard.js"; import { runSource, type Outcome } from "./execute.js"; import { - assertThrowsSource, directory, exclusion, harnessSource, propertyHelperSource, matchesExpectation, matchesParseNegative, metadata, pin, sha256, snapshotDigest, summarize, + assertThrowsSource, directory, exclusion, harnessSource, propertyHelperSource, typedArrayHelperSource, matchesExpectation, matchesParseNegative, metadata, pin, sha256, snapshotDigest, summarize, testPaths, variants, vendorRoot, verifyVendor, } from "./support.mjs"; @@ -104,8 +104,8 @@ Other variants remain visible as exclusions.`); const report = { schema: "scriptc.test262.v1", revision: pin.commit, snapshotSha256: pin.snapshotSha256, - profile: "static-adapted-v5", - harnessSha256: sha256(harnessSource + assertThrowsSource + propertyHelperSource), + profile: "static-adapted-v6", + harnessSha256: sha256(harnessSource + assertThrowsSource + propertyHelperSource + typedArrayHelperSource), requestedBackend: backend, sanitize: process.env.SCRIPTC_SAN === "1", host: `${process.platform}-${process.arch}`, node: process.version, compilerVersion: JSON.parse(readFileSync(join(directory, "../../packages/compiler/package.json"), "utf8")).version, diff --git a/tests/test262/support.mjs b/tests/test262/support.mjs index e56612457..52739f91a 100644 --- a/tests/test262/support.mjs +++ b/tests/test262/support.mjs @@ -12,6 +12,8 @@ export const completion = "__scriptc_test262_complete__"; export const harnessSource = readFileSync(join(directory, "harness.ts"), "utf8"); export const assertThrowsSource = readFileSync(join(directory, "assert-throws.js"), "utf8"); export const propertyHelperSource = readFileSync(join(directory, "property-helper.js"), "utf8"); +export const typedArrayHelperSource = readFileSync(join(directory, "test-typed-array.js"), "utf8"); +const adaptedIncludes = new Set(["compareArray.js", "propertyHelper.js", "testTypedArray.js"]); export const expectations = JSON.parse(readFileSync(join(directory, "expectations.json"), "utf8")); export const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex"); @@ -131,11 +133,12 @@ export function exclusion(source, meta, variant) { } if (variant !== "strict" && variant !== "sloppy") return `execution:${variant}`; if (meta.flags.some((flag) => flag.startsWith("CanBlock"))) return "host:agents"; - const unsupportedIncludes = meta.includes.filter((name) => name !== "compareArray.js" && name !== "propertyHelper.js"); + const unsupportedIncludes = meta.includes.filter((name) => !adaptedIncludes.has(name)); if (unsupportedIncludes.length) return `harness-includes:${unsupportedIncludes.join(",")}`; const sf = ts.createSourceFile("test.js", source, ts.ScriptTarget.Latest, true, ts.ScriptKind.JS); let reason; const propertyHelper = meta.includes.includes("propertyHelper.js"); + const typedArrayHelper = meta.includes.includes("testTypedArray.js"); const forbidden = new Set(["$262", "$DONE", "$DONOTEVALUATE", "globalThis", "eval", "Function", "print", "process", "require"]); if (variant === "sloppy") for (const name of ["module", "exports", "__dirname", "__filename"]) forbidden.add(name); if (meta.flags.includes("async")) forbidden.delete("$DONE"); @@ -170,6 +173,9 @@ export function exclusion(source, meta, variant) { reason = "harness:compareArray-surface"; } else if (ts.isIdentifier(node) && node.text === "Test262Error") { if (!ts.isNewExpression(node.parent) || node.parent.expression !== node) reason = "harness:Test262Error-surface"; + } else if (typedArrayHelper && ts.isIdentifier(node) && node.text === "TypedArray") { + // The adapted testTypedArray.js leaves out %TypedArray%. + reason = "harness:TypedArray-intrinsic"; } else if (propertyHelper && ts.isIdentifier(node)) { if (unsupportedPropertyHelpers.has(node.text)) reason = "harness:propertyHelper-surface"; else if (node.text === "verifyProperty") { @@ -188,7 +194,9 @@ export function exclusion(source, meta, variant) { export function prepare(source, asyncTest = false, marker = completion, variant = "strict", includes = []) { const names = asyncTest ? "assert, Test262Error, $DONE" : "assert, Test262Error"; const end = asyncTest ? "" : `;console.log(${JSON.stringify(marker)});\n`; - const helpers = includes.includes("propertyHelper.js") ? `${propertyHelperSource}\n` : ""; + const helpers = + (includes.includes("propertyHelper.js") ? `${propertyHelperSource}\n` : "") + + (includes.includes("testTypedArray.js") ? `${typedArrayHelperSource}\n` : ""); if (variant === "sloppy") return `const { ${names} } = require("./harness.ts");\n${helpers}${source}\n${end}`; return `"use strict";\nimport { ${names} } from "./harness.ts";\n${helpers}${source}\n${end}`; } diff --git a/tests/test262/test-typed-array.js b/tests/test262/test-typed-array.js new file mode 100644 index 000000000..f7ee73c84 --- /dev/null +++ b/tests/test262/test-typed-array.js @@ -0,0 +1,313 @@ +// Copyright (C) 2015 André Bargull. All rights reserved. +// This adaptation of Test262's testTypedArray.js is governed by the BSD license in vendor/LICENSE. +// It keeps the upstream constructor lists and argument factories. The host +// assert.js is replaced by harness.ts, so isPrimitive is defined here, and +// Test262Error is a class that needs `new`. Factories bind through a +// closure, and feature checks use typeof (and tolerate an ArrayBuffer +// without a reachable prototype, which then has no resizable or immutable +// factories). %TypedArray% (`TypedArray`) is left out: tests that name it +// are excluded. + +function isPrimitive(value) { + return !value || (typeof value !== "object" && typeof value !== "function"); +} + +var floatArrayConstructors = [ + Float64Array, + Float32Array +]; + +var nonClampedIntArrayConstructors = [ + Int32Array, + Int16Array, + Int8Array, + Uint32Array, + Uint16Array, + Uint8Array +]; + +var intArrayConstructors = nonClampedIntArrayConstructors.concat([Uint8ClampedArray]); + +// Float16Array is a newer feature +// adding it to this list unconditionally would cause implementations lacking it to fail every test which uses it +if (typeof Float16Array !== "undefined") { + floatArrayConstructors.push(Float16Array); +} + +var bigIntArrayConstructors = []; +if (typeof BigInt64Array !== "undefined") { + bigIntArrayConstructors.push(BigInt64Array); +} +if (typeof BigUint64Array !== "undefined") { + bigIntArrayConstructors.push(BigUint64Array); +} + +var typedArrayConstructors = floatArrayConstructors.concat(intArrayConstructors); + +var allTypedArrayConstructors = typedArrayConstructors.concat(bigIntArrayConstructors); + +function makePassthrough(TA, primitiveOrIterable) { + return primitiveOrIterable; +} + +function makeArray(TA, primitiveOrIterable) { + if (isPrimitive(primitiveOrIterable)) { + var n = Number(primitiveOrIterable); + // Only values between 0 and 2**53 - 1 inclusive can get mapped into TA contents. + if (!(n >= 0 && n < 9007199254740992)) return primitiveOrIterable; + return Array.from({ length: n }, function() { return "0"; }); + } + return Array.from(primitiveOrIterable); +} + +function makeArrayLike(TA, primitiveOrIterable) { + var arr = makeArray(TA, primitiveOrIterable); + if (isPrimitive(arr)) return arr; + var obj = { length: arr.length }; + for (var i = 0; i < obj.length; i++) obj[i] = arr[i]; + return obj; +} + +var makeIterable; +if (typeof Symbol !== "undefined") { + makeIterable = function makeIterable(TA, primitiveOrIterable) { + var src = makeArray(TA, primitiveOrIterable); + if (isPrimitive(src)) return src; + var obj = {}; + obj[Symbol.iterator] = function() { return src[Symbol.iterator](); }; + return obj; + }; +} + +function makeArrayBuffer(TA, primitiveOrIterable) { + var arr = makeArray(TA, primitiveOrIterable); + if (isPrimitive(arr)) return arr; + return new TA(arr).buffer; +} + +var makeResizableArrayBuffer, makeGrownArrayBuffer, makeShrunkArrayBuffer, makeImmutableArrayBuffer; +if (typeof ArrayBuffer.prototype === "object" && typeof ArrayBuffer.prototype.resize === "function") { + var copyIntoArrayBuffer = function(destBuffer, srcBuffer) { + var destView = new Uint8Array(destBuffer); + var srcView = new Uint8Array(srcBuffer); + for (var i = 0; i < srcView.length; i++) destView[i] = srcView[i]; + return destBuffer; + }; + + makeResizableArrayBuffer = function makeResizableArrayBuffer(TA, primitiveOrIterable) { + if (isPrimitive(primitiveOrIterable)) { + var n = Number(primitiveOrIterable) * TA.BYTES_PER_ELEMENT; + if (!(n >= 0 && n < 9007199254740992)) return primitiveOrIterable; + return new ArrayBuffer(n, { maxByteLength: n * 2 }); + } + var fixed = makeArrayBuffer(TA, primitiveOrIterable); + var byteLength = fixed.byteLength; + var resizable = new ArrayBuffer(byteLength, { maxByteLength: byteLength * 2 }); + return copyIntoArrayBuffer(resizable, fixed); + }; + + makeGrownArrayBuffer = function makeGrownArrayBuffer(TA, primitiveOrIterable) { + if (isPrimitive(primitiveOrIterable)) { + var n = Number(primitiveOrIterable) * TA.BYTES_PER_ELEMENT; + if (!(n >= 0 && n < 9007199254740992)) return primitiveOrIterable; + var grown = new ArrayBuffer(Math.floor(n / 2), { maxByteLength: n }); + grown.resize(n); + } + var fixed = makeArrayBuffer(TA, primitiveOrIterable); + var byteLength = fixed.byteLength; + var grown = new ArrayBuffer(Math.floor(byteLength / 2), { maxByteLength: byteLength }); + grown.resize(byteLength); + return copyIntoArrayBuffer(grown, fixed); + }; + + makeShrunkArrayBuffer = function makeShrunkArrayBuffer(TA, primitiveOrIterable) { + if (isPrimitive(primitiveOrIterable)) { + var n = Number(primitiveOrIterable) * TA.BYTES_PER_ELEMENT; + if (!(n >= 0 && n < 9007199254740992)) return primitiveOrIterable; + var shrunk = new ArrayBuffer(n * 2, { maxByteLength: n * 2 }); + shrunk.resize(n); + } + var fixed = makeArrayBuffer(TA, primitiveOrIterable); + var byteLength = fixed.byteLength; + var shrunk = new ArrayBuffer(byteLength * 2, { maxByteLength: byteLength * 2 }); + copyIntoArrayBuffer(shrunk, fixed); + shrunk.resize(byteLength); + return shrunk; + }; +} +if (typeof ArrayBuffer.prototype === "object" && typeof ArrayBuffer.prototype.transferToImmutable === "function") { + makeImmutableArrayBuffer = function makeImmutableArrayBuffer(TA, primitiveOrIterable) { + if (isPrimitive(primitiveOrIterable)) { + var n = Number(primitiveOrIterable) * TA.BYTES_PER_ELEMENT; + if (!(n >= 0 && n < 9007199254740992)) return primitiveOrIterable; + return (new ArrayBuffer(n)).transferToImmutable(); + } + var mutable = makeArrayBuffer(TA, primitiveOrIterable); + return mutable.transferToImmutable(); + }; +} + +var typedArrayCtorArgFactories = [makePassthrough, makeArray, makeArrayLike]; +if (makeIterable) typedArrayCtorArgFactories.push(makeIterable); +typedArrayCtorArgFactories.push(makeArrayBuffer); +if (makeResizableArrayBuffer) typedArrayCtorArgFactories.push(makeResizableArrayBuffer); +if (makeGrownArrayBuffer) typedArrayCtorArgFactories.push(makeGrownArrayBuffer); +if (makeShrunkArrayBuffer) typedArrayCtorArgFactories.push(makeShrunkArrayBuffer); +if (makeImmutableArrayBuffer) typedArrayCtorArgFactories.push(makeImmutableArrayBuffer); + + +function ctorArgFactoryMatchesSome(argFactory, features) { + for (var i = 0; i < features.length; ++i) { + switch (features[i]) { + case "passthrough": + if (argFactory === makePassthrough) return true; + break; + case "arraylike": + if (argFactory === makeArray || argFactory === makeArrayLike) return true; + break; + case "iterable": + if (argFactory === makeIterable) return true; + break; + case "arraybuffer": + if ( + argFactory === makeArrayBuffer || + argFactory === makeResizableArrayBuffer || + argFactory === makeGrownArrayBuffer || + argFactory === makeShrunkArrayBuffer || + argFactory === makeImmutableArrayBuffer + ) { + return true; + } + break; + case "resizable": + if ( + argFactory === makeResizableArrayBuffer || + argFactory === makeGrownArrayBuffer || + argFactory === makeShrunkArrayBuffer + ) { + return true; + } + break; + case "immutable": + if (argFactory === makeImmutableArrayBuffer) return true; + break; + default: + throw new Test262Error("unknown feature: " + features[i]); + } + } + return false; +} + + +function bindArgFactory(argFactory, constructor) { + return function(primitiveOrIterable) { + return argFactory(constructor, primitiveOrIterable); + }; +} + +function testWithAllTypedArrayConstructors(f, constructors, includeArgFactories, excludeArgFactories) { + var ctors = constructors || allTypedArrayConstructors; + var ctorArgFactories = typedArrayCtorArgFactories; + if (includeArgFactories) { + ctorArgFactories = []; + for (var i = 0; i < typedArrayCtorArgFactories.length; ++i) { + if (ctorArgFactoryMatchesSome(typedArrayCtorArgFactories[i], includeArgFactories)) { + ctorArgFactories.push(typedArrayCtorArgFactories[i]); + } + } + } + if (excludeArgFactories) { + ctorArgFactories = ctorArgFactories.slice(); + for (var i = ctorArgFactories.length - 1; i >= 0; --i) { + if (ctorArgFactoryMatchesSome(ctorArgFactories[i], excludeArgFactories)) { + ctorArgFactories.splice(i, 1); + } + } + } + if (ctorArgFactories.length === 0) { + throw new Test262Error("no arg factories match include " + includeArgFactories + " and exclude " + excludeArgFactories); + } + for (var k = 0; k < ctorArgFactories.length; ++k) { + var argFactory = ctorArgFactories[k]; + for (var i = 0; i < ctors.length; ++i) { + var constructor = ctors[i]; + var boundArgFactory = bindArgFactory(argFactory, constructor); + try { + f(constructor, boundArgFactory); + } catch (e) { + e.message += " (Testing with " + constructor.name + " and " + argFactory.name + ".)"; + throw e; + } + } + } +} + +function testWithTypedArrayConstructors(f, constructors, includeArgFactories, excludeArgFactories) { + var ctors = constructors || typedArrayConstructors; + testWithAllTypedArrayConstructors(f, ctors, includeArgFactories, excludeArgFactories); +} + +function testWithBigIntTypedArrayConstructors(f, constructors, includeArgFactories, excludeArgFactories) { + var ctors = constructors || [BigInt64Array, BigUint64Array]; + testWithAllTypedArrayConstructors(f, ctors, includeArgFactories, excludeArgFactories); +} + +var nonAtomicsFriendlyTypedArrayConstructors = floatArrayConstructors.concat([Uint8ClampedArray]); +function testWithNonAtomicsFriendlyTypedArrayConstructors(f, includeArgFactories, excludeArgFactories) { + testWithAllTypedArrayConstructors( + f, + nonAtomicsFriendlyTypedArrayConstructors, + includeArgFactories, + excludeArgFactories + ); +} + +function testWithAtomicsFriendlyTypedArrayConstructors(f, includeArgFactories, excludeArgFactories) { + testWithAllTypedArrayConstructors( + f, + [ + Int32Array, + Int16Array, + Int8Array, + Uint32Array, + Uint16Array, + Uint8Array, + ], + includeArgFactories, + excludeArgFactories + ); +} + +function testTypedArrayConversions(byteConversionValues, fn) { + var values = byteConversionValues.values; + var expected = byteConversionValues.expected; + + testWithTypedArrayConstructors(function(TA) { + var name = TA.name.slice(0, -5); + + return values.forEach(function(value, index) { + var exp = expected[name][index]; + var initial = 0; + if (exp === 0) { + initial = 1; + } + fn(TA, value, exp, initial); + }); + }, null, ["passthrough"]); +} + +function isFloatTypedArrayConstructor(arg) { + return floatArrayConstructors.indexOf(arg) !== -1; +} + +function floatTypedArrayConstructorPrecision(FA) { + if (typeof Float16Array !== "undefined" && FA === Float16Array) { + return "half"; + } else if (FA === Float32Array) { + return "single"; + } else if (FA === Float64Array) { + return "double"; + } else { + throw new Error("Malformed test - floatTypedArrayConstructorPrecision called with non-float TypedArray"); + } +} diff --git a/tests/test262/upstream.json b/tests/test262/upstream.json index 1dc97ba20..025576e72 100644 --- a/tests/test262/upstream.json +++ b/tests/test262/upstream.json @@ -169,6 +169,12 @@ "test/built-ins/String/prototype/trim/15.5.4.20-2-47.js", "test/built-ins/String/prototype/trim/15.5.4.20-2-50.js", "test/built-ins/String/prototype/trim/15.5.4.20-2-6.js", + "test/built-ins/TypedArray/prototype/byteOffset/return-byteoffset.js", + "test/built-ins/TypedArray/prototype/subarray/return-abrupt-from-begin-symbol.js", + "test/built-ins/TypedArrayConstructors/ctors/buffer-arg/byteoffset-is-negative-zero-sab.js", + "test/built-ins/TypedArrayConstructors/ctors/length-arg/is-infinity-throws-rangeerror.js", + "test/built-ins/TypedArrayConstructors/ctors/typedarray-arg/undefined-newtarget-throws.js", + "test/built-ins/TypedArrayConstructors/internals/DefineOwnProperty/key-is-numericindex-desc-not-configurable-throws.js", "test/language/arguments-object/func-decl-args-trailing-comma-null.js", "test/language/arguments-object/func-expr-args-trailing-comma-undefined.js", "test/language/expressions/addition/S11.6.1_A4_T1.js", @@ -214,6 +220,7 @@ "harness/assert.js": "206e274ca325eb8a652e3911c3fbd090e2480d11ed7579dc17a5d17a2360ed48", "harness/propertyHelper.js": "08bc29114d804c4f24f906bac950ee4dc9c93c659d93258e7b44a54283ba9bb8", "harness/sta.js": "1930c54af79455c484799f43e9a28e2b2f15c40d0917c9941ca54e26db243f35", + "harness/testTypedArray.js": "51eb50e53e6215f31f14d64d249bea2d08ad943c77835f63ba52ec694de2189a", "test/annexB/language/function-code/function-redeclaration-block.js": "8d4a3f259dabceeee101959aac117a19f5bfe9e374e1178b45f45a1142195d9e", "test/annexB/language/function-code/if-decl-else-decl-b-func-existing-fn-no-init.js": "ccd95922fd6c784012f7299445f0859db04812232a2266dd90070b026244bbf5", "test/built-ins/Array/prototype/at/returns-item-relative-index.js": "71f53cf86f1387101b19b6ae9939d47ced34497d3cee12f954eef454f912ba88", @@ -377,6 +384,12 @@ "test/built-ins/String/prototype/trim/15.5.4.20-2-47.js": "3e1a745af465177a96a29601e56299afc044060dbc2ac3061a59032be451cfde", "test/built-ins/String/prototype/trim/15.5.4.20-2-50.js": "38b84997faa99e422408917eeac7f3575bd6064b429ea390a9d041fd77fce532", "test/built-ins/String/prototype/trim/15.5.4.20-2-6.js": "3d5857f24117f65f07a70edd0d254dfa07f593c76effed94500847ccf8564c06", + "test/built-ins/TypedArray/prototype/byteOffset/return-byteoffset.js": "04cd6e31c8223319bbc6f5e5630ee476708a3897ead70e9cec34d40fb3ebd062", + "test/built-ins/TypedArray/prototype/subarray/return-abrupt-from-begin-symbol.js": "51fd475317d19c863a85c26f1ffa2070b10abf123b49a3243bb3829218b0b7a9", + "test/built-ins/TypedArrayConstructors/ctors/buffer-arg/byteoffset-is-negative-zero-sab.js": "96741e885f333125077884cb59c23c297fad61a4834941a5ac38720227094d7d", + "test/built-ins/TypedArrayConstructors/ctors/length-arg/is-infinity-throws-rangeerror.js": "ea340740e8fb3227b19fff023508c38d3bd6b2ff45f7a623d1e4460fad9b32c1", + "test/built-ins/TypedArrayConstructors/ctors/typedarray-arg/undefined-newtarget-throws.js": "19d2ebe0d3db51eb9f978b9b2462bb38ff5cf4702f32e3163d8be46e851d0f8f", + "test/built-ins/TypedArrayConstructors/internals/DefineOwnProperty/key-is-numericindex-desc-not-configurable-throws.js": "711cf2873e49c9d48b3246f8dd067380ac902d286907b620735f5f4dbd36c943", "test/language/arguments-object/func-decl-args-trailing-comma-null.js": "80aecea8edd0d7bac1ddcfcd6cb97dc9a513b25a9283bb9cedd21fd77dd64d50", "test/language/arguments-object/func-expr-args-trailing-comma-undefined.js": "cf39515a9031db4c81db231a157516bdc97617c392e4ef28890b2ae74777a99e", "test/language/expressions/addition/S11.6.1_A4_T1.js": "12928eb319dcf0cc785e399580effb0b7495709296dce9c0a58e8408616a959f", diff --git a/tests/test262/vendor/harness/testTypedArray.js b/tests/test262/vendor/harness/testTypedArray.js new file mode 100644 index 000000000..a6418c2f7 --- /dev/null +++ b/tests/test262/vendor/harness/testTypedArray.js @@ -0,0 +1,431 @@ +// Copyright (C) 2015 André Bargull. All rights reserved. +// This code is governed by the BSD license found in the LICENSE file. +/*--- +description: | + Collection of functions used to assert the correctness of TypedArray objects. +defines: + - floatArrayConstructors + - nonClampedIntArrayConstructors + - intArrayConstructors + - typedArrayConstructors + - TypedArray + - testWithAllTypedArrayConstructors + - testWithTypedArrayConstructors + - testWithBigIntTypedArrayConstructors + - nonAtomicsFriendlyTypedArrayConstructors + - testWithAtomicsFriendlyTypedArrayConstructors + - testWithNonAtomicsFriendlyTypedArrayConstructors + - testTypedArrayConversions +---*/ + +var floatArrayConstructors = [ + Float64Array, + Float32Array +]; + +var nonClampedIntArrayConstructors = [ + Int32Array, + Int16Array, + Int8Array, + Uint32Array, + Uint16Array, + Uint8Array +]; + +var intArrayConstructors = nonClampedIntArrayConstructors.concat([Uint8ClampedArray]); + +// Float16Array is a newer feature +// adding it to this list unconditionally would cause implementations lacking it to fail every test which uses it +if (typeof Float16Array !== "undefined") { + floatArrayConstructors.push(Float16Array); +} + +var bigIntArrayConstructors = []; +if (typeof BigInt64Array !== "undefined") { + bigIntArrayConstructors.push(BigInt64Array); +} +if (typeof BigUint64Array !== "undefined") { + bigIntArrayConstructors.push(BigUint64Array); +} + +/** + * Array containing every non-bigint typed array constructor. + */ +var typedArrayConstructors = floatArrayConstructors.concat(intArrayConstructors); + +/** + * Array containing every typed array constructor, including those with bigint values. + */ +var allTypedArrayConstructors = typedArrayConstructors.concat(bigIntArrayConstructors); + +/** + * The %TypedArray% intrinsic constructor function. + */ +var TypedArray = Object.getPrototypeOf(Int8Array); + +function makePassthrough(TA, primitiveOrIterable) { + return primitiveOrIterable; +} + +function makeArray(TA, primitiveOrIterable) { + if (isPrimitive(primitiveOrIterable)) { + var n = Number(primitiveOrIterable); + // Only values between 0 and 2**53 - 1 inclusive can get mapped into TA contents. + if (!(n >= 0 && n < 9007199254740992)) return primitiveOrIterable; + return Array.from({ length: n }, function() { return "0"; }); + } + return Array.from(primitiveOrIterable); +} + +function makeArrayLike(TA, primitiveOrIterable) { + var arr = makeArray(TA, primitiveOrIterable); + if (isPrimitive(arr)) return arr; + var obj = { length: arr.length }; + for (var i = 0; i < obj.length; i++) obj[i] = arr[i]; + return obj; +} + +var makeIterable; +if (typeof Symbol !== "undefined" && Symbol.iterator) { + makeIterable = function makeIterable(TA, primitiveOrIterable) { + var src = makeArray(TA, primitiveOrIterable); + if (isPrimitive(src)) return src; + var obj = {}; + obj[Symbol.iterator] = function() { return src[Symbol.iterator](); }; + return obj; + }; +} + +function makeArrayBuffer(TA, primitiveOrIterable) { + var arr = makeArray(TA, primitiveOrIterable); + if (isPrimitive(arr)) return arr; + return new TA(arr).buffer; +} + +var makeResizableArrayBuffer, makeGrownArrayBuffer, makeShrunkArrayBuffer, makeImmutableArrayBuffer; +if (ArrayBuffer.prototype.resize) { + var copyIntoArrayBuffer = function(destBuffer, srcBuffer) { + var destView = new Uint8Array(destBuffer); + var srcView = new Uint8Array(srcBuffer); + for (var i = 0; i < srcView.length; i++) destView[i] = srcView[i]; + return destBuffer; + }; + + makeResizableArrayBuffer = function makeResizableArrayBuffer(TA, primitiveOrIterable) { + if (isPrimitive(primitiveOrIterable)) { + var n = Number(primitiveOrIterable) * TA.BYTES_PER_ELEMENT; + if (!(n >= 0 && n < 9007199254740992)) return primitiveOrIterable; + return new ArrayBuffer(n, { maxByteLength: n * 2 }); + } + var fixed = makeArrayBuffer(TA, primitiveOrIterable); + var byteLength = fixed.byteLength; + var resizable = new ArrayBuffer(byteLength, { maxByteLength: byteLength * 2 }); + return copyIntoArrayBuffer(resizable, fixed); + }; + + makeGrownArrayBuffer = function makeGrownArrayBuffer(TA, primitiveOrIterable) { + if (isPrimitive(primitiveOrIterable)) { + var n = Number(primitiveOrIterable) * TA.BYTES_PER_ELEMENT; + if (!(n >= 0 && n < 9007199254740992)) return primitiveOrIterable; + var grown = new ArrayBuffer(Math.floor(n / 2), { maxByteLength: n }); + grown.resize(n); + } + var fixed = makeArrayBuffer(TA, primitiveOrIterable); + var byteLength = fixed.byteLength; + var grown = new ArrayBuffer(Math.floor(byteLength / 2), { maxByteLength: byteLength }); + grown.resize(byteLength); + return copyIntoArrayBuffer(grown, fixed); + }; + + makeShrunkArrayBuffer = function makeShrunkArrayBuffer(TA, primitiveOrIterable) { + if (isPrimitive(primitiveOrIterable)) { + var n = Number(primitiveOrIterable) * TA.BYTES_PER_ELEMENT; + if (!(n >= 0 && n < 9007199254740992)) return primitiveOrIterable; + var shrunk = new ArrayBuffer(n * 2, { maxByteLength: n * 2 }); + shrunk.resize(n); + } + var fixed = makeArrayBuffer(TA, primitiveOrIterable); + var byteLength = fixed.byteLength; + var shrunk = new ArrayBuffer(byteLength * 2, { maxByteLength: byteLength * 2 }); + copyIntoArrayBuffer(shrunk, fixed); + shrunk.resize(byteLength); + return shrunk; + }; +} +if (ArrayBuffer.prototype.transferToImmutable) { + makeImmutableArrayBuffer = function makeImmutableArrayBuffer(TA, primitiveOrIterable) { + if (isPrimitive(primitiveOrIterable)) { + var n = Number(primitiveOrIterable) * TA.BYTES_PER_ELEMENT; + if (!(n >= 0 && n < 9007199254740992)) return primitiveOrIterable; + return (new ArrayBuffer(n)).transferToImmutable(); + } + var mutable = makeArrayBuffer(TA, primitiveOrIterable); + return mutable.transferToImmutable(); + }; +} + +var typedArrayCtorArgFactories = [makePassthrough, makeArray, makeArrayLike]; +if (makeIterable) typedArrayCtorArgFactories.push(makeIterable); +typedArrayCtorArgFactories.push(makeArrayBuffer); +if (makeResizableArrayBuffer) typedArrayCtorArgFactories.push(makeResizableArrayBuffer); +if (makeGrownArrayBuffer) typedArrayCtorArgFactories.push(makeGrownArrayBuffer); +if (makeShrunkArrayBuffer) typedArrayCtorArgFactories.push(makeShrunkArrayBuffer); +if (makeImmutableArrayBuffer) typedArrayCtorArgFactories.push(makeImmutableArrayBuffer); + +/** + * @typedef {"passthrough" | "arraylike" | "iterable" | "arraybuffer" | "resizable" | "immutable"} typedArrayArgFactoryFeature + */ + +/** + * A predicate for testing whether a TypedArray argument factory from this file + * matches any of the provided features. + * + * @param {Function} argFactory + * @param {typedArrayArgFactoryFeature[]} features + * @returns {boolean} + */ +function ctorArgFactoryMatchesSome(argFactory, features) { + for (var i = 0; i < features.length; ++i) { + switch (features[i]) { + case "passthrough": + if (argFactory === makePassthrough) return true; + break; + case "arraylike": + if (argFactory === makeArray || argFactory === makeArrayLike) return true; + break; + case "iterable": + if (argFactory === makeIterable) return true; + break; + case "arraybuffer": + if ( + argFactory === makeArrayBuffer || + argFactory === makeResizableArrayBuffer || + argFactory === makeGrownArrayBuffer || + argFactory === makeShrunkArrayBuffer || + argFactory === makeImmutableArrayBuffer + ) { + return true; + } + break; + case "resizable": + if ( + argFactory === makeResizableArrayBuffer || + argFactory === makeGrownArrayBuffer || + argFactory === makeShrunkArrayBuffer + ) { + return true; + } + break; + case "immutable": + if (argFactory === makeImmutableArrayBuffer) return true; + break; + default: + throw Test262Error("unknown feature: " + features[i]); + } + } + return false; +} + +/** + * Callback for testing a typed array constructor. + * + * @callback typedArrayConstructorCallback + * @param {Function} TypedArrayConstructor the constructor object to test with + * @param {Function} [TypedArrayConstructorArgFactory] a function for making + * a TypedArrayConstructor argument from a primitive (usually a number) or + * iterable (usually an array) + */ + +/** + * Calls the provided function with (typedArrayCtor, typedArrayCtorArgFactory) + * pairs, where typedArrayCtor is Uint8Array/Int8Array/BigInt64Array/etc. and + * typedArrayCtorArgFactory is a function for mapping a primitive (usually a + * number) or iterable (usually an array) into a value suitable as the first + * argument of typedArrayCtor (an Array, arraylike, iterable, or ArrayBuffer). + * + * @param {typedArrayConstructorCallback} f - the function to call + * @param {Array} [constructors] - an explicit list of TypedArray constructors + * @param {typedArrayArgFactoryFeature[]} [includeArgFactories] - for selecting + * initial constructor argument factory functions, rather than starting with + * all argument factories + * @param {typedArrayArgFactoryFeature[]} [excludeArgFactories] - for excluding + * constructor argument factory functions, after an initial selection + */ +function testWithAllTypedArrayConstructors(f, constructors, includeArgFactories, excludeArgFactories) { + var ctors = constructors || allTypedArrayConstructors; + var ctorArgFactories = typedArrayCtorArgFactories; + if (includeArgFactories) { + ctorArgFactories = []; + for (var i = 0; i < typedArrayCtorArgFactories.length; ++i) { + if (ctorArgFactoryMatchesSome(typedArrayCtorArgFactories[i], includeArgFactories)) { + ctorArgFactories.push(typedArrayCtorArgFactories[i]); + } + } + } + if (excludeArgFactories) { + ctorArgFactories = ctorArgFactories.slice(); + for (var i = ctorArgFactories.length - 1; i >= 0; --i) { + if (ctorArgFactoryMatchesSome(ctorArgFactories[i], excludeArgFactories)) { + ctorArgFactories.splice(i, 1); + } + } + } + if (ctorArgFactories.length === 0) { + throw Test262Error("no arg factories match include " + includeArgFactories + " and exclude " + excludeArgFactories); + } + for (var k = 0; k < ctorArgFactories.length; ++k) { + var argFactory = ctorArgFactories[k]; + for (var i = 0; i < ctors.length; ++i) { + var constructor = ctors[i]; + var boundArgFactory = argFactory.bind(undefined, constructor); + try { + f(constructor, boundArgFactory); + } catch (e) { + e.message += " (Testing with " + constructor.name + " and " + argFactory.name + ".)"; + throw e; + } + } + } +} + +/** + * Calls the provided function with (typedArrayCtor, typedArrayCtorArgFactory) + * pairs, where typedArrayCtor is Uint8Array/Int8Array/BigInt64Array/etc. and + * typedArrayCtorArgFactory is a function for mapping a primitive (usually a + * number) or iterable (usually an array) into a value suitable as the first + * argument of typedArrayCtor (an Array, arraylike, iterable, or ArrayBuffer). + * + * typedArrayCtor will not be BigInt64Array or BigUint64Array unless one or both + * of those are explicitly provided. + * + * @param {typedArrayConstructorCallback} f - the function to call + * @param {Array} [constructors] - an explicit list of TypedArray constructors + * @param {typedArrayArgFactoryFeature[]} [includeArgFactories] - for selecting + * initial constructor argument factory functions, rather than starting with + * all argument factories + * @param {typedArrayArgFactoryFeature[]} [excludeArgFactories] - for excluding + * constructor argument factory functions, after an initial selection + */ +function testWithTypedArrayConstructors(f, constructors, includeArgFactories, excludeArgFactories) { + var ctors = constructors || typedArrayConstructors; + testWithAllTypedArrayConstructors(f, ctors, includeArgFactories, excludeArgFactories); +} + +/** + * Calls the provided function for every BigInt typed array constructor. + * + * @param {typedArrayConstructorCallback} f - the function to call + * @param {Array} [constructors] - an explicit list of TypedArray constructors + * @param {typedArrayArgFactoryFeature[]} [includeArgFactories] - for selecting + * initial constructor argument factory functions, rather than starting with + * all argument factories + * @param {typedArrayArgFactoryFeature[]} [excludeArgFactories] - for excluding + * constructor argument factory functions, after an initial selection + */ +function testWithBigIntTypedArrayConstructors(f, constructors, includeArgFactories, excludeArgFactories) { + var ctors = constructors || [BigInt64Array, BigUint64Array]; + testWithAllTypedArrayConstructors(f, ctors, includeArgFactories, excludeArgFactories); +} + +var nonAtomicsFriendlyTypedArrayConstructors = floatArrayConstructors.concat([Uint8ClampedArray]); +/** + * Calls the provided function for every non-"Atomics Friendly" typed array constructor. + * + * @param {typedArrayConstructorCallback} f - the function to call for each typed array constructor. + * @param {Array} selected - An optional Array with filtered typed arrays + * @param {typedArrayArgFactoryFeature[]} [includeArgFactories] - for selecting + * initial constructor argument factory functions, rather than starting with + * all argument factories + * @param {typedArrayArgFactoryFeature[]} [excludeArgFactories] - for excluding + * constructor argument factory functions, after an initial selection + */ +function testWithNonAtomicsFriendlyTypedArrayConstructors(f, includeArgFactories, excludeArgFactories) { + testWithAllTypedArrayConstructors( + f, + nonAtomicsFriendlyTypedArrayConstructors, + includeArgFactories, + excludeArgFactories + ); +} + +/** + * Calls the provided function for every "Atomics Friendly" typed array constructor. + * + * @param {typedArrayConstructorCallback} f - the function to call for each typed array constructor. + * @param {Array} selected - An optional Array with filtered typed arrays + * @param {typedArrayArgFactoryFeature[]} [includeArgFactories] - for selecting + * initial constructor argument factory functions, rather than starting with + * all argument factories + * @param {typedArrayArgFactoryFeature[]} [excludeArgFactories] - for excluding + * constructor argument factory functions, after an initial selection + */ +function testWithAtomicsFriendlyTypedArrayConstructors(f, includeArgFactories, excludeArgFactories) { + testWithAllTypedArrayConstructors( + f, + [ + Int32Array, + Int16Array, + Int8Array, + Uint32Array, + Uint16Array, + Uint8Array, + ], + includeArgFactories, + excludeArgFactories + ); +} + +/** + * Helper for conversion operations on TypedArrays, the expected values + * properties are indexed in order to match the respective value for each + * TypedArray constructor + * @param {Function} fn - the function to call for each constructor and value. + * will be called with the constructor, value, expected + * value, and a initial value that can be used to avoid + * a false positive with an equivalent expected value. + */ +function testTypedArrayConversions(byteConversionValues, fn) { + var values = byteConversionValues.values; + var expected = byteConversionValues.expected; + + testWithTypedArrayConstructors(function(TA) { + var name = TA.name.slice(0, -5); + + return values.forEach(function(value, index) { + var exp = expected[name][index]; + var initial = 0; + if (exp === 0) { + initial = 1; + } + fn(TA, value, exp, initial); + }); + }, null, ["passthrough"]); +} + +/** + * Checks if the given argument is one of the float-based TypedArray constructors. + * + * @param {constructor} ctor - the value to check + * @returns {boolean} + */ +function isFloatTypedArrayConstructor(arg) { + return floatArrayConstructors.indexOf(arg) !== -1; +} + +/** + * Determines the precision of the given float-based TypedArray constructor. + * + * @param {constructor} ctor - the value to check + * @returns {string} "half", "single", or "double" for Float16Array, Float32Array, and Float64Array respectively. + */ +function floatTypedArrayConstructorPrecision(FA) { + if (typeof Float16Array !== "undefined" && FA === Float16Array) { + return "half"; + } else if (FA === Float32Array) { + return "single"; + } else if (FA === Float64Array) { + return "double"; + } else { + throw new Error("Malformed test - floatTypedArrayConstructorPrecision called with non-float TypedArray"); + } +} diff --git a/tests/test262/vendor/test/built-ins/TypedArray/prototype/byteOffset/return-byteoffset.js b/tests/test262/vendor/test/built-ins/TypedArray/prototype/byteOffset/return-byteoffset.js new file mode 100644 index 000000000..cbca8c3ec --- /dev/null +++ b/tests/test262/vendor/test/built-ins/TypedArray/prototype/byteOffset/return-byteoffset.js @@ -0,0 +1,31 @@ +// Copyright (C) 2016 the V8 project authors. All rights reserved. +// This code is governed by the BSD license found in the LICENSE file. +/*--- +esid: sec-get-%typedarray%.prototype.byteoffset +description: > + Return value from [[ByteOffset]] internal slot +info: | + 22.2.3.3 get %TypedArray%.prototype.byteOffset + + ... + 6. Let offset be the value of O's [[ByteOffset]] internal slot. + 7. Return size. +includes: [testTypedArray.js] +features: [TypedArray] +---*/ + +testWithTypedArrayConstructors(function(TA, makeCtorArg) { + var ta1 = new TA(); + assert.sameValue(ta1.byteOffset, 0, "Regular typedArray"); + + var offset = 4 * TA.BYTES_PER_ELEMENT; + + var buffer1 = makeCtorArg(8); + var ta2 = new TA(buffer1, offset); + assert.sameValue(ta2.byteOffset, offset, "TA(buffer, offset)"); + + var buffer2 = makeCtorArg(8); + var sample = new TA(buffer2, offset); + var ta3 = new TA(sample); + assert.sameValue(ta3.byteOffset, 0, "TA(typedArray)"); +}, null, ["arraybuffer"]); diff --git a/tests/test262/vendor/test/built-ins/TypedArray/prototype/subarray/return-abrupt-from-begin-symbol.js b/tests/test262/vendor/test/built-ins/TypedArray/prototype/subarray/return-abrupt-from-begin-symbol.js new file mode 100644 index 000000000..1e54741d3 --- /dev/null +++ b/tests/test262/vendor/test/built-ins/TypedArray/prototype/subarray/return-abrupt-from-begin-symbol.js @@ -0,0 +1,24 @@ +// Copyright (C) 2016 the V8 project authors. All rights reserved. +// This code is governed by the BSD license found in the LICENSE file. +/*--- +esid: sec-%typedarray%.prototype.subarray +description: Return abrupt from ToInteger(begin), begin is symbol +info: | + 22.2.3.27 %TypedArray%.prototype.subarray( begin , end ) + + ... + 7. Let relativeBegin be ? ToInteger(begin). + ... +includes: [testTypedArray.js] +features: [Symbol, TypedArray] +---*/ + +var s = Symbol("1"); + +testWithTypedArrayConstructors(function(TA, makeCtorArg) { + var sample = new TA(makeCtorArg(0)); + + assert.throws(TypeError, function() { + sample.subarray(s); + }); +}); diff --git a/tests/test262/vendor/test/built-ins/TypedArrayConstructors/ctors/buffer-arg/byteoffset-is-negative-zero-sab.js b/tests/test262/vendor/test/built-ins/TypedArrayConstructors/ctors/buffer-arg/byteoffset-is-negative-zero-sab.js new file mode 100644 index 000000000..b4bf00149 --- /dev/null +++ b/tests/test262/vendor/test/built-ins/TypedArrayConstructors/ctors/buffer-arg/byteoffset-is-negative-zero-sab.js @@ -0,0 +1,23 @@ +// Copyright (C) 2015 André Bargull. All rights reserved. +// This code is governed by the BSD license found in the LICENSE file. + +/*--- +esid: pending +description: > + TypedArray's [[ByteOffset]] internal slot is always a positive number, test with negative zero. +info: | + %TypedArray% ( buffer [ , byteOffset [ , length ] ] ) + + ... + 6. Let offset be ? ToInteger(byteOffset). + 7. If offset < 0, throw a RangeError exception. + 8. If offset is -0, let offset be +0. + ... +includes: [testTypedArray.js] +features: [SharedArrayBuffer, TypedArray] +---*/ + +testWithTypedArrayConstructors(function(TAConstructor) { + var typedArray = new TAConstructor(new SharedArrayBuffer(8), -0); + assert.sameValue(typedArray.byteOffset, +0); +}, null, ["passthrough"]); diff --git a/tests/test262/vendor/test/built-ins/TypedArrayConstructors/ctors/length-arg/is-infinity-throws-rangeerror.js b/tests/test262/vendor/test/built-ins/TypedArrayConstructors/ctors/length-arg/is-infinity-throws-rangeerror.js new file mode 100644 index 000000000..c04f79f76 --- /dev/null +++ b/tests/test262/vendor/test/built-ins/TypedArrayConstructors/ctors/length-arg/is-infinity-throws-rangeerror.js @@ -0,0 +1,27 @@ +// Copyright (C) 2016 the V8 project authors. All rights reserved. +// This code is governed by the BSD license found in the LICENSE file. +/*--- +esid: sec-typedarray-length +description: > + Throws a RangeError if length is a Infinity value +info: | + 22.2.4.2 TypedArray ( length ) + + This description applies only if the TypedArray function is called with at + least one argument and the Type of the first argument is not Object. + + ... + 4. Let numberLength be ? ToNumber(length). + 5. Let elementLength be ToLength(numberLength). + 6. If SameValueZero(numberLength, elementLength) is false, throw a RangeError + exception. + ... +includes: [testTypedArray.js] +features: [TypedArray] +---*/ + +testWithTypedArrayConstructors(function(TA) { + assert.throws(RangeError, function() { + new TA(Infinity); + }); +}, null, ["passthrough"]); diff --git a/tests/test262/vendor/test/built-ins/TypedArrayConstructors/ctors/typedarray-arg/undefined-newtarget-throws.js b/tests/test262/vendor/test/built-ins/TypedArrayConstructors/ctors/typedarray-arg/undefined-newtarget-throws.js new file mode 100644 index 000000000..e96b6d1d4 --- /dev/null +++ b/tests/test262/vendor/test/built-ins/TypedArrayConstructors/ctors/typedarray-arg/undefined-newtarget-throws.js @@ -0,0 +1,27 @@ +// Copyright (C) 2016 the V8 project authors. All rights reserved. +// This code is governed by the BSD license found in the LICENSE file. +/*--- +esid: sec-typedarray-typedarray +description: > + Throws a TypeError if NewTarget is undefined. +info: | + 22.2.4.3 TypedArray ( typedArray ) + + This description applies only if the TypedArray function is called with at + least one argument and the Type of the first argument is Object and that + object has a [[TypedArrayName]] internal slot. + + ... + 2. If NewTarget is undefined, throw a TypeError exception. + ... +includes: [testTypedArray.js] +features: [TypedArray] +---*/ + +testWithTypedArrayConstructors(function(TA, makeCtorArg) { + var typedArray = new TA(makeCtorArg(4)); + + assert.throws(TypeError, function() { + TA(typedArray); + }); +}); diff --git a/tests/test262/vendor/test/built-ins/TypedArrayConstructors/internals/DefineOwnProperty/key-is-numericindex-desc-not-configurable-throws.js b/tests/test262/vendor/test/built-ins/TypedArrayConstructors/internals/DefineOwnProperty/key-is-numericindex-desc-not-configurable-throws.js new file mode 100644 index 000000000..898cf1abc --- /dev/null +++ b/tests/test262/vendor/test/built-ins/TypedArrayConstructors/internals/DefineOwnProperty/key-is-numericindex-desc-not-configurable-throws.js @@ -0,0 +1,39 @@ +// Copyright (C) 2021 Alexey Shvayka. All rights reserved. +// This code is governed by the BSD license found in the LICENSE file. +/*--- +esid: sec-integer-indexed-exotic-objects-defineownproperty-p-desc +description: > + Throws TypeError for valid index & non-configurable descriptor. +info: | + [[DefineOwnProperty]] ( P, Desc ) + + [...] + 3. If Type(P) is String, then + a. Let numericIndex be ! CanonicalNumericIndexString(P). + b. If numericIndex is not undefined, then + [...] + ii. If Desc has a [[Configurable]] field and if Desc.[[Configurable]] is false, return false. +includes: [testTypedArray.js] +features: [align-detached-buffer-semantics-with-web-reality, TypedArray] +---*/ + +testWithTypedArrayConstructors(function(TA, makeCtorArg) { + var sample = new TA(makeCtorArg([0])); + + assert.throws(TypeError, function() { + Object.defineProperty(sample, "0", { + configurable: false, + }); + }, "partial descriptor"); + + assert.throws(TypeError, function() { + Object.defineProperty(sample, "0", { + value: 42, + writable: true, + enumerable: true, + configurable: false, + }); + }, "complete descriptor"); + + assert.sameValue(sample[0], 0, "side effect check"); +}, null, ["passthrough"]); From 46b00f69c97629ba4df012f30583a3d5262e1b66 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 14:30:29 -0500 Subject: [PATCH 094/184] Give glibc's per-thread heaps huge pages Blocks above 512 bytes come from glibc, which serves each worker thread from a 64 MiB heap of its own and makes it accessible with mprotect a page-rounded step at a time. Every 2 MiB range of a heap is first touched while part of it is still inaccessible, so the heaps never get transparent huge pages: on a 16-checker type check of VS Code they hold about 0.9 GB on small pages, close to half of the process's page faults. Once a worker thread has allocated 16 MiB of large blocks, the heap of each large block it gets is made accessible whole and advised MADV_HUGEPAGE, once per heap. glibc's calloc still clears only what lies below its own high-water mark, so no page is touched early. glibc's mmap and trim thresholds are set to 4 MiB and 8 MiB, so that the largest blocks get mappings of their own and a heap's free top goes back to the system: with huge pages, a heap otherwise keeps resident memory it never writes. The brk heap policy's own thresholds take precedence once it switches on. The policy is off unless THP is in madvise mode and malloc is glibc's, and when the glibc.malloc.hugetlb tunable is set or the system forbids overcommit. A heap is advised only when /proc/self/maps shows anonymous mappings covering exactly its range. Library archives leave their embedder's heaps alone. --- packages/runtime/src/scr_alloc.c | 101 ++++++++++++++++++++++++++ packages/runtime/src/scr_alloc.test.c | 53 ++++++++++++++ 2 files changed, 154 insertions(+) diff --git a/packages/runtime/src/scr_alloc.c b/packages/runtime/src/scr_alloc.c index 3a36f672e..f3d6c0893 100644 --- a/packages/runtime/src/scr_alloc.c +++ b/packages/runtime/src/scr_alloc.c @@ -567,7 +567,108 @@ static _Atomic(uintptr_t) scr_sa_heap_page; /* the page size, read with the mode * These loads run only on blocks too large for the small allocator, next to * a glibc malloc call: plain loads on x86-64, LDAR (or LDAPR) on arm64. */ +#if defined(SCR_WORKERS) && !defined(SCR_LIB) +/* glibc's per-thread heaps (worker executables; library archives run on + * their embedder's threads and leave its heaps alone). glibc serves a thread's + * blocks from a heap of its own: a 64 MiB mapping, aligned to its size, + * that it makes accessible with mprotect as it grows, a page-rounded step + * at a time, so every 2 MiB range is first touched while part of it is + * still inaccessible and gets small pages. Once a thread has allocated + * SCR_SA_THREAD_HUGE_AFTER bytes of large blocks (threads with few keep + * small pages), the heap of each large block it gets is made accessible + * whole and advised huge pages, once per heap; glibc's calloc still clears + * only what lies below its own high-water mark, so this touches nothing. + * glibc's mmap and trim thresholds are fixed at SCR_SA_THREAD_MMAP_THRESHOLD + * and SCR_SA_THREAD_TRIM_THRESHOLD: lower than the maxima glibc's adaptive + * thresholds reach, so that blocks of 4 MiB and more get mappings of their + * own and a heap's free top goes back once it passes 8 MiB, while huge + * pages keep resident memory that a heap would otherwise leave untouched. + * The brk heap policy below sets its own (higher) values when the main heap + * grows past SCR_SA_HEAP_HUGE_AFTER, and those take precedence. The + * policy is off unless THP is in madvise mode, when the glibc.malloc.hugetlb + * tunable is set (glibc then advises its heaps itself, or sizes them by + * hugetlbfs pages), when the system forbids overcommit (making a heap + * accessible would commit all of it), and when malloc is not glibc's (a + * sanitizer or a preloaded allocator: blocks have no glibc chunk header). + * A heap is only advised when /proc/self/maps shows the anonymous mappings + * of exactly its range. */ +#define SCR_SA_GLIBC_HEAP ((uintptr_t)64 << 20) /* glibc's heap size and alignment */ +#define SCR_SA_THREAD_HUGE_AFTER ((size_t)16 << 20) +#define SCR_SA_THREAD_MMAP_THRESHOLD (4 << 20) +#define SCR_SA_THREAD_TRIM_THRESHOLD (8 << 20) +static _Thread_local size_t scr_sa_thread_large_bytes; +static _Thread_local uintptr_t scr_sa_thread_heap; /* the heap this thread advised last */ +static _Atomic(int) scr_sa_thread_heaps; /* 0 undecided, 1 advised, 2 off */ +extern void *__libc_malloc(size_t); + +/* Whether [heap, heap + SCR_SA_GLIBC_HEAP) is covered exactly by anonymous + * mappings, as a glibc heap is (its accessible part and the rest). */ +static bool scr_sa_glibc_heap_mapped(uintptr_t heap) { + FILE *f = fopen("/proc/self/maps", "r"); + if (!f) return false; + char line[512]; + uintptr_t next = heap, end = heap + SCR_SA_GLIBC_HEAP; + while (next < end && fgets(line, sizeof line, f)) { + unsigned long lo, hi, inode; + char path[2] = {0}; + if (sscanf(line, "%lx-%lx %*s %*s %*s %lu %1s", &lo, &hi, &inode, path) < 3) continue; + if (hi <= next) continue; + if (lo != next || inode != 0 || path[0]) break; + next = hi; + } + fclose(f); + return next == end; +} + +static void scr_sa_thread_heap_note(void *p, size_t n) { + if (scr_sa_thread_large_bytes < SCR_SA_THREAD_HUGE_AFTER) { + scr_sa_thread_large_bytes += n; + return; + } + int mode = atomic_load_explicit(&scr_sa_thread_heaps, memory_order_relaxed); + if (mode == 2) return; + if (mode == 0) { + int undecided = 0; + const char *tunables = getenv("GLIBC_TUNABLES"); + FILE *f = fopen("/proc/sys/vm/overcommit_memory", "r"); + int overcommit = f ? fgetc(f) : '0'; + if (f) fclose(f); + /* Loaded through volatile: the compiler takes two declared functions + * to have different addresses. */ + void *(*volatile used)(size_t) = malloc, *(*volatile own)(size_t) = __libc_malloc; + bool glibc_malloc = used == own; + mode = glibc_malloc && scr_sa_thp_mode() == 'm' && !(tunables && strstr(tunables, "glibc.malloc.hugetlb")) && + overcommit != '2' + ? 1 + : 2; + if (atomic_compare_exchange_strong(&scr_sa_thread_heaps, &undecided, mode) && mode == 1 && + atomic_load_explicit(&scr_sa_heap_state, memory_order_relaxed) != 1) { + (void)mallopt(M_MMAP_THRESHOLD, SCR_SA_THREAD_MMAP_THRESHOLD); + (void)mallopt(M_TRIM_THRESHOLD, SCR_SA_THREAD_TRIM_THRESHOLD); + /* The brk heap policy marks itself on before it sets its values: if it + * did so meanwhile, its values are set again. */ + if (atomic_load(&scr_sa_heap_state) == 1) { + (void)mallopt(M_MMAP_THRESHOLD, SCR_SA_HEAP_MMAP_THRESHOLD); + (void)mallopt(M_TRIM_THRESHOLD, SCR_SA_HEAP_TRIM_THRESHOLD); + } + } + mode = atomic_load_explicit(&scr_sa_thread_heaps, memory_order_relaxed); + if (mode != 1) return; + } + /* glibc's chunk header: NON_MAIN_ARENA (4) set, IS_MMAPPED (2) clear. */ + if ((((const size_t *)p)[-1] & 6) != 4) return; + uintptr_t heap = (uintptr_t)p & ~(SCR_SA_GLIBC_HEAP - 1); + if (heap == scr_sa_thread_heap) return; + scr_sa_thread_heap = heap; + if (scr_sa_glibc_heap_mapped(heap) && mprotect((void *)heap, SCR_SA_GLIBC_HEAP, PROT_READ | PROT_WRITE) == 0) + (void)madvise((void *)heap, SCR_SA_GLIBC_HEAP, MADV_HUGEPAGE); +} +#endif + static void scr_sa_large_note(void *p, size_t n) { +#if defined(SCR_WORKERS) && !defined(SCR_LIB) + if (p) scr_sa_thread_heap_note(p, n); +#endif int state = atomic_load_explicit(&scr_sa_heap_state, memory_order_relaxed); if (state == 2 || !p) return; uintptr_t top = (uintptr_t)sbrk(0); diff --git a/packages/runtime/src/scr_alloc.test.c b/packages/runtime/src/scr_alloc.test.c index e20c0733b..6ccb1495b 100644 --- a/packages/runtime/src/scr_alloc.test.c +++ b/packages/runtime/src/scr_alloc.test.c @@ -343,6 +343,56 @@ static void *reclaim_arena(void *arg) { return NULL; } +#if defined(__linux__) && defined(__GLIBC__) +/* A thread whose large blocks pass 16 MiB gets them from a glibc heap of + * its own, which is then advised huge pages where THP is in madvise mode. */ +enum { HEAP_BLOCKS = 320, HEAP_BLOCK = 64 << 10 }; +static void *heap_blocks[HEAP_BLOCKS]; +static char heap_flags[256]; +static void *thread_heap_allocate(void *arg) { + (void)arg; + for (unsigned i = 0; i < HEAP_BLOCKS; i++) { + heap_blocks[i] = scr_mem_alloc(HEAP_BLOCK); + assert(heap_blocks[i] && !owned(heap_blocks[i])); + ((unsigned char *)heap_blocks[i])[0] = (unsigned char)i; + } + vm_flags(heap_blocks[HEAP_BLOCKS - 1], heap_flags, sizeof heap_flags); + for (unsigned i = 0; i < HEAP_BLOCKS; i++) { + assert(((unsigned char *)heap_blocks[i])[0] == (unsigned char)i); + scr_mem_free(heap_blocks[i]); + } + scr_sa_thread_park(); + return NULL; +} + +static void check_thread_heap_huge_pages(void) { + pthread_t t; + assert(pthread_create(&t, NULL, thread_heap_allocate, NULL) == 0); + assert(pthread_join(t, NULL) == 0); + char mode[96] = {0}; + FILE *thp = fopen("/sys/kernel/mm/transparent_hugepage/enabled", "r"); + if (thp) { + size_t n = fread(mode, 1, sizeof mode - 1, thp); + mode[n] = '\0'; + fclose(thp); + } + FILE *f = fopen("/proc/sys/vm/overcommit_memory", "r"); + int overcommit = f ? fgetc(f) : '0'; + if (f) fclose(f); + /* ThreadSanitizer replaces malloc, which turns the policy off. */ + bool glibc_malloc = true; +#if defined(__SANITIZE_THREAD__) + glibc_malloc = false; +#elif defined(__has_feature) +#if __has_feature(thread_sanitizer) + glibc_malloc = false; +#endif +#endif + if (glibc_malloc && strstr(mode, "[madvise]") && !getenv("GLIBC_TUNABLES") && overcommit != '2') + assert(strstr(heap_flags, " hg")); +} +#endif + static void check_threads(void) { void *mine = scr_mem_alloc(48); assert(owned(mine)); @@ -1047,6 +1097,9 @@ int main(void) { setenv("SCR_SA_DISCARD_DELAY_MS", "100", 1); setenv("SCR_SA_DISCARD_DELAY_MAX_MS", "0", 1); check_threads(); +#if defined(__linux__) && defined(__GLIBC__) + check_thread_heap_huge_pages(); +#endif #ifdef __linux__ check_trim(); check_release_frontiers(); From 19d6736b01b65bfaa75f3be92f1bcd49c0acc74d Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 15:04:56 -0500 Subject: [PATCH 095/184] Look up exports whose names start with "__" by their escaped key TypeScript keys a symbol table entry whose name starts with `__` with an extra `_` (`__x` is stored as `___x`; `symbolName` removes it). Native ESM link checking looked an import up by its own spelling, so a program that imports such an export compiled to Node's startup SyntaxError: // dep.ts export const __x = 1; // main.ts import { __x } from "./dep.ts"; // The requested module './dep.ts' // does not provide an export named '__x' `escapedSymbolName`, the inverse of `symbolName`, gives a name's table key. Link checking, external type lookups, CommonJS export lookups and the `this` check of ES module exports now look names up through it. `Object.keys` over a module namespace skipped every key starting with `__` to drop TypeScript's internal names, so it also dropped these exports; it now skips only internal names and lists exports by their own names. Closes #1582. Co-authored-by: thekevinbrown <1531750+thekevinbrown@users.noreply.github.com> --- .../frontend/lowering/cjs-exports-object.ts | 2 +- .../src/frontend/lowering/lower-calls.ts | 6 ++++-- .../compiler/src/frontend/lowering/lowerer.ts | 4 ++-- packages/compiler/src/frontend/program.ts | 2 +- .../src/frontend/ts7/semantic-model.test.ts | 6 ++++++ .../src/frontend/ts7/semantic-model.ts | 6 ++++++ .../src/frontend/ts7/semantic-types.ts | 2 +- .../corpus/7306-dunder-exports/main.ts.json | 10 ++++++++++ tests/corpus/7306-dunder-exports/dep.ts | 19 ++++++++++++++++++ tests/corpus/7306-dunder-exports/facade.ts | 2 ++ tests/corpus/7306-dunder-exports/main.ts | 20 +++++++++++++++++++ tests/corpus/7306-dunder-exports/star.ts | 4 ++++ tests/corpus/7306-dunder-exports/table.cjs | 5 +++++ .../7306-dunder-exports/main.ts.json | 1 + 14 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7306-dunder-exports/main.ts.json create mode 100644 tests/corpus/7306-dunder-exports/dep.ts create mode 100644 tests/corpus/7306-dunder-exports/facade.ts create mode 100644 tests/corpus/7306-dunder-exports/main.ts create mode 100644 tests/corpus/7306-dunder-exports/star.ts create mode 100644 tests/corpus/7306-dunder-exports/table.cjs create mode 100644 tests/harness/corpus-costs/7306-dunder-exports/main.ts.json diff --git a/packages/compiler/src/frontend/lowering/cjs-exports-object.ts b/packages/compiler/src/frontend/lowering/cjs-exports-object.ts index ae8cce4ca..5e5f24e32 100644 --- a/packages/compiler/src/frontend/lowering/cjs-exports-object.ts +++ b/packages/compiler/src/frontend/lowering/cjs-exports-object.ts @@ -381,7 +381,7 @@ function esModuleExportIgnoresThis( depth: number, ): boolean { const moduleSym = lowerer.checker.getSymbolAtLocation(sf); - const exported = moduleSym?.getExports().get(name as ts.__String); + const exported = moduleSym?.getExports().get(ts.escapedSymbolName(name)); if (exported !== undefined) { const target = exported.flags & ts.SymbolFlags.Alias ? lowerer.checker.getAliasedSymbol(exported) : exported; diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 31276263d..5366ca9d0 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -8777,11 +8777,13 @@ function lowerObjectStaticCall( const moduleSymbol = source ? lowerer.checker.getSymbolAtLocation(source) : undefined; const names: string[] = []; moduleSymbol?.getExports().forEach((symbol, key) => { + // Internal names (`__export`) start with `__`; an exported name that + // does is keyed with an extra `_` (escapedSymbolName). const name = String(key); - if (name.startsWith("__") || name === "export=") return; + if ((name.startsWith("__") && !name.startsWith("___")) || name === "export=") return; const target = symbol.flags & ts.SymbolFlags.Alias ? lowerer.checker.getAliasedSymbol(symbol) : symbol; - if (target.flags & ts.SymbolFlags.Value) names.push(name); + if (target.flags & ts.SymbolFlags.Value) names.push(ts.symbolName(symbol)); }); names.sort(); const loc = locOf(call); diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index fff4a043b..7faba254f 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -4219,7 +4219,7 @@ export class Lowerer { // exact mappings, while star exports resolve to the mapped declaration // owner through externalTypeSpecifiersByFile. const moduleSymbol = this.checker.getSymbolAtLocation(sf); - const exported = moduleSymbol?.getExports().get(exportName as ts.__String); + const exported = moduleSymbol?.getExports().get(ts.escapedSymbolName(exportName)); return this.externalTypeSpecifierOfSymbol(exported, seenSymbols, seenExports); } @@ -4362,7 +4362,7 @@ export class Lowerer { cjsModuleExportSymbol(sf: ts.SourceFile, name: string): ts.Symbol | undefined { const moduleSym = this.checker.getSymbolAtLocation(sf); const exports = moduleSym?.getExports(); - const direct = exports?.get(name as ts.__String); + const direct = exports?.get(ts.escapedSymbolName(name)); if (direct) return direct; // tsgo represents `module.exports = { ... }` as one `export=` symbol // and does not duplicate the object literal's properties into the diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index 6d8ffe4c0..e92251554 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -6437,7 +6437,7 @@ function analyzeEsmNamedImportLinks( const nextSeen = new Set(seenModules); nextSeen.add(dep); const module = checker.getSymbolAtLocation(dep); - const exported = module?.getExports().get(name as ts.__String); + const exported = module?.getExports().get(ts.escapedSymbolName(name)); if (exported !== undefined) { let resolved = exported; const seenSymbols = new Set(); diff --git a/packages/compiler/src/frontend/ts7/semantic-model.test.ts b/packages/compiler/src/frontend/ts7/semantic-model.test.ts index 9d0f61a4e..bbcd53d8b 100644 --- a/packages/compiler/src/frontend/ts7/semantic-model.test.ts +++ b/packages/compiler/src/frontend/ts7/semantic-model.test.ts @@ -28,6 +28,7 @@ import { SemanticSnapshot, SemanticType, symbolNameOf, + escapedSymbolName, } from "./semantic-model.js"; import type { __String } from "./ast-types.js"; import { @@ -313,6 +314,11 @@ test("symbol tables keep escaped keys, display names, canonical context and cach const exports = parent.getExports(); expect(symbolNameOf(members.get("___x" as __String))).toBe("__x"); expect(symbolNameOf(exports.get("__x" as __String))).toBe("__x"); + // A name that starts with `__` is keyed with an extra `_`. + expect(symbolNameOf(members.get(escapedSymbolName("__x")))).toBe("__x"); + expect(escapedSymbolName("x")).toBe("x"); + expect(escapedSymbolName("_x")).toBe("_x"); + expect(escapedSymbolName("___y")).toBe("____y"); expect(parent.getMembers()).toBe(members); expect(parent.getExports()).toBe(exports); expect(parent.getParent()).toBe(first.symbol(symbol(2))); diff --git a/packages/compiler/src/frontend/ts7/semantic-model.ts b/packages/compiler/src/frontend/ts7/semantic-model.ts index d3d5fe61a..405f4775b 100644 --- a/packages/compiler/src/frontend/ts7/semantic-model.ts +++ b/packages/compiler/src/frontend/ts7/semantic-model.ts @@ -454,6 +454,12 @@ export function symbolName(symbol: SemanticSymbol): string { return name.startsWith("___") ? name.slice(1) : name; } +/** The key of the name `name` in a symbol table (`getExports()`, `getMembers()`): the name, with + * an extra `_` when it starts with `__`. The inverse of `symbolName`. */ +export function escapedSymbolName(name: string): __String { + return (name.startsWith("__") ? `_${name}` : name) as __String; +} + /** `symbolName(symbol)`, or undefined without a symbol. */ export function symbolNameOf(symbol: SemanticSymbol | null | undefined): string | undefined { return symbol === undefined || symbol === null ? undefined : symbolName(symbol); diff --git a/packages/compiler/src/frontend/ts7/semantic-types.ts b/packages/compiler/src/frontend/ts7/semantic-types.ts index ea43e3885..5a597ae4e 100644 --- a/packages/compiler/src/frontend/ts7/semantic-types.ts +++ b/packages/compiler/src/frontend/ts7/semantic-types.ts @@ -30,7 +30,7 @@ export type { SemanticSignature as Signature, SemanticSymbol as Symbol, } from "./semantic-model.js"; -export { symbolName, symbolNameOf } from "./semantic-model.js"; +export { escapedSymbolName, symbolName, symbolNameOf } from "./semantic-model.js"; export type { SemanticIndexInfo as IndexInfo, SemanticTypePredicate as TypePredicate, diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7306-dunder-exports/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7306-dunder-exports/main.ts.json new file mode 100644 index 000000000..fd14c0dfe --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7306-dunder-exports/main.ts.json @@ -0,0 +1,10 @@ +{ + "order": [ + "/tests/corpus/7306-dunder-exports/dep.ts", + "/tests/corpus/7306-dunder-exports/star.ts", + "/tests/corpus/7306-dunder-exports/facade.ts", + "/tests/corpus/7306-dunder-exports/table.cjs", + "/tests/corpus/7306-dunder-exports/main.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7306-dunder-exports/dep.ts b/tests/corpus/7306-dunder-exports/dep.ts new file mode 100644 index 000000000..6d54eb365 --- /dev/null +++ b/tests/corpus/7306-dunder-exports/dep.ts @@ -0,0 +1,19 @@ +export const __x = 1; +export const ___y = "three"; +export let __count = 0; +export function __bump(): number { + __count++; + return __count; +} +export class __Point { + a: number; + b: number; + constructor(a: number, b: number) { + this.a = a; + this.b = b; + } + sum(): number { + return this.a + this.b; + } +} +export const plain = "plain"; diff --git a/tests/corpus/7306-dunder-exports/facade.ts b/tests/corpus/7306-dunder-exports/facade.ts new file mode 100644 index 000000000..89ab6eb32 --- /dev/null +++ b/tests/corpus/7306-dunder-exports/facade.ts @@ -0,0 +1,2 @@ +export { __x, __x as __alias, ___y as renamed } from "./dep.ts"; +export * from "./star.ts"; diff --git a/tests/corpus/7306-dunder-exports/main.ts b/tests/corpus/7306-dunder-exports/main.ts new file mode 100644 index 000000000..415bc4a77 --- /dev/null +++ b/tests/corpus/7306-dunder-exports/main.ts @@ -0,0 +1,20 @@ +// Exports whose names start with "__": named imports of a const, a function, +// a class and a live let, a name with three underscores, aliased re-exports, +// names reached through `export *`, namespace member reads and keys, and +// CommonJS exports read through require. +import { __x, ___y, __count, __bump, __Point, plain } from "./dep.ts"; +import { __alias, renamed, __starred, __twice } from "./facade.ts"; +import * as facade from "./facade.ts"; +import * as depNs from "./dep.ts"; +import { createRequire } from "node:module"; + +console.log(__x, ___y, plain); +console.log(__count, __bump(), __bump(), __count); +console.log(new __Point(2, 3).sum()); +console.log(__alias, renamed, __starred, __twice(21)); +console.log(facade.__x, facade.__starred, facade.__twice(2)); +console.log(Object.keys(depNs).join(",")); + +const require = createRequire(import.meta.url); +const table = require("./table.cjs"); +console.log(table.__a, table.__hello("dunder"), table.b); diff --git a/tests/corpus/7306-dunder-exports/star.ts b/tests/corpus/7306-dunder-exports/star.ts new file mode 100644 index 000000000..6ca21d73d --- /dev/null +++ b/tests/corpus/7306-dunder-exports/star.ts @@ -0,0 +1,4 @@ +export const __starred = "from export *"; +export function __twice(n: number): number { + return n * 2; +} diff --git a/tests/corpus/7306-dunder-exports/table.cjs b/tests/corpus/7306-dunder-exports/table.cjs new file mode 100644 index 000000000..e775925ff --- /dev/null +++ b/tests/corpus/7306-dunder-exports/table.cjs @@ -0,0 +1,5 @@ +exports.__a = 10; +exports.__hello = function (name) { + return "hello " + name; +}; +exports.b = 20; diff --git a/tests/harness/corpus-costs/7306-dunder-exports/main.ts.json b/tests/harness/corpus-costs/7306-dunder-exports/main.ts.json new file mode 100644 index 000000000..0477ef4d5 --- /dev/null +++ b/tests/harness/corpus-costs/7306-dunder-exports/main.ts.json @@ -0,0 +1 @@ +{ "plain": 2400, "san": 3600 } From e75174aeea3d01e31ce4d8f2b3f2f2adeba679bd Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 15:13:03 -0500 Subject: [PATCH 096/184] Compile string-key destructuring of any values and cast-guarded Buffer aliases Co-authored-by: patlux <4481570+patlux@users.noreply.github.com> Co-authored-by: jlucaso1 <55464917+jlucaso1@users.noreply.github.com> --- .../lowering/expressions/identifiers.ts | 11 ++++ .../frontend/lowering/expressions/typeof.ts | 6 ++ .../frontend/lowering/expressions/updates.ts | 4 ++ .../src/frontend/lowering/lower-exprs.ts | 4 ++ .../src/frontend/lowering/lower-stmts.ts | 9 ++- .../compiler/src/frontend/lowering/lowerer.ts | 4 ++ .../src/frontend/lowering/surfaces.ts | 32 ++++++++++ .../7309-globalthis-buffer-aliases.ts.json | 6 ++ .../7310-any-destructure-string-keys.ts.json | 6 ++ .../any-destructure-unspelled-keys.ts.json | 6 ++ .../globalthis-buffer-alias-values.ts.json | 6 ++ .../corpus/7309-globalthis-buffer-aliases.ts | 31 ++++++++++ .../7310-any-destructure-string-keys.ts | 60 +++++++++++++++++++ .../any-destructure-unspelled-keys.ts | 12 ++++ .../globalthis-buffer-alias-values.ts | 12 ++++ .../any-destructure-unspelled-keys.ts.txt | 27 +++++++++ .../globalthis-buffer-alias-values.ts.txt | 17 ++++++ .../7309-globalthis-buffer-aliases.ts.json | 1 + .../7310-any-destructure-string-keys.ts.json | 1 + 19 files changed, 254 insertions(+), 1 deletion(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7309-globalthis-buffer-aliases.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7310-any-destructure-string-keys.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/any-destructure-unspelled-keys.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/globalthis-buffer-alias-values.ts.json create mode 100644 tests/corpus/7309-globalthis-buffer-aliases.ts create mode 100644 tests/corpus/7310-any-destructure-string-keys.ts create mode 100644 tests/diagnostics/any-destructure-unspelled-keys.ts create mode 100644 tests/diagnostics/globalthis-buffer-alias-values.ts create mode 100644 tests/harness/__snapshots__/any-destructure-unspelled-keys.ts.txt create mode 100644 tests/harness/__snapshots__/globalthis-buffer-alias-values.ts.txt create mode 100644 tests/harness/corpus-costs/7309-globalthis-buffer-aliases.ts.json create mode 100644 tests/harness/corpus-costs/7310-any-destructure-string-keys.ts.json diff --git a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts index ef0d8a2d1..b1116647a 100644 --- a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts +++ b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts @@ -744,6 +744,17 @@ export function lowerIdentifierExpr(lowerer: Lowerer, expr: ts.Identifier, loc: return { kind: "libCall", fn: "webcrypto.object", args: [], type: DYN, loc }; const nativeAlias = stdlibGlobalNameOf(lowerer, expr); if (nativeAlias === "Buffer" && expr.text !== "Buffer") { + // An alias through a cast of globalThis carries the cast's static type, + // which the identity token below does not have. + const aliasSymbol = identifierReadSymbol(lowerer, expr); + if (aliasSymbol && lowerer.castGlobalAliases.has(aliasSymbol)) { + lowerer.unsupported( + "SC1090", + expr, + "the Buffer constructor read through a cast of globalThis as a value", + "test it in a condition, read its typeof, or call its members", + ); + } return { kind: "strLit", value: "[builtin Buffer]", type: STRING, loc }; } if (lowerer.isStdlibGlobal(expr, "fetch")) { diff --git a/packages/compiler/src/frontend/lowering/expressions/typeof.ts b/packages/compiler/src/frontend/lowering/expressions/typeof.ts index cc3ac06fb..fbbc068b9 100644 --- a/packages/compiler/src/frontend/lowering/expressions/typeof.ts +++ b/packages/compiler/src/frontend/lowering/expressions/typeof.ts @@ -88,6 +88,12 @@ export function lowerTypeofExpr(lowerer: Lowerer, expr: ts.TypeOfExpression, loc if (builtinClassRefOf(lowerer, expr.expression) !== null) { return { kind: "strLit", value: "function", type: STRING, loc }; } + // The Buffer constructor and its aliases (`const B = globalThis.Buffer`, + // `(globalThis as { Buffer?: B }).Buffer`): a value read of an alias is an + // identity token, so its typeof is decided here. + if (stdlibGlobalNameOf(lowerer, expr.expression) === "Buffer") { + return { kind: "strLit", value: "function", type: STRING, loc }; + } if (ts.isPropertyAccessExpression(expr.expression)) { const member = expr.expression; const prototype = member.expression; diff --git a/packages/compiler/src/frontend/lowering/expressions/updates.ts b/packages/compiler/src/frontend/lowering/expressions/updates.ts index 04ca23a96..7cc77a2db 100644 --- a/packages/compiler/src/frontend/lowering/expressions/updates.ts +++ b/packages/compiler/src/frontend/lowering/expressions/updates.ts @@ -191,6 +191,10 @@ export function lowerPrefixUnary(lowerer: Lowerer, expr: ts.PrefixUnaryExpressio operatorUnsupported(lowerer, expr, "+", [raw.type]); } case ts.SyntaxKind.ExclamationToken: { + // `!runtimeBuffer` over an alias of the Buffer global: Node always + // defines Buffer (the condition twin is lowerCondition's). + if (lowerer.isStdlibGlobal(expr.operand, "Buffer")) + return { kind: "boolLit", value: false, type: BOOL, loc }; // `!x` is ToBoolean-then-negate: f64/string operands go through toBool. const operand = lowerer.ensureBool( deferredSlotOr( diff --git a/packages/compiler/src/frontend/lowering/lower-exprs.ts b/packages/compiler/src/frontend/lowering/lower-exprs.ts index 2bde5f2aa..c1c5ef8c8 100644 --- a/packages/compiler/src/frontend/lowering/lower-exprs.ts +++ b/packages/compiler/src/frontend/lowering/lower-exprs.ts @@ -1087,6 +1087,10 @@ export function lowerCondition(lowerer: Lowerer, expr: ts.Expression): IrExpr { lowerer.isStdlibGlobal(e.expression, "Object") ) return { kind: "boolLit", value: true, type: BOOL, loc: locOf(e) }; + // `if (runtimeBuffer)` over an alias of the Buffer global (a + // capability probe): Node always defines Buffer. + if (lowerer.isStdlibGlobal(e, "Buffer")) + return { kind: "boolLit", value: true, type: BOOL, loc: locOf(e) }; if (ts.isPropertyAccessExpression(e)) { const presence = lowerPromiseThenPresence(lowerer, e, { kind: "boolLit", diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index e66d0e855..e11f79c25 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -8749,13 +8749,20 @@ function destrSpellingOf(rhs: ts.Expression): string | null { return base === null ? null : `${base}.${e.name.text}`; } // `table[key]` and `list[0]`; V8 prints a numeric key in its canonical - // number form (`o[1.50]` reads "o[1.5]"). + // number form (`o[1.50]` reads "o[1.5]"), and a string key as a property + // name with its text decoded (`o["a.b"]` and `o[\`a.b\`]` read "o.a.b"). + // The message is a C string, so a key holding NUL keeps the refusal. if (ts.isElementAccessExpression(e) && !e.questionDotToken) { const base = destrSpellingOf(e.expression); const key = e.argumentExpression; if (base === null) return null; if (ts.isIdentifier(key)) return `${base}[${key.text}]`; if (ts.isNumericLiteral(key)) return `${base}[${String(Number(key.text))}]`; + if ( + (ts.isStringLiteral(key) || ts.isNoSubstitutionTemplateLiteral(key)) && + !key.text.includes("\0") + ) + return `${base}.${key.text}`; return null; } if (ts.isCallExpression(e) && !e.questionDotToken) { diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index fff4a043b..441b313f0 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -2937,6 +2937,10 @@ export class Lowerer { * checks resolve through this map (stdlibGlobalNameOf), declarations * emit nothing. */ readonly stdlibGlobalAliases = new Map(); + /** The stdlibGlobalAliases bound through a cast of the global object + * (`(globalThis as { Buffer?: B }).Buffer`). Their static type is the + * cast's, so their value reads are refused (identifiers.ts). */ + readonly castGlobalAliases = new Set(); /** CJS export-table ACCESSORS (`module.exports = { get path() {...} }`), * lifted lazily as module-level functions and interned per accessor * declaration: member reads call the getter (lower-exprs). */ diff --git a/packages/compiler/src/frontend/lowering/surfaces.ts b/packages/compiler/src/frontend/lowering/surfaces.ts index 44564f2f1..ed615b276 100644 --- a/packages/compiler/src/frontend/lowering/surfaces.ts +++ b/packages/compiler/src/frontend/lowering/surfaces.ts @@ -6,6 +6,20 @@ function unparenthesized(e: ts.Expression): ts.Expression { while (ts.isParenthesizedExpression(e)) e = e.expression; return e; } + +/** `e` without parentheses and type-only wrappers (as, satisfies, type + * assertions, non-null assertions), which change no runtime value. */ +function withoutTypeCasts(e: ts.Expression): ts.Expression { + while ( + ts.isParenthesizedExpression(e) || + ts.isAsExpression(e) || + ts.isSatisfiesExpression(e) || + ts.isTypeAssertion(e) || + ts.isNonNullExpression(e) + ) + e = e.expression; + return e; +} import { UNSUPPORTED } from "../../diagnostics/diagnostic.js"; import { BOOL, @@ -3440,6 +3454,17 @@ export function stdlibGlobalNameOf(lowerer: Lowerer, expr: ts.Expression): strin return ts.symbolName(resolved) === "global" ? "globalThis" : ts.symbolName(resolved); } if (ts.isPropertyAccessExpression(expr) && !expr.questionDotToken) { + // A cast of the global object (`(globalThis as { Buffer?: B }).Buffer`, + // `(globalThis as any).Buffer`) changes only its static type: the + // member is the global object's own. + const uncast = withoutTypeCasts(expr.expression); + if (uncast !== unparenthesized(expr.expression)) { + if (!ts.isIdentifier(uncast) || stdlibGlobalNameOf(lowerer, uncast) !== "globalThis") + return null; + const member = lowerer.checker.getPropertyOfType(lowerer.typeOf(uncast), expr.name.text); + if (!member || !lowerer.isStdlibSymbol(member)) return null; + return ts.symbolName(member) === "global" ? "globalThis" : ts.symbolName(member); + } if (stdlibGlobalNameOf(lowerer, expr.expression) !== "globalThis") return null; // Through a detection chain the checker types the receiver as the // union of every arm; the selected arm (the global object) owns the @@ -3572,6 +3597,13 @@ export function stdlibGlobalAliasDecl( const symbol = lowerer.checker.getSymbolAtLocation(nameNode); if (!symbol) return false; lowerer.stdlibGlobalAliases.set(symbol, name); + const access = init !== undefined ? unparenthesized(init) : undefined; + if ( + access !== undefined && + ts.isPropertyAccessExpression(access) && + withoutTypeCasts(access.expression) !== unparenthesized(access.expression) + ) + lowerer.castGlobalAliases.add(symbol); return true; } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7309-globalthis-buffer-aliases.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7309-globalthis-buffer-aliases.ts.json new file mode 100644 index 000000000..94d458e11 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7309-globalthis-buffer-aliases.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7309-globalthis-buffer-aliases.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7310-any-destructure-string-keys.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7310-any-destructure-string-keys.ts.json new file mode 100644 index 000000000..45a9f62ff --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7310-any-destructure-string-keys.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7310-any-destructure-string-keys.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/any-destructure-unspelled-keys.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/any-destructure-unspelled-keys.ts.json new file mode 100644 index 000000000..f38d71f47 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/any-destructure-unspelled-keys.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/diagnostics/any-destructure-unspelled-keys.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/globalthis-buffer-alias-values.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/globalthis-buffer-alias-values.ts.json new file mode 100644 index 000000000..4f6fe0c8c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/globalthis-buffer-alias-values.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/diagnostics/globalthis-buffer-alias-values.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7309-globalthis-buffer-aliases.ts b/tests/corpus/7309-globalthis-buffer-aliases.ts new file mode 100644 index 000000000..39d896463 --- /dev/null +++ b/tests/corpus/7309-globalthis-buffer-aliases.ts @@ -0,0 +1,31 @@ +// Aliases of the global Buffer constructor read through a cast of +// globalThis, the capability-probe spelling of code that also runs where +// Buffer is absent: typed and `as any` casts, `if`, ternary, typeof and `!` +// guards, member calls through the alias, and typeof of the alias. +type RuntimeBuffer = { + byteLength(value: string, encoding: "utf8"): number; + from(value: string, encoding: "utf8"): Uint8Array; + isBuffer(value: unknown): boolean; +}; + +const runtimeBuffer = (globalThis as { Buffer?: RuntimeBuffer }).Buffer; +const anyBuffer = (globalThis as any).Buffer; +const text = "héllo wörld"; + +if (runtimeBuffer) { + console.log("typed", runtimeBuffer.byteLength(text, "utf8")); +} else { + console.log("typed: no Buffer"); +} +if (typeof anyBuffer === "function") { + console.log("any", anyBuffer.byteLength(text, "utf8")); +} +const bytes = runtimeBuffer ? runtimeBuffer.from(text, "utf8") : undefined; +console.log("from", bytes ? bytes.length : -1); +console.log("isBuffer", runtimeBuffer ? runtimeBuffer.isBuffer(bytes) : false); +console.log("typeof", typeof runtimeBuffer, typeof anyBuffer, !runtimeBuffer); + +function utf8Length(value: string): number { + return runtimeBuffer ? runtimeBuffer.byteLength(value, "utf8") : value.length; +} +console.log("function", utf8Length("✓✓"), utf8Length("abc")); diff --git a/tests/corpus/7310-any-destructure-string-keys.ts b/tests/corpus/7310-any-destructure-string-keys.ts new file mode 100644 index 000000000..4ab24ec77 --- /dev/null +++ b/tests/corpus/7310-any-destructure-string-keys.ts @@ -0,0 +1,60 @@ +// Destructuring an `any` source read through a string-literal key +// (`const { value } = events["a.b"]`): reads, renames, defaults, rest, a +// nested pattern, the assignment form, a template-literal key, chained +// string keys, and V8's TypeError for undefined and null sources, whose +// message spells the key as a property name with its text decoded. +const events: any = { + "a.b": { value: 1, label: "first", extra: true, nested: { deep: "d" } }, + "x-y": { value: 2 }, + outer: { "in.ner": { value: 3 } }, + "nul": null, +}; + +const { value } = events["a.b"]; +const { value: renamed, missing = "fallback", ...rest } = events["a.b"]; +const { + nested: { deep }, +} = events["a.b"]; +console.log(value, renamed, missing, JSON.stringify(rest), deep); + +let assigned: unknown; +({ value: assigned } = events[`x-y`]); +const { value: chained } = events["outer"]["in.ner"]; +console.log(assigned, chained); + +function attempt(label: string, f: () => unknown): void { + try { + console.log(label, f()); + } catch (e) { + console.log(label, e instanceof TypeError, (e as Error).message); + } +} +attempt("missing", () => { + const { value } = events["no.such"]; + return value; +}); +attempt("null", () => { + const { value } = events["nul"]; + return value; +}); +attempt("quotes", () => { + const { value } = events["it's \"quoted\""]; + return value; +}); +attempt("escapes", () => { + const { value } = events["tab\there\nnext"]; + return value; +}); +attempt("accents", () => { + const { value } = events["été ✓"]; + return value; +}); +attempt("template", () => { + const { value } = events[`plain`]; + return value; +}); +attempt("assignment", () => { + let target: unknown; + ({ value: target } = events["gone"]); + return target; +}); diff --git a/tests/diagnostics/any-destructure-unspelled-keys.ts b/tests/diagnostics/any-destructure-unspelled-keys.ts new file mode 100644 index 000000000..6680de854 --- /dev/null +++ b/tests/diagnostics/any-destructure-unspelled-keys.ts @@ -0,0 +1,12 @@ +// Destructuring an `any` source keeps SC1031 where the source has no +// compiled spelling for V8's TypeError: a key computed by an expression, an +// optional element read, a template key with a substitution, and a string key +// holding NUL (the message is a C string). String-literal keys compile +// (tests/corpus/7310-any-destructure-string-keys.ts). +const events: any = { ab: { value: 1 } }; +const k = "a"; +const { value: computed } = events["a" + "b"]; +const { value: optional } = events?.["ab"]; +const { value: template } = events[`${k}b`]; +const { value: nul } = events["a\0b"]; +console.log(computed, optional, template, nul); diff --git a/tests/diagnostics/globalthis-buffer-alias-values.ts b/tests/diagnostics/globalthis-buffer-alias-values.ts new file mode 100644 index 000000000..97b49f920 --- /dev/null +++ b/tests/diagnostics/globalthis-buffer-alias-values.ts @@ -0,0 +1,12 @@ +// An alias of the Buffer global read through a cast of globalThis compiles +// in conditions, typeof and member calls +// (tests/corpus/7309-globalthis-buffer-aliases.ts). Its other value uses +// keep a named SC1090 refusal: the alias has the cast's static type. +type RuntimeBuffer = { from(value: string, encoding: "utf8"): Uint8Array }; +const runtimeBuffer = (globalThis as { Buffer?: RuntimeBuffer }).Buffer; +function keep(value: unknown): unknown { + return value; +} +const bytes = runtimeBuffer && runtimeBuffer.from("abc", "utf8"); +keep(runtimeBuffer); +keep(bytes); diff --git a/tests/harness/__snapshots__/any-destructure-unspelled-keys.ts.txt b/tests/harness/__snapshots__/any-destructure-unspelled-keys.ts.txt new file mode 100644 index 000000000..872b717cf --- /dev/null +++ b/tests/harness/__snapshots__/any-destructure-unspelled-keys.ts.txt @@ -0,0 +1,27 @@ +any-destructure-unspelled-keys.ts:8:29 - error SC1031: destructuring declaration from an `any` source whose expression spelling is unsupported (use a variable, ordinary property chain, or ordinary call) is not supported yet + + 7 | const k = "a"; + 8 | const { value: computed } = events["a" + "b"]; + | ^~~~~~~~~~~~~~~~~ + 9 | const { value: optional } = events?.["ab"]; + +any-destructure-unspelled-keys.ts:9:29 - error SC1031: destructuring declaration from an `any` source whose expression spelling is unsupported (use a variable, ordinary property chain, or ordinary call) is not supported yet + + 8 | const { value: computed } = events["a" + "b"]; + 9 | const { value: optional } = events?.["ab"]; + | ^~~~~~~~~~~~~~ + 10 | const { value: template } = events[`${k}b`]; + +any-destructure-unspelled-keys.ts:10:29 - error SC1031: destructuring declaration from an `any` source whose expression spelling is unsupported (use a variable, ordinary property chain, or ordinary call) is not supported yet + + 9 | const { value: optional } = events?.["ab"]; + 10 | const { value: template } = events[`${k}b`]; + | ^~~~~~~~~~~~~~~ + 11 | const { value: nul } = events["a\0b"]; + +any-destructure-unspelled-keys.ts:11:24 - error SC1031: destructuring declaration from an `any` source whose expression spelling is unsupported (use a variable, ordinary property chain, or ordinary call) is not supported yet + + 10 | const { value: template } = events[`${k}b`]; + 11 | const { value: nul } = events["a\0b"]; + | ^~~~~~~~~~~~~~ + 12 | console.log(computed, optional, template, nul); \ No newline at end of file diff --git a/tests/harness/__snapshots__/globalthis-buffer-alias-values.ts.txt b/tests/harness/__snapshots__/globalthis-buffer-alias-values.ts.txt new file mode 100644 index 000000000..f55423db9 --- /dev/null +++ b/tests/harness/__snapshots__/globalthis-buffer-alias-values.ts.txt @@ -0,0 +1,17 @@ +globalthis-buffer-alias-values.ts:10:15 - error SC1090: the Buffer constructor read through a cast of globalThis as a value is not supported yet + + 9 | } + 10 | const bytes = runtimeBuffer && runtimeBuffer.from("abc", "utf8"); + | ^~~~~~~~~~~~~ + 11 | keep(runtimeBuffer); + + hint: test it in a condition, read its typeof, or call its members + +globalthis-buffer-alias-values.ts:11:6 - error SC1090: the Buffer constructor read through a cast of globalThis as a value is not supported yet + + 10 | const bytes = runtimeBuffer && runtimeBuffer.from("abc", "utf8"); + 11 | keep(runtimeBuffer); + | ^~~~~~~~~~~~~ + 12 | keep(bytes); + + hint: test it in a condition, read its typeof, or call its members \ No newline at end of file diff --git a/tests/harness/corpus-costs/7309-globalthis-buffer-aliases.ts.json b/tests/harness/corpus-costs/7309-globalthis-buffer-aliases.ts.json new file mode 100644 index 000000000..f1c2cc30b --- /dev/null +++ b/tests/harness/corpus-costs/7309-globalthis-buffer-aliases.ts.json @@ -0,0 +1 @@ +{ "plain": 2600, "san": 3400 } diff --git a/tests/harness/corpus-costs/7310-any-destructure-string-keys.ts.json b/tests/harness/corpus-costs/7310-any-destructure-string-keys.ts.json new file mode 100644 index 000000000..df39abc08 --- /dev/null +++ b/tests/harness/corpus-costs/7310-any-destructure-string-keys.ts.json @@ -0,0 +1 @@ +{ "plain": 1500, "san": 3900 } From 97b64fae1cc66582b2e0ffe13008d817acfc4966 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 15:13:36 -0500 Subject: [PATCH 097/184] Turn the thread-heap huge-page policy off under memory tagging --- packages/runtime/src/scr_alloc.c | 26 +++++++++++++++++++++++--- packages/runtime/src/scr_alloc.test.c | 6 +++++- packages/runtime/src/scr_alloc.test.ts | 20 ++++++++++++++------ 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/packages/runtime/src/scr_alloc.c b/packages/runtime/src/scr_alloc.c index f3d6c0893..a27df378a 100644 --- a/packages/runtime/src/scr_alloc.c +++ b/packages/runtime/src/scr_alloc.c @@ -568,6 +568,9 @@ static _Atomic(uintptr_t) scr_sa_heap_page; /* the page size, read with the mode * a glibc malloc call: plain loads on x86-64, LDAR (or LDAPR) on arm64. */ #if defined(SCR_WORKERS) && !defined(SCR_LIB) +#if defined(__aarch64__) +#include +#endif /* glibc's per-thread heaps (worker executables; library archives run on * their embedder's threads and leave its heaps alone). glibc serves a thread's * blocks from a heap of its own: a 64 MiB mapping, aligned to its size, @@ -588,8 +591,11 @@ static _Atomic(uintptr_t) scr_sa_heap_page; /* the page size, read with the mode * policy is off unless THP is in madvise mode, when the glibc.malloc.hugetlb * tunable is set (glibc then advises its heaps itself, or sizes them by * hugetlbfs pages), when the system forbids overcommit (making a heap - * accessible would commit all of it), and when malloc is not glibc's (a - * sanitizer or a preloaded allocator: blocks have no glibc chunk header). + * accessible would commit all of it), when malloc is not glibc's (a + * sanitizer or a preloaded allocator: blocks have no glibc chunk header), + * and when memory tagging is on (arm64 MTE, the glibc.mem.tagging tunable): + * a chunk header then carries another tag than the block's pointer, and + * reading it through that pointer faults. * A heap is only advised when /proc/self/maps shows the anonymous mappings * of exactly its range. */ #define SCR_SA_GLIBC_HEAP ((uintptr_t)64 << 20) /* glibc's heap size and alignment */ @@ -601,6 +607,20 @@ static _Thread_local uintptr_t scr_sa_thread_heap; /* the heap this thread advis static _Atomic(int) scr_sa_thread_heaps; /* 0 undecided, 1 advised, 2 off */ extern void *__libc_malloc(size_t); +/* Whether glibc tags heap memory: the glibc.mem.tagging tunable is set to a + * nonzero value, or (arm64) the kernel checks tags for this process. */ +static bool scr_sa_mem_tagging(const char *tunables) { + const char *t = tunables ? strstr(tunables, "glibc.mem.tagging=") : NULL; + if (t && t[sizeof "glibc.mem.tagging=" - 1] != '0') return true; +#if defined(__aarch64__) + /* PR_GET_TAGGED_ADDR_CTRL (56); PR_MTE_TCF_MASK is bits 1-2. A kernel + * without the call returns -1: no tag checks. */ + long ctrl = prctl(56, 0, 0, 0, 0); + if (ctrl > 0 && (ctrl & (3L << 1))) return true; +#endif + return false; +} + /* Whether [heap, heap + SCR_SA_GLIBC_HEAP) is covered exactly by anonymous * mappings, as a glibc heap is (its accessible part and the rest). */ static bool scr_sa_glibc_heap_mapped(uintptr_t heap) { @@ -638,7 +658,7 @@ static void scr_sa_thread_heap_note(void *p, size_t n) { void *(*volatile used)(size_t) = malloc, *(*volatile own)(size_t) = __libc_malloc; bool glibc_malloc = used == own; mode = glibc_malloc && scr_sa_thp_mode() == 'm' && !(tunables && strstr(tunables, "glibc.malloc.hugetlb")) && - overcommit != '2' + overcommit != '2' && !scr_sa_mem_tagging(tunables) ? 1 : 2; if (atomic_compare_exchange_strong(&scr_sa_thread_heaps, &undecided, mode) && mode == 1 && diff --git a/packages/runtime/src/scr_alloc.test.c b/packages/runtime/src/scr_alloc.test.c index 6ccb1495b..dfdd9bf5c 100644 --- a/packages/runtime/src/scr_alloc.test.c +++ b/packages/runtime/src/scr_alloc.test.c @@ -388,7 +388,11 @@ static void check_thread_heap_huge_pages(void) { glibc_malloc = false; #endif #endif - if (glibc_malloc && strstr(mode, "[madvise]") && !getenv("GLIBC_TUNABLES") && overcommit != '2') + /* With glibc's memory tagging tunable set the heap is left alone: under + * arm64 MTE, reading its chunk headers through block pointers faults. */ + const char *tunables = getenv("GLIBC_TUNABLES"); + if (tunables && strstr(tunables, "glibc.mem.tagging=3")) assert(!strstr(heap_flags, " hg")); + else if (glibc_malloc && strstr(mode, "[madvise]") && !tunables && overcommit != '2') assert(strstr(heap_flags, " hg")); } #endif diff --git a/packages/runtime/src/scr_alloc.test.ts b/packages/runtime/src/scr_alloc.test.ts index 187ffce51..0ceda33cd 100644 --- a/packages/runtime/src/scr_alloc.test.ts +++ b/packages/runtime/src/scr_alloc.test.ts @@ -11,13 +11,20 @@ const exec = promisify(execFile); // UBSan-only build is the one that exercises it; the ASan build pins the // system-allocator bypass the sanitizer lane relies on. // Worker executables (-DSCR_WORKERS) give each script thread its own arena; -// the ThreadSanitizer build checks the cross-thread handoff for races. +// the ThreadSanitizer build checks the cross-thread handoff for races. The +// last build runs with glibc's memory tagging tunable set, which turns the +// thread-heap huge-page policy off. test.each([ - ["size-class", ["-fsanitize=undefined"]], - ["system", ["-fsanitize=address,undefined"]], - ["size-class workers", ["-fsanitize=undefined", "-DSCR_WORKERS", "-pthread"]], - ["size-class workers", ["-fsanitize=thread", "-DSCR_WORKERS", "-pthread"]], -])("small-object allocator contracts hold (%s, %j)", async (mode, sanitize) => { + ["size-class", ["-fsanitize=undefined"], {}], + ["system", ["-fsanitize=address,undefined"], {}], + ["size-class workers", ["-fsanitize=undefined", "-DSCR_WORKERS", "-pthread"], {}], + ["size-class workers", ["-fsanitize=thread", "-DSCR_WORKERS", "-pthread"], {}], + [ + "size-class workers", + ["-fsanitize=undefined", "-DSCR_WORKERS", "-pthread"], + { GLIBC_TUNABLES: "glibc.mem.tagging=3" }, + ], +])("small-object allocator contracts hold (%s, %j, %j)", async (mode, sanitize, extraEnv) => { const scratch = await mkdtemp(join(tmpdir(), "scriptc-alloc-")); try { const binary = join(scratch, "alloc"); @@ -41,6 +48,7 @@ test.each([ ASAN_OPTIONS: "halt_on_error=1", TSAN_OPTIONS: "halt_on_error=1", UBSAN_OPTIONS: "halt_on_error=1", + ...extraEnv, }, }); expect(result.stdout).toBe(`allocator checks passed: ${mode}\n`); From fa80286f9b164b01921457f14983953f8d7f5845 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 15:21:21 -0500 Subject: [PATCH 098/184] Lower borrowed typed array set calls and keep method calls direct after dictionary stores Co-authored-by: ainslec <201870+ainslec@users.noreply.github.com> Co-authored-by: MathurAditya724 <57684218+MathurAditya724@users.noreply.github.com> --- .../lowering/class-method-mutations.ts | 17 ++- .../src/frontend/lowering/containers/bytes.ts | 128 +++++++++++++++--- .../src/frontend/lowering/lower-calls.ts | 3 +- .../test/class-method-mutations.test.ts | 32 +++++ .../7940-typed-array-prototype-set.mjs.json | 6 + .../7941-typed-array-prototype-set.ts.json | 6 + ...7942-dictionary-store-method-calls.ts.json | 6 + .../corpus/7940-typed-array-prototype-set.mjs | 40 ++++++ .../corpus/7941-typed-array-prototype-set.ts | 21 +++ .../7942-dictionary-store-method-calls.ts | 45 ++++++ 10 files changed, 282 insertions(+), 22 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7940-typed-array-prototype-set.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7941-typed-array-prototype-set.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7942-dictionary-store-method-calls.ts.json create mode 100644 tests/corpus/7940-typed-array-prototype-set.mjs create mode 100644 tests/corpus/7941-typed-array-prototype-set.ts create mode 100644 tests/corpus/7942-dictionary-store-method-calls.ts diff --git a/packages/compiler/src/frontend/lowering/class-method-mutations.ts b/packages/compiler/src/frontend/lowering/class-method-mutations.ts index 035c2c2a5..a07a4d3e1 100644 --- a/packages/compiler/src/frontend/lowering/class-method-mutations.ts +++ b/packages/compiler/src/frontend/lowering/class-method-mutations.ts @@ -125,11 +125,13 @@ export function collectClassMethodMutations( }; // Whether no function value can inhabit `type`. Primitives qualify, and so // do non-callable object types a function is not assignable to: one that - // requires a property functions lack, or a weak (all-optional) type that + // requires a property functions lack, a weak (all-optional) type that // shares no property with functions (TypeScript's weak-type rule rejects - // callable sources too). - const functionFree = (type: ts.Type): boolean => { - if (type.isUnionType()) return ts.constituentTypes(type).every(functionFree); + // callable sources too), or a dictionary with no properties whose index + // signatures hold function-free values (functions have no index + // signature). `seen` stops recursive dictionary types. + const functionFree = (type: ts.Type, seen: Set = new Set()): boolean => { + if (type.isUnionType()) return ts.constituentTypes(type).every((t) => functionFree(t, seen)); if ( (type.flags & (ts.TypeFlags.StringLike | @@ -149,7 +151,12 @@ export function collectClassMethodMutations( ) return false; const properties = lowerer.checker.getPropertiesOfType(type); - if (properties.length === 0) return false; + if (properties.length === 0) { + if (seen.has(type)) return true; + seen.add(type); + const indexes = lowerer.checker.getIndexInfosOfType(type); + return indexes.length > 0 && indexes.every((index) => functionFree(index.type, seen)); + } const required = properties.filter( (property) => (property.flags & ts.SymbolFlags.Optional) === 0, ); diff --git a/packages/compiler/src/frontend/lowering/containers/bytes.ts b/packages/compiler/src/frontend/lowering/containers/bytes.ts index 995cb177a..36e7ecd22 100644 --- a/packages/compiler/src/frontend/lowering/containers/bytes.ts +++ b/packages/compiler/src/frontend/lowering/containers/bytes.ts @@ -1,5 +1,5 @@ import { keyedHelperName } from "../stable-names.js"; -import { boolLit, dynUndefinedExpr, numLit, varRef } from "../../../ir/build.js"; +import { boolLit, dynUndefinedExpr, nodeThrowExpr, numLit, varRef } from "../../../ir/build.js"; import { BUF_NUM_METHODS } from "./buffer-numeric-methods.js"; import * as ts from "../../ts7/adapter.js"; import { @@ -903,6 +903,116 @@ function lowerDataViewNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr { return lowerArrayBufferView(lowerer, args, "dv", locOf(expr)); } +/** `receiver.set(src, offset?)` on a lowered typed-array receiver. */ +function bytesSetIntrinsic( + lowerer: Lowerer, + receiver: IrExpr, + argNodes: readonly ts.Expression[], + loc: SrcLoc, +): IrExpr { + const src = lowerer.lowerExpr(argNodes[0]!); + const method = src.type.kind === "bytes" ? "setFrom" : "setFromDyn"; + const args = [src.type.kind === "bytes" ? src : lowerer.coerceInto(argNodes[0]!, src, DYN)]; + if (argNodes.length === 2) + args.push( + lowerOptionalArgument(lowerer, argNodes[1]!, F64, { + kind: "numLit", + value: 0, + type: F64, + loc, + }), + ); + return { kind: "bytesIntrinsic", method, receiver, args, type: VOID, loc }; +} + +/** `Uint8Array.prototype.set.call(target, src, offset?)`, through any typed + * array constructor's prototype (they share %TypedArray%.prototype.set). A + * typed-array target lowers like `target.set(src, offset)`. Any other + * target is checked at run time: a typed array calls its `set`, and + * anything else throws Node's TypeError "this is not a typed array.", + * after every argument is evaluated. */ +export function lowerBytesPrototypeSetCall( + lowerer: Lowerer, + call: ts.CallExpression, + access: ts.PropertyAccessExpression, +): IrExpr | null { + if ( + call.questionDotToken || + access.questionDotToken || + access.name.text !== "call" || + !ts.isPropertyAccessExpression(access.expression) + ) + return null; + const method = access.expression; + const proto = method.expression; + if ( + method.questionDotToken || + method.name.text !== "set" || + !ts.isPropertyAccessExpression(proto) || + proto.questionDotToken || + proto.name.text !== "prototype" || + !ts.isIdentifier(proto.expression) || + !lowerer.isStdlibMember(method) + ) + return null; + const ctor = lowerer.resolveValueSymbol(proto.expression); + if (!ctor || !lowerer.isStdlibSymbol(ctor) || !own(BYTES_CTORS, ts.symbolName(ctor))) return null; + const argNodes = call.arguments.nodes; + if (argNodes.length < 2 || argNodes.length > 3 || argNodes.some(ts.isSpreadElement)) return null; + const loc = locOf(call); + const target = lowerer.lowerExpr(argNodes[0]!); + if (target.type.kind === "bytes") + return bytesSetIntrinsic(lowerer, target, argNodes.slice(1), loc); + const slot = (value: IrExpr): { decl: IrStmt; read: IrExpr } => { + const local = lowerer.declareHiddenLocal("%setArg", DYN); + return { + decl: { kind: "varDecl", localId: local.id, init: value, loc }, + read: varRef(local.id, DYN, loc), + }; + }; + const slots = [ + slot(lowerer.coerceInto(argNodes[0]!, target, DYN)), + ...argNodes.slice(1).map((arg) => slot(lowerer.lowerExprExpecting(arg, DYN))), + ]; + const isTypedArray: IrExpr = { + kind: "libCall", + fn: "util.typeIs", + args: [slots[0]!.read, { kind: "strLit", value: "isTypedArray", type: STRING, loc }], + type: BOOL, + loc, + }; + return { + kind: "seqExpr", + stmts: [ + ...slots.map((s) => s.decl), + { + kind: "if", + cond: { kind: "unary", op: "!", operand: isTypedArray, type: BOOL, loc }, + then: [ + { + kind: "exprStmt", + expr: nodeThrowExpr(1, "", "this is not a typed array.", VOID, loc), + loc, + }, + ], + else_: null, + loc, + }, + ], + result: { + kind: "dynInvoke", + recv: slots[0]!.read, + method: "set", + calleeName: ts.getTextOfNode(method), + args: slots.slice(1).map((s) => s.read), + type: DYN, + loc, + }, + type: DYN, + loc, + }; +} + /** Method calls on typed-array/Buffer receivers: slice (copy), subarray * (shared view), toReversed, copyWithin, set(src, * offset?), and the u8-only Buffer surface: toString(enc?) plus the @@ -1106,21 +1216,7 @@ export function lowerBytesMethodCall( lowerer.noLowering(`.set with ${nArgs} arguments on typed arrays`, call); } const receiver = lowerer.lowerExprExpecting(access.expression, receiverIr); - const src = lowerer.lowerExpr(call.arguments.nodes[0]!); - const method = src.type.kind === "bytes" ? "setFrom" : "setFromDyn"; - const args = [ - src.type.kind === "bytes" ? src : lowerer.coerceInto(call.arguments.nodes[0]!, src, DYN), - ]; - if (nArgs === 2) - args.push( - lowerOptionalArgument(lowerer, call.arguments.nodes[1]!, F64, { - kind: "numLit", - value: 0, - type: F64, - loc, - }), - ); - return { kind: "bytesIntrinsic", method, receiver, args, type: VOID, loc }; + return bytesSetIntrinsic(lowerer, receiver, call.arguments.nodes, loc); } if (name === "toString") { // Buffer's toString(encoding?) — utf8 by default. A 0-arg toString diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 31276263d..c071d5a83 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -119,7 +119,7 @@ import { } from "./containers/indexed-objects.js"; import { lowerTupleReadMethodCall } from "./containers/array-methods.js"; import { tryFuseIndexRead } from "./array-values.js"; -import { lowerBytesStaticCall } from "./containers/bytes.js"; +import { lowerBytesPrototypeSetCall, lowerBytesStaticCall } from "./containers/bytes.js"; import { lowerRegexMethodCall, lowerStringIndexCall, @@ -1579,6 +1579,7 @@ export function lowerCall(lowerer: Lowerer, expr: ts.CallExpression): IrExpr { lowerer.lowerTestMethodCall(expr, expr.expression) ?? lowerer.lowerTimeoutMethodCall(expr, expr.expression) ?? lowerArrayPrototypeBorrowCall(lowerer, expr, expr.expression) ?? + lowerBytesPrototypeSetCall(lowerer, expr, expr.expression) ?? lowerObjectOwnPrototypeCall(lowerer, expr, expr.expression) ?? lowerObjectPrototypeCall(lowerer, expr, expr.expression) ?? lowerStringPrototypeCall(lowerer, expr, expr.expression) ?? diff --git a/packages/compiler/test/class-method-mutations.test.ts b/packages/compiler/test/class-method-mutations.test.ts index b508a3949..4871dda2b 100644 --- a/packages/compiler/test/class-method-mutations.test.ts +++ b/packages/compiler/test/class-method-mutations.test.ts @@ -135,3 +135,35 @@ test("a reflection call with a well-known symbol key keeps checked dispatch", as const callers = await callersOf(`${range}${write}`, "%Range.sym:iterator"); expect(callers.filter((name) => !name.startsWith("%method.value:"))).toEqual([]); }); + +const dictionaries = ` +class Counter { + n = 0; + add(): number { return ++this.n; } +} +interface Settings { groups: Record> } +interface Tree { [key: string]: Tree } +function addGroup(settings: Settings, name: string): void { + settings.groups[name] = {}; +} +function grow(tree: Tree, name: string): void { + tree[name] = {}; +} +const settings: Settings = { groups: {} }; +addGroup(settings, "colours"); +grow({}, "a"); +const counter = new Counter(); +console.log(counter.add(), Object.keys(settings.groups)); +`; + +test("a store into a dictionary of dictionaries leaves method calls direct", async () => { + expect(await callersOf(dictionaries, "%dyn.class.properties")).toEqual([]); +}); + +test("a store into a dictionary whose values can be functions keeps checked dispatch", async () => { + const source = dictionaries.replace( + "Record>", + "Record | (() => number)>", + ); + expect(await callersOf(source, "%dyn.class.properties")).not.toEqual([]); +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7940-typed-array-prototype-set.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7940-typed-array-prototype-set.mjs.json new file mode 100644 index 000000000..3c59407b3 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7940-typed-array-prototype-set.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7940-typed-array-prototype-set.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7941-typed-array-prototype-set.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7941-typed-array-prototype-set.ts.json new file mode 100644 index 000000000..8fbec6a90 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7941-typed-array-prototype-set.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7941-typed-array-prototype-set.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7942-dictionary-store-method-calls.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7942-dictionary-store-method-calls.ts.json new file mode 100644 index 000000000..f04478490 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7942-dictionary-store-method-calls.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7942-dictionary-store-method-calls.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7940-typed-array-prototype-set.mjs b/tests/corpus/7940-typed-array-prototype-set.mjs new file mode 100644 index 000000000..4f1b2301d --- /dev/null +++ b/tests/corpus/7940-typed-array-prototype-set.mjs @@ -0,0 +1,40 @@ +// Uint8Array.prototype.set borrowed through .call on untyped receivers. +let cached = null; +function memory() { + if (cached === null) cached = new Uint8Array(4); + return cached; +} +function copy(target, source, offset) { + return Uint8Array.prototype.set.call(target, source, offset); +} +console.log(copy(memory(), new Uint8Array([3, 4]), 1)); +console.log(String(memory())); +copy(memory(), [9], undefined); +console.log(String(memory())); + +// Every typed array shares %TypedArray%.prototype.set. +const wide = new Int16Array(3); +copy(wide, [1, -2], 1); +console.log(String(wide)); +Float64Array.prototype.set.call(memory(), new Uint8Array([7]), 3); +console.log(String(memory())); + +const order = []; +const arg = (label, value) => { + order.push(label); + return value; +}; +for (const target of [{}, [0, 0], "ab", null, new DataView(new ArrayBuffer(2))]) { + order.length = 0; + try { + Uint8Array.prototype.set.call(arg("target", target), arg("source", [1]), arg("offset", 0)); + console.log("set"); + } catch (e) { + console.log(e instanceof TypeError, e.message, order.join(",")); + } +} +try { + copy(memory(), [1, 2], 3); +} catch (e) { + console.log(e instanceof RangeError, e.message); +} diff --git a/tests/corpus/7941-typed-array-prototype-set.ts b/tests/corpus/7941-typed-array-prototype-set.ts new file mode 100644 index 000000000..db880e38a --- /dev/null +++ b/tests/corpus/7941-typed-array-prototype-set.ts @@ -0,0 +1,21 @@ +// Uint8Array.prototype.set borrowed through .call on typed receivers. +const target = new Uint8Array(4); +Uint8Array.prototype.set.call(target, new Uint8Array([3, 4]), 1); +console.log(String(target)); +Uint8Array.prototype.set.call(target, [5]); +console.log(String(target)); + +const wide = new Int32Array(3); +Uint8Array.prototype.set.call(wide, new Uint8Array([200, 100]), 1); +console.log(String(wide)); + +function fill(view: Uint8Array, bytes: number[], at: number): void { + Uint8Array.prototype.set.call(view, bytes, at); +} +fill(target, [8, 9], 2); +console.log(String(target)); +try { + fill(target, [1, 2, 3], 2); +} catch (e) { + console.log(e instanceof RangeError, (e as Error).message); +} diff --git a/tests/corpus/7942-dictionary-store-method-calls.ts b/tests/corpus/7942-dictionary-store-method-calls.ts new file mode 100644 index 000000000..0056c26f6 --- /dev/null +++ b/tests/corpus/7942-dictionary-store-method-calls.ts @@ -0,0 +1,45 @@ +// Stores into dictionaries whose values cannot be functions leave class +// methods in place. +interface Data { + items: string[]; +} +class Store { + data: Data = { items: [] }; + get(): Data { + return this.data; + } +} +class Counter { + n = 0; + add(): number { + return ++this.n; + } +} +interface Settings { + groups: Record>; +} +interface Tree { + [key: string]: Tree; +} +function addGroup(settings: Settings, name: string): void { + settings.groups[name] = {}; +} +function grow(tree: Tree, name: string): Tree { + tree[name] = {}; + return tree[name]!; +} +const settings: Settings = { groups: {} }; +addGroup(settings, "colours"); +addGroup(settings, "fonts"); +settings.groups["colours"]!["fg"] = "red"; +console.log(Object.keys(settings.groups).join(","), JSON.stringify(settings.groups)); +const tree: Tree = {}; +grow(grow(tree, "a"), "b"); +console.log(JSON.stringify(tree)); +const store = new Store(); +store.get().items.push("a"); +store.get().items.push("b"); +const counter = new Counter(); +let total = 0; +for (let i = 0; i < 1000; i++) total += counter.add(); +console.log(store.data.items.length, total, typeof counter.add); From b630efd450f066dabd3013e9d48b40a2b87ae0d1 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 13:41:29 -0700 Subject: [PATCH 099/184] Report inferred shapes for untyped npm JavaScript (shape inference, stage 1) SCRIPTC_DEBUG_SHAPES=1 runs a whole-program allocation-site points-to analysis over opted-in npm packages' JavaScript and the user files that call into them, and prints which object and array literals a native record would represent exactly, the inferred class field types, the census of writes and reflection on untracked receivers, and the hot functions' parameter and return types. Report only: nothing reaches the lowering, and the analysis runs only when the variable is set. Design: results/design-shape-inference/DESIGN.md (scriptc-perf). Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/frontend/program.ts | 5 + .../src/frontend/shape-inference/analyze.ts | 2864 +++++++++++++++++ .../src/frontend/shape-inference/domain.ts | 153 + .../src/frontend/shape-inference/index.ts | 21 + .../src/frontend/shape-inference/report.ts | 374 +++ .../compiler/test/shape-inference.test.ts | 166 + 6 files changed, 3583 insertions(+) create mode 100644 packages/compiler/src/frontend/shape-inference/analyze.ts create mode 100644 packages/compiler/src/frontend/shape-inference/domain.ts create mode 100644 packages/compiler/src/frontend/shape-inference/index.ts create mode 100644 packages/compiler/src/frontend/shape-inference/report.ts create mode 100644 packages/compiler/test/shape-inference.test.ts diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index 6d8ffe4c0..a1f440648 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -140,6 +140,7 @@ import { } from "./type-only-imports.js"; import { moduleSourceCandidates } from "./module-source-candidates.js"; import { planDeclProjection, type DeclProjection } from "./decl-projection.js"; +import { reportShapes } from "./shape-inference/index.js"; import { inferredJsDiagnosticSuppressed } from "./inferred-js-diagnostics.js"; import { containsJsx, @@ -1260,6 +1261,10 @@ export function loadProgram( npmStaticPackages.length > 0, projector, ); + // Shape inference, stage S1: a report only (SCRIPTC_DEBUG_SHAPES). + const shapes = process.env["SCRIPTC_DEBUG_SHAPES"]; + if (shapes !== undefined && shapes !== "" && shapes !== "0" && npmStaticPackages.length > 0) + reportShapes(load.program, shapes); return { ...load, services, diff --git a/packages/compiler/src/frontend/shape-inference/analyze.ts b/packages/compiler/src/frontend/shape-inference/analyze.ts new file mode 100644 index 000000000..55e0e0107 --- /dev/null +++ b/packages/compiler/src/frontend/shape-inference/analyze.ts @@ -0,0 +1,2864 @@ +/* Shape inference (results/design-shape-inference/DESIGN.md), stage S1: + * a whole-program allocation-site points-to analysis over the JavaScript of + * opted-in npm packages and the user files that call into them. + * + * Stage S1 is report-only: the analysis runs when SCRIPTC_DEBUG_SHAPES is + * set and prints its plan (report.ts). Nothing it computes reaches the + * lowering yet, so every program compiles exactly as before. + * + * The analysis (DESIGN.md §4): + * - Andersen-style inclusion constraints, solved with a worklist. Locals + * and parameters are one value node per binding (flow-insensitive in S1); + * heap fields are one node per (atom, name); arrays one element node. + * - Calls resolve by the callee's atoms: functions, class constructors, + * methods by the receiver's class chain (inherited members, getters and + * setters, `this.constructor`), closures stored in fields, `bind`, and + * builtin summaries for the common library surface. + * - Escape: an atom escapes when it reaches code the analysis cannot see + * (a ⊤ callee, a builtin without a summary, a store into a ⊤ object). + * Escaped functions run with ⊤ arguments; everything reachable from an + * escaped atom escapes. + * - Census: writes and reflection whose receiver may be ⊤ apply to every + * escaped atom (§5.4); the report uses them to decide which allocation + * sites could become native records (§5.5). */ +import * as ts from "../ts7/adapter.js"; +import { isJsSourceFile } from "../program.js"; +import { npmStaticPackageOfPath } from "../npm-static.js"; +import { AtomTable, MODELED_GLOBALS, STRING_METHOD_RESULTS, type Atom } from "./domain.js"; + +type Node = ts.Node; + +export interface ValueNode { + readonly id: number; + readonly atoms: Set; + readonly succ: Set; + /** Constraints to run for each atom that arrives. */ + readonly onAtom: ((atom: Atom) => void)[]; + /** Every atom that arrives escapes. */ + escapeSink: string | null; +} + +export interface FunctionInfo { + readonly node: Node; + /** The analysis context: "" or a method's receiver (`C`). */ + readonly ctx: string; + /** The function whose clone the context is (bindings inside it are per context). */ + readonly root: Node | null; + /** Bodies are walked on first call: code that never runs adds no facts. */ + walked: boolean; + readonly label: string; + readonly params: number[]; + /** Index of a rest parameter, or -1. */ + readonly rest: number; + readonly thisNode: number; + readonly ret: number; + readonly isArrow: boolean; + readonly isAsync: boolean; + readonly isGenerator: boolean; + /** Reads `arguments` other than through `...arguments` forwarding. */ + usesArguments: boolean; + /** `f(...arguments)` forwarding: one node per argument position. */ + argPos: number[] | null; + /** The class whose member this function is (method, accessor, constructor). */ + readonly ownerClass: Node | null; + readonly inPackage: boolean; +} + +export interface LiteralSite { + readonly atom: Atom; + readonly node: Node; + readonly kind: "object" | "array"; + /** Static keys in creation order; null when a spread or computed key makes them unknown. */ + readonly keys: string[] | null; + readonly hasAccessor: boolean; + readonly hasProto: boolean; + readonly hasComputedKey: boolean; + readonly spreads: number[]; + readonly inPackage: boolean; +} + +export interface Op { + readonly kind: + | "read" + | "write" + | "computedRead" + | "computedWrite" + | "delete" + | "freeze" + | "seal" + | "defineProperty" + | "setPrototypeOf" + | "assign" + | "proxy" + | "lengthWrite"; + readonly recv: number; + readonly name: string | null; + readonly node: Node; +} + +export interface CallEdge { + readonly caller: FunctionInfo | null; + readonly callee: FunctionInfo; + readonly inLoop: boolean; +} + +export interface ShapeAnalysis { + readonly atoms: AtomTable; + readonly nodes: ValueNode[]; + readonly functions: Map; + readonly sites: LiteralSite[]; + readonly ops: Op[]; + readonly calls: CallEdge[]; + readonly escaped: Map; + readonly topWriteNames: Map; + readonly topComputedWrites: Node[]; + /** Fields per atom: name → node id. */ + readonly fields: Map>; + readonly elements: Map; + readonly top: Atom; + readonly files: readonly ts.SourceFile[]; + readonly packageFiles: number; + readonly checker: ts.TypeChecker; + readonly budgetExceeded: boolean; + readonly steps: number; + readonly topCause: ReadonlyMap; + readonly nodeLabel: ReadonlyMap; +} + +/** Array.prototype members: other names on an array are own properties. */ +const ARRAY_METHODS: ReadonlySet = new Set([ + "at", + "concat", + "copyWithin", + "entries", + "every", + "fill", + "filter", + "find", + "findIndex", + "findLast", + "findLastIndex", + "flat", + "flatMap", + "forEach", + "includes", + "indexOf", + "join", + "keys", + "lastIndexOf", + "map", + "pop", + "push", + "reduce", + "reduceRight", + "reverse", + "shift", + "slice", + "some", + "sort", + "splice", + "toLocaleString", + "toReversed", + "toSorted", + "toSpliced", + "toString", + "unshift", + "values", + "with", +]); +const MAX_STEPS = 20_000_000; + +function list(arr: { readonly nodes: readonly T[] } | undefined): readonly T[] { + return arr?.nodes ?? []; +} + +function isStatic(node: Node): boolean { + const mods = (node as { modifiers?: { nodes: readonly Node[] } }).modifiers; + return list(mods).some((m) => m.kind === ts.SyntaxKind.StaticKeyword); +} + +function unparen(node: ts.Expression): ts.Expression { + let n = node; + for (;;) { + if (ts.isParenthesizedExpression(n)) n = n.expression; + else if (ts.isNonNullExpression(n)) n = n.expression; + else if (ts.isAsExpression(n) || ts.isSatisfiesExpression(n)) n = n.expression; + else return n; + } +} + +function propertyNameText(name: Node): string | null { + if (ts.isIdentifier(name) || ts.isPrivateIdentifier(name)) return name.text; + if (ts.isStringLiteral(name) || ts.isNoSubstitutionTemplateLiteral(name)) return name.text; + if (ts.isNumericLiteral(name)) return String(Number(name.text)); + if (ts.isComputedPropertyName(name)) { + const e = unparen(name.expression); + if (ts.isStringLiteral(e) || ts.isNoSubstitutionTemplateLiteral(e)) return e.text; + if (ts.isNumericLiteral(e)) return String(Number(e.text)); + } + return null; +} + +function shortFile(fileName: string): string { + const norm = fileName.split("\\").join("/"); + const nm = norm.lastIndexOf("/node_modules/"); + if (nm >= 0) { + const rest = norm.slice(nm + "/node_modules/".length).split("/"); + const pkgLen = rest[0]?.startsWith("@") ? 2 : 1; + const pkg = rest.slice(0, pkgLen).join("/"); + return `${pkg}/${rest[rest.length - 1]}`; + } + return norm.slice(norm.lastIndexOf("/") + 1); +} + +export function locLabel(node: Node): string { + const sf = node.getSourceFile(); + const { line } = ts.getLineAndCharacterOfPosition(sf, node.getStart(sf)); + return `${shortFile(sf.fileName)}:${line + 1}`; +} + +/** Whether a source file belongs to an opted-in package's JavaScript. */ +export function isPackageJsFile(sf: ts.SourceFile): boolean { + return isJsSourceFile(sf) && npmStaticPackageOfPath(sf.fileName) !== null; +} + +function isLoop(node: Node): boolean { + return ( + ts.isForStatement(node) || + ts.isForOfStatement(node) || + ts.isForInStatement(node) || + ts.isWhileStatement(node) || + ts.isDoStatement(node) + ); +} + +const LOOP_CALLBACK_METHODS: ReadonlySet = new Set([ + "map", + "forEach", + "filter", + "some", + "every", + "find", + "findIndex", + "findLast", + "findLastIndex", + "reduce", + "reduceRight", + "flatMap", + "sort", +]); + +export function analyzeShapes(program: ts.Program, checker: ts.TypeChecker): ShapeAnalysis { + return new Analyzer(program, checker).run(); +} + +class Analyzer { + readonly atoms = new AtomTable(); + readonly nodes: ValueNode[] = []; + readonly functions = new Map(); + readonly sites: LiteralSite[] = []; + private readonly siteAtoms = new Set(); + readonly ops: Op[] = []; + readonly calls: CallEdge[] = []; + readonly escaped = new Map(); + readonly topWriteNames = new Map(); + readonly topComputedWrites: Node[] = []; + readonly fields = new Map>(); + readonly elements = new Map(); + readonly top: Atom; + private readonly bindings = new Map(); + private readonly exprMemo = new Map(); + /** Current analysis context and the function whose clone it is. */ + private ctx = ""; + private ctxRoot: Node | null = null; + private readonly classInfo = new Map(); + private readonly boundCalls = new Set(); + private readonly escapedFunctionCalls = new Set(); + private readonly worklist: [number, Atom][] = []; + private steps = 0; + private budgetExceeded = false; + private currentFn: FunctionInfo | null = null; + private loopDepth = 0; + private files: ts.SourceFile[] = []; + private packageFiles = 0; + private readonly prims = new Map(); + /** Per call site: the calling function and whether the site is in a loop + * (constraints fire during solving, after the walk has moved on). */ + private readonly siteCtx = new Map(); + private forceLoop = false; + + noteSite(site: Node): void { + if (!this.siteCtx.has(site)) + this.siteCtx.set(site, { caller: this.currentFn, inLoop: this.loopDepth > 0 }); + } + + edgeFor(site: Node, callee: FunctionInfo): void { + const ctx = this.siteCtx.get(site); + this.calls.push({ + caller: ctx?.caller ?? null, + callee, + inLoop: (ctx?.inLoop ?? false) || this.forceLoop, + }); + } + + constructor( + private readonly program: ts.Program, + private readonly checker: ts.TypeChecker, + ) { + this.top = this.atoms.intern("top", () => ({ kind: "top", label: "⊤" })); + } + + run(): ShapeAnalysis { + for (const sf of this.program.getSourceFiles()) { + if (sf.isDeclarationFile) continue; + const name = sf.fileName.split("\\").join("/"); + const inPackage = isPackageJsFile(sf); + if (!inPackage && name.includes("/node_modules/")) continue; + this.files.push(sf); + if (inPackage) this.packageFiles++; + } + for (const sf of this.files) this.checker.prefetchSourceFile(sf); + for (const sf of this.files) this.moduleBody(sf); + this.finish(); + return { + atoms: this.atoms, + nodes: this.nodes, + functions: this.functions, + sites: this.sites, + ops: this.ops, + calls: this.calls, + escaped: this.escaped, + topWriteNames: this.topWriteNames, + topComputedWrites: this.topComputedWrites, + fields: this.fields, + elements: this.elements, + top: this.top, + files: this.files, + packageFiles: this.packageFiles, + checker: this.checker, + budgetExceeded: this.budgetExceeded, + steps: this.steps, + topCause: this.topCause, + nodeLabel: this.nodeLabel, + }; + } + + // ---- value nodes ------------------------------------------------------- + + newNode(): number { + const id = this.nodes.length; + this.nodes.push({ id, atoms: new Set(), succ: new Set(), onAtom: [], escapeSink: null }); + return id; + } + + /** Debugging (SCRIPTC_DEBUG_SHAPES_WHY): where each node's ⊤ came from. */ + readonly topCause = new Map(); + readonly nodeLabel = new Map(); + private cause: number | null = null; + private readonly traceTop = process.env["SCRIPTC_DEBUG_SHAPES_WHY"] !== undefined; + + add(node: number, atom: Atom): void { + const n = this.nodes[node]!; + if (n.atoms.has(atom)) return; + n.atoms.add(atom); + if (this.traceTop && atom === this.top && !this.topCause.has(node)) + this.topCause.set( + node, + this.cause ?? + (new Error().stack ?? "") + .split("\n") + .slice(2, 4) + .map((l) => l.trim()) + .join(" < "), + ); + this.worklist.push([node, atom]); + } + + edge(from: number, to: number): void { + if (from === to) return; + const n = this.nodes[from]!; + if (n.succ.has(to)) return; + n.succ.add(to); + const saved = this.cause; + this.cause = from; + for (const a of n.atoms) this.add(to, a); + this.cause = saved; + } + + on(node: number, fn: (atom: Atom) => void): void { + const n = this.nodes[node]!; + n.onAtom.push(fn); + for (const a of [...n.atoms]) fn(a); + } + + sink(node: number, reason: string): void { + const n = this.nodes[node]!; + if (n.escapeSink !== null) return; + n.escapeSink = reason; + for (const a of [...n.atoms]) this.escape(a, reason); + } + + constNode(atom: Atom): number { + const n = this.newNode(); + this.add(n, atom); + return n; + } + + prim(name: string): Atom { + let a = this.prims.get(name); + if (a === undefined) { + a = this.atoms.intern(`prim:${name}`, () => ({ kind: "prim", label: name })); + this.prims.set(name, a); + } + return a; + } + + builtin(path: string, receiver?: Atom): Atom { + const key = receiver === undefined ? `builtin:${path}` : `builtin:${path}@${receiver.id}`; + return this.atoms.intern(key, () => ({ kind: "builtin", label: path, path, receiver })); + } + + solve(): void { + while (this.worklist.length > 0) { + if (++this.steps > MAX_STEPS) { + this.budgetExceeded = true; + return; + } + const [id, atom] = this.worklist.pop()!; + const n = this.nodes[id]!; + this.cause = id; + for (const s of n.succ) this.add(s, atom); + this.cause = null; + for (const f of n.onAtom) f(atom); + if (n.escapeSink !== null) this.escape(atom, n.escapeSink); + } + } + + // ---- heap ---------------------------------------------------------------- + + field(atom: Atom, name: string): number { + let m = this.fields.get(atom); + if (m === undefined) { + m = new Map(); + this.fields.set(atom, m); + } + let id = m.get(name); + if (id === undefined) { + id = this.newNode(); + m.set(name, id); + if (this.traceTop) this.nodeLabel.set(id, `${atom.label}.${name}`); + const reason = this.escaped.get(atom); + if (reason !== undefined) { + this.sink(id, reason); + if (this.topComputedWrites.length > 0 || this.topWriteNames.has(name)) + this.add(id, this.top); + } + } + return id; + } + + element(atom: Atom): number { + let id = this.elements.get(atom); + if (id === undefined) { + id = this.newNode(); + this.elements.set(atom, id); + const reason = this.escaped.get(atom); + if (reason !== undefined) { + this.sink(id, reason); + if (this.topComputedWrites.length > 0) this.add(id, this.top); + } + } + return id; + } + + escape(atom: Atom, reason: string): void { + if (atom.kind === "prim" || atom.kind === "top") return; + if (atom.kind === "builtin" && atom.receiver === undefined) return; + if (this.escaped.has(atom)) return; + this.escaped.set(atom, reason); + const via = `via ${atom.label}`; + const fm = this.fields.get(atom); + if (fm !== undefined) { + for (const [name, id] of fm) { + this.sink(id, via); + if (this.topComputedWrites.length > 0 || this.topWriteNames.has(name)) + this.add(id, this.top); + } + } + const el = this.elements.get(atom); + if (el !== undefined) { + this.sink(el, via); + if (this.topComputedWrites.length > 0) this.add(el, this.top); + } + if (atom.receiver !== undefined) this.escape(atom.receiver, via); + if (atom.kind === "function") { + const info = this.infoOfAtom(atom); + if (info !== undefined) this.escapeFunction(info, via); + } + if ((atom.kind === "instance" || atom.kind === "class") && atom.node !== undefined) { + // Unknown code may call any member with any arguments. + for (const member of this.classMembers(atom.node, atom.kind === "class")) { + const info = this.functionInfo(member, this.ownerOf(member), this.receiverCtx(atom)); + this.add(info.thisNode, atom); + this.escapeFunction(info, via); + } + } + if (atom.kind === "promise") this.sink(this.field(atom, "value"), via); + } + + /** A function unknown code may call: ⊤ arguments and receiver; its + * results escape. */ + escapeFunction(info: FunctionInfo, reason: string): void { + if (this.escapedFunctionCalls.has(info)) return; + this.escapedFunctionCalls.add(info); + this.activate(info); + for (const p of info.params) this.add(p, this.top); + if (!info.isArrow && info.ownerClass === null) this.add(info.thisNode, this.top); + this.sink(info.ret, reason); + } + + /** Property names that untyped code writes on receivers that may be ⊤ (census). */ + noteTopWrite(name: string | null, node: Node): void { + if (name === null) { + const first = this.topComputedWrites.length === 0; + this.topComputedWrites.push(node); + if (first) { + for (const atom of [...this.escaped.keys()]) { + for (const id of (this.fields.get(atom) ?? new Map()).values()) + this.add(id, this.top); + const el = this.elements.get(atom); + if (el !== undefined) this.add(el, this.top); + } + } + return; + } + if (this.topWriteNames.has(name)) return; + this.topWriteNames.set(name, node); + for (const atom of [...this.escaped.keys()]) { + const id = this.fields.get(atom)?.get(name); + if (id !== undefined) this.add(id, this.top); + } + } + + // ---- classes --------------------------------------------------------------- + + /** The base class declaration of `decl`, via the checker; null at a + * builtin base or none; undefined when unresolved. */ + baseClass(decl: Node): Node | null | undefined { + const heritage = list((decl as ts.ClassLikeDeclaration).heritageClauses).find( + (c) => c.token === ts.SyntaxKind.ExtendsKeyword, + )?.types?.nodes[0]; + if (heritage === undefined) return null; + const e = unparen(heritage.expression); + if (!ts.isIdentifier(e) && !ts.isPropertyAccessExpression(e)) return undefined; + let d = this.resolve(ts.isIdentifier(e) ? e : e.name); + // `var C = class C { … }` (bundler output) + if ( + d !== undefined && + ts.isVariableDeclaration(d) && + d.initializer !== undefined && + ts.isClassExpression(unparen(d.initializer)) + ) + d = unparen(d.initializer); + if (d !== undefined && ts.isClassLikeDeclaration(d)) return d; + if (d !== undefined && d.getSourceFile().isDeclarationFile) return null; + return undefined; + } + + classChain(decl: Node): Node[] { + const out: Node[] = []; + let cur: Node | null | undefined = decl; + while (cur && out.length < 64 && !out.includes(cur)) { + out.push(cur); + cur = this.baseClass(cur); + } + return out; + } + + /** Method, accessor and constructor declarations of a class chain. */ + classMembers(decl: Node, statics: boolean): Node[] { + const out: Node[] = []; + for (const c of this.classChain(decl)) { + for (const m of list((c as ts.ClassLikeDeclaration).members)) { + if (isStatic(m) !== statics) continue; + if ( + ts.isMethodDeclaration(m) || + ts.isGetAccessorDeclaration(m) || + ts.isSetAccessorDeclaration(m) + ) + out.push(m); + else if (ts.isPropertyDeclaration(m) && m.initializer !== undefined) { + const init = unparen(m.initializer); + if (ts.isArrowFunction(init) || ts.isFunctionExpression(init)) out.push(init); + } + } + } + return out; + } + + lookupMember( + decl: Node, + name: string, + statics: boolean, + ): { + kind: "method" | "getter" | "setter" | "both"; + get?: Node; + set?: Node; + method?: Node; + } | null { + for (const c of this.classChain(decl)) { + let get: Node | undefined; + let set: Node | undefined; + for (const m of list((c as ts.ClassLikeDeclaration).members)) { + if (isStatic(m) !== statics) continue; + const mn = (m as { name?: Node }).name; + if (mn === undefined || propertyNameText(mn) !== name) continue; + if (ts.isMethodDeclaration(m)) return { kind: "method", method: m }; + if (ts.isGetAccessorDeclaration(m)) get = m; + if (ts.isSetAccessorDeclaration(m)) set = m; + if (ts.isPropertyDeclaration(m)) return null; // an own field: data + } + if (get || set) + return { + kind: get && set ? "both" : get ? "getter" : "setter", + ...(get ? { get } : {}), + ...(set ? { set } : {}), + }; + } + return null; + } + + // ---- symbols ----------------------------------------------------------------- + + resolve(id: Node): Node | undefined { + let sym = this.checker.getSymbolAtLocation(id); + if (sym === undefined) return undefined; + if (sym.flags & ts.SymbolFlags.Alias) { + try { + sym = this.checker.getAliasedSymbol(sym); + } catch { + return undefined; + } + } + return sym.valueDeclaration ?? sym.declarations[0]; + } + + binding(decl: Node): number { + const key = this.ctxKey(decl); + let id = this.bindings.get(key); + if (id === undefined) { + id = this.newNode(); + this.bindings.set(key, id); + if (this.traceTop) + this.nodeLabel.set( + id, + `binding ${(decl as { name?: { text?: string } }).name?.text ?? "?"}@${locLabel(decl)}${this.ctx !== "" ? ` [${this.ctx}]` : ""}`, + ); + } + return id; + } + + /** Bindings and expressions inside the function being cloned belong to + * the clone's context; everything else to the default context. */ + ctxKey(node: Node): string { + const root = this.ctxRoot; + const inside = + root !== null && + node.getSourceFile() === root.getSourceFile() && + node.pos >= root.pos && + node.end <= root.end; + return `${inside ? this.ctx : ""}|${this.nodeKey(node)}`; + } + + receiverCtx(atom: Atom): string { + return atom.kind === "instance" || atom.kind === "class" ? `C${atom.id}` : ""; + } + + infoOfAtom(atom: Atom): FunctionInfo | undefined { + if (atom.node === undefined) return undefined; + return this.functions.get(`${atom.ctx ?? ""}|${this.nodeKey(atom.node)}`); + } + + // ---- functions --------------------------------------------------------------- + + functionInfo(node: Node, ownerClass: Node | null, ctxArg = ""): FunctionInfo { + // A nested closure walked inside a clone shares the clone's context. + const inherit = + ctxArg === "" && + this.ctxRoot !== null && + node.getSourceFile() === this.ctxRoot.getSourceFile() && + node.pos >= this.ctxRoot.pos && + node.end <= this.ctxRoot.end; + const ctx = ctxArg !== "" ? ctxArg : inherit ? this.ctx : ""; + const root = ctxArg !== "" ? node : inherit ? this.ctxRoot : null; + const fkey = `${ctx}|${this.nodeKey(node)}`; + const existing = this.functions.get(fkey); + if (existing !== undefined) return existing; + const savedCtx = this.ctx; + const savedRoot = this.ctxRoot; + this.ctx = ctx; + this.ctxRoot = root; + const fn = node as ts.FunctionLikeDeclaration; + const isArrow = ts.isArrowFunction(node); + const params: number[] = []; + let rest = -1; + for (const p of list(fn.parameters)) { + const id = this.binding(p); + if (p.dotDotDotToken !== undefined) rest = params.length; + params.push(id); + } + const mods = list((fn as { modifiers?: { nodes: readonly Node[] } }).modifiers); + const sf = node.getSourceFile(); + const nameNode = (fn as { name?: Node }).name; + const name = nameNode !== undefined ? (propertyNameText(nameNode) ?? "") : null; + const owner = ownerClass !== null ? this.className(ownerClass) : null; + const label = `${owner !== null ? `${owner}.` : ""}${name ?? (ts.isConstructorDeclaration(node) ? "constructor" : "")}@${locLabel(node)}`; + const info: FunctionInfo = { + node, + ctx, + root, + walked: false, + label: ctx === "" ? label : `${label} [${this.ctxLabel(ctx)}]`, + params, + rest, + thisNode: isArrow && this.currentFn !== null ? this.currentFn.thisNode : this.newNode(), + ret: this.newNode(), + isArrow, + isAsync: mods.some((m) => m.kind === ts.SyntaxKind.AsyncKeyword), + isGenerator: (fn as { asteriskToken?: Node }).asteriskToken !== undefined, + usesArguments: false, + argPos: null, + ownerClass, + inPackage: isPackageJsFile(sf), + }; + this.functions.set(fkey, info); + this.ctx = savedCtx; + this.ctxRoot = savedRoot; + return info; + } + + /** Walks a function's body the first time it can run. */ + activate(info: FunctionInfo): void { + if (info.walked) return; + info.walked = true; + if (ts.isSourceFile(info.node) || ts.isClassLikeDeclaration(info.node)) return; + const fn = info.node as ts.FunctionLikeDeclaration; + const savedCtx = this.ctx; + const savedRoot = this.ctxRoot; + const saved = this.currentFn; + const savedLoop = this.loopDepth; + this.ctx = info.ctx; + this.ctxRoot = info.root; + this.currentFn = info; + this.loopDepth = 0; + for (const p of list(fn.parameters)) { + if (p.initializer !== undefined) this.edge(this.expr(p.initializer), this.binding(p)); + if (p.dotDotDotToken !== undefined) { + const arr = this.atoms.intern(`rest:${this.nodes.length}`, () => ({ + kind: "array", + label: `rest@${locLabel(p)}`, + node: p, + })); + this.add(this.binding(p), arr); + } + this.bindPattern(p.name, this.binding(p)); + } + const body = fn.body; + if (body !== undefined) { + if (ts.isBlock(body)) this.stmts(list(body.statements)); + else this.edge(this.expr(body as ts.Expression), info.ret); + } + this.currentFn = saved; + this.loopDepth = savedLoop; + this.ctx = savedCtx; + this.ctxRoot = savedRoot; + if (info.usesArguments) { + for (const p of info.params) this.add(p, this.top); + } + } + + /** The atom of a function node (memoized by its info). */ + fnAtom(node: Node, ownerClass: Node | null, ctx = ""): Atom { + const info = this.functionInfo(node, ownerClass, ctx); + return this.atoms.intern(`fn:${info.ret}`, () => ({ + kind: "function", + label: info.label, + node, + ctx: info.ctx, + })); + } + + ctxLabel(ctx: string): string { + if (ctx.startsWith("C")) return `this=${this.atoms.atoms[Number(ctx.slice(1))]?.label ?? ctx}`; + return ctx; + } + + className(decl: Node): string { + const own = (decl as ts.ClassLikeDeclaration).name?.text; + if (own !== undefined) return own; + const p = decl.parent; + if (p !== undefined && ts.isVariableDeclaration(p) && ts.isIdentifier(p.name)) + return p.name.text; + return ``; + } + + classAtoms(decl: Node): { cls: Atom; inst: Atom } { + const name = this.className(decl); + const cls = this.atoms.intern(`class:${this.nodeKey(decl)}`, () => ({ + kind: "class", + label: `class ${name}`, + node: decl, + })); + const inst = this.atoms.intern(`inst:${this.nodeKey(decl)}`, () => ({ + kind: "instance", + label: name, + node: decl, + })); + this.ensureClass(decl); + return { cls, inst }; + } + + private readonly nodeKeys = new Map(); + nodeKey(node: Node): number { + let k = this.nodeKeys.get(node); + if (k === undefined) { + k = this.nodeKeys.size; + this.nodeKeys.set(node, k); + } + return k; + } + + /** Registers a class: member functions, field initializers (as a pseudo + * function run at construction) and static initializers. */ + ensureClass(decl: Node): { initFn: FunctionInfo; staticInit: FunctionInfo } { + const existing = this.classInfo.get(decl); + if (existing !== undefined) return existing; + const c = decl as ts.ClassLikeDeclaration; + const label = this.className(decl); + const mk = (suffix: string): FunctionInfo => ({ + node: decl, + ctx: "", + root: null, + walked: true, + label: `${label}.${suffix}@${locLabel(decl)}`, + params: [], + rest: -1, + thisNode: this.newNode(), + ret: this.newNode(), + isArrow: false, + isAsync: false, + isGenerator: false, + usesArguments: false, + argPos: null, + ownerClass: decl, + inPackage: isPackageJsFile(decl.getSourceFile()), + }); + const entry = { initFn: mk(""), staticInit: mk("") }; + this.classInfo.set(decl, entry); + const cls = this.atoms.intern(`class:${this.nodeKey(decl)}`, () => ({ + kind: "class", + label: `class ${label}`, + node: decl, + })); + this.add(entry.staticInit.thisNode, cls); + const saved = this.currentFn; + for (const m of list(c.members)) { + const statics = isStatic(m); + if ( + ts.isMethodDeclaration(m) || + ts.isGetAccessorDeclaration(m) || + ts.isSetAccessorDeclaration(m) || + ts.isConstructorDeclaration(m) + ) { + continue; + } + if (ts.isPropertyDeclaration(m)) { + const name = propertyNameText(m.name); + const init = statics ? entry.staticInit : entry.initFn; + this.currentFn = init; + const value = + m.initializer !== undefined + ? this.exprIn(m.initializer, decl) + : this.constNode(this.prim("undefined")); + if (name !== null) this.storeConstraint(init.thisNode, name, value, m); + continue; + } + if (ts.isClassStaticBlockDeclaration(m)) { + this.currentFn = entry.staticInit; + this.stmts(list(m.body.statements)); + } + } + this.currentFn = saved; + return entry; + } + + /** An initializer expression: arrow functions in field initializers take + * the instance as `this` and belong to the class. */ + exprIn(e: ts.Expression, ownerClass: Node): number { + const u = unparen(e); + if (ts.isArrowFunction(u) || ts.isFunctionExpression(u)) { + const n = this.newNode(); + this.add(n, this.fnAtom(u, ts.isArrowFunction(u) ? null : ownerClass)); + return n; + } + return this.expr(e); + } + + /** `new C(args)` on a class atom: the instance, the field initializers of + * every class in the chain, and the constructor (or the implicit one). */ + construct( + decl: Node, + args: number[], + spread: boolean, + dst: number, + site: Node, + inst?: Atom, + ): void { + const atoms = this.classAtoms(decl); + const instance = inst ?? atoms.inst; + this.add(dst, instance); + this.callConstructorChain(decl, instance, args, spread, site); + } + + callConstructorChain( + decl: Node, + instance: Atom, + args: number[], + spread: boolean, + site: Node, + ): void { + const key = `ctor:${this.nodeKey(decl)}:${instance.id}:${this.nodeKey(site)}`; + if (this.boundCalls.has(key)) return; + this.boundCalls.add(key); + const entry = this.ensureClass(decl); + this.add(entry.initFn.thisNode, instance); + const ctor = list((decl as ts.ClassLikeDeclaration).members).find((m) => + ts.isConstructorDeclaration(m), + ); + if (ctor !== undefined) { + const info = this.functionInfo(ctor, decl, this.receiverCtx(instance)); + this.activate(info); + this.add(info.thisNode, instance); + this.bindArgs(info, args, spread); + this.edgeFor(site, info); + return; + } + const base = this.baseClass(decl); + if (base) this.callConstructorChain(base, instance, args, spread, site); + } + + bindArgs(info: FunctionInfo, args: number[], spread: boolean): void { + if (info.argPos !== null) { + for (let i = 0; i < info.argPos.length; i++) { + if (spread || i >= info.argPos.length - 1) { + for (const a of args.slice(i)) this.edge(a, info.argPos[i]!); + if (spread) this.add(info.argPos[i]!, this.top); + } else if (i < args.length) this.edge(args[i]!, info.argPos[i]!); + else this.add(info.argPos[i]!, this.prim("undefined")); + } + } + if (spread) { + for (const p of info.params) this.add(p, this.top); + for (const a of args) this.sink(a, `spread into ${info.label}`); + return; + } + for (let i = 0; i < info.params.length; i++) { + if (i === info.rest) { + const restAtoms = [...this.nodes[info.params[i]!]!.atoms]; + for (const ra of restAtoms) + for (let j = i; j < args.length; j++) this.edge(args[j]!, this.element(ra)); + break; + } + if (i < args.length) this.edge(args[i]!, info.params[i]!); + else this.add(info.params[i]!, this.prim("undefined")); + } + } + + /** Call of a function atom with a receiver node. */ + callFunction( + info: FunctionInfo, + thisNode: number | null, + args: number[], + spread: boolean, + dst: number, + site: Node, + ): void { + // `() => Object.create(null)` helpers: each call site gets its own + // dictionary, as if the helper were inlined. + if (this.isFactory(info.node)) { + const d = this.atoms.intern(`dict:${this.nodeKey(site)}`, () => ({ + kind: "dictionary", + label: `Object.create@${locLabel(site)}`, + node: site, + })); + this.add(dst, d); + return; + } + this.activate(info); + const fnNode = info.node; + const key = `call:${info.ctx}|${this.nodeKey(fnNode)}:${this.nodeKey(site)}:${thisNode ?? -1}:${dst}`; + if (this.boundCalls.has(key)) return; + this.boundCalls.add(key); + if (thisNode !== null && !info.isArrow) this.edge(thisNode, info.thisNode); + this.bindArgs(info, args, spread); + if (info.isAsync) { + const p = this.promiseAtom(fnNode, `async ${info.label}`); + this.awaitInto(info.ret, this.field(p, "value")); + this.add(dst, p); + } else if (info.isGenerator) { + this.add(dst, this.top); + } else this.edge(info.ret, dst); + this.edgeFor(site, info); + } + + private readonly factories = new Map(); + isFactory(node: Node): boolean { + let r = this.factories.get(node); + if (r === undefined) { + r = false; + const fn = node as ts.FunctionLikeDeclaration; + let e: Node | undefined = fn.body; + if (e !== undefined && ts.isBlock(e)) { + const st = e.statements.nodes; + e = st.length === 1 && ts.isReturnStatement(st[0]!) ? st[0]!.expression : undefined; + } + if (e !== undefined && !ts.isSourceFile(node)) { + const u = unparen(e as ts.Expression); + r = + ts.isCallExpression(u) && + u.expression.getText() === "Object.create" && + u.arguments.nodes.length === 1 && + u.arguments.nodes[0]!.kind === ts.SyntaxKind.NullKeyword; + } + this.factories.set(node, r); + } + return r; + } + + promiseAtom(site: Node, label: string): Atom { + return this.atoms.intern(`promise:${this.nodeKey(site)}`, () => ({ + kind: "promise", + label: `Promise@${label}`, + node: site, + })); + } + + /** dst ⊇ awaited(src). */ + awaitInto(src: number, dst: number): void { + this.on(src, (a) => { + if (a.kind === "promise") this.edge(this.field(a, "value"), dst); + else this.add(dst, a); + }); + } + + syntheticArray(site: Node, tag: string): Atom { + return this.atoms.intern(`arr:${tag}:${this.nodeKey(site)}`, () => ({ + kind: "array", + label: `${tag}[]@${locLabel(site)}`, + node: site, + })); + } + + // ---- statements ---------------------------------------------------------------- + + moduleBody(sf: ts.SourceFile): void { + const info: FunctionInfo = { + node: sf, + ctx: "", + root: null, + walked: true, + label: ``, + params: [], + rest: -1, + thisNode: this.constNode(this.prim("undefined")), + ret: this.newNode(), + isArrow: false, + isAsync: false, + isGenerator: false, + usesArguments: false, + argPos: null, + ownerClass: null, + inPackage: isPackageJsFile(sf), + }; + this.functions.set(`|${this.nodeKey(sf)}`, info); + this.currentFn = info; + this.stmts(list(sf.statements)); + this.currentFn = null; + } + + stmts(stmts: readonly Node[]): void { + for (const s of stmts) this.stmt(s); + } + + stmt(s: Node): void { + if (ts.isVariableStatement(s)) { + for (const d of list(s.declarationList.declarations)) this.varDecl(d); + return; + } + if (ts.isExpressionStatement(s)) { + this.expr(s.expression); + return; + } + if (ts.isReturnStatement(s)) { + if (s.expression !== undefined && this.currentFn !== null) + this.edge(this.expr(s.expression), this.currentFn.ret); + return; + } + if (ts.isFunctionDeclaration(s)) { + const b = this.binding(s); + const saved = this.currentFn; + this.currentFn = null; + this.add(b, this.fnAtom(s, null)); + this.currentFn = saved; + return; + } + if (ts.isClassDeclaration(s)) { + const b = this.binding(s); + this.add(b, this.classAtoms(s).cls); + return; + } + if (isLoop(s)) { + if (ts.isForOfStatement(s) || ts.isForInStatement(s)) { + const src = this.expr(s.expression); + const target = this.newNode(); + if (ts.isForInStatement(s)) this.add(target, this.prim("string")); + else this.iterateInto(src, target, s); + const init = s.initializer; + if (ts.isVariableDeclarationList(init)) { + for (const d of list(init.declarations)) { + this.edge(target, this.binding(d)); + this.bindPattern(d.name, this.binding(d)); + } + } else this.assignTo(init as ts.Expression, target); + } + this.loopDepth++; + if (ts.isForStatement(s)) { + if (s.initializer !== undefined) { + if (ts.isVariableDeclarationList(s.initializer)) + for (const d of list(s.initializer.declarations)) this.varDecl(d); + else this.expr(s.initializer as ts.Expression); + } + if (s.condition !== undefined) this.expr(s.condition); + if (s.incrementor !== undefined) this.expr(s.incrementor); + } + if (ts.isWhileStatement(s) || ts.isDoStatement(s)) this.expr(s.expression); + const body = (s as { statement?: Node }).statement; + if (body !== undefined) this.stmt(body); + this.loopDepth--; + return; + } + if (ts.isTryStatement(s)) { + this.stmts(list(s.tryBlock.statements)); + if (s.catchClause !== undefined) { + const v = s.catchClause.variableDeclaration; + if (v !== undefined) { + this.add(this.binding(v), this.top); + this.bindPattern(v.name, this.binding(v)); + } + this.stmts(list(s.catchClause.block.statements)); + } + if (s.finallyBlock !== undefined) this.stmts(list(s.finallyBlock.statements)); + return; + } + if (ts.isThrowStatement(s)) { + this.sink(this.expr(s.expression), `thrown at ${locLabel(s)}`); + return; + } + if ( + ts.isImportDeclaration(s) || + ts.isExportDeclaration(s) || + ts.isInterfaceDeclaration(s) || + ts.isTypeAliasDeclaration(s) + ) + return; + if (ts.isExportAssignment(s)) { + this.expr(s.expression); + return; + } + // Blocks, ifs, switches, labels: walk children, evaluating expressions. + ts.forEachChild(s, (child) => { + if (ts.isExpression(child)) this.expr(child); + else this.stmt(child); + return undefined; + }); + } + + varDecl(d: ts.VariableDeclaration): void { + const b = this.binding(d); + if (d.initializer !== undefined) this.edge(this.expr(d.initializer), b); + this.bindPattern(d.name, b); + } + + /** Destructuring: binding elements read from the source node. */ + bindPattern(name: Node, src: number): void { + if (ts.isIdentifier(name)) return; + if (ts.isObjectBindingPattern(name)) { + for (const el of list(name.elements)) { + const b = this.binding(el); + if (el.dotDotDotToken !== undefined) { + this.add(b, this.top); + continue; + } + const keyNode = el.propertyName ?? el.name; + const key = keyNode !== undefined ? propertyNameText(keyNode) : null; + if (key !== null) this.loadConstraint(src, key, b, el); + else this.add(b, this.top); + if (el.initializer !== undefined) this.edge(this.expr(el.initializer), b); + if (el.name !== undefined) this.bindPattern(el.name, b); + } + return; + } + if (ts.isArrayBindingPattern(name)) { + let i = 0; + for (const el of list(name.elements)) { + if (ts.isBindingElement(el)) { + const b = this.binding(el); + if (el.dotDotDotToken !== undefined) this.add(b, this.top); + else this.iterateInto(src, b, el, i); + if (el.initializer !== undefined) this.edge(this.expr(el.initializer), b); + if (el.name !== undefined) this.bindPattern(el.name, b); + } + i++; + } + } + } + + /** Element values of an iterable. */ + iterateInto(src: number, dst: number, site: Node, _index?: number): void { + this.on(src, (a) => { + if (a.kind === "array" || a.kind === "collection") this.edge(this.element(a), dst); + else if (a.kind === "prim" && a.label === "string") this.add(dst, this.prim("string")); + else if (a.kind === "prim" || a.kind === "object" || a.kind === "dictionary") { + /* not iterable (a TypeError): no value. Literals with a + Symbol.iterator member have a computed key; they are G sites. */ + } else { + this.add(dst, this.top); + if (a.kind !== "top") this.escape(a, `iterated at ${locLabel(site)}`); + } + }); + } + + // ---- expressions --------------------------------------------------------------- + + expr(e: ts.Expression): number { + const key = this.ctxKey(e); + const memo = this.exprMemo.get(key); + if (memo !== undefined) return memo; + const id = this.exprUncached(e); + this.exprMemo.set(key, id); + if (this.traceTop && !this.nodeLabel.has(id)) + this.nodeLabel.set( + id, + `expr ${e.getText().slice(0, 40).replace(/\s+/g, " ")}@${locLabel(e)}`, + ); + return id; + } + + private exprUncached(e: ts.Expression): number { + const u = unparen(e); + if (u !== e) return this.expr(u); + switch (e.kind) { + case ts.SyntaxKind.StringLiteral: + case ts.SyntaxKind.NoSubstitutionTemplateLiteral: + case ts.SyntaxKind.TemplateExpression: + if (ts.isTemplateExpression(e)) + for (const span of list(e.templateSpans)) this.expr(span.expression); + return this.constNode(this.prim("string")); + case ts.SyntaxKind.NumericLiteral: + return this.constNode(this.prim("number")); + case ts.SyntaxKind.BigIntLiteral: + return this.constNode(this.prim("bigint")); + case ts.SyntaxKind.TrueKeyword: + case ts.SyntaxKind.FalseKeyword: + return this.constNode(this.prim("boolean")); + case ts.SyntaxKind.NullKeyword: + return this.constNode(this.prim("null")); + case ts.SyntaxKind.RegularExpressionLiteral: + return this.constNode(this.builtin("RegExp.instance")); + case ts.SyntaxKind.ThisKeyword: + return this.currentFn !== null + ? this.currentFn.thisNode + : this.constNode(this.prim("undefined")); + case ts.SyntaxKind.SuperKeyword: + return this.constNode(this.top); + } + if (ts.isIdentifier(e)) return this.identifier(e); + if (ts.isObjectLiteralExpression(e)) return this.objectLiteral(e); + if (ts.isArrayLiteralExpression(e)) return this.arrayLiteral(e); + if (ts.isArrowFunction(e) || ts.isFunctionExpression(e)) { + const n = this.newNode(); + this.add(n, this.fnAtom(e, null)); + if (ts.isFunctionExpression(e) && e.name !== undefined) + this.add(this.binding(e), this.fnAtom(e, null)); + return n; + } + if (ts.isClassExpression(e)) { + const n = this.newNode(); + this.add(n, this.classAtoms(e).cls); + return n; + } + if (ts.isPropertyAccessExpression(e)) { + this.noteSite(e); + const recv = this.expr(e.expression); + const name = e.name.text; + const dst = this.newNode(); + this.loadConstraint(recv, name, dst, e); + this.ops.push({ kind: "read", recv, name, node: e }); + return dst; + } + if (ts.isElementAccessExpression(e)) { + const recv = this.expr(e.expression); + const key = unparen(e.argumentExpression); + const dst = this.newNode(); + const name = + ts.isStringLiteral(key) || ts.isNoSubstitutionTemplateLiteral(key) ? key.text : null; + if (name !== null) { + this.loadConstraint(recv, name, dst, e); + this.ops.push({ kind: "read", recv, name, node: e }); + } else { + this.expr(e.argumentExpression); + this.computedLoad(recv, dst, e); + this.ops.push({ kind: "computedRead", recv, name: null, node: e }); + } + return dst; + } + if (ts.isCallExpression(e)) return this.call(e); + if (ts.isNewExpression(e)) return this.newExpr(e); + if (ts.isBinaryExpression(e)) return this.binary(e); + if (ts.isConditionalExpression(e)) { + this.expr(e.condition); + const n = this.newNode(); + this.edge(this.expr(e.whenTrue), n); + this.edge(this.expr(e.whenFalse), n); + return n; + } + if (ts.isAwaitExpression(e)) { + const n = this.newNode(); + this.awaitInto(this.expr(e.expression), n); + return n; + } + if (ts.isTypeOfExpression(e)) { + this.expr(e.expression); + return this.constNode(this.prim("string")); + } + if (ts.isVoidExpression(e)) { + this.expr(e.expression); + return this.constNode(this.prim("undefined")); + } + if (ts.isDeleteExpression(e)) { + const target = unparen(e.expression); + if (ts.isPropertyAccessExpression(target) || ts.isElementAccessExpression(target)) { + const recv = this.expr(target.expression); + const name = ts.isPropertyAccessExpression(target) ? target.name.text : null; + this.ops.push({ kind: "delete", recv, name, node: e }); + } + return this.constNode(this.prim("boolean")); + } + if (ts.isPrefixUnaryExpression(e) || ts.isPostfixUnaryExpression(e)) { + const operand = e.operand; + this.expr(operand); + if ( + e.operator === ts.SyntaxKind.PlusPlusToken || + e.operator === ts.SyntaxKind.MinusMinusToken + ) + this.assignTo(operand, this.constNode(this.prim("number"))); + if (ts.isPrefixUnaryExpression(e) && e.operator === ts.SyntaxKind.ExclamationToken) + return this.constNode(this.prim("boolean")); + return this.constNode(this.prim("number")); + } + if (ts.isSpreadElement(e)) return this.expr(e.expression); + if (ts.isTaggedTemplateExpression(e)) { + this.expr(e.tag); + return this.constNode(this.top); + } + if (ts.isYieldExpression(e)) { + if (e.expression !== undefined) + this.sink(this.expr(e.expression), `yielded at ${locLabel(e)}`); + return this.constNode(this.top); + } + if (ts.isMetaProperty(e)) return this.constNode(this.top); + // Anything else: evaluate children, answer ⊤. + ts.forEachChild(e, (child) => { + if (ts.isExpression(child)) this.expr(child); + return undefined; + }); + return this.constNode(this.top); + } + + identifier(id: ts.Identifier): number { + if (id.text === "undefined") return this.constNode(this.prim("undefined")); + if (id.text === "arguments") { + const fn = this.enclosingNonArrow(); + if (fn !== null) { + const parent = id.parent; + const forwarding = parent !== undefined && ts.isSpreadElement(parent); + if (!forwarding) fn.usesArguments = true; + const arr = this.syntheticArray(id, "arguments"); + for (const p of fn.params) this.edge(p, this.element(arr)); + return this.constNode(arr); + } + } + const decl = this.resolve(id); + if (decl === undefined) return this.globalNode(id.text); + if (decl.getSourceFile().isDeclarationFile) return this.globalNode(id.text); + if (!this.files.includes(decl.getSourceFile())) return this.constNode(this.top); + if (ts.isFunctionDeclaration(decl)) { + const b = this.binding(decl); + const saved = this.currentFn; + this.currentFn = null; + this.add(b, this.fnAtom(decl, null)); + this.currentFn = saved; + return b; + } + if (ts.isClassDeclaration(decl) || ts.isClassExpression(decl)) { + const b = this.binding(decl); + this.add(b, this.classAtoms(decl).cls); + return b; + } + if (ts.isFunctionExpression(decl)) return this.binding(decl); + if ( + ts.isImportSpecifier(decl) || + ts.isImportClause(decl) || + ts.isNamespaceImport(decl) || + ts.isImportEqualsDeclaration(decl) + ) + return this.constNode(this.top); + if (ts.isSourceFile(decl) || ts.isModuleDeclaration(decl)) return this.namespaceNode(decl); + return this.narrowed(id, decl, this.binding(decl)); + } + + /** Flow narrowing at guarded references (S1 handles the guards hot code + * uses: `x instanceof C` and calls of predicate functions whose body is + * `… && x instanceof C`, in the true arm of an `if`, `?:` or `&&`). The + * narrowed value keeps the guard's class arms and ⊤. */ + narrowed(id: ts.Identifier, decl: Node, value: number): number { + let child: Node = id; + let p = id.parent; + while (p !== undefined && !ts.isFunctionLike(p) && !ts.isSourceFile(p)) { + let cond: ts.Expression | undefined; + if (ts.isIfStatement(p) && p.thenStatement === child) cond = p.expression; + else if (ts.isConditionalExpression(p) && p.whenTrue === child) cond = p.condition; + else if ( + ts.isBinaryExpression(p) && + p.operatorToken.kind === ts.SyntaxKind.AmpersandAmpersandToken && + p.right === child + ) + cond = p.left; + if (cond !== undefined) { + const cls = this.guardClass(cond, decl); + if (cls !== null) { + const out = this.newNode(); + this.on(value, (a) => { + if (a.kind === "top") this.add(out, a); + else if ( + cls === "Promise" + ? a.kind === "promise" + : a.kind === "instance" && + a.node !== undefined && + this.classChain(a.node).includes(cls) + ) + this.add(out, a); + }); + return out; + } + } + child = p; + p = p.parent; + } + return value; + } + + /** The class a condition proves `decl`'s value an instance of, if any. */ + guardClass(cond: ts.Expression, decl: Node): Node | "Promise" | null { + const e = unparen(cond); + if (ts.isBinaryExpression(e)) { + if (e.operatorToken.kind === ts.SyntaxKind.AmpersandAmpersandToken) + return this.guardClass(e.left, decl) ?? this.guardClass(e.right, decl); + if (e.operatorToken.kind !== ts.SyntaxKind.InstanceOfKeyword) return null; + const l = unparen(e.left); + if (!ts.isIdentifier(l) || this.resolve(l) !== decl) return null; + return this.instanceofTarget(e.right); + } + if ( + ts.isCallExpression(e) && + ts.isIdentifier(unparen(e.expression)) && + e.arguments.nodes.length === 1 + ) { + const arg = unparen(e.arguments.nodes[0]!); + if (!ts.isIdentifier(arg) || this.resolve(arg) !== decl) return null; + const fnDecl = this.resolve(unparen(e.expression)); + let fn: Node | undefined = fnDecl; + if (fn !== undefined && ts.isVariableDeclaration(fn) && fn.initializer !== undefined) + fn = unparen(fn.initializer); + if ( + fn === undefined || + !(ts.isArrowFunction(fn) || ts.isFunctionDeclaration(fn) || ts.isFunctionExpression(fn)) + ) + return null; + const param = fn.parameters.nodes[0]; + if (param === undefined || !ts.isIdentifier(param.name)) return null; + let body: Node | undefined = fn.body; + if (body !== undefined && ts.isBlock(body)) { + const stmts = body.statements.nodes; + body = + stmts.length === 1 && ts.isReturnStatement(stmts[0]!) ? stmts[0]!.expression : undefined; + } + if (body === undefined) return null; + return this.predicateClass(body as ts.Expression, param); + } + return null; + } + + /** `… && p instanceof C` in a predicate body over parameter `p`. */ + predicateClass(e: ts.Expression, param: Node): Node | "Promise" | null { + const u = unparen(e); + if (!ts.isBinaryExpression(u)) return null; + if (u.operatorToken.kind === ts.SyntaxKind.AmpersandAmpersandToken) + return this.predicateClass(u.right, param) ?? this.predicateClass(u.left, param); + if (u.operatorToken.kind !== ts.SyntaxKind.InstanceOfKeyword) return null; + const l = unparen(u.left); + if (!ts.isIdentifier(l) || this.resolve(l) !== param) return null; + return this.instanceofTarget(u.right); + } + + instanceofTarget(right: ts.Expression): Node | "Promise" | null { + const r = unparen(right); + if (!ts.isIdentifier(r)) return null; + const d = this.resolve(r); + if (d === undefined || d.getSourceFile().isDeclarationFile) + return r.text === "Promise" ? "Promise" : null; + return ts.isClassLikeDeclaration(d) ? d : null; + } + + private readonly namespaces = new Map(); + /** `import * as ns`: an object whose fields are the module's exports. */ + namespaceNode(mod: Node): number { + const existing = this.namespaces.get(mod); + if (existing !== undefined) return existing; + const n = this.newNode(); + this.namespaces.set(mod, n); + const atom = this.atoms.intern(`ns:${this.nodeKey(mod)}`, () => ({ + kind: "object", + label: `namespace ${shortFile((mod as ts.SourceFile).fileName ?? "")}`, + node: mod, + })); + this.add(n, atom); + const sym = this.checker.getSymbolAtLocation(mod); + if (sym !== undefined) { + for (const ex of this.checker.getExportsOfModule(sym)) { + let target = ex; + if (target.flags & ts.SymbolFlags.Alias) { + try { + target = this.checker.getAliasedSymbol(target); + } catch { + continue; + } + } + const d = target.valueDeclaration ?? target.declarations[0]; + if (d === undefined) continue; + const value = this.declValue(d); + if (value !== null) this.edge(value, this.field(atom, String(ex.escapedName))); + } + } + return n; + } + + declValue(d: Node): number | null { + if (d.getSourceFile().isDeclarationFile || !this.files.includes(d.getSourceFile())) return null; + if (ts.isFunctionDeclaration(d)) { + const b = this.binding(d); + const saved = this.currentFn; + this.currentFn = null; + this.add(b, this.fnAtom(d, null)); + this.currentFn = saved; + return b; + } + if (ts.isClassDeclaration(d)) { + const b = this.binding(d); + this.add(b, this.classAtoms(d).cls); + return b; + } + if (ts.isVariableDeclaration(d) || ts.isBindingElement(d) || ts.isParameter(d)) + return this.binding(d); + if (ts.isSourceFile(d)) return this.namespaceNode(d); + return null; + } + + globalNode(name: string): number { + if (MODELED_GLOBALS.has(name)) return this.constNode(this.builtin(name)); + if (name === "NaN" || name === "Infinity") return this.constNode(this.prim("number")); + return this.constNode(this.top); + } + + enclosingNonArrow(): FunctionInfo | null { + const fn = this.currentFn; + if (fn === null) return null; + if (!fn.isArrow) return fn; + // An arrow's `arguments` is its enclosing function's. + let p: Node | undefined = fn.node.parent; + while (p !== undefined) { + if (ts.isFunctionLike(p) && !ts.isArrowFunction(p)) + return ( + this.functions.get(this.ctxKey(p)) ?? this.functions.get(`|${this.nodeKey(p)}`) ?? null + ); + p = p.parent; + } + return null; + } + + objectLiteral(e: ts.ObjectLiteralExpression): number { + const keys: string[] = []; + let keysKnown = true; + let hasAccessor = false; + let hasProto = false; + let hasComputedKey = false; + const spreads: number[] = []; + const inPackage = isPackageJsFile(e.getSourceFile()); + const label = (): string => + `{${keys.slice(0, 6).join(",")}${keys.length > 6 ? ",…" : ""}${keysKnown ? "" : ",…"}}@${locLabel(e)}`; + // keys first, so the label is complete + for (const p of list(e.properties)) { + if (ts.isSpreadAssignment(p)) { + keysKnown = false; + continue; + } + const name = (p as { name?: Node }).name; + const key = name !== undefined ? propertyNameText(name) : null; + if (key === null) { + hasComputedKey = true; + keysKnown = false; + continue; + } + if (key === "__proto__" && ts.isPropertyAssignment(p) && !ts.isComputedPropertyName(p.name)) + hasProto = true; + if (!keys.includes(key)) keys.push(key); + } + const atom = this.atoms.intern(`obj:${this.nodeKey(e)}`, () => ({ + kind: "object", + label: label(), + node: e, + })); + const n = this.newNode(); + this.add(n, atom); + for (const p of list(e.properties)) { + if (ts.isSpreadAssignment(p)) { + const src = this.expr(p.expression); + spreads.push(src); + this.copyFields(src, atom, p); + continue; + } + if (ts.isGetAccessorDeclaration(p) || ts.isSetAccessorDeclaration(p)) { + hasAccessor = true; + const info = this.functionInfo(p, null); + this.activate(info); + this.add(info.thisNode, atom); + const key = propertyNameText(p.name); + if (key !== null && ts.isGetAccessorDeclaration(p)) + this.edge(info.ret, this.field(atom, key)); + continue; + } + const name = (p as { name?: Node }).name; + const key = name !== undefined ? propertyNameText(name) : null; + if (name !== undefined && ts.isComputedPropertyName(name)) this.expr(name.expression); + let value: number; + if (ts.isPropertyAssignment(p)) value = this.expr(p.initializer); + else if (ts.isShorthandPropertyAssignment(p)) { + const sym = this.checker.getShorthandAssignmentValueSymbol(p); + const d = sym?.valueDeclaration ?? sym?.declarations[0]; + const id = p.name as ts.Identifier; + value = d !== undefined ? (this.declValue(d) ?? this.identifier(id)) : this.identifier(id); + } else if (ts.isMethodDeclaration(p)) { + const fa = this.fnAtom(p, null); + this.add(this.infoOfAtom(fa)!.thisNode, atom); + value = this.constNode(fa); + } else continue; + if (key !== null) this.edge(value, this.field(atom, key)); + else this.edge(value, this.anyField(atom)); + } + if (!this.siteAtoms.has(atom) && this.siteAtoms.add(atom)) + this.sites.push({ + atom, + node: e, + kind: "object", + keys: keysKnown ? keys : null, + hasAccessor, + hasProto, + hasComputedKey, + spreads, + inPackage, + }); + return n; + } + + /** Values written under unknown keys. */ + anyField(atom: Atom): number { + return this.field(atom, "*"); + } + + copyFields(src: number, dstAtom: Atom, site: Node): void { + this.on(src, (a) => { + if (a.kind === "top") { + this.edge(this.constNode(this.top), this.anyField(dstAtom)); + return; + } + if (a.kind === "prim" || a.kind === "builtin") return; + if (a.kind === "array") { + this.edge(this.element(a), this.anyField(dstAtom)); + return; + } + // Copy every own field the source has, now and later. + const fm = this.fields.get(a); + const copied = new Set(); + const copy = (): void => { + for (const [name, id] of this.fields.get(a) ?? new Map()) { + if (copied.has(name)) continue; + copied.add(name); + this.edge(id, this.field(dstAtom, name)); + } + }; + copy(); + if (fm === undefined) this.fields.set(a, new Map()); + this.lateCopies.push(copy); + if (a.kind === "instance" || a.kind === "dictionary" || a.kind === "collection") + this.edge(this.constNode(this.top), this.anyField(dstAtom)); + void site; + }); + } + + /** Field copies re-run after solving rounds (fields created later). */ + private readonly lateCopies: (() => void)[] = []; + + arrayLiteral(e: ts.ArrayLiteralExpression): number { + const atom = this.atoms.intern(`arr:${this.nodeKey(e)}`, () => ({ + kind: "array", + label: `[]@${locLabel(e)}`, + node: e, + })); + const n = this.newNode(); + this.add(n, atom); + const el = this.element(atom); + let hasHole = false; + for (const item of list(e.elements)) { + if (ts.isOmittedExpression(item)) { + hasHole = true; + continue; + } + if (ts.isSpreadElement(item)) this.iterateInto(this.expr(item.expression), el, item); + else this.edge(this.expr(item), el); + } + if (!this.siteAtoms.has(atom) && this.siteAtoms.add(atom)) + this.sites.push({ + atom, + node: e, + kind: "array", + keys: null, + hasAccessor: false, + hasProto: hasHole, + hasComputedKey: false, + spreads: [], + inPackage: isPackageJsFile(e.getSourceFile()), + }); + return n; + } + + binary(e: ts.BinaryExpression): number { + const op = e.operatorToken.kind; + if (op === ts.SyntaxKind.EqualsToken) { + const value = this.expr(e.right); + this.assignTo(e.left, value); + return value; + } + if ( + op === ts.SyntaxKind.BarBarEqualsToken || + op === ts.SyntaxKind.QuestionQuestionEqualsToken || + op === ts.SyntaxKind.AmpersandAmpersandEqualsToken + ) { + const n = this.newNode(); + this.edge(this.expr(e.left), n); + const value = this.expr(e.right); + this.edge(value, n); + this.assignTo(e.left, value); + return n; + } + if (op >= ts.SyntaxKind.FirstCompoundAssignment && op <= ts.SyntaxKind.LastCompoundAssignment) { + this.expr(e.left); + this.expr(e.right); + const result = this.constNode( + op === ts.SyntaxKind.PlusEqualsToken ? this.top : this.prim("number"), + ); + if (op === ts.SyntaxKind.PlusEqualsToken) { + this.nodes[result]!.atoms.clear(); + this.add(result, this.prim("string")); + this.add(result, this.prim("number")); + } + this.assignTo(e.left, result); + return result; + } + if ( + op === ts.SyntaxKind.BarBarToken || + op === ts.SyntaxKind.QuestionQuestionToken || + op === ts.SyntaxKind.AmpersandAmpersandToken + ) { + const n = this.newNode(); + this.edge(this.expr(e.left), n); + this.edge(this.expr(e.right), n); + return n; + } + if (op === ts.SyntaxKind.CommaToken) { + this.expr(e.left); + return this.expr(e.right); + } + const l = this.expr(e.left); + const r = this.expr(e.right); + switch (op) { + case ts.SyntaxKind.EqualsEqualsEqualsToken: + case ts.SyntaxKind.ExclamationEqualsEqualsToken: + case ts.SyntaxKind.EqualsEqualsToken: + case ts.SyntaxKind.ExclamationEqualsToken: + case ts.SyntaxKind.LessThanToken: + case ts.SyntaxKind.LessThanEqualsToken: + case ts.SyntaxKind.GreaterThanToken: + case ts.SyntaxKind.GreaterThanEqualsToken: + case ts.SyntaxKind.InstanceOfKeyword: + case ts.SyntaxKind.InKeyword: + if (op === ts.SyntaxKind.InKeyword) + this.ops.push({ kind: "computedRead", recv: r, name: null, node: e }); + return this.constNode(this.prim("boolean")); + case ts.SyntaxKind.PlusToken: { + const n = this.newNode(); + const strOrNum = (a: Atom): void => { + this.add( + n, + a.kind === "prim" && a.label === "number" ? this.prim("number") : this.prim("string"), + ); + if ( + a.kind === "prim" && + (a.label === "boolean" || a.label === "null" || a.label === "undefined") + ) + this.add(n, this.prim("number")); + if (a.kind === "top") this.add(n, this.prim("number")); + }; + this.on(l, strOrNum); + this.on(r, strOrNum); + return n; + } + default: + return this.constNode(this.prim("number")); + } + } + + assignTo(target: ts.Expression, value: number): void { + const t = unparen(target); + if (ts.isIdentifier(t)) { + const decl = this.resolve(t); + if (decl === undefined || decl.getSourceFile().isDeclarationFile) { + this.sink(value, `global write at ${locLabel(t)}`); + return; + } + this.edge(value, this.binding(decl)); + return; + } + if (ts.isPropertyAccessExpression(t)) { + const recv = this.expr(t.expression); + this.storeConstraint(recv, t.name.text, value, t); + this.ops.push({ + kind: t.name.text === "length" ? "lengthWrite" : "write", + recv, + name: t.name.text, + node: t, + }); + return; + } + if (ts.isElementAccessExpression(t)) { + const recv = this.expr(t.expression); + const key = unparen(t.argumentExpression); + if (ts.isStringLiteral(key) || ts.isNoSubstitutionTemplateLiteral(key)) { + this.storeConstraint(recv, key.text, value, t); + this.ops.push({ kind: "write", recv, name: key.text, node: t }); + } else { + this.expr(t.argumentExpression); + this.computedStore(recv, value, t); + this.ops.push({ kind: "computedWrite", recv, name: null, node: t }); + } + return; + } + if (ts.isObjectLiteralExpression(t)) { + for (const p of list(t.properties)) { + if (ts.isPropertyAssignment(p)) { + const key = propertyNameText(p.name); + const n = this.newNode(); + if (key !== null) this.loadConstraint(value, key, n, p); + else this.add(n, this.top); + this.assignTo(p.initializer, n); + } else if (ts.isShorthandPropertyAssignment(p)) { + const n = this.newNode(); + const id = p.name as ts.Identifier; + this.loadConstraint(value, id.text, n, p); + this.assignTo(id, n); + } + } + return; + } + if (ts.isArrayLiteralExpression(t)) { + for (const el of list(t.elements)) { + if (ts.isOmittedExpression(el)) continue; + const n = this.newNode(); + this.iterateInto(value, n, el); + this.assignTo(ts.isSpreadElement(el) ? el.expression : el, n); + } + } + } + + // ---- loads and stores ------------------------------------------------------------ + + loadConstraint(recv: number, name: string, dst: number, site: Node): void { + this.on(recv, (a) => this.loadAtom(a, name, dst, site)); + } + + loadAtom(a: Atom, name: string, dst: number, site: Node): void { + switch (a.kind) { + case "top": + this.add(dst, this.top); + return; + case "prim": + if (name === "length" && a.label === "string") this.add(dst, this.prim("number")); + else if (a.label === "string" || a.label === "number" || a.label === "boolean") + this.add(dst, this.builtin(`${a.label}.${name}`, a)); + return; + case "instance": { + const decl = a.node!; + if (name === "constructor") { + this.add(dst, this.classAtoms(decl).cls); + return; + } + const member = this.lookupMember(decl, name, false); + if (member?.kind === "method") + this.add( + dst, + this.fnAtom(member.method!, this.ownerOf(member.method!), this.receiverCtx(a)), + ); + if (member !== null && member.get !== undefined) { + const g = this.functionInfo(member.get, this.ownerOf(member.get), this.receiverCtx(a)); + this.activate(g); + this.add(g.thisNode, a); + this.edge(g.ret, dst); + this.edgeFor(site, g); + return; + } + if (this.classChain(decl).some((c) => this.baseClass(c) === undefined)) + this.add(dst, this.top); + this.edge(this.field(a, name), dst); + return; + } + case "class": { + const decl = a.node!; + if (name === "prototype") { + this.add(dst, this.top); + return; + } + if (name === "name") { + this.add(dst, this.prim("string")); + return; + } + const member = this.lookupMember(decl, name, true); + if (member?.kind === "method") + this.add( + dst, + this.fnAtom(member.method!, this.ownerOf(member.method!), this.receiverCtx(a)), + ); + if (member?.get !== undefined) { + const g = this.functionInfo(member.get, this.ownerOf(member.get), this.receiverCtx(a)); + this.activate(g); + this.add(g.thisNode, a); + this.edge(g.ret, dst); + return; + } + // static fields of the class and its bases + for (const c of this.classChain(decl)) { + const cls = this.classAtoms(c).cls; + this.edge(this.field(cls, name), dst); + } + return; + } + case "function": + if (name === "call" || name === "apply" || name === "bind") + this.add(dst, this.builtin(`function.${name}`, a)); + else if (name === "prototype") this.add(dst, this.top); + else if (name === "name") this.add(dst, this.prim("string")); + else if (name === "length") this.add(dst, this.prim("number")); + else this.edge(this.field(a, name), dst); + return; + case "array": + if (name === "length") this.add(dst, this.prim("number")); + else if (/^\d+$/.test(name)) this.edge(this.element(a), dst); + else if (ARRAY_METHODS.has(name)) this.add(dst, this.builtin(`array.${name}`, a)); + else this.edge(this.field(a, name), dst); + return; + case "object": + this.edge(this.field(a, name), dst); + this.edge(this.anyField(a), dst); + if (name === "constructor" || name === "hasOwnProperty" || name === "toString") + this.add(dst, this.builtin(`object.${name}`, a)); + return; + case "dictionary": + this.edge(this.anyField(a), dst); + this.edge(this.field(a, name), dst); + return; + case "promise": + if (name === "then" || name === "catch" || name === "finally") + this.add(dst, this.builtin(`promise.${name}`, a)); + return; + case "collection": + if (name === "size") this.add(dst, this.prim("number")); + else this.add(dst, this.builtin(`collection.${name}`, a)); + return; + case "builtin": + this.loadBuiltin(a, name, dst); + return; + } + void site; + } + + ownerOf(member: Node): Node | null { + const p = member.parent; + return p !== undefined && ts.isClassLikeDeclaration(p) ? p : null; + } + + loadBuiltin(a: Atom, name: string, dst: number): void { + const path = a.path ?? ""; + if ( + path === "Number" && + (name === "MAX_SAFE_INTEGER" || + name === "MIN_SAFE_INTEGER" || + name === "EPSILON" || + name === "MAX_VALUE" || + name === "MIN_VALUE" || + name === "POSITIVE_INFINITY" || + name === "NEGATIVE_INFINITY" || + name === "NaN") + ) + this.add(dst, this.prim("number")); + else if (path === "Math" && /^[A-Z0-9_]+$/.test(name)) this.add(dst, this.prim("number")); + else if (path === "Symbol") this.add(dst, this.prim("symbol")); + else if (path === "Error.instance" || path === "error") { + if (name === "message" || name === "stack" || name === "name") + this.add(dst, this.prim("string")); + else this.add(dst, this.top); + } else if ( + a.receiver === undefined && + MODELED_GLOBALS.has(path.split(".")[0]!) && + path.split(".").length < 4 + ) + this.add(dst, this.builtin(`${path}.${name}`)); + else if (path === "RegExp.instance") this.add(dst, this.builtin(`regexp.${name}`, a)); + else this.add(dst, this.top); + } + + storeConstraint(recv: number, name: string, value: number, site: Node): void { + this.on(recv, (a) => { + switch (a.kind) { + case "top": + this.noteTopWrite(name, site); + this.sink(value, `stored into ⊤.${name} at ${locLabel(site)}`); + return; + case "instance": { + const member = this.lookupMember(a.node!, name, false); + if (member?.set !== undefined) { + const s = this.functionInfo(member.set, this.ownerOf(member.set), this.receiverCtx(a)); + this.activate(s); + this.add(s.thisNode, a); + if (s.params[0] !== undefined) this.edge(value, s.params[0]); + return; + } + this.edge(value, this.field(a, name)); + return; + } + case "class": + case "function": + case "object": + case "dictionary": + this.edge(value, this.field(a, name)); + return; + case "array": + if (/^\d+$/.test(name)) this.edge(value, this.element(a)); + return; + case "prim": + return; + default: + this.sink(value, `stored into ${a.label} at ${locLabel(site)}`); + } + }); + } + + computedLoad(recv: number, dst: number, site: Node): void { + this.on(recv, (a) => { + switch (a.kind) { + case "top": + this.add(dst, this.top); + return; + case "array": + case "collection": + this.edge(this.element(a), dst); + return; + case "prim": + if (a.label === "string") this.add(dst, this.prim("string")); + return; + case "object": + case "dictionary": + case "instance": + case "class": + case "function": { + // every field, now and later + const seen = new Set(); + const copy = (): void => { + for (const [name, id] of this.fields.get(a) ?? new Map()) { + if (seen.has(name)) continue; + seen.add(name); + this.edge(id, dst); + } + }; + copy(); + this.lateCopies.push(copy); + if (a.kind === "instance" || a.kind === "class") this.add(dst, this.top); + return; + } + default: + this.add(dst, this.top); + } + void site; + }); + } + + computedStore(recv: number, value: number, site: Node): void { + this.on(recv, (a) => { + switch (a.kind) { + case "top": + this.noteTopWrite(null, site); + this.sink(value, `stored into ⊤[k] at ${locLabel(site)}`); + return; + case "array": + this.edge(value, this.element(a)); + return; + case "prim": + return; + default: + this.edge(value, this.anyField(a)); + } + }); + } + + // ---- calls --------------------------------------------------------------------------- + + args(e: ts.CallExpression | ts.NewExpression): { args: number[]; spread: boolean } { + const out: number[] = []; + let spread = false; + for (const a of list(e.arguments)) { + if (ts.isSpreadElement(a)) { + const inner = unparen(a.expression); + if (ts.isIdentifier(inner) && inner.text === "arguments") { + // `f(...arguments)`: forward the enclosing function's parameters. + const fn = this.enclosingNonArrow(); + if (fn !== null && out.length === 0) { + if (fn.argPos === null) fn.argPos = [0, 1, 2, 3, 4, 5].map(() => this.newNode()); + out.push(...fn.argPos); + continue; + } + } + spread = true; + out.push(this.expr(a.expression)); + continue; + } + out.push(this.expr(a)); + } + return { args: out, spread }; + } + + call(e: ts.CallExpression): number { + this.noteSite(e); + const callee = unparen(e.expression); + const dst = this.newNode(); + if (callee.kind === ts.SyntaxKind.SuperKeyword) { + const { args, spread } = this.args(e); + const fn = this.enclosingNonArrow(); + const owner = fn?.ownerClass ?? null; + const base = owner !== null ? this.baseClass(owner) : undefined; + if (fn !== null && base) { + this.on(fn.thisNode, (inst) => { + if (inst.kind === "instance") this.callConstructorChain(base, inst, args, spread, e); + }); + } else if (base === undefined) + for (const a of args) this.sink(a, `super(...) at ${locLabel(e)}`); + return dst; + } + if (ts.isPropertyAccessExpression(callee) || ts.isElementAccessExpression(callee)) { + const recvExpr = unparen(callee.expression); + const { args, spread } = this.args(e); + if (recvExpr.kind === ts.SyntaxKind.SuperKeyword && ts.isPropertyAccessExpression(callee)) { + // super.m(...): the base class member with the current receiver. + const fn = this.enclosingNonArrow(); + const owner = fn?.ownerClass ?? null; + const base = owner !== null ? this.baseClass(owner) : undefined; + if (fn !== null && base) { + const member = this.lookupMember(base, callee.name.text, isStatic(fn.node)); + if (member?.method !== undefined) { + this.callFunction( + this.functionInfo(member.method, this.ownerOf(member.method), fn.ctx), + fn.thisNode, + args, + spread, + dst, + e, + ); + } + } + return dst; + } + const recv = this.expr(recvExpr); + let name: string | null = null; + if (ts.isPropertyAccessExpression(callee)) name = callee.name.text; + else { + const k = unparen(callee.argumentExpression); + if (ts.isStringLiteral(k) || ts.isNoSubstitutionTemplateLiteral(k)) name = k.text; + else this.expr(callee.argumentExpression); + } + if (name !== null) this.ops.push({ kind: "read", recv, name, node: callee }); + // Per receiver atom: the member it resolves to runs with that atom as `this`. + const member = name; + this.on(recv, (a) => { + const fnNode = this.newNode(); + const self = this.constNode(a); + if (member !== null) this.loadAtom(a, member, fnNode, callee); + else this.computedLoad(self, fnNode, callee); + this.invoke(fnNode, self, args, spread, dst, e); + }); + return dst; + } + const calleeNode = this.expr(callee); + const { args, spread } = this.args(e); + this.invoke(calleeNode, null, args, spread, dst, e); + return dst; + } + + invoke( + callee: number, + recv: number | null, + args: number[], + spread: boolean, + dst: number, + site: ts.CallExpression, + ): void { + const undefNode = recv === null ? this.constNode(this.prim("undefined")) : recv; + this.on(callee, (a) => { + switch (a.kind) { + case "function": + this.callFunction(this.infoOfAtom(a)!, undefNode, args, spread, dst, site); + return; + case "builtin": + this.callBuiltin(a, args, spread, dst, site, recv); + return; + case "class": + return; // calling a class throws + case "prim": + return; + default: + for (const x of args) this.sink(x, `argument of ⊤ call at ${locLabel(site)}`); + if (recv !== null && a.kind === "top") + this.sink(recv, `receiver of ⊤ call at ${locLabel(site)}`); + this.add(dst, this.top); + } + }); + } + + newExpr(e: ts.NewExpression): number { + this.noteSite(e); + const dst = this.newNode(); + const callee = this.expr(e.expression); + const { args, spread } = this.args(e); + this.on(callee, (a) => { + switch (a.kind) { + case "class": + this.construct(a.node!, args, spread, dst, e); + return; + case "function": { + // An ES5 constructor function: one instance atom per function. + const inst = this.atoms.intern(`fninst:${this.nodeKey(a.node!)}`, () => ({ + kind: "object", + label: `new ${a.label}`, + node: a.node, + })); + this.add(dst, inst); + const info = this.infoOfAtom(a); + if (info !== undefined) { + this.activate(info); + this.add(info.thisNode, inst); + this.bindArgs(info, args, spread); + this.edge(info.ret, dst); + } + return; + } + case "builtin": + this.constructBuiltin(a.path ?? "", args, dst, e); + return; + default: + for (const x of args) this.sink(x, `argument of ⊤ constructor at ${locLabel(e)}`); + this.add(dst, this.top); + } + }); + return dst; + } + + constructBuiltin(path: string, args: number[], dst: number, site: Node): void { + switch (path) { + case "Map": + case "Set": + case "WeakMap": + case "WeakSet": { + const c = this.atoms.intern(`coll:${this.nodeKey(site)}`, () => ({ + kind: "collection", + label: `${path}@${locLabel(site)}`, + node: site, + })); + this.add(dst, c); + if (args[0] !== undefined) this.iterateInto(args[0], this.element(c), site); + return; + } + case "Error": + case "TypeError": + case "RangeError": + case "SyntaxError": + this.add(dst, this.builtin("Error.instance")); + return; + case "Promise": { + const p = this.promiseAtom(site, locLabel(site)); + this.add(this.field(p, "value"), this.top); + this.add(dst, p); + if (args[0] !== undefined) { + const resolver = this.constNode(this.builtin("promise.resolver")); + this.on(args[0], (f) => { + if (f.kind === "function") + this.callFunction( + this.infoOfAtom(f)!, + null, + [resolver, resolver], + false, + this.newNode(), + site, + ); + }); + } + return; + } + case "Date": + this.add(dst, this.builtin("Date.instance")); + return; + case "RegExp": + this.add(dst, this.builtin("RegExp.instance")); + return; + case "Array": { + this.add(dst, this.syntheticArray(site, "Array")); + return; + } + default: + for (const x of args) this.sink(x, `argument of new ${path} at ${locLabel(site)}`); + this.add(dst, this.top); + } + } + + /** Builtin call summaries; anything unmodeled makes its arguments escape. */ + callBuiltin( + a: Atom, + args: number[], + spread: boolean, + dst: number, + site: ts.CallExpression, + recvNode: number | null, + ): void { + const path = a.path ?? ""; + const recv = a.receiver; + const arg = (i: number): number => args[i] ?? this.constNode(this.prim("undefined")); + const prim = (p: string): void => this.add(dst, this.prim(p)); + const callback = (i: number, cbArgs: number[], ret: number | null): void => { + if (args[i] === undefined) return; + this.on(args[i]!, (f) => { + if (f.kind === "function") { + const saved = this.forceLoop; + this.forceLoop = LOOP_CALLBACK_METHODS.has(path.slice(path.indexOf(".") + 1)); + this.callFunction(this.infoOfAtom(f)!, null, cbArgs, false, ret ?? this.newNode(), site); + this.forceLoop = saved; + } else if (f.kind === "top" || f.kind === "builtin") { + for (const x of cbArgs) this.sink(x, `argument of ⊤ callback at ${locLabel(site)}`); + if (ret !== null) this.add(ret, this.top); + } + }); + }; + const escapeAll = (): void => { + for (const x of args) this.sink(x, `argument of ${path} at ${locLabel(site)}`); + this.add(dst, this.top); + }; + // Methods of builtin receivers. + if (recv !== undefined) { + const method = path.slice(path.indexOf(".") + 1); + if (path.startsWith("array.")) { + const el = this.element(recv); + const num = this.constNode(this.prim("number")); + const recvN = this.constNode(recv); + switch (method) { + case "push": + case "unshift": + for (const x of args) { + if (spread) this.iterateInto(x, el, site); + else this.edge(x, el); + } + prim("number"); + return; + case "pop": + case "shift": + case "at": + this.edge(el, dst); + this.add(dst, this.prim("undefined")); + return; + case "find": + case "findLast": + callback(0, [el, num, recvN], null); + this.edge(el, dst); + this.add(dst, this.prim("undefined")); + return; + case "forEach": + callback(0, [el, num, recvN], null); + prim("undefined"); + return; + case "some": + case "every": + case "includes": + callback(0, [el, num, recvN], null); + prim("boolean"); + return; + case "findIndex": + case "findLastIndex": + case "indexOf": + case "lastIndexOf": + callback(0, [el, num, recvN], null); + prim("number"); + return; + case "join": + case "toString": + prim("string"); + return; + case "map": + case "flatMap": { + const out = this.syntheticArray(site, method); + const outEl = this.element(out); + if (method === "map") callback(0, [el, num, recvN], outEl); + else { + const r = this.newNode(); + callback(0, [el, num, recvN], r); + this.iterateInto(r, outEl, site); + } + this.add(dst, out); + return; + } + case "filter": + case "slice": + case "concat": + case "reverse": + case "sort": + case "splice": + case "toSorted": + case "toReversed": + case "flat": { + if (method === "filter") callback(0, [el, num, recvN], null); + if (method === "sort" || method === "toSorted") callback(0, [el, el], null); + if (method === "reverse" || method === "sort") { + this.add(dst, recv); + return; + } + const out = this.syntheticArray(site, method); + this.edge(el, this.element(out)); + if (method === "concat") + for (const x of args) this.iterateInto(x, this.element(out), site); + if (method === "splice") for (const x of args.slice(2)) this.edge(x, el); + this.add(dst, out); + return; + } + case "reduce": + case "reduceRight": { + const acc = this.newNode(); + if (args[1] !== undefined) this.edge(args[1], acc); + else this.edge(el, acc); + callback(0, [acc, el, num, recvN], acc); + this.edge(acc, dst); + return; + } + case "keys": + case "values": + case "entries": { + const out = this.syntheticArray(site, method); + if (method === "values") this.edge(el, this.element(out)); + else if (method === "keys") this.add(this.element(out), this.prim("number")); + else { + const pair = this.syntheticArray(site, "entry"); + this.add(this.element(pair), this.prim("number")); + this.edge(el, this.element(pair)); + this.add(this.element(out), pair); + } + this.add(dst, out); + return; + } + case "fill": + for (const x of args) this.edge(x, el); + this.add(dst, recv); + return; + } + escapeAll(); + this.escape(recv, `array.${method} at ${locLabel(site)}`); + return; + } + if (path.startsWith("string.") || path.startsWith("number.") || path.startsWith("boolean.")) { + const r = STRING_METHOD_RESULTS.get(method); + if (r === "array") { + const out = this.syntheticArray(site, method); + this.add(this.element(out), this.prim("string")); + this.add(this.element(out), this.prim("undefined")); + this.add(dst, out); + this.add(dst, this.prim("null")); + } else if (r !== undefined) prim(r); + else if (method === "replace" || method === "replaceAll") prim("string"); + else escapeAll(); + if (method === "replace" || method === "replaceAll") + callback(1, [this.constNode(this.prim("string"))], null); + return; + } + if (path.startsWith("regexp.")) { + if (method === "test") prim("boolean"); + else if (method === "exec") { + const out = this.syntheticArray(site, "exec"); + this.add(this.element(out), this.prim("string")); + this.add(dst, out); + prim("null"); + } else escapeAll(); + return; + } + if (path.startsWith("promise.")) { + const out = this.promiseAtom(site, locLabel(site)); + const value = this.field(recv, "value"); + const ret = this.newNode(); + if (method === "then") { + callback(0, [value], ret); + callback(1, [this.constNode(this.top)], ret); + } else if (method === "catch") { + callback(0, [this.constNode(this.top)], ret); + this.edge(value, ret); + } else { + callback(0, [], null); + this.edge(value, ret); + } + this.awaitInto(ret, this.field(out, "value")); + this.add(dst, out); + return; + } + if (path.startsWith("collection.")) { + const el = this.element(recv); + switch (method) { + case "add": + case "set": + for (const x of args) this.edge(x, el); + this.add(dst, recv); + return; + case "get": + this.edge(el, dst); + prim("undefined"); + return; + case "has": + case "delete": + prim("boolean"); + return; + case "clear": + prim("undefined"); + return; + case "forEach": + callback(0, [el, el], null); + prim("undefined"); + return; + case "keys": + case "values": + case "entries": { + const out = this.syntheticArray(site, method); + this.edge(el, this.element(out)); + if (method === "entries") { + const pair = this.syntheticArray(site, "entry"); + this.edge(el, this.element(pair)); + this.nodes[this.element(out)]!.atoms.clear(); + this.add(this.element(out), pair); + } + this.add(dst, out); + return; + } + } + escapeAll(); + return; + } + if (path.startsWith("function.")) { + if (recv.kind !== "function" || recv.node === undefined) { + escapeAll(); + return; + } + if (method === "bind") { + const info = this.infoOfAtom(recv); + if (info !== undefined && args[0] !== undefined && !info.isArrow) + this.edge(args[0], info.thisNode); + if (info !== undefined && args.length > 1) { + this.activate(info); + for (const p of info.params) this.add(p, this.top); + for (const x of args.slice(1)) this.sink(x, `bound argument at ${locLabel(site)}`); + } + this.add(dst, recv); + return; + } + if (method === "call") { + this.callFunction( + this.infoOfAtom(recv)!, + args[0] ?? this.constNode(this.prim("undefined")), + args.slice(1), + spread, + dst, + site, + ); + return; + } + // apply: arguments from an array + const info = this.infoOfAtom(recv); + if (info !== undefined) { + this.activate(info); + if (args[0] !== undefined && !info.isArrow) this.edge(args[0], info.thisNode); + for (const p of info.params) + if (args[1] !== undefined) this.iterateInto(args[1], p, site); + this.edge(info.ret, dst); + } + return; + } + if (path.startsWith("object.")) { + if (method === "hasOwnProperty") prim("boolean"); + else if (method === "toString") prim("string"); + else escapeAll(); + return; + } + escapeAll(); + return; + } + // `Object.prototype.m.call(o, …)`-style calls of builtin prototype methods. + if (/^(Object|Array|String|Function)\.prototype\.\w+\.(call|apply)$/.test(path)) { + const m = path.split(".")[2]!; + if (m === "hasOwnProperty" || m === "propertyIsEnumerable" || m === "isPrototypeOf") + prim("boolean"); + else if (m === "toString") prim("string"); + else escapeAll(); + return; + } + switch (path) { + case "Object.keys": + case "Object.getOwnPropertyNames": { + const out = this.syntheticArray(site, "keys"); + this.add(this.element(out), this.prim("string")); + this.add(dst, out); + return; + } + case "Object.values": + case "Object.entries": { + const out = this.syntheticArray(site, path === "Object.values" ? "values" : "entries"); + const vals = this.newNode(); + if (args[0] !== undefined) this.computedLoad(args[0], vals, site); + if (path === "Object.values") this.edge(vals, this.element(out)); + else { + const pair = this.syntheticArray(site, "entry"); + this.add(this.element(pair), this.prim("string")); + this.edge(vals, this.element(pair)); + this.add(this.element(out), pair); + } + this.add(dst, out); + return; + } + case "Object.freeze": + case "Object.seal": + case "Object.preventExtensions": + this.ops.push({ + kind: path === "Object.freeze" ? "freeze" : "seal", + recv: arg(0), + name: null, + node: site, + }); + this.edge(arg(0), dst); + return; + case "Object.isFrozen": + case "Object.isSealed": + case "Object.isExtensible": + case "Array.isArray": + case "Number.isInteger": + case "Number.isNaN": + case "Number.isFinite": + case "Number.isSafeInteger": + case "isNaN": + case "isFinite": + prim("boolean"); + return; + case "Object.assign": { + this.ops.push({ kind: "assign", recv: arg(0), name: null, node: site }); + this.on(arg(0), (t) => { + for (const s of args.slice(1)) this.copyFields(s, t, site); + if (t.kind === "top") + for (const s of args.slice(1)) + this.sink(s, `Object.assign into ⊤ at ${locLabel(site)}`); + }); + this.edge(arg(0), dst); + return; + } + case "Object.defineProperty": + case "Object.defineProperties": + case "Reflect.defineProperty": + this.ops.push({ kind: "defineProperty", recv: arg(0), name: null, node: site }); + this.sink(arg(2), `property descriptor at ${locLabel(site)}`); + this.sink(arg(1), `property descriptor at ${locLabel(site)}`); + this.edge(arg(0), dst); + return; + case "Object.setPrototypeOf": + case "Reflect.setPrototypeOf": + this.ops.push({ kind: "setPrototypeOf", recv: arg(0), name: null, node: site }); + this.sink(arg(1), `prototype at ${locLabel(site)}`); + this.edge(arg(0), dst); + return; + case "Object.create": { + const d = this.atoms.intern(`dict:${this.nodeKey(site)}`, () => ({ + kind: "dictionary", + label: `Object.create@${locLabel(site)}`, + node: site, + })); + this.add(dst, d); + return; + } + case "Object.fromEntries": { + const d = this.atoms.intern(`dict:${this.nodeKey(site)}`, () => ({ + kind: "dictionary", + label: `fromEntries@${locLabel(site)}`, + node: site, + })); + const pairs = this.newNode(); + if (args[0] !== undefined) this.iterateInto(args[0], pairs, site); + const vals = this.newNode(); + this.iterateInto(pairs, vals, site); + this.edge(vals, this.anyField(d)); + this.add(dst, d); + return; + } + case "Object.getPrototypeOf": + case "Reflect.getPrototypeOf": + this.add(dst, this.top); + return; + case "Object.is": + prim("boolean"); + return; + case "Array.from": + case "Array.of": { + const out = this.syntheticArray(site, path); + if (path === "Array.of") for (const x of args) this.edge(x, this.element(out)); + else if (args[1] !== undefined) { + const els = this.newNode(); + if (args[0] !== undefined) this.iterateInto(args[0], els, site); + callback(1, [els, this.constNode(this.prim("number"))], this.element(out)); + } else if (args[0] !== undefined) this.iterateInto(args[0], this.element(out), site); + this.add(dst, out); + return; + } + case "JSON.parse": + this.add(dst, this.top); + return; + case "JSON.stringify": + prim("string"); + prim("undefined"); + return; + case "String": + case "encodeURIComponent": + case "decodeURIComponent": + case "encodeURI": + case "decodeURI": + prim("string"); + return; + case "Number": + case "parseInt": + case "parseFloat": + case "Number.parseInt": + case "Number.parseFloat": + prim("number"); + return; + case "Boolean": + prim("boolean"); + return; + case "Symbol": + case "Symbol.for": + prim("symbol"); + return; + case "BigInt": + prim("bigint"); + return; + case "Promise.resolve": { + const p = this.promiseAtom(site, locLabel(site)); + if (args[0] !== undefined) this.awaitInto(args[0], this.field(p, "value")); + else this.add(this.field(p, "value"), this.prim("undefined")); + this.add(dst, p); + return; + } + case "Promise.reject": { + const p = this.promiseAtom(site, locLabel(site)); + this.add(dst, p); + if (args[0] !== undefined) this.sink(args[0], `rejection at ${locLabel(site)}`); + return; + } + case "Promise.all": + case "Promise.allSettled": + case "Promise.race": + case "Promise.any": { + const p = this.promiseAtom(site, locLabel(site)); + const items = this.newNode(); + if (args[0] !== undefined) this.iterateInto(args[0], items, site); + if (path === "Promise.all") { + const out = this.syntheticArray(site, "all"); + this.awaitInto(items, this.element(out)); + this.add(this.field(p, "value"), out); + } else if (path === "Promise.race" || path === "Promise.any") + this.awaitInto(items, this.field(p, "value")); + else this.add(this.field(p, "value"), this.top); + this.add(dst, p); + return; + } + case "promise.resolver": + for (const x of args) this.sink(x, `promise resolution at ${locLabel(site)}`); + return; + } + if (path.startsWith("console.")) { + prim("undefined"); + return; + } + if (path.startsWith("Math.")) { + prim("number"); + return; + } + if (path === "Date.now") { + prim("number"); + return; + } + if (path.startsWith("Date.instance") || path === "Error.instance") { + escapeAll(); + return; + } + void recvNode; + escapeAll(); + } + + /** Re-run deferred field copies until nothing new flows. */ + finish(): void { + for (;;) { + const before = this.steps; + for (const c of this.lateCopies) c(); + this.solve(); + if (this.steps === before || this.budgetExceeded) return; + } + } +} diff --git a/packages/compiler/src/frontend/shape-inference/domain.ts b/packages/compiler/src/frontend/shape-inference/domain.ts new file mode 100644 index 000000000..36978d4c9 --- /dev/null +++ b/packages/compiler/src/frontend/shape-inference/domain.ts @@ -0,0 +1,153 @@ +/* Shape inference (results/design-shape-inference/DESIGN.md), stage S1: + * the abstract domain of the allocation-site points-to analysis. + * + * An abstract value is a set of atoms. An atom names a set of runtime + * values: a primitive kind, an allocation site (object or array literal, or + * a value a builtin creates at one call site), the instances of one class, + * one class value, one function (closure site), a builtin object, or ⊤ — + * every value the analysis does not track. Value nodes hold atom sets and + * flow into each other along inclusion edges; the solver (analyze.ts) + * attaches load, store and call constraints that fire as atoms arrive. */ +import type * as ts from "../ts7/adapter.js"; + +export type AtomKind = + | "prim" + | "top" + | "object" + | "array" + | "instance" + | "class" + | "function" + | "builtin" + | "promise" + | "collection" + | "dictionary"; + +export interface Atom { + readonly id: number; + readonly kind: AtomKind; + /** Short display name, e.g. `string`, `ZodString`, `{status,value}@types.js:804`. */ + readonly label: string; + /** The syntax that creates the atom: the literal, class or function + * declaration, or the builtin call. Undefined for prims, ⊤ and builtins. */ + readonly node?: ts.Node | undefined; + /** builtin atoms: the dotted path (`Object.keys`, `array.push`). */ + readonly path?: string | undefined; + /** method atoms of builtin receivers: the receiver atom. */ + readonly receiver?: Atom | undefined; + /** function atoms: the analysis context of the function's clone (a + * method's receiver class), "" for the context-insensitive default. */ + readonly ctx?: string | undefined; +} + +export class AtomTable { + readonly atoms: Atom[] = []; + private readonly byKey = new Map(); + + intern(key: string, make: (id: number) => Omit): Atom { + let atom = this.byKey.get(key); + if (atom === undefined) { + const id = this.atoms.length; + atom = { id, ...make(id) }; + this.atoms.push(atom); + this.byKey.set(key, atom); + } + return atom; + } +} + +/** Object.prototype members: a read of one of these names on a plain + * object resolves on the prototype, never to an absent own key. */ +export const OBJECT_PROTOTYPE_MEMBERS: ReadonlySet = new Set([ + "constructor", + "hasOwnProperty", + "isPrototypeOf", + "propertyIsEnumerable", + "toLocaleString", + "toString", + "valueOf", + "__defineGetter__", + "__defineSetter__", + "__lookupGetter__", + "__lookupSetter__", + "__proto__", +]); + +/** Global names the analysis models as builtin objects. Any other global is ⊤. */ +export const MODELED_GLOBALS: ReadonlySet = new Set([ + "Object", + "Array", + "JSON", + "Math", + "Number", + "String", + "Boolean", + "Symbol", + "Promise", + "Map", + "Set", + "WeakMap", + "WeakSet", + "Error", + "TypeError", + "RangeError", + "SyntaxError", + "Date", + "RegExp", + "console", + "Reflect", + "parseInt", + "parseFloat", + "isNaN", + "isFinite", + "encodeURIComponent", + "decodeURIComponent", + "encodeURI", + "decodeURI", + "BigInt", +]); + +/** Results of string (and number) methods, by name. Anything else is ⊤. */ +type MethodResult = "string" | "number" | "boolean" | "array"; +export const STRING_METHOD_RESULTS: ReadonlyMap = new Map< + string, + MethodResult +>([ + ["at", "string"], + ["charAt", "string"], + ["charCodeAt", "number"], + ["codePointAt", "number"], + ["concat", "string"], + ["endsWith", "boolean"], + ["includes", "boolean"], + ["indexOf", "number"], + ["lastIndexOf", "number"], + ["localeCompare", "number"], + ["match", "array"], + ["matchAll", "array"], + ["normalize", "string"], + ["padEnd", "string"], + ["padStart", "string"], + ["repeat", "string"], + ["replace", "string"], + ["replaceAll", "string"], + ["search", "number"], + ["slice", "string"], + ["split", "array"], + ["startsWith", "boolean"], + ["substring", "string"], + ["substr", "string"], + ["toLowerCase", "string"], + ["toUpperCase", "string"], + ["toLocaleLowerCase", "string"], + ["toLocaleUpperCase", "string"], + ["toString", "string"], + ["trim", "string"], + ["trimEnd", "string"], + ["trimStart", "string"], + ["valueOf", "string"], + ["test", "boolean"], + ["toFixed", "string"], + ["toPrecision", "string"], + ["toExponential", "string"], +]); diff --git a/packages/compiler/src/frontend/shape-inference/index.ts b/packages/compiler/src/frontend/shape-inference/index.ts new file mode 100644 index 000000000..88c2100ee --- /dev/null +++ b/packages/compiler/src/frontend/shape-inference/index.ts @@ -0,0 +1,21 @@ +/* Shape inference for untyped npm JavaScript + * (results/design-shape-inference/DESIGN.md). Stage S1: report only. */ +import type * as ts from "../ts7/adapter.js"; +import { analyzeShapes } from "./analyze.js"; +import { buildShapeReport } from "./report.js"; + +export { analyzeShapes } from "./analyze.js"; +export { buildShapeReport } from "./report.js"; + +/** SCRIPTC_DEBUG_SHAPES: `1` reports package code in hot functions, + * `verbose` every package site, `all` user files too, any other value only + * files whose path contains it. Writes to stderr; changes nothing else. */ +export function reportShapes(program: ts.Program, filter: string): void { + const start = performance.now(); + const analysis = analyzeShapes(program, program.getTypeChecker()); + const report = buildShapeReport(analysis, filter); + const ms = (performance.now() - start).toFixed(0); + process.stderr.write( + `[shapes] ${ms} ms\n${report.lines.map((l) => `[shapes] ${l}`).join("\n")}\n`, + ); +} diff --git a/packages/compiler/src/frontend/shape-inference/report.ts b/packages/compiler/src/frontend/shape-inference/report.ts new file mode 100644 index 000000000..c18d9eb56 --- /dev/null +++ b/packages/compiler/src/frontend/shape-inference/report.ts @@ -0,0 +1,374 @@ +/* Shape inference stage S1: the plan the analysis would hand the lowering, + * as a report (DESIGN.md §4.5, §5.5). Nothing here changes compilation. + * + * Per allocation site of package code, the verdict: + * P local — a native record would be exact: no reflection, no added + * keys, no computed writes, and its objects never escape; + * P escaping — exact as a record behind identity-keeping capsules: it + * escapes, but the census leaves every later operation + * answerable (fields untyped code may write become dyn); + * G — keeps today's dynamic object, with the rule that decided. + * Plus class field types, the census, and the hot functions' value types. */ +import * as ts from "../ts7/adapter.js"; +import { OBJECT_PROTOTYPE_MEMBERS, type Atom } from "./domain.js"; +import { locLabel, type FunctionInfo, type LiteralSite, type ShapeAnalysis } from "./analyze.js"; + +export interface SiteVerdict { + readonly site: LiteralSite; + readonly label: string; + readonly verdict: "P-local" | "P-escaping" | "G"; + readonly reasons: string[]; + /** P-escaping: fields untyped code may write (they become dyn slots). */ + readonly writableByTop: string[]; + readonly freezable: boolean; + readonly brand: boolean; +} + +export interface ShapeReport { + readonly sites: SiteVerdict[]; + readonly hot: Set; + readonly lines: string[]; +} + +const MAX_ATOMS_SHOWN = 6; + +function fieldKeys(fields: ReadonlyMap | undefined): string[] { + return fields === undefined ? [] : [...fields.keys()]; +} + +function fieldEntries(fields: ReadonlyMap | undefined): [string, number][] { + return fields === undefined ? [] : [...fields.entries()]; +} + +export function describeValue(analysis: ShapeAnalysis, node: number): string { + const atoms = [...analysis.nodes[node]!.atoms]; + if (atoms.length === 0) return "∅"; + const labels = [...new Set(atoms.map((a) => a.label))]; + labels.sort((a, b) => (a === "⊤" ? 1 : b === "⊤" ? -1 : a.localeCompare(b))); + const shown = labels.slice(0, MAX_ATOMS_SHOWN).join(" | "); + if (labels.length <= MAX_ATOMS_SHOWN) return shown; + return `${shown} | +${labels.length - MAX_ATOMS_SHOWN}${labels.includes("⊤") ? " | ⊤" : ""}`; +} + +function atomsOf(analysis: ShapeAnalysis, node: number): Set { + return analysis.nodes[node]!.atoms; +} + +/** Functions reached from call sites in loops (or loop callbacks), and + * everything they call: the static hotness heuristic (DESIGN.md §8.1). */ +export function hotFunctions(analysis: ShapeAnalysis): Set { + const hot = new Set(); + const callees = new Map(); + const work: FunctionInfo[] = []; + for (const c of analysis.calls) { + let l = callees.get(c.caller); + if (l === undefined) callees.set(c.caller, (l = [])); + l.push(c.callee); + if (c.inLoop && !hot.has(c.callee)) { + hot.add(c.callee); + work.push(c.callee); + } + } + while (work.length > 0) { + const f = work.pop()!; + for (const g of callees.get(f) ?? []) { + if (!hot.has(g)) { + hot.add(g); + work.push(g); + } + } + } + return hot; +} + +export function buildShapeReport(analysis: ShapeAnalysis, filter: string): ShapeReport { + const lines: string[] = []; + const all = filter === "all" || filter === "verbose"; + const verbose = filter === "verbose"; + const fileFilter = filter !== "1" && !all ? filter : null; + const wanted = (n: ts.Node, inPackage: boolean): boolean => { + if (fileFilter !== null) return n.getSourceFile().fileName.includes(fileFilter); + return all || inPackage; + }; + + // Census facts: operations whose receiver may be ⊤ reach every escaped atom. + + // Index ops by the site atoms they can reach directly. + const byAtom = new Map(); + const topOps: typeof analysis.ops = []; + for (const o of analysis.ops) { + const atoms = atomsOf(analysis, o.recv); + for (const a of atoms) { + if (a.kind !== "object" && a.kind !== "array") continue; + let l = byAtom.get(a); + if (l === undefined) byAtom.set(a, (l = [])); + l.push(o); + } + if (atoms.has(analysis.top)) topOps.push(o); + } + + // Key-order groups: sites with the same key set and different orders need brands. + const orders = new Map>(); + for (const s of analysis.sites) { + if (s.kind !== "object" || s.keys === null) continue; + const set = [...s.keys].sort().join(","); + let o = orders.get(set); + if (o === undefined) orders.set(set, (o = new Set())); + o.add(s.keys.join(",")); + } + + const verdicts: SiteVerdict[] = []; + for (const s of analysis.sites) { + if (!s.inPackage && !all && fileFilter === null) continue; + const reasons: string[] = []; + const escapedWhy = analysis.escaped.get(s.atom); + const escapes = escapedWhy !== undefined; + if (s.hasAccessor) reasons.push("accessor in literal"); + if (s.hasProto) reasons.push(s.kind === "array" ? "hole in literal" : "__proto__ in literal"); + if (s.hasComputedKey) reasons.push("computed key in literal"); + for (const sp of s.spreads) + if (atomsOf(analysis, sp).has(analysis.top)) reasons.push("spread of ⊤"); + const direct = byAtom.get(s.atom) ?? []; + const ops = escapes ? [...direct, ...topOps] : direct; + const keys = new Set(s.keys ?? []); + const fieldsOf = analysis.fields.get(s.atom); + const knownKeys = + s.keys !== null ? keys : new Set([...fieldKeys(fieldsOf)].filter((k) => k !== "*")); + let freezable = false; + const seen = new Set(); + const note = (r: string): void => { + if (!seen.has(r)) { + seen.add(r); + reasons.push(r); + } + }; + for (const o of ops) { + const viaTop = !atomsOf(analysis, o.recv).has(s.atom); + const where = `${locLabel(o.node)}${viaTop ? " via ⊤" : ""}`; + switch (o.kind) { + case "delete": + case "defineProperty": + case "setPrototypeOf": + case "seal": + case "proxy": + note(`${o.kind} at ${where}`); + break; + case "assign": + note(`Object.assign target at ${where}`); + break; + case "freeze": + freezable = true; + break; + case "computedWrite": + if (!viaTop) + note(s.kind === "array" ? `index write at ${where}` : `computed write at ${where}`); + break; + case "lengthWrite": + if (s.kind === "array") note(`length write at ${where}`); + break; + case "write": + if (s.kind === "object" && !viaTop && o.name !== null && !knownKeys.has(o.name)) + note(`adds key '${o.name}' at ${where}`); + break; + case "read": + if ( + s.kind === "object" && + !underPromiseGuard(o.node) && + escapes && + o.name !== null && + !viaTop && + !knownKeys.has(o.name) && + !OBJECT_PROTOTYPE_MEMBERS.has(o.name) + ) + note(`reads absent key '${o.name}' at ${where} (escapes)`); + break; + default: + break; + } + } + let brand = false; + if (s.kind === "object" && s.keys !== null) { + const o = orders.get([...s.keys].sort().join(",")); + if (o !== undefined && o.size > 1) brand = true; + } + const writableByTop: string[] = []; + if (escapes) { + if (analysis.topComputedWrites.length > 0) writableByTop.push("*"); + else for (const k of knownKeys) if (analysis.topWriteNames.has(k)) writableByTop.push(k); + } + const verdict: SiteVerdict["verdict"] = + reasons.length > 0 ? "G" : escapes ? "P-escaping" : "P-local"; + const label = s.atom.label; + if (escapes) reasons.push(`escapes: ${escapedWhy}`); + verdicts.push({ site: s, label, verdict, reasons, writableByTop, freezable, brand }); + } + + const hot = hotFunctions(analysis); + const hotNodes = new Set([...hot].map((f) => f.node)); + + // ---- print ---- + const pkgFns = [...analysis.functions.values()].filter((f) => f.inPackage); + lines.push( + `analyzed ${analysis.files.length} files (${analysis.packageFiles} package JS), ${analysis.functions.size} functions (${pkgFns.length} in packages), ${analysis.atoms.atoms.length} atoms, ${analysis.nodes.length} value nodes, ${analysis.steps} steps${analysis.budgetExceeded ? " — BUDGET EXCEEDED, results incomplete" : ""}`, + ); + const topNames = [...analysis.topWriteNames.entries()].map( + ([n, node]) => `${n} (${locLabel(node)})`, + ); + lines.push( + `census: writes on ⊤ receivers: ${topNames.length === 0 ? "none" : topNames.slice(0, 20).join(", ")}${topNames.length > 20 ? ` +${topNames.length - 20}` : ""}`, + ); + lines.push( + `census: computed writes on ⊤ receivers: ${analysis.topComputedWrites.length === 0 ? "none" : analysis.topComputedWrites.slice(0, 8).map(locLabel).join(", ")}`, + ); + const topReflect = topOps.filter( + (o) => + o.kind !== "read" && + o.kind !== "computedRead" && + o.kind !== "write" && + o.kind !== "computedWrite", + ); + lines.push( + `census: reflection on ⊤ receivers: ${topReflect.length === 0 ? "none" : topReflect.map((o) => `${o.kind} (${locLabel(o.node)})`).join(", ")}`, + ); + const escapedPkg = [...analysis.escaped.keys()].filter( + (a) => + a.node !== undefined && + wanted(a.node, a.node.getSourceFile().fileName.includes("/node_modules/")), + ); + lines.push( + `escaped atoms: ${analysis.escaped.size} (${escapedPkg.length} shown below where relevant)`, + ); + const firstEscapes = [...analysis.escaped.entries()] + .filter(([, why]) => !why.startsWith("via ")) + .slice(0, 12); + for (const [atom, why] of firstEscapes) lines.push(` escape root: ${atom.label}: ${why}`); + + const counts = { "P-local": 0, "P-escaping": 0, G: 0 }; + const hotCounts = { "P-local": 0, "P-escaping": 0, G: 0 }; + const inHot = (n: ts.Node): boolean => { + let p: ts.Node | undefined = n.parent; + while (p !== undefined) { + if (hotNodes.has(p)) return true; + p = p.parent; + } + return false; + }; + for (const v of verdicts) { + counts[v.verdict]++; + if (inHot(v.site.node)) hotCounts[v.verdict]++; + } + lines.push( + `sites: ${verdicts.length} (P local ${counts["P-local"]}, P escaping ${counts["P-escaping"]}, G ${counts.G}); in hot functions: P local ${hotCounts["P-local"]}, P escaping ${hotCounts["P-escaping"]}, G ${hotCounts.G}`, + ); + for (const v of verdicts) { + if (!wanted(v.site.node, v.site.inPackage)) continue; + if (!verbose && !inHot(v.site.node)) continue; + const extra: string[] = []; + if (v.brand) extra.push("needs brand (key order)"); + if (v.freezable) extra.push("freezable"); + if (v.writableByTop.length > 0) extra.push(`⊤-writable: ${v.writableByTop.join(",")}`); + const fields = analysis.fields.get(v.site.atom); + const ftypes = + v.site.kind === "array" + ? `elem: ${analysis.elements.has(v.site.atom) ? describeValue(analysis, analysis.elements.get(v.site.atom)!) : "∅"}` + : fieldEntries(fields) + .filter(([k]) => k !== "*") + .map(([k, id]) => `${k}: ${describeValue(analysis, id)}`) + .join("; "); + lines.push( + `site ${v.label}${inHot(v.site.node) ? " [hot]" : ""}: ${v.verdict}${v.reasons.length > 0 ? ` — ${v.reasons.join("; ")}` : ""}${extra.length > 0 ? ` [${extra.join("; ")}]` : ""}`, + ); + if (ftypes.length > 0) lines.push(` ${ftypes}`); + } + + // Class fields. + for (const atom of analysis.atoms.atoms) { + if (atom.kind !== "instance" || atom.node === undefined) continue; + const inPackage = atom.node.getSourceFile().fileName.includes("/node_modules/"); + if (!wanted(atom.node, inPackage)) continue; + const fm = analysis.fields.get(atom); + if (fm === undefined || fm.size === 0) continue; + const esc = analysis.escaped.get(atom); + const parts = [...fm.entries()].map(([k, id]) => { + const w = + esc !== undefined && + !k.startsWith("#") && + (analysis.topComputedWrites.length > 0 || analysis.topWriteNames.has(k)) + ? " (⊤-writable)" + : ""; + return `${k}: ${describeValue(analysis, id)}${w}`; + }); + lines.push( + `class ${atom.label}${esc !== undefined ? ` (escapes: ${esc})` : ""}: ${parts.join("; ")}`, + ); + } + + // Hot functions. + const hotPkg = [...hot].filter((info) => wanted(info.node, info.inPackage)); + lines.push(`hot functions: ${hotPkg.length} (clones per receiver class counted separately)`); + for (const info of hotPkg) { + const n = info.node; + const params = info.params.map((p, i) => `p${i}=${describeValue(analysis, p)}`).join(", "); + const flags = [ + info.usesArguments ? "uses arguments" : "", + analysis.escaped.size > 0 && [...analysis.escaped.keys()].some((a) => a.node === n) + ? "escapes" + : "", + ].filter((x) => x !== ""); + lines.push( + `fn ${info.label}${flags.length > 0 ? ` [${flags.join(", ")}]` : ""}: this=${describeValue(analysis, info.thisNode)}${params.length > 0 ? `, ${params}` : ""}, ret=${describeValue(analysis, info.ret)}`, + ); + } + const why = process.env["SCRIPTC_DEBUG_SHAPES_WHY"]; + if (why !== undefined) { + for (const info of analysis.functions.values()) { + if (!info.label.startsWith(why)) continue; + for (const [what, node] of [ + ["ret", info.ret], + ...info.params.map((p, i) => [`p${i}`, p] as const), + ] as const) { + if (!analysis.nodes[node]!.atoms.has(analysis.top)) continue; + lines.push(`why ⊤ in ${info.label} ${what}:`); + let cur: number | string | undefined = node; + for (let i = 0; i < 40 && typeof cur === "number"; i++) { + lines.push(` ${analysis.nodeLabel.get(cur) ?? `node ${cur}`}`); + cur = analysis.topCause.get(cur); + } + if (typeof cur === "string") lines.push(` seed: ${cur}`); + } + } + } + return { sites: verdicts, hot, lines }; +} + +/** A read inside the true arm of a promise test (`isAsync(x)`, + * `x instanceof Promise`): flow-sensitive narrowing removes the record + * arms there. Stage S1's locals are flow-insensitive, so the report + * recognizes the guard syntactically. */ +function underPromiseGuard(node: ts.Node): boolean { + let child: ts.Node = node; + let p = node.parent; + while (p !== undefined && !ts.isFunctionLike(p)) { + let cond: ts.Expression | undefined; + if (ts.isIfStatement(p) && p.thenStatement === child) cond = p.expression; + if (ts.isConditionalExpression(p) && p.whenTrue === child) cond = p.condition; + if (cond !== undefined && isPromiseTest(cond)) return true; + child = p; + p = p.parent; + } + return false; +} + +function isPromiseTest(e: ts.Expression): boolean { + if (ts.isParenthesizedExpression(e)) return isPromiseTest(e.expression); + if (ts.isBinaryExpression(e)) { + if (e.operatorToken.kind === ts.SyntaxKind.InstanceOfKeyword) + return ts.isIdentifier(e.right) && e.right.text === "Promise"; + if (e.operatorToken.kind === ts.SyntaxKind.AmpersandAmpersandToken) + return isPromiseTest(e.left) || isPromiseTest(e.right); + return false; + } + if (ts.isCallExpression(e) && ts.isIdentifier(e.expression)) + return /^is(Async|Promise|Thenable)$/.test(e.expression.text); + return false; +} diff --git a/packages/compiler/test/shape-inference.test.ts b/packages/compiler/test/shape-inference.test.ts new file mode 100644 index 000000000..3cd6e809d --- /dev/null +++ b/packages/compiler/test/shape-inference.test.ts @@ -0,0 +1,166 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import { compile } from "../src/index.js"; + +// A package shaped like a validation library: schema classes with a virtual +// `_parse`, per-field context and result literals, a class holding the +// context, and an output object built with computed keys. +const packageSource = ` +export class Status { + constructor() { this.value = "valid"; } + dirty() { if (this.value === "valid") this.value = "dirty"; } +} +class LazyPath { + constructor(parent, data, key) { this.parent = parent; this.data = data; this.key = key; } + get path() { return [this.key]; } +} +export class Base { + constructor(def) { this._def = def; } + ctxOf(input) { return { common: input.parent.common, data: input.data, parent: input.parent }; } + parse(data) { + const root = { common: { issues: [] }, data, parent: null }; + const result = this._parse({ data, parent: root }); + return result.value; + } +} +export class Str extends Base { + _parse(input) { + const status = new Status(); + if (typeof input.data !== "string") status.dirty(); + return { status: status.value, value: input.data }; + } +} +export class Obj extends Base { + _parse(input) { + const ctx = this.ctxOf(input); + const pairs = []; + for (const key of Object.keys(this._def.shape)) { + const field = this._def.shape[key]; + pairs.push({ key, value: field._parse(new LazyPath(ctx, ctx.data[key], key)) }); + } + const out = {}; + for (const pair of pairs) out[pair.key] = pair.value.value; + return { status: "valid", value: out }; + } +} +export const str = () => new Str({ checks: [] }); +export const obj = (shape) => new Obj({ shape }); +export function leak(value) { globalThis.leaked = value; return value; } +export function reflect() { const o = { a: 1 }; delete o.a; return o; } +`; + +const packageTypes = ` +export declare class Base { parse(data: unknown): unknown; } +export declare class Str extends Base {} +export declare class Obj extends Base {} +export declare const str: () => Str; +export declare const obj: (shape: Record) => Obj; +export declare function leak(value: T): T; +export declare function reflect(): object; +`; + +const mainSource = ` +import { leak, obj, reflect, str } from "shapes-fixture"; +const schema = obj({ a: str(), b: str() }); +const input: unknown = JSON.parse('{"a":"x","b":"y"}'); +console.log(JSON.stringify(schema.parse(input))); +console.log(JSON.stringify(leak({ kept: 1 })), JSON.stringify(reflect())); +`; + +let dir: string; +let saved: string | undefined; +beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), "scriptc-shapes-")); + const pkg = join(dir, "node_modules", "shapes-fixture"); + await mkdir(pkg, { recursive: true }); + await writeFile( + join(pkg, "package.json"), + JSON.stringify({ + name: "shapes-fixture", + version: "1.0.0", + type: "module", + main: "index.js", + types: "index.d.ts", + }), + ); + await writeFile(join(pkg, "index.js"), packageSource); + await writeFile(join(pkg, "index.d.ts"), packageTypes); + await writeFile( + join(dir, "package.json"), + JSON.stringify({ type: "module", dependencies: { "shapes-fixture": "1.0.0" } }), + ); + await writeFile(join(dir, "main.ts"), mainSource); + saved = process.env["SCRIPTC_DEBUG_SHAPES"]; +}); +afterEach(async () => { + if (saved === undefined) delete process.env["SCRIPTC_DEBUG_SHAPES"]; + else process.env["SCRIPTC_DEBUG_SHAPES"] = saved; + vi.restoreAllMocks(); + await rm(dir, { recursive: true, force: true }); +}); + +async function report(): Promise { + process.env["SCRIPTC_DEBUG_SHAPES"] = "verbose"; + const lines: string[] = []; + vi.spyOn(process.stderr, "write").mockImplementation((chunk: string | Uint8Array) => { + const text = typeof chunk === "string" ? chunk : Buffer.from(chunk).toString("utf8"); + for (const l of text.split("\n")) if (l.startsWith("[shapes] ")) lines.push(l.slice(9)); + return true; + }); + const entry = join(dir, "main.ts"); + await compile(entry, { + outDir: dir, + outPath: join(dir, "main.ir.json"), + outputKind: "ir", + npmStatic: ["shapes-fixture"], + }); + return lines; +} + +function site(lines: string[], label: RegExp): string { + const line = lines.find((l) => l.startsWith("site ") && label.test(l)); + if (line === undefined) throw new Error(`no site ${label}:\n${lines.join("\n")}`); + return line; +} + +test("S1 report: per-field temporaries are provable records, dynamic objects stay G", async () => { + const lines = await report(); + // result literals of both _parse implementations, the pair literal, the + // context and the root context never escape: native records would be exact + // the leaf schema's result: in a function called from a loop (hot) + expect(site(lines, /^site \{status,value\}@shapes-fixture\/index\.js:23 \[hot\]/)).toMatch( + /: P-local/, + ); + expect(site(lines, /^site \{status,value\}@shapes-fixture\/index\.js:36/)).toMatch(/: P-local/); + expect(site(lines, /^site \{key,value\}@/)).toMatch(/: P-local/); + expect(site(lines, /^site \{common,data,parent\}@shapes-fixture\/index\.js:12/)).toMatch( + /: P-local/, + ); + expect(site(lines, /^site \[\]@shapes-fixture\/index\.js:29/)).toMatch(/: P-local/); + // the output object is built with computed keys + expect(site(lines, /^site \{\}@shapes-fixture\/index\.js:34/)).toMatch(/: G — computed write/); + // a deleted key and a value stored into the global object + expect(site(lines, /^site \{a\}@/)).toMatch(/: G — delete at/); + expect(site(lines, /^site \{kept\}@main\.ts/)).toMatch(/: P-escaping — escapes: stored into ⊤/); + // receiver-typed dispatch: the field's validators are the schema classes + expect(lines.find((l) => l.startsWith("fn Str._parse@"))).toMatch(/p0=LazyPath/); + expect(lines.find((l) => l.startsWith("class LazyPath"))).toMatch( + /parent: \{common,data,parent\}@shapes-fixture\/index\.js:12;/, + ); + expect(lines.find((l) => l.startsWith("class Str"))).toMatch(/_def: \{checks\}@/); +}); + +test("S1 report is off by default and changes nothing", async () => { + delete process.env["SCRIPTC_DEBUG_SHAPES"]; + const writes = vi.spyOn(process.stderr, "write"); + const entry = join(dir, "main.ts"); + await compile(entry, { + outDir: dir, + outPath: join(dir, "main.ir.json"), + outputKind: "ir", + npmStatic: ["shapes-fixture"], + }); + expect(writes.mock.calls.some(([c]) => String(c).includes("[shapes]"))).toBe(false); +}); From e0d0e03eed999bcbc4564986632659937cecd030 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 15:47:49 -0500 Subject: [PATCH 100/184] Support Symbol.toStringTag class fields and getters --- .../symbol-fields.mdx | 1 + .../src/backend/llvm/expr-stream-bridges.ts | 71 +++++++++--- .../lowering/class-dynamic-dispatch.ts | 84 +++++++++++++- .../lowering/class-method-mutations.ts | 14 ++- .../frontend/lowering/class-to-primitive.ts | 103 ++++++++++++++++-- .../frontend/lowering/cycle-initialization.ts | 10 +- .../src/frontend/lowering/lower-calls.ts | 6 +- .../src/frontend/lowering/lower-classes.ts | 5 +- .../src/frontend/lowering/lower-inspect.ts | 87 ++++++++++++++- .../src/frontend/lowering/symbol-fields.ts | 49 +++++++++ .../src/ir/function-references.test.ts | 6 + .../compiler/src/ir/function-references.ts | 1 + packages/compiler/src/ir/ir.ts | 9 +- packages/compiler/src/ir/validate.ts | 25 ++++- ...751-class-well-known-symbol-fields.ts.json | 6 + packages/runtime/src/scr_dyn.h | 8 ++ packages/runtime/src/scr_inspect.c | 30 ++++- packages/runtime/src/scr_json.c | 71 +++++++++++- packages/runtime/src/scr_symbol.c | 5 + .../7751-class-well-known-symbol-fields.ts | 76 +++++++++++++ ...751-class-well-known-symbol-fields.ts.json | 1 + 21 files changed, 620 insertions(+), 48 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7751-class-well-known-symbol-fields.ts.json create mode 100644 tests/corpus/7751-class-well-known-symbol-fields.ts create mode 100644 tests/harness/corpus-costs/7751-class-well-known-symbol-fields.ts.json diff --git a/docs/content/limitations/types-and-class-definitions/symbol-fields.mdx b/docs/content/limitations/types-and-class-definitions/symbol-fields.mdx index baff4d190..45659b493 100644 --- a/docs/content/limitations/types-and-class-definitions/symbol-fields.mdx +++ b/docs/content/limitations/types-and-class-definitions/symbol-fields.mdx @@ -1,4 +1,5 @@ - **Symbol fields:** Instance fields require a stable module-level `Symbol()` or `Symbol.for()` binding with a literal description or constant string binding. + A `[Symbol.toStringTag]` field or getter with a string value sets the tag that `String()`, `Object.prototype.toString` and `util.inspect` print; fields keyed by other well-known symbols are unsupported. Reassigned keys, runtime-computed descriptions, distinct symbols with the same description in one class, and copying symbol fields with rest bindings or `Object.assign` are unsupported. JavaScript object spreads copy represented own symbol fields. Equal registry keys share a field across modules. diff --git a/packages/compiler/src/backend/llvm/expr-stream-bridges.ts b/packages/compiler/src/backend/llvm/expr-stream-bridges.ts index 70a8dcdee..f4532b21a 100644 --- a/packages/compiler/src/backend/llvm/expr-stream-bridges.ts +++ b/packages/compiler/src/backend/llvm/expr-stream-bridges.ts @@ -20,6 +20,7 @@ import { isRefCounted, shapeHasAccessorSlots, typeKey, + type IrClassDef, } from "../../ir/ir.js"; import { mangleFunction, mangleGlobal, mangleRecordStruct } from "../mangle.js"; import { BlockBuilder } from "./blocks.js"; @@ -277,9 +278,7 @@ export function streamTypedRefCommitAdapter( ); }); } - const symbols = new Map( - (meta.def.symbolFields ?? []).map((symbol) => [symbol.field, symbol.globalId]), - ); + const symbols = new Map((meta.def.symbolFields ?? []).map((symbol) => [symbol.field, symbol])); const fields = meta.def.fields.filter( (field) => (isClassOwnEnumerableFieldName(field.name) || symbols.has(field.name)) && @@ -299,13 +298,15 @@ export function streamTypedRefCommitAdapter( const inline = host.inlineUnions.classField(t.className, field.name); const fieldTy = int32 ? "i32" : llFieldType(field.type, inline); const checkTy = host.llType(field.type); - const symbolGlobal = symbols.get(field.name); - if (symbolGlobal) { + const symbolKey = symbols.get(field.name); + if (symbolKey) { host.declare(`declare ptr @scr_dyn_symbol_key_get(ptr, ptr, i1 zeroext)`); host.declare(`declare void @scr_dyn_release_v(ptr)`); + const keyLines = symbolKeyLines(host, symbolKey, `f${index}_keyraw`); lines.push( - ` %f${index}_keyraw = load ptr, ptr @${mangleGlobal(symbolGlobal)}`, + ...keyLines.load, ` %f${index}_key = call ptr @${host.dyn.toDynHelper(SYMBOL_T)}(ptr %f${index}_keyraw)`, + ...keyLines.release, ); if (field.type.kind === "symbol") { // A base constructor can mutate the instance before this @@ -333,7 +334,7 @@ export function streamTypedRefCommitAdapter( lines.push( ` %${raw} = call ptr @scr_dyn_obj_get(ptr %d, ptr ${host.cstr(field.name)}, ${host.sizeType} ${Buffer.byteLength(field.name, "utf8")})`, ); - if (meta.def.tracksOwnFields && !symbolGlobal) + if (meta.def.tracksOwnFields && !symbolKey) lines.push( ` %f${index}_present = icmp ne ptr %${raw}, null`, ` br i1 %f${index}_present, label %f${index}_read, label %${after}`, @@ -344,7 +345,7 @@ export function streamTypedRefCommitAdapter( ` %${undef} = call ptr @scr_dyn_undefined()`, ` %${input} = select i1 %${missing}, ptr %${undef}, ptr %${raw}`, ` %${next} = call ${field.type.kind === "bool" ? "zeroext " : ""}${checkTy} @${host.dyn.dynCheckHelper(field.type)}(ptr %${input}, ptr null)`, - ...(symbolGlobal ? [` call void @scr_dyn_release_v(ptr %${raw})`] : []), + ...(symbolKey ? [` call void @scr_dyn_release_v(ptr %${raw})`] : []), ...host.pendingTestLines(`%${pending}`), ` br i1 %${pending}, label %done, label %${store}`, `${store}:`, @@ -499,16 +500,18 @@ export function streamTypedRefCommitAdapter( ? `declare void @scr_dyn_symbol_key_placeholder(ptr, ptr)` : `declare void @scr_dyn_key_delete_computed(ptr, ptr, i1 zeroext)`, ); - [...symbols.values()].forEach((globalId, index) => + [...symbols.values()].forEach((symbol, index) => { + const keyLines = symbolKeyLines(host, symbol, `bag_symraw${index}`); lines.push( - ` %bag_symraw${index} = load ptr, ptr @${mangleGlobal(globalId)}`, + ...keyLines.load, ` %bag_sym${index} = call ptr @${host.dyn.toDynHelper(SYMBOL_T)}(ptr %bag_symraw${index})`, + ...keyLines.release, meta.def.tracksOwnFields ? ` call void @scr_dyn_symbol_key_placeholder(ptr %bag, ptr %bag_sym${index})` : ` call void @scr_dyn_key_delete_computed(ptr %bag, ptr %bag_sym${index}, i1 zeroext false)`, ` call void @scr_dyn_release_v(ptr %bag_sym${index})`, - ), - ); + ); + }); const { index } = classFieldIndex(meta, DYN_CLASS_PROPERTIES); lines.push( ` %bag_ptr = getelementptr inbounds %${classStructSym(t.className)}, ptr %target, i64 0, i32 ${index}`, @@ -705,6 +708,27 @@ export function liveDynUnionRefAdapter( return sym; } +/** The raw ScrSym of a class's symbol field key into `%name`: a module + * key is borrowed from its global, a well-known one comes from the + * runtime's registry (+1, released by the `release` lines once the key + * value holds it). */ +function symbolKeyLines( + host: LlvmEmitterContext, + symbol: NonNullable[number], + name: string, +): { load: string[]; release: string[] } { + if ("globalId" in symbol) + return { load: [` %${name} = load ptr, ptr @${mangleGlobal(symbol.globalId)}`], release: [] }; + host.declare("declare ptr @scr_sym_well_known(ptr)"); + host.declare("declare void @scr_sym_release(ptr)"); + return { + load: [ + ` %${name} = call ptr @scr_sym_well_known(ptr ${host.internLiteral(symbol.wellKnown)})`, + ], + release: [` call void @scr_sym_release(ptr %${name})`], + }; +} + export function streamTypedRefBoxValue( host: LlvmEmitterContext, B: BlockBuilder, @@ -929,8 +953,10 @@ export function streamTypedRefMaterializeAdapter( B.line(`${fieldValue} = sitofp i32 ${raw} to double`); } else if (inline) fieldValue = loadStructField(B, host.sizeType, fieldPtr, field.type, true); else B.line(`${fieldValue} = load ${llFieldType(field.type, false)}, ptr ${fieldPtr}`); - B.line(`${rawKey} = load ptr, ptr @${mangleGlobal(symbol.globalId)}`); + const keyLines = symbolKeyLines(host, symbol, rawKey.slice(1)); + for (const line of keyLines.load) B.line(line.trim()); const key = host.streamTypedRefBoxValue(B, SYMBOL_T, rawKey); + for (const line of keyLines.release) B.line(line.trim()); const afterSymbol = meta.def.tracksOwnFields || field.type.kind === "symbol" ? B.newLabel("symbol.after") @@ -1473,9 +1499,26 @@ function liveClassOps(host: LlvmEmitterContext, snapshot: string, meta: LlClassM // The constructor's name, as util.inspect prints before the fields. const jsName = meta.def.jsName ?? meta.def.name.replace(/^%/, ""); const className = jsName === "" ? "null" : host.internLiteral(jsName); + // Its Symbol.toStringTag getter's string, for Object.prototype.toString + // and util.inspect. + let stringTag = "null"; + if (meta.def.stringTagHelper) { + // The runtime lends the instance; the helper takes its own reference. + stringTag = `@${snapshot}_ctag`; + const retain = vAdapters(host.shapeHost, { kind: "object", className: meta.def.name }).retain; + host.resolveThunkDefs.push( + `define internal ptr ${stringTag}(ptr %p) ${FN_ATTRS} { ; Symbol.toStringTag of ${meta.def.name}`, + `entry:`, + ` %owned = call ptr ${retain}(ptr %p)`, + ` %tag = call ptr @${mangleFunction(meta.def.stringTagHelper)}(ptr %owned)`, + ` ret ptr %tag`, + `}`, + ``, + ); + } host.resolveThunkDefs.push( `@${names} = internal constant [${members.length} x ptr] [${literals.join(", ")}]`, - `@${ops} = internal constant { ptr, ${host.sizeType}, ptr, ptr } { ptr ${exact}, ${host.sizeType} ${members.length}, ptr @${names}, ptr ${className} }`, + `@${ops} = internal constant { ptr, ${host.sizeType}, ptr, ptr, ptr } { ptr ${exact}, ${host.sizeType} ${members.length}, ptr @${names}, ptr ${className}, ptr ${stringTag} }`, ``, ); } diff --git a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts index abf03811e..ad5f4a620 100644 --- a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts +++ b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts @@ -1,4 +1,5 @@ import { primitiveConstructorOr } from "./lower-exprs.js"; +import { TO_STRING_TAG_FIELD, wellKnownSymbolOfField } from "./symbol-fields.js"; import { InternalCompilerError } from "../../errors.js"; import { dynUndefinedExpr, helperLoc, nodeThrowExpr, varRef } from "../../ir/build.js"; import { constructedValue } from "./constructor-returns.js"; @@ -26,6 +27,7 @@ import { isUnitType, typeEquals, typeKey, + type IrClassDef, type IrExpr, type IrFunction, type IrParam, @@ -153,6 +155,41 @@ const STREAM_NUM_PROPERTIES = new Set([ const streamProperty = (name: string): boolean => STREAM_BOOL_PROPERTIES.has(name) || STREAM_NUM_PROPERTIES.has(name); +/** `(instance) => string`, the string a class's inherited + * Symbol.toStringTag getter answers, which the runtime reads for + * Object.prototype.toString and util.inspect of an instance seen as + * `unknown`. None for a class without a string getter, or whose own tag + * field shadows it. */ +function stringTagHelper(lowerer: Lowerer, info: ClassInfo, loc: SrcLoc): IrFunction | null { + if (info.fields.has(TO_STRING_TAG_FIELD)) return null; + const getter = findMethodOn(lowerer, info, "get:sym:toStringTag"); + if (getter?.sig.ret.kind !== "string") return null; + const type: IrType = { kind: "object", className: info.def.name }; + const owner = getter.declarer.def.name; + return { + name: `%class.stringTag:${info.def.name}`, + params: [{ localId: "this", name: "this", type }], + locals: [{ id: "this", name: "this", type, mutable: false }], + returnType: STRING, + body: [ + { + kind: "return", + value: accessorCall( + lowerer, + owner, + "get:sym:toStringTag", + lowerer.upcastTo(varRef("this", type, loc), owner), + [], + STRING, + loc, + ), + loc, + }, + ], + loc, + }; +} + function symbolMemberKey( lowerer: Lowerer, info: ClassInfo, @@ -1276,10 +1313,27 @@ export class ClassDynamicDispatch { const info = lowerer.classes.get(className); if (!info || info.builtinEmitter || info.builtinStream || info.builtinError) continue; if (info.symbolFields && info.def.symbolFields === undefined) { - info.def.symbolFields = [...info.symbolFields].flatMap(([symbol, field]) => { - const global = lowerer.globalsBySymbol.get(symbol); - return global?.type.kind === "symbol" ? [{ field, globalId: global.id }] : []; - }); + info.def.symbolFields = [...info.symbolFields].flatMap( + ([symbol, field]): NonNullable => { + const global = lowerer.globalsBySymbol.get(symbol); + if (global?.type.kind === "symbol") return [{ field, globalId: global.id }]; + const wellKnown = wellKnownSymbolOfField(field); + return wellKnown === null ? [] : [{ field, wellKnown }]; + }, + ); + } + const tagged = (current: ClassInfo): ClassInfo[] => [ + current, + ...current.subclasses.flatMap(tagged), + ]; + for (const current of tagged(info)) { + if (current.def.stringTagHelper !== undefined) continue; + const fn = stringTagHelper(lowerer, current, functions[0]!.loc); + if (!fn) continue; + current.def.stringTagHelper = fn.name; + this.generated.add(fn); + lowerer.liftedFns.push(fn); + changed = true; } if (this.prototypeHelper && !this.prototypeClasses.has(className)) { this.prototypeClasses.add(className); @@ -3554,10 +3608,29 @@ export class ClassDynamicDispatch { name, lowerer.coerceToExpected(varRef(global.id, global.type, loc), DYN), ); + } else if (wellKnownSymbolOfField(name) !== null) { + // A field keyed by a well-known symbol (Symbol.toStringTag). + symbols.set(name, { + kind: "dynFrom", + type: DYN, + loc, + value: { + kind: "libCall", + fn: "sym.wellKnown", + args: [ + { kind: "strLit", value: wellKnownSymbolOfField(name)!, type: STRING, loc }, + ], + type: SYMBOL_T, + loc, + }, + }); } } for (let owner: ClassInfo | null = info; owner; owner = owner.base) { - for (const name of [...owner.methods.keys()]) + for (const method of [...owner.methods.keys()]) { + // Accessors key by the property they define (`get [Symbol.toStringTag]`). + const name = + method.startsWith("get:") || method.startsWith("set:") ? method.slice(4) : method; if (name.startsWith("sym:")) { symbols.set(name, { kind: "dynFrom", @@ -3572,6 +3645,7 @@ export class ClassDynamicDispatch { }, }); } + } } const branches: IrStmt[] = []; if (!dispatch.write && !dispatch.reflect && !info.def.tracksOwnFields) { diff --git a/packages/compiler/src/frontend/lowering/class-method-mutations.ts b/packages/compiler/src/frontend/lowering/class-method-mutations.ts index 035c2c2a5..9a2632ed3 100644 --- a/packages/compiler/src/frontend/lowering/class-method-mutations.ts +++ b/packages/compiler/src/frontend/lowering/class-method-mutations.ts @@ -394,7 +394,19 @@ export function collectClassMethodMutations( (ts.isElementAccessExpression(base) && ts.isStringLiteralLike(base.argumentExpression) && base.argumentExpression.text === "prototype"); - if (prototype) { + // `Object.prototype.toString.call(value)` calls the builtin and + // keeps no reference to it, so it replaces nothing. + const builtinCall = + ts.isPropertyAccessExpression(node) && + ts.isPropertyAccessExpression(base) && + ts.isIdentifier(unwrap(base.expression)) && + lowerer.isStdlibGlobal(unwrap(base.expression) as ts.Identifier, "Object") && + ts.isPropertyAccessExpression(node.parent) && + node.parent.expression === node && + (node.parent.name.text === "call" || node.parent.name.text === "apply") && + ts.isCallExpression(node.parent.parent) && + node.parent.parent.expression === node.parent; + if (prototype && !builtinCall) { if (ts.isPropertyAccessExpression(node)) lowerer.noteMethodAccess(node.name.text, node); else if (ts.isStringLiteralLike(node.argumentExpression)) lowerer.noteMethodAccess(node.argumentExpression.text, node); diff --git a/packages/compiler/src/frontend/lowering/class-to-primitive.ts b/packages/compiler/src/frontend/lowering/class-to-primitive.ts index 0b107ab4e..a260116f8 100644 --- a/packages/compiler/src/frontend/lowering/class-to-primitive.ts +++ b/packages/compiler/src/frontend/lowering/class-to-primitive.ts @@ -13,6 +13,7 @@ import * as ts from "../ts7/adapter.js"; import type { Lowerer } from "./lowerer.js"; import { accessorCall, findMethodOn, type ClassInfo } from "./lower-classes.js"; import { isClassCallback } from "./class-callbacks.js"; +import { TO_STRING_TAG_FIELD } from "./symbol-fields.js"; import { isSafeToRepeat } from "./expressions/evaluation-safety.js"; import { DATE_TIME_FIELD, dateRootOf } from "./builtins/date-classes.js"; import { dateValueMethod } from "./builtins/dates.js"; @@ -29,7 +30,15 @@ type ConversionMethod = "sym:toPrimitive" | "valueOf" | "toString"; * Date`, or through Object.prototype.toString ("[object Object]"). */ type Plan = | { method: ConversionMethod; ret: IrType } - | { method: null; date?: "toString" | "valueOf" }; + | { method: null; date?: "toString" | "valueOf"; tag?: ToStringTag }; + +/** Where Object.prototype.toString finds a class's Symbol.toStringTag: a + * string field (`[Symbol.toStringTag] = "LRUCache"`) or a getter that + * returns a string, declared on `owner`. */ +interface ToStringTag { + owner: ClassInfo; + getter: boolean; +} const PRIMITIVE_KINDS = new Set(["f64", "string", "bool", "bigint", "undefinedT", "nullT"]); @@ -52,6 +61,17 @@ export function classToStringConversion( ); } +/** Object.prototype.toString of a program class instance: "[object " + + * its Symbol.toStringTag + "]" (an own tag field, else an inherited + * getter), or "[object Object]". The class's own conversion methods do not + * take part. Null as for classToStringConversion, and for Date subclasses. */ +export function classObjectTag(lowerer: Lowerer, value: IrExpr, node: ts.Node): IrExpr | null { + if (value.type.kind !== "object") return null; + const info = lowerer.classes.get(value.type.className); + if (!info || dateRootOf(info) !== null) return null; + return classConversion(lowerer, value, "tag", node, STRING, (tag) => tag); +} + /** ToNumber(ToPrimitive(value, "number")) of a program class instance * (`Number(x)`): `[Symbol.toPrimitive]("number")`, then `valueOf` and then * `toString`, and NaN for "[object Object]". Null as for @@ -122,7 +142,7 @@ function mayBeBigint(lowerer: Lowerer, type: IrType): boolean { function classConversion( lowerer: Lowerer, value: IrExpr, - hint: ConversionHint, + hint: ConversionHint | "tag", node: ts.Node, resultType: IrType, finish: (primitive: IrExpr) => IrExpr, @@ -131,7 +151,7 @@ function classConversion( const info = lowerer.classes.get(value.type.className); if (!info || !plainProgramClass(info)) return null; const classes = subtree(lowerer, info); - for (const cls of classes) refuseReplacedConversions(lowerer, cls, node); + if (hint !== "tag") for (const cls of classes) refuseReplacedConversions(lowerer, cls, node); const plans = new Map(classes.map((cls) => [cls, planOf(lowerer, cls, hint, node)])); // A class whose plan differs from its base's starts a group; the deepest // groups are tested first, so each instance takes its own class's plan. @@ -164,6 +184,41 @@ function classConversion( }; return finish(dateValueMethod(plan.date, time, loc)!); } + if (plan.method === null && plan.tag) { + // Object.prototype.toString: "[object " + the tag + "]". + const owner = plan.tag.owner.def.name; + const tag: IrExpr = plan.tag.getter + ? accessorCall( + lowerer, + owner, + "get:sym:toStringTag", + lowerer.upcastTo(object, owner), + [], + STRING, + loc, + ) + : { + kind: "fieldGet", + obj: lowerer.upcastTo(object, owner), + className: owner, + field: TO_STRING_TAG_FIELD, + type: STRING, + loc, + }; + return finish({ + kind: "strConcat", + left: { + kind: "strConcat", + left: { kind: "strLit", value: "[object ", type: STRING, loc }, + right: tag, + type: STRING, + loc, + }, + right: { kind: "strLit", value: "]", type: STRING, loc }, + type: STRING, + loc, + }); + } if (plan.method === null) return finish({ kind: "strLit", value: "[object Object]", type: STRING, loc }); const args: IrExpr[] = @@ -228,10 +283,28 @@ function depth(info: ClassInfo): number { } function samePlan(a: Plan, b: Plan): boolean { - return ( - a.method === b.method && - (a.method !== null || b.method !== null || a.date === (b as { date?: string }).date) - ); + if (a.method !== null || b.method !== null) return a.method === b.method; + return a.date === b.date && a.tag?.owner === b.tag?.owner && a.tag?.getter === b.tag?.getter; +} + +/** The class's Symbol.toStringTag, when it has one: the class at the top + * of the chain whose layout holds the tag field (an own property, which + * shadows any getter), or its getter. Other tag types would make + * Object.prototype.toString answer "Object" or the value depending on the + * run, and are refused. */ +function toStringTagOf(lowerer: Lowerer, info: ClassInfo, node: ts.Node): ToStringTag | null { + const type = info.fields.get(TO_STRING_TAG_FIELD); + if (type !== undefined) { + if (type.kind !== "string") refuse(lowerer, node, info, "a non-string Symbol.toStringTag"); + let owner = info; + while (owner.base?.fields.has(TO_STRING_TAG_FIELD)) owner = owner.base; + return { owner, getter: false }; + } + const getter = findMethodOn(lowerer, info, "get:sym:toStringTag"); + if (!getter) return null; + if (getter.sig.ret.kind !== "string") + refuse(lowerer, node, info, "a non-string Symbol.toStringTag"); + return { owner: getter.declarer, getter: true }; } /** A conversion method the class declares or inherits (a field of that @@ -246,7 +319,16 @@ function declared( return found ? { ret: found.sig.ret } : null; } -function planOf(lowerer: Lowerer, info: ClassInfo, hint: ConversionHint, node: ts.Node): Plan { +function planOf( + lowerer: Lowerer, + info: ClassInfo, + hint: ConversionHint | "tag", + node: ts.Node, +): Plan { + if (hint === "tag") { + const tag = toStringTagOf(lowerer, info, node); + return tag ? { method: null, tag } : { method: null }; + } const exotic = declared(lowerer, info, "sym:toPrimitive"); if (exotic) { if (!primitive(lowerer, exotic.ret)) @@ -271,7 +353,8 @@ function planOf(lowerer: Lowerer, info: ClassInfo, hint: ConversionHint, node: t refuse(lowerer, node, info, `a '${method}' method that can return an object`); return { method, ret: found.ret }; } - return { method: null }; + const tag = toStringTagOf(lowerer, info, node); + return tag ? { method: null, tag } : { method: null }; } function primitive(lowerer: Lowerer, type: IrType): boolean { @@ -292,8 +375,6 @@ function refuseReplacedConversions(lowerer: Lowerer, info: ClassInfo, node: ts.N if (lowerer.prototypeMethodAccesses.has(method)) refuse(lowerer, node, info, `a '${method}' the program reaches through a prototype`); } - if (findMethodOn(lowerer, info, "get:sym:toStringTag") || info.fields.has("sym:toStringTag")) - refuse(lowerer, node, info, "a Symbol.toStringTag"); } function refuse(lowerer: Lowerer, node: ts.Node, info: ClassInfo, what: string): never { diff --git a/packages/compiler/src/frontend/lowering/cycle-initialization.ts b/packages/compiler/src/frontend/lowering/cycle-initialization.ts index 4b1e74fcf..8c8ce06e6 100644 --- a/packages/compiler/src/frontend/lowering/cycle-initialization.ts +++ b/packages/compiler/src/frontend/lowering/cycle-initialization.ts @@ -707,11 +707,17 @@ function rewriteFunction( * unchecked. */ export function refuseUncheckedCycleReads( lowerer: Lowerer, - classes: readonly { baseValueGlobal?: string; symbolFields?: { globalId: string }[] }[], + classes: readonly { + baseValueGlobal?: string; + symbolFields?: readonly ({ globalId: string } | { wellKnown: string })[]; + }[], ): void { if (lowerer.cycleInitFlags.size === 0) return; for (const cls of classes) { - const ids = [cls.baseValueGlobal, ...(cls.symbolFields ?? []).map((f) => f.globalId)]; + const ids = [ + cls.baseValueGlobal, + ...(cls.symbolFields ?? []).map((f) => ("globalId" in f ? f.globalId : undefined)), + ]; for (const id of ids) { const flag = id === undefined ? undefined : lowerer.cycleInitFlags.get(id); if (flag === undefined) continue; diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 5366ca9d0..c9416df46 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -183,7 +183,7 @@ import { import { isSafeToDiscard } from "./expressions/evaluation-safety.js"; import { tryLowerExpression } from "./expressions/try-lower-expression.js"; import { lowerChildMethodValue, lowerChildPropertyCall } from "./handle-properties.js"; -import { classToNumberConversion } from "./class-to-primitive.js"; +import { classObjectTag, classToNumberConversion } from "./class-to-primitive.js"; import { hasRuntimeStatics } from "./class-runtime-statics.js"; import { structuralReadType } from "./class-unions.js"; import { @@ -3103,6 +3103,10 @@ function lowerObjectPrototypeCall( const immediate = immediateObjectTagReceiver(lowerer, receiverNode); const receiver = immediate?.effect ?? lowerer.lowerExpr(receiverNode); const tag = immediate?.tag ?? objectToStringTag(lowerer, receiver.type); + if (tag === null && immediate === null && call.arguments.nodes.length === 1) { + const classTag = classObjectTag(lowerer, receiver, receiverNode); + if (classTag) return classTag; + } // A generator object answers its tag through its checked capsule. if (tag === null && receiver.type.kind !== "dyn" && receiver.type.kind !== "generator") { lowerer.noLowering( diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index 9452cdf25..fb8c07a99 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -8156,7 +8156,10 @@ export function classMemberNameOf(lowerer: Lowerer, name: ts.PropertyName): stri member === "asyncIterator" || member === "dispose" || member === "asyncDispose" || - member === "toPrimitive" + member === "toPrimitive" || + // A toStringTag field keeps its symbol field name; its accessor and + // method forms take the method slot name. + (member === "toStringTag" && !ts.isPropertyDeclaration(name.parent)) ) { return `sym:${member}`; } diff --git a/packages/compiler/src/frontend/lowering/lower-inspect.ts b/packages/compiler/src/frontend/lowering/lower-inspect.ts index 5266d5c4e..ce9c35277 100644 --- a/packages/compiler/src/frontend/lowering/lower-inspect.ts +++ b/packages/compiler/src/frontend/lowering/lower-inspect.ts @@ -44,9 +44,9 @@ import { shapeKeyOrders, typeKey, } from "../../ir/ir.js"; -import type { ClassInfo } from "./lower-classes.js"; +import { accessorCall, findMethodOn, type ClassInfo } from "./lower-classes.js"; import { isSafeToDiscard, isSafeToRepeat } from "./expressions/evaluation-safety.js"; -import { symbolFieldDisplayName } from "./symbol-fields.js"; +import { symbolFieldDisplayName, TO_STRING_TAG_FIELD } from "./symbol-fields.js"; import { eventTargetRootOf } from "./lower-event-target.js"; import { dateRootOf } from "./builtins/date-classes.js"; import { lowerSystemErrorMap } from "./lower-system-errors.js"; @@ -1870,8 +1870,70 @@ function inspectHelper(lowerer: Lowerer, t: IrType, loc: SrcLoc): string { // observable way — Node's inspect omits them entirely (verified), // so a class whose only fields are private prints as `C {}`. const visible = info.def.fields.filter((f) => isClassOwnEnumerableFieldName(f.name)); + // A Symbol.toStringTag getter (an own tag field is listed instead) + // brackets the tag after the name, `Name [tag] {`, when it is a + // nonempty string other than the name. + const tagGetter = info.fields.has(TO_STRING_TAG_FIELD) + ? null + : findMethodOn(lowerer, info, "get:sym:toStringTag"); + let named: (plain: string, make: (head: IrExpr) => IrExpr) => IrExpr = (plain) => + strLit(plain, loc); + if (tagGetter?.sig.ret.kind === "string") { + const owner = tagGetter.declarer.def.name; + locals.push({ id: "tag.0", name: "tag", type: STRING, mutable: false }); + body.push({ + kind: "varDecl", + localId: "tag.0", + init: accessorCall( + lowerer, + owner, + "get:sym:toStringTag", + lowerer.upcastTo(v(), owner), + [], + STRING, + loc, + ), + loc, + }); + const tag = (): IrExpr => varRef("tag.0", STRING, loc); + const shown: IrExpr = { + kind: "logical", + op: "&&", + left: { + kind: "strEq", + negated: true, + left: tag(), + right: strLit("", loc), + type: BOOL, + loc, + }, + right: { + kind: "strEq", + negated: true, + left: tag(), + right: strLit(display, loc), + type: BOOL, + loc, + }, + type: BOOL, + loc, + }; + const head = concatAll( + [strLit(`${display === "" ? "Object" : display} [`, loc), tag(), strLit("]", loc)], + loc, + ); + named = (plain, make) => ({ + kind: "ternary", + cond: shown, + then: make(head), + else_: strLit(plain, loc), + type: STRING, + loc, + }); + } + const after = (text: string) => (head: IrExpr) => concatAll([head, strLit(text, loc)], loc); if (visible.length === 0) { - body = [ret(strLit(display === "" ? "{}" : `${display} {}`, loc))]; + body.push(ret(named(display === "" ? "{}" : `${display} {}`, after(" {}")))); break; } const get = (field: string, type: IrType): IrExpr => ({ @@ -1882,7 +1944,22 @@ function inspectHelper(lowerer: Lowerer, t: IrType, loc: SrcLoc): string { type, loc, }); - body = [depthGate(display === "" ? "[Object]" : `[${display}]`), ...begin()]; + body.push( + { + kind: "if", + cond: { kind: "bin", op: ">", left: r(), right: d(), type: BOOL, loc }, + then: [ + ret( + named(display === "" ? "[Object]" : `[${display}]`, (head) => + concatAll([strLit("[", loc), head, strLit("]", loc)], loc), + ), + ), + ], + else_: null, + loc, + }, + ...begin(), + ); // def.fields carries layout order: the base chain first, then own — // exactly the own-property insertion order of a constructor that // assigns in declaration order (SEMANTICS.md 36's stance). SYMBOL- @@ -1911,7 +1988,7 @@ function inspectHelper(lowerer: Lowerer, t: IrType, loc: SrcLoc): string { ret( end( strLit("", loc), - strLit(display === "" ? "{" : `${display} {`, loc), + named(display === "" ? "{" : `${display} {`, after(" {")), strLit("}", loc), false, boolLit(false, loc), diff --git a/packages/compiler/src/frontend/lowering/symbol-fields.ts b/packages/compiler/src/frontend/lowering/symbol-fields.ts index 56718615e..cdfb9e1b8 100644 --- a/packages/compiler/src/frontend/lowering/symbol-fields.ts +++ b/packages/compiler/src/frontend/lowering/symbol-fields.ts @@ -77,6 +77,8 @@ export function fenceSymbolFieldCopy( } export function classSymbolKeyOf(lowerer: Lowerer, key: ts.Expression): ClassSymbolKey | null { + const wellKnown = wellKnownSymbolKeyOf(lowerer, key); + if (wellKnown !== undefined) return wellKnown; if ( !ts.isIdentifier(key) && (!ts.isPropertyAccessExpression(key) || key.questionDotToken || !ts.isIdentifier(key.name)) @@ -90,6 +92,53 @@ export function classSymbolKeyOf(lowerer: Lowerer, key: ts.Expression): ClassSym return sym ? classSymbolKeyOfSymbol(lowerer, sym) : null; } +/** Well-known symbols a class field may be keyed by: those no protocol + * consults on an instance, or whose protocol the lowering reads from the + * field (Symbol.toStringTag, by the class string conversions). A field + * keyed by any other well-known symbol (Symbol.iterator, toPrimitive, + * hasInstance, ...) would be skipped by its protocol, so it keeps the + * computed-key refusal. */ +const FIELD_WELL_KNOWN_SYMBOLS: ReadonlySet = new Set(["toStringTag"]); + +/** The key of `Symbol.` on the standard global: undefined when the + * expression is not one, null when it is a well-known symbol a field may + * not be keyed by. The identity is the lib's declaration of the property, + * one symbol for every program reference. */ +function wellKnownSymbolKeyOf( + lowerer: Lowerer, + key: ts.Expression, +): ClassSymbolKey | null | undefined { + while (ts.isParenthesizedExpression(key)) key = key.expression; + if ( + !ts.isPropertyAccessExpression(key) || + key.questionDotToken || + !ts.isIdentifier(key.expression) || + !ts.isIdentifier(key.name) || + !lowerer.isStdlibGlobal(key.expression, "Symbol") + ) + return undefined; + const name = key.name.text; + const sym = lowerer.checker.getSymbolAtLocation(key.name); + if (!sym || !(lowerer.typeOf(key).flags & ts.TypeFlags.UniqueESSymbol)) return undefined; + if (!FIELD_WELL_KNOWN_SYMBOLS.has(name)) return null; + return { sym, identity: sym, fieldName: wellKnownSymbolFieldName(name) }; +} + +/** The layout name of a field keyed by the well-known `Symbol.`: a + * reserved symbol-field name, printed as Node spells the key. */ +export function wellKnownSymbolFieldName(name: string): string { + return `${FIELD_PREFIX}Symbol(Symbol.${name})`; +} + +/** The well-known symbol a field name is keyed by (`toStringTag`), or null. */ +export function wellKnownSymbolOfField(field: string): string | null { + const match = /^%symbol:Symbol\(Symbol\.(\w+)\)$/.exec(field); + return match === null ? null : match[1]!; +} + +/** The layout name of a `Symbol.toStringTag` field. */ +export const TO_STRING_TAG_FIELD = wellKnownSymbolFieldName("toStringTag"); + /** A literal Symbol()/Symbol.for() initialized once at module scope. The * whole-file proof also covers var declarations emitted by bundlers. Keys * cannot be used before initialization: a computed class field evaluates diff --git a/packages/compiler/src/ir/function-references.test.ts b/packages/compiler/src/ir/function-references.test.ts index 91e1d5f05..db872a525 100644 --- a/packages/compiler/src/ir/function-references.test.ts +++ b/packages/compiler/src/ir/function-references.test.ts @@ -157,10 +157,12 @@ const MODULE_FIELDS: Record>, Role> = { "classes[].errorNameAfter": "other", "classes[].prototypeDataHelper": "reference", "classes[].instancePrototypeHelper": "reference", + "classes[].stringTagHelper": "reference", "classes[].localCaptures[].name": "other", "classes[].localCaptures[].localId": "other", "classes[].symbolFields[].field": "other", "classes[].symbolFields[].globalId": "other", + "classes[].symbolFields[].wellKnown": "other", "classes[].fields[].name": "other", "classes[].methods[]": "method", "classes[].prototypeNames[]": "other", @@ -281,6 +283,10 @@ const REFERENCE_FIXTURES: Record IrModule> = { ...module(), classes: [{ name: "%m0.Cell", fields: [], instancePrototypeHelper: TARGET, loc }], }), + "classes[].stringTagHelper": () => ({ + ...module(), + classes: [{ name: "%m0.Cell", fields: [], stringTagHelper: TARGET, loc }], + }), "lib.exports[].fnName": () => ({ ...module(), lib: { diff --git a/packages/compiler/src/ir/function-references.ts b/packages/compiler/src/ir/function-references.ts index db474e9bb..6696a3c11 100644 --- a/packages/compiler/src/ir/function-references.ts +++ b/packages/compiler/src/ir/function-references.ts @@ -24,6 +24,7 @@ export function moduleFunctionReferences(mod: IrModule): string[] { for (const cls of mod.classes ?? []) { if (cls.prototypeDataHelper !== undefined) names.push(cls.prototypeDataHelper); if (cls.instancePrototypeHelper !== undefined) names.push(cls.instancePrototypeHelper); + if (cls.stringTagHelper !== undefined) names.push(cls.stringTagHelper); } return names; } diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 0ec0043ce..4119ddcf3 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -1477,6 +1477,9 @@ export interface IrClassDef { prototypeDataHelper?: string; /** Native instance to its actual prototype, including factory captures and subclasses. */ instancePrototypeHelper?: string; + /** Native instance to the string its inherited Symbol.toStringTag getter + * answers, read by the runtime for an instance seen as `unknown`. */ + stringTagHelper?: string; /** Public JavaScript fields record their creation in the shared property * bag. Its descriptors preserve presence and insertion order independently * of native layout order; field values remain in their native slots. */ @@ -1486,8 +1489,10 @@ export interface IrClassDef { * class of the chain sets `name` during construction, and then an own * `name` follows every other own property. */ errorNameAfter?: string | null; - /** Stable module symbol identities for public instance layout fields. */ - symbolFields?: { field: string; globalId: string }[]; + /** Stable symbol identities for public instance layout fields: the + * module global holding the key, or the name of the well-known symbol + * (`toStringTag` for `Symbol.toStringTag`). */ + symbolFields?: ({ field: string } & ({ globalId: string } | { wellKnown: string }))[]; /** RUNTIME-PROVIDED class (the builtin Error hierarchy): the struct, RC * helpers, and vtable live in the runtime (ScrError / scr_error_*), so * backends emit no definitions for it — only the preorder-interval diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index c5b7e38f8..bd79d965b 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -501,10 +501,29 @@ export function validateModule(mod: IrModule): IrValidationError[] { }); } } - for (const symbol of cls.symbolFields ?? []) { - const global = mod.globals?.find((global) => global.id === symbol.globalId); + if (cls.stringTagHelper !== undefined) { + const helper = functionsByName.get(cls.stringTagHelper); + const receiver = helper?.params[0]?.type; if ( - global?.type.kind !== "symbol" || + !helper || + helper.params.length !== 1 || + receiver?.kind !== "object" || + receiver.className !== cls.name || + helper.returnType.kind !== "string" || + helper.captures !== undefined || + helper.classCaptures !== undefined + ) { + errors.push({ + message: `class ${cls.name}: string tag helper must be a noncapturing (instance) => string function`, + loc: cls.loc, + }); + } + } + for (const symbol of cls.symbolFields ?? []) { + const global = + "globalId" in symbol ? mod.globals?.find((global) => global.id === symbol.globalId) : null; + if ( + ("globalId" in symbol ? global?.type.kind !== "symbol" : symbol.wellKnown === "") || !cls.fields.some((field) => field.name === symbol.field) ) { errors.push({ message: `class ${cls.name}: invalid symbol field metadata`, loc: cls.loc }); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7751-class-well-known-symbol-fields.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7751-class-well-known-symbol-fields.ts.json new file mode 100644 index 000000000..2985bdf05 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7751-class-well-known-symbol-fields.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7751-class-well-known-symbol-fields.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 9f7ddc527..21fed207e 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -398,6 +398,10 @@ typedef struct ScrDynTypedClassOps { size_t count; ScrStr *const *names; /* immortal member-name strings */ ScrStr *class_name; /* immortal constructor name; NULL for an anonymous class */ + /* The string an inherited Symbol.toStringTag getter answers for the + * instance (+1), or NULL with a pending exception from the getter; + * NULL when the class has no such getter. */ + ScrStr *(*string_tag)(void *instance); } ScrDynTypedClassOps; /* A compiled JavaScript class whose instances record their own properties @@ -915,6 +919,7 @@ void scr_dyn_install_iterator_symbol(ScrDyn *key); void scr_dyn_install_has_instance_symbol(ScrDyn *key); void scr_dyn_install_async_iterator_symbol(ScrDyn *key); void scr_dyn_install_async_dispose_symbol(ScrDyn *key); +void scr_dyn_install_to_string_tag_symbol(ScrDyn *key); /* True when key is Symbol.asyncDispose (registered by the symbol unit when * the program names it); handle symbol_get ops outside that unit use it. */ bool scr_dyn_is_async_dispose_symbol(const ScrDyn *key); @@ -1072,6 +1077,9 @@ ScrDyn *scr_record_source_apply(ScrDyn *snapshot, const void *rec); /* Attach the prototype member names of a class instance capsule * (idempotent), so `in` sees its methods and accessors. */ void scr_dyn_typed_ref_bind_class(ScrDyn *d, const ScrDynTypedClassOps *ops); +/* A class instance capsule's inherited Symbol.toStringTag string from its + * class's getter (+1); NULL when it has none or the getter threw. */ +ScrStr *scr_dyn_typed_ref_class_tag(const ScrDyn *d); /* Attach the bag and prototype hooks of a capsule of a bag-tracked class * instance (idempotent). */ void scr_dyn_typed_ref_bind_bag_class(ScrDyn *d, const ScrDynTypedBagClassOps *ops); diff --git a/packages/runtime/src/scr_inspect.c b/packages/runtime/src/scr_inspect.c index 1dce8d3c2..6777ef1ec 100644 --- a/packages/runtime/src/scr_inspect.c +++ b/packages/runtime/src/scr_inspect.c @@ -1207,6 +1207,19 @@ static bool insp_tag_names_class(const char *tag, const char *cls) { memcmp(tag + tag_len - cls_len, cls, cls_len) == 0; } +/* The Symbol.toStringTag a class instance inherits from its class's + * getter (+1), which Node brackets after the class name when it is a + * nonempty string other than the name. An own tag field is listed with + * the fields instead. */ +static ScrStr *insp_inherited_tag(const ScrDyn *instance, const ScrStr *name) { + ScrStr *tag = scr_dyn_typed_ref_class_tag(instance); + if (tag && (tag->len == 0 || (tag->len == name->len && memcmp(tag->data, name->data, name->len) == 0))) { + scr_str_release(tag); + return NULL; + } + return tag; +} + /* A native Set or Map held as a checked-dynamic value: `Set(n) { a, b }`, * `Map(n) { k => v }`, at most 100 entries like Node's maxArrayLength. */ /* Node's formatWeakCollection: the entries of a WeakMap or WeakSet are not @@ -1725,10 +1738,23 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { free(cls); } else if (name && materialized->kind == SCR_DYN_OBJ && scr_dyn_obj_get(materialized, "%error", 6) == NULL) { - char *cls = malloc(name->len + 1); + ScrStr *tag = insp_inherited_tag(d, name); + if (scr_exc_pending()) { + scr_dyn_release(materialized); + return scr_str_new("", 0); + } + size_t tag_len = tag ? tag->len + 3 : 0; + char *cls = malloc(name->len + tag_len + 1); if (!cls) scr_trap("scriptc: out of memory\n"); memcpy(cls, name->data, name->len); - cls[name->len] = '\0'; + if (tag) { + /* Node's `Name [tag]` prefix. */ + memcpy(cls + name->len, " [", 2); + memcpy(cls + name->len + 2, tag->data, tag->len); + cls[name->len + tag_len - 1] = ']'; + scr_str_release(tag); + } + cls[name->len + tag_len] = '\0'; size_t nseen = g_nseen; g_pending_seen = instance; out = scr_insp_obj(materialized, recurse, depth, cls); diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index a036e0dfd..fe890febd 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -202,6 +202,19 @@ void scr_dyn_install_has_instance_symbol(ScrDyn *key) { scr_atexit(scr_has_instance_symbol_cleanup); } +/* Symbol.toStringTag, installed when the program first evaluates it: no + * object holds the key before then. */ +static SCR_TL ScrDyn *scr_to_string_tag_symbol; +static void scr_to_string_tag_symbol_cleanup(void) { + scr_dyn_release(scr_to_string_tag_symbol); + scr_to_string_tag_symbol = NULL; +} +void scr_dyn_install_to_string_tag_symbol(ScrDyn *key) { + if (scr_to_string_tag_symbol) return; + scr_to_string_tag_symbol = scr_dyn_retain(key); + scr_atexit(scr_to_string_tag_symbol_cleanup); +} + void scr_dyn_install_iterator_symbol(ScrDyn *key) { if (scr_iterator_symbol) return; scr_iterator_symbol = scr_dyn_retain(key); @@ -3884,6 +3897,12 @@ static bool scr_dyn_name_in_strs(const ScrStr *key, ScrStr *const *names, size_t return false; } +ScrStr *scr_dyn_typed_ref_class_tag(const ScrDyn *d) { + if (!d || d->kind != SCR_DYN_TYPED_REF || d->prototype) return NULL; + const ScrDynTypedClassOps *ops = scr_typed_ref_class_ops(d); + return ops && ops->string_tag ? ops->string_tag(d->v.typed_ref.ptr) : NULL; +} + bool scr_dyn_typed_ref_class_has(const ScrDyn *d, const ScrStr *key) { if (!d || d->kind != SCR_DYN_TYPED_REF || d->prototype) return false; const ScrDynTypedClassOps *ops = scr_typed_ref_class_ops(d); @@ -7616,11 +7635,45 @@ void scr_dyn_throw(ScrDyn *d) { scr_dyn_trace_v, scr_dyn_is_object(d)); } +static ScrStr *scr_dyn_tag_string(const ScrStr *tag) { + ScrJsonBuf b; + scr_jb_init(&b); + scr_jb_puts(&b, "[object "); + scr_jb_write(&b, tag->data, tag->len); + scr_jb_putc(&b, ']'); + return scr_jb_finish(&b); +} + +/* Object.prototype.toString of an object whose own or inherited + * @@toStringTag is a string (+1), NULL otherwise or with a pending + * exception from its getter. */ +static ScrStr *scr_dyn_string_tag(const ScrDyn *d) { + ScrStr *getter_tag = scr_dyn_typed_ref_class_tag(d); + if (getter_tag) { + ScrStr *out = scr_dyn_tag_string(getter_tag); + scr_str_release(getter_tag); + return out; + } + if (scr_exc_pending() || !scr_to_string_tag_symbol) return NULL; + ScrDyn *tag = scr_dyn_symbol_key_get(d, scr_to_string_tag_symbol, true); + if (!tag) return NULL; + ScrStr *out = tag->kind == SCR_DYN_STR ? scr_dyn_tag_string(tag->v.str) : NULL; + scr_dyn_release(tag); + return out; +} + ScrStr *scr_dyn_object_tag(const ScrDyn *d) { if (d->kind == SCR_DYN_PROXY) { scr_dyn_proxy_unsupported("Symbol.toStringTag lookup"); return NULL; } + /* Object.prototype.toString answers a string @@toStringTag over the + * builtin tag of a plain object or a class instance. */ + if ((d->kind == SCR_DYN_OBJ || (d->kind == SCR_DYN_TYPED_REF && !scr_dyn_ref_is_error(d) && + !scr_dyn_generator(d)))) { + ScrStr *tag = scr_dyn_string_tag(d); + if (tag || scr_exc_pending()) return tag; + } if (d->kind == SCR_DYN_TYPED_REF) { if (scr_dyn_ref_is_error(d)) return scr_str_new("[object Error]", 14); if (scr_dyn_generator(d)) { @@ -8838,6 +8891,22 @@ ScrStr *scr_dyn_string_coerce_js(const ScrDyn *d) { return NULL; } if (d->kind == SCR_DYN_TYPED_REF) { + /* A class instance without its own conversion methods converts with + * Object.prototype.toString, which reads its toStringTag getter. */ + ScrStr *to_string_key = scr_str_new("toString", 8), *value_of_key = scr_str_new("valueOf", 7); + bool own_conversion = scr_dyn_typed_ref_class_has(d, to_string_key) || + scr_dyn_typed_ref_class_has(d, value_of_key); + scr_str_release(to_string_key); + scr_str_release(value_of_key); + if (!own_conversion) { + ScrStr *tag = scr_dyn_typed_ref_class_tag(d); + if (tag) { + ScrStr *out = scr_dyn_tag_string(tag); + scr_str_release(tag); + return out; + } + if (scr_exc_pending()) return NULL; + } ScrDyn *materialized = scr_dyn_typed_ref_string_view(d); ScrStr *out = scr_dyn_string_coerce_js(materialized); scr_dyn_release(materialized); @@ -8858,7 +8927,7 @@ ScrStr *scr_dyn_string_coerce_js(const ScrDyn *d) { scr_error_release(error); return text; } - return scr_str_new("[object Object]", 15); + return scr_dyn_object_tag(d); } if (!m || m->kind != SCR_DYN_FUNC) { scr_dyn_release(m); continue; } /* OrdinaryToPrimitive performs a method call, not a bare function diff --git a/packages/runtime/src/scr_symbol.c b/packages/runtime/src/scr_symbol.c index 647c31dc3..727e008b5 100644 --- a/packages/runtime/src/scr_symbol.c +++ b/packages/runtime/src/scr_symbol.c @@ -90,6 +90,11 @@ ScrSym *scr_sym_well_known(ScrStr *name) { scr_dyn_install_has_instance_symbol(key); scr_dyn_release(key); } + if (name->len == 11 && !memcmp(name->data, "toStringTag", 11)) { + ScrDyn *key = scr_dyn_new_symbol(symbol); + scr_dyn_install_to_string_tag_symbol(key); + scr_dyn_release(key); + } if (name->len == 13 && !memcmp(name->data, "asyncIterator", 13)) { ScrDyn *key = scr_dyn_new_symbol(symbol); scr_dyn_install_async_iterator_symbol(key); diff --git a/tests/corpus/7751-class-well-known-symbol-fields.ts b/tests/corpus/7751-class-well-known-symbol-fields.ts new file mode 100644 index 000000000..445af40d6 --- /dev/null +++ b/tests/corpus/7751-class-well-known-symbol-fields.ts @@ -0,0 +1,76 @@ +// Class fields keyed by well-known symbols, Symbol.toStringTag first (the +// lru-cache shape), beside toStringTag getters and module symbol keys. +// String conversion, Object.prototype.toString and util.inspect read the +// tag as Node does, through typed values and through `unknown`. + +const limit = Symbol("limit"); + +class Cache { + size = 0; + [limit] = 10; + [Symbol.toStringTag] = "LRUCache"; +} + +class Sub extends Cache { + extra = 1; +} + +class Retag extends Cache { + constructor() { + super(); + this[Symbol.toStringTag] = "Retagged"; + } +} + +class Tagged { + x = 1; + get [Symbol.toStringTag](): string { + return "Custom"; + } +} + +class TaggedSub extends Tagged { + y = 2; +} + +class Named { + get [Symbol.toStringTag](): string { + return "Named"; + } +} + +class Blank { + get [Symbol.toStringTag](): string { + return ""; + } +} + +class Plain { + n = 3; +} + +const c = new Cache(); +console.log(String(c), `${c}`, Object.prototype.toString.call(c), c[Symbol.toStringTag]); +console.log(c); +console.log(Object.keys(c), JSON.stringify(c)); +console.log(new Sub(), String(new Sub())); +const r: Cache = new Retag(); +console.log(String(r), r); +c[Symbol.toStringTag] = "Other"; +console.log(`${c}`, c); + +const t = new Tagged(); +console.log(String(t), `${t}`, t, [t], { deep: { deeper: { t } } }); +const ts: Tagged = new TaggedSub(); +console.log(String(ts), ts); +console.log(new Named(), new Blank(), String(new Blank())); +console.log(String(new Plain()), new Plain()); + +const values: unknown[] = [new Cache(), new Tagged(), new TaggedSub(), new Plain()]; +for (const v of values) { + console.log(Object.prototype.toString.call(v), String(v), (v as { [Symbol.toStringTag]?: unknown })[Symbol.toStringTag]); +} +console.log(values); + +const literal: unknown = { [Symbol.toStringTag]: "Literal", a: 1 }; +console.log(Object.prototype.toString.call(literal), String(literal), `${literal}`); diff --git a/tests/harness/corpus-costs/7751-class-well-known-symbol-fields.ts.json b/tests/harness/corpus-costs/7751-class-well-known-symbol-fields.ts.json new file mode 100644 index 000000000..5ed28c9dd --- /dev/null +++ b/tests/harness/corpus-costs/7751-class-well-known-symbol-fields.ts.json @@ -0,0 +1 @@ +{ "plain": 3800, "san": 4300 } From 9eb14207a63e971c46a66b76c5a796d3702a7a92 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 15:53:36 -0500 Subject: [PATCH 101/184] Revert "Cache untyped property access in programs that declare classes (reland)" This reverts #2017 (adb8690f1), which made main CI fail. --- benchmarks/runtime/README.md | 1 - .../runtime/workloads/dynamic-properties.json | 6 - .../runtime/workloads/dynamic-properties.mjs | 56 ------ packages/compiler/src/ir/js-ic.test.ts | 188 ------------------ packages/compiler/src/ir/js-ic.ts | 92 +-------- packages/compiler/test/js-ic-emission.test.ts | 44 ---- packages/runtime/src/scr_js.h | 11 +- packages/runtime/src/scr_js_object.c | 32 +-- packages/runtime/src/scr_js_object.test.c | 26 +-- scripts/zero-cost-compare.mjs | 5 +- tests/harness/library-mode.test.ts | 20 -- tests/library-mode/js-ic-sessions/lib.ts | 6 - tests/library-mode/js-ic-sessions/probe.c | 25 --- .../library-mode/js-ic-sessions/profile.json | 14 -- tests/library-mode/js-ic-sessions/work.mjs | 26 --- 15 files changed, 31 insertions(+), 521 deletions(-) delete mode 100644 benchmarks/runtime/workloads/dynamic-properties.json delete mode 100644 benchmarks/runtime/workloads/dynamic-properties.mjs delete mode 100644 tests/library-mode/js-ic-sessions/lib.ts delete mode 100644 tests/library-mode/js-ic-sessions/probe.c delete mode 100644 tests/library-mode/js-ic-sessions/profile.json delete mode 100644 tests/library-mode/js-ic-sessions/work.mjs diff --git a/benchmarks/runtime/README.md b/benchmarks/runtime/README.md index 04c90f7bb..4f152e4bb 100644 --- a/benchmarks/runtime/README.md +++ b/benchmarks/runtime/README.md @@ -13,7 +13,6 @@ Application-shaped workloads for measuring the speed of compiled executables. Ea | `cyclic-graphs` | cpu | object graphs with back references built and dropped: trees with parent and sibling links, rings of records sharing a long-lived registry, closures capturing their owner | | `dictionary-walks` | cpu | `Record` dictionaries walked with for-in and `Object.keys`/`values`/`entries` while entries are added, updated and deleted | | `dynamic-methods` | cpu | untyped JavaScript calling builtin methods on values it does not type: string `slice`/`indexOf`/`toUpperCase`/`padStart`, array `push`/`pop`/`includes`, `Map` and `Set`, and an object's own methods | -| `dynamic-properties` | cpu | untyped JavaScript reading and writing properties of values it does not type: records built by object literals in two key orders, nested records, and instances of a class and its subclass | | `functional-pipeline` | cpu | `filter`/`map`/`reduce`/`flatMap` chains, closures, small records | | `intern-table`, `intern-table-map`, `intern-table-1`, `intern-table-map-1` | cpu | worker threads interning the same type-like objects under string keys and records under number keys, then hits and misses: one shared `InternTable` per kind of key, or a module-level `Map` per thread (the `-map` entries), with 8 threads or 1 (the `-1` entries); under Node.js every thread has its own tables | | `json-records` | cpu | `JSON.stringify`, typed `JSON.parse`, Map aggregation | diff --git a/benchmarks/runtime/workloads/dynamic-properties.json b/benchmarks/runtime/workloads/dynamic-properties.json deleted file mode 100644 index 398981ef3..000000000 --- a/benchmarks/runtime/workloads/dynamic-properties.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "entry": "workloads/dynamic-properties.mjs", - "args": [ - "1000000" - ] -} diff --git a/benchmarks/runtime/workloads/dynamic-properties.mjs b/benchmarks/runtime/workloads/dynamic-properties.mjs deleted file mode 100644 index d0ee3802c..000000000 --- a/benchmarks/runtime/workloads/dynamic-properties.mjs +++ /dev/null @@ -1,56 +0,0 @@ -// Untyped JavaScript reading and writing properties of values it never -// declares types for: records built by object literals in two key orders, -// nested records, property updates, and instances of a class and its -// subclass read and written the same way. Argument: the number of -// iterations. -const rounds = Number(process.argv[2] ?? "1000000"); -class Cursor { - constructor(x, y) { - this.x = x; - this.y = y; - } - step(dx, dy) { - this.x += dx; - this.y += dy; - return this; - } -} -class Marker extends Cursor { - constructor(x, y, label) { - super(x, y); - this.label = label; - } -} -function makeItem(i) { - if (i % 3 === 0) - return { id: i, price: i % 17, qty: i % 5, meta: { tag: i % 7, weight: i % 11 } }; - return { id: i, qty: i % 5, price: i % 13, meta: { weight: i % 9, tag: i % 5 } }; -} -function cost(item) { - return item.price * item.qty + item.meta.weight; -} -function restock(item, round) { - item.qty = (item.qty + round) % 7; - item.meta.tag = (item.meta.tag + 1) % 7; -} -function distance(a, b) { - return Math.abs(a.x - b.x) + Math.abs(a.y - b.y); -} -const items = []; -for (let i = 0; i < 1024; i++) items.push(makeItem(i)); -const cursors = []; -for (let i = 0; i < 64; i++) - cursors.push(i % 2 === 0 ? new Cursor(i - 32, 7 - i) : new Marker(i - 32, 7 - i, "m" + i)); -const origin = { x: 0, y: 0 }; -let checksum = 0; -for (let r = 0; r < rounds; r++) { - const item = items[r & 1023]; - checksum += cost(item); - restock(item, r & 3); - checksum += item.meta.tag; - const cursor = cursors[r & 63]; - checksum += distance(cursor.step((r % 3) - 1, (r % 5) - 2), origin); - if (cursor.x > 100 || cursor.x < -100) cursor.x = 0; - if (cursor.y > 100 || cursor.y < -100) cursor.y = 0; -} -console.log(checksum, items[5].qty, items[6].meta.tag, cursors[3].x, cursors[3].y, cursors[3].label); diff --git a/packages/compiler/src/ir/js-ic.test.ts b/packages/compiler/src/ir/js-ic.test.ts index f5391b83e..f71a1e9db 100644 --- a/packages/compiler/src/ir/js-ic.test.ts +++ b/packages/compiler/src/ir/js-ic.test.ts @@ -126,140 +126,6 @@ test("property helpers whose class branches only test capsules are cache sites", expect(sites.site("%dyn.class.property.1")).toEqual({ kind: "set", key: "y" }); }); -test("bags converted between dyn and jsval are still the receiver", () => { - // A program that declares a class returns its bags as jsvals, and the - // helpers read them through js.toDyn. - const toDyn = (value: IrExpr): IrExpr => ({ - kind: "libCall", - fn: "js.toDyn", - args: [value], - type: DYN, - loc, - }); - const fromDyn = (value: IrExpr): IrExpr => ({ - kind: "libCall", - fn: "js.fromDyn", - args: [value], - type: { kind: "jsval" }, - loc, - }); - const jsvalBag = helper( - "%dyn.class.readPropertiesOrEmpty.js", - ["v"], - [capsuleBranch("v"), ret(fromDyn(ref("v")))], - ); - const staticBag = helper("%dyn.class.staticProperties", ["v"], [ret(ref("v"))]); - const getter = getHelper( - "%dyn.class.property.0", - "x", - [capsuleBranch("p.0")], - call(staticBag.name, toDyn(call(jsvalBag.name, ref("p.0")))), - ); - const setter = setHelper("%dyn.class.property.1", "y"); - const write = setter.body[0] as Extract; - const keySet = write.expr as Extract; - keySet.args[0] = toDyn(call(jsvalBag.name, ref("p.0"))); - // A converted value that is not the receiver stays a helper call. - const foreign = getHelper( - "%dyn.class.property.2", - "x", - [], - toDyn(fromDyn({ kind: "dynObjLit", type: DYN, loc } as IrExpr)), - ); - const sites = sitesOf(jsvalBag, staticBag, getter, setter, foreign); - expect(sites.site("%dyn.class.property.0")).toEqual({ kind: "get", key: "x" }); - expect(sites.site("%dyn.class.property.1")).toEqual({ kind: "set", key: "y" }); - expect(sites.site("%dyn.class.property.2")).toBeNull(); -}); - -test("a bag helper may first rebind its parameter to a bag of it", () => { - // Programs with subclasses normalize the value before the class branches. - const normalize = helper("%dyn.class.normalize", ["v"], [capsuleBranch("v"), ret(ref("v"))]); - const rebind = (value: IrExpr): IrStmt => ({ kind: "assign", localId: "v", value, loc }); - const normalized = helper( - "%dyn.class.readPropertiesOrEmpty.normalized", - ["v"], - [rebind(call(normalize.name, ref("v"))), capsuleBranch("v"), ret(ref("v"))], - ); - const replaced = helper( - "%dyn.class.readPropertiesOrEmpty.replaced", - ["v"], - [rebind({ kind: "dynObjLit", type: DYN, loc } as IrExpr), ret(ref("v"))], - ); - const sites = sitesOf( - normalize, - normalized, - replaced, - getHelper("%dyn.class.property.0", "x", [], call(normalized.name, ref("p.0"))), - getHelper("%dyn.class.property.1", "x", [], call(replaced.name, ref("p.0"))), - // The property helper itself normalizes its receiver first, too. - getHelper("%dyn.class.property.2", "x", [ - { kind: "assign", localId: "p.0", value: call(normalize.name, ref("p.0")), loc }, - capsuleBranch("p.0"), - ]), - ); - expect(sites.site("%dyn.class.property.0")).toEqual({ kind: "get", key: "x" }); - expect(sites.site("%dyn.class.property.1")).toBeNull(); - expect(sites.site("%dyn.class.property.2")).toEqual({ kind: "get", key: "x" }); -}); - -test("keyed helpers may test capsules between the key's coercion and the read", () => { - const read = (object: IrExpr, key: string): IrExpr => ({ - kind: "libCall", - fn: "js.fromDyn", - args: [{ kind: "dynKeyGet", value: object, key: ref(key), type: DYN, loc }], - type: { kind: "jsval" }, - loc, - }); - const guard = (test: "symbol" | "nullish", tested: string, object: IrExpr): IrStmt => ({ - kind: "if", - cond: { kind: "dynTest", test, value: ref(tested), type: BOOL, loc }, - then: [ret(read(object, "p.key"))], - else_: null, - loc, - }); - const keyed = (name: string, between: IrStmt[]): IrFunction => - helper( - name, - ["p.0", "p.key"], - [ - capsuleBranch("p.0"), - guard("symbol", "p.key", call(identityBag.name, ref("p.0"))), - guard("nullish", "p.0", ref("p.0")), - { - kind: "varDecl", - localId: "key.string", - init: { - kind: "libCall", - fn: "dyn.toStringCoerce", - args: [ref("p.key")], - type: STRING, - loc, - }, - loc, - }, - ...between, - ret(read(call(identityBag.name, ref("p.0")), "key.string")), - ], - ["key.string"], - ); - const errorBranch: IrStmt = { - kind: "if", - cond: { kind: "dynTest", test: "error", value: ref("p.0"), type: BOOL, loc }, - then: [ret(ref("p.0"))], - else_: null, - loc, - }; - const sites = sitesOf( - keyed("%dyn.class.computed.0", []), - keyed("%dyn.class.computed.1", [capsuleBranch("p.0"), capsuleBranch("p.0", "dyn.classIs")]), - keyed("%dyn.class.computed.2", [errorBranch]), - ); - expect(sites.site("%dyn.class.computed.0")).toEqual({ kind: "keyed" }); - expect(sites.site("%dyn.class.computed.1")).toEqual({ kind: "keyed" }); - expect(sites.site("%dyn.class.computed.2")).toBeNull(); -}); - test("any other prefix or tail keeps the helper call", () => { const errorBranch: IrStmt = { kind: "if", @@ -371,57 +237,3 @@ test("bag helpers that exit early for values no class branch matches are still t expect(sites.site("%dyn.class.property.2")).toBeNull(); expect(sites.site("%dyn.class.property.3")).toBeNull(); }); - -test("helpers that answer plain values first with the same operation are cache sites", () => { - const negated = (test: "classCapsule" | "funcKind"): IrExpr => ({ - kind: "dynTest", - test, - negated: true, - value: ref("p.0"), - type: BOOL, - loc, - }); - const exit = (body: IrStmt[], test: IrExpr = negated("classCapsule")): IrStmt => ({ - kind: "if", - cond: test, - then: [{ kind: "if", cond: negated("funcKind"), then: body, else_: null, loc }], - else_: null, - loc, - }); - const store = (key: string): IrStmt[] => [ - { - kind: "exprStmt", - expr: { - kind: "libCall", - fn: "dyn.keySet", - args: [ref("p.0"), str(key), ref("p.1")], - type: VOID, - loc, - }, - loc, - }, - ret(null), - ]; - const load = (key: string): IrStmt[] => [ - ret({ kind: "dynKeyGet", value: ref("p.0"), key: str(key), type: DYN, loc }), - ]; - const sites = sitesOf( - setHelper("%dyn.class.property.0", "x", [exit(store("x")), capsuleBranch("p.0")]), - getHelper("%dyn.class.property.1", "x", [exit(load("x")), capsuleBranch("p.0")]), - // An exit for another key, or one plain values do not take. - setHelper("%dyn.class.property.2", "x", [exit(store("y"))]), - setHelper("%dyn.class.property.3", "x", [ - exit(store("x"), { - kind: "dynTest", - test: "classCapsule", - value: ref("p.0"), - type: BOOL, - loc, - }), - ]), - ); - expect(sites.site("%dyn.class.property.0")).toEqual({ kind: "set", key: "x" }); - expect(sites.site("%dyn.class.property.1")).toEqual({ kind: "get", key: "x" }); - expect(sites.site("%dyn.class.property.2")).toBeNull(); - expect(sites.site("%dyn.class.property.3")).toBeNull(); -}); diff --git a/packages/compiler/src/ir/js-ic.ts b/packages/compiler/src/ir/js-ic.ts index 48019636a..752316db3 100644 --- a/packages/compiler/src/ir/js-ic.ts +++ b/packages/compiler/src/ir/js-ic.ts @@ -61,68 +61,13 @@ export class JsIcSites { if (fn.params.some((p) => p.type.kind !== "dyn")) return null; const receiver = fn.params[0]?.localId; if (receiver === undefined) return null; - const second = fn.params[1]?.localId; - const exits: JsIcSite[] = []; let i = 0; - for (; i < fn.body.length; i++) { - const s = fn.body[i]!; - if (this.capsuleBranch(s, receiver) || this.rebinding(s, receiver)) continue; - const exit = this.plainExit(s, receiver, second); - if (exit === null) break; - exits.push(exit); - } + while (i < fn.body.length && this.capsuleBranch(fn.body[i]!, receiver)) i++; const tail = fn.body.slice(i); - let site: JsIcSite | null; - if (fn.params.length === 1) site = this.getTail(tail, receiver); - else if (fn.params.length !== 2 || second === undefined) return null; - else site = this.setTail(tail, receiver, second) ?? this.keyedTail(tail, receiver, second); - if (site === null || site.kind === "keyed") return exits.length === 0 ? site : null; - // An exit answers plain values with the operation the site caches. - return exits.every((exit) => exit.kind === site.kind && exit.key === site.key) ? site : null; - } - - /** `if (!classCapsule(receiver)) { [if (!funcKind(receiver))] OP }`, no - * else: an exit that runs the helper's generic operation directly on a - * value no class table matches. OP is `return [js.fromDyn](dynKeyGet(R, - * K))` or `dyn.keySet(R, K, value); return`, with R the receiver (or a - * bag of it). */ - private plainExit(s: IrStmt, receiver: string, value: string | undefined): JsIcSite | null { - const negated = (e: IrExpr, test: "classCapsule" | "funcKind"): boolean => - e.kind === "dynTest" && e.test === test && e.negated === true && isVar(e.value, receiver); - if (s.kind !== "if" || s.else_ !== null || !negated(s.cond, "classCapsule")) return null; - let body = s.then; - const inner = body[0]; - if ( - body.length === 1 && - inner?.kind === "if" && - inner.else_ === null && - negated(inner.cond, "funcKind") - ) - body = inner.then; - const [first, ret] = body; - if (body.length === 1 && first?.kind === "return") { - const read = unwrapFromDyn(first.value); - if ( - read?.kind !== "dynKeyGet" || - read.key.kind !== "strLit" || - !this.bagOf(read.value, receiver) - ) - return null; - return { kind: "get", key: read.key.value }; - } - if ( - body.length !== 2 || - first?.kind !== "exprStmt" || - ret?.kind !== "return" || - ret.value !== null - ) - return null; - const call = first.expr; - if (call.kind !== "libCall" || call.fn !== "dyn.keySet" || call.args.length !== 3) return null; - const [bag, key, stored] = call.args; - if (!bag || !this.bagOf(bag, receiver) || key?.kind !== "strLit") return null; - if (value === undefined || !isVar(stored, value)) return null; - return { kind: "set", key: key.value }; + if (fn.params.length === 1) return this.getTail(tail, receiver); + if (fn.params.length !== 2) return null; + const second = fn.params[1]!.localId; + return this.setTail(tail, receiver, second) ?? this.keyedTail(tail, receiver, second); } /** `varDecl bag = BAG(receiver); return [js.fromDyn](bag === receiver ? @@ -164,14 +109,10 @@ export class JsIcSites { /** `if (typeof key === "symbol") return dynKeyGet(BAG(receiver), key); * if (receiver == null) return dynKeyGet(receiver, key); * const s = String(key); return dynKeyGet(BAG(receiver), s)`, each read - * possibly under js.fromDyn. Capsule branches may precede the final read: - * a program that declares classes looks the string key up in each class's - * key table there. */ + * possibly under js.fromDyn. */ private keyedTail(tail: IrStmt[], receiver: string, key: string): JsIcSite | null { - const [symbol, nullish, decl] = tail; - const ret = tail[tail.length - 1]; - if (tail.length < 4) return null; - if (!tail.slice(3, -1).every((s) => this.capsuleBranch(s, receiver))) return null; + const [symbol, nullish, decl, ret] = tail; + if (tail.length !== 4) return null; if (!this.guard(symbol, "symbol", key, (read) => this.bagOf(read, receiver), key)) return null; if (!this.guard(nullish, "nullish", receiver, (read) => isVar(read, receiver), key)) return null; @@ -218,12 +159,9 @@ export class JsIcSites { } /** Whether `e` is the receiver itself for every ordinary object or array: - * the receiver, or a chain of property-bag helpers applied to it. A - * conversion between the dyn and jsval forms keeps the object it holds. */ + * the receiver, or a chain of property-bag helpers applied to it. */ private bagOf(e: IrExpr, receiver: string): boolean { if (isVar(e, receiver)) return true; - if (e.kind === "libCall" && (e.fn === "js.toDyn" || e.fn === "js.fromDyn")) - return e.args.length === 1 && this.bagOf(e.args[0]!, receiver); return ( e.kind === "call" && e.args.length === 1 && @@ -244,13 +182,6 @@ export class JsIcSites { return ret.kind === "return" && ret.value !== null && this.bagOf(ret.value, param); } - /** `param = `, such as the subclass normalization - * `value = %dyn.class.normalize(value)`, which leaves an ordinary object - * or array in `param`. */ - private rebinding(s: IrStmt, param: string): boolean { - return s.kind === "assign" && s.localId === param && this.bagOf(s.value, param); - } - /** A property-bag helper that returns its argument unless a capsule * branch is taken. */ private identityHelper(name: string): boolean { @@ -273,10 +204,7 @@ export class JsIcSites { identity = fn.body .slice(0, -1) - .every( - (s) => - this.capsuleBranch(s, param) || this.kindExit(s, param) || this.rebinding(s, param), - ) && + .every((s) => this.capsuleBranch(s, param) || this.kindExit(s, param)) && last?.kind === "return" && last.value !== null && this.bagOf(last.value, param); diff --git a/packages/compiler/test/js-ic-emission.test.ts b/packages/compiler/test/js-ic-emission.test.ts index 54ea13d26..47d641bff 100644 --- a/packages/compiler/test/js-ic-emission.test.ts +++ b/packages/compiler/test/js-ic-emission.test.ts @@ -64,50 +64,6 @@ console.log(read(o), at(dyn([5, 6]), 1), call(o), o.n); } }); -test("property sites keep their caches in programs that declare classes and subclasses", async () => { - const dir = await mkdtemp(join(tmpdir(), "scriptc-js-ic-class-")); - try { - const entry = join(dir, "main.js"); - const outPath = join(dir, "main.ir.json"); - // Declared classes give every property helper class branches and - // property-bag lookups, and a subclass adds a normalization step; plain - // objects still read through the caches. - await writeFile( - entry, - ` -class Point { - constructor(x) { this.x = x; } -} -class Tagged extends Point { - constructor(x, tag) { super(x); this.tag = tag; } -} -function dyn(v) { return v; } -function read(o) { return o.x; } -function write(o, v) { o.x = v; } -function at(a, i) { return a[i]; } -var o = dyn({ x: 1 }); -var p = dyn(new Point(4)); -var t = dyn(new Tagged(6, "t")); -write(o, 3); -write(p, 5); -write(t, 7); -console.log(read(o), read(p), read(t), at(dyn([5, 6]), 1), at(p, "x"), at(t, "tag")); -`, - ); - const result = await compile(entry, { outDir: dir, outPath, outputKind: "ir" }); - if (!result.ok) - throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); - const mod = deserializeModule(await readFile(outPath, "utf8")); - expect(validateModule(mod)).toEqual([]); - const llvm = emitLlvmModule(mod); - expect(body(llvm, "read")).toContain("@scr_js_load_probe("); - expect(body(llvm, "write")).toContain("@scr_js_store_probe("); - expect(body(llvm, "at")).toContain("@scr_js_keyed_probe("); - } finally { - await rm(dir, { recursive: true, force: true }); - } -}); - test("typed programs have no cache sites and keep the ordinary runtime", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-js-ic-typed-")); try { diff --git a/packages/runtime/src/scr_js.h b/packages/runtime/src/scr_js.h index 1d874b31e..6fa20fdb8 100644 --- a/packages/runtime/src/scr_js.h +++ b/packages/runtime/src/scr_js.h @@ -195,13 +195,10 @@ bool scr_js_truthy(ScrJsVal value); * index and which keys an object lacks. * * Each property site of compiled dynamic-tier code owns one zeroed, - * thread-local ScrJsIc with static storage: the reset that frees a - * thread's shapes (at thread exit, and when a library session ends) clears - * every record that holds entries. The code at the site is fixed: it tests - * entry 0 inline and calls a probe for everything else. Probes never run - * user code: they answer from plain data properties or return a miss - * (NULL / false) with nothing done, and the site then runs its generic - * path. + * thread-local ScrJsIc. The code at the site is fixed: it tests entry 0 + * inline and calls a probe for everything else. Probes never run user + * code: they answer from plain data properties or return a miss (NULL / + * false) with nothing done, and the site then runs its generic path. * * load entry: shape = receiver shape; target = NULL for an own property * (slot indexes the receiver's entries), else the shape of diff --git a/packages/runtime/src/scr_js_object.c b/packages/runtime/src/scr_js_object.c index 0e0ec0908..63929ab57 100644 --- a/packages/runtime/src/scr_js_object.c +++ b/packages/runtime/src/scr_js_object.c @@ -18,7 +18,6 @@ #include "scr_key.h" #include -#include /* The compiler's inline cache tests read these in place (64-bit targets; * backend/llvm/js-ic.ts). */ @@ -38,10 +37,10 @@ _Static_assert(sizeof(ScrJsIcEntry) == 40 && sizeof(ScrJsIc) == 168, "cache layo * take the generic path and its hash index. */ #define SCR_JS_SHAPE_MAX 64 -/* Shapes live until the thread exits or a library session ends, so a thread - * creates at most this many, holding at most this many key bytes. Once - * either budget is spent, objects whose key sequence has no shape yet stay - * unshaped; shapes that exist keep serving. */ +/* Shapes live until the thread exits, so a thread creates at most this + * many, holding at most this many key bytes. Once either budget is spent, + * objects whose key sequence has no shape yet stay unshaped; shapes that + * exist keep serving. */ #define SCR_JS_SHAPE_BUDGET ((size_t)1 << 16) #define SCR_JS_SHAPE_KEY_BYTES_BUDGET ((size_t)4 << 20) @@ -64,25 +63,12 @@ static SCR_TL size_t scr_js_transition_count; static SCR_TL size_t scr_js_shape_key_bytes; static SCR_TL bool scr_js_shapes_registered; -/* The cache records that hold entries. Entries name shapes and prototype - * objects, so the reset that frees the shapes also clears these records: a - * library's next session runs the same sites with the same records. */ -static SCR_TL ScrJsIc **scr_js_filled_ics; -static SCR_TL size_t scr_js_filled_count; -static SCR_TL size_t scr_js_filled_capacity; - /* The shape word of an object whose key sequence has no shape within the * budgets. No cache entry names it, so its accesses take the probes, * which answer with a miss until the key set changes. */ static ScrJsShape scr_js_unshaped; static void scr_js_shapes_reset(void) { - for (size_t i = 0; i < scr_js_filled_count; i++) - memset(scr_js_filled_ics[i], 0, sizeof *scr_js_filled_ics[i]); - free(scr_js_filled_ics); - scr_js_filled_ics = NULL; - scr_js_filled_count = 0; - scr_js_filled_capacity = 0; while (scr_js_all_shapes) { ScrJsShape *shape = scr_js_all_shapes; scr_js_all_shapes = shape->all_next; @@ -206,16 +192,6 @@ static size_t scr_js_shape_find(const ScrJsShape *shape, const char *key, size_t /* Fills a cache entry: `stale` (a matching entry whose hit failed) when * there is one, else a free or replaceable one. */ static void scr_js_ic_add(ScrJsIc *ic, uint32_t stale, ScrJsIcEntry entry) { - if (ic->count == 0) { - if (scr_js_filled_count == scr_js_filled_capacity) { - size_t capacity = scr_js_filled_capacity ? scr_js_filled_capacity * 2 : 64; - ScrJsIc **grown = realloc(scr_js_filled_ics, capacity * sizeof *grown); - if (!grown) scr_trap("scriptc: out of memory\n"); - scr_js_filled_ics = grown; - scr_js_filled_capacity = capacity; - } - scr_js_filled_ics[scr_js_filled_count++] = ic; - } uint32_t index; if (stale < ic->count) { index = stale; diff --git a/packages/runtime/src/scr_js_object.test.c b/packages/runtime/src/scr_js_object.test.c index 6b6934dea..a14aa9ff6 100644 --- a/packages/runtime/src/scr_js_object.test.c +++ b/packages/runtime/src/scr_js_object.test.c @@ -24,17 +24,11 @@ static ScrDyn *object_with(const ScrStr *key, double n) { return object; } -/* Cache records have static storage, as the compiled sites' do: the reset - * at exit clears every record that holds entries. Each check starts from - * an empty record. */ -static ScrJsIc load_ic; -static ScrJsIc store_ic; - /* Whether a load probe answers `object.key` and returns `n`. */ static bool probe_reads(ScrDyn *object, const ScrStr *key, double n) { - ScrJsIc *ic = &load_ic; - memset(ic, 0, sizeof *ic); - ScrDyn *value = scr_js_load_probe(object, key, ic); + ScrJsIc ic; + memset(&ic, 0, sizeof ic); + ScrDyn *value = scr_js_load_probe(object, key, &ic); if (!value) return false; assert(value->kind == SCR_DYN_NUM && value->v.num == n); scr_dyn_release(value); @@ -72,13 +66,14 @@ static void budget_bounds_shapes(size_t key_len, size_t budget) { /* A store probe that would add a new shape leaves the add to the generic * store; one that replaces through an existing shape still runs. */ - memset(&store_ic, 0, sizeof store_ic); + ScrJsIc ic; + memset(&ic, 0, sizeof ic); ScrStr *added = scr_str_new("added", 5); ScrDyn *value = scr_dyn_new_num(8); - assert(!scr_js_store_probe(object, added, value, &store_ic)); + assert(!scr_js_store_probe(object, added, value, &ic)); assert(object->v.obj.len == 1); - memset(&store_ic, 0, sizeof store_ic); - assert(scr_js_store_probe(object, first, value, &store_ic)); + memset(&ic, 0, sizeof ic); + assert(scr_js_store_probe(object, first, value, &ic)); assert(probe_reads(object, first, 8)); scr_dyn_release(value); scr_str_release(added); @@ -95,9 +90,10 @@ static void immortal_objects_stay_unshaped(void) { object->rc = SIZE_MAX; assert(!probe_reads(object, key, 3)); assert(object->v.obj.shape == NULL); - memset(&store_ic, 0, sizeof store_ic); + ScrJsIc ic; + memset(&ic, 0, sizeof ic); ScrDyn *value = scr_dyn_new_num(4); - assert(!scr_js_store_probe(object, key, value, &store_ic)); + assert(!scr_js_store_probe(object, key, value, &ic)); assert(object->v.obj.shape == NULL); scr_dyn_release(value); scr_str_release(key); diff --git a/scripts/zero-cost-compare.mjs b/scripts/zero-cost-compare.mjs index f202c2ab7..160e33438 100644 --- a/scripts/zero-cost-compare.mjs +++ b/scripts/zero-cost-compare.mjs @@ -394,8 +394,8 @@ export function dynamicTierSymbols(names) { * not yet traced to its typed root. bench/thread-slots, * bench/channel-messages and bench/worker-messages, added later, link them * through worker messages: - * node:worker_threads values are dyn objects. bench/dynamic-methods and - * bench/dynamic-properties are untyped JavaScript. Every other bench workload is fully typed and must link none. The gate and + * node:worker_threads values are dyn objects. bench/dynamic-methods is + * untyped JavaScript. Every other bench workload is fully typed and must link none. The gate and * tests/harness/bench-dynamic-tier.test.ts enforce that, and the test also * fails when a listed workload stops linking them, so remove a workload here * when a leak fix makes its binary clean. */ @@ -410,7 +410,6 @@ export const DYNAMIC_WORKLOADS = new Set([ "bench/thread-slots", "bench/channel-messages", "bench/dynamic-methods", - "bench/dynamic-properties", "bench/worker-messages", "bench/typed-array-scan-workers", "bench/log-summary", diff --git a/tests/harness/library-mode.test.ts b/tests/harness/library-mode.test.ts index b60c3ac54..db4fabb3e 100644 --- a/tests/harness/library-mode.test.ts +++ b/tests/harness/library-mode.test.ts @@ -363,17 +363,6 @@ globals: undefined fenced expect(run.stdout).toBe(SESSION + SESSION + SESSION); }); - test("K4: untyped property caches start empty in every session", async () => { - // The shapes a session's cache entries name are freed when it ends; - // the next session's sites must not find those entries. - const { archive, outDir } = await buildLibrary("js-ic-sessions", emission); - const probe = buildProbe("js-ic-sessions", archive, outDir); - const run = runProbe(probe); - expect(run.signal).toBeNull(); - expect(run.status).toBe(0); - expect(run.stdout).toBe("run: 20000\n".repeat(3)); - }); - /* ── K5/K6/K7: the trap channel ──────────────────────────────────────── */ test("K5: a trap delivers to the sink exactly once, host survives", async () => { @@ -593,15 +582,6 @@ survived, sink_calls=1 expect(run.stdout).toBe(SESSION + SESSION + SESSION); }); - platformTest("K10: property caches across sessions under ASan", async () => { - const { archive, outDir } = await buildLibrary("js-ic-sessions", emission, { sanitize: true }); - const probe = buildProbe("js-ic-sessions", archive, outDir, { sanitize: true }); - const run = runProbe(probe); - expect(run.signal).toBeNull(); - expect(run.status).toBe(0); - expect(run.stdout).toBe("run: 20000\n".repeat(3)); - }); - platformTest("K10: K5/K7 under ASan", async () => { const { archive, outDir } = await buildLibrary("traps", emission, { sanitize: true }); const probe = buildProbe("traps", archive, outDir, { sanitize: true }); diff --git a/tests/library-mode/js-ic-sessions/lib.ts b/tests/library-mode/js-ic-sessions/lib.ts deleted file mode 100644 index ebea59a54..000000000 --- a/tests/library-mode/js-ic-sessions/lib.ts +++ /dev/null @@ -1,6 +0,0 @@ -// The library's typed export over untyped code (work.mjs). -import { runAll } from "./work.mjs"; - -export function run(): number { - return runAll() as number; -} diff --git a/tests/library-mode/js-ic-sessions/probe.c b/tests/library-mode/js-ic-sessions/probe.c deleted file mode 100644 index 7dbcb56bf..000000000 --- a/tests/library-mode/js-ic-sessions/probe.c +++ /dev/null @@ -1,25 +0,0 @@ -/* Three sessions of a library whose untyped code reads and writes through - * property caches: each prints the same result, and under the sanitize - * flavor no session reads a cache entry the previous session freed. */ -#include -#include - -extern void kj_init(void); -extern void kj_set_panic_sink(void (*fn)(void *, const uint8_t *, size_t, uint64_t), void *ctx); -extern void kj_collect(void); -extern double kj_run(void); - -static void sink(void *ctx, const uint8_t *msg, size_t len, uint64_t addr) { - (void)ctx; (void)addr; - printf("UNEXPECTED SINK: %.*s", (int)len, (const char *)msg); -} - -int main(void) { - kj_set_panic_sink(sink, NULL); - for (int session = 0; session < 3; session++) { - kj_init(); - printf("run: %.0f\n", kj_run()); - kj_collect(); - } - return 0; -} diff --git a/tests/library-mode/js-ic-sessions/profile.json b/tests/library-mode/js-ic-sessions/profile.json deleted file mode 100644 index fd802b42c..000000000 --- a/tests/library-mode/js-ic-sessions/profile.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "profile_format": 1, - "name": "conformance-js-ic-sessions", - "entry": "lib.ts", - "emission": "llvm", - "abi": { - "prefix": "kj_", - "init_symbol": "kj_init", - "sink_register_symbol": "kj_set_panic_sink", - "collect_symbol": "kj_collect", - "result_reset_symbol": null - }, - "exports": [{ "export": "run", "symbol": "kj_run", "params": [], "returns": "f64" }] -} diff --git a/tests/library-mode/js-ic-sessions/work.mjs b/tests/library-mode/js-ic-sessions/work.mjs deleted file mode 100644 index 5c1afcd18..000000000 --- a/tests/library-mode/js-ic-sessions/work.mjs +++ /dev/null @@ -1,26 +0,0 @@ -// Dynamic-tier property caches across library sessions: objects built empty -// and filled through untyped stores, read back through untyped loads, and a -// class instance at the same kind of sites. Each session must start with -// empty caches, since the shapes their entries named are freed when the -// previous session ends. -class Box { - constructor(v) { - this.v = v; - } -} -function fill(o, n) { - o.a = n; - o.b = n + 1; - return o; -} -function sum(o) { - return o.a + o.b; -} -function unbox(b) { - return b.v; -} -export function runAll() { - let total = 0; - for (let i = 0; i < 100; i++) total += sum(fill({}, i)); - return total + unbox(new Box(total)); -} From fc229822e7ac3ba33c0e31da0a044854cba5dbf7 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 15:56:08 -0500 Subject: [PATCH 102/184] Support X25519 keys and crypto.diffieHellman X25519 KeyObjects come from generateKeyPair(Sync)("x25519"), which keeps the private key clamped as OpenSSL does, and from createPrivateKey and createPublicKey of PKCS#8, SubjectPublicKeyInfo, JWK (OKP, X25519) and raw keys. They export in the same forms. sign, verify, createSign and createVerify throw OpenSSL's "operation not supported for this keytype", with its library, reason and code properties. X25519 uses mbedTLS's Curve25519 group. crypto.diffieHellman({ privateKey, publicKey }[, callback]) computes the X25519 and ECDH secrets of two keys of one type, given as KeyObjects or in any form createPrivateKey and createPublicKey read. The checks run in Node's order: the options, the callback, both keys present, the key types when both are KeyObjects ("Incompatible key types for Diffie-Hellman"), then the public and private key inputs. Mismatched curves, a small-order X25519 point and key types other than EC and X25519 throw OpenSSL's errors. The callback form passes the secret, or the error without the OpenSSL properties, in a later macrotask, as Node's DHBitsJob does. createPublicKey also reads a private key's DER encodings (pkcs8 and sec1), which Node accepts there. --- .../limitations/builtin-apis/crypto.mdx | 12 +- .../src/generated/node-v24-compatibility.json | 6 +- .../generated/node-v24-internal.json | 15 +- .../ambient/scriptc-node-fallback.d.ts | 23 ++ .../compiler/src/backend/llvm/lib-shared.ts | 2 + .../src/frontend/lowering/builtins/crypto.ts | 4 +- .../src/frontend/lowering/surfaces.ts | 4 +- packages/compiler/src/frontend/type-mapper.ts | 1 + packages/compiler/src/ir/builtin-effects.ts | 1 + .../compiler/src/ir/builtin-signatures.ts | 1 + packages/compiler/src/ir/runtime-features.ts | 1 + packages/compiler/src/library/ir-policy.ts | 1 + packages/compiler/surface-manifest.json | 4 +- .../tests/corpus/7450-crypto-x25519.mjs.json | 6 + .../corpus/7451-crypto-diffie-hellman.ts.json | 6 + ...7452-crypto-diffie-hellman-errors.mjs.json | 6 + .../node-types/diffie-hellman.ts.json | 6 + packages/runtime/src/scr_crypto.h | 3 + packages/runtime/src/scr_pki.c | 339 ++++++++++++++++-- tests/corpus/7450-crypto-x25519.mjs | 90 +++++ tests/corpus/7451-crypto-diffie-hellman.ts | 55 +++ .../7452-crypto-diffie-hellman-errors.mjs | 61 ++++ tests/fixtures/node-types/diffie-hellman.ts | 19 + tests/fixtures/node-types/fenced.ts | 7 +- .../__snapshots__/node-types-fenced.txt | 35 +- .../corpus-costs/7450-crypto-x25519.mjs.json | 1 + .../7451-crypto-diffie-hellman.ts.json | 1 + ...7452-crypto-diffie-hellman-errors.mjs.json | 1 + tests/harness/project-config.test.ts | 18 + 29 files changed, 656 insertions(+), 73 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7450-crypto-x25519.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7451-crypto-diffie-hellman.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7452-crypto-diffie-hellman-errors.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/node-types/diffie-hellman.ts.json create mode 100644 tests/corpus/7450-crypto-x25519.mjs create mode 100644 tests/corpus/7451-crypto-diffie-hellman.ts create mode 100644 tests/corpus/7452-crypto-diffie-hellman-errors.mjs create mode 100644 tests/fixtures/node-types/diffie-hellman.ts create mode 100644 tests/harness/corpus-costs/7450-crypto-x25519.mjs.json create mode 100644 tests/harness/corpus-costs/7451-crypto-diffie-hellman.ts.json create mode 100644 tests/harness/corpus-costs/7452-crypto-diffie-hellman-errors.mjs.json diff --git a/docs/content/limitations/builtin-apis/crypto.mdx b/docs/content/limitations/builtin-apis/crypto.mdx index 964ca4ec9..329aa6809 100644 --- a/docs/content/limitations/builtin-apis/crypto.mdx +++ b/docs/content/limitations/builtin-apis/crypto.mdx @@ -27,22 +27,22 @@ A call on a DiffieHellman value whose arguments no @types/node overload accepts, or whose output encoding is typed `any`, is refused at compile time, as are its `Object.prototype` methods; other property reads, and property writes, throw a coded SC2020 error. `Object.keys()` and `in` do not see a DiffieHellman's own `verifyError` property, and `JSON.stringify()` and `instanceof` are refused. After `createDiffieHellman(primeLength)` throws `Invalid DH parameters`, the next OpenSSL error Node throws can list the failed generation's errors in `opensslErrorStack`; errors in a compiled binary list only their own. - `crypto.diffieHellman`, which takes DH, EC, or X25519 KeyObjects, and the `DiffieHellman` and `DiffieHellmanGroup` constructors are not available. + `crypto.diffieHellman` computes ECDH and X25519 secrets from EC and X25519 keys, given as KeyObjects or in any form `createPrivateKey` and `createPublicKey` read, with Node's errors; DH keys have no KeyObjects here, and the `DiffieHellman` and `DiffieHellmanGroup` constructors are not available. `checkPrime`, `checkPrimeSync`, `generatePrime`, and `generatePrimeSync` support bigint, ArrayBuffer, typed-array, Buffer, and DataView values and the `checks`, `safe`, `add`, `rem`, and `bigint` options on native targets, with Node's errors; the callback forms compute before they return and call back in a later macrotask. Candidates and sizes longer than 262144 bits throw a coded SC2020 error, and wasm32-wasi builds are refused. Node calls their callbacks with `undefined` as the error, which a callback whose error parameter is typed `Error | null`, as @types/node declares it, receives as `null`. - RSA, EC, and Ed25519 private and public KeyObjects come from `generateKeyPairSync`, `generateKeyPair`, `createPrivateKey`, and `createPublicKey`, on native targets. + RSA, EC, Ed25519, and X25519 private and public KeyObjects come from `generateKeyPairSync`, `generateKeyPair`, `createPrivateKey`, and `createPublicKey`, on native targets. `generateKeyPairSync` and `generateKeyPair` check their arguments in Node's order, with Node's errors, including the OpenSSL errors for RSA modulus lengths below 512 or above 2147483647 bits and for public exponents of 0, 1, or an even number, which `generateKeyPair` passes to its callback. RSA keys generate with even modulus lengths from 1024 to 8192 bits and public exponents up to 2147483647; other lengths and exponents that Node accepts, and `paramEncoding: 'explicit'`, throw a coded SC2020 error. The EC curves are prime256v1 (P-256), secp384r1 (P-384), secp521r1 (P-521), and secp256k1. - Keys read and write PEM and DER as PKCS#1, SEC1, PKCS#8, and SubjectPublicKeyInfo, and JWK (Ed25519 keys use PKCS#8, SubjectPublicKeyInfo, and JWK); `createPublicKey` also takes a private key or an X.509 certificate. - EC and Ed25519 keys also read and write the `raw-public` and `raw-private` formats, including compressed EC points. - Raw X25519, X448, Ed448, and post-quantum keys, EC scalars of 0 or at least the curve order, and the EC point at infinity, which Node accepts, throw a coded SC2020 error. + Keys read and write PEM and DER as PKCS#1, SEC1, PKCS#8, and SubjectPublicKeyInfo, and JWK (Ed25519 and X25519 keys use PKCS#8, SubjectPublicKeyInfo, and JWK); `createPublicKey` also takes a private key or an X.509 certificate. + EC, Ed25519, and X25519 keys also read and write the `raw-public` and `raw-private` formats, including compressed EC points. + Raw X448, Ed448, and post-quantum keys, EC scalars of 0 or at least the curve order, and the EC point at infinity, which Node accepts, throw a coded SC2020 error. They support `type`, `asymmetricKeyType`, `asymmetricKeyDetails`, `export()`, `equals()`, and `instanceof KeyObject`. `crypto.sign`, `crypto.verify`, `createSign`, and `createVerify` support RSA PKCS#1 v1.5 and PSS padding and ECDSA with DER and IEEE P1363 encodings over SHA-1 and SHA-2 digests, and Ed25519 through `crypto.sign` and `crypto.verify` with a `null` or `undefined` algorithm. Ed25519 signing, verification, and private keys link when a string literal in the program names `ed25519` (in any case); without one they throw a coded SC2020 error, while Ed25519 public keys still import and export. Sign and Verify values accumulate `update()` data and are not writable streams. - Encrypted private keys (`passphrase`, `cipher`), the other key types (`rsa-pss`, `dsa`, `ed448`, `x25519`, and the rest), and wasm32-wasi builds throw a coded SC2020 error or are refused. + Encrypted private keys (`passphrase`, `cipher`), the other key types (`rsa-pss`, `dsa`, `ed448`, `x448`, and the rest), and wasm32-wasi builds throw a coded SC2020 error or are refused. Comparing keys of different types with `equals()` answers false; Node also leaves an OpenSSL error queued, which a later key operation can report. `publicEncrypt`/`privateDecrypt` remain unsupported. The WebCrypto `crypto` global, which node:crypto also exports as `webcrypto`, supports `randomUUID()` and `getRandomValues()`, also read as values from untyped JavaScript. diff --git a/docs/src/generated/node-v24-compatibility.json b/docs/src/generated/node-v24-compatibility.json index fc5826666..1039bc3d8 100644 --- a/docs/src/generated/node-v24-compatibility.json +++ b/docs/src/generated/node-v24-compatibility.json @@ -11476,9 +11476,9 @@ "inherited": true }, "static": { - "status": "refused", - "detail": "Explicitly rejected by a named compiler diagnostic.", - "verification": "explicit-refusal" + "status": "partial", + "detail": "Implemented for the call shapes accepted by the compiler lowering.", + "verification": "test-backed" } }, { diff --git a/internal/compatibility/generated/node-v24-internal.json b/internal/compatibility/generated/node-v24-internal.json index 83c318a0a..b3d91e453 100644 --- a/internal/compatibility/generated/node-v24-internal.json +++ b/internal/compatibility/generated/node-v24-internal.json @@ -20345,8 +20345,19 @@ "inherited": true }, "static": { - "status": "refused", - "evidence": "surface-manifest:unsupported:node-builtin.crypto.diffieHellman" + "status": "partial", + "evidence": "surface-manifest:node-builtin.crypto.diffieHellman", + "tests": [ + "tests/corpus/1358-crypto-random.ts", + "tests/corpus/1534-crypto-hash-chain.ts", + "tests/corpus/2556-crypto-introspection.ts", + "tests/corpus/2890-crypto-static-utility.ts", + "tests/harness/npm-static.test.ts", + "tests/corpus/node24-crypto-sha2.ts", + "tests/corpus/node24-crypto-kdf.ts", + "tests/corpus/node24-crypto-kdf-errors.ts", + "tests/corpus/node24-crypto-kdf-options.ts" + ] }, "anchorSource": "exact" }, diff --git a/packages/compiler/ambient/scriptc-node-fallback.d.ts b/packages/compiler/ambient/scriptc-node-fallback.d.ts index 10ed35e68..cb08a4965 100644 --- a/packages/compiler/ambient/scriptc-node-fallback.d.ts +++ b/packages/compiler/ambient/scriptc-node-fallback.d.ts @@ -2992,6 +2992,24 @@ declare module "crypto" { options: ED25519KeyPairOptions<"der", "der">, ): KeyPairSyncResult; export function generateKeyPairSync(type: "ed25519", options?: {}): KeyPairKeyObjectResult; + export interface X25519KeyPairOptions { + publicKeyEncoding: { type: "spki"; format: PubF }; + privateKeyEncoding: { type: "pkcs8"; format: PrivF }; + } + export function generateKeyPairSync( + type: "x25519", + options: X25519KeyPairOptions<"pem", "pem">, + ): KeyPairSyncResult; + export function generateKeyPairSync( + type: "x25519", + options: X25519KeyPairOptions<"der", "der">, + ): KeyPairSyncResult; + export function generateKeyPairSync(type: "x25519", options?: {}): KeyPairKeyObjectResult; + export function generateKeyPair( + type: "x25519", + options: {} | undefined, + callback: (err: Error | null, publicKey: KeyObject, privateKey: KeyObject) => void, + ): void; export function generateKeyPair( type: "ed25519", options: {} | undefined, @@ -3020,6 +3038,11 @@ declare module "crypto" { export interface VerifyKeyObjectInput extends SigningOptions { key: KeyObject; } + export function diffieHellman(options: { privateKey: KeyObject; publicKey: KeyObject }): Buffer; + export function diffieHellman( + options: { privateKey: KeyObject; publicKey: KeyObject }, + callback: (err: Error | null, secret: Buffer) => void, + ): void; export function sign( algorithm: string | null | undefined, data: ArrayBufferView, diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 9cdc36cf5..ade8aea5e 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1070,6 +1070,7 @@ export const LIB_FN_SYMS: Record = { "crypto.generateKeyPairSync": "scr_crypto_generate_key_pair_sync", "crypto.generateKeyPairCb": "scr_crypto_generate_key_pair_async", "crypto.signOneshot": "scr_crypto_sign_oneshot", + "crypto.diffieHellman": "scr_crypto_diffie_hellman", "crypto.verifyOneshot": "scr_crypto_verify_oneshot", "crypto.createSign": "scr_crypto_create_sign", "crypto.createVerify": "scr_crypto_create_verify", @@ -1654,6 +1655,7 @@ export function mockTimerSym( export const USES_TIMERS_LIB_FNS = new Set([ "crypto.generateKeyPairCb", + "crypto.diffieHellman", "crypto.checkPrimeCb", "crypto.generatePrimeCb", "process.emitWarning", diff --git a/packages/compiler/src/frontend/lowering/builtins/crypto.ts b/packages/compiler/src/frontend/lowering/builtins/crypto.ts index 051ec18c4..15c057462 100644 --- a/packages/compiler/src/frontend/lowering/builtins/crypto.ts +++ b/packages/compiler/src/frontend/lowering/builtins/crypto.ts @@ -1454,6 +1454,7 @@ function lowerPkiCall( ], sign: ["crypto.signOneshot", 3, 3, "sign(algorithm, data, key)"], verify: ["crypto.verifyOneshot", 4, 4, "verify(algorithm, data, key, signature)"], + diffieHellman: ["crypto.diffieHellman", 1, 2, "diffieHellman(options[, callback])"], createSign: ["crypto.createSign", 1, 1, "createSign(algorithm)"], createVerify: ["crypto.createVerify", 1, 1, "createVerify(algorithm)"], }; @@ -1474,6 +1475,7 @@ function lowerPkiCall( return { kind: "libCall", fn, args: values, type: VOID, loc }; const call: IrExpr = { kind: "libCall", fn, args: values, type: DYN, loc }; const result = lowerer.mapTypeOf(lowerer.typeOf(expr)); - if (result === null || result.kind === "dyn") return call; + // diffieHellman's callback form answers undefined. + if (result === null || result.kind === "dyn" || result.kind === "void") return call; return lowerer.coerceInto(expr, call, result); } diff --git a/packages/compiler/src/frontend/lowering/surfaces.ts b/packages/compiler/src/frontend/lowering/surfaces.ts index 44564f2f1..a54b41fd6 100644 --- a/packages/compiler/src/frontend/lowering/surfaces.ts +++ b/packages/compiler/src/frontend/lowering/surfaces.ts @@ -1563,6 +1563,7 @@ export const BUILTIN_MODULE_FNS: Record< generateKeyPair: { fn: "crypto.generateKeyPairCb", params: [], result: VOID }, sign: { fn: "crypto.signOneshot", params: [], result: DYN }, verify: { fn: "crypto.verifyOneshot", params: [], result: DYN }, + diffieHellman: { fn: "crypto.diffieHellman", params: [], result: DYN }, pbkdf2Sync: { fn: "crypto.pbkdf2", params: [BYTES_U8, BYTES_U8, F64, F64, STRING], @@ -3101,9 +3102,6 @@ export const BUILTIN_MODULE_FENCE_HINTS: Record< "createSign, createVerify, createECDH, and createDiffieHellman", ]), ), - diffieHellman: - "crypto.diffieHellman takes DH, EC or X25519 KeyObjects and has no lowering; " + - "the objects of createDiffieHellman, getDiffieHellman and createECDH compute shared secrets", setFips: "a compiled binary has no FIPS provider to enable, and Node itself throws on setFips(true) " + "in a non-FIPS build — getFips() answers 0 here", diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index 45b9572b6..781c032e7 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -183,6 +183,7 @@ export const CRYPTO_KEY_OPTION_DYN_TYPES: ReadonlySet = new Set([ "PrivateKeyInput", "PublicKeyInput", "RSAKeyPairKeyObjectOptions", + "X25519KeyPairOptions", "RSAKeyPairOptions", "SignKeyObjectInput", "SignPrivateKeyInput", diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index b517d43a1..5dd437c7a 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -1031,6 +1031,7 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "crypto.generateKeyPairCb", "crypto.signOneshot", "crypto.verifyOneshot", + "crypto.diffieHellman", "crypto.createSign", "crypto.createVerify", ]); diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index e174238e2..2703f4014 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -4874,6 +4874,7 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "crypto.generateKeyPairSync": { argTypes: [DYN, DYN], result: DYN }, "crypto.generateKeyPairCb": { argTypes: [DYN, DYN, DYN], result: VOID }, "crypto.signOneshot": { argTypes: [DYN, DYN, DYN], result: DYN }, + "crypto.diffieHellman": { argTypes: [DYN, DYN], result: DYN }, "crypto.verifyOneshot": { argTypes: [DYN, DYN, DYN, DYN], result: DYN }, "crypto.createSign": { argTypes: [DYN], result: DYN }, "crypto.createVerify": { argTypes: [DYN], result: DYN }, diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index ca7fb7129..d5d2626c5 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -732,6 +732,7 @@ export const PKI_LIB_FNS: ReadonlySet = new Set([ "crypto.generateKeyPairSync", "crypto.signOneshot", "crypto.verifyOneshot", + "crypto.diffieHellman", ]); /** The WebCrypto algorithms scr_webcrypto_pki.c implements over the pki diff --git a/packages/compiler/src/library/ir-policy.ts b/packages/compiler/src/library/ir-policy.ts index e8fad6dd1..ac5d5e47b 100644 --- a/packages/compiler/src/library/ir-policy.ts +++ b/packages/compiler/src/library/ir-policy.ts @@ -63,6 +63,7 @@ const LIB_MODE_REFUSED_PREFIXES: readonly [string, string][] = [ ["crypto.generateKeyPair", "asymmetric KeyObjects, generateKeyPair, sign and verify"], ["crypto.signOneshot", "asymmetric KeyObjects, generateKeyPair, sign and verify"], ["crypto.verifyOneshot", "asymmetric KeyObjects, generateKeyPair, sign and verify"], + ["crypto.diffieHellman", "crypto.diffieHellman"], ["webcrypto.", "WebCrypto (crypto.subtle)"], ["fs.watch", "fs.watch"], ["watcher.", "fs.watch"], diff --git a/packages/compiler/surface-manifest.json b/packages/compiler/surface-manifest.json index 3387217b5..75541b8e5 100644 --- a/packages/compiler/surface-manifest.json +++ b/packages/compiler/surface-manifest.json @@ -568,9 +568,7 @@ "id": "node-builtin.crypto.diffieHellman", "kind": "node-builtin", "name": "crypto.diffieHellman", - "status": "unsupported", - "code": "SC2020", - "note": "crypto.diffieHellman takes DH, EC or X25519 KeyObjects and has no lowering; the objects of createDiffieHellman, getDiffieHellman and createECDH compute shared secrets" + "status": "static" }, { "id": "node-builtin.crypto.generateKey", diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7450-crypto-x25519.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7450-crypto-x25519.mjs.json new file mode 100644 index 000000000..f156f7530 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7450-crypto-x25519.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7450-crypto-x25519.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7451-crypto-diffie-hellman.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7451-crypto-diffie-hellman.ts.json new file mode 100644 index 000000000..593054dd5 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7451-crypto-diffie-hellman.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7451-crypto-diffie-hellman.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7452-crypto-diffie-hellman-errors.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7452-crypto-diffie-hellman-errors.mjs.json new file mode 100644 index 000000000..7cb78758b --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7452-crypto-diffie-hellman-errors.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7452-crypto-diffie-hellman-errors.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/node-types/diffie-hellman.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/node-types/diffie-hellman.ts.json new file mode 100644 index 000000000..fb8a7a740 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/node-types/diffie-hellman.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/node-types/diffie-hellman.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_crypto.h b/packages/runtime/src/scr_crypto.h index f61d27406..03a9dd109 100644 --- a/packages/runtime/src/scr_crypto.h +++ b/packages/runtime/src/scr_crypto.h @@ -298,6 +298,9 @@ ScrDyn *scr_crypto_generate_key_pair_sync(ScrDyn *type, ScrDyn *options); void scr_crypto_generate_key_pair_async(ScrDyn *type, ScrDyn *options, ScrDyn *callback); ScrDyn *scr_crypto_sign_oneshot(ScrDyn *algorithm, ScrDyn *data, ScrDyn *key); ScrDyn *scr_crypto_verify_oneshot(ScrDyn *algorithm, ScrDyn *data, ScrDyn *key, ScrDyn *signature); +/* crypto.diffieHellman(options[, callback]): the shared secret, or undefined + * with the callback, which receives it in a later macrotask. */ +ScrDyn *scr_crypto_diffie_hellman(ScrDyn *options, ScrDyn *callback); ScrDyn *scr_crypto_create_sign(ScrDyn *algorithm); ScrDyn *scr_crypto_create_verify(ScrDyn *algorithm); bool scr_crypto_is_asymmetric_key(const ScrDyn *value); diff --git a/packages/runtime/src/scr_pki.c b/packages/runtime/src/scr_pki.c index 94a0000e7..a1d5b98a6 100644 --- a/packages/runtime/src/scr_pki.c +++ b/packages/runtime/src/scr_pki.c @@ -1,7 +1,8 @@ /* node:crypto asymmetric keys and signatures over the vendored mbedTLS * public-key layer: createPrivateKey, createPublicKey, generateKeyPairSync - * and generateKeyPair for "rsa", "ec" and "ed25519" keys, the - * PrivateKeyObject and PublicKeyObject handles (type, asymmetricKeyType, + * and generateKeyPair for "rsa", "ec", "ed25519" and "x25519" keys, + * crypto.diffieHellman (ECDH and X25519), the PrivateKeyObject and + * PublicKeyObject handles (type, asymmetricKeyType, * asymmetricKeyDetails, export(), equals()), crypto.sign and crypto.verify, * and the Sign and Verify objects of createSign and createVerify. Keys read * and write PEM and DER as PKCS#1, SEC1, PKCS#8 and SubjectPublicKeyInfo, @@ -23,6 +24,7 @@ #include "mbedtls/asn1write.h" #include "mbedtls/base64.h" #include "mbedtls/bignum.h" +#include "mbedtls/ecdh.h" #include "mbedtls/ecdsa.h" #include "mbedtls/ecp.h" #include "mbedtls/md.h" @@ -40,6 +42,30 @@ static void sp_throw(int kind, const char *code, const char *message) { scr_throw_error_msg_code(kind, message, strlen(message), code); } +/* An OpenSSL error as Node's ThrowCryptoError reports it, + * "error::::", with own library, reason and code + * properties in Node's key order. A crypto job's callback gets the error + * without them (`props` false), as from Node's async jobs. */ +static void sp_throw_openssl(const char *packed, const char *library, const char *reason, const char *code, + bool props) { + char message[160]; + snprintf(message, sizeof message, "error:%s:%s::%s", packed, library, reason); + ScrStr *text = scr_str_new(message, strlen(message)); + ScrError *e = scr_error_new(SCR_ERR_ERROR, text); + scr_str_release(text); + if (props) { + const char *keys[] = {"library", "reason", "code"}; + const char *values[] = {library, reason, code}; + for (size_t i = 0; i < 3; i++) { + if (i == 2) scr_error_set_code(e, code); + ScrStr *value = scr_str_new(values[i], strlen(values[i])); + scr_error_set_prop(e, keys[i], strlen(keys[i]), scr_dyn_new_str(value)); + scr_str_release(value); + } + } + scr_throw_obj(e, &scr_error_retain_v, &scr_error_release_v, scr_error_trace_arg()); +} + void scr_pki_throw_fmt(int kind, const char *code, const char *fmt, ...) { va_list ap, sized; va_start(ap, fmt); @@ -157,6 +183,8 @@ typedef struct SpKey { mbedtls_pk_context pk; /* RSA and EC keys */ bool private_key; bool ed25519; /* an Ed25519 key: ed_seed (private keys) and ed_pub */ + bool x25519; /* an X25519 key, in the same two fields: the private key's + * bytes as given, before clamping, and the public key */ uint8_t ed_seed[32]; uint8_t ed_pub[32]; } SpKey; @@ -215,6 +243,43 @@ static bool sp_is_ec(const SpKey *k) { return mbedtls_pk_get_type(&k->pk) == MBE static bool sp_is_ed(const SpKey *k) { return k->ed25519; } +static bool sp_is_x(const SpKey *k) { return k->x25519; } + +/* An Ed25519 or X25519 key: 32-byte private and public keys, no details. */ +static bool sp_is_okp(const SpKey *k) { return k->ed25519 || k->x25519; } + +/* X25519(scalar, point) per RFC 7748 over mbedTLS's Curve25519 group, or + * of the base point when point is NULL. False when the result is the + * all-zero value (a small-order point), which OpenSSL refuses. */ +static bool sp_x25519(const uint8_t scalar[32], const uint8_t *point, uint8_t out[32]) { + mbedtls_ecp_group grp; + mbedtls_ecp_point P, R; + mbedtls_mpi d; + mbedtls_ecp_group_init(&grp); + mbedtls_ecp_point_init(&P); + mbedtls_ecp_point_init(&R); + mbedtls_mpi_init(&d); + uint8_t k[32]; + memcpy(k, scalar, 32); + k[0] &= 248; + k[31] &= 127; + k[31] |= 64; + size_t n = 0; + bool ok = mbedtls_ecp_group_load(&grp, MBEDTLS_ECP_DP_CURVE25519) == 0 && + mbedtls_mpi_read_binary_le(&d, k, 32) == 0 && + (point ? mbedtls_ecp_point_read_binary(&grp, &P, point, 32) : mbedtls_ecp_copy(&P, &grp.G)) == 0 && + mbedtls_ecp_mul(&grp, &R, &d, &P, sp_random, NULL) == 0 && + mbedtls_ecp_point_write_binary(&grp, &R, MBEDTLS_ECP_PF_UNCOMPRESSED, &n, out, 32) == 0 && n == 32; + uint8_t any = 0; + for (size_t i = 0; ok && i < 32; i++) any |= out[i]; + mbedtls_platform_zeroize(k, sizeof k); + mbedtls_mpi_free(&d); + mbedtls_ecp_point_free(&R); + mbedtls_ecp_point_free(&P); + mbedtls_ecp_group_free(&grp); + return ok && any != 0; +} + typedef struct { const char *name; /* OpenSSL short name (namedCurve) */ const char *nist; /* EC_curve_nist2nid alias, or NULL */ @@ -241,7 +306,7 @@ static const SpCurve *sp_curve_of(const SpKey *k) { /* A parsed key is usable when it is RSA or EC on a curve above. */ static bool sp_key_supported(const SpKey *k) { - return sp_is_ed(k) || sp_is_rsa(k) || (sp_is_ec(k) && sp_curve_of(k) != NULL); + return sp_is_okp(k) || sp_is_rsa(k) || (sp_is_ec(k) && sp_curve_of(k) != NULL); } /* ── DER writing ──────────────────────────────────────────────────────── */ @@ -260,6 +325,11 @@ static const uint8_t sp_ed_pkcs8_prefix[16] = {0x30, 0x2e, 0x02, 0x01, 0x00, 0x3 0x03, 0x2b, 0x65, 0x70, 0x04, 0x22, 0x04, 0x20}; static const uint8_t sp_ed_spki_prefix[12] = {0x30, 0x2a, 0x30, 0x05, 0x06, 0x03, 0x2b, 0x65, 0x70, 0x03, 0x21, 0x00}; +/* X25519's, which differ in the OID's last byte (1.3.101.110). */ +static const uint8_t sp_x_pkcs8_prefix[16] = {0x30, 0x2e, 0x02, 0x01, 0x00, 0x30, 0x05, 0x06, + 0x03, 0x2b, 0x65, 0x6e, 0x04, 0x22, 0x04, 0x20}; +static const uint8_t sp_x_spki_prefix[12] = {0x30, 0x2a, 0x30, 0x05, 0x06, 0x03, + 0x2b, 0x65, 0x6e, 0x03, 0x21, 0x00}; /* Writes prefix || 32 bytes at the end of buf, as the mbedtls writers do. */ static int sp_write_ed_der(const uint8_t *prefix, size_t prefix_len, const uint8_t key[32], @@ -272,7 +342,8 @@ static int sp_write_ed_der(const uint8_t *prefix, size_t prefix_len, const uint8 } static int sp_write_spki_der(SpKey *k, unsigned char *buf, size_t size) { - if (sp_is_ed(k)) return sp_write_ed_der(sp_ed_spki_prefix, sizeof sp_ed_spki_prefix, k->ed_pub, buf, size); + if (sp_is_okp(k)) + return sp_write_ed_der(sp_is_x(k) ? sp_x_spki_prefix : sp_ed_spki_prefix, 12, k->ed_pub, buf, size); return mbedtls_pk_write_pubkey_der(&k->pk, buf, size); } @@ -287,8 +358,8 @@ static int sp_write_pkcs1_public_der(SpKey *k, unsigned char *buf, size_t size) * parameters, or id-ecPublicKey with the curve OID), and the inner key, * which for EC omits the parameters field. */ static int sp_write_pkcs8_der(SpKey *k, unsigned char *buf, size_t size) { - if (sp_is_ed(k)) - return sp_write_ed_der(sp_ed_pkcs8_prefix, sizeof sp_ed_pkcs8_prefix, k->ed_seed, buf, size); + if (sp_is_okp(k)) + return sp_write_ed_der(sp_is_x(k) ? sp_x_pkcs8_prefix : sp_ed_pkcs8_prefix, 16, k->ed_seed, buf, size); unsigned char inner[SP_DER_MAX]; size_t inner_len; if (sp_is_rsa(k)) { @@ -473,9 +544,9 @@ static ScrDyn *sp_b64url(const uint8_t *raw, size_t len) { static ScrDyn *sp_export_jwk(SpKey *k) { ScrDyn *jwk = scr_dyn_new_obj(); - if (sp_is_ed(k)) { + if (sp_is_okp(k)) { sp_set(jwk, "kty", sp_dyn_str(scr_str_new("OKP", 3))); - sp_set(jwk, "crv", sp_dyn_str(scr_str_new("Ed25519", 7))); + sp_set(jwk, "crv", sp_dyn_str(sp_is_x(k) ? scr_str_new("X25519", 6) : scr_str_new("Ed25519", 7))); sp_set(jwk, "x", sp_b64url(k->ed_pub, 32)); if (k->private_key) sp_set(jwk, "d", sp_b64url(k->ed_seed, 32)); return jwk; @@ -587,7 +658,8 @@ static SpKey *sp_key_from_jwk_plain(const ScrDyn *jwk, bool want_private) { bool ok = false; if (sp_is_str(kty, "OKP")) { const ScrDyn *crv = sp_opt(jwk, "crv"); - if (!sp_is_str(crv, "Ed25519")) { + bool x = sp_is_str(crv, "X25519"); + if (!x && !sp_is_str(crv, "Ed25519")) { sp_key_release(k); if (crv->kind != SCR_DYN_STR) { scr_dyn_prop_type_fail("key.crv", "of type string", crv); @@ -596,10 +668,13 @@ static SpKey *sp_key_from_jwk_plain(const ScrDyn *jwk, bool want_private) { sp_not_supported("JWK keys of this crv"); return NULL; } - k->ed25519 = true; + k->ed25519 = !x; + k->x25519 = x; if (private_key) { ok = sp_jwk_bytes(jwk, "d", k->ed_seed); - if (ok) { + if (ok && x) { + ok = sp_x25519(k->ed_seed, NULL, k->ed_pub); + } else if (ok) { const ScrPkiEd25519 *ed = sp_ed(); if (!ed) { sp_key_release(k); @@ -678,9 +753,10 @@ static SpKey *sp_key_from_jwk_plain(const ScrDyn *jwk, bool want_private) { /* The private key of a public SpKey derived from `k`. */ static SpKey *sp_public_of(SpKey *k) { - if (sp_is_ed(k)) { + if (sp_is_okp(k)) { SpKey *pub = sp_key_alloc(false); - pub->ed25519 = true; + pub->ed25519 = k->ed25519; + pub->x25519 = k->x25519; memcpy(pub->ed_pub, k->ed_pub, 32); return pub; } @@ -770,6 +846,24 @@ static bool sp_cert_spki(const unsigned char *der, size_t len, const unsigned ch * as public answers its public key. */ static SpKey *sp_ed_from_der(const unsigned char *der, size_t len, bool private_key, bool *failed) { *failed = false; + /* X25519 keys, which need no other unit. */ + if (len == 48 && memcmp(der, sp_x_pkcs8_prefix, 16) == 0) { + SpKey *k = sp_key_alloc(private_key); + k->x25519 = true; + memcpy(k->ed_seed, der + 16, 32); + if (!sp_x25519(k->ed_seed, NULL, k->ed_pub)) { + sp_key_release(k); + return NULL; + } + if (!private_key) mbedtls_platform_zeroize(k->ed_seed, 32); + return k; + } + if (!private_key && len == 44 && memcmp(der, sp_x_spki_prefix, 12) == 0) { + SpKey *k = sp_key_alloc(false); + k->x25519 = true; + memcpy(k->ed_pub, der + 12, 32); + return k; + } if (len == 48 && memcmp(der, sp_ed_pkcs8_prefix, 16) == 0) { const ScrPkiEd25519 *ed = sp_ed(); if (!ed) { @@ -945,9 +1039,9 @@ static ScrDyn *sp_create_key(ScrDyn *input, bool private_key) { return NULL; } if (sp_is_str(format, "der")) { - bool valid = private_key ? sp_is_str(type, "pkcs1") || sp_is_str(type, "pkcs8") || - sp_is_str(type, "sec1") - : sp_is_str(type, "pkcs1") || sp_is_str(type, "spki"); + /* createPublicKey also reads a private key's encodings. */ + bool valid = sp_is_str(type, "pkcs1") || sp_is_str(type, "pkcs8") || sp_is_str(type, "sec1") || + (!private_key && sp_is_str(type, "spki")); if (!valid) { sp_property_invalid("key.type", type); return NULL; @@ -997,15 +1091,14 @@ static ScrDyn *sp_export_raw(SpKey *k, bool private_part, bool seed, bool compre sp_incompatible_key_type(); return NULL; } - if (sp_is_ed(k)) return sp_dyn_buffer(private_part ? k->ed_seed : k->ed_pub, 32); + if (sp_is_okp(k)) return sp_dyn_buffer(private_part ? k->ed_seed : k->ed_pub, 32); mbedtls_ecp_keypair *ec = mbedtls_pk_ec(k->pk); const mbedtls_ecp_group *grp = &ec->MBEDTLS_PRIVATE(grp); unsigned char buf[1 + 2 * 66]; size_t n = (grp->nbits + 7) / 8; + int form = compressed ? MBEDTLS_ECP_PF_COMPRESSED : MBEDTLS_ECP_PF_UNCOMPRESSED; int rc = private_part ? mbedtls_mpi_write_binary(&ec->MBEDTLS_PRIVATE(d), buf, n) - : mbedtls_ecp_point_write_binary( - grp, &ec->MBEDTLS_PRIVATE(Q), - compressed ? MBEDTLS_ECP_PF_COMPRESSED : MBEDTLS_ECP_PF_UNCOMPRESSED, &n, buf, sizeof buf); + : mbedtls_ecp_point_write_binary(grp, &ec->MBEDTLS_PRIVATE(Q), form, &n, buf, sizeof buf); ScrDyn *out = rc == 0 ? sp_dyn_buffer(buf, n) : NULL; mbedtls_platform_zeroize(buf, sizeof buf); if (!out) sp_unsupported_input(); @@ -1013,10 +1106,9 @@ static ScrDyn *sp_export_raw(SpKey *k, bool private_part, bool seed, bool compre } /* Node's named key types that this runtime does not generate or read: - * the other OKP curves and the post-quantum types. */ + * X448, Ed448 and the post-quantum types. */ static const char *const sp_other_named_types[] = { "ed448", - "x25519", "x448", "ml-dsa-44", "ml-dsa-65", @@ -1071,15 +1163,16 @@ static SpKey *sp_key_from_raw(const ScrDyn *input, const ScrDyn *data, const Scr sp_incompatible_key_type(); return NULL; } + bool x = sp_is_str(type, "x25519"); bool other = false; for (size_t i = 0; i < sizeof sp_other_named_types / sizeof sp_other_named_types[0]; i++) other = other || sp_is_str(type, sp_other_named_types[i]); - if (!ec && !ed && !other) { + if (!ec && !ed && !x && !other) { ScrStr *s = type->v.str; scr_pki_throw_fmt(SCR_ERR_TYPE, "ERR_INVALID_ARG_VALUE", "Invalid asymmetricKeyType: %.*s", (int)s->len, s->data); return NULL; } - bool ml = !ec && !ed && type->v.str->len > 3 && memcmp(type->v.str->data, "ml-", 3) == 0; + bool ml = other && type->v.str->len > 3 && memcmp(type->v.str->data, "ml-", 3) == 0; if (seed && !ml) { sp_incompatible_key_type(); return NULL; @@ -1099,7 +1192,16 @@ static SpKey *sp_key_from_raw(const ScrDyn *input, const ScrDyn *data, const Scr size_t len = (size_t)scr_bytes_byte_len(bytes); SpKey *k = sp_key_alloc(!raw_public); bool ok = false; - if (ed) { + if (x) { + k->x25519 = true; + if (len == 32 && raw_public) { + memcpy(k->ed_pub, p, 32); + ok = true; + } else if (len == 32) { + memcpy(k->ed_seed, p, 32); + ok = sp_x25519(k->ed_seed, NULL, k->ed_pub); + } + } else if (ed) { k->ed25519 = true; if (len == 32 && raw_public) { memcpy(k->ed_pub, p, 32); @@ -1254,7 +1356,7 @@ static ScrDyn *sp_export_plain(SpKey *k, const ScrDyn *options) { static ScrDyn *sp_key_details(SpKey *k) { ScrDyn *details = scr_dyn_new_obj(); - if (sp_is_ed(k)) return details; + if (sp_is_okp(k)) return details; if (sp_is_rsa(k)) { mbedtls_rsa_context *rsa = mbedtls_pk_rsa(k->pk); mbedtls_mpi e; @@ -1280,7 +1382,7 @@ static ScrDyn *sp_key_details(SpKey *k) { /* The DER that identifies a key for equals(): its private encoding for a * private key, its SubjectPublicKeyInfo otherwise. */ static int sp_identity_der(SpKey *k, unsigned char *buf, size_t size) { - if (sp_is_ed(k)) return k->private_key ? sp_write_pkcs8_der(k, buf, size) : sp_write_spki_der(k, buf, size); + if (sp_is_okp(k)) return k->private_key ? sp_write_pkcs8_der(k, buf, size) : sp_write_spki_der(k, buf, size); return k->private_key ? sp_write_private_der(k, buf, size) : sp_write_spki_der(k, buf, size); } @@ -1326,6 +1428,7 @@ static ScrDyn *sp_key_get(void *h, const char *name, size_t len) { return k->private_key ? sp_dyn_str(scr_str_new("private", 7)) : sp_dyn_str(scr_str_new("public", 6)); if (len == 17 && memcmp(name, "asymmetricKeyType", 17) == 0) return sp_is_ed(k) ? sp_dyn_str(scr_str_new("ed25519", 7)) + : sp_is_x(k) ? sp_dyn_str(scr_str_new("x25519", 6)) : sp_is_rsa(k) ? sp_dyn_str(scr_str_new("rsa", 3)) : sp_dyn_str(scr_str_new("ec", 2)); if (len == 20 && memcmp(name, "asymmetricKeyDetails", 20) == 0) return sp_key_details(k); @@ -1543,7 +1646,7 @@ static bool sp_check_encoding_plain(const ScrDyn *enc, const char *name, const S /* The key generateKeyPair makes, as sp_keygen_check reads it. */ typedef struct { - enum { SP_GEN_RSA, SP_GEN_EC, SP_GEN_ED25519 } kind; + enum { SP_GEN_RSA, SP_GEN_EC, SP_GEN_ED25519, SP_GEN_X25519 } kind; double bits, exponent; /* RSA */ const SpCurve *curve; /* EC */ } SpKeyGen; @@ -1604,6 +1707,8 @@ static bool sp_keygen_check(const ScrDyn *type, const ScrDyn *options, SpKeyGen } } else if (sp_is_str(type, "ed25519")) { gen->kind = SP_GEN_ED25519; + } else if (sp_is_str(type, "x25519")) { + gen->kind = SP_GEN_X25519; } else { bool named = false; for (size_t i = 0; i < sizeof sp_other_named_types / sizeof sp_other_named_types[0]; i++) @@ -1646,6 +1751,20 @@ static SpKey *sp_keygen_run(const SpKeyGen *gen, bool *job_error) { scr_pki_ed25519->public_key(k->ed_pub, k->ed_seed); return k; } + if (gen->kind == SP_GEN_X25519) { + /* OpenSSL stores a generated X25519 key clamped. */ + k->x25519 = true; + sp_random(NULL, k->ed_seed, 32); + k->ed_seed[0] &= 248; + k->ed_seed[31] &= 127; + k->ed_seed[31] |= 64; + if (!sp_x25519(k->ed_seed, NULL, k->ed_pub)) { + sp_key_release(k); + sp_unsupported_input(); + return NULL; + } + return k; + } if (gen->kind == SP_GEN_RSA) { const char *failure = sp_rsa_keygen_failure(gen->bits, gen->exponent); if (failure) { @@ -1977,6 +2096,16 @@ static bool sp_verify_hash(SpSigKey *key, mbedtls_md_type_t md, const unsigned c return mbedtls_pk_verify(&key->key->pk, md, hash, hash_len, sig, sig_len) == 0; } +/* X25519 keys neither sign nor verify: OpenSSL's error, after the key + * checks. Releases the key and answers true when it rejects. */ +static bool sp_reject_x25519(SpSigKey *k) { + if (!sp_is_x(k->key)) return false; + sp_key_release(k->key); + sp_throw_openssl("03000096", "digital envelope routines", "operation not supported for this keytype", + "ERR_OSSL_EVP_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE", true); + return true; +} + static const char sp_data_types[] = "an instance of Buffer, TypedArray, or DataView"; /* Ed25519 signs the message itself: an algorithm other than null or @@ -1997,6 +2126,7 @@ ScrDyn *scr_crypto_sign_oneshot(ScrDyn *algorithm, ScrDyn *data, ScrDyn *key) { } SpSigKey k; if (!sp_sig_key(key, true, &k)) return NULL; + if (sp_reject_x25519(&k)) return NULL; ScrBytes *bytes = scr_bytes_buffer_source(data); if (sp_is_ed(k.key)) { ScrDyn *result = NULL; @@ -2034,6 +2164,7 @@ ScrDyn *scr_crypto_verify_oneshot(ScrDyn *algorithm, ScrDyn *data, ScrDyn *key, scr_dyn_arg_type_fail("signature", sp_data_types, signature); return NULL; } + if (sp_reject_x25519(&k)) return NULL; ScrBytes *bytes = scr_bytes_buffer_source(data); ScrBytes *sig = scr_bytes_buffer_source(signature); if (sp_is_ed(k.key)) { @@ -2120,6 +2251,7 @@ static ScrDyn *sp_signer_invoke(void *h, ScrDyn *self, const char *method, ScrDy } SpSigKey k; if (!sp_sig_key(argc ? args[0] : scr_dyn_undefined(), true, &k)) return NULL; + if (sp_reject_x25519(&k)) return NULL; if (sp_is_ed(k.key)) { sp_key_release(k.key); sp_throw(SCR_ERR_ERROR, "ERR_CRYPTO_UNSUPPORTED_OPERATION", "Unsupported crypto operation"); @@ -2157,6 +2289,7 @@ static ScrDyn *sp_signer_invoke(void *h, ScrDyn *self, const char *method, ScrDy signature); return NULL; } + if (sp_reject_x25519(&k)) return NULL; const ScrStr *encoding = sp_encoding_arg(argc > 2 ? args[2] : NULL); ScrDyn *encoding_value = encoding ? scr_dyn_new_str((ScrStr *)encoding) : NULL; ScrBytes *sig = sp_source_bytes(signature, encoding_value); @@ -2323,6 +2456,156 @@ void scr_crypto_generate_key_pair_async(ScrDyn *type, ScrDyn *options, ScrDyn *c scr_loop_defer(c); } +/* ── diffieHellman ────────────────────────────────────────────────────── */ + +/* The keys an options.privateKey or options.publicKey value can be. */ +static const char sp_dh_private_types[] = + "of type string or an instance of ArrayBuffer, Buffer, TypedArray, DataView, KeyObject, CryptoKey, or URL"; +static const char sp_dh_public_types[] = + "of type string or an instance of ArrayBuffer, Buffer, TypedArray, DataView, KeyObject, or CryptoKey"; + +/* Node's message names: a KeyObject's asymmetricKeyType. */ +static const char *sp_type_name(const SpKey *k) { + return sp_is_ed(k) ? "ed25519" : sp_is_x(k) ? "x25519" : sp_is_rsa(k) ? "rsa" : "ec"; +} + +/* A key input of diffieHellman's options: a primitive other than a string + * throws Node's ERR_INVALID_ARG_TYPE for `name`; other values read as + * sign and verify read their keys. */ +static bool sp_dh_key(const ScrDyn *value, const char *name, bool private_key, SpSigKey *out) { + out->key = NULL; + /* Node's key preparation destructures the value before it checks it. */ + if (value->kind == SCR_DYN_NULL) { + scr_pki_throw_fmt(SCR_ERR_TYPE, NULL, "Cannot destructure property 'key' of 'key' as it is null."); + return false; + } + bool keyish = value->kind == SCR_DYN_STR || value->kind == SCR_DYN_BYTES || scr_buffer_storage_is(value) || + (value->kind != SCR_DYN_NULL && value->kind != SCR_DYN_FUNC && scr_dyn_is_object(value)); + if (!keyish) { + scr_dyn_prop_type_fail(name, private_key ? sp_dh_private_types : sp_dh_public_types, value); + return false; + } + return sp_sig_key(value, private_key, out); +} + +/* The shared secret of two keys of one type: X25519, or ECDH's x + * coordinate at the field's size. NULL with OpenSSL's error pending. */ +static void sp_dh_failed(bool props) { + sp_throw_openssl("1C8000A4", "Provider routines", "failed during derivation", "ERR_OSSL_FAILED_DURING_DERIVATION", + props); +} + +static ScrDyn *sp_dh_compute(SpKey *priv, SpKey *pub, bool props) { + if (sp_is_x(priv) && sp_is_x(pub)) { + uint8_t out[32]; + if (!sp_x25519(priv->ed_seed, pub->ed_pub, out)) { + sp_dh_failed(props); + return NULL; + } + ScrDyn *secret = sp_dyn_buffer(out, 32); + mbedtls_platform_zeroize(out, sizeof out); + return secret; + } + if (!sp_is_ec(priv) || !sp_is_ec(pub)) { + sp_throw_openssl("03000096", "digital envelope routines", "operation not supported for this keytype", + "ERR_OSSL_EVP_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE", props); + return NULL; + } + mbedtls_ecp_keypair *a = mbedtls_pk_ec(priv->pk); + const mbedtls_ecp_keypair *b = mbedtls_pk_ec(pub->pk); + if (a->MBEDTLS_PRIVATE(grp).id != b->MBEDTLS_PRIVATE(grp).id) { + sp_throw_openssl("1C8000CB", "Provider routines", "mismatching domain parameters", + "ERR_OSSL_MISMATCHING_DOMAIN_PARAMETERS", props); + return NULL; + } + mbedtls_mpi z; + mbedtls_mpi_init(&z); + unsigned char out[66]; + size_t n = (a->MBEDTLS_PRIVATE(grp).pbits + 7) / 8; + bool ok = mbedtls_ecdh_compute_shared(&a->MBEDTLS_PRIVATE(grp), &z, &b->MBEDTLS_PRIVATE(Q), + &a->MBEDTLS_PRIVATE(d), sp_random, NULL) == 0 && + mbedtls_mpi_write_binary(&z, out, n) == 0; + mbedtls_mpi_free(&z); + ScrDyn *secret = ok ? sp_dyn_buffer(out, n) : NULL; + mbedtls_platform_zeroize(out, sizeof out); + if (!secret) sp_dh_failed(props); + return secret; +} + +static void sp_dh_deliver(ScrClosure *self) { + ScrDyn *callback = sp_cap(self, 0); + ScrDyn *secret = sp_cap(self, 1); + ScrDyn *args[2] = {scr_dyn_new_null(), secret}; + ScrDyn *result = scr_dyn_call(callback, args, 2, "callback"); + if (result) scr_dyn_release(result); + scr_dyn_release(args[0]); + scr_dyn_release(secret); + scr_dyn_release(callback); +} + +/* crypto.diffieHellman(options[, callback]) in Node's order: the options + * object, the callback, both keys present, the key types when both are + * KeyObjects, then the public and private key inputs. With a callback, + * the secret, or the error of computing it, arrives in a later macrotask. */ +ScrDyn *scr_crypto_diffie_hellman(ScrDyn *options, ScrDyn *callback) { + ScrDyn *plain = sp_plain(options); + if (!plain) return NULL; + ScrDyn *result = NULL; + SpSigKey pub = {0}, priv = {0}; + if (!sp_check_object(plain, "options")) goto done; + bool async = !sp_absent(callback); + if (async && !scr_dyn_is_callable(callback)) { + scr_dyn_arg_type_fail("callback", "of type function", callback); + goto done; + } + const ScrDyn *private_value = sp_opt(plain, "privateKey"); + const ScrDyn *public_value = sp_opt(plain, "publicKey"); + if (sp_absent(private_value)) { + sp_property_invalid("options.privateKey", private_value); + goto done; + } + if (sp_absent(public_value)) { + sp_property_invalid("options.publicKey", public_value); + goto done; + } + SpKey *a = sp_key_of(private_value), *b = sp_key_of(public_value); + if (a && b && a->private_key) { + const char *ta = sp_type_name(a), *tb = sp_type_name(b); + if (strcmp(ta, tb) != 0 || (!sp_is_ec(a) && !sp_is_x(a))) { + scr_pki_throw_fmt(SCR_ERR_ERROR, "ERR_CRYPTO_INCOMPATIBLE_KEY", + "Incompatible key types for Diffie-Hellman: %s and %s", ta, tb); + goto done; + } + } + if (!sp_dh_key(public_value, "options.publicKey", false, &pub) || + !sp_dh_key(private_value, "options.privateKey", true, &priv)) + goto done; + ScrDyn *secret = sp_dh_compute(priv.key, pub.key, !async); + if (!async) { + result = secret; + goto done; + } + void *deliver = (void *)sp_dh_deliver; + if (!secret) { + ScrCaught *caught = scr_exc_take(); + secret = scr_caught_to_dyn(caught); + scr_caught_release(caught); + deliver = (void *)sp_key_pair_fail; + } + ScrClosure *c = scr_closure_new(deliver, 2); + c->caps[0] = scr_box_new_obj(scr_dyn_retain_v, scr_dyn_release_v, NULL); + scr_box_set_ref(c->caps[0], scr_dyn_retain(callback)); + c->caps[1] = scr_box_new_obj(scr_dyn_retain_v, scr_dyn_release_v, NULL); + scr_box_set_ref(c->caps[1], secret); + scr_loop_defer(c); + result = scr_dyn_retain(scr_dyn_undefined()); +done: + if (pub.key) sp_key_release(pub.key); + if (priv.key) sp_key_release(priv.key); + scr_dyn_release(plain); + return result; +} + /* ── the core for the WebCrypto layer (scr_pki.h) ─────────────────────── */ ScrPkiKey *scr_pki_key_retain(ScrPkiKey *key) { return sp_key_retain(key); } diff --git a/tests/corpus/7450-crypto-x25519.mjs b/tests/corpus/7450-crypto-x25519.mjs new file mode 100644 index 000000000..180221390 --- /dev/null +++ b/tests/corpus/7450-crypto-x25519.mjs @@ -0,0 +1,90 @@ +// X25519 KeyObjects: generateKeyPair, PKCS#8, SubjectPublicKeyInfo, JWK +// and raw keys, RFC 7748's test vectors through crypto.diffieHellman, and +// Node's errors for signatures and incompatible encodings. +import * as crypto from "node:crypto"; + +function describe(value) { + if (Buffer.isBuffer(value)) return `Buffer(${value.length})`; + if (value instanceof crypto.KeyObject) return `${value.type} ${value.asymmetricKeyType} ${JSON.stringify(value.asymmetricKeyDetails)}`; + if (typeof value === "string") return value.split("\n")[0]; + if (value && typeof value === "object") return Object.entries(value).map(([key, field]) => `${key}=${describe(field)}`).join(" "); + return String(value); +} + +function attempt(label, run) { + try { + console.log(label, describe(run())); + } catch (e) { + console.log(label, "throws", e.name, e.code, e.message); + } +} + +// RFC 7748, section 6.1 +const alicePrivate = Buffer.from("77076d0a7318a57d3c16c17251b26645df4c2f87ebc0992ab177fba51db92c2a", "hex"); +const alicePublic = "8520f0098930a754748b7ddcb43ef75a0dbf3a0d26381af4eba4a98eaa9b4e6a"; +const bobPrivate = Buffer.from("5dab087e624a8a4b79e17f8b83800ee66f3bb1292618b6fd1c2f8b27ff88e0eb", "hex"); +const bobPublic = "de9edb7d7b7dc1b4d35b61c2ece435373f8343c85b78674dadfc7e146f882b4f"; +const alice = crypto.createPrivateKey({ key: alicePrivate, format: "raw-private", asymmetricKeyType: "x25519" }); +const bob = crypto.createPrivateKey({ key: bobPrivate, format: "raw-private", asymmetricKeyType: "x25519" }); +console.log("alice public", Buffer.from(crypto.createPublicKey(alice).export({ format: "raw-public" })).toString("hex") === alicePublic); +console.log("bob public", Buffer.from(crypto.createPublicKey(bob).export({ format: "raw-public" })).toString("hex") === bobPublic); +const shared = crypto.diffieHellman({ privateKey: alice, publicKey: crypto.createPublicKey(bob) }); +console.log("shared", shared.toString("hex")); +console.log("both ways", crypto.diffieHellman({ privateKey: bob, publicKey: crypto.createPublicKey(alice) }).equals(shared)); +console.log("raw-private kept as given", Buffer.from(alice.export({ format: "raw-private" })).equals(alicePrivate)); + +// encodings +console.log("details", describe(alice), describe(crypto.createPublicKey(alice))); +const spki = alice.export({ format: "der", type: "pkcs8" }); +console.log("pkcs8", spki.length, spki.subarray(0, 16).toString("hex")); +const pub = crypto.createPublicKey(alice); +const spkiDer = pub.export({ format: "der", type: "spki" }); +console.log("spki", spkiDer.length, spkiDer.subarray(0, 12).toString("hex")); +console.log("pem", describe(alice.export({ format: "pem", type: "pkcs8" })), describe(pub.export({ format: "pem", type: "spki" }))); +const jwk = alice.export({ format: "jwk" }); +console.log("jwk", Object.keys(jwk).join(), jwk.kty, jwk.crv, jwk.x === Buffer.from(alicePublic, "hex").toString("base64url"), jwk.d === alicePrivate.toString("base64url")); +console.log("public jwk", JSON.stringify(pub.export({ format: "jwk" }))); +const fromPem = crypto.createPrivateKey(alice.export({ format: "pem", type: "pkcs8" })); +console.log("pem round trip", fromPem.equals(alice), crypto.createPublicKey(pub.export({ format: "pem", type: "spki" })).equals(pub)); +console.log("jwk round trip", crypto.createPrivateKey({ key: jwk, format: "jwk" }).equals(alice), crypto.createPublicKey({ key: pub.export({ format: "jwk" }), format: "jwk" }).equals(pub)); +console.log("public of pkcs8", crypto.createPublicKey({ key: spki, format: "der", type: "pkcs8" }).equals(pub)); +console.log("not ed25519", alice.equals(crypto.generateKeyPairSync("ed25519").privateKey)); + +// generateKeyPair +const pair = crypto.generateKeyPairSync("x25519"); +console.log("generated", describe(pair)); +const clamped = Buffer.from(pair.privateKey.export({ format: "raw-private" })); +console.log("generated clamped", (clamped[0] & 7) === 0, (clamped[31] & 0xc0) === 0x40); +attempt("pem pair", () => + crypto.generateKeyPairSync("x25519", { + publicKeyEncoding: { type: "spki", format: "pem" }, + privateKeyEncoding: { type: "pkcs8", format: "der" }, + }), +); +attempt("raw pair", () => + crypto.generateKeyPairSync("x25519", { publicKeyEncoding: { format: "raw-public" }, privateKeyEncoding: { format: "raw-private" } }), +); +attempt("pkcs1", () => crypto.generateKeyPairSync("x25519", { publicKeyEncoding: { type: "pkcs1", format: "pem" } })); +attempt("sec1", () => crypto.generateKeyPairSync("x25519", { privateKeyEncoding: { type: "sec1", format: "pem" } })); +attempt("raw-seed", () => crypto.generateKeyPairSync("x25519", { privateKeyEncoding: { format: "raw-seed" } })); +attempt("X25519 spelling", () => crypto.generateKeyPairSync("X25519")); + +// what X25519 keys cannot do +const data = Buffer.from("x25519"); +attempt("sign", () => crypto.sign(null, data, alice)); +attempt("sign sha256", () => crypto.sign("sha256", data, alice)); +attempt("verify", () => crypto.verify(null, data, pub, Buffer.alloc(64))); +attempt("createSign", () => crypto.createSign("sha256").update(data).sign(alice)); +attempt("createVerify", () => crypto.createVerify("sha256").update(data).verify(pub, Buffer.alloc(64))); +attempt("hmac", () => crypto.createHmac("sha256", alice)); +attempt("export pkcs1", () => pub.export({ format: "pem", type: "pkcs1" })); +attempt("export raw-seed", () => alice.export({ format: "raw-seed" })); +attempt("short raw key", () => crypto.createPublicKey({ key: Buffer.alloc(31), format: "raw-public", asymmetricKeyType: "x25519" })); + +crypto.generateKeyPair("x25519", (err, publicKey, privateKey) => { + console.log("async", err, describe(publicKey), describe(privateKey)); + crypto.diffieHellman({ privateKey, publicKey: crypto.createPublicKey(bob) }, (err2, secret) => { + console.log("async secret", err2, describe(secret)); + }); +}); +console.log("returned"); diff --git a/tests/corpus/7451-crypto-diffie-hellman.ts b/tests/corpus/7451-crypto-diffie-hellman.ts new file mode 100644 index 000000000..9648cd32b --- /dev/null +++ b/tests/corpus/7451-crypto-diffie-hellman.ts @@ -0,0 +1,55 @@ +// crypto.diffieHellman with typed KeyObjects: ECDH on every supported +// curve, checked against createECDH, X25519, the callback form, and +// Node's errors for incompatible key types, mismatched curves and a +// small-order X25519 public key. +import * as crypto from "node:crypto"; + +function attempt(label: string, run: () => Buffer): void { + try { + const secret = run(); + console.log(label, secret.length); + } catch (e) { + const error = e as NodeJS.ErrnoException; + console.log(label, "throws", error.name, error.code, error.message); + } +} + +for (const namedCurve of ["prime256v1", "secp384r1", "secp521r1", "secp256k1"]) { + const a = crypto.generateKeyPairSync("ec", { namedCurve }); + const ecdh = crypto.createECDH(namedCurve); + const ecdhPublic = ecdh.generateKeys(); + const peer = crypto.createPublicKey({ key: ecdhPublic, format: "raw-public", asymmetricKeyType: "ec", namedCurve } as unknown as crypto.PublicKeyInput); + const secret = crypto.diffieHellman({ privateKey: a.privateKey, publicKey: peer }); + // The SubjectPublicKeyInfo ends with the uncompressed point. + const spki = a.publicKey.export({ format: "der", type: "spki" }); + const own = spki.subarray(spki.length - ecdhPublic.length); + console.log(namedCurve, secret.length, secret.equals(ecdh.computeSecret(own))); +} + +const x = crypto.generateKeyPairSync("x25519"); +const y = crypto.generateKeyPairSync("x25519"); +const xy = crypto.diffieHellman({ privateKey: x.privateKey, publicKey: y.publicKey }); +console.log("x25519", xy.length, xy.equals(crypto.diffieHellman({ privateKey: y.privateKey, publicKey: x.publicKey }))); +attempt("public key given as the private key", () => crypto.diffieHellman({ privateKey: x.privateKey, publicKey: y.privateKey })); + +const p256 = crypto.generateKeyPairSync("ec", { namedCurve: "P-256" }); +const p384 = crypto.generateKeyPairSync("ec", { namedCurve: "P-384" }); +const rsa = crypto.generateKeyPairSync("rsa", { modulusLength: 1024 }); +const ed = crypto.generateKeyPairSync("ed25519"); +attempt("x25519 and ec", () => crypto.diffieHellman({ privateKey: x.privateKey, publicKey: p256.publicKey })); +attempt("ec and x25519", () => crypto.diffieHellman({ privateKey: p256.privateKey, publicKey: x.publicKey })); +attempt("rsa", () => crypto.diffieHellman({ privateKey: rsa.privateKey, publicKey: rsa.publicKey })); +attempt("ed25519", () => crypto.diffieHellman({ privateKey: ed.privateKey, publicKey: ed.publicKey })); +attempt("P-256 and P-384", () => crypto.diffieHellman({ privateKey: p256.privateKey, publicKey: p384.publicKey })); +attempt("public as private", () => crypto.diffieHellman({ privateKey: x.publicKey, publicKey: y.publicKey })); +const zero = crypto.createPublicKey({ key: Buffer.alloc(32), format: "raw-public", asymmetricKeyType: "x25519" } as unknown as crypto.PublicKeyInput); +attempt("small-order point", () => crypto.diffieHellman({ privateKey: x.privateKey, publicKey: zero })); + +crypto.diffieHellman({ privateKey: x.privateKey, publicKey: y.publicKey }, (err, secret) => { + console.log("callback", err, secret.equals(xy)); + crypto.diffieHellman({ privateKey: x.privateKey, publicKey: zero }, (...args: unknown[]) => { + const error = args[0] as NodeJS.ErrnoException; + console.log("callback error", args.length, error.code, error.message); + }); +}); +console.log("returned"); diff --git a/tests/corpus/7452-crypto-diffie-hellman-errors.mjs b/tests/corpus/7452-crypto-diffie-hellman-errors.mjs new file mode 100644 index 000000000..d8adc6442 --- /dev/null +++ b/tests/corpus/7452-crypto-diffie-hellman-errors.mjs @@ -0,0 +1,61 @@ +// crypto.diffieHellman's argument checks in Node's order, key inputs other +// than KeyObjects, and the own properties of the OpenSSL errors it and +// X25519 signatures throw. +import * as crypto from "node:crypto"; + +function attempt(label, run) { + try { + const result = run(); + console.log(label, Buffer.isBuffer(result) ? `Buffer(${result.length})` : String(result)); + } catch (e) { + console.log(label, "throws", e.name, e.code, e.message, JSON.stringify(Object.keys(e))); + } +} + +const x = crypto.generateKeyPairSync("x25519"); +const y = crypto.generateKeyPairSync("x25519"); +const ec = crypto.generateKeyPairSync("ec", { namedCurve: "P-256" }); +const other = crypto.generateKeyPairSync("ec", { namedCurve: "P-384" }); +const dh = (options, callback) => () => + callback === undefined ? crypto.diffieHellman(options) : crypto.diffieHellman(options, callback); + +attempt("no options", dh()); +attempt("number options", dh(5)); +attempt("array options", dh([])); +attempt("number callback", dh({ privateKey: x.privateKey, publicKey: y.publicKey }, 5)); +attempt("callback before keys", dh({}, 5)); +attempt("no private key", dh({ publicKey: y.publicKey })); +attempt("no public key", dh({ privateKey: x.privateKey })); +attempt("number public key", dh({ privateKey: x.privateKey, publicKey: 2 })); +attempt("number private key", dh({ privateKey: 1, publicKey: y.publicKey })); +attempt("null private key", dh({ privateKey: null, publicKey: y.publicKey })); +attempt("public key object as private", dh({ privateKey: x.publicKey, publicKey: y.publicKey })); +attempt("types checked first", dh({ privateKey: x.privateKey, publicKey: ec.publicKey })); + +// key inputs other than KeyObjects +const pkcs8 = x.privateKey.export({ format: "pem", type: "pkcs8" }); +const spki = y.publicKey.export({ format: "pem", type: "spki" }); +attempt("pem keys", dh({ privateKey: pkcs8, publicKey: spki })); +attempt("der key objects", dh({ + privateKey: { key: x.privateKey.export({ format: "der", type: "pkcs8" }), format: "der", type: "pkcs8" }, + publicKey: { key: y.publicKey.export({ format: "jwk" }), format: "jwk" }, +})); +attempt("public key from a private pem", dh({ privateKey: x.privateKey, publicKey: pkcs8 })); +attempt("same result", () => + crypto.diffieHellman({ privateKey: pkcs8, publicKey: spki }).equals(crypto.diffieHellman({ privateKey: x.privateKey, publicKey: y.publicKey })), +); +attempt("mixed types from pem", dh({ privateKey: x.privateKey, publicKey: ec.publicKey.export({ format: "pem", type: "spki" }) })); +attempt("mismatched curves", dh({ privateKey: ec.privateKey, publicKey: other.publicKey })); +const zero = crypto.createPublicKey({ key: Buffer.alloc(32), format: "raw-public", asymmetricKeyType: "x25519" }); +attempt("small-order point", dh({ privateKey: x.privateKey, publicKey: zero })); +attempt("sign", () => crypto.sign(null, Buffer.from("x"), x.privateKey)); + +// the callback form: argument errors throw, job errors arrive without +// the OpenSSL properties +attempt("callback returns", dh({ privateKey: x.privateKey, publicKey: y.publicKey }, (err, secret) => { + console.log("callback", err, secret.length); + crypto.diffieHellman({ privateKey: ec.privateKey, publicKey: other.publicKey }, (error) => { + console.log("callback mismatch", error.code, error.message, JSON.stringify(Object.keys(error))); + }); +})); +attempt("callback with a bad key throws", dh({ privateKey: 1, publicKey: y.publicKey }, () => console.log("never"))); diff --git a/tests/fixtures/node-types/diffie-hellman.ts b/tests/fixtures/node-types/diffie-hellman.ts new file mode 100644 index 000000000..d14fd5ffc --- /dev/null +++ b/tests/fixtures/node-types/diffie-hellman.ts @@ -0,0 +1,19 @@ +import { type KeyObject, createPublicKey, diffieHellman, generateKeyPair, generateKeyPairSync } from "node:crypto"; + +const alice = generateKeyPairSync("x25519"); +const bob = generateKeyPairSync("x25519"); +const secret: Buffer = diffieHellman({ privateKey: alice.privateKey, publicKey: bob.publicKey }); +console.log(alice.publicKey.asymmetricKeyType, secret.length, secret.equals(diffieHellman({ privateKey: bob.privateKey, publicKey: alice.publicKey }))); +const exported = alice.publicKey.export({ type: "spki", format: "pem" }); +const pem = typeof exported === "string" ? exported : exported.toString(); +const reread: KeyObject = createPublicKey(pem); +console.log(reread.equals(alice.publicKey), pem.split("\n")[0]); +const ec = generateKeyPairSync("ec", { namedCurve: "P-384" }); +const peer = generateKeyPairSync("ec", { namedCurve: "P-384" }); +console.log("ecdh", diffieHellman({ privateKey: ec.privateKey, publicKey: peer.publicKey }).length); +generateKeyPair("x25519", undefined, (err, publicKey, privateKey) => { + console.log("generated", err, publicKey.type, privateKey.asymmetricKeyType); + diffieHellman({ privateKey, publicKey: bob.publicKey }, (error, shared) => { + console.log("callback", error, shared.length); + }); +}); diff --git a/tests/fixtures/node-types/fenced.ts b/tests/fixtures/node-types/fenced.ts index 7c468de01..0ddf0ea3f 100644 --- a/tests/fixtures/node-types/fenced.ts +++ b/tests/fixtures/node-types/fenced.ts @@ -99,12 +99,9 @@ void callbackified; import { createECDH } from "node:crypto"; createECDH("sect163k1"); createECDH("prime256v1").computeSecret(Buffer.alloc(65), "hex"); -/* DiffieHellman's Object.prototype members, and crypto.diffieHellman over - * asymmetric KeyObjects. */ -import { createDiffieHellman, diffieHellman, type KeyObject } from "node:crypto"; +/* DiffieHellman's Object.prototype members. */ +import { createDiffieHellman } from "node:crypto"; createDiffieHellman(Buffer.from([23])).toString(); -declare const dhKey: KeyObject; -diffieHellman({ privateKey: dhKey, publicKey: dhKey }); /* WebCrypto beyond the secret-key slice: @types/node declares the whole * SubtleCrypto surface, and the methods outside the slice, such as * getPublicKey, fence by name. */ diff --git a/tests/harness/__snapshots__/node-types-fenced.txt b/tests/harness/__snapshots__/node-types-fenced.txt index 562733f53..d0aa4a140 100644 --- a/tests/harness/__snapshots__/node-types-fenced.txt +++ b/tests/harness/__snapshots__/node-types-fenced.txt @@ -219,42 +219,33 @@ fenced.ts:101:1 - error SC2020: 'ECDH.computeSecret(key, encoding) with a Buffer 100 | createECDH("sect163k1"); 101 | createECDH("prime256v1").computeSecret(Buffer.alloc(65), "hex"); | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - 102 | /* DiffieHellman's Object.prototype members, and crypto.diffieHellman over + 102 | /* DiffieHellman's Object.prototype members. */ hint: Node reads the second argument as the key's input encoding and answers a Buffer; pass the output encoding third: computeSecret(key, undefined, encoding) -fenced.ts:105:1 - error SC2020: 'DiffieHellman.toString' is part of the standard library types but has no scriptc lowering yet +fenced.ts:104:1 - error SC2020: 'DiffieHellman.toString' is part of the standard library types but has no scriptc lowering yet - 104 | import { createDiffieHellman, diffieHellman, type KeyObject } from "node:crypto"; - 105 | createDiffieHellman(Buffer.from([23])).toString(); + 103 | import { createDiffieHellman } from "node:crypto"; + 104 | createDiffieHellman(Buffer.from([23])).toString(); | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - 106 | declare const dhKey: KeyObject; + 105 | /* WebCrypto beyond the secret-key slice: @types/node declares the whole hint: generateKeys(), computeSecret(), getPrime(), getGenerator(), getPublicKey(), getPrivateKey(), setPublicKey(), setPrivateKey() and verifyError are supported -fenced.ts:107:1 - error SC2020: 'crypto.diffieHellman' is typed by @types/node but has no scriptc lowering yet +fenced.ts:109:9 - error SC2020: 'crypto.subtle.getPublicKey' is part of the standard library types but has no scriptc lowering yet - 106 | declare const dhKey: KeyObject; - 107 | diffieHellman({ privateKey: dhKey, publicKey: dhKey }); - | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - 108 | /* WebCrypto beyond the secret-key slice: @types/node declares the whole - - hint: crypto.diffieHellman takes DH, EC or X25519 KeyObjects and has no lowering; the objects of createDiffieHellman, getDiffieHellman and createECDH compute shared secrets - -fenced.ts:112:9 - error SC2020: 'crypto.subtle.getPublicKey' is part of the standard library types but has no scriptc lowering yet - - 111 | async function publicKeyWithWebCrypto(key: import("node:crypto").webcrypto.CryptoKey): Promise { - 112 | await crypto.subtle.getPublicKey(key, ["verify"]); + 108 | async function publicKeyWithWebCrypto(key: import("node:crypto").webcrypto.CryptoKey): Promise { + 109 | await crypto.subtle.getPublicKey(key, ["verify"]); | ^~~~~~~~~~~~~~~~~~~~~~~~~~ - 113 | } + 110 | } hint: the native WebCrypto surface is crypto.subtle's importKey, generateKey and exportKey for HMAC and AES keys (importKey also for PBKDF2 and HKDF keys), encrypt and decrypt with AES-GCM, AES-CBC or AES-CTR, wrapKey and unwrapKey of secret keys with AES-KW or those ciphers, deriveBits and deriveKey with PBKDF2 or HKDF, sign and verify with HMAC, generateKey, importKey, exportKey, sign and verify for ECDSA, Ed25519, RSASSA-PKCS1-v1_5 and RSA-PSS, and digest with SHA-1, SHA-256, SHA-384 or SHA-512, each called directly on crypto.subtle -fenced.ts:117:15 - error SC2020: 'navigator.hardwareConcurrency' is typed by @types/node but has no scriptc lowering yet +fenced.ts:114:15 - error SC2020: 'navigator.hardwareConcurrency' is typed by @types/node but has no scriptc lowering yet - 116 | * compile; the other members fence by name. */ - 117 | const cores = navigator.hardwareConcurrency; + 113 | * compile; the other members fence by name. */ + 114 | const cores = navigator.hardwareConcurrency; | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - 118 | void cores; + 115 | void cores; hint: of Node's navigator global only typeof navigator and navigator.userAgent compile \ No newline at end of file diff --git a/tests/harness/corpus-costs/7450-crypto-x25519.mjs.json b/tests/harness/corpus-costs/7450-crypto-x25519.mjs.json new file mode 100644 index 000000000..727e2cdb6 --- /dev/null +++ b/tests/harness/corpus-costs/7450-crypto-x25519.mjs.json @@ -0,0 +1 @@ +{ "plain": 3330, "san": 4877 } diff --git a/tests/harness/corpus-costs/7451-crypto-diffie-hellman.ts.json b/tests/harness/corpus-costs/7451-crypto-diffie-hellman.ts.json new file mode 100644 index 000000000..a9347d13c --- /dev/null +++ b/tests/harness/corpus-costs/7451-crypto-diffie-hellman.ts.json @@ -0,0 +1 @@ +{ "plain": 3265, "san": 6438 } diff --git a/tests/harness/corpus-costs/7452-crypto-diffie-hellman-errors.mjs.json b/tests/harness/corpus-costs/7452-crypto-diffie-hellman-errors.mjs.json new file mode 100644 index 000000000..72961a07f --- /dev/null +++ b/tests/harness/corpus-costs/7452-crypto-diffie-hellman-errors.mjs.json @@ -0,0 +1 @@ +{ "plain": 2917, "san": 4460 } diff --git a/tests/harness/project-config.test.ts b/tests/harness/project-config.test.ts index 4676499eb..fd2077a48 100644 --- a/tests/harness/project-config.test.ts +++ b/tests/harness/project-config.test.ts @@ -1160,6 +1160,24 @@ test("node-types: asymmetric keys and signatures match Node under @types/node", expect(comparableStderr(native.stderr)).toBe(node.stderr); }); +test("node-types: X25519 keys and diffieHellman match Node under @types/node", async () => { + const outDir = outDirFor("node-diffie-hellman"); + const entry = join(nodeTypesDir, "diffie-hellman.ts"); + const result = await compile(entry, { + outPath: join(outDir, "diffie-hellman"), + outDir, + sanitize, + }); + expect(result.ok, !result.ok ? JSON.stringify(result.diagnostics, null, 2) : "").toBe(true); + if (!result.ok) return; + const [native, node] = await Promise.all([ + execFileAsync(result.binaryPath), + execFileAsync(process.execPath, ["--disable-warning=MODULE_TYPELESS_PACKAGE_JSON", entry]), + ]); + expect(native.stdout).toBe(node.stdout); + expect(comparableStderr(native.stderr)).toBe(node.stderr); +}); + test("node-types: WebCrypto ECDSA, Ed25519 and RSA match Node under @types/node", async () => { const outDir = outDirFor("node-webcrypto-ecdsa"); const entry = join(nodeTypesDir, "webcrypto-ecdsa.ts"); From dddcbed335c5ff5cbead198cd2f8c5137cada11f Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 14:09:39 -0700 Subject: [PATCH 103/184] Project zod 4 and @hono/zod-validator values at the declaration boundary The declaration-typed boundary already reaches zod 4 schemas and hono's c.req.valid(): zod 4's z.infer and hono's validator Input generics resolve to plain object types in the declaration world. Three gaps remained: - Boolean literal types printed as `false` (the TS7 facade leaves intrinsicName unset), so `{ ok: true } | { ok: false }` lost its true arm and `boolean | undefined` became `undefined | false`. The printer reads the literal's value. - `r.data` after `const r = S.safeParse(x); if (r.success)` never crossed: zod 4's result union is not data (its error arm is a class), and reads on projectable receivers were skipped on the assumption that something else types the receiver. Such reads now cross when the receiver is a package call whose type does not project, or a const bound to one. - Reads through bindings annotated with a projected type no longer get a redundant cast, and aliases are named only for casts that survive. New npm cases zod4-boundary and hono-zod-validator run real zod 4.6.7, hono 4.13.13 and @hono/zod-validator 0.9.1 (vendored file closures) under Node and natively; decl-boundary.test.ts checks that c.req.valid() is a checked cast with a path TypeError. Co-Authored-By: Claude Opus 5.5 --- .../compiler/src/frontend/decl-projection.ts | 111 +- .../npm/cases/hono-zod-validator/main.ts.json | 61 + .../npm/cases/zod4-boundary/main.ts.json | 6 + .../npm/cases/hono-zod-validator/lying.ts | 42 + .../npm/cases/hono-zod-validator/main.ts | 96 + .../npm/cases/hono-zod-validator/package.json | 1 + .../fixtures/npm/cases/zod4-boundary/main.ts | 146 + .../npm/cases/zod4-boundary/package.json | 1 + .../@hono/zod-validator/dist/index.d.mts | 43 + .../@hono/zod-validator/dist/index.mjs | 28 + .../@hono/zod-validator/package.json | 14 + .../npm/node_modules/hono/dist/compose.js | 50 + .../npm/node_modules/hono/dist/context.js | 381 +++ .../hono/dist/helper/cookie/index.js | 82 + .../npm/node_modules/hono/dist/hono-base.js | 350 +++ .../npm/node_modules/hono/dist/hono.js | 28 + .../node_modules/hono/dist/http-exception.js | 55 + .../npm/node_modules/hono/dist/index.js | 20 + .../npm/node_modules/hono/dist/request.js | 335 +++ .../hono/dist/request/constants.js | 4 + .../npm/node_modules/hono/dist/router.js | 35 + .../hono/dist/router/reg-exp-router/index.js | 3 + .../dist/router/reg-exp-router/matcher.js | 19 + .../hono/dist/router/reg-exp-router/node.js | 91 + .../router/reg-exp-router/prepared-router.js | 94 + .../hono/dist/router/reg-exp-router/router.js | 101 + .../hono/dist/router/reg-exp-router/trie.js | 73 + .../hono/dist/router/smart-router/router.js | 49 + .../hono/dist/router/trie-router/index.js | 2 + .../hono/dist/router/trie-router/node.js | 134 + .../hono/dist/router/trie-router/router.js | 15 + .../node_modules/hono/dist/router/utils.js | 4 + .../hono/dist/types/client/types.d.ts | 204 ++ .../node_modules/hono/dist/types/context.d.ts | 459 +++ .../hono/dist/types/hono-base.d.ts | 221 ++ .../node_modules/hono/dist/types/hono.d.ts | 20 + .../node_modules/hono/dist/types/index.d.ts | 6 + .../node_modules/hono/dist/types/request.d.ts | 326 ++ .../hono/dist/types/request/constants.d.ts | 3 + .../node_modules/hono/dist/types/router.d.ts | 98 + .../node_modules/hono/dist/types/types.d.ts | 214 ++ .../hono/dist/types/utils/body.d.ts | 75 + .../hono/dist/types/utils/headers.d.ts | 10 + .../hono/dist/types/utils/http-status.d.ts | 34 + .../hono/dist/types/utils/mime.d.ts | 73 + .../hono/dist/types/utils/types.d.ts | 62 + .../hono/dist/types/validator/index.d.ts | 3 + .../hono/dist/types/validator/utils.d.ts | 27 + .../hono/dist/types/validator/validator.d.ts | 18 + .../npm/node_modules/hono/dist/utils/body.js | 96 + .../node_modules/hono/dist/utils/buffer.js | 51 + .../node_modules/hono/dist/utils/constants.js | 7 + .../node_modules/hono/dist/utils/cookie.js | 125 + .../node_modules/hono/dist/utils/crypto.js | 34 + .../npm/node_modules/hono/dist/utils/html.js | 103 + .../npm/node_modules/hono/dist/utils/url.js | 201 ++ .../node_modules/hono/dist/validator/index.js | 2 + .../hono/dist/validator/validator.js | 64 + .../npm/node_modules/hono/package.json | 10 +- tests/fixtures/npm/node_modules/zod/LICENSE | 21 + .../fixtures/npm/node_modules/zod/index.d.cts | 3 + tests/fixtures/npm/node_modules/zod/index.js | 7 + .../npm/node_modules/zod/package.json | 23 + .../npm/node_modules/zod/v3/ZodError.d.cts | 164 + .../npm/node_modules/zod/v3/errors.d.cts | 5 + .../npm/node_modules/zod/v3/external.d.cts | 6 + .../zod/v3/helpers/enumUtil.d.cts | 8 + .../zod/v3/helpers/errorUtil.d.cts | 9 + .../zod/v3/helpers/parseUtil.d.cts | 78 + .../zod/v3/helpers/partialUtil.d.cts | 8 + .../zod/v3/helpers/typeAliases.d.cts | 2 + .../node_modules/zod/v3/helpers/util.d.cts | 85 + .../npm/node_modules/zod/v3/index.d.cts | 4 + .../npm/node_modules/zod/v3/locales/en.d.cts | 3 + .../node_modules/zod/v3/standard-schema.d.cts | 102 + .../npm/node_modules/zod/v3/types.d.cts | 1034 +++++++ .../node_modules/zod/v4/classic/checks.d.cts | 1 + .../npm/node_modules/zod/v4/classic/checks.js | 1 + .../node_modules/zod/v4/classic/coerce.d.cts | 17 + .../npm/node_modules/zod/v4/classic/coerce.js | 17 + .../node_modules/zod/v4/classic/compat.d.cts | 48 + .../npm/node_modules/zod/v4/classic/compat.js | 31 + .../zod/v4/classic/deep-partial.d.cts | 14 + .../zod/v4/classic/deep-partial.js | 13 + .../node_modules/zod/v4/classic/errors.d.cts | 29 + .../npm/node_modules/zod/v4/classic/errors.js | 57 + .../zod/v4/classic/external.d.cts | 19 + .../node_modules/zod/v4/classic/external.js | 17 + .../zod/v4/classic/from-json-schema.d.cts | 12 + .../zod/v4/classic/from-json-schema.js | 881 ++++++ .../node_modules/zod/v4/classic/in-out.d.cts | 8 + .../npm/node_modules/zod/v4/classic/in-out.js | 36 + .../node_modules/zod/v4/classic/index.d.cts | 3 + .../npm/node_modules/zod/v4/classic/iso.d.cts | 7 + .../npm/node_modules/zod/v4/classic/iso.js | 15 + .../node_modules/zod/v4/classic/parse.d.cts | 44 + .../npm/node_modules/zod/v4/classic/parse.js | 16 + .../node_modules/zod/v4/classic/schemas.d.cts | 816 +++++ .../node_modules/zod/v4/classic/schemas.js | 1535 ++++++++++ .../npm/node_modules/zod/v4/core/api.d.cts | 335 +++ .../npm/node_modules/zod/v4/core/api.js | 1107 +++++++ .../npm/node_modules/zod/v4/core/checks.d.cts | 294 ++ .../npm/node_modules/zod/v4/core/checks.js | 516 ++++ .../node_modules/zod/v4/core/compile.d.cts | 65 + .../npm/node_modules/zod/v4/core/compile.js | 1866 ++++++++++++ .../npm/node_modules/zod/v4/core/core.d.cts | 82 + .../npm/node_modules/zod/v4/core/core.js | 141 + .../npm/node_modules/zod/v4/core/doc.d.cts | 16 + .../npm/node_modules/zod/v4/core/doc.js | 38 + .../npm/node_modules/zod/v4/core/errors.d.cts | 225 ++ .../npm/node_modules/zod/v4/core/errors.js | 267 ++ .../npm/node_modules/zod/v4/core/index.d.cts | 19 + .../npm/node_modules/zod/v4/core/index.js | 19 + .../zod/v4/core/json-schema-generator.d.cts | 65 + .../zod/v4/core/json-schema-generator.js | 99 + .../zod/v4/core/json-schema-processors.d.cts | 65 + .../zod/v4/core/json-schema-processors.js | 843 ++++++ .../zod/v4/core/json-schema.d.cts | 89 + .../node_modules/zod/v4/core/json-schema.js | 1 + .../node_modules/zod/v4/core/memoizer.d.cts | 15 + .../npm/node_modules/zod/v4/core/memoizer.js | 311 ++ .../npm/node_modules/zod/v4/core/parse.d.cts | 52 + .../npm/node_modules/zod/v4/core/parse.js | 175 ++ .../node_modules/zod/v4/core/regexes.d.cts | 90 + .../npm/node_modules/zod/v4/core/regexes.js | 156 + .../node_modules/zod/v4/core/registries.d.cts | 35 + .../node_modules/zod/v4/core/registries.js | 51 + .../node_modules/zod/v4/core/schemas.d.cts | 1275 ++++++++ .../npm/node_modules/zod/v4/core/schemas.js | 2636 +++++++++++++++++ .../zod/v4/core/standard-schema.d.cts | 126 + .../zod/v4/core/to-json-schema.d.cts | 154 + .../zod/v4/core/to-json-schema.js | 651 ++++ .../npm/node_modules/zod/v4/core/util.d.cts | 280 ++ .../npm/node_modules/zod/v4/core/util.js | 873 ++++++ .../node_modules/zod/v4/core/versions.d.cts | 5 + .../npm/node_modules/zod/v4/core/versions.js | 5 + .../npm/node_modules/zod/v4/core/visit.js | 169 ++ .../npm/node_modules/zod/v4/index.d.cts | 3 + .../npm/node_modules/zod/v4/locales/ar.d.cts | 5 + .../npm/node_modules/zod/v4/locales/ar.js | 111 + .../npm/node_modules/zod/v4/locales/az.d.cts | 5 + .../npm/node_modules/zod/v4/locales/az.js | 110 + .../npm/node_modules/zod/v4/locales/be.d.cts | 5 + .../npm/node_modules/zod/v4/locales/be.js | 168 ++ .../npm/node_modules/zod/v4/locales/bg.d.cts | 5 + .../npm/node_modules/zod/v4/locales/bg.js | 125 + .../npm/node_modules/zod/v4/locales/bn.d.cts | 5 + .../npm/node_modules/zod/v4/locales/bn.js | 113 + .../npm/node_modules/zod/v4/locales/ca.d.cts | 5 + .../npm/node_modules/zod/v4/locales/ca.js | 112 + .../npm/node_modules/zod/v4/locales/ckb.d.cts | 5 + .../npm/node_modules/zod/v4/locales/ckb.js | 133 + .../npm/node_modules/zod/v4/locales/cs.d.cts | 5 + .../npm/node_modules/zod/v4/locales/cs.js | 116 + .../npm/node_modules/zod/v4/locales/da.d.cts | 5 + .../npm/node_modules/zod/v4/locales/da.js | 120 + .../npm/node_modules/zod/v4/locales/de.d.cts | 5 + .../npm/node_modules/zod/v4/locales/de.js | 113 + .../npm/node_modules/zod/v4/locales/el.d.cts | 5 + .../npm/node_modules/zod/v4/locales/el.js | 112 + .../npm/node_modules/zod/v4/locales/en.d.cts | 5 + .../npm/node_modules/zod/v4/locales/en.js | 125 + .../npm/node_modules/zod/v4/locales/eo.d.cts | 5 + .../npm/node_modules/zod/v4/locales/eo.js | 114 + .../npm/node_modules/zod/v4/locales/es.d.cts | 5 + .../npm/node_modules/zod/v4/locales/es.js | 136 + .../npm/node_modules/zod/v4/locales/fa.d.cts | 5 + .../npm/node_modules/zod/v4/locales/fa.js | 119 + .../npm/node_modules/zod/v4/locales/fi.d.cts | 5 + .../npm/node_modules/zod/v4/locales/fi.js | 117 + .../node_modules/zod/v4/locales/fr-CA.d.cts | 5 + .../npm/node_modules/zod/v4/locales/fr-CA.js | 112 + .../npm/node_modules/zod/v4/locales/fr.d.cts | 5 + .../npm/node_modules/zod/v4/locales/fr.js | 129 + .../npm/node_modules/zod/v4/locales/gu.d.cts | 5 + .../npm/node_modules/zod/v4/locales/gu.js | 113 + .../npm/node_modules/zod/v4/locales/he.d.cts | 5 + .../npm/node_modules/zod/v4/locales/he.js | 221 ++ .../npm/node_modules/zod/v4/locales/hi.d.cts | 5 + .../npm/node_modules/zod/v4/locales/hi.js | 111 + .../npm/node_modules/zod/v4/locales/hr.d.cts | 5 + .../npm/node_modules/zod/v4/locales/hr.js | 126 + .../npm/node_modules/zod/v4/locales/hu.d.cts | 5 + .../npm/node_modules/zod/v4/locales/hu.js | 113 + .../npm/node_modules/zod/v4/locales/hy.d.cts | 5 + .../npm/node_modules/zod/v4/locales/hy.js | 158 + .../npm/node_modules/zod/v4/locales/id.d.cts | 5 + .../npm/node_modules/zod/v4/locales/id.js | 111 + .../node_modules/zod/v4/locales/index.d.cts | 63 + .../npm/node_modules/zod/v4/locales/index.js | 63 + .../npm/node_modules/zod/v4/locales/is.d.cts | 5 + .../npm/node_modules/zod/v4/locales/is.js | 114 + .../npm/node_modules/zod/v4/locales/it.d.cts | 5 + .../npm/node_modules/zod/v4/locales/it.js | 113 + .../npm/node_modules/zod/v4/locales/ja.d.cts | 5 + .../npm/node_modules/zod/v4/locales/ja.js | 112 + .../npm/node_modules/zod/v4/locales/ka.d.cts | 5 + .../npm/node_modules/zod/v4/locales/ka.js | 117 + .../npm/node_modules/zod/v4/locales/kh.d.cts | 5 + .../npm/node_modules/zod/v4/locales/kh.js | 5 + .../npm/node_modules/zod/v4/locales/km.d.cts | 5 + .../npm/node_modules/zod/v4/locales/km.js | 115 + .../npm/node_modules/zod/v4/locales/kn.d.cts | 5 + .../npm/node_modules/zod/v4/locales/kn.js | 116 + .../npm/node_modules/zod/v4/locales/ko.d.cts | 5 + .../npm/node_modules/zod/v4/locales/ko.js | 116 + .../npm/node_modules/zod/v4/locales/lt.d.cts | 5 + .../npm/node_modules/zod/v4/locales/lt.js | 207 ++ .../npm/node_modules/zod/v4/locales/mk.d.cts | 5 + .../npm/node_modules/zod/v4/locales/mk.js | 114 + .../npm/node_modules/zod/v4/locales/ms.d.cts | 5 + .../npm/node_modules/zod/v4/locales/ms.js | 112 + .../npm/node_modules/zod/v4/locales/ne.d.cts | 5 + .../npm/node_modules/zod/v4/locales/ne.js | 111 + .../npm/node_modules/zod/v4/locales/nl.d.cts | 5 + .../npm/node_modules/zod/v4/locales/nl.js | 115 + .../npm/node_modules/zod/v4/locales/nn.d.cts | 5 + .../npm/node_modules/zod/v4/locales/nn.js | 113 + .../npm/node_modules/zod/v4/locales/no.d.cts | 5 + .../npm/node_modules/zod/v4/locales/no.js | 113 + .../npm/node_modules/zod/v4/locales/ota.d.cts | 5 + .../npm/node_modules/zod/v4/locales/ota.js | 114 + .../npm/node_modules/zod/v4/locales/pl.d.cts | 5 + .../npm/node_modules/zod/v4/locales/pl.js | 114 + .../npm/node_modules/zod/v4/locales/ps.d.cts | 5 + .../npm/node_modules/zod/v4/locales/ps.js | 119 + .../node_modules/zod/v4/locales/pt-BR.d.cts | 5 + .../npm/node_modules/zod/v4/locales/pt-BR.js | 143 + .../npm/node_modules/zod/v4/locales/pt.d.cts | 5 + .../npm/node_modules/zod/v4/locales/pt.js | 142 + .../npm/node_modules/zod/v4/locales/ro.d.cts | 5 + .../npm/node_modules/zod/v4/locales/ro.js | 122 + .../npm/node_modules/zod/v4/locales/ru.d.cts | 5 + .../npm/node_modules/zod/v4/locales/ru.js | 168 ++ .../npm/node_modules/zod/v4/locales/sk.d.cts | 5 + .../npm/node_modules/zod/v4/locales/sk.js | 116 + .../npm/node_modules/zod/v4/locales/sl.d.cts | 5 + .../npm/node_modules/zod/v4/locales/sl.js | 114 + .../npm/node_modules/zod/v4/locales/sv.d.cts | 5 + .../npm/node_modules/zod/v4/locales/sv.js | 115 + .../npm/node_modules/zod/v4/locales/ta.d.cts | 5 + .../npm/node_modules/zod/v4/locales/ta.js | 115 + .../npm/node_modules/zod/v4/locales/tg.d.cts | 5 + .../npm/node_modules/zod/v4/locales/tg.js | 117 + .../npm/node_modules/zod/v4/locales/th.d.cts | 5 + .../npm/node_modules/zod/v4/locales/th.js | 115 + .../npm/node_modules/zod/v4/locales/tk.d.cts | 5 + .../npm/node_modules/zod/v4/locales/tk.js | 107 + .../npm/node_modules/zod/v4/locales/tr.d.cts | 5 + .../npm/node_modules/zod/v4/locales/tr.js | 110 + .../npm/node_modules/zod/v4/locales/ua.d.cts | 5 + .../npm/node_modules/zod/v4/locales/ua.js | 5 + .../npm/node_modules/zod/v4/locales/uk.d.cts | 5 + .../npm/node_modules/zod/v4/locales/uk.js | 113 + .../npm/node_modules/zod/v4/locales/ur.d.cts | 5 + .../npm/node_modules/zod/v4/locales/ur.js | 115 + .../npm/node_modules/zod/v4/locales/uz.d.cts | 5 + .../npm/node_modules/zod/v4/locales/uz.js | 113 + .../npm/node_modules/zod/v4/locales/vi.d.cts | 5 + .../npm/node_modules/zod/v4/locales/vi.js | 113 + .../npm/node_modules/zod/v4/locales/yo.d.cts | 5 + .../npm/node_modules/zod/v4/locales/yo.js | 112 + .../node_modules/zod/v4/locales/zh-CN.d.cts | 5 + .../npm/node_modules/zod/v4/locales/zh-CN.js | 114 + .../node_modules/zod/v4/locales/zh-TW.d.cts | 5 + .../npm/node_modules/zod/v4/locales/zh-TW.js | 112 + tests/harness/decl-boundary.test.ts | 28 +- tests/harness/npm.test.ts | 11 +- 268 files changed, 31356 insertions(+), 16 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hono-zod-validator/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/zod4-boundary/main.ts.json create mode 100644 tests/fixtures/npm/cases/hono-zod-validator/lying.ts create mode 100644 tests/fixtures/npm/cases/hono-zod-validator/main.ts create mode 100644 tests/fixtures/npm/cases/hono-zod-validator/package.json create mode 100644 tests/fixtures/npm/cases/zod4-boundary/main.ts create mode 100644 tests/fixtures/npm/cases/zod4-boundary/package.json create mode 100644 tests/fixtures/npm/node_modules/@hono/zod-validator/dist/index.d.mts create mode 100644 tests/fixtures/npm/node_modules/@hono/zod-validator/dist/index.mjs create mode 100644 tests/fixtures/npm/node_modules/@hono/zod-validator/package.json create mode 100644 tests/fixtures/npm/node_modules/hono/dist/compose.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/context.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/helper/cookie/index.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/hono-base.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/hono.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/http-exception.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/index.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/request.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/request/constants.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/router.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/index.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/matcher.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/node.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/prepared-router.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/router.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/trie.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/router/smart-router/router.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/router/trie-router/index.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/router/trie-router/node.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/router/trie-router/router.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/router/utils.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/types/client/types.d.ts create mode 100644 tests/fixtures/npm/node_modules/hono/dist/types/context.d.ts create mode 100644 tests/fixtures/npm/node_modules/hono/dist/types/hono-base.d.ts create mode 100644 tests/fixtures/npm/node_modules/hono/dist/types/hono.d.ts create mode 100644 tests/fixtures/npm/node_modules/hono/dist/types/index.d.ts create mode 100644 tests/fixtures/npm/node_modules/hono/dist/types/request.d.ts create mode 100644 tests/fixtures/npm/node_modules/hono/dist/types/request/constants.d.ts create mode 100644 tests/fixtures/npm/node_modules/hono/dist/types/router.d.ts create mode 100644 tests/fixtures/npm/node_modules/hono/dist/types/types.d.ts create mode 100644 tests/fixtures/npm/node_modules/hono/dist/types/utils/body.d.ts create mode 100644 tests/fixtures/npm/node_modules/hono/dist/types/utils/headers.d.ts create mode 100644 tests/fixtures/npm/node_modules/hono/dist/types/utils/http-status.d.ts create mode 100644 tests/fixtures/npm/node_modules/hono/dist/types/utils/mime.d.ts create mode 100644 tests/fixtures/npm/node_modules/hono/dist/types/utils/types.d.ts create mode 100644 tests/fixtures/npm/node_modules/hono/dist/types/validator/index.d.ts create mode 100644 tests/fixtures/npm/node_modules/hono/dist/types/validator/utils.d.ts create mode 100644 tests/fixtures/npm/node_modules/hono/dist/types/validator/validator.d.ts create mode 100644 tests/fixtures/npm/node_modules/hono/dist/utils/body.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/utils/buffer.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/utils/constants.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/utils/cookie.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/utils/crypto.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/utils/html.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/utils/url.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/validator/index.js create mode 100644 tests/fixtures/npm/node_modules/hono/dist/validator/validator.js create mode 100644 tests/fixtures/npm/node_modules/zod/LICENSE create mode 100644 tests/fixtures/npm/node_modules/zod/index.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/index.js create mode 100644 tests/fixtures/npm/node_modules/zod/package.json create mode 100644 tests/fixtures/npm/node_modules/zod/v3/ZodError.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v3/errors.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v3/external.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v3/helpers/enumUtil.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v3/helpers/errorUtil.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v3/helpers/parseUtil.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v3/helpers/partialUtil.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v3/helpers/typeAliases.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v3/helpers/util.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v3/index.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v3/locales/en.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v3/standard-schema.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v3/types.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/classic/checks.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/classic/checks.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/classic/coerce.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/classic/coerce.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/classic/compat.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/classic/compat.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/classic/deep-partial.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/classic/deep-partial.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/classic/errors.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/classic/errors.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/classic/external.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/classic/external.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/classic/from-json-schema.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/classic/from-json-schema.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/classic/in-out.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/classic/in-out.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/classic/index.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/classic/iso.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/classic/iso.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/classic/parse.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/classic/parse.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/classic/schemas.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/classic/schemas.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/api.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/api.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/checks.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/checks.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/compile.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/compile.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/core.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/core.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/doc.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/doc.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/errors.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/errors.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/index.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/index.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/json-schema-generator.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/json-schema-generator.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/json-schema-processors.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/json-schema-processors.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/json-schema.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/json-schema.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/memoizer.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/memoizer.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/parse.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/parse.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/regexes.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/regexes.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/registries.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/registries.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/schemas.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/schemas.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/standard-schema.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/to-json-schema.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/to-json-schema.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/util.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/util.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/versions.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/versions.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/core/visit.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/index.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ar.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ar.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/az.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/az.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/be.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/be.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/bg.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/bg.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/bn.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/bn.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ca.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ca.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ckb.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ckb.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/cs.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/cs.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/da.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/da.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/de.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/de.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/el.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/el.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/en.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/en.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/eo.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/eo.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/es.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/es.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/fa.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/fa.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/fi.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/fi.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/fr-CA.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/fr-CA.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/fr.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/fr.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/gu.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/gu.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/he.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/he.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/hi.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/hi.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/hr.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/hr.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/hu.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/hu.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/hy.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/hy.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/id.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/id.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/index.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/index.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/is.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/is.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/it.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/it.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ja.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ja.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ka.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ka.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/kh.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/kh.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/km.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/km.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/kn.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/kn.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ko.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ko.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/lt.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/lt.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/mk.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/mk.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ms.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ms.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ne.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ne.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/nl.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/nl.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/nn.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/nn.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/no.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/no.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ota.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ota.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/pl.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/pl.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ps.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ps.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/pt-BR.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/pt-BR.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/pt.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/pt.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ro.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ro.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ru.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ru.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/sk.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/sk.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/sl.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/sl.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/sv.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/sv.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ta.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ta.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/tg.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/tg.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/th.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/th.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/tk.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/tk.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/tr.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/tr.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ua.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ua.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/uk.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/uk.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ur.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/ur.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/uz.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/uz.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/vi.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/vi.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/yo.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/yo.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/zh-CN.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/zh-CN.js create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/zh-TW.d.cts create mode 100644 tests/fixtures/npm/node_modules/zod/v4/locales/zh-TW.js diff --git a/packages/compiler/src/frontend/decl-projection.ts b/packages/compiler/src/frontend/decl-projection.ts index eaf7486df..02a300996 100644 --- a/packages/compiler/src/frontend/decl-projection.ts +++ b/packages/compiler/src/frontend/decl-projection.ts @@ -185,8 +185,24 @@ class TypePrinter { readonly declarations: string[] = []; private readonly inProgress = new Map(); + /** While set, printing registers no recursive alias (projectable()). */ + private dry = false; + constructor(private readonly checker: ts.TypeChecker) {} + /** Whether a type projects, without naming it in the generated file. */ + projectable(type: ts.Type): boolean { + if (type.flags & ANYISH) return false; + const was = this.dry; + this.dry = true; + try { + const text = this.print(type, 0); + return text !== null && text !== "unknown"; + } finally { + this.dry = was; + } + } + /** The alias naming a projectable type, or null. Top-level any/unknown * never projects. */ project(type: ts.Type): string | null { @@ -212,7 +228,12 @@ class TypePrinter { if (f & ts.TypeFlags.Null) return "null"; if (f & ts.TypeFlags.Undefined) return "undefined"; if (f & ts.TypeFlags.EnumLiteral || f & ts.TypeFlags.Enum) return null; - if (f & ts.TypeFlags.BooleanLiteral) return type.intrinsicName === "true" ? "true" : "false"; + // A boolean literal type carries its value (`intrinsicName` is not + // set for it by every checker facade). + if (f & ts.TypeFlags.BooleanLiteral) + return (type as ts.BooleanLiteralType).value === true || type.intrinsicName === "true" + ? "true" + : "false"; if (f & ts.TypeFlags.StringLiteral) return JSON.stringify(String(type.value)); if (f & ts.TypeFlags.NumberLiteral) { const v = Number(type.value); @@ -341,7 +362,7 @@ class TypePrinter { if (props.length > 0 && readonlyCount === props.length) return null; const text = `{ ${members.join("; ")} }`; if (frame.name !== null) { - this.declarations.push(`type ${frame.name} = ${text};`); + if (!this.dry) this.declarations.push(`type ${frame.name} = ${text};`); return frame.name; } return text; @@ -517,9 +538,17 @@ export function planDeclProjection(input: DeclProjectionInput): DeclProjection | const declSf = input.declProgram.getSourceFile(file); if (jsSf === undefined || declSf === undefined || jsSf.text !== declSf.text) continue; const fileEdits: Edit[] = []; - /** Candidate expressions (by js node) with their alias. */ - const exprCandidates = new Map(); + /** Candidate expressions (by js node) with their declared type, named + * only for the casts that survive suppression. */ + const exprCandidates = new Map(); const exprOrder: ts.Node[] = []; + /** Reads whose projectable receiver the JavaScript world types `any`: + * kept only when nothing types the receiver. */ + const deferred = new Set(); + /** Package calls whose declared type does not project. */ + const unprojected = new Set(); + /** The declaration-world twin of every walked node. */ + const twins = new Map(); const considerType = (jsNode: ts.Node, declNode: ts.Node): boolean => { if ( @@ -603,8 +632,18 @@ export function planDeclProjection(input: DeclProjectionInput): DeclProjection | } else if (ts.isPropertyAccessExpression(d) || ts.isElementAccessExpression(d)) { const recvDecl = typeAt(declChecker, d.expression); const recvJs = typeAt(jsChecker, (jsNode as ts.PropertyAccessExpression).expression); - const recvProjectable = recvDecl !== null && printer.project(recvDecl) !== null; + const recvProjectable = recvDecl !== null && printer.projectable(recvDecl); produced = !recvProjectable || (recvJs !== null && !isAnyish(recvJs)); + // A projectable receiver the JavaScript world types `any` is + // usually typed by another candidate. One that is a package value + // nothing projects stays `any`, such as a safeParse result narrowed + // to its success arm (the whole result is not data: its error arm + // is a class): the read crosses here instead, if the suppression + // below confirms the receiver stays untyped. + if (!produced) { + produced = true; + deferred.add(jsNode); + } } else if (ts.isAwaitExpression(d)) { const inner = unwrapTransparent(d.expression); produced = @@ -633,6 +672,12 @@ export function planDeclProjection(input: DeclProjectionInput): DeclProjection | declaredInPackage(resolveAlias(raw)); } if (!produced) return; + if (isCall && !deferred.has(jsNode)) { + // Recorded before the contextual-type test below: a package value + // whose type does not project stays `any` wherever it flows. + const t = typeAt(declChecker, d); + if (t === null || !printer.projectable(t)) unprojected.add(jsNode); + } // A consumer that takes anything needs no conversion. try { const ctx = declChecker.getContextualType(d); @@ -641,10 +686,8 @@ export function planDeclProjection(input: DeclProjectionInput): DeclProjection | // no contextual type } const declType = typeAt(declChecker, d); - if (declType === null) return; - const alias = printer.project(declType); - if (alias === null) return; - exprCandidates.set(jsNode, alias); + if (declType === null || !printer.projectable(declType)) return; + exprCandidates.set(jsNode, declType); exprOrder.push(jsNode); }; @@ -703,6 +746,7 @@ export function planDeclProjection(input: DeclProjectionInput): DeclProjection | const walk = (jsNode: ts.Node, declNode: ts.Node): void => { if (jsNode.kind !== declNode.kind) return; + twins.set(jsNode, declNode); if (ts.isTypeNode(jsNode) || ts.isExpressionWithTypeArguments(jsNode)) { if (considerType(jsNode, declNode)) return; } else { @@ -749,21 +793,67 @@ export function planDeclProjection(input: DeclProjectionInput): DeclProjection | result = willBeTyped(decl.initializer, depth + 1); else if (ts.isParameter(decl) && annotated.has(decl)) result = true; else if (ts.isBindingElement(decl)) result = false; + else if ( + (ts.isVariableDeclaration(decl) || ts.isParameter(decl)) && + decl.type !== undefined + ) + result = annotationWillBeTyped(decl.type); } symbolTyped.set(sym, result); return result; } return false; }; + /** An annotation in this file whose declaration-world type projects: + * after the rewrite it names the projected structure (a package type + * is replaced by its alias, a user alias of one has its right-hand + * side replaced). */ + const annotationWillBeTyped = (type: ts.TypeNode): boolean => { + const twin = twins.get(type); + if (twin === undefined) return false; + let declType: ts.Type | null; + try { + declType = declChecker.getTypeFromTypeNode(twin as ts.TypeNode); + } catch { + return false; + } + return declType !== null && printer.projectable(declType); + }; const receiverOf = (e: ts.Node): ts.Node | null => { if (ts.isPropertyAccessExpression(e) || ts.isElementAccessExpression(e)) return e.expression; if (ts.isIdentifier(e)) return null; return null; }; const kept: ts.Node[] = []; + /** A receiver that stays `any` after the rewrite: a package call whose + * type does not project, or a user const bound to one. */ + const staysUntyped = (e: ts.Node): boolean => { + const u = unwrapTransparent(e as ts.Expression); + if (unprojected.has(u)) return true; + if (!ts.isIdentifier(u)) return false; + let sym: ts.Symbol | undefined; + try { + sym = jsChecker.getSymbolAtLocation(u); + } catch { + return false; + } + const decls = sym?.declarations ?? []; + if (decls.length !== 1) return false; + const decl = decls[0]!; + return ( + ts.isVariableDeclaration(decl) && + decl.type === undefined && + decl.initializer !== undefined && + decl.parent !== undefined && + ts.isVariableDeclarationList(decl.parent) && + (decl.parent.flags & ts.NodeFlags.Const) !== 0 && + unprojected.has(unwrapTransparent(decl.initializer)) + ); + }; for (const e of exprOrder) { const recv = receiverOf(e); if (recv !== null && willBeTyped(recv)) continue; + if (deferred.has(e) && (recv === null || !staysUntyped(recv))) continue; // An identifier candidate that is a user binding never gets here // (only package imports are candidates). kept.push(e); @@ -800,7 +890,8 @@ export function planDeclProjection(input: DeclProjectionInput): DeclProjection | ) continue; } - const alias = exprCandidates.get(e)!; + const alias = printer.project(exprCandidates.get(e)!); + if (alias === null) continue; const start = ts.tokenStartOfNode(e); const stmt = e.parent; let prefix = "("; diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hono-zod-validator/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hono-zod-validator/main.ts.json new file mode 100644 index 000000000..72c2578e6 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/hono-zod-validator/main.ts.json @@ -0,0 +1,61 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/hono-zod-validator/main.ts" + ], + "diags": [ + { + "code": "SC0001", + "message": "Cannot find name 'WebSocket'.", + "loc": { + "file": "/tests/fixtures/npm/node_modules/hono/dist/types/client/types.d.ts", + "start": 1398, + "end": 1407 + } + }, + { + "code": "SC0001", + "message": "Cannot find name 'WebSocket'.", + "loc": { + "file": "/tests/fixtures/npm/node_modules/hono/dist/types/client/types.d.ts", + "start": 1413, + "end": 1422 + } + }, + { + "code": "SC0001", + "message": "Cannot find name 'WebSocket'.", + "loc": { + "file": "/tests/fixtures/npm/node_modules/hono/dist/types/client/types.d.ts", + "start": 3716, + "end": 3725 + } + }, + { + "code": "SC0001", + "message": "Cannot find name 'FormData'.", + "loc": { + "file": "/tests/fixtures/npm/node_modules/hono/dist/types/client/types.d.ts", + "start": 4798, + "end": 4806 + } + }, + { + "code": "SC0001", + "message": "Cannot find name 'FormData'.", + "loc": { + "file": "/tests/fixtures/npm/node_modules/hono/dist/types/request.d.ts", + "start": 549, + "end": 557 + } + }, + { + "code": "SC0001", + "message": "Cannot find name 'FormData'.", + "loc": { + "file": "/tests/fixtures/npm/node_modules/hono/dist/types/request.d.ts", + "start": 5572, + "end": 5580 + } + } + ] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/zod4-boundary/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/zod4-boundary/main.ts.json new file mode 100644 index 000000000..0d656e939 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/zod4-boundary/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/zod4-boundary/main.ts" + ], + "diags": [] +} diff --git a/tests/fixtures/npm/cases/hono-zod-validator/lying.ts b/tests/fixtures/npm/cases/hono-zod-validator/lying.ts new file mode 100644 index 000000000..fa44c2e12 --- /dev/null +++ b/tests/fixtures/npm/cases/hono-zod-validator/lying.ts @@ -0,0 +1,42 @@ +// Native-only (decl-boundary.test.ts): `c.req.valid()` is a checked +// boundary. z.custom checks nothing, so a body its declared type +// misdescribes fails the cast with a path TypeError, which hono's error +// handler reports; Node does not check. A union discriminated by boolean +// literals keeps both arms. +import { zValidator } from "@hono/zod-validator"; +import { Hono } from "hono"; +import { z } from "zod"; + +const Liar = z.custom<{ n: number }>(); +const Outcome = z.discriminatedUnion("ok", [ + z.object({ ok: z.literal(true), value: z.number() }), + z.object({ ok: z.literal(false), reason: z.string() }), +]); + +const app = new Hono(); +app.onError((err, c) => c.text(`${err instanceof TypeError} ${err.message}`, 500)); +app.post("/liar", zValidator("json", Liar), (c) => c.text(String(c.req.valid("json").n + 1))); +app.post("/outcome", zValidator("json", Outcome), (c) => { + const outcome = c.req.valid("json"); + return c.text(outcome.ok ? `value ${outcome.value + 1}` : `reason ${outcome.reason}`); +}); + +const requests: [string, unknown][] = [ + ["/liar", { n: 1 }], + ["/liar", { n: "x" }], + ["/outcome", { ok: true, value: 41 }], + ["/outcome", { ok: false, reason: "late" }], +]; + +async function main(): Promise { + for (const [path, body] of requests) { + const res = await app.request(path, { + method: "POST", + body: JSON.stringify(body), + headers: { "content-type": "application/json" }, + }); + console.log(path, res.status, await res.text()); + } +} + +void main(); diff --git a/tests/fixtures/npm/cases/hono-zod-validator/main.ts b/tests/fixtures/npm/cases/hono-zod-validator/main.ts new file mode 100644 index 000000000..9de69c482 --- /dev/null +++ b/tests/fixtures/npm/cases/hono-zod-validator/main.ts @@ -0,0 +1,96 @@ +// A Hono app validated by @hono/zod-validator and zod 4.6, all compiled +// from their JavaScript: `c.req.valid(target)` hands the handler the +// parsed value, typed by the schema's z.infer through hono's declarations. +// Requests go through app.request, so no socket is involved. +import { zValidator } from "@hono/zod-validator"; +import { Hono, type Context } from "hono"; +import { z } from "zod"; + +const Book = z.strictObject({ + title: z.string().trim().min(1), + year: z.number().int().min(1450), + tags: z.array(z.string()).max(4).default([]), + rating: z.number().min(0).max(5).optional(), +}); +type Book = z.infer; +const ListQuery = z.object({ + minYear: z.coerce.number().int().optional(), + sort: z.enum(["title", "year"]).default("title"), + limit: z.coerce.number().int().min(1).max(10).default(3), +}); +const IdParam = z.object({ id: z.string().regex(/^b\d+$/) }); + +// Failed checks answer with the issues' paths and codes. +const reject = ( + result: { success: true } | { success: false; error: z.core.$ZodError }, + c: Context, +): Response | undefined => { + if (result.success) return undefined; + const paths = result.error.issues.map((i) => `${i.path.join(".")}:${i.code}`); + return c.json({ error: "invalid", paths }, 400); +}; + +const books = new Map(); +let nextId = 1; + +const app = new Hono(); +app.post( + "/books", + zValidator("json", Book, (result, c) => { + if (!result.success) { + const paths = result.error.issues.map((i) => `${i.path.join(".")}:${i.code}`); + return c.json({ error: "invalid", paths }, 400); + } + }), + (c) => { + const book = c.req.valid("json"); + book.tags.push("new"); + const id = `b${nextId++}`; + books.set(id, book); + const score = (book.rating ?? 0) * 2 + book.tags.length; + return c.json({ id, title: book.title.toUpperCase(), year: book.year, score }, 201); + }, +); +app.get("/books", zValidator("query", ListQuery, reject), (c) => { + const { minYear, sort, limit } = c.req.valid("query"); + const rows = [...books.values()] + .filter((b) => minYear === undefined || b.year >= minYear) + .sort((a, b) => (sort === "year" ? a.year - b.year : a.title < b.title ? -1 : 1)) + .slice(0, limit) + .map((b) => `${b.title}/${b.year}`); + return c.json({ sort, limit, rows }); +}); +app.get("/books/:id", zValidator("param", IdParam, reject), (c) => { + const id = c.req.valid("param").id; + const book = books.get(id); + if (book === undefined) return c.json({ error: "missing", id }, 404); + return c.json({ id, ...book }); +}); + +async function call(method: string, path: string, body?: unknown): Promise { + const init: RequestInit = { method }; + if (body !== undefined) { + init.body = JSON.stringify(body); + init.headers = { "content-type": "application/json" }; + } + const res = await app.request(path, init); + console.log(method, path, res.status, await res.text()); +} + +async function main(): Promise { + await call("POST", "/books", { title: " Dune ", year: 1965, tags: ["sf"], rating: 4.5 }); + await call("POST", "/books", { title: "Emma", year: 1815 }); + await call("POST", "/books", { title: "Ulysses", year: 1922, tags: ["modern"] }); + await call("POST", "/books", { title: "", year: 1200, extra: true }); + await call("POST", "/books", { title: "Bad", year: "x", tags: [1, 2, 3, 4, 5] }); + await call("GET", "/books"); + await call("GET", "/books?minYear=1900&sort=year&limit=5"); + await call("GET", "/books?limit=99"); + await call("GET", "/books/b2"); + await call("GET", "/books/b9"); + await call("GET", "/books/nope"); + const kept = books.get("b1")!; + console.log("stored", kept.tags.join(","), kept.title); +} + +void main(); diff --git a/tests/fixtures/npm/cases/hono-zod-validator/package.json b/tests/fixtures/npm/cases/hono-zod-validator/package.json new file mode 100644 index 000000000..27e5f1b4b --- /dev/null +++ b/tests/fixtures/npm/cases/hono-zod-validator/package.json @@ -0,0 +1 @@ +{ "name": "hono-zod-validator-case", "private": true, "type": "module" } diff --git a/tests/fixtures/npm/cases/zod4-boundary/main.ts b/tests/fixtures/npm/cases/zod4-boundary/main.ts new file mode 100644 index 000000000..b4079d1be --- /dev/null +++ b/tests/fixtures/npm/cases/zod4-boundary/main.ts @@ -0,0 +1,146 @@ +// Values that cross from zod 4.6 (compiled from its JavaScript) into code +// typed by zod's declarations: z.infer, z.input and z.output, identity, +// aliasing and mutation after parse, values zod hands back unchanged, +// callbacks zod calls, typed array work, and caught failures. +import { z } from "zod"; + +const show = (label: string, value: unknown): void => { + console.log(label + " " + JSON.stringify(value)); +}; + +// Typed records: field reads, writes and array work on parsed data. +const Item = z.object({ + sku: z.string(), + price: z.number().nonnegative(), + qty: z.number().int(), + tags: z.array(z.string()).default([]), + note: z.string().optional(), + owner: z.string().nullable(), +}); +type Item = z.infer; +const items: Item[] = z.array(Item).parse([ + { sku: "a", price: 2.5, qty: 4, tags: ["x"], owner: null }, + { sku: "b", price: 10, qty: 1, owner: "ann", note: "fragile" }, + { sku: "c", price: 0.5, qty: 10, tags: ["x", "y"], owner: null }, +]); +const total = items.reduce((sum, it) => sum + it.price * it.qty, 0); +console.log("total", total); +const byTag = new Map(); +for (const it of items) for (const t of it.tags) byTag.set(t, (byTag.get(t) ?? 0) + it.qty); +console.log("byTag", [...byTag].map(([k, v]) => `${k}=${v}`).join(",")); +const sorted = [...items].sort((a, b) => b.price - a.price).map((it) => it.sku); +console.log("sorted", sorted.join(",")); +console.log("optional", items.map((it) => `${it.note ?? "-"}/${it.owner ?? "nobody"}`).join(",")); +console.log("keys", items.map((it) => Object.keys(it).join("+")).join(" ")); + +// Mutation after parse is visible through every reference. +const first = items[0]!; +first.qty += 1; +first.tags.push("z"); +console.log("mutated", items[0]!.qty, items[0]!.tags.join("|"), first === items[0]); +show("item", items[0]); + +// Strict objects reject unknown keys; the error is zod's, caught. +const Strict = z.strictObject({ name: z.string().trim().min(1), email: z.email() }); +type Strict = z.infer; +const person: Strict = Strict.parse({ name: " Ada ", email: "ada@example.com" }); +console.log("strict", person.name.length, person.email.toUpperCase()); +const rejected = Strict.safeParse({ name: "x", email: "x@example.com", extra: 1 }); +console.log("strict bad", rejected.success, rejected.error?.issues.map((i) => i.code).join(",")); + +// z.input and z.output differ across a transform and a default. +const Config = z.object({ + port: z.string().transform((s) => Number(s)), + host: z.string().default("localhost"), + debug: z.enum(["true", "false"]).transform((v) => v === "true"), +}); +type ConfigIn = z.input; +type ConfigOut = z.output; +const raw: ConfigIn = { port: "8080", debug: "true" }; +const config: ConfigOut = Config.parse(raw); +console.log("config", config.port + 1, config.host, config.debug, typeof config.port); + +// A value zod hands back unchanged keeps its identity and its mutations. +const Point = z.custom<{ x: number; y: number }>(); +const original = { x: 1, y: 2 }; +const same = Point.parse(original); +same.x = 10; +console.log("custom", original.x, same === original); +original.y = 20; +console.log("custom back", same.y); + +// A shared object returned by a transform: every parse sees the same one. +const shared = { hits: 0 }; +const Counter = z.string().transform(() => shared); +const c1 = Counter.parse("a"); +const c2 = Counter.parse("b"); +c1.hits += 1; +console.log("shared", c2.hits, shared.hits, c1 === c2); + +// A refine callback reads typed fields of the value zod checks. +const Range = z + .object({ lo: z.number(), hi: z.number() }) + .refine((r) => r.lo <= r.hi, { message: "lo must not exceed hi" }); +console.log("range ok", JSON.stringify(Range.parse({ lo: 1, hi: 3 }))); +const bad = Range.safeParse({ lo: 5, hi: 3 }); +console.log("range bad", bad.success ? "?" : bad.error.issues[0]?.message); + +// safeParse data, a discriminated union, and loose objects' extra keys. +const Event = z.discriminatedUnion("type", [ + z.object({ type: z.literal("click"), x: z.number(), y: z.number() }), + z.object({ type: z.literal("key"), key: z.string() }), +]); +type Event = z.infer; +const describe = (e: Event): string => (e.type === "click" ? `click@${e.x},${e.y}` : `key:${e.key}`); +const r = Event.safeParse({ type: "key", key: "Enter" }); +if (r.success) console.log("event", describe(r.data)); +const events = z.array(Event).parse([ + { type: "click", x: 1, y: 2 }, + { type: "key", key: "a" }, +]); +console.log("events", events.map(describe).join(" ")); +const Loose = z.looseObject({ id: z.number() }); +const loose = Loose.parse({ extra: true, id: 3, more: "m" }); +console.log("loose", loose.id, Object.keys(loose).join(","), JSON.stringify(loose)); + +// A union discriminated by boolean literals, read through a safeParse +// result narrowed to its success arm. +const Outcome = z.discriminatedUnion("ok", [ + z.object({ ok: z.literal(true), value: z.number() }), + z.object({ ok: z.literal(false), reason: z.string() }), +]); +type Outcome = z.infer; +const outcomes: Outcome[] = []; +for (const input of [{ ok: true, value: 41 }, { ok: false, reason: "late" }, { ok: "maybe" }]) { + const parsed = Outcome.safeParse(input); + if (parsed.success) outcomes.push(parsed.data); + else console.log("outcome bad", parsed.error.issues.length); +} +console.log("outcomes", outcomes.map((o) => (o.ok ? `value ${o.value + 1}` : `reason ${o.reason}`)).join(", ")); +const Flags = z.object({ beta: z.boolean().optional(), mode: z.enum(["a", "b"]) }); +const flagged = Flags.safeParse({ beta: true, mode: "b" }); +if (flagged.success) { + const flags = flagged.data; + console.log("flags", flags.beta === true ? "beta on" : "beta off", flags.mode); +} + +// Tuples, brands and coerced query-style input. +const Pair = z.tuple([z.string(), z.number()]); +const [label, count] = Pair.parse(["n", 4]); +console.log("tuple", label, count + 1); +const UserId = z.string().brand<"UserId">(); +const uid = UserId.parse("u_1"); +console.log("brand", uid.length, uid); +const Query = z.object({ page: z.coerce.number().int().min(1).default(1), q: z.string().optional() }); +const query = Query.parse({ page: "3" }); +console.log("query", query.page * 10, query.q === undefined, "q" in query); + +// async parse +async function main(): Promise { + const it = await Item.parseAsync({ sku: "d", price: 1, qty: 2, owner: null }); + console.log("async", it.sku, it.price * it.qty, it.tags.length); + const failed = await Item.safeParseAsync({ sku: 1, price: -1, qty: 0.5, owner: null }); + console.log("async bad", failed.success, failed.error?.issues.map((i) => i.path.join(".")).join(",")); +} + +void main(); diff --git a/tests/fixtures/npm/cases/zod4-boundary/package.json b/tests/fixtures/npm/cases/zod4-boundary/package.json new file mode 100644 index 000000000..39bd81280 --- /dev/null +++ b/tests/fixtures/npm/cases/zod4-boundary/package.json @@ -0,0 +1 @@ +{ "name": "zod4-boundary-case", "private": true, "type": "module" } diff --git a/tests/fixtures/npm/node_modules/@hono/zod-validator/dist/index.d.mts b/tests/fixtures/npm/node_modules/@hono/zod-validator/dist/index.d.mts new file mode 100644 index 000000000..6028477a3 --- /dev/null +++ b/tests/fixtures/npm/node_modules/@hono/zod-validator/dist/index.d.mts @@ -0,0 +1,43 @@ +import { InferInput } from "hono/validator"; +import { Context, Env, Input, MiddlewareHandler, TypedResponse, ValidationTargets } from "hono"; +import * as v3 from "zod/v3"; +import { ZodSafeParseResult } from "zod/v4"; +import * as v4 from "zod/v4/core"; +//#region src/index.d.ts +type ZodSchema = v3.ZodType | v4.$ZodType; +type ZodError = T extends v4.$ZodType ? v4.$ZodError> : v3.ZodError; +type ZodSafeParseResult$1 = T3 extends v4.$ZodType ? ZodSafeParseResult : v3.SafeParseReturnType; +type zInput = T extends v3.ZodType ? v3.input : T extends v4.$ZodType ? v4.input : never; +type zOutput = T extends v3.ZodType ? v3.output : T extends v4.$ZodType ? v4.output : never; +type zInfer = T extends v3.ZodType ? v3.infer : T extends v4.$ZodType ? v4.infer : never; +type ZodValidatorFailureBody = T extends v4.$ZodType ? Extract>, { + success: false; +}> : T extends v3.ZodType ? Extract, v3.output>, { + success: false; +}> : never; +type ZodValidatorFailureResponse = Response & TypedResponse, 400, "json">; +type Hook = (result: ({ + success: true; + data: T; +} | { + success: false; + error: ZodError; + data: T; +}) & { + target: Target; +}, c: Context) => Response | void | TypedResponse | Promise>; +type HasUndefined = undefined extends T ? true : false; +type ExtractValidationResponse = VF extends ((value: any, c: any) => infer R) ? R extends Promise ? PR extends TypedResponse ? Response & TypedResponse : PR extends Response ? PR : PR extends undefined ? never : never : R extends TypedResponse ? Response & TypedResponse : R extends Response ? R : R extends undefined ? never : never : never; +type ValidationFunctionOption = { + validationFunction: (schema: T, value: ValidationTargets[Target]) => ZodSafeParseResult$1 | Promise>; +}; +type DefaultInput = { + in: HasUndefined extends true ? { [K in Target]?: [In] extends [ValidationTargets[K]] ? In : InferInput; } : { [K in Target]: [In] extends [ValidationTargets[K]] ? In : InferInput; }; + out: { [K in Target]: Out; }; +}; +declare function zValidatorFunction, Out = zOutput, I extends Input = DefaultInput, V extends I = I>(target: Target, schema: T, hook?: undefined, options?: ValidationFunctionOption): MiddlewareHandler>; +declare function zValidatorFunction, In = zInput, Out = zOutput, I extends Input = DefaultInput, V extends I = I, InferredValue = zInfer>(target: Target, schema: T, hook?: HookFn, options?: ValidationFunctionOption): MiddlewareHandler>; +declare const zValidator: typeof zValidatorFunction; +//#endregion +export { Hook, zValidator }; +//# sourceMappingURL=index.d.mts.map \ No newline at end of file diff --git a/tests/fixtures/npm/node_modules/@hono/zod-validator/dist/index.mjs b/tests/fixtures/npm/node_modules/@hono/zod-validator/dist/index.mjs new file mode 100644 index 000000000..946d23d2a --- /dev/null +++ b/tests/fixtures/npm/node_modules/@hono/zod-validator/dist/index.mjs @@ -0,0 +1,28 @@ +import { validator } from "hono/validator"; +//#region src/index.ts +function zValidatorFunction(target, schema, hook, options) { + const caseInsensitiveKeymap = target === "header" && ("_def" in schema || "_zod" in schema) ? Object.fromEntries(Object.keys("in" in schema ? schema.in.shape : schema.shape).map((key) => [key.toLowerCase(), key])) : void 0; + return validator(target, async (value, c) => { + let validatorValue = value; + if (caseInsensitiveKeymap) validatorValue = Object.fromEntries(Object.entries(value).map(([key, value]) => [caseInsensitiveKeymap[key] || key, value])); + const result = options && options.validationFunction ? await options.validationFunction(schema, validatorValue) : await schema.safeParseAsync(validatorValue); + if (hook) { + const hookResult = await hook({ + data: validatorValue, + ...result, + target + }, c); + if (hookResult) { + if (hookResult instanceof Response) return hookResult; + if ("response" in hookResult) return hookResult.response; + } + } + if (!result.success) return c.json(result, 400); + return result.data; + }); +} +const zValidator = zValidatorFunction; +//#endregion +export { zValidator }; + +//# sourceMappingURL=index.mjs.map \ No newline at end of file diff --git a/tests/fixtures/npm/node_modules/@hono/zod-validator/package.json b/tests/fixtures/npm/node_modules/@hono/zod-validator/package.json new file mode 100644 index 000000000..31355f555 --- /dev/null +++ b/tests/fixtures/npm/node_modules/@hono/zod-validator/package.json @@ -0,0 +1,14 @@ +{ + "name": "@hono/zod-validator", + "version": "0.9.1", + "description": "Test fixture: the ESM build and its declarations.", + "type": "module", + "license": "MIT", + "types": "./dist/index.d.mts", + "exports": { + ".": { + "types": "./dist/index.d.mts", + "import": "./dist/index.mjs" + } + } +} diff --git a/tests/fixtures/npm/node_modules/hono/dist/compose.js b/tests/fixtures/npm/node_modules/hono/dist/compose.js new file mode 100644 index 000000000..ec576df0f --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/compose.js @@ -0,0 +1,50 @@ +//#region src/compose.ts +/** +* Compose middleware functions into a single function based on `koa-compose` package. +* +* @template E - The environment type. +* +* @param {[[Function, unknown], unknown][] | [[Function]][]} middleware - An array of middleware functions and their corresponding parameters. +* @param {ErrorHandler} [onError] - An optional error handler function. +* @param {NotFoundHandler} [onNotFound] - An optional not-found handler function. +* +* @returns {(context: Context, next?: Next) => Promise} - A composed middleware function. +*/ +const compose = (middleware, onError, onNotFound) => { + return (context, next) => { + let index = -1; + return dispatch(0); + /** + * Dispatch the middleware functions. + * + * @param {number} i - The current index in the middleware array. + * + * @returns {Promise} - A promise that resolves to the context. + */ + async function dispatch(i) { + if (i <= index) throw new Error("next() called multiple times"); + index = i; + let res; + let isError = false; + let handler; + if (middleware[i]) { + handler = middleware[i][0][0]; + context.req.routeIndex = i; + } else handler = i === middleware.length && next || void 0; + if (handler) try { + res = await handler(context, () => dispatch(i + 1)); + } catch (err) { + if (err instanceof Error && onError) { + context.error = err; + res = await onError(err, context); + isError = true; + } else throw err; + } + else if (context.finalized === false && onNotFound) res = await onNotFound(context); + if (res && (context.finalized === false || isError)) context.res = res; + return context; + } + }; +}; +//#endregion +export { compose }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/context.js b/tests/fixtures/npm/node_modules/hono/dist/context.js new file mode 100644 index 000000000..7f8184af5 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/context.js @@ -0,0 +1,381 @@ +import { HonoRequest } from "./request.js"; +import { HtmlEscapedCallbackPhase, resolveCallback } from "./utils/html.js"; +//#region src/context.ts +const TEXT_PLAIN = "text/plain; charset=UTF-8"; +const setDefaultContentType = (contentType, headers) => { + return { + "Content-Type": contentType, + ...headers + }; +}; +const createResponseInstance = (body, init) => new Response(body, init); +var Context = class { + #rawRequest; + #req; + /** + * `.env` can get bindings (environment variables, secrets, KV namespaces, D1 database, R2 bucket etc.) in Cloudflare Workers. + * + * @see {@link https://hono.dev/docs/api/context#env} + * + * @example + * ```ts + * // Environment object for Cloudflare Workers + * app.get('*', async c => { + * const counter = c.env.COUNTER + * }) + * ``` + */ + env = {}; + #var; + finalized = false; + /** + * `.error` can get the error object from the middleware if the Handler throws an error. + * + * @see {@link https://hono.dev/docs/api/context#error} + * + * @example + * ```ts + * app.use('*', async (c, next) => { + * await next() + * if (c.error) { + * // do something... + * } + * }) + * ``` + */ + error; + #status; + #executionCtx; + #res; + #layout; + #renderer; + #notFoundHandler; + #preparedHeaders; + #matchResult; + #path; + /** + * Creates an instance of the Context class. + * + * @param req - The Request object. + * @param options - Optional configuration options for the context. + */ + constructor(req, options) { + this.#rawRequest = req; + if (options) { + this.#executionCtx = options.executionCtx; + this.env = options.env; + this.#notFoundHandler = options.notFoundHandler; + this.#path = options.path; + this.#matchResult = options.matchResult; + } + } + /** + * `.req` is the instance of {@link HonoRequest}. + */ + get req() { + this.#req ??= new HonoRequest(this.#rawRequest, this.#path, this.#matchResult); + return this.#req; + } + /** + * @see {@link https://hono.dev/docs/api/context#event} + * The FetchEvent associated with the current request. + * + * @throws Will throw an error if the context does not have a FetchEvent. + */ + get event() { + if (this.#executionCtx && "respondWith" in this.#executionCtx) return this.#executionCtx; + else throw Error("This context has no FetchEvent"); + } + /** + * @see {@link https://hono.dev/docs/api/context#executionctx} + * The ExecutionContext associated with the current request. + * + * @throws Will throw an error if the context does not have an ExecutionContext. + */ + get executionCtx() { + if (this.#executionCtx) return this.#executionCtx; + else throw Error("This context has no ExecutionContext"); + } + /** + * @see {@link https://hono.dev/docs/api/context#res} + * The Response object for the current request. + */ + get res() { + return this.#res ||= createResponseInstance(null, { headers: this.#preparedHeaders ??= new Headers() }); + } + /** + * Sets the Response object for the current request. + * + * @param _res - The Response object to set. + */ + set res(_res) { + if (this.#res && _res) { + _res = createResponseInstance(_res.body, _res); + for (const [k, v] of this.#res.headers.entries()) { + if (k === "content-type") continue; + if (k === "set-cookie") { + const cookies = this.#res.headers.getSetCookie(); + _res.headers.delete("set-cookie"); + for (const cookie of cookies) _res.headers.append("set-cookie", cookie); + } else _res.headers.set(k, v); + } + } + this.#res = _res; + this.finalized = true; + } + /** + * `.render()` can create a response within a layout. + * + * @see {@link https://hono.dev/docs/api/context#render-setrenderer} + * + * @example + * ```ts + * app.get('/', (c) => { + * return c.render('Hello!') + * }) + * ``` + */ + render = (...args) => { + this.#renderer ??= (content) => this.html(content); + return this.#renderer(...args); + }; + /** + * Sets the layout for the response. + * + * @param layout - The layout to set. + * @returns The layout function. + */ + setLayout = (layout) => this.#layout = layout; + /** + * Gets the current layout for the response. + * + * @returns The current layout function. + */ + getLayout = () => this.#layout; + /** + * `.setRenderer()` can set the layout in the custom middleware. + * + * @see {@link https://hono.dev/docs/api/context#render-setrenderer} + * + * @example + * ```tsx + * app.use('*', async (c, next) => { + * c.setRenderer((content) => { + * return c.html( + * + * + *

{content}

+ * + * + * ) + * }) + * await next() + * }) + * ``` + */ + setRenderer = (renderer) => { + this.#renderer = renderer; + }; + /** + * `.header()` can set headers. + * + * @see {@link https://hono.dev/docs/api/context#header} + * + * @example + * ```ts + * app.get('/welcome', (c) => { + * // Set headers + * c.header('X-Message', 'Hello!') + * c.header('Content-Type', 'text/plain') + * + * // Append multiple headers using the append option (e.g. Vary) + * c.header('Vary', 'Accept-Encoding', { append: true }) + * c.header('Vary', 'User-Agent', { append: true }) + * + * return c.body('Thank you for coming') + * }) + * ``` + */ + header = (name, value, options) => { + if (this.finalized) this.#res = createResponseInstance(this.#res.body, this.#res); + const headers = this.#res ? this.#res.headers : this.#preparedHeaders ??= new Headers(); + if (value === void 0) headers.delete(name); + else if (options?.append) headers.append(name, value); + else headers.set(name, value); + }; + status = (status) => { + this.#status = status; + }; + /** + * `.set()` can set the value specified by the key. + * + * @see {@link https://hono.dev/docs/api/context#set-get} + * + * @example + * ```ts + * app.use('*', async (c, next) => { + * c.set('message', 'Hono is hot!!') + * await next() + * }) + * ``` + */ + set = (key, value) => { + this.#var ??= /* @__PURE__ */ new Map(); + this.#var.set(key, value); + }; + /** + * `.get()` can use the value specified by the key. + * + * @see {@link https://hono.dev/docs/api/context#set-get} + * + * @example + * ```ts + * app.get('/', (c) => { + * const message = c.get('message') + * return c.text(`The message is "${message}"`) + * }) + * ``` + */ + get = (key) => { + return this.#var ? this.#var.get(key) : void 0; + }; + /** + * `.var` can access the value of a variable. + * + * @see {@link https://hono.dev/docs/api/context#var} + * + * @example + * ```ts + * const result = c.var.client.oneMethod() + * ``` + */ + get var() { + if (!this.#var) return {}; + return Object.fromEntries(this.#var); + } + #newResponse(data, arg, headers) { + let responseHeaders = this.#res ? new Headers(this.#res.headers) : this.#preparedHeaders; + if (typeof arg === "object" && arg.headers) { + responseHeaders ??= new Headers(); + for (const [key, value] of new Headers(arg.headers)) if (key === "set-cookie") responseHeaders.append(key, value); + else responseHeaders.set(key, value); + } + if (headers) { + if (!responseHeaders) { + let count = 0; + for (const k in headers) if (++count > 1 || typeof headers[k] !== "string") { + responseHeaders = new Headers(); + break; + } + } + if (responseHeaders) for (const k in headers) { + const v = headers[k]; + if (typeof v === "string") responseHeaders.set(k, v); + else { + responseHeaders.delete(k); + for (const v2 of v) responseHeaders.append(k, v2); + } + } + } + const status = typeof arg === "number" ? arg : arg?.status ?? this.#status; + return createResponseInstance(data, { + status, + headers: responseHeaders ?? headers + }); + } + newResponse = (...args) => this.#newResponse(...args); + /** + * `.body()` can return the HTTP response. + * You can set headers with `.header()` and set HTTP status code with `.status`. + * This can also be set in `.text()`, `.json()` and so on. + * + * @see {@link https://hono.dev/docs/api/context#body} + * + * @example + * ```ts + * app.get('/welcome', (c) => { + * // Set headers + * c.header('X-Message', 'Hello!') + * c.header('Content-Type', 'text/plain') + * // Set HTTP status code + * c.status(201) + * + * // Return the response body + * return c.body('Thank you for coming') + * }) + * ``` + */ + body = (data, arg, headers) => this.#newResponse(data, arg, headers); + /** + * `.text()` can render text as `Content-Type:text/plain`. + * + * @see {@link https://hono.dev/docs/api/context#text} + * + * @example + * ```ts + * app.get('/say', (c) => { + * return c.text('Hello!') + * }) + * ``` + */ + text = (text, arg, headers) => { + return !this.#preparedHeaders && !this.#status && !arg && !headers && !this.finalized ? new Response(text) : this.#newResponse(text, arg, setDefaultContentType(TEXT_PLAIN, headers)); + }; + /** + * `.json()` can render JSON as `Content-Type:application/json`. + * + * @see {@link https://hono.dev/docs/api/context#json} + * + * @example + * ```ts + * app.get('/api', (c) => { + * return c.json({ message: 'Hello!' }) + * }) + * ``` + */ + json = (object, arg, headers) => { + return this.#newResponse(JSON.stringify(object), arg, setDefaultContentType("application/json", headers)); + }; + html = (html, arg, headers) => { + const res = (html) => this.#newResponse(html, arg, setDefaultContentType("text/html; charset=UTF-8", headers)); + return typeof html === "object" ? resolveCallback(html, HtmlEscapedCallbackPhase.Stringify, false, {}).then(res) : res(html); + }; + /** + * `.redirect()` can Redirect, default status code is 302. + * + * @see {@link https://hono.dev/docs/api/context#redirect} + * + * @example + * ```ts + * app.get('/redirect', (c) => { + * return c.redirect('/') + * }) + * app.get('/redirect-permanently', (c) => { + * return c.redirect('/', 301) + * }) + * ``` + */ + redirect = (location, status) => { + const locationString = String(location); + this.header("Location", !/[^\x00-\xFF]/.test(locationString) ? locationString : encodeURI(locationString)); + return this.newResponse(null, status ?? 302); + }; + /** + * `.notFound()` can return the Not Found Response. + * + * @see {@link https://hono.dev/docs/api/context#notfound} + * + * @example + * ```ts + * app.get('/notfound', (c) => { + * return c.notFound() + * }) + * ``` + */ + notFound = () => { + this.#notFoundHandler ??= () => createResponseInstance(); + return this.#notFoundHandler(this); + }; +}; +//#endregion +export { Context, TEXT_PLAIN }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/helper/cookie/index.js b/tests/fixtures/npm/node_modules/hono/dist/helper/cookie/index.js new file mode 100644 index 000000000..48f9cbfb7 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/helper/cookie/index.js @@ -0,0 +1,82 @@ +import { parse, parseSigned, serialize, serializeSigned } from "../../utils/cookie.js"; +//#region src/helper/cookie/index.ts +const getCookie = (c, key, prefix) => { + const cookie = c.req.raw.headers.get("Cookie"); + if (typeof key === "string") { + if (!cookie) return; + let finalKey = key; + if (prefix === "secure") finalKey = "__Secure-" + key; + else if (prefix === "host") finalKey = "__Host-" + key; + return parse(cookie, finalKey)[finalKey]; + } + if (!cookie) return {}; + return parse(cookie); +}; +const getSignedCookie = async (c, secret, key, prefix) => { + const cookie = c.req.raw.headers.get("Cookie"); + if (typeof key === "string") { + if (!cookie) return; + let finalKey = key; + if (prefix === "secure") finalKey = "__Secure-" + key; + else if (prefix === "host") finalKey = "__Host-" + key; + return (await parseSigned(cookie, secret, finalKey))[finalKey]; + } + if (!cookie) return {}; + return await parseSigned(cookie, secret); +}; +const generateCookie = (name, value, opt) => { + let cookie; + if (opt?.prefix === "secure") cookie = serialize("__Secure-" + name, value, { + path: "/", + ...opt, + secure: true + }); + else if (opt?.prefix === "host") cookie = serialize("__Host-" + name, value, { + ...opt, + path: "/", + secure: true, + domain: void 0 + }); + else cookie = serialize(name, value, { + path: "/", + ...opt + }); + return cookie; +}; +const setCookie = (c, name, value, opt) => { + const cookie = generateCookie(name, value, opt); + c.header("Set-Cookie", cookie, { append: true }); +}; +const generateSignedCookie = async (name, value, secret, opt) => { + let cookie; + if (opt?.prefix === "secure") cookie = await serializeSigned("__Secure-" + name, value, secret, { + path: "/", + ...opt, + secure: true + }); + else if (opt?.prefix === "host") cookie = await serializeSigned("__Host-" + name, value, secret, { + ...opt, + path: "/", + secure: true, + domain: void 0 + }); + else cookie = await serializeSigned(name, value, secret, { + path: "/", + ...opt + }); + return cookie; +}; +const setSignedCookie = async (c, name, value, secret, opt) => { + const cookie = await generateSignedCookie(name, value, secret, opt); + c.header("set-cookie", cookie, { append: true }); +}; +const deleteCookie = (c, name, opt) => { + const deletedCookie = getCookie(c, name, opt?.prefix); + setCookie(c, name, "", { + ...opt, + maxAge: 0 + }); + return deletedCookie; +}; +//#endregion +export { deleteCookie, generateCookie, generateSignedCookie, getCookie, getSignedCookie, setCookie, setSignedCookie }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/hono-base.js b/tests/fixtures/npm/node_modules/hono/dist/hono-base.js new file mode 100644 index 000000000..7bc399a89 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/hono-base.js @@ -0,0 +1,350 @@ +import { compose } from "./compose.js"; +import { getPath, getPathNoStrict, mergePath } from "./utils/url.js"; +import { Context } from "./context.js"; +import { METHODS } from "./router.js"; +import { COMPOSED_HANDLER } from "./utils/constants.js"; +//#region src/hono-base.ts +/** +* @module +* This module is the base module for the Hono object. +*/ +const notFoundHandler = (c) => { + return c.text("404 Not Found", 404); +}; +const errorHandler = (err, c) => { + if ("getResponse" in err) { + const res = err.getResponse(); + return c.newResponse(res.body, res); + } + console.error(err); + return c.text("Internal Server Error", 500); +}; +var Hono = class Hono { + get; + post; + put; + delete; + options; + patch; + query; + all; + on; + use; + router; + getPath; + _basePath = "/"; + #path = "/"; + routes = []; + constructor(options = {}) { + [...METHODS, "all"].forEach((method) => { + this[method] = (args1, ...args) => { + const methodName = method.toUpperCase(); + if (typeof args1 === "string") this.#path = args1; + else this.#addRoute(methodName, this.#path, args1); + args.forEach((handler) => { + this.#addRoute(methodName, this.#path, handler); + }); + return this; + }; + }); + this.on = (method, path, ...handlers) => { + for (const p of [path].flat()) { + this.#path = p; + for (const m of [method].flat()) { + const methodName = m.toUpperCase(); + for (const handler of handlers) this.#addRoute(methodName, this.#path, handler); + } + } + return this; + }; + this.use = (arg1, ...handlers) => { + if (typeof arg1 === "string") this.#path = arg1; + else { + this.#path = "*"; + handlers.unshift(arg1); + } + handlers.forEach((handler) => { + this.#addRoute("ALL", this.#path, handler); + }); + return this; + }; + const { strict, ...optionsWithoutStrict } = options; + Object.assign(this, optionsWithoutStrict); + this.getPath = strict ?? true ? options.getPath ?? getPath : getPathNoStrict; + } + #clone() { + const clone = new Hono({ + router: this.router, + getPath: this.getPath + }); + clone.errorHandler = this.errorHandler; + clone.#notFoundHandler = this.#notFoundHandler; + clone.routes = this.routes; + return clone; + } + #notFoundHandler = notFoundHandler; + errorHandler = errorHandler; + /** + * `.route()` allows grouping other Hono instance in routes. + * + * @see {@link https://hono.dev/docs/api/routing#grouping} + * + * @param {string} path - base Path + * @param {Hono} app - other Hono instance + * @returns {Hono} routed Hono instance + * + * @example + * ```ts + * const app = new Hono() + * const app2 = new Hono() + * + * app2.get("/user", (c) => c.text("user")) + * app.route("/api", app2) // GET /api/user + * ``` + */ + route(path, app) { + const subApp = this.basePath(path); + app.routes.map((r) => { + let handler; + if (app.errorHandler === errorHandler) handler = r.handler; + else { + handler = async (c, next) => (await compose([], app.errorHandler)(c, () => r.handler(c, next))).res; + handler[COMPOSED_HANDLER] = r.handler; + } + subApp.#addRoute(r.method, r.path, handler, r.basePath); + }); + return this; + } + /** + * `.basePath()` allows base paths to be specified. + * + * @see {@link https://hono.dev/docs/api/routing#base-path} + * + * @param {string} path - base Path + * @returns {Hono} changed Hono instance + * + * @example + * ```ts + * const api = new Hono().basePath('/api') + * ``` + */ + basePath(path) { + const subApp = this.#clone(); + subApp._basePath = mergePath(this._basePath, path); + return subApp; + } + /** + * `.onError()` handles an error and returns a customized Response. + * + * @see {@link https://hono.dev/docs/api/hono#error-handling} + * + * @param {ErrorHandler} handler - request Handler for error + * @returns {Hono} changed Hono instance + * + * @example + * ```ts + * app.onError((err, c) => { + * console.error(`${err}`) + * return c.text('Custom Error Message', 500) + * }) + * ``` + */ + onError = (handler) => { + this.errorHandler = handler; + return this; + }; + /** + * `.notFound()` allows you to customize a Not Found Response. + * + * @see {@link https://hono.dev/docs/api/hono#not-found} + * + * @param {NotFoundHandler} handler - request handler for not-found + * @returns {Hono} changed Hono instance + * + * @example + * ```ts + * app.notFound((c) => { + * return c.text('Custom 404 Message', 404) + * }) + * ``` + */ + notFound = (handler) => { + this.#notFoundHandler = handler; + return this; + }; + /** + * `.mount()` allows you to mount applications built with other frameworks into your Hono application. + * + * @deprecated Use `mount()` from `hono/mount` instead. `.mount()` will be removed in v5. + * + * @see {@link https://hono.dev/docs/api/hono#mount} + * + * @param {string} path - base Path + * @param {Function} applicationHandler - other Request Handler + * @param {MountOptions} [options] - options of `.mount()` + * @returns {Hono} mounted Hono instance + * + * @example + * ```ts + * import { Router as IttyRouter } from 'itty-router' + * import { Hono } from 'hono' + * // Create itty-router application + * const ittyRouter = IttyRouter() + * // GET /itty-router/hello + * ittyRouter.get('/hello', () => new Response('Hello from itty-router')) + * + * const app = new Hono() + * app.mount('/itty-router', ittyRouter.handle) + * ``` + * + * @example + * ```ts + * const app = new Hono() + * // Send the request to another application without modification. + * app.mount('/app', anotherApp, { + * replaceRequest: (req) => req, + * }) + * ``` + */ + mount(path, applicationHandler, options) { + let replaceRequest; + let optionHandler; + if (options) { + if (typeof options === "function") optionHandler = options; + else { + optionHandler = options.optionHandler; + if (options.replaceRequest === false) replaceRequest = (request) => request; + else replaceRequest = options.replaceRequest; + } + } + const getOptions = optionHandler ? (c) => { + const options = optionHandler(c); + return Array.isArray(options) ? options : [options]; + } : (c) => { + let executionContext = void 0; + try { + executionContext = c.executionCtx; + } catch {} + return [c.env, executionContext]; + }; + replaceRequest ||= (() => { + const mergedPath = mergePath(this._basePath, path); + const pathPrefixLength = mergedPath === "/" ? 0 : mergedPath.length; + return (request) => { + const url = new URL(request.url); + url.pathname = this.getPath(request).slice(pathPrefixLength) || "/"; + return new Request(url, request); + }; + })(); + const handler = async (c, next) => { + const res = await applicationHandler(replaceRequest(c.req.raw), ...getOptions(c)); + if (res) return res; + await next(); + }; + this.#addRoute("ALL", mergePath(path, "*"), handler); + return this; + } + #addRoute(method, path, handler, baseRoutePath) { + path = mergePath(this._basePath, path); + const r = { + basePath: baseRoutePath !== void 0 ? mergePath(this._basePath, baseRoutePath) : this._basePath, + path, + method, + handler + }; + this.router.add(method, path, [handler, r]); + this.routes.push(r); + } + #handleError(err, c) { + if (err instanceof Error) return this.errorHandler(err, c); + throw err; + } + #dispatch(request, executionCtx, env, method) { + if (method === "HEAD") return (async () => new Response(null, await this.#dispatch(request, executionCtx, env, "GET")))(); + const path = this.getPath(request, { env }); + const matchResult = this.router.match(method, path); + const c = new Context(request, { + path, + matchResult, + env, + executionCtx, + notFoundHandler: this.#notFoundHandler + }); + if (matchResult[0].length === 1) { + let res; + try { + res = matchResult[0][0][0][0](c, async () => { + c.res = await this.#notFoundHandler(c); + }); + } catch (err) { + return this.#handleError(err, c); + } + return res instanceof Promise ? res.then((resolved) => resolved || (c.finalized ? c.res : this.#notFoundHandler(c))).catch((err) => this.#handleError(err, c)) : res ?? this.#notFoundHandler(c); + } + const composed = compose(matchResult[0], this.errorHandler, this.#notFoundHandler); + return (async () => { + try { + const context = await composed(c); + if (!context.finalized) throw new Error("Context is not finalized. Did you forget to return a Response object or `await next()`?"); + return context.res; + } catch (err) { + return this.#handleError(err, c); + } + })(); + } + /** + * `.fetch()` will be entry point of your app. + * + * @see {@link https://hono.dev/docs/api/hono#fetch} + * + * @param {Request} request - request Object of request + * @param {Env} env - env Object + * @param {ExecutionContext} executionCtx - context of execution + * @returns {Response | Promise} response of request + * + */ + fetch = (request, ...rest) => { + return this.#dispatch(request, rest[1], rest[0], request.method); + }; + /** + * `.request()` is a useful method for testing. + * You can pass a URL or pathname to send a GET request. + * app will return a Response object. + * ```ts + * test('GET /hello is ok', async () => { + * const res = await app.request('/hello') + * expect(res.status).toBe(200) + * }) + * ``` + * @see https://hono.dev/docs/api/hono#request + */ + request = (input, requestInit, Env, executionCtx) => { + if (input instanceof Request) return this.fetch(requestInit ? new Request(input, requestInit) : input, Env, executionCtx); + input = input.toString(); + return this.fetch(new Request(/^https?:\/\//.test(input) ? input : `http://localhost${mergePath("/", input)}`, requestInit), Env, executionCtx); + }; + /** + * `.fire()` automatically adds a global fetch event listener. + * This can be useful for environments that adhere to the Service Worker API, such as non-ES module Cloudflare Workers. + * @deprecated + * Use `fire` from `hono/service-worker` instead. + * ```ts + * import { Hono } from 'hono' + * import { fire } from 'hono/service-worker' + * + * const app = new Hono() + * // ... + * fire(app) + * ``` + * @see https://hono.dev/docs/api/hono#fire + * @see https://developer.mozilla.org/en-US/docs/Web/API/Service_Worker_API + * @see https://developers.cloudflare.com/workers/reference/migrate-to-module-workers/ + */ + fire = () => { + addEventListener("fetch", (event) => { + event.respondWith(this.#dispatch(event.request, event, void 0, event.request.method)); + }); + }; +}; +//#endregion +export { Hono as HonoBase }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/hono.js b/tests/fixtures/npm/node_modules/hono/dist/hono.js new file mode 100644 index 000000000..9602aafc3 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/hono.js @@ -0,0 +1,28 @@ +import { HonoBase as Hono$1 } from "./hono-base.js"; +import { RegExpRouter } from "./router/reg-exp-router/router.js"; +import "./router/reg-exp-router/index.js"; +import { SmartRouter } from "./router/smart-router/router.js"; +import { TrieRouter } from "./router/trie-router/router.js"; +import "./router/trie-router/index.js"; +//#region src/hono.ts +/** +* The Hono class extends the functionality of the HonoBase class. +* It sets up routing and allows for custom options to be passed. +* +* @template E - The environment type. +* @template S - The schema type. +* @template BasePath - The base path type. +*/ +var Hono = class extends Hono$1 { + /** + * Creates an instance of the Hono class. + * + * @param options - Optional configuration options for the Hono instance. + */ + constructor(options = {}) { + super(options); + this.router = options.router ?? new SmartRouter({ routers: [new RegExpRouter(), new TrieRouter()] }); + } +}; +//#endregion +export { Hono }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/http-exception.js b/tests/fixtures/npm/node_modules/hono/dist/http-exception.js new file mode 100644 index 000000000..800b530cd --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/http-exception.js @@ -0,0 +1,55 @@ +//#region src/http-exception.ts +/** +* `HTTPException` must be used when a fatal error such as authentication failure occurs. +* +* @see {@link https://hono.dev/docs/api/exception} +* +* @param {StatusCode} status - status code of HTTPException +* @param {HTTPExceptionOptions} options - options of HTTPException +* @param {HTTPExceptionOptions["res"]} options.res - response of options of HTTPException +* @param {HTTPExceptionOptions["message"]} options.message - message of options of HTTPException +* @param {HTTPExceptionOptions["cause"]} options.cause - cause of options of HTTPException +* +* @example +* ```ts +* import { HTTPException } from 'hono/http-exception' +* +* // ... +* +* app.post('/auth', async (c, next) => { +* // authentication +* if (authorized === false) { +* throw new HTTPException(401, { message: 'Custom error message' }) +* } +* await next() +* }) +* ``` +*/ +var HTTPException = class extends Error { + res; + status; + /** + * Creates an instance of `HTTPException`. + * @param status - HTTP status code for the exception. Defaults to 500. + * @param options - Additional options for the exception. + */ + constructor(status = 500, options) { + super(options?.message, { cause: options?.cause }); + this.res = options?.res; + this.status = status; + } + /** + * Returns the response object associated with the exception. + * If a response object is not provided, a new response is created with the error message and status code. + * @returns The response object. + */ + getResponse() { + if (this.res) return new Response(this.res.body, { + status: this.status, + headers: this.res.headers + }); + return new Response(this.message, { status: this.status }); + } +}; +//#endregion +export { HTTPException }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/index.js b/tests/fixtures/npm/node_modules/hono/dist/index.js new file mode 100644 index 000000000..20c96572c --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/index.js @@ -0,0 +1,20 @@ +import { Context } from "./context.js"; +import { Hono } from "./hono.js"; +//#region src/index.ts +/** +* @module +* +* Hono - Web Framework built on Web Standards +* +* @example +* ```ts +* import { Hono } from 'hono' +* const app = new Hono() +* +* app.get('/', (c) => c.text('Hono!')) +* +* export default app +* ``` +*/ +//#endregion +export { Context, Hono }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/request.js b/tests/fixtures/npm/node_modules/hono/dist/request.js new file mode 100644 index 000000000..bf130a518 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/request.js @@ -0,0 +1,335 @@ +import { HTTPException } from "./http-exception.js"; +import { GET_MATCH_RESULT } from "./request/constants.js"; +import { parseBody } from "./utils/body.js"; +import { getQueryParam, getQueryParams, tryDecodeURIComponent } from "./utils/url.js"; +//#region src/request.ts +var HonoRequest = class { + /** + * `.raw` can get the raw Request object. + * + * @see {@link https://hono.dev/docs/api/request#raw} + * + * @example + * ```ts + * // For Cloudflare Workers + * app.post('/', async (c) => { + * const metadata = c.req.raw.cf?.hostMetadata? + * ... + * }) + * ``` + */ + raw; + #validatedData; + #matchResult; + routeIndex = 0; + /** + * `.path` can get the pathname of the request. + * + * @see {@link https://hono.dev/docs/api/request#path} + * + * @example + * ```ts + * app.get('/about/me', (c) => { + * const pathname = c.req.path // `/about/me` + * }) + * ``` + */ + path; + bodyCache = {}; + constructor(request, path = "/", matchResult = [[]]) { + this.raw = request; + this.path = path; + this.#matchResult = matchResult; + } + param(key) { + return key ? this.#getDecodedParam(key) : this.#getAllDecodedParams(); + } + #getDecodedParam(key) { + const paramKey = this.#matchResult[0][this.routeIndex]?.[1][key]; + const param = this.#getParamValue(paramKey); + return param && tryDecodeURIComponent(param); + } + #getAllDecodedParams() { + const decoded = {}; + const keys = Object.keys(this.#matchResult[0][this.routeIndex]?.[1] ?? {}); + for (const key of keys) { + const value = this.#getParamValue(this.#matchResult[0][this.routeIndex][1][key]); + if (value !== void 0) decoded[key] = tryDecodeURIComponent(value); + } + return decoded; + } + #getParamValue(paramKey) { + return this.#matchResult[1] ? this.#matchResult[1][paramKey] : paramKey; + } + query(key) { + return getQueryParam(this.url, key); + } + queries(key) { + return getQueryParams(this.url, key); + } + header(name) { + if (name) return this.raw.headers.get(name) ?? void 0; + const headerData = Object.create(null); + this.raw.headers.forEach((value, key) => { + headerData[key] = value; + }); + return headerData; + } + async parseBody(options) { + return parseBody(this, options); + } + #cachedBody = (key) => { + const { bodyCache, raw } = this; + const cachedBody = bodyCache[key]; + if (cachedBody) return cachedBody; + for (const anyCachedKey in bodyCache) return bodyCache[anyCachedKey].then((body) => { + if (anyCachedKey === "json") body = JSON.stringify(body); + const contentType = anyCachedKey === "formData" ? void 0 : raw.headers.get("content-type"); + return new Response(body, { headers: contentType ? { "Content-Type": contentType } : void 0 })[key](); + }); + return bodyCache[key] = raw[key](); + }; + /** + * `.json()` can parse Request body of type `application/json` + * + * @see {@link https://hono.dev/docs/api/request#json} + * + * @example + * ```ts + * app.post('/entry', async (c) => { + * const body = await c.req.json() + * }) + * ``` + */ + json() { + return this.#cachedBody("text").then((text) => JSON.parse(text)); + } + /** + * `.text()` can parse Request body of type `text/plain` + * + * @see {@link https://hono.dev/docs/api/request#text} + * + * @example + * ```ts + * app.post('/entry', async (c) => { + * const body = await c.req.text() + * }) + * ``` + */ + text() { + return this.#cachedBody("text"); + } + /** + * `.arrayBuffer()` parse Request body as an `ArrayBuffer` + * + * @see {@link https://hono.dev/docs/api/request#arraybuffer} + * + * @example + * ```ts + * app.post('/entry', async (c) => { + * const body = await c.req.arrayBuffer() + * }) + * ``` + */ + arrayBuffer() { + return this.#cachedBody("arrayBuffer"); + } + /** + * `.bytes()` parses the request body as a `Uint8Array`. + * + * @see {@link https://hono.dev/docs/api/request#bytes} + * + * @example + * ```ts + * app.post('/entry', async (c) => { + * const body = await c.req.bytes() + * }) + * ``` + */ + bytes() { + return this.#cachedBody("arrayBuffer").then((buffer) => new Uint8Array(buffer)); + } + /** + * Parses the request body as a `Blob`. + * @example + * ```ts + * app.post('/entry', async (c) => { + * const body = await c.req.blob(); + * }); + * ``` + * @see https://hono.dev/docs/api/request#blob + */ + blob() { + return this.#cachedBody("blob"); + } + /** + * Parses the request body as `FormData`. + * @example + * ```ts + * app.post('/entry', async (c) => { + * const body = await c.req.formData(); + * }); + * ``` + * @see https://hono.dev/docs/api/request#formdata + */ + formData() { + return this.#cachedBody("formData"); + } + /** + * Adds validated data to the request. + * + * @param target - The target of the validation. + * @param data - The validated data to add. + */ + addValidatedData(target, data) { + (this.#validatedData ??= {})[target] = data; + } + valid(target) { + return this.#validatedData?.[target]; + } + /** + * `.url` can get the request url strings. + * + * @see {@link https://hono.dev/docs/api/request#url} + * + * @example + * ```ts + * app.get('/about/me', (c) => { + * const url = c.req.url // `http://localhost:8787/about/me` + * ... + * }) + * ``` + */ + get url() { + return this.raw.url; + } + /** + * `.method` can get the method name of the request. + * + * @see {@link https://hono.dev/docs/api/request#method} + * + * @example + * ```ts + * app.get('/about/me', (c) => { + * const method = c.req.method // `GET` + * }) + * ``` + */ + get method() { + return this.raw.method; + } + get [GET_MATCH_RESULT]() { + return this.#matchResult; + } + /** + * `.matchedRoutes` can return a matched route in the handler + * + * @deprecated + * + * Use matchedRoutes helper defined in "hono/route" instead. + * + * @see {@link https://hono.dev/docs/api/request#matchedroutes} + * + * @example + * ```ts + * app.use('*', async function logger(c, next) { + * await next() + * c.req.matchedRoutes.forEach(({ handler, method, path }, i) => { + * const name = handler.name || (handler.length < 2 ? '[handler]' : '[middleware]') + * console.log( + * method, + * ' ', + * path, + * ' '.repeat(Math.max(10 - path.length, 0)), + * name, + * i === c.req.routeIndex ? '<- respond from here' : '' + * ) + * }) + * }) + * ``` + */ + get matchedRoutes() { + return this.#matchResult[0].map(([[, route]]) => route); + } + /** + * `.routePath` can retrieve the path registered within the handler + * + * @deprecated + * + * Use routePath helper defined in "hono/route" instead. + * + * @see {@link https://hono.dev/docs/api/request#routepath} + * + * @example + * ```ts + * app.get('/posts/:id', (c) => { + * return c.json({ path: c.req.routePath }) + * }) + * ``` + */ + get routePath() { + return this.#matchResult[0].map(([[, route]]) => route)[this.routeIndex].path; + } +}; +/** +* Clones a HonoRequest's underlying raw Request object. +* +* This utility handles both consumed and unconsumed request bodies: +* - If the request body hasn't been consumed, it uses the native `clone()` method +* - If the request body has been consumed, it reconstructs a new Request using cached body data +* +* This is particularly useful when you need to: +* - Process the same request body multiple times +* - Pass requests to external services after validation +* +* @param req - The HonoRequest object to clone +* @returns A Promise that resolves to a new Request object with the same properties +* @throws {HTTPException} If the request body was consumed directly via `req.raw` +* without using HonoRequest methods (e.g., `req.json()`, `req.text()`), making it +* impossible to reconstruct the body from cache +* +* @example +* ```ts +* // Clone after consuming the body (e.g., after validation) +* app.post('/forward', +* validator('json', (data) => data), +* async (c) => { +* const validated = c.req.valid('json') +* // Body has been consumed, but cloneRawRequest still works +* const clonedReq = await cloneRawRequest(c.req) +* return fetch('http://backend-service.com', clonedReq) +* } +* ) +* ``` +*/ +const cloneRawRequest = async (req) => { + if (!req.raw.bodyUsed) return req.raw.clone(); + const cacheKey = Object.keys(req.bodyCache)[0]; + if (!cacheKey) throw new HTTPException(500, { message: "Cannot clone request: body was already consumed and not cached. Please use HonoRequest methods (e.g., req.json(), req.text()) instead of consuming req.raw directly." }); + let body = await req[cacheKey](); + const headers = req.header(); + if (cacheKey === "json") { + body = JSON.stringify(body); + delete headers["content-length"]; + } else if (body instanceof FormData) { + delete headers["content-type"]; + delete headers["content-length"]; + } + const requestInit = { + body, + cache: req.raw.cache, + credentials: req.raw.credentials, + headers, + integrity: req.raw.integrity, + keepalive: req.raw.keepalive, + method: req.method, + mode: req.raw.mode, + redirect: req.raw.redirect, + referrer: req.raw.referrer, + referrerPolicy: req.raw.referrerPolicy, + signal: req.raw.signal + }; + return new Request(req.url, requestInit); +}; +//#endregion +export { HonoRequest, cloneRawRequest }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/request/constants.js b/tests/fixtures/npm/node_modules/hono/dist/request/constants.js new file mode 100644 index 000000000..0f2734437 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/request/constants.js @@ -0,0 +1,4 @@ +//#region src/request/constants.ts +const GET_MATCH_RESULT = Symbol(); +//#endregion +export { GET_MATCH_RESULT }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/router.js b/tests/fixtures/npm/node_modules/hono/dist/router.js new file mode 100644 index 000000000..52ea7f5c2 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/router.js @@ -0,0 +1,35 @@ +//#region src/router.ts +/** +* @module +* This module provides types definitions and variables for the routers. +*/ +/** +* Constant representing all HTTP methods in uppercase. +*/ +const METHOD_NAME_ALL = "ALL"; +/** +* Constant representing all HTTP methods in lowercase. +*/ +const METHOD_NAME_ALL_LOWERCASE = "all"; +/** +* Array of supported HTTP methods. +*/ +const METHODS = [ + "get", + "post", + "put", + "delete", + "options", + "patch", + "query" +]; +/** +* Error message indicating that a route cannot be added because the matcher is already built. +*/ +const MESSAGE_MATCHER_IS_ALREADY_BUILT = "Can not add a route since the matcher is already built."; +/** +* Error class representing an unsupported path error. +*/ +var UnsupportedPathError = class extends Error {}; +//#endregion +export { MESSAGE_MATCHER_IS_ALREADY_BUILT, METHODS, METHOD_NAME_ALL, METHOD_NAME_ALL_LOWERCASE, UnsupportedPathError }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/index.js b/tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/index.js new file mode 100644 index 000000000..38dfdd48a --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/index.js @@ -0,0 +1,3 @@ +import { RegExpRouter } from "./router.js"; +import { PreparedRegExpRouter, buildInitParams, serializeInitParams } from "./prepared-router.js"; +export { PreparedRegExpRouter, RegExpRouter, buildInitParams, serializeInitParams }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/matcher.js b/tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/matcher.js new file mode 100644 index 000000000..97ac8435c --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/matcher.js @@ -0,0 +1,19 @@ +import "../../router.js"; +//#region src/router/reg-exp-router/matcher.ts +const emptyParam = []; +function match(method, path) { + const matchers = this.buildAllMatchers(); + const match = ((method, path) => { + const matcher = matchers[method] || matchers["ALL"]; + const staticMatch = matcher[2][path]; + if (staticMatch) return staticMatch; + const match = path.match(matcher[0]); + if (!match) return [[], emptyParam]; + const index = match.indexOf("", 1); + return [matcher[1][index], match]; + }); + this.match = match; + return match(method, path); +} +//#endregion +export { emptyParam, match }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/node.js b/tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/node.js new file mode 100644 index 000000000..6738b9e98 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/node.js @@ -0,0 +1,91 @@ +import { createNullObject } from "../utils.js"; +//#region src/router/reg-exp-router/node.ts +const LABEL_REG_EXP_STR = "[^/]+"; +const ONLY_WILDCARD_REG_EXP_STR = ".*"; +const TAIL_WILDCARD_REG_EXP_STR = "(?:|/.*)"; +const PATH_ERROR = Symbol(); +const regExpMetaChars = /* @__PURE__ */ new Set(".\\+*[^]$()"); +/** +* Sort order: +* 1. literal +* 2. special pattern (e.g. :label{[0-9]+}) +* 3. common label pattern (e.g. :label) +* 4. wildcard +*/ +function compareKey(a, b) { + if (a.length === 1) return b.length === 1 ? a < b ? -1 : 1 : -1; + if (b.length === 1) return 1; + if (a === ".*" || a === "(?:|/.*)") return b === "(?:|/.*)" ? -1 : 1; + else if (b === ".*" || b === "(?:|/.*)") return -1; + if (a === "[^/]+") return 1; + else if (b === "[^/]+") return -1; + return a.length === b.length ? a < b ? -1 : 1 : b.length - a.length; +} +var Node = class Node { + #index; + #varIndex; + #children = createNullObject(); + insert(tokens, index, paramMap, context, isStatic) { + let node = this; + for (let i = 0, len = tokens.length; i < len; i++) { + const token = tokens[i]; + const pattern = token.length === 1 ? token === "*" ? i === len - 1 ? [ + "", + "", + ".*" + ] : [ + "", + "", + LABEL_REG_EXP_STR + ] : null : token === "/*" ? [ + "", + "", + TAIL_WILDCARD_REG_EXP_STR + ] : token.match(/^\:([^\{\}]+)(?:\{(.+)\})?$/); + let nextNode; + if (pattern) { + const name = pattern[1]; + let regexpStr = pattern[2] || "[^/]+"; + if (name && pattern[2]) { + if (regexpStr === ".*") throw PATH_ERROR; + regexpStr = regexpStr.replace(/^\((?!\?:)(?=[^)]+\)$)/, "(?:"); + if (/\((?!\?:)/.test(regexpStr)) throw PATH_ERROR; + if (regexpStr.length === 1 && regExpMetaChars.has(regexpStr)) throw PATH_ERROR; + } + nextNode = node.#children[regexpStr]; + if (!nextNode) { + if (regexpStr !== ".*" && regexpStr !== "(?:|/.*)") { + for (const k in node.#children) if ((regexpStr.length > 1 || k.length > 1) && k !== ".*" && k !== "(?:|/.*)") throw PATH_ERROR; + } + nextNode = node.#children[regexpStr] = new Node(); + } + if (name !== "") { + nextNode.#varIndex ??= context.varIndex++; + paramMap.push([name, nextNode.#varIndex]); + } + } else { + nextNode = node.#children[token]; + if (!nextNode) { + for (const k in node.#children) if (k.length > 1 && k !== ".*" && k !== "(?:|/.*)") throw PATH_ERROR; + nextNode = node.#children[token] = new Node(); + } + } + node = nextNode; + } + if (node.#index !== void 0) throw PATH_ERROR; + node.#index = isStatic ? -1 : index; + } + buildRegExpStr() { + const strList = Object.keys(this.#children).sort(compareKey).map((k) => { + const c = this.#children[k]; + const childStr = c.buildRegExpStr(); + return childStr === "" ? "" : (typeof c.#varIndex === "number" ? `(${k})@${c.#varIndex}` : regExpMetaChars.has(k) ? `\\${k}` : k) + childStr; + }).filter(Boolean); + if (typeof this.#index === "number" && this.#index !== -1) strList.unshift(`#${this.#index}`); + if (strList.length === 0) return ""; + if (strList.length === 1) return strList[0]; + return "(?:" + strList.join("|") + ")"; + } +}; +//#endregion +export { LABEL_REG_EXP_STR, Node, ONLY_WILDCARD_REG_EXP_STR, PATH_ERROR, TAIL_WILDCARD_REG_EXP_STR }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/prepared-router.js b/tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/prepared-router.js new file mode 100644 index 000000000..c9a8c1314 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/prepared-router.js @@ -0,0 +1,94 @@ +import "../../router.js"; +import { emptyParam, match } from "./matcher.js"; +import { RegExpRouter } from "./router.js"; +//#region src/router/reg-exp-router/prepared-router.ts +var PreparedRegExpRouter = class { + name = "PreparedRegExpRouter"; + #matchers; + #relocateMap; + constructor(matchers, relocateMap) { + this.#matchers = matchers; + this.#relocateMap = relocateMap; + } + #addWildcard(method, handlerData) { + const matcher = this.#matchers[method]; + matcher[1].forEach((list) => list && list.push(handlerData)); + Object.values(matcher[2]).forEach((list) => list[0].push(handlerData)); + } + #addPath(method, path, handler, indexes, map) { + const matcher = this.#matchers[method]; + if (!map) matcher[2][path][0].push([handler, {}]); + else indexes.forEach((index) => { + if (typeof index === "number") matcher[1][index].push([handler, map]); + else matcher[2][index || path][0].push([handler, map]); + }); + } + add(method, path, handler) { + if (!this.#matchers[method]) { + const all = this.#matchers["ALL"]; + const staticMap = {}; + for (const key in all[2]) staticMap[key] = [all[2][key][0].slice(), emptyParam]; + this.#matchers[method] = [ + all[0], + all[1].map((list) => Array.isArray(list) ? list.slice() : 0), + staticMap + ]; + } + if (path === "/*" || path === "*") { + const handlerData = [handler, {}]; + if (method === "ALL") for (const m in this.#matchers) this.#addWildcard(m, handlerData); + else this.#addWildcard(method, handlerData); + return; + } + const data = this.#relocateMap[path]; + if (!data) throw new Error(`Path ${path} is not registered`); + for (const [indexes, map] of data) if (method === "ALL") for (const m in this.#matchers) this.#addPath(m, path, handler, indexes, map); + else this.#addPath(method, path, handler, indexes, map); + } + buildAllMatchers() { + return this.#matchers; + } + match = match; +}; +const buildInitParams = ({ paths }) => { + const RegExpRouterWithMatcherExport = class extends RegExpRouter { + buildAndExportAllMatchers() { + return this.buildAllMatchers(); + } + }; + const router = new RegExpRouterWithMatcherExport(); + for (const path of paths) router.add("ALL", path, path); + const matchers = router.buildAndExportAllMatchers(); + const all = matchers["ALL"]; + const relocateMap = {}; + for (const path of paths) { + if (path === "/*" || path === "*") continue; + all[1].forEach((list, i) => { + list.forEach(([p, map]) => { + if (p === path) { + if (relocateMap[path]) relocateMap[path][0][1] = { + ...relocateMap[path][0][1], + ...map + }; + else relocateMap[path] = [[[], map]]; + if (relocateMap[path][0][0].findIndex((j) => j === i) === -1) relocateMap[path][0][0].push(i); + } + }); + }); + for (const path2 in all[2]) all[2][path2][0].forEach(([p]) => { + if (p === path) { + relocateMap[path] ||= [[[]]]; + const value = path2 === path ? "" : path2; + if (relocateMap[path][0][0].findIndex((v) => v === value) === -1) relocateMap[path][0][0].push(value); + } + }); + } + for (let i = 0, len = all[1].length; i < len; i++) all[1][i] = all[1][i] ? [] : 0; + for (const path in all[2]) all[2][path][0] = []; + return [matchers, relocateMap]; +}; +const serializeInitParams = ([matchers, relocateMap]) => { + return `[${JSON.stringify(matchers, (_, value) => value instanceof RegExp ? `##${value.toString()}##` : value).replace(/"##(.+?)##"/g, (_, str) => str.replace(/\\\\/g, "\\"))},${JSON.stringify(relocateMap)}]`; +}; +//#endregion +export { PreparedRegExpRouter, buildInitParams, serializeInitParams }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/router.js b/tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/router.js new file mode 100644 index 000000000..21ef235d1 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/router.js @@ -0,0 +1,101 @@ +import { checkOptionalParameter } from "../../utils/url.js"; +import { MESSAGE_MATCHER_IS_ALREADY_BUILT, UnsupportedPathError } from "../../router.js"; +import { createNullObject } from "../utils.js"; +import { emptyParam, match } from "./matcher.js"; +import { LABEL_REG_EXP_STR, PATH_ERROR, TAIL_WILDCARD_REG_EXP_STR } from "./node.js"; +import { Trie } from "./trie.js"; +//#region src/router/reg-exp-router/router.ts +let wildcardRegExpCache = createNullObject(); +function buildWildcardRegExp(path) { + return wildcardRegExpCache[path] ??= new RegExp(`^${path.replace(/\/:[^/{}]+(?:\{\[\^\/]\+})?(?=[/{]|$)|\/?\*$|([.\\+*[^\]$()?{}|])/g, (match, metaChar) => metaChar ? `\\${metaChar}` : match === "/*" ? TAIL_WILDCARD_REG_EXP_STR : match === "*" ? ".*" : `/:${LABEL_REG_EXP_STR}`)}$`); +} +function findMiddleware(middleware, path) { + for (const k of Object.keys(middleware).sort((a, b) => b.length - a.length)) if (buildWildcardRegExp(k).test(path)) return [...middleware[k]]; +} +var RegExpRouter = class { + name = "RegExpRouter"; + #middleware; + #routes; + #tries; + constructor() { + this.#middleware = { ["ALL"]: createNullObject() }; + this.#routes = { ["ALL"]: createNullObject() }; + this.#tries = { ["ALL"]: new Trie() }; + } + #insertPath(method, path) { + try { + this.#tries[method].insert(path, !/\*|\/:/.test(path)); + } catch (e) { + throw e === PATH_ERROR ? new UnsupportedPathError(path) : e; + } + } + add(method, path, handler) { + const middleware = this.#middleware; + const routes = this.#routes; + if (!middleware) throw new Error(MESSAGE_MATCHER_IS_ALREADY_BUILT); + if (!middleware[method]) { + this.#tries[method] = new Trie(); + for (const handlerMap of [middleware, routes]) { + handlerMap[method] = createNullObject(); + for (const p in handlerMap["ALL"]) { + handlerMap[method][p] = [...handlerMap["ALL"][p]]; + this.#insertPath(method, p); + } + } + } + if (path === "/*") path = "*"; + const methods = method === "ALL" ? Object.keys(middleware) : [method]; + if (/\*$/.test(path)) { + const re = buildWildcardRegExp(path); + for (const m of methods) if (!middleware[m][path]) { + this.#insertPath(m, path); + middleware[m][path] = findMiddleware(middleware[m], path) || findMiddleware(middleware["ALL"], path) || []; + } + for (const handlerMap of [middleware, routes]) for (const m of methods) for (const p in handlerMap[m]) re.test(p) && handlerMap[m][p].push([handler, path]); + return; + } + const paths = checkOptionalParameter(path) || [path]; + for (const path of paths) for (const m of methods) { + if (!routes[m][path]) { + this.#insertPath(m, path); + routes[m][path] = findMiddleware(middleware[m], path) || findMiddleware(middleware["ALL"], path) || []; + } + routes[m][path].push([handler, path]); + } + } + match = match; + buildAllMatchers() { + const matchers = createNullObject(); + for (const method of Object.keys(this.#routes)) matchers[method] = this.#buildMatcher(method); + this.#middleware = this.#routes = this.#tries = void 0; + wildcardRegExpCache = createNullObject(); + return matchers; + } + #buildMatcher(method) { + const middleware = this.#middleware[method]; + const routes = this.#routes[method]; + const trie = this.#tries[method]; + const staticMap = createNullObject(); + const handlerData = []; + const [regexp, indexReplacementMap, paramReplacementMap] = trie.buildRegExp(); + for (const r of [middleware, routes]) for (const path in r) { + const handlers = r[path]; + const pathData = trie.paths[path]; + if (!pathData) { + staticMap[path] = [handlers.map(([h]) => [h, createNullObject()]), emptyParam]; + continue; + } + handlerData[pathData[0]] = handlers.map(([h, handlerPath]) => [h, trie.paths[handlerPath][1].reduceRight((map, [key], i) => { + map[key] = paramReplacementMap[pathData[1][i][1]]; + return map; + }, createNullObject())]); + } + return [ + regexp, + indexReplacementMap.map((i) => handlerData[i]), + staticMap + ]; + } +}; +//#endregion +export { RegExpRouter }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/trie.js b/tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/trie.js new file mode 100644 index 000000000..a995678c0 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/router/reg-exp-router/trie.js @@ -0,0 +1,73 @@ +import { createNullObject } from "../utils.js"; +import { Node } from "./node.js"; +//#region src/router/reg-exp-router/trie.ts +var Trie = class { + #context = { varIndex: 0 }; + #root = new Node(); + #index = 0; + paths = createNullObject(); + insert(path, isStatic) { + if (isStatic) { + this.#root.insert(path.split(""), 0, [], this.#context, true); + return; + } + const paramAssoc = []; + const groups = []; + let markedPath = path; + for (let i = 0;;) { + let replaced = false; + markedPath = markedPath.replace(/\{[^}]+\}/g, (m) => { + const mark = `@\\${i}`; + groups[i] = [mark, m]; + i++; + replaced = true; + return mark; + }); + if (!replaced) break; + } + /** + * - pattern (:label, :label{0-9]+}, ...) + * - /* wildcard + * - character + */ + const tokens = markedPath.match(/(?::[^\/]+)|(?:\/\*$)|./g) || []; + for (let i = groups.length - 1; i >= 0; i--) { + const [mark] = groups[i]; + for (let j = tokens.length - 1; j >= 0; j--) if (tokens[j].indexOf(mark) !== -1) { + tokens[j] = tokens[j].replace(mark, groups[i][1]); + break; + } + } + this.#root.insert(tokens, this.#index, paramAssoc, this.#context, false); + this.paths[path] = [this.#index++, paramAssoc]; + } + buildRegExp() { + let regexp = this.#root.buildRegExpStr(); + if (regexp === "") return [ + /^$/, + [], + [] + ]; + let captureIndex = 0; + const indexReplacementMap = []; + const paramReplacementMap = []; + regexp = regexp.replace(/#(\d+)|@(\d+)|\.\*\$/g, (_, handlerIndex, paramIndex) => { + if (handlerIndex !== void 0) { + indexReplacementMap[++captureIndex] = Number(handlerIndex); + return "$()"; + } + if (paramIndex !== void 0) { + paramReplacementMap[Number(paramIndex)] = ++captureIndex; + return ""; + } + return ""; + }); + return [ + new RegExp(`^${regexp}`), + indexReplacementMap, + paramReplacementMap + ]; + } +}; +//#endregion +export { Trie }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/router/smart-router/router.js b/tests/fixtures/npm/node_modules/hono/dist/router/smart-router/router.js new file mode 100644 index 000000000..a115ab938 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/router/smart-router/router.js @@ -0,0 +1,49 @@ +import { MESSAGE_MATCHER_IS_ALREADY_BUILT, UnsupportedPathError } from "../../router.js"; +//#region src/router/smart-router/router.ts +var SmartRouter = class { + name = "SmartRouter"; + #routers = []; + #routes = []; + constructor(init) { + this.#routers = init.routers; + } + add(method, path, handler) { + if (!this.#routes) throw new Error(MESSAGE_MATCHER_IS_ALREADY_BUILT); + this.#routes.push([ + method, + path, + handler + ]); + } + match(method, path) { + if (!this.#routes) throw new Error("Fatal error"); + const routers = this.#routers; + const routes = this.#routes; + const len = routers.length; + let i = 0; + let res; + for (; i < len; i++) { + const router = routers[i]; + try { + for (let i = 0, len = routes.length; i < len; i++) router.add(...routes[i]); + res = router.match(method, path); + } catch (e) { + if (e instanceof UnsupportedPathError) continue; + throw e; + } + this.match = router.match.bind(router); + this.#routers = [router]; + this.#routes = void 0; + break; + } + if (i === len) throw new Error("Fatal error"); + this.name = `SmartRouter + ${this.activeRouter.name}`; + return res; + } + get activeRouter() { + if (this.#routes || this.#routers.length !== 1) throw new Error("No active router has been determined yet."); + return this.#routers[0]; + } +}; +//#endregion +export { SmartRouter }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/router/trie-router/index.js b/tests/fixtures/npm/node_modules/hono/dist/router/trie-router/index.js new file mode 100644 index 000000000..8ea5b3150 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/router/trie-router/index.js @@ -0,0 +1,2 @@ +import { TrieRouter } from "./router.js"; +export { TrieRouter }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/router/trie-router/node.js b/tests/fixtures/npm/node_modules/hono/dist/router/trie-router/node.js new file mode 100644 index 000000000..c33c7af1e --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/router/trie-router/node.js @@ -0,0 +1,134 @@ +import { getPattern, splitPath, splitRoutingPath } from "../../utils/url.js"; +import "../../router.js"; +import { createNullObject } from "../utils.js"; +//#region src/router/trie-router/node.ts +const emptyParams = createNullObject(); +let order = 0; +var Node = class Node { + #methods = []; + #children = createNullObject(); + #patterns = []; + #pattern; + #params = emptyParams; + insert(method, path, handler) { + let curNode = this; + const parts = splitRoutingPath(path); + const possibleKeys = /* @__PURE__ */ new Set(); + let i = 0; + for (const p of parts) { + const nextP = parts[++i]; + const pattern = getPattern(p, nextP) || (nextP === void 0 && p && p.indexOf("*") === p.length - 1 ? p : null); + const isParam = Array.isArray(pattern); + const key = isParam ? pattern[0] : pattern || p; + const child = curNode.#children[key] ||= new Node(); + if (pattern && !child.#pattern) { + child.#pattern = pattern; + curNode.#patterns.push(child); + } + curNode = child; + if (isParam) possibleKeys.add(pattern[1]); + } + curNode.#methods.push({ [method]: { + handler, + possibleKeys: [...possibleKeys], + score: ++order + } }); + } + #pushHandlerSets(handlerSets, node, method, nodeParams, params) { + for (let i = 0, len = node.#methods.length; i < len; i++) { + const m = node.#methods[i]; + const handlerSet = m[method] || m["ALL"]; + if (handlerSet) { + handlerSet.params = createNullObject(); + handlerSets.push(handlerSet); + for (let i = 0, len = handlerSet.possibleKeys.length; i < len; i++) { + const key = handlerSet.possibleKeys[i]; + handlerSet.params[key] = params?.[key] && !i ? params[key] : nodeParams[key] ?? params?.[key]; + } + } + } + } + search(method, path) { + const handlerSets = []; + this.#params = emptyParams; + let curNodes = [this]; + const parts = splitPath(path); + const curNodesQueue = []; + const len = parts.length; + let partOffsets = null; + for (let i = 0; i < len; i++) { + const part = parts[i]; + const isLast = i === len - 1; + const tempNodes = []; + for (let j = 0, len2 = curNodes.length; j < len2; j++) { + const node = curNodes[j]; + const nextNode = node.#children[part]; + if (nextNode) { + nextNode.#params = node.#params; + if (isLast) { + if (nextNode.#children["*"]) this.#pushHandlerSets(handlerSets, nextNode.#children["*"], method, node.#params); + this.#pushHandlerSets(handlerSets, nextNode, method, node.#params); + } else tempNodes.push(nextNode); + } + for (const child of node.#patterns) { + const pattern = child.#pattern; + const params = node.#params === emptyParams ? {} : { ...node.#params }; + if (typeof pattern === "string") { + if (pattern === "*" || part.startsWith(pattern.slice(0, -1))) { + this.#pushHandlerSets(handlerSets, child, method, node.#params); + if (pattern === "*") { + child.#params = params; + tempNodes.push(child); + } + } + continue; + } + const [, name, matcher] = pattern; + if (!part && matcher === true) continue; + if (matcher !== true) { + if (!partOffsets) { + partOffsets = []; + let offset = path[0] === "/" ? 1 : 0; + for (let p = 0; p < len; p++) { + partOffsets[p] = offset; + offset += parts[p].length + 1; + } + } + const restPathString = path.slice(partOffsets[i]); + const m = matcher.exec(restPathString); + if (m) { + params[name] = m[0]; + this.#pushHandlerSets(handlerSets, child, method, node.#params, params); + if (m[0].length === restPathString.length && child.#children["*"]) this.#pushHandlerSets(handlerSets, child.#children["*"], method, node.#params, params); + for (const _ in child.#children) { + child.#params = params; + const componentCount = m[0].match(/\//g)?.length ?? 0; + (curNodesQueue[componentCount] ||= []).push(child); + break; + } + continue; + } + } + if (matcher === true || matcher.test(part)) { + params[name] = part; + if (isLast) { + this.#pushHandlerSets(handlerSets, child, method, params, node.#params); + if (child.#children["*"]) this.#pushHandlerSets(handlerSets, child.#children["*"], method, params, node.#params); + } else { + child.#params = params; + tempNodes.push(child); + } + } + } + } + const shifted = curNodesQueue.shift(); + curNodes = shifted ? tempNodes.concat(shifted) : tempNodes; + } + if (handlerSets[1]) handlerSets.sort((a, b) => { + return a.score - b.score; + }); + return [handlerSets.map(({ handler, params }) => [handler, params])]; + } +}; +//#endregion +export { Node }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/router/trie-router/router.js b/tests/fixtures/npm/node_modules/hono/dist/router/trie-router/router.js new file mode 100644 index 000000000..96f4c9351 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/router/trie-router/router.js @@ -0,0 +1,15 @@ +import { checkOptionalParameter } from "../../utils/url.js"; +import { Node } from "./node.js"; +//#region src/router/trie-router/router.ts +var TrieRouter = class { + name = "TrieRouter"; + #node = new Node(); + add(method, path, handler) { + for (const result of checkOptionalParameter(path) || [path]) this.#node.insert(method, result, handler); + } + match(method, path) { + return this.#node.search(method, path); + } +}; +//#endregion +export { TrieRouter }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/router/utils.js b/tests/fixtures/npm/node_modules/hono/dist/router/utils.js new file mode 100644 index 000000000..3f1e9d328 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/router/utils.js @@ -0,0 +1,4 @@ +//#region src/router/utils.ts +const createNullObject = () => Object.create(null); +//#endregion +export { createNullObject }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/types/client/types.d.ts b/tests/fixtures/npm/node_modules/hono/dist/types/client/types.d.ts new file mode 100644 index 000000000..81dc47313 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/types/client/types.d.ts @@ -0,0 +1,204 @@ +import { METHODS, METHOD_NAME_ALL_LOWERCASE } from "../router.js"; +import { HonoBase as Hono } from "../hono-base.js"; +import { StatusCode, SuccessStatusCode } from "../utils/http-status.js"; +import { HasRequiredKeys } from "../utils/types.js"; +import { Endpoint, ExtractSchema, KnownResponseFormat, ResponseFormat, Schema } from "../types.js"; +import { Hono as Hono$1 } from "../hono.js"; +//#region src/client/types.d.ts +/** + * Type representing the '$all' method name + */ +type MethodNameAll = `$${typeof METHOD_NAME_ALL_LOWERCASE}`; +/** + * Type representing all standard HTTP methods prefixed with '$' + * e.g., '$get' | '$post' | '$put' | '$delete' | '$options' | '$patch' | '$query' + */ +type StandardMethods = `$${(typeof METHODS)[number]}`; +/** + * Expands '$all' into all standard HTTP methods. + * If the schema contains '$all', it creates a type where all standard HTTP methods + * point to the same endpoint definition as '$all', while removing '$all' itself. + */ +type ExpandAllMethod = MethodNameAll extends keyof S ? { [M in StandardMethods]: S[MethodNameAll]; } & Omit : S; +type HonoRequest = (typeof Hono$1.prototype)['request']; +export type BuildSearchParamsFn = (query: Record) => URLSearchParams; +export type ClientRequestOptions = { + fetch?: typeof fetch | HonoRequest; + webSocket?: (...args: ConstructorParameters) => WebSocket; + /** + * Standard `RequestInit`, caution that this take highest priority + * and could be used to overwrite things that Hono sets for you, like `body | method | headers`. + * + * If you want to add some headers, use in `headers` instead of `init` + */ + init?: RequestInit; + /** + * Custom function to serialize query parameters into URLSearchParams. + * By default, arrays are serialized as multiple parameters with the same key (e.g., `key=a&key=b`). + * You can provide a custom function to change this behavior, for example to use bracket notation (e.g., `key[]=a&key[]=b`). + * + * @example + * ```ts + * const client = hc('http://localhost', { + * buildSearchParams: (query) => { + * return new URLSearchParams(qs.stringify(query)) + * } + * }) + * ``` + */ + buildSearchParams?: BuildSearchParamsFn; +} & (keyof T extends never ? { + headers?: Record | (() => Record | Promise>); +} : { + headers: T | (() => T | Promise); +}); +export type ClientRequest = { [M in keyof ExpandAllMethod]: ExpandAllMethod[M] extends Endpoint & { + input: infer R; +} ? R extends object ? HasRequiredKeys extends true ? (args: R, options?: ClientRequestOptions) => Promise[M]>> : (args?: R, options?: ClientRequestOptions) => Promise[M]>> : never : never; } & { + $url: (arg?: Arg) => HonoURL; + $path: (arg?: Arg) => BuildPath; +} & (S['$get'] extends { + outputFormat: 'ws'; +} ? S['$get'] extends { + input: infer I; +} ? { + $ws: (args?: I) => WebSocket; +} : {} : {}); +type ClientResponseOfEndpoint = T extends { + output: infer O; + outputFormat: infer F; + status: infer S; +} ? ClientResponse : never; +export interface ClientResponse { + readonly body: ReadableStream | null; + readonly bodyUsed: boolean; + ok: U extends SuccessStatusCode ? true : U extends Exclude ? false : boolean; + redirected: boolean; + status: U; + statusText: string; + type: 'basic' | 'cors' | 'default' | 'error' | 'opaque' | 'opaqueredirect'; + headers: Headers; + url: string; + redirect(url: string, status: number): Response; + clone(): Response; + bytes(): Promise>; + json(): F extends 'text' ? Promise : F extends 'json' ? Promise : Promise; + text(): F extends 'text' ? (T extends string ? Promise : Promise) : Promise; + blob(): Promise; + formData(): Promise; + arrayBuffer(): Promise; +} +type BuildSearch = Arg extends { [K in Key]: infer Query; } ? IsEmptyObject extends true ? '' : `?${string}` : ''; +type BuildPathname

= Arg extends { + param: infer Param; +} ? `${ApplyParam, Param>}` : `/${TrimStartSlash

}`; +type BuildPath

= `${BuildPathname}${BuildSearch}`; +type BuildTypedURL = TypedURL<`${Protocol}:`, Host, Port, BuildPathname, BuildSearch>; +type HonoURL = IsLiteral extends true ? TrimEndSlash extends `${infer Protocol}://${infer Rest}` ? Rest extends `${infer Hostname}/${infer P}` ? ParseHostName extends [infer Host extends string, infer Port extends string] ? BuildTypedURL : never : ParseHostName extends [infer Host extends string, infer Port extends string] ? BuildTypedURL : never : URL : URL; +type ParseHostName = T extends `${infer Host}:${infer Port}` ? [Host, Port] : [T, '']; +type TrimStartSlash = T extends `/${infer R}` ? TrimStartSlash : T; +type TrimEndSlash = T extends `${infer R}/` ? TrimEndSlash : T; +type IsLiteral = [T] extends [never] ? false : string extends T ? false : true; +type ApplyParam = Path extends `${infer Head}/${infer Rest}` ? Head extends `:${infer Param}` ? P extends Record ? IsLiteral extends true ? ApplyParam : ApplyParam : ApplyParam : ApplyParam : Path extends `:${infer Param}` ? P extends Record ? IsLiteral extends true ? `${Result}/${Value & string}` : `${Result}/${Path}` : `${Result}/${Path}` : `${Result}/${Path}`; +type IsEmptyObject = keyof T extends never ? true : false; +export interface TypedURL extends URL { + protocol: Protocol; + hostname: Hostname; + port: Port; + host: Port extends '' ? Hostname : `${Hostname}:${Port}`; + origin: `${Protocol}//${Hostname}${Port extends '' ? '' : `:${Port}`}`; + pathname: Pathname; + search: Search; + href: `${Protocol}//${Hostname}${Port extends '' ? '' : `:${Port}`}${Pathname}${Search}`; +} +export interface Response extends ClientResponse {} +export type Fetch = (args?: InferRequestType, opt?: ClientRequestOptions) => Promise>>; +type InferEndpointType = T extends ((args: infer R, options: any | undefined) => Promise) ? U extends ClientResponse ? { + input: NonNullable; + output: O; + outputFormat: F; + status: S; +} extends Endpoint ? { + input: NonNullable; + output: O; + outputFormat: F; + status: S; +} : never : never : never; +export type InferResponseType = InferResponseTypeFromEndpoint, U>; +type InferResponseTypeFromEndpoint = T extends { + output: infer O; + status: infer S; +} ? S extends U ? O : never : never; +export type InferRequestType = T extends ((args: infer R, options: any | undefined) => Promise>) ? NonNullable : never; +export type InferRequestOptionsType = T extends ((args: any, options: infer R) => Promise>) ? NonNullable : never; +/** + * Filter a ClientResponse type so it only includes responses of specific status codes. + */ +export type FilterClientResponseByStatusCode, U extends number = StatusCode> = T extends ClientResponse ? RC extends U ? ClientResponse : never : never; +type PathToChain = Path extends `/${infer P}` ? PathToChain : Path extends `${infer P}/${infer R}` ? { [K in P]: PathToChain; } : { [K in Path extends '' ? 'index' : Path]: ClientRequest ? E[Original] : never>; }; +export type Client = T extends Hono ? S extends Record ? K extends string ? PathToChain : never : never : never; +export type Callback = (opts: CallbackOptions) => unknown; +interface CallbackOptions { + path: string[]; + args: any[]; +} +export type ObjectType = { + [key: string]: T; +}; +type GlobalResponseDefinition = { [S in StatusCode]?: { [F in KnownResponseFormat]?: unknown; }; }; +type ToEndpoints = { [S in keyof Def & StatusCode]: { [F in keyof Def[S] & KnownResponseFormat]: Omit & { + output: Def[S][F]; + status: S; + outputFormat: F; +}; }[keyof Def[S] & KnownResponseFormat]; }[keyof Def & StatusCode]; +type ModRoute = R extends Endpoint ? R | ToEndpoints : R; +type ModSchema = { [K in keyof D]: { [M in keyof D[K]]: ModRoute; }; }; +export type ApplyGlobalResponse = App extends Hono ? ModSchema, Def> extends (infer S extends Schema) ? Hono$1 : never : never; +type PickRoute = R extends Endpoint ? R extends { + status: U; +} ? R : never : R; +type PickSchema = { [K in keyof D]: { [M in keyof D[K]]: PickRoute; }; }; +/** + * Keep only specific status code responses from all routes of an app. + * Useful when error responses are handled centrally (e.g., via custom fetch) + * and you want the client to only expose success response types. + * + * @example + * ```ts + * type AppSuccessOnly = PickResponseByStatusCode + * const client = hc('http://localhost') + * ``` + */ +export type PickResponseByStatusCode = App extends Hono ? PickSchema, U> extends (infer S extends Schema) ? Hono$1 : never : never; +//#endregion +export {}; diff --git a/tests/fixtures/npm/node_modules/hono/dist/types/context.d.ts b/tests/fixtures/npm/node_modules/hono/dist/types/context.d.ts new file mode 100644 index 000000000..163c86b8c --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/types/context.d.ts @@ -0,0 +1,459 @@ +import { Result } from "./router.js"; +import { ResponseHeader } from "./utils/headers.js"; +import { ContentfulStatusCode, RedirectStatusCode, StatusCode } from "./utils/http-status.js"; +import { InvalidJSONValue, IsAny, JSONParsed, JSONValue } from "./utils/types.js"; +import { Env, FetchEventLike, H, Input, NotFoundHandler, RouterRoute, TypedResponse } from "./types.js"; +import { HonoRequest } from "./request.js"; +import { BaseMime } from "./utils/mime.js"; +//#region src/context.d.ts +type HeaderRecord = Record<'Content-Type', BaseMime> | Record | Record; +/** + * Data type can be a string, ArrayBuffer, Blob, Uint8Array (buffer), or ReadableStream. + */ +export type Data = string | ArrayBuffer | Blob | ReadableStream | Uint8Array; +/** + * Interface for the execution context in a web worker or similar environment. + */ +export interface ExecutionContext { + /** + * Extends the lifetime of the event callback until the promise is settled. + * + * @param promise - A promise to wait for. + */ + waitUntil(promise: Promise): void; + /** + * Allows the event to be passed through to subsequent event listeners. + */ + passThroughOnException(): void; + /** + * For compatibility with Wrangler 4.x. + */ + props: any; + /** + * For compatibility with Wrangler 4.x. + */ + exports?: any; +} +/** + * Interface for context variable mapping. + */ +export interface ContextVariableMap {} +/** + * Interface for context renderer. + */ +export interface ContextRenderer {} +/** + * Interface representing a renderer for content. + * + * @interface DefaultRenderer + * @param {string | Promise} content - The content to be rendered, which can be either a string or a Promise resolving to a string. + * @returns {Response | Promise} - The response after rendering the content, which can be either a Response or a Promise resolving to a Response. + */ +interface DefaultRenderer { + (content: string | Promise): Response | Promise; +} +/** + * Renderer type which can either be a ContextRenderer or DefaultRenderer. + */ +export type Renderer = ContextRenderer extends Function ? ContextRenderer : DefaultRenderer; +/** + * Extracts the props for the renderer. + */ +export type PropsForRenderer = [...Required>] extends [unknown, infer Props] ? Props : unknown; +export type Layout> = (props: T) => any; +/** + * Interface for getting context variables. + * + * @template E - Environment type. + */ +interface Get { + (key: Key): E['Variables'][Key]; + (key: Key): ContextVariableMap[Key]; +} +/** + * Interface for setting context variables. + * + * @template E - Environment type. + */ +interface Set { + (key: Key, value: E['Variables'][Key]): void; + (key: Key, value: ContextVariableMap[Key]): void; +} +/** + * Interface for creating a new response. + */ +interface NewResponse { + (data: Data | null, status?: StatusCode, headers?: HeaderRecord): Response; + (data: Data | null, init?: ResponseOrInit): Response; +} +/** + * Interface for responding with a body. + */ +interface BodyRespond { + (data: T, status?: U, headers?: HeaderRecord): Response & TypedResponse; + (data: T, init?: ResponseOrInit): Response & TypedResponse; + (data: T, status?: U, headers?: HeaderRecord): Response & TypedResponse; + (data: T, init?: ResponseOrInit): Response & TypedResponse; +} +/** + * Interface for responding with text. + * + * @interface TextRespond + * @template T - The type of the text content. + * @template U - The type of the status code. + * + * @param {T} text - The text content to be included in the response. + * @param {U} [status] - An optional status code for the response. + * @param {HeaderRecord} [headers] - An optional record of headers to include in the response. + * + * @returns {Response & TypedResponse} - The response after rendering the text content, typed with the provided text and status code types. + */ +interface TextRespond { + (text: T, status?: U, headers?: HeaderRecord): Response & TypedResponse; + (text: T, init?: ResponseOrInit): Response & TypedResponse; +} +/** + * Interface for responding with JSON. + * + * @interface JSONRespond + * @template T - The type of the JSON value or simplified unknown type. + * @template U - The type of the status code. + * + * @param {T} object - The JSON object to be included in the response. + * @param {U} [status] - An optional status code for the response. + * @param {HeaderRecord} [headers] - An optional record of headers to include in the response. + * + * @returns {JSONRespondReturn} - The response after rendering the JSON object, typed with the provided object and status code types. + */ +interface JSONRespond { + (object: T, status?: U, headers?: HeaderRecord): JSONRespondReturn; + (object: T, init?: ResponseOrInit): JSONRespondReturn; +} +/** + * @template T - The type of the JSON value or simplified unknown type. + * @template U - The type of the status code. + * + * @returns {Response & TypedResponse, U, 'json'>} - The response after rendering the JSON object, typed with the provided object and status code types. + */ +type JSONRespondReturn = Response & TypedResponse, U, 'json'>; +/** + * Interface representing a function that responds with HTML content. + * + * @param html - The HTML content to respond with, which can be a string or a Promise that resolves to a string. + * @param status - (Optional) The HTTP status code for the response. + * @param headers - (Optional) A record of headers to include in the response. + * @param init - (Optional) The response initialization object. + * + * @returns A Response object or a Promise that resolves to a Response object. + */ +interface HTMLRespond { + >(html: T, status?: ContentfulStatusCode, headers?: HeaderRecord): T extends string ? Response : Promise; + >(html: T, init?: ResponseOrInit): T extends string ? Response : Promise; +} +/** + * Options for configuring the context. + * + * @template E - Environment type. + */ +type ContextOptions = { + /** + * Bindings for the environment. + */ + env: E['Bindings']; + /** + * Execution context for the request. + */ + executionCtx?: FetchEventLike | ExecutionContext | undefined; + /** + * Handler for not found responses. + */ + notFoundHandler?: NotFoundHandler; + matchResult?: Result<[H, RouterRoute]>; + path?: string; +}; +interface SetHeadersOptions { + append?: boolean; +} +interface SetHeaders { + (name: 'Content-Type', value?: BaseMime, options?: SetHeadersOptions): void; + (name: ResponseHeader, value?: string, options?: SetHeadersOptions): void; + (name: string, value?: string, options?: SetHeadersOptions): void; +} +type ResponseHeadersInit = [string, string][] | Record<'Content-Type', BaseMime> | Record | Record | Headers; +interface ResponseInit { + headers?: ResponseHeadersInit; + status?: T; + statusText?: string; +} +type ResponseOrInit = ResponseInit | Response; +export declare const TEXT_PLAIN = "text/plain; charset=UTF-8"; +export declare class Context { + + /** + * `.env` can get bindings (environment variables, secrets, KV namespaces, D1 database, R2 bucket etc.) in Cloudflare Workers. + * + * @see {@link https://hono.dev/docs/api/context#env} + * + * @example + * ```ts + * // Environment object for Cloudflare Workers + * app.get('*', async c => { + * const counter = c.env.COUNTER + * }) + * ``` + */ + env: E['Bindings']; + finalized: boolean; + /** + * `.error` can get the error object from the middleware if the Handler throws an error. + * + * @see {@link https://hono.dev/docs/api/context#error} + * + * @example + * ```ts + * app.use('*', async (c, next) => { + * await next() + * if (c.error) { + * // do something... + * } + * }) + * ``` + */ + error: Error | undefined; + /** + * Creates an instance of the Context class. + * + * @param req - The Request object. + * @param options - Optional configuration options for the context. + */ + constructor(req: Request, options?: ContextOptions); + /** + * `.req` is the instance of {@link HonoRequest}. + */ + get req(): HonoRequest; + /** + * @see {@link https://hono.dev/docs/api/context#event} + * The FetchEvent associated with the current request. + * + * @throws Will throw an error if the context does not have a FetchEvent. + */ + get event(): FetchEventLike; + /** + * @see {@link https://hono.dev/docs/api/context#executionctx} + * The ExecutionContext associated with the current request. + * + * @throws Will throw an error if the context does not have an ExecutionContext. + */ + get executionCtx(): ExecutionContext; + /** + * @see {@link https://hono.dev/docs/api/context#res} + * The Response object for the current request. + */ + get res(): Response; + /** + * Sets the Response object for the current request. + * + * @param _res - The Response object to set. + */ + set res(_res: Response | undefined); + /** + * `.render()` can create a response within a layout. + * + * @see {@link https://hono.dev/docs/api/context#render-setrenderer} + * + * @example + * ```ts + * app.get('/', (c) => { + * return c.render('Hello!') + * }) + * ``` + */ + render: Renderer; + /** + * Sets the layout for the response. + * + * @param layout - The layout to set. + * @returns The layout function. + */ + setLayout: (layout: Layout) => Layout; + /** + * Gets the current layout for the response. + * + * @returns The current layout function. + */ + getLayout: () => Layout | undefined; + /** + * `.setRenderer()` can set the layout in the custom middleware. + * + * @see {@link https://hono.dev/docs/api/context#render-setrenderer} + * + * @example + * ```tsx + * app.use('*', async (c, next) => { + * c.setRenderer((content) => { + * return c.html( + * + * + *

{content}

+ * + * + * ) + * }) + * await next() + * }) + * ``` + */ + setRenderer: (renderer: Renderer) => void; + /** + * `.header()` can set headers. + * + * @see {@link https://hono.dev/docs/api/context#header} + * + * @example + * ```ts + * app.get('/welcome', (c) => { + * // Set headers + * c.header('X-Message', 'Hello!') + * c.header('Content-Type', 'text/plain') + * + * // Append multiple headers using the append option (e.g. Vary) + * c.header('Vary', 'Accept-Encoding', { append: true }) + * c.header('Vary', 'User-Agent', { append: true }) + * + * return c.body('Thank you for coming') + * }) + * ``` + */ + header: SetHeaders; + status: (status: StatusCode) => void; + /** + * `.set()` can set the value specified by the key. + * + * @see {@link https://hono.dev/docs/api/context#set-get} + * + * @example + * ```ts + * app.use('*', async (c, next) => { + * c.set('message', 'Hono is hot!!') + * await next() + * }) + * ``` + */ + set: Set extends true ? { + Variables: ContextVariableMap & Record; + } : E>; + /** + * `.get()` can use the value specified by the key. + * + * @see {@link https://hono.dev/docs/api/context#set-get} + * + * @example + * ```ts + * app.get('/', (c) => { + * const message = c.get('message') + * return c.text(`The message is "${message}"`) + * }) + * ``` + */ + get: Get extends true ? { + Variables: ContextVariableMap & Record; + } : E>; + /** + * `.var` can access the value of a variable. + * + * @see {@link https://hono.dev/docs/api/context#var} + * + * @example + * ```ts + * const result = c.var.client.oneMethod() + * ``` + */ + get var(): Readonly extends true ? Record : E['Variables'])>; + newResponse: NewResponse; + /** + * `.body()` can return the HTTP response. + * You can set headers with `.header()` and set HTTP status code with `.status`. + * This can also be set in `.text()`, `.json()` and so on. + * + * @see {@link https://hono.dev/docs/api/context#body} + * + * @example + * ```ts + * app.get('/welcome', (c) => { + * // Set headers + * c.header('X-Message', 'Hello!') + * c.header('Content-Type', 'text/plain') + * // Set HTTP status code + * c.status(201) + * + * // Return the response body + * return c.body('Thank you for coming') + * }) + * ``` + */ + body: BodyRespond; + /** + * `.text()` can render text as `Content-Type:text/plain`. + * + * @see {@link https://hono.dev/docs/api/context#text} + * + * @example + * ```ts + * app.get('/say', (c) => { + * return c.text('Hello!') + * }) + * ``` + */ + text: TextRespond; + /** + * `.json()` can render JSON as `Content-Type:application/json`. + * + * @see {@link https://hono.dev/docs/api/context#json} + * + * @example + * ```ts + * app.get('/api', (c) => { + * return c.json({ message: 'Hello!' }) + * }) + * ``` + */ + json: JSONRespond; + html: HTMLRespond; + /** + * `.redirect()` can Redirect, default status code is 302. + * + * @see {@link https://hono.dev/docs/api/context#redirect} + * + * @example + * ```ts + * app.get('/redirect', (c) => { + * return c.redirect('/') + * }) + * app.get('/redirect-permanently', (c) => { + * return c.redirect('/', 301) + * }) + * ``` + */ + redirect: (location: string | URL, status?: T) => Response & TypedResponse; + /** + * `.notFound()` can return the Not Found Response. + * + * @see {@link https://hono.dev/docs/api/context#notfound} + * + * @example + * ```ts + * app.get('/notfound', (c) => { + * return c.notFound() + * }) + * ``` + */ + notFound: () => ReturnType; +} +//#endregion +export {}; diff --git a/tests/fixtures/npm/node_modules/hono/dist/types/hono-base.d.ts b/tests/fixtures/npm/node_modules/hono/dist/types/hono-base.d.ts new file mode 100644 index 000000000..85f57dfc6 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/types/hono-base.d.ts @@ -0,0 +1,221 @@ +import { Router } from "./router.js"; +import { Env, ErrorHandler, H, HandlerInterface, MergePath, MergeSchemaPath, MiddlewareHandlerInterface, NotFoundHandler, OnHandlerInterface, RouterRoute, Schema } from "./types.js"; +import { Context, ExecutionContext } from "./context.js"; +//#region src/hono-base.d.ts +type GetPath = (request: Request, options?: { + env?: E['Bindings']; +}) => string; +export type HonoOptions = { + /** + * `strict` option specifies whether to distinguish whether the last path is a directory or not. + * + * @see {@link https://hono.dev/docs/api/hono#strict-mode} + * + * @default true + */ + strict?: boolean; + /** + * `router` option specifies which router to use. + * + * @see {@link https://hono.dev/docs/api/hono#router-option} + * + * @example + * ```ts + * const app = new Hono({ router: new RegExpRouter() }) + * ``` + */ + router?: Router<[H, RouterRoute]>; + /** + * `getPath` can handle the host header value. + * + * @see {@link https://hono.dev/docs/api/routing#routing-with-host-header-value} + * + * @example + * ```ts + * const app = new Hono({ + * getPath: (req) => + * '/' + req.headers.get('host') + req.url.replace(/^https?:\/\/[^/]+(\/[^?]*)/, '$1'), + * }) + * + * app.get('/www1.example.com/hello', () => c.text('hello www1')) + * + * // A following request will match the route: + * // new Request('http://www1.example.com/hello', { + * // headers: { host: 'www1.example.com' }, + * // }) + * ``` + */ + getPath?: GetPath; +}; +type MountOptionHandler = (c: Context) => unknown; +type MountReplaceRequest = (originalRequest: Request) => Request; +type MountOptions = MountOptionHandler | { + optionHandler?: MountOptionHandler; + replaceRequest?: MountReplaceRequest | false; +}; +declare class Hono { + + get: HandlerInterface; + post: HandlerInterface; + put: HandlerInterface; + delete: HandlerInterface; + options: HandlerInterface; + patch: HandlerInterface; + query: HandlerInterface; + all: HandlerInterface; + on: OnHandlerInterface; + use: MiddlewareHandlerInterface; + router: Router<[H, RouterRoute]>; + readonly getPath: GetPath; + private _basePath; + routes: RouterRoute[]; + constructor(options?: HonoOptions); + private errorHandler; + /** + * `.route()` allows grouping other Hono instance in routes. + * + * @see {@link https://hono.dev/docs/api/routing#grouping} + * + * @param {string} path - base Path + * @param {Hono} app - other Hono instance + * @returns {Hono} routed Hono instance + * + * @example + * ```ts + * const app = new Hono() + * const app2 = new Hono() + * + * app2.get("/user", (c) => c.text("user")) + * app.route("/api", app2) // GET /api/user + * ``` + */ + route(path: SubPath, app: Hono): Hono> | S, BasePath, CurrentPath>; + /** + * `.basePath()` allows base paths to be specified. + * + * @see {@link https://hono.dev/docs/api/routing#base-path} + * + * @param {string} path - base Path + * @returns {Hono} changed Hono instance + * + * @example + * ```ts + * const api = new Hono().basePath('/api') + * ``` + */ + basePath(path: SubPath): Hono, MergePath>; + /** + * `.onError()` handles an error and returns a customized Response. + * + * @see {@link https://hono.dev/docs/api/hono#error-handling} + * + * @param {ErrorHandler} handler - request Handler for error + * @returns {Hono} changed Hono instance + * + * @example + * ```ts + * app.onError((err, c) => { + * console.error(`${err}`) + * return c.text('Custom Error Message', 500) + * }) + * ``` + */ + onError: (handler: ErrorHandler) => Hono; + /** + * `.notFound()` allows you to customize a Not Found Response. + * + * @see {@link https://hono.dev/docs/api/hono#not-found} + * + * @param {NotFoundHandler} handler - request handler for not-found + * @returns {Hono} changed Hono instance + * + * @example + * ```ts + * app.notFound((c) => { + * return c.text('Custom 404 Message', 404) + * }) + * ``` + */ + notFound: (handler: NotFoundHandler) => Hono; + /** + * `.mount()` allows you to mount applications built with other frameworks into your Hono application. + * + * @deprecated Use `mount()` from `hono/mount` instead. `.mount()` will be removed in v5. + * + * @see {@link https://hono.dev/docs/api/hono#mount} + * + * @param {string} path - base Path + * @param {Function} applicationHandler - other Request Handler + * @param {MountOptions} [options] - options of `.mount()` + * @returns {Hono} mounted Hono instance + * + * @example + * ```ts + * import { Router as IttyRouter } from 'itty-router' + * import { Hono } from 'hono' + * // Create itty-router application + * const ittyRouter = IttyRouter() + * // GET /itty-router/hello + * ittyRouter.get('/hello', () => new Response('Hello from itty-router')) + * + * const app = new Hono() + * app.mount('/itty-router', ittyRouter.handle) + * ``` + * + * @example + * ```ts + * const app = new Hono() + * // Send the request to another application without modification. + * app.mount('/app', anotherApp, { + * replaceRequest: (req) => req, + * }) + * ``` + */ + mount(path: string, applicationHandler: (request: Request, ...args: any) => Response | Promise, options?: MountOptions): Hono; + /** + * `.fetch()` will be entry point of your app. + * + * @see {@link https://hono.dev/docs/api/hono#fetch} + * + * @param {Request} request - request Object of request + * @param {Env} env - env Object + * @param {ExecutionContext} executionCtx - context of execution + * @returns {Response | Promise} response of request + * + */ + fetch: (request: Request, env?: E['Bindings'] | {}, executionCtx?: ExecutionContext) => Response | Promise; + /** + * `.request()` is a useful method for testing. + * You can pass a URL or pathname to send a GET request. + * app will return a Response object. + * ```ts + * test('GET /hello is ok', async () => { + * const res = await app.request('/hello') + * expect(res.status).toBe(200) + * }) + * ``` + * @see https://hono.dev/docs/api/hono#request + */ + request: (input: Request | string | URL, requestInit?: RequestInit, Env?: E["Bindings"] | {}, executionCtx?: ExecutionContext) => Response | Promise; + /** + * `.fire()` automatically adds a global fetch event listener. + * This can be useful for environments that adhere to the Service Worker API, such as non-ES module Cloudflare Workers. + * @deprecated + * Use `fire` from `hono/service-worker` instead. + * ```ts + * import { Hono } from 'hono' + * import { fire } from 'hono/service-worker' + * + * const app = new Hono() + * // ... + * fire(app) + * ``` + * @see https://hono.dev/docs/api/hono#fire + * @see https://developer.mozilla.org/en-US/docs/Web/API/Service_Worker_API + * @see https://developers.cloudflare.com/workers/reference/migrate-to-module-workers/ + */ + fire: () => void; +} +//#endregion +export { Hono as HonoBase }; +export {}; diff --git a/tests/fixtures/npm/node_modules/hono/dist/types/hono.d.ts b/tests/fixtures/npm/node_modules/hono/dist/types/hono.d.ts new file mode 100644 index 000000000..52ce78cec --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/types/hono.d.ts @@ -0,0 +1,20 @@ +import { HonoBase as Hono$1, HonoOptions } from "./hono-base.js"; +import { BlankEnv, BlankSchema, Env, Schema } from "./types.js"; +//#region src/hono.d.ts +/** + * The Hono class extends the functionality of the HonoBase class. + * It sets up routing and allows for custom options to be passed. + * + * @template E - The environment type. + * @template S - The schema type. + * @template BasePath - The base path type. + */ +export declare class Hono extends Hono$1 { + /** + * Creates an instance of the Hono class. + * + * @param options - Optional configuration options for the Hono instance. + */ + constructor(options?: HonoOptions); +} +//#endregion \ No newline at end of file diff --git a/tests/fixtures/npm/node_modules/hono/dist/types/index.d.ts b/tests/fixtures/npm/node_modules/hono/dist/types/index.d.ts new file mode 100644 index 000000000..12bbd7aaf --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/types/index.d.ts @@ -0,0 +1,6 @@ +import { Env, ErrorHandler, Handler, Input, MiddlewareHandler, Next, NotFoundHandler, NotFoundResponse, Schema, ToSchema, TypedResponse, ValidationTargets } from "./types.js"; +import { HonoRequest } from "./request.js"; +import { Context, ContextRenderer, ContextVariableMap, ExecutionContext } from "./context.js"; +import { Hono } from "./hono.js"; +import { ClientRequestOptions, InferRequestType, InferResponseType } from "./client/types.js"; +export { type ClientRequestOptions, Context, type ContextRenderer, type ContextVariableMap, type Env, type ErrorHandler, type ExecutionContext, type Handler, Hono, type HonoRequest, type InferRequestType, type InferResponseType, type Input, type MiddlewareHandler, type Next, type NotFoundHandler, type NotFoundResponse, type Schema, type ToSchema, type TypedResponse, type ValidationTargets }; \ No newline at end of file diff --git a/tests/fixtures/npm/node_modules/hono/dist/types/request.d.ts b/tests/fixtures/npm/node_modules/hono/dist/types/request.d.ts new file mode 100644 index 000000000..03d14a385 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/types/request.d.ts @@ -0,0 +1,326 @@ +import { GET_MATCH_RESULT } from "./request/constants.js"; +import { Result } from "./router.js"; +import { CustomHeader, RequestHeader } from "./utils/headers.js"; +import { Simplify, UnionToIntersection } from "./utils/types.js"; +import { Input, InputToDataByTarget, ParamKeyToRecord, ParamKeys, RemoveQuestion, RouterRoute, ValidationTargets } from "./types.js"; +import { BodyData, ParseBodyOptions } from "./utils/body.js"; +//#region src/request.d.ts +type Body = { + json: any; + text: string; + arrayBuffer: ArrayBuffer; + blob: Blob; + formData: FormData; +}; +type BodyCache = Partial; +export declare class HonoRequest

{ + + /** + * `.raw` can get the raw Request object. + * + * @see {@link https://hono.dev/docs/api/request#raw} + * + * @example + * ```ts + * // For Cloudflare Workers + * app.post('/', async (c) => { + * const metadata = c.req.raw.cf?.hostMetadata? + * ... + * }) + * ``` + */ + raw: Request; + routeIndex: number; + /** + * `.path` can get the pathname of the request. + * + * @see {@link https://hono.dev/docs/api/request#path} + * + * @example + * ```ts + * app.get('/about/me', (c) => { + * const pathname = c.req.path // `/about/me` + * }) + * ``` + */ + path: string; + bodyCache: BodyCache; + constructor(request: Request, path?: string, matchResult?: Result<[unknown, RouterRoute]>); + /** + * `.req.param()` gets the path parameters. + * + * @see {@link https://hono.dev/docs/api/routing#path-parameter} + * + * @example + * ```ts + * const name = c.req.param('name') + * // or all parameters at once + * const { id, comment_id } = c.req.param() + * ``` + */ + param = ParamKeys

>(key: string extends P ? never : P2 extends `${infer _}?` ? never : P2): string; + param> = RemoveQuestion>>(key: P2): string | undefined; + param(key: string): string | undefined; + param(): Simplify>>>; + /** + * `.query()` can get querystring parameters. + * + * @see {@link https://hono.dev/docs/api/request#query} + * + * @example + * ```ts + * // Query params + * app.get('/search', (c) => { + * const query = c.req.query('q') + * }) + * + * // Get all params at once + * app.get('/search', (c) => { + * const { q, limit, offset } = c.req.query() + * }) + * ``` + */ + query(key: string): string | undefined; + query(): Record; + /** + * `.queries()` can get multiple querystring parameter values, e.g. /search?tags=A&tags=B + * + * @see {@link https://hono.dev/docs/api/request#queries} + * + * @example + * ```ts + * app.get('/search', (c) => { + * // tags will be string[] + * const tags = c.req.queries('tags') + * }) + * ``` + */ + queries(key: string): string[] | undefined; + queries(): Record; + /** + * `.header()` can get the request header value. + * + * @see {@link https://hono.dev/docs/api/request#header} + * + * @example + * ```ts + * app.get('/', (c) => { + * const userAgent = c.req.header('User-Agent') + * }) + * ``` + */ + header(name: RequestHeader): string | undefined; + header(name: string): string | undefined; + header(): Record; + /** + * `.parseBody()` can parse Request body of type `multipart/form-data` or `application/x-www-form-urlencoded` + * + * @see {@link https://hono.dev/docs/api/request#parsebody} + * + * @example + * ```ts + * app.post('/entry', async (c) => { + * const body = await c.req.parseBody() + * }) + * ``` + */ + parseBody, T extends BodyData>(options?: Options): Promise; + parseBody(options?: Partial): Promise; + /** + * `.json()` can parse Request body of type `application/json` + * + * @see {@link https://hono.dev/docs/api/request#json} + * + * @example + * ```ts + * app.post('/entry', async (c) => { + * const body = await c.req.json() + * }) + * ``` + */ + json(): Promise; + /** + * `.text()` can parse Request body of type `text/plain` + * + * @see {@link https://hono.dev/docs/api/request#text} + * + * @example + * ```ts + * app.post('/entry', async (c) => { + * const body = await c.req.text() + * }) + * ``` + */ + text(): Promise; + /** + * `.arrayBuffer()` parse Request body as an `ArrayBuffer` + * + * @see {@link https://hono.dev/docs/api/request#arraybuffer} + * + * @example + * ```ts + * app.post('/entry', async (c) => { + * const body = await c.req.arrayBuffer() + * }) + * ``` + */ + arrayBuffer(): Promise; + /** + * `.bytes()` parses the request body as a `Uint8Array`. + * + * @see {@link https://hono.dev/docs/api/request#bytes} + * + * @example + * ```ts + * app.post('/entry', async (c) => { + * const body = await c.req.bytes() + * }) + * ``` + */ + bytes(): Promise; + /** + * Parses the request body as a `Blob`. + * @example + * ```ts + * app.post('/entry', async (c) => { + * const body = await c.req.blob(); + * }); + * ``` + * @see https://hono.dev/docs/api/request#blob + */ + blob(): Promise; + /** + * Parses the request body as `FormData`. + * @example + * ```ts + * app.post('/entry', async (c) => { + * const body = await c.req.formData(); + * }); + * ``` + * @see https://hono.dev/docs/api/request#formdata + */ + formData(): Promise; + /** + * Adds validated data to the request. + * + * @param target - The target of the validation. + * @param data - The validated data to add. + */ + addValidatedData(target: keyof ValidationTargets, data: {}): void; + /** + * Gets validated data from the request. + * + * @param target - The target of the validation. + * @returns The validated data. + * + * @see https://hono.dev/docs/api/request#valid + */ + valid(target: T): InputToDataByTarget; + /** + * `.url` can get the request url strings. + * + * @see {@link https://hono.dev/docs/api/request#url} + * + * @example + * ```ts + * app.get('/about/me', (c) => { + * const url = c.req.url // `http://localhost:8787/about/me` + * ... + * }) + * ``` + */ + get url(): string; + /** + * `.method` can get the method name of the request. + * + * @see {@link https://hono.dev/docs/api/request#method} + * + * @example + * ```ts + * app.get('/about/me', (c) => { + * const method = c.req.method // `GET` + * }) + * ``` + */ + get method(): string; + get [GET_MATCH_RESULT](): Result<[unknown, RouterRoute]>; + /** + * `.matchedRoutes` can return a matched route in the handler + * + * @deprecated + * + * Use matchedRoutes helper defined in "hono/route" instead. + * + * @see {@link https://hono.dev/docs/api/request#matchedroutes} + * + * @example + * ```ts + * app.use('*', async function logger(c, next) { + * await next() + * c.req.matchedRoutes.forEach(({ handler, method, path }, i) => { + * const name = handler.name || (handler.length < 2 ? '[handler]' : '[middleware]') + * console.log( + * method, + * ' ', + * path, + * ' '.repeat(Math.max(10 - path.length, 0)), + * name, + * i === c.req.routeIndex ? '<- respond from here' : '' + * ) + * }) + * }) + * ``` + */ + get matchedRoutes(): RouterRoute[]; + /** + * `.routePath` can retrieve the path registered within the handler + * + * @deprecated + * + * Use routePath helper defined in "hono/route" instead. + * + * @see {@link https://hono.dev/docs/api/request#routepath} + * + * @example + * ```ts + * app.get('/posts/:id', (c) => { + * return c.json({ path: c.req.routePath }) + * }) + * ``` + */ + get routePath(): string; +} +/** + * Clones a HonoRequest's underlying raw Request object. + * + * This utility handles both consumed and unconsumed request bodies: + * - If the request body hasn't been consumed, it uses the native `clone()` method + * - If the request body has been consumed, it reconstructs a new Request using cached body data + * + * This is particularly useful when you need to: + * - Process the same request body multiple times + * - Pass requests to external services after validation + * + * @param req - The HonoRequest object to clone + * @returns A Promise that resolves to a new Request object with the same properties + * @throws {HTTPException} If the request body was consumed directly via `req.raw` + * without using HonoRequest methods (e.g., `req.json()`, `req.text()`), making it + * impossible to reconstruct the body from cache + * + * @example + * ```ts + * // Clone after consuming the body (e.g., after validation) + * app.post('/forward', + * validator('json', (data) => data), + * async (c) => { + * const validated = c.req.valid('json') + * // Body has been consumed, but cloneRawRequest still works + * const clonedReq = await cloneRawRequest(c.req) + * return fetch('http://backend-service.com', clonedReq) + * } + * ) + * ``` + */ +export declare const cloneRawRequest: (req: HonoRequest) => Promise; +//#endregion +export {}; diff --git a/tests/fixtures/npm/node_modules/hono/dist/types/request/constants.d.ts b/tests/fixtures/npm/node_modules/hono/dist/types/request/constants.d.ts new file mode 100644 index 000000000..0899a5d2b --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/types/request/constants.d.ts @@ -0,0 +1,3 @@ +//#region src/request/constants.d.ts +export declare const GET_MATCH_RESULT: unique symbol; +//#endregion \ No newline at end of file diff --git a/tests/fixtures/npm/node_modules/hono/dist/types/router.d.ts b/tests/fixtures/npm/node_modules/hono/dist/types/router.d.ts new file mode 100644 index 000000000..f00b40662 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/types/router.d.ts @@ -0,0 +1,98 @@ +//#region src/router.d.ts +/** + * @module + * This module provides types definitions and variables for the routers. + */ +/** + * Constant representing all HTTP methods in uppercase. + */ +export declare const METHOD_NAME_ALL: "ALL"; +/** + * Constant representing all HTTP methods in lowercase. + */ +export declare const METHOD_NAME_ALL_LOWERCASE: "all"; +/** + * Array of supported HTTP methods. + */ +export declare const METHODS: readonly ["get", "post", "put", "delete", "options", "patch", "query"]; +/** + * Error message indicating that a route cannot be added because the matcher is already built. + */ +export declare const MESSAGE_MATCHER_IS_ALREADY_BUILT = "Can not add a route since the matcher is already built."; +/** + * Interface representing a router. + * + * @template T - The type of the handler. + */ +export interface Router { + /** + * The name of the router. + */ + name: string; + /** + * Adds a route to the router. + * + * @param method - The HTTP method (e.g., 'get', 'post'). + * @param path - The path for the route. + * @param handler - The handler for the route. + */ + add(method: string, path: string, handler: T): void; + /** + * Matches a route based on the given method and path. + * + * @param method - The HTTP method (e.g., 'get', 'post'). + * @param path - The path to match. + * @returns The result of the match. + */ + match(method: string, path: string): Result; +} +/** + * Type representing a map of parameter indices. + */ +export type ParamIndexMap = Record; +/** + * Type representing a stash of parameters. + */ +export type ParamStash = string[]; +/** + * Type representing a map of parameters. + */ +export type Params = Record; +/** + * Type representing the result of a route match. + * + * The result can be in one of two formats: + * 1. An array of handlers with their corresponding parameter index maps, followed by a parameter stash. + * 2. An array of handlers with their corresponding parameter maps. + * + * Example: + * + * [[handler, paramIndexMap][], paramArray] + * ```typescript + * [ + * [ + * [middlewareA, {}], // '*' + * [funcA, {'id': 0}], // '/user/:id/*' + * [funcB, {'id': 0, 'action': 1}], // '/user/:id/:action' + * ], + * ['123', 'abc'] + * ] + * ``` + * + * [[handler, params][]] + * ```typescript + * [ + * [ + * [middlewareA, {}], // '*' + * [funcA, {'id': '123'}], // '/user/:id/*' + * [funcB, {'id': '123', 'action': 'abc'}], // '/user/:id/:action' + * ] + * ] + * ``` + */ +export type Result = [[T, ParamIndexMap][], ParamStash] | [[T, Params][]]; +/** + * Error class representing an unsupported path error. + */ +export declare class UnsupportedPathError extends Error {} +//#endregion \ No newline at end of file diff --git a/tests/fixtures/npm/node_modules/hono/dist/types/types.d.ts b/tests/fixtures/npm/node_modules/hono/dist/types/types.d.ts new file mode 100644 index 000000000..4a0be1dea --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/types/types.d.ts @@ -0,0 +1,214 @@ +import { HonoBase as Hono } from "./hono-base.js"; +import { CustomHeader, RequestHeader } from "./utils/headers.js"; +import { StatusCode } from "./utils/http-status.js"; +import { IfAnyThenEmptyObject, IsAny, JSONValue, RemoveBlankRecord, Simplify, UnionToIntersection } from "./utils/types.js"; +import { Context } from "./context.js"; +//#region src/types.d.ts +export type Bindings = object; +export type Variables = object; +export type BlankEnv = {}; +export type Env = { + Bindings?: Bindings; + Variables?: Variables; +}; +export type Next = () => Promise; +export type ExtractInput = I extends Input ? unknown extends I['in'] ? {} : I['in'] : I; +export type Input = { + in?: {}; + out?: {}; + outputFormat?: ResponseFormat; +}; +export type BlankSchema = {}; +export type BlankInput = {}; +export interface RouterRoute { + basePath: string; + path: string; + method: string; + handler: H; +} +export type HandlerResponse = Response | TypedResponse | Promise> | Promise; +export type Handler = any> = (c: Context, next: Next) => R; +export type MiddlewareHandler = Response> = (c: Context, next: Next) => Promise; +export type H = any> = Handler | MiddlewareHandler; +/** + * You can extend this interface to define a custom `c.notFound()` Response type. + * + * @example + * declare module 'hono' { + * interface NotFoundResponse extends Response, TypedResponse {} + * } + */ +export interface NotFoundResponse {} +export type NotFoundHandler = (c: Context) => NotFoundResponse extends Response ? NotFoundResponse | Promise : Response | Promise; +export interface HTTPResponseError extends Error { + getResponse: () => Response; +} +export type ErrorHandler = (err: Error | HTTPResponseError, c: Context) => Response | Promise; +export interface HandlerInterface { +

= any, E2 extends Env = E>(handler: H): Hono, S & ToSchema>, BasePath, CurrentPath>; +

= any, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, M1 extends H = H>(...handlers: [H & M1, H]): Hono, S & ToSchema | MergeMiddlewareResponse>, BasePath, CurrentPath>; +

, R extends HandlerResponse = any, I extends Input = BlankInput, E2 extends Env = E>(path: P, handler: H): Hono, S, M, P, I, BasePath>, BasePath, MergePath>; +

= any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, M1 extends H = H, M2 extends H = H>(...handlers: [H & M1, H & M2, H]): Hono, S & ToSchema | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, CurrentPath>; +

, R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, M1 extends H = H>(path: P, ...handlers: [H & M1, H]): Hono | MergeMiddlewareResponse, S, M, P, I2, BasePath>, BasePath, MergePath>; +

= any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, M1 extends H = H, M2 extends H = H, M3 extends H = H>(...handlers: [H & M1, H & M2, H & M3, H]): Hono, S & ToSchema | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, CurrentPath>; +

, R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, M1 extends H = H, M2 extends H = H>(path: P, ...handlers: [H & M1, H & M2, H]): Hono | MergeMiddlewareResponse | MergeMiddlewareResponse, S, M, P, I3, BasePath>, BasePath, MergePath>; +

= any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H>(...handlers: [H & M1, H & M2, H & M3, H & M4, H]): Hono, S & ToSchema | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, CurrentPath>; +

, R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, M1 extends H = H, M2 extends H = H, M3 extends H = H>(path: P, ...handlers: [H & M1, H & M2, H & M3, H]): Hono | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse, S, M, P, I4, BasePath>, BasePath, MergePath>; +

= any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, I6 extends Input = I & I2 & I3 & I4 & I5, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H, M5 extends H = H>(...handlers: [H & M1, H & M2, H & M3, H & M4, H & M5, H]): Hono, S & ToSchema | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, CurrentPath>; +

, R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H>(path: P, ...handlers: [H & M1, H & M2, H & M3, H & M4, H]): Hono | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse, S, M, P, I5, BasePath>, BasePath, MergePath>; +

= any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, I6 extends Input = I & I2 & I3 & I4 & I5, I7 extends Input = I & I2 & I3 & I4 & I5 & I6, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, E8 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H, M5 extends H = H, M6 extends H = H>(...handlers: [H & M1, H & M2, H & M3, H & M4, H & M5, H & M6, H]): Hono, S & ToSchema | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, CurrentPath>; +

, R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, I6 extends Input = I & I2 & I3 & I4 & I5, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H, M5 extends H = H>(path: P, ...handlers: [H & M1, H & M2, H & M3, H & M4, H & M5, H]): Hono | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse, S, M, P, I6, BasePath>, BasePath, MergePath>; +

= any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, I6 extends Input = I & I2 & I3 & I4 & I5, I7 extends Input = I & I2 & I3 & I4 & I5 & I6, I8 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, E8 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7]>, E9 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H, M5 extends H = H, M6 extends H = H, M7 extends H = H>(...handlers: [H & M1, H & M2, H & M3, H & M4, H & M5, H & M6, H & M7, H]): Hono, S & ToSchema | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, CurrentPath>; +

, R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, I6 extends Input = I & I2 & I3 & I4 & I5, I7 extends Input = I & I2 & I3 & I4 & I5 & I6, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, E8 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H, M5 extends H = H, M6 extends H = H>(path: P, ...handlers: [H & M1, H & M2, H & M3, H & M4, H & M5, H & M6, H]): Hono | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse, S, M, P, I7, BasePath>, BasePath, MergePath>; +

= any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, I6 extends Input = I & I2 & I3 & I4 & I5, I7 extends Input = I & I2 & I3 & I4 & I5 & I6, I8 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7, I9 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7 & I8, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, E8 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7]>, E9 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8]>, E10 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8, E9]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H, M5 extends H = H, M6 extends H = H, M7 extends H = H, M8 extends H = H>(...handlers: [H & M1, H & M2, H & M3, H & M4, H & M5, H & M6, H & M7, H & M8, H]): Hono, S & ToSchema | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, CurrentPath>; +

, R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, I6 extends Input = I & I2 & I3 & I4 & I5, I7 extends Input = I & I2 & I3 & I4 & I5 & I6, I8 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, E8 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7]>, E9 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H, M5 extends H = H, M6 extends H = H, M7 extends H = H>(path: P, ...handlers: [H & M1, H & M2, H & M3, H & M4, H & M5, H & M6, H & M7, H]): Hono | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse, S, M, P, I8, BasePath>, BasePath, MergePath>; +

= any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, I6 extends Input = I & I2 & I3 & I4 & I5, I7 extends Input = I & I2 & I3 & I4 & I5 & I6, I8 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7, I9 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7 & I8, I10 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7 & I8 & I9, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, E8 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7]>, E9 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8]>, E10 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8, E9]>, E11 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8, E9, E10]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H, M5 extends H = H, M6 extends H = H, M7 extends H = H, M8 extends H = H, M9 extends H = H>(...handlers: [H & M1, H & M2, H & M3, H & M4, H & M5, H & M6, H & M7, H & M8, H & M9, H]): Hono, S & ToSchema | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, CurrentPath>; +

, R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, I6 extends Input = I & I2 & I3 & I4 & I5, I7 extends Input = I & I2 & I3 & I4 & I5 & I6, I8 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7, I9 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7 & I8, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, E8 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7]>, E9 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8]>, E10 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8, E9]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H, M5 extends H = H, M6 extends H = H, M7 extends H = H, M8 extends H = H>(path: P, ...handlers: [H & M1, H & M2, H & M3, H & M4, H & M5, H & M6, H & M7, H & M8, H]): Hono | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse, S, M, P, I9, BasePath>, BasePath, MergePath>; +

, R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, I6 extends Input = I & I2 & I3 & I4 & I5, I7 extends Input = I & I2 & I3 & I4 & I5 & I6, I8 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7, I9 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7 & I8, I10 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7 & I8 & I9, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, E8 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7]>, E9 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8]>, E10 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8, E9]>, E11 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8, E9, E10]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H, M5 extends H = H, M6 extends H = H, M7 extends H = H, M8 extends H = H, M9 extends H = H>(path: P, ...handlers: [H & M1, H & M2, H & M3, H & M4, H & M5, H & M6, H & M7, H & M8, H & M9, H]): Hono | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse, S, M, P, I10, BasePath>, BasePath, MergePath>; +

= any>(...handlers: H[]): Hono>, BasePath, CurrentPath>; +

= any>(path: P, ...handlers: [H, I, R>, ...H, I, R>[]]): Hono, I, MergeTypedResponse>, BasePath, MergePath>; +

= any, I extends Input = BlankInput>(path: P): Hono, I, MergeTypedResponse>, BasePath, MergePath>; +} +export interface MiddlewareHandlerInterface { + (...handlers: MiddlewareHandler>[]): Hono, S, BasePath, MergePath>; + (handler: MiddlewareHandler>): Hono, S, BasePath, MergePath>; + , P extends string = MergePath>(...handlers: [MiddlewareHandler, MiddlewareHandler]): Hono, S, BasePath, P>; +

, E2 extends Env = E>(path: P, handler: MiddlewareHandler): Hono, S, BasePath, MergedPath>; + , E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, P extends string = MergePath>(...handlers: [MiddlewareHandler, MiddlewareHandler, MiddlewareHandler]): Hono, S, BasePath, P>; +

, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>>(path: P, ...handlers: [MiddlewareHandler, MiddlewareHandler]): Hono, S, BasePath, MergedPath>; + , E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, P extends string = MergePath>(...handlers: [MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler]): Hono, S, BasePath, P>; +

, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>>(path: P, ...handlers: [MiddlewareHandler, MiddlewareHandler, MiddlewareHandler]): Hono, S, BasePath, MergedPath>; + , E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, P extends string = MergePath>(...handlers: [MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler]): Hono, S, BasePath, P>; +

, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>>(path: P, ...handlers: [MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler]): Hono, S, BasePath, MergedPath>; + , E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, P extends string = MergePath>(...handlers: [MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler]): Hono, S, BasePath, P>; +

, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>>(path: P, ...handlers: [MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler]): Hono, S, BasePath, MergedPath>; + , E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, E8 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7]>, P extends string = MergePath>(...handlers: [MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler]): Hono, S, BasePath, P>; +

, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>>(path: P, ...handlers: [MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler]): Hono, S, BasePath, MergedPath>; + , E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, E8 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7]>, E9 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8]>, P extends string = MergePath>(...handlers: [MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler]): Hono, S, BasePath, P>; +

, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, E8 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7]>>(path: P, ...handlers: [MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler]): Hono, S, BasePath, MergedPath>; + , E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, E8 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7]>, E9 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8]>, E10 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8, E9]>, P extends string = MergePath>(...handlers: [MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler]): Hono, S, BasePath, P>; +

, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, E8 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7]>, E9 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8]>>(path: P, ...handlers: [MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler]): Hono, S, BasePath, MergedPath>; + , E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, E8 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7]>, E9 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8]>, E10 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8, E9]>, E11 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8, E9, E10]>, P extends string = MergePath>(...handlers: [MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler]): Hono, S, BasePath, P>; +

, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, E8 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7]>, E9 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8]>, E10 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8, E9]>>(path: P, ...handlers: [MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler, MiddlewareHandler]): Hono, S, BasePath, MergedPath>; +

(path: P, ...handlers: MiddlewareHandler>[]): Hono>; +} +export interface OnHandlerInterface { + , R extends HandlerResponse = any, I extends Input = BlankInput, E2 extends Env = E>(method: M, path: P, handler: H): Hono, S & ToSchema, I, MergeTypedResponse>, BasePath, MergePath>; + , R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, M1 extends H = H>(method: M, path: P, ...handlers: [H & M1, H]): Hono, S & ToSchema, I2, MergeTypedResponse | MergeMiddlewareResponse>, BasePath, MergePath>; + , R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, M1 extends H = H, M2 extends H = H>(method: M, path: P, ...handlers: [H & M1, H & M2, H]): Hono, S & ToSchema, I3, MergeTypedResponse | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, MergePath>; + , R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, M1 extends H = H, M2 extends H = H, M3 extends H = H>(method: M, path: P, ...handlers: [H & M1, H & M2, H & M3, H]): Hono, S & ToSchema, I4, MergeTypedResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, MergePath>; + , R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H>(method: M, path: P, ...handlers: [H & M1, H & M2, H & M3, H & M4, H]): Hono, S & ToSchema, I5, MergeTypedResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, MergePath>; + , R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, I6 extends Input = I & I2 & I3 & I4 & I5, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H, M5 extends H = H>(method: M, path: P, ...handlers: [H & M1, H & M2, H & M3, H & M4, H & M5, H]): Hono, S & ToSchema, I6, MergeTypedResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, MergePath>; + , R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, I6 extends Input = I & I2 & I3 & I4 & I5, I7 extends Input = I & I2 & I3 & I4 & I5 & I6, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, E8 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H, M5 extends H = H, M6 extends H = H>(method: M, path: P, ...handlers: [H & M1, H & M2, H & M3, H & M4, H & M5, H & M6, H]): Hono, S & ToSchema, I7, MergeTypedResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, MergePath>; + , R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, I6 extends Input = I & I2 & I3 & I4 & I5, I7 extends Input = I & I2 & I3 & I4 & I5 & I6, I8 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, E8 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7]>, E9 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H, M5 extends H = H, M6 extends H = H, M7 extends H = H>(method: M, path: P, ...handlers: [H & M1, H & M2, H & M3, H & M4, H & M5, H & M6, H & M7, H]): Hono, S & ToSchema, I8, MergeTypedResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, MergePath>; + , R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, I6 extends Input = I & I2 & I3 & I4 & I5, I7 extends Input = I & I2 & I3 & I4 & I5 & I6, I8 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7, I9 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7 & I8, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, E8 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7]>, E9 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8]>, E10 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8, E9]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H, M5 extends H = H, M6 extends H = H, M7 extends H = H, M8 extends H = H>(method: M, path: P, ...handlers: [H & M1, H & M2, H & M3, H & M4, H & M5, H & M6, H & M7, H & M8, H]): Hono, S & ToSchema, I9, MergeTypedResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, MergePath>; + , R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, I6 extends Input = I & I2 & I3 & I4 & I5, I7 extends Input = I & I2 & I3 & I4 & I5 & I6, I8 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7, I9 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7 & I8, I10 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7 & I8 & I9, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, E8 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7]>, E9 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8]>, E10 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8, E9]>, E11 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8, E9, E10]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H, M5 extends H = H, M6 extends H = H, M7 extends H = H, M8 extends H = H, M9 extends H = H>(method: M, path: P, ...handlers: [H & M1, H & M2, H & M3, H & M4, H & M5, H & M6, H & M7, H & M8, H & M9, H]): Hono, S & ToSchema, I10, MergeTypedResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, MergePath>; + = any, I extends Input = BlankInput>(method: M, path: P, ...handlers: [H, I, R>, ...H, I, R>[]]): Hono, I, MergeTypedResponse>, BasePath, MergePath>; + , R extends HandlerResponse = any, I extends Input = BlankInput, E2 extends Env = E>(methods: M[], path: P, handler: H): Hono, S & ToSchema, I, MergeTypedResponse>, BasePath, MergePath>; + , R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, M1 extends H = H>(methods: M[], path: P, ...handlers: [H & M1, H]): Hono, S & ToSchema, I2, MergeTypedResponse | MergeMiddlewareResponse>, BasePath, MergePath>; + , R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, M1 extends H = H, M2 extends H = H>(methods: M[], path: P, ...handlers: [H & M1, H & M2, H]): Hono, S & ToSchema, I3, MergeTypedResponse | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, MergePath>; + , R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, M1 extends H = H, M2 extends H = H, M3 extends H = H>(methods: M[], path: P, ...handlers: [H & M1, H & M2, H & M3, H]): Hono, S & ToSchema, I4, MergeTypedResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, MergePath>; + , R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H>(methods: M[], path: P, ...handlers: [H & M1, H & M2, H & M3, H & M4, H]): Hono, S & ToSchema, I5, MergeTypedResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, MergePath>; + , R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, I6 extends Input = I & I2 & I3 & I4 & I5, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H, M5 extends H = H>(methods: M[], path: P, ...handlers: [H & M1, H & M2, H & M3, H & M4, H & M5, H]): Hono, S & ToSchema, I6, MergeTypedResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, MergePath>; + , R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, I6 extends Input = I & I2 & I3 & I4 & I5, I7 extends Input = I & I2 & I3 & I4 & I5 & I6, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, E8 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H, M5 extends H = H, M6 extends H = H>(methods: M[], path: P, ...handlers: [H & M1, H & M2, H & M3, H & M4, H & M5, H & M6, H]): Hono, S & ToSchema, I7, MergeTypedResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, MergePath>; + , R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, I6 extends Input = I & I2 & I3 & I4 & I5, I7 extends Input = I & I2 & I3 & I4 & I5 & I6, I8 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, E8 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7]>, E9 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H, M5 extends H = H, M6 extends H = H, M7 extends H = H>(methods: M[], path: P, ...handlers: [H & M1, H & M2, H & M3, H & M4, H & M5, H & M6, H & M7, H]): Hono, S & ToSchema, I8, MergeTypedResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, MergePath>; + , R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, I6 extends Input = I & I2 & I3 & I4 & I5, I7 extends Input = I & I2 & I3 & I4 & I5 & I6, I8 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7, I9 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7 & I8, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, E8 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7]>, E9 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8]>, E10 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8, E9]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H, M5 extends H = H, M6 extends H = H, M7 extends H = H, M8 extends H = H>(methods: M[], path: P, ...handlers: [H & M1, H & M2, H & M3, H & M4, H & M5, H & M6, H & M7, H & M8, H]): Hono, S & ToSchema, I9, MergeTypedResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, MergePath>; + , R extends HandlerResponse = any, I extends Input = BlankInput, I2 extends Input = I, I3 extends Input = I & I2, I4 extends Input = I & I2 & I3, I5 extends Input = I & I2 & I3 & I4, I6 extends Input = I & I2 & I3 & I4 & I5, I7 extends Input = I & I2 & I3 & I4 & I5 & I6, I8 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7, I9 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7 & I8, I10 extends Input = I & I2 & I3 & I4 & I5 & I6 & I7 & I8 & I9, E2 extends Env = E, E3 extends Env = IntersectNonAnyTypes<[E, E2]>, E4 extends Env = IntersectNonAnyTypes<[E, E2, E3]>, E5 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4]>, E6 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5]>, E7 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6]>, E8 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7]>, E9 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8]>, E10 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8, E9]>, E11 extends Env = IntersectNonAnyTypes<[E, E2, E3, E4, E5, E6, E7, E8, E9, E10]>, M1 extends H = H, M2 extends H = H, M3 extends H = H, M4 extends H = H, M5 extends H = H, M6 extends H = H, M7 extends H = H, M8 extends H = H, M9 extends H = H>(methods: M[], path: P, ...handlers: [H & M1, H & M2, H & M3, H & M4, H & M5, H & M6, H & M7, H & M8, H & M9, H]): Hono, S & ToSchema, I10, MergeTypedResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse | MergeMiddlewareResponse>, BasePath, MergePath>; + = any, I extends Input = BlankInput>(methods: M[], path: P, ...handlers: [H, I, R>, ...H, I, R>[]]): Hono, I, MergeTypedResponse>, BasePath, MergePath>; + = any, E2 extends Env = E>(methods: M | M[], paths: Ps, ...handlers: H, I, R>[]): Hono, I, MergeTypedResponse>, BasePath, Ps extends [...string[], infer LastPath extends string] ? MergePath : never>; +} +type ToSchemaOutput = RorO extends TypedResponse ? { + output: unknown extends T ? {} : T; + outputFormat: I extends { + outputFormat: string; + } ? I['outputFormat'] : F; + status: U; +} : { + output: unknown extends RorO ? {} : RorO; + outputFormat: unknown extends RorO ? 'json' : I extends { + outputFormat: string; + } ? I['outputFormat'] : 'json'; + status: StatusCode; +}; +export type ToSchema = IsAny extends true ? { [K in P]: { [K2 in M as AddDollar]: { + input: AddParam, P>; + output: {}; + outputFormat: ResponseFormat; + status: StatusCode; +}; }; } : [RorO] extends [never] ? {} : [RorO] extends [Promise] ? {} : { [K in P]: { [K2 in M as AddDollar]: Simplify<{ + input: AddParam, P>; +} & ToSchemaOutput>; }; }; +export type Schema = { + [Path: string]: { + [Method: `$${Lowercase}`]: Endpoint; + }; +}; +type AddSchemaIfHasResponse = [Merged] extends [Promise] ? S : S & ToSchema, I, Merged>; +export type Endpoint = { + input: any; + output: any; + outputFormat: ResponseFormat; + status: StatusCode; +}; +type ExtractParams = string extends Path ? Record : Path extends `${infer _Start}:${infer Param}/${infer Rest}` ? { [K in Param | keyof ExtractParams<`/${Rest}`>]: string; } : Path extends `${infer _Start}:${infer Param}` ? { [K in Param]: string; } : never; +type FlattenIfIntersect = T extends (infer O) ? { [K in keyof O]: O[K]; } : never; +export type MergeSchemaPath = { [P in keyof OrigSchema as MergePath]: [OrigSchema[P]] extends [Record] ? { [M in keyof OrigSchema[P]]: MergeEndpointParamsWithPath; } : never; }; +type MergeEndpointParamsWithPath = T extends unknown ? { + input: T['input'] extends { + param: infer _; + } ? ExtractParams extends never ? T['input'] : FlattenIfIntersect as K extends `${infer Prefix}{${infer _}}` ? Prefix : K]: string; }; + }> : RemoveBlankRecord> extends never ? T['input'] : T['input'] & { + param: { [K in keyof ExtractParams as K extends `${infer Prefix}{${infer _}}` ? Prefix : K]: string; }; + }; + output: T['output']; + outputFormat: T['outputFormat']; + status: T['status']; +} : never; +export type AddParam = ParamKeys

extends never ? I : I extends { + param: infer _; +} ? I : I & { + param: UnionToIntersection>>; +}; +type AddDollar = `$${Lowercase}`; +export type MergePath = B extends '' ? MergePath : A extends '' ? B : A extends '/' ? B : A extends `${infer P}/` ? B extends `/${infer Q}` ? `${P}/${Q}` : `${P}/${B}` : B extends `/${infer Q}` ? Q extends '' ? A : `${A}/${Q}` : `${A}/${B}`; +export type KnownResponseFormat = 'json' | 'text' | 'redirect'; +export type ResponseFormat = KnownResponseFormat | string; +export type TypedResponse = { + _data: T; + _status: U; + _format: F; +}; +type MergeTypedResponse = T extends Promise ? T : T extends Promise ? T2 extends TypedResponse ? T2 : TypedResponse : T extends TypedResponse ? T : TypedResponse; +type ExtractTypedResponseOnly = T extends TypedResponse ? T : never; +type MergeMiddlewareResponse = T extends ((c: any, next: any) => Promise) ? Exclude extends never ? never : Exclude extends Response | TypedResponse ? ExtractTypedResponseOnly> : never : T extends ((c: any, next: any) => infer R) ? R extends Response | TypedResponse ? ExtractTypedResponseOnly : never : never; +export type FormValue = string | Blob; +export type ParsedFormValue = string | File; +export type ValidationTargets = { + json: any; + form: Record; + query: Record; + param: Record; + header: Record; + cookie: Record; +}; +type ParamKey = Component extends `:${infer NameWithPattern}` ? NameWithPattern extends `${infer Name}{${infer Rest}` ? Rest extends `${infer _Pattern}?` ? `${Name}?` : Name : NameWithPattern : never; +export type ParamKeys = Path extends `${infer Component}/${infer Rest}` ? ParamKey | ParamKeys : ParamKey; +export type ParamKeyToRecord = T extends `${infer R}?` ? Record : { [K in T]: string; }; +export type InputToDataByTarget = T extends { [K in Target]: infer R; } ? R : never; +export type RemoveQuestion = T extends `${infer R}?` ? R : T; +export type ExtractSchema = UnionToIntersection ? S : never>; +export type ExtractSchemaForStatusCode = { [Path in keyof ExtractSchema]: { [Method in keyof ExtractSchema[Path]]: Extract[Path][Method], { + status: Status; +}>; }; }; +export type ExtractHandlerResponse = T extends ((c: any, next: any) => Promise) ? Exclude extends never ? never : Exclude extends Response | TypedResponse ? Exclude : never : T extends ((c: any, next: any) => infer R) ? R extends Response | TypedResponse ? R : never : never; +type ProcessHead = IfAnyThenEmptyObject; +export type IntersectNonAnyTypes = T extends [infer Head, ...infer Rest] ? ProcessHead & IntersectNonAnyTypes : {}; +export declare abstract class FetchEventLike { + abstract readonly request: Request; + abstract respondWith(promise: Response | Promise): void; + abstract passThroughOnException(): void; + abstract waitUntil(promise: Promise): void; +} +//#endregion +export {}; diff --git a/tests/fixtures/npm/node_modules/hono/dist/types/utils/body.d.ts b/tests/fixtures/npm/node_modules/hono/dist/types/utils/body.d.ts new file mode 100644 index 000000000..9bbf8cafd --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/types/utils/body.d.ts @@ -0,0 +1,75 @@ +import { HonoRequest } from "../request.js"; +//#region src/utils/body.d.ts +type BodyDataValueDot = { + [x: string]: string | File | BodyDataValueDot; +}; +type BodyDataValueDotAll = { + [x: string]: string | File | (string | File)[] | BodyDataValueDotAll; +}; +type SimplifyBodyData = { [K in keyof T]: string | File | (string | File)[] | BodyDataValueDotAll extends T[K] ? string | File | (string | File)[] | BodyDataValueDotAll : string | File | BodyDataValueDot extends T[K] ? string | File | BodyDataValueDot : string | File | (string | File)[] extends T[K] ? string | File | (string | File)[] : string | File; } & {}; +type BodyDataValueComponent = string | File | (T extends { + all: false; +} ? never : T extends { + all: true; +} | { + all: boolean; +} ? (string | File)[] : never); +type BodyDataValueObject = { + [key: string]: BodyDataValueComponent | BodyDataValueObject; +}; +type BodyDataValue = BodyDataValueComponent | (T extends { + dot: false; +} ? never : T extends { + dot: true; +} | { + dot: boolean; +} ? BodyDataValueObject : never); +export type BodyData = {}> = SimplifyBodyData>>; +export type ParseBodyOptions = { + /** + * Determines whether all fields with multiple values should be parsed as arrays. + * @default false + * @example + * const data = new FormData() + * data.append('file', 'aaa') + * data.append('file', 'bbb') + * data.append('message', 'hello') + * + * If all is false: + * parseBody should return { file: 'bbb', message: 'hello' } + * + * If all is true: + * parseBody should return { file: ['aaa', 'bbb'], message: 'hello' } + */ + all: boolean; + /** + * Determines whether all fields with dot notation should be parsed as nested objects. + * @default false + * @example + * const data = new FormData() + * data.append('obj.key1', 'value1') + * data.append('obj.key2', 'value2') + * + * If dot is false: + * parseBody should return { 'obj.key1': 'value1', 'obj.key2': 'value2' } + * + * If dot is true: + * parseBody should return { obj: { key1: 'value1', key2: 'value2' } } + */ + dot: boolean; +}; +/** + * Parses the body of a request based on the provided options. + * + * @template T - The type of the parsed body data. + * @param {HonoRequest | Request} request - The request object to parse. + * @param {Partial} [options] - Options for parsing the body. + * @returns {Promise} The parsed body data. + */ +interface ParseBody { + , T extends BodyData>(request: HonoRequest | Request, options?: Options): Promise; + (request: HonoRequest | Request, options?: Partial): Promise; +} +export declare const parseBody: ParseBody; +//#endregion +export {}; diff --git a/tests/fixtures/npm/node_modules/hono/dist/types/utils/headers.d.ts b/tests/fixtures/npm/node_modules/hono/dist/types/utils/headers.d.ts new file mode 100644 index 000000000..d3edf8c77 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/types/utils/headers.d.ts @@ -0,0 +1,10 @@ +//#region src/utils/headers.d.ts +/** + * @module + * HTTP Headers utility. + */ +export type RequestHeader = 'A-IM' | 'Accept' | 'Accept-Additions' | 'Accept-CH' | 'Accept-Charset' | 'Accept-Datetime' | 'Accept-Encoding' | 'Accept-Features' | 'Accept-Language' | 'Accept-Patch' | 'Accept-Post' | 'Accept-Query' | 'Accept-Ranges' | 'Accept-Signature' | 'Access-Control' | 'Access-Control-Allow-Credentials' | 'Access-Control-Allow-Headers' | 'Access-Control-Allow-Methods' | 'Access-Control-Allow-Origin' | 'Access-Control-Expose-Headers' | 'Access-Control-Max-Age' | 'Access-Control-Request-Headers' | 'Access-Control-Request-Method' | 'Activate-Storage-Access' | 'Age' | 'Allow' | 'ALPN' | 'Alt-Svc' | 'Alt-Used' | 'Alternates' | 'AMP-Cache-Transform' | 'Apply-To-Redirect-Ref' | 'Authentication-Control' | 'Authentication-Info' | 'Authorization' | 'Available-Dictionary' | 'C-Ext' | 'C-Man' | 'C-Opt' | 'C-PEP' | 'C-PEP-Info' | 'Cache-Control' | 'Cache-Group-Invalidation' | 'Cache-Groups' | 'Cache-Status' | 'Cal-Managed-ID' | 'CalDAV-Timezones' | 'Capsule-Protocol' | 'CDN-Cache-Control' | 'CDN-Loop' | 'Cert-Not-After' | 'Cert-Not-Before' | 'Clear-Site-Data' | 'Client-Cert' | 'Client-Cert-Chain' | 'Close' | 'CMCD-Object' | 'CMCD-Request' | 'CMCD-Session' | 'CMCD-Status' | 'CMSD-Dynamic' | 'CMSD-Static' | 'Concealed-Auth-Export' | 'Configuration-Context' | 'Connection' | 'Content-Base' | 'Content-Digest' | 'Content-Disposition' | 'Content-Encoding' | 'Content-ID' | 'Content-Language' | 'Content-Length' | 'Content-Location' | 'Content-MD5' | 'Content-Range' | 'Content-Script-Type' | 'Content-Security-Policy' | 'Content-Security-Policy-Report-Only' | 'Content-Style-Type' | 'Content-Type' | 'Content-Version' | 'Cookie' | 'Cookie2' | 'Cross-Origin-Embedder-Policy' | 'Cross-Origin-Embedder-Policy-Report-Only' | 'Cross-Origin-Opener-Policy' | 'Cross-Origin-Opener-Policy-Report-Only' | 'Cross-Origin-Resource-Policy' | 'CTA-Common-Access-Token' | 'DASL' | 'Date' | 'DAV' | 'Default-Style' | 'Delta-Base' | 'Deprecation' | 'Depth' | 'Derived-From' | 'Destination' | 'Detached-JWS' | 'Differential-ID' | 'Dictionary-ID' | 'Digest' | 'DPoP' | 'DPoP-Nonce' | 'Early-Data' | 'EDIINT-Features' | 'ETag' | 'Expect' | 'Expect-CT' | 'Expires' | 'Ext' | 'Forwarded' | 'From' | 'GetProfile' | 'Hobareg' | 'Host' | 'HTTP2-Settings' | 'If' | 'If-Match' | 'If-Modified-Since' | 'If-None-Match' | 'If-Range' | 'If-Schedule-Tag-Match' | 'If-Unmodified-Since' | 'IM' | 'Include-Referred-Token-Binding-ID' | 'Incremental' | 'Isolation' | 'Keep-Alive' | 'Label' | 'Last-Event-ID' | 'Last-Modified' | 'Link' | 'Link-Template' | 'Location' | 'Lock-Token' | 'Man' | 'Max-Forwards' | 'Memento-Datetime' | 'Meter' | 'Method-Check' | 'Method-Check-Expires' | 'MIME-Version' | 'Negotiate' | 'NEL' | 'OData-EntityId' | 'OData-Isolation' | 'OData-MaxVersion' | 'OData-Version' | 'Opt' | 'Optional-WWW-Authenticate' | 'Ordering-Type' | 'Origin' | 'Origin-Agent-Cluster' | 'OSCORE' | 'OSLC-Core-Version' | 'Overwrite' | 'P3P' | 'PEP' | 'PEP-Info' | 'Permissions-Policy' | 'PICS-Label' | 'Ping-From' | 'Ping-To' | 'Position' | 'Pragma' | 'Prefer' | 'Preference-Applied' | 'Priority' | 'ProfileObject' | 'Protocol' | 'Protocol-Info' | 'Protocol-Query' | 'Protocol-Request' | 'Proxy-Authenticate' | 'Proxy-Authentication-Info' | 'Proxy-Authorization' | 'Proxy-Features' | 'Proxy-Instruction' | 'Proxy-Status' | 'Public' | 'Public-Key-Pins' | 'Public-Key-Pins-Report-Only' | 'Range' | 'Redirect-Ref' | 'Referer' | 'Referer-Root' | 'Referrer-Policy' | 'Refresh' | 'Repeatability-Client-ID' | 'Repeatability-First-Sent' | 'Repeatability-Request-ID' | 'Repeatability-Result' | 'Replay-Nonce' | 'Reporting-Endpoints' | 'Repr-Digest' | 'Retry-After' | 'Safe' | 'Schedule-Reply' | 'Schedule-Tag' | 'Sec-Fetch-Dest' | 'Sec-Fetch-Mode' | 'Sec-Fetch-Site' | 'Sec-Fetch-Storage-Access' | 'Sec-Fetch-User' | 'Sec-GPC' | 'Sec-Purpose' | 'Sec-Token-Binding' | 'Sec-WebSocket-Accept' | 'Sec-WebSocket-Extensions' | 'Sec-WebSocket-Key' | 'Sec-WebSocket-Protocol' | 'Sec-WebSocket-Version' | 'Security-Scheme' | 'Server' | 'Server-Timing' | 'Set-Cookie' | 'Set-Cookie2' | 'Set-Txn' | 'SetProfile' | 'Signature' | 'Signature-Input' | 'SLUG' | 'SoapAction' | 'Status-URI' | 'Strict-Transport-Security' | 'Sunset' | 'Surrogate-Capability' | 'Surrogate-Control' | 'TCN' | 'TE' | 'Timeout' | 'Timing-Allow-Origin' | 'Topic' | 'Traceparent' | 'Tracestate' | 'Trailer' | 'Transfer-Encoding' | 'TTL' | 'Unencoded-Digest' | 'Upgrade' | 'Urgency' | 'URI' | 'Use-As-Dictionary' | 'User-Agent' | 'Variant-Vary' | 'Vary' | 'Via' | 'Want-Content-Digest' | 'Want-Digest' | 'Want-Repr-Digest' | 'Want-Unencoded-Digest' | 'Warning' | 'WWW-Authenticate' | 'X-Content-Type-Options' | 'X-Frame-Options'; +export type ResponseHeader = 'Accept-Query' | 'Access-Control-Allow-Credentials' | 'Access-Control-Allow-Headers' | 'Access-Control-Allow-Methods' | 'Access-Control-Allow-Origin' | 'Access-Control-Expose-Headers' | 'Access-Control-Max-Age' | 'Activate-Storage-Access' | 'Age' | 'Allow' | 'Cache-Control' | 'Cache-Group-Invalidation' | 'Cache-Groups' | 'Clear-Site-Data' | 'Content-Disposition' | 'Content-Encoding' | 'Content-Language' | 'Content-Length' | 'Content-Location' | 'Content-Range' | 'Content-Security-Policy' | 'Content-Security-Policy-Report-Only' | 'Content-Type' | 'Cookie' | 'Cross-Origin-Embedder-Policy' | 'Cross-Origin-Opener-Policy' | 'Cross-Origin-Resource-Policy' | 'Date' | 'ETag' | 'Expires' | 'Incremental' | 'Last-Modified' | 'Location' | 'Permissions-Policy' | 'Pragma' | 'Retry-After' | 'Save-Data' | 'Sec-CH-Prefers-Color-Scheme' | 'Sec-CH-Prefers-Reduced-Motion' | 'Sec-CH-UA' | 'Sec-CH-UA-Arch' | 'Sec-CH-UA-Bitness' | 'Sec-CH-UA-Form-Factor' | 'Sec-CH-UA-Full-Version' | 'Sec-CH-UA-Full-Version-List' | 'Sec-CH-UA-Mobile' | 'Sec-CH-UA-Model' | 'Sec-CH-UA-Platform' | 'Sec-CH-UA-Platform-Version' | 'Sec-CH-UA-WoW64' | 'Sec-Fetch-Dest' | 'Sec-Fetch-Mode' | 'Sec-Fetch-Site' | 'Sec-Fetch-User' | 'Sec-GPC' | 'Server' | 'Server-Timing' | 'Service-Worker-Navigation-Preload' | 'Set-Cookie' | 'Set-Txn' | 'Strict-Transport-Security' | 'Timing-Allow-Origin' | 'Trailer' | 'Transfer-Encoding' | 'Unencoded-Digest' | 'Upgrade' | 'Vary' | 'Want-Unencoded-Digest' | 'Warning' | 'WWW-Authenticate' | 'X-Content-Type-Options' | 'X-DNS-Prefetch-Control' | 'X-Frame-Options' | 'X-Permitted-Cross-Domain-Policies' | 'X-Powered-By' | 'X-Robots-Tag' | 'X-XSS-Protection'; +export type AcceptHeader = 'Accept' | 'Accept-Charset' | 'Accept-Encoding' | 'Accept-Language' | 'Accept-Patch' | 'Accept-Post' | 'Accept-Ranges'; +export type CustomHeader = string & {}; +//#endregion \ No newline at end of file diff --git a/tests/fixtures/npm/node_modules/hono/dist/types/utils/http-status.d.ts b/tests/fixtures/npm/node_modules/hono/dist/types/utils/http-status.d.ts new file mode 100644 index 000000000..4c48304e2 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/types/utils/http-status.d.ts @@ -0,0 +1,34 @@ +//#region src/utils/http-status.d.ts +/** + * @module + * HTTP Status utility. + */ +export type InfoStatusCode = 100 | 101 | 102 | 103; +export type SuccessStatusCode = 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 226; +export type DeprecatedStatusCode = 305 | 306; +export type RedirectStatusCode = 300 | 301 | 302 | 303 | 304 | DeprecatedStatusCode | 307 | 308; +export type ClientErrorStatusCode = 400 | 401 | 402 | 403 | 404 | 405 | 406 | 407 | 408 | 409 | 410 | 411 | 412 | 413 | 414 | 415 | 416 | 417 | 418 | 421 | 422 | 423 | 424 | 425 | 426 | 428 | 429 | 431 | 451; +export type ServerErrorStatusCode = 500 | 501 | 502 | 503 | 504 | 505 | 506 | 507 | 508 | 510 | 511; +/** + * `UnofficialStatusCode` can be used to specify an unofficial status code. + * @example + * + * ```ts + * app.get('/unknown', (c) => { + * return c.text("Unknown Error", 520 as UnofficialStatusCode) + * }) + * ``` + */ +export type UnofficialStatusCode = -1; +/** + * @deprecated + * Use `UnofficialStatusCode` instead. + */ +export type UnOfficalStatusCode = UnofficialStatusCode; +/** + * If you want to use an unofficial status, use `UnofficialStatusCode`. + */ +export type StatusCode = InfoStatusCode | SuccessStatusCode | RedirectStatusCode | ClientErrorStatusCode | ServerErrorStatusCode | UnofficialStatusCode; +export type ContentlessStatusCode = 101 | 204 | 205 | 304; +export type ContentfulStatusCode = Exclude; +//#endregion \ No newline at end of file diff --git a/tests/fixtures/npm/node_modules/hono/dist/types/utils/mime.d.ts b/tests/fixtures/npm/node_modules/hono/dist/types/utils/mime.d.ts new file mode 100644 index 000000000..b4ea3597e --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/types/utils/mime.d.ts @@ -0,0 +1,73 @@ +//#region src/utils/mime.d.ts +/** + * @module + * MIME utility. + */ +export declare const getMimeType: (filename: string, mimes?: Record) => string | undefined; +export declare const getExtension: (mimeType: string) => string | undefined; +/** + * Union types for BaseMime + */ +export type BaseMime = (typeof _baseMimes)[keyof typeof _baseMimes]; +declare const _baseMimes: { + readonly aac: "audio/aac"; + readonly avi: "video/x-msvideo"; + readonly avif: "image/avif"; + readonly av1: "video/av1"; + readonly bin: "application/octet-stream"; + readonly bmp: "image/bmp"; + readonly css: "text/css; charset=utf-8"; + readonly csv: "text/csv; charset=utf-8"; + readonly eot: "application/vnd.ms-fontobject"; + readonly epub: "application/epub+zip"; + readonly gif: "image/gif"; + readonly gz: "application/gzip"; + readonly htm: "text/html; charset=utf-8"; + readonly html: "text/html; charset=utf-8"; + readonly ico: "image/x-icon"; + readonly ics: "text/calendar; charset=utf-8"; + readonly jpeg: "image/jpeg"; + readonly jpg: "image/jpeg"; + readonly js: "text/javascript; charset=utf-8"; + readonly json: "application/json"; + readonly jsonld: "application/ld+json"; + readonly map: "application/json"; + readonly mid: "audio/x-midi"; + readonly midi: "audio/x-midi"; + readonly mjs: "text/javascript; charset=utf-8"; + readonly mp3: "audio/mpeg"; + readonly mp4: "video/mp4"; + readonly mpeg: "video/mpeg"; + readonly oga: "audio/ogg"; + readonly ogv: "video/ogg"; + readonly ogx: "application/ogg"; + readonly opus: "audio/opus"; + readonly otf: "font/otf"; + readonly pdf: "application/pdf"; + readonly png: "image/png"; + readonly rtf: "application/rtf"; + readonly svg: "image/svg+xml; charset=utf-8"; + readonly tif: "image/tiff"; + readonly tiff: "image/tiff"; + readonly ts: "video/mp2t"; + readonly ttf: "font/ttf"; + readonly txt: "text/plain; charset=utf-8"; + readonly wasm: "application/wasm"; + readonly webm: "video/webm"; + readonly weba: "audio/webm"; + readonly webmanifest: "application/manifest+json"; + readonly webp: "image/webp"; + readonly woff: "font/woff"; + readonly woff2: "font/woff2"; + readonly xhtml: "application/xhtml+xml; charset=utf-8"; + readonly xml: "application/xml; charset=utf-8"; + readonly zip: "application/zip"; + readonly '3gp': "video/3gpp"; + readonly '3g2': "video/3gpp2"; + readonly gltf: "model/gltf+json"; + readonly glb: "model/gltf-binary"; +}; +declare const baseMimes: Record; +//#endregion +export { baseMimes as mimes }; +export {}; diff --git a/tests/fixtures/npm/node_modules/hono/dist/types/utils/types.d.ts b/tests/fixtures/npm/node_modules/hono/dist/types/utils/types.d.ts new file mode 100644 index 000000000..2a4ad3781 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/types/utils/types.d.ts @@ -0,0 +1,62 @@ +//#region src/utils/types.d.ts +/** + * @module + * Types utility. + */ +export type Expect = T; +export type Equal = (() => T extends X ? 1 : 2) extends (() => T extends Y ? 1 : 2) ? true : false; +export type NotEqual = true extends Equal ? false : true; +export type UnionToIntersection = (U extends any ? (k: U) => void : never) extends ((k: infer I) => void) ? I : never; +export type RemoveBlankRecord = T extends Record ? (K extends string ? T : never) : never; +export type IfAnyThenEmptyObject = 0 extends 1 & T ? {} : T; +export type JSONPrimitive = string | boolean | number | null; +export type JSONArray = (JSONPrimitive | JSONObject | JSONArray)[]; +export type JSONObject = { + [key: string]: JSONPrimitive | JSONArray | JSONObject | object | InvalidJSONValue; +}; +export type InvalidJSONValue = undefined | symbol | ((...args: unknown[]) => unknown); +type InvalidToNull = T extends InvalidJSONValue ? null : T; +type IsInvalid = T extends InvalidJSONValue ? true : false; +/** + * symbol keys are omitted through `JSON.stringify` + */ +type OmitSymbolKeys = { [K in keyof T as K extends symbol ? never : K]: T[K]; }; +export type JSONValue = JSONObject | JSONArray | JSONPrimitive; +/** + * Convert a type to a JSON-compatible type. + * + * Non-JSON values such as `Date` implement `.toJSON()`, + * so they can be transformed to a value assignable to `JSONObject` + * + * `JSON.stringify()` throws a `TypeError` when it encounters a `bigint` value, + * unless a custom `replacer` function or `.toJSON()` method is provided. + * + * This behavior can be controlled by the `TError` generic type parameter, + * which defaults to `bigint | ReadonlyArray`. + * You can set it to `never` to disable this check. + */ +export type JSONParsed> = T extends { + toJSON(): infer J; +} ? (() => J) extends (() => JSONPrimitive) ? J : (() => J) extends (() => { + toJSON(): unknown; +}) ? {} : JSONParsed : T extends JSONPrimitive ? T : T extends InvalidJSONValue ? never : T extends ReadonlyArray ? { [K in keyof T]: JSONParsed, TError>; } extends (infer A) ? A extends ReadonlyArray ? A : JSONParsed, TError>[] : never : T extends Set | Map | Record ? {} : T extends object ? T[keyof T] extends TError ? never : { [K in keyof OmitSymbolKeys as IsInvalid extends true ? never : K]: boolean extends IsInvalid ? JSONParsed | undefined : JSONParsed; } : T extends unknown ? T extends TError ? never : JSONValue : never; +/** + * Useful to flatten the type output to improve type hints shown in editors. And also to transform an interface into a type to aide with assignability. + * @copyright from sindresorhus/type-fest + */ +export type Simplify = { [KeyType in keyof T]: T[KeyType]; } & {}; +/** + * A simple extension of Simplify that will deeply traverse array elements. + */ +export type SimplifyDeepArray = T extends any[] ? { [E in keyof T]: SimplifyDeepArray; } : Simplify; +export type InterfaceToType = T extends Function ? T : { [K in keyof T]: InterfaceToType; }; +export type RequiredKeysOf = Exclude<{ [Key in keyof BaseType]: BaseType extends Record ? Key : never; }[keyof BaseType], undefined>; +export type HasRequiredKeys = RequiredKeysOf extends never ? false : true; +export type IsAny = boolean extends (T extends never ? true : false) ? true : false; +/** + * String literal types with auto-completion + * @see https://github.com/Microsoft/TypeScript/issues/29729 + */ +export type StringLiteralUnion = T | (string & Record); +//#endregion +export {}; diff --git a/tests/fixtures/npm/node_modules/hono/dist/types/validator/index.d.ts b/tests/fixtures/npm/node_modules/hono/dist/types/validator/index.d.ts new file mode 100644 index 000000000..3a494895e --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/types/validator/index.d.ts @@ -0,0 +1,3 @@ +import { InferInput } from "./utils.js"; +import { ValidationFunction, validator } from "./validator.js"; +export { type InferInput, type ValidationFunction, validator }; \ No newline at end of file diff --git a/tests/fixtures/npm/node_modules/hono/dist/types/validator/utils.d.ts b/tests/fixtures/npm/node_modules/hono/dist/types/validator/utils.d.ts new file mode 100644 index 000000000..13159be3c --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/types/validator/utils.d.ts @@ -0,0 +1,27 @@ +import { UnionToIntersection } from "../utils/types.js"; +import { FormValue, ParsedFormValue, ValidationTargets } from "../types.js"; +//#region src/validator/utils.d.ts +/** + * Checks if T is a literal union type (e.g., 'asc' | 'desc') + * that should be preserved in input types. + * Returns true for union literals, false for single literals or wide types. + */ +export type IsLiteralUnion = [Exclude] extends [Base] ? [Exclude] extends [UnionToIntersection>] ? false : true : false; +type IsOptionalUnion = [unknown] extends [T] ? false : undefined extends T ? true : false; +type SimplifyDeep = { [K in keyof T]: T[K]; } & {}; +type InferInputInner = SimplifyDeep<{ [K in keyof Output]: IsLiteralUnion extends true ? Output[K] : IsOptionalUnion extends true ? Output[K] : Target extends 'form' ? T | T[] : Target extends 'query' ? string | string[] : Target extends 'param' ? string : Target extends 'header' ? string : Target extends 'cookie' ? string : unknown; }>; +/** + * Utility type to infer input types for validation targets. + * Preserves literal union types (e.g., 'asc' | 'desc') while using + * the default ValidationTargets type for other values. + * + * @example + * ```ts + * // In @hono/zod-validator or similar: + * type Input = InferInput, 'query'> + * // { orderBy: 'asc' | 'desc', page: string | string[] } + * ``` + */ +export type InferInput = [Exclude] extends [never] ? {} : [Exclude] extends [object] ? undefined extends Output ? SimplifyDeep, Target, T>> | undefined : SimplifyDeep> : {}; +//#endregion +export {}; diff --git a/tests/fixtures/npm/node_modules/hono/dist/types/validator/validator.d.ts b/tests/fixtures/npm/node_modules/hono/dist/types/validator/validator.d.ts new file mode 100644 index 000000000..85e09a1a3 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/types/validator/validator.d.ts @@ -0,0 +1,18 @@ +import { Env, FormValue, MiddlewareHandler, TypedResponse, ValidationTargets } from "../types.js"; +import { Context } from "../context.js"; +import { InferInput } from "./utils.js"; +//#region src/validator/validator.d.ts +type ValidationTargetKeysWithBody = 'form' | 'json'; +type ValidationTargetByMethod = M extends 'get' | 'head' ? Exclude : keyof ValidationTargets; +export type ValidationFunction = (value: InputType, c: Context) => OutputType | TypedResponse | Promise | Promise; +export type ExtractValidationResponse = VF extends ((value: any, c: any) => infer R) ? R extends Promise ? PR extends TypedResponse ? TypedResponse : PR extends Response ? PR : PR extends undefined ? never : never : R extends TypedResponse ? TypedResponse : R extends Response ? R : R extends undefined ? never : never : never; +export declare const validator: , P2 extends string = P, VF extends (value: unknown extends InputType ? ValidationTargets[U] : InputType, c: Context) => any = (value: unknown extends InputType ? ValidationTargets[U] : InputType, c: Context) => any, V extends { + in: { [K in U]: K extends "json" ? unknown extends InputType ? ExtractValidatorOutput : InputType : InferInput, K, FormValue>; }; + out: { [K in U]: ExtractValidatorOutput; }; +} = { + in: { [K in U]: K extends "json" ? unknown extends InputType ? ExtractValidatorOutput : InputType : InferInput, K, FormValue>; }; + out: { [K in U]: ExtractValidatorOutput; }; +}, E extends Env = any>(target: U, validationFunc: VF) => MiddlewareHandler>; +export type ExtractValidatorOutput = VF extends ((value: any, c: any) => infer R) ? R extends Promise ? PR extends Response | TypedResponse ? never : PR : R extends Response | TypedResponse ? never : R : never; +//#endregion +export {}; diff --git a/tests/fixtures/npm/node_modules/hono/dist/utils/body.js b/tests/fixtures/npm/node_modules/hono/dist/utils/body.js new file mode 100644 index 000000000..6fe8e3031 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/utils/body.js @@ -0,0 +1,96 @@ +import { bufferToFormData } from "./buffer.js"; +//#region src/utils/body.ts +const MAX_NESTED_OBJECTS = 1e4; +const isRawRequest = (request) => "headers" in request; +const parseBody = async (request, options = Object.create(null)) => { + const { all = false, dot = false } = options; + const mediaType = (isRawRequest(request) ? request.headers : request.raw.headers).get("Content-Type")?.split(";")[0].trim().toLowerCase(); + if (mediaType === "multipart/form-data" || mediaType === "application/x-www-form-urlencoded") return parseFormData(request, { + all, + dot + }); + return {}; +}; +/** +* Parses form data from a request. +* +* @template T - The type of the parsed body data. +* @param {HonoRequest | Request} request - The request object containing form data. +* @param {ParseBodyOptions} options - Options for parsing the form data. +* @returns {Promise} The parsed body data. +*/ +async function parseFormData(request, options) { + if (!isRawRequest(request) && request.bodyCache.formData) return convertFormDataToBodyData(await request.bodyCache.formData, options); + const headers = isRawRequest(request) ? request.headers : request.raw.headers; + const arrayBuffer = await request.arrayBuffer(); + const formDataPromise = bufferToFormData(arrayBuffer, headers.get("Content-Type") || ""); + if (!isRawRequest(request)) request.bodyCache.formData = formDataPromise; + const formData = await formDataPromise; + if (formData) return convertFormDataToBodyData(formData, options); + return {}; +} +/** +* Converts form data to body data based on the provided options. +* +* @template T - The type of the parsed body data. +* @param {FormData} formData - The form data to convert. +* @param {ParseBodyOptions} options - Options for parsing the form data. +* @returns {T} The converted body data. +*/ +function convertFormDataToBodyData(formData, options) { + const form = Object.create(null); + const nestingState = { count: 0 }; + formData.forEach((value, key) => { + if (!(options.all || key.endsWith("[]"))) form[key] = value; + else handleParsingAllValues(form, key, value); + }); + if (options.dot) Object.entries(form).forEach(([key, value]) => { + if (key.includes(".")) { + handleParsingNestedValues(form, key, value, nestingState); + delete form[key]; + } + }); + return form; +} +/** +* Handles parsing all values for a given key, supporting multiple values as arrays. +* +* @param {BodyData} form - The form data object. +* @param {string} key - The key to parse. +* @param {FormDataEntryValue} value - The value to assign. +*/ +const handleParsingAllValues = (form, key, value) => { + if (form[key] !== void 0) { + if (Array.isArray(form[key])) form[key].push(value); + else form[key] = [form[key], value]; + } else if (!key.endsWith("[]")) form[key] = value; + else form[key] = [value]; +}; +/** +* Handles parsing nested values using dot notation keys. +* +* @param {BodyData} form - The form data object. +* @param {string} key - The dot notation key. +* @param {BodyDataValue} value - The value to assign. +*/ +const handleParsingNestedValues = (form, key, value, state) => { + if (/(?:^|\.)__proto__\./.test(key)) return; + let nestedForm = form; + const keys = key.split(".", 34); + if (keys.length > 33) throwNestingLimitExceeded(); + keys.forEach((key, index) => { + if (index === keys.length - 1) nestedForm[key] = value; + else { + if (!nestedForm[key] || typeof nestedForm[key] !== "object" || Array.isArray(nestedForm[key]) || nestedForm[key] instanceof File) { + if (state.count++ >= MAX_NESTED_OBJECTS) throwNestingLimitExceeded(); + nestedForm[key] = Object.create(null); + } + nestedForm = nestedForm[key]; + } + }); +}; +const throwNestingLimitExceeded = () => { + throw new Error("Nesting limit exceeded"); +}; +//#endregion +export { parseBody }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/utils/buffer.js b/tests/fixtures/npm/node_modules/hono/dist/utils/buffer.js new file mode 100644 index 000000000..8c6506452 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/utils/buffer.js @@ -0,0 +1,51 @@ +import { sha256 } from "./crypto.js"; +//#region src/utils/buffer.ts +/** +* @module +* Buffer utility. +*/ +const equal = (a, b) => { + if (a === b) return true; + if (a.byteLength !== b.byteLength) return false; + const va = new DataView(a); + const vb = new DataView(b); + let i = va.byteLength; + while (i--) if (va.getUint8(i) !== vb.getUint8(i)) return false; + return true; +}; +const constantTimeEqualString = (a, b) => { + const aLen = a.length; + const bLen = b.length; + const maxLen = Math.max(aLen, bLen); + let out = aLen ^ bLen; + for (let i = 0; i < maxLen; i++) { + const aChar = i < aLen ? a.charCodeAt(i) : 0; + const bChar = i < bLen ? b.charCodeAt(i) : 0; + out |= aChar ^ bChar; + } + return out === 0; +}; +const timingSafeEqualString = async (a, b, hashFunction) => { + if (!hashFunction) hashFunction = sha256; + const [sa, sb] = await Promise.all([hashFunction(a), hashFunction(b)]); + if (sa == null || sb == null || typeof sa !== "string" || typeof sb !== "string") return false; + const hashEqual = constantTimeEqualString(sa, sb); + const originalEqual = constantTimeEqualString(a, b); + return hashEqual && originalEqual; +}; +const timingSafeEqual = async (a, b, hashFunction) => { + if (typeof a === "string" && typeof b === "string") return timingSafeEqualString(a, b, hashFunction); + if (!hashFunction) hashFunction = sha256; + const [sa, sb] = await Promise.all([hashFunction(a), hashFunction(b)]); + if (!sa || !sb || typeof sa !== "string" || typeof sb !== "string") return false; + return timingSafeEqualString(sa, sb); +}; +const bufferToString = (buffer) => { + if (buffer instanceof ArrayBuffer) return new TextDecoder("utf-8").decode(buffer); + return buffer; +}; +const bufferToFormData = (arrayBuffer, contentType) => { + return new Response(arrayBuffer, { headers: { "Content-Type": contentType.replace(/^[^;]+/, (mediaType) => mediaType.toLowerCase()) } }).formData(); +}; +//#endregion +export { bufferToFormData, bufferToString, equal, timingSafeEqual }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/utils/constants.js b/tests/fixtures/npm/node_modules/hono/dist/utils/constants.js new file mode 100644 index 000000000..8b0615b69 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/utils/constants.js @@ -0,0 +1,7 @@ +//#region src/utils/constants.ts +/** +* Constant used to mark a composed handler. +*/ +const COMPOSED_HANDLER = "__COMPOSED_HANDLER"; +//#endregion +export { COMPOSED_HANDLER }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/utils/cookie.js b/tests/fixtures/npm/node_modules/hono/dist/utils/cookie.js new file mode 100644 index 000000000..fd541131c --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/utils/cookie.js @@ -0,0 +1,125 @@ +import { tryDecodeURIComponent } from "./url.js"; +//#region src/utils/cookie.ts +/** +* @module +* Cookie utility. +*/ +const algorithm = { + name: "HMAC", + hash: "SHA-256" +}; +const getCryptoKey = async (secret) => { + const secretBuf = typeof secret === "string" ? new TextEncoder().encode(secret) : secret; + return await crypto.subtle.importKey("raw", secretBuf, algorithm, false, ["sign", "verify"]); +}; +const makeSignature = async (value, secret) => { + const key = await getCryptoKey(secret); + const signature = await crypto.subtle.sign(algorithm.name, key, new TextEncoder().encode(value)); + return btoa(String.fromCharCode(...new Uint8Array(signature))); +}; +const verifySignature = async (base64Signature, value, secret) => { + try { + const signatureBinStr = atob(base64Signature); + const signature = new Uint8Array(signatureBinStr.length); + for (let i = 0, len = signatureBinStr.length; i < len; i++) signature[i] = signatureBinStr.charCodeAt(i); + return await crypto.subtle.verify(algorithm, secret, signature, new TextEncoder().encode(value)); + } catch { + return false; + } +}; +const validCookieNameRegEx = /^[\w!#$%&'*.^`|~+-]+$/; +const relaxedCookieNameRegEx = /^[!#-:<>-[\]-~]+$/; +const validCookieValueRegEx = /^[ !#-:<-[\]-~]*$/; +const trimCookieWhitespace = (value) => { + let start = 0; + let end = value.length; + while (start < end) { + const charCode = value.charCodeAt(start); + if (charCode !== 32 && charCode !== 9) break; + start++; + } + while (end > start) { + const charCode = value.charCodeAt(end - 1); + if (charCode !== 32 && charCode !== 9) break; + end--; + } + return start === 0 && end === value.length ? value : value.slice(start, end); +}; +const parse = (cookie, name) => { + if (name && cookie.indexOf(name) === -1) return {}; + const pairs = cookie.split(";"); + const parsedCookie = Object.create(null); + for (const pairStr of pairs) { + const valueStartPos = pairStr.indexOf("="); + if (valueStartPos === -1) continue; + const cookieName = trimCookieWhitespace(pairStr.substring(0, valueStartPos)); + if (name && name !== cookieName || !relaxedCookieNameRegEx.test(cookieName) || cookieName in parsedCookie) continue; + let cookieValue = trimCookieWhitespace(pairStr.substring(valueStartPos + 1)); + if (cookieValue.startsWith("\"") && cookieValue.endsWith("\"")) cookieValue = cookieValue.slice(1, -1); + if (validCookieValueRegEx.test(cookieValue)) { + parsedCookie[cookieName] = tryDecodeURIComponent(cookieValue); + if (name) break; + } + } + return parsedCookie; +}; +const parseSigned = async (cookie, secret, name) => { + const parsedCookie = Object.create(null); + const secretKey = await getCryptoKey(secret); + for (const [key, value] of Object.entries(parse(cookie, name))) { + const signatureStartPos = value.lastIndexOf("."); + if (signatureStartPos < 0) continue; + const signedValue = value.substring(0, signatureStartPos); + const signature = value.substring(signatureStartPos + 1); + if (signature.length !== 44 || !signature.endsWith("=")) continue; + parsedCookie[key] = await verifySignature(signature, signedValue, secretKey) ? signedValue : false; + } + return parsedCookie; +}; +const _serialize = (name, value, opt = {}) => { + if (!validCookieNameRegEx.test(name)) throw new Error("Invalid cookie name"); + let cookie = `${name}=${value}`; + if (name.startsWith("__Secure-") && !opt.secure) throw new Error("__Secure- Cookie must have Secure attributes"); + if (name.startsWith("__Host-")) { + if (!opt.secure) throw new Error("__Host- Cookie must have Secure attributes"); + if (opt.path !== "/") throw new Error("__Host- Cookie must have Path attributes with \"/\""); + if (opt.domain) throw new Error("__Host- Cookie must not have Domain attributes"); + } + for (const key of [ + "domain", + "path", + "sameSite", + "priority" + ]) if (opt[key] && /[;\r\n]/.test(opt[key])) throw new Error(`${key} must not contain ";", "\\r", or "\\n"`); + if (opt && typeof opt.maxAge === "number" && opt.maxAge >= 0) { + if (opt.maxAge > 3456e4) throw new Error("Cookies Max-Age SHOULD NOT be greater than 400 days (34560000 seconds) in duration."); + cookie += `; Max-Age=${opt.maxAge | 0}`; + } + if (opt.domain && opt.prefix !== "host") cookie += `; Domain=${opt.domain}`; + if (opt.path) cookie += `; Path=${opt.path}`; + if (opt.expires) { + if (opt.expires.getTime() - Date.now() > 3456e7) throw new Error("Cookies Expires SHOULD NOT be greater than 400 days (34560000 seconds) in the future."); + cookie += `; Expires=${opt.expires.toUTCString()}`; + } + if (opt.httpOnly) cookie += "; HttpOnly"; + if (opt.secure) cookie += "; Secure"; + if (opt.sameSite) cookie += `; SameSite=${opt.sameSite.charAt(0).toUpperCase() + opt.sameSite.slice(1)}`; + if (opt.priority) cookie += `; Priority=${opt.priority.charAt(0).toUpperCase() + opt.priority.slice(1)}`; + if (opt.partitioned) { + if (!opt.secure) throw new Error("Partitioned Cookie must have Secure attributes"); + cookie += "; Partitioned"; + } + return cookie; +}; +const serialize = (name, value, opt) => { + value = encodeURIComponent(value); + return _serialize(name, value, opt); +}; +const serializeSigned = async (name, value, secret, opt = {}) => { + const signature = await makeSignature(value, secret); + value = `${value}.${signature}`; + value = encodeURIComponent(value); + return _serialize(name, value, opt); +}; +//#endregion +export { parse, parseSigned, serialize, serializeSigned }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/utils/crypto.js b/tests/fixtures/npm/node_modules/hono/dist/utils/crypto.js new file mode 100644 index 000000000..548867c49 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/utils/crypto.js @@ -0,0 +1,34 @@ +//#region src/utils/crypto.ts +const sha256 = async (data) => { + return await createHash(data, { + name: "SHA-256", + alias: "sha256" + }); +}; +const sha1 = async (data) => { + return await createHash(data, { + name: "SHA-1", + alias: "sha1" + }); +}; +const md5 = async (data) => { + return await createHash(data, { + name: "MD5", + alias: "md5" + }); +}; +const createHash = async (data, algorithm) => { + let sourceBuffer; + if (ArrayBuffer.isView(data) || data instanceof ArrayBuffer) sourceBuffer = data; + else { + if (typeof data === "object") data = JSON.stringify(data); + sourceBuffer = new TextEncoder().encode(String(data)); + } + if (crypto && crypto.subtle) { + const buffer = await crypto.subtle.digest({ name: algorithm.name }, sourceBuffer); + return Array.prototype.map.call(new Uint8Array(buffer), (x) => ("00" + x.toString(16)).slice(-2)).join(""); + } + return null; +}; +//#endregion +export { createHash, md5, sha1, sha256 }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/utils/html.js b/tests/fixtures/npm/node_modules/hono/dist/utils/html.js new file mode 100644 index 000000000..7615ebdbd --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/utils/html.js @@ -0,0 +1,103 @@ +//#region src/utils/html.ts +/** +* @module +* HTML utility. +*/ +const HtmlEscapedCallbackPhase = { + Stringify: 1, + BeforeStream: 2, + Stream: 3 +}; +const raw = (value, callbacks) => { + const escapedString = new String(value); + escapedString.isEscaped = true; + escapedString.callbacks = callbacks; + return escapedString; +}; +const escapeRe = /[&<>'"]/; +const stringBufferToString = async (buffer, callbacks) => { + let str = ""; + callbacks ||= []; + const resolvedBuffer = await Promise.all(buffer); + for (let i = resolvedBuffer.length - 1;; i--) { + str += resolvedBuffer[i]; + i--; + if (i < 0) break; + let r = resolvedBuffer[i]; + if (typeof r === "object") callbacks.push(...r.callbacks || []); + const isEscaped = r.isEscaped; + r = await (typeof r === "object" ? r.toString() : r); + if (typeof r === "object") callbacks.push(...r.callbacks || []); + if (r.isEscaped ?? isEscaped) str += r; + else { + const buf = [str]; + escapeToBuffer(r, buf); + str = buf[0]; + } + } + return raw(str, callbacks); +}; +const escapeToBuffer = (str, buffer) => { + const match = str.search(escapeRe); + if (match === -1) { + buffer[0] += str; + return; + } + let escape; + let index; + let lastIndex = 0; + for (index = match; index < str.length; index++) { + switch (str.charCodeAt(index)) { + case 34: + escape = """; + break; + case 39: + escape = "'"; + break; + case 38: + escape = "&"; + break; + case 60: + escape = "<"; + break; + case 62: + escape = ">"; + break; + default: continue; + } + buffer[0] += str.substring(lastIndex, index) + escape; + lastIndex = index + 1; + } + buffer[0] += str.substring(lastIndex, index); +}; +const resolveCallbackSync = (str) => { + const callbacks = str.callbacks; + if (!callbacks?.length) return str; + const buffer = [str]; + const context = {}; + callbacks.forEach((c) => c({ + phase: HtmlEscapedCallbackPhase.Stringify, + buffer, + context + })); + return buffer[0]; +}; +const resolveCallback = async (str, phase, preserveCallbacks, context, buffer) => { + if (typeof str === "object" && !(str instanceof String)) { + if (!(str instanceof Promise)) str = str.toString(); + if (str instanceof Promise) str = await str; + } + const callbacks = str.callbacks; + if (!callbacks?.length) return Promise.resolve(str); + if (buffer) buffer[0] += str; + else buffer = [str]; + const resStr = Promise.all(callbacks.map((c) => c({ + phase, + buffer, + context + }))).then((res) => Promise.all(res.filter(Boolean).map((str) => resolveCallback(str, phase, false, context, buffer))).then(() => buffer[0])); + if (preserveCallbacks) return raw(await resStr, callbacks); + else return resStr; +}; +//#endregion +export { HtmlEscapedCallbackPhase, escapeToBuffer, raw, resolveCallback, resolveCallbackSync, stringBufferToString }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/utils/url.js b/tests/fixtures/npm/node_modules/hono/dist/utils/url.js new file mode 100644 index 000000000..8433d4cb9 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/utils/url.js @@ -0,0 +1,201 @@ +//#region src/utils/url.ts +const splitPath = (path) => { + const paths = path.split("/"); + if (paths[0] === "") paths.shift(); + return paths; +}; +const splitRoutingPath = (routePath) => { + const { groups, path } = extractGroupsFromPath(routePath); + const paths = splitPath(path); + return replaceGroupMarks(paths, groups); +}; +const extractGroupsFromPath = (path) => { + const groups = []; + path = path.replace(/\{[^}]+\}/g, (match, index) => { + const mark = `@${index}`; + groups.push([mark, match]); + return mark; + }); + return { + groups, + path + }; +}; +const replaceGroupMarks = (paths, groups) => { + for (let i = groups.length - 1; i >= 0; i--) { + const [mark] = groups[i]; + for (let j = paths.length - 1; j >= 0; j--) if (paths[j].includes(mark)) { + paths[j] = paths[j].replace(mark, groups[i][1]); + break; + } + } + return paths; +}; +const patternCache = {}; +const getPattern = (label, next) => { + if (label === "*") return "*"; + const match = label.match(/^\:([^\{\}]+)(?:\{(.+)\})?$/); + if (match) { + const cacheKey = `${label}#${next}`; + if (!patternCache[cacheKey]) { + if (match[2]) patternCache[cacheKey] = next && next[0] !== ":" && next[0] !== "*" ? [ + cacheKey, + match[1], + new RegExp(`^${match[2]}(?=/${next})`) + ] : [ + label, + match[1], + new RegExp(`^${match[2]}$`) + ]; + else patternCache[cacheKey] = [ + label, + match[1], + true + ]; + } + return patternCache[cacheKey]; + } + return null; +}; +const tryDecode = (str, decoder) => { + try { + return decoder(str); + } catch { + return str.replace(/(?:%[0-9A-Fa-f]{2})+/g, (match) => { + try { + return decoder(match); + } catch { + return match; + } + }); + } +}; +/** +* Try to apply decodeURI() to given string. +* If it fails, skip invalid percent encoding or invalid UTF-8 sequences, and apply decodeURI() to the rest as much as possible. +* @param str The string to decode. +* @returns The decoded string that sometimes contains undecodable percent encoding. +* @example +* tryDecodeURI('Hello%20World') // 'Hello World' +* tryDecodeURI('Hello%20World/%A4%A2') // 'Hello World/%A4%A2' +*/ +const tryDecodeURI = (str) => tryDecode(str, decodeURI); +const getPath = (request) => { + const url = request.url; + const start = url.indexOf("/", url.indexOf(":") + 4); + let i = start; + for (; i < url.length; i++) { + const charCode = url.charCodeAt(i); + if (charCode === 37) { + const queryIndex = url.indexOf("?", i); + const hashIndex = url.indexOf("#", i); + const end = queryIndex === -1 ? hashIndex === -1 ? void 0 : hashIndex : hashIndex === -1 ? queryIndex : Math.min(queryIndex, hashIndex); + const path = url.slice(start, end); + return tryDecodeURI(path.includes("%25") ? path.replace(/%25/g, "%2525") : path); + } else if (charCode === 63 || charCode === 35) break; + } + return url.slice(start, i); +}; +/** +* @deprecated +* Use the `URL` API instead. +*/ +const getQueryStrings = (url) => { + const queryIndex = url.indexOf("?", 8); + if (queryIndex === -1) return ""; + const hashIndex = url.indexOf("#", 8); + return hashIndex === -1 ? url.slice(queryIndex) : queryIndex < hashIndex ? url.slice(queryIndex, hashIndex) : ""; +}; +const getPathNoStrict = (request) => { + const result = getPath(request); + return result.length > 1 && result.at(-1) === "/" ? result.slice(0, -1) : result; +}; +/** +* Merge paths. +* @param {string[]} ...paths - The paths to merge. +* @returns {string} The merged path. +* @example +* mergePath('/api', '/users') // '/api/users' +* mergePath('/api/', '/users') // '/api/users' +* mergePath('/api', '/') // '/api' +* mergePath('/api/', '/') // '/api/' +*/ +const mergePath = (base, sub, ...rest) => { + if (rest.length) sub = mergePath(sub, ...rest); + return `${base?.[0] === "/" ? "" : "/"}${base}${sub === "/" ? "" : `${base?.at(-1) === "/" ? "" : "/"}${sub?.[0] === "/" ? sub.slice(1) : sub}`}`; +}; +const checkOptionalParameter = (path) => { + if (path.charCodeAt(path.length - 1) !== 63 || !path.includes(":")) return null; + const segments = path.split("/"); + const results = []; + let basePath = ""; + segments.forEach((segment) => { + if (segment !== "" && !/\:/.test(segment)) basePath += "/" + segment; + else if (/\:/.test(segment)) { + if (segment.charCodeAt(segment.length - 1) === 63) { + if (results.length === 0 && basePath === "") results.push("/"); + else results.push(basePath); + const optionalSegment = segment.slice(0, -1); + basePath += "/" + optionalSegment; + results.push(basePath); + } else basePath += "/" + segment; + } + }); + return results.filter((v, i, a) => a.indexOf(v) === i); +}; +const tryDecodeURIComponent = (str) => str.indexOf("%") !== -1 ? tryDecode(str, decodeURIComponent_) : str; +const _decodeURI = (value) => { + if (value.indexOf("+") !== -1) value = value.replace(/\+/g, " "); + return tryDecodeURIComponent(value); +}; +const _getQueryParam = (url, key, multiple) => { + const hashIndex = url.indexOf("#", 8); + if (hashIndex !== -1) url = url.slice(0, hashIndex); + let encoded; + if (!multiple && key && key.indexOf("%") === -1 && key.indexOf("+") === -1) { + let keyIndex = url.indexOf("?", 8); + if (keyIndex === -1) return; + if (!url.startsWith(key, keyIndex + 1)) keyIndex = url.indexOf(`&${key}`, keyIndex + 1); + while (keyIndex !== -1) { + const trailingKeyCode = url.charCodeAt(keyIndex + key.length + 1); + if (trailingKeyCode === 61) { + const valueIndex = keyIndex + key.length + 2; + const endIndex = url.indexOf("&", valueIndex); + return _decodeURI(url.slice(valueIndex, endIndex === -1 ? void 0 : endIndex)); + } else if (trailingKeyCode == 38 || isNaN(trailingKeyCode)) return ""; + keyIndex = url.indexOf(`&${key}`, keyIndex + 1); + } + encoded = /[%+]/.test(url); + if (!encoded) return; + } + const results = Object.create(null); + encoded ??= /[%+]/.test(url); + let keyIndex = url.indexOf("?", 8); + while (keyIndex !== -1) { + const nextKeyIndex = url.indexOf("&", keyIndex + 1); + let valueIndex = url.indexOf("=", keyIndex); + if (valueIndex > nextKeyIndex && nextKeyIndex !== -1) valueIndex = -1; + let name = url.slice(keyIndex + 1, valueIndex === -1 ? nextKeyIndex === -1 ? void 0 : nextKeyIndex : valueIndex); + if (encoded) name = _decodeURI(name); + keyIndex = nextKeyIndex; + if (name === "") continue; + let value; + if (valueIndex === -1) value = ""; + else { + value = url.slice(valueIndex + 1, nextKeyIndex === -1 ? void 0 : nextKeyIndex); + if (encoded) value = _decodeURI(value); + } + if (multiple) { + if (!(results[name] && Array.isArray(results[name]))) results[name] = []; + results[name].push(value); + } else results[name] ??= value; + } + return key ? results[key] : results; +}; +const getQueryParam = _getQueryParam; +const getQueryParams = (url, key) => { + return _getQueryParam(url, key, true); +}; +const decodeURIComponent_ = decodeURIComponent; +//#endregion +export { checkOptionalParameter, decodeURIComponent_, getPath, getPathNoStrict, getPattern, getQueryParam, getQueryParams, getQueryStrings, mergePath, splitPath, splitRoutingPath, tryDecode, tryDecodeURI, tryDecodeURIComponent }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/validator/index.js b/tests/fixtures/npm/node_modules/hono/dist/validator/index.js new file mode 100644 index 000000000..6e744aca6 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/validator/index.js @@ -0,0 +1,2 @@ +import { validator } from "./validator.js"; +export { validator }; diff --git a/tests/fixtures/npm/node_modules/hono/dist/validator/validator.js b/tests/fixtures/npm/node_modules/hono/dist/validator/validator.js new file mode 100644 index 000000000..1cc282071 --- /dev/null +++ b/tests/fixtures/npm/node_modules/hono/dist/validator/validator.js @@ -0,0 +1,64 @@ +import { HTTPException } from "../http-exception.js"; +import { bufferToFormData } from "../utils/buffer.js"; +import { getCookie } from "../helper/cookie/index.js"; +//#region src/validator/validator.ts +const jsonRegex = /^application\/([a-z-\.]+\+)?json(;\s*[a-zA-Z0-9\-]+\=([^;]+))*$/i; +const multipartRegex = /^multipart\/form-data(;\s?boundary=[a-zA-Z0-9'"()+_,\-./:=?]+)?$/i; +const urlencodedRegex = /^application\/x-www-form-urlencoded(;\s*[a-zA-Z0-9\-]+\=([^;]+))*$/i; +const validator = (target, validationFunc) => { + return async (c, next) => { + let value = {}; + const contentType = c.req.header("Content-Type"); + switch (target) { + case "json": + if (!contentType || !jsonRegex.test(contentType)) break; + try { + value = await c.req.json(); + } catch { + throw new HTTPException(400, { message: "Malformed JSON in request body" }); + } + break; + case "form": { + if (!contentType || !(multipartRegex.test(contentType) || urlencodedRegex.test(contentType))) break; + let formData; + if (c.req.bodyCache.formData) formData = await c.req.bodyCache.formData; + else try { + const arrayBuffer = await c.req.arrayBuffer(); + formData = await bufferToFormData(arrayBuffer, contentType); + c.req.bodyCache.formData = formData; + } catch (e) { + let message = "Malformed FormData request."; + message += e instanceof Error ? ` ${e.message}` : ` ${String(e)}`; + throw new HTTPException(400, { message }); + } + const form = Object.create(null); + formData.forEach((value, key) => { + if (key.endsWith("[]")) (form[key] ??= []).push(value); + else if (Array.isArray(form[key])) form[key].push(value); + else if (Object.hasOwn(form, key)) form[key] = [form[key], value]; + else form[key] = value; + }); + value = form; + break; + } + case "query": + value = Object.fromEntries(Object.entries(c.req.queries()).map(([k, v]) => { + return v.length === 1 ? [k, v[0]] : [k, v]; + })); + break; + case "param": + value = c.req.param(); + break; + case "header": + value = c.req.header(); + break; + case "cookie": value = getCookie(c); + } + const res = await validationFunc(value, c); + if (res instanceof Response) return res; + c.req.addValidatedData(target, res); + return await next(); + }; +}; +//#endregion +export { validator }; diff --git a/tests/fixtures/npm/node_modules/hono/package.json b/tests/fixtures/npm/node_modules/hono/package.json index 6264c824a..6c7dec6e9 100644 --- a/tests/fixtures/npm/node_modules/hono/package.json +++ b/tests/fixtures/npm/node_modules/hono/package.json @@ -1,10 +1,18 @@ { "name": "hono", "version": "4.13.13", - "description": "Test fixture: only the hono/ws helper that @hono/node-server's index imports.", + "description": "Test fixture: the hono core and validator (with their declarations) and the hono/ws helper that @hono/node-server's index imports.", "type": "module", "license": "MIT", "exports": { + ".": { + "types": "./dist/types/index.d.ts", + "import": "./dist/index.js" + }, + "./validator": { + "types": "./dist/types/validator/index.d.ts", + "import": "./dist/validator/index.js" + }, "./ws": { "import": "./dist/helper/websocket/index.js" } diff --git a/tests/fixtures/npm/node_modules/zod/LICENSE b/tests/fixtures/npm/node_modules/zod/LICENSE new file mode 100644 index 000000000..c06579627 --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Colin McDonnell + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tests/fixtures/npm/node_modules/zod/index.d.cts b/tests/fixtures/npm/node_modules/zod/index.d.cts new file mode 100644 index 000000000..b8f3021a8 --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/index.d.cts @@ -0,0 +1,3 @@ +import * as z from "./v4/classic/external.cjs"; +export * from "./v4/classic/external.cjs"; +export { z, z as default }; diff --git a/tests/fixtures/npm/node_modules/zod/index.js b/tests/fixtures/npm/node_modules/zod/index.js new file mode 100644 index 000000000..4102a4715 --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/index.js @@ -0,0 +1,7 @@ +import * as z from "./v4/classic/external.js"; +export * from "./v4/classic/external.js"; +// Aliasing the binding, instead of `export default z`, keeps the default export a +// re-export of `z` rather than a fresh namespace value. Rollup and Webpack can then +// shake `import z from "zod"` as they already shake `import { z } from "zod"` — without +// this they pull in every locale. See #6050. +export { z, z as default }; diff --git a/tests/fixtures/npm/node_modules/zod/package.json b/tests/fixtures/npm/node_modules/zod/package.json new file mode 100644 index 000000000..58b9916bd --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/package.json @@ -0,0 +1,23 @@ +{ + "name": "zod", + "version": "4.6.7", + "description": "Test fixture: zod 4's classic API, its core and locales, and the declarations @hono/zod-validator reads (the files the npm cases load).", + "type": "module", + "license": "MIT", + "types": "./index.d.cts", + "exports": { + ".": { + "types": "./index.d.cts", + "import": "./index.js" + }, + "./v3": { + "types": "./v3/index.d.cts" + }, + "./v4": { + "types": "./v4/index.d.cts" + }, + "./v4/core": { + "types": "./v4/core/index.d.cts" + } + } +} diff --git a/tests/fixtures/npm/node_modules/zod/v3/ZodError.d.cts b/tests/fixtures/npm/node_modules/zod/v3/ZodError.d.cts new file mode 100644 index 000000000..384b36487 --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v3/ZodError.d.cts @@ -0,0 +1,164 @@ +import type { Primitive } from "./helpers/typeAliases.cjs"; +import { util, type ZodParsedType } from "./helpers/util.cjs"; +import type { TypeOf, ZodType } from "./index.cjs"; +type allKeys = T extends any ? keyof T : never; +export type inferFlattenedErrors, U = string> = typeToFlattenedError, U>; +export type typeToFlattenedError = { + formErrors: U[]; + fieldErrors: { + [P in allKeys]?: U[]; + }; +}; +export declare const ZodIssueCode: { + invalid_type: "invalid_type"; + too_big: "too_big"; + too_small: "too_small"; + not_multiple_of: "not_multiple_of"; + unrecognized_keys: "unrecognized_keys"; + invalid_union: "invalid_union"; + custom: "custom"; + invalid_literal: "invalid_literal"; + invalid_union_discriminator: "invalid_union_discriminator"; + invalid_enum_value: "invalid_enum_value"; + invalid_arguments: "invalid_arguments"; + invalid_return_type: "invalid_return_type"; + invalid_date: "invalid_date"; + invalid_string: "invalid_string"; + invalid_intersection_types: "invalid_intersection_types"; + not_finite: "not_finite"; +}; +export type ZodIssueCode = keyof typeof ZodIssueCode; +export type ZodIssueBase = { + path: (string | number)[]; + message?: string | undefined; +}; +export interface ZodInvalidTypeIssue extends ZodIssueBase { + code: typeof ZodIssueCode.invalid_type; + expected: ZodParsedType; + received: ZodParsedType; +} +export interface ZodInvalidLiteralIssue extends ZodIssueBase { + code: typeof ZodIssueCode.invalid_literal; + expected: unknown; + received: unknown; +} +export interface ZodUnrecognizedKeysIssue extends ZodIssueBase { + code: typeof ZodIssueCode.unrecognized_keys; + keys: string[]; +} +export interface ZodInvalidUnionIssue extends ZodIssueBase { + code: typeof ZodIssueCode.invalid_union; + unionErrors: ZodError[]; +} +export interface ZodInvalidUnionDiscriminatorIssue extends ZodIssueBase { + code: typeof ZodIssueCode.invalid_union_discriminator; + options: Primitive[]; +} +export interface ZodInvalidEnumValueIssue extends ZodIssueBase { + received: string | number; + code: typeof ZodIssueCode.invalid_enum_value; + options: (string | number)[]; +} +export interface ZodInvalidArgumentsIssue extends ZodIssueBase { + code: typeof ZodIssueCode.invalid_arguments; + argumentsError: ZodError; +} +export interface ZodInvalidReturnTypeIssue extends ZodIssueBase { + code: typeof ZodIssueCode.invalid_return_type; + returnTypeError: ZodError; +} +export interface ZodInvalidDateIssue extends ZodIssueBase { + code: typeof ZodIssueCode.invalid_date; +} +export type StringValidation = "email" | "url" | "emoji" | "uuid" | "nanoid" | "regex" | "cuid" | "cuid2" | "ulid" | "datetime" | "date" | "time" | "duration" | "ip" | "cidr" | "base64" | "jwt" | "base64url" | { + includes: string; + position?: number | undefined; +} | { + startsWith: string; +} | { + endsWith: string; +}; +export interface ZodInvalidStringIssue extends ZodIssueBase { + code: typeof ZodIssueCode.invalid_string; + validation: StringValidation; +} +export interface ZodTooSmallIssue extends ZodIssueBase { + code: typeof ZodIssueCode.too_small; + minimum: number | bigint; + inclusive: boolean; + exact?: boolean; + type: "array" | "string" | "number" | "set" | "date" | "bigint"; +} +export interface ZodTooBigIssue extends ZodIssueBase { + code: typeof ZodIssueCode.too_big; + maximum: number | bigint; + inclusive: boolean; + exact?: boolean; + type: "array" | "string" | "number" | "set" | "date" | "bigint"; +} +export interface ZodInvalidIntersectionTypesIssue extends ZodIssueBase { + code: typeof ZodIssueCode.invalid_intersection_types; +} +export interface ZodNotMultipleOfIssue extends ZodIssueBase { + code: typeof ZodIssueCode.not_multiple_of; + multipleOf: number | bigint; +} +export interface ZodNotFiniteIssue extends ZodIssueBase { + code: typeof ZodIssueCode.not_finite; +} +export interface ZodCustomIssue extends ZodIssueBase { + code: typeof ZodIssueCode.custom; + params?: { + [k: string]: any; + }; +} +export type DenormalizedError = { + [k: string]: DenormalizedError | string[]; +}; +export type ZodIssueOptionalMessage = ZodInvalidTypeIssue | ZodInvalidLiteralIssue | ZodUnrecognizedKeysIssue | ZodInvalidUnionIssue | ZodInvalidUnionDiscriminatorIssue | ZodInvalidEnumValueIssue | ZodInvalidArgumentsIssue | ZodInvalidReturnTypeIssue | ZodInvalidDateIssue | ZodInvalidStringIssue | ZodTooSmallIssue | ZodTooBigIssue | ZodInvalidIntersectionTypesIssue | ZodNotMultipleOfIssue | ZodNotFiniteIssue | ZodCustomIssue; +export type ZodIssue = ZodIssueOptionalMessage & { + fatal?: boolean | undefined; + message: string; +}; +export declare const quotelessJson: (obj: any) => string; +type recursiveZodFormattedError = T extends [any, ...any[]] ? { + [K in keyof T]?: ZodFormattedError; +} : T extends any[] ? { + [k: number]: ZodFormattedError; +} : T extends object ? { + [K in keyof T]?: ZodFormattedError; +} : unknown; +export type ZodFormattedError = { + _errors: U[]; +} & recursiveZodFormattedError>; +export type inferFormattedError, U = string> = ZodFormattedError, U>; +export declare class ZodError extends Error { + issues: ZodIssue[]; + get errors(): ZodIssue[]; + constructor(issues: ZodIssue[]); + format(): ZodFormattedError; + format(mapper: (issue: ZodIssue) => U): ZodFormattedError; + static create: (issues: ZodIssue[]) => ZodError; + static assert(value: unknown): asserts value is ZodError; + toString(): string; + get message(): string; + get isEmpty(): boolean; + addIssue: (sub: ZodIssue) => void; + addIssues: (subs?: ZodIssue[]) => void; + flatten(): typeToFlattenedError; + flatten(mapper?: (issue: ZodIssue) => U): typeToFlattenedError; + get formErrors(): typeToFlattenedError; +} +type stripPath = T extends any ? util.OmitKeys : never; +export type IssueData = stripPath & { + path?: (string | number)[]; + fatal?: boolean | undefined; +}; +export type ErrorMapCtx = { + defaultError: string; + data: any; +}; +export type ZodErrorMap = (issue: ZodIssueOptionalMessage, _ctx: ErrorMapCtx) => { + message: string; +}; +export {}; diff --git a/tests/fixtures/npm/node_modules/zod/v3/errors.d.cts b/tests/fixtures/npm/node_modules/zod/v3/errors.d.cts new file mode 100644 index 000000000..205f3a096 --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v3/errors.d.cts @@ -0,0 +1,5 @@ +import type { ZodErrorMap } from "./ZodError.cjs"; +import defaultErrorMap from "./locales/en.cjs"; +export { defaultErrorMap }; +export declare function setErrorMap(map: ZodErrorMap): void; +export declare function getErrorMap(): ZodErrorMap; diff --git a/tests/fixtures/npm/node_modules/zod/v3/external.d.cts b/tests/fixtures/npm/node_modules/zod/v3/external.d.cts new file mode 100644 index 000000000..ced9c1a85 --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v3/external.d.cts @@ -0,0 +1,6 @@ +export * from "./errors.cjs"; +export * from "./helpers/parseUtil.cjs"; +export * from "./helpers/typeAliases.cjs"; +export * from "./helpers/util.cjs"; +export * from "./types.cjs"; +export * from "./ZodError.cjs"; diff --git a/tests/fixtures/npm/node_modules/zod/v3/helpers/enumUtil.d.cts b/tests/fixtures/npm/node_modules/zod/v3/helpers/enumUtil.d.cts new file mode 100644 index 000000000..fad28a711 --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v3/helpers/enumUtil.d.cts @@ -0,0 +1,8 @@ +export declare namespace enumUtil { + type UnionToIntersectionFn = (T extends unknown ? (k: () => T) => void : never) extends (k: infer Intersection) => void ? Intersection : never; + type GetUnionLast = UnionToIntersectionFn extends () => infer Last ? Last : never; + type UnionToTuple = [T] extends [never] ? Tuple : UnionToTuple>, [GetUnionLast, ...Tuple]>; + type CastToStringTuple = T extends [string, ...string[]] ? T : never; + export type UnionToTupleString = CastToStringTuple>; + export {}; +} diff --git a/tests/fixtures/npm/node_modules/zod/v3/helpers/errorUtil.d.cts b/tests/fixtures/npm/node_modules/zod/v3/helpers/errorUtil.d.cts new file mode 100644 index 000000000..bc9b033ff --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v3/helpers/errorUtil.d.cts @@ -0,0 +1,9 @@ +export declare namespace errorUtil { + type ErrMessage = string | { + message?: string | undefined; + }; + const errToObj: (message?: ErrMessage) => { + message?: string | undefined; + }; + const toString: (message?: ErrMessage) => string | undefined; +} diff --git a/tests/fixtures/npm/node_modules/zod/v3/helpers/parseUtil.d.cts b/tests/fixtures/npm/node_modules/zod/v3/helpers/parseUtil.d.cts new file mode 100644 index 000000000..f34a63d7f --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v3/helpers/parseUtil.d.cts @@ -0,0 +1,78 @@ +import type { IssueData, ZodErrorMap, ZodIssue } from "../ZodError.cjs"; +import type { ZodParsedType } from "./util.cjs"; +export declare const makeIssue: (params: { + data: any; + path: (string | number)[]; + errorMaps: ZodErrorMap[]; + issueData: IssueData; +}) => ZodIssue; +export type ParseParams = { + path: (string | number)[]; + errorMap: ZodErrorMap; + async: boolean; +}; +export type ParsePathComponent = string | number; +export type ParsePath = ParsePathComponent[]; +export declare const EMPTY_PATH: ParsePath; +export interface ParseContext { + readonly common: { + readonly issues: ZodIssue[]; + readonly contextualErrorMap?: ZodErrorMap | undefined; + readonly async: boolean; + }; + readonly path: ParsePath; + readonly schemaErrorMap?: ZodErrorMap | undefined; + readonly parent: ParseContext | null; + readonly data: any; + readonly parsedType: ZodParsedType; +} +export type ParseInput = { + data: any; + path: (string | number)[]; + parent: ParseContext; +}; +export declare function addIssueToContext(ctx: ParseContext, issueData: IssueData): void; +export type ObjectPair = { + key: SyncParseReturnType; + value: SyncParseReturnType; +}; +export declare class ParseStatus { + value: "aborted" | "dirty" | "valid"; + dirty(): void; + abort(): void; + static mergeArray(status: ParseStatus, results: SyncParseReturnType[]): SyncParseReturnType; + static mergeObjectAsync(status: ParseStatus, pairs: { + key: ParseReturnType; + value: ParseReturnType; + }[]): Promise>; + static mergeObjectSync(status: ParseStatus, pairs: { + key: SyncParseReturnType; + value: SyncParseReturnType; + alwaysSet?: boolean; + }[]): SyncParseReturnType; +} +export interface ParseResult { + status: "aborted" | "dirty" | "valid"; + data: any; +} +export type INVALID = { + status: "aborted"; +}; +export declare const INVALID: INVALID; +export type DIRTY = { + status: "dirty"; + value: T; +}; +export declare const DIRTY: (value: T) => DIRTY; +export type OK = { + status: "valid"; + value: T; +}; +export declare const OK: (value: T) => OK; +export type SyncParseReturnType = OK | DIRTY | INVALID; +export type AsyncParseReturnType = Promise>; +export type ParseReturnType = SyncParseReturnType | AsyncParseReturnType; +export declare const isAborted: (x: ParseReturnType) => x is INVALID; +export declare const isDirty: (x: ParseReturnType) => x is OK | DIRTY; +export declare const isValid: (x: ParseReturnType) => x is OK; +export declare const isAsync: (x: ParseReturnType) => x is AsyncParseReturnType; diff --git a/tests/fixtures/npm/node_modules/zod/v3/helpers/partialUtil.d.cts b/tests/fixtures/npm/node_modules/zod/v3/helpers/partialUtil.d.cts new file mode 100644 index 000000000..3f9e46a2a --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v3/helpers/partialUtil.d.cts @@ -0,0 +1,8 @@ +import type { ZodArray, ZodNullable, ZodObject, ZodOptional, ZodRawShape, ZodTuple, ZodTupleItems, ZodTypeAny } from "../types.cjs"; +export declare namespace partialUtil { + type DeepPartial = T extends ZodObject ? ZodObject<{ + [k in keyof T["shape"]]: ZodOptional>; + }, T["_def"]["unknownKeys"], T["_def"]["catchall"]> : T extends ZodArray ? ZodArray, Card> : T extends ZodOptional ? ZodOptional> : T extends ZodNullable ? ZodNullable> : T extends ZodTuple ? { + [k in keyof Items]: Items[k] extends ZodTypeAny ? DeepPartial : never; + } extends infer PI ? PI extends ZodTupleItems ? ZodTuple : never : never : T; +} diff --git a/tests/fixtures/npm/node_modules/zod/v3/helpers/typeAliases.d.cts b/tests/fixtures/npm/node_modules/zod/v3/helpers/typeAliases.d.cts new file mode 100644 index 000000000..32df022ca --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v3/helpers/typeAliases.d.cts @@ -0,0 +1,2 @@ +export type Primitive = string | number | symbol | bigint | boolean | null | undefined; +export type Scalars = Primitive | Primitive[]; diff --git a/tests/fixtures/npm/node_modules/zod/v3/helpers/util.d.cts b/tests/fixtures/npm/node_modules/zod/v3/helpers/util.d.cts new file mode 100644 index 000000000..eeaaeee1c --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v3/helpers/util.d.cts @@ -0,0 +1,85 @@ +export declare namespace util { + type AssertEqual = (() => V extends T ? 1 : 2) extends () => V extends U ? 1 : 2 ? true : false; + export type isAny = 0 extends 1 & T ? true : false; + export const assertEqual: (_: AssertEqual) => void; + export function assertIs(_arg: T): void; + export function assertNever(_x: never): never; + export type Omit = Pick>; + export type OmitKeys = Pick>; + export type MakePartial = Omit & Partial>; + export type Exactly = T & Record, never>; + export type InexactPartial = { + [k in keyof T]?: T[k] | undefined; + }; + export const arrayToEnum: (items: U) => { [k in U[number]]: k; }; + export const getValidEnumValues: (obj: any) => any[]; + export const objectValues: (obj: any) => any[]; + export const objectKeys: ObjectConstructor["keys"]; + export const find: (arr: T[], checker: (arg: T) => any) => T | undefined; + export type identity = objectUtil.identity; + export type flatten = objectUtil.flatten; + export type noUndefined = T extends undefined ? never : T; + export const isInteger: NumberConstructor["isInteger"]; + export function joinValues(array: T, separator?: string): string; + export const jsonStringifyReplacer: (_: string, value: any) => any; + export {}; +} +export declare namespace objectUtil { + export type MergeShapes = keyof U & keyof V extends never ? U & V : { + [k in Exclude]: U[k]; + } & V; + type optionalKeys = { + [k in keyof T]: undefined extends T[k] ? k : never; + }[keyof T]; + type requiredKeys = { + [k in keyof T]: undefined extends T[k] ? never : k; + }[keyof T]; + export type addQuestionMarks = { + [K in requiredKeys]: T[K]; + } & { + [K in optionalKeys]?: T[K]; + } & { + [k in keyof T]?: unknown; + }; + export type identity = T; + export type flatten = identity<{ + [k in keyof T]: T[k]; + }>; + export type noNeverKeys = { + [k in keyof T]: [T[k]] extends [never] ? never : k; + }[keyof T]; + export type noNever = identity<{ + [k in noNeverKeys]: k extends keyof T ? T[k] : never; + }>; + export const mergeShapes: (first: U, second: T) => T & U; + export type extendShape = keyof A & keyof B extends never ? A & B : { + [K in keyof A as K extends keyof B ? never : K]: A[K]; + } & { + [K in keyof B]: B[K]; + }; + export {}; +} +export declare const ZodParsedType: { + string: "string"; + nan: "nan"; + number: "number"; + integer: "integer"; + float: "float"; + boolean: "boolean"; + date: "date"; + bigint: "bigint"; + symbol: "symbol"; + function: "function"; + undefined: "undefined"; + null: "null"; + array: "array"; + object: "object"; + unknown: "unknown"; + promise: "promise"; + void: "void"; + never: "never"; + map: "map"; + set: "set"; +}; +export type ZodParsedType = keyof typeof ZodParsedType; +export declare const getParsedType: (data: any) => ZodParsedType; diff --git a/tests/fixtures/npm/node_modules/zod/v3/index.d.cts b/tests/fixtures/npm/node_modules/zod/v3/index.d.cts new file mode 100644 index 000000000..b0662babe --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v3/index.d.cts @@ -0,0 +1,4 @@ +import * as z from "./external.cjs"; +export * from "./external.cjs"; +export { z }; +export default z; diff --git a/tests/fixtures/npm/node_modules/zod/v3/locales/en.d.cts b/tests/fixtures/npm/node_modules/zod/v3/locales/en.d.cts new file mode 100644 index 000000000..e146a11c5 --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v3/locales/en.d.cts @@ -0,0 +1,3 @@ +import { type ZodErrorMap } from "../ZodError.cjs"; +declare const errorMap: ZodErrorMap; +export = errorMap; diff --git a/tests/fixtures/npm/node_modules/zod/v3/standard-schema.d.cts b/tests/fixtures/npm/node_modules/zod/v3/standard-schema.d.cts new file mode 100644 index 000000000..7ddff3b7f --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v3/standard-schema.d.cts @@ -0,0 +1,102 @@ +/** + * The Standard Schema interface. + */ +export type StandardSchemaV1 = { + /** + * The Standard Schema properties. + */ + readonly "~standard": StandardSchemaV1.Props; +}; +export declare namespace StandardSchemaV1 { + /** + * The Standard Schema properties interface. + */ + export interface Props { + /** + * The version number of the standard. + */ + readonly version: 1; + /** + * The vendor name of the schema library. + */ + readonly vendor: string; + /** + * Validates unknown input values. + */ + readonly validate: (value: unknown) => Result | Promise>; + /** + * Inferred types associated with the schema. + */ + readonly types?: Types | undefined; + } + /** + * The result interface of the validate function. + */ + export type Result = SuccessResult | FailureResult; + /** + * The result interface if validation succeeds. + */ + export interface SuccessResult { + /** + * The typed output value. + */ + readonly value: Output; + /** + * The non-existent issues. + */ + readonly issues?: undefined; + } + /** + * The result interface if validation fails. + */ + export interface FailureResult { + /** + * The issues of failed validation. + */ + readonly issues: ReadonlyArray; + } + /** + * The issue interface of the failure output. + */ + export interface Issue { + /** + * The error message of the issue. + */ + readonly message: string; + /** + * The path of the issue, if any. + */ + readonly path?: ReadonlyArray | undefined; + } + /** + * The path segment interface of the issue. + */ + export interface PathSegment { + /** + * The key representing a path segment. + */ + readonly key: PropertyKey; + } + /** + * The Standard Schema types interface. + */ + export interface Types { + /** + * The input type of the schema. + */ + readonly input: Input; + /** + * The output type of the schema. + */ + readonly output: Output; + } + /** + * Infers the input type of a Standard Schema. + */ + export type InferInput = NonNullable["input"]; + /** + * Infers the output type of a Standard Schema. + */ + export type InferOutput = NonNullable["output"]; + export {}; +} diff --git a/tests/fixtures/npm/node_modules/zod/v3/types.d.cts b/tests/fixtures/npm/node_modules/zod/v3/types.d.cts new file mode 100644 index 000000000..f1013a74f --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v3/types.d.cts @@ -0,0 +1,1034 @@ +import { type IssueData, type StringValidation, type ZodCustomIssue, ZodError, type ZodErrorMap } from "./ZodError.cjs"; +import type { enumUtil } from "./helpers/enumUtil.cjs"; +import { errorUtil } from "./helpers/errorUtil.cjs"; +import { type AsyncParseReturnType, INVALID, type ParseContext, type ParseInput, type ParseParams, type ParseReturnType, ParseStatus, type SyncParseReturnType } from "./helpers/parseUtil.cjs"; +import type { partialUtil } from "./helpers/partialUtil.cjs"; +import type { Primitive } from "./helpers/typeAliases.cjs"; +import { util, type objectUtil } from "./helpers/util.cjs"; +import type { StandardSchemaV1 } from "./standard-schema.cjs"; +export interface RefinementCtx { + addIssue: (arg: IssueData) => void; + path: (string | number)[]; +} +export type ZodRawShape = { + [k: string]: ZodTypeAny; +}; +export type ZodTypeAny = ZodType; +export type TypeOf> = T["_output"]; +export type input> = T["_input"]; +export type output> = T["_output"]; +export type { TypeOf as infer }; +export type CustomErrorParams = Partial>; +export interface ZodTypeDef { + errorMap?: ZodErrorMap | undefined; + description?: string | undefined; +} +export type RawCreateParams = { + errorMap?: ZodErrorMap | undefined; + invalid_type_error?: string | undefined; + required_error?: string | undefined; + message?: string | undefined; + description?: string | undefined; +} | undefined; +export type ProcessedCreateParams = { + errorMap?: ZodErrorMap | undefined; + description?: string | undefined; +}; +export type SafeParseSuccess = { + success: true; + data: Output; + error?: never; +}; +export type SafeParseError = { + success: false; + error: ZodError; + data?: never; +}; +export type SafeParseReturnType = SafeParseSuccess | SafeParseError; +export declare abstract class ZodType { + readonly _type: Output; + readonly _output: Output; + readonly _input: Input; + readonly _def: Def; + get description(): string | undefined; + "~standard": StandardSchemaV1.Props; + abstract _parse(input: ParseInput): ParseReturnType; + _getType(input: ParseInput): string; + _getOrReturnCtx(input: ParseInput, ctx?: ParseContext | undefined): ParseContext; + _processInputParams(input: ParseInput): { + status: ParseStatus; + ctx: ParseContext; + }; + _parseSync(input: ParseInput): SyncParseReturnType; + _parseAsync(input: ParseInput): AsyncParseReturnType; + parse(data: unknown, params?: util.InexactPartial): Output; + safeParse(data: unknown, params?: util.InexactPartial): SafeParseReturnType; + "~validate"(data: unknown): StandardSchemaV1.Result | Promise>; + parseAsync(data: unknown, params?: util.InexactPartial): Promise; + safeParseAsync(data: unknown, params?: util.InexactPartial): Promise>; + /** Alias of safeParseAsync */ + spa: (data: unknown, params?: util.InexactPartial) => Promise>; + refine(check: (arg: Output) => arg is RefinedOutput, message?: string | CustomErrorParams | ((arg: Output) => CustomErrorParams)): ZodEffects; + refine(check: (arg: Output) => unknown | Promise, message?: string | CustomErrorParams | ((arg: Output) => CustomErrorParams)): ZodEffects; + refinement(check: (arg: Output) => arg is RefinedOutput, refinementData: IssueData | ((arg: Output, ctx: RefinementCtx) => IssueData)): ZodEffects; + refinement(check: (arg: Output) => boolean, refinementData: IssueData | ((arg: Output, ctx: RefinementCtx) => IssueData)): ZodEffects; + _refinement(refinement: RefinementEffect["refinement"]): ZodEffects; + superRefine(refinement: (arg: Output, ctx: RefinementCtx) => arg is RefinedOutput): ZodEffects; + superRefine(refinement: (arg: Output, ctx: RefinementCtx) => void): ZodEffects; + superRefine(refinement: (arg: Output, ctx: RefinementCtx) => Promise): ZodEffects; + constructor(def: Def); + optional(): ZodOptional; + nullable(): ZodNullable; + nullish(): ZodOptional>; + array(): ZodArray; + promise(): ZodPromise; + or(option: T): ZodUnion<[this, T]>; + and(incoming: T): ZodIntersection; + transform(transform: (arg: Output, ctx: RefinementCtx) => NewOut | Promise): ZodEffects; + default(def: util.noUndefined): ZodDefault; + default(def: () => util.noUndefined): ZodDefault; + brand(brand?: B): ZodBranded; + catch(def: Output): ZodCatch; + catch(def: (ctx: { + error: ZodError; + input: Input; + }) => Output): ZodCatch; + describe(description: string): this; + pipe(target: T): ZodPipeline; + readonly(): ZodReadonly; + isOptional(): boolean; + isNullable(): boolean; +} +export type IpVersion = "v4" | "v6"; +export type ZodStringCheck = { + kind: "min"; + value: number; + message?: string | undefined; +} | { + kind: "max"; + value: number; + message?: string | undefined; +} | { + kind: "length"; + value: number; + message?: string | undefined; +} | { + kind: "email"; + message?: string | undefined; +} | { + kind: "url"; + message?: string | undefined; +} | { + kind: "emoji"; + message?: string | undefined; +} | { + kind: "uuid"; + message?: string | undefined; +} | { + kind: "nanoid"; + message?: string | undefined; +} | { + kind: "cuid"; + message?: string | undefined; +} | { + kind: "includes"; + value: string; + position?: number | undefined; + message?: string | undefined; +} | { + kind: "cuid2"; + message?: string | undefined; +} | { + kind: "ulid"; + message?: string | undefined; +} | { + kind: "startsWith"; + value: string; + message?: string | undefined; +} | { + kind: "endsWith"; + value: string; + message?: string | undefined; +} | { + kind: "regex"; + regex: RegExp; + message?: string | undefined; +} | { + kind: "trim"; + message?: string | undefined; +} | { + kind: "toLowerCase"; + message?: string | undefined; +} | { + kind: "toUpperCase"; + message?: string | undefined; +} | { + kind: "jwt"; + alg?: string; + message?: string | undefined; +} | { + kind: "datetime"; + offset: boolean; + local: boolean; + precision: number | null; + message?: string | undefined; +} | { + kind: "date"; + message?: string | undefined; +} | { + kind: "time"; + precision: number | null; + message?: string | undefined; +} | { + kind: "duration"; + message?: string | undefined; +} | { + kind: "ip"; + version?: IpVersion | undefined; + message?: string | undefined; +} | { + kind: "cidr"; + version?: IpVersion | undefined; + message?: string | undefined; +} | { + kind: "base64"; + message?: string | undefined; +} | { + kind: "base64url"; + message?: string | undefined; +}; +export interface ZodStringDef extends ZodTypeDef { + checks: ZodStringCheck[]; + typeName: ZodFirstPartyTypeKind.ZodString; + coerce: boolean; +} +export declare function datetimeRegex(args: { + precision?: number | null; + offset?: boolean; + local?: boolean; +}): RegExp; +export declare class ZodString extends ZodType { + _parse(input: ParseInput): ParseReturnType; + protected _regex(regex: RegExp, validation: StringValidation, message?: errorUtil.ErrMessage): ZodEffects; + _addCheck(check: ZodStringCheck): ZodString; + email(message?: errorUtil.ErrMessage): ZodString; + url(message?: errorUtil.ErrMessage): ZodString; + emoji(message?: errorUtil.ErrMessage): ZodString; + uuid(message?: errorUtil.ErrMessage): ZodString; + nanoid(message?: errorUtil.ErrMessage): ZodString; + cuid(message?: errorUtil.ErrMessage): ZodString; + cuid2(message?: errorUtil.ErrMessage): ZodString; + ulid(message?: errorUtil.ErrMessage): ZodString; + base64(message?: errorUtil.ErrMessage): ZodString; + base64url(message?: errorUtil.ErrMessage): ZodString; + jwt(options?: { + alg?: string; + message?: string | undefined; + }): ZodString; + ip(options?: string | { + version?: IpVersion; + message?: string | undefined; + }): ZodString; + cidr(options?: string | { + version?: IpVersion; + message?: string | undefined; + }): ZodString; + datetime(options?: string | { + message?: string | undefined; + precision?: number | null; + offset?: boolean; + local?: boolean; + }): ZodString; + date(message?: string): ZodString; + time(options?: string | { + message?: string | undefined; + precision?: number | null; + }): ZodString; + duration(message?: errorUtil.ErrMessage): ZodString; + regex(regex: RegExp, message?: errorUtil.ErrMessage): ZodString; + includes(value: string, options?: { + message?: string; + position?: number; + }): ZodString; + startsWith(value: string, message?: errorUtil.ErrMessage): ZodString; + endsWith(value: string, message?: errorUtil.ErrMessage): ZodString; + min(minLength: number, message?: errorUtil.ErrMessage): ZodString; + max(maxLength: number, message?: errorUtil.ErrMessage): ZodString; + length(len: number, message?: errorUtil.ErrMessage): ZodString; + /** + * Equivalent to `.min(1)` + */ + nonempty(message?: errorUtil.ErrMessage): ZodString; + trim(): ZodString; + toLowerCase(): ZodString; + toUpperCase(): ZodString; + get isDatetime(): boolean; + get isDate(): boolean; + get isTime(): boolean; + get isDuration(): boolean; + get isEmail(): boolean; + get isURL(): boolean; + get isEmoji(): boolean; + get isUUID(): boolean; + get isNANOID(): boolean; + get isCUID(): boolean; + get isCUID2(): boolean; + get isULID(): boolean; + get isIP(): boolean; + get isCIDR(): boolean; + get isBase64(): boolean; + get isBase64url(): boolean; + get minLength(): number | null; + get maxLength(): number | null; + static create: (params?: RawCreateParams & { + coerce?: true; + }) => ZodString; +} +export type ZodNumberCheck = { + kind: "min"; + value: number; + inclusive: boolean; + message?: string | undefined; +} | { + kind: "max"; + value: number; + inclusive: boolean; + message?: string | undefined; +} | { + kind: "int"; + message?: string | undefined; +} | { + kind: "multipleOf"; + value: number; + message?: string | undefined; +} | { + kind: "finite"; + message?: string | undefined; +}; +export interface ZodNumberDef extends ZodTypeDef { + checks: ZodNumberCheck[]; + typeName: ZodFirstPartyTypeKind.ZodNumber; + coerce: boolean; +} +export declare class ZodNumber extends ZodType { + _parse(input: ParseInput): ParseReturnType; + static create: (params?: RawCreateParams & { + coerce?: boolean; + }) => ZodNumber; + gte(value: number, message?: errorUtil.ErrMessage): ZodNumber; + min: (value: number, message?: errorUtil.ErrMessage) => ZodNumber; + gt(value: number, message?: errorUtil.ErrMessage): ZodNumber; + lte(value: number, message?: errorUtil.ErrMessage): ZodNumber; + max: (value: number, message?: errorUtil.ErrMessage) => ZodNumber; + lt(value: number, message?: errorUtil.ErrMessage): ZodNumber; + protected setLimit(kind: "min" | "max", value: number, inclusive: boolean, message?: string): ZodNumber; + _addCheck(check: ZodNumberCheck): ZodNumber; + int(message?: errorUtil.ErrMessage): ZodNumber; + positive(message?: errorUtil.ErrMessage): ZodNumber; + negative(message?: errorUtil.ErrMessage): ZodNumber; + nonpositive(message?: errorUtil.ErrMessage): ZodNumber; + nonnegative(message?: errorUtil.ErrMessage): ZodNumber; + multipleOf(value: number, message?: errorUtil.ErrMessage): ZodNumber; + step: (value: number, message?: errorUtil.ErrMessage) => ZodNumber; + finite(message?: errorUtil.ErrMessage): ZodNumber; + safe(message?: errorUtil.ErrMessage): ZodNumber; + get minValue(): number | null; + get maxValue(): number | null; + get isInt(): boolean; + get isFinite(): boolean; +} +export type ZodBigIntCheck = { + kind: "min"; + value: bigint; + inclusive: boolean; + message?: string | undefined; +} | { + kind: "max"; + value: bigint; + inclusive: boolean; + message?: string | undefined; +} | { + kind: "multipleOf"; + value: bigint; + message?: string | undefined; +}; +export interface ZodBigIntDef extends ZodTypeDef { + checks: ZodBigIntCheck[]; + typeName: ZodFirstPartyTypeKind.ZodBigInt; + coerce: boolean; +} +export declare class ZodBigInt extends ZodType { + _parse(input: ParseInput): ParseReturnType; + _getInvalidInput(input: ParseInput): INVALID; + static create: (params?: RawCreateParams & { + coerce?: boolean; + }) => ZodBigInt; + gte(value: bigint, message?: errorUtil.ErrMessage): ZodBigInt; + min: (value: bigint, message?: errorUtil.ErrMessage) => ZodBigInt; + gt(value: bigint, message?: errorUtil.ErrMessage): ZodBigInt; + lte(value: bigint, message?: errorUtil.ErrMessage): ZodBigInt; + max: (value: bigint, message?: errorUtil.ErrMessage) => ZodBigInt; + lt(value: bigint, message?: errorUtil.ErrMessage): ZodBigInt; + protected setLimit(kind: "min" | "max", value: bigint, inclusive: boolean, message?: string): ZodBigInt; + _addCheck(check: ZodBigIntCheck): ZodBigInt; + positive(message?: errorUtil.ErrMessage): ZodBigInt; + negative(message?: errorUtil.ErrMessage): ZodBigInt; + nonpositive(message?: errorUtil.ErrMessage): ZodBigInt; + nonnegative(message?: errorUtil.ErrMessage): ZodBigInt; + multipleOf(value: bigint, message?: errorUtil.ErrMessage): ZodBigInt; + get minValue(): bigint | null; + get maxValue(): bigint | null; +} +export interface ZodBooleanDef extends ZodTypeDef { + typeName: ZodFirstPartyTypeKind.ZodBoolean; + coerce: boolean; +} +export declare class ZodBoolean extends ZodType { + _parse(input: ParseInput): ParseReturnType; + static create: (params?: RawCreateParams & { + coerce?: boolean; + }) => ZodBoolean; +} +export type ZodDateCheck = { + kind: "min"; + value: number; + message?: string | undefined; +} | { + kind: "max"; + value: number; + message?: string | undefined; +}; +export interface ZodDateDef extends ZodTypeDef { + checks: ZodDateCheck[]; + coerce: boolean; + typeName: ZodFirstPartyTypeKind.ZodDate; +} +export declare class ZodDate extends ZodType { + _parse(input: ParseInput): ParseReturnType; + _addCheck(check: ZodDateCheck): ZodDate; + min(minDate: Date, message?: errorUtil.ErrMessage): ZodDate; + max(maxDate: Date, message?: errorUtil.ErrMessage): ZodDate; + get minDate(): Date | null; + get maxDate(): Date | null; + static create: (params?: RawCreateParams & { + coerce?: boolean; + }) => ZodDate; +} +export interface ZodSymbolDef extends ZodTypeDef { + typeName: ZodFirstPartyTypeKind.ZodSymbol; +} +export declare class ZodSymbol extends ZodType { + _parse(input: ParseInput): ParseReturnType; + static create: (params?: RawCreateParams) => ZodSymbol; +} +export interface ZodUndefinedDef extends ZodTypeDef { + typeName: ZodFirstPartyTypeKind.ZodUndefined; +} +export declare class ZodUndefined extends ZodType { + _parse(input: ParseInput): ParseReturnType; + params?: RawCreateParams; + static create: (params?: RawCreateParams) => ZodUndefined; +} +export interface ZodNullDef extends ZodTypeDef { + typeName: ZodFirstPartyTypeKind.ZodNull; +} +export declare class ZodNull extends ZodType { + _parse(input: ParseInput): ParseReturnType; + static create: (params?: RawCreateParams) => ZodNull; +} +export interface ZodAnyDef extends ZodTypeDef { + typeName: ZodFirstPartyTypeKind.ZodAny; +} +export declare class ZodAny extends ZodType { + _any: true; + _parse(input: ParseInput): ParseReturnType; + static create: (params?: RawCreateParams) => ZodAny; +} +export interface ZodUnknownDef extends ZodTypeDef { + typeName: ZodFirstPartyTypeKind.ZodUnknown; +} +export declare class ZodUnknown extends ZodType { + _unknown: true; + _parse(input: ParseInput): ParseReturnType; + static create: (params?: RawCreateParams) => ZodUnknown; +} +export interface ZodNeverDef extends ZodTypeDef { + typeName: ZodFirstPartyTypeKind.ZodNever; +} +export declare class ZodNever extends ZodType { + _parse(input: ParseInput): ParseReturnType; + static create: (params?: RawCreateParams) => ZodNever; +} +export interface ZodVoidDef extends ZodTypeDef { + typeName: ZodFirstPartyTypeKind.ZodVoid; +} +export declare class ZodVoid extends ZodType { + _parse(input: ParseInput): ParseReturnType; + static create: (params?: RawCreateParams) => ZodVoid; +} +export interface ZodArrayDef extends ZodTypeDef { + type: T; + typeName: ZodFirstPartyTypeKind.ZodArray; + exactLength: { + value: number; + message?: string | undefined; + } | null; + minLength: { + value: number; + message?: string | undefined; + } | null; + maxLength: { + value: number; + message?: string | undefined; + } | null; +} +export type ArrayCardinality = "many" | "atleastone"; +export type arrayOutputType = Cardinality extends "atleastone" ? [T["_output"], ...T["_output"][]] : T["_output"][]; +export declare class ZodArray extends ZodType, ZodArrayDef, Cardinality extends "atleastone" ? [T["_input"], ...T["_input"][]] : T["_input"][]> { + _parse(input: ParseInput): ParseReturnType; + get element(): T; + min(minLength: number, message?: errorUtil.ErrMessage): this; + max(maxLength: number, message?: errorUtil.ErrMessage): this; + length(len: number, message?: errorUtil.ErrMessage): this; + nonempty(message?: errorUtil.ErrMessage): ZodArray; + static create: (schema: El, params?: RawCreateParams) => ZodArray; +} +export type ZodNonEmptyArray = ZodArray; +export type UnknownKeysParam = "passthrough" | "strict" | "strip"; +export interface ZodObjectDef extends ZodTypeDef { + typeName: ZodFirstPartyTypeKind.ZodObject; + shape: () => T; + catchall: Catchall; + unknownKeys: UnknownKeys; +} +export type mergeTypes = { + [k in keyof A | keyof B]: k extends keyof B ? B[k] : k extends keyof A ? A[k] : never; +}; +export type objectOutputType = objectUtil.flatten>> & CatchallOutput & PassthroughType; +export type baseObjectOutputType = { + [k in keyof Shape]: Shape[k]["_output"]; +}; +export type objectInputType = objectUtil.flatten> & CatchallInput & PassthroughType; +export type baseObjectInputType = objectUtil.addQuestionMarks<{ + [k in keyof Shape]: Shape[k]["_input"]; +}>; +export type CatchallOutput = ZodType extends T ? unknown : { + [k: string]: T["_output"]; +}; +export type CatchallInput = ZodType extends T ? unknown : { + [k: string]: T["_input"]; +}; +export type PassthroughType = T extends "passthrough" ? { + [k: string]: unknown; +} : unknown; +export type deoptional = T extends ZodOptional ? deoptional : T extends ZodNullable ? ZodNullable> : T; +export type SomeZodObject = ZodObject; +export type noUnrecognized = { + [k in keyof Obj]: k extends keyof Shape ? Obj[k] : never; +}; +export declare class ZodObject, Input = objectInputType> extends ZodType, Input> { + _cached: { + shape: T; + keys: string[]; + } | null; + _getCached(): { + shape: T; + keys: string[]; + }; + _parse(input: ParseInput): ParseReturnType; + get shape(): T; + strict(message?: errorUtil.ErrMessage): ZodObject; + strip(): ZodObject; + passthrough(): ZodObject; + /** + * @deprecated In most cases, this is no longer needed - unknown properties are now silently stripped. + * If you want to pass through unknown properties, use `.passthrough()` instead. + */ + nonstrict: () => ZodObject; + extend(augmentation: Augmentation): ZodObject, UnknownKeys, Catchall>; + /** + * @deprecated Use `.extend` instead + * */ + augment: (augmentation: Augmentation) => ZodObject, UnknownKeys, Catchall>; + /** + * Prior to zod@1.0.12 there was a bug in the + * inferred type of merged objects. Please + * upgrade if you are experiencing issues. + */ + merge(merging: Incoming): ZodObject, Incoming["_def"]["unknownKeys"], Incoming["_def"]["catchall"]>; + setKey(key: Key, schema: Schema): ZodObject; + catchall(index: Index): ZodObject; + pick>(mask: Mask): ZodObject>, UnknownKeys, Catchall>; + omit>(mask: Mask): ZodObject, UnknownKeys, Catchall>; + /** + * @deprecated + */ + deepPartial(): partialUtil.DeepPartial; + partial(): ZodObject<{ + [k in keyof T]: ZodOptional; + }, UnknownKeys, Catchall>; + partial>(mask: Mask): ZodObject : T[k]; + }>, UnknownKeys, Catchall>; + required(): ZodObject<{ + [k in keyof T]: deoptional; + }, UnknownKeys, Catchall>; + required>(mask: Mask): ZodObject : T[k]; + }>, UnknownKeys, Catchall>; + keyof(): ZodEnum>; + static create: (shape: Shape, params?: RawCreateParams) => ZodObject, objectInputType>; + static strictCreate: (shape: Shape, params?: RawCreateParams) => ZodObject; + static lazycreate: (shape: () => Shape, params?: RawCreateParams) => ZodObject; +} +export type AnyZodObject = ZodObject; +export type ZodUnionOptions = Readonly<[ZodTypeAny, ...ZodTypeAny[]]>; +export interface ZodUnionDef> extends ZodTypeDef { + options: T; + typeName: ZodFirstPartyTypeKind.ZodUnion; +} +export declare class ZodUnion extends ZodType, T[number]["_input"]> { + _parse(input: ParseInput): ParseReturnType; + get options(): T; + static create: >(types: Options, params?: RawCreateParams) => ZodUnion; +} +export type ZodDiscriminatedUnionOption = ZodObject<{ + [key in Discriminator]: ZodTypeAny; +} & ZodRawShape, UnknownKeysParam, ZodTypeAny>; +export interface ZodDiscriminatedUnionDef[] = ZodDiscriminatedUnionOption[]> extends ZodTypeDef { + discriminator: Discriminator; + options: Options; + optionsMap: Map>; + typeName: ZodFirstPartyTypeKind.ZodDiscriminatedUnion; +} +export declare class ZodDiscriminatedUnion[]> extends ZodType, ZodDiscriminatedUnionDef, input> { + _parse(input: ParseInput): ParseReturnType; + get discriminator(): Discriminator; + get options(): Options; + get optionsMap(): Map>; + /** + * The constructor of the discriminated union schema. Its behaviour is very similar to that of the normal z.union() constructor. + * However, it only allows a union of objects, all of which need to share a discriminator property. This property must + * have a different value for each object in the union. + * @param discriminator the name of the discriminator property + * @param types an array of object schemas + * @param params + */ + static create, + ...ZodDiscriminatedUnionOption[] + ]>(discriminator: Discriminator, options: Types, params?: RawCreateParams): ZodDiscriminatedUnion; +} +export interface ZodIntersectionDef extends ZodTypeDef { + left: T; + right: U; + typeName: ZodFirstPartyTypeKind.ZodIntersection; +} +export declare class ZodIntersection extends ZodType, T["_input"] & U["_input"]> { + _parse(input: ParseInput): ParseReturnType; + static create: (left: TSchema, right: USchema, params?: RawCreateParams) => ZodIntersection; +} +export type ZodTupleItems = [ZodTypeAny, ...ZodTypeAny[]]; +export type AssertArray = T extends any[] ? T : never; +export type OutputTypeOfTuple = AssertArray<{ + [k in keyof T]: T[k] extends ZodType ? T[k]["_output"] : never; +}>; +export type OutputTypeOfTupleWithRest = Rest extends ZodTypeAny ? [...OutputTypeOfTuple, ...Rest["_output"][]] : OutputTypeOfTuple; +export type InputTypeOfTuple = AssertArray<{ + [k in keyof T]: T[k] extends ZodType ? T[k]["_input"] : never; +}>; +export type InputTypeOfTupleWithRest = Rest extends ZodTypeAny ? [...InputTypeOfTuple, ...Rest["_input"][]] : InputTypeOfTuple; +export interface ZodTupleDef extends ZodTypeDef { + items: T; + rest: Rest; + typeName: ZodFirstPartyTypeKind.ZodTuple; +} +export type AnyZodTuple = ZodTuple<[ZodTypeAny, ...ZodTypeAny[]] | [], ZodTypeAny | null>; +export declare class ZodTuple extends ZodType, ZodTupleDef, InputTypeOfTupleWithRest> { + _parse(input: ParseInput): ParseReturnType; + get items(): T; + rest(rest: RestSchema): ZodTuple; + static create: (schemas: Items, params?: RawCreateParams) => ZodTuple; +} +export interface ZodRecordDef extends ZodTypeDef { + valueType: Value; + keyType: Key; + typeName: ZodFirstPartyTypeKind.ZodRecord; +} +export type KeySchema = ZodType; +export type RecordType = [string] extends [K] ? Record : [number] extends [K] ? Record : [symbol] extends [K] ? Record : [BRAND] extends [K] ? Record : Partial>; +export declare class ZodRecord extends ZodType, ZodRecordDef, RecordType> { + get keySchema(): Key; + get valueSchema(): Value; + _parse(input: ParseInput): ParseReturnType; + get element(): Value; + static create(valueType: Value, params?: RawCreateParams): ZodRecord; + static create(keySchema: Keys, valueType: Value, params?: RawCreateParams): ZodRecord; +} +export interface ZodMapDef extends ZodTypeDef { + valueType: Value; + keyType: Key; + typeName: ZodFirstPartyTypeKind.ZodMap; +} +export declare class ZodMap extends ZodType, ZodMapDef, Map> { + get keySchema(): Key; + get valueSchema(): Value; + _parse(input: ParseInput): ParseReturnType; + static create: (keyType: KeySchema, valueType: ValueSchema, params?: RawCreateParams) => ZodMap; +} +export interface ZodSetDef extends ZodTypeDef { + valueType: Value; + typeName: ZodFirstPartyTypeKind.ZodSet; + minSize: { + value: number; + message?: string | undefined; + } | null; + maxSize: { + value: number; + message?: string | undefined; + } | null; +} +export declare class ZodSet extends ZodType, ZodSetDef, Set> { + _parse(input: ParseInput): ParseReturnType; + min(minSize: number, message?: errorUtil.ErrMessage): this; + max(maxSize: number, message?: errorUtil.ErrMessage): this; + size(size: number, message?: errorUtil.ErrMessage): this; + nonempty(message?: errorUtil.ErrMessage): ZodSet; + static create: (valueType: ValueSchema, params?: RawCreateParams) => ZodSet; +} +export interface ZodFunctionDef = ZodTuple, Returns extends ZodTypeAny = ZodTypeAny> extends ZodTypeDef { + args: Args; + returns: Returns; + typeName: ZodFirstPartyTypeKind.ZodFunction; +} +export type OuterTypeOfFunction, Returns extends ZodTypeAny> = Args["_input"] extends Array ? (...args: Args["_input"]) => Returns["_output"] : never; +export type InnerTypeOfFunction, Returns extends ZodTypeAny> = Args["_output"] extends Array ? (...args: Args["_output"]) => Returns["_input"] : never; +export declare class ZodFunction, Returns extends ZodTypeAny> extends ZodType, ZodFunctionDef, InnerTypeOfFunction> { + _parse(input: ParseInput): ParseReturnType; + parameters(): Args; + returnType(): Returns; + args[0]>(...items: Items): ZodFunction, Returns>; + returns>(returnType: NewReturnType): ZodFunction; + implement>(func: F): ReturnType extends Returns["_output"] ? (...args: Args["_input"]) => ReturnType : OuterTypeOfFunction; + strictImplement(func: InnerTypeOfFunction): InnerTypeOfFunction; + validate: >(func: F) => ReturnType extends Returns["_output"] ? (...args: Args["_input"]) => ReturnType : OuterTypeOfFunction; + static create(): ZodFunction, ZodUnknown>; + static create>(args: T): ZodFunction; + static create(args: T, returns: U): ZodFunction; + static create, U extends ZodTypeAny = ZodUnknown>(args: T, returns: U, params?: RawCreateParams): ZodFunction; +} +export interface ZodLazyDef extends ZodTypeDef { + getter: () => T; + typeName: ZodFirstPartyTypeKind.ZodLazy; +} +export declare class ZodLazy extends ZodType, ZodLazyDef, input> { + get schema(): T; + _parse(input: ParseInput): ParseReturnType; + static create: (getter: () => Inner, params?: RawCreateParams) => ZodLazy; +} +export interface ZodLiteralDef extends ZodTypeDef { + value: T; + typeName: ZodFirstPartyTypeKind.ZodLiteral; +} +export declare class ZodLiteral extends ZodType, T> { + _parse(input: ParseInput): ParseReturnType; + get value(): T; + static create: (value: Value, params?: RawCreateParams) => ZodLiteral; +} +export type ArrayKeys = keyof any[]; +export type Indices = Exclude; +export type EnumValues = readonly [T, ...T[]]; +export type Values = { + [k in T[number]]: k; +}; +export interface ZodEnumDef extends ZodTypeDef { + values: T; + typeName: ZodFirstPartyTypeKind.ZodEnum; +} +export type Writeable = { + -readonly [P in keyof T]: T[P]; +}; +export type FilterEnum = Values extends [] ? [] : Values extends [infer Head, ...infer Rest] ? Head extends ToExclude ? FilterEnum : [Head, ...FilterEnum] : never; +export type typecast = A extends T ? A : never; +declare function createZodEnum>(values: T, params?: RawCreateParams): ZodEnum>; +declare function createZodEnum(values: T, params?: RawCreateParams): ZodEnum; +export declare class ZodEnum extends ZodType, T[number]> { + _cache: Set | undefined; + _parse(input: ParseInput): ParseReturnType; + get options(): T; + get enum(): Values; + get Values(): Values; + get Enum(): Values; + extract(values: ToExtract, newDef?: RawCreateParams): ZodEnum>; + exclude(values: ToExclude, newDef?: RawCreateParams): ZodEnum>, [string, ...string[]]>>; + static create: typeof createZodEnum; +} +export interface ZodNativeEnumDef extends ZodTypeDef { + values: T; + typeName: ZodFirstPartyTypeKind.ZodNativeEnum; +} +export type EnumLike = { + [k: string]: string | number; + [nu: number]: string; +}; +export declare class ZodNativeEnum extends ZodType, T[keyof T]> { + _cache: Set | undefined; + _parse(input: ParseInput): ParseReturnType; + get enum(): T; + static create: (values: Elements, params?: RawCreateParams) => ZodNativeEnum; +} +export interface ZodPromiseDef extends ZodTypeDef { + type: T; + typeName: ZodFirstPartyTypeKind.ZodPromise; +} +export declare class ZodPromise extends ZodType, ZodPromiseDef, Promise> { + unwrap(): T; + _parse(input: ParseInput): ParseReturnType; + static create: (schema: Inner, params?: RawCreateParams) => ZodPromise; +} +export type Refinement = (arg: T, ctx: RefinementCtx) => any; +export type SuperRefinement = (arg: T, ctx: RefinementCtx) => void | Promise; +export type RefinementEffect = { + type: "refinement"; + refinement: (arg: T, ctx: RefinementCtx) => any; +}; +export type TransformEffect = { + type: "transform"; + transform: (arg: T, ctx: RefinementCtx) => any; +}; +export type PreprocessEffect = { + type: "preprocess"; + transform: (arg: T, ctx: RefinementCtx) => any; +}; +export type Effect = RefinementEffect | TransformEffect | PreprocessEffect; +export interface ZodEffectsDef extends ZodTypeDef { + schema: T; + typeName: ZodFirstPartyTypeKind.ZodEffects; + effect: Effect; +} +export declare class ZodEffects, Input = input> extends ZodType, Input> { + innerType(): T; + sourceType(): T; + _parse(input: ParseInput): ParseReturnType; + static create: (schema: I, effect: Effect, params?: RawCreateParams) => ZodEffects; + static createWithPreprocess: (preprocess: (arg: unknown, ctx: RefinementCtx) => unknown, schema: I, params?: RawCreateParams) => ZodEffects; +} +export { ZodEffects as ZodTransformer }; +export interface ZodOptionalDef extends ZodTypeDef { + innerType: T; + typeName: ZodFirstPartyTypeKind.ZodOptional; +} +export type ZodOptionalType = ZodOptional; +export declare class ZodOptional extends ZodType, T["_input"] | undefined> { + _parse(input: ParseInput): ParseReturnType; + unwrap(): T; + static create: (type: Inner, params?: RawCreateParams) => ZodOptional; +} +export interface ZodNullableDef extends ZodTypeDef { + innerType: T; + typeName: ZodFirstPartyTypeKind.ZodNullable; +} +export type ZodNullableType = ZodNullable; +export declare class ZodNullable extends ZodType, T["_input"] | null> { + _parse(input: ParseInput): ParseReturnType; + unwrap(): T; + static create: (type: Inner, params?: RawCreateParams) => ZodNullable; +} +export interface ZodDefaultDef extends ZodTypeDef { + innerType: T; + defaultValue: () => util.noUndefined; + typeName: ZodFirstPartyTypeKind.ZodDefault; +} +export declare class ZodDefault extends ZodType, ZodDefaultDef, T["_input"] | undefined> { + _parse(input: ParseInput): ParseReturnType; + removeDefault(): T; + static create: (type: Inner, params: RawCreateParams & { + default: Inner["_input"] | (() => util.noUndefined); + }) => ZodDefault; +} +export interface ZodCatchDef extends ZodTypeDef { + innerType: T; + catchValue: (ctx: { + error: ZodError; + input: unknown; + }) => T["_input"]; + typeName: ZodFirstPartyTypeKind.ZodCatch; +} +export declare class ZodCatch extends ZodType, unknown> { + _parse(input: ParseInput): ParseReturnType; + removeCatch(): T; + static create: (type: Inner, params: RawCreateParams & { + catch: Inner["_output"] | (() => Inner["_output"]); + }) => ZodCatch; +} +export interface ZodNaNDef extends ZodTypeDef { + typeName: ZodFirstPartyTypeKind.ZodNaN; +} +export declare class ZodNaN extends ZodType { + _parse(input: ParseInput): ParseReturnType; + static create: (params?: RawCreateParams) => ZodNaN; +} +export interface ZodBrandedDef extends ZodTypeDef { + type: T; + typeName: ZodFirstPartyTypeKind.ZodBranded; +} +export declare const BRAND: unique symbol; +export type BRAND = { + [BRAND]: { + [k in T]: true; + }; +}; +export declare class ZodBranded extends ZodType, ZodBrandedDef, T["_input"]> { + _parse(input: ParseInput): ParseReturnType; + unwrap(): T; +} +export interface ZodPipelineDef extends ZodTypeDef { + in: A; + out: B; + typeName: ZodFirstPartyTypeKind.ZodPipeline; +} +export declare class ZodPipeline extends ZodType, A["_input"]> { + _parse(input: ParseInput): ParseReturnType; + static create(a: ASchema, b: BSchema): ZodPipeline; +} +type BuiltIn = (((...args: any[]) => any) | (new (...args: any[]) => any)) | { + readonly [Symbol.toStringTag]: string; +} | Date | Error | Generator | Promise | RegExp; +type MakeReadonly = T extends Map ? ReadonlyMap : T extends Set ? ReadonlySet : T extends [infer Head, ...infer Tail] ? readonly [Head, ...Tail] : T extends Array ? ReadonlyArray : T extends BuiltIn ? T : Readonly; +export interface ZodReadonlyDef extends ZodTypeDef { + innerType: T; + typeName: ZodFirstPartyTypeKind.ZodReadonly; +} +export declare class ZodReadonly extends ZodType, ZodReadonlyDef, MakeReadonly> { + _parse(input: ParseInput): ParseReturnType; + static create: (type: Inner, params?: RawCreateParams) => ZodReadonly; + unwrap(): T; +} +type CustomParams = CustomErrorParams & { + fatal?: boolean; +}; +export declare function custom(check?: (data: any) => any, _params?: string | CustomParams | ((input: any) => CustomParams), +/** + * @deprecated + * + * Pass `fatal` into the params object instead: + * + * ```ts + * z.string().custom((val) => val.length > 5, { fatal: false }) + * ``` + * + */ +fatal?: boolean): ZodType; +export { ZodType as Schema, ZodType as ZodSchema }; +export declare const late: { + object: (shape: () => Shape, params?: RawCreateParams) => ZodObject; +}; +export declare enum ZodFirstPartyTypeKind { + ZodString = "ZodString", + ZodNumber = "ZodNumber", + ZodNaN = "ZodNaN", + ZodBigInt = "ZodBigInt", + ZodBoolean = "ZodBoolean", + ZodDate = "ZodDate", + ZodSymbol = "ZodSymbol", + ZodUndefined = "ZodUndefined", + ZodNull = "ZodNull", + ZodAny = "ZodAny", + ZodUnknown = "ZodUnknown", + ZodNever = "ZodNever", + ZodVoid = "ZodVoid", + ZodArray = "ZodArray", + ZodObject = "ZodObject", + ZodUnion = "ZodUnion", + ZodDiscriminatedUnion = "ZodDiscriminatedUnion", + ZodIntersection = "ZodIntersection", + ZodTuple = "ZodTuple", + ZodRecord = "ZodRecord", + ZodMap = "ZodMap", + ZodSet = "ZodSet", + ZodFunction = "ZodFunction", + ZodLazy = "ZodLazy", + ZodLiteral = "ZodLiteral", + ZodEnum = "ZodEnum", + ZodEffects = "ZodEffects", + ZodNativeEnum = "ZodNativeEnum", + ZodOptional = "ZodOptional", + ZodNullable = "ZodNullable", + ZodDefault = "ZodDefault", + ZodCatch = "ZodCatch", + ZodPromise = "ZodPromise", + ZodBranded = "ZodBranded", + ZodPipeline = "ZodPipeline", + ZodReadonly = "ZodReadonly" +} +export type ZodFirstPartySchemaTypes = ZodString | ZodNumber | ZodNaN | ZodBigInt | ZodBoolean | ZodDate | ZodUndefined | ZodNull | ZodAny | ZodUnknown | ZodNever | ZodVoid | ZodArray | ZodObject | ZodUnion | ZodDiscriminatedUnion | ZodIntersection | ZodTuple | ZodRecord | ZodMap | ZodSet | ZodFunction | ZodLazy | ZodLiteral | ZodEnum | ZodEffects | ZodNativeEnum | ZodOptional | ZodNullable | ZodDefault | ZodCatch | ZodPromise | ZodBranded | ZodPipeline | ZodReadonly | ZodSymbol; +declare abstract class Class { + constructor(..._: any[]); +} +declare const instanceOfType: (cls: T, params?: CustomParams) => ZodType, ZodTypeDef, InstanceType>; +declare const stringType: (params?: RawCreateParams & { + coerce?: true; +}) => ZodString; +declare const numberType: (params?: RawCreateParams & { + coerce?: boolean; +}) => ZodNumber; +declare const nanType: (params?: RawCreateParams) => ZodNaN; +declare const bigIntType: (params?: RawCreateParams & { + coerce?: boolean; +}) => ZodBigInt; +declare const booleanType: (params?: RawCreateParams & { + coerce?: boolean; +}) => ZodBoolean; +declare const dateType: (params?: RawCreateParams & { + coerce?: boolean; +}) => ZodDate; +declare const symbolType: (params?: RawCreateParams) => ZodSymbol; +declare const undefinedType: (params?: RawCreateParams) => ZodUndefined; +declare const nullType: (params?: RawCreateParams) => ZodNull; +declare const anyType: (params?: RawCreateParams) => ZodAny; +declare const unknownType: (params?: RawCreateParams) => ZodUnknown; +declare const neverType: (params?: RawCreateParams) => ZodNever; +declare const voidType: (params?: RawCreateParams) => ZodVoid; +declare const arrayType: (schema: El, params?: RawCreateParams) => ZodArray; +declare const objectType: (shape: Shape, params?: RawCreateParams) => ZodObject, objectInputType>; +declare const strictObjectType: (shape: Shape, params?: RawCreateParams) => ZodObject; +declare const unionType: >(types: Options, params?: RawCreateParams) => ZodUnion; +declare const discriminatedUnionType: typeof ZodDiscriminatedUnion.create; +declare const intersectionType: (left: TSchema, right: USchema, params?: RawCreateParams) => ZodIntersection; +declare const tupleType: (schemas: Items, params?: RawCreateParams) => ZodTuple; +declare const recordType: typeof ZodRecord.create; +declare const mapType: (keyType: KeySchema, valueType: ValueSchema, params?: RawCreateParams) => ZodMap; +declare const setType: (valueType: ValueSchema, params?: RawCreateParams) => ZodSet; +declare const functionType: typeof ZodFunction.create; +declare const lazyType: (getter: () => Inner, params?: RawCreateParams) => ZodLazy; +declare const literalType: (value: Value, params?: RawCreateParams) => ZodLiteral; +declare const enumType: typeof createZodEnum; +declare const nativeEnumType: (values: Elements, params?: RawCreateParams) => ZodNativeEnum; +declare const promiseType: (schema: Inner, params?: RawCreateParams) => ZodPromise; +declare const effectsType: (schema: I, effect: Effect, params?: RawCreateParams) => ZodEffects; +declare const optionalType: (type: Inner, params?: RawCreateParams) => ZodOptional; +declare const nullableType: (type: Inner, params?: RawCreateParams) => ZodNullable; +declare const preprocessType: (preprocess: (arg: unknown, ctx: RefinementCtx) => unknown, schema: I, params?: RawCreateParams) => ZodEffects; +declare const pipelineType: typeof ZodPipeline.create; +declare const ostring: () => ZodOptional; +declare const onumber: () => ZodOptional; +declare const oboolean: () => ZodOptional; +export declare const coerce: { + string: (typeof ZodString)["create"]; + number: (typeof ZodNumber)["create"]; + boolean: (typeof ZodBoolean)["create"]; + bigint: (typeof ZodBigInt)["create"]; + date: (typeof ZodDate)["create"]; +}; +export { anyType as any, arrayType as array, bigIntType as bigint, booleanType as boolean, dateType as date, discriminatedUnionType as discriminatedUnion, effectsType as effect, enumType as enum, functionType as function, instanceOfType as instanceof, intersectionType as intersection, lazyType as lazy, literalType as literal, mapType as map, nanType as nan, nativeEnumType as nativeEnum, neverType as never, nullType as null, nullableType as nullable, numberType as number, objectType as object, oboolean, onumber, optionalType as optional, ostring, pipelineType as pipeline, preprocessType as preprocess, promiseType as promise, recordType as record, setType as set, strictObjectType as strictObject, stringType as string, symbolType as symbol, effectsType as transformer, tupleType as tuple, undefinedType as undefined, unionType as union, unknownType as unknown, voidType as void, }; +export declare const NEVER: never; diff --git a/tests/fixtures/npm/node_modules/zod/v4/classic/checks.d.cts b/tests/fixtures/npm/node_modules/zod/v4/classic/checks.d.cts new file mode 100644 index 000000000..757d60e6e --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v4/classic/checks.d.cts @@ -0,0 +1 @@ +export { _lt as lt, _lte as lte, _gt as gt, _gte as gte, _positive as positive, _negative as negative, _nonpositive as nonpositive, _nonnegative as nonnegative, _multipleOf as multipleOf, _maxSize as maxSize, _minSize as minSize, _size as size, _maxLength as maxLength, _minLength as minLength, _length as length, _regex as regex, _lowercase as lowercase, _uppercase as uppercase, _includes as includes, _startsWith as startsWith, _endsWith as endsWith, _property as property, _properties as properties, _mime as mime, _overwrite as overwrite, _normalize as normalize, _trim as trim, _toLowerCase as toLowerCase, _toUpperCase as toUpperCase, _slugify as slugify, type $RefinementCtx as RefinementCtx, } from "../core/index.cjs"; diff --git a/tests/fixtures/npm/node_modules/zod/v4/classic/checks.js b/tests/fixtures/npm/node_modules/zod/v4/classic/checks.js new file mode 100644 index 000000000..50a08b884 --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v4/classic/checks.js @@ -0,0 +1 @@ +export { _lt as lt, _lte as lte, _gt as gt, _gte as gte, _positive as positive, _negative as negative, _nonpositive as nonpositive, _nonnegative as nonnegative, _multipleOf as multipleOf, _maxSize as maxSize, _minSize as minSize, _size as size, _maxLength as maxLength, _minLength as minLength, _length as length, _regex as regex, _lowercase as lowercase, _uppercase as uppercase, _includes as includes, _startsWith as startsWith, _endsWith as endsWith, _property as property, _properties as properties, _mime as mime, _overwrite as overwrite, _normalize as normalize, _trim as trim, _toLowerCase as toLowerCase, _toUpperCase as toUpperCase, _slugify as slugify, } from "../core/index.js"; diff --git a/tests/fixtures/npm/node_modules/zod/v4/classic/coerce.d.cts b/tests/fixtures/npm/node_modules/zod/v4/classic/coerce.d.cts new file mode 100644 index 000000000..231b91745 --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v4/classic/coerce.d.cts @@ -0,0 +1,17 @@ +import * as core from "../core/index.cjs"; +import * as schemas from "./schemas.cjs"; +export interface ZodCoercedString extends schemas._ZodString> { +} +export declare function string(params?: string | core.$ZodStringParams): ZodCoercedString; +export interface ZodCoercedNumber extends schemas._ZodNumber> { +} +export declare function number(params?: string | core.$ZodNumberParams): ZodCoercedNumber; +export interface ZodCoercedBoolean extends schemas._ZodBoolean> { +} +export declare function boolean(params?: string | core.$ZodBooleanParams): ZodCoercedBoolean; +export interface ZodCoercedBigInt extends schemas._ZodBigInt> { +} +export declare function bigint(params?: string | core.$ZodBigIntParams): ZodCoercedBigInt; +export interface ZodCoercedDate extends schemas._ZodDate> { +} +export declare function date(params?: string | core.$ZodDateParams): ZodCoercedDate; diff --git a/tests/fixtures/npm/node_modules/zod/v4/classic/coerce.js b/tests/fixtures/npm/node_modules/zod/v4/classic/coerce.js new file mode 100644 index 000000000..141cf8936 --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v4/classic/coerce.js @@ -0,0 +1,17 @@ +import * as core from "../core/index.js"; +import * as schemas from "./schemas.js"; +export function string(params) { + return core._coercedString(schemas.ZodString, params); +} +export function number(params) { + return core._coercedNumber(schemas.ZodNumber, params); +} +export function boolean(params) { + return core._coercedBoolean(schemas.ZodBoolean, params); +} +export function bigint(params) { + return core._coercedBigint(schemas.ZodBigInt, params); +} +export function date(params) { + return core._coercedDate(schemas.ZodDate, params); +} diff --git a/tests/fixtures/npm/node_modules/zod/v4/classic/compat.d.cts b/tests/fixtures/npm/node_modules/zod/v4/classic/compat.d.cts new file mode 100644 index 000000000..8f1b3d8c8 --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v4/classic/compat.d.cts @@ -0,0 +1,48 @@ +import * as core from "../core/index.cjs"; +import type { ZodType } from "./schemas.cjs"; +/** @deprecated Use `z.output` instead. */ +export type TypeOf = core.output; +/** @deprecated Use `z.output` instead. */ +export type Infer = core.output; +/** @deprecated Use `z.core.$ZodTypes` instead */ +export type ZodFirstPartySchemaTypes = core.$ZodTypes; +/** @deprecated Use the raw string literal codes instead, e.g. "invalid_type". */ +export declare const ZodIssueCode: { + readonly invalid_type: "invalid_type"; + readonly too_big: "too_big"; + readonly too_small: "too_small"; + readonly invalid_format: "invalid_format"; + readonly not_multiple_of: "not_multiple_of"; + readonly unrecognized_keys: "unrecognized_keys"; + readonly invalid_union: "invalid_union"; + readonly invalid_key: "invalid_key"; + readonly invalid_element: "invalid_element"; + readonly invalid_value: "invalid_value"; + readonly custom: "custom"; +}; +/** @deprecated Use `z.core.$ZodFlattenedError>` instead; it takes an inferred type, not a schema. */ +export type inferFlattenedErrors = core.$ZodFlattenedError, U>; +/** @deprecated Use `z.core.$ZodFormattedError>` instead; it takes an inferred type, not a schema. */ +export type inferFormattedError, U = string> = core.$ZodFormattedError, U>; +/** Use `z.$brand` instead */ +export type BRAND = { + [core.$brand]: { + [k in T]: true; + }; +}; +export { $brand, config } from "../core/index.cjs"; +/** @deprecated Use `z.config(params)` instead. */ +export declare function setErrorMap(map: core.$ZodErrorMap): void; +/** @deprecated Use `z.config()` instead. */ +export declare function getErrorMap(): core.$ZodErrorMap | undefined; +/** @deprecated Use z.ZodType (without generics) instead. */ +export type ZodTypeAny = core.$ZodTypeInternals> = ZodType; +/** @deprecated Use `z.ZodType` */ +export type ZodSchema = core.$ZodTypeInternals> = ZodType; +/** @deprecated Use `z.ZodType` */ +export type Schema = core.$ZodTypeInternals> = ZodType; +/** Included for Zod 3 compatibility */ +export type ZodRawShape = core.$ZodShape; +/** @deprecated Do not use. Stub definition, only included for zod-to-json-schema compatibility. */ +export declare enum ZodFirstPartyTypeKind { +} diff --git a/tests/fixtures/npm/node_modules/zod/v4/classic/compat.js b/tests/fixtures/npm/node_modules/zod/v4/classic/compat.js new file mode 100644 index 000000000..2179ea904 --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v4/classic/compat.js @@ -0,0 +1,31 @@ +// Zod 3 compat layer +import * as core from "../core/index.js"; +/** @deprecated Use the raw string literal codes instead, e.g. "invalid_type". */ +export const ZodIssueCode = { + invalid_type: "invalid_type", + too_big: "too_big", + too_small: "too_small", + invalid_format: "invalid_format", + not_multiple_of: "not_multiple_of", + unrecognized_keys: "unrecognized_keys", + invalid_union: "invalid_union", + invalid_key: "invalid_key", + invalid_element: "invalid_element", + invalid_value: "invalid_value", + custom: "custom", +}; +export { $brand, config } from "../core/index.js"; +/** @deprecated Use `z.config(params)` instead. */ +export function setErrorMap(map) { + core.config({ + customError: map, + }); +} +/** @deprecated Use `z.config()` instead. */ +export function getErrorMap() { + return core.config().customError; +} +/** @deprecated Do not use. Stub definition, only included for zod-to-json-schema compatibility. */ +export var ZodFirstPartyTypeKind; +(function (ZodFirstPartyTypeKind) { +})(ZodFirstPartyTypeKind || (ZodFirstPartyTypeKind = {})); diff --git a/tests/fixtures/npm/node_modules/zod/v4/classic/deep-partial.d.cts b/tests/fixtures/npm/node_modules/zod/v4/classic/deep-partial.d.cts new file mode 100644 index 000000000..674ecf18e --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v4/classic/deep-partial.d.cts @@ -0,0 +1,14 @@ +import type * as core from "../core/index.cjs"; +import * as schemas from "./schemas.cjs"; +/** Distributes `.partial()` through every object in a schema type, preserving the wrappers around them. */ +export type DeepPartial = T extends schemas.ZodObject ? schemas.ZodObject<{ + -readonly [K in keyof Shape]: schemas.ZodOptional>; +}, Config> : T extends schemas.ZodDiscriminatedUnion ? schemas.ZodUnion<{ + -readonly [I in keyof Options]: Options[I] extends core.SomeType ? DeepPartial : Options[I]; +}> : T extends schemas.ZodUnion ? schemas.ZodUnion<{ + -readonly [I in keyof Options]: Options[I] extends core.SomeType ? DeepPartial : Options[I]; +}> : T extends schemas.ZodArray ? schemas.ZodArray> : T extends schemas.ZodTuple ? schemas.ZodTuple<{ + -readonly [K in keyof Items]: Items[K] extends core.SomeType ? DeepPartial : Items[K]; +}, Rest extends core.SomeType ? DeepPartial : Rest> : T extends schemas.ZodIntersection ? schemas.ZodIntersection, DeepPartial> : T extends schemas.ZodRecord ? schemas.ZodRecord> : T extends schemas.ZodMap ? schemas.ZodMap> : T extends schemas.ZodSet ? schemas.ZodSet> : T extends schemas.ZodOptional ? schemas.ZodOptional> : T extends schemas.ZodNullable ? schemas.ZodNullable> : T extends schemas.ZodDefault ? schemas.ZodDefault> : T extends schemas.ZodPrefault ? schemas.ZodPrefault> : T extends schemas.ZodNonOptional ? schemas.ZodNonOptional> : T extends schemas.ZodCatch ? schemas.ZodCatch> : T extends schemas.ZodReadonly ? schemas.ZodReadonly> : T extends schemas.ZodSuccess ? schemas.ZodSuccess> : T extends schemas.ZodPromise ? schemas.ZodPromise> : T extends schemas.ZodCodec ? schemas.ZodCodec, DeepPartial> : T extends schemas.ZodPreprocess ? schemas.ZodPreprocess> : T extends schemas.ZodPipe ? schemas.ZodPipe, DeepPartial> : T extends schemas.ZodLazy ? schemas.ZodLazy> : T; +/** Returns a copy of the schema with every nested object's properties made optional. */ +export declare function deepPartial(schema: T): DeepPartial; diff --git a/tests/fixtures/npm/node_modules/zod/v4/classic/deep-partial.js b/tests/fixtures/npm/node_modules/zod/v4/classic/deep-partial.js new file mode 100644 index 000000000..a87bb625c --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v4/classic/deep-partial.js @@ -0,0 +1,13 @@ +import { visit } from "../core/visit.js"; +import * as schemas from "./schemas.js"; +/** Returns a copy of the schema with every nested object's properties made optional. */ +export function deepPartial(schema) { + return visit(schema, { + object: (s) => s.partial(), + // Every partialed option now admits `undefined`, which the constructor rejects as a duplicate. + union: (s) => { + const def = s._zod.def; + return def.discriminator === undefined ? s : schemas.union(def.options); + }, + }); +} diff --git a/tests/fixtures/npm/node_modules/zod/v4/classic/errors.d.cts b/tests/fixtures/npm/node_modules/zod/v4/classic/errors.d.cts new file mode 100644 index 000000000..472102f6a --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v4/classic/errors.d.cts @@ -0,0 +1,29 @@ +import * as core from "../core/index.cjs"; +import { $ZodError } from "../core/index.cjs"; +/** @deprecated Use `z.core.$ZodIssue` from `@zod/core` instead, especially if you are building a library on top of Zod. */ +export type ZodIssue = core.$ZodIssue; +/** An Error-like class used to store Zod validation issues. */ +export interface ZodError extends $ZodError { + /** @deprecated Use the `z.treeifyError(err)` function instead. */ + format(): core.$ZodFormattedError; + format(mapper: (issue: core.$ZodIssue) => U): core.$ZodFormattedError; + /** @deprecated Use the `z.treeifyError(err)` function instead. */ + flatten(): core.$ZodFlattenedError; + flatten(mapper: (issue: core.$ZodIssue) => U): core.$ZodFlattenedError; + /** @deprecated Push directly to `.issues` instead. */ + addIssue(issue: core.$ZodIssue): void; + /** @deprecated Push directly to `.issues` instead. */ + addIssues(issues: core.$ZodIssue[]): void; + /** @deprecated Check `err.issues.length === 0` instead. */ + isEmpty: boolean; +} +export declare const ZodError: core.$constructor; +export declare const ZodRealError: core.$constructor; +/** @deprecated Use `z.core.$ZodFlattenedError` instead. */ +export type ZodFlattenedError = core.$ZodFlattenedError; +/** @deprecated Use `z.core.$ZodFormattedError` instead. */ +export type ZodFormattedError = core.$ZodFormattedError; +/** @deprecated Use `z.core.$ZodErrorMap` instead. */ +export type ZodErrorMap = core.$ZodErrorMap; +/** @deprecated Use `z.core.$ZodRawIssue` instead. */ +export type IssueData = core.$ZodRawIssue; diff --git a/tests/fixtures/npm/node_modules/zod/v4/classic/errors.js b/tests/fixtures/npm/node_modules/zod/v4/classic/errors.js new file mode 100644 index 000000000..e3c02587a --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v4/classic/errors.js @@ -0,0 +1,57 @@ +import * as core from "../core/index.js"; +import { $ZodError } from "../core/index.js"; +import * as util from "../core/util.js"; +/* Prototypes that already carry the lazy helper methods. Seeded with the + * intrinsics so that `init` on a foreign object — it accepts any object — + * can never install an accessor onto a prototype we do not own. */ +const _installedErrorProtos = /* @__PURE__ */ new WeakSet([Object.prototype, Error.prototype]); +/* Helper methods live as non-enumerable lazy getters on the shared + * prototype instead of own properties on every instance. On first + * access the getter allocates the per-instance closure and caches it + * as a non-enumerable own property, so detached usage still works and + * the allocation only happens for methods actually touched. */ +function _lazyMethod(proto, key, make) { + Object.defineProperty(proto, key, { + configurable: true, + enumerable: false, + get() { + const value = make(this); + Object.defineProperty(this, key, { value, configurable: true, writable: true }); + return value; + }, + set(value) { + Object.defineProperty(this, key, { value, configurable: true, writable: true }); + }, + }); +} +const initializer = (inst, issues) => { + $ZodError.init(inst, issues); + inst.name = "ZodError"; + const proto = Object.getPrototypeOf(inst); + if (_installedErrorProtos.has(proto)) + return; + _installedErrorProtos.add(proto); + _lazyMethod(proto, "format", (self) => (mapper) => core.formatError(self, mapper)); + _lazyMethod(proto, "flatten", (self) => (mapper) => core.flattenError(self, mapper)); + _lazyMethod(proto, "addIssue", (self) => (issue) => { + self.issues.push(issue); + self.message = JSON.stringify(self.issues, util.jsonStringifyReplacer, 2); + }); + _lazyMethod(proto, "addIssues", (self) => (issues) => { + self.issues.push(...issues); + self.message = JSON.stringify(self.issues, util.jsonStringifyReplacer, 2); + }); + Object.defineProperty(proto, "isEmpty", { + configurable: true, + enumerable: false, + get() { + return this.issues.length === 0; + }, + }); +}; +export const ZodError = /*@__PURE__*/ core.$constructor("ZodError", initializer); +export const ZodRealError = /*@__PURE__*/ core.$constructor("ZodError", initializer, undefined, { + Parent: Error, +}); +// /** @deprecated Use `z.core.$ZodErrorMapCtx` instead. */ +// export type ErrorMapCtx = core.$ZodErrorMapCtx; diff --git a/tests/fixtures/npm/node_modules/zod/v4/classic/external.d.cts b/tests/fixtures/npm/node_modules/zod/v4/classic/external.d.cts new file mode 100644 index 000000000..7e9ce5e74 --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v4/classic/external.d.cts @@ -0,0 +1,19 @@ +export * as core from "../core/index.cjs"; +export * from "./schemas.cjs"; +export * from "./checks.cjs"; +export * from "./errors.cjs"; +export * from "./parse.cjs"; +export * from "./compat.cjs"; +export type { infer } from "../core/index.cjs"; +export type { JSONType } from "../core/util.cjs"; +export type { CompileOptions } from "../core/index.cjs"; +export { globalRegistry, type GlobalMeta, registry, config, memoizer, $output, $input, $brand, clone, regexes, treeifyError, prettifyError, formatError, flattenError, TimePrecision, util, NEVER, INVALID, toZod, compile, withParser, ZodCompileAsyncError, ZodCompileUnsupportedError, getDiscriminatedOption, } from "../core/index.cjs"; +export { toJSONSchema } from "../core/json-schema-processors.cjs"; +export { fromJSONSchema } from "./from-json-schema.cjs"; +export { deepPartial } from "./deep-partial.cjs"; +export { input, output } from "./in-out.cjs"; +export * as locales from "../locales/index.cjs"; +export { ZodISODateTime, ZodISODate, ZodISOTime, ZodISODuration } from "./iso.cjs"; +export * as iso from "./iso.cjs"; +export type { ZodCoercedString, ZodCoercedNumber, ZodCoercedBigInt, ZodCoercedBoolean, ZodCoercedDate, } from "./coerce.cjs"; +export * as coerce from "./coerce.cjs"; diff --git a/tests/fixtures/npm/node_modules/zod/v4/classic/external.js b/tests/fixtures/npm/node_modules/zod/v4/classic/external.js new file mode 100644 index 000000000..435be167f --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v4/classic/external.js @@ -0,0 +1,17 @@ +export * as core from "../core/index.js"; +export * from "./schemas.js"; +export * from "./checks.js"; +export * from "./errors.js"; +export * from "./parse.js"; +export * from "./compat.js"; +export { globalRegistry, registry, config, memoizer, $output, $input, $brand, clone, regexes, treeifyError, prettifyError, formatError, flattenError, TimePrecision, util, NEVER, INVALID, toZod, compile, withParser, ZodCompileAsyncError, ZodCompileUnsupportedError, getDiscriminatedOption, } from "../core/index.js"; +export { toJSONSchema } from "../core/json-schema-processors.js"; +export { fromJSONSchema } from "./from-json-schema.js"; +export { deepPartial } from "./deep-partial.js"; +// Types and values both, from one module — re-exporting the types from core would collide. +export { input, output } from "./in-out.js"; +export * as locales from "../locales/index.js"; +// iso must be exported from top-level https://github.com/colinhacks/zod/issues/4491 +export { ZodISODateTime, ZodISODate, ZodISOTime, ZodISODuration } from "./iso.js"; +export * as iso from "./iso.js"; +export * as coerce from "./coerce.js"; diff --git a/tests/fixtures/npm/node_modules/zod/v4/classic/from-json-schema.d.cts b/tests/fixtures/npm/node_modules/zod/v4/classic/from-json-schema.d.cts new file mode 100644 index 000000000..dc028bc01 --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v4/classic/from-json-schema.d.cts @@ -0,0 +1,12 @@ +import type * as JSONSchema from "../core/json-schema.cjs"; +import { type $ZodRegistry } from "../core/registries.cjs"; +import type { ZodType } from "./schemas.cjs"; +type JSONSchemaVersion = "draft-2020-12" | "draft-7" | "draft-4" | "openapi-3.0"; +interface FromJSONSchemaParams { + defaultTarget?: JSONSchemaVersion; + registry?: $ZodRegistry; +} +/** + * Converts a JSON Schema to a Zod schema. This function should be considered semi-experimental. It's behavior is liable to change. */ +export declare function fromJSONSchema(schema: JSONSchema.JSONSchema | boolean, params?: FromJSONSchemaParams): ZodType; +export {}; diff --git a/tests/fixtures/npm/node_modules/zod/v4/classic/from-json-schema.js b/tests/fixtures/npm/node_modules/zod/v4/classic/from-json-schema.js new file mode 100644 index 000000000..1f79b5c85 --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v4/classic/from-json-schema.js @@ -0,0 +1,881 @@ +import { globalRegistry } from "../core/registries.js"; +import { assignProp, isPlainObject } from "../core/util.js"; +import * as _checks from "./checks.js"; +import * as _iso from "./iso.js"; +import * as _schemas from "./schemas.js"; +// Local z object to avoid circular dependency with ../index.js +const z = { + ..._schemas, + ..._checks, + iso: _iso, +}; +// Keys that are recognized and handled by the conversion logic +const RECOGNIZED_KEYS = /*@__PURE__*/ new Set([ + // Schema identification + "$schema", + "$ref", + "$defs", + "definitions", + // Core schema keywords + "$id", + "id", + "$comment", + "$anchor", + "$vocabulary", + "$dynamicRef", + "$dynamicAnchor", + // Type + "type", + "enum", + "const", + // Composition + "anyOf", + "oneOf", + "allOf", + "not", + // Object + "properties", + "required", + "additionalProperties", + "patternProperties", + "propertyNames", + "minProperties", + "maxProperties", + // Array + "items", + "prefixItems", + "additionalItems", + "minItems", + "maxItems", + "uniqueItems", + "contains", + "minContains", + "maxContains", + // String + "minLength", + "maxLength", + "pattern", + "format", + // Number + "minimum", + "maximum", + "exclusiveMinimum", + "exclusiveMaximum", + "multipleOf", + // Already handled metadata + "description", + "default", + // Content + "contentEncoding", + "contentMediaType", + "contentSchema", + // Unsupported (error-throwing) + "unevaluatedItems", + "unevaluatedProperties", + "if", + "then", + "else", + "dependentSchemas", + "dependentRequired", + // OpenAPI + "nullable", + "readOnly", +]); +function detectVersion(schema, defaultTarget) { + const $schema = schema.$schema; + if ($schema === "https://json-schema.org/draft/2020-12/schema") { + return "draft-2020-12"; + } + if ($schema === "http://json-schema.org/draft-07/schema#") { + return "draft-7"; + } + if ($schema === "http://json-schema.org/draft-04/schema#") { + return "draft-4"; + } + // Use defaultTarget if provided, otherwise default to draft-2020-12 + return defaultTarget ?? "draft-2020-12"; +} +// Positional schemas constrain the elements that are present; only minItems makes them required. +function applyMinItems(items, minItems) { + return items.map((item, index) => (index < minItems ? item : item.optional())); +} +// Inverse of the encoding applied to $ref pointer segments in to-json-schema.ts. Per RFC 6901 the `~1` replacement must run before `~0`. +function decodeJSONPointerSegment(segment) { + return segment.replace(/~1/g, "/").replace(/~0/g, "~"); +} +function resolveRef(ref, ctx) { + if (!ref.startsWith("#")) { + throw new Error("External $ref is not supported, only local refs (#/...) are allowed"); + } + const path = ref.slice(1).split("/").filter(Boolean); + // Handle root reference "#" + if (path.length === 0) { + return ctx.rootSchema; + } + const defsKey = ctx.version === "draft-2020-12" ? "$defs" : "definitions"; + if (path[0] === defsKey) { + const key = path[1] === undefined ? undefined : decodeJSONPointerSegment(path[1]); + if (!key || !ctx.defs[key]) { + throw new Error(`Reference not found: ${ref}`); + } + return ctx.defs[key]; + } + throw new Error(`Reference not found: ${ref}`); +} +/** + * Enforces the keywords that constrain an object's own keys — `propertyNames`, + * `minProperties`, `maxProperties` — before `objectSchema` runs. The guard has + * to see the raw input: an object parse drops `__proto__` and can add keys from + * a property `default`, so its output is not the set of names the instance + * actually carried. + */ +function checkObjectGuards(objectSchema, guards) { + // An identity transform, not z.any(), so `toJSONSchema` reports the object on both the input and the output side of the pipe. + const guard = z + .transform((value) => value) + .check((payload) => { + const value = payload.value; + if (typeof value !== "object" || value === null || Array.isArray(value)) + return; + const keys = Object.getOwnPropertyNames(value); + if (guards.minProperties !== undefined && keys.length < guards.minProperties) { + payload.issues.push({ + origin: "object", + code: "too_small", + minimum: guards.minProperties, + inclusive: true, + message: `Too small: expected object to have >=${guards.minProperties} properties`, + input: value, + inst: objectSchema, + continue: true, + }); + } + if (guards.maxProperties !== undefined && keys.length > guards.maxProperties) { + payload.issues.push({ + origin: "object", + code: "too_big", + maximum: guards.maxProperties, + inclusive: true, + message: `Too big: expected object to have <=${guards.maxProperties} properties`, + input: value, + inst: objectSchema, + continue: true, + }); + } + if (guards.keySchema) { + for (const key of keys) { + const result = guards.keySchema.safeParse(key); + if (result.success) + continue; + payload.issues.push({ + code: "invalid_key", + origin: "record", + issues: result.error.issues, + input: key, + path: [key], + continue: true, + }); + } + } + }); + return guard.pipe(objectSchema); +} +/** + * An injective string for JSON data: two values share a key exactly when `uniqueItems` calls them + * equal, so duplicate detection is a Map lookup rather than a pairwise walk. Strings and keys are + * length-prefixed so no value can spell another one. `null` means "never equal to anything" — a + * cycle or a NaN, neither of which JSON can express. + */ +function canonicalKey(value, seen) { + if (value === null) + return "z"; + const type = typeof value; + if (type !== "object") { + if (type === "number" && Number.isNaN(value)) + return null; + const raw = String(value); + return `${type[0]}${raw.length}:${raw}`; + } + if (seen.has(value)) + return null; + seen.add(value); + try { + if (Array.isArray(value)) { + const parts = []; + for (const item of value) { + const key = canonicalKey(item, seen); + if (key === null) + return null; + parts.push(key); + } + return `a${parts.length}:[${parts.join(",")}]`; + } + const keys = Object.keys(value).sort(); + const parts = []; + for (const k of keys) { + const key = canonicalKey(value[k], seen); + if (key === null) + return null; + parts.push(`${k.length}:${k}=${key}`); + } + return `o${parts.length}:{${parts.join(",")}}`; + } + finally { + seen.delete(value); + } +} +// keywords whose value is a schema or an array of them +const SCHEMA_KEYWORDS = /*@__PURE__*/ new Set([ + "items", + "prefixItems", + "additionalItems", + "additionalProperties", + "contains", + "propertyNames", + "not", + "if", + "then", + "else", + "allOf", + "anyOf", + "oneOf", + "unevaluatedItems", + "unevaluatedProperties", + "contentSchema", +]); +// keywords whose value maps names to schemas. draft-7 `dependencies` also allows an array of property names, which holds no schema to walk +const SCHEMA_MAP_KEYWORDS = /*@__PURE__*/ new Set([ + "properties", + "patternProperties", + "dependentSchemas", + "dependencies", + "$defs", + "definitions", +]); +/** + * True when a subschema holds a `$ref` anywhere a schema can appear. Only those positions are + * walked: `$ref` under `default` or an `x-` annotation is instance data with an ordinary key, and + * reading it as a reference would drop a constraint that resolves perfectly well. + */ +function containsRef(value) { + if (typeof value !== "object" || value === null) + return false; + if (Array.isArray(value)) + return value.some(containsRef); + if (typeof value.$ref === "string") + return true; + return Object.entries(value).some(([key, sub]) => { + if (SCHEMA_KEYWORDS.has(key)) + return containsRef(sub); + if (!SCHEMA_MAP_KEYWORDS.has(key) || typeof sub !== "object" || sub === null) + return false; + return Object.values(sub).some(containsRef); + }); +} +function plural(n) { + return n === 1 ? "element" : "elements"; +} +/** + * Enforces `uniqueItems` and the `contains` family before `arraySchema` runs, for the same reason + * `checkObjectGuards` runs pre-pipe: an item parse can apply a nested `default`, so the parsed + * array is not the instance the keywords are defined over. A guard issue aborts the pipe, so + * `minItems`/`maxItems` are not also reported on the same parse — instance correctness is worth + * more than co-reporting two issues. + */ +function checkArrayGuards(arraySchema, guards) { + // An identity transform, not z.any(), so `toJSONSchema` reports the array on both the input and the output side of the pipe. + const guard = z + .transform((value) => value) + .check((payload) => { + const items = payload.value; + if (!Array.isArray(items)) + return; + if (guards.uniqueItems === true) { + const firstSeen = new Map(); + for (let i = 0; i < items.length; i++) { + const key = canonicalKey(items[i], new Set()); + if (key === null) + continue; + const first = firstSeen.get(key); + if (first === undefined) { + firstSeen.set(key, i); + continue; + } + payload.issues.push({ + code: "custom", + message: `Array items must be unique: element at index ${i} duplicates the one at index ${first}`, + input: items, + path: [i], + continue: true, + }); + } + } + if (guards.containsSchema) { + const minContains = guards.minContains ?? 1; + // one past the ceiling already proves the failure, and nothing above it changes the verdict + const ceiling = guards.maxContains !== undefined ? guards.maxContains + 1 : Number.POSITIVE_INFINITY; + let matches = 0; + for (const item of items) { + if (guards.containsSchema.safeParse(item).success && ++matches >= ceiling) + break; + } + if (matches < minContains) { + // the scan ran to the end, so this count is exact + payload.issues.push({ + code: "custom", + message: `Array must contain at least ${minContains} matching ${plural(minContains)}; found ${matches}`, + input: items, + continue: true, + }); + } + if (guards.maxContains !== undefined && matches > guards.maxContains) { + payload.issues.push({ + code: "custom", + message: `Array must contain at most ${guards.maxContains} matching ${plural(guards.maxContains)}`, + input: items, + continue: true, + }); + } + } + }); + return guard.pipe(arraySchema); +} +function getTupleRest(restSchema, ctx) { + if (restSchema === false) { + return undefined; + } + if (restSchema === undefined || restSchema === true) { + return z.any(); + } + return convertSchema(restSchema, ctx); +} +// the RFC 3339 full-time keyword, narrower than `z.iso.time()`; local because sharing it via `timeSource` pins that builder into every bundle (+139 B gzipped on a mini `z.boolean()`) +const fullTime = /^(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|[+-](?:[01]\d|2[0-3]):[0-5]\d)$/; +function convertBaseSchema(schema, ctx) { + // Handle unsupported features + if (schema.not !== undefined) { + // Special case: { not: {} } represents never + if (typeof schema.not === "object" && Object.keys(schema.not).length === 0) { + return z.never(); + } + throw new Error("not is not supported in Zod (except { not: {} } for never)"); + } + if (schema.unevaluatedItems !== undefined) { + throw new Error("unevaluatedItems is not supported"); + } + if (schema.unevaluatedProperties !== undefined) { + throw new Error("unevaluatedProperties is not supported"); + } + if (schema.if !== undefined || schema.then !== undefined || schema.else !== undefined) { + throw new Error("Conditional schemas (if/then/else) are not supported"); + } + if (schema.dependentSchemas !== undefined || schema.dependentRequired !== undefined) { + throw new Error("dependentSchemas and dependentRequired are not supported"); + } + // Handle $ref + if (schema.$ref) { + const refPath = schema.$ref; + if (ctx.refs.has(refPath)) { + return ctx.refs.get(refPath); + } + if (ctx.processing.has(refPath)) { + // Circular reference - use lazy + return z.lazy(() => { + if (!ctx.refs.has(refPath)) { + throw new Error(`Circular reference not resolved: ${refPath}`); + } + return ctx.refs.get(refPath); + }); + } + ctx.processing.add(refPath); + const resolved = resolveRef(refPath, ctx); + const zodSchema = convertSchema(resolved, ctx); + ctx.refs.set(refPath, zodSchema); + ctx.processing.delete(refPath); + return zodSchema; + } + // Handle enum + if (schema.enum !== undefined) { + const enumValues = schema.enum; + // Special case: OpenAPI 3.0 null representation { type: "string", nullable: true, enum: [null] } + if (ctx.version === "openapi-3.0" && + schema.nullable === true && + enumValues.length === 1 && + enumValues[0] === null) { + return z.null(); + } + if (enumValues.length === 0) { + return z.never(); + } + if (enumValues.length === 1) { + return z.literal(enumValues[0]); + } + // Check if all values are strings + if (enumValues.every((v) => typeof v === "string")) { + return z.enum(enumValues); + } + // Mixed types - use union of literals + const literalSchemas = enumValues.map((v) => z.literal(v)); + if (literalSchemas.length < 2) { + return literalSchemas[0]; + } + return z.union([literalSchemas[0], literalSchemas[1], ...literalSchemas.slice(2)]); + } + // Handle const + if (schema.const !== undefined) { + return z.literal(schema.const); + } + // Handle type + const type = schema.type; + if (Array.isArray(type)) { + // Expand type array into anyOf union + const typeSchemas = type.map((t) => { + const typeSchema = { ...schema, type: t }; + return convertBaseSchema(typeSchema, ctx); + }); + if (typeSchemas.length === 0) { + return z.never(); + } + if (typeSchemas.length === 1) { + return typeSchemas[0]; + } + return z.union(typeSchemas); + } + if (!type) { + // No type specified - empty schema (any) + return z.any(); + } + let zodSchema; + switch (type) { + case "string": { + let stringSchema = z.string(); + // Apply format using .check() with Zod format functions + if (schema.format) { + const format = schema.format; + // Map common formats to Zod check functions + if (format === "email") { + stringSchema = stringSchema.check(z.email()); + } + else if (format === "uri" || format === "uri-reference") { + stringSchema = stringSchema.check(z.url()); + } + else if (format === "uuid" || format === "guid") { + stringSchema = stringSchema.check(z.uuid()); + } + else if (format === "date-time") { + stringSchema = stringSchema.check(z.iso.datetime({ offset: true })); + } + else if (format === "date") { + stringSchema = stringSchema.check(z.iso.date()); + } + else if (format === "time") { + stringSchema = stringSchema.check(z.regex(fullTime)); + } + else if (format === "duration") { + stringSchema = stringSchema.check(z.iso.duration()); + } + else if (format === "hostname") { + stringSchema = stringSchema.check(z.hostname()); + } + else if (format === "ipv4") { + stringSchema = stringSchema.check(z.ipv4()); + } + else if (format === "ipv6") { + stringSchema = stringSchema.check(z.ipv6()); + } + else if (format === "mac") { + stringSchema = stringSchema.check(z.mac()); + } + else if (format === "cidr") { + stringSchema = stringSchema.check(z.cidrv4()); + } + else if (format === "cidr-v6") { + stringSchema = stringSchema.check(z.cidrv6()); + } + else if (format === "base64") { + stringSchema = stringSchema.check(z.base64()); + } + else if (format === "base64url") { + stringSchema = stringSchema.check(z.base64url()); + } + else if (format === "e164") { + stringSchema = stringSchema.check(z.e164()); + } + else if (format === "credit_card") { + stringSchema = stringSchema.check(z.creditCard()); + } + else if (format === "iban") { + stringSchema = stringSchema.check(z.iban()); + } + else if (format === "jwt") { + stringSchema = stringSchema.check(z.jwt()); + } + else if (format === "emoji") { + stringSchema = stringSchema.check(z.emoji()); + } + else if (format === "nanoid") { + stringSchema = stringSchema.check(z.nanoid()); + } + else if (format === "cuid") { + stringSchema = stringSchema.check(z.cuid()); + } + else if (format === "cuid2") { + stringSchema = stringSchema.check(z.cuid2()); + } + else if (format === "ulid") { + stringSchema = stringSchema.check(z.ulid()); + } + else if (format === "xid") { + stringSchema = stringSchema.check(z.xid()); + } + else if (format === "ksuid") { + stringSchema = stringSchema.check(z.ksuid()); + } + // Note: json-string format is not currently supported by Zod + // Custom formats are ignored - keep as plain string + } + // Apply constraints + if (typeof schema.minLength === "number") { + stringSchema = stringSchema.min(schema.minLength); + } + if (typeof schema.maxLength === "number") { + stringSchema = stringSchema.max(schema.maxLength); + } + if (schema.pattern) { + // JSON Schema patterns are not implicitly anchored (match anywhere in string) + stringSchema = stringSchema.regex(new RegExp(schema.pattern)); + } + zodSchema = stringSchema; + break; + } + case "number": + case "integer": { + let numberSchema = type === "integer" ? z.number().int() : z.number(); + // Apply constraints + // In draft-04, `exclusiveMinimum: true` makes the sibling `minimum` exclusive rather than an independent bound, so the inclusive `.min()` is skipped; emitting it too would be dominated by the exclusive check and report a second, weaker issue. + if (typeof schema.minimum === "number" && schema.exclusiveMinimum !== true) { + numberSchema = numberSchema.min(schema.minimum); + } + if (typeof schema.maximum === "number" && schema.exclusiveMaximum !== true) { + numberSchema = numberSchema.max(schema.maximum); + } + if (typeof schema.exclusiveMinimum === "number") { + numberSchema = numberSchema.gt(schema.exclusiveMinimum); + } + else if (schema.exclusiveMinimum === true && typeof schema.minimum === "number") { + numberSchema = numberSchema.gt(schema.minimum); + } + if (typeof schema.exclusiveMaximum === "number") { + numberSchema = numberSchema.lt(schema.exclusiveMaximum); + } + else if (schema.exclusiveMaximum === true && typeof schema.maximum === "number") { + numberSchema = numberSchema.lt(schema.maximum); + } + if (typeof schema.multipleOf === "number") { + numberSchema = numberSchema.multipleOf(schema.multipleOf); + } + zodSchema = numberSchema; + break; + } + case "boolean": { + zodSchema = z.boolean(); + break; + } + case "null": { + zodSchema = z.null(); + break; + } + case "object": { + const shape = {}; + const properties = schema.properties || {}; + const requiredSet = new Set(schema.required || []); + const additionalSchema = typeof schema.additionalProperties === "object" + ? convertSchema(schema.additionalProperties, ctx) + : undefined; + // Convert properties - mark optional ones + for (const [key, propSchema] of Object.entries(properties)) { + const propZodSchema = convertSchema(propSchema, ctx); + // If not in required array, make it optional. assignProp so a __proto__ key becomes an own property instead of hitting the inherited setter + assignProp(shape, key, requiredSet.has(key) ? propZodSchema : propZodSchema.optional()); + } + // Handle patternProperties + if (schema.patternProperties) { + // patternProperties: keys matching pattern must satisfy corresponding schema. Use loose records so non-matching keys pass through + const patternProps = schema.patternProperties; + const patternKeys = Object.keys(patternProps); + const looseRecords = []; + for (const pattern of patternKeys) { + const patternValue = convertSchema(patternProps[pattern], ctx); + const keySchema = z.string().regex(new RegExp(pattern)); + looseRecords.push(z.looseRecord(keySchema, patternValue)); + } + // Build intersection: object schema + all pattern property records + const schemasToIntersect = []; + if (Object.keys(shape).length > 0) { + // Use passthrough so patternProperties can validate additional keys + schemasToIntersect.push(z.object(shape).passthrough()); + } + schemasToIntersect.push(...looseRecords); + if (schemasToIntersect.length === 0) { + zodSchema = z.object({}).passthrough(); + } + else if (schemasToIntersect.length === 1) { + zodSchema = schemasToIntersect[0]; + } + else { + // Chain intersections: (A & B) & C & D ... + let result = z.intersection(schemasToIntersect[0], schemasToIntersect[1]); + for (let i = 2; i < schemasToIntersect.length; i++) { + result = z.intersection(result, schemasToIntersect[i]); + } + zodSchema = result; + } + // When additionalProperties is false, reject keys that are neither defined in properties nor matched by any patternProperty. + if (schema.additionalProperties === false) { + const propertyKeys = Object.keys(shape); + const patterns = patternKeys.map((p) => new RegExp(p)); + const basePatternSchema = zodSchema; + zodSchema = zodSchema.check((payload) => { + if (!isPlainObject(payload.value)) + return; + const unrecognized = []; + for (const key of Object.keys(payload.value)) { + if (propertyKeys.includes(key)) + continue; + if (patterns.some((regex) => regex.test(key))) + continue; + unrecognized.push(key); + } + if (unrecognized.length) { + payload.issues.push({ + code: "unrecognized_keys", + keys: unrecognized, + input: payload.value, + inst: basePatternSchema, + }); + } + }); + } + } + else { + // Handle additionalProperties. In JSON Schema, additionalProperties defaults to true (allow any extra properties). In Zod, objects strip unknown keys by default, so we need to handle this explicitly + const objectSchema = z.object(shape); + if (schema.additionalProperties === false) { + // Strict mode - no extra properties allowed + zodSchema = objectSchema.strict(); + } + else if (additionalSchema) { + // Extra properties must match the specified schema + zodSchema = objectSchema.catchall(additionalSchema); + } + else { + // additionalProperties is true or undefined - allow any extra properties (passthrough) + zodSchema = objectSchema.passthrough(); + } + } + // propertyNames constrains key *names* only, and says nothing about which keys are required or how their values validate. Layering it on top of the result keeps properties/patternProperties/additionalProperties composing underneath. `true` allows every name, so it needs no guard. + const hasKeyGuard = schema.propertyNames !== undefined && schema.propertyNames !== true; + const minProperties = typeof schema.minProperties === "number" ? schema.minProperties : undefined; + const maxProperties = typeof schema.maxProperties === "number" ? schema.maxProperties : undefined; + if (hasKeyGuard || minProperties !== undefined || maxProperties !== undefined) { + let keySchema; + if (hasKeyGuard) { + // Keys are always strings, so a propertyNames subschema that omits `type` still constrains them — without this it would convert to z.any(). + const keyJSONSchema = typeof schema.propertyNames === "object" && schema.propertyNames.type === undefined + ? { type: "string", ...schema.propertyNames } + : schema.propertyNames; + keySchema = convertSchema(keyJSONSchema, ctx); + } + zodSchema = checkObjectGuards(zodSchema, { keySchema, minProperties, maxProperties }); + } + break; + } + case "array": { + // Check if this is a tuple (prefixItems or items as array) + const prefixItems = schema.prefixItems; + const items = schema.items; + if (prefixItems && Array.isArray(prefixItems)) { + // Tuple with prefixItems (draft-2020-12) + const minItems = typeof schema.minItems === "number" ? schema.minItems : 0; + const tupleItems = prefixItems.map((item) => convertSchema(item, ctx)); + const positionalItems = applyMinItems(tupleItems, minItems); + const rest = !Array.isArray(items) ? getTupleRest(items, ctx) : undefined; + const tupleSchema = z.tuple(positionalItems); + zodSchema = rest ? tupleSchema.rest(rest) : tupleSchema; + // Apply minItems/maxItems constraints to tuples + if (typeof schema.minItems === "number") { + zodSchema = zodSchema.check(z.minLength(schema.minItems)); + } + if (typeof schema.maxItems === "number") { + zodSchema = zodSchema.check(z.maxLength(schema.maxItems)); + } + } + else if (Array.isArray(items)) { + // Tuple with items array (draft-7) + const minItems = typeof schema.minItems === "number" ? schema.minItems : 0; + const tupleItems = items.map((item) => convertSchema(item, ctx)); + const positionalItems = applyMinItems(tupleItems, minItems); + const rest = getTupleRest(schema.additionalItems, ctx); + const tupleSchema = z.tuple(positionalItems); + zodSchema = rest ? tupleSchema.rest(rest) : tupleSchema; + // Apply minItems/maxItems constraints to tuples + if (typeof schema.minItems === "number") { + zodSchema = zodSchema.check(z.minLength(schema.minItems)); + } + if (typeof schema.maxItems === "number") { + zodSchema = zodSchema.check(z.maxLength(schema.maxItems)); + } + } + else if (items !== undefined) { + // Regular array + const element = convertSchema(items, ctx); + let arraySchema = z.array(element); + // Apply constraints + if (typeof schema.minItems === "number") { + arraySchema = arraySchema.min(schema.minItems); + } + if (typeof schema.maxItems === "number") { + arraySchema = arraySchema.max(schema.maxItems); + } + zodSchema = arraySchema; + } + else { + // No items specified - array of any + zodSchema = z.array(z.any()); + } + // minContains/maxContains only constrain anything when `contains` itself is present + if (schema.uniqueItems === true || schema.contains !== undefined) { + zodSchema = checkArrayGuards(zodSchema, { + uniqueItems: schema.uniqueItems === true, + containsSchema: schema.contains !== undefined ? convertSchema(schema.contains, ctx) : undefined, + minContains: typeof schema.minContains === "number" ? schema.minContains : undefined, + maxContains: typeof schema.maxContains === "number" ? schema.maxContains : undefined, + }); + } + break; + } + default: + throw new Error(`Unsupported type: ${type}`); + } + return zodSchema; +} +function convertSchema(schema, ctx) { + if (typeof schema === "boolean") { + return schema ? z.any() : z.never(); + } + // Convert base schema first (ignoring composition keywords) + let baseSchema = convertBaseSchema(schema, ctx); + const hasExplicitType = schema.type || schema.enum !== undefined || schema.const !== undefined; + // Process composition keywords LAST (they can appear together) + // Handle anyOf - wrap base schema with union + if (schema.anyOf && Array.isArray(schema.anyOf)) { + const options = schema.anyOf.map((s) => convertSchema(s, ctx)); + const anyOfUnion = z.union(options); + baseSchema = hasExplicitType ? z.intersection(baseSchema, anyOfUnion) : anyOfUnion; + } + // Handle oneOf - exclusive union (exactly one must match) + if (schema.oneOf && Array.isArray(schema.oneOf)) { + const options = schema.oneOf.map((s) => convertSchema(s, ctx)); + const oneOfUnion = z.xor(options); + baseSchema = hasExplicitType ? z.intersection(baseSchema, oneOfUnion) : oneOfUnion; + } + // Handle allOf - wrap base schema with intersection + if (schema.allOf && Array.isArray(schema.allOf)) { + if (schema.allOf.length === 0) { + baseSchema = hasExplicitType ? baseSchema : z.any(); + } + else { + let result = hasExplicitType ? baseSchema : convertSchema(schema.allOf[0], ctx); + const startIdx = hasExplicitType ? 0 : 1; + for (let i = startIdx; i < schema.allOf.length; i++) { + result = z.intersection(result, convertSchema(schema.allOf[i], ctx)); + } + baseSchema = result; + } + } + // Handle nullable (OpenAPI 3.0) + if (schema.nullable === true && ctx.version === "openapi-3.0") { + baseSchema = z.nullable(baseSchema); + } + // Handle readOnly + if (schema.readOnly === true) { + baseSchema = z.readonly(baseSchema); + } + // Apply `default` so it wraps the fully-composed schema. This ensures `parse(undefined) -> default` works regardless of which branch of `convertBaseSchema` produced the inner schema (enum/const/not/typed/etc.). + if (schema.default !== undefined) { + baseSchema = baseSchema.default(schema.default); + } + // Collect non-description annotation metadata into the user-supplied registry. Description is handled separately below via `.describe()` to preserve the contract that `schema.description` reads from globalRegistry. + const extraMeta = {}; + const coreMetadataKeys = ["$id", "id", "$comment", "$anchor", "$vocabulary", "$dynamicRef", "$dynamicAnchor"]; + for (const key of coreMetadataKeys) { + if (key in schema) { + extraMeta[key] = schema[key]; + } + } + const contentMetadataKeys = ["contentEncoding", "contentMediaType", "contentSchema"]; + for (const key of contentMetadataKeys) { + if (key in schema) { + extraMeta[key] = schema[key]; + } + } + // `propertyNames` and `contains` are enforced by a guard, which `toJSONSchema` cannot infer, so the original keyword is carried as metadata to keep the round-trip lossless. Only where it was actually applied: on any other type it is inert, and on a `$ref` the metadata would land on the target every reference shares. A carried subschema is copied verbatim while the root `$defs` stay behind, so one holding a `$ref` is dropped rather than emitted as a dangling reference. + if (schema.type === "object" && schema.$ref === undefined) { + if (schema.propertyNames !== undefined && !containsRef(schema.propertyNames)) { + extraMeta.propertyNames = schema.propertyNames; + } + for (const key of ["minProperties", "maxProperties"]) { + if (schema[key] !== undefined) + extraMeta[key] = schema[key]; + } + } + if (schema.type === "array" && schema.$ref === undefined) { + if (schema.contains !== undefined && !containsRef(schema.contains)) { + extraMeta.contains = schema.contains; + } + for (const key of ["uniqueItems", "minContains", "maxContains"]) { + if (schema[key] !== undefined) + extraMeta[key] = schema[key]; + } + } + for (const key of Object.keys(schema)) { + if (!RECOGNIZED_KEYS.has(key)) { + assignProp(extraMeta, key, schema[key]); + } + } + if (Object.keys(extraMeta).length > 0) { + ctx.registry.add(baseSchema, extraMeta); + } + // Apply description last. `.describe()` clones the schema and sets `_zod.parent` on the clone, so registry lookups on the returned reference still resolve `extraMeta` via parent inheritance. + if (schema.description) { + baseSchema = baseSchema.describe(schema.description); + } + return baseSchema; +} +/** + * Converts a JSON Schema to a Zod schema. This function should be considered semi-experimental. It's behavior is liable to change. */ +export function fromJSONSchema(schema, params) { + // Handle boolean schemas + if (typeof schema === "boolean") { + return schema ? z.any() : z.never(); + } + // Normalize input via a JSON round-trip. This guarantees the converter walks a plain, finite, JSON-valid object graph: cyclic inputs fail here, getter/Proxy-based properties are materialized into static values, and class instances collapse to plain objects. + let normalized; + try { + normalized = JSON.parse(JSON.stringify(schema)); + } + catch { + throw new Error("fromJSONSchema input is not valid JSON (possibly cyclic); use $defs/$ref for recursive schemas"); + } + const version = detectVersion(normalized, params?.defaultTarget); + const defs = (normalized.$defs || normalized.definitions || {}); + const ctx = { + version, + defs, + refs: new Map(), + processing: new Set(), + rootSchema: normalized, + registry: params?.registry ?? globalRegistry, + }; + return convertSchema(normalized, ctx); +} diff --git a/tests/fixtures/npm/node_modules/zod/v4/classic/in-out.d.cts b/tests/fixtures/npm/node_modules/zod/v4/classic/in-out.d.cts new file mode 100644 index 000000000..09b530fe2 --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v4/classic/in-out.d.cts @@ -0,0 +1,8 @@ +import type * as core from "../core/index.cjs"; +import * as schemas from "./schemas.cjs"; +export type input = core.input; +export type output = core.output; +/** Returns a copy of the schema with every pipe replaced by its input side. A codec's checks are dropped: they constrain the decoded value the input side never produces. */ +export declare function input(schema: T): schemas.ZodType, core.input>; +/** Returns a copy of the schema with every pipe replaced by its output side, carrying over the pipe's own checks. */ +export declare function output(schema: T): schemas.ZodType, core.output>; diff --git a/tests/fixtures/npm/node_modules/zod/v4/classic/in-out.js b/tests/fixtures/npm/node_modules/zod/v4/classic/in-out.js new file mode 100644 index 000000000..4a0002688 --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v4/classic/in-out.js @@ -0,0 +1,36 @@ +import { clone, mergeDefs } from "../core/util.js"; +import { visit } from "../core/visit.js"; +import * as schemas from "./schemas.js"; +/** Appends a pipe's own checks to the side that replaces it, mirroring how `.check()` clones. */ +function withChecks(side, checks) { + if (!checks?.length) + return side; + const def = side._zod.def; + return clone(side, mergeDefs(def, { checks: [...(def.checks ?? []), ...checks] }), { parent: true }); +} +/** The out side, carrying the pipe's own checks: those run against the decoded value, which is what `out` produces. */ +function outSide(def) { + return withChecks(def.out, def.checks); +} +/** The in side. `z.preprocess` pipes a transform into a schema, and a bare transform validates nothing, so the schema it feeds is the real input side — the resolution `toJSONSchema` already makes for this case. */ +function inSide(def) { + return def.in._zod.traits.has("$ZodTransform") ? outSide(def) : def.in; +} +/** Returns a copy of the schema with every pipe replaced by its input side. A codec's checks are dropped: they constrain the decoded value the input side never produces. */ +export function input(schema) { + return visit(schema, { + pipe: (s) => inSide(s._zod.def), + // A default value belongs to the output side, so a rewritten inner type leaves it stranded. `.default()` widens the declared input type with `undefined`, and `optional` is what carries that across. + default: (s, rewritten) => (rewritten ? schemas.optional(s._zod.def.innerType) : s), + // A catch value is output-side too, but `.catch()` leaves the declared input type alone, so the inner schema stands on its own. + catch: (s, rewritten) => (rewritten ? s._zod.def.innerType : s), + }); +} +/** Returns a copy of the schema with every pipe replaced by its output side, carrying over the pipe's own checks. */ +export function output(schema) { + return visit(schema, { + pipe: (s) => outSide(s._zod.def), + // A prefault value is fed through the schema, which makes it input-side, so a rewritten inner type leaves it stranded. + prefault: (s, rewritten) => (rewritten ? s._zod.def.innerType : s), + }); +} diff --git a/tests/fixtures/npm/node_modules/zod/v4/classic/index.d.cts b/tests/fixtures/npm/node_modules/zod/v4/classic/index.d.cts new file mode 100644 index 000000000..b5fc2a17b --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v4/classic/index.d.cts @@ -0,0 +1,3 @@ +import * as z from "./external.cjs"; +export * from "./external.cjs"; +export { z, z as default }; diff --git a/tests/fixtures/npm/node_modules/zod/v4/classic/iso.d.cts b/tests/fixtures/npm/node_modules/zod/v4/classic/iso.d.cts new file mode 100644 index 000000000..db764d9f8 --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v4/classic/iso.d.cts @@ -0,0 +1,7 @@ +import * as core from "../core/index.cjs"; +import { ZodISODate, ZodISODateTime, ZodISODuration, ZodISOTime } from "./schemas.cjs"; +export { ZodISODate, ZodISODateTime, ZodISODuration, ZodISOTime } from "./schemas.cjs"; +export declare function datetime(params?: string | core.$ZodISODateTimeParams): ZodISODateTime; +export declare function date(params?: string | core.$ZodISODateParams): ZodISODate; +export declare function time(params?: string | core.$ZodISOTimeParams): ZodISOTime; +export declare function duration(params?: string | core.$ZodISODurationParams): ZodISODuration; diff --git a/tests/fixtures/npm/node_modules/zod/v4/classic/iso.js b/tests/fixtures/npm/node_modules/zod/v4/classic/iso.js new file mode 100644 index 000000000..d38f24ddd --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v4/classic/iso.js @@ -0,0 +1,15 @@ +import * as core from "../core/index.js"; +import { ZodISODate, ZodISODateTime, ZodISODuration, ZodISOTime } from "./schemas.js"; +export { ZodISODate, ZodISODateTime, ZodISODuration, ZodISOTime } from "./schemas.js"; +export function datetime(params) { + return core._isoDateTime(ZodISODateTime, params); +} +export function date(params) { + return core._isoDate(ZodISODate, params); +} +export function time(params) { + return core._isoTime(ZodISOTime, params); +} +export function duration(params) { + return core._isoDuration(ZodISODuration, params); +} diff --git a/tests/fixtures/npm/node_modules/zod/v4/classic/parse.d.cts b/tests/fixtures/npm/node_modules/zod/v4/classic/parse.d.cts new file mode 100644 index 000000000..408566b52 --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v4/classic/parse.d.cts @@ -0,0 +1,44 @@ +import * as core from "../core/index.cjs"; +import { type ZodError } from "./errors.cjs"; +export type ZodSafeParseResult = ZodSafeParseSuccess | ZodSafeParseError; +export type ZodSafeParseSuccess = { + success: true; + data: T; + error?: never; +}; +export type ZodSafeParseError = { + success: false; + data?: never; + error: ZodError; +}; +export declare const parse: (schema: T, value: unknown, _ctx?: core.ParseContext, _params?: { + callee?: core.util.AnyFunc; + Err?: core.$ZodErrorClass; +}) => core.output; +export declare const parseAsync: (schema: T, value: unknown, _ctx?: core.ParseContext, _params?: { + callee?: core.util.AnyFunc; + Err?: core.$ZodErrorClass; +}) => Promise>; +export declare const safeParse: (schema: T, value: unknown, _ctx?: core.ParseContext) => ZodSafeParseResult>; +export declare const safeParseAsync: (schema: T, value: unknown, _ctx?: core.ParseContext) => Promise>>; +export { validate, validateAsync } from "../core/index.cjs"; +export declare const encode: (schema: T, value: core.output, _ctx?: core.ParseContext, _params?: { + callee?: core.util.AnyFunc; + Err?: core.$ZodErrorClass; +}) => core.input; +export declare const decode: (schema: T, value: core.input, _ctx?: core.ParseContext, _params?: { + callee?: core.util.AnyFunc; + Err?: core.$ZodErrorClass; +}) => core.output; +export declare const encodeAsync: (schema: T, value: core.output, _ctx?: core.ParseContext, _params?: { + callee?: core.util.AnyFunc; + Err?: core.$ZodErrorClass; +}) => Promise>; +export declare const decodeAsync: (schema: T, value: core.input, _ctx?: core.ParseContext, _params?: { + callee?: core.util.AnyFunc; + Err?: core.$ZodErrorClass; +}) => Promise>; +export declare const safeEncode: (schema: T, value: core.output, _ctx?: core.ParseContext) => ZodSafeParseResult>; +export declare const safeDecode: (schema: T, value: core.input, _ctx?: core.ParseContext) => ZodSafeParseResult>; +export declare const safeEncodeAsync: (schema: T, value: core.output, _ctx?: core.ParseContext) => Promise>>; +export declare const safeDecodeAsync: (schema: T, value: core.input, _ctx?: core.ParseContext) => Promise>>; diff --git a/tests/fixtures/npm/node_modules/zod/v4/classic/parse.js b/tests/fixtures/npm/node_modules/zod/v4/classic/parse.js new file mode 100644 index 000000000..514aacc64 --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v4/classic/parse.js @@ -0,0 +1,16 @@ +import * as core from "../core/index.js"; +import { ZodRealError } from "./errors.js"; +export const parse = /* @__PURE__ */ core._parse(ZodRealError); +export const parseAsync = /* @__PURE__ */ core._parseAsync(ZodRealError); +export const safeParse = /* @__PURE__ */ core._safeParse(ZodRealError); +export const safeParseAsync = /* @__PURE__ */ core._safeParseAsync(ZodRealError); +export { validate, validateAsync } from "../core/index.js"; +// Codec functions +export const encode = /* @__PURE__ */ core._encode(ZodRealError); +export const decode = /* @__PURE__ */ core._decode(ZodRealError); +export const encodeAsync = /* @__PURE__ */ core._encodeAsync(ZodRealError); +export const decodeAsync = /* @__PURE__ */ core._decodeAsync(ZodRealError); +export const safeEncode = /* @__PURE__ */ core._safeEncode(ZodRealError); +export const safeDecode = /* @__PURE__ */ core._safeDecode(ZodRealError); +export const safeEncodeAsync = /* @__PURE__ */ core._safeEncodeAsync(ZodRealError); +export const safeDecodeAsync = /* @__PURE__ */ core._safeDecodeAsync(ZodRealError); diff --git a/tests/fixtures/npm/node_modules/zod/v4/classic/schemas.d.cts b/tests/fixtures/npm/node_modules/zod/v4/classic/schemas.d.cts new file mode 100644 index 000000000..226a3ae25 --- /dev/null +++ b/tests/fixtures/npm/node_modules/zod/v4/classic/schemas.d.cts @@ -0,0 +1,816 @@ +import type { $ZodBigIntFormats } from "../core/checks.cjs"; +import * as core from "../core/index.cjs"; +import { util, type $ZodNumberFormats } from "../core/index.cjs"; +import type { StandardSchemaWithJSONProps } from "../core/standard-schema.cjs"; +import * as parse from "./parse.cjs"; +export type ZodStandardSchemaWithJSON = StandardSchemaWithJSONProps, core.output>; +export interface ZodType = core.$ZodTypeInternals> extends core.$ZodType { + def: Internals["def"]; + type: Internals["def"]["type"]; + /** @deprecated Use `.def` instead. */ + _def: Internals["def"]; + /** @deprecated Use `z.output` instead. */ + _output: Internals["output"]; + /** @deprecated Use `z.input` instead. */ + _input: Internals["input"]; + "~standard": ZodStandardSchemaWithJSON; + /** Converts this schema to a JSON Schema representation. */ + toJSONSchema(params?: core.ToJSONSchemaParams): core.ZodStandardJSONSchemaPayload; + check(...checks: (core.CheckFn> | core.$ZodCheck>)[]): this; + with(...checks: (core.CheckFn> | core.$ZodCheck>)[]): this; + clone(def?: Internals["def"], params?: { + parent: boolean; + }): this; + register(registry: R, ...meta: this extends R["_schema"] ? undefined extends R["_meta"] ? [core.$replace?] : [core.$replace] : ["Incompatible schema"]): this; + brand(value?: T): PropertyKey extends T ? this : core.$ZodBranded; + parse(data: unknown, params?: core.ParseContext): core.output; + safeParse(data: unknown, params?: core.ParseContext): parse.ZodSafeParseResult>; + parseAsync(data: unknown, params?: core.ParseContext): Promise>; + safeParseAsync(data: unknown, params?: core.ParseContext): Promise>>; + spa: (data: unknown, params?: core.ParseContext) => Promise>>; + validate(data: unknown, params?: core.ParseContext): data is core.input; + validateAsync(data: unknown, params?: core.ParseContext): Promise; + encode(data: core.output, params?: core.ParseContext): core.input; + decode(data: core.input, params?: core.ParseContext): core.output; + encodeAsync(data: core.output, params?: core.ParseContext): Promise>; + decodeAsync(data: core.input, params?: core.ParseContext): Promise>; + safeEncode(data: core.output, params?: core.ParseContext): parse.ZodSafeParseResult>; + safeDecode(data: core.input, params?: core.ParseContext): parse.ZodSafeParseResult>; + safeEncodeAsync(data: core.output, params?: core.ParseContext): Promise>>; + safeDecodeAsync(data: core.input, params?: core.ParseContext): Promise>>; + refine) => unknown | Promise>(check: Ch, params?: string | core.$ZodCustomParams): Ch extends (arg: any) => arg is infer R ? this & ZodType> : this; + superRefine(refinement: (arg: core.output, ctx: core.$RefinementCtx>) => void | Promise, params?: core.$ZodSuperRefineParams): this; + overwrite(fn: (x: core.output) => core.output): this; + optional(): ZodOptional; + exactOptional(): ZodExactOptional; + nonoptional(params?: string | core.$ZodNonOptionalParams): ZodNonOptional; + nullable(): ZodNullable; + nullish(): ZodOptional>; + default(def: util.NoUndefined>): ZodDefault; + default(def: () => util.NoUndefined>): ZodDefault; + prefault(def: () => core.input): ZodPrefault; + prefault(def: core.input): ZodPrefault; + array(): ZodArray; + or(option: T): ZodUnion<[this, T]>; + and(incoming: T): ZodIntersection; + transform(transform: (arg: core.output, ctx: core.$RefinementCtx>) => NewOut | Promise): ZodPipe, core.output>>; + catch(def: core.output): ZodCatch; + catch(def: (ctx: core.$ZodCatchCtx) => core.output): ZodCatch; + pipe>>(target: T | core.$ZodType>): ZodPipe; + readonly(): ZodReadonly; + /** Returns a new instance that has been registered in `z.globalRegistry` with the specified description */ + describe(description: string): this; + description?: string; + /** Returns the metadata associated with this instance in `z.globalRegistry` */ + meta(): core.$replace | undefined; + /** Returns a new instance that has been registered in `z.globalRegistry` with the specified metadata */ + meta(data: core.$replace): this; + /** @deprecated Try safe-parsing `undefined` (this is what `isOptional` does internally): + * + * ```ts + * const schema = z.string().optional(); + * const isOptional = schema.safeParse(undefined).success; // true + * ``` + */ + isOptional(): boolean; + /** + * @deprecated Try safe-parsing `null` (this is what `isNullable` does internally): + * + * ```ts + * const schema = z.string().nullable(); + * const isNullable = schema.safeParse(null).success; // true + * ``` + */ + isNullable(): boolean; + apply(fn: (schema: this, ...args: TArgs) => T, ...args: TArgs): T; +} +export interface _ZodType extends ZodType { +} +export declare const ZodType: core.$constructor; +export interface _ZodString = core.$ZodStringInternals> extends _ZodType { + format: string | null; + minLength: number | null; + maxLength: number | null; + regex(regex: RegExp, params?: string | core.$ZodCheckRegexParams): this; + includes(value: string, params?: string | core.$ZodCheckIncludesParams): this; + startsWith(value: string, params?: string | core.$ZodCheckStartsWithParams): this; + endsWith(value: string, params?: string | core.$ZodCheckEndsWithParams): this; + min(minLength: number, params?: string | core.$ZodCheckMinLengthParams): this; + max(maxLength: number, params?: string | core.$ZodCheckMaxLengthParams): this; + length(len: number, params?: string | core.$ZodCheckLengthEqualsParams): this; + nonempty(params?: string | core.$ZodCheckMinLengthParams): this; + lowercase(params?: string | core.$ZodCheckLowerCaseParams): this; + uppercase(params?: string | core.$ZodCheckUpperCaseParams): this; + trim(): this; + normalize(form?: "NFC" | "NFD" | "NFKC" | "NFKD" | (string & {})): this; + toLowerCase(): this; + toUpperCase(): this; + slugify(): this; +} +/** @internal */ +export declare const _ZodString: core.$constructor<_ZodString>; +export interface ZodString extends _ZodString> { + /** @deprecated Use `z.email()` instead. */ + email(params?: string | core.$ZodCheckEmailParams): this; + /** @deprecated Use `z.url()` instead. */ + url(params?: string | core.$ZodCheckURLParams): this; + /** @deprecated Use `z.jwt()` instead. */ + jwt(params?: string | core.$ZodCheckJWTParams): this; + /** @deprecated Use `z.emoji()` instead. */ + emoji(params?: string | core.$ZodCheckEmojiParams): this; + /** @deprecated Use `z.guid()` instead. */ + guid(params?: string | core.$ZodCheckGUIDParams): this; + /** @deprecated Use `z.uuid()` instead. */ + uuid(params?: string | core.$ZodCheckUUIDParams): this; + /** @deprecated Use `z.uuid()` instead. */ + uuidv4(params?: string | core.$ZodCheckUUIDParams): this; + /** @deprecated Use `z.uuid()` instead. */ + uuidv6(params?: string | core.$ZodCheckUUIDParams): this; + /** @deprecated Use `z.uuid()` instead. */ + uuidv7(params?: string | core.$ZodCheckUUIDParams): this; + /** @deprecated Use `z.nanoid()` instead. */ + nanoid(params?: string | core.$ZodCheckNanoIDParams): this; + /** @deprecated Use `z.guid()` instead. */ + guid(params?: string | core.$ZodCheckGUIDParams): this; + /** + * @deprecated CUID v1 is deprecated by its authors due to information leakage + * (timestamps embedded in the id). Use `z.cuid2()` instead. + * See https://github.com/paralleldrive/cuid. + */ + cuid(params?: string | core.$ZodCheckCUIDParams): this; + /** @deprecated Use `z.cuid2()` instead. */ + cuid2(params?: string | core.$ZodCheckCUID2Params): this; + /** @deprecated Use `z.ulid()` instead. */ + ulid(params?: string | core.$ZodCheckULIDParams): this; + /** @deprecated Use `z.base64()` instead. */ + base64(params?: string | core.$ZodCheckBase64Params): this; + /** @deprecated Use `z.base64url()` instead. */ + base64url(params?: string | core.$ZodCheckBase64URLParams): this; + /** @deprecated Use `z.xid()` instead. */ + xid(params?: string | core.$ZodCheckXIDParams): this; + /** @deprecated Use `z.ksuid()` instead. */ + ksuid(params?: string | core.$ZodCheckKSUIDParams): this; + /** @deprecated Use `z.ipv4()` instead. */ + ipv4(params?: string | core.$ZodCheckIPv4Params): this; + /** @deprecated Use `z.ipv6()` instead. */ + ipv6(params?: string | core.$ZodCheckIPv6Params): this; + /** @deprecated Use `z.cidrv4()` instead. */ + cidrv4(params?: string | core.$ZodCheckCIDRv4Params): this; + /** @deprecated Use `z.cidrv6()` instead. */ + cidrv6(params?: string | core.$ZodCheckCIDRv6Params): this; + /** @deprecated Use `z.e164()` instead. */ + e164(params?: string | core.$ZodCheckE164Params): this; + /** @deprecated Use `z.iso.datetime()` instead. */ + datetime(params?: string | core.$ZodCheckISODateTimeParams): this; + /** @deprecated Use `z.iso.date()` instead. */ + date(params?: string | core.$ZodCheckISODateParams): this; + /** @deprecated Use `z.iso.time()` instead. */ + time(params?: string | core.$ZodCheckISOTimeParams): this; + /** @deprecated Use `z.iso.duration()` instead. */ + duration(params?: string | core.$ZodCheckISODurationParams): this; +} +export declare const ZodString: core.$constructor; +export declare function string(params?: string | core.$ZodStringParams): ZodString; +export declare function string(params?: string | core.$ZodStringParams): core.$ZodType; +export interface ZodStringFormat extends _ZodString> { +} +export declare const ZodStringFormat: core.$constructor; +export interface ZodISODateTime extends ZodStringFormat { + _zod: core.$ZodISODateTimeInternals; +} +export declare const ZodISODateTime: core.$constructor; +export interface ZodISODate extends ZodStringFormat { + _zod: core.$ZodISODateInternals; +} +export declare const ZodISODate: core.$constructor; +export interface ZodISOTime extends ZodStringFormat { + _zod: core.$ZodISOTimeInternals; +} +export declare const ZodISOTime: core.$constructor; +export interface ZodISODuration extends ZodStringFormat { + _zod: core.$ZodISODurationInternals; +} +export declare const ZodISODuration: core.$constructor; +export interface ZodEmail extends ZodStringFormat<"email"> { + _zod: core.$ZodEmailInternals; +} +export declare const ZodEmail: core.$constructor; +export declare function email(params?: string | core.$ZodEmailParams): ZodEmail; +export interface ZodGUID extends ZodStringFormat<"guid"> { + _zod: core.$ZodGUIDInternals; +} +export declare const ZodGUID: core.$constructor; +export declare function guid(params?: string | core.$ZodGUIDParams): ZodGUID; +export interface ZodUUID extends ZodStringFormat<"uuid"> { + _zod: core.$ZodUUIDInternals; +} +export declare const ZodUUID: core.$constructor; +export declare function uuid(params?: string | core.$ZodUUIDParams): ZodUUID; +export declare function uuidv4(params?: string | core.$ZodUUIDv4Params): ZodUUID; +export declare function uuidv6(params?: string | core.$ZodUUIDv6Params): ZodUUID; +export declare function uuidv7(params?: string | core.$ZodUUIDv7Params): ZodUUID; +export interface ZodURL extends ZodStringFormat<"url"> { + _zod: core.$ZodURLInternals; +} +export declare const ZodURL: core.$constructor; +export declare function url(params?: string | core.$ZodURLParams): ZodURL; +export declare function httpUrl(params?: string | Omit): ZodURL; +export interface ZodEmoji extends ZodStringFormat<"emoji"> { + _zod: core.$ZodEmojiInternals; +} +export declare const ZodEmoji: core.$constructor; +export declare function emoji(params?: string | core.$ZodEmojiParams): ZodEmoji; +export interface ZodNanoID extends ZodStringFormat<"nanoid"> { + _zod: core.$ZodNanoIDInternals; +} +export declare const ZodNanoID: core.$constructor; +export declare function nanoid(params?: string | core.$ZodNanoIDParams): ZodNanoID; +/** + * @deprecated CUID v1 is deprecated by its authors due to information leakage + * (timestamps embedded in the id). Use {@link ZodCUID2} instead. + * See https://github.com/paralleldrive/cuid. + */ +export interface ZodCUID extends ZodStringFormat<"cuid"> { + _zod: core.$ZodCUIDInternals; +} +/** + * @deprecated CUID v1 is deprecated by its authors due to information leakage + * (timestamps embedded in the id). Use {@link ZodCUID2} instead. + * See https://github.com/paralleldrive/cuid. + */ +export declare const ZodCUID: core.$constructor; +/** + * Validates a CUID v1 string. + * + * @deprecated CUID v1 is deprecated by its authors due to information leakage + * (timestamps embedded in the id). Use {@link cuid2 | `z.cuid2()`} instead. + * See https://github.com/paralleldrive/cuid. + */ +export declare function cuid(params?: string | core.$ZodCUIDParams): ZodCUID; +export interface ZodCUID2 extends ZodStringFormat<"cuid2"> { + _zod: core.$ZodCUID2Internals; +} +export declare const ZodCUID2: core.$constructor; +export declare function cuid2(params?: string | core.$ZodCUID2Params): ZodCUID2; +export interface ZodULID extends ZodStringFormat<"ulid"> { + _zod: core.$ZodULIDInternals; +} +export declare const ZodULID: core.$constructor; +export declare function ulid(params?: string | core.$ZodULIDParams): ZodULID; +export interface ZodXID extends ZodStringFormat<"xid"> { + _zod: core.$ZodXIDInternals; +} +export declare const ZodXID: core.$constructor; +export declare function xid(params?: string | core.$ZodXIDParams): ZodXID; +export interface ZodKSUID extends ZodStringFormat<"ksuid"> { + _zod: core.$ZodKSUIDInternals; +} +export declare const ZodKSUID: core.$constructor; +export declare function ksuid(params?: string | core.$ZodKSUIDParams): ZodKSUID; +export interface ZodIPv4 extends ZodStringFormat<"ipv4"> { + _zod: core.$ZodIPv4Internals; +} +export declare const ZodIPv4: core.$constructor; +export declare function ipv4(params?: string | core.$ZodIPv4Params): ZodIPv4; +export interface ZodMAC extends ZodStringFormat<"mac"> { + _zod: core.$ZodMACInternals; +} +export declare const ZodMAC: core.$constructor; +export declare function mac(params?: string | core.$ZodMACParams): ZodMAC; +export interface ZodIPv6 extends ZodStringFormat<"ipv6"> { + _zod: core.$ZodIPv6Internals; +} +export declare const ZodIPv6: core.$constructor; +export declare function ipv6(params?: string | core.$ZodIPv6Params): ZodIPv6; +export interface ZodCIDRv4 extends ZodStringFormat<"cidrv4"> { + _zod: core.$ZodCIDRv4Internals; +} +export declare const ZodCIDRv4: core.$constructor; +export declare function cidrv4(params?: string | core.$ZodCIDRv4Params): ZodCIDRv4; +export interface ZodCIDRv6 extends ZodStringFormat<"cidrv6"> { + _zod: core.$ZodCIDRv6Internals; +} +export declare const ZodCIDRv6: core.$constructor; +export declare function cidrv6(params?: string | core.$ZodCIDRv6Params): ZodCIDRv6; +export interface ZodBase64 extends ZodStringFormat<"base64"> { + _zod: core.$ZodBase64Internals; +} +export declare const ZodBase64: core.$constructor; +export declare function base64(params?: string | core.$ZodBase64Params): ZodBase64; +export interface ZodBase64URL extends ZodStringFormat<"base64url"> { + _zod: core.$ZodBase64URLInternals; +} +export declare const ZodBase64URL: core.$constructor; +export declare function base64url(params?: string | core.$ZodBase64URLParams): ZodBase64URL; +export interface ZodE164 extends ZodStringFormat<"e164"> { + _zod: core.$ZodE164Internals; +} +export declare const ZodE164: core.$constructor; +export declare function e164(params?: string | core.$ZodE164Params): ZodE164; +export interface ZodCreditCard extends ZodStringFormat<"credit_card"> { + _zod: core.$ZodCreditCardInternals; +} +export declare const ZodCreditCard: core.$constructor; +export declare function creditCard(params?: string | core.$ZodCreditCardParams): ZodCreditCard; +export interface ZodIBAN extends ZodStringFormat<"iban"> { + _zod: core.$ZodIBANInternals; +} +export declare const ZodIBAN: core.$constructor; +export declare function iban(params?: string | core.$ZodIBANParams): ZodIBAN; +export interface ZodJWT extends ZodStringFormat<"jwt"> { + _zod: core.$ZodJWTInternals; +} +export declare const ZodJWT: core.$constructor; +export declare function jwt(params?: string | core.$ZodJWTParams): ZodJWT; +export interface ZodCustomStringFormat extends ZodStringFormat, core.$ZodCustomStringFormat { + _zod: core.$ZodCustomStringFormatInternals; + "~standard": ZodStandardSchemaWithJSON; +} +export declare const ZodCustomStringFormat: core.$constructor; +export declare function stringFormat(format: Format, fnOrRegex: ((arg: string) => util.MaybeAsync) | RegExp, _params?: string | core.$ZodStringFormatParams): ZodCustomStringFormat; +export declare function hostname(_params?: string | core.$ZodStringFormatParams): ZodCustomStringFormat<"hostname">; +export declare function hex(_params?: string | core.$ZodStringFormatParams): ZodCustomStringFormat<"hex">; +export declare function currencyCode(_params?: string | core.$ZodStringFormatParams): ZodCustomStringFormat<"currency_code">; +export declare function hash(alg: Alg, params?: { + enc?: Enc; +} & core.$ZodStringFormatParams): ZodCustomStringFormat<`${Alg}_${Enc}`>; +export interface _ZodNumber extends _ZodType { + gt(value: number, params?: string | core.$ZodCheckGreaterThanParams): this; + /** Identical to .min() */ + gte(value: number, params?: string | core.$ZodCheckGreaterThanParams): this; + min(value: number, params?: string | core.$ZodCheckGreaterThanParams): this; + lt(value: number, params?: string | core.$ZodCheckLessThanParams): this; + /** Identical to .max() */ + lte(value: number, params?: string | core.$ZodCheckLessThanParams): this; + max(value: number, params?: string | core.$ZodCheckLessThanParams): this; + /** Consider `z.int()` instead. This API is considered *legacy*; it will never be removed but a better alternative exists. */ + int(params?: string | core.$ZodCheckNumberFormatParams): this; + /** @deprecated This is now identical to `.int()`. Only numbers in the safe integer range are accepted. */ + safe(params?: string | core.$ZodCheckNumberFormatParams): this; + positive(params?: string | core.$ZodCheckGreaterThanParams): this; + nonnegative(params?: string | core.$ZodCheckGreaterThanParams): this; + negative(params?: string | core.$ZodCheckLessThanParams): this; + nonpositive(params?: string | core.$ZodCheckLessThanParams): this; + multipleOf(value: number, params?: string | core.$ZodCheckMultipleOfParams): this; + /** @deprecated Use `.multipleOf()` instead. */ + step(value: number, params?: string | core.$ZodCheckMultipleOfParams): this; + /** @deprecated In v4 and later, z.number() does not allow infinite values by default. This is a no-op. */ + finite(params?: unknown): this; + minValue: number | null; + maxValue: number | null; + /** @deprecated Check the `format` property instead. */ + isInt: boolean; + /** @deprecated Number schemas no longer accept infinite values, so this always returns `true`. */ + isFinite: boolean; + format: string | null; +} +export interface ZodNumber extends _ZodNumber> { +} +export declare const ZodNumber: core.$constructor; +export declare function number(params?: string | core.$ZodNumberParams): ZodNumber; +export interface ZodNumberFormat extends ZodNumber { + _zod: core.$ZodNumberFormatInternals; +} +export declare const ZodNumberFormat: core.$constructor; +export interface ZodInt extends ZodNumberFormat<"safeint"> { +} +export declare function int(params?: string | core.$ZodCheckNumberFormatParams): ZodInt; +export interface ZodFloat32 extends ZodNumberFormat<"float32"> { +} +export declare function float32(params?: string | core.$ZodCheckNumberFormatParams): ZodFloat32; +export interface ZodFloat64 extends ZodNumberFormat<"float64"> { +} +export declare function float64(params?: string | core.$ZodCheckNumberFormatParams): ZodFloat64; +export interface ZodInt32 extends ZodNumberFormat<"int32"> { +} +export declare function int32(params?: string | core.$ZodCheckNumberFormatParams): ZodInt32; +export interface ZodUInt32 extends ZodNumberFormat<"uint32"> { +} +export declare function uint32(params?: string | core.$ZodCheckNumberFormatParams): ZodUInt32; +export interface _ZodBoolean extends _ZodType { +} +export interface ZodBoolean extends _ZodBoolean> { +} +export declare const ZodBoolean: core.$constructor; +export declare function boolean(params?: string | core.$ZodBooleanParams): ZodBoolean; +export interface _ZodBigInt extends _ZodType { + gte(value: bigint, params?: string | core.$ZodCheckGreaterThanParams): this; + /** Alias of `.gte()` */ + min(value: bigint, params?: string | core.$ZodCheckGreaterThanParams): this; + gt(value: bigint, params?: string | core.$ZodCheckGreaterThanParams): this; + /** Alias of `.lte()` */ + lte(value: bigint, params?: string | core.$ZodCheckLessThanParams): this; + max(value: bigint, params?: string | core.$ZodCheckLessThanParams): this; + lt(value: bigint, params?: string | core.$ZodCheckLessThanParams): this; + positive(params?: string | core.$ZodCheckGreaterThanParams): this; + negative(params?: string | core.$ZodCheckLessThanParams): this; + nonpositive(params?: string | core.$ZodCheckLessThanParams): this; + nonnegative(params?: string | core.$ZodCheckGreaterThanParams): this; + multipleOf(value: bigint, params?: string | core.$ZodCheckMultipleOfParams): this; + minValue: bigint | null; + maxValue: bigint | null; + format: string | null; +} +export interface ZodBigInt extends _ZodBigInt> { +} +export declare const ZodBigInt: core.$constructor; +export declare function bigint(params?: string | core.$ZodBigIntParams): ZodBigInt; +export interface ZodBigIntFormat extends ZodBigInt { + _zod: core.$ZodBigIntFormatInternals; +} +export declare const ZodBigIntFormat: core.$constructor; +export interface ZodInt64 extends ZodBigIntFormat<"int64"> { +} +export declare function int64(params?: string | core.$ZodBigIntFormatParams): ZodInt64; +export interface ZodUInt64 extends ZodBigIntFormat<"uint64"> { +} +export declare function uint64(params?: string | core.$ZodBigIntFormatParams): ZodUInt64; +export interface ZodSymbol extends _ZodType { +} +export declare const ZodSymbol: core.$constructor; +export declare function symbol(params?: string | core.$ZodSymbolParams): ZodSymbol; +export interface ZodUndefined extends _ZodType { +} +export declare const ZodUndefined: core.$constructor; +declare function _undefined(params?: string | core.$ZodUndefinedParams): ZodUndefined; +export { _undefined as undefined }; +export interface ZodNull extends _ZodType { +} +export declare const ZodNull: core.$constructor; +declare function _null(params?: string | core.$ZodNullParams): ZodNull; +export { _null as null }; +export interface ZodAny extends _ZodType { +} +export declare const ZodAny: core.$constructor; +export declare function any(): ZodAny; +export interface ZodUnknown extends _ZodType { +} +export declare const ZodUnknown: core.$constructor; +export declare function unknown(): ZodUnknown; +export interface ZodNever extends _ZodType { +} +export declare const ZodNever: core.$constructor; +export declare function never(params?: string | core.$ZodNeverParams): ZodNever; +export interface ZodVoid extends _ZodType { +} +export declare const ZodVoid: core.$constructor; +declare function _void(params?: string | core.$ZodVoidParams): ZodVoid; +export { _void as void }; +export interface _ZodDate extends _ZodType { + min(value: number | Date, params?: string | core.$ZodCheckGreaterThanParams): this; + max(value: number | Date, params?: string | core.$ZodCheckLessThanParams): this; + /** @deprecated Not recommended. */ + minDate: Date | null; + /** @deprecated Not recommended. */ + maxDate: Date | null; +} +export interface ZodDate extends _ZodDate> { +} +export declare const ZodDate: core.$constructor; +export declare function date(params?: string | core.$ZodDateParams): ZodDate; +export interface ZodArray extends _ZodType>, core.$ZodArray { + element: T; + min(minLength: number, params?: string | core.$ZodCheckMinLengthParams): this; + nonempty(params?: string | core.$ZodCheckMinLengthParams): this; + max(maxLength: number, params?: string | core.$ZodCheckMaxLengthParams): this; + length(len: number, params?: string | core.$ZodCheckLengthEqualsParams): this; + unwrap(): T; + "~standard": ZodStandardSchemaWithJSON; +} +export declare const ZodArray: core.$constructor; +export declare function array(element: T, params?: string | core.$ZodArrayParams): ZodArray; +export declare function keyof(schema: T): ZodEnum>; +export type SafeExtendShape = { + [K in keyof Ext]: K extends keyof Base ? core.output extends core.output ? core.input extends core.input ? Ext[K] : never : never : Ext[K]; +}; +export interface ZodObject< +/** @ts-ignore Cast variance */ +out Shape extends core.$ZodShape = core.$ZodLooseShape, out Config extends core.$ZodObjectConfig = core.$strip> extends _ZodType>, core.$ZodObject { + "~standard": ZodStandardSchemaWithJSON; + shape: Shape; + keyof(): ZodEnum>; + /** Define a schema to validate all unrecognized keys. This overrides the existing strict/loose behavior. */ + catchall(schema: T): ZodObject>; + /** @deprecated Use `z.looseObject()` or `.loose()` instead. */ + passthrough(): ZodObject; + /** Consider `z.looseObject(A.shape)` instead */ + loose(): ZodObject; + /** Consider `z.strictObject(A.shape)` instead */ + strict(): ZodObject; + /** This is the default behavior. This method call is likely unnecessary. */ + strip(): ZodObject; + extend(shape: U): ZodObject>, Config>; + safeExtend(shape: SafeExtendShape & Partial>): ZodObject>, Config>; + /** + * @deprecated Use [`A.extend(B.shape)`](https://zod.dev/api?id=extend) instead. + */ + merge(other: U): ZodObject, U["_zod"]["config"]>; + pick>(mask: M & Record, never>): ZodObject>>, Config>; + omit>(mask: M & Record, never>): ZodObject>>, Config>; + partial(): ZodObject<{ + -readonly [k in keyof Shape]: ZodOptional; + }, Config>; + partial>(mask: M & Record, never>): ZodObject<{ + -readonly [k in keyof Shape]: k extends keyof M ? ZodOptional : Shape[k]; + }, Config>; + exactPartial(): ZodObject<{ + -readonly [k in keyof Shape]: ZodExactOptional; + }, Config>; + exactPartial>(mask: M & Record, never>): ZodObject<{ + -readonly [k in keyof Shape]: k extends keyof M ? ZodExactOptional : Shape[k]; + }, Config>; + required(): ZodObject<{ + -readonly [k in keyof Shape]: ZodNonOptional; + }, Config>; + required>(mask: M & Record, never>): ZodObject<{ + -readonly [k in keyof Shape]: k extends keyof M ? ZodNonOptional : Shape[k]; + }, Config>; +} +export declare const ZodObject: core.$constructor; +export declare function object>>(shape?: T, params?: string | core.$ZodObjectParams): ZodObject, core.$strip>; +export declare function strictObject(shape: T, params?: string | core.$ZodObjectParams): ZodObject, core.$strict>; +export declare function looseObject(shape: T, params?: string | core.$ZodObjectParams): ZodObject, core.$loose>; +export interface ZodUnion extends _ZodType>, core.$ZodUnion { + "~standard": ZodStandardSchemaWithJSON; + options: T; +} +export declare const ZodUnion: core.$constructor; +export declare function union(options: T, params?: string | core.$ZodUnionParams): ZodUnion; +export interface ZodXor extends _ZodType>, core.$ZodXor { + "~standard": ZodStandardSchemaWithJSON; + options: T; +} +export declare const ZodXor: core.$constructor; +/** Creates an exclusive union (XOR) where exactly one option must match. + * Unlike regular unions that succeed when any option matches, xor fails if + * zero or more than one option matches the input. */ +export declare function xor(options: T, params?: string | core.$ZodXorParams): ZodXor; +export interface ZodDiscriminatedUnion extends ZodUnion, core.$ZodDiscriminatedUnion { + "~standard": ZodStandardSchemaWithJSON; + _zod: core.$ZodDiscriminatedUnionInternals; + def: core.$ZodDiscriminatedUnionDef; +} +export declare const ZodDiscriminatedUnion: core.$constructor; +export declare function discriminatedUnion(discriminator: Disc, options: Types, params?: string | core.$ZodDiscriminatedUnionParams): ZodDiscriminatedUnion; +export interface ZodIntersection extends _ZodType>, core.$ZodIntersection { + "~standard": ZodStandardSchemaWithJSON; +} +export declare const ZodIntersection: core.$constructor; +export declare function intersection(left: T, right: U): ZodIntersection; +export interface ZodTuple extends _ZodType>, core.$ZodTuple { + "~standard": ZodStandardSchemaWithJSON; + rest(rest: Rest): ZodTuple; + partial(): ZodTuple<{ + -readonly [k in keyof T]: ZodOptional; + }, Rest>; +} +export declare const ZodTuple: core.$constructor; +export declare function tuple(items: T, params?: string | core.$ZodTupleParams): ZodTuple; +export declare function tuple(items: T, rest: Rest, params?: string | core.$ZodTupleParams): ZodTuple; +export declare function tuple(items: [], params?: string | core.$ZodTupleParams): ZodTuple<[], null>; +export interface ZodRecord extends _ZodType>, core.$ZodRecord { + "~standard": ZodStandardSchemaWithJSON; + keyType: Key; + valueType: Value; +} +export declare const ZodRecord: core.$constructor; +export declare function record(keyType: Key, valueType: Value, params?: string | core.$ZodRecordParams): ZodRecord; +export declare function partialRecord(keyType: Key, valueType: Value, params?: string | core.$ZodRecordParams): ZodRecord; +export declare function looseRecord(keyType: Key, valueType: Value, params?: string | core.$ZodRecordParams): ZodRecord; +export interface ZodMap extends _ZodType>, core.$ZodMap { + "~standard": ZodStandardSchemaWithJSON; + keyType: Key; + valueType: Value; + min(minSize: number, params?: string | core.$ZodCheckMinSizeParams): this; + nonempty(params?: string | core.$ZodCheckMinSizeParams): this; + max(maxSize: number, params?: string | core.$ZodCheckMaxSizeParams): this; + size(size: number, params?: string | core.$ZodCheckSizeEqualsParams): this; +} +export declare const ZodMap: core.$constructor; +export declare function map(keyType: Key, valueType: Value, params?: string | core.$ZodMapParams): ZodMap; +export interface ZodSet extends _ZodType>, core.$ZodSet { + "~standard": ZodStandardSchemaWithJSON; + min(minSize: number, params?: string | core.$ZodCheckMinSizeParams): this; + nonempty(params?: string | core.$ZodCheckMinSizeParams): this; + max(maxSize: number, params?: string | core.$ZodCheckMaxSizeParams): this; + size(size: number, params?: string | core.$ZodCheckSizeEqualsParams): this; +} +export declare const ZodSet: core.$constructor; +export declare function set(valueType: Value, params?: string | core.$ZodSetParams): ZodSet; +export interface ZodEnum< +/** @ts-ignore Cast variance */ +out T extends util.EnumLike = util.EnumLike> extends _ZodType>, core.$ZodEnum { + "~standard": ZodStandardSchemaWithJSON; + enum: T; + options: Array; + extract(values: U, params?: string | core.$ZodEnumParams): ZodEnum>>; + exclude(values: U, params?: string | core.$ZodEnumParams): ZodEnum>>; +} +export declare const ZodEnum: core.$constructor; +declare function _enum(values: T, params?: string | core.$ZodEnumParams): ZodEnum>; +declare function _enum(entries: T, params?: string | core.$ZodEnumParams): ZodEnum; +export { _enum as enum }; +/** @deprecated This API has been merged into `z.enum()`. Use `z.enum()` instead. + * + * ```ts + * enum Colors { red, green, blue } + * z.enum(Colors); + * ``` + */ +export declare function nativeEnum(entries: T, params?: string | core.$ZodEnumParams): ZodEnum; +export interface ZodLiteral extends _ZodType>, core.$ZodLiteral { + "~standard": ZodStandardSchemaWithJSON; + values: Set; + /** @legacy Use `.values` instead. Accessing this property will throw an error if the literal accepts multiple values. */ + value: T; +} +export declare const ZodLiteral: core.$constructor; +export declare function literal>(value: T, params?: string | core.$ZodLiteralParams): ZodLiteral; +export declare function literal(value: T, params?: string | core.$ZodLiteralParams): ZodLiteral; +export interface ZodFile extends _ZodType, core.$ZodFile { + "~standard": ZodStandardSchemaWithJSON; + min(size: number, params?: string | core.$ZodCheckMinSizeParams): this; + max(size: number, params?: string | core.$ZodCheckMaxSizeParams): this; + mime(types: util.MimeTypes | Array, params?: string | core.$ZodCheckMimeTypeParams): this; +} +export declare const ZodFile: core.$constructor; +export declare function file(params?: string | core.$ZodFileParams): ZodFile; +export interface ZodTransform extends _ZodType>, core.$ZodTransform { + "~standard": ZodStandardSchemaWithJSON; +} +export declare const ZodTransform: core.$constructor; +export declare function transform(fn: (input: I, ctx: core.$RefinementCtx) => O): ZodTransform, I>; +export interface ZodOptional extends _ZodType>, core.$ZodOptional { + "~standard": ZodStandardSchemaWithJSON; + unwrap(): T; +} +export declare const ZodOptional: core.$constructor; +export declare function optional(innerType: T): ZodOptional; +export interface ZodExactOptional extends _ZodType>, core.$ZodExactOptional { + "~standard": ZodStandardSchemaWithJSON; + unwrap(): T; +} +export declare const ZodExactOptional: core.$constructor; +export declare function exactOptional(innerType: T): ZodExactOptional; +export interface ZodNullable extends _ZodType>, core.$ZodNullable { + "~standard": ZodStandardSchemaWithJSON; + unwrap(): T; +} +export declare const ZodNullable: core.$constructor; +export declare function nullable(innerType: T): ZodNullable; +export declare function nullish(innerType: T): ZodOptional>; +export interface ZodDefault extends _ZodType>, core.$ZodDefault { + "~standard": ZodStandardSchemaWithJSON; + unwrap(): T; + /** @deprecated Use `.unwrap()` instead. */ + removeDefault(): T; +} +export declare const ZodDefault: core.$constructor; +export declare function _default(innerType: T, defaultValue: util.NoUndefined> | (() => util.NoUndefined>)): ZodDefault; +export interface ZodPrefault extends _ZodType>, core.$ZodPrefault { + "~standard": ZodStandardSchemaWithJSON; + unwrap(): T; +} +export declare const ZodPrefault: core.$constructor; +export declare function prefault(innerType: T, defaultValue: core.input | (() => core.input)): ZodPrefault; +export interface ZodNonOptional extends _ZodType>, core.$ZodNonOptional { + "~standard": ZodStandardSchemaWithJSON; + unwrap(): T; +} +export declare const ZodNonOptional: core.$constructor; +export declare function nonoptional(innerType: T, params?: string | core.$ZodNonOptionalParams): ZodNonOptional; +export interface ZodSuccess extends _ZodType>, core.$ZodSuccess { + "~standard": ZodStandardSchemaWithJSON; + unwrap(): T; +} +export declare const ZodSuccess: core.$constructor; +export declare function success(innerType: T): ZodSuccess; +export interface ZodCatch extends _ZodType>, core.$ZodCatch { + "~standard": ZodStandardSchemaWithJSON; + unwrap(): T; + /** @deprecated Use `.unwrap()` instead. */ + removeCatch(): T; +} +export declare const ZodCatch: core.$constructor; +declare function _catch(innerType: T, catchValue: core.output | ((ctx: core.$ZodCatchCtx) => core.output)): ZodCatch; +export { _catch as catch }; +export interface ZodNaN extends _ZodType, core.$ZodNaN { + "~standard": ZodStandardSchemaWithJSON; +} +export declare const ZodNaN: core.$constructor; +export declare function nan(params?: string | core.$ZodNaNParams): ZodNaN; +export interface ZodPipe extends _ZodType>, core.$ZodPipe { + "~standard": ZodStandardSchemaWithJSON; + in: A; + out: B; +} +export declare const ZodPipe: core.$constructor; +export declare function pipe> = core.$ZodType>>(in_: A, out: B | core.$ZodType>): ZodPipe; +export interface ZodCodec extends ZodPipe, core.$ZodCodec { + "~standard": ZodStandardSchemaWithJSON; + _zod: core.$ZodCodecInternals; + def: core.$ZodCodecDef; +} +export declare const ZodCodec: core.$constructor; +export declare function codec(in_: A, out: B, params: { + decode: (value: core.output, payload: core.ParsePayload>) => core.util.MaybeAsync>; + encode: (value: core.input, payload: core.ParsePayload>) => core.util.MaybeAsync>; +}): ZodCodec; +export declare function invertCodec(codec: ZodCodec): ZodCodec; +export interface ZodPreprocess extends ZodPipe, B>, core.$ZodPreprocess { + "~standard": ZodStandardSchemaWithJSON; + _zod: core.$ZodPreprocessInternals; + def: core.$ZodPreprocessDef; +} +export declare const ZodPreprocess: core.$constructor; +export interface ZodReadonly extends _ZodType>, core.$ZodReadonly { + "~standard": ZodStandardSchemaWithJSON; + unwrap(): T; +} +export declare const ZodReadonly: core.$constructor; +export declare function readonly(innerType: T): ZodReadonly; +export interface ZodTemplateLiteral