From c2ec0b989d65ebbf526870419902fc7f7ff83f9a Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 06:52:18 -0500 Subject: [PATCH 01/54] Capture stack traces with Error.captureStackTrace as Node does --- .../ambient/scriptc-node-fallback.d.ts | 1 + .../compiler/src/backend/llvm/lib-process.ts | 26 +++ .../compiler/src/backend/llvm/lib-shared.ts | 1 - .../src/frontend/lowering/builtins/errors.ts | 175 ++++++++++++------ .../src/frontend/lowering/lower-calls.ts | 8 + packages/compiler/src/ir/builtin-effects.ts | 4 +- .../compiler/src/ir/builtin-signatures.ts | 2 +- .../6873-error-capture-stack-trace.ts.json | 6 + ...error-capture-stack-trace-objects.cjs.json | 6 + .../error-capture-stack-trace-fences.ts.json | 6 + packages/runtime/src/scr_dyn.h | 2 +- packages/runtime/src/scr_exception.c | 24 ++- packages/runtime/src/scr_exception.h | 1 + packages/runtime/src/scr_json.c | 150 ++++++++++----- .../corpus/6873-error-capture-stack-trace.ts | 87 +++++++++ ...6874-error-capture-stack-trace-objects.cjs | 92 +++++++++ .../error-capture-stack-trace-fences.ts | 21 +++ .../error-capture-stack-trace-fences.ts.txt | 26 +++ .../6873-error-capture-stack-trace.ts.json | 1 + ...error-capture-stack-trace-objects.cjs.json | 1 + 20 files changed, 530 insertions(+), 110 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6873-error-capture-stack-trace.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6874-error-capture-stack-trace-objects.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/error-capture-stack-trace-fences.ts.json create mode 100644 tests/corpus/6873-error-capture-stack-trace.ts create mode 100644 tests/corpus/6874-error-capture-stack-trace-objects.cjs create mode 100644 tests/diagnostics/error-capture-stack-trace-fences.ts create mode 100644 tests/harness/__snapshots__/error-capture-stack-trace-fences.ts.txt create mode 100644 tests/harness/corpus-costs/6873-error-capture-stack-trace.ts.json create mode 100644 tests/harness/corpus-costs/6874-error-capture-stack-trace-objects.cjs.json diff --git a/packages/compiler/ambient/scriptc-node-fallback.d.ts b/packages/compiler/ambient/scriptc-node-fallback.d.ts index b3d2954a8..c7dd06541 100644 --- a/packages/compiler/ambient/scriptc-node-fallback.d.ts +++ b/packages/compiler/ambient/scriptc-node-fallback.d.ts @@ -7586,4 +7586,5 @@ declare module "node:stream/web" { } interface ErrorConstructor { stackTraceLimit: number; + captureStackTrace(targetObject: object, constructorOpt?: Function): void; } diff --git a/packages/compiler/src/backend/llvm/lib-process.ts b/packages/compiler/src/backend/llvm/lib-process.ts index 3e8a9bff5..e60307fa0 100644 --- a/packages/compiler/src/backend/llvm/lib-process.ts +++ b/packages/compiler/src/backend/llvm/lib-process.ts @@ -677,6 +677,32 @@ export function emitProcessLibCall(host: LlvmEmitterContext, e: LibCallExpr): Ll export function emitErrorsEventsLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue { const B = host.B; + if (e.fn === "error.captureStackTrace") { + // The second argument names the function constructorOpt skips through; + // its frame text is the constant its own frame pushes (emit-function.ts), + // or null when it has no frame, which leaves no frames. + const target = host.emitExpr(e.args[0]!); + const named = e.args[1]!; + const skip = e.args[2]!.kind === "boolLit" && e.args[2]!.value; + let through = "null"; + if (named.kind === "strLit" && named.value !== "") { + const fn = host.mod.functions.find((f) => f.name === named.value); + if ( + fn !== undefined && + host.stackTraces && + !host.wasi && + fn.sourceName !== undefined && + !host.unframedFunctions.has(fn.name) + ) + through = host.cstr(` at ${fn.sourceName} (${fn.loc.file})`); + } + host.declare(`declare void @scr_error_capture_stack_trace(ptr, ptr, i1)`); + B.line( + `call void @scr_error_capture_stack_trace(ptr ${target.name}, ptr ${through}, i1 ${skip ? "true" : "false"})`, + ); + host.emitPendingCheck(); + return { name: "", type: e.type }; + } if (e.fn === "error.argTypeThrow") { // Always throws with the runtime-rendered Received tail (the // error.nodeThrow dummy pattern). Borrows all three. diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 53c715108..bfe6a4ed6 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1506,7 +1506,6 @@ export const LIB_FN_SYMS: Record = { "dyn.objectPrototype": "scr_dyn_object_prototype", "dyn.errorPrototype": "scr_dyn_error_prototype", "dyn.inheritsErrorPrototype": "scr_dyn_inherits_error_prototype", - "dyn.errorCaptureStackTrace": "scr_dyn_error_capture_stack_trace", "dyn.errorCaptureStackTraceFunction": "scr_dyn_error_capture_stack_trace_function", "dyn.functionApply": "scr_dyn_function_apply", "dyn.builtinMethod": "scr_dyn_builtin_method", diff --git a/packages/compiler/src/frontend/lowering/builtins/errors.ts b/packages/compiler/src/frontend/lowering/builtins/errors.ts index 1371769d6..ce95d4a46 100644 --- a/packages/compiler/src/frontend/lowering/builtins/errors.ts +++ b/packages/compiler/src/frontend/lowering/builtins/errors.ts @@ -1,7 +1,6 @@ import * as ts from "../../ts7/adapter.js"; import { type Lowerer } from "../lowerer.js"; -import { locOf } from "../../program.js"; -import { tryLowerExpression } from "../expressions/try-lower-expression.js"; +import { isJsSourceFile, locOf } from "../../program.js"; import { dynUndefinedExpr } from "../../../ir/build.js"; import { BOOL, @@ -13,6 +12,7 @@ import { type IrStmt, type IrType, STRING, + VOID, } from "../../../ir/ir.js"; const ASSERTION_ERROR_PROPERTIES: ReadonlySet = new Set([ @@ -324,60 +324,6 @@ export function lowerErrorFunctionCall(lowerer: Lowerer, call: ts.CallExpression : error; } -/** `Error.captureStackTrace(target[, constructorOpt])` (also called as - * `Error.captureStackTrace?.(...)`, which always calls it). On a - * checked-dynamic target (`this` in a JavaScript constructor function) the - * runtime defines target's `stack`. An instance of a class that extends - * Error derives its stack from its name and message when the stack is - * read, so the call only evaluates its arguments. Both arguments are - * evaluated; the second only selects frames in Node. Null for other - * targets and callees. */ -export function lowerErrorCaptureStackTraceCall( - lowerer: Lowerer, - call: ts.CallExpression, - access: ts.PropertyAccessExpression, -): IrExpr | null { - if (access.questionDotToken) return null; - if (lowerer.stdlibGlobalMember(access, "Error") !== "captureStackTrace") return null; - const args = call.arguments.nodes; - if (args.length < 1 || args.length > 2 || args.some(ts.isSpreadElement)) return null; - const probed = tryLowerExpression(lowerer, args[0]!); - const loc = locOf(call); - if ( - probed?.type.kind === "object" && - (probed.type.className === "%Error" || lowerer.isSubclassOf(probed.type.className, "%Error")) - ) { - const stmts: IrStmt[] = []; - for (const arg of args) { - const value = lowerer.lowerExpr(arg); - if (value.kind !== "varRef") stmts.push({ kind: "exprStmt", expr: value, loc: value.loc }); - } - const result = dynUndefinedExpr(loc); - return stmts.length > 0 ? { kind: "seqExpr", stmts, result, type: DYN, loc } : result; - } - if (probed?.type.kind !== "dyn") return null; - const target = lowerer.lowerExpr(args[0]!); - const result: IrExpr = { - kind: "libCall", - fn: "dyn.errorCaptureStackTrace", - args: [target], - type: DYN, - loc, - }; - if (args.length === 1) return result; - const local = lowerer.declareHiddenLocal("%captureTarget", DYN); - return { - kind: "seqExpr", - stmts: [ - { kind: "varDecl", localId: local.id, init: target, loc }, - { kind: "exprStmt", expr: lowerer.lowerExprExpecting(args[1]!, DYN), loc }, - ], - result: { ...result, args: [{ kind: "varRef", localId: local.id, type: DYN, loc }] }, - type: DYN, - loc, - }; -} - /** `Error.isError(value)`. A catch binding tests its payload against the * Error class, an `unknown` value tests the checked-dynamic tree's error * encoding, and a union tests its runtime tag against the arms whose class @@ -470,3 +416,120 @@ export function lowerErrorIsErrorCall( loc, }; } + +/** The class whose constructor body `node` is directly in (not inside a + * nested function): while it runs, `new` of that class is on the stack. */ +function enclosingConstructorClass(node: ts.Node): ts.ClassLikeDeclaration | undefined { + for (let n = node.parent; n !== undefined; n = n.parent) { + if (ts.isConstructorDeclaration(n)) + return ts.isClassDeclaration(n.parent) || ts.isClassExpression(n.parent) + ? n.parent + : undefined; + if (ts.isFunctionLike(n) && !ts.isArrowFunction(n)) return undefined; + } + return undefined; +} + +/** `Error.captureStackTrace(target, constructorOpt?)`, also through `?.`: + * target.stack becomes the target's heading and the current stack frames, + * as V8 does it (scr_error_capture_stack_trace). The target is an error, a + * checked-dynamic object, or a primitive (a TypeError at run time). + * constructorOpt leaves out the frames up to and including its topmost call: + * a function declaration skips through its own frame; `this.constructor`, + * `new.target` or the class's own name inside a class constructor leave the + * frames from the code that ran `new`, which are the current frames + * (constructors have no frames of their own). Other targets are refused, + * and so are other constructorOpt values in TypeScript; JavaScript keeps all + * frames for them. Null when the callee is not the + * standard library's Error.captureStackTrace. */ +export function lowerErrorCaptureStackTraceCall( + lowerer: Lowerer, + call: ts.CallExpression, + access: ts.PropertyAccessExpression, +): IrExpr | null { + if (lowerer.stdlibGlobalMember(access, "Error") !== "captureStackTrace") return null; + const args = call.arguments.nodes; + const loc = locOf(call); + if (args.length < 1 || args.length > 2 || args.some(ts.isSpreadElement)) { + lowerer.noLowering( + "Error.captureStackTrace arguments", + call, + "pass a target and an optional constructor", + ); + } + const target = lowerer.lowerExpr(args[0]!); + // Primitives (and unions of them) reach the runtime, which throws V8's + // TypeError for them; objects without a checked-dynamic view are refused, + // since a boxed copy would not show the stack. + const primitive = + ["f64", "string", "bool", "nullT", "undefinedT", "bigint"].includes(target.type.kind) || + (target.type.kind === "union" && + lowerer.dynConvertible(target.type) && + (lowerer.unions.get(target.type.unionId)?.arms ?? []).every((arm) => + ["f64", "string", "bool", "nullT", "undefinedT", "bigint"].includes(arm.kind), + )); + if (target.type.kind !== "dyn" && !primitive && !isErrorClassType(lowerer, target.type)) { + lowerer.noLowering( + `Error.captureStackTrace on '${lowerer.fmt(target.type)}' values`, + args[0]!, + "pass an Error instance, or an untyped object in JavaScript", + ); + } + let through = ""; + let skip = false; + const ctor = args[1]; + if (ctor !== undefined) { + let node: ts.Expression = ctor; + while (ts.isParenthesizedExpression(node)) node = node.expression; + const symbol = ts.isIdentifier(node) ? lowerer.checker.getSymbolAtLocation(node) : undefined; + const resolved = + symbol !== undefined && symbol.flags & ts.SymbolFlags.Alias + ? lowerer.checker.getAliasedSymbol(symbol) + : symbol; + const cls = enclosingConstructorClass(call); + const constructing = + (ts.isPropertyAccessExpression(node) && + node.expression.kind === ts.SyntaxKind.ThisKeyword && + node.name.text === "constructor") || + (ts.isMetaProperty(node) && node.name.text === "target") || + (resolved !== undefined && + cls?.name !== undefined && + lowerer.checker.getSymbolAtLocation(cls.name) === resolved); + const sig = resolved !== undefined ? lowerer.fnSigsBySymbol.get(resolved) : undefined; + if (constructing && cls !== undefined) { + skip = false; + } else if (sig !== undefined) { + lowerer.noteEdge(sig.name); + through = sig.name; + skip = true; + } else if (isJsSourceFile(ts.getSourceFileOfNode(call))) { + // JavaScript (undici's `captureStackTrace(err, module.exports.fetch)`) + // keeps every frame: the same frames V8 keeps, and the ones above them. + skip = false; + } else { + lowerer.noLowering( + "Error.captureStackTrace with this constructor argument", + ctor, + "pass a function declaration, or this.constructor or the class inside its constructor", + ); + } + } + const capture: IrExpr = { + kind: "libCall", + fn: "error.captureStackTrace", + args: [ + lowerer.coerceToExpected(target, DYN), + { kind: "strLit", value: through, type: STRING, loc }, + { kind: "boolLit", value: skip, type: BOOL, loc }, + ], + type: VOID, + loc, + }; + return { + kind: "seqExpr", + stmts: [{ kind: "exprStmt", expr: capture, loc }], + result: dynUndefinedExpr(loc), + type: DYN, + loc, + }; +} diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 39b7207c0..b9ec9418e 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -418,6 +418,14 @@ export function lowerCall(lowerer: Lowerer, expr: ts.CallExpression): IrExpr { const timeoutOptional = lowerer.lowerTimeoutMethodCall(expr, expr.expression); if (timeoutOptional) return timeoutOptional; } + // `Error.captureStackTrace?.(this, this.constructor)`: Node always has it. + if ( + ts.isPropertyAccessExpression(expr.expression) && + (expr.questionDotToken || expr.expression.questionDotToken) + ) { + const captured = lowerErrorCaptureStackTraceCall(lowerer, expr, expr.expression); + if (captured) return captured; + } // `req.session?.m(...)` remains absent on compatibility requests. Live // `req.stream` calls now pass through the normal optional-chain and h2 // stream lowering using lowerServerProperty's checked getter. diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index c202b8538..40b9ff1ab 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -443,6 +443,7 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "js.toNumberCoerce", // Error messages use the same coercion protocol before installing cause. "error.newOptions", + "error.captureStackTrace", "error.ctorOptions", "error.newAggregate", "error.newSuppressed", @@ -610,9 +611,6 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "dyn.objCreateWithProperties", "dyn.getPrototype", "dyn.setPrototype", - // Error.captureStackTrace: Node's TypeError for a primitive target, and - // the error string conversion of name and message - "dyn.errorCaptureStackTrace", // a proxy in the prototype chain can throw "dyn.builtinInstanceOf", "dyn.functionInstanceOf", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index f37e8809f..a3bb620f1 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -3369,6 +3369,7 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ * Options retain cause presence; constructor calls borrow their args. * Message coercion may throw before the cause is installed. */ "error.newOptions": { argTypes: [DYN, DYN], result: VOID }, + "error.captureStackTrace": { argTypes: [DYN, STRING, BOOL], result: VOID }, "error.ctorOptions": { argTypes: [null, DYN, DYN], result: VOID }, /** Borrowed Error receiver. cause returns an owned dyn value (undefined * when absent); hasCause distinguishes absence from present undefined. */ @@ -3682,7 +3683,6 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "dyn.objectPrototype": { argTypes: [], result: DYN }, "dyn.errorPrototype": { argTypes: [], result: DYN }, "dyn.inheritsErrorPrototype": { argTypes: [DYN], result: BOOL }, - "dyn.errorCaptureStackTrace": { argTypes: [DYN], result: DYN }, "dyn.errorCaptureStackTraceFunction": { argTypes: [], result: DYN }, "dyn.functionApply": { argTypes: [], result: DYN }, "dyn.builtinMethod": { argTypes: [STRING, STRING], result: DYN }, diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6873-error-capture-stack-trace.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6873-error-capture-stack-trace.ts.json new file mode 100644 index 000000000..8544af112 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6873-error-capture-stack-trace.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6873-error-capture-stack-trace.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6874-error-capture-stack-trace-objects.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6874-error-capture-stack-trace-objects.cjs.json new file mode 100644 index 000000000..3f25ca0aa --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6874-error-capture-stack-trace-objects.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6874-error-capture-stack-trace-objects.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/error-capture-stack-trace-fences.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/error-capture-stack-trace-fences.ts.json new file mode 100644 index 000000000..9159ca9cd --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/error-capture-stack-trace-fences.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/diagnostics/error-capture-stack-trace-fences.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index facf9f2ff..ef6f57926 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -1092,7 +1092,6 @@ ScrStr *scr_dyn_to_string_method(const ScrDyn *d, const ScrStr *enc, const ScrSt ScrDyn *scr_dyn_object_prototype(void); ScrDyn *scr_dyn_error_prototype(void); bool scr_dyn_inherits_error_prototype(const ScrDyn *value); -ScrDyn *scr_dyn_error_capture_stack_trace(ScrDyn *target); ScrDyn *scr_dyn_error_capture_stack_trace_function(void); ScrDyn *scr_dyn_function_apply(void); ScrDyn *scr_dyn_reflect_apply(ScrDyn *target, ScrDyn *receiver, ScrDyn *arguments_list); @@ -1195,6 +1194,7 @@ ScrError *scr_error_from_dyn(const ScrDyn *d); /* scr_async_dyn.c (gated) */ /* The cache's runtime-internal access pair (scr_json.c owns the storage; * the gated extraction reads/writes through these). */ ScrError *scr_errdyn_err_of(const ScrDyn *d); /* +1 or NULL */ +void scr_error_capture_stack_trace(ScrDyn *target, const char *through, bool skip); void scr_errdyn_put(ScrError *e, ScrDyn *d); /* retains both sides */ /* Sets an own property on an error: its checked-dynamic node, which casts * and util.inspect read. Moves value. */ diff --git a/packages/runtime/src/scr_exception.c b/packages/runtime/src/scr_exception.c index c7ff71349..9dcad5b67 100644 --- a/packages/runtime/src/scr_exception.c +++ b/packages/runtime/src/scr_exception.c @@ -196,19 +196,21 @@ void scr_stack_limit_set(double limit) { scr_stack_limit = limit; } -ScrStr *scr_stack_capture(void) { - if (!SCR_EXC_CUR()->stack) return NULL; +/* The frames from `first` down, at most Error.stackTraceLimit of them, one + * "\n at ..." line each; NULL when there are none. */ +static ScrStr *scr_stack_capture_from(ScrStackFrame *first) { + if (!first) return NULL; const double limit = floor(scr_stack_limit); if (!(limit > 0)) return NULL; size_t length = 0; size_t count = 0; - for (ScrStackFrame *frame = SCR_EXC_CUR()->stack; frame && (double)count++ < limit; frame = frame->previous) + for (ScrStackFrame *frame = first; frame && (double)count++ < limit; frame = frame->previous) length += 1 + strlen(frame->text); char *text = malloc(length); if (!text) scr_trap("scriptc: out of memory\n"); size_t offset = 0; count = 0; - for (ScrStackFrame *frame = SCR_EXC_CUR()->stack; frame && (double)count++ < limit; frame = frame->previous) { + for (ScrStackFrame *frame = first; frame && (double)count++ < limit; frame = frame->previous) { text[offset++] = '\n'; size_t size = strlen(frame->text); memcpy(text + offset, frame->text, size); @@ -219,6 +221,20 @@ ScrStr *scr_stack_capture(void) { return out; } +ScrStr *scr_stack_capture(void) { + return scr_stack_capture_from(SCR_EXC_CUR()->stack); +} + +/* The frames below the topmost frame whose text is `through` (a function's + * " at name (file)" line), as Error.captureStackTrace's constructorOpt + * leaves them; NULL when no such frame is on the stack. */ +ScrStr *scr_stack_capture_after(const char *through) { + for (ScrStackFrame *frame = SCR_EXC_CUR()->stack; frame; frame = frame->previous) + if (frame->text == through || strcmp(frame->text, through) == 0) + return scr_stack_capture_from(frame->previous); + return NULL; +} + #define scr_exc_kind (SCR_EXC_CUR()->kind) #define scr_exc_f64 (SCR_EXC_CUR()->f64) #define scr_exc_bool (SCR_EXC_CUR()->b) diff --git a/packages/runtime/src/scr_exception.h b/packages/runtime/src/scr_exception.h index e99f84243..95153a4a8 100644 --- a/packages/runtime/src/scr_exception.h +++ b/packages/runtime/src/scr_exception.h @@ -68,6 +68,7 @@ typedef struct ScrStackFrame { void scr_stack_enter(ScrStackFrame *frame, const char *text); void scr_stack_leave(ScrStackFrame *frame); ScrStr *scr_stack_capture(void); /* +1, NULL when no source frames exist */ +ScrStr *scr_stack_capture_after(const char *through); /* +1, NULL when none remain */ double scr_stack_limit_get(void); void scr_stack_limit_set(double limit); diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index f610ffdbb..d7c084ae6 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -1950,52 +1950,13 @@ ScrDyn *scr_dyn_error_prototype(void) { return scr_dyn_retain(scr_builtin_error_prototype); } -/* Error.captureStackTrace(target): defines a non-enumerable own `stack` - * on target. Node's value is the Error.prototype.toString text of target - * (read without calling an overriding toString) followed by stack frames; - * here, as for scriptc's own Error instances, it is the text alone. - * Answers undefined, or NULL with the exception pending. */ -ScrDyn *scr_dyn_error_capture_stack_trace(ScrDyn *target) { - /* An instance of a class that extends Error derives its stack from its - * name and message when the stack is read. */ - if (target->kind == SCR_DYN_TYPED_REF) { - ScrDyn *view = scr_dyn_typed_ref_materialize(target); - bool error = view && view->kind == SCR_DYN_OBJ && scr_dyn_obj_get(view, "%error", 6) != NULL; - scr_dyn_release(view); - if (scr_exc_pending()) return NULL; - if (error) return scr_dyn_retain(scr_dyn_undefined()); - } - if (target->kind != SCR_DYN_OBJ) { - if (target->kind == SCR_DYN_ARR || target->kind == SCR_DYN_FUNC || target->kind == SCR_DYN_TYPED_REF || - target->kind == SCR_DYN_HANDLE || target->kind == SCR_DYN_PROXY) { - static const char message[] = "Error.captureStackTrace on this object is not supported yet"; - scr_throw_error_msg_code(SCR_ERR_ERROR, message, sizeof message - 1, "SC2020"); - } else { - static const char message[] = "invalid_argument"; - scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); - } - return NULL; - } - ScrDynEntry *existing = scr_dyn_find_entry(target, "stack", 5); - if (existing ? existing->accessor || !existing->configurable : target->non_extensible) { - static const char message[] = "Cannot redefine property: stack"; - static const char extensible[] = "Cannot define property stack, object is not extensible"; - if (existing) scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); - else scr_throw_error_msg(SCR_ERR_TYPE, extensible, sizeof extensible - 1); - return NULL; - } - ScrStr *text = scr_dyn_error_string(target); - if (!text) return NULL; - scr_dyn_obj_set_field(target, "stack", 5, scr_dyn_new_str(text)); - scr_str_release(text); - ScrDynEntry *entry = scr_dyn_find_entry(target, "stack", 5); - entry->enumerable = false; - entry->writable = entry->configurable = true; - return scr_dyn_retain(scr_dyn_undefined()); -} +/* Error.captureStackTrace called through its function value. The + * constructor argument's frame cannot be found from a function value, so + * every frame is kept: the frames V8 keeps and the ones above them. */ static ScrDyn *scr_error_capture_stack_trace_call(ScrClosure *closure, ScrDyn *const *args, size_t argc) { (void)closure; - return scr_dyn_error_capture_stack_trace(argc > 0 ? args[0] : scr_dyn_undefined()); + scr_error_capture_stack_trace(argc > 0 ? args[0] : scr_dyn_undefined(), NULL, false); + return scr_exc_pending() ? NULL : scr_dyn_retain(scr_dyn_undefined()); } /* Error.captureStackTrace read as a value: one function per thread. */ static SCR_TL ScrDyn *scr_error_capture_stack_trace_fn; @@ -15430,3 +15391,104 @@ bool scr_error_has_key(ScrError *e, const ScrStr *key) { return true; return scr_dyn_object_proto_name(key); } + +/* The `stack` accessor Error.captureStackTrace gives an object that is not + * an error. caps[0] holds the captured frames until the first read, then the + * stack; caps[1] is nonzero once caps[0] holds the stack. The first read + * formats it with the receiver's name and message then, as V8 does. */ +static ScrDyn *scr_captured_stack_get_call(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + (void)args; (void)argc; + ScrDyn *value = scr_box_get_ref(closure->caps[0]); + if (scr_box_get_f64(closure->caps[1]) != 0) return value; + ScrDyn *self = scr_dyn_this_get(); + ScrStr *heading = scr_dyn_error_string(self); + scr_dyn_release(self); + if (!heading) { + scr_dyn_release(value); + return NULL; + } + ScrStr *stack = scr_str_concat(heading, value->v.str); + scr_str_release(heading); + scr_dyn_release(value); + ScrDyn *out = scr_dyn_new_str(stack); + scr_str_release(stack); + scr_box_set_ref(closure->caps[0], scr_dyn_retain(out)); + scr_box_set_f64(closure->caps[1], 1); + return out; +} + +static ScrDyn *scr_captured_stack_set_call(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + scr_box_set_ref(closure->caps[0], scr_dyn_retain(argc > 0 ? args[0] : scr_dyn_undefined())); + scr_box_set_f64(closure->caps[1], 1); + return scr_dyn_retain(scr_dyn_undefined()); +} + +/* Error.captureStackTrace(target, constructorOpt), as V8 does it: target.stack + * becomes the target's heading and the current stack frames, formatted when + * first read. With `skip`, the frames up to and including the topmost one + * whose text is `through` are left out (none remain when `through` is NULL or + * not on the stack). An error keeps the frames as its own; another object + * gets an own non-enumerable, configurable `stack` accessor. A primitive + * target throws `TypeError: invalid_argument` and a non-extensible object + * V8's define-property TypeError. */ +void scr_error_capture_stack_trace(ScrDyn *target, const char *through, bool skip) { + switch (target->kind) { + case SCR_DYN_UNDEF: + case SCR_DYN_NULL: + case SCR_DYN_BOOL: + case SCR_DYN_NUM: + case SCR_DYN_STR: + case SCR_DYN_BIGINT: + case SCR_DYN_SYMBOL: { + static const char msg[] = "invalid_argument"; + scr_throw_error_msg(SCR_ERR_TYPE, msg, sizeof msg - 1); + return; + } + default: + break; + } + ScrStr *frames = !skip ? scr_stack_capture() : through ? scr_stack_capture_after(through) : NULL; + ScrError *error = scr_errdyn_err_of(target); + if (error) { + scr_str_release(error->stack_frames); + error->stack_frames = frames; + scr_str_release(error->stack); + error->stack = NULL; + scr_error_release(error); + return; + } + if (target->kind != SCR_DYN_OBJ && target->kind != SCR_DYN_ARR) { + static const char msg[] = "Error.captureStackTrace on this object is not supported yet"; + scr_throw_error_msg_code(SCR_ERR_ERROR, msg, sizeof msg - 1, "SC2020"); + scr_str_release(frames); + return; + } + if (!scr_dyn_is_extensible(target)) { + static const char msg[] = "Cannot define property stack, object is not extensible"; + scr_throw_error_msg(SCR_ERR_TYPE, msg, sizeof msg - 1); + scr_str_release(frames); + return; + } + if (!frames) frames = scr_str_new("", 0); + /* The getter and setter share their boxes. */ + ScrClosure *state[2] = {scr_closure_new(NULL, 2), scr_closure_new(NULL, 2)}; + state[0]->caps[0] = scr_box_new_obj(scr_dyn_retain_v, scr_dyn_release_v, scr_dyn_trace_v); + state[0]->caps[1] = scr_box_new(SCR_BOX_F64); + scr_box_set_ref(state[0]->caps[0], scr_dyn_new_str(frames)); + scr_str_release(frames); + for (int i = 0; i < 2; i++) state[1]->caps[i] = scr_box_retain(state[0]->caps[i]); + ScrDyn *descriptor = scr_dyn_new_obj(); + scr_dyn_obj_set(descriptor, "get", 3, + scr_dyn_new_func(state[0], scr_captured_stack_get_call, 0, "native:error.stack", NULL)); + scr_dyn_obj_set(descriptor, "set", 3, + scr_dyn_new_func(state[1], scr_captured_stack_set_call, 1, "native:error.stack", NULL)); + scr_dyn_obj_set(descriptor, "enumerable", 10, scr_dyn_new_bool(false)); + scr_dyn_obj_set(descriptor, "configurable", 12, scr_dyn_new_bool(true)); + ScrStr *key_text = scr_str_new("stack", 5); + ScrDyn *key = scr_dyn_new_str(key_text); + scr_str_release(key_text); + ScrDyn *result = scr_dyn_define_property(target, key, descriptor); + scr_dyn_release(result); + scr_dyn_release(key); + scr_dyn_release(descriptor); +} diff --git a/tests/corpus/6873-error-capture-stack-trace.ts b/tests/corpus/6873-error-capture-stack-trace.ts new file mode 100644 index 000000000..62db5572d --- /dev/null +++ b/tests/corpus/6873-error-capture-stack-trace.ts @@ -0,0 +1,87 @@ +// Error.captureStackTrace on errors, as Node does it: the error's stack +// becomes its heading and the current frames, starting at the function that +// called it; a constructor argument leaves out the frames up to and +// including that function's topmost call, and every frame when it is not on +// the stack. Error subclasses that call it with this.constructor (also +// through ?.) keep the frames from the code that ran `new`. +function frames(stack: string | undefined): string { + const lines = (stack ?? "").split("\n"); + const wanted = ["inner", "middle", "outer", "makeExpired", "skipped", "caller"]; + const names = lines.slice(1).map((line) => line.trim().split(" ")[1] ?? ""); + return `${lines[0]} | ${names.filter((name) => wanted.includes(name)).join(",")}`; +} + +function inner(target: Error): Error { + Error.captureStackTrace(target); + return target; +} +function middle(target: Error): Error { + return inner(target); +} +function outer(target: Error): Error { + return middle(target); +} +console.log("error", frames(outer(new Error("plain")).stack)); +console.log("type error", frames(outer(new TypeError("typed")).stack)); + +function skipped(target: Error): Error { + Error.captureStackTrace(target, skipped); + return target; +} +function caller(target: Error): Error { + return skipped(target); +} +console.log("constructorOpt", frames(caller(new Error("skip")).stack)); +function unrelated(): void {} +const missing = new Error("missing"); +Error.captureStackTrace(missing, unrelated); +console.log("not on the stack", JSON.stringify(missing.stack)); + +class AppError extends Error { + constructor(message: string) { + super(message); + this.name = "AppError"; + Error.captureStackTrace(this, this.constructor); + } +} +class Expired extends AppError { + constructor(message: string) { + super(message); + this.name = "Expired"; + } +} +class Optional extends Error { + constructor(message: string) { + super(message); + this.name = "Optional"; + Error.captureStackTrace?.(this, this.constructor); + } +} +class Named extends Error { + constructor(message: string) { + super(message); + Error.captureStackTrace(this, Named); + } +} +function makeExpired(): Error { + return new Expired("token expired"); +} +const expired = makeExpired(); +console.log("subclass", frames(expired.stack), expired instanceof AppError, expired.message); +console.log("optional call", frames(new Optional("maybe").stack)); +console.log("class name", frames(new Named("named").stack)); + +const renamed = new RangeError("range"); +renamed.name = "Custom"; +Error.captureStackTrace(renamed); +console.log("renamed", frames(renamed.stack)); +const late = new Error("late"); +Error.captureStackTrace(late); +late.name = "Late"; +console.log("renamed after", frames(late.stack)); + +Error.stackTraceLimit = 1; +console.log("limit", frames(outer(new Error("limited")).stack)); +Error.stackTraceLimit = 0; +console.log("limit 0", JSON.stringify(outer(new Error("none")).stack)); +Error.stackTraceLimit = 10; diff --git a/tests/corpus/6874-error-capture-stack-trace-objects.cjs b/tests/corpus/6874-error-capture-stack-trace-objects.cjs new file mode 100644 index 000000000..97d322659 --- /dev/null +++ b/tests/corpus/6874-error-capture-stack-trace-objects.cjs @@ -0,0 +1,92 @@ +// Error.captureStackTrace on objects in JavaScript, as Node does it: an own, +// non-enumerable, configurable `stack` accessor giving the object's heading +// (its name and message when first read, through Error.prototype.toString's +// rules, "Error" by default) and the frames from the caller; TypeErrors for a +// primitive and a non-extensible target. +'use strict'; +function frames(stack) { + const lines = String(stack).split('\n'); + const names = lines.slice(1).map((line) => line.trim().split(' ')[1]); + return `${lines[0]} | ${names.filter((name) => ['inner', 'middle', 'outer', 'skipped', 'caller'].includes(name)).join(',')}`; +} +function inner(target) { + Error.captureStackTrace(target); + return target; +} +function middle(target) { + return inner(target); +} +function outer(target) { + return middle(target); +} +console.log('plain', frames(outer({}).stack)); +console.log('named', frames(outer({ name: 'Parse', message: 'bad token' }).stack)); +console.log('message only', frames(outer({ message: 'just this' }).stack)); +console.log('empty name', frames(outer({ name: '', message: 'no name' }).stack)); +console.log('number fields', frames(outer({ name: 7, message: 8 }).stack)); +const own = inner({ kept: 1 }); +console.log('keys', JSON.stringify(Object.keys(own)), typeof own.stack); +const descriptor = Object.getOwnPropertyDescriptor(own, 'stack'); +console.log('descriptor', typeof descriptor.get, typeof descriptor.set, descriptor.enumerable, descriptor.configurable); +own.stack = 'replaced'; +console.log('writable', own.stack); +const later = inner({ message: 'first' }); +later.name = 'Renamed'; +later.message = 'second'; +console.log('formatted when read', frames(later.stack)); +later.name = 'Again'; +console.log('formatted once', frames(later.stack)); +const counted = { message: 'm' }; +let reads = 0; +Object.defineProperty(counted, 'name', { get() { reads++; return 'Counted'; } }); +inner(counted); +console.log('reads before stack', reads); +console.log('counted', frames(counted.stack), reads); +function skipped(target) { + Error.captureStackTrace(target, skipped); + return target; +} +function caller(target) { + return skipped(target); +} +console.log('constructorOpt', frames(caller({}).stack)); +const arr = []; +Error.captureStackTrace(arr); +console.log('array', frames(arr.stack), arr.length); +for (const value of [5, 'text', undefined, null, true]) { + try { + Error.captureStackTrace(value); + console.log('primitive', String(value), 'no error'); + } catch (error) { + console.log('primitive', String(value), error.name, error.message); + } +} +try { + Error.captureStackTrace(Object.freeze({})); + console.log('frozen', 'no error'); +} catch (error) { + console.log('frozen', error.name, error.message); +} +const api = { + run(target) { + Error.captureStackTrace(target, api.run); + return target; + }, +}; +function viaApi(target) { + return api.run(target); +} +console.log('member constructorOpt', frames(viaApi({ message: 'member' }).stack).split(' | ')[0]); +const captureFn = Error.captureStackTrace; +function viaValue(target) { + captureFn(target); + return target; +} +const fromValue = viaValue({ message: 'value' }); +fromValue.message = 'read later'; +console.log('function value', frames(fromValue.stack), captureFn.name, captureFn.length); +try { + captureFn(7); +} catch (error) { + console.log('function value primitive', error.name, error.message); +} diff --git a/tests/diagnostics/error-capture-stack-trace-fences.ts b/tests/diagnostics/error-capture-stack-trace-fences.ts new file mode 100644 index 000000000..b30afb7b7 --- /dev/null +++ b/tests/diagnostics/error-capture-stack-trace-fences.ts @@ -0,0 +1,21 @@ +// Error.captureStackTrace gives a stack to errors and untyped objects; a +// typed object has no stack property to receive it, and a constructor +// argument other than a function declaration or the constructing class +// cannot be found on the stack. +interface Report { + message: string; +} +const report: Report = { message: "typed" }; +Error.captureStackTrace(report); + +const arrow = (): void => {}; +Error.captureStackTrace(new Error("arrow"), arrow); + +class Other {} +class Wrapped extends Error { + constructor() { + super("wrapped"); + Error.captureStackTrace(this, Other); + } +} +console.log(new Wrapped().stack); diff --git a/tests/harness/__snapshots__/error-capture-stack-trace-fences.ts.txt b/tests/harness/__snapshots__/error-capture-stack-trace-fences.ts.txt new file mode 100644 index 000000000..b12866eb2 --- /dev/null +++ b/tests/harness/__snapshots__/error-capture-stack-trace-fences.ts.txt @@ -0,0 +1,26 @@ +error-capture-stack-trace-fences.ts:9:25 - error SC2020: 'Error.captureStackTrace on '{ message: string }' values' is part of the standard library types but has no scriptc lowering yet + + 8 | const report: Report = { message: "typed" }; + 9 | Error.captureStackTrace(report); + | ^~~~~~ + 10 | + + hint: pass an Error instance, or an untyped object in JavaScript + +error-capture-stack-trace-fences.ts:12:45 - error SC2020: 'Error.captureStackTrace with this constructor argument' is part of the standard library types but has no scriptc lowering yet + + 11 | const arrow = (): void => {}; + 12 | Error.captureStackTrace(new Error("arrow"), arrow); + | ^~~~~ + 13 | + + hint: pass a function declaration, or this.constructor or the class inside its constructor + +error-capture-stack-trace-fences.ts:18:35 - error SC2020: 'Error.captureStackTrace with this constructor argument' is part of the standard library types but has no scriptc lowering yet + + 17 | super("wrapped"); + 18 | Error.captureStackTrace(this, Other); + | ^~~~~ + 19 | } + + hint: pass a function declaration, or this.constructor or the class inside its constructor \ No newline at end of file diff --git a/tests/harness/corpus-costs/6873-error-capture-stack-trace.ts.json b/tests/harness/corpus-costs/6873-error-capture-stack-trace.ts.json new file mode 100644 index 000000000..21b254d87 --- /dev/null +++ b/tests/harness/corpus-costs/6873-error-capture-stack-trace.ts.json @@ -0,0 +1 @@ +{ "plain": 2446, "san": 2267 } diff --git a/tests/harness/corpus-costs/6874-error-capture-stack-trace-objects.cjs.json b/tests/harness/corpus-costs/6874-error-capture-stack-trace-objects.cjs.json new file mode 100644 index 000000000..a151f11d0 --- /dev/null +++ b/tests/harness/corpus-costs/6874-error-capture-stack-trace-objects.cjs.json @@ -0,0 +1 @@ +{ "plain": 2286, "san": 3360 } From 3121f6909db422e8429a4d2696cb6d6178385879 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 18:43:07 -0500 Subject: [PATCH 02/54] Evaluate a JavaScript constructorOpt, and keep a non-extensible error's stack - Error.captureStackTrace in JavaScript discarded a constructorOpt it does not recognize (a getter, a call), so its side effects and exceptions did not happen. It now evaluates after the target and before the capture, as V8 evaluates the call's arguments; the target is held in a local when evaluating it twice would not be safe. - Capturing on an error replaced its saved stack without the checks V8 makes when it defines `stack`. A frozen, sealed or non-extensible error now throws "Cannot define property stack, object is not extensible", and one whose `stack` is non-configurable throws "Cannot redefine property: stack"; both keep their stack. Corpus 6874 checks the evaluation order and a constructorOpt that throws. --- .../src/frontend/lowering/builtins/errors.ts | 29 +++++++++++++--- packages/runtime/src/scr_json.c | 21 ++++++++++-- ...6874-error-capture-stack-trace-objects.cjs | 33 ++++++++++++++++++- 3 files changed, 75 insertions(+), 8 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/builtins/errors.ts b/packages/compiler/src/frontend/lowering/builtins/errors.ts index ce95d4a46..1eb2686fd 100644 --- a/packages/compiler/src/frontend/lowering/builtins/errors.ts +++ b/packages/compiler/src/frontend/lowering/builtins/errors.ts @@ -2,6 +2,7 @@ import * as ts from "../../ts7/adapter.js"; import { type Lowerer } from "../lowerer.js"; import { isJsSourceFile, locOf } from "../../program.js"; import { dynUndefinedExpr } from "../../../ir/build.js"; +import { isSafeToDiscard, isSafeToRepeat } from "../expressions/evaluation-safety.js"; import { BOOL, CAUGHT, @@ -439,9 +440,9 @@ function enclosingConstructorClass(node: ts.Node): ts.ClassLikeDeclaration | und * `new.target` or the class's own name inside a class constructor leave the * frames from the code that ran `new`, which are the current frames * (constructors have no frames of their own). Other targets are refused, - * and so are other constructorOpt values in TypeScript; JavaScript keeps all - * frames for them. Null when the callee is not the - * standard library's Error.captureStackTrace. */ + * and so are other constructorOpt values in TypeScript; JavaScript evaluates + * them and keeps all frames. Null when the callee is not the standard + * library's Error.captureStackTrace. */ export function lowerErrorCaptureStackTraceCall( lowerer: Lowerer, call: ts.CallExpression, @@ -477,6 +478,11 @@ export function lowerErrorCaptureStackTraceCall( } let through = ""; let skip = false; + // A JavaScript constructorOpt that is not a recognized form still + // evaluates, after the target and before the capture, as V8 evaluates + // the call's arguments: its getters and calls run and its exceptions + // propagate. + let evaluated: IrExpr | null = null; const ctor = args[1]; if (ctor !== undefined) { let node: ts.Expression = ctor; @@ -506,6 +512,8 @@ export function lowerErrorCaptureStackTraceCall( // JavaScript (undici's `captureStackTrace(err, module.exports.fetch)`) // keeps every frame: the same frames V8 keeps, and the ones above them. skip = false; + const value = lowerer.lowerExpr(ctor); + if (!isSafeToDiscard(value)) evaluated = value; } else { lowerer.noLowering( "Error.captureStackTrace with this constructor argument", @@ -514,20 +522,31 @@ export function lowerErrorCaptureStackTraceCall( ); } } + const stmts: IrStmt[] = []; + let receiver = lowerer.coerceToExpected(target, DYN); + if (evaluated !== null) { + if (!isSafeToRepeat(receiver)) { + const local = lowerer.declareHiddenLocal("%captureTarget", DYN); + stmts.push({ kind: "varDecl", localId: local.id, init: receiver, loc }); + receiver = { kind: "varRef", localId: local.id, type: DYN, loc }; + } + stmts.push({ kind: "exprStmt", expr: evaluated, loc }); + } const capture: IrExpr = { kind: "libCall", fn: "error.captureStackTrace", args: [ - lowerer.coerceToExpected(target, DYN), + receiver, { kind: "strLit", value: through, type: STRING, loc }, { kind: "boolLit", value: skip, type: BOOL, loc }, ], type: VOID, loc, }; + stmts.push({ kind: "exprStmt", expr: capture, loc }); return { kind: "seqExpr", - stmts: [{ kind: "exprStmt", expr: capture, loc }], + stmts, result: dynUndefinedExpr(loc), type: DYN, loc, diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 03e198cc8..ed3df36bc 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -16050,8 +16050,9 @@ static ScrDyn *scr_captured_stack_set_call(ScrClosure *closure, ScrDyn *const *a * whose text is `through` are left out (none remain when `through` is NULL or * not on the stack). An error keeps the frames as its own; another object * gets an own non-enumerable, configurable `stack` accessor. A primitive - * target throws `TypeError: invalid_argument` and a non-extensible object - * V8's define-property TypeError. */ + * target throws `TypeError: invalid_argument`. A non-extensible target (an + * error or another object), and an error whose `stack` is non-configurable, + * throw V8's define-property TypeErrors and keep their stack. */ void scr_error_capture_stack_trace(ScrDyn *target, const char *through, bool skip) { switch (target->kind) { case SCR_DYN_UNDEF: @@ -16071,6 +16072,22 @@ void scr_error_capture_stack_trace(ScrDyn *target, const char *through, bool ski ScrStr *frames = !skip ? scr_stack_capture() : through ? scr_stack_capture_after(through) : NULL; ScrError *error = scr_errdyn_err_of(target); if (error) { + /* V8 defines the error's own `stack`, so a frozen, sealed or + * non-extensible error throws, and so does one whose `stack` was made + * non-configurable; the saved stack stays. */ + bool extensible = scr_dyn_is_extensible(target); + const ScrDynEntry *own = target->kind == SCR_DYN_OBJ ? scr_dyn_find_entry(target, "stack", 5) : NULL; + if (scr_exc_pending() || !extensible || (own && !own->configurable)) { + static const char not_extensible[] = "Cannot define property stack, object is not extensible"; + static const char redefine[] = "Cannot redefine property: stack"; + if (!scr_exc_pending() && !extensible) + scr_throw_error_msg(SCR_ERR_TYPE, not_extensible, sizeof not_extensible - 1); + else if (!scr_exc_pending()) + scr_throw_error_msg(SCR_ERR_TYPE, redefine, sizeof redefine - 1); + scr_str_release(frames); + scr_error_release(error); + return; + } scr_str_release(error->stack_frames); error->stack_frames = frames; scr_str_release(error->stack); diff --git a/tests/corpus/6874-error-capture-stack-trace-objects.cjs b/tests/corpus/6874-error-capture-stack-trace-objects.cjs index 97d322659..94a47680c 100644 --- a/tests/corpus/6874-error-capture-stack-trace-objects.cjs +++ b/tests/corpus/6874-error-capture-stack-trace-objects.cjs @@ -2,7 +2,8 @@ // non-enumerable, configurable `stack` accessor giving the object's heading // (its name and message when first read, through Error.prototype.toString's // rules, "Error" by default) and the frames from the caller; TypeErrors for a -// primitive and a non-extensible target. +// primitive and a non-extensible target; and a constructorOpt getter or call +// that runs after the target and before the capture. 'use strict'; function frames(stack) { const lines = String(stack).split('\n'); @@ -90,3 +91,33 @@ try { } catch (error) { console.log('function value primitive', error.name, error.message); } +const order = []; +const holder = { + get ctor() { + order.push('getter'); + return inner; + }, +}; +function pick() { + order.push('call'); + return inner; +} +function note(label, value) { + order.push(label); + return value; +} +Error.captureStackTrace(note('target', {}), holder.ctor); +Error.captureStackTrace(note('target', {}), pick()); +console.log('constructorOpt evaluated', order.join(',')); +const throwing = { + get ctor() { + throw new RangeError('from constructorOpt'); + }, +}; +const untouched = {}; +try { + Error.captureStackTrace(untouched, throwing.ctor); + console.log('throwing constructorOpt', 'no error'); +} catch (caught) { + console.log('throwing constructorOpt', caught.name, caught.message, 'stack' in untouched); +} From c81758a08d7a5e79f65d91a73febcf52acf441cb Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 21:30:03 -0700 Subject: [PATCH 03/54] Map a failing type exactly from its canonical form where localized failure would degrade it Stage N2 of the localized-failure design (canonical type normalization). At localized failure's commit points, before a failing type degrades to its pushed-down or whole-slot dynamic form, its canonical form maps through mapCanonical under the current bindings. When that gives a static IR type, the type maps exactly: no dynamic slot, no SC6104. Opaque and unbound forms, and forms that map only to the checked dynamic, degrade as before. The coverage remainder re-derives canonical forms in its own tables. Because it runs only at commit points, a program that compiles today never consults it. Co-Authored-By: Claude Opus 5.5 --- .../compiler/src/frontend/lowering/lowerer.ts | 64 ++++++++++++++++--- 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index ecc10a5a4..7d3b4d38d 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -7,7 +7,8 @@ import { arrayRootOf, lowerWithArraySpecies, } from "./builtins/array-classes.js"; -import { canonShadowHook } from "../canon-map.js"; +import { canonShadowHook, mapCanonical } from "../canon-map.js"; +import { Normalizer } from "../canon.js"; import { compactDeclOrigins, inPackageFile, @@ -750,6 +751,9 @@ export interface DynamicSlot { /** The slot keeps its type's compilable shape; only the failing * components are checked-dynamic (push-down, stage L3). */ partial?: true; + /** The type mapped exactly from its canonical form (stage N2): a static + * slot, never reported as dynamic. */ + canonical?: true; } /** Localized failure: rounds of the declaration flush retry. */ @@ -1007,6 +1011,9 @@ export interface DegradedType { lenient: IrType | null; code: string; reason: string; + /** The type maps exactly from its canonical form (stage N2): its slots + * are static. */ + canonical?: true; } /** The Lowerer's pass configuration (see lowerToIr). */ @@ -4178,9 +4185,12 @@ export class Lowerer { // checked dynamic cuts the knot while it does. if (degraded.lenient === null) { degraded.lenient = DYN; - degraded.lenient = this.lenientType(type); + degraded.lenient = + (degraded.canonical === true ? this.canonicalType(type) : null) ?? + this.lenientType(type); } - if (degraded.lenient.kind !== "dyn") this.partialHits[this.partialHits.length - 1]?.add(id); + if (degraded.lenient.kind !== "dyn" && degraded.canonical !== true) + this.partialHits[this.partialHits.length - 1]?.add(id); return degraded.lenient; }, ...(shadow !== undefined ? { canonShadow: shadow } : {}), @@ -9833,8 +9843,10 @@ export class Lowerer { } if (committed === undefined) return null; this.degradeType(type, committed.code, committed.message); + const canonical = this.isCanonicalDegraded(id); const partial = this.degradedTypes.get(id)?.lenient?.kind !== "dyn"; return { + ...(canonical ? { canonical: true as const } : {}), ...(partial ? { partial: true as const } : {}), loc: committed.loc, failure, @@ -9966,6 +9978,14 @@ export class Lowerer { this.degradedTypes.set(id, { lenient: DYN, code, reason }); return true; } + // Canonical normalization (stage N2): a type the per-form rules miss + // maps exactly from its canonical form when that form maps; no dynamic + // slot then. Otherwise the lenient (pushed-down) form. + const canonical = this.canonicalType(type); + if (canonical !== null) { + this.degradedTypes.set(id, { lenient: canonical, code, reason, canonical: true }); + return true; + } this.degradedTypes.set(id, { lenient: this.wholeSlotDegradation ? DYN : this.lenientType(type), code, @@ -9974,10 +9994,38 @@ export class Lowerer { return true; } + /** The IR type of a type's canonical form (canon-map.ts), or null when + * the form is irreducible, unbound, or only the checked dynamic. Runs + * only at localized failure's commit points, so a program that compiles + * today never consults it. */ + canonicalType(type: ts.Type): IrType | null { + try { + const resolve = this.typeCtx.resolveTypeParamTs; + const c = new Normalizer(this.checker).canon(type, resolve ? (t) => resolve(t) : new Map()); + if (c.k === "opaque" || c.k === "param") return null; + const { typeMemo: _strictMemo, ...strict } = this.typeCtx; + const mapped = mapCanonical(c, { ...strict, canMemoizeType: () => false }); + return mapped === null || mapped.kind === "void" || mapped.kind === "dyn" ? null : mapped; + } catch { + return null; + } + } + + /** The degraded type of `id` maps exactly (canonical form): its slots + * are static, not dynamic. */ + isCanonicalDegraded(id: number): boolean { + return this.degradedTypes.get(id)?.canonical === true; + } + /** Some degraded type kept a pushed-down (partial) shape. */ hasPartialDegradedTypes(): boolean { for (const degraded of this.degradedTypes.values()) - if (degraded.lenient !== null && degraded.lenient.kind !== "dyn") return true; + if ( + degraded.canonical !== true && + degraded.lenient !== null && + degraded.lenient.kind !== "dyn" + ) + return true; return false; } @@ -10011,6 +10059,7 @@ export class Lowerer { /** Records a dynamic slot whose unit lowered with it (one per site). */ noteDynamicSlot(slot: DynamicSlot): void { + if (slot.canonical === true) return; const seen = this.dynamicSlots.some( (s) => s.loc.file === slot.loc.file && @@ -13166,18 +13215,17 @@ export class Lowerer { const prior = this.degradedTypes.get(id); degraded.push({ id, - prior: - prior === undefined - ? undefined - : { lenient: prior.lenient, code: prior.code, reason: prior.reason }, + prior: prior === undefined ? undefined : { ...prior }, }); this.degradeType(type, diag.code, diag.message); + const canonical = this.isCanonicalDegraded(id); const partial = this.degradedTypes.get(id)?.lenient?.kind !== "dyn"; slots.push({ loc: diag.loc, failure, code: diag.code, reason: diag.message, + ...(canonical ? { canonical: true as const } : {}), ...(partial ? { partial: true as const } : {}), }); } From 08cc15d87295500d46e3bb16e5c14262e79bf48e Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 26 Oct 2026 07:07:50 -0700 Subject: [PATCH 04/54] Keep canonical degraded entries when a declaration retry rolls back Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/frontend/lowering/lowerer.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 7d3b4d38d..eee126efb 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -13215,7 +13215,15 @@ export class Lowerer { const prior = this.degradedTypes.get(id); degraded.push({ id, - prior: prior === undefined ? undefined : { ...prior }, + prior: + prior === undefined + ? undefined + : { + lenient: prior.lenient, + code: prior.code, + reason: prior.reason, + ...(prior.canonical === true ? { canonical: true as const } : {}), + }, }); this.degradeType(type, diag.code, diag.message); const canonical = this.isCanonicalDegraded(id); From a17b727e91277753635729cb0814239744968ec4 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 10:51:18 -0500 Subject: [PATCH 05/54] Address review feedback --- packages/compiler/src/frontend/canon-map.ts | 4 ++++ packages/compiler/src/frontend/canon.ts | 16 ++++++++++++-- .../compiler/test/canon-normalize.test.ts | 22 ++++++++++++++----- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/packages/compiler/src/frontend/canon-map.ts b/packages/compiler/src/frontend/canon-map.ts index 2f40af21b..b6e786bd5 100644 --- a/packages/compiler/src/frontend/canon-map.ts +++ b/packages/compiler/src/frontend/canon-map.ts @@ -75,6 +75,10 @@ export function mapCanonical(c: Canon, ctx: TypeMapperCtx, depth = 0): IrType | return viaChecker(); if (c.index().length > 0) return viaChecker(); const props = c.props(); + // Accessor members are closure slots (`%get:x`, `%set:x`), and a + // getter and setter of different types have no single slot type: + // mapType's rules decide them, not a data field of the read type. + if (props.some((p) => p.accessor)) return viaChecker(); // `{}` holds any non-nullish value: the checked dynamic, as mapType. if (props.length === 0) return DYN; const fields: { name: string; type: IrType }[] = []; diff --git a/packages/compiler/src/frontend/canon.ts b/packages/compiler/src/frontend/canon.ts index 706ec1e89..0942a1a7d 100644 --- a/packages/compiler/src/frontend/canon.ts +++ b/packages/compiler/src/frontend/canon.ts @@ -21,6 +21,8 @@ export interface CanonProp { name: string; optional: boolean; readonly: boolean; + /** A get or set accessor, not a data property. */ + accessor: boolean; type: () => Canon; } @@ -131,7 +133,10 @@ function canonKeyInner(c: Canon, depth: number, budget: { left: number }): strin } const props = c .props() - .map((p) => `${p.name}${p.optional ? "?" : ""}: ${canonKey(p.type(), depth + 1)}`); + .map( + (p) => + `${p.accessor ? "get " : ""}${p.name}${p.optional ? "?" : ""}: ${canonKey(p.type(), depth + 1)}`, + ); const sig = (s: CanonSig): string => `(${s.params .map( @@ -251,7 +256,13 @@ function sameCanon(a: Canon, b: Canon, assumed: Assumptions): boolean { if (ap.length !== op.length) return false; for (const p of ap) { const q = op.find((x) => x.name === p.name); - if (q === undefined || q.optional !== p.optional || !eq(p.type(), q.type())) return false; + if ( + q === undefined || + q.optional !== p.optional || + q.accessor !== p.accessor || + !eq(p.type(), q.type()) + ) + return false; } const ai = a.index(); const oi = o.index(); @@ -501,6 +512,7 @@ export class Normalizer { name: ts.symbolName(p), optional: (p.flags & ts.SymbolFlags.Optional) !== 0, readonly: false, + accessor: (p.flags & (ts.SymbolFlags.GetAccessor | ts.SymbolFlags.SetAccessor)) !== 0, type: () => this.canon(checker.getTypeOfSymbol(p), env), }))), index: () => diff --git a/packages/compiler/test/canon-normalize.test.ts b/packages/compiler/test/canon-normalize.test.ts index c9140c6f3..515d895f3 100644 --- a/packages/compiler/test/canon-normalize.test.ts +++ b/packages/compiler/test/canon-normalize.test.ts @@ -73,6 +73,9 @@ beforeAll(() => { export type Loose = [string, number?, ...boolean[]]; // Arrays of two instantiations of one generic class. export type Maps = Map[] | Map[]; + // Accessor members: one type, and a getter and setter of different types. + export interface Gauge { get level(): number; set level(next: number) } + export interface Cell { get value(): string; set value(next: string | number) } `, ); load = loadProgram(entry); @@ -291,11 +294,20 @@ describe("mapCanonical", () => { isExternalTypeFile: () => false, isProgramFile: (sf) => load.moduleOrder.includes(sf), }); - test.for(["Normalized", "Same", "Flat", "Id", "Pair"])("%s maps like mapType", (name) => { - const strict = mapType(aliases.get(name)!, ctx()); - const canonical = mapCanonical(new Normalizer(checker).canon(aliases.get(name)!), ctx()); - expect(strict).not.toBeNull(); - expect(canonical && typeKey(canonical)).toBe(typeKey(strict!)); + test.for(["Normalized", "Same", "Flat", "Id", "Pair", "Gauge"])( + "%s maps like mapType", + (name) => { + const strict = mapType(aliases.get(name)!, ctx()); + const canonical = mapCanonical(new Normalizer(checker).canon(aliases.get(name)!), ctx()); + expect(strict).not.toBeNull(); + expect(canonical && typeKey(canonical)).toBe(typeKey(strict!)); + }, + ); + test("accessors are not data fields: a getter and setter of different types do not map", () => { + const cell = aliases.get("Cell")!; + expect(mapType(cell, ctx())).toBeNull(); + expect(mapCanonical(new Normalizer(checker).canon(cell), ctx())).toBeNull(); + expect(key(new Normalizer(checker).canon(cell))).toContain("get value"); }); test("a generic alias maps under bindings, where mapType needs the instance table", () => { const { type, params } = asTypes.get("Catalog")!; From 6c63d8cd10ca8e099bc93eb3665a34e54bf16055 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 21:39:37 -0500 Subject: [PATCH 06/54] Run a constructor getter that this.constructor reads in Error.captureStackTrace Inside a class constructor, Error.captureStackTrace(this, this.constructor) now reads this.constructor before the capture when the class or a base class declares a constructor getter, as V8 evaluates the argument: the getter runs and its exception propagates. --- .../src/frontend/lowering/builtins/errors.ts | 16 ++++++++++ .../corpus/6873-error-capture-stack-trace.ts | 29 +++++++++++++++++++ ...6874-error-capture-stack-trace-objects.cjs | 29 +++++++++++++++++++ 3 files changed, 74 insertions(+) diff --git a/packages/compiler/src/frontend/lowering/builtins/errors.ts b/packages/compiler/src/frontend/lowering/builtins/errors.ts index 1eb2686fd..7e2bb6d37 100644 --- a/packages/compiler/src/frontend/lowering/builtins/errors.ts +++ b/packages/compiler/src/frontend/lowering/builtins/errors.ts @@ -431,6 +431,16 @@ function enclosingConstructorClass(node: ts.Node): ts.ClassLikeDeclaration | und return undefined; } +/** Whether `access` (`this.constructor`) reads a `constructor` accessor + * declared on the class or a base class. */ +function readsConstructorAccessor(lowerer: Lowerer, access: ts.PropertyAccessExpression): boolean { + const property = lowerer.checker.getPropertyOfType( + lowerer.typeOf(access.expression), + "constructor", + ); + return property !== undefined && (property.flags & ts.SymbolFlags.GetAccessor) !== 0; +} + /** `Error.captureStackTrace(target, constructorOpt?)`, also through `?.`: * target.stack becomes the target's heading and the current stack frames, * as V8 does it (scr_error_capture_stack_trace). The target is an error, a @@ -504,6 +514,12 @@ export function lowerErrorCaptureStackTraceCall( const sig = resolved !== undefined ? lowerer.fnSigsBySymbol.get(resolved) : undefined; if (constructing && cls !== undefined) { skip = false; + // `this.constructor` runs a `constructor` getter of the class, whose + // calls and exceptions are observable. + if (ts.isPropertyAccessExpression(node) && readsConstructorAccessor(lowerer, node)) { + const value = lowerer.lowerExpr(ctor); + if (!isSafeToDiscard(value)) evaluated = value; + } } else if (sig !== undefined) { lowerer.noteEdge(sig.name); through = sig.name; diff --git a/tests/corpus/6873-error-capture-stack-trace.ts b/tests/corpus/6873-error-capture-stack-trace.ts index 62db5572d..922588870 100644 --- a/tests/corpus/6873-error-capture-stack-trace.ts +++ b/tests/corpus/6873-error-capture-stack-trace.ts @@ -85,3 +85,32 @@ console.log("limit", frames(outer(new Error("limited")).stack)); Error.stackTraceLimit = 0; console.log("limit 0", JSON.stringify(outer(new Error("none")).stack)); Error.stackTraceLimit = 10; + +// `this.constructor` in a constructor runs a `constructor` getter, and its +// exception propagates. +class Watched extends Error { + constructor() { + super("watched"); + Error.captureStackTrace(this, this.constructor); + } + get ["constructor"](): typeof Watched { + console.log("constructor getter ran"); + return Watched; + } +} +console.log("getter", typeof new Watched().stack); +class Throwing extends Error { + constructor() { + super("throwing"); + Error.captureStackTrace(this, this.constructor); + } + get ["constructor"](): typeof Throwing { + throw new RangeError("constructor getter threw"); + } +} +try { + new Throwing(); + console.log("getter did not throw"); +} catch (e: any) { + console.log("getter threw", e instanceof RangeError, e.message); +} diff --git a/tests/corpus/6874-error-capture-stack-trace-objects.cjs b/tests/corpus/6874-error-capture-stack-trace-objects.cjs index 94a47680c..456874b4e 100644 --- a/tests/corpus/6874-error-capture-stack-trace-objects.cjs +++ b/tests/corpus/6874-error-capture-stack-trace-objects.cjs @@ -121,3 +121,32 @@ try { } catch (caught) { console.log('throwing constructorOpt', caught.name, caught.message, 'stack' in untouched); } + +// `this.constructor` in a constructor runs a `constructor` getter, and its +// exception propagates. +class Watched extends Error { + constructor() { + super('watched'); + Error.captureStackTrace(this, this.constructor); + } + get ['constructor']() { + console.log('constructor getter ran'); + return Watched; + } +} +console.log('getter', typeof new Watched().stack); +class Throwing extends Error { + constructor() { + super('throwing'); + Error.captureStackTrace(this, this.constructor); + } + get ['constructor']() { + throw new RangeError('constructor getter threw'); + } +} +try { + new Throwing(); + console.log('getter did not throw'); +} catch (e) { + console.log('getter threw', e instanceof RangeError, e.message); +} From ed54f4f0c8c15ab9ed308b3311f3e3772a6b782e Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 02:59:35 -0500 Subject: [PATCH 07/54] Leave the dispose list out of arena constructors when every arena is closed An arena-aware constructor of a record shape or class with reference fields tests whether the current arena is closed on every allocation, and appends a dispose entry when it is not. When every root and call region in the program opens a closed arena, the current arena is closed or absent, so planArenas sets ArenaPlan.allClosed and those constructors leave out the test and the dispose list. --- .../compiler/src/backend/llvm/arenas.test.ts | 114 ++++++++++++++++++ packages/compiler/src/backend/llvm/arenas.ts | 7 +- packages/compiler/src/backend/llvm/classes.ts | 3 +- packages/compiler/src/backend/llvm/emitter.ts | 1 + packages/compiler/src/backend/llvm/shapes.ts | 6 +- 5 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 packages/compiler/src/backend/llvm/arenas.test.ts diff --git a/packages/compiler/src/backend/llvm/arenas.test.ts b/packages/compiler/src/backend/llvm/arenas.test.ts new file mode 100644 index 000000000..1ab4b010a --- /dev/null +++ b/packages/compiler/src/backend/llvm/arenas.test.ts @@ -0,0 +1,114 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile } from "../../index.js"; + +async function llvmFor(source: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "scriptc-arenas-")); + try { + const entry = join(dir, "main.ts"), + output = join(dir, "main.ll"); + await writeFile(entry, source); + const result = await compile(entry, { outDir: dir, outPath: output, outputKind: "llvm" }); + if (!result.ok) + throw new Error(result.diagnostics.map((item) => `${item.code}: ${item.message}`).join("\n")); + return await readFile(output, "utf8"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +/** The bodies of the module's arena-aware constructors. */ +function arenaConstructors(llvm: string): string[] { + return [...llvm.matchAll(/^define internal ptr @[^(]*\.r\(\)[^\n]*\n[\s\S]*?^}/gm)].map( + (m) => m[0], + ); +} + +const TREE = ` +class Node { + left: Node | null; + right: Node | null; + constructor(left: Node | null, right: Node | null) { + this.left = left; + this.right = right; + } + count(): number { + return 1 + (this.left === null ? 0 : this.left.count()) + (this.right === null ? 0 : this.right.count()); + } +} +function tree(depth: number): Node { + return depth === 0 ? new Node(null, null) : new Node(tree(depth - 1), tree(depth - 1)); +} +interface Leaf { + left: Leaf | null; + right: Leaf | null; +} +function leaves(depth: number): Leaf { + return depth === 0 ? { left: null, right: null } : { left: leaves(depth - 1), right: leaves(depth - 1) }; +} +function size(leaf: Leaf): number { + return leaf.left === null || leaf.right === null ? 1 : 1 + size(leaf.left) + size(leaf.right); +} +`; + +test("when every arena is closed, arena-aware constructors take no dispose list", async () => { + const llvm = await llvmFor(`${TREE} +let total = 0; +for (let d = 0; d < 4; d++) total += tree(d).count() + size(leaves(d)); +console.log(total); +`); + expect(llvm).toMatch(/@scr_arena_open\(ptr %[\w.]+, i32 1\)/); + expect(llvm).not.toMatch(/@scr_arena_open\(ptr %[\w.]+, i32 0\)/); + const constructors = arenaConstructors(llvm); + expect(constructors.length).toBe(2); + for (const body of constructors) { + expect(body).not.toContain("ScrArena.closed"); + expect(body).not.toContain("@scr_arena_disposal"); + } +}); + +test("an arena that is not closed keeps the dispose list in every arena-aware constructor", async () => { + // The rings hold strings and outside categories, so their groups are not closed. + const llvm = await llvmFor(`${TREE} +class Category { + name: string; + constructor(name: string) { + this.name = name; + } +} +class Member { + next: Member | undefined = undefined; + category: Category; + label: string; + constructor(category: Category, label: string) { + this.category = category; + this.label = label; + } +} +function ring(categories: Category[], size: number, seed: number): Member { + const first = new Member(categories[seed % categories.length]!, \`m\${seed}\`); + let last = first; + for (let i = 1; i < size; i++) { + const member = new Member(categories[(seed + i) % categories.length]!, \`m\${seed}-\${i}\`); + last.next = member; + last = member; + } + last.next = first; + return first; +} +const categories = [new Category("a"), new Category("b")]; +let total = 0; +for (let round = 0; round < 4; round++) { + const r = ring(categories, 30, round); + total += r.next!.label.length + r.category.name.length + tree(round).count(); +} +console.log(total); +`); + expect(llvm).toMatch(/@scr_arena_open\(ptr %[\w.]+, i32 1\)/); + expect(llvm).toMatch(/@scr_arena_open\(ptr %[\w.]+, i32 0\)/); + const withFields = arenaConstructors(llvm).filter((body) => body.includes(".dispose")); + expect(withFields.length).toBeGreaterThan(0); + for (const body of withFields) expect(body).toContain("ScrArena.closed"); +}); diff --git a/packages/compiler/src/backend/llvm/arenas.ts b/packages/compiler/src/backend/llvm/arenas.ts index ee5ea8ab0..3e3954856 100644 --- a/packages/compiler/src/backend/llvm/arenas.ts +++ b/packages/compiler/src/backend/llvm/arenas.ts @@ -47,6 +47,10 @@ export interface ArenaPlan { /** Record shapes and classes that get an arena-aware constructor. */ shapes: Set; classes: Set; + /** Every root and call region opens a closed arena, so the current arena + * is always closed or absent, and the arena-aware constructors of record + * shapes and classes add no dispose entries. */ + allClosed: boolean; } export interface ArenaEligibility { @@ -150,7 +154,8 @@ export function planArenas(mod: IrModule, eligible: ArenaEligibility): ArenaPlan }; const keptSite = (e: Node): boolean => sites.arenaSites.has(e) || groups.roots.has(e); for (const r of analyzeClosedGroups(mod, groups, arenaType, keptSite)) groups.closed.add(r); - return { groups, sites, shapes, classes }; + const allClosed = [...groups.roots, ...groups.calls].every((r) => groups.closed.has(r)); + return { groups, sites, shapes, classes, allClosed }; } /** The functions in `reachable` that can allocate a cycle-capable object diff --git a/packages/compiler/src/backend/llvm/classes.ts b/packages/compiler/src/backend/llvm/classes.ts index 695acc273..93f75df58 100644 --- a/packages/compiler/src/backend/llvm/classes.ts +++ b/packages/compiler/src/backend/llvm/classes.ts @@ -1527,7 +1527,8 @@ export function emitClassShapes( ? ` store i64 ${(BigInt(meta.pre + 1) * 2n ** 32n + BigInt(ARENA_RC)).toString()}, ptr %o.vt` : ` store ${host.sizeType} ${ARENA_RC}, ptr %o`; const newSym = mangleClassNew(cls.name); - const dispose = refFields.length > 0 ? arenaDisposeSym(newSym) : null; + const dispose = + refFields.length > 0 && host.arenasClosed !== true ? arenaDisposeSym(newSym) : null; defs.push( ...arenaNewLines( host, diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 2d2ebfb8d..6d023be87 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -2511,6 +2511,7 @@ export class LlEmitter { if (this.arenas !== null) { this.shapeHost.arenaShapes = this.arenas.shapes; this.shapeHost.arenaClasses = this.arenas.classes; + this.shapeHost.arenasClosed = this.arenas.allClosed; } } for (const [name, rec] of RUNTIME_ERROR_CLASSES) { diff --git a/packages/compiler/src/backend/llvm/shapes.ts b/packages/compiler/src/backend/llvm/shapes.ts index b5b1e735a..05f1915b3 100644 --- a/packages/compiler/src/backend/llvm/shapes.ts +++ b/packages/compiler/src/backend/llvm/shapes.ts @@ -149,6 +149,9 @@ export interface ShapeHost { * (arenas.ts). */ arenaShapes?: ReadonlySet; arenaClasses?: ReadonlySet; + /** Every arena the program opens is closed (ArenaPlan.allClosed): the + * arena-aware constructors take no dispose list. */ + arenasClosed?: boolean; /** The target C ABI's `size_t` / refcount integer type. */ readonly sizeType: "i32" | "i64"; /** Worker executables: the inline allocator state is thread-local. */ @@ -2344,7 +2347,8 @@ export function emitRecordShapes( defs.push(...nw); if (host.arenaShapes?.has(shape.id) === true && !shape.indexValue) { const newSym = mangleRecordNew(shape.id); - const dispose = refMembers.length > 0 ? arenaDisposeSym(newSym) : null; + const dispose = + refMembers.length > 0 && host.arenasClosed !== true ? arenaDisposeSym(newSym) : null; const init = [` store ${host.sizeType} ${ARENA_RC}, ptr %o`]; if (shapeNumber !== 0) init.push( From d4185bcd8e5b0cad683e38ae37f94fd78d0e58f3 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 03:34:16 -0500 Subject: [PATCH 08/54] Add an opt-in --static flag for Linux executables - `scriptc build --static` (and the `staticLink` compile option) links glibc executables as static PIE on the build host's own glibc target; musl executables are already static and accept the flag. - An executable that keeps glibc's NSS module loader (host, user and group name lookups) is refused with SC3002, since a static glibc executable loads those modules from the run host. - Other targets, cross-compiled glibc targets, --sanitize and non-executable output are refused; static and dynamic builds use separate cache entries. --- docs/content/docs/cli.mdx | 7 + docs/content/docs/platforms.mdx | 2 +- packages/cli/src/bootstrap.ts | 2 + packages/cli/test/output-options.test.ts | 2 + packages/cli/test/static-link.test.ts | 132 ++++++++++++++++++ .../compiler/src/backend/link-plan-core.ts | 4 + packages/compiler/src/backend/link-plan.ts | 5 + packages/compiler/src/backend/linker.ts | 18 ++- .../compiler/src/backend/runtime-pack.test.ts | 26 +++- packages/compiler/src/backend/targets.test.ts | 56 ++++++++ packages/compiler/src/backend/targets.ts | 65 +++++++++ packages/compiler/src/cli/command.ts | 10 +- packages/compiler/src/cli/output-options.ts | 4 + packages/compiler/src/cli/usage.ts | 6 + packages/compiler/src/compile-types.ts | 5 + .../src/executable/executable-cache.ts | 4 + packages/compiler/src/index.ts | 40 +++++- packages/compiler/src/native/compiler.ts | 33 ++++- 18 files changed, 409 insertions(+), 12 deletions(-) create mode 100644 packages/cli/test/static-link.test.ts diff --git a/docs/content/docs/cli.mdx b/docs/content/docs/cli.mdx index 499192352..b1583a617 100644 --- a/docs/content/docs/cli.mdx +++ b/docs/content/docs/cli.mdx @@ -151,6 +151,9 @@ With `--sparse`, the profile leaves out the functions the runs never reached. Th
--strip
Remove symbol and debug payload when linking an executable, including cross-compiled targets. This flag preserves code optimization.
+
--static
+
On Linux, link the C library and the other system libraries into the executable. The executable starts without the dynamic loader, which shortens its startup, and doesn't depend on the run host's glibc version. glibc executables link as static PIE, are about 1 MB larger, and need glibc's static libraries on the build machine (libc6-dev on Debian and Ubuntu, glibc-static on Fedora); they link only for the build machine's own glibc target. A program whose executable includes host, user, or group name lookups, which dns.lookup, net and http connections, fetch, and os.userInfo() use, is refused with SC3002: a static glibc executable loads the NSS modules for those lookups from the run host, which glibc supports only for the glibc version it was linked with. Build such a program without --static, or for an <arch>-linux-musl target. musl executables are always linked statically, and accept the flag. Valid only for executable builds.
+
--optimization <release|dev|speed>
Select native optimization for executable, assembly, or object output. The default is release (-O2), which emits string retains and the retains and releases of dynamic values inline. dev uses -O0 and emits source locations for TypeScript and JavaScript breakpoints and stack frames. LLVM builds include source variable names and native storage descriptions; see Native debugging for the supported representations. macOS executable builds also produce an adjacent .dSYM bundle. speed is -O2 plus optimizations that trade binary size and build time for run-time speed: it emits the remaining reference counting inline and, on x86-64, starts every runtime function on a cache-line boundary. Expect larger executables and longer cold builds than release; how much faster a program runs depends on how much of its time is spent in the runtime. Where a runtime pack has no speed runtime, speed links the release runtime and keeps inline reference counting. Selecting speed never changes what release builds produce, and the two never share cached artifacts. Library profiles accept "optimization": "speed" for inline reference counting. In executables large enough to be divided into partitions, dev compiles each partition on its own, and release and speed add a step that lets partitions import small functions from each other; see Build parallelism.
@@ -373,6 +376,10 @@ For source breakpoints in Xcode or LLDB, build with --optimization=devscriptc run on wasm32-wasi Node.js 24 or later on PATH + + --static on glibc Linux + glibc's static libraries (libc6-dev on Debian and Ubuntu, glibc-static on Fedora) + --sanitize A C compiler and LLVM toolchain diff --git a/docs/content/docs/platforms.mdx b/docs/content/docs/platforms.mdx index 96b91fc2a..72cd65a3c 100644 --- a/docs/content/docs/platforms.mdx +++ b/docs/content/docs/platforms.mdx @@ -49,7 +49,7 @@ $ file fib-linux fib-linux: ELF 64-bit LSB executable, ARM aarch64, version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-aarch64.so.1, for GNU/Linux 2.0.0, with debug_info, not stripped ``` -`x86_64-linux-gnu.2.36` works the same way. For Alpine and other musl-based systems, use `aarch64-linux-musl` or `x86_64-linux-musl`; these produce statically linked executables. +`x86_64-linux-gnu.2.36` works the same way. For Alpine and other musl-based systems, use `aarch64-linux-musl` or `x86_64-linux-musl`; these produce statically linked executables. On a glibc-based Linux machine, [`--static`](/docs/cli#options) links a static executable for the machine's own target. Linux executables use the native Linux implementations of the event loop (epoll), servers, TLS (with the system's CA bundle), and `fs.watch`, and they are tested against Node.js on Linux, including Alpine. diff --git a/packages/cli/src/bootstrap.ts b/packages/cli/src/bootstrap.ts index f6efec134..ec3d881c2 100644 --- a/packages/cli/src/bootstrap.ts +++ b/packages/cli/src/bootstrap.ts @@ -140,6 +140,7 @@ async function tryFastPath(): Promise { backend: "llvm", ...(optimization === "dev" || optimization === "speed" ? { optimization } : {}), ...(values.strip ? { strip: true as const } : {}), + ...(values.static ? { staticLink: true as const } : {}), ...(values["detached-exit"] ? { detachedExit: true as const } : {}), ...(values["large-pages"] ? { largePages: true as const } : {}), ...(values["cpu-dispatch"] === true ? { cpuDispatch: true as const } : {}), @@ -225,6 +226,7 @@ async function coverageFastPath( (failOn !== undefined && failOn !== "blockers" && failOn !== "divergences") || values.emit !== undefined || values.strip || + values.static || values["detached-exit"] || values["large-pages"] || values["target-cpu"] !== undefined || diff --git a/packages/cli/test/output-options.test.ts b/packages/cli/test/output-options.test.ts index a00a61667..45d5c55b8 100644 --- a/packages/cli/test/output-options.test.ts +++ b/packages/cli/test/output-options.test.ts @@ -46,6 +46,8 @@ describe("output option compatibility", () => { [{ emit: "llvm", windowsSubsystem: "gui" }, /windows-subsystem/], [{ emit: "obj", strip: true }, /strip/], [{ emit: "llvm", strip: true }, /strip/], + [{ emit: "obj", staticLink: true }, /--static/], + [{ emit: "llvm", staticLink: true }, /--static/], [{ emit: "obj", detachedExit: true }, /detached-exit/], [{ emit: "llvm", detachedExit: true }, /detached-exit/], [{ emit: "obj", largePages: true }, /large-pages/], diff --git a/packages/cli/test/static-link.test.ts b/packages/cli/test/static-link.test.ts new file mode 100644 index 000000000..5d64e9d01 --- /dev/null +++ b/packages/cli/test/static-link.test.ts @@ -0,0 +1,132 @@ +import { execFile, execFileSync } from "node:child_process"; +import { createRequire } from "node:module"; +import { existsSync } from "node:fs"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, isAbsolute, join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; + +const execFileAsync = promisify(execFile); +const require = createRequire(import.meta.url); +const repoRoot = join(import.meta.dirname, "../../.."); +const cliEntry = join(repoRoot, "packages/cli/src/main.ts"); +const bootstrapEntry = join(repoRoot, "packages/cli/dist/bootstrap.js"); +const tsxLoader = join(dirname(require.resolve("tsx/package.json")), "dist/loader.mjs"); + +const glibcHost = + process.platform === "linux" && + typeof (process.report?.getReport() as { header?: { glibcVersionRuntime?: unknown } })?.header + ?.glibcVersionRuntime === "string"; + +/** Whether the host's clang finds glibc's static C library. */ +function hostHasStaticLibc(): boolean { + try { + const path = execFileSync("clang", ["-print-file-name=libc.a"], { encoding: "utf8" }).trim(); + return isAbsolute(path) && existsSync(path); + } catch { + return false; + } +} + +/** The ELF type and whether the executable names a program interpreter. */ +async function elfLinkage(path: string): Promise<{ type: number; interpreter: boolean }> { + const bytes = await readFile(path); + const type = bytes.readUInt16LE(16); + const phoff = Number(bytes.readBigUInt64LE(32)); + const phentsize = bytes.readUInt16LE(54); + const phnum = bytes.readUInt16LE(56); + let interpreter = false; + for (let i = 0; i < phnum; i++) { + if (bytes.readUInt32LE(phoff + i * phentsize) === 3 /* PT_INTERP */) interpreter = true; + } + return { type, interpreter }; +} + +async function withProject(run: (dir: string, env: NodeJS.ProcessEnv) => Promise) { + const dir = await mkdtemp( + join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-static-"), + ); + try { + return await run(dir, { ...process.env, SCRIPTC_CACHE_DIR: join(dir, "cache") }); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +function cli(args: string[], env: NodeJS.ProcessEnv, bootstrap = false) { + return execFileAsync( + process.execPath, + [...(bootstrap ? [bootstrapEntry] : ["--import", tsxLoader, cliEntry]), ...args], + { env, maxBuffer: 4 * 1024 * 1024 }, + ); +} + +test.skipIf(!glibcHost || !hostHasStaticLibc())( + "--static links a glibc executable as static PIE without mixing cache variants", + async () => { + await withProject(async (dir, env) => { + const entry = join(dir, "main.ts"); + const outPath = join(dir, "program"); + await writeFile(entry, 'console.log("static-ok", [1, 2, 3].map((n) => n * 2).join(","));\n'); + const build = async (staticLink: boolean, bootstrap = false) => { + await cli( + ["build", entry, "-o", outPath, ...(staticLink ? ["--static"] : [])], + env, + bootstrap, + ); + expect((await execFileAsync(outPath)).stdout).toBe("static-ok 2,4,6\n"); + return elfLinkage(outPath); + }; + // ET_DYN with a program interpreter: the default dynamic PIE link. + expect(await build(false)).toEqual({ type: 3, interpreter: true }); + // ET_DYN without one: static PIE. + expect(await build(true)).toEqual({ type: 3, interpreter: false }); + expect(await build(true, true)).toEqual({ type: 3, interpreter: false }); + expect(await build(false, true)).toEqual({ type: 3, interpreter: true }); + }); + }, + 180_000, +); + +test.skipIf(!glibcHost || !hostHasStaticLibc())( + "--static refuses a glibc executable that includes user name lookups", + async () => { + await withProject(async (dir, env) => { + const entry = join(dir, "main.ts"); + const outPath = join(dir, "program"); + await writeFile( + entry, + 'import { userInfo } from "node:os";\nconsole.log(userInfo().username);\n', + ); + const failure = await cli(["build", entry, "-o", outPath, "--static"], env).then( + () => null, + (error: { code?: number; stderr?: string }) => error, + ); + expect(failure?.code).toBe(1); + expect(failure?.stderr).toContain("error SC3002: --static cannot link this program"); + expect(existsSync(outPath)).toBe(false); + // The same program links without --static. + await cli(["build", entry, "-o", outPath], env); + expect((await execFileAsync(outPath)).stdout).not.toBe(""); + }); + }, + 180_000, +); + +test.skipIf(process.platform === "linux")( + "--static is refused for executables that do not target Linux", + async () => { + await withProject(async (dir, env) => { + const entry = join(dir, "main.ts"); + await writeFile(entry, 'console.log("x");\n'); + const failure = await cli(["build", entry, "-o", join(dir, "program"), "--static"], env).then( + () => null, + (error: { code?: number; stderr?: string }) => error, + ); + expect(failure?.code).toBe(1); + expect(failure?.stderr).toContain("--static is only supported for Linux executables"); + }); + }, + 180_000, +); diff --git a/packages/compiler/src/backend/link-plan-core.ts b/packages/compiler/src/backend/link-plan-core.ts index f8ad20e6f..f810a4a17 100644 --- a/packages/compiler/src/backend/link-plan-core.ts +++ b/packages/compiler/src/backend/link-plan-core.ts @@ -2,6 +2,7 @@ import type { NativeOptimization } from "./optimization.js"; import { executableOptimizationLinkerArgs, + executableStaticLinkerArgs, executableStripLinkerArgs, windowsSubsystemLinkerArgs, type NativeTargetSpec, @@ -28,6 +29,8 @@ export function executableLinkInputs(options: { runtimeSystemLibraries: readonly string[]; optimization: NativeOptimization; strip?: boolean; + /** Link the C library and the other system libraries statically (`--static`). */ + staticLink?: boolean; windowsSubsystem?: WindowsSubsystem; /** Instrumented for profile-guided optimization. */ profileGenerate?: boolean; @@ -65,6 +68,7 @@ export function executableLinkInputs(options: { ), ...executableOptimizationLinkerArgs(options.target.platform, options.optimization), ...executableStripLinkerArgs(options.target.platform, options.strip ?? false), + ...executableStaticLinkerArgs(options.target, options.staticLink ?? false), ...windowsSubsystemLinkerArgs(options.target.platform, options.windowsSubsystem), ...profileGenerateLinkerArgs(options.profileGenerate ?? false), ], diff --git a/packages/compiler/src/backend/link-plan.ts b/packages/compiler/src/backend/link-plan.ts index f0be965be..73139d22a 100644 --- a/packages/compiler/src/backend/link-plan.ts +++ b/packages/compiler/src/backend/link-plan.ts @@ -28,6 +28,8 @@ export interface NativeLinkPlan { programObjectDependencies: NativeArtifactDependency[]; runtimePack: RuntimePackSelection; darwinDebugSymbols?: boolean; + /** Linked with `--static`: the linked executable is checked before use. */ + staticLink?: boolean; } export async function createNativeLinkPlan(options: { @@ -39,6 +41,7 @@ export async function createNativeLinkPlan(options: { ffi: FfiProfile | null; optimization: NativeOptimization; strip?: boolean; + staticLink?: boolean; windowsSubsystem?: WindowsSubsystem; profileGenerate?: boolean; programObjectDependencies?: readonly NativeArtifactDependency[]; @@ -70,6 +73,7 @@ export async function createNativeLinkPlan(options: { runtimeSystemLibraries: runtimePack.systemLibraries, optimization: options.optimization, ...(options.strip === undefined ? {} : { strip: options.strip }), + ...(options.staticLink === true ? { staticLink: true } : {}), ...(options.windowsSubsystem === undefined ? {} : { windowsSubsystem: options.windowsSubsystem }), @@ -78,5 +82,6 @@ export async function createNativeLinkPlan(options: { dependencyPaths: [...runtimePack.dependencyPaths, ...(options.ffi?.libraries ?? [])], programObjectDependencies: [...(options.programObjectDependencies ?? [])], runtimePack, + ...(options.staticLink === true ? { staticLink: true } : {}), }; } diff --git a/packages/compiler/src/backend/linker.ts b/packages/compiler/src/backend/linker.ts index a29a7c947..7391acae0 100644 --- a/packages/compiler/src/backend/linker.ts +++ b/packages/compiler/src/backend/linker.ts @@ -7,11 +7,13 @@ import { execFile } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; import { existsSync, realpathSync, statSync } from "node:fs"; -import { mkdtemp, rename, rm, stat } from "node:fs/promises"; +import { mkdtemp, readFile, rename, rm, stat } from "node:fs/promises"; import { delimiter } from "node:path"; import { basename, dirname, isAbsolute, join, resolve } from "node:path"; import { promisify } from "node:util"; import { CcCompileError, subprocessFailureDetail } from "./native/process.js"; +import { NativeCodegenError } from "./native-codegen-core.js"; +import { staticExecutableRefusal, staticLinkFailureNote } from "./targets.js"; import { nativeArtifactDependenciesStillMatch } from "./native/artifact-stamps.js"; import { parseLinkTraceFiles } from "./native/linker-fingerprint.js"; import { @@ -293,6 +295,10 @@ export async function linkNativeExecutable( }); const output = await stat(privateOut); if (!output.isFile() || output.size === 0) throw new Error("linker produced no executable"); + if (plan.staticLink) { + const refusal = staticExecutableRefusal(plan.target, true, await readFile(privateOut)); + if (refusal !== null) throw new NativeCodegenError("SC3002", refusal); + } if (plan.darwinDebugSymbols) { await createDarwinDebugSymbols(privateOut); await installDarwinDebugSymbols(await readDarwinDebugSymbols(privateOut), plan.outputPath); @@ -312,12 +318,18 @@ export async function linkNativeExecutable( await options.onArtifactReady({ dependencies: preLinkDependencies }).catch(() => undefined); } } catch (error) { - if (error instanceof CcCompileError || error instanceof RuntimePackError) throw error; + if ( + error instanceof CcCompileError || + error instanceof RuntimePackError || + error instanceof NativeCodegenError + ) + throw error; const detail = subprocessFailureDetail(error); throw new CcCompileError( linker, detail, - `${linker} failed linking ${basename(plan.outputPath)} from the precompiled runtime pack.\n${detail}`, + `${linker} failed linking ${basename(plan.outputPath)} from the precompiled runtime pack.\n${detail}` + + staticLinkFailureNote(plan.target, plan.staticLink === true), ); } finally { await Promise.all([ diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index b6e3f0ef7..a12c137b2 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -21,7 +21,12 @@ import { platformLinkerSupportsPersistentCache, resolvePlatformLinker, } from "./linker.js"; -import { MACOS_ARM64_TARGET, WINDOWS_X64_MSVC_TARGET, type NativeTargetSpec } from "./targets.js"; +import { + LINUX_X64_GNU_TARGET, + MACOS_ARM64_TARGET, + WINDOWS_X64_MSVC_TARGET, + type NativeTargetSpec, +} from "./targets.js"; const VERSION = compilerReleaseVersion(); @@ -242,6 +247,25 @@ describe("runtime pack manifests", () => { expect(guiPlan.driverFlags).toEqual([...defaultPlan.driverFlags, "-Wl,--subsystem,windows"]); }); + test("Linux runtime-pack links are static PIE only when requested", async () => { + const { packagePath, root } = await fixture(LINUX_X64_GNU_TARGET); + const options = { + target: LINUX_X64_GNU_TARGET, + programObject: join(root, "program.o"), + outPath: join(root, "program"), + features: BASE, + ffi: null, + optimization: "release" as const, + resolver: () => packagePath, + }; + const defaultPlan = await createNativeLinkPlan(options); + const staticPlan = await createNativeLinkPlan({ ...options, staticLink: true }); + expect(defaultPlan.driverFlags).not.toContain("-static-pie"); + expect(defaultPlan.staticLink).toBeUndefined(); + expect(staticPlan.driverFlags).toEqual([...defaultPlan.driverFlags, "-static-pie"]); + expect(staticPlan.staticLink).toBe(true); + }); + test("selection chooses the most-specific variant and feature archive", async () => { const { packagePath } = await fixture(); const resolver = () => packagePath; diff --git a/packages/compiler/src/backend/targets.test.ts b/packages/compiler/src/backend/targets.test.ts index 24d2e31a7..599f660db 100644 --- a/packages/compiler/src/backend/targets.test.ts +++ b/packages/compiler/src/backend/targets.test.ts @@ -13,9 +13,14 @@ import { WASM32_WASI_TARGET, WINDOWS_X64_MSVC_TARGET, executableOptimizationLinkerArgs, + executableStaticLinkerArgs, ffiExtendsNarrowIntegers, nativeCodegenTarget, nativeCodegenTargetRefusal, + selectNativeTarget, + staticExecutableRefusal, + staticLinkFailureNote, + staticLinkRefusal, windowsSubsystemLinkerArgs, } from "./targets.js"; @@ -232,4 +237,55 @@ describe("native code-generation targets", () => { "unknown Windows subsystem", ); }); + + test("links host glibc executables as static PIE only for --static", () => { + expect(executableStaticLinkerArgs(LINUX_X64_GNU_TARGET, false)).toEqual([]); + expect(executableStaticLinkerArgs(LINUX_X64_GNU_TARGET, true)).toEqual(["-static-pie"]); + expect(executableStaticLinkerArgs(LINUX_ARM64_GNU_TARGET, true)).toEqual(["-static-pie"]); + // musl executables are static already. + expect(executableStaticLinkerArgs(LINUX_X64_MUSL_TARGET, true)).toEqual([]); + expect(executableStaticLinkerArgs(MACOS_ARM64_TARGET, false)).toEqual([]); + expect(() => executableStaticLinkerArgs(MACOS_ARM64_TARGET, true)).toThrow( + "only supported for Linux executables", + ); + }); + + test("refuses --static where glibc cannot be linked statically", () => { + const request = { staticLink: true, outputKind: "exe", sanitize: false }; + expect(staticLinkRefusal({ ...request, staticLink: undefined, target: null })).toBeNull(); + expect(staticLinkRefusal({ ...request, target: LINUX_X64_GNU_TARGET })).toBeNull(); + expect(staticLinkRefusal({ ...request, target: LINUX_X64_MUSL_TARGET })).toBeNull(); + expect(staticLinkRefusal({ ...request, outputKind: "obj", target: LINUX_X64_GNU_TARGET })).toBe( + "--static is only supported for executable output", + ); + expect(staticLinkRefusal({ ...request, sanitize: true, target: LINUX_X64_GNU_TARGET })).toMatch( + "cannot be combined with --sanitize", + ); + for (const target of [MACOS_ARM64_TARGET, WINDOWS_X64_MSVC_TARGET, WASM32_WASI_TARGET, null]) + expect(staticLinkRefusal({ ...request, target })).toBe( + "--static is only supported for Linux executables", + ); + // zig links cross-compiled glibc targets, and links glibc dynamically. + const cross = selectNativeTarget("aarch64-linux-gnu", LINUX_X64_GNU_TARGET, "linux")!; + expect(staticLinkRefusal({ ...request, target: cross })).toMatch("-linux-musl"); + const crossMusl = selectNativeTarget("aarch64-linux-musl", LINUX_X64_GNU_TARGET, "linux")!; + expect(staticLinkRefusal({ ...request, target: crossMusl })).toBeNull(); + }); + + test("refuses static glibc executables that contain the NSS module loader", () => { + const loader = Buffer.from("\x7fELF\0libnss_%s.so%s\0"); + const plain = Buffer.from("\x7fELF\0hello\0"); + expect(staticExecutableRefusal(LINUX_X64_GNU_TARGET, true, plain)).toBeNull(); + expect(staticExecutableRefusal(LINUX_X64_GNU_TARGET, false, loader)).toBeNull(); + expect(staticExecutableRefusal(LINUX_X64_MUSL_TARGET, true, loader)).toBeNull(); + expect(staticExecutableRefusal(LINUX_X64_GNU_TARGET, true, loader)).toMatch( + /^--static cannot link this program: its executable includes host, user or group name lookups/, + ); + }); + + test("notes glibc's static libraries on failed --static links", () => { + expect(staticLinkFailureNote(LINUX_X64_GNU_TARGET, true)).toMatch("libc.a"); + expect(staticLinkFailureNote(LINUX_X64_GNU_TARGET, false)).toBe(""); + expect(staticLinkFailureNote(LINUX_X64_MUSL_TARGET, true)).toBe(""); + }); }); diff --git a/packages/compiler/src/backend/targets.ts b/packages/compiler/src/backend/targets.ts index a6713aee3..d1f0da43d 100644 --- a/packages/compiler/src/backend/targets.ts +++ b/packages/compiler/src/backend/targets.ts @@ -91,6 +91,71 @@ export function executableStripLinkerArgs(platform: string, strip: boolean): str } } +/** Why a `--static` request cannot be linked, or null when it can. A glibc + * executable links statically only through the host's clang driver: zig, + * which links cross-compiled glibc targets, links glibc dynamically. */ +export function staticLinkRefusal(options: { + staticLink: boolean | undefined; + outputKind: string; + sanitize: boolean; + target: Pick | null; +}): string | null { + if (options.staticLink !== true) return null; + if (options.outputKind !== "exe") return "--static is only supported for executable output"; + if (options.sanitize) + return "--static cannot be combined with --sanitize; AddressSanitizer requires a dynamically linked executable"; + const target = options.target; + if (target === null || target.platform !== "linux") + return "--static is only supported for Linux executables"; + if (!target.name.endsWith("-musl") && target.defaultLinker === "zig") + return "--static links glibc from the build host and is only supported for the host's own glibc target; use an -linux-musl target to cross-compile a static executable"; + return null; +} + +/** Link the C library and the other system libraries into the executable. + * glibc executables link as static PIE, so the kernel still loads them at a + * random address; musl executables are always linked statically. */ +export function executableStaticLinkerArgs( + target: Pick, + staticLink: boolean, +): string[] { + if (!staticLink) return []; + const refusal = staticLinkRefusal({ staticLink, outputKind: "exe", sanitize: false, target }); + if (refusal !== null) throw new Error(refusal); + return target.name.endsWith("-musl") ? [] : ["-static-pie"]; +} + +/** glibc's NSS loader names each module's shared library with this literal + * (`libnss_%s.so%s` in nss_module.c, glibc 2.33 and later; nsswitch.c + * before). Section garbage collection keeps it in a static executable only + * when the loader's code is live. */ +const GLIBC_NSS_LOADER_TEXT = Buffer.from("libnss_"); + +/** Why a linked `--static` executable is refused, or null. A static glibc + * executable answers host, user and group lookups with glibc's built-in + * `files` and `dns` sources, and loads every other module the run host's + * nsswitch.conf names from that host, which glibc supports only for the + * glibc version the executable was linked with. */ +export function staticExecutableRefusal( + target: Pick, + staticLink: boolean, + executable: Buffer, +): string | null { + if (!staticLink || target.name.endsWith("-musl")) return null; + if (executable.indexOf(GLIBC_NSS_LOADER_TEXT) === -1) return null; + return "--static cannot link this program: its executable includes host, user or group name lookups (which dns.lookup, net and http connections, fetch and os.userInfo use), and a statically linked glibc executable loads the NSS modules for those lookups from the run host, which glibc supports only for the glibc version it was linked with. Build without --static, or use an -linux-musl target"; +} + +/** The note a failed `--static` glibc link adds to the linker's error. */ +export function staticLinkFailureNote( + target: Pick, + staticLink: boolean, +): string { + return staticLink && !target.name.endsWith("-musl") + ? "\n--static links glibc's static libraries (libc.a, libm.a), which the libc6-dev package installs on Debian and Ubuntu and glibc-static on Fedora" + : ""; +} + export interface NativeTargetSpec { /** Stable scriptc-facing identity used in cache keys and diagnostics. */ name: NativeTargetName; diff --git a/packages/compiler/src/cli/command.ts b/packages/compiler/src/cli/command.ts index eb72ca14d..fcaaf1d02 100644 --- a/packages/compiler/src/cli/command.ts +++ b/packages/compiler/src/cli/command.ts @@ -115,6 +115,7 @@ async function main(args: string[], host: CliHost): Promise { values.ffi !== undefined || values.profile !== undefined || values.strip || + values.static || values["windows-subsystem"] !== undefined || values["detached-exit"] || values["large-pages"] || @@ -176,6 +177,7 @@ async function main(args: string[], host: CliHost): Promise { values.ffi !== undefined || values.profile !== undefined || values.strip || + values.static || values.sanitize || values["windows-subsystem"] !== undefined || values["detached-exit"] || @@ -269,6 +271,7 @@ async function main(args: string[], host: CliHost): Promise { values["fail-on"] !== undefined || values.optimization !== undefined || values.strip || + values.static || values.ffi !== undefined || values["windows-subsystem"] !== undefined || values["detached-exit"] || @@ -284,7 +287,7 @@ async function main(args: string[], host: CliHost): Promise { hasNodeIslandFlags(values) ) { fail( - "scriptc build --lib takes no --backend/--emit/--print/--optimization/--strip/--windows-subsystem/--detached-exit/--large-pages/--target-cpu/--cpu-dispatch/--profile-generate/--profile-use/--npm-static/--ffi/--external-types/--tier/--explain-dynamic: the profile pins the emission and optimization, npm imports are judged automatically, outbound FFI belongs to executable builds, external type mappings belong to coverage, and library artifacts are static-tier only", + "scriptc build --lib takes no --backend/--emit/--print/--optimization/--strip/--static/--windows-subsystem/--detached-exit/--large-pages/--target-cpu/--cpu-dispatch/--profile-generate/--profile-use/--npm-static/--ffi/--external-types/--tier/--explain-dynamic: the profile pins the emission and optimization, npm imports are judged automatically, outbound FFI belongs to executable builds, external type mappings belong to coverage, and library artifacts are static-tier only", ); } const profilePath = resolve(profileArg); @@ -327,6 +330,9 @@ async function main(args: string[], host: CliHost): Promise { if (command === "coverage" && values.strip) { fail(`--strip is a build/run option\n\n${USAGE}`); } + if (command === "coverage" && values.static) { + fail(`--static is a build/run option\n\n${USAGE}`); + } if (command === "coverage" && values["target-cpu"] !== undefined) { fail(`--target-cpu is a build/run option\n\n${USAGE}`); } @@ -515,6 +521,7 @@ async function main(args: string[], host: CliHost): Promise { sanitize: values.sanitize, ...(values.optimization === undefined ? {} : { optimization: values.optimization }), strip: values.strip, + staticLink: values.static, detachedExit: values["detached-exit"], largePages: values["large-pages"], ...(values["cpu-dispatch"] === undefined ? {} : { cpuDispatch: values["cpu-dispatch"] }), @@ -635,6 +642,7 @@ async function main(args: string[], host: CliHost): Promise { ...(backend !== undefined ? { backend } : {}), ...(optimization !== undefined ? { optimization } : {}), ...(values.strip ? { strip: true } : {}), + ...(values.static ? { staticLink: true } : {}), ...(windowsSubsystem !== undefined ? { windowsSubsystem } : {}), ...(values["detached-exit"] ? { detachedExit: true } : {}), ...(values["large-pages"] ? { largePages: true } : {}), diff --git a/packages/compiler/src/cli/output-options.ts b/packages/compiler/src/cli/output-options.ts index d684c884f..207878692 100644 --- a/packages/compiler/src/cli/output-options.ts +++ b/packages/compiler/src/cli/output-options.ts @@ -9,6 +9,7 @@ export interface OutputOptionValues { sanitize: boolean; optimization?: string; strip?: boolean; + staticLink?: boolean; windowsSubsystem?: string; detachedExit?: boolean; largePages?: boolean; @@ -74,6 +75,9 @@ export function resolveOutputOptions( if (values.strip && emit !== "exe") { return { ok: false, message: `--strip is only supported with --emit=exe` }; } + if (values.staticLink && emit !== "exe") { + return { ok: false, message: `--static is only supported with --emit=exe` }; + } if (values.detachedExit && emit !== "exe") { return { ok: false, message: `--detached-exit is only supported with --emit=exe` }; } diff --git a/packages/compiler/src/cli/usage.ts b/packages/compiler/src/cli/usage.ts index 6d01502b5..9f2b62d51 100644 --- a/packages/compiler/src/cli/usage.ts +++ b/packages/compiler/src/cli/usage.ts @@ -56,6 +56,11 @@ Options: counting, cache-line-aligned runtime code on x86-64) --strip remove symbol/debug payload from the linked executable for smaller builds (opt in; --emit=exe only) + --static Linux: link the C library into the executable, which + then starts without the dynamic loader. glibc builds + need its static libraries (libc.a) on the build host + and refuse executables that include host, user or + group name lookups (opt in; --emit=exe only) --windows-subsystem Windows executable subsystem (default: console). gui prevents Windows from opening a console window @@ -176,6 +181,7 @@ export const CLI_OPTIONS = { backend: { type: "string" }, optimization: { type: "string" }, strip: { type: "boolean", default: false }, + static: { type: "boolean", default: false }, "windows-subsystem": { type: "string" }, "detached-exit": { type: "boolean", default: false }, "large-pages": { type: "boolean", default: false }, diff --git a/packages/compiler/src/compile-types.ts b/packages/compiler/src/compile-types.ts index 2ef7a703a..6ea532b67 100644 --- a/packages/compiler/src/compile-types.ts +++ b/packages/compiler/src/compile-types.ts @@ -31,6 +31,11 @@ export interface CompileBaseOptions { optimization?: NativeOptimization; /** Remove symbol/debug payload from an executable at link time. */ strip?: boolean; + /** Link the C library and the other system libraries into the executable + * (`--static`). glibc executables link as static PIE and need the C + * library's static archives on the build host; musl executables are always + * static. Linux executables only. */ + staticLink?: boolean; /** Windows PE executable subsystem. Console is the default; GUI suppresses * automatic console-window creation. Only valid for Windows executables. */ windowsSubsystem?: WindowsSubsystem; diff --git a/packages/compiler/src/executable/executable-cache.ts b/packages/compiler/src/executable/executable-cache.ts index 6124a6edf..5e60d7443 100644 --- a/packages/compiler/src/executable/executable-cache.ts +++ b/packages/compiler/src/executable/executable-cache.ts @@ -148,6 +148,8 @@ export interface EarlyExecutableCacheOptions { optimization?: "dev" | "speed"; /** Omitted retains the unstripped executable's historical cache key. */ strip?: true; + /** Omitted unless --static is set. */ + staticLink?: true; /** Omitted for the default Windows console subsystem. */ windowsSubsystem?: "gui"; /** Omitted unless --detached-exit is set. */ @@ -315,6 +317,7 @@ function cacheKey(options: EarlyExecutableCacheOptions): string { options.backend, ...optimizationKeyParts(options.optimization), ...(options.strip ? ["strip"] : []), + ...(options.staticLink ? ["static-link"] : []), ...(options.windowsSubsystem === "gui" ? ["windows-subsystem-gui"] : []), ...(options.detachedExit ? ["detached-exit"] : []), ...(options.largePages ? ["large-pages"] : []), @@ -361,6 +364,7 @@ function routeKey(options: Omit = null, programObjectDependencies: readonly NativeArtifactDependency[] = [], onArtifactReady?: NonNullable[0]["onArtifactReady"]>, @@ -625,6 +630,7 @@ async function compileExecutableNative( ffi, optimization: features.optimization ?? "release", ...(strip ? { strip: true } : {}), + ...(staticLink ? { staticLink: true } : {}), ...(windowsSubsystem === undefined ? {} : { windowsSubsystem }), ...(profileGenerate ? { profileGenerate: true } : {}), programObjectDependencies, @@ -650,6 +656,8 @@ async function compileExecutableNative( : null); if (programIsObject) throw new InternalCompilerError("program objects must link against a precompiled runtime pack"); + if (staticLink) + throw new InternalCompilerError("--static executables link against a precompiled runtime pack"); await compileExternalC({ ...(llvmVisiblePath === undefined || llvmVisiblePath === llvmPath ? { cPath: llvmPath } @@ -1631,6 +1639,19 @@ async function compileTracked( sourceTexts: new Map(), }; } + const staticRefusal = staticLinkRefusal({ + staticLink: opts.staticLink, + outputKind, + sanitize: opts.sanitize ?? false, + target: nativeCodegenTarget(), + }); + if (staticRefusal !== null) { + return { + ok: false, + diagnostics: [nativeCodegenDiag("SC3002", staticRefusal, entryPath)], + sourceTexts: new Map(), + }; + } try { windowsSubsystemLinkerArgs(buildPlatform, opts.windowsSubsystem); } catch (err) { @@ -1685,6 +1706,7 @@ async function compileTracked( backend: "llvm", ...optimizationField(opts.optimization), ...(opts.strip ? { strip: true as const } : {}), + ...(opts.staticLink === true ? { staticLink: true as const } : {}), ...(opts.windowsSubsystem === "gui" ? { windowsSubsystem: "gui" as const } : {}), ...(opts.detachedExit === true ? { detachedExit: true as const } : {}), ...(opts.largePages === true ? { largePages: true as const } : {}), @@ -1831,6 +1853,7 @@ async function compileTracked( ffi, opts.windowsSubsystem, opts.strip, + opts.staticLink === true, null, nativeProgramObject?.dependencies, opts.nativeProgramObject === true @@ -1872,6 +1895,13 @@ async function compileTracked( sourceTexts: new Map(), }; } + if (err instanceof NativeCodegenError) { + return { + ok: false, + diagnostics: [nativeCodegenDiag(err.diagnosticCode, err.message, entryPath)], + sourceTexts: new Map(), + }; + } throw err; } finally { if (nativeProgramObject !== null) { @@ -1970,6 +2000,7 @@ async function compileTracked( ffi, opts.windowsSubsystem, opts.strip, + opts.staticLink === true, programSplit, nativeProgramObject?.dependencies, opts.nativeProgramObject === true @@ -2019,6 +2050,13 @@ async function compileTracked( sourceTexts, }; } + if (err instanceof NativeCodegenError) { + return { + ok: false, + diagnostics: [nativeCodegenDiag(err.diagnosticCode, err.message, entryPath)], + sourceTexts, + }; + } throw err; } finally { if (nativeProgramObject !== null) { diff --git a/packages/compiler/src/native/compiler.ts b/packages/compiler/src/native/compiler.ts index 1cf4f7eac..22cee0ad5 100644 --- a/packages/compiler/src/native/compiler.ts +++ b/packages/compiler/src/native/compiler.ts @@ -75,6 +75,11 @@ import { inputsWithoutLld, lldLinkerArgs } from "../backend/lld.js"; import { openNativeExecutableCache } from "./executable-cache.js"; import { LinkerJobs } from "./linker-job.js"; import { nativeFileIdentity } from "./file-identity.js"; +import { + staticExecutableRefusal, + staticLinkFailureNote, + staticLinkRefusal, +} from "../backend/targets.js"; /** The first partition keeps the whole-object key used before partitioning. */ function partitionCacheKey(key: string, index: number): string { @@ -404,6 +409,13 @@ export class NativeCompiler { platform: target.platform, }); if (profileRefusal !== null) throw new NativeCodegenError("SC3002", profileRefusal); + const staticRefusal = staticLinkRefusal({ + staticLink: options.staticLink, + outputKind, + sanitize: options.sanitize ?? false, + target, + }); + if (staticRefusal !== null) throw new NativeCodegenError("SC3002", staticRefusal); const profile = nativeProfile(options); let ffi: FfiProfile | null = null; if (options.ffiProfilePath !== undefined) { @@ -571,6 +583,7 @@ export class NativeCompiler { runtimeSystemLibraries: runtime.systemLibraries, optimization, strip: options.strip ?? false, + staticLink: options.staticLink ?? false, ...(options.windowsSubsystem === undefined ? {} : { windowsSubsystem: options.windowsSubsystem }), @@ -598,16 +611,26 @@ export class NativeCompiler { if (!cacheable && (job?.run(linkArgs, true) ?? null) === null) { const driver = options.sanitize ? sanitizerDriver(toolchain) : toolchain.linker; try { - runNativeTool(driver, linkArgs); + try { + runNativeTool(driver, linkArgs); + } catch (error) { + // A failed link with lld runs again with the driver's default + // linker, whose error (if any) is the one reported. + if (lld.length === 0) throw error; + runNativeTool(driver, linkArgsWith(false)); + } } catch (error) { - // A failed link with lld runs again with the driver's default - // linker, whose error (if any) is the one reported. - if (lld.length === 0) throw error; - runNativeTool(driver, linkArgsWith(false)); + const note = staticLinkFailureNote(target, options.staticLink === true); + if (note === "") throw error; + throw new Error(`${error instanceof Error ? error.message : String(error)}${note}`); } } timing("native-link"); requireNativeArtifact(stagedOutput); + if (options.staticLink) { + const refusal = staticExecutableRefusal(target, true, readFileSync(stagedOutput)); + if (refusal !== null) throw new NativeCodegenError("SC3002", refusal); + } if (target.platform === "darwin" && optimization === "dev" && !options.strip) { runNativeTool(toolchain.dsymutil, [stagedOutput, "-o", stagedOutput + ".dSYM"]); } From 8cdf7830237b929bd2a2d85260513745b709699f Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 04:00:21 -0500 Subject: [PATCH 09/54] Find glibc's NSS loader in the --static link map The NSS check scanned the linked executable for the `libnss_` literal, so a program with that text in a string of its own was refused. A --static glibc link now writes a link map, and the executable is refused when the map places code from libc.a's nss_module.o or nsswitch.o in it (lld's kept input sections, or GNU ld's memory map with a nonzero size). --- packages/cli/test/static-link.test.ts | 10 +++- packages/compiler/src/backend/linker.ts | 9 ++- packages/compiler/src/backend/targets.test.ts | 58 ++++++++++++++++--- packages/compiler/src/backend/targets.ts | 41 ++++++++++--- packages/compiler/src/native/compiler.ts | 13 ++++- 5 files changed, 107 insertions(+), 24 deletions(-) diff --git a/packages/cli/test/static-link.test.ts b/packages/cli/test/static-link.test.ts index 5d64e9d01..68fbda064 100644 --- a/packages/cli/test/static-link.test.ts +++ b/packages/cli/test/static-link.test.ts @@ -68,14 +68,20 @@ test.skipIf(!glibcHost || !hostHasStaticLibc())( await withProject(async (dir, env) => { const entry = join(dir, "main.ts"); const outPath = join(dir, "program"); - await writeFile(entry, 'console.log("static-ok", [1, 2, 3].map((n) => n * 2).join(","));\n'); + // The NSS module file name in a string of the program's own is no lookup. + await writeFile( + entry, + 'console.log("static-ok", [1, 2, 3].map((n) => n * 2).join(","), "/lib/libnss_files.so.2");\n', + ); const build = async (staticLink: boolean, bootstrap = false) => { await cli( ["build", entry, "-o", outPath, ...(staticLink ? ["--static"] : [])], env, bootstrap, ); - expect((await execFileAsync(outPath)).stdout).toBe("static-ok 2,4,6\n"); + expect((await execFileAsync(outPath)).stdout).toBe( + "static-ok 2,4,6 /lib/libnss_files.so.2\n", + ); return elfLinkage(outPath); }; // ET_DYN with a program interpreter: the default dynamic PIE link. diff --git a/packages/compiler/src/backend/linker.ts b/packages/compiler/src/backend/linker.ts index 7391acae0..fe476360d 100644 --- a/packages/compiler/src/backend/linker.ts +++ b/packages/compiler/src/backend/linker.ts @@ -13,7 +13,7 @@ import { basename, dirname, isAbsolute, join, resolve } from "node:path"; import { promisify } from "node:util"; import { CcCompileError, subprocessFailureDetail } from "./native/process.js"; import { NativeCodegenError } from "./native-codegen-core.js"; -import { staticExecutableRefusal, staticLinkFailureNote } from "./targets.js"; +import { staticExecutableRefusal, staticLinkFailureNote, staticLinkMapArgs } from "./targets.js"; import { nativeArtifactDependenciesStillMatch } from "./native/artifact-stamps.js"; import { parseLinkTraceFiles } from "./native/linker-fingerprint.js"; import { @@ -240,6 +240,8 @@ export async function linkNativeExecutable( join(dirname(plan.outputPath), ".scriptc-runtime-pack-link-"), ); const privateOut = join(privateOutRoot, basename(plan.outputPath)); + const linkMap = join(privateOutRoot, "link.map"); + const mapArgs = staticLinkMapArgs(plan.target, plan.staticLink === true, linkMap); let stagedRoot: string | null = null; try { const staged = await stageRuntimePackArtifacts(plan.runtimePack); @@ -252,6 +254,7 @@ export async function linkNativeExecutable( ...(explicitLinker === undefined ? plan.target.defaultLinkerArgs : []), ...(withLld ? lld : []), ...plan.driverFlags, + ...mapArgs, ...(withLld ? inputs : inputsWithoutLld(plan.target.platform, inputs)), ...systemLibraryLinkerArgs(plan.target, plan.systemLibraries), "-o", @@ -295,8 +298,8 @@ export async function linkNativeExecutable( }); const output = await stat(privateOut); if (!output.isFile() || output.size === 0) throw new Error("linker produced no executable"); - if (plan.staticLink) { - const refusal = staticExecutableRefusal(plan.target, true, await readFile(privateOut)); + if (mapArgs.length !== 0) { + const refusal = staticExecutableRefusal(plan.target, true, await readFile(linkMap, "utf8")); if (refusal !== null) throw new NativeCodegenError("SC3002", refusal); } if (plan.darwinDebugSymbols) { diff --git a/packages/compiler/src/backend/targets.test.ts b/packages/compiler/src/backend/targets.test.ts index 599f660db..4fb275140 100644 --- a/packages/compiler/src/backend/targets.test.ts +++ b/packages/compiler/src/backend/targets.test.ts @@ -20,6 +20,7 @@ import { selectNativeTarget, staticExecutableRefusal, staticLinkFailureNote, + staticLinkMapArgs, staticLinkRefusal, windowsSubsystemLinkerArgs, } from "./targets.js"; @@ -272,15 +273,56 @@ describe("native code-generation targets", () => { expect(staticLinkRefusal({ ...request, target: crossMusl })).toBeNull(); }); - test("refuses static glibc executables that contain the NSS module loader", () => { - const loader = Buffer.from("\x7fELF\0libnss_%s.so%s\0"); - const plain = Buffer.from("\x7fELF\0hello\0"); - expect(staticExecutableRefusal(LINUX_X64_GNU_TARGET, true, plain)).toBeNull(); - expect(staticExecutableRefusal(LINUX_X64_GNU_TARGET, false, loader)).toBeNull(); - expect(staticExecutableRefusal(LINUX_X64_MUSL_TARGET, true, loader)).toBeNull(); - expect(staticExecutableRefusal(LINUX_X64_GNU_TARGET, true, loader)).toMatch( - /^--static cannot link this program: its executable includes host, user or group name lookups/, + test("refuses static glibc executables whose link map keeps the NSS loader", () => { + const libc = "/lib/x86_64-linux-gnu/libc.a"; + // lld lists the input sections it kept. + const lld = (member: string) => + [ + " VMA LMA Size Align Out In Symbol", + ` 11e1a0 11e1a0 640 16 ${libc}(${member}):(.text)`, + ` 11e7e0 11e7e0 38 1 ${libc}(${member}):(.rodata)`, + ].join("\n"); + const lldPlain = [ + " VMA LMA Size Align Out In Symbol", + ` 2c62c0 2c62c0 a10 16 ${libc}(dl-open.o):(.text)`, + ].join("\n"); + // GNU ld lists discarded sections first, and sections relaxation emptied with size 0. + const bfd = (text: boolean) => + [ + "Discarded input sections", + "", + ` .text 0x0000000000000000 0x640 ${libc}(nss_module.o)`, + "", + "Linker script and memory map", + "", + ` .eh_frame 0x00000000000db750 0x0 ${libc}(nss_module.o)`, + " 0x1b8 (size before relaxing)", + ...(text ? [` .text 0x00000000000a81b0 0x640 ${libc}(nsswitch.o)`] : []), + ].join("\n"); + const refusal = + /^--static cannot link this program: its executable includes host, user or group name lookups/; + expect(staticExecutableRefusal(LINUX_X64_GNU_TARGET, true, lld("nss_module.o"))).toMatch( + refusal, ); + expect(staticExecutableRefusal(LINUX_X64_GNU_TARGET, true, lld("nsswitch.o"))).toMatch(refusal); + expect(staticExecutableRefusal(LINUX_X64_GNU_TARGET, true, bfd(true))).toMatch(refusal); + expect(staticExecutableRefusal(LINUX_X64_GNU_TARGET, true, bfd(false))).toBeNull(); + expect(staticExecutableRefusal(LINUX_X64_GNU_TARGET, true, lldPlain)).toBeNull(); + // A program's own string never counts. + expect(staticExecutableRefusal(LINUX_X64_GNU_TARGET, true, "libnss_files.so.2")).toBeNull(); + expect(staticExecutableRefusal(LINUX_X64_GNU_TARGET, false, lld("nss_module.o"))).toBeNull(); + expect(staticExecutableRefusal(LINUX_X64_MUSL_TARGET, true, lld("nss_module.o"))).toBeNull(); + }); + + test("writes a link map only for --static glibc links", () => { + expect(staticLinkMapArgs(LINUX_X64_GNU_TARGET, true, "/stage/link.map")).toEqual([ + "-Xlinker", + "-Map", + "-Xlinker", + "/stage/link.map", + ]); + expect(staticLinkMapArgs(LINUX_X64_GNU_TARGET, false, "/stage/link.map")).toEqual([]); + expect(staticLinkMapArgs(LINUX_X64_MUSL_TARGET, true, "/stage/link.map")).toEqual([]); }); test("notes glibc's static libraries on failed --static links", () => { diff --git a/packages/compiler/src/backend/targets.ts b/packages/compiler/src/backend/targets.ts index d1f0da43d..e4ce3ab5f 100644 --- a/packages/compiler/src/backend/targets.ts +++ b/packages/compiler/src/backend/targets.ts @@ -125,24 +125,49 @@ export function executableStaticLinkerArgs( return target.name.endsWith("-musl") ? [] : ["-static-pie"]; } -/** glibc's NSS loader names each module's shared library with this literal - * (`libnss_%s.so%s` in nss_module.c, glibc 2.33 and later; nsswitch.c - * before). Section garbage collection keeps it in a static executable only - * when the loader's code is live. */ -const GLIBC_NSS_LOADER_TEXT = Buffer.from("libnss_"); +/** The link map a `--static` glibc link writes to `path`, which + * staticExecutableRefusal reads. `-Xlinker` keeps the path one argument + * through the driver and into the linker's job. */ +export function staticLinkMapArgs( + target: Pick, + staticLink: boolean, + path: string, +): string[] { + return staticLink && !target.name.endsWith("-musl") ? ["-Xlinker", "-Map", "-Xlinker", path] : []; +} + +/** The libc.a members of glibc's NSS loader: nss_module.o (glibc 2.33 and + * later) and nsswitch.o. */ +const GLIBC_NSS_LOADER_MEMBER = /libc\.a\((?:nss_module|nsswitch)\.o\)/; + +/** Whether a link map line places code from an archive member in the + * output: lld lists the input sections it kept (`… libc.a(m.o):(.text)`), + * GNU ld each input section with its size (` .text 0x… 0x… …libc.a(m.o)`), + * and lists sections that relaxation emptied with size 0. */ +function keepsText(line: string): boolean { + return /\):\(\.text\)\s*$/.test(line) || /^\s*\.text\s+0x[0-9a-f]+\s+0x0*[1-9a-f]/.test(line); +} /** Why a linked `--static` executable is refused, or null. A static glibc * executable answers host, user and group lookups with glibc's built-in * `files` and `dns` sources, and loads every other module the run host's * nsswitch.conf names from that host, which glibc supports only for the - * glibc version the executable was linked with. */ + * glibc version the executable was linked with. `linkMap` is the link's + * map (staticLinkMapArgs): the executable has the NSS loader when the map + * places the loader's code in it. GNU ld lists the input sections that + * garbage collection discarded before its memory map. */ export function staticExecutableRefusal( target: Pick, staticLink: boolean, - executable: Buffer, + linkMap: string, ): string | null { if (!staticLink || target.name.endsWith("-musl")) return null; - if (executable.indexOf(GLIBC_NSS_LOADER_TEXT) === -1) return null; + const memoryMap = linkMap.indexOf("Linker script and memory map"); + const kept = memoryMap === -1 ? linkMap : linkMap.slice(memoryMap); + const loader = kept + .split("\n") + .some((line) => GLIBC_NSS_LOADER_MEMBER.test(line) && keepsText(line)); + if (!loader) return null; return "--static cannot link this program: its executable includes host, user or group name lookups (which dns.lookup, net and http connections, fetch and os.userInfo use), and a statically linked glibc executable loads the NSS modules for those lookups from the run host, which glibc supports only for the glibc version it was linked with. Build without --static, or use an -linux-musl target"; } diff --git a/packages/compiler/src/native/compiler.ts b/packages/compiler/src/native/compiler.ts index 22cee0ad5..efe4fc5c0 100644 --- a/packages/compiler/src/native/compiler.ts +++ b/packages/compiler/src/native/compiler.ts @@ -78,6 +78,7 @@ import { nativeFileIdentity } from "./file-identity.js"; import { staticExecutableRefusal, staticLinkFailureNote, + staticLinkMapArgs, staticLinkRefusal, } from "../backend/targets.js"; @@ -591,10 +592,13 @@ export class NativeCompiler { }); // Linux links with lld when the driver finds it. const lld = options.sanitize ? [] : lldLinkerArgs(target.platform, toolchain.linker); + const linkMap = join(stage, "link.map"); + const mapArgs = staticLinkMapArgs(target, options.staticLink === true, linkMap); const linkArgsWith = (withLld: boolean): string[] => [ ...(options.sanitize ? ["-fsanitize=address"] : toolchain.linkerArgs), ...(withLld ? lld : []), ...plan.driverFlags, + ...mapArgs, ...(withLld ? plan.inputs : inputsWithoutLld(target.platform, plan.inputs)), ...systemLibraryLinkerArgs(target, plan.systemLibraries), "-o", @@ -603,7 +607,10 @@ export class NativeCompiler { const linkArgs = linkArgsWith(lld.length !== 0); // Host Linux links with lld run the linker job the driver would. const jobs = LinkerJobs.open(this.cache, toolchain, lld, options.sanitize ?? false); - const job = jobs?.link([...plan.inputs, stagedOutput], stage); + const job = jobs?.link( + [...plan.inputs, stagedOutput, ...(mapArgs.length === 0 ? [] : [linkMap])], + stage, + ); const cacheable = executableCache?.trace(linkArgs, stage, job) ?? false; timing("link-inputs"); // A successful dependency trace already produced this executable. @@ -627,8 +634,8 @@ export class NativeCompiler { } timing("native-link"); requireNativeArtifact(stagedOutput); - if (options.staticLink) { - const refusal = staticExecutableRefusal(target, true, readFileSync(stagedOutput)); + if (mapArgs.length !== 0) { + const refusal = staticExecutableRefusal(target, true, readFileSync(linkMap, "utf8")); if (refusal !== null) throw new NativeCodegenError("SC3002", refusal); } if (target.platform === "darwin" && optimization === "dev" && !options.strip) { From f65c5518bf808b3ecea69a8eed2781530b41c09e Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 06:43:19 -0500 Subject: [PATCH 10/54] Record each reachable-emit unit's lowering as plain data, checked by a serial mode A unit's record holds its IR in the compact encoding, the entries it added to the helper, lifted-function and other appendable lowerer tables, and the reachability, generic demand, statement and flag events it made. A unit has no record when it grew a table no record carries, changed a field, replaced an entry it did not add, or emitted a diagnostic. Every lowerer collection and scalar field is classified, and a test fails on one that is not. SCRIPTC_LOWER_JOURNAL=serial records every unit and replaces its results with the ones its record carries. scripts/recorded-lowering.mjs (pnpm test:recorded-lowering) compares corpus programs' LLVM text with and without the mode. SCRIPTC_LOWER_JOURNAL_DUMP= also turns the mode on and appends each unit's record to /records.jsonl. Without either, lowering is unchanged. --- package.json | 1 + .../lowering/deferred-module-initializers.ts | 7 + .../frontend/lowering/key-order-witness.ts | 11 + .../compiler/src/frontend/lowering/lowerer.ts | 686 +++++++++++++++++- .../frontend/lowering/unit-journal.test.ts | 282 +++++++ .../src/frontend/lowering/unit-journal.ts | 474 ++++++++++++ packages/compiler/src/frontend/type-mapper.ts | 41 ++ packages/compiler/src/ir/serialize.ts | 11 +- .../compiler/test/recorded-lowering.test.ts | 70 ++ scripts/recorded-lowering.mjs | 145 ++++ 10 files changed, 1718 insertions(+), 10 deletions(-) create mode 100644 packages/compiler/src/frontend/lowering/unit-journal.test.ts create mode 100644 packages/compiler/src/frontend/lowering/unit-journal.ts create mode 100644 packages/compiler/test/recorded-lowering.test.ts create mode 100644 scripts/recorded-lowering.mjs diff --git a/package.json b/package.json index 9b0213b44..e54ee164f 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "test:test262:fetch": "node tests/test262/fetch.mjs", "test:cache-identity": "node tests/harness/cache-identity.mjs", "test:lowering-order": "node scripts/lowering-order.mjs --fail-on names", + "test:recorded-lowering": "node scripts/recorded-lowering.mjs", "test:sandbox": "node scripts/sandbox-test.mjs", "test:sandbox:image": "node scripts/sandbox-image.mjs", "test:watch": "vitest", diff --git a/packages/compiler/src/frontend/lowering/deferred-module-initializers.ts b/packages/compiler/src/frontend/lowering/deferred-module-initializers.ts index 3af054dd6..4d1674127 100644 --- a/packages/compiler/src/frontend/lowering/deferred-module-initializers.ts +++ b/packages/compiler/src/frontend/lowering/deferred-module-initializers.ts @@ -26,6 +26,13 @@ export class DeferredModuleInitializers { /** The last scan of each function (scanOf). */ private readonly scans = new Map(); + /** The sizes of the deferral tables (unit-journal.ts). */ + journalSizes(): number[] { + let initializers = 0; + for (const list of this.pending.values()) initializers += list.length; + return [this.pending.size, initializers, this.registryGlobals.size, this.deferredKeys.size]; + } + defer(lowerer: Lowerer, statement: ts.Statement): IrStmt | null { let key: string; if ( diff --git a/packages/compiler/src/frontend/lowering/key-order-witness.ts b/packages/compiler/src/frontend/lowering/key-order-witness.ts index a5e635e04..8da0f2edf 100644 --- a/packages/compiler/src/frontend/lowering/key-order-witness.ts +++ b/packages/compiler/src/frontend/lowering/key-order-witness.ts @@ -140,6 +140,17 @@ export class KeyOrderNotes { /** Shapes whose table this pass replaced. */ readonly changed = new Set(); + /** The sizes of the notes' tables (unit-journal.ts). */ + journalSizes(): number[] { + return [ + this.literals.size, + this.copies.size, + this.methodResults.size, + this.observed.size, + this.changed.size, + ]; + } + literal(node: IrExpr, segments: KeyOrderSegment[]): IrExpr { this.literals.set(node, segments); return node; diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 0c63997df..2b40f64ef 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -20,6 +20,26 @@ import { } from "./decl-origin.js"; import { isDeclBoundaryFile } from "../decl-projection.js"; import { relieveHeapPressure } from "../../heap-pressure.js"; +import { + checkedTable, + countTable, + fixedScalar, + flagScalar, + functionTable, + irDataTable, + journalDump, + journalMode, + lowerWaveRecorded, + mapTable, + setTable, + waveStats, + writeRecords, + type JournalEvent, + type JournalScalar, + type JournalTable, + type UnitEvents, + type WaveHost, +} from "./unit-journal.js"; import { moduleNamespaceObject, moduleNamespaceValue } from "./module-namespace-elements.js"; import { keyedHelperName, moduleTagNumber, portableRelativePath } from "./stable-names.js"; import { positionTag, StructuralPositions } from "./structural-positions.js"; @@ -2213,6 +2233,7 @@ export class Lowerer { } noteGenericInstanceDemand(inst: GenericInstance): void { + this.unitEvents?.record("functionDemand", inst.name); const owner = this.genericDemandOwner; owner?.functionDemands.push(inst); const ref = this.genericDemandPriority.get(inst) ?? { rank: [4, Number.MAX_SAFE_INTEGER] }; @@ -2228,6 +2249,7 @@ export class Lowerer { } noteGenericClassInstanceDemand(info: ClassInfo): void { + this.unitEvents?.record("classDemand", info.def.name); const owner = this.genericDemandOwner; owner?.classDemands.push(info); const ref = this.genericClassDemandPriority.get(info) ?? { rank: [4, Number.MAX_SAFE_INTEGER] }; @@ -3766,6 +3788,9 @@ export class Lowerer { * promise operand (returnedPromiseAwait). */ private returnAwaitedPromise = false; private readonly virtualEdges = new Map>(); + /** The events of the unit being recorded (unit-journal.ts); null + * otherwise. */ + unitEvents: UnitEvents | null = null; classCanBeConstructed(info: ClassInfo): boolean { return ( @@ -3903,6 +3928,517 @@ export class Lowerer { } return name === undefined ? undefined : this.liftedFnsByName.get(name); } + + /** The tables lowering a unit can grow (unit-journal.ts): the + * appendable tables whose entries a unit's record carries, then the + * tables whose growth leaves the unit without a record. + * Lowerer.unjournaled lists the collections that are in neither. */ + journalTables(): JournalTable[] { + const sizes = (name: string, of: () => readonly number[]): JournalTable => + checkedTable(name, () => { + let hash = 0; + for (const size of of()) hash = (Math.imul(hash, 31) + size) | 0; + return hash; + }); + return [ + functionTable("implicitFns", this.implicitFns, (fn) => + this.implicitFns.some((other) => other.name === fn), + ), + mapTable("equalityHelpers", this.equalityHelpers), + mapTable("arrHofHelpers", this.arrHofHelpers), + irDataTable("arrHofInfo", this.arrHofInfo), + mapTable("unionCallHelpers", this.unionCallHelpers), + mapTable("indexedUnionNumericTwins", this.indexedUnionNumericTwins), + setTable("coercions.checkedNarrows", this.coercions.checkedNarrows), + irDataTable("coercions.checkedNarrowArms", this.coercions.checkedNarrowArms), + mapTable("coercions.retags", this.coercions.retags), + mapTable("coercions.narrows", this.coercions.narrows), + mapTable("coercions.widthNarrows", this.coercions.widthNarrows), + setTable("coercions.widenedReturns", this.coercions.widenedReturns), + mapTable("valueHelpers", this.valueHelpers), + countTable("keyedNameKeys", this.keyedNameKeys), + mapTable("assertHelpers", this.assertHelpers), + mapTable("inspectHelpers", this.inspectHelpers), + setTable("classMethodValueHelpers", this.classMethodValueHelpers), + setTable("freshClosureAdapters", this.freshClosureAdapters), + mapTable("iterDrainHelpers", this.iterDrainHelpers), + mapTable("mapHofHelpers", this.mapHofHelpers), + mapTable("setHofHelpers", this.setHofHelpers), + mapTable("spHofHelpers", this.spHofHelpers), + mapTable("primitiveCtorFns", this.primitiveCtorFns), + mapTable("builtinCallableValueFns", this.builtinCallableValueFns), + mapTable("builtinCallableValueNames", this.builtinCallableValueNames), + countTable("builtinCallableValueLengths", this.builtinCallableValueLengths), + mapTable("regionRunHelpers", this.regionRunHelpers), + mapTable("internTableHelpers", this.internTableHelpers), + functionTable( + "liftedFns", + this.liftedFns, + (fn) => this.liftedFnNamed(fn) !== undefined, + () => { + this.liftedFnsByName.clear(); + this.liftedFnsIndexed = 0; + }, + ), + countTable("closureNameCounts", this.closureNameCounts), + setTable("sharedHelperNames", this.sharedHelperNames), + setTable("sloppyRecordWrites", this.sloppyRecordWrites), + setTable("recordFieldDeletes", this.recordFieldDeletes), + setTable("freezableArrays", this.freezableArrays), + checkedTable("moduleOrder", () => this.moduleOrder.length), + checkedTable("diags", () => this.diags.length), + checkedTable("fnSigsBySymbol", () => this.fnSigsBySymbol.size), + checkedTable("cjsExportsAssignments", () => this.cjsExportsAssignments.size), + checkedTable("inlineRequireCycleCalls", () => this.inlineRequireCycleCalls.size), + checkedTable("staticCallables", () => this.staticCallables.size), + checkedTable("genericFnsBySymbol", () => this.genericFnsBySymbol.size), + checkedTable("objLitGenericFns", () => this.objLitGenericFns.size), + checkedTable("bindingGenericFns", () => this.bindingGenericFns.size), + checkedTable("bindingIdentifiers", () => this.bindingIdentifiers.size), + checkedTable("ownToStringTargets", () => this.ownToStringTargets.size), + checkedTable("neverReassignedCache", () => this.neverReassignedCache.size), + checkedTable("jsBindingWritesByOwner", () => this.jsBindingWritesByOwner.size), + checkedTable("rebindableFunctions", () => this.rebindableFunctions.size), + checkedTable("trapBindings", () => this.trapBindings.size), + checkedTable("deadBindings", () => this.deadBindings.size), + checkedTable("instantiationQueue", () => this.instantiationQueue.length), + checkedTable("genericFunctionDemandOwner", () => this.genericFunctionDemandOwner.size), + checkedTable("genericClassDemandOwner", () => this.genericClassDemandOwner.size), + checkedTable("erasingLambdas", () => this.erasingLambdas.size), + checkedTable("typedArgumentsLambdas", () => this.typedArgumentsLambdas.size), + checkedTable("tupleAliasSymbols", () => this.tupleAliasSymbols.size), + checkedTable("nullishFuncResults", () => this.nullishFuncResults.size), + checkedTable("localClassInstantiations", () => this.localClassInstantiations.length), + checkedTable("localClassInstanceTypes", () => this.localClassInstanceTypes.size), + checkedTable("localClassValueTypes", () => this.localClassValueTypes.size), + checkedTable("dynParamLambdas", () => this.dynParamLambdas.size), + checkedTable("aliasNarrowTypes", () => this.aliasNarrowTypes.size), + checkedTable("classSymbolKeys", () => this.classSymbolKeys.size), + checkedTable("registeredClassSymbols", () => this.registeredClassSymbols.size), + checkedTable("symbolFieldIdentities", () => this.symbolFieldIdentities.size), + checkedTable("httpClientFnBindings", () => this.httpClientFnBindings.size), + checkedTable("emitterComputedPatterns", () => this.emitterComputedPatterns.length), + checkedTable("unstableSwitchBindings", () => this.unstableSwitchBindings.size), + checkedTable("earlyReadVars", () => this.earlyReadVars.size), + checkedTable("runtimeOptionalGlobals", () => this.runtimeOptionalGlobals.size), + checkedTable("cjsExportsThis", () => this.cjsExportsThis.size), + checkedTable("inlineRequireInitDone", () => this.inlineRequireInitDone.size), + checkedTable( + "runtimeOptionalArithmeticGlobals", + () => this.runtimeOptionalArithmeticGlobals.size, + ), + checkedTable("runtimeOptionalBindingTypes", () => this.runtimeOptionalBindingTypes.size), + checkedTable("stageSourceTypes", () => this.stageSourceTypes.size), + checkedTable( + "runtimeOptionalAssignedSymbols", + () => this.runtimeOptionalAssignedSymbols.size, + ), + checkedTable("runtimeOptionalGenericReturns", () => this.runtimeOptionalGenericReturns.size), + checkedTable( + "runtimeOptionalFunctionReturns", + () => this.runtimeOptionalFunctionReturns.size, + ), + checkedTable("contextualFunctionReturns", () => this.contextualFunctionReturns.size), + checkedTable( + "runtimeOptionalArithmeticTypes", + () => this.runtimeOptionalArithmeticTypes.size, + ), + checkedTable("runtimeOptionalFields", () => this.runtimeOptionalFields.size), + checkedTable("dateJsonBindings", () => this.dateJsonBindings.size), + checkedTable( + "runtimeOptionalRecordsKeptDeclared", + () => this.runtimeOptionalRecordsKeptDeclared.size, + ), + checkedTable("runtimeOptionalPatternTypes", () => this.runtimeOptionalPatternTypes.size), + checkedTable("runtimeOptionalReduceTypes", () => this.runtimeOptionalReduceTypes.size), + checkedTable("runtimeOptionalAssertions", () => this.runtimeOptionalAssertions.size), + checkedTable("presentElementCalls", () => this.presentElementCalls.size), + checkedTable("presentElementLoops", () => this.presentElementLoops.size), + checkedTable("errorFirstLiterals", () => this.errorFirstLiterals.size), + checkedTable("anySymbols", () => this.anySymbols.size), + checkedTable("knownErrors", () => this.knownErrors.size), + checkedTable("shapeOrderMetadataFinalizers", () => this.shapeOrderMetadataFinalizers.length), + checkedTable("receiverTypeOverrides", () => this.receiverTypeOverrides.size), + checkedTable("shapeOrderHelperFinalizers", () => this.shapeOrderHelperFinalizers.length), + checkedTable("emitSpecDone", () => this.emitSpecDone.size), + checkedTable("emitSpecQueue", () => this.emitSpecQueue.length), + checkedTable("structuralClassPeers", () => this.structuralClassPeers.size), + checkedTable("methodLambdasInProgress", () => this.methodLambdasInProgress.size), + checkedTable("classMethodValueSelections", () => this.classMethodValueSelections.size), + checkedTable("stdlibGlobalAliases", () => this.stdlibGlobalAliases.size), + checkedTable("cjsAccessorFns", () => this.cjsAccessorFns.size), + checkedTable("cjsAccessorSnapshots", () => this.cjsAccessorSnapshots.size), + checkedTable("cjsSnapshotReady", () => this.cjsSnapshotReady.size), + checkedTable("matchAllIndexBindings", () => this.matchAllIndexBindings.size), + checkedTable("matchAllDrainIndexes", () => this.matchAllDrainIndexes.size), + checkedTable("numericIterators", () => this.numericIterators.size), + checkedTable("classes", () => this.classes.size), + checkedTable("classBySymbol", () => this.classBySymbol.size), + checkedTable("prototypeMethodAccesses", () => this.prototypeMethodAccesses.size), + checkedTable("replacedSymbolMethods", () => this.replacedSymbolMethods.size), + checkedTable("protocolDispatches", () => this.protocolDispatches.size), + checkedTable("protocolOwnedBindings", () => this.protocolOwnedBindings.size), + checkedTable("receiverFreeCallbackFields", () => this.receiverFreeCallbackFields.size), + checkedTable("virtualJsMethods", () => this.virtualJsMethods.size), + checkedTable("inheritedJsFieldWrites", () => this.inheritedJsFieldWrites.size), + checkedTable("virtualJsMethodArity", () => this.virtualJsMethodArity.size), + checkedTable("checkedVirtualJsMethods", () => this.checkedVirtualJsMethods.size), + checkedTable("unionReturnOverrides", () => this.unionReturnOverrides.size), + checkedTable("checkedVirtualJsAccessors", () => this.checkedVirtualJsAccessors.size), + checkedTable("checkedVirtualJsValueReturns", () => this.checkedVirtualJsValueReturns.size), + checkedTable("globalsBySymbol", () => this.globalsBySymbol.size), + checkedTable("expandoMembers", () => this.expandoMembers.size), + checkedTable("globalsByDeclNode", () => this.globalsByDeclNode.size), + checkedTable("globalsList", () => this.globalsList.length), + checkedTable("npmInitActions", () => this.npmInitActions.size), + checkedTable("jsonInitActions", () => this.jsonInitActions.size), + checkedTable("jsonRequireModules", () => this.jsonRequireModules.size), + checkedTable("jsonRequireCalls", () => this.jsonRequireCalls.size), + checkedTable("dynImports", () => this.dynImports.size), + checkedTable("createRequireImports", () => this.createRequireImports.size), + checkedTable("dynNsBuilders", () => this.dynNsBuilders.size), + checkedTable("checkedCallbackParams", () => this.checkedCallbackParams.size), + checkedTable("valueStreamReceivers", () => this.valueStreamReceivers.size), + checkedTable("stagedArguments", () => this.stagedArguments.size), + checkedTable("moduleNsParamOverrides", () => this.moduleNsParamOverrides.size), + checkedTable("fileTag", () => this.fileTag.size), + checkedTable("forkTargets", () => this.forkTargets.length), + checkedTable("workerTargets", () => this.workerTargets.length), + checkedTable("workerTargetIdByPath", () => this.workerTargetIdByPath.size), + checkedTable("experimentalFiles", () => this.experimentalFiles.size), + checkedTable("internTableNames", () => this.internTableNames.size), + checkedTable("threadSlotNames", () => this.threadSlotNames.size), + checkedTable("threadSlotFamilies", () => this.threadSlotFamilies.size), + checkedTable("threadSlotKeys", () => this.threadSlotKeys.size), + checkedTable("forkTargetIdByPath", () => this.forkTargetIdByPath.size), + checkedTable("nsBlocks", () => this.nsBlocks.size), + checkedTable("initNameOf", () => this.initNameOf.size), + checkedTable("cjsModuleIdOf", () => this.cjsModuleIdOf.size), + checkedTable("cjsModuleFiles", () => this.cjsModuleFiles.length), + checkedTable("moduleGuardOf", () => this.moduleGuardOf.size), + checkedTable("asyncInitFiles", () => this.asyncInitFiles.size), + checkedTable("modulePromiseOf", () => this.modulePromiseOf.size), + checkedTable("asyncCycleRepresentativeOf", () => this.asyncCycleRepresentativeOf.size), + checkedTable("asyncCyclePromiseOf", () => this.asyncCyclePromiseOf.size), + checkedTable("runtimeClassValues", () => this.runtimeClassValues.size), + checkedTable("preparedClassFactories", () => this.preparedClassFactories.size), + checkedTable("preparingClassFactories", () => this.preparingClassFactories.size), + checkedTable("varGlobalEntryInits", () => this.varGlobalEntryInits.size), + checkedTable("cycleInitFlags", () => this.cycleInitFlags.size), + checkedTable("classInitFlags", () => this.classInitFlags.size), + checkedTable("cycleInitDeclarations", () => this.cycleInitDeclarations.size), + checkedTable("uninitializedLexicalGlobals", () => this.uninitializedLexicalGlobals.size), + checkedTable("requireInitFlags", () => this.requireInitFlags.size), + checkedTable("requireEarlyReferences", () => this.requireEarlyReferences.length), + checkedTable("sharedHelpers", () => this.sharedHelpers.size), + checkedTable("dynamicFunctions", () => this.dynamicFunctions.length), + checkedTable("dynamicBodies", () => this.dynamicBodies.length), + checkedTable("deferredDiags", () => this.deferredDiags.size), + checkedTable("deferredClassByName", () => this.deferredClassByName.size), + checkedTable("flushedSymbols", () => this.flushedSymbols.size), + checkedTable("alreadyFlushed", () => this.alreadyFlushed.size), + checkedTable("pendingFlushes", () => this.pendingFlushes.size), + checkedTable("checkedJsFiles", () => this.checkedJsFiles.size), + checkedTable("ffiImports", () => this.ffiImports.length), + checkedTable("ffiImportsByName", () => this.ffiImportsByName.size), + checkedTable("externalTypes", () => this.externalTypes.size), + checkedTable("externalTypeSpecifiersByFile", () => this.externalTypeSpecifiersByFile.size), + checkedTable("blockedBindings", () => this.blockedBindings.size), + checkedTable("runtimeFences", () => this.runtimeFences.length), + checkedTable("provenanceElided", () => this.provenanceElided.length), + checkedTable("statsByFile", () => this.statsByFile.size), + checkedTable( + "reachedBlockedBodyDiagnostics", + () => this.reachedBlockedBodyDiagnostics.length, + ), + checkedTable("exprClasses", () => this.exprClasses.length), + checkedTable("exprClassInfoByName", () => this.exprClassInfoByName.size), + checkedTable("collectingExprClasses", () => this.collectingExprClasses.size), + checkedTable("pendingClassExprInits", () => this.pendingClassExprInits.length), + checkedTable("computedClassBases", () => this.computedClassBases.size), + checkedTable("computedCallableBases", () => this.computedCallableBases.size), + checkedTable("mixinInstanceByCall", () => this.mixinInstanceByCall.size), + checkedTable("mixinCollectingCalls", () => this.mixinCollectingCalls.size), + checkedTable("mixinOrdinals", () => this.mixinOrdinals.size), + checkedTable("mixinInstancesByClassNode", () => this.mixinInstancesByClassNode.size), + checkedTable("genericClassByDecl", () => this.genericClassByDecl.size), + checkedTable("collectingClassDeclarations", () => this.collectingClassDeclarations.size), + checkedTable("genericClassInstances", () => this.genericClassInstances.length), + checkedTable("mainPrelude", () => this.mainPrelude.length), + checkedTable("structuralFnInfos", () => this.structuralFnInfos.size), + checkedTable("preservedLayoutTypes", () => this.preservedLayoutTypes.size), + checkedTable("layoutKeptNodes", () => this.layoutKeptNodes.size), + checkedTable("boundAliasInstances", () => this.boundAliasInstances.size), + checkedTable("castGlobalAliases", () => this.castGlobalAliases.size), + checkedTable("methodAccessSites", () => this.methodAccessSites.size), + checkedTable("checkedJsOverrideMethods", () => this.checkedJsOverrideMethods.size), + checkedTable("untypedLoops", () => this.untypedLoops.size), + checkedTable("classBagInitializers", () => this.classBagInitializers.size), + checkedTable("degradedTypes", () => this.degradedTypes.size), + checkedTable("rejectedGroups", () => this.rejectedGroups.size), + checkedTable("lenientPieces", () => this.lenientPieces.size), + checkedTable("dynamicSlots", () => this.dynamicSlots.length), + checkedTable("retriedSymbols", () => this.retriedSymbols.size), + checkedTable("skippedBodies", () => this.skippedBodies.size), + checkedTable("retriedDeclarations", () => this.retriedDeclarations.length), + checkedTable("loweredOnceDiags", () => this.loweredOnceDiags.size), + checkedTable("staticInitsPassed", () => this.staticInitsPassed.size), + checkedTable("deferredCollectors", () => this.deferredCollectors.size), + checkedTable("retriableDiags", () => this.retriableDiags.size), + checkedTable("failedTypes", () => this.failedTypes.size), + checkedTable("retryDegraded", () => this.retryDegraded.length), + checkedTable("declOrigins", () => this.declOrigins.length), + checkedTable("declOriginIds", () => this.declOriginIds.size), + sizes("shapes", () => this.shapes.journalSizes()), + sizes("unions", () => this.unions.journalSizes()), + sizes("keyOrder", () => this.keyOrder.journalSizes()), + sizes("deferredModuleInitializers", () => + this.deferredModuleInitializers === null + ? [] + : this.deferredModuleInitializers.journalSizes(), + ), + ]; + } + + /** The fields a unit's record need not carry: settings, caches of + * program facts that are computed again when needed, state of the unit + * being lowered (keyed by its own IR, nodes or frames), scratch that + * unwinds before the unit ends, and state that applying the unit's events + * rebuilds. */ + static readonly unjournaled: readonly string[] = [ + // settings + "frontendServices", + "ambient", + "overridesAmbient", + "remainder", + "tierMode", + "generalNodeStreams", + "dynNodeStreams", + "targetPlatform", + "startupCrash", + "projectCheckJs", + "libraryCallbacks", + "ffiBindingSymbols", + "priorRetriedSymbols", + "wholeSlotDegradation", + "inferredCollectionTypes", + "foreignElementKeys", + // caches made when first needed + "nanCodedReads", + "mentionsProcessVersions", + "stringIndexBoundsCache", + "functionLocalSubclasses", + "socketEncodingReachable", + "thenablesMayBeDefined", + "unixListenReachable", + "streamPropMutatedSyms", + "readsExitCode", + "requestInitPropMutatedSyms", + "classProtocols", + "cjsSplitRequireBindings", + "cjsWholeExportSymbols", + "cjsLiveMemberExports", + "cjsNamespaces", + "createRequireBindingTargets", + "createRequireElementTargets", + "computedGlobalReads", + "timeValueDateDecisions", + "timeValueDateWrites", + "dynamicOverloadAnswers", + "argumentCountSlots", + "coercions.widenedReturnTypes", + "packageFileMemo", + "slotOriginMemo", + "recordOriginMemo", + "boundaryCastOrder", + "declBoundaryMemo", + "structuredCloneReadCounts", + // state of the unit being lowered + "assertedValues", + "timeValueDateLocals", + "speculatedClosures", + "fsPathPlans", + "fallibleFsCalls", + "failureTouches", + "partialHits", + // sized by journalTables, indexes over journaled tables, and state that + // applying a unit's events rebuilds + "deferredModuleInitializers", + "liftedFnsIndexed", + "liftedFnsByName", + "reachable", + "reachableForArtifacts", + "constructedClasses", + "virtualEdges", + "countedStatements", + "failedStatements", + "genericDemandRoots", + "genericDemandPriority", + "genericClassDemandPriority", + "unitEvents", + // scratch of the lowering in progress, and memos keyed by nodes + "nullishBindings", + "implicitLocalFns", + "builtinClassAliases", + "runtimeOptionalLocals.active", + "runtimeOptionalLocals.scopes", + "narrowing.facts", + "narrowing.verdicts", + "narrowing.effects", + "narrowing.capturedMutable", + "narrowing.staleBindings", + "narrowing.identifierWrites", + "narrowing.widened", + "narrowing.plainContainers", + "runtimeOptionalStorageLocals", + "runtimeOptionalArithmeticLocals", + "runtimeOptionalRoots", + "coercions.planning", + "coercions.copyRetags", + "assertInstances", + "unionLiteralOwners", + "strippedUndefinedArms", + "addedUndefinedArms", + "stdlibSymbols", + "chainRecvByNode", + "neverValued", + "chainNarrowedType", + "chainHandled", + "splitIdentifierUses", + "typeMemo", + "fnStack", + "positions.nodesByKey", + "positions.paths", + "activeStmtLists", + "constructorReturnPlans", + "mixinFnShapes", + ]; + + /** The fields a unit must leave as they were, and the flags it may set + * (unit-journal.ts). */ + journalScalars(): JournalScalar[] { + const flag = flagScalar; + const fixed = fixedScalar; + return [ + flag( + "handleEventReader", + () => this.handleEventReader, + () => (this.handleEventReader = true), + ), + flag( + "writesOntoChildren", + () => this.writesOntoChildren, + () => (this.writesOntoChildren = true), + ), + flag( + "regexGroupResults", + () => this.regexGroupResults, + () => (this.regexGroupResults = true), + ), + flag( + "requiresHistoricalOrderRelower", + () => this.requiresHistoricalOrderRelower, + () => (this.requiresHistoricalOrderRelower = true), + ), + flag( + "requiresGeneralNodeStreams", + () => this.requiresGeneralNodeStreams, + () => (this.requiresGeneralNodeStreams = true), + ), + flag( + "requiresDynNodeStreams", + () => this.requiresDynNodeStreams, + () => (this.requiresDynNodeStreams = true), + ), + flag( + "errorPrototypeValues", + () => this.errorPrototypeValues, + () => (this.errorPrototypeValues = true), + ), + flag( + "usesWorkers", + () => this.usesWorkers, + () => (this.usesWorkers = true), + ), + flag( + "usesWebAssembly", + () => this.usesWebAssembly, + () => (this.usesWebAssembly = true), + ), + flag( + "usesWasmCustomSections", + () => this.usesWasmCustomSections, + () => (this.usesWasmCustomSections = true), + ), + flag( + "usesLifetimeRegions", + () => this.usesLifetimeRegions, + () => (this.usesLifetimeRegions = true), + ), + flag( + "cjsModuleGraphEnabled", + () => this.cjsModuleGraphEnabled, + () => (this.cjsModuleGraphEnabled = true), + ), + flag( + "globalThisComputedWrite", + () => this.globalThisComputedWrite, + () => (this.globalThisComputedWrite = true), + ), + flag( + "untypedIntegrity", + () => this.untypedIntegrity, + () => (this.untypedIntegrity = true), + ), + flag( + "partialDegradedTypesMapped", + () => this.partialDegradedTypesMapped, + () => (this.partialDegradedTypesMapped = true), + ), + fixed("errorAccessorReflection", () => this.errorAccessorReflection), + fixed("subclassLinks", () => this.subclassLinks), + fixed("errorPathCopies", () => this.errorPathCopies), + fixed("errorMethodDispatchRevision", () => this.errorMethodDispatchRevision), + fixed("errorPropertyDispatchRevision", () => this.errorPropertyDispatchRevision), + fixed("childAdditions", () => this.childAdditions), + fixed("emitterEventTable", () => this.emitterEventTable), + fixed("globalThisWrittenNames", () => this.globalThisWrittenNames), + fixed("importRetryModules", () => this.importRetryModules), + fixed("localSubclassIndex", () => this.localSubclassIndex), + // State of the lowering in progress, which a unit restores. + fixed("genericInstantiationPath", () => this.genericInstantiationPath), + fixed("genericDemandOwner", () => this.genericDemandOwner), + fixed("typeParamBindings", () => this.typeParamBindings), + fixed("staticFnNames", () => this.staticFnNames), + fixed("erasingDepth", () => this.erasingDepth), + fixed("instanceResultClass", () => this.instanceResultClass), + fixed("typeParamTsBindings", () => this.typeParamTsBindings), + fixed("implicitParamTypes", () => this.implicitParamTypes), + fixed("fspGlobHead", () => this.fspGlobHead), + fixed("instantiationContext", () => this.instantiationContext), + fixed("suppressStats", () => this.suppressStats), + fixed("emitSpecCtx", () => this.emitSpecCtx), + fixed("assertCallOptions", () => this.assertCallOptions), + fixed("currentClass", () => this.currentClass), + fixed("closureRoot", () => this.closureRoot), + fixed("superConstructorEdge", () => this.superConstructorEdge), + fixed("returnAwaitedPromise", () => this.returnAwaitedPromise), + fixed("collecting", () => this.collecting), + fixed("diagSink", () => this.diagSink), + fixed("speculativeTypeQueryDepth", () => this.speculativeTypeQueryDepth), + fixed("mixinTypeContext", () => this.mixinTypeContext), + fixed("pendingLabels", () => this.pendingLabels), + fixed("onEdge", () => this.onEdge), + fixed("onExprClassCollected", () => this.onExprClassCollected), + fixed("onLateClassCollected", () => this.onLateClassCollected), + fixed("onDeclarationRegistered", () => this.onDeclarationRegistered), + fixed("liftDynFieldPolicy", () => this.liftDynFieldPolicy), + fixed("coercionOriginNode", () => this.coercionOriginNode), + fixed("coercionOriginUnion", () => this.coercionOriginUnion), + fixed("coercionFieldOrigin", () => this.coercionFieldOrigin), + ]; + } + /** Coverage remainder mode: the reachability gate inverts (see wantBody) * and no module is built. */ readonly remainder: boolean; @@ -4053,6 +4589,12 @@ export class Lowerer { countStatement(node: ts.Node, file: string | undefined): void { if (this.countedStatements.has(node)) return; this.countedStatements.add(node); + this.unitEvents?.record( + "statement", + ts.getSourceFileOfNode(node).fileName, + file ?? null, + node.index, + ); this.stats.statementsTotal++; if (file !== undefined) this.bumpFileStat(file, "total"); } @@ -4062,6 +4604,12 @@ export class Lowerer { countStatementFailed(node: ts.Node, file: string | undefined): void { if (this.failedStatements.has(node)) return; this.failedStatements.add(node); + this.unitEvents?.record( + "failedStatement", + ts.getSourceFileOfNode(node).fileName, + file ?? null, + node.index, + ); this.stats.statementsFailed++; if (file !== undefined) this.bumpFileStat(file, "failed"); } @@ -9500,25 +10048,36 @@ export class Lowerer { const rootEdge = this.onEdge; for (const root of extraRoots ?? []) rootEdge?.(root); const shuffle = lowerShuffle(); + const lowerUnitBody = (name: string): IrFunction | null => { + const unit = units.get(name)!; + const priority = metadataPriority.get(name) ?? [3, expressionMetadataOrder++]; + const owner = demandOwner(priority); + return this.withGenericDemandOwner(owner, () => + this.shapes.withDeclaredOrderPriority(priority, () => + this.withClosureRoot(name, unit.lower), + ), + ); + }; + const recorded = journalMode() === "serial" ? waveStats() : null; + const host = recorded === null ? null : this.waveHost(loweredUnits, lowerUnitBody); + const dump = recorded === null ? null : journalDump(); + const recordLines: string[] = []; const drainUnits = (): void => { while (queue.length > 0) { const wave = queue.splice(0); shuffle?.(wave); this.checker.prefetchRoots(wave.flatMap((name) => units.get(name)!.roots)); + if (host !== null && recorded !== null) { + lowerWaveRecorded(host, wave, recorded, dump === null ? null : recordLines); + continue; + } for (const name of wave) { // A body-level poison outside the per-statement catches (a fenced // constructor/method parameter default lowered by declareParams): // every edge fired before poisoning remains retained; the diagnostic // stays recorded and the member stays omitted. try { - const unit = units.get(name)!; - const priority = metadataPriority.get(name) ?? [3, expressionMetadataOrder++]; - const owner = demandOwner(priority); - const fn = this.withGenericDemandOwner(owner, () => - this.shapes.withDeclaredOrderPriority(priority, () => - this.withClosureRoot(name, unit.lower), - ), - ); + const fn = lowerUnitBody(name); if (fn) loweredUnits.set(name, fn); relieveHeapPressure(); } catch (e) { @@ -9569,6 +10128,11 @@ export class Lowerer { this.restoreGenericClassInstanceOrder(clsInstLowered); } classDispatch.finalize(this); + if (dump !== null) writeRecords(dump, recordLines); + if (recorded !== null && process.env["SCRIPTC_TIMING"] === "1") + process.stderr.write( + `scriptc lowering ${JSON.stringify({ phase: "unit-records", ...recorded })}\n`, + ); const orderedUnits = unitsInOrder(); this.settleGenericDemandPriorities(); this.shapes.settleDeclaredOrderPriorities(); @@ -9613,12 +10177,101 @@ export class Lowerer { return { reachable, result: this.finishModule(functions) }; } + /** The lowerer's side of a recorded wave (unit-journal.ts). + * `lowerUnitBody` lowers a unit as reachable emit does. */ + private waveHost( + loweredUnits: Map, + lowerUnitBody: (name: string) => IrFunction | null, + ): WaveHost { + const scalars = this.journalScalars(); + const flags = new Map(scalars.map((scalar) => [scalar.name, scalar])); + let instances = new Map(); + let indexed = -1; + const instanceNamed = (name: string): GenericInstance | undefined => { + if (indexed !== this.instantiationQueue.length) { + instances = new Map(this.instantiationQueue.map(({ inst }) => [inst.name, inst])); + indexed = this.instantiationQueue.length; + } + return instances.get(name); + }; + const classNamed = (name: string): ClassInfo | undefined => { + const info = this.classes.get(name); + return info !== undefined && info.def.name === name ? info : undefined; + }; + const files = new Map(); + const statementNode = (event: JournalEvent): ts.Node | undefined => { + let sf = files.get(event.name); + if (sf === undefined && !files.has(event.name)) { + sf = this.program.getSourceFile(event.name); + files.set(event.name, sf); + } + if (sf === undefined) return undefined; + try { + return sf.file.node(event.index); + } catch { + return undefined; + } + }; + return { + tables: this.journalTables(), + scalars, + lowerRecorded: (name, events) => { + this.unitEvents = events; + try { + const fn = lowerUnitBody(name); + relieveHeapPressure(); + return { fn, poisoned: false }; + } catch (e) { + if (!(e instanceof PoisonError)) throw e; + return { fn: null, poisoned: true }; + } finally { + this.unitEvents = null; + } + }, + store: (name, fn) => { + if (fn) loweredUnits.set(name, fn); + }, + statementCounts: () => ({ + total: this.stats.statementsTotal, + failed: this.stats.statementsFailed, + other: this.stats.functionsSkipped + this.diags.length, + }), + resolves: (event) => { + switch (event.kind) { + case "virtualEdge": + case "classDemand": + return classNamed(event.name) !== undefined; + case "functionDemand": + return instanceNamed(event.name) !== undefined; + case "statement": + case "failedStatement": + return statementNode(event) !== undefined; + case "flag": + return flags.get(event.name)?.set !== undefined; + default: + return true; + } + }, + }; + } + /** Reachability hook (see emitReachable): fires when lowering resolves a * reference to a lowerable body. */ noteEdge(name: string): void { // Called as a plain function: the instance is not its receiver. const onEdge = this.onEdge; - if (onEdge) onEdge(name); + const events = this.unitEvents; + if (events === null) { + if (onEdge) onEdge(name); + return; + } + events.record(this.superConstructorEdge ? "superConstructorEdge" : "edge", name); + events.depth++; + try { + if (onEdge) onEdge(name); + } finally { + events.depth--; + } } /** A subclass constructor's super() edge: the base constructor runs, but @@ -9639,6 +10292,21 @@ export class Lowerer { * them). */ noteVirtualEdge(info: ClassInfo, method: string): void { if (!this.onEdge) return; + const events = this.unitEvents; + if (events === null) { + this.virtualEdgeOf(info, method); + return; + } + events.record("virtualEdge", info.def.name, method); + events.depth++; + try { + this.virtualEdgeOf(info, method); + } finally { + events.depth--; + } + } + + private virtualEdgeOf(info: ClassInfo, method: string): void { if (this.constructedClasses) { const methods = this.virtualEdges.get(info) ?? new Set(); methods.add(method); diff --git a/packages/compiler/src/frontend/lowering/unit-journal.test.ts b/packages/compiler/src/frontend/lowering/unit-journal.test.ts new file mode 100644 index 000000000..74dc57b1a --- /dev/null +++ b/packages/compiler/src/frontend/lowering/unit-journal.test.ts @@ -0,0 +1,282 @@ +import { expect, test } from "vitest"; +import type { Program, SourceFile } from "../ts7/adapter.js"; +import { Lowerer } from "./lowerer.js"; +import { + checkedTable, + flagScalar, + functionTable, + lowerWaveRecorded, + mapTable, + recordUnit, + setTable, + waveStats, + type WaveHost, +} from "./unit-journal.js"; +import type { IrFunction } from "../../ir/ir.js"; + +const lowerer = (): Lowerer => + new Lowerer( + { getTypeChecker: () => ({}), isSourceFileDefaultLibrary: () => false } as unknown as Program, + { fileName: "/island/entry.ts" } as SourceFile, + [], + ); + +const isCollection = (value: unknown): boolean => + value instanceof Map || value instanceof Set || Array.isArray(value); + +/** The registries whose collections journalSizes covers, with the + * collections each one's journalSizes reads. */ +const sized: Record = { + shapes: [ + "keyedNameKeys", + "byKey", + "byId", + "declaredOrderPriority", + "declaredOrderCandidates", + "derivedDeclaredOrderFinalizers", + "shapes", + "recIds", + "pendingRec", + "extendedNames", + ], + unions: ["keyedNameKeys", "byKey", "byId", "armTags", "unions", "recIds", "pendingRec"], + keyOrder: ["literals", "copies", "methodResults", "observed", "changed"], +}; + +/** Scratch of the registries' own mapping (it unwinds before mapType returns). */ +const scratch = new Set([ + "shapes.inProgress", + "shapes.currentDeclaredOrderPriority", + "unions.inProgress", +]); + +/** Owned objects whose collections are listed one by one. */ +const owned = new Set(["coercions", "narrowing", "runtimeOptionalLocals", "positions"]); + +test("every collection of the lowerer is journaled, checked or left out by name", () => { + const subject = lowerer(); + const named = new Set([ + ...subject.journalTables().map((table) => table.name), + ...Lowerer.unjournaled, + ]); + const missing: string[] = []; + const fields = subject as unknown as Record; + for (const field of Object.keys(fields)) { + const value = fields[field]; + if (isCollection(value)) { + if (!named.has(field)) missing.push(field); + continue; + } + if (value === null || typeof value !== "object") continue; + const inner = value as Record; + const covered = sized[field]; + if (covered === undefined && !owned.has(field)) continue; + for (const key of Object.keys(inner)) { + const path = `${field}.${key}`; + if (!isCollection(inner[key]) || scratch.has(path)) continue; + if (covered !== undefined ? !covered.includes(key) : !named.has(path)) missing.push(path); + } + } + expect(missing).toEqual([]); +}); + +test("every scalar field of the lowerer is journaled or left out by name", () => { + const subject = lowerer(); + const named = new Set([ + ...subject.journalScalars().map((scalar) => scalar.name), + ...Lowerer.unjournaled, + ]); + const fields = subject as unknown as Record; + const missing = Object.keys(fields).filter((field) => { + const value = fields[field]; + if (isCollection(value) || typeof value === "function") return false; + const scalar = value === null || value === undefined || typeof value !== "object"; + return scalar && !named.has(field); + }); + expect(missing).toEqual([]); +}); + +test("every name the lists hold is a field of the lowerer", () => { + const subject = lowerer(); + const fields = subject as unknown as Record; + const names = [ + ...subject.journalTables().map((table) => table.name), + ...subject.journalScalars().map((scalar) => scalar.name), + ...Lowerer.unjournaled, + ]; + const missing = names.filter((name) => { + const [head, key] = name.split("."); + if (!(head! in fields)) return true; + if (key === undefined) return false; + const inner = fields[head!]; + return inner === null || typeof inner !== "object" || !(key in inner); + }); + expect(missing).toEqual([]); +}); + +const fn = (name: string, ret: "f64" | "string" = "f64"): IrFunction => + ({ + name, + params: [], + returnType: { kind: ret }, + locals: [], + body: [], + loc: { file: "a.ts", start: 0, end: 0 }, + }) as unknown as IrFunction; + +test("a map table hands over the entries added after a size", () => { + const map = new Map([["a", "%h.1"]]); + const table = mapTable("helpers", map); + map.set("b", "%h.2"); + map.set("c", "%h.3"); + const entries = table.entries!(1)!; + expect(entries.length).toBe(2); + const parent = new Map([ + ["a", "%h.1"], + ["c", "%h.3"], + ]); + mapTable("helpers", parent).apply!(entries); + expect([...parent]).toEqual([ + ["a", "%h.1"], + ["c", "%h.3"], + ["b", "%h.2"], + ]); + table.truncate!(1); + expect([...map]).toEqual([["a", "%h.1"]]); +}); + +test("a held table notices a replaced entry", () => { + const map = new Map([["a", "%h.1"]]); + const held = mapTable("helpers", map).hold!(); + map.set("b", "%h.2"); + expect(held.same()).toBe(true); + map.set("a", "%h.9"); + expect(held.same()).toBe(false); + const set = new Set(["x"]); + const heldSet = setTable("names", set).hold!(); + set.delete("x"); + set.add("x"); + expect(heldSet.same()).toBe(true); + set.delete("x"); + expect(heldSet.same()).toBe(false); +}); + +test("a function table keeps the function an earlier unit made", () => { + const worker = [fn("%a")]; + const table = functionTable("lifted", worker, () => false); + worker.push(fn("%b"), fn("%c")); + const entries = table.entries!(1)!; + const parent = [fn("%c", "string")]; + const target = functionTable("lifted", parent, (name) => + parent.some((other) => other.name === name), + ); + target.apply!(entries); + expect(parent.map((f) => [f.name, f.returnType.kind])).toEqual([ + ["%c", "string"], + ["%b", "f64"], + ]); +}); + +test("a set table adds what the parent lacks", () => { + const set = new Set(["x"]); + const table = setTable("names", set); + set.add("y"); + const parent = new Set(); + setTable("names", parent).apply!(table.entries!(1)!); + expect([...parent]).toEqual(["y"]); +}); + +/** A wave host over one helper map, one checked list and one flag, whose + * units are lowered by `lower`. */ +function host(lower: (name: string, state: HostState) => IrFunction | null): { + host: WaveHost; + state: HostState; +} { + const state: HostState = { helpers: new Map(), checked: [], flag: false, stored: new Map() }; + return { + state, + host: { + tables: [ + mapTable("helpers", state.helpers), + checkedTable("checked", () => state.checked.length), + ], + scalars: [ + flagScalar( + "flag", + () => state.flag, + () => (state.flag = true), + ), + ], + lowerRecorded: (name, events) => { + events.record("edge", `%callee.${name}`); + return { fn: lower(name, state), poisoned: false }; + }, + store: (name, unitFn) => { + if (unitFn !== null) state.stored.set(name, unitFn); + }, + statementCounts: () => ({ total: 0, failed: 0, other: 0 }), + resolves: () => true, + }, + }; +} + +interface HostState { + helpers: Map; + checked: string[]; + flag: boolean; + stored: Map; +} + +test("a recorded wave keeps what the records carry, in wave order", () => { + const made = new Map(); + const { host: subject, state } = host((name, held) => { + held.helpers.set(`key.${name}`, `%h.${name}`); + if (name === "b") held.flag = true; + if (name === "c") held.checked.push(name); + const unitFn = fn(`%${name}`); + made.set(name, unitFn); + return unitFn; + }); + const stats = waveStats(); + lowerWaveRecorded(subject, ["a", "b", "c"], stats); + expect([...state.helpers.keys()]).toEqual(["key.a", "key.b", "key.c"]); + expect(stats).toMatchObject({ + waves: 1, + units: 3, + recorded: 2, + unrecorded: 1, + reasons: { checked: 1 }, + }); + // Recorded units keep the function decoded from their record; the unit + // without one keeps the function lowering made. + expect(state.stored.get("a")).not.toBe(made.get("a")); + expect(state.stored.get("a")).toEqual(made.get("a")); + expect(state.stored.get("c")).toBe(made.get("c")); +}); + +test("a unit's record lists its events, flags and added entries", () => { + const { host: subject } = host((name, held) => { + held.helpers.set(`key.${name}`, `%h.${name}`); + held.flag = true; + return fn(`%${name}`); + }); + const { record } = recordUnit(subject, "a"); + expect(record).not.toBeNull(); + expect(record!.events.map((event) => [event.kind, event.name])).toEqual([ + ["edge", "%callee.a"], + ["flag", "flag"], + ]); + expect(record!.tables).toEqual({ helpers: [`"key.a"\n"%h.a"`] }); + expect(JSON.parse(JSON.stringify(record))).toEqual(record); +}); + +test("a unit that changes an entry it did not add has no record", () => { + const { host: subject, state } = host((name, held) => { + held.helpers.set("shared", `%h.${name}`); + return null; + }); + state.helpers.set("shared", "%h.first"); + const unit = recordUnit(subject, "a"); + expect(unit.record).toBeNull(); + expect(unit.reasons).toEqual(["helpers:replaced"]); +}); diff --git a/packages/compiler/src/frontend/lowering/unit-journal.ts b/packages/compiler/src/frontend/lowering/unit-journal.ts new file mode 100644 index 000000000..8c8de3001 --- /dev/null +++ b/packages/compiler/src/frontend/lowering/unit-journal.ts @@ -0,0 +1,474 @@ +/* Per-unit records of reachable-emit lowering (Lowerer.emitReachable). + * + * A unit's record is what another process needs to reach the state the + * unit's lowering reached: the unit's IR, the entries it added to the + * tables that collect lowering results (Lowerer.journalTables), and the + * calls it made into reachability, generic demand and statement counting + * (JournalEvent). Records hold only strings, numbers and names, so they can + * be written to a file and read by another process or another build. + * + * A unit has no record when its lowering changed state that no record + * carries: a table that is not appendable grew, a field changed, an entry + * held before the unit was replaced, or a diagnostic was emitted. Such a + * unit lowers again wherever its record would have been applied. + * + * SCRIPTC_LOWER_JOURNAL=serial records every unit while it lowers, and + * replaces the unit's results with the ones its record carries: the + * function decoded from the record and the table entries added again from + * it. The build's output is then the output of applying the records, which + * tests compare with ordinary lowering. SCRIPTC_LOWER_JOURNAL_DUMP= + * turns the serial mode on and appends each unit's record to + * /records.jsonl, one JSON line per unit in lowering order; a unit + * without a record has a line with its name and reasons. */ +import { appendFileSync, mkdirSync } from "node:fs"; +import { join } from "node:path"; +import type { IrFunction } from "../../ir/ir.js"; +import { deserializeFunction, encodeFunction } from "../../ir/serialize.js"; + +/** A function as records hold it, or null when it does not encode. Every + * function a record holds goes through this pair. */ +export function encodeUnitFunction(fn: IrFunction): string | null { + return encodeFunction(fn) ?? null; +} + +/** A function from its record encoding (encodeUnitFunction). */ +export function decodeUnitFunction(text: string): IrFunction { + return deserializeFunction(text); +} + +/** A call a unit made while it lowered, which applying its record makes + * again. */ +export interface JournalEvent { + readonly kind: + | "edge" + | "superConstructorEdge" + | "virtualEdge" + | "functionDemand" + | "classDemand" + | "statement" + | "failedStatement" + | "flag"; + /** The edge's body name, the class of a virtual edge or class demand, the + * generic instance's name, a counted statement's source file, or the + * flag's field. */ + readonly name: string; + /** The virtual edge's method, or the file a statement counts toward. */ + readonly detail: string | null; + /** A counted statement's node index in its source file. */ + readonly index: number; +} + +/** One unit's record. */ +export interface UnitRecord { + readonly name: string; + /** The unit's function in the compact IR encoding, or null when the unit + * lowered to nothing. */ + readonly fn: string | null; + /** True when the unit's lowering ended in a poison: its diagnostics and + * the edges it fired before poisoning stay, and it has no function. */ + readonly poisoned: boolean; + readonly events: readonly JournalEvent[]; + /** The encoded entries the unit added, by table name, for the tables it + * grew. */ + readonly tables: Readonly>; +} + +/** A table a unit's lowering can grow. An appendable table only adds + * entries at the end (Map and Set insertion order, array pushes), so a + * unit's additions are the entries past the size the table had when the + * unit started. A checked table has only `size`: when it grows, the unit + * has no record. */ +export interface JournalTable { + readonly name: string; + size(): number; + /** The encoded entries from position `from` on, or null when one of + * them does not encode. */ + readonly entries?: (from: number) => string[] | null; + /** Adds the encoded entries whose keys this process's table lacks. */ + readonly apply?: (entries: readonly string[]) => void; + /** Removes the entries from position `from` on. */ + readonly truncate?: (from: number) => void; + /** The entries the table holds now. */ + readonly hold?: () => HeldEntries; +} + +/** A table's entries at one point (JournalTable.hold). */ +export interface HeldEntries { + /** Whether the table's first entries are still these keys and values. */ + same(): boolean; +} + +/** A field a unit's lowering must leave as it found it (fixedScalar), or a + * flag it may set, which applying its record sets too (flagScalar). */ +export interface JournalScalar { + readonly name: string; + /** Takes the field's value; the result answers how the field changed. */ + mark(): ScalarMark; + readonly set?: () => void; +} + +/** A field's value when a unit started. */ +export interface ScalarMark { + /** Not at all, set (a flag that went from false to true), or otherwise. */ + change(): "same" | "set" | "changed"; +} + +/** A field a unit's lowering must leave as it found it. */ +export function fixedScalar(name: string, get: () => T): JournalScalar { + return { + name, + mark: () => { + const start = get(); + return { change: () => (get() === start ? "same" : "changed") }; + }, + }; +} + +/** A flag a unit's lowering may set. */ +export function flagScalar(name: string, get: () => boolean, set: () => void): JournalScalar { + return { + name, + mark: () => { + const start = get(); + return { + change: () => { + const now = get(); + return now === start ? "same" : now && !start ? "set" : "changed"; + }, + }; + }, + set, + }; +} + +/** The events of the unit being recorded (Lowerer.unitEvents). */ +export class UnitEvents { + readonly events: JournalEvent[] = []; + /** Calls into reachability nest (an edge can fire virtual edges). Only + * the outermost call is recorded, because making it again makes the + * nested ones. */ + depth = 0; + + record(kind: JournalEvent["kind"], name: string, detail: string | null = null, index = 0): void { + if (this.depth === 0) this.events.push({ kind, name, detail, index }); + } +} + +/** The lowerer's side of a recorded wave (Lowerer.emitReachable). */ +export interface WaveHost { + readonly tables: readonly JournalTable[]; + readonly scalars: readonly JournalScalar[]; + /** Lowers `name` as reachable emit does, with `events` recording: the + * function, or null, and whether the unit poisoned. Throws what lowering + * throws other than a poison. */ + lowerRecorded(name: string, events: UnitEvents): { fn: IrFunction | null; poisoned: boolean }; + /** Keeps a unit's function. */ + store(name: string, fn: IrFunction | null): void; + /** Statement totals, and the count of diagnostics and skipped functions. */ + statementCounts(): { total: number; failed: number; other: number }; + /** Whether `event` names something this process holds. */ + resolves(event: JournalEvent): boolean; +} + +/** Whether reachable emit records its units (SCRIPTC_LOWER_JOURNAL=serial, + * or a SCRIPTC_LOWER_JOURNAL_DUMP directory). */ +export function journalMode(): "serial" | "off" { + return process.env["SCRIPTC_LOWER_JOURNAL"] === "serial" || journalDump() !== null + ? "serial" + : "off"; +} + +/** The directory recorded waves write their records to + * (SCRIPTC_LOWER_JOURNAL_DUMP), or null. */ +export function journalDump(): string | null { + const dir = process.env["SCRIPTC_LOWER_JOURNAL_DUMP"]; + return dir === undefined || dir === "" ? null : dir; +} + +/** Appends `lines` to `dir`/records.jsonl. */ +export function writeRecords(dir: string, lines: readonly string[]): void { + if (lines.length === 0) return; + mkdirSync(dir, { recursive: true }); + appendFileSync(join(dir, "records.jsonl"), `${lines.join("\n")}\n`); +} + +/** What a build's recorded waves did (SCRIPTC_TIMING). */ +export interface WaveStats { + waves: number; + units: number; + /** Units whose record stands for them. */ + recorded: number; + /** Units without a record. */ + unrecorded: number; + /** Per table, field or check, the units it left without a record. */ + reasons: Record; +} + +export function waveStats(): WaveStats { + return { waves: 0, units: 0, recorded: 0, unrecorded: 0, reasons: {} }; +} + +/** What lowering one unit with its events recording produced. */ +export interface RecordedUnit { + /** The unit's record, or null when the unit has none. */ + readonly record: UnitRecord | null; + /** Why the unit has no record. */ + readonly reasons: readonly string[]; + /** The function lowering made. */ + readonly fn: IrFunction | null; + /** Each table's size before the unit. */ + readonly before: readonly number[]; +} + +/** Lowers `name` with its events recording, and makes its record. The unit + * has a record when every table that grew is appendable and holds the + * entries it held before, every field is as it was or a flag that was set, + * statement totals grew by the recorded statements alone, no diagnostic was + * emitted, every event resolves and the function encodes. */ +export function recordUnit(host: WaveHost, name: string): RecordedUnit { + const tables = host.tables; + const before = tables.map((table) => table.size()); + const held = tables.map((table) => (table.hold === undefined ? null : table.hold())); + const marks = host.scalars.map((scalar) => scalar.mark()); + const counts = host.statementCounts(); + const recorder = new UnitEvents(); + const { fn, poisoned } = host.lowerRecorded(name, recorder); + const reasons: string[] = []; + const grown: number[] = []; + tables.forEach((table, index) => { + const check = held[index] ?? null; + if (check !== null && !check.same()) reasons.push(`${table.name}:replaced`); + if (table.size() === before[index]) return; + if (table.entries === undefined) reasons.push(table.name); + else grown.push(index); + }); + const events = [...recorder.events]; + host.scalars.forEach((scalar, index) => { + const change = marks[index]!.change(); + if (change === "set") events.push({ kind: "flag", name: scalar.name, detail: null, index: 0 }); + else if (change === "changed") reasons.push(scalar.name); + }); + let statements = 0; + let failed = 0; + for (const event of events) { + if (event.kind === "statement") statements++; + else if (event.kind === "failedStatement") failed++; + if (!host.resolves(event)) reasons.push(`event:${event.kind}`); + } + const now = host.statementCounts(); + if (now.total - counts.total !== statements || now.failed - counts.failed !== failed) + reasons.push("statements"); + if (now.other !== counts.other) reasons.push("diagnostics"); + const text = fn === null ? null : encodeUnitFunction(fn); + if (fn !== null && text === null) reasons.push("encoding"); + const added: Record = {}; + if (reasons.length === 0) + for (const index of grown) { + const table = tables[index]!; + const entries = table.entries!(before[index]!); + if (entries === null) reasons.push(`${table.name}:encoding`); + else added[table.name] = entries; + } + if (reasons.length > 0) return { record: null, reasons, fn, before }; + return { record: { name, fn: text, poisoned, events, tables: added }, reasons, fn, before }; +} + +/** Lowers `wave` with every unit recorded. A unit with a record keeps the + * results its record carries: the entries it added are taken out of their + * tables and added again from the record, and its function is the one + * decoded from the record. A unit without one keeps what lowering made. + * `lines`, when given, receives each unit's record as a JSON line. */ +export function lowerWaveRecorded( + host: WaveHost, + wave: readonly string[], + stats: WaveStats, + lines: string[] | null = null, +): void { + stats.waves++; + const tables = host.tables; + for (const name of wave) { + const unit = recordUnit(host, name); + stats.units++; + const record = unit.record; + if (record === null) { + lines?.push(JSON.stringify({ name, reasons: unit.reasons })); + stats.unrecorded++; + for (const reason of unit.reasons) stats.reasons[reason] = (stats.reasons[reason] ?? 0) + 1; + host.store(name, unit.fn); + continue; + } + lines?.push(JSON.stringify(record)); + stats.recorded++; + tables.forEach((table, index) => { + const entries = record.tables[table.name]; + if (entries === undefined) return; + table.truncate!(unit.before[index]!); + table.apply!(entries); + }); + host.store(name, record.fn === null ? null : decodeUnitFunction(record.fn)); + } +} + +/** An appendable table over a Map from strings to strings, such as helper + * names by key. */ +export function mapTable(name: string, map: Map): JournalTable { + return codecTable( + name, + map, + (value) => JSON.stringify(value), + (text) => JSON.parse(text) as string, + ); +} + +/** An appendable table over a Map from strings to numbers. */ +export function countTable(name: string, map: Map): JournalTable { + return codecTable( + name, + map, + (value) => String(value), + (text) => Number(text), + ); +} + +/** An appendable table over a Map from strings to IR data, encoded as + * JSON. JSON writes a number that is not finite as null, so a value whose + * encoding holds a null does not encode. */ +export function irDataTable(name: string, map: Map): JournalTable { + return codecTable( + name, + map, + (value) => { + const text = JSON.stringify(value); + return /[:,[]null\b/.test(text) ? null : text; + }, + (text) => JSON.parse(text) as V, + ); +} + +/** An appendable table over a Map from strings to values that `encode` + * writes, or answers null for, and `decode` reads back. An entry is its key + * as a JSON string, a line break and its value: a JSON string holds no line + * break. A value that does not encode leaves its unit without a record. */ +export function codecTable( + name: string, + map: Map, + encode: (value: V) => string | null, + decode: (text: string) => V, +): JournalTable { + return { + name, + size: () => map.size, + entries: (from) => { + const out: string[] = []; + let index = 0; + for (const [key, value] of map) { + if (index++ < from) continue; + const text = encode(value); + if (text === null) return null; + out.push(`${JSON.stringify(key)}\n${text}`); + } + return out; + }, + apply: (entries) => { + for (const entry of entries) { + const split = entry.indexOf("\n"); + const key = JSON.parse(entry.slice(0, split)) as string; + if (!map.has(key)) map.set(key, decode(entry.slice(split + 1))); + } + }, + truncate: (from) => { + const keys = [...map.keys()].slice(from); + for (const key of keys) map.delete(key); + }, + hold: () => { + const keys = [...map.keys()]; + const values = [...map.values()]; + return { + same: () => { + let index = 0; + for (const [key, value] of map) { + if (index >= keys.length) break; + if (key !== keys[index] || value !== values[index]) return false; + index++; + } + return index === keys.length; + }, + }; + }, + }; +} + +/** An appendable table over a Set of strings. */ +export function setTable(name: string, set: Set): JournalTable { + return { + name, + size: () => set.size, + entries: (from) => [...set].slice(from), + apply: (entries) => { + for (const entry of entries) set.add(entry); + }, + truncate: (from) => { + const entries = [...set].slice(from); + for (const entry of entries) set.delete(entry); + }, + hold: () => { + const entries = [...set]; + return { + same: () => { + let index = 0; + for (const entry of set) { + if (index >= entries.length) break; + if (entry !== entries[index]) return false; + index++; + } + return index === entries.length; + }, + }; + }, + }; +} + +/** An appendable list of functions this process holds one of per name. A + * function whose name the list already holds is the same function, made by + * an earlier unit. `changed` runs after the list changes, for indexes over + * it. */ +export function functionTable( + name: string, + functions: IrFunction[], + has: (name: string) => boolean, + changed: () => void = () => {}, +): JournalTable { + return { + name, + size: () => functions.length, + entries: (from) => { + const out: string[] = []; + for (const fn of functions.slice(from)) { + const text = encodeUnitFunction(fn); + if (text === null) return null; + out.push(text); + } + return out; + }, + apply: (entries) => { + for (const entry of entries) { + const fn = decodeUnitFunction(entry); + if (!has(fn.name)) functions.push(fn); + } + changed(); + }, + truncate: (from) => { + while (functions.length > from) functions.pop(); + changed(); + }, + hold: () => { + const held = [...functions]; + return { same: () => held.every((fn, index) => functions[index] === fn) }; + }, + }; +} + +/** A table whose growth leaves the unit without a record. */ +export function checkedTable(name: string, size: () => number): JournalTable { + return { name, size }; +} diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index 40ca10b38..1d505f08f 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -324,6 +324,30 @@ export class ShapeRegistry { private currentDeclaredOrderPriority: DeclaredOrderPriority = [0, 0]; /** All interned shapes in first-seen order. */ readonly shapes: IrRecordShape[] = []; + + /** The sizes of the registry's tables, which only grow while shapes are + * interned (unit-journal.ts). */ + journalSizes(): number[] { + let candidates = 0; + for (const list of this.declaredOrderCandidates.values()) candidates += list.length; + let extended = 0; + for (const names of this.extendedNames.values()) extended += names.length; + return [ + this.revision, + this.shapes.length, + this.byKey.size, + this.byId.size, + this.keyedNameKeys.size, + this.declaredOrderPriority.size, + this.declaredOrderCandidates.size, + candidates, + this.derivedDeclaredOrderFinalizers.length, + this.recIds.size, + this.pendingRec.size, + this.extendedNames.size, + extended, + ]; + } /** ts.Types currently being mapped — a BACK-REFERENCE to one of these is * the recursive knot (`interface TreeNode { children: TreeNode[] }`): * mapType answers a NAMED RECURSIVE SHAPE (recursiveRef) whose fields @@ -675,6 +699,23 @@ export class UnionRegistry { private readonly armTags = new Map>(); /** All interned unions in first-seen order. */ readonly unions: IrUnionDef[] = []; + + /** The sizes of the registry's tables (ShapeRegistry.journalSizes). */ + journalSizes(): number[] { + let tags = 0; + for (const byArm of this.armTags.values()) tags += byArm.size; + return [ + this.revision, + this.unions.length, + this.byKey.size, + this.byId.size, + this.keyedNameKeys.size, + this.armTags.size, + tags, + this.recIds.size, + this.pendingRec.size, + ]; + } /** ts.Types currently being mapped — a back-reference to one is the * recursive knot passing through a union (`type Tree = Leaf | Branch` * whose Branch arm carries `Tree[]`, the optional field `a?: A` of a diff --git a/packages/compiler/src/ir/serialize.ts b/packages/compiler/src/ir/serialize.ts index 314f258d6..185472b89 100644 --- a/packages/compiler/src/ir/serialize.ts +++ b/packages/compiler/src/ir/serialize.ts @@ -42,7 +42,8 @@ function encodeNumber(_key: string, value: unknown): unknown { return value; } -function encodeFunction(fn: IrFunction): string | undefined { +/** One function in the compact encoding (deserializeFunction reads it). */ +export function encodeFunction(fn: IrFunction): string | undefined { return ordinaryNumbers(fn) ? JSON.stringify(fn) : JSON.stringify(fn, encodeNumber); } @@ -127,6 +128,14 @@ function checkedVersion(mod: IrModule): IrModule { return mod; } +/** A function from its compact encoding (encodeFunction). */ +export function deserializeFunction(json: string): IrFunction { + // Only an encoding that holds a number sentinel needs the reviver. + return ( + json.includes('"$nonfinite"') ? JSON.parse(json, decodeNumber) : JSON.parse(json) + ) as IrFunction; +} + export function deserializeModule(json: string): IrModule { return checkedVersion(JSON.parse(json, decodeNumber) as IrModule); } diff --git a/packages/compiler/test/recorded-lowering.test.ts b/packages/compiler/test/recorded-lowering.test.ts new file mode 100644 index 000000000..a40f33964 --- /dev/null +++ b/packages/compiler/test/recorded-lowering.test.ts @@ -0,0 +1,70 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { compile } from "../src/index.js"; + +// Programs whose units add the entries unit records carry (lifted closures +// and helpers, interned helper names, statement counts, generic demands, +// virtual edges) and grow tables that leave a unit without a record (new +// records and unions, generic instances, key-order notes). +const PROGRAMS = [ + "1003-json-parse-unions.ts", + "1200-regex-test-basics.ts", + "1304-errors-uncaught.ts", + "1573-tdz-scalar-forward-capture.ts", + "1631-inspect-arrays.ts", + "4900-console-instances.ts", + "5047-generator-delegation-iterables-typed.ts", + "5390-interface-loop-element-lifetimes.ts", + "6496-key-order-built-and-parsed.ts", + "6916-index-record-spread-overrides.ts", + "6951-unknown-cast-class-methods.ts", +]; + +async function lower(program: string, dir: string, label: string): Promise { + const entry = join(import.meta.dirname, "../../../tests/corpus", program); + const outPath = join(dir, `${program}.${label}.ir.json`); + const result = await compile(entry, { outDir: dir, outPath, outputKind: "ir" }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + return readFile(outPath, "utf8"); +} + +afterEach(() => { + delete process.env["SCRIPTC_LOWER_JOURNAL"]; +}); + +test("units that keep what their records carry give the IR lowering gives", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-recorded-lowering-")); + try { + for (const program of PROGRAMS) { + delete process.env["SCRIPTC_LOWER_JOURNAL"]; + const plain = await lower(program, dir, "plain"); + process.env["SCRIPTC_LOWER_JOURNAL"] = "serial"; + const recorded = await lower(program, dir, "recorded"); + expect(recorded, program).toBe(plain); + } + } finally { + await rm(dir, { recursive: true, force: true }); + } +}, 300_000); + +test("two builds of a program write the same records", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-recorded-lowering-dump-")); + try { + const dumps: string[] = []; + for (const label of ["first", "second"]) { + process.env["SCRIPTC_LOWER_JOURNAL_DUMP"] = join(dir, label); + await lower("1573-tdz-scalar-forward-capture.ts", dir, label); + dumps.push(await readFile(join(dir, label, "records.jsonl"), "utf8")); + } + expect(dumps[0]!.length).toBeGreaterThan(0); + expect(dumps[1]).toBe(dumps[0]); + for (const line of dumps[0]!.trimEnd().split("\n")) + expect(typeof (JSON.parse(line) as { name: unknown }).name).toBe("string"); + } finally { + delete process.env["SCRIPTC_LOWER_JOURNAL_DUMP"]; + await rm(dir, { recursive: true, force: true }); + } +}, 300_000); diff --git a/scripts/recorded-lowering.mjs b/scripts/recorded-lowering.mjs new file mode 100644 index 000000000..ec9a528da --- /dev/null +++ b/scripts/recorded-lowering.mjs @@ -0,0 +1,145 @@ +#!/usr/bin/env node +/* Checks that unit records stand for the units they record. + * + * Each corpus program is compiled to LLVM text twice: as usual, and with + * SCRIPTC_LOWER_JOURNAL=serial, where every reachable-emit unit that has a + * record keeps the function and table entries its record carries + * (frontend/lowering/unit-journal.ts). The two texts must be the same. + * + * node scripts/recorded-lowering.mjs [--stride 1] [--offset 0] [--limit N] + * [--jobs 8] [--entry ]... [--out ] + * + * --stride and --offset sample the corpus entries in name order. Needs a + * built compiler (pnpm build). Writes /results.json, prints the programs + * whose texts differ and the units each mode recorded, and exits with status + * 1 when a program's texts differ or only one mode builds it. */ +import { spawn } from "node:child_process"; +import { globSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { cpus, tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { parseArgs } from "node:util"; + +const root = fileURLToPath(new URL("..", import.meta.url)); +const dist = join(root, "packages/compiler/dist"); + +if (process.argv[2] === "--child") { + const [, , , entry, outPath] = process.argv; + const { compile } = await import(pathToFileURL(join(dist, "index.js")).href); + const outDir = join(tmpdir(), `scriptc-recorded-lowering-${process.pid}`); + try { + const result = await compile(entry, { outDir, outPath, outputKind: "llvm" }); + if (!result.ok) + process.stderr.write( + `${result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")}\n`, + ); + process.exitCode = result.ok ? 0 : 3; + } finally { + rmSync(outDir, { recursive: true, force: true }); + } +} else { + await main(); +} + +async function main() { + const { values } = parseArgs({ + options: { + stride: { type: "string", default: "1" }, + offset: { type: "string", default: "0" }, + limit: { type: "string", default: "100000" }, + jobs: { type: "string", default: String(Math.max(1, Math.floor(cpus().length / 2))) }, + entry: { type: "string", multiple: true, default: [] }, + out: { type: "string", default: join(tmpdir(), "scriptc-recorded-lowering") }, + }, + }); + const out = resolve(values.out); + mkdirSync(out, { recursive: true }); + const corpus = join(root, "tests/corpus"); + const sampled = + values.entry.length > 0 + ? values.entry.map((entry) => resolve(entry)) + : ["ts", "js", "mjs", "cjs"] + .flatMap((ext) => [ + ...globSync(join(corpus, `*.${ext}`)), + ...globSync(join(corpus, `*/main.${ext}`)), + ]) + .sort() + .filter((_, index) => index % Number(values.stride) === Number(values.offset)) + .slice(0, Number(values.limit)); + + const child = (entry, outPath, journal) => + new Promise((done) => { + const env = { ...process.env, SCRIPTC_NO_CACHE: "1", SCRIPTC_TIMING: "1" }; + if (journal) env["SCRIPTC_LOWER_JOURNAL"] = "serial"; + else delete env["SCRIPTC_LOWER_JOURNAL"]; + const proc = spawn( + process.execPath, + ["--max-old-space-size=8192", fileURLToPath(import.meta.url), "--child", entry, outPath], + { stdio: ["ignore", "ignore", "pipe"], env }, + ); + let stderr = ""; + proc.stderr.on("data", (chunk) => (stderr += chunk)); + proc.on("close", (code) => done({ ok: code === 0, stderr })); + }); + + /** The unit-records line SCRIPTC_TIMING prints, or null. */ + const recordStats = (stderr) => { + const line = stderr.split("\n").find((text) => text.includes('"phase":"unit-records"')); + return line === undefined ? null : JSON.parse(line.slice(line.indexOf("{"))); + }; + + const results = []; + let next = 0; + const work = async () => { + for (;;) { + const index = next++; + if (index >= sampled.length) return; + const entry = sampled[index]; + const name = entry.slice(corpus.length + 1).replaceAll("/", "__"); + const plainPath = join(out, `${name}.plain.ll`); + const recordedPath = join(out, `${name}.recorded.ll`); + const plain = await child(entry, plainPath, false); + const recorded = await child(entry, recordedPath, true); + const result = { entry, plain: plain.ok, recorded: recorded.ok, stats: null, same: false }; + result.stats = recordStats(recorded.stderr); + if (plain.ok && recorded.ok) + result.same = readFileSync(plainPath, "utf8") === readFileSync(recordedPath, "utf8"); + if (result.same || (!plain.ok && !recorded.ok)) { + rmSync(plainPath, { force: true }); + rmSync(recordedPath, { force: true }); + } + if (plain.ok !== recorded.ok) + result.error = (plain.ok ? recorded.stderr : plain.stderr).slice(-2000); + results.push(result); + } + }; + await Promise.all(Array.from({ length: Number(values.jobs) }, work)); + results.sort((a, b) => (a.entry < b.entry ? -1 : a.entry > b.entry ? 1 : 0)); + writeFileSync(join(out, "results.json"), JSON.stringify(results, null, 2)); + + const built = results.filter((result) => result.plain && result.recorded); + const differ = built.filter((result) => !result.same); + const oneMode = results.filter((result) => result.plain !== result.recorded); + const totals = { units: 0, recorded: 0, unrecorded: 0, reasons: {} }; + for (const { stats } of results) { + if (stats === null) continue; + totals.units += stats.units; + totals.recorded += stats.recorded; + totals.unrecorded += stats.unrecorded; + for (const [reason, count] of Object.entries(stats.reasons)) + totals.reasons[reason] = (totals.reasons[reason] ?? 0) + count; + } + console.log( + `${results.length} programs, ${built.length} built, ${differ.length} differ, ${oneMode.length} build in one mode only`, + ); + console.log( + `units: ${totals.units}, recorded ${totals.recorded}, without a record ${totals.unrecorded}`, + ); + for (const [reason, count] of Object.entries(totals.reasons).sort((a, b) => b[1] - a[1])) + console.log(` ${reason}: ${count}`); + for (const result of differ) console.log(`differs: ${result.entry}`); + for (const result of oneMode) + console.log(`builds ${result.plain ? "plain" : "recorded"} only: ${result.entry}`); + console.log(`results: ${join(out, "results.json")}`); + if (differ.length > 0 || oneMode.length > 0) process.exitCode = 1; +} From 9ca9c7af9dbecae8817d5ce36a35dd8cb76c6848 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 06:53:26 -0500 Subject: [PATCH 11/54] Run JavaScript web streams on a native ReadableStream In JavaScript sources, ReadableStream, ReadableStreamDefaultReader, ReadableStreamDefaultController, CountQueuingStrategy and ByteLengthQueuingStrategy (globals and node:stream/web exports) are class values over a new runtime unit, scr_webstreams.c, which follows Node 24's webstreams step by step: underlying sources, queuing strategies, default readers, tee, ReadableStream.from and async iteration settle their promises in Node's order. scr_dyn_handle_class.c gives a dyn handle tag a constructor value with a prototype, brand checks and Node's constructor errors. Stream state carries cycle headers, so the collector reclaims streams whose callbacks close over their controllers. Promise steps run as native promise reactions, a variant of scr_async.c that only programs with web streams link; every other program links the default object. TypeScript sources keep their static web stream lowerings. Byte streams, BYOB readers, pipeTo and pipeThrough throw SC2020. node24-web-stream-readers and node24-web-stream-readable-lifecycle now match Node. --- .../builtin-apis/streams-and-readline.mdx | 5 +- .../generated/node-v24-internal.json | 2 + internal/compatibility/static-support.json | 2 +- .../ambient/node-fallback/stream-web.d.ts | 25 +- .../ambient/scriptc-node-fallback.d.ts | 85 + .../src/backend/executable-features.ts | 1 + .../compiler/src/backend/llvm/lib-dispatch.ts | 5 + .../compiler/src/backend/llvm/lib-shared.ts | 2 + .../src/backend/native-link-info-core.ts | 3 + .../compiler/src/backend/native/contracts.ts | 3 + .../compiler/src/backend/native/executable.ts | 4 + .../compiler/src/backend/runtime-pack-core.ts | 2 + .../src/executable/executable-cache.ts | 1 + .../lowering/builtins/module-bindings.ts | 7 + .../lowering/builtins/web-streams-dyn.ts | 187 ++ .../frontend/lowering/expressions/binary.ts | 6 +- .../lowering/expressions/identifiers.ts | 5 + .../lowering/expressions/narrowing.ts | 13 + .../lowering/expressions/property-access.ts | 6 + .../lowering/expressions/relational.ts | 3 + .../src/frontend/lowering/lower-calls.ts | 18 + .../src/frontend/lowering/lower-stdlib.ts | 22 +- .../src/frontend/lowering/lower-stmts.ts | 12 + .../compiler/src/frontend/lowering/lowerer.ts | 3 + .../src/frontend/lowering/surfaces.ts | 4 + packages/compiler/src/index.ts | 1 + .../compiler/src/ir/builtin-signatures.ts | 2 + packages/compiler/src/ir/runtime-features.ts | 14 + .../compiler/src/native/sanitizer.test.ts | 1 + .../web-stream-pull-after-script.mjs.json | 6 + .../web-stream-readable-untyped.mjs.json | 6 + .../runtime-darwin-arm64/scripts/build.mjs | 1 + .../runtime-pack-matrix.mjs | 11 + .../runtime-pack-common/scripts/build.mjs | 1 + packages/runtime/src/scr_async.c | 63 +- packages/runtime/src/scr_async.h | 14 + packages/runtime/src/scr_dyn.h | 63 +- packages/runtime/src/scr_dyn_handle_class.c | 241 ++ packages/runtime/src/scr_fetch.c | 15 + packages/runtime/src/scr_generator.h | 10 + packages/runtime/src/scr_json.c | 15 + packages/runtime/src/scr_stream.c | 6 +- packages/runtime/src/scr_stream_from_web.c | 3 +- packages/runtime/src/scr_webstreams.c | 2508 +++++++++++++++++ tests/corpus/web-stream-pull-after-script.mjs | 14 + tests/corpus/web-stream-readable-untyped.mjs | 130 + .../web-stream-pull-after-script.mjs.json | 1 + .../web-stream-readable-untyped.mjs.json | 1 + tests/harness/npm.test.ts | 14 +- 49 files changed, 3537 insertions(+), 30 deletions(-) create mode 100644 packages/compiler/src/frontend/lowering/builtins/web-streams-dyn.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/web-stream-pull-after-script.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/web-stream-readable-untyped.mjs.json create mode 100644 packages/runtime/src/scr_dyn_handle_class.c create mode 100644 packages/runtime/src/scr_webstreams.c create mode 100644 tests/corpus/web-stream-pull-after-script.mjs create mode 100644 tests/corpus/web-stream-readable-untyped.mjs create mode 100644 tests/harness/corpus-costs/web-stream-pull-after-script.mjs.json create mode 100644 tests/harness/corpus-costs/web-stream-readable-untyped.mjs.json diff --git a/docs/content/limitations/builtin-apis/streams-and-readline.mdx b/docs/content/limitations/builtin-apis/streams-and-readline.mdx index 25427a307..25e944994 100644 --- a/docs/content/limitations/builtin-apis/streams-and-readline.mdx +++ b/docs/content/limitations/builtin-apis/streams-and-readline.mdx @@ -60,4 +60,7 @@ The `readline/promises` `Readline` class writes to the same outputs. `readline.emitKeypressEvents` emits `'keypress'` events on `process.stdin` or a readable stream, shared with terminal interfaces on that stream as in Node; a listener's first parameter must admit `undefined`, which it receives for escape sequences. Chained listener registration is unsupported. - `node:stream/web` exports the same `ReadableStream` class as the global; its other classes are unsupported. + `node:stream/web` exports the same `ReadableStream` class as the global; in TypeScript, its other classes are unsupported. + In JavaScript, `ReadableStream`, `ReadableStreamDefaultReader`, `ReadableStreamDefaultController`, `CountQueuingStrategy`, and `ByteLengthQueuingStrategy`, as globals and as `node:stream/web` exports, are constructor values with Node's prototypes, brand checks, and errors. + Their underlying sources (`start`, `pull`, `cancel`), queuing strategies with a `size` function, readers, `tee`, `ReadableStream.from`, and async iteration settle promises in Node's order. + In JavaScript, a byte stream (`type: 'bytes'`), `pipeTo`, `pipeThrough`, `Object.getPrototypeOf` on these streams, readers, controllers, and strategies, their use as `WeakMap` and `WeakSet` keys, `ReadableStream.from` of a function that is not iterable, and the other `node:stream/web` classes are unsupported. diff --git a/internal/compatibility/generated/node-v24-internal.json b/internal/compatibility/generated/node-v24-internal.json index aac6cfb14..43aa5e38b 100644 --- a/internal/compatibility/generated/node-v24-internal.json +++ b/internal/compatibility/generated/node-v24-internal.json @@ -111928,6 +111928,7 @@ "tests/corpus/7743-transform-stream-identity.ts", "tests/corpus/stream-web-readable.ts", "tests/corpus/stream-web-require.cjs", + "tests/corpus/web-stream-readable-untyped.mjs", "tests/harness/fetch-conformance.test.ts" ] }, @@ -112200,6 +112201,7 @@ "evidence": "compiler-feature:webstreams.ReadableStream", "tests": [ "tests/corpus/stream-web-readable.ts", + "tests/corpus/web-stream-readable-untyped.mjs", "tests/corpus/stream-web-require.cjs", "tests/corpus/2862-web-readable-async-iteration.ts" ] diff --git a/internal/compatibility/static-support.json b/internal/compatibility/static-support.json index 033e199c7..25a693c7c 100644 --- a/internal/compatibility/static-support.json +++ b/internal/compatibility/static-support.json @@ -433,7 +433,7 @@ { "chapter": "zlib", "signatures": ["`zlib.constants`"], "status": "partial", "evidence": ["tests/corpus/5376-zlib-constants.cjs", "tests/corpus/5374-zlib-stream-flush-params.ts"] }, { "chapter": "dns", "symbols": ["dns.lookup", "dns.lookupService", "dns.getServers", "dns.getDefaultResultOrder", "dns.setDefaultResultOrder"], "status": "partial", "evidence": ["tests/corpus/5648-dns-lookup-forms.ts", "tests/corpus/5649-dns-service-servers.ts", "tests/corpus/5650-dns-lookup-errors-untyped.cjs"] }, { "chapter": "dns", "symbols": ["dnsPromises.lookup", "dnsPromises.lookupService", "dnsPromises.getDefaultResultOrder"], "status": "partial", "evidence": ["tests/corpus/dns-promises-lookup.ts", "tests/corpus/dns-promises-untyped.cjs"] }, - { "chapter": "webstreams", "symbols": ["ReadableStream"], "status": "partial", "evidence": ["tests/corpus/stream-web-readable.ts", "tests/corpus/stream-web-require.cjs", "tests/corpus/2862-web-readable-async-iteration.ts"] }, + { "chapter": "webstreams", "symbols": ["ReadableStream"], "status": "partial", "evidence": ["tests/corpus/stream-web-readable.ts", "tests/corpus/web-stream-readable-untyped.mjs", "tests/corpus/stream-web-require.cjs", "tests/corpus/2862-web-readable-async-iteration.ts"] }, { "chapter": "webstreams", "symbols": ["TransformStream"], "status": "partial", "evidence": ["tests/corpus/7743-transform-stream-identity.ts"] }, { "chapter": "globals", "signatures": ["Class: `TransformStream`"], "status": "partial", "evidence": ["tests/corpus/7743-transform-stream-identity.ts"] } ], diff --git a/packages/compiler/ambient/node-fallback/stream-web.d.ts b/packages/compiler/ambient/node-fallback/stream-web.d.ts index f947e667d..6761186cc 100644 --- a/packages/compiler/ambient/node-fallback/stream-web.d.ts +++ b/packages/compiler/ambient/node-fallback/stream-web.d.ts @@ -1,13 +1,32 @@ /* node:stream/web — the module exports the web stream globals themselves - * (require("stream/web").ReadableStream === globalThis.ReadableStream). - * ReadableStream is the implemented class; its reader and controller are - * exported as types. */ + * (require("stream/web").ReadableStream === globalThis.ReadableStream). */ declare module "stream/web" { export const ReadableStream: typeof globalThis.ReadableStream; export type ReadableStream = globalThis.ReadableStream; + export const ReadableStreamDefaultReader: typeof globalThis.ReadableStreamDefaultReader; export type ReadableStreamDefaultReader = globalThis.ReadableStreamDefaultReader; + export const ReadableStreamDefaultController: typeof globalThis.ReadableStreamDefaultController; export type ReadableStreamDefaultController = globalThis.ReadableStreamDefaultController; + export const WritableStream: typeof globalThis.WritableStream; + export type WritableStream = globalThis.WritableStream; + export const WritableStreamDefaultWriter: typeof globalThis.WritableStreamDefaultWriter; + export type WritableStreamDefaultWriter = globalThis.WritableStreamDefaultWriter; + export const WritableStreamDefaultController: typeof globalThis.WritableStreamDefaultController; + export type WritableStreamDefaultController = globalThis.WritableStreamDefaultController; + export const TransformStream: typeof globalThis.TransformStream; + export type TransformStream = globalThis.TransformStream; + export const TransformStreamDefaultController: typeof globalThis.TransformStreamDefaultController; + export type TransformStreamDefaultController = + globalThis.TransformStreamDefaultController; + export const CountQueuingStrategy: typeof globalThis.CountQueuingStrategy; + export type CountQueuingStrategy = globalThis.CountQueuingStrategy; + export const ByteLengthQueuingStrategy: typeof globalThis.ByteLengthQueuingStrategy; + export type ByteLengthQueuingStrategy = globalThis.ByteLengthQueuingStrategy; + export const TextEncoderStream: typeof globalThis.TextEncoderStream; + export type TextEncoderStream = globalThis.TextEncoderStream; + export const TextDecoderStream: typeof globalThis.TextDecoderStream; + export type TextDecoderStream = globalThis.TextDecoderStream; export type UnderlyingSource = globalThis.UnderlyingSource; export type QueuingStrategy = globalThis.QueuingStrategy; } diff --git a/packages/compiler/ambient/scriptc-node-fallback.d.ts b/packages/compiler/ambient/scriptc-node-fallback.d.ts index dff6b6b05..51b1f5ceb 100644 --- a/packages/compiler/ambient/scriptc-node-fallback.d.ts +++ b/packages/compiler/ambient/scriptc-node-fallback.d.ts @@ -1084,6 +1084,12 @@ interface ReadableStreamDefaultController { close(): void; error(reason?: unknown): void; } +interface StreamPipeOptions { + preventAbort?: boolean; + preventCancel?: boolean; + preventClose?: boolean; + signal?: AbortSignal; +} interface UnderlyingSource { start?(controller: ReadableStreamDefaultController): unknown; pull?(controller: ReadableStreamDefaultController): unknown; @@ -1093,6 +1099,12 @@ interface ReadableStream { readonly locked: boolean; cancel(reason?: unknown): Promise; getReader(): ReadableStreamDefaultReader; + tee(): [ReadableStream, ReadableStream]; + pipeTo(destination: WritableStream, options?: StreamPipeOptions): Promise; + pipeThrough( + transform: { readable: ReadableStream; writable: WritableStream }, + options?: StreamPipeOptions, + ): ReadableStream; [Symbol.asyncIterator](options?: { preventCancel?: boolean }): AsyncIterableIterator; values(options?: { preventCancel?: boolean }): AsyncIterableIterator; } @@ -1131,6 +1143,79 @@ declare var TransformStream: { new (): TransformStream; readonly prototype: TransformStream; }; +/* The other web stream interfaces. JavaScript sources construct and read + * them as checked-dynamic values (scr_webstreams.c); TypeScript sources + * have no static lowering for them. */ +declare var ReadableStreamDefaultReader: { + new (stream: ReadableStream): ReadableStreamDefaultReader; + readonly prototype: ReadableStreamDefaultReader; +}; +declare var ReadableStreamDefaultController: { + new (): ReadableStreamDefaultController; + readonly prototype: ReadableStreamDefaultController; +}; +interface WritableStreamDefaultController { + readonly signal: AbortSignal; + error(reason?: unknown): void; +} +declare var WritableStream: { + new (sink?: object, strategy?: QueuingStrategy): WritableStream; + readonly prototype: WritableStream; +}; +declare var WritableStreamDefaultWriter: { + new (stream: WritableStream): WritableStreamDefaultWriter; + readonly prototype: WritableStreamDefaultWriter; +}; +declare var WritableStreamDefaultController: { + new (): WritableStreamDefaultController; + readonly prototype: WritableStreamDefaultController; +}; +interface TransformStreamDefaultController { + readonly desiredSize: number | null; + enqueue(chunk?: O): void; + error(reason?: unknown): void; + terminate(): void; +} +declare var TransformStreamDefaultController: { + new (): TransformStreamDefaultController; + readonly prototype: TransformStreamDefaultController; +}; +interface CountQueuingStrategy extends QueuingStrategy { + readonly highWaterMark: number; + readonly size: QueuingStrategySize; +} +declare var CountQueuingStrategy: { + new (init: { highWaterMark: number }): CountQueuingStrategy; + readonly prototype: CountQueuingStrategy; +}; +interface ByteLengthQueuingStrategy extends QueuingStrategy { + readonly highWaterMark: number; + readonly size: QueuingStrategySize; +} +declare var ByteLengthQueuingStrategy: { + new (init: { highWaterMark: number }): ByteLengthQueuingStrategy; + readonly prototype: ByteLengthQueuingStrategy; +}; +interface TextEncoderStream { + readonly encoding: "utf-8"; + readonly readable: ReadableStream; + readonly writable: WritableStream; +} +declare var TextEncoderStream: { + new (): TextEncoderStream; + readonly prototype: TextEncoderStream; +}; +interface TextDecoderStream { + readonly encoding: string; + readonly fatal: boolean; + readonly ignoreBOM: boolean; + readonly readable: ReadableStream; + readonly writable: WritableStream; +} +declare var TextDecoderStream: { + new (label?: string, options?: { fatal?: boolean; ignoreBOM?: boolean }): TextDecoderStream; + readonly prototype: TextDecoderStream; +}; /* fetch, typed as the native JSON + readable-stream slice: one URL, an * optional init with the members CLI requests carry, and a Response diff --git a/packages/compiler/src/backend/executable-features.ts b/packages/compiler/src/backend/executable-features.ts index 1d95285b4..acc47b326 100644 --- a/packages/compiler/src/backend/executable-features.ts +++ b/packages/compiler/src/backend/executable-features.ts @@ -36,6 +36,7 @@ export function executableLinkFeatures(mod: IrModule, largePages = false): Nativ ...(features.dynForAwait ? { dynForAwait: true } : {}), ...(features.eventsOnceSignal ? { eventsOnceSignal: true } : {}), ...(features.mime ? { mime: true } : {}), + ...(features.webStreams ? { webStreams: true } : {}), ...(features.utilTiming ? { utilTiming: true } : {}), ...(features.dynBuffer ? { dynBuffer: true } : {}), ...(features.builtinValues ? { builtinValues: true } : {}), diff --git a/packages/compiler/src/backend/llvm/lib-dispatch.ts b/packages/compiler/src/backend/llvm/lib-dispatch.ts index c3acf766f..aac165743 100644 --- a/packages/compiler/src/backend/llvm/lib-dispatch.ts +++ b/packages/compiler/src/backend/llvm/lib-dispatch.ts @@ -1239,6 +1239,11 @@ export function emitLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue { case "navigator": case "mime": return host.emitGenericLibCall(e); + case "webstream": + // A stream's start, pull and cancel steps run as promise jobs, so + // a program with the classes runs its event loop. + if (e.fn === "webstream.class") host.usesTimers = true; + return host.emitGenericLibCall(e); case "test": return emitTestLibCall(host, e); case "tls": diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 747c9c883..090d84154 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -118,6 +118,8 @@ export const LIB_FN_SYMS: Record = { "util.diff": "scr_util_diff", "util.debounce": "scr_util_debounce", "util.throttle": "scr_util_throttle", + "webstream.class": "scr_webstream_class", + "webstream.instanceOf": "scr_webstream_instanceof", "mime.typeNew": "scr_mime_type_new", "mime.typeParse": "scr_mime_type_parse", "mime.paramsNew": "scr_mime_params_new", diff --git a/packages/compiler/src/backend/native-link-info-core.ts b/packages/compiler/src/backend/native-link-info-core.ts index a1ecc66a3..9cbc4b477 100644 --- a/packages/compiler/src/backend/native-link-info-core.ts +++ b/packages/compiler/src/backend/native-link-info-core.ts @@ -67,6 +67,9 @@ export interface NativeLinkFeatures { /** `--large-pages`: scr_text_pages.c, which moves the executable's code * onto transparent huge pages at startup. Absent (false) without it. */ largePages?: boolean; + /** Web streams in checked-dynamic code: scr_webstreams.c and + * scr_dyn_handle_class.c. */ + webStreams?: boolean; /** util.debounce and util.throttle: scr_util_timing.c. */ utilTiming?: boolean; /** Buffer.prototype methods called on checked-dynamic values link diff --git a/packages/compiler/src/backend/native/contracts.ts b/packages/compiler/src/backend/native/contracts.ts index ee0533ce3..fc7059b8e 100644 --- a/packages/compiler/src/backend/native/contracts.ts +++ b/packages/compiler/src/backend/native/contracts.ts @@ -131,6 +131,9 @@ export interface CcOptions { mime?: boolean; /** `--large-pages`: compiles scr_text_pages.c. */ largePages?: boolean; + /** Web streams in checked-dynamic code: compiles scr_webstreams.c and + * scr_dyn_handle_class.c. */ + webStreams?: boolean; /** util.debounce and util.throttle: compiles scr_util_timing.c. */ utilTiming?: boolean; /** Buffer.prototype methods called on checked-dynamic values: diff --git a/packages/compiler/src/backend/native/executable.ts b/packages/compiler/src/backend/native/executable.ts index 22e7cf2cb..cb0a49359 100644 --- a/packages/compiler/src/backend/native/executable.ts +++ b/packages/compiler/src/backend/native/executable.ts @@ -248,6 +248,8 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [ unit((u) => u.opts.eventsOnceSignal === true, "scr_events_once_signal.c"), unit((u) => u.opts.mime === true, "scr_mime.c"), unit((u) => u.opts.largePages === true, "scr_text_pages.c"), + unit((u) => u.opts.webStreams === true, "scr_webstreams.c"), + unit((u) => u.opts.webStreams === true, "scr_dyn_handle_class.c"), unit((u) => u.opts.utilTiming === true, "scr_util_timing.c"), // The checked-dynamic HANDLE support unit (listener gate + runtime // adapter closures): every referencing unit is one of the emitter or @@ -390,6 +392,8 @@ const RUNTIME_DEFINES: readonly { { define: "SCR_JS_TIER", when: (opts) => opts.dynTier === true }, { define: "SCR_WARNING_REPLACE", when: (opts) => opts.warningReplace }, { define: "SCR_STDIO_EXTENDED", when: (opts) => opts.stdioExtended }, + // scr_async.c's native promise reactions, which the web streams unit uses. + { define: "SCR_NATIVE_REACTIONS", when: (opts) => opts.webStreams }, { define: "SCR_YIN", when: (opts) => opts.youngNursery }, ]; diff --git a/packages/compiler/src/backend/runtime-pack-core.ts b/packages/compiler/src/backend/runtime-pack-core.ts index 1ac7e2718..9888c600b 100644 --- a/packages/compiler/src/backend/runtime-pack-core.ts +++ b/packages/compiler/src/backend/runtime-pack-core.ts @@ -92,6 +92,7 @@ export interface RuntimeFeatureSet extends NativeLinkFeatures { eventsOnceSignal: boolean; mime: boolean; largePages: boolean; + webStreams: boolean; utilTiming: boolean; dynBuffer: boolean; builtinValues: boolean; @@ -297,6 +298,7 @@ export function effectiveRuntimeFeatures( eventsOnceSignal: features.eventsOnceSignal === true, mime: features.mime === true, largePages: features.largePages === true, + webStreams: features.webStreams === true, utilTiming: features.utilTiming === true, dynBuffer: features.dynBuffer === true, builtinValues: features.builtinValues === true, diff --git a/packages/compiler/src/executable/executable-cache.ts b/packages/compiler/src/executable/executable-cache.ts index 737a78b98..dff207b02 100644 --- a/packages/compiler/src/executable/executable-cache.ts +++ b/packages/compiler/src/executable/executable-cache.ts @@ -81,6 +81,7 @@ export interface EarlyExecutableNativeFeatures { dynForAwait?: boolean; eventsOnceSignal?: boolean; mime?: boolean; + webStreams?: boolean; utilTiming?: boolean; dynBuffer?: boolean; builtinValues?: boolean; diff --git a/packages/compiler/src/frontend/lowering/builtins/module-bindings.ts b/packages/compiler/src/frontend/lowering/builtins/module-bindings.ts index 98cc188f8..47aedfee9 100644 --- a/packages/compiler/src/frontend/lowering/builtins/module-bindings.ts +++ b/packages/compiler/src/frontend/lowering/builtins/module-bindings.ts @@ -31,6 +31,13 @@ export function builtinImportOf( return binding?.module === "stream/web" ? null : binding; } +/** The member a node:stream/web import binding names (`import { + * WritableStream } from "node:stream/web"`, or its CommonJS twin), or null. */ +export function streamWebImportOf(lowerer: Lowerer, ident: ts.Identifier): string | null { + const binding = builtinModuleBindingOf(lowerer, ident); + return binding?.module === "stream/web" ? binding.member : null; +} + function builtinModuleBindingOf( lowerer: Lowerer, ident: ts.Identifier, diff --git a/packages/compiler/src/frontend/lowering/builtins/web-streams-dyn.ts b/packages/compiler/src/frontend/lowering/builtins/web-streams-dyn.ts new file mode 100644 index 000000000..c465c2920 --- /dev/null +++ b/packages/compiler/src/frontend/lowering/builtins/web-streams-dyn.ts @@ -0,0 +1,187 @@ +import { numLit } from "../../../ir/build.js"; +import { BOOL, DYN, STRING, type IrExpr, type SrcLoc } from "../../../ir/ir.js"; +import * as ts from "../../ts7/adapter.js"; +import { isJsSourceFile, locOf } from "../../program.js"; +import { constituentTypes } from "../../ts7/checker.js"; +import type { Lowerer } from "../lowerer.js"; +import { stdlibGlobalNameOf } from "../surfaces.js"; +import { streamWebImportOf } from "./module-bindings.js"; + +/* The web stream classes in JavaScript sources: the globals and the + * node:stream/web exports, which are the same constructors. Each reference + * is the class's constructor value (scr_webstreams.c, webstream.class), so + * construction, static members, prototypes and instanceof go through the + * checked-dynamic paths over that value. TypeScript sources keep their + * static lowerings (lower-stdlib.ts). */ + +/** The classes in scr_webstream_class's order. */ +export const WEB_STREAM_CLASSES = [ + "ReadableStream", + "ReadableStreamDefaultReader", + "ReadableStreamDefaultController", + "CountQueuingStrategy", + "ByteLengthQueuingStrategy", +] as const; + +const CLASS_NAMES: readonly string[] = WEB_STREAM_CLASSES; + +function unparenthesized(node: ts.Expression): ts.Expression { + let current = node; + while (ts.isParenthesizedExpression(current)) current = current.expression; + return current; +} + +/** The class a JavaScript expression names (an identifier bound to the + * global or to a node:stream/web import, or a member of the global + * object), as its index in WEB_STREAM_CLASSES; null otherwise. */ +export function webStreamClassKindOf(lowerer: Lowerer, node: ts.Expression): number | null { + const expr = unparenthesized(node); + if (!ts.isIdentifier(expr) && !ts.isPropertyAccessExpression(expr)) return null; + if (!isJsSourceFile(ts.getSourceFileOfNode(expr))) return null; + const member = ts.isPropertyAccessExpression(expr) ? lowerer.builtinMemberOf(expr) : null; + const name = + (ts.isIdentifier(expr) ? streamWebImportOf(lowerer, expr) : null) ?? + (member?.module === "stream/web" ? member.member : null) ?? + stdlibGlobalNameOf(lowerer, expr); + if (name === null) return null; + const index = CLASS_NAMES.indexOf(name); + return index < 0 ? null : index; +} + +export function webStreamClassValue(kind: number, loc: SrcLoc): IrExpr { + return { kind: "libCall", fn: "webstream.class", args: [numLit(kind, loc)], type: DYN, loc }; +} + +/** A JavaScript read of one of the classes as a value; null otherwise. */ +export function lowerWebStreamClassRead(lowerer: Lowerer, node: ts.Expression): IrExpr | null { + const kind = webStreamClassKindOf(lowerer, node); + return kind === null ? null : webStreamClassValue(kind, locOf(node)); +} + +/** `new X(...args)` on one of the classes in JavaScript. */ +export function lowerWebStreamNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr | null { + const kind = webStreamClassKindOf(lowerer, expr.expression); + if (kind === null) return null; + const nodes = expr.arguments?.nodes ?? []; + if (nodes.some(ts.isSpreadElement)) return null; + const loc = locOf(expr); + return { + kind: "libCall", + fn: "dyn.construct", + args: [ + webStreamClassValue(kind, loc), + { + kind: "dynArrLit", + elems: nodes.map((arg) => lowerer.lowerExprExpecting(arg, DYN)), + type: DYN, + loc, + }, + { kind: "strLit", value: CLASS_NAMES[kind]!, type: STRING, loc }, + ], + type: DYN, + loc, + }; +} + +/** `X.from(…)` and other calls of a class's static members in JavaScript: + * a member call on the constructor value. */ +export function lowerWebStreamStaticCall(lowerer: Lowerer, call: ts.CallExpression): IrExpr | null { + const access = call.expression; + if (call.questionDotToken || !ts.isPropertyAccessExpression(access) || access.questionDotToken) + return null; + const kind = webStreamClassKindOf(lowerer, access.expression); + if (kind === null) return null; + const nodes = call.arguments.nodes; + if (nodes.some(ts.isSpreadElement)) return null; + const loc = locOf(call); + return { + kind: "dynInvoke", + recv: webStreamClassValue(kind, locOf(access.expression)), + method: access.name.text, + calleeName: ts.getTextOfNode(access), + args: nodes.map((arg) => lowerer.lowerExprExpecting(arg, DYN)), + type: DYN, + loc, + }; +} + +/** `value instanceof X` on one of the classes in JavaScript. */ +export function lowerWebStreamInstanceof( + lowerer: Lowerer, + expr: ts.BinaryExpression, +): IrExpr | null { + const kind = webStreamClassKindOf(lowerer, expr.right); + if (kind === null) return null; + const loc = locOf(expr); + return { + kind: "libCall", + fn: "webstream.instanceOf", + args: [lowerer.lowerExprExpecting(expr.left, DYN), numLit(kind, loc)], + type: BOOL, + loc, + }; +} + +/** The ambient web stream interfaces whose members JavaScript reads from + * checked-dynamic values. */ +const WEB_STREAM_INTERFACES: ReadonlySet = new Set([ + ...WEB_STREAM_CLASSES, + "ReadableStreamReadResult", + "ReadableStreamReadValueResult", + "ReadableStreamReadDoneResult", + "UnderlyingSource", + "QueuingStrategy", + "QueuingStrategySize", +]); + +/** Whether a JavaScript member read names a member of a web stream + * interface, which the receiver's checked-dynamic value answers. */ +export function isWebStreamMemberInJavaScript( + lowerer: Lowerer, + access: ts.PropertyAccessExpression, +): boolean { + if (!isJsSourceFile(ts.getSourceFileOfNode(access))) return false; + const member = lowerer.checker.getSymbolAtLocation(access.name); + const declaration = member?.declarations[0]; + let owner: ts.Node | undefined = declaration?.parent; + while ( + owner !== undefined && + !ts.isInterfaceDeclaration(owner) && + !ts.isTypeAliasDeclaration(owner) + ) + owner = owner.parent; + return owner !== undefined && WEB_STREAM_INTERFACES.has(owner.name.text); +} + +/** Whether a checker type is, or is a promise of, a web stream interface + * (a read result, a reader's closed promise). JavaScript keeps such values + * checked-dynamic, as their handles answered them: a typed copy would lose + * the value's identity (a chunk object, a promise). */ +export function typeInvolvesWebStream(lowerer: Lowerer, type: ts.Type, depth = 0): boolean { + if (depth > 4) return false; + if ((type.flags & ts.TypeFlags.Union) !== 0) + return constituentTypes(type).some((t) => typeInvolvesWebStream(lowerer, t, depth + 1)); + for (const symbol of [type.getSymbol(), type.getAliasSymbol()]) { + if (symbol === undefined) continue; + const name = ts.symbolName(symbol); + if (WEB_STREAM_INTERFACES.has(name) && lowerer.isStdlibSymbol(symbol)) return true; + if (name === "Promise") { + const args = lowerer.checker.getTypeArguments(type as ts.TypeReference); + if (args.some((t) => typeInvolvesWebStream(lowerer, t, depth + 1))) return true; + } + } + return false; +} + +/** Whether a JavaScript expression produces a web stream value: a member of + * a web stream interface (`reader.closed`), a call of one (`reader.read()`), + * an await of either, or a value whose type involves one. */ +export function isWebStreamValueInJavaScript(lowerer: Lowerer, node: ts.Expression): boolean { + if (!isJsSourceFile(ts.getSourceFileOfNode(node))) return false; + let expr = unparenthesized(node); + if (ts.isAwaitExpression(expr)) expr = unparenthesized(expr.expression); + if (ts.isCallExpression(expr)) expr = unparenthesized(expr.expression); + if (ts.isPropertyAccessExpression(expr) && isWebStreamMemberInJavaScript(lowerer, expr)) + return true; + return typeInvolvesWebStream(lowerer, lowerer.typeOf(node)); +} diff --git a/packages/compiler/src/frontend/lowering/expressions/binary.ts b/packages/compiler/src/frontend/lowering/expressions/binary.ts index 5c326ab1e..bfff4b954 100644 --- a/packages/compiler/src/frontend/lowering/expressions/binary.ts +++ b/packages/compiler/src/frontend/lowering/expressions/binary.ts @@ -1599,9 +1599,9 @@ export function lowerBinary(lowerer: Lowerer, expr: ts.BinaryExpression): IrExpr reference.kind === "map" || reference.kind === "set" || reference.kind === "union" || - // A promise of dyn boxes its own promise, so identity holds; - // other promise types box a fresh adapter. - (reference.kind === "promise" && reference.inner.kind === "dyn") || + // A promise of dyn boxes its own promise; other promise types + // box an adapter that shares the promise's identity. + reference.kind === "promise" || DYN_HANDLE_KINDS.has(reference.kind)) && lowerer.dynConvertible(reference) ) { diff --git a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts index 1b21bf0c7..88b822803 100644 --- a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts +++ b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts @@ -3,6 +3,7 @@ import { globalMemberValue } from "../global-values.js"; import { predeclareForwardClass } from "../lower-stmts.js"; import { cjsMemberAliasValueOf } from "../cjs-exports-object.js"; import { builtinClassValue, builtinMemberClassKind } from "../builtins/builtin-class-dyn.js"; +import { lowerWebStreamClassRead } from "../builtins/web-streams-dyn.js"; import * as ts from "../../ts7/adapter.js"; import { InternalCompilerError } from "../../../errors.js"; import * as posix from "node:path/posix"; @@ -770,6 +771,10 @@ export function lowerIdentifierExpr(lowerer: Lowerer, expr: ts.Identifier, loc: // global, one token; what a token cannot do (be called, answer // typeof "function") meets per-site fences/divergences, and // SEMANTICS.md documents the stance. + { + const webStream = lowerWebStreamClassRead(lowerer, expr); + if (webStream) return webStream; + } if (stdlibGlobalNameOf(lowerer, expr) === "globalThis") { return lowerGlobalValue(lowerer, expr); } diff --git a/packages/compiler/src/frontend/lowering/expressions/narrowing.ts b/packages/compiler/src/frontend/lowering/expressions/narrowing.ts index 91b63cabc..24c3e80c6 100644 --- a/packages/compiler/src/frontend/lowering/expressions/narrowing.ts +++ b/packages/compiler/src/frontend/lowering/expressions/narrowing.ts @@ -1,5 +1,9 @@ /* Type narrowing of lowered values, conditional and as-expressions, and runtime-optional receiver reads. */ import { declBoundaryOrigin } from "../../decl-projection.js"; +import { + isWebStreamMemberInJavaScript, + typeInvolvesWebStream, +} from "../builtins/web-streams-dyn.js"; import * as ts from "../../ts7/adapter.js"; import { type Lowerer, dynFallbackType } from "../lowerer.js"; import { @@ -427,6 +431,15 @@ export function maybeNarrow(lowerer: Lowerer, expr: IrExpr, node: ts.Node): IrEx // A process.versions entry declared as a string can be absent here. if (isOptionalProcessVersionsRead(lowerer, node)) return lowerer.coerceToExpected(expr, lowerer.withUndefinedArm(STRING)); + // A web stream member in JavaScript (a reader's closed promise, a read + // result's value) keeps the checked-dynamic value its handle answered. + if (ts.isPropertyAccessExpression(node) && isWebStreamMemberInJavaScript(lowerer, node)) + return expr; + if ( + isJsSourceFile(ts.getSourceFileOfNode(node)) && + typeInvolvesWebStream(lowerer, lowerer.typeOf(node)) + ) + return expr; // An inferred JS member's storage may be deliberately wider than its // declaration (prototype-selected buffers, null-initialized scratch // objects). Extract at the consuming operation, not at the read. diff --git a/packages/compiler/src/frontend/lowering/expressions/property-access.ts b/packages/compiler/src/frontend/lowering/expressions/property-access.ts index 58f07f890..e15204324 100644 --- a/packages/compiler/src/frontend/lowering/expressions/property-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/property-access.ts @@ -11,6 +11,7 @@ import { indexedUnionArms, lowerIndexedUnionMember } from "../indexed-unions.js" import { keyedHelperName } from "../stable-names.js"; import { builtinNamespaceValue } from "../builtin-namespaces.js"; import { staticallySelected } from "../static-typeof.js"; +import { lowerWebStreamClassRead } from "../builtins/web-streams-dyn.js"; import * as ts from "../../ts7/adapter.js"; import { lowerHistogramProperty } from "../builtins/performance.js"; import { staticFunctionNameRead } from "../static-function-names.js"; @@ -595,6 +596,11 @@ export function lowerPropertyAccessExpr( if (lowerer.stdlibGlobalMember(expr, "globalThis") === "console") { return { kind: "libCall", fn: "console.native", args: [], type: DYN, loc }; } + { + // `globalThis.ReadableStream` in JavaScript: the class value. + const webStream = lowerWebStreamClassRead(lowerer, expr); + if (webStream) return webStream; + } if (lowerer.stdlibGlobalMember(expr, "console") !== null) { return { kind: "dynKeyGet", diff --git a/packages/compiler/src/frontend/lowering/expressions/relational.ts b/packages/compiler/src/frontend/lowering/expressions/relational.ts index d29226685..7cc573085 100644 --- a/packages/compiler/src/frontend/lowering/expressions/relational.ts +++ b/packages/compiler/src/frontend/lowering/expressions/relational.ts @@ -2,6 +2,7 @@ import { lowerArrayClassInstanceOf } from "../builtins/array-classes.js"; import { dateRootOf } from "../builtins/date-classes.js"; import { keyedHelperName } from "../stable-names.js"; +import { lowerWebStreamInstanceof } from "../builtins/web-streams-dyn.js"; import * as ts from "../../ts7/adapter.js"; import { builtinClassInstanceOfAny, builtinClassKind } from "../builtins/builtin-class-dyn.js"; import { @@ -386,6 +387,8 @@ export function lowerInstanceOf(lowerer: Lowerer, expr: ts.BinaryExpression, loc loc, }; } + const webStream = lowerWebStreamInstanceof(lowerer, expr); + if (webStream) return webStream; if (lowerer.isStdlibGlobal(expr.right, "ReadableStream")) { const value = lowerer.lowerExprExpecting(expr.left, DYN); return { kind: "libCall", fn: "fetch.streamIs", args: [value], type: BOOL, loc }; diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 5f151c7a9..91113b4e8 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -6102,6 +6102,24 @@ export function lowerPromiseMethodCall( const fulfilledAbsent = isAbsentHandler(fulfilledNode); const rejectedAbsent = isAbsentHandler(rejectedNode); const resultT = lowerer.mapTypeOf(lowerer.typeOf(call)); + // An untyped call in JavaScript whose receiver arrived as a typed + // promise (a function instantiated with one): Promise.prototype.then + // over the promise as a checked-dynamic value, which shares its + // identity. + if (resultT?.kind !== "promise" && isJsSourceFile(ts.getSourceFileOfNode(call))) { + return { + kind: "dynInvoke", + recv: lowerer.coerceToExpected(receiver, DYN), + method: "then", + calleeName: ts.getTextOfNode(access), + args: [ + lowerer.lowerExprExpecting(fulfilledNode, DYN), + lowerer.lowerExprExpecting(rejectedNode, DYN), + ], + type: DYN, + loc, + }; + } if (resultT?.kind !== "promise") { lowerer.noLowering( "then with these handlers", diff --git a/packages/compiler/src/frontend/lowering/lower-stdlib.ts b/packages/compiler/src/frontend/lowering/lower-stdlib.ts index fe748bb93..889d0aeb4 100644 --- a/packages/compiler/src/frontend/lowering/lower-stdlib.ts +++ b/packages/compiler/src/frontend/lowering/lower-stdlib.ts @@ -1,6 +1,7 @@ import { dynUndefinedExpr, nodeThrowExpr } from "../../ir/build.js"; import { InternalCompilerError } from "../../errors.js"; +import { webStreamClassKindOf } from "./builtins/web-streams-dyn.js"; import * as ts from "../ts7/adapter.js"; import { lowerToNumberArguments } from "./lower-exprs.js"; import type { Lowerer } from "./lowerer.js"; @@ -31,7 +32,13 @@ import { unsupportedNativeApiDiag, unsupportedPackageValueDiag, } from "../../diagnostics/diagnostic.js"; -import { esmNamedImportLinkCrash, isCjsJsFile, locOf, npmPackageNameOf } from "../program.js"; +import { + esmNamedImportLinkCrash, + isCjsJsFile, + isJsSourceFile, + locOf, + npmPackageNameOf, +} from "../program.js"; import { foldedStringKeyOf, lowerDynObjectLiteral } from "./expressions/object-literals.js"; import { noteEventListeners } from "./lower-event-target.js"; import { PoisonError, newFnCtx, own } from "./lowerer.js"; @@ -1026,6 +1033,12 @@ export function fenceStaticAbortControllerMemberRead( ); } +/** JavaScript sources read web stream values through their handles' + * checked-dynamic surfaces (scr_webstreams.c), not the static lowerings. */ +function inJavaScript(node: ts.Node): boolean { + return isJsSourceFile(ts.getSourceFileOfNode(node)); +} + function isStdlibFetchInterface(lowerer: Lowerer, node: ts.Node, owner: string): ts.Symbol | null { const type = lowerer.checker.getBaseTypeOfLiteralType(lowerer.typeOf(node)); for (const symbol of [type.getSymbol(), type.getAliasSymbol()]) { @@ -1377,6 +1390,7 @@ export function fenceStaticReadableStreamMember( access: StaticResponseAccess, use: "read" | "call", ): IrExpr | null { + if (inJavaScript(access.expression)) return null; const sym = isStdlibFetchInterface(lowerer, access.expression, "ReadableStream"); if (!sym) return null; const members = fetchAccessMemberNames(lowerer, access); @@ -1481,6 +1495,9 @@ export function lowerStaticFetchCompanionCall( return null; } const root = call.expression.expression; + // JavaScript reads the web stream classes as constructor values + // (web-streams-dyn.ts). + if (webStreamClassKindOf(lowerer, root) !== null) return null; const symbol = lowerer.resolveValueSymbol(root); if ((!symbol || !lowerer.isStdlibSymbol(symbol)) && !isStreamWebReadableStream(lowerer, root)) return null; @@ -1683,6 +1700,7 @@ export function lowerStaticReadableStreamControllerCall( ): IrExpr | null { const access = call.expression; if ( + inJavaScript(call) || call.questionDotToken || call.arguments.nodes.length > 1 || (!ts.isPropertyAccessExpression(access) && !ts.isElementAccessExpression(access)) || @@ -1739,6 +1757,7 @@ export function lowerStaticReadableStreamCancelCall( ): IrExpr | null { const access = call.expression; if ( + inJavaScript(call) || call.questionDotToken || call.arguments.nodes.length > 1 || call.arguments.nodes.some((argument) => ts.isSpreadElement(argument)) || @@ -2163,6 +2182,7 @@ export function lowerStaticReadableStreamReaderCall( ): IrExpr | null { const access = call.expression; if ( + inJavaScript(call) || call.questionDotToken || call.arguments.nodes.length !== 0 || (!ts.isPropertyAccessExpression(access) && !ts.isElementAccessExpression(access)) || diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index d43aaabf8..08f468850 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -34,6 +34,7 @@ import { InternalCompilerError } from "../../errors.js"; * declarations including destructuring patterns, scoped blocks, control * flow (if/while/for/for-of/do, switch, try/catch, abrupt completions), * explicit resource management, and blocked-binding poisoning. */ +import { isWebStreamValueInJavaScript, typeInvolvesWebStream } from "./builtins/web-streams-dyn.js"; import * as ts from "../ts7/adapter.js"; import { lowerOptionalIndex } from "./undefined-index.js"; import { processDataMemberValue, programReadsExitCode } from "./builtins/process.js"; @@ -6110,6 +6111,17 @@ export function lowerVarDecl( // Promise.prototype is an ordinary object the checker types Promise. if (!decl.type && !hasJsTypeAnnotation(decl) && isPromisePrototypeRead(lowerer, decl.initializer)) type = DYN; + // A web stream value in JavaScript keeps its checked-dynamic identity. + if ( + init.type.kind === "dyn" && + !decl.type && + !hasJsTypeAnnotation(decl) && + decl.initializer !== undefined && + (isWebStreamValueInJavaScript(lowerer, decl.initializer) || + (isJsSourceFile(ts.getSourceFileOfNode(decl)) && + typeInvolvesWebStream(lowerer, lowerer.typeOf(decl.name)))) + ) + type = DYN; // A const whose initializer has another record or array layout than the // binding's type keeps the initializer's layout: converting would copy the // value, so the binding would not be the value its initializer names. diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 0c63997df..e67687a7e 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -90,6 +90,7 @@ import { fenceWebCryptoMember, lowerWebCryptoCall } from "./builtins/webcrypto.j import { valueStreamOf } from "./builtins/hash-stream.js"; import { lowerNetAddressCall, lowerNetAddressNew } from "./builtins/net-address.js"; import { lowerMimeCall, lowerMimeNew } from "./builtins/mime.js"; +import { lowerWebStreamNew, lowerWebStreamStaticCall } from "./builtins/web-streams-dyn.js"; import { dynUndefinedExpr, nodeThrowExpr, numLit, varRef } from "../../ir/build.js"; import { iteratorSlotAdapter } from "./iterator-adapters.js"; import { iterableSlotValue } from "./iterable-values.js"; @@ -15082,6 +15083,7 @@ export class Lowerer { lowerStaticFetchCompanionCall(this, expr) ?? lowerStaticAbortControllerCall(this, expr) ?? lowerStaticAbortSignalListenerCall(this, expr) ?? + lowerWebStreamStaticCall(this, expr) ?? lowerStaticReadableStreamCancelCall(this, expr) ?? lowerStaticReadableStreamControllerCall(this, expr) ?? lowerStaticReadableStreamReaderCall(this, expr) ?? @@ -15334,6 +15336,7 @@ export class Lowerer { lowerBlobNew(this, expr) ?? lowerNetAddressNew(this, expr) ?? lowerMimeNew(this, expr) ?? + lowerWebStreamNew(this, expr) ?? lowerStaticReadableStreamNew(this, expr) ?? lowerStaticTransformStreamNew(this, expr) ?? lowerConsoleNew(this, expr); diff --git a/packages/compiler/src/frontend/lowering/surfaces.ts b/packages/compiler/src/frontend/lowering/surfaces.ts index 3520e3aea..97fd3ccda 100644 --- a/packages/compiler/src/frontend/lowering/surfaces.ts +++ b/packages/compiler/src/frontend/lowering/surfaces.ts @@ -1,3 +1,4 @@ +import { isWebStreamMemberInJavaScript } from "./builtins/web-streams-dyn.js"; import * as ts from "../ts7/adapter.js"; import type { Lowerer } from "./lowerer.js"; import { staticCondition, staticallySelected } from "./static-typeof.js"; @@ -3279,6 +3280,9 @@ export function stdlibMemberFence(lowerer: Lowerer, access: ts.PropertyAccessExp // below this fence answers, member-or-refusal ladder, so no compile // fence belongs here. if (lowerer.mapTypeOf(lowerer.typeOf(recv))?.kind === "dyn") return; + // JavaScript reads web stream values (read results, sources, strategies) + // as checked-dynamic objects (web-streams-dyn.ts). + if (isWebStreamMemberInJavaScript(lowerer, access)) return; // Name the container the way the source reads: the global's name when // the receiver IS a stdlib global (Math, process), the dotted path for // a member of one (process.stdout — its TYPE text would be the useless diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index a41a6b4c7..f92d7c455 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -702,6 +702,7 @@ async function compileExecutableNative( ...(features.eventsOnceSignal ? { eventsOnceSignal: true } : {}), ...(features.mime ? { mime: true } : {}), ...(features.largePages ? { largePages: true } : {}), + ...(features.webStreams ? { webStreams: true } : {}), ...(features.utilTiming ? { utilTiming: true } : {}), ...(features.dynBuffer ? { dynBuffer: true } : {}), ...(features.builtinValues ? { builtinValues: true } : {}), diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index e50e61a92..a088ccdeb 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -1178,6 +1178,8 @@ function buildLibFnSignatures() { "util.diff": { argTypes: [DYN, DYN], result: DYN }, "util.debounce": { argTypes: [DYN, DYN, DYN], result: DYN }, "util.throttle": { argTypes: [DYN, DYN, DYN, DYN], result: DYN }, + "webstream.class": { argTypes: [F64], result: DYN }, + "webstream.instanceOf": { argTypes: [DYN, F64], result: BOOL }, "mime.typeNew": { argTypes: [DYN, DYN], result: DYN }, "mime.typeParse": { argTypes: [DYN], result: DYN }, "mime.paramsNew": { argTypes: [], result: DYN }, diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index 0fb4ce815..99a98a66c 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -80,6 +80,9 @@ const DYN_ASYNC_LIB_FNS: ReadonlySet = new Set([ "crypto.native", // The crypto global's object extends crypto.native's. "webcrypto.object", + // Web streams settle their promises with dyn values. + "webstream.class", + "webstream.instanceOf", "dyn.promiseAll", "dyn.promiseAllSettled", "dyn.promiseAny", @@ -384,6 +387,9 @@ const BUILTIN_CLASS_LIB_FNS: readonly IrLibFn[] = [ /** Entry points that link the symbol unit, besides the sym. and disposable. families. */ const SYMBOL_LIB_FNS: ReadonlySet = new Set([ + // Web streams answer async iteration through Symbol.asyncIterator. + "webstream.class", + "webstream.instanceOf", // EventEmitter's kShapeMode and kCapture properties ...BUILTIN_CLASS_LIB_FNS, "dyn.usingDispose", @@ -945,6 +951,9 @@ const RUNTIME_FEATURE_RULES = { ...BUILTIN_CLASS_LIB_FNS, "util.promisifyDyn", "buffer.prototypeMember", + // scr_webstreams.c calls the iterators ReadableStream.from reads. + "webstream.class", + "webstream.instanceOf", ], // Constant-key reads of Object.prototype member names answer the // inherited member through scr_dyn_invoke.c. @@ -1033,6 +1042,8 @@ const RUNTIME_FEATURE_RULES = { }), /** util.MIMEType and util.MIMEParams (scr_mime.c). */ mime: rule({ prefixes: ["mime."] }), + /** Web streams in checked-dynamic code (scr_webstreams.c). */ + webStreams: rule({ prefixes: ["webstream."] }), /** util.debounce and util.throttle (scr_util_timing.c). */ utilTiming: rule({ fns: ["util.debounce", "util.throttle"] }), /** events.once(emitter, name, { signal }) (scr_events_once_signal.c). */ @@ -1060,6 +1071,8 @@ const RUNTIME_FEATURE_RULES = { inspect: rule({ fns: [ "global.native", + // ERR_ARG_NOT_ITERABLE renders the value as util.format's %s does. + "webstream.class", "util.styleText", "crypto.cipherNew", "webcrypto.encrypt", @@ -1254,6 +1267,7 @@ function noRuntimeFeatures(): RuntimeFeatures { inspect: false, legacyTextDecoder: false, mime: false, + webStreams: false, utilTiming: false, mockTimers: false, net: false, diff --git a/packages/compiler/src/native/sanitizer.test.ts b/packages/compiler/src/native/sanitizer.test.ts index c4b1bd651..826623e93 100644 --- a/packages/compiler/src/native/sanitizer.test.ts +++ b/packages/compiler/src/native/sanitizer.test.ts @@ -23,6 +23,7 @@ const features = { eventsOnceSignal: false, mime: false, largePages: false, + webStreams: false, utilTiming: false, dynBuffer: false, builtinValues: false, diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/web-stream-pull-after-script.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/web-stream-pull-after-script.mjs.json new file mode 100644 index 000000000..ee8d71f5e --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/web-stream-pull-after-script.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/web-stream-pull-after-script.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/web-stream-readable-untyped.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/web-stream-readable-untyped.mjs.json new file mode 100644 index 000000000..8752b14a6 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/web-stream-readable-untyped.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/web-stream-readable-untyped.mjs" + ], + "diags": [] +} diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index 95fe91ab3..60b1527f3 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -376,6 +376,7 @@ async function build() { "SCR_TEXT_DECODER_LEGACY", "SCR_WARNING_REPLACE", "SCR_STDIO_EXTENDED", + "SCR_NATIVE_REACTIONS", "SCR_YIN", ], excluded: ["SCR_RC_AUDIT", "SCR_ASAN_FIBERS"], diff --git a/packages/runtime-pack-common/runtime-pack-matrix.mjs b/packages/runtime-pack-common/runtime-pack-matrix.mjs index 0f2a69c49..5ddbc7ce3 100644 --- a/packages/runtime-pack-common/runtime-pack-matrix.mjs +++ b/packages/runtime-pack-common/runtime-pack-matrix.mjs @@ -103,6 +103,8 @@ export const OPTIONAL_RUNTIME_SOURCES = [ ["scr_mime.c", "mime"], // `scriptc build --large-pages`: executable code on transparent huge pages (Linux). ["scr_text_pages.c", "largePages"], + ["scr_webstreams.c", "webStreams"], + ["scr_dyn_handle_class.c", "webStreams"], ["scr_util_timing.c", "utilTiming"], ["scr_dyn_handle.c", any("emitter", "netEffective")], ["scr_symbol.c", "symbol"], @@ -212,6 +214,15 @@ export function variantsFor(source) { defines: ["SCR_WARNING_REPLACE"], }); } + // Programs with web streams link the promise unit with native promise + // reactions; every other program keeps the default object. + if (source === "scr_async.c") { + variants.push({ + id: "native-reactions", + when: { webStreams: true }, + defines: ["SCR_NATIVE_REACTIONS"], + }); + } // stdin setEncoding, string data listeners and stdio 'resize' events. if (source === "scr_events.c") { variants.push({ diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index aebf1b017..62fdfbbf1 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -479,6 +479,7 @@ async function build() { "SCR_TEXT_DECODER_LEGACY", "SCR_WARNING_REPLACE", "SCR_STDIO_EXTENDED", + "SCR_NATIVE_REACTIONS", "SCR_YIN", ], excluded: ["SCR_RC_AUDIT", "SCR_ASAN_FIBERS"], diff --git a/packages/runtime/src/scr_async.c b/packages/runtime/src/scr_async.c index fc06a9f06..2319a6e35 100644 --- a/packages/runtime/src/scr_async.c +++ b/packages/runtime/src/scr_async.c @@ -211,12 +211,20 @@ struct ScrPromise { * Combinator entries (Promise.all / Promise.race) are reaction jobs: * `job_at` is the number of waiters parked before the entry subscribed, * so settlement queues the job in subscription order among the waking - * fibers. Conversions (SCR_CB_SYNC) run inside the settle itself. */ + * fibers. Conversions (SCR_CB_SYNC) run inside the settle itself. + * Native reactions (scr_promise_react) are reaction jobs with no dst: + * `reaction` and `ctx` take the places of adapt and all_idx. */ struct ScrPromiseCbWaiter { - void (*adapt)(ScrPromise *dst, ScrPromise *src); + union { + void (*adapt)(ScrPromise *dst, ScrPromise *src); + const ScrPromiseReaction *reaction; + }; ScrPromise *dst; struct ScrAllState *all; - size_t all_idx; + union { + size_t all_idx; + void *ctx; + }; size_t job_at; } *cbs; size_t ncbs, cbs_cap; @@ -288,6 +296,24 @@ static void scr_promise_add_waiter(ScrPromise *p, ScrFiber *f) { p->waiters[p->nwaiters++] = f; } +#ifdef SCR_NATIVE_REACTIONS +/* A native reaction's steps (scr_promise_react). Only the build for + * programs with web streams has them (runtime-pack-matrix.mjs). */ +static void scr_promise_reaction_drop(struct ScrPromiseCbWaiter *cb) { + if (cb->reaction->drop) cb->reaction->drop(cb->ctx); +} + +static void scr_promise_reaction_run(struct ScrPromiseCbWaiter *cb, ScrPromise *src) { + cb->reaction->react(cb->ctx, src); + scr_promise_reaction_drop(cb); +} +#define SCR_PROMISE_IS_REACTION(cb) ((cb)->dst == NULL) +#else +#define SCR_PROMISE_IS_REACTION(cb) false +#define scr_promise_reaction_drop(cb) ((void)0) +#define scr_promise_reaction_run(cb, src) ((void)0) +#endif + static bool scr_promise_has_traced_children(const ScrPromise *p) { return p->ncbs != 0 || ((p->payload_kind == SCR_EXC_REF || p->payload_kind == SCR_EXC_OBJ || @@ -324,7 +350,8 @@ static void scr_promise_gcfree(void *o) { * is reachable only through them) — release like the untraced payload * above; the cb destinations stay untouched (trace visits them). */ for (size_t i = 0; i < p->ncbs; i++) { - if (p->cbs[i].all) scr_promise_all_state_release(p->cbs[i].all); + if (SCR_PROMISE_IS_REACTION(&p->cbs[i])) scr_promise_reaction_drop(&p->cbs[i]); + else if (p->cbs[i].all) scr_promise_all_state_release(p->cbs[i].all); } scr_promise_free_side(p); if (p->cbs) free(p->cbs); @@ -416,6 +443,10 @@ void scr_promise_release(ScrPromise *p) { scr_promise_release_payload(p); scr_promise_free_side(p); for (size_t i = 0; i < p->ncbs; i++) { + if (SCR_PROMISE_IS_REACTION(&p->cbs[i])) { + scr_promise_reaction_drop(&p->cbs[i]); + continue; + } scr_promise_release(p->cbs[i].dst); if (p->cbs[i].all) scr_promise_all_state_release(p->cbs[i].all); } @@ -1635,6 +1666,10 @@ static inline void scr_promise_wake_one(ScrFiber *f) { /* Runs one callback waiter against its settled source and drops the * waiter's references (its destination and countdown state). */ static void scr_promise_cb_run(struct ScrPromiseCbWaiter *cb, ScrPromise *src) { + if (SCR_PROMISE_IS_REACTION(cb)) { + scr_promise_reaction_run(cb, src); + return; + } if (cb->all) { scr_promise_all_settle(cb->all, cb->dst, cb->all_idx, src); scr_promise_all_state_release(cb->all); @@ -1695,6 +1730,15 @@ static void scr_promise_cb_subscribe(ScrPromise *in, struct ScrPromiseCbWaiter c } } +#ifdef SCR_NATIVE_REACTIONS +void scr_promise_react(ScrPromise *p, const ScrPromiseReaction *reaction, void *ctx) { + struct ScrPromiseCbWaiter cb = {0}; + cb.reaction = reaction; + cb.ctx = ctx; + scr_promise_cb_subscribe(p, cb); +} +#endif + /* Settlement with callback waiters: combinator jobs enter the microtask * queue interleaved with the waking fibers in subscription order, then * the conversions run in place (they settle their destinations now, so @@ -1741,7 +1785,7 @@ static void scr_promise_settle_wake(ScrPromise *p) { void scr_promise_race_join(ScrPromise *race, ScrPromise *in, void (*adapt)(ScrPromise *dst, ScrPromise *src)) { scr_promise_cb_subscribe( - in, (struct ScrPromiseCbWaiter){ adapt, scr_promise_retain(race), NULL, 0, 0 }); + in, (struct ScrPromiseCbWaiter){ .adapt = adapt, .dst = scr_promise_retain(race) }); } void scr_promise_race_add(ScrPromise *race, ScrPromise *in, @@ -1752,7 +1796,7 @@ void scr_promise_race_add(ScrPromise *race, ScrPromise *in, return; } *scr_promise_cb_push(in) = - (struct ScrPromiseCbWaiter){ adapt, scr_promise_retain(race), NULL, 0, SCR_CB_SYNC }; + (struct ScrPromiseCbWaiter){ .adapt = adapt, .dst = scr_promise_retain(race), .job_at = SCR_CB_SYNC }; } /* ── Promise.all / allSettled / any ─────────────────────────────────── @@ -1805,7 +1849,7 @@ __attribute__((always_inline)) static inline void scr_combine_add(ScrCombine *c, if (in->state == SCR_PROM_PENDING) { st->rc++; scr_promise_cb_subscribe( - in, (struct ScrPromiseCbWaiter){ NULL, scr_promise_retain(c->result), st, i, 0 }); + in, (struct ScrPromiseCbWaiter){ .dst = scr_promise_retain(c->result), .all = st, .all_idx = i }); return; } scr_prom_observe(in); @@ -1825,8 +1869,8 @@ __attribute__((always_inline)) static inline void scr_combine_flush(ScrCombine * c->st->rc++; job->cb_job = true; job->cb_src = c->decided; /* the reference moves in */ - job->cb = (struct ScrPromiseCbWaiter){ NULL, scr_promise_retain(c->result), c->st, c->batched, - SCR_CB_BATCH }; + job->cb = (struct ScrPromiseCbWaiter){ + .dst = scr_promise_retain(c->result), .all = c->st, .all_idx = c->batched, .job_at = SCR_CB_BATCH}; scr_ready_push(job); c->decided = NULL; c->batched = 0; @@ -6386,6 +6430,7 @@ static void scr_context_ready_discard(void) { scr_gen_release(generator); scr_mem_free(fiber); } else if (fiber->cb_job) { + if (SCR_PROMISE_IS_REACTION(&fiber->cb)) scr_promise_reaction_drop(&fiber->cb); scr_promise_release(fiber->cb.dst); if (fiber->cb.all) scr_promise_all_state_release(fiber->cb.all); scr_promise_release(fiber->cb_src); diff --git a/packages/runtime/src/scr_async.h b/packages/runtime/src/scr_async.h index d2e16be43..dd92c8594 100644 --- a/packages/runtime/src/scr_async.h +++ b/packages/runtime/src/scr_async.h @@ -157,6 +157,20 @@ ScrDyn *scr_await_dyn_value(ScrDyn *v); * settles a fresh result promise, answered BOXED (+1). Microtask-exact * ordering rides the fiber machinery. */ ScrDyn *scr_dyn_promise_then(ScrPromise *src, ScrDyn *onf, ScrDyn *onr, ScrDyn *onfin); +/* A native reaction to a promise's settlement, as PerformPromiseThen with + * native steps: react(ctx, settled) runs as a reaction job once the promise + * settles (from the next microtask checkpoint when it already has), in + * subscription order among the promise's other reactions, and borrows the + * settled promise. drop(ctx), when non-NULL, runs after react, or when the + * promise goes away unsettled. The collector does not trace the context: + * the reaction's references count as outside references while it waits. */ +typedef struct ScrPromiseReaction { + void (*react)(void *ctx, ScrPromise *settled); + void (*drop)(void *ctx); +} ScrPromiseReaction; +/* Subscribes a native reaction to `p` (the descriptor outlives it) and + * marks `p` handled, as a `then` does. */ +void scr_promise_react(ScrPromise *p, const ScrPromiseReaction *reaction, void *ctx); /* ── AsyncLocalStorage (node:async_hooks — scr_async.c) ─────────────── * Stores are process-lived f64 handles; the CONTEXT is an immutable * refcounted snapshot of (store → dyn value) entries riding the fiber diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 3a1e83308..c63d608a2 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -239,7 +239,12 @@ typedef enum { X(WORKER, 35) /* Tags added since the last pin, by name only (see above). */ -#define SCR_DYNH_NEW_TAGS(Y) +#define SCR_DYNH_NEW_TAGS(Y) \ + Y(WS_BYTE_STRATEGY) /* ByteLengthQueuingStrategy (scr_webstreams.c) */ \ + Y(WS_COUNT_STRATEGY) /* CountQueuingStrategy (scr_webstreams.c) */ \ + Y(WS_RCONTROLLER) /* ReadableStreamDefaultController (scr_webstreams.c) */ \ + Y(WS_READABLE) /* ReadableStream for checked-dynamic code (scr_webstreams.c) */ \ + Y(WS_READER) /* ReadableStreamDefaultReader (scr_webstreams.c) */ /* Values below the largest pinned one that no tag holds, each with the * reason. Never reuse one without removing it here. */ @@ -1199,6 +1204,10 @@ void scr_dyn_install_species_symbol(ScrDyn *key); /* Records each well-known symbol the symbol unit creates. */ void scr_dyn_install_well_known_symbol(ScrDyn *key); void scr_dyn_install_to_string_tag_symbol(ScrDyn *key); +/* Whether key is Symbol.asyncIterator. */ +bool scr_dyn_is_async_iterator_symbol(const ScrDyn *key); +/* Symbol.asyncIterator (+1), or NULL before the symbol unit installed it. */ +ScrDyn *scr_dyn_async_iterator_symbol_value(void); /* True when key is Symbol.asyncDispose (registered by the symbol unit when * the program names it); handle symbol_get ops outside that unit use it. */ bool scr_dyn_is_async_dispose_symbol(const ScrDyn *key); @@ -1999,6 +2008,58 @@ typedef struct ScrDynHandleOps { ScrDyn *(*rejected)(void); } ScrDynHandleOps; +/* A WebIDL interface over a handle tag: its constructor value, whose + * prototype holds the tag's methods (ScrDynHandleOps.methods, the same + * function values a member read answers) and the getters below, each + * enumerable and configurable and brand-checked against the tag. Handles of + * the tag answer the prototype from Object.getPrototypeOf, find its other + * members through it, and pass `instanceof` against the constructor. */ +typedef struct ScrDynHandleAccessor { + const char *name; + /* The getter on another receiver answers a rejected promise instead of + * throwing (a promise-valued attribute such as a reader's `closed`). */ + bool rejects; +} ScrDynHandleAccessor; + +typedef struct ScrDynHandleStatic { + const char *name; + uint32_t length; + ScrDynThunk thunk; +} ScrDynHandleStatic; + +typedef struct ScrDynHandleClass { + const char *name; + uint32_t length; + /* `new X(...args)`: the new handle (+1), or NULL with the exception + * pending. NULL = the interface has no constructor ("Illegal + * constructor"). Arguments are borrowed. */ + ScrDyn *(*construct)(ScrDyn *const *args, size_t argc); + /* Getters, read through ScrDynHandleOps.get; NULL name terminates. */ + const ScrDynHandleAccessor *accessors; + /* Static methods of the constructor; NULL name terminates. */ + const ScrDynHandleStatic *statics; + /* The prototype's methods and getters in definition order (the order + * Object.keys reports); NULL = the methods, then the getters. */ + const char *const *order; +} ScrDynHandleClass; + +/* Installs a tag's ops with the class its handles belong to; both outlive + * the program. */ +void scr_dyn_handle_class_install(ScrDynHandleTag tag, const ScrDynHandleOps *ops, const ScrDynHandleClass *klass); +/* The tag's constructor value (+1), created on first use; the tag must be + * installed with a class. */ +ScrDyn *scr_dyn_handle_class_value(ScrDynHandleTag tag); +/* The method `name` of the tag's class as the function value a member + * read answers (+1), or NULL when the class has no such method. */ +ScrDyn *scr_dyn_handle_class_method(ScrDynHandleTag tag, const char *name); +/* The tag's prototype (+1), created on first use. */ +ScrDyn *scr_dyn_handle_class_prototype(ScrDynHandleTag tag); +/* For a class's ScrDynHandleOps.get: a member that is not one of the + * class's methods, from the prototype chain with a handle over `h` as the + * receiver (+1); NULL when the chain has none or with the exception + * pending. */ +ScrDyn *scr_dyn_handle_class_member(ScrDynHandleTag tag, void *h, const char *key, size_t len); + /* The operations a Transform handle gives the stream unit (scr_stream.c): * the slot where the handle keeps its stream (+1) with the stream's * release, update() and final() over bytes (+1 Buffer, or NULL with the diff --git a/packages/runtime/src/scr_dyn_handle_class.c b/packages/runtime/src/scr_dyn_handle_class.c new file mode 100644 index 000000000..2539a54d4 --- /dev/null +++ b/packages/runtime/src/scr_dyn_handle_class.c @@ -0,0 +1,241 @@ +/* WebIDL interfaces over dyn handles (ScrDynHandleClass): the constructor + * value of a handle tag, with a prototype that holds the tag's methods and + * getters. Each member is brand-checked against the tag: on another + * receiver a method or getter raises the class's ERR_INVALID_THIS TypeError, + * or answers a promise rejected with it for the promise-valued ones, as + * Node's implementations do. The constructor runs the class's construct + * entry under `new`, throws Node's "Class constructor X cannot be invoked + * without 'new'" TypeError when called, and throws the ERR_ILLEGAL_CONSTRUCTOR + * TypeError for interfaces without a constructor (controllers). + * + * One constructor and one prototype per tag and thread, so every read of + * the class value is the same function. */ +#include "scr_runtime.h" + +#include +#include +#include + +static SCR_TL const ScrDynHandleClass *hc_classes[SCR_DYNH_CAPACITY]; +static SCR_TL ScrDyn *hc_ctors[SCR_DYNH_CAPACITY]; +static SCR_TL ScrDyn *hc_prototypes[SCR_DYNH_CAPACITY]; +static SCR_TL bool hc_teardown_registered; + +static const ScrDynHandleOps *hc_ops(ScrDynHandleTag tag) { + ScrDyn probe; + probe.kind = SCR_DYN_HANDLE; + probe.v.handle.tag = tag; + probe.v.handle.ptr = NULL; + return scr_dyn_handle_ops_of(&probe); +} + +static ScrDyn *hc_string(const char *text) { + ScrStr *s = scr_str_new(text, strlen(text)); + ScrDyn *d = scr_dyn_new_str(s); + scr_str_release(s); + return d; +} + +/* Defines `name` on target as a data property with the given attributes; + * the value moves in. */ +static void hc_define(ScrDyn *target, const char *name, ScrDyn *value, bool writable, bool enumerable, + bool configurable) { + ScrDyn *descriptor = scr_dyn_new_obj(); + scr_dyn_obj_set(descriptor, "value", 5, value); + scr_dyn_obj_set(descriptor, "writable", 8, scr_dyn_new_bool(writable)); + scr_dyn_obj_set(descriptor, "enumerable", 10, scr_dyn_new_bool(enumerable)); + scr_dyn_obj_set(descriptor, "configurable", 12, scr_dyn_new_bool(configurable)); + ScrDyn *key = hc_string(name); + ScrDyn *defined = scr_dyn_define_property(target, key, descriptor); + scr_dyn_release(key); + scr_dyn_release(descriptor); + if (defined) scr_dyn_release(defined); +} + +static void hc_invalid_this(const ScrDynHandleOps *ops) { + const char *message = ops->invalid_this ? ops->invalid_this : "Illegal invocation"; + scr_throw_error_msg_code(SCR_ERR_TYPE, message, strlen(message), "ERR_INVALID_THIS"); +} + +static ScrDynHandleTag hc_tag_of(ScrClosure *closure) { + return (ScrDynHandleTag)scr_box_get_f64(closure->caps[0]); +} + +/* A getter: closure->fn is the accessor, caps[0] the tag. */ +static ScrDyn *hc_getter(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + (void)args; + (void)argc; + const ScrDynHandleAccessor *accessor = closure->fn; + ScrDynHandleTag tag = hc_tag_of(closure); + const ScrDynHandleOps *ops = hc_ops(tag); + ScrDyn *self = scr_dyn_this_get(); + ScrDyn *result = NULL; + if (self->kind == SCR_DYN_HANDLE && self->v.handle.tag == tag) { + result = ops->get(self->v.handle.ptr, accessor->name, strlen(accessor->name)); + if (!result && !scr_exc_pending()) result = scr_dyn_retain(scr_dyn_undefined()); + } else { + hc_invalid_this(ops); + if (accessor->rejects && ops->rejected) result = ops->rejected(); + } + scr_dyn_release(self); + return result; +} + +/* The constructor: caps[0] is the tag. scr_dyn_construct calls it with a + * fresh receiver whose prototype is the class's; any other receiver means + * a call without `new`. */ +static ScrDyn *hc_construct(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + ScrDynHandleTag tag = hc_tag_of(closure); + const ScrDynHandleClass *klass = hc_classes[tag]; + ScrDyn *self = scr_dyn_this_get(); + bool constructing = false; + if (self->kind == SCR_DYN_OBJ && hc_prototypes[tag]) { + ScrDyn *prototype = scr_dyn_get_prototype(self); + constructing = prototype && prototype == hc_prototypes[tag]; + if (prototype) scr_dyn_release(prototype); + if (scr_exc_pending()) scr_exc_clear(); + } + scr_dyn_release(self); + if (!constructing) { + size_t n = strlen(klass->name) + 64; + char *message = malloc(n); + if (!message) scr_trap("scriptc: out of memory\n"); + int len = snprintf(message, n, "Class constructor %s cannot be invoked without 'new'", klass->name); + scr_throw_error_msg(SCR_ERR_TYPE, message, (size_t)len); + free(message); + return NULL; + } + if (!klass->construct) { + static const char message[] = "Illegal constructor"; + scr_throw_error_msg_code(SCR_ERR_TYPE, message, sizeof message - 1, "ERR_ILLEGAL_CONSTRUCTOR"); + return NULL; + } + return klass->construct(args, argc); +} + +static void hc_teardown(void) { + for (size_t tag = 0; tag < SCR_DYNH_CAPACITY; tag++) { + if (!hc_ctors[tag]) continue; + /* prototype.constructor points back at the constructor. */ + ScrStr *key = scr_str_new("constructor", 11); + scr_dyn_key_set(hc_prototypes[tag], key, scr_dyn_undefined()); + scr_str_release(key); + if (scr_exc_pending()) scr_exc_clear(); + } + for (size_t tag = 0; tag < SCR_DYNH_CAPACITY; tag++) { + if (hc_prototypes[tag]) scr_dyn_release(hc_prototypes[tag]); + if (hc_ctors[tag]) scr_dyn_release(hc_ctors[tag]); + hc_prototypes[tag] = NULL; + hc_ctors[tag] = NULL; + } +} + +static void hc_create(ScrDynHandleTag tag); + +ScrDyn *scr_dyn_handle_class_prototype(ScrDynHandleTag tag) { + if (!hc_prototypes[tag]) { + if (!hc_classes[tag]) return NULL; + hc_create(tag); + } + return scr_dyn_retain(hc_prototypes[tag]); +} + +ScrDyn *scr_dyn_handle_class_method(ScrDynHandleTag tag, const char *name) { + /* A member read on a handle answers the method's one function value; the + * class's get answers no method name, so a handle without a referent + * reads it. */ + ScrDyn probe; + memset(&probe, 0, sizeof probe); + probe.kind = SCR_DYN_HANDLE; + probe.v.handle.tag = tag; + ScrStr *key = scr_str_new(name, strlen(name)); + ScrDyn *value = scr_dyn_handle_key_get(&probe, key); + scr_str_release(key); + if (value && !scr_dyn_is_callable(value)) { + scr_dyn_release(value); + return NULL; + } + return value; +} + +ScrDyn *scr_dyn_handle_class_member(ScrDynHandleTag tag, void *h, const char *key, size_t len) { + const ScrDynHandleMethod *methods = hc_ops(tag)->methods; + for (size_t i = 0; methods && methods[i].name; i++) + if (strlen(methods[i].name) == len && memcmp(methods[i].name, key, len) == 0) return NULL; + ScrDyn *prototype = scr_dyn_handle_class_prototype(tag); + if (!prototype) return NULL; + ScrStr *text = scr_str_new(key, len); + ScrDyn *name = scr_dyn_new_str(text); + scr_str_release(text); + /* Handles compare by referent, so a fresh box over h is the receiver. */ + ScrDyn *receiver = scr_dyn_new_handle(h, tag); + ScrDyn *value = scr_dyn_reflect_get(prototype, name, receiver); + scr_dyn_release(receiver); + scr_dyn_release(name); + scr_dyn_release(prototype); + if (value && value->kind == SCR_DYN_UNDEF) { + scr_dyn_release(value); + return NULL; + } + return value; +} + +static void hc_define_method(ScrDyn *prototype, ScrDynHandleTag tag, const char *name) { + ScrDyn *fn = scr_dyn_handle_class_method(tag, name); + if (fn) hc_define(prototype, name, fn, true, true, true); +} + +static void hc_define_getter(ScrDyn *prototype, ScrDynHandleTag tag, const ScrDynHandleAccessor *a) { + ScrClosure *getter_closure = scr_closure_new((void *)a, 1); + getter_closure->caps[0] = scr_box_new(SCR_BOX_F64); + scr_box_set_f64(getter_closure->caps[0], (double)tag); + ScrDyn *getter = scr_dyn_new_func(getter_closure, hc_getter, 0, "", a->name); + scr_dyn_obj_set_accessor(prototype, a->name, strlen(a->name), getter, NULL); +} + +static void hc_create(ScrDynHandleTag tag) { + const ScrDynHandleOps *ops = hc_ops(tag); + const ScrDynHandleClass *klass = hc_classes[tag]; + if (!klass) scr_trap("scriptc: internal error: dyn handle has no class value\n"); + ScrClosure *closure = scr_closure_new(NULL, 1); + closure->function_kind = SCR_CLOSURE_KIND_OWN_PROTOTYPE; + closure->caps[0] = scr_box_new(SCR_BOX_F64); + scr_box_set_f64(closure->caps[0], (double)tag); + ScrDyn *ctor = scr_dyn_new_func(closure, hc_construct, klass->length, "native:constructor", klass->name); + ScrDyn *prototype = scr_dyn_new_obj(); + hc_ctors[tag] = ctor; + hc_prototypes[tag] = prototype; + hc_define(prototype, "constructor", scr_dyn_retain(ctor), true, false, true); + hc_define(ctor, "prototype", scr_dyn_retain(prototype), false, false, false); + if (klass->order) { + for (const char *const *name = klass->order; *name; name++) { + const ScrDynHandleAccessor *accessor = NULL; + for (const ScrDynHandleAccessor *a = klass->accessors; a && a->name; a++) + if (strcmp(a->name, *name) == 0) accessor = a; + if (accessor) hc_define_getter(prototype, tag, accessor); + else hc_define_method(prototype, tag, *name); + } + } else { + for (const ScrDynHandleMethod *m = ops->methods; m && m->name; m++) hc_define_method(prototype, tag, m->name); + for (const ScrDynHandleAccessor *a = klass->accessors; a && a->name; a++) hc_define_getter(prototype, tag, a); + } + for (const ScrDynHandleStatic *st = klass->statics; st && st->name; st++) { + ScrDyn *fn = scr_dyn_new_func(scr_closure_new(NULL, 0), st->thunk, st->length, "", st->name); + hc_define(ctor, st->name, fn, true, false, true); + } + if (scr_exc_pending()) scr_exc_clear(); + if (!hc_teardown_registered) { + hc_teardown_registered = true; + scr_atexit(hc_teardown); + } +} + +void scr_dyn_handle_class_install(ScrDynHandleTag tag, const ScrDynHandleOps *ops, const ScrDynHandleClass *klass) { + scr_dyn_handle_install(tag, ops); + hc_classes[tag] = klass; +} + +ScrDyn *scr_dyn_handle_class_value(ScrDynHandleTag tag) { + if (!hc_ctors[tag]) hc_create(tag); + return scr_dyn_retain(hc_ctors[tag]); +} diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index 1b30c451d..5f2122486 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -2266,6 +2266,9 @@ ScrDyn *scr_fetch_stream_from_string(ScrStr *iterable) { } bool scr_fetch_reader_read_into(ScrDyn *reader, const ScrFetchReadSink *sink, void *ctx) { + if (scr_web_reader_read_into_fn && reader && reader->kind == SCR_DYN_HANDLE && + reader->v.handle.tag == SCR_DYNH_WS_READER) + return scr_web_reader_read_into_fn(reader, sink, ctx); if (!reader || reader->kind != SCR_DYN_HANDLE || reader->v.handle.tag != SCR_DYNH_WEB_READER) { sf_type_error("Expected a ReadableStreamDefaultReader"); @@ -2275,6 +2278,18 @@ bool scr_fetch_reader_read_into(ScrDyn *reader, const ScrFetchReadSink *sink, vo } ScrPromise *scr_fetch_reader_read(ScrDyn *reader) { + /* A checked-dynamic web stream's reader answers its own read(). */ + if (reader && reader->kind == SCR_DYN_HANDLE && reader->v.handle.tag == SCR_DYNH_WS_READER) { + ScrDyn *result = scr_dyn_invoke(reader, "read", NULL, 0, "reader.read"); + if (!result) { + ScrPromise *p = scr_promise_new(); + scr_promise_reject_pending(p); + return p; + } + ScrPromise *p = scr_promise_retain(result->v.promise); + scr_dyn_release(result); + return p; + } if (!reader || reader->kind != SCR_DYN_HANDLE || reader->v.handle.tag != SCR_DYNH_WEB_READER) { sf_type_error("Expected a ReadableStreamDefaultReader"); diff --git a/packages/runtime/src/scr_generator.h b/packages/runtime/src/scr_generator.h index 4f43892b4..722ee1f88 100644 --- a/packages/runtime/src/scr_generator.h +++ b/packages/runtime/src/scr_generator.h @@ -223,5 +223,15 @@ typedef struct ScrFetchReadSink { /* Queue a sink read request on a reader handle (borrowed). false with a * TypeError pending for a value that is not a reader or a released one. */ bool scr_fetch_reader_read_into(ScrDyn *reader, const ScrFetchReadSink *sink, void *ctx); +/* The same for the readers of checked-dynamic web streams + * (scr_webstreams.c installs it); NULL in programs without them. */ +extern SCR_TL bool (*scr_web_reader_read_into_fn)(ScrDyn *reader, const ScrFetchReadSink *sink, void *ctx); +/* The web stream class constructors for checked-dynamic code, by kind + * (scr_webstreams.c: ReadableStream, ReadableStreamDefaultReader, + * ReadableStreamDefaultController, CountQueuingStrategy, + * ByteLengthQueuingStrategy); +1. */ +ScrDyn *scr_webstream_class(double kind); +/* `value instanceof ` for those classes (value borrowed). */ +bool scr_webstream_instanceof(ScrDyn *value, double kind); #endif /* SCR_GENERATOR_H */ diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index eb40100cf..b88271d3a 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -6981,6 +6981,9 @@ static ScrDyn *scr_dynh_method_call(ScrClosure *closure, ScrDyn *const *args, si return result; } +/* scr_generator.h: the web stream readers' sink reads (scr_webstreams.c). */ +SCR_TL bool (*scr_web_reader_read_into_fn)(ScrDyn *reader, const ScrFetchReadSink *sink, void *ctx) = NULL; + /* The method `key` of the handle's class as a function value (+1), or NULL * when the class has no such method. */ static ScrDyn *scr_dynh_method_value(ScrDynHandleTag tag, const ScrDynHandleOps *ops, const char *key, size_t len) { @@ -19622,3 +19625,15 @@ bool scr_dyn_is_class_capsule(const ScrDyn *d) { return d && d->kind == SCR_DYN_TYPED_REF && d->v.typed_ref.type_key_len > 7 && memcmp(d->v.typed_ref.type_key, "object:", 7) == 0; } + +/* Symbol.asyncIterator for units that answer async iteration (the web + * streams): the symbol value (+1), or NULL before it exists, and whether a + * key is it. */ +ScrDyn *scr_dyn_async_iterator_symbol_value(void) { + return scr_async_iterator_symbol ? scr_dyn_retain(scr_async_iterator_symbol) : NULL; +} + +bool scr_dyn_is_async_iterator_symbol(const ScrDyn *key) { + return scr_async_iterator_symbol && key->kind == SCR_DYN_SYMBOL && + key->v.symbol.value == scr_async_iterator_symbol->v.symbol.value; +} diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index fdd7179e5..872586976 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -4379,7 +4379,8 @@ static int scr_pipe_dest_dyn_kind(const ScrPipeStages *fs, ScrDyn *r, void **out int kind = scr_dyn_iterable_kind(r); if (kind < 0) return -1; if (r->kind == SCR_DYN_HANDLE && - (r->v.handle.tag == SCR_DYNH_WEB_STREAM || r->v.handle.tag == SCR_DYNH_WEB_TRANSFORM)) { + (r->v.handle.tag == SCR_DYNH_WEB_STREAM || r->v.handle.tag == SCR_DYNH_WEB_TRANSFORM || + r->v.handle.tag == SCR_DYNH_WS_READABLE)) { scr_stream_throw_web_stage(); return -1; } @@ -8840,7 +8841,8 @@ static bool scr_stream_dyn_has_method(const ScrDyn *value, const char *key) { static bool scr_stream_dyn_stream_like(const ScrDyn *value) { if (value->kind == SCR_DYN_HANDLE) { int tag = value->v.handle.tag; - return tag == SCR_DYNH_WEB_STREAM || tag == SCR_DYNH_WEB_TRANSFORM || tag == SCR_DYNH_WEB_WRITABLE; + return tag == SCR_DYNH_WEB_STREAM || tag == SCR_DYNH_WEB_TRANSFORM || tag == SCR_DYNH_WEB_WRITABLE || + tag == SCR_DYNH_WS_READABLE; } if (value->kind != SCR_DYN_OBJ) return false; static const char *const state_keys[] = {"_readableState", "_writableState"}; diff --git a/packages/runtime/src/scr_stream_from_web.c b/packages/runtime/src/scr_stream_from_web.c index f88826335..f627605fe 100644 --- a/packages/runtime/src/scr_stream_from_web.c +++ b/packages/runtime/src/scr_stream_from_web.c @@ -207,7 +207,8 @@ static void sfw_closed_entry(ScrFiber *fiber, void *arg) { * pending. */ ScrStream *scr_stream_readable_from_web(ScrDyn *web, bool object_mode, double hwm, ScrStr *encoding) { if (encoding && encoding->len == 0) encoding = NULL; - if (!web || web->kind != SCR_DYN_HANDLE || web->v.handle.tag != SCR_DYNH_WEB_STREAM) { + if (!web || web->kind != SCR_DYN_HANDLE || + (web->v.handle.tag != SCR_DYNH_WEB_STREAM && web->v.handle.tag != SCR_DYNH_WS_READABLE)) { scr_dyn_arg_type_fail("readableStream", "an instance of ReadableStream", web); return NULL; } diff --git a/packages/runtime/src/scr_webstreams.c b/packages/runtime/src/scr_webstreams.c new file mode 100644 index 000000000..a9b6a0392 --- /dev/null +++ b/packages/runtime/src/scr_webstreams.c @@ -0,0 +1,2508 @@ +/* WHATWG streams for checked-dynamic code: ReadableStream with its default + * reader and controller, and the queuing strategies, as untyped JavaScript + * reads them (globals and node:stream/web). The algorithms follow Node 24's + * lib/internal/webstreams step by step, including where Node departs from + * the specification's promise usage (its read fast path, the lazily + * materialized closed promises), so promise jobs run in Node's order. + * + * Every interface is a dyn handle class (scr_dyn_handle_class.c): its + * constructor value, prototype members and brand checks come from the + * handle tables at the end of this file. Algorithm state is refcounted C + * structs the cycle collector traces (WsObj); user callbacks and values are + * dyn values. + * + * Promise steps: "upon settlement" is scr_promise_react, which runs as a + * reaction job in subscription order, as PromisePrototypeThen does. */ +#include "scr_runtime.h" + +#include +#include +#include +#include + +typedef struct WsReadable WsReadable; +typedef struct WsReader WsReader; +typedef struct WsRController WsRController; +typedef struct WsReadRequest WsReadRequest; +typedef struct WsTee WsTee; + +static void ws_oom(void) { scr_trap("scriptc: out of memory\n"); } + +static void *ws_alloc(size_t size) { + void *p = calloc(1, size); + if (!p) ws_oom(); + return p; +} + +/* ── collected state ───────────────────────────────────────────────── */ + +/* Every algorithm state starts with WsObj and carries a cycle header: a + * stream, its controller and its reader point at one another, as Node's + * objects do, and user callbacks close over their handles, so the cycle + * collector reclaims them. A type's trace visits every reference the state + * holds to another state, a promise or a dyn value (every heap dyn value + * carries a header); drop releases those references, or in a collection + * (gc) none of them, since the collector settles what trace visits, and + * frees the state's own buffers. */ +typedef struct WsObjType { + void (*trace)(void *obj, ScrTraceVisit visit, void *ctx); + void (*drop)(void *obj, bool gc); +} WsObjType; + +typedef struct WsObj { + size_t rc; + const WsObjType *type; +} WsObj; + +static void ws_obj_trace(void *obj, ScrTraceVisit visit, void *ctx) { + ((WsObj *)obj)->type->trace(obj, visit, ctx); +} + +static void ws_obj_gcfree(void *obj) { + ((WsObj *)obj)->type->drop(obj, true); + scr_obj_free_note(); + scr_cyc_free(obj); +} + +static void ws_obj_destroy(void *obj) { + ((WsObj *)obj)->type->drop(obj, false); + scr_obj_free_note(); + scr_cyc_free(obj); +} + +static ScrCycType ws_cyc_type = {ws_obj_trace, ws_obj_gcfree, SCR_CYC_KIND_NONE, 0}; + +static void *ws_obj_new(size_t size, const WsObjType *type) { + WsObj *o = scr_cyc_alloc(size, &ws_cyc_type); + o->rc = 1; + o->type = type; + scr_obj_alloc_note(); + return o; +} + +static void *ws_retain(void *obj) { + ((WsObj *)obj)->rc++; + return obj; +} + +static void ws_release(void *obj) { + WsObj *o = obj; + if (!o) return; + if (--o->rc == 0) { + scr_cyc_dead(o); + scr_rc_destroy(o, ws_obj_destroy); + } else { + scr_cyc_released(o); + } +} + +static void ws_visit(const void *child, ScrTraceVisit visit, void *ctx) { + if (child) visit((void *)child, ctx); +} + +static void ws_drop_dyn(ScrDyn *d, bool gc) { + if (d && !gc) scr_dyn_release(d); +} + +static void ws_drop_promise(ScrPromise *p, bool gc) { + if (p && !gc) scr_promise_release(p); +} + +static void ws_drop_obj(void *o, bool gc) { + if (o && !gc) ws_release(o); +} + +/* ── values and errors ─────────────────────────────────────────────── */ + +static ScrDyn *ws_undef(void) { return scr_dyn_retain(scr_dyn_undefined()); } + +static bool ws_is_object(const ScrDyn *v) { + return v && v->kind != SCR_DYN_NULL && (v->kind == SCR_DYN_FUNC || scr_dyn_is_object(v)); +} + +/* The pending exception as a dyn value (+1); the cell is cleared. */ +static ScrDyn *ws_take_pending(void) { + ScrCaught *c = scr_exc_take(); + ScrDyn *d = scr_caught_to_dyn(c); + scr_caught_release(c); + return d; +} + +/* ERR_INVALID_STATE as Node's ERR_INVALID_STATE.TypeError: pending. */ +static void ws_state_error(const char *message) { + char buf[256]; + int n = snprintf(buf, sizeof buf, "Invalid state: %s", message); + scr_throw_error_msg_code(SCR_ERR_TYPE, buf, (size_t)n, "ERR_INVALID_STATE"); +} + +static ScrDyn *ws_state_error_value(const char *message) { + ws_state_error(message); + return ws_take_pending(); +} + +/* ERR_INVALID_ARG_VALUE.RangeError: "The property 'name' is invalid. + * Received ". */ +static void ws_range_arg_value(const char *name, const ScrDyn *got) { + char inspected[64]; + const char *text = scr_dyn_inspect_lite(got, inspected, sizeof inspected); + char buf[256]; + int n = snprintf(buf, sizeof buf, "The %s '%s' is invalid. Received %s", + strchr(name, '.') ? "property" : "argument", name, text); + scr_throw_error_msg_code(SCR_ERR_RANGE, buf, (size_t)n, "ERR_INVALID_ARG_VALUE"); +} + +/* validateObject with Node's kValidateObjectAllowObjects (null refused) or + * kValidateObjectAllowObjectsAndNull. */ +static bool ws_validate_object(const ScrDyn *value, const char *name, bool allow_null) { + if (value->kind == SCR_DYN_NULL ? allow_null : ws_is_object(value)) return true; + scr_dyn_arg_type_fail(name, "of type object", value); + return false; +} + +/* value[name] through Get (getters run); +1, or NULL with the exception + * pending. undefined and null answer undefined, as optional chaining does. */ +static ScrDyn *ws_get(const ScrDyn *object, const char *name) { + if (object->kind == SCR_DYN_UNDEF || object->kind == SCR_DYN_NULL) return ws_undef(); + if (object->kind == SCR_DYN_HANDLE || object->kind == SCR_DYN_BYTES) { + ScrStr *key = scr_str_new(name, strlen(name)); + ScrDyn *value = object->kind == SCR_DYN_HANDLE ? scr_dyn_handle_key_get(object, key) + : scr_dyn_bytes_key_get(object, key); + scr_str_release(key); + return value; + } + ScrStr *key = scr_str_new(name, strlen(name)); + ScrDyn *k = scr_dyn_new_str(key); + scr_str_release(key); + ScrDyn *value = scr_dyn_reflect_get((ScrDyn *)object, k, (ScrDyn *)object); + scr_dyn_release(k); + if (!value && !scr_exc_pending()) value = ws_undef(); + return value; +} + +/* fn.call(self, ...args): +1, or NULL with the exception pending. */ +static ScrDyn *ws_call(const ScrDyn *fn, const ScrDyn *self, ScrDyn *const *args, size_t argc, const char *what) { + scr_dyn_this_push_dyn((ScrDyn *)self); + ScrDyn *result = scr_dyn_call(fn, args, argc, what); + scr_dyn_this_pop(); + return result; +} + +/* validateFunction(value, name): ERR_INVALID_ARG_TYPE for any value that + * is not callable. */ +static bool ws_validate_function(const ScrDyn *value, const char *name) { + if (scr_dyn_is_callable(value)) return true; + scr_dyn_arg_type_fail(name, "of type function", value); + return false; +} + +/* ── promises ──────────────────────────────────────────────────────── */ + +static SCR_TL ScrPromise *ws_resolved_promise; /* kResolvedPromise */ + +static ScrPromise *ws_resolved_singleton(void) { + if (!ws_resolved_promise) { + ws_resolved_promise = scr_promise_new(); + scr_promise_fulfill_ref(ws_resolved_promise, ws_undef(), scr_dyn_retain_v, scr_dyn_release_v, scr_dyn_trace_v); + } + return ws_resolved_promise; +} + +static void ws_fulfill(ScrPromise *p, ScrDyn *value) { + scr_promise_fulfill_ref(p, scr_dyn_retain(value), scr_dyn_retain_v, scr_dyn_release_v, scr_dyn_trace_v); +} + +static void ws_reject(ScrPromise *p, ScrDyn *reason) { + scr_dyn_throw(scr_dyn_retain(reason)); + scr_promise_reject_pending(p); +} + +/* PromiseResolve() and PromiseReject(e): fresh settled promises (+1). */ +static ScrPromise *ws_promise_resolved(ScrDyn *value) { + ScrPromise *p = scr_promise_new(); + ws_fulfill(p, value); + return p; +} + +static ScrPromise *ws_promise_rejected(ScrDyn *reason) { + ScrPromise *p = scr_promise_new(); + ws_reject(p, reason); + return p; +} + +/* A rejected promise for the pending exception (+1). */ +static ScrPromise *ws_promise_rejected_pending(void) { + ScrPromise *p = scr_promise_new(); + scr_promise_reject_pending(p); + return p; +} + +/* setPromiseHandled. */ +static void ws_set_handled(ScrPromise *p) { scr_promise_mark_handled(p); } + +/* The settled value or reason of p as a dyn value (+1). */ +static ScrDyn *ws_settled_value(ScrPromise *p) { return scr_promise_reason_dyn(p); } + +/* The promise a `new Promise((r) => r(value))` makes, or an async + * function's result promise when it returns value: a promise adopts through + * its then job, a thenable through its resolve job, and any other value + * fulfills. +1. */ +static ScrPromise *ws_promise_resolving(ScrDyn *value) { + if (value->kind == SCR_DYN_PROMISE) { + ScrPromise *p = scr_promise_new(); + scr_promise_resolve_dyn(p, scr_dyn_retain(value)); + return p; + } + if (ws_is_object(value)) return scr_dyn_promise_resolve(value, NULL); + return ws_promise_resolved(value); +} + +/* PromiseResolve(value): a promise is itself. +1. */ +static ScrPromise *ws_promise_resolve(ScrDyn *value) { + if (value->kind == SCR_DYN_PROMISE) return scr_promise_retain(value->v.promise); + return scr_dyn_promise_resolve(value, NULL); +} + +/* Native promise reactions over collected contexts (WsObj, or NULL). ok and err borrow the + * context and the value; a NULL step passes the settlement on (and marks + * nothing). */ +typedef void (*WsStep)(void *ctx, ScrDyn *value); + +/* A step whose result promise the then() result adopts (+1, or NULL with + * the exception pending). */ +typedef ScrPromise *(*WsPStep)(void *ctx, ScrDyn *value); + +typedef struct WsReaction { + WsStep ok; + WsStep err; + WsPStep pok; + WsPStep perr; + void *ctx; /* a WsObj (+1), or NULL */ + ScrPromise *derived; /* the then() result, settled by the steps' outcome */ +} WsReaction; + +static void ws_reaction_run(void *opaque, ScrPromise *settled) { + WsReaction *r = opaque; + bool rejected = scr_promise_is_rejected(settled); + WsStep step = rejected ? r->err : r->ok; + WsPStep pstep = rejected ? r->perr : r->pok; + ScrDyn *value = ws_settled_value(settled); + if (pstep) { + ScrPromise *adopted = pstep(r->ctx, value); + if (!adopted) { + scr_promise_reject_pending(r->derived); + } else { + ScrDyn *box = scr_dyn_new_promise(adopted); + scr_promise_release(adopted); + scr_promise_resolve_dyn(r->derived, box); + } + scr_dyn_release(value); + return; + } + if (step) step(r->ctx, value); + if (r->derived) { + if (scr_exc_pending()) scr_promise_reject_pending(r->derived); + else if (rejected && !step) ws_reject(r->derived, value); + else { + ScrDyn *u = ws_undef(); + ws_fulfill(r->derived, u); + scr_dyn_release(u); + } + } else if (scr_exc_pending()) { + /* A step's own throw in a reaction nobody observes: Node's then + * result rejects unhandled, which reports. */ + ScrPromise *lost = ws_promise_rejected_pending(); + scr_promise_release(lost); + } + scr_dyn_release(value); +} + +static void ws_reaction_drop(void *opaque) { + WsReaction *r = opaque; + ws_release(r->ctx); + if (r->derived) scr_promise_release(r->derived); + free(r); +} + +static const ScrPromiseReaction ws_reaction = {ws_reaction_run, ws_reaction_drop}; + +/* PromisePrototypeThen(p, ok, err) with native steps; the caller's +1 on + * ctx moves in. Answers the derived promise (+1) when want_derived. */ +static ScrPromise *ws_then(ScrPromise *p, WsStep ok, WsStep err, void *ctx, bool want_derived) { + WsReaction *r = ws_alloc(sizeof *r); + r->ok = ok; + r->err = err; + r->ctx = ctx; + ScrPromise *derived = NULL; + if (want_derived) { + derived = scr_promise_new(); + r->derived = scr_promise_retain(derived); + } + scr_promise_react(p, &ws_reaction, r); + return derived; +} + +/* PromisePrototypeThen(p, pok, perr) where the handlers return promises + * the result adopts: the derived promise (+1). */ +static ScrPromise *ws_then_adopt(ScrPromise *p, WsPStep pok, WsPStep perr, void *ctx) { + WsReaction *r = ws_alloc(sizeof *r); + r->pok = pok; + r->perr = perr; + r->ctx = ctx; + ScrPromise *derived = scr_promise_new(); + r->derived = scr_promise_retain(derived); + scr_promise_react(p, &ws_reaction, r); + return derived; +} + +/* `then(p, () => {})`: a promise fulfilled with undefined after p fulfills, + * rejected with p's reason (+1). */ +static ScrPromise *ws_then_void(ScrPromise *p) { return ws_then(p, NULL, NULL, NULL, true); } + +/* thenAlgorithmResult (util.js): a non-object result continues from the + * shared resolved promise; any other result through PromiseResolve. ctx + * is borrowed. */ +static void ws_then_algorithm_result(ScrDyn *result, WsStep ok, WsStep err, void *ctx) { + if (!ws_is_object(result)) { + ws_then(ws_resolved_singleton(), ok, NULL, ws_retain(ctx), false); + return; + } + ScrPromise *p = ws_promise_resolve(result); + if (!p) { + /* PromiseResolve's then read threw: the algorithm result rejects. */ + p = ws_promise_rejected_pending(); + } + ws_then(p, ok, err, ws_retain(ctx), false); + scr_promise_release(p); +} + +/* queueMicrotask(fn) with a native step; the caller's +1 on ctx moves in. */ +static void ws_queue_microtask(WsStep step, void *ctx) { + ws_then(ws_resolved_singleton(), step, NULL, ctx, false); +} + +/* process.nextTick(fn) with a native step; the caller's +1 on ctx moves + * in. The tick queue holds it until the step runs. */ +typedef struct WsTick { + WsStep step; + void *ctx; +} WsTick; + +static void *ws_tick_retain(void *p) { return p; } +static void ws_tick_release(void *p) { + WsTick *t = p; + ws_release(t->ctx); + free(t); +} + +static void ws_tick_run(ScrClosure *self) { + WsTick *t = scr_box_get_ref(self->caps[0]); + t->step(t->ctx, NULL); +} + +static void ws_next_tick(WsStep step, void *ctx) { + WsTick *t = ws_alloc(sizeof *t); + t->step = step; + t->ctx = ctx; + ScrClosure *cb = scr_closure_new((void *)ws_tick_run, 1); + cb->caps[0] = scr_box_new_obj(ws_tick_retain, ws_tick_release, NULL); + scr_box_set_ref(cb->caps[0], t); + scr_next_tick(cb); +} + +/* A promise with its resolvers, settled at most once. */ +typedef struct WsDeferred { + ScrPromise *promise; + bool settled; +} WsDeferred; + +static void ws_deferred_init(WsDeferred *d) { + d->promise = scr_promise_new(); + d->settled = false; +} + +static void ws_deferred_resolve(WsDeferred *d, ScrDyn *value) { + if (d->settled) return; + d->settled = true; + ws_fulfill(d->promise, value); +} + +static void ws_deferred_resolve_undefined(WsDeferred *d) { + ScrDyn *u = ws_undef(); + ws_deferred_resolve(d, u); + scr_dyn_release(u); +} + +/* resolve(promiseValue): adopts a promise the way a resolver called with + * one does. */ +static void ws_deferred_resolve_promise(WsDeferred *d, ScrPromise *value) { + if (d->settled) return; + d->settled = true; + ScrDyn *box = scr_dyn_new_promise(value); + scr_promise_resolve_dyn(d->promise, box); +} + +static void ws_deferred_reject(WsDeferred *d, ScrDyn *reason) { + if (d->settled) return; + d->settled = true; + ws_reject(d->promise, reason); +} + +/* ── the queue (Queue's pairs, util.js) ───────────────────────────── */ + +typedef struct WsQueue { + ScrDyn **values; + double *sizes; + size_t head, length, capacity; +} WsQueue; + +static void ws_queue_push(WsQueue *q, ScrDyn *value, double size) { + if (q->length == q->capacity) { + size_t capacity = q->capacity ? q->capacity * 2 : 8; + ScrDyn **values = malloc(capacity * sizeof *values); + double *sizes = malloc(capacity * sizeof *sizes); + if (!values || !sizes) ws_oom(); + for (size_t i = 0; i < q->length; i++) { + values[i] = q->values[(q->head + i) % q->capacity]; + sizes[i] = q->sizes[(q->head + i) % q->capacity]; + } + free(q->values); + free(q->sizes); + q->values = values; + q->sizes = sizes; + q->head = 0; + q->capacity = capacity; + } + size_t at = (q->head + q->length) % q->capacity; + q->values[at] = scr_dyn_retain(value); + q->sizes[at] = size; + q->length++; +} + +/* The front value (+1 moves out) and its size. */ +static ScrDyn *ws_queue_shift(WsQueue *q, double *size) { + ScrDyn *value = q->values[q->head]; + *size = q->sizes[q->head]; + q->head = (q->head + 1) % q->capacity; + q->length--; + if (q->length == 0) q->head = 0; + return value; +} + +static void ws_queue_drop(WsQueue *q, bool gc) { + if (!gc) + for (size_t i = 0; i < q->length; i++) scr_dyn_release(q->values[(q->head + i) % q->capacity]); + free(q->values); + free(q->sizes); + memset(q, 0, sizeof *q); +} + +static void ws_queue_reset(WsQueue *q) { ws_queue_drop(q, false); } + +static void ws_queue_trace(const WsQueue *q, ScrTraceVisit visit, void *ctx) { + for (size_t i = 0; i < q->length; i++) visit(q->values[(q->head + i) % q->capacity], ctx); +} + +/* ── read requests ─────────────────────────────────────────────────── */ + +typedef struct WsReadRequestOps { + /* Each step consumes the request. Values are borrowed. */ + void (*chunk)(WsReadRequest *r, ScrDyn *chunk); + void (*close)(WsReadRequest *r); + void (*error)(WsReadRequest *r, ScrDyn *error); +} WsReadRequestOps; + +struct WsReadRequest { + const WsReadRequestOps *ops; + WsReadRequest *next; + ScrPromise *promise; /* +1, or NULL */ + void *ctx; /* a WsObj (+1), or NULL */ +}; + +static ScrDyn *ws_read_result(ScrDyn *value, bool done) { + ScrDyn *result = scr_dyn_new_obj(); + scr_dyn_obj_set(result, "value", 5, scr_dyn_retain(value)); + scr_dyn_obj_set(result, "done", 4, scr_dyn_new_bool(done)); + return result; +} + +/* DefaultReadRequest. */ +static void ws_default_chunk(WsReadRequest *r, ScrDyn *chunk) { + ScrDyn *result = ws_read_result(chunk, false); + ws_fulfill(r->promise, result); + scr_dyn_release(result); + scr_promise_release(r->promise); + free(r); +} + +static void ws_default_close(WsReadRequest *r) { + ScrDyn *u = ws_undef(); + ScrDyn *result = ws_read_result(u, true); + scr_dyn_release(u); + ws_fulfill(r->promise, result); + scr_dyn_release(result); + scr_promise_release(r->promise); + free(r); +} + +static void ws_default_error(WsReadRequest *r, ScrDyn *error) { + ws_reject(r->promise, error); + scr_promise_release(r->promise); + free(r); +} + +static const WsReadRequestOps ws_default_request_ops = {ws_default_chunk, ws_default_close, ws_default_error}; + +/* ── ReadableStream state ──────────────────────────────────────────── */ + +typedef enum { WS_STATE_READABLE, WS_STATE_CLOSED, WS_STATE_ERRORED } WsState; + +/* Native algorithms for streams the runtime makes (tee branches, from()): + * pull answers the algorithm's result (+1 dyn, or NULL with the exception + * pending, or the parked sentinel), cancel a promise (+1). The context is + * a WsObj the controller holds until it clears its algorithms. */ +typedef struct WsSourceOps { + ScrDyn *(*pull)(void *ctx, WsRController *c); + ScrPromise *(*cancel)(void *ctx, WsRController *c, ScrDyn *reason); +} WsSourceOps; + +struct WsReadable { + WsObj obj; + WsState state; + bool disturbed; + ScrDyn *stored_error; + WsReader *reader; /* +1 while locked */ + WsRController *controller; /* +1 */ + WsDeferred closed; /* kIsClosedPromise, materialized on use */ + bool closed_materialized; +}; + +struct WsRController { + WsObj obj; + WsReadable *stream; /* +1 */ + WsQueue queue; + double queue_total_size; + double high_water_mark; + bool started, close_requested, pull_again, pulling, algorithms_cleared; + /* The underlying source's algorithms (user source), or native ones. */ + ScrDyn *source; + ScrDyn *pull_fn; + ScrDyn *cancel_fn; + ScrDyn *size_fn; /* NULL = the default size algorithm */ + const WsSourceOps *native; + void *native_ctx; /* +1 */ + int native_branch; /* a tee branch's number */ +}; + +struct WsReader { + WsObj obj; + WsReadable *stream; /* +1 while attached */ + WsReadRequest *head, *tail; + size_t requests; + /* The closed promise record, materialized on first observation: the + * promise, and whether its resolvers are still live (PromiseWithResolvers) + * or it was made settled. */ + ScrPromise *closed; + bool closed_live; + /* The tee reading through this reader (+1), which reacts to the + * stream's error as Node's reaction to the closed promise does. */ + WsTee *tee; +}; + +static SCR_TL ScrDyn *ws_released_error; /* "Reader released" */ +static SCR_TL ScrDyn *ws_releasing_error; /* "Releasing reader" */ +static SCR_TL bool ws_teardown_registered; + +static void ws_teardown(void) { + if (ws_released_error) scr_dyn_release(ws_released_error); + if (ws_releasing_error) scr_dyn_release(ws_releasing_error); + if (ws_resolved_promise) scr_promise_release(ws_resolved_promise); + ws_released_error = ws_releasing_error = NULL; + ws_resolved_promise = NULL; +} + +static void ws_register_teardown(void) { + if (ws_teardown_registered) return; + ws_teardown_registered = true; + scr_atexit(ws_teardown); +} + +static ScrDyn *ws_reader_released_error(void) { + if (!ws_released_error) { + ws_register_teardown(); + ws_released_error = ws_state_error_value("Reader released"); + } + return ws_released_error; +} + +static ScrDyn *ws_reader_releasing_error(void) { + if (!ws_releasing_error) { + ws_register_teardown(); + ws_releasing_error = ws_state_error_value("Releasing reader"); + } + return ws_releasing_error; +} + +static WsReadable *ws_readable_retain(WsReadable *s) { return ws_retain(s); } +static WsRController *ws_rcontroller_retain(WsRController *c) { return ws_retain(c); } +static WsReader *ws_reader_retain(WsReader *r) { return ws_retain(r); } +static void ws_readable_release(WsReadable *s) { ws_release(s); } +static void ws_reader_release(WsReader *r) { ws_release(r); } + +static void ws_readable_trace(void *obj, ScrTraceVisit visit, void *ctx) { + WsReadable *s = obj; + ws_visit(s->stored_error, visit, ctx); + ws_visit(s->reader, visit, ctx); + ws_visit(s->controller, visit, ctx); + if (s->closed_materialized) ws_visit(s->closed.promise, visit, ctx); +} + +static void ws_readable_drop(void *obj, bool gc) { + WsReadable *s = obj; + ws_drop_dyn(s->stored_error, gc); + ws_drop_obj(s->reader, gc); + ws_drop_obj(s->controller, gc); + if (s->closed_materialized) ws_drop_promise(s->closed.promise, gc); +} + +static const WsObjType ws_readable_type = {ws_readable_trace, ws_readable_drop}; + +static void ws_rcontroller_trace(void *obj, ScrTraceVisit visit, void *ctx) { + WsRController *c = obj; + ws_visit(c->stream, visit, ctx); + ws_queue_trace(&c->queue, visit, ctx); + ws_visit(c->source, visit, ctx); + ws_visit(c->pull_fn, visit, ctx); + ws_visit(c->cancel_fn, visit, ctx); + ws_visit(c->size_fn, visit, ctx); + ws_visit(c->native_ctx, visit, ctx); +} + +static void ws_rcontroller_drop(void *obj, bool gc) { + WsRController *c = obj; + ws_queue_drop(&c->queue, gc); + ws_drop_dyn(c->source, gc); + ws_drop_dyn(c->pull_fn, gc); + ws_drop_dyn(c->cancel_fn, gc); + ws_drop_dyn(c->size_fn, gc); + ws_drop_obj(c->native_ctx, gc); + ws_drop_obj(c->stream, gc); +} + +static const WsObjType ws_rcontroller_type = {ws_rcontroller_trace, ws_rcontroller_drop}; + +static void ws_reader_trace(void *obj, ScrTraceVisit visit, void *ctx) { + WsReader *r = obj; + ws_visit(r->stream, visit, ctx); + ws_visit(r->closed, visit, ctx); + ws_visit(r->tee, visit, ctx); + for (WsReadRequest *q = r->head; q; q = q->next) { + ws_visit(q->promise, visit, ctx); + ws_visit(q->ctx, visit, ctx); + } +} + +static void ws_reader_drop(void *obj, bool gc) { + WsReader *r = obj; + while (r->head) { + WsReadRequest *q = r->head; + r->head = q->next; + ws_drop_promise(q->promise, gc); + ws_drop_obj(q->ctx, gc); + free(q); + } + ws_drop_promise(r->closed, gc); + ws_drop_obj(r->tee, gc); + ws_drop_obj(r->stream, gc); +} + +static const WsObjType ws_reader_type = {ws_reader_trace, ws_reader_drop}; + +/* ── handles ───────────────────────────────────────────────────────── */ + +/* A handle over a state (+1); the collector traces the handle's edge to + * it. */ +static ScrDyn *ws_box(void *state, ScrDynHandleTag tag) { + ScrDyn *d = scr_dyn_new_handle(state, tag); + d->v.handle.traced = true; + return d; +} + +static ScrDyn *ws_stream_box(WsReadable *s) { return ws_box(s, SCR_DYNH_WS_READABLE); } +static ScrDyn *ws_reader_box(WsReader *r) { return ws_box(r, SCR_DYNH_WS_READER); } +static ScrDyn *ws_rcontroller_box(WsRController *c) { return ws_box(c, SCR_DYNH_WS_RCONTROLLER); } + +static WsReadable *ws_stream_of(const ScrDyn *d) { + return d && d->kind == SCR_DYN_HANDLE && d->v.handle.tag == SCR_DYNH_WS_READABLE ? d->v.handle.ptr : NULL; +} + +static void ws_install(void); + +/* ── ReadableStream abstract operations ───────────────────────────── */ + +static bool ws_readable_locked(WsReadable *s) { return s->reader != NULL; } + +static void ws_rcontroller_clear_algorithms(WsRController *c) { + if (c->algorithms_cleared) return; + c->algorithms_cleared = true; + ScrDyn *source = c->source, *pull = c->pull_fn, *cancel = c->cancel_fn, *size = c->size_fn; + void *native_ctx = c->native_ctx; + c->source = c->pull_fn = c->cancel_fn = c->size_fn = NULL; + c->native_ctx = NULL; + if (source) scr_dyn_release(source); + if (pull) scr_dyn_release(pull); + if (cancel) scr_dyn_release(cancel); + if (size) scr_dyn_release(size); + ws_release(native_ctx); +} + +static void ws_readable_close(WsReadable *s) { + s->state = WS_STATE_CLOSED; + if (s->closed_materialized) ws_deferred_resolve_undefined(&s->closed); + WsReader *r = s->reader; + if (!r) return; + if (r->closed && r->closed_live) { + r->closed_live = false; + ScrDyn *u = ws_undef(); + ws_fulfill(r->closed, u); + scr_dyn_release(u); + } + ws_reader_retain(r); + while (r->head) { + WsReadRequest *q = r->head; + r->head = q->next; + if (!r->head) r->tail = NULL; + r->requests--; + q->ops->close(q); + } + ws_reader_release(r); +} + +static void ws_tee_source_error(WsTee *t, ScrDyn *error); + +static void ws_readable_error(WsReadable *s, ScrDyn *error) { + s->state = WS_STATE_ERRORED; + s->stored_error = scr_dyn_retain(error); + if (s->closed_materialized) { + ws_set_handled(s->closed.promise); + ws_deferred_reject(&s->closed, error); + } + WsReader *r = s->reader; + if (!r) return; + if (r->closed && r->closed_live) { + r->closed_live = false; + ws_set_handled(r->closed); + ws_reject(r->closed, error); + } + if (r->tee) ws_tee_source_error(r->tee, error); + ws_reader_retain(r); + while (r->head) { + WsReadRequest *q = r->head; + r->head = q->next; + if (!r->head) r->tail = NULL; + r->requests--; + q->ops->error(q, error); + } + ws_reader_release(r); +} + +static void ws_readable_add_read_request(WsReadable *s, WsReadRequest *q) { + WsReader *r = s->reader; + q->next = NULL; + if (r->tail) r->tail->next = q; + else r->head = q; + r->tail = q; + r->requests++; +} + +static void ws_readable_fulfill_read_request(WsReadable *s, ScrDyn *chunk) { + WsReader *r = s->reader; + WsReadRequest *q = r->head; + r->head = q->next; + if (!r->head) r->tail = NULL; + r->requests--; + q->ops->chunk(q, chunk); +} + +static ScrPromise *ws_rcontroller_cancel_steps(WsRController *c, ScrDyn *reason); + +/* readableStreamCancel: +1 promise. */ +static ScrPromise *ws_readable_cancel(WsReadable *s, ScrDyn *reason) { + s->disturbed = true; + if (s->state == WS_STATE_CLOSED) { + ScrDyn *u = ws_undef(); + ScrPromise *p = ws_promise_resolved(u); + scr_dyn_release(u); + return p; + } + if (s->state == WS_STATE_ERRORED) return ws_promise_rejected(s->stored_error); + ws_readable_close(s); + ScrPromise *source = ws_rcontroller_cancel_steps(s->controller, reason); + ScrPromise *result = ws_then_void(source); + scr_promise_release(source); + return result; +} + +/* readerClosedPromise: the reader's closed promise, materialized. Borrowed. */ +static ScrPromise *ws_reader_closed_promise(WsReader *r) { + if (r->closed) return r->closed; + WsReadable *s = r->stream; + if (!s) { + r->closed = ws_promise_rejected(ws_reader_released_error()); + ws_set_handled(r->closed); + r->closed_live = false; + } else if (s->state == WS_STATE_READABLE) { + r->closed = scr_promise_new(); + r->closed_live = true; + } else if (s->state == WS_STATE_CLOSED) { + ScrDyn *u = ws_undef(); + r->closed = ws_promise_resolved(u); + scr_dyn_release(u); + r->closed_live = false; + } else { + r->closed = ws_promise_rejected(s->stored_error); + ws_set_handled(r->closed); + r->closed_live = false; + } + return r->closed; +} + +/* readableStreamReaderGenericRelease. */ +static void ws_reader_generic_release(WsReader *r) { + WsReadable *s = r->stream; + if (s->state == WS_STATE_READABLE) { + if (r->closed) { + if (r->closed_live) { + r->closed_live = false; + ws_reject(r->closed, ws_reader_released_error()); + } + ws_set_handled(r->closed); + } + } else if (r->closed) { + scr_promise_release(r->closed); + r->closed = NULL; + } + s->reader = NULL; + r->stream = NULL; + ws_reader_release(r); /* the stream's reference */ + ws_readable_release(s); +} + +/* readableStreamDefaultReaderRelease. */ +static void ws_reader_release_lock(WsReader *r) { + ws_reader_retain(r); + ws_reader_generic_release(r); + while (r->head) { + WsReadRequest *q = r->head; + r->head = q->next; + if (!r->head) r->tail = NULL; + r->requests--; + q->ops->error(q, ws_reader_releasing_error()); + } + ws_reader_release(r); +} + +/* setupReadableStreamDefaultReader: false with the exception pending. */ +/* setupReadableStreamDefaultReader: the new reader (+1), or NULL with the + * exception pending. */ +static WsReader *ws_reader_new(WsReadable *s) { + if (ws_readable_locked(s)) { + ws_state_error("ReadableStream is locked"); + return NULL; + } + WsReader *r = ws_obj_new(sizeof *r, &ws_reader_type); + r->stream = ws_readable_retain(s); + s->reader = ws_reader_retain(r); + return r; +} + +/* ── the default controller ────────────────────────────────────────── */ + +static bool ws_rcontroller_can_close_or_enqueue(WsRController *c) { + return !c->close_requested && c->stream->state == WS_STATE_READABLE; +} + +static void ws_rcontroller_pull(WsRController *c); + +static bool ws_rcontroller_should_call_pull(WsRController *c) { + WsReadable *s = c->stream; + if (c->close_requested || s->state != WS_STATE_READABLE || !c->started) return false; + if (s->reader && s->reader->requests) return true; + return c->high_water_mark - c->queue_total_size > 0; +} + +static void ws_rcontroller_call_pull_if_needed(WsRController *c) { + if (ws_rcontroller_should_call_pull(c)) ws_rcontroller_pull(c); +} + +static void ws_rcontroller_error(WsRController *c, ScrDyn *error) { + WsReadable *s = c->stream; + if (s->state != WS_STATE_READABLE) return; + ws_queue_reset(&c->queue); + c->queue_total_size = 0; + ws_rcontroller_clear_algorithms(c); + ws_readable_error(s, error); +} + +static void ws_rcontroller_close(WsRController *c) { + if (!ws_rcontroller_can_close_or_enqueue(c)) return; + c->close_requested = true; + if (!c->queue.length) { + ws_rcontroller_clear_algorithms(c); + ws_readable_close(c->stream); + } +} + +/* readableStreamDefaultControllerEnqueue: false with the exception pending + * (the size algorithm threw, or its size was invalid; the stream errors). */ +static bool ws_rcontroller_enqueue(WsRController *c, ScrDyn *chunk) { + WsReadable *s = c->stream; + if (c->close_requested || s->state != WS_STATE_READABLE) return true; + if (s->reader && s->reader->requests) { + ws_readable_fulfill_read_request(s, chunk); + } else if (!c->size_fn) { + ws_queue_push(&c->queue, chunk, 1); + c->queue_total_size++; + if (c->started && c->high_water_mark - c->queue_total_size > 0) ws_rcontroller_pull(c); + return true; + } else { + ScrDyn *size_fn = scr_dyn_retain(c->size_fn); + ScrDyn *size = ws_call(size_fn, scr_dyn_undefined(), &chunk, 1, "strategy.size"); + scr_dyn_release(size_fn); + double coerced = NAN; + if (size) { + coerced = scr_dyn_number_coerce(size); + if (!scr_exc_pending() && (isnan(coerced) || coerced < 0 || isinf(coerced))) + ws_range_arg_value("size", size); + scr_dyn_release(size); + } + if (scr_exc_pending()) { + ScrDyn *error = ws_take_pending(); + ws_rcontroller_error(c, error); + scr_dyn_throw(error); + return false; + } + ws_queue_push(&c->queue, chunk, coerced); + c->queue_total_size += coerced; + } + ws_rcontroller_call_pull_if_needed(c); + return true; +} + +static ScrDyn *ws_rcontroller_desired_size(WsRController *c) { + switch (c->stream->state) { + case WS_STATE_ERRORED: return scr_dyn_new_null(); + case WS_STATE_CLOSED: return scr_dyn_new_num(0); + default: return scr_dyn_new_num(c->high_water_mark - c->queue_total_size); + } +} + +static void ws_pull_fulfilled(void *ctx, ScrDyn *value) { + (void)value; + WsRController *c = ctx; + c->pulling = false; + if (c->pull_again) { + c->pull_again = false; + ws_rcontroller_call_pull_if_needed(c); + } +} + +static void ws_pull_rejected(void *ctx, ScrDyn *error) { + ws_rcontroller_error(ctx, error); +} + +/* The parked algorithm result (kParkedAlgorithmResult): the algorithm + * settles the pull itself. */ +static ScrDyn ws_parked_value; +#define WS_PARKED (&ws_parked_value) + +static void ws_rcontroller_pull(WsRController *c) { + if (c->pulling) { + c->pull_again = true; + return; + } + c->pulling = true; + ScrDyn *result; + if (c->native) { + result = c->native->pull(c->native_ctx, c); + } else if (c->pull_fn) { + ScrDyn *controller = ws_rcontroller_box(c); + ScrDyn *pull = scr_dyn_retain(c->pull_fn); + ScrDyn *source = scr_dyn_retain(c->source); + result = ws_call(pull, source, &controller, 1, "source.pull"); + scr_dyn_release(pull); + scr_dyn_release(source); + scr_dyn_release(controller); + } else { + result = ws_undef(); /* nonOpCallback */ + } + if (result == WS_PARKED) return; + if (!result) { + ScrPromise *rejected = ws_promise_rejected_pending(); + ws_then(rejected, ws_pull_fulfilled, ws_pull_rejected, ws_rcontroller_retain(c), false); + scr_promise_release(rejected); + return; + } + ws_then_algorithm_result(result, ws_pull_fulfilled, ws_pull_rejected, c); + scr_dyn_release(result); +} + +/* readableStreamDefaultControllerCancelSteps: the cancel algorithm's + * promise (+1). */ +static ScrPromise *ws_rcontroller_cancel_steps(WsRController *c, ScrDyn *reason) { + ws_queue_reset(&c->queue); + c->queue_total_size = 0; + ScrPromise *result; + if (c->native) { + result = c->native->cancel(c->native_ctx, c, reason); + } else if (c->cancel_fn) { + /* createPromiseCallback1Param: an async arrow around the call. */ + ScrDyn *cancel = scr_dyn_retain(c->cancel_fn); + ScrDyn *source = scr_dyn_retain(c->source); + ScrDyn *r = ws_call(cancel, source, &reason, 1, "source.cancel"); + scr_dyn_release(cancel); + scr_dyn_release(source); + if (!r) { + result = ws_promise_rejected_pending(); + } else { + result = ws_promise_resolving(r); + scr_dyn_release(r); + } + } else { + ScrDyn *u = ws_undef(); /* nonOpCancel */ + result = ws_promise_resolved(u); + scr_dyn_release(u); + } + ws_rcontroller_clear_algorithms(c); + return result; +} + +/* readableStreamDefaultControllerPullSteps. */ +static void ws_rcontroller_pull_steps(WsRController *c, WsReadRequest *q) { + WsReadable *s = c->stream; + if (c->queue.length) { + double size; + ScrDyn *chunk = ws_queue_shift(&c->queue, &size); + c->queue_total_size = fmax(0, c->queue_total_size - size); + if (c->close_requested && !c->queue.length) { + ws_rcontroller_clear_algorithms(c); + ws_readable_close(s); + } else if (!c->close_requested && c->started && c->high_water_mark - c->queue_total_size > 0) { + ws_rcontroller_pull(c); + } + q->ops->chunk(q, chunk); + scr_dyn_release(chunk); + return; + } + ws_readable_add_read_request(s, q); + if (c->started) ws_rcontroller_pull(c); +} + +/* readableStreamDefaultReaderRead. */ +static void ws_reader_read_request(WsReader *r, WsReadRequest *q) { + WsReadable *s = r->stream; + s->disturbed = true; + switch (s->state) { + case WS_STATE_CLOSED: q->ops->close(q); break; + case WS_STATE_ERRORED: q->ops->error(q, s->stored_error); break; + default: ws_rcontroller_pull_steps(s->controller, q); break; + } +} + +static void ws_start_fulfilled(void *ctx, ScrDyn *value) { + (void)value; + WsRController *c = ctx; + c->started = true; + ws_rcontroller_call_pull_if_needed(c); +} + +static void ws_start_rejected(void *ctx, ScrDyn *error) { + ws_rcontroller_error(ctx, error); +} + +/* setupReadableStreamDefaultController after the algorithms are in place: + * runs the start algorithm's result. */ +static void ws_rcontroller_started_with(WsRController *c, ScrDyn *start_result) { + if (!ws_is_object(start_result)) { + ws_queue_microtask(ws_start_fulfilled, ws_rcontroller_retain(c)); + return; + } + ScrPromise *p = ws_promise_resolving(start_result); + if (!p) p = ws_promise_rejected_pending(); + ws_then(p, ws_start_fulfilled, ws_start_rejected, ws_rcontroller_retain(c), false); + scr_promise_release(p); +} + +static WsReadable *ws_readable_alloc(void) { + WsReadable *s = ws_obj_new(sizeof *s, &ws_readable_type); + s->state = WS_STATE_READABLE; + return s; +} + +static WsRController *ws_rcontroller_attach(WsReadable *s, double high_water_mark) { + WsRController *c = ws_obj_new(sizeof *c, &ws_rcontroller_type); + c->stream = ws_readable_retain(s); + c->high_water_mark = high_water_mark; + s->controller = c; /* the stream's reference */ + return c; +} + +/* createReadableStream with native algorithms and the nonOp start: the new + * stream (+1). The native context's reference moves in. */ +static WsReadable *ws_readable_create_native(const WsSourceOps *ops, void *ctx, int branch, double high_water_mark) { + ws_install(); + WsReadable *s = ws_readable_alloc(); + WsRController *c = ws_rcontroller_attach(s, high_water_mark); + c->native = ops; + c->native_ctx = ctx; + c->native_branch = branch; + ScrDyn *u = ws_undef(); + ws_rcontroller_started_with(c, u); + scr_dyn_release(u); + return s; +} + +/* extractHighWaterMark: false with the exception pending. */ +static bool ws_extract_high_water_mark(ScrDyn *value, double fallback, double *out) { + if (value->kind == SCR_DYN_UNDEF) { + *out = fallback; + return true; + } + double coerced = scr_dyn_number_coerce(value); + if (scr_exc_pending()) return false; + if (isnan(coerced) || coerced < 0) { + ws_range_arg_value("strategy.highWaterMark", value); + return false; + } + *out = coerced; + return true; +} + +/* new ReadableStream(source, strategy): +1 handle, or NULL with the + * exception pending. */ +static ScrDyn *ws_readable_construct(ScrDyn *const *args, size_t argc) { + ScrDyn *source = argc > 0 && args[0]->kind != SCR_DYN_UNDEF ? args[0] : NULL; + ScrDyn *strategy = argc > 1 && args[1]->kind != SCR_DYN_UNDEF ? args[1] : NULL; + ScrDyn *empty = NULL; + if (!source || !strategy) empty = scr_dyn_new_obj(); + if (!source) source = empty; + if (!strategy) strategy = empty; + ScrDyn *result = NULL; + ScrDyn *size = NULL, *hwm = NULL, *type = NULL, *start = NULL, *pull = NULL, *cancel = NULL; + if (!ws_validate_object(source, "source", false) || !ws_validate_object(strategy, "strategy", true)) goto done; + if (!(size = ws_get(strategy, "size"))) goto done; + if (!(hwm = ws_get(strategy, "highWaterMark"))) goto done; + if (!(type = ws_get(source, "type"))) goto done; + if (type->kind != SCR_DYN_UNDEF) { + ScrStr *text = scr_dyn_string_coerce_js(type); + if (!text) goto done; + bool bytes = text->len == 5 && memcmp(text->data, "bytes", 5) == 0; + scr_str_release(text); + if (bytes) { + static const char message[] = "ReadableStream byte streams ({ type: 'bytes' }) are not supported yet"; + scr_throw_error_msg_code(SCR_ERR_ERROR, message, sizeof message - 1, "SC2020"); + goto done; + } + scr_dyn_arg_value_fail("source.type", NULL, type); + goto done; + } + double high_water_mark; + if (!ws_extract_high_water_mark(hwm, 1, &high_water_mark)) goto done; + if (size->kind != SCR_DYN_UNDEF && !ws_validate_function(size, "strategy.size")) goto done; + if (!(start = ws_get(source, "start"))) goto done; + if (!(pull = ws_get(source, "pull"))) goto done; + if (!(cancel = ws_get(source, "cancel"))) goto done; + bool has_pull = scr_dyn_truthy(pull), has_cancel = scr_dyn_truthy(cancel); + if (has_pull && !ws_validate_function(pull, "source.pull")) goto done; + if (has_cancel && !ws_validate_function(cancel, "source.cancel")) goto done; + ws_install(); + WsReadable *s = ws_readable_alloc(); + WsRController *c = ws_rcontroller_attach(s, high_water_mark); + c->source = scr_dyn_retain(source); + if (has_pull) c->pull_fn = scr_dyn_retain(pull); + if (has_cancel) c->cancel_fn = scr_dyn_retain(cancel); + if (size->kind != SCR_DYN_UNDEF) c->size_fn = scr_dyn_retain(size); + ScrDyn *start_result; + if (scr_dyn_truthy(start)) { + ScrDyn *controller = ws_rcontroller_box(c); + start_result = ws_call(start, source, &controller, 1, "source.start"); + scr_dyn_release(controller); + } else { + start_result = ws_undef(); + } + if (!start_result) { + ws_readable_release(s); + goto done; + } + ws_rcontroller_started_with(c, start_result); + scr_dyn_release(start_result); + result = ws_stream_box(s); + ws_readable_release(s); +done: + if (empty) scr_dyn_release(empty); + if (size) scr_dyn_release(size); + if (hwm) scr_dyn_release(hwm); + if (type) scr_dyn_release(type); + if (start) scr_dyn_release(start); + if (pull) scr_dyn_release(pull); + if (cancel) scr_dyn_release(cancel); + return result; +} + +/* ── tee (readableStreamDefaultTee) ───────────────────────────────── */ + +struct WsTee { + WsObj obj; + WsReader *reader; + bool reading, canceled1, canceled2; + ScrDyn *reason1, *reason2; + WsReadable *branch1, *branch2; /* +1 each */ + WsDeferred cancel_promise; + ScrDyn *pending_chunk; + ScrDyn *source_error; /* the error the source stream errored with */ + bool request_live; +}; + +static void ws_tee_trace(void *obj, ScrTraceVisit visit, void *ctx) { + WsTee *t = obj; + ws_visit(t->reader, visit, ctx); + ws_visit(t->reason1, visit, ctx); + ws_visit(t->reason2, visit, ctx); + ws_visit(t->pending_chunk, visit, ctx); + ws_visit(t->source_error, visit, ctx); + ws_visit(t->branch1, visit, ctx); + ws_visit(t->branch2, visit, ctx); + ws_visit(t->cancel_promise.promise, visit, ctx); +} + +static void ws_tee_drop(void *obj, bool gc) { + WsTee *t = obj; + ws_drop_obj(t->reader, gc); + ws_drop_dyn(t->reason1, gc); + ws_drop_dyn(t->reason2, gc); + ws_drop_dyn(t->pending_chunk, gc); + ws_drop_dyn(t->source_error, gc); + ws_drop_obj(t->branch1, gc); + ws_drop_obj(t->branch2, gc); + ws_drop_promise(t->cancel_promise.promise, gc); +} + +static const WsObjType ws_tee_type = {ws_tee_trace, ws_tee_drop}; + +static void ws_tee_release(WsTee *t) { ws_release(t); } + +static void ws_tee_forward_chunk(void *ctx, ScrDyn *unused) { + (void)unused; + WsTee *t = ctx; + t->reading = false; + ScrDyn *value = t->pending_chunk; + t->pending_chunk = NULL; + if (!t->canceled1) ws_rcontroller_enqueue(t->branch1->controller, value); + if (scr_exc_pending()) scr_exc_clear(); + if (!t->canceled2) ws_rcontroller_enqueue(t->branch2->controller, value); + if (scr_exc_pending()) scr_exc_clear(); + scr_dyn_release(value); +} + +static void ws_tee_request_chunk(WsReadRequest *q, ScrDyn *chunk) { + WsTee *t = q->ctx; + if (t->pending_chunk) scr_dyn_release(t->pending_chunk); + t->pending_chunk = scr_dyn_retain(chunk); + ws_then(ws_resolved_singleton(), ws_tee_forward_chunk, NULL, t, false); /* the request's reference moves */ + free(q); +} + +static void ws_tee_close_tick(void *ctx, ScrDyn *unused) { + (void)unused; + WsTee *t = ctx; + t->reading = false; + if (!t->canceled1) ws_rcontroller_close(t->branch1->controller); + if (!t->canceled2) ws_rcontroller_close(t->branch2->controller); + if (!t->canceled1 || !t->canceled2) ws_deferred_resolve_undefined(&t->cancel_promise); +} + +static void ws_tee_request_close(WsReadRequest *q) { + WsTee *t = q->ctx; + ws_next_tick(ws_tee_close_tick, t); /* the request's reference moves */ + free(q); +} + +static void ws_tee_request_error(WsReadRequest *q, ScrDyn *error) { + (void)error; + WsTee *t = q->ctx; + t->reading = false; + ws_tee_release(t); + free(q); +} + +static const WsReadRequestOps ws_tee_request_ops = {ws_tee_request_chunk, ws_tee_request_close, ws_tee_request_error}; + +static ScrDyn *ws_tee_pull(void *ctx, WsRController *c) { + (void)c; + WsTee *t = ctx; + /* `async function pullAlgorithm`: a promise fulfilled with undefined. */ + if (!t->reading) { + t->reading = true; + WsReadRequest *q = ws_alloc(sizeof *q); + q->ops = &ws_tee_request_ops; + q->ctx = ws_retain(t); + ws_reader_read_request(t->reader, q); + } + ScrDyn *u = ws_undef(); + ScrPromise *p = ws_promise_resolved(u); + scr_dyn_release(u); + ScrDyn *box = scr_dyn_new_promise(p); + scr_promise_release(p); + return box; +} + +static ScrPromise *ws_tee_cancel(void *ctx, WsRController *c, ScrDyn *reason) { + WsTee *t = ctx; + if (c->native_branch == 1) { + t->canceled1 = true; + t->reason1 = scr_dyn_retain(reason); + } else { + t->canceled2 = true; + t->reason2 = scr_dyn_retain(reason); + } + if (t->canceled1 && t->canceled2) { + ScrDyn *composite = scr_dyn_new_arr(); + scr_dyn_arr_push(composite, scr_dyn_retain(t->reason1)); + scr_dyn_arr_push(composite, scr_dyn_retain(t->reason2)); + ScrPromise *cancel = ws_readable_cancel(t->reader->stream, composite); + scr_dyn_release(composite); + ws_deferred_resolve_promise(&t->cancel_promise, cancel); + scr_promise_release(cancel); + } + return scr_promise_retain(t->cancel_promise.promise); +} + +static const WsSourceOps ws_tee_source_ops = {ws_tee_pull, ws_tee_cancel}; + +/* The reaction to the reader's closed promise rejecting (Node's tee). */ +static void ws_tee_source_errored(void *ctx, ScrDyn *unused) { + (void)unused; + WsTee *t = ctx; + ScrDyn *error = t->source_error; + ws_rcontroller_error(t->branch1->controller, error); + ws_rcontroller_error(t->branch2->controller, error); + if (!t->canceled1 || !t->canceled2) ws_deferred_resolve_undefined(&t->cancel_promise); +} + +/* The source stream errored: the reaction job runs where the closed + * promise's would. */ +static void ws_tee_source_error(WsTee *t, ScrDyn *error) { + if (t->source_error) return; + t->source_error = scr_dyn_retain(error); + ws_queue_microtask(ws_tee_source_errored, ws_retain(t)); +} + +/* readableStreamDefaultTee: false with the exception pending. */ +static bool ws_readable_tee(WsReadable *s, WsReadable **out1, WsReadable **out2) { + WsReader *reader = ws_reader_new(s); + if (!reader) return false; + WsTee *t = ws_obj_new(sizeof *t, &ws_tee_type); + t->reader = reader; + ws_deferred_init(&t->cancel_promise); + /* Each branch's controller holds the tee as its source until it clears + * its algorithms, and the tee holds the branches. */ + t->branch1 = ws_readable_create_native(&ws_tee_source_ops, ws_retain(t), 1, 1); + t->branch2 = ws_readable_create_native(&ws_tee_source_ops, ws_retain(t), 2, 1); + reader->tee = ws_retain(t); + if (s->state == WS_STATE_ERRORED) ws_tee_source_error(t, s->stored_error); + *out1 = ws_readable_retain(t->branch1); + *out2 = ws_readable_retain(t->branch2); + ws_tee_release(t); + return true; +} + +/* ── ReadableStream.from (readableStreamFromIterable) ─────────────── */ + +typedef struct WsFrom { + WsObj obj; + WsReadable *stream; /* +1 until the stream clears its algorithms */ + ScrDyn *iterator; + ScrDyn *next_method; + ScrDyn *pending_result; +} WsFrom; + +static void ws_from_trace(void *obj, ScrTraceVisit visit, void *ctx) { + WsFrom *f = obj; + ws_visit(f->stream, visit, ctx); + ws_visit(f->iterator, visit, ctx); + ws_visit(f->next_method, visit, ctx); + ws_visit(f->pending_result, visit, ctx); +} + +static void ws_from_drop(void *obj, bool gc) { + WsFrom *f = obj; + ws_drop_obj(f->stream, gc); + ws_drop_dyn(f->iterator, gc); + ws_drop_dyn(f->next_method, gc); + ws_drop_dyn(f->pending_result, gc); +} + +static const WsObjType ws_from_type = {ws_from_trace, ws_from_drop}; + +/* The stream's controller while it still holds this source; NULL once it + * cleared its algorithms (closed, errored or canceled). */ +static WsRController *ws_from_controller(WsFrom *f) { + WsRController *c = f->stream->controller; + return c->algorithms_cleared ? NULL : c; +} + +static void ws_from_reject_pull(void *ctx, ScrDyn *error) { + WsRController *c = ws_from_controller(ctx); + if (c) ws_rcontroller_error(c, error); +} + +static void ws_from_pull_fulfilled(WsFrom *f) { + WsRController *c = ws_from_controller(f); + if (c) ws_pull_fulfilled(c, NULL); +} + +static void ws_from_enqueue_value(void *ctx, ScrDyn *value) { + WsFrom *f = ctx; + WsRController *c = ws_from_controller(f); + if (!c) return; + if (!ws_rcontroller_enqueue(c, value)) { + ScrDyn *error = ws_take_pending(); + ws_from_reject_pull(f, error); + scr_dyn_release(error); + return; + } + ws_from_pull_fulfilled(f); +} + +static void ws_from_process_result(void *ctx, ScrDyn *result) { + WsFrom *f = ctx; + WsRController *c = ws_from_controller(f); + if (!c) return; + if (!ws_is_object(result)) { + ScrDyn *error = ws_state_error_value( + "The promise returned by the iterator.next() method must fulfill with an object"); + ws_from_reject_pull(f, error); + scr_dyn_release(error); + return; + } + ScrDyn *done = ws_get(result, "done"); + if (!done) goto threw; + bool finished = scr_dyn_truthy(done); + scr_dyn_release(done); + if (finished) { + ws_rcontroller_close(c); + } else { + ScrDyn *value = ws_get(result, "value"); + if (!value) goto threw; + if (ws_is_object(value)) { + ScrPromise *p = ws_promise_resolve(value); + scr_dyn_release(value); + if (!p) goto threw; + ws_then(p, ws_from_enqueue_value, ws_from_reject_pull, ws_retain(f), false); + scr_promise_release(p); + return; + } + bool ok = ws_rcontroller_enqueue(c, value); + scr_dyn_release(value); + if (!ok) goto threw; + } + ws_from_pull_fulfilled(f); + return; +threw: { + ScrDyn *error = ws_take_pending(); + ws_from_reject_pull(f, error); + scr_dyn_release(error); +} +} + +static void ws_from_process_pending(void *ctx, ScrDyn *unused) { + (void)unused; + WsFrom *f = ctx; + ScrDyn *result = f->pending_result; + f->pending_result = NULL; + ws_from_process_result(f, result); + scr_dyn_release(result); +} + +static ScrDyn *ws_from_pull(void *ctx, WsRController *c) { + (void)c; + WsFrom *f = ctx; + /* A native iterator (an array's) answers next as a member call. */ + ScrDyn *next = scr_dyn_is_callable(f->next_method) + ? ws_call(f->next_method, f->iterator, NULL, 0, "iterator.next") + : scr_dyn_invoke(f->iterator, "next", NULL, 0, "iterator.next"); + if (!next) return NULL; + if (ws_is_object(next)) { + ScrPromise *p = ws_promise_resolve(next); + scr_dyn_release(next); + if (!p) return NULL; + ws_then(p, ws_from_process_result, ws_from_reject_pull, ws_retain(f), false); + scr_promise_release(p); + return WS_PARKED; + } + if (f->pending_result) scr_dyn_release(f->pending_result); + f->pending_result = next; + ws_then(ws_resolved_singleton(), ws_from_process_pending, NULL, ws_retain(f), false); + return WS_PARKED; +} + +/* `async function cancelAlgorithm(reason)`: awaits iterator.return. */ +typedef struct WsFromCancel { + WsObj obj; + WsDeferred result; +} WsFromCancel; + +static void ws_from_cancel_trace(void *obj, ScrTraceVisit visit, void *ctx) { + ws_visit(((WsFromCancel *)obj)->result.promise, visit, ctx); +} + +static void ws_from_cancel_drop(void *obj, bool gc) { ws_drop_promise(((WsFromCancel *)obj)->result.promise, gc); } + +static const WsObjType ws_from_cancel_type = {ws_from_cancel_trace, ws_from_cancel_drop}; + +static void ws_from_cancel_returned(void *ctx, ScrDyn *iter_result) { + WsDeferred *d = &((WsFromCancel *)ctx)->result; + if (!ws_is_object(iter_result)) { + ScrDyn *error = ws_state_error_value( + "The promise returned by the iterator.return() method must fulfill with an object"); + ws_deferred_reject(d, error); + scr_dyn_release(error); + return; + } + ws_deferred_resolve_undefined(d); +} + +static void ws_from_cancel_failed(void *ctx, ScrDyn *error) { + ws_deferred_reject(&((WsFromCancel *)ctx)->result, error); +} + +static ScrPromise *ws_from_cancel(void *ctx, WsRController *c, ScrDyn *reason) { + (void)c; + WsFrom *f = ctx; + ScrDyn *method = ws_get(f->iterator, "return"); + if (!method) return ws_promise_rejected_pending(); + if (method->kind == SCR_DYN_UNDEF) { + scr_dyn_release(method); + ScrDyn *u = ws_undef(); + ScrPromise *p = ws_promise_resolved(u); + scr_dyn_release(u); + return p; + } + ScrDyn *returned = ws_call(method, f->iterator, &reason, 1, "iterator.return"); + scr_dyn_release(method); + if (!returned) return ws_promise_rejected_pending(); + /* `await returned`, then the check. */ + ScrPromise *awaited = ws_promise_resolve(returned); + scr_dyn_release(returned); + if (!awaited) return ws_promise_rejected_pending(); + WsFromCancel *wait = ws_obj_new(sizeof *wait, &ws_from_cancel_type); + ws_deferred_init(&wait->result); + ScrPromise *result = scr_promise_retain(wait->result.promise); + ws_then(awaited, ws_from_cancel_returned, ws_from_cancel_failed, wait, false); + scr_promise_release(awaited); + return result; +} + +static const WsSourceOps ws_from_source_ops = {ws_from_pull, ws_from_cancel}; + +static ScrDyn *ws_readable_from(ScrDyn *iterable) { + /* `iterable[SymbolAsyncIterator]` on undefined or null. */ + if (iterable->kind == SCR_DYN_UNDEF || iterable->kind == SCR_DYN_NULL) { + char buf[96]; + int n = snprintf(buf, sizeof buf, "Cannot read properties of %s (reading 'Symbol(Symbol.asyncIterator)')", + iterable->kind == SCR_DYN_NULL ? "null" : "undefined"); + scr_throw_error_msg(SCR_ERR_TYPE, buf, (size_t)n); + return NULL; + } + int kind = 0; + ScrDyn *iterator = scr_dyn_open_iterator(iterable, &kind, true); + if (kind < 0) return NULL; + if (!iterator) { + /* ERR_ARG_NOT_ITERABLE's '%s must be iterable', which renders a + * function by its source text. */ + if (scr_dyn_is_callable(iterable)) { + static const char message[] = "ReadableStream.from of a function that is not iterable is not supported yet"; + scr_throw_error_msg_code(SCR_ERR_ERROR, message, sizeof message - 1, "SC2020"); + return NULL; + } + ScrStr *text = scr_insp_dyn_format_s(iterable, 0); + if (!text) return NULL; + size_t n = text->len + 17; + char *buf = malloc(n); + if (!buf) ws_oom(); + memcpy(buf, text->data, text->len); + memcpy(buf + text->len, " must be iterable", 17); + scr_str_release(text); + scr_throw_error_msg_code(SCR_ERR_TYPE, buf, n, "ERR_ARG_NOT_ITERABLE"); + free(buf); + return NULL; + } + if (!ws_is_object(iterator)) { + scr_dyn_release(iterator); + ws_state_error("The iterator method must return an object"); + return NULL; + } + ScrDyn *next = ws_get(iterator, "next"); + if (!next) { + scr_dyn_release(iterator); + return NULL; + } + WsFrom *f = ws_obj_new(sizeof *f, &ws_from_type); + f->iterator = iterator; + f->next_method = next; + WsReadable *s = ws_readable_create_native(&ws_from_source_ops, f, 0, 0); + f->stream = ws_readable_retain(s); + ScrDyn *box = ws_stream_box(s); + ws_readable_release(s); + return box; +} + +static ScrDyn *ws_readable_from_thunk(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + (void)closure; + return ws_readable_from(argc > 0 ? args[0] : scr_dyn_undefined()); +} + +/* ── async iteration (ReadableStream.prototype.values) ────────────── */ + +typedef struct WsIter { + WsObj obj; + WsReader *reader; + bool done, started, prevent_cancel; + ScrPromise *current; /* state.current, +1 when set */ +} WsIter; + +static void ws_iter_trace(void *obj, ScrTraceVisit visit, void *ctx) { + WsIter *it = obj; + ws_visit(it->reader, visit, ctx); + ws_visit(it->current, visit, ctx); +} + +static void ws_iter_drop(void *obj, bool gc) { + WsIter *it = obj; + ws_drop_obj(it->reader, gc); + ws_drop_promise(it->current, gc); +} + +static const WsObjType ws_iter_type = {ws_iter_trace, ws_iter_drop}; + +static void ws_iter_release(WsIter *it) { ws_release(it); } + +static void ws_iter_set_current(WsIter *it, ScrPromise *p) { + if (it->current) scr_promise_release(it->current); + it->current = p; +} + +static ScrDyn *ws_iter_result(bool done, ScrDyn *value) { + ScrDyn *result = scr_dyn_new_obj(); + scr_dyn_obj_set(result, "done", 4, scr_dyn_new_bool(done)); + scr_dyn_obj_set(result, "value", 5, scr_dyn_retain(value)); + return result; +} + +static void ws_iter_chunk(WsReadRequest *q, ScrDyn *chunk) { + WsIter *it = q->ctx; + ws_iter_set_current(it, NULL); + ScrDyn *result = ws_read_result(chunk, false); + ws_fulfill(q->promise, result); + scr_dyn_release(result); + scr_promise_release(q->promise); + ws_iter_release(it); + free(q); +} + +static void ws_iter_close(WsReadRequest *q) { + WsIter *it = q->ctx; + ws_iter_set_current(it, NULL); + it->done = true; + if (it->reader->stream) ws_reader_generic_release(it->reader); + ScrDyn *u = ws_undef(); + ScrDyn *result = ws_iter_result(true, u); + scr_dyn_release(u); + ws_fulfill(q->promise, result); + scr_dyn_release(result); + scr_promise_release(q->promise); + ws_iter_release(it); + free(q); +} + +static void ws_iter_error(WsReadRequest *q, ScrDyn *error) { + WsIter *it = q->ctx; + ws_iter_set_current(it, NULL); + it->done = true; + if (it->reader->stream) ws_reader_generic_release(it->reader); + ws_reject(q->promise, error); + scr_promise_release(q->promise); + ws_iter_release(it); + free(q); +} + +static const WsReadRequestOps ws_iter_request_ops = {ws_iter_chunk, ws_iter_close, ws_iter_error}; + +/* nextSteps: +1 promise. */ +static ScrPromise *ws_iter_next_steps(void *ctx, ScrDyn *unused) { + (void)unused; + WsIter *it = ctx; + if (it->done) { + ScrDyn *u = ws_undef(); + ScrDyn *result = ws_iter_result(true, u); + scr_dyn_release(u); + ScrPromise *p = ws_promise_resolved(result); + scr_dyn_release(result); + return p; + } + if (!it->reader->stream) { + ScrDyn *error = ws_state_error_value("The reader is not bound to a ReadableStream"); + ScrPromise *p = ws_promise_rejected(error); + scr_dyn_release(error); + return p; + } + WsReadRequest *q = ws_alloc(sizeof *q); + q->ops = &ws_iter_request_ops; + q->promise = scr_promise_new(); + q->ctx = ws_retain(it); + ScrPromise *result = scr_promise_retain(q->promise); + ws_reader_read_request(it->reader, q); + return result; +} + +/* The read() and next() fast path: a queued chunk, dequeued now. */ +static bool ws_dequeue_fast(WsReadable *s, ScrDyn **chunk_out) { + if (!s || s->state != WS_STATE_READABLE) return false; + WsRController *c = s->controller; + if (!c->queue.length) return false; + s->disturbed = true; + double size; + ScrDyn *chunk = ws_queue_shift(&c->queue, &size); + c->queue_total_size = fmax(0, c->queue_total_size - size); + if (c->close_requested && !c->queue.length) { + ws_rcontroller_clear_algorithms(c); + ws_readable_close(s); + } else if (!c->close_requested && c->started && c->high_water_mark - c->queue_total_size > 0) { + ws_rcontroller_pull(c); + } + *chunk_out = chunk; + return true; +} + +static WsIter *ws_iter_of(ScrClosure *closure) { return scr_box_get_ref(closure->caps[0]); } + +static ScrDyn *ws_iter_next(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + (void)args; + (void)argc; + WsIter *it = ws_iter_of(closure); + ScrPromise *result; + if (!it->started) { + ScrDyn *u = ws_undef(); + ws_iter_set_current(it, ws_promise_resolved(u)); + scr_dyn_release(u); + it->started = true; + } + ScrDyn *chunk; + if (it->current) { + ScrPromise *chained = ws_then_adopt(it->current, ws_iter_next_steps, ws_iter_next_steps, ws_retain(it)); + ws_iter_set_current(it, chained); + result = scr_promise_retain(chained); + } else if (!it->done && ws_dequeue_fast(it->reader->stream, &chunk)) { + ScrDyn *record = ws_read_result(chunk, false); + scr_dyn_release(chunk); + result = ws_promise_resolved(record); + scr_dyn_release(record); + } else { + result = ws_iter_next_steps(it, NULL); + ws_iter_set_current(it, scr_promise_retain(result)); + } + ws_iter_release(it); + ScrDyn *box = scr_dyn_new_promise(result); + scr_promise_release(result); + return box; +} + +/* returnSteps(value): `async function`, so the result is a promise. */ +typedef struct WsIterReturn { + WsObj obj; + WsIter *it; + ScrDyn *value; +} WsIterReturn; + +static void ws_iter_return_trace(void *obj, ScrTraceVisit visit, void *ctx) { + WsIterReturn *r = obj; + ws_visit(r->it, visit, ctx); + ws_visit(r->value, visit, ctx); +} + +static void ws_iter_return_drop(void *obj, bool gc) { + WsIterReturn *r = obj; + ws_drop_obj(r->it, gc); + ws_drop_dyn(r->value, gc); +} + +static const WsObjType ws_iter_return_type = {ws_iter_return_trace, ws_iter_return_drop}; + +/* `await result; return record` inside returnSteps. */ +typedef struct WsAwaitReturn { + WsObj obj; + ScrPromise *out; + ScrDyn *record; +} WsAwaitReturn; + +static void ws_await_return_trace(void *obj, ScrTraceVisit visit, void *ctx) { + WsAwaitReturn *a = obj; + ws_visit(a->out, visit, ctx); + ws_visit(a->record, visit, ctx); +} + +static void ws_await_return_drop(void *obj, bool gc) { + WsAwaitReturn *a = obj; + ws_drop_promise(a->out, gc); + ws_drop_dyn(a->record, gc); +} + +static const WsObjType ws_await_return_type = {ws_await_return_trace, ws_await_return_drop}; + +static void ws_await_return_ok(void *ctx, ScrDyn *unused) { + (void)unused; + WsAwaitReturn *a = ctx; + ws_fulfill(a->out, a->record); +} + +static void ws_await_return_err(void *ctx, ScrDyn *reason) { + WsAwaitReturn *a = ctx; + ws_reject(a->out, reason); +} + +static ScrPromise *ws_iter_return_steps(WsIter *it, ScrDyn *value) { + ScrDyn *record = ws_iter_result(true, value); + ScrPromise *out; + if (it->done) { + out = ws_promise_resolved(record); + scr_dyn_release(record); + return out; + } + it->done = true; + if (!it->reader->stream) { + scr_dyn_release(record); + ScrDyn *error = ws_state_error_value("The reader is not bound to a ReadableStream"); + out = ws_promise_rejected(error); + scr_dyn_release(error); + return out; + } + if (!it->prevent_cancel) { + ScrPromise *cancel = ws_readable_cancel(it->reader->stream, value); + ws_reader_generic_release(it->reader); + WsAwaitReturn *a = ws_obj_new(sizeof *a, &ws_await_return_type); + out = scr_promise_new(); + a->out = scr_promise_retain(out); + a->record = record; + ws_then(cancel, ws_await_return_ok, ws_await_return_err, a, false); + scr_promise_release(cancel); + return out; + } + ws_reader_generic_release(it->reader); + out = ws_promise_resolved(record); + scr_dyn_release(record); + return out; +} + +static ScrPromise *ws_iter_return_pstep(void *ctx, ScrDyn *unused) { + (void)unused; + WsIterReturn *r = ctx; + return ws_iter_return_steps(r->it, r->value); +} + +static ScrDyn *ws_iter_return(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + WsIter *it = ws_iter_of(closure); + ScrDyn *value = argc > 0 ? args[0] : scr_dyn_undefined(); + it->started = true; + ScrPromise *result; + if (it->current) { + WsIterReturn *r = ws_obj_new(sizeof *r, &ws_iter_return_type); + r->it = ws_retain(it); + r->value = scr_dyn_retain(value); + result = ws_then_adopt(it->current, ws_iter_return_pstep, ws_iter_return_pstep, r); + } else { + result = ws_iter_return_steps(it, value); + } + ws_iter_set_current(it, scr_promise_retain(result)); + ws_iter_release(it); + ScrDyn *box = scr_dyn_new_promise(result); + scr_promise_release(result); + return box; +} + +static ScrDyn *ws_iter_method(WsIter *it, ScrDynThunk thunk, const char *name) { + ScrClosure *closure = scr_closure_new(NULL, 1); + /* A box with a trace visits its payload, which carries a header. */ + closure->caps[0] = scr_box_new_obj(ws_retain, ws_release, ws_obj_trace); + scr_box_set_ref(closure->caps[0], ws_retain(it)); + return scr_dyn_new_func(closure, thunk, 0, "", name); +} + +static ScrDyn *ws_self_thunk(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + (void)closure; + (void)args; + (void)argc; + return scr_dyn_this_get(); +} + +/* values(options): the async iterator object, +1. */ +static ScrDyn *ws_readable_values(WsReadable *s, ScrDyn *options) { + if (!ws_validate_object(options, "options", true)) return NULL; + ScrDyn *prevent = ws_get(options, "preventCancel"); + if (!prevent) return NULL; + bool prevent_cancel = scr_dyn_truthy(prevent); + scr_dyn_release(prevent); + WsReader *reader = ws_reader_new(s); + if (!reader) return NULL; + WsIter *it = ws_obj_new(sizeof *it, &ws_iter_type); + it->reader = reader; + it->prevent_cancel = prevent_cancel; + ScrDyn *object = scr_dyn_new_obj(); + scr_dyn_obj_set(object, "next", 4, ws_iter_method(it, ws_iter_next, "next")); + scr_dyn_obj_set(object, "return", 6, ws_iter_method(it, ws_iter_return, "return")); + ScrDyn *self_fn = scr_dyn_new_func(scr_closure_new(NULL, 0), ws_self_thunk, 0, "", "[Symbol.asyncIterator]"); + ScrDyn *key = scr_dyn_async_iterator_symbol_value(); + if (key) { + scr_dyn_symbol_key_set(object, key, self_fn); + scr_dyn_release(key); + } + scr_dyn_release(self_fn); + ws_iter_release(it); + return object; +} + +/* ── sink read requests (Readable.fromWeb) ─────────────────────────── */ + +typedef struct WsSinkRequest { + WsReadRequest base; + const ScrFetchReadSink *sink; + void *sink_ctx; +} WsSinkRequest; + +static void ws_sink_chunk(WsReadRequest *q, ScrDyn *chunk) { + WsSinkRequest *r = (WsSinkRequest *)q; + r->sink->chunk(r->sink_ctx, chunk); + free(r); +} + +static void ws_sink_close(WsReadRequest *q) { + WsSinkRequest *r = (WsSinkRequest *)q; + r->sink->close(r->sink_ctx); + free(r); +} + +static void ws_sink_error(WsReadRequest *q, ScrDyn *error) { + WsSinkRequest *r = (WsSinkRequest *)q; + r->sink->error(r->sink_ctx, error); + free(r); +} + +static const WsReadRequestOps ws_sink_request_ops = {ws_sink_chunk, ws_sink_close, ws_sink_error}; + +static bool ws_reader_read_into(ScrDyn *reader, const ScrFetchReadSink *sink, void *ctx) { + WsReader *r = reader->v.handle.ptr; + if (!r->stream) { + ws_state_error("The reader is not attached to a stream"); + return false; + } + WsSinkRequest *q = ws_alloc(sizeof *q); + q->base.ops = &ws_sink_request_ops; + q->sink = sink; + q->sink_ctx = ctx; + ws_reader_read_request(r, &q->base); + return true; +} + +/* ── the handle surfaces ───────────────────────────────────────────── */ + +static bool ws_name(const char *key, size_t len, const char *want) { + return strlen(want) == len && memcmp(key, want, len) == 0; +} + +/* A member call of a name that is no method of the class: the member's + * value (an accessor such as a strategy's size), called with the handle as + * its receiver; " is not a function" when it is not callable. */ +static ScrDyn *ws_invoke_member(ScrDyn *self, const char *method, ScrDyn *const *args, size_t argc, + const char *what) { + ScrStr *key = scr_str_new(method, strlen(method)); + ScrDyn *member = scr_dyn_handle_key_get(self, key); + scr_str_release(key); + if (!member) return NULL; + if (!scr_dyn_is_callable(member)) { + scr_dyn_release(member); + char buf[256]; + int n = snprintf(buf, sizeof buf, "%s is not a function", what && *what ? what : method); + scr_throw_error_msg(SCR_ERR_TYPE, buf, (size_t)n); + return NULL; + } + ScrDyn *result = ws_call(member, self, args, argc, what); + scr_dyn_release(member); + return result; +} + +static ScrDyn *ws_arg(ScrDyn *const *args, size_t argc, size_t i) { + return i < argc ? args[i] : scr_dyn_undefined(); +} + +static ScrDyn *ws_promise_box(ScrPromise *p) { + ScrDyn *box = scr_dyn_new_promise(p); + scr_promise_release(p); + return box; +} + +static ScrDyn *ws_rejected_pending_box(void) { return ws_promise_box(ws_promise_rejected_pending()); } + +/* ReadableStream */ + +static ScrDyn *ws_readable_get_reader(WsReadable *s, ScrDyn *options) { + if (!ws_validate_object(options, "options", true)) return NULL; + ScrDyn *mode = ws_get(options, "mode"); + if (!mode) return NULL; + if (mode->kind != SCR_DYN_UNDEF) { + ScrStr *text = scr_dyn_string_coerce_js(mode); + if (!text) { + scr_dyn_release(mode); + return NULL; + } + bool byob = text->len == 4 && memcmp(text->data, "byob", 4) == 0; + scr_str_release(text); + if (byob) { + scr_dyn_release(mode); + static const char message[] = "ReadableStreamBYOBReader is not supported yet"; + scr_throw_error_msg_code(SCR_ERR_ERROR, message, sizeof message - 1, "SC2020"); + return NULL; + } + scr_dyn_arg_value_fail("options.mode", NULL, mode); + scr_dyn_release(mode); + return NULL; + } + scr_dyn_release(mode); + WsReader *r = ws_reader_new(s); + if (!r) return NULL; + ScrDyn *box = ws_reader_box(r); + ws_reader_release(r); + return box; +} + +static ScrDyn *ws_readable_invoke(void *ptr, ScrDyn *self, const char *method, ScrDyn *const *args, size_t argc, + const char *what) { + (void)self; + (void)what; + WsReadable *s = ptr; + size_t len = strlen(method); + if (ws_name(method, len, "cancel")) { + if (ws_readable_locked(s)) { + ScrDyn *error = ws_state_error_value("ReadableStream is locked"); + ScrPromise *p = ws_promise_rejected(error); + scr_dyn_release(error); + return ws_promise_box(p); + } + return ws_promise_box(ws_readable_cancel(s, ws_arg(args, argc, 0))); + } + if (ws_name(method, len, "getReader")) { + ScrDyn *options = ws_arg(args, argc, 0); + ScrDyn *empty = options->kind == SCR_DYN_UNDEF ? scr_dyn_new_obj() : NULL; + ScrDyn *reader = ws_readable_get_reader(s, empty ? empty : options); + if (empty) scr_dyn_release(empty); + return reader; + } + if (ws_name(method, len, "tee")) { + WsReadable *a, *b; + if (!ws_readable_tee(s, &a, &b)) return NULL; + ScrDyn *pair = scr_dyn_new_arr(); + scr_dyn_arr_push(pair, ws_stream_box(a)); + scr_dyn_arr_push(pair, ws_stream_box(b)); + ws_readable_release(a); + ws_readable_release(b); + return pair; + } + if (ws_name(method, len, "values")) { + ScrDyn *options = ws_arg(args, argc, 0); + ScrDyn *empty = options->kind == SCR_DYN_UNDEF ? scr_dyn_new_obj() : NULL; + ScrDyn *iterator = ws_readable_values(s, empty ? empty : options); + if (empty) scr_dyn_release(empty); + return iterator; + } + if (ws_name(method, len, "pipeTo") || ws_name(method, len, "pipeThrough")) { + static const char message[] = "ReadableStream.prototype.pipeTo and pipeThrough are not supported yet"; + scr_throw_error_msg_code(SCR_ERR_ERROR, message, sizeof message - 1, "SC2020"); + return NULL; + } + return ws_invoke_member(self, method, args, argc, what); +} + +static ScrDyn *ws_readable_get(void *ptr, const char *key, size_t len) { + WsReadable *s = ptr; + if (ws_name(key, len, "locked")) return scr_dyn_new_bool(ws_readable_locked(s)); + return scr_dyn_handle_class_member(SCR_DYNH_WS_READABLE, ptr, key, len); +} + +/* Symbol.asyncIterator is values. */ +static ScrDyn *ws_readable_symbol_get(void *ptr, const ScrDyn *key) { + (void)ptr; + if (!scr_dyn_is_async_iterator_symbol(key)) return NULL; + return scr_dyn_handle_class_method(SCR_DYNH_WS_READABLE, "values"); +} + +static ScrDyn *ws_rejected_value(void) { return ws_rejected_pending_box(); } + +static const ScrDynHandleMethod ws_readable_methods[] = { + {"cancel", 0, true, NULL}, {"getReader", 0, false, NULL}, {"pipeThrough", 1, false, NULL}, + {"pipeTo", 1, true, NULL}, {"tee", 0, false, NULL}, {"values", 0, false, NULL}, + {NULL, 0, false, NULL}, +}; + +static const ScrDynHandleAccessor ws_readable_accessors[] = {{"locked", false}, {NULL, false}}; + +static const ScrDynHandleStatic ws_readable_statics[] = {{"from", 1, ws_readable_from_thunk}, {NULL, 0, NULL}}; + +static const char *const ws_readable_order[] = {"locked", "cancel", "getReader", "pipeThrough", "pipeTo", + "tee", "values", NULL}; + +static const ScrDynHandleClass ws_readable_class = { + "ReadableStream", 0, ws_readable_construct, ws_readable_accessors, ws_readable_statics, ws_readable_order, +}; + +/* ReadableStreamDefaultReader */ + +static ScrDyn *ws_reader_construct(ScrDyn *const *args, size_t argc) { + ScrDyn *stream = ws_arg(args, argc, 0); + WsReadable *s = ws_stream_of(stream); + if (!s) { + scr_dyn_arg_type_fail("stream", "an instance of ReadableStream", stream); + return NULL; + } + WsReader *r = ws_reader_new(s); + if (!r) return NULL; + ScrDyn *box = ws_reader_box(r); + ws_reader_release(r); + return box; +} + +static ScrDyn *ws_reader_detached(void) { + ScrDyn *error = ws_state_error_value("The reader is not attached to a stream"); + ScrPromise *p = ws_promise_rejected(error); + scr_dyn_release(error); + return ws_promise_box(p); +} + +static ScrDyn *ws_reader_read(WsReader *r) { + if (!r->stream) return ws_reader_detached(); + ScrDyn *chunk; + if (ws_dequeue_fast(r->stream, &chunk)) { + ScrDyn *record = ws_read_result(chunk, false); + scr_dyn_release(chunk); + ScrPromise *p = ws_promise_resolved(record); + scr_dyn_release(record); + return ws_promise_box(p); + } + WsReadRequest *q = ws_alloc(sizeof *q); + q->ops = &ws_default_request_ops; + q->promise = scr_promise_new(); + ScrPromise *result = scr_promise_retain(q->promise); + ws_reader_read_request(r, q); + return ws_promise_box(result); +} + +static ScrDyn *ws_reader_invoke(void *ptr, ScrDyn *self, const char *method, ScrDyn *const *args, size_t argc, + const char *what) { + (void)self; + (void)what; + WsReader *r = ptr; + size_t len = strlen(method); + if (ws_name(method, len, "read")) return ws_reader_read(r); + if (ws_name(method, len, "releaseLock")) { + if (r->stream) ws_reader_release_lock(r); + return ws_undef(); + } + if (ws_name(method, len, "cancel")) { + if (!r->stream) return ws_reader_detached(); + return ws_promise_box(ws_readable_cancel(r->stream, ws_arg(args, argc, 0))); + } + return ws_invoke_member(self, method, args, argc, what); +} + +static ScrDyn *ws_reader_get(void *ptr, const char *key, size_t len) { + WsReader *r = ptr; + if (ws_name(key, len, "closed")) return scr_dyn_new_promise(ws_reader_closed_promise(r)); + return scr_dyn_handle_class_member(SCR_DYNH_WS_READER, ptr, key, len); +} + +static const ScrDynHandleMethod ws_reader_methods[] = { + {"read", 0, true, NULL}, {"releaseLock", 0, false, NULL}, {"cancel", 0, true, NULL}, {NULL, 0, false, NULL}, +}; + +static const ScrDynHandleAccessor ws_reader_accessors[] = {{"closed", true}, {NULL, false}}; + +static const char *const ws_reader_order[] = {"read", "releaseLock", "closed", "cancel", NULL}; + +static const ScrDynHandleClass ws_reader_class = { + "ReadableStreamDefaultReader", 1, ws_reader_construct, ws_reader_accessors, NULL, ws_reader_order, +}; + +/* ReadableStreamDefaultController */ + +static ScrDyn *ws_rcontroller_invoke(void *ptr, ScrDyn *self, const char *method, ScrDyn *const *args, + size_t argc, const char *what) { + (void)self; + (void)what; + WsRController *c = ptr; + size_t len = strlen(method); + if (ws_name(method, len, "close")) { + if (!ws_rcontroller_can_close_or_enqueue(c)) { + ws_state_error("Controller is already closed"); + return NULL; + } + ws_rcontroller_close(c); + return ws_undef(); + } + if (ws_name(method, len, "enqueue")) { + if (!ws_rcontroller_can_close_or_enqueue(c)) { + ws_state_error("Controller is already closed"); + return NULL; + } + if (!ws_rcontroller_enqueue(c, ws_arg(args, argc, 0))) return NULL; + return ws_undef(); + } + if (ws_name(method, len, "error")) { + ws_rcontroller_error(c, ws_arg(args, argc, 0)); + return ws_undef(); + } + return ws_invoke_member(self, method, args, argc, what); +} + +static ScrDyn *ws_rcontroller_get(void *ptr, const char *key, size_t len) { + if (ws_name(key, len, "desiredSize")) return ws_rcontroller_desired_size(ptr); + return scr_dyn_handle_class_member(SCR_DYNH_WS_RCONTROLLER, ptr, key, len); +} + +static const ScrDynHandleMethod ws_rcontroller_methods[] = { + {"close", 0, false, NULL}, {"enqueue", 0, false, NULL}, {"error", 0, false, NULL}, {NULL, 0, false, NULL}, +}; + +static const ScrDynHandleAccessor ws_rcontroller_accessors[] = {{"desiredSize", false}, {NULL, false}}; + +static const char *const ws_rcontroller_order[] = {"desiredSize", "close", "enqueue", "error", NULL}; + +static const ScrDynHandleClass ws_rcontroller_class = { + "ReadableStreamDefaultController", 0, NULL, ws_rcontroller_accessors, NULL, ws_rcontroller_order, +}; + +/* ── queuing strategies ────────────────────────────────────────────── */ + +typedef struct WsStrategy { + size_t rc; + bool bytes; + double high_water_mark; +} WsStrategy; + +static void *ws_strategy_retain(void *p) { + ((WsStrategy *)p)->rc++; + return p; +} +static void ws_strategy_release(void *p) { + WsStrategy *q = p; + if (--q->rc == 0) free(q); +} + +static ScrDyn *ws_count_size(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + (void)closure; + (void)args; + (void)argc; + return scr_dyn_new_num(1); +} + +/* (chunk) => chunk.byteLength */ +static ScrDyn *ws_byte_size(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + (void)closure; + return ws_get(ws_arg(args, argc, 0), "byteLength"); +} + +static SCR_TL ScrDyn *ws_size_functions[2]; + +static void ws_size_functions_teardown(void) { + for (int i = 0; i < 2; i++) { + if (ws_size_functions[i]) scr_dyn_release(ws_size_functions[i]); + ws_size_functions[i] = NULL; + } +} + +static ScrDyn *ws_size_function(bool bytes) { + int i = bytes ? 1 : 0; + if (!ws_size_functions[i]) { + if (!ws_size_functions[0] && !ws_size_functions[1]) scr_atexit(ws_size_functions_teardown); + ws_size_functions[i] = scr_dyn_new_func(scr_closure_new(NULL, 0), bytes ? ws_byte_size : ws_count_size, + bytes ? 1 : 0, "", "size"); + } + return scr_dyn_retain(ws_size_functions[i]); +} + +static ScrDyn *ws_strategy_construct(ScrDyn *const *args, size_t argc, bool bytes) { + ScrDyn *init = ws_arg(args, argc, 0); + /* validateObject without options: arrays and functions are refused too. */ + if (init->kind == SCR_DYN_ARR || init->kind == SCR_DYN_FUNC || !ws_validate_object(init, "init", false)) { + if (!scr_exc_pending()) scr_dyn_arg_type_fail("init", "of type object", init); + return NULL; + } + ScrDyn *hwm = ws_get(init, "highWaterMark"); + if (!hwm) return NULL; + if (hwm->kind == SCR_DYN_UNDEF) { + scr_dyn_release(hwm); + static const char message[] = "init.highWaterMark is required"; + scr_throw_error_msg_code(SCR_ERR_TYPE, message, sizeof message - 1, "ERR_MISSING_OPTION"); + return NULL; + } + double value = scr_dyn_number_coerce(hwm); + scr_dyn_release(hwm); + if (scr_exc_pending()) return NULL; + ws_install(); + WsStrategy *q = ws_alloc(sizeof *q); + q->rc = 1; + q->bytes = bytes; + q->high_water_mark = value; + ScrDyn *box = scr_dyn_new_handle(q, bytes ? SCR_DYNH_WS_BYTE_STRATEGY : SCR_DYNH_WS_COUNT_STRATEGY); + ws_strategy_release(q); + return box; +} + +static ScrDyn *ws_count_strategy_construct(ScrDyn *const *args, size_t argc) { + return ws_strategy_construct(args, argc, false); +} + +static ScrDyn *ws_byte_strategy_construct(ScrDyn *const *args, size_t argc) { + return ws_strategy_construct(args, argc, true); +} + +static ScrDyn *ws_strategy_get(void *ptr, const char *key, size_t len) { + WsStrategy *q = ptr; + if (ws_name(key, len, "highWaterMark")) return scr_dyn_new_num(q->high_water_mark); + if (ws_name(key, len, "size")) return ws_size_function(q->bytes); + return scr_dyn_handle_class_member(q->bytes ? SCR_DYNH_WS_BYTE_STRATEGY : SCR_DYNH_WS_COUNT_STRATEGY, ptr, key, len); +} + +static ScrDyn *ws_strategy_invoke(void *ptr, ScrDyn *self, const char *method, ScrDyn *const *args, size_t argc, + const char *what) { + (void)ptr; + return ws_invoke_member(self, method, args, argc, what); +} + +static const ScrDynHandleAccessor ws_strategy_accessors[] = {{"highWaterMark", false}, {"size", false}, {NULL, false}}; + +static const ScrDynHandleClass ws_count_strategy_class = { + "CountQueuingStrategy", 1, ws_count_strategy_construct, ws_strategy_accessors, NULL, NULL, +}; + +static const ScrDynHandleClass ws_byte_strategy_class = { + "ByteLengthQueuingStrategy", 1, ws_byte_strategy_construct, ws_strategy_accessors, NULL, NULL, +}; + +/* ── installation and the compiler's entry points ─────────────────── */ + +static bool ws_set_noop(void *h, const char *key, size_t len, const ScrDyn *value) { + (void)h; + (void)key; + (void)len; + (void)value; + return false; +} + +#define WS_OPS(NAME, CLS, RETAIN, RELEASE, INVOKE, GET, METHODS, INVALID) \ + static const ScrDynHandleOps NAME = { \ + .cls = CLS, \ + .retain = RETAIN, \ + .release = RELEASE, \ + .invoke = INVOKE, \ + .get = GET, \ + .set = ws_set_noop, \ + .json_empty_object = true, \ + .to_string_tag = CLS, \ + .methods = METHODS, \ + .invalid_this = INVALID, \ + .rejected = ws_rejected_value, \ + } + +static const ScrDynHandleOps ws_readable_ops = { + .cls = "ReadableStream", + .retain = ws_retain, + .release = ws_release, + .invoke = ws_readable_invoke, + .get = ws_readable_get, + .set = ws_set_noop, + .json_empty_object = true, + .to_string_tag = "ReadableStream", + .symbol_get = ws_readable_symbol_get, + .methods = ws_readable_methods, + .invalid_this = "Value of \"this\" must be of type ReadableStream", + .rejected = ws_rejected_value, +}; + +WS_OPS(ws_reader_ops, "ReadableStreamDefaultReader", ws_retain, ws_release, ws_reader_invoke, + ws_reader_get, ws_reader_methods, + "Value of \"this\" must be of type ReadableStreamDefaultReader"); +WS_OPS(ws_rcontroller_ops, "ReadableStreamDefaultController", ws_retain, ws_release, + ws_rcontroller_invoke, ws_rcontroller_get, ws_rcontroller_methods, + "Value of \"this\" must be of type ReadableStreamDefaultController"); +WS_OPS(ws_count_strategy_ops, "CountQueuingStrategy", ws_strategy_retain, ws_strategy_release, ws_strategy_invoke, + ws_strategy_get, NULL, "Value of \"this\" must be of type CountQueuingStrategy"); +WS_OPS(ws_byte_strategy_ops, "ByteLengthQueuingStrategy", ws_strategy_retain, ws_strategy_release, + ws_strategy_invoke, ws_strategy_get, NULL, + "Value of \"this\" must be of type ByteLengthQueuingStrategy"); + +static SCR_TL bool ws_installed; + +static void ws_install(void) { + if (ws_installed) return; + ws_installed = true; + /* Symbol.asyncIterator, which the streams answer async iteration by + * (scr_dyn_iterable_kind, for await). */ + ScrStr *name = scr_str_new("asyncIterator", 13); + scr_sym_release(scr_sym_well_known(name)); + scr_str_release(name); + scr_web_reader_read_into_fn = ws_reader_read_into; + scr_dyn_handle_class_install(SCR_DYNH_WS_READABLE, &ws_readable_ops, &ws_readable_class); + scr_dyn_handle_class_install(SCR_DYNH_WS_READER, &ws_reader_ops, &ws_reader_class); + scr_dyn_handle_class_install(SCR_DYNH_WS_RCONTROLLER, &ws_rcontroller_ops, &ws_rcontroller_class); + scr_dyn_handle_class_install(SCR_DYNH_WS_COUNT_STRATEGY, &ws_count_strategy_ops, &ws_count_strategy_class); + scr_dyn_handle_class_install(SCR_DYNH_WS_BYTE_STRATEGY, &ws_byte_strategy_ops, &ws_byte_strategy_class); +} + +/* The classes in the frontend's WEB_STREAM_CLASSES order. */ +static ScrDynHandleTag ws_class_tag(double kind) { + static const ScrDynHandleTag tags[] = { + SCR_DYNH_WS_READABLE, SCR_DYNH_WS_READER, SCR_DYNH_WS_RCONTROLLER, SCR_DYNH_WS_COUNT_STRATEGY, + SCR_DYNH_WS_BYTE_STRATEGY, + }; + int index = (int)kind; + if (index < 0 || index >= (int)(sizeof tags / sizeof tags[0])) scr_trap("scriptc: unknown web stream class\n"); + return tags[index]; +} + +/* The web stream class `kind` names as a constructor value (+1): the + * compiler's webstream.class. */ +ScrDyn *scr_webstream_class(double kind) { + ws_install(); + return scr_dyn_handle_class_value(ws_class_tag(kind)); +} + +/* `value instanceof ` (webstream.instanceOf): a handle of the class, + * or an object with the class's prototype on its chain. */ +bool scr_webstream_instanceof(ScrDyn *value, double kind) { + ws_install(); + ScrDynHandleTag tag = ws_class_tag(kind); + if (value->kind == SCR_DYN_HANDLE) return value->v.handle.tag == tag; + if (value->kind != SCR_DYN_OBJ && value->kind != SCR_DYN_ARR && value->kind != SCR_DYN_FUNC) return false; + ScrDyn *prototype = scr_dyn_handle_class_prototype(tag); + bool found = false; + ScrDyn *current = scr_dyn_get_prototype(value); + while (current && current->kind != SCR_DYN_NULL && current->kind == SCR_DYN_OBJ) { + if (current == prototype) { + found = true; + break; + } + ScrDyn *next = scr_dyn_get_prototype(current); + scr_dyn_release(current); + current = next; + } + if (current) scr_dyn_release(current); + if (scr_exc_pending()) scr_exc_clear(); + scr_dyn_release(prototype); + return found; +} diff --git a/tests/corpus/web-stream-pull-after-script.mjs b/tests/corpus/web-stream-pull-after-script.mjs new file mode 100644 index 000000000..59ad9e653 --- /dev/null +++ b/tests/corpus/web-stream-pull-after-script.mjs @@ -0,0 +1,14 @@ +// A stream's start step runs during construction and its first pull as a +// promise job after the script, in a program with no other asynchronous +// work. +const stream = new ReadableStream({ + start() { + console.log("start"); + }, + pull(controller) { + console.log("pull", controller.desiredSize, stream.locked); + controller.enqueue("chunk"); + console.log("queued", controller.desiredSize); + }, +}); +console.log("end", stream.locked); diff --git a/tests/corpus/web-stream-readable-untyped.mjs b/tests/corpus/web-stream-readable-untyped.mjs new file mode 100644 index 000000000..8e1267a52 --- /dev/null +++ b/tests/corpus/web-stream-readable-untyped.mjs @@ -0,0 +1,130 @@ +// Web ReadableStream from untyped JavaScript: the classes as constructor +// values (globals and node:stream/web), their prototypes and brand checks, +// queuing strategies, default readers and controllers, tee, from() and +// async iteration, in Node's promise order. +import { + ReadableStream as WebReadableStream, + ReadableStreamDefaultReader, + ReadableStreamDefaultController, + CountQueuingStrategy, + ByteLengthQueuingStrategy, +} from "node:stream/web"; + +const log = []; +const out = (...args) => log.push(args.map(String).join(" ")); +const caught = (fn) => { + try { + fn(); + return "ok"; + } catch (e) { + return `${e.name}:${e.code ?? ""}:${e.message}`; + } +}; +const settled = (p) => p.then((v) => `ok:${v}`, (e) => `${e.name}:${e.code ?? ""}:${e.message}`); + +out("classes", typeof ReadableStream, ReadableStream === WebReadableStream, ReadableStream.name, ReadableStream.length); +out("illegal", caught(() => new ReadableStreamDefaultController()), caught(() => /** @type {any} */ (ReadableStream)())); +out("prototype", Object.keys(ReadableStreamDefaultReader.prototype).join(","), Object.keys(ReadableStreamDefaultController.prototype).join(",")); + +{ + /** @type {any} */ + let controller; + const stream = new ReadableStream({ start(c) { controller = c; } }, new CountQueuingStrategy({ highWaterMark: 2 })); + out("strategy", controller.desiredSize, stream instanceof ReadableStream, controller instanceof ReadableStreamDefaultController); + controller.enqueue("a"); + controller.enqueue("b"); + controller.enqueue("c"); + out("queued", controller.desiredSize, Object.prototype.toString.call(stream), Object.prototype.toString.call(controller)); + const reader = new ReadableStreamDefaultReader(stream); + out("locked", stream.locked, caught(() => stream.getReader()), reader instanceof ReadableStreamDefaultReader); + const first = await reader.read(); + out("read", first.value, first.done, controller.desiredSize); + controller.close(); + const rest = []; + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + rest.push(value); + } + out("rest", rest.join(","), await settled(reader.closed), controller.desiredSize); + reader.releaseLock(); + out("released", stream.locked, await settled(reader.read())); +} + +{ + const sizes = new ByteLengthQueuingStrategy({ highWaterMark: 8 }); + /** @type {any} */ + let controller; + const stream = new ReadableStream({ start(c) { controller = c; } }, sizes); + controller.enqueue(new Uint8Array(3)); + out("bytes", sizes.highWaterMark, sizes.size(new Uint8Array(5)), controller.desiredSize, sizes.size.name); + controller.error(new Error("broken")); + out("errored", controller.desiredSize, await settled(stream.getReader().read())); +} + +{ + const order = []; + const stream = new ReadableStream({ + start() { order.push("start"); }, + pull(c) { + order.push(`pull ${c.desiredSize}`); + c.enqueue(order.length); + if (order.length > 3) c.close(); + }, + cancel(reason) { order.push(`cancel ${reason}`); }, + }); + order.push("constructed"); + await Promise.resolve(); + order.push("tick"); + const values = []; + for await (const v of stream) values.push(v); + out("pulls", order.join(" | "), values.join(",")); +} + +{ + const [a, b] = ReadableStream.from(["x", "y", "z"]).tee(); + const left = []; + for await (const v of a) left.push(v); + const right = []; + const reader = b.getReader(); + for (let r = await reader.read(); !r.done; r = await reader.read()) right.push(r.value); + out("tee", left.join(""), right.join("")); +} + +{ + const canceled = []; + const stream = ReadableStream.from({ + [Symbol.iterator]() { + let n = 0; + return { + next: () => ({ value: n++, done: false }), + return: (reason) => { + canceled.push(`return ${reason}`); + return { done: true }; + }, + }; + }, + }); + const taken = []; + for await (const v of stream) { + taken.push(v); + if (v === 2) break; + } + out("from-break", taken.join(","), canceled.join(","), stream.locked); +} + +out( + "brands", + caught(() => ReadableStream.prototype.getReader.call({})), + await settled(ReadableStream.prototype.cancel.call({})), + await settled(/** @type {any} */ (Object.getOwnPropertyDescriptor(ReadableStreamDefaultReader.prototype, "closed")).get.call({})), + caught(() => /** @type {any} */ (Object.getOwnPropertyDescriptor(ReadableStream.prototype, "locked")).get.call(1)), +); +out( + "invalid", + caught(() => new ReadableStream(null)), + caught(() => new ReadableStream({}, { highWaterMark: -1 })), + caught(() => new ReadableStream({ pull: 1 })), + caught(() => new CountQueuingStrategy({})), +); +console.log(log.join("\n")); diff --git a/tests/harness/corpus-costs/web-stream-pull-after-script.mjs.json b/tests/harness/corpus-costs/web-stream-pull-after-script.mjs.json new file mode 100644 index 000000000..46c7f7832 --- /dev/null +++ b/tests/harness/corpus-costs/web-stream-pull-after-script.mjs.json @@ -0,0 +1 @@ +{ "plain": 3300, "san": 4500 } diff --git a/tests/harness/corpus-costs/web-stream-readable-untyped.mjs.json b/tests/harness/corpus-costs/web-stream-readable-untyped.mjs.json new file mode 100644 index 000000000..f6af337c6 --- /dev/null +++ b/tests/harness/corpus-costs/web-stream-readable-untyped.mjs.json @@ -0,0 +1 @@ +{ "plain": 5200, "san": 7000 } diff --git a/tests/harness/npm.test.ts b/tests/harness/npm.test.ts index 9623b7563..984d870ab 100644 --- a/tests/harness/npm.test.ts +++ b/tests/harness/npm.test.ts @@ -87,16 +87,14 @@ const KNOWN: Record = { "node24-url-mutation": { run: "SC1090" }, "node24-util-adapters": { run: "Object.create with this value is not supported yet" }, "node24-util-types": { run: "SC1090" }, - "node24-web-stream-encoding": { run: "SC1090" }, + "node24-web-stream-encoding": { run: "SC2020" }, "node24-web-stream-errors": { run: "SC1090" }, - "node24-web-stream-lifecycle": { run: "SC1090" }, + "node24-web-stream-lifecycle": { run: "SC2020" }, "node24-web-stream-pipe-abort": { run: "SC2020" }, - "node24-web-stream-pipe-errors": { run: "SC1090" }, - "node24-web-stream-pipes": { run: "SC1090" }, - "node24-web-stream-queues": { run: "SC1090" }, - "node24-web-stream-readable-lifecycle": { run: "SC2020" }, - "node24-web-stream-readers": { run: "SC1090" }, - "node24-web-stream-tee": { run: "SC2020" }, + "node24-web-stream-pipe-errors": { run: "SC2020" }, + "node24-web-stream-pipes": { run: "SC2020" }, + "node24-web-stream-queues": { run: "SC2020" }, + "node24-web-stream-tee": { run: "SC1090" }, "node24-web-stream-transforms": { run: "SC1090" }, "package-imports": { build: "SC2013" }, "promise-bridge": { build: "SC1090" }, From e84d3322641859de7bbb7ca76f4b03bf6d996036 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 07:05:47 -0500 Subject: [PATCH 12/54] Link AbortController without the fetch unit AbortController and AbortSignal lowered to fetch.abort* entry points, so a program that only used them linked the fetch unit with its socket, HTTP, TLS and zlib dependencies. They now lower to the abort.* entry points on every target, a new abortSignal feature links scr_abort.c (the fetch unit's signal implementation built without its transport, which WASI already ships) and the EventTarget unit, and programs that also fetch keep the fetch unit's copy. --- .../src/backend/executable-features.ts | 1 + .../compiler/src/backend/llvm/lib-shared.ts | 7 ------- .../src/backend/native-link-info-core.ts | 3 +++ .../compiler/src/backend/native/contracts.ts | 3 +++ .../compiler/src/backend/native/executable.ts | 6 +++++- .../compiler/src/backend/runtime-pack-core.ts | 6 ++++++ .../src/backend/target-diagnostics.test.ts | 2 +- .../src/backend/target-diagnostics.ts | 20 +------------------ .../src/executable/executable-cache.ts | 1 + .../src/frontend/lowering/lower-stdlib.ts | 10 +++++----- .../src/frontend/lowering/lower-stream.ts | 8 ++++---- packages/compiler/src/index.ts | 1 + packages/compiler/src/ir/builtin-effects.ts | 2 -- .../compiler/src/ir/builtin-signatures.ts | 4 ---- packages/compiler/src/ir/runtime-features.ts | 3 +++ .../compiler/src/native/sanitizer.test.ts | 2 ++ .../runtime-pack-matrix.mjs | 3 ++- .../runtime-wasm32-wasi/scripts/build.mjs | 1 + 18 files changed, 39 insertions(+), 44 deletions(-) diff --git a/packages/compiler/src/backend/executable-features.ts b/packages/compiler/src/backend/executable-features.ts index 1d95285b4..e43a64780 100644 --- a/packages/compiler/src/backend/executable-features.ts +++ b/packages/compiler/src/backend/executable-features.ts @@ -36,6 +36,7 @@ export function executableLinkFeatures(mod: IrModule, largePages = false): Nativ ...(features.dynForAwait ? { dynForAwait: true } : {}), ...(features.eventsOnceSignal ? { eventsOnceSignal: true } : {}), ...(features.mime ? { mime: true } : {}), + ...(features.abortSignal ? { abortSignal: true } : {}), ...(features.utilTiming ? { utilTiming: true } : {}), ...(features.dynBuffer ? { dynBuffer: true } : {}), ...(features.builtinValues ? { builtinValues: true } : {}), diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 747c9c883..dc5ec61e0 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1030,10 +1030,6 @@ export const LIB_FN_SYMS: Record = { "fetch.responseJson": "scr_fetch_response_json", "fetch.responseText": "scr_fetch_response_text", "fetch.responseBytes": "scr_fetch_response_bytes", - "fetch.abortControllerNew": "scr_fetch_abort_controller_new", - "fetch.abortTimeout": "scr_fetch_abort_timeout", - "fetch.abortNow": "scr_fetch_abort_now", - "fetch.abortAny": "scr_fetch_abort_any", "fetch.webIs": "scr_fetch_web_is", "fetch.streamIs": "scr_fetch_stream_is", "zlib.streamIs": "scr_zlib_stream_is", @@ -1838,14 +1834,11 @@ export const USES_TIMERS_LIB_FNS = new Set([ "fetch.input", "fetch.function", "fetch.requestNew", - "fetch.abortTimeout", "abort.timeout", // EventTarget listeners report their exceptions from a nextTick. "event.targetNew", "abort.controllerNew", "abort.any", - "fetch.abortControllerNew", - "fetch.abortAny", "fetch.streamNew", "fetch.streamFrom", "fetch.transformNew", diff --git a/packages/compiler/src/backend/native-link-info-core.ts b/packages/compiler/src/backend/native-link-info-core.ts index a1ecc66a3..e467f39c4 100644 --- a/packages/compiler/src/backend/native-link-info-core.ts +++ b/packages/compiler/src/backend/native-link-info-core.ts @@ -67,6 +67,9 @@ export interface NativeLinkFeatures { /** `--large-pages`: scr_text_pages.c, which moves the executable's code * onto transparent huge pages at startup. Absent (false) without it. */ largePages?: boolean; + /** AbortController and AbortSignal: scr_abort.c, unless the program + * links the fetch unit, which has them. */ + abortSignal?: boolean; /** util.debounce and util.throttle: scr_util_timing.c. */ utilTiming?: boolean; /** Buffer.prototype methods called on checked-dynamic values link diff --git a/packages/compiler/src/backend/native/contracts.ts b/packages/compiler/src/backend/native/contracts.ts index ee0533ce3..56b44e505 100644 --- a/packages/compiler/src/backend/native/contracts.ts +++ b/packages/compiler/src/backend/native/contracts.ts @@ -131,6 +131,9 @@ export interface CcOptions { mime?: boolean; /** `--large-pages`: compiles scr_text_pages.c. */ largePages?: boolean; + /** AbortController and AbortSignal: compiles scr_abort.c, unless the + * program compiles the fetch unit, which has them. */ + abortSignal?: boolean; /** util.debounce and util.throttle: compiles scr_util_timing.c. */ utilTiming?: boolean; /** Buffer.prototype methods called on checked-dynamic values: diff --git a/packages/compiler/src/backend/native/executable.ts b/packages/compiler/src/backend/native/executable.ts index 22e7cf2cb..60e61ff0b 100644 --- a/packages/compiler/src/backend/native/executable.ts +++ b/packages/compiler/src/backend/native/executable.ts @@ -195,7 +195,10 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [ unit((u) => u.opts.computedRequire === true, "scr_require.c"), unit((u) => u.opts.dc, "scr_dc.c"), unit((u) => u.opts.dynTier, "scr_js_value.c", "scr_js_object.c"), - unit((u) => u.opts.eventTarget || u.nativeFetch, "scr_event_target.c"), + unit( + (u) => u.opts.eventTarget || u.nativeFetch || u.opts.abortSignal === true, + "scr_event_target.c", + ), unit((u) => u.opts.wasm, "scr_wasm.c"), // Module namespace objects as checked-dynamic values. unit((u) => u.opts.moduleNamespace === true, "scr_module_namespace.c"), @@ -248,6 +251,7 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [ unit((u) => u.opts.eventsOnceSignal === true, "scr_events_once_signal.c"), unit((u) => u.opts.mime === true, "scr_mime.c"), unit((u) => u.opts.largePages === true, "scr_text_pages.c"), + unit((u) => u.opts.abortSignal === true && !u.nativeFetch, "scr_abort.c"), unit((u) => u.opts.utilTiming === true, "scr_util_timing.c"), // The checked-dynamic HANDLE support unit (listener gate + runtime // adapter closures): every referencing unit is one of the emitter or diff --git a/packages/compiler/src/backend/runtime-pack-core.ts b/packages/compiler/src/backend/runtime-pack-core.ts index 1ac7e2718..589e3963f 100644 --- a/packages/compiler/src/backend/runtime-pack-core.ts +++ b/packages/compiler/src/backend/runtime-pack-core.ts @@ -92,6 +92,7 @@ export interface RuntimeFeatureSet extends NativeLinkFeatures { eventsOnceSignal: boolean; mime: boolean; largePages: boolean; + abortSignal: boolean; utilTiming: boolean; dynBuffer: boolean; builtinValues: boolean; @@ -101,6 +102,7 @@ export interface RuntimeFeatureSet extends NativeLinkFeatures { brotli: boolean; moduleNamespace: boolean; nativeFetch: boolean; + abortSignalOnly: boolean; netEffective: boolean; httpEffective: boolean; tlsEffective: boolean; @@ -297,6 +299,7 @@ export function effectiveRuntimeFeatures( eventsOnceSignal: features.eventsOnceSignal === true, mime: features.mime === true, largePages: features.largePages === true, + abortSignal: features.abortSignal === true, utilTiming: features.utilTiming === true, dynBuffer: features.dynBuffer === true, builtinValues: features.builtinValues === true, @@ -306,6 +309,9 @@ export function effectiveRuntimeFeatures( brotli: features.brotli === true, moduleNamespace: features.moduleNamespace === true, nativeFetch, + // The fetch unit has AbortController and AbortSignal; scr_abort.c is + // the same implementation without the fetch transport. + abortSignalOnly: features.abortSignal === true && !nativeFetch, netEffective, httpEffective, tlsEffective, diff --git a/packages/compiler/src/backend/target-diagnostics.test.ts b/packages/compiler/src/backend/target-diagnostics.test.ts index e1b344f37..d0bef0cf7 100644 --- a/packages/compiler/src/backend/target-diagnostics.test.ts +++ b/packages/compiler/src/backend/target-diagnostics.test.ts @@ -117,7 +117,7 @@ test("prototype tables store optional methods without invoking their host capabi test("local abort signals remain available without a reflection dispatcher", () => { const mod = module(); mod.functions = [ - fn("%main", [{ kind: "libCall", fn: "fetch.abortControllerNew", args: [], type: DYN, loc }]), + fn("%main", [{ kind: "libCall", fn: "abort.controllerNew", args: [], type: DYN, loc }]), ]; fenceSpeculativeWasiFunctions(mod); expect(moduleWasiUnavailableSurface(mod)).toBeNull(); diff --git a/packages/compiler/src/backend/target-diagnostics.ts b/packages/compiler/src/backend/target-diagnostics.ts index 236ce12cf..e98b671b2 100644 --- a/packages/compiler/src/backend/target-diagnostics.ts +++ b/packages/compiler/src/backend/target-diagnostics.ts @@ -1,4 +1,4 @@ -import { everyModuleNode, everyStmtList, transformStmtList } from "../ir/traverse.js"; +import { everyModuleNode, everyStmtList } from "../ir/traverse.js"; import type { LlvmUnsupportedError } from "./llvm/emitter.js"; import type { ScrDiagnostic } from "../diagnostics/diagnostic.js"; import { type IrModule, type SrcLoc } from "../ir/ir.js"; @@ -14,24 +14,6 @@ import { * calls, escaped callables, constructors, and exported roots still retain * the ordinary target diagnostic. ABI types remain checked by the gate. */ export function fenceSpeculativeWasiFunctions(mod: IrModule): void { - for (const fn of mod.functions) - fn.body = transformStmtList(fn.body, { - stmt: (statement) => statement, - expr: (expr) => { - if (expr.kind !== "libCall") return expr; - const portable = - expr.fn === "fetch.abortControllerNew" - ? "abort.controllerNew" - : expr.fn === "fetch.abortTimeout" - ? "abort.timeout" - : expr.fn === "fetch.abortNow" - ? "abort.now" - : expr.fn === "fetch.abortAny" - ? "abort.any" - : null; - return portable ? { ...expr, fn: portable } : expr; - }, - }); if (!mod.functions.some((fn) => fn.speculativeDispatch)) return; const functions = new Map(mod.functions.map((fn) => [fn.name, fn])); const classes = new Map((mod.classes ?? []).map((info) => [info.name, info])); diff --git a/packages/compiler/src/executable/executable-cache.ts b/packages/compiler/src/executable/executable-cache.ts index 737a78b98..4bd6a2b4a 100644 --- a/packages/compiler/src/executable/executable-cache.ts +++ b/packages/compiler/src/executable/executable-cache.ts @@ -81,6 +81,7 @@ export interface EarlyExecutableNativeFeatures { dynForAwait?: boolean; eventsOnceSignal?: boolean; mime?: boolean; + abortSignal?: boolean; utilTiming?: boolean; dynBuffer?: boolean; builtinValues?: boolean; diff --git a/packages/compiler/src/frontend/lowering/lower-stdlib.ts b/packages/compiler/src/frontend/lowering/lower-stdlib.ts index fe748bb93..068e3a1dc 100644 --- a/packages/compiler/src/frontend/lowering/lower-stdlib.ts +++ b/packages/compiler/src/frontend/lowering/lower-stdlib.ts @@ -1500,7 +1500,7 @@ export function lowerStaticFetchCompanionCall( : dynUndefinedExpr(loc), (first) => ({ kind: "libCall", - fn: "fetch.abortTimeout", + fn: "abort.timeout", args: [first], type: DYN, loc, @@ -1510,7 +1510,7 @@ export function lowerStaticFetchCompanionCall( if (call.arguments.nodes.length === 0) { return { kind: "libCall", - fn: "fetch.abortNow", + fn: "abort.now", args: [dynUndefinedExpr(loc)], type: DYN, loc, @@ -1523,7 +1523,7 @@ export function lowerStaticFetchCompanionCall( lowerLiveWebValue(lowerer, call.arguments.nodes[0]!), (first) => ({ kind: "libCall", - fn: "fetch.abortNow", + fn: "abort.now", args: [first], type: DYN, loc, @@ -1541,7 +1541,7 @@ export function lowerStaticFetchCompanionCall( : dynUndefinedExpr(loc), (first) => ({ kind: "libCall", - fn: "fetch.abortAny", + fn: "abort.any", args: [first], type: DYN, loc, @@ -1886,7 +1886,7 @@ export function lowerAbortControllerNew(lowerer: Lowerer, expr: ts.NewExpression const result: IrExpr = { kind: "libCall", - fn: "fetch.abortControllerNew", + fn: "abort.controllerNew", args: [], type: DYN, loc, diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index 2ae952aaa..d8c370511 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -5176,7 +5176,7 @@ function lowerPipelineStages( { kind: "varDecl", localId: ac.id, - init: { kind: "libCall", fn: "fetch.abortControllerNew", args: [], type: DYN, loc }, + init: { kind: "libCall", fn: "abort.controllerNew", args: [], type: DYN, loc }, loc, }, ], @@ -5502,7 +5502,7 @@ function duplexFromBody(lowerer: Lowerer, node: ts.Expression, loc: SrcLoc): IrE { kind: "varDecl", localId: acLocal.id, - init: { kind: "libCall", fn: "fetch.abortControllerNew", args: [], type: DYN, loc }, + init: { kind: "libCall", fn: "abort.controllerNew", args: [], type: DYN, loc }, loc, }, ]; @@ -5954,7 +5954,7 @@ export function operatorSignalOptions(lowerer: Lowerer, options: IrExpr, loc: Sr type: DYN, loc, }; - const signal: IrExpr = { kind: "libCall", fn: "fetch.abortAny", args: [list], type: DYN, loc }; + const signal: IrExpr = { kind: "libCall", fn: "abort.any", args: [list], type: DYN, loc }; return { kind: "libCall", fn: "readable.opSignalOpts", args: [signal], type: DYN, loc }; } @@ -5978,7 +5978,7 @@ export function reduceSignalSetup( { kind: "varDecl", localId: acLocal.id, - init: { kind: "libCall", fn: "fetch.abortControllerNew", args: [], type: DYN, loc }, + init: { kind: "libCall", fn: "abort.controllerNew", args: [], type: DYN, loc }, loc, }, { diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index a41a6b4c7..e81e4353a 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -702,6 +702,7 @@ async function compileExecutableNative( ...(features.eventsOnceSignal ? { eventsOnceSignal: true } : {}), ...(features.mime ? { mime: true } : {}), ...(features.largePages ? { largePages: true } : {}), + ...(features.abortSignal ? { abortSignal: true } : {}), ...(features.utilTiming ? { utilTiming: true } : {}), ...(features.dynBuffer ? { dynBuffer: true } : {}), ...(features.builtinValues ? { builtinValues: true } : {}), diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index a0d75c864..e97d89419 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -151,8 +151,6 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "fetch.headersNew", "fetch.responseNew", "fetch.responseJsonStatic", - "fetch.abortTimeout", - "fetch.abortAny", "abort.timeout", "abort.any", "event.new", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index e50e61a92..784d08216 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -159,10 +159,6 @@ function buildLibFnSignatures() { "fetch.responseJson": { argTypes: [DYN], result: { kind: "promise", inner: DYN } }, "fetch.responseText": { argTypes: [DYN], result: { kind: "promise", inner: STRING } }, "fetch.responseBytes": { argTypes: [DYN], result: { kind: "promise", inner: BYTES_U8 } }, - "fetch.abortControllerNew": { argTypes: [], result: DYN }, - "fetch.abortTimeout": { argTypes: [DYN], result: DYN }, - "fetch.abortNow": { argTypes: [DYN], result: DYN }, - "fetch.abortAny": { argTypes: [DYN], result: DYN }, "fetch.streamNew": { argTypes: [DYN], result: DYN }, "fetch.transformNew": { argTypes: [], result: DYN }, // Program-dependent iterable: typed arrays/bytes/string stay intact so diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index 0fb4ce815..99610eaf6 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -1033,6 +1033,8 @@ const RUNTIME_FEATURE_RULES = { }), /** util.MIMEType and util.MIMEParams (scr_mime.c). */ mime: rule({ prefixes: ["mime."] }), + /** AbortController and AbortSignal (scr_abort.c, or the fetch unit). */ + abortSignal: rule({ prefixes: ["abort."] }), /** util.debounce and util.throttle (scr_util_timing.c). */ utilTiming: rule({ fns: ["util.debounce", "util.throttle"] }), /** events.once(emitter, name, { signal }) (scr_events_once_signal.c). */ @@ -1254,6 +1256,7 @@ function noRuntimeFeatures(): RuntimeFeatures { inspect: false, legacyTextDecoder: false, mime: false, + abortSignal: false, utilTiming: false, mockTimers: false, net: false, diff --git a/packages/compiler/src/native/sanitizer.test.ts b/packages/compiler/src/native/sanitizer.test.ts index c4b1bd651..c44cc239d 100644 --- a/packages/compiler/src/native/sanitizer.test.ts +++ b/packages/compiler/src/native/sanitizer.test.ts @@ -23,6 +23,7 @@ const features = { eventsOnceSignal: false, mime: false, largePages: false, + abortSignal: false, utilTiming: false, dynBuffer: false, builtinValues: false, @@ -74,6 +75,7 @@ const features = { webcrypto: false, perfLoop: false, nativeFetch: false, + abortSignalOnly: false, netEffective: false, httpEffective: false, tlsEffective: false, diff --git a/packages/runtime-pack-common/runtime-pack-matrix.mjs b/packages/runtime-pack-common/runtime-pack-matrix.mjs index 0f2a69c49..fae696d1b 100644 --- a/packages/runtime-pack-common/runtime-pack-matrix.mjs +++ b/packages/runtime-pack-common/runtime-pack-matrix.mjs @@ -80,7 +80,7 @@ export const OPTIONAL_RUNTIME_SOURCES = [ // programs whose IR has dynamic-tier operations. ["scr_js_value.c", "dynTier"], ["scr_js_object.c", "dynTier"], - ["scr_event_target.c", any("eventTarget", "nativeFetch")], + ["scr_event_target.c", any("eventTarget", "nativeFetch", "abortSignal")], ["scr_async_dyn.c", any("dynAsync", "dynInvoke", "dc", "fileHandle", "nativeFetch")], // The crypto global's object extends scr_async_dyn.c's crypto object. ["scr_webcrypto_object.c", all("webcrypto", "dynAsync")], @@ -184,6 +184,7 @@ export const OPTIONAL_RUNTIME_SOURCES = [ ["scr_tls_x509.c", all("tlsEffective", "x509")], // WebCrypto's AES ciphers run over the cipher feature's mbedTLS archive. ["scr_webcrypto_aes.c", all("cipher", "webcrypto")], + ["scr_abort.c", "abortSignalOnly"], ["scr_fetch.c", "nativeFetch"], ["scr_stream_from_web.c", all("stream", "nativeFetch")], ]; diff --git a/packages/runtime-wasm32-wasi/scripts/build.mjs b/packages/runtime-wasm32-wasi/scripts/build.mjs index 2a2766f4b..285be6641 100644 --- a/packages/runtime-wasm32-wasi/scripts/build.mjs +++ b/packages/runtime-wasm32-wasi/scripts/build.mjs @@ -8,6 +8,7 @@ process.env.SCRIPTC_RUNTIME_PACK_CONFIG = JSON.stringify({ runtimeDefines: ["_GNU_SOURCE", "_WASI_EMULATED_SIGNAL", "_WASI_EMULATED_PROCESS_CLOCKS"], omitRuntimeSources: ["scr_child.c"], omitOptionalSources: [ + "scr_abort.c", "scr_event_target.c", "scr_loop_kqueue.c", "scr_loop_epoll.c", From 635f732a34fe954a27ec2a52aa1978ab06d2dcd4 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 07:17:08 -0500 Subject: [PATCH 13/54] Free a worker's group-arena pool when the worker exits Each thread keeps up to 8 MB of arena chunks and up to 64 dropped arenas for reuse (scr_arena.c). Nothing freed them when a worker thread finished, so every worker that used a group arena left its pool behind: 40 workers run one after another, each dropping a 12 MB arena, kept 328 MB resident. scr_arena_thread_exit frees the calling thread's pooled arenas and chunks. The worker thread calls it after its context cleanup, before it posts the exit event. The new harness test runs 24 such workers in sequence and checks that the resident set grows by less than 32 MiB over the last 20 (160 MiB before this change, 1 MiB after). --- packages/runtime/src/scr_arena.c | 18 +++++ packages/runtime/src/scr_arena.h | 4 + packages/runtime/src/scr_worker.c | 3 + tests/harness/worker-arena-pools.test.ts | 99 ++++++++++++++++++++++++ 4 files changed, 124 insertions(+) create mode 100644 tests/harness/worker-arena-pools.test.ts diff --git a/packages/runtime/src/scr_arena.c b/packages/runtime/src/scr_arena.c index e04a6ed35..73901f2c1 100644 --- a/packages/runtime/src/scr_arena.c +++ b/packages/runtime/src/scr_arena.c @@ -251,6 +251,24 @@ static void scr_arena_free(ScrArena *a) { free(a); } +void scr_arena_thread_exit(void) { + while (scr_arena_pool) { + ScrArena *a = scr_arena_pool; + scr_arena_pool = a->next; + free(a->disposal_base); + free(a); + } + scr_arena_pooled = 0; +#if SCR_SMALL_ALLOC + for (ScrArenaChunk *c = scr_arena_chunk_pool, *next; c; c = next) { + next = c->next; + free(c); + } + scr_arena_chunk_pool = NULL; + scr_arena_chunk_bytes = 0; +#endif +} + void scr_arena_drop(void **list) { ScrArena *a = (ScrArena *)*list; *list = NULL; diff --git a/packages/runtime/src/scr_arena.h b/packages/runtime/src/scr_arena.h index cdfe5a476..4c704eeff 100644 --- a/packages/runtime/src/scr_arena.h +++ b/packages/runtime/src/scr_arena.h @@ -79,6 +79,10 @@ void scr_arena_drop(void **list); void *scr_arena_obj(void *arena, size_t size, void *cyc); /* Appends `obj` to `arena`'s dispose list, which the drop runs. */ void scr_arena_disposal(void *arena, void *obj, ScrArenaDispose dispose); +/* Frees the arenas and chunks the calling thread keeps for reuse. A worker + * thread calls it when its script has finished, after every arena it + * opened has been dropped. */ +void scr_arena_thread_exit(void); /* The leading fields of an arena (scr_arena.c's ScrArena), for the * runtime's own inline allocation. */ diff --git a/packages/runtime/src/scr_worker.c b/packages/runtime/src/scr_worker.c index 59e245ea0..5eed2469d 100644 --- a/packages/runtime/src/scr_worker.c +++ b/packages/runtime/src/scr_worker.c @@ -331,6 +331,9 @@ static void scr_worker_thread_run(ScrWorkerThread *worker) { scr_mailbox_close(worker->inbox); scr_stdin_fed_clear(); /* chunks the worker never read */ scr_context_cleanup(); + /* Frees the group arenas and chunks the thread kept for reuse before the + * parent sees the exit. */ + scr_arena_thread_exit(); if (stats) { uint64_t now = scr_hrtime_ns(); fprintf(stderr, "scriptc worker %llu: started %.2f ms after spawn, script %.2f ms, exit %.2f ms\n", diff --git a/tests/harness/worker-arena-pools.test.ts b/tests/harness/worker-arena-pools.test.ts new file mode 100644 index 000000000..a512d5a7d --- /dev/null +++ b/tests/harness/worker-arena-pools.test.ts @@ -0,0 +1,99 @@ +/* Group arenas (runtime scr_arena.c): each thread keeps up to 8 MB of + * arena chunks and its dropped arenas for reuse, and a worker thread frees + * them when its script finishes. This suite runs workers one after another, + * each building and dropping trees of about 12 MB in a group arena, and + * checks that the resident set stays flat across them: a worker that kept + * its pool would add about 8 MB each. Under SCRIPTC_SAN=1 the runtime + * keeps every worker's memory, so the check does not apply. */ +import { execFile } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; + +const exec = promisify(execFile); +const sanitize = process.env["SCRIPTC_SAN"] === "1"; + +const PROGRAM = ` +import { isMainThread, parentPort, Worker, workerData } from "node:worker_threads"; + +interface TreeNode { + left: TreeNode | null; + right: TreeNode | null; +} + +function tree(depth: number): TreeNode { + return depth > 0 ? { left: tree(depth - 1), right: tree(depth - 1) } : { left: null, right: null }; +} + +function count(node: TreeNode): number { + if (node.left === null || node.right === null) return 1; + return 1 + count(node.left) + count(node.right); +} + +function work(depth: number): number { + let total = 0; + for (let i = 0; i < 2; i++) total += count(tree(depth)); + return total; +} + +function runWorker(depth: number): Promise { + return new Promise((resolve) => { + let total = 0; + const worker = new Worker(new URL(import.meta.url), { workerData: depth }); + worker.on("message", (message: number) => { + total = message; + }); + worker.on("exit", () => resolve(total)); + }); +} + +async function main(): Promise { + let total = 0; + let settled = 0; + for (let i = 0; i < 24; i++) { + total += await runWorker(18); + if (i === 3) settled = process.memoryUsage.rss(); + } + const growth = Math.round((process.memoryUsage.rss() - settled) / (1024 * 1024)); + console.log(total, growth); +} + +if (isMainThread) { + await main(); +} else { + parentPort!.postMessage(work(workerData as number)); +} +`; + +test.skipIf(process.platform !== "linux" || sanitize)( + "workers that use group arenas free their arena pools when they exit", + { timeout: 300_000 }, + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-worker-arena-pools-")); + try { + const source = join(dir, "pools.ts"); + await writeFile(source, PROGRAM); + const result = await compile(source, { + backend: "llvm", + outDir: dir, + outPath: join(dir, "pools"), + }); + expect(result.ok, JSON.stringify(result.diagnostics)).toBe(true); + if (!result.ok) return; + const { stdout } = await exec(result.binaryPath, [], { + env: { ...process.env, SCR_SA_DISCARD_DELAY_MS: "0" }, + timeout: 120_000, + }); + const [total, growth] = stdout.trim().split(" ").map(Number); + // 24 workers, each with 2 trees of 2^19 - 1 nodes. + expect(total).toBe(24 * 2 * (2 ** 19 - 1)); + // 20 workers that kept their pools would add about 160 MiB. + expect(growth).toBeLessThan(32); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, +); From 041565d7f2c1d0394adc9f9ab9e01d3b908fa13d Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 26 Oct 2026 23:24:47 -0700 Subject: [PATCH 14/54] Split the slowest sanitized suites by case and raise the sanitized bootstrap's stack Under ASan, self-hosting-llvm-emitter, three-library, npm-static and npm each take 20 to 40 minutes as one Vitest file, so a sharded run cannot finish before them. - self-hosting-llvm-emitter splits into its emitter-cases-and-programs and fixed-point halves; a shard builds the seed once for the halves it owns. three-library skips the optimization its shard does not own. Both put their tests in a describe block, which skipUnownedTest partitions. - CI runs both in every shard's case-sharded side queue (scripts/ci-test-plan.mjs); pnpm test:sandbox case-shards them with npm and npm-static. - npm.test.ts keeps a skipped placeholder in a shard that owns no program, so an empty shard does not fail the file. The sanitized native-driver bootstrap ran its instrumented compiler with the default 8 MiB stack. Its frontend phase overflows it (SC3004) on main as well, outside CI's ulimit for the emitter phase. The test now raises the soft stack limit to the hard one for every sanitized compiler process it starts. Co-Authored-By: Claude Opus 5.5 --- scripts/ci-test-costs.json | 4 - scripts/ci-test-plan.mjs | 7 +- scripts/sandbox-test.mjs | 6 + tests/harness/npm.test.ts | 2 + .../harness/self-hosting-llvm-emitter.test.ts | 377 ++++++++++-------- .../self-hosting-native-driver.test.ts | 13 +- tests/harness/three-library.test.ts | 137 ++++--- 7 files changed, 299 insertions(+), 247 deletions(-) diff --git a/scripts/ci-test-costs.json b/scripts/ci-test-costs.json index 99d47e972..6eb491a41 100644 --- a/scripts/ci-test-costs.json +++ b/scripts/ci-test-costs.json @@ -384,7 +384,6 @@ "tests/harness/self-hosting-frontend-program.test.ts": 94350, "tests/harness/self-hosting-frontend-services.test.ts": 31589, "tests/harness/self-hosting-integer-ranges.test.ts": 23959, - "tests/harness/self-hosting-llvm-emitter.test.ts": 297588, "tests/harness/self-hosting-lowering-context.test.ts": 54669, "tests/harness/self-hosting-native-cache.test.ts": 19497, "tests/harness/self-hosting-native-frontend-checks.test.ts": 363000, @@ -412,7 +411,6 @@ "tests/harness/teardown-segments.test.ts": 2600, "tests/harness/terminal-geometry.test.ts": 2730, "tests/harness/test262-runner.test.ts": 207, - "tests/harness/three-library.test.ts": 251122, "tests/harness/ts7-baselines.test.ts": 110, "tests/harness/unicode-host-filesystem.test.ts": 1874, "tests/harness/workspace-source.test.ts": 1744, @@ -804,7 +802,6 @@ "tests/harness/self-hosting-frontend-program.test.ts": 130169, "tests/harness/self-hosting-frontend-services.test.ts": 37096, "tests/harness/self-hosting-integer-ranges.test.ts": 25745, - "tests/harness/self-hosting-llvm-emitter.test.ts": 429842, "tests/harness/self-hosting-lowering-context.test.ts": 47647, "tests/harness/self-hosting-native-cache.test.ts": 29299, "tests/harness/self-hosting-native-frontend-checks.test.ts": 381000, @@ -832,7 +829,6 @@ "tests/harness/teardown-segments.test.ts": 5, "tests/harness/terminal-geometry.test.ts": 3270, "tests/harness/test262-runner.test.ts": 230, - "tests/harness/three-library.test.ts": 361470, "tests/harness/ts7-baselines.test.ts": 123, "tests/harness/unicode-host-filesystem.test.ts": 3878, "tests/harness/workspace-source.test.ts": 2546, diff --git a/scripts/ci-test-plan.mjs b/scripts/ci-test-plan.mjs index 88a77e32e..03a56eaaf 100644 --- a/scripts/ci-test-plan.mjs +++ b/scripts/ci-test-plan.mjs @@ -18,14 +18,17 @@ const cacheFile = "packages/compiler/src/backend/native-toolchain.test.ts"; export const isolatedFiles = [ ["self-hosting serialization", "tests/harness/self-hosting-serialization.test.ts"], ["self-hosting", "tests/harness/self-hosting.test.ts"], - ["self-hosting LLVM emitter", "tests/harness/self-hosting-llvm-emitter.test.ts"], ["fetch", "tests/harness/fetch.test.ts"], ["surface manifest", "tests/harness/surface-manifest.test.ts"], ]; // Suites that split their own tests across the shards (SCRIPTC_TEST_SHARD, // skipUnownedTest in tests/harness/shard.ts): every shard runs its share in // its own vitest process. -export const caseShardedSideFiles = [["npm static", "tests/harness/npm-static.test.ts"]]; +export const caseShardedSideFiles = [ + ["npm static", "tests/harness/npm-static.test.ts"], + ["self-hosting LLVM emitter", "tests/harness/self-hosting-llvm-emitter.test.ts"], + ["three.js library", "tests/harness/three-library.test.ts"], +]; export const fileExclusions = [ ...corpusFiles, ...effectFiles, diff --git a/scripts/sandbox-test.mjs b/scripts/sandbox-test.mjs index 3fe063075..c1e3fcf1a 100644 --- a/scripts/sandbox-test.mjs +++ b/scripts/sandbox-test.mjs @@ -34,6 +34,12 @@ const laneCaseShardedFiles = [ "tests/harness/server.test.ts", "tests/harness/test262.test.ts", "tests/harness/three.test.ts", + // These split their own cases by SCRIPTC_TEST_SHARD too, as in CI + // (scripts/ci-test-plan.mjs); whole, each takes 20-40 minutes under ASan. + "tests/harness/npm.test.ts", + "tests/harness/npm-static.test.ts", + "tests/harness/self-hosting-llvm-emitter.test.ts", + "tests/harness/three-library.test.ts", ]; // Coverage analysis is frontend-only: SCRIPTC_SAN cannot change its result. // It still case-shards across the selected lane so every corpus entry is diff --git a/tests/harness/npm.test.ts b/tests/harness/npm.test.ts index 9623b7563..9a8cc9c2e 100644 --- a/tests/harness/npm.test.ts +++ b/tests/harness/npm.test.ts @@ -220,6 +220,8 @@ test("every KNOWN entry names a case", () => { }); describe(`npm native (${cases.length} programs${sanitize ? ", sanitized" : ""}${shardSuffix()})`, () => { + // With many shards some own no program; an empty suite would fail the file. + if (cases.length === 0) test.skip("no npm program falls in this shard", () => {}); for (const c of cases) { const known = KNOWN[c.name]; if (known !== undefined && "divergence" in known) { diff --git a/tests/harness/self-hosting-llvm-emitter.test.ts b/tests/harness/self-hosting-llvm-emitter.test.ts index a81bac16c..7964db31b 100644 --- a/tests/harness/self-hosting-llvm-emitter.test.ts +++ b/tests/harness/self-hosting-llvm-emitter.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { promisify } from "node:util"; -import { expect, test } from "vitest"; +import { afterAll, beforeEach, describe, expect, test } from "vitest"; import { compile, compileC, @@ -21,6 +21,7 @@ import { llvmEmitterRequest, type LlvmEmitterRequest, } from "./self-hosting-llvm-emitter-cases.js"; +import { skipUnownedTest } from "./shard.js"; const root = fileURLToPath(new URL("../..", import.meta.url)); const entry = join(root, "tests/fixtures/self-hosting/llvm-emitter.ts"); @@ -105,196 +106,222 @@ const programs = [ // (typed-array loop versioning, which caches null index shapes). const withoutDebugInfo = new Set(["typed-array-masked-index-stores.ts"]); -for (const backend of ["llvm"] as const) { - test( - `the complete LLVM emitter bootstraps natively (${backend})`, - async () => { - const dir = mkdtempSync( - join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-llvm-emitter-"), - ); - const executable = (name: string) => - join(dir, name + (process.platform === "win32" ? ".exe" : "")); - const stage = executable("emitter"); - const input = join(dir, "input.json"); - const output = join(dir, "native.ll"); - const config = join(dir, "config.json"); - const started = performance.now(); - const progress = (phase: string) => - console.log( - `LLVM bootstrap: ${phase} (${((performance.now() - started) / 1000).toFixed(1)}s)`, - ); - try { - // Lower the full production graph in a child so synchronous frontend - // work cannot block Vitest's worker RPC while it compiles the seed. - const api = pathToFileURL(join(root, "packages/compiler/src/index.ts")).href; - progress("build seed"); - const { stdout } = await execFileAsync( - process.execPath, - [ - "--import", - "tsx", - "--input-type=module", - "--eval", - `import { compile } from ${JSON.stringify(api)}; +const backend = "llvm" as const; +const title = `the complete LLVM emitter bootstraps natively (${backend})`; + +// The bootstrap's two halves are separate tests so SCRIPTC_TEST_SHARD can run +// them on different shards (tests/harness/shard.ts). Both need the seed: a +// shard builds it once for whichever halves it owns. Milliseconds measured +// under ASan, where the fixed point takes three times the cases' share. +const halfCosts = { + [`${title}: emitter cases and programs`]: 232_000, + [`${title}: fixed point`]: 741_000, +}; + +const dir = mkdtempSync( + join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-llvm-emitter-"), +); +afterAll(() => rmSync(dir, { recursive: true, force: true })); +const executable = (name: string) => join(dir, name + (process.platform === "win32" ? ".exe" : "")); +const stage = executable("emitter"); +const input = join(dir, "input.json"); +const output = join(dir, "native.ll"); +const config = join(dir, "config.json"); +const started = performance.now(); +const progress = (phase: string) => + console.log(`LLVM bootstrap: ${phase} (${((performance.now() - started) / 1000).toFixed(1)}s)`); + +let seed: Promise | undefined; +/** Build the native emitter seed once per process and return its own IR. */ +function buildSeed(): Promise { + seed ??= (async () => { + // Lower the full production graph in a child so synchronous frontend + // work cannot block Vitest's worker RPC while it compiles the seed. + const api = pathToFileURL(join(root, "packages/compiler/src/index.ts")).href; + progress("build seed"); + const { stdout } = await execFileAsync( + process.execPath, + [ + "--import", + "tsx", + "--input-type=module", + "--eval", + `import { compile } from ${JSON.stringify(api)}; const result = await compile(process.argv[1], { outDir: process.argv[2], outPath: process.argv[3], backend: process.argv[4], optimization: 'dev', sanitize: process.argv[5] === '1', emitIr: true, }); console.log(JSON.stringify(result));`, - entry, - dir, - stage, - backend, - sanitize ? "1" : "0", - ], - { ...runOptions, timeout: 600_000 }, - ); - const built = JSON.parse(stdout) as Awaited>; - if (!built.ok) - throw new Error(built.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); - if (!("binaryPath" in built)) throw new Error("bootstrap did not produce an executable"); - expect(built.backend).toBe(backend); + entry, + dir, + stage, + backend, + sanitize ? "1" : "0", + ], + { ...runOptions, timeout: 600_000 }, + ); + const built = JSON.parse(stdout) as Awaited>; + if (!built.ok) + throw new Error(built.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + if (!("binaryPath" in built)) throw new Error("bootstrap did not produce an executable"); + expect(built.backend).toBe(backend); + expect(built.irPath).toBeDefined(); + const ownIr = deserializeModule(readFileSync(built.irPath!, "utf8")); + expect(ownIr.functions.length).toBeGreaterThan(800); + expect(validateModule(ownIr)).toEqual([]); + progress("seed built"); + return ownIr; + })(); + return seed; +} - expect(built.irPath).toBeDefined(); - const ownIr = deserializeModule(readFileSync(built.irPath!, "utf8")); - expect(ownIr.functions.length).toBeGreaterThan(800); - expect(validateModule(ownIr)).toEqual([]); - progress("seed built; compare emitter cases"); +const emit = async ( + mod: IrModule, + name: string, + request = llvmEmitterRequest(), +): Promise => { + writeFileSync(input, serializeModule(mod, true)); + writeFileSync(config, JSON.stringify(request)); + const result = await execFileAsync(stage, [input, output, config], { + ...runOptions, + timeout: emitTimeout, + }).catch((cause: unknown) => { + throw new Error(`native LLVM emission failed for ${name}`, { cause }); + }); + expect(result.stdout, name).toBe(""); + expect(result.stderr, name).toBe(""); + const text = readFileSync(output, "utf8"); + const expected = emitLlvmModule(mod, llvmEmitterOptions(request)); + expect(text, name).toBe(expected); + return text; +}; - const emit = async ( - mod: IrModule, - name: string, - request = llvmEmitterRequest(), - ): Promise => { - writeFileSync(input, serializeModule(mod, true)); - writeFileSync(config, JSON.stringify(request)); - const result = await execFileAsync(stage, [input, output, config], { - ...runOptions, - timeout: emitTimeout, - }).catch((cause: unknown) => { - throw new Error(`native LLVM emission failed for ${name}`, { cause }); - }); - expect(result.stdout, name).toBe(""); - expect(result.stderr, name).toBe(""); - const text = readFileSync(output, "utf8"); - const expected = emitLlvmModule(mod, llvmEmitterOptions(request)); - expect(text, name).toBe(expected); - return text; - }; +// skipUnownedTest partitions a describe block's tests. +describe(title, () => { + beforeEach((context) => skipUnownedTest(context, halfCosts)); - for (const item of llvmEmitterCases()) { - expect(validateModule(item.module), item.name).toEqual([]); - const text = await emit(item.module, item.name, item.request); - for (const fragment of item.contains) expect(text, item.name).toContain(fragment); - for (const fragment of item.excludes) expect(text, item.name).not.toContain(fragment); - } + test( + `${title}: emitter cases and programs`, + async () => { + await buildSeed(); + progress("compare emitter cases"); + for (const item of llvmEmitterCases()) { + expect(validateModule(item.module), item.name).toEqual([]); + const text = await emit(item.module, item.name, item.request); + for (const fragment of item.contains) expect(text, item.name).toContain(fragment); + for (const fragment of item.excludes) expect(text, item.name).not.toContain(fragment); + } - // A refusal must leave the requested output untouched, with the same - // exception category/message under Node and the compiled emitter. - const rejected = llvmEmitterCases()[0]!.module; - rejected.classes = [ - { - name: "UnimplementedNative", - fields: [], - runtime: true, - loc: { file: rejected.sourceFile, start: 0, end: 0 }, - }, - ]; - writeFileSync(input, serializeModule(rejected, true)); - writeFileSync(config, JSON.stringify(llvmEmitterRequest())); - writeFileSync(output, "untouched"); - const args = [input, output, config]; - const oracleRefusal = spawnSync( - process.execPath, - ["--import", "tsx", entry, ...args], - runOptions, - ); - const nativeRefusal = spawnSync(stage, args, runOptions); - for (const result of [oracleRefusal, nativeRefusal]) { - expect(result.error).toBeUndefined(); - expect(result.signal).toBeNull(); - expect(result.status).toBe(2); - expect(result.stderr.toString()).toBe(""); - } - expect(nativeRefusal.stdout).toEqual(oracleRefusal.stdout); - expect(nativeRefusal.stdout.toString()).toContain("classDef:UnimplementedNative"); - expect(readFileSync(output, "utf8")).toBe("untouched"); + // A refusal must leave the requested output untouched, with the same + // exception category/message under Node and the compiled emitter. + const rejected = llvmEmitterCases()[0]!.module; + rejected.classes = [ + { + name: "UnimplementedNative", + fields: [], + runtime: true, + loc: { file: rejected.sourceFile, start: 0, end: 0 }, + }, + ]; + writeFileSync(input, serializeModule(rejected, true)); + writeFileSync(config, JSON.stringify(llvmEmitterRequest())); + writeFileSync(output, "untouched"); + const args = [input, output, config]; + const oracleRefusal = spawnSync( + process.execPath, + ["--import", "tsx", entry, ...args], + runOptions, + ); + const nativeRefusal = spawnSync(stage, args, runOptions); + for (const result of [oracleRefusal, nativeRefusal]) { + expect(result.error).toBeUndefined(); + expect(result.signal).toBeNull(); + expect(result.status).toBe(2); + expect(result.stderr.toString()).toBe(""); + } + expect(nativeRefusal.stdout).toEqual(oracleRefusal.stdout); + expect(nativeRefusal.stdout.toString()).toContain("classDef:UnimplementedNative"); + expect(readFileSync(output, "utf8")).toBe("untouched"); - for (const source of programs) { - progress(`compile and run ${source}`); - const sourcePath = join(root, "tests/corpus", source); - const irPath = join(dir, "program.ir.json"); - const lowered = await compile(sourcePath, { - outDir: dir, - outPath: irPath, - outputKind: "ir", - }); - if (!lowered.ok) - throw new Error(`${source}: ${lowered.diagnostics.map((d) => d.message).join("\n")}`); - const mod = deserializeModule(readFileSync(irPath, "utf8")); - const request: LlvmEmitterRequest = withoutDebugInfo.has(source) - ? llvmEmitterRequest() - : llvmEmitterRequest({ - debug: true, - sources: [{ file: sourcePath, text: readFileSync(sourcePath, "utf8") }], - }); - const text = await emit(mod, source, request); - const llvmPath = join(dir, "program.ll"); - const outPath = executable("program"); - writeFileSync(llvmPath, text); - await compileC({ - cPath: llvmPath, - outPath, - // Emitted modules are generated programs closed over the runtime - // tree, as in compile(): reuse cached runtime objects instead of - // rebuilding every runtime unit for each program. - cacheIdentity: generatedProgram, - sanitize, - optimization: "dev", - ...moduleLinkFeatures(mod), - }); - const node = spawnSync(process.execPath, [sourcePath], runOptions); - const native = spawnSync(outPath, [], runOptions); - for (const result of [node, native]) { - expect(result.error, source).toBeUndefined(); - expect(result.signal, `${source}: ${result.stderr}`).toBeNull(); - expect(result.status, `${source}: ${result.stderr}`).toBe(0); - } - expect(native.stdout, source).toEqual(node.stdout); - expect(programStderr(native.stderr), source).toBe(programStderr(node.stderr)); - } - - // The seed emits its own complete IR, including the production - // optimizer and LLVM expression/library graph. Compile those bytes - // and require a second native generation to reproduce the module. - progress("emit own IR"); - const self = await emit(ownIr, "LLVM emitter emits itself"); - const selfPath = join(dir, "self.ll"); - const secondStage = executable("emitter-second"); - writeFileSync(selfPath, self); - progress("compile second generation"); + for (const source of programs) { + progress(`compile and run ${source}`); + const sourcePath = join(root, "tests/corpus", source); + const irPath = join(dir, "program.ir.json"); + const lowered = await compile(sourcePath, { + outDir: dir, + outPath: irPath, + outputKind: "ir", + }); + if (!lowered.ok) + throw new Error(`${source}: ${lowered.diagnostics.map((d) => d.message).join("\n")}`); + const mod = deserializeModule(readFileSync(irPath, "utf8")); + const request: LlvmEmitterRequest = withoutDebugInfo.has(source) + ? llvmEmitterRequest() + : llvmEmitterRequest({ + debug: true, + sources: [{ file: sourcePath, text: readFileSync(sourcePath, "utf8") }], + }); + const text = await emit(mod, source, request); + const llvmPath = join(dir, "program.ll"); + const outPath = executable("program"); + writeFileSync(llvmPath, text); await compileC({ - cPath: selfPath, - outPath: secondStage, + cPath: llvmPath, + outPath, + // Emitted modules are generated programs closed over the runtime + // tree, as in compile(): reuse cached runtime objects instead of + // rebuilding every runtime unit for each program. cacheIdentity: generatedProgram, sanitize, optimization: "dev", - ...moduleLinkFeatures(ownIr), + ...moduleLinkFeatures(mod), }); - const again = join(dir, "again.ll"); - progress("second generation emits own IR"); - const second = await execFileAsync(secondStage, [input, again, config], { - ...runOptions, - timeout: emitTimeout, - }); - expect(second.stdout).toBe(""); - expect(second.stderr).toBe(""); - expect(readFileSync(again, "utf8")).toBe(self); - progress("fixed point verified"); - } finally { - rmSync(dir, { recursive: true, force: true }); + const node = spawnSync(process.execPath, [sourcePath], runOptions); + const native = spawnSync(outPath, [], runOptions); + for (const result of [node, native]) { + expect(result.error, source).toBeUndefined(); + expect(result.signal, `${source}: ${result.stderr}`).toBeNull(); + expect(result.status, `${source}: ${result.stderr}`).toBe(0); + } + expect(native.stdout, source).toEqual(node.stdout); + expect(programStderr(native.stderr), source).toBe(programStderr(node.stderr)); } }, testTimeout, ); -} + + test( + `${title}: fixed point`, + async () => { + // The seed emits its own complete IR, including the production + // optimizer and LLVM expression/library graph. Compile those bytes + // and require a second native generation to reproduce the module. + const ownIr = await buildSeed(); + progress("emit own IR"); + const self = await emit(ownIr, "LLVM emitter emits itself"); + const selfPath = join(dir, "self.ll"); + const secondStage = executable("emitter-second"); + writeFileSync(selfPath, self); + progress("compile second generation"); + await compileC({ + cPath: selfPath, + outPath: secondStage, + cacheIdentity: generatedProgram, + sanitize, + optimization: "dev", + ...moduleLinkFeatures(ownIr), + }); + const again = join(dir, "again.ll"); + progress("second generation emits own IR"); + const second = await execFileAsync(secondStage, [input, again, config], { + ...runOptions, + timeout: emitTimeout, + }); + expect(second.stdout).toBe(""); + expect(second.stderr).toBe(""); + expect(readFileSync(again, "utf8")).toBe(self); + progress("fixed point verified"); + }, + testTimeout, + ); +}); diff --git a/tests/harness/self-hosting-native-driver.test.ts b/tests/harness/self-hosting-native-driver.test.ts index 60969d0b2..842d817a5 100644 --- a/tests/harness/self-hosting-native-driver.test.ts +++ b/tests/harness/self-hosting-native-driver.test.ts @@ -177,7 +177,18 @@ test( expectedStderr = "", env: Record = {}, ) => { - const result = await exec(compiler, args, { + // An ASan-instrumented compiler's frames are several times larger, and + // its recursion over the compiler's own deep ASTs overflows the + // default 8 MiB stack (SC3004). Raise the soft limit to the hard limit + // for each sanitized compiler process, as a shell's ulimit would. + const [file, argv] = + sanitize && process.platform !== "win32" + ? [ + "/bin/sh", + ["-c", 'ulimit -s "$(ulimit -H -s)" && exec "$0" "$@"', compiler, ...args], + ] + : [compiler, args]; + const result = await exec(file, argv, { ...nativeOptions, env: { ...nativeOptions.env, ...env }, }).catch((error: unknown) => { diff --git a/tests/harness/three-library.test.ts b/tests/harness/three-library.test.ts index 49ae825fb..23aa4c5e6 100644 --- a/tests/harness/three-library.test.ts +++ b/tests/harness/three-library.test.ts @@ -2,58 +2,64 @@ import { spawnSync } from "node:child_process"; import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { compileLibrary } from "@scriptc/compiler"; -import { expect, test } from "vitest"; +import { beforeEach, describe, expect, test } from "vitest"; +import { skipUnownedTest } from "./shard.js"; const fixture = join(import.meta.dirname, "../library-mode/wasm"); const hasClang = spawnSync("clang", ["--version"]).status === 0; -test.skipIf(!hasClang).each(["dev", "release"] as const)( - "three.js sends animated geometry, materials and resource lifecycle to a native host in %s", - async (optimization) => { - const directory = await mkdtemp("/tmp/scriptc-three-library-"); - const sanitize = process.env["SCRIPTC_SAN"] === "1"; - try { - const profile = JSON.parse(await readFile(join(fixture, "three.json"), "utf8")); - const profilePath = join(directory, "profile.json"); - await writeFile( - profilePath, - JSON.stringify({ - ...profile, - name: "three-native-embedding", - entry: join(fixture, "three.mjs"), - optimization, - }), - ); - const built = await compileLibrary({ profilePath, outDir: directory, sanitize }); - if (!built.ok) - throw new Error( - built.diagnostics - .map((diagnostic) => `${diagnostic.code}: ${diagnostic.message}`) - .join("\n"), +// The two optimizations take similar time (about 10 minutes each under ASan): +// SCRIPTC_TEST_SHARD runs them on different shards (tests/harness/shard.ts). +// skipUnownedTest partitions a describe block's tests. +describe("three.js native library", () => { + beforeEach((context) => skipUnownedTest(context, {})); + test.skipIf(!hasClang).each(["dev", "release"] as const)( + "three.js sends animated geometry, materials and resource lifecycle to a native host in %s", + async (optimization) => { + const directory = await mkdtemp("/tmp/scriptc-three-library-"); + const sanitize = process.env["SCRIPTC_SAN"] === "1"; + try { + const profile = JSON.parse(await readFile(join(fixture, "three.json"), "utf8")); + const profilePath = join(directory, "profile.json"); + await writeFile( + profilePath, + JSON.stringify({ + ...profile, + name: "three-native-embedding", + entry: join(fixture, "three.mjs"), + optimization, + }), ); - const binary = join(directory, "host"); - const linked = spawnSync( - "clang", - [ - "-std=c11", - ...(sanitize ? ["-fsanitize=address,undefined"] : []), - join(fixture, "three-native.c"), - built.archivePath, - "-lm", - "-o", - binary, - ], - { encoding: "utf8" }, - ); - expect(linked.status, linked.stderr).toBe(0); - const actual = spawnSync(binary, [], { encoding: "utf8", timeout: 30_000 }); - expect(actual.status, actual.stderr).toBe(0); - const reference = spawnSync( - process.execPath, - [ - "--no-warnings", - "--input-type=module", - "-e", - ` + const built = await compileLibrary({ profilePath, outDir: directory, sanitize }); + if (!built.ok) + throw new Error( + built.diagnostics + .map((diagnostic) => `${diagnostic.code}: ${diagnostic.message}`) + .join("\n"), + ); + const binary = join(directory, "host"); + const linked = spawnSync( + "clang", + [ + "-std=c11", + ...(sanitize ? ["-fsanitize=address,undefined"] : []), + join(fixture, "three-native.c"), + built.archivePath, + "-lm", + "-o", + binary, + ], + { encoding: "utf8" }, + ); + expect(linked.status, linked.stderr).toBe(0); + const actual = spawnSync(binary, [], { encoding: "utf8", timeout: 30_000 }); + expect(actual.status, actual.stderr).toBe(0); + const reference = spawnSync( + process.execPath, + [ + "--no-warnings", + "--input-type=module", + "-e", + ` const events=[]; for (const name of ["vertex","surface","resource"]) globalThis[name]=(...values)=>events.push([name,...values.map(value=>Number(value.toFixed(9)))]); const {frame,dispose}=await import(${JSON.stringify(join(fixture, "three.mjs"))}); @@ -61,20 +67,21 @@ test.skipIf(!hasClang).each(["dev", "release"] as const)( dispose(); console.log(JSON.stringify(events)); `, - ], - { encoding: "utf8" }, - ); - expect(reference.status, reference.stderr).toBe(0); - const events = JSON.parse(reference.stdout); - expect( - actual.stdout - .trim() - .split("\n") - .map((line) => JSON.parse(line)), - ).toEqual([...events, ...events]); - } finally { - await rm(directory, { recursive: true, force: true }); - } - }, - 600_000, -); + ], + { encoding: "utf8" }, + ); + expect(reference.status, reference.stderr).toBe(0); + const events = JSON.parse(reference.stdout); + expect( + actual.stdout + .trim() + .split("\n") + .map((line) => JSON.parse(line)), + ).toEqual([...events, ...events]); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }, + 600_000, + ); +}); From fb58a34ca190c63b4d66e309874610c383560941 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 07:44:34 -0500 Subject: [PATCH 15/54] Version for loops on the positions they advance - Loop-entry versioning covers innermost for loops as well as while loops, and a typed array index that the assumed entry ranges prove qualifies a loop as a remainder does. In `for (let j = 0; j < line; j++) out[o++] = f(src[i++])` the test checks `line`, `o` and `i` once per entry, and the fast copy bounds both positions by the loop's passes. - Proven integer loops step their shadowed bindings in expression position too (`out[o++] = v`), so those positions run as i64 counters in the fast copy and the typed array accesses use them without a round trip. - An increment of a binding the enclosing for loop bounds takes the loop range where a surrounding statement has discarded its fact. --- .../src/backend/llvm/emit-statements.ts | 7 +- packages/compiler/src/backend/llvm/emitter.ts | 3 + .../src/backend/llvm/expr-primitives.ts | 23 ++++ .../src/backend/llvm/loop-entry-versions.ts | 6 +- .../src/backend/llvm/speculative-integers.ts | 33 +++-- packages/compiler/src/ir/integer-ranges.ts | 15 ++- .../compiler/src/ir/loop-entry-ranges.test.ts | 76 +++++++++++- packages/compiler/src/ir/loop-entry-ranges.ts | 110 ++++++++++++----- .../compiler/test/loop-entry-versions.test.ts | 30 +++++ .../corpus/for-entry-range-cursors.ts.json | 6 + tests/corpus/for-entry-range-cursors.ts | 116 ++++++++++++++++++ .../for-entry-range-cursors.ts.json | 1 + 12 files changed, 381 insertions(+), 45 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/for-entry-range-cursors.ts.json create mode 100644 tests/corpus/for-entry-range-cursors.ts create mode 100644 tests/harness/corpus-costs/for-entry-range-cursors.ts.json diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index 7732fb8be..08f2267bd 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -2992,7 +2992,12 @@ function emitProvenLoop( } const deferred = new Set(); for (const id of plan.locals.keys()) deferred.add(id); - emitter.speculation = { writes: plan.writes, resume: new Map(), deferred }; + emitter.speculation = { + writes: plan.writes, + resume: new Map(), + deferred, + increments: plan.increments, + }; try { const leave = B.newLabel("integer.leave"); emitCopy(leave); diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index e6c1b98c5..2b9b45ed0 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -1775,6 +1775,9 @@ export class LlEmitter { ran?: string; /** Writes that speculate a rewritten value (rewriteSpeculable). */ values?: ReadonlyMap; + /** A proven integer loop's expression-position `++`/`--` of its + * shadowed bindings (ProvenLoopPlan.increments). */ + increments?: ReadonlySet; } | null = null; /** Speculative loop sites numbered so far in the module. */ speculativeLoopSites = 0; diff --git a/packages/compiler/src/backend/llvm/expr-primitives.ts b/packages/compiler/src/backend/llvm/expr-primitives.ts index 088d392d6..63db3d67b 100644 --- a/packages/compiler/src/backend/llvm/expr-primitives.ts +++ b/packages/compiler/src/backend/llvm/expr-primitives.ts @@ -517,6 +517,29 @@ export function emitOperatorExpr( // Expression-position ++/-- over an f64 binding (locals, module // globals, capture boxes): read, write ±1, yield old (postfix) or // new (prefix). + const shadow = host.integerLoopBindings.get(e.localId); + if (shadow !== undefined && host.speculation?.increments?.has(e)) { + // A proven integer loop's shadow: the loop range holds every value + // the binding takes, so the step cannot overflow. + const old = B.tmp(); + const next = B.tmp(); + const number = B.tmp(); + B.line(`${old} = load i64, ptr ${shadow.slot}`); + B.line(`${next} = ${e.op === "+" ? "add" : "sub"} nsw i64 ${old}, 1`); + B.line(`store i64 ${next}, ptr ${shadow.slot}`); + const value = e.prefix ? next : old; + B.line(`${number} = sitofp i64 ${value} to double`); + return { + name: number, + type: e.type, + integer: { + name: value, + type: "i64", + signed: true, + range: shadow.range ?? { min: -Number.MAX_SAFE_INTEGER, max: Number.MAX_SAFE_INTEGER }, + }, + }; + } const b = host.binding(e.localId); if (host.int32Locals.has(e.localId)) { if (!host.int32LocalWrites.has(e)) diff --git a/packages/compiler/src/backend/llvm/loop-entry-versions.ts b/packages/compiler/src/backend/llvm/loop-entry-versions.ts index 2c448cae4..662146cb6 100644 --- a/packages/compiler/src/backend/llvm/loop-entry-versions.ts +++ b/packages/compiler/src/backend/llvm/loop-entry-versions.ts @@ -1,4 +1,4 @@ -/* Loop-entry range versions (ir/loop-entry-ranges.ts): a while loop whose +/* Loop-entry range versions (ir/loop-entry-ranges.ts): a while or for loop whose * integer work depends on locals entering it without range facts runs as * two copies. A test on entry checks that each such local holds an integer * in its assumed range; the fast copy is emitted under the facts that hold @@ -16,7 +16,7 @@ interface LoopEntryState { fn: IrFunction | null; facts: IntegerRangeFacts | null; slots: IntegerSlotFacts; - /** Found on the first while loop the function emits. */ + /** Found on the first while or for loop the function emits. */ versions: Map | null; /** Loops whose copies are being emitted: a loop is versioned once. */ readonly versioning: Set; @@ -61,7 +61,7 @@ export function beginLoopEntryVersions( * its locals: plain double slots of the current function, outside a * speculative copy and a coroutine. */ export function loopEntryVersion(emitter: LlEmitter, loop: IrStmt): LoopEntryVersion | null { - if (loop.kind !== "while") return null; + if (loop.kind !== "while" && loop.kind !== "for") return null; const state = ENTRY_VERSIONS.of(emitter); if (state.fn === null || state.facts === null || state.versioning.has(loop)) return null; if ( diff --git a/packages/compiler/src/backend/llvm/speculative-integers.ts b/packages/compiler/src/backend/llvm/speculative-integers.ts index 971bd00cb..1a2b220b0 100644 --- a/packages/compiler/src/backend/llvm/speculative-integers.ts +++ b/packages/compiler/src/backend/llvm/speculative-integers.ts @@ -773,6 +773,9 @@ export interface ProvenLoopPlan { locals: Map; /** Statement-level writes computed as integers. */ writes: Set; + /** Expression-position `++`/`--` of shadowed bindings (`out[o++] = v`), + * computed on the shadows. */ + increments: Set; } /** Plan a proven integer loop. `cursors` are the loop's bounded bindings, @@ -791,10 +794,17 @@ export function planProvenIntegerLoop( ): ProvenLoopPlan | null { if (cursors.size === 0) return null; const writes = new Map(); + const increments = new Map(); const refused = new Set(); let rejected = false; const expr = (e: IrExpr): boolean => { - if (e.kind === "assignExpr" || e.kind === "incDec") refused.add(e.localId); + // Every value a cursor holds in the loop lies in its loop range, so an + // increment of one writes an integer of that range. + if (e.kind === "incDec" && cursors.has(e.localId)) { + const list = increments.get(e.localId) ?? []; + list.push(e); + increments.set(e.localId, list); + } else if (e.kind === "assignExpr" || e.kind === "incDec") refused.add(e.localId); return everyExprChild(e, expr, nested); }; const nested = (s: IrStmt): boolean => { @@ -827,12 +837,13 @@ export function planProvenIntegerLoop( for (const s of loop.body) stmt(s); if (rejected) return null; const locals = new Map(); - for (const [id, list] of writes) { + for (const id of new Set([...writes.keys(), ...increments.keys()])) { if (refused.has(id) || !eligible(id)) continue; - const exact = list.every((s) => { - const value = speculativeWriteTarget(s)!.value!; - return isSpeculableValue(value, false) && proven(value); - }); + const exact = + (writes.get(id) ?? []).every((s) => { + const value = speculativeWriteTarget(s)!.value!; + return isSpeculableValue(value, false) && proven(value); + }) && (increments.get(id) ?? []).every(proven); if (exact) locals.set(id, cursors.get(id)!); } // Every binding a shadowed write reads is shadowed, already an integer, @@ -844,15 +855,19 @@ export function planProvenIntegerLoop( for (let changed = true; changed;) { changed = false; for (const id of [...locals.keys()]) - if (!writes.get(id)!.every((s) => grounded(speculativeWriteTarget(s)!.value!))) { + if (!(writes.get(id) ?? []).every((s) => grounded(speculativeWriteTarget(s)!.value!))) { locals.delete(id); changed = true; } } if (locals.size === 0) return null; const planned = new Set(); - for (const id of locals.keys()) for (const s of writes.get(id)!) planned.add(s); - return { locals, writes: planned }; + const plannedIncrements = new Set(); + for (const id of locals.keys()) { + for (const s of writes.get(id) ?? []) planned.add(s); + for (const e of increments.get(id) ?? []) plannedIncrements.add(e); + } + return { locals, writes: planned, increments: plannedIncrements }; } const SAFE = Number.MAX_SAFE_INTEGER; diff --git a/packages/compiler/src/ir/integer-ranges.ts b/packages/compiler/src/ir/integer-ranges.ts index 23e2df90a..f71532381 100644 --- a/packages/compiler/src/ir/integer-ranges.ts +++ b/packages/compiler/src/ir/integer-ranges.ts @@ -23,7 +23,8 @@ export interface IntegerRangeFacts { ranges: IntegerRanges; loopRanges: ReadonlyMap>; } -/** Ranges assumed for locals on entry to while loops (loop-entry-ranges.ts): +/** Ranges assumed for locals on entry to while and for loops + * (loop-entry-ranges.ts), after a for loop's initializer: * the facts inside such a loop hold in a copy of it that runs only after a * test proves every assumed local is an integer in its range. After the * loop, its assumed locals and its writes carry no facts. */ @@ -744,8 +745,11 @@ export function analyzeIntegerRangeFacts( set(facts, e.localId, range); break; case "incDec": { - const before = facts.get(e.localId) ?? null; + // A binding a for loop bounds holds a value in its loop range at + // every point of the loop, also where a statement around this + // increment has discarded its fact. const bound = counterRanges.get(e.localId); + const before = facts.get(e.localId) ?? bound ?? null; const sum = before ? integerArithmeticRange(e.op, before, { min: 1, max: 1 }) : null; const after = bound && sum @@ -1005,6 +1009,9 @@ export function analyzeIntegerRangeFacts( } case "for": { if (s.init) body([s.init], facts); + // The initializer writes none of the assumed locals. + const assumed = entries?.get(s); + if (assumed) for (const [id, range] of assumed) if (!facts.has(id)) set(facts, id, range); const counter = induction(s, facts); const bounded = boundedIntegerLoopFacts(s, loopLocals, facts); const loop = new Map(facts); @@ -1076,6 +1083,10 @@ export function analyzeIntegerRangeFacts( body([s.update], entry); } havoc(facts, [s], facts); + if (assumed) { + invalidate(facts, [s]); + for (const id of assumed.keys()) facts.delete(id); + } break; } case "while": diff --git a/packages/compiler/src/ir/loop-entry-ranges.test.ts b/packages/compiler/src/ir/loop-entry-ranges.test.ts index ad1b78ef6..1c15e1f42 100644 --- a/packages/compiler/src/ir/loop-entry-ranges.test.ts +++ b/packages/compiler/src/ir/loop-entry-ranges.test.ts @@ -1,7 +1,7 @@ import { expect, test } from "vitest"; import { analyzeIntegerRangeFacts } from "./integer-ranges.js"; import { findLoopEntryVersions, LOOP_ENTRY_RANGE } from "./loop-entry-ranges.js"; -import { BOOL, F64, type IrExpr, type IrFunction, type IrStmt } from "./ir.js"; +import { BOOL, BYTES_U8, F64, type IrExpr, type IrFunction, type IrStmt } from "./ir.js"; const loc = { file: "test.ts", start: 0, end: 0 }; const num = (value: number): IrExpr => ({ loc, kind: "numLit", value, type: F64 }); @@ -141,3 +141,77 @@ test("loops without a remainder the assumption proves keep one copy", () => { const known = fn([set("acc", ref("unknown")), set("i", num(0)), proven], ["acc", "i"]); expect(findLoopEntryVersions(known, analyzeIntegerRangeFacts(known)).size).toBe(0); }); + +const step = (localId: string): IrExpr => ({ + loc, + kind: "incDec", + op: "+", + prefix: false, + localId, + type: F64, +}); +const bytes = (localId: string): IrExpr => ({ loc, kind: "varRef", localId, type: BYTES_U8 }); + +/** i = unknown; o = unknown; line = unknown; + * for (let j = 0; j < line; j = j + 1) out[o++] = src[i++]; + * observe(o) */ +function cursors(init?: IrStmt) { + const read: IrExpr = { + loc, + kind: "bytesIntrinsic", + method: "get", + receiver: bytes("src"), + args: [step("i")], + type: F64, + }; + const store: IrStmt = { loc, kind: "bytesSet", arr: bytes("out"), index: step("o"), value: read }; + const loop: IrStmt = { + loc, + kind: "for", + init: init ?? { loc, kind: "varDecl", localId: "j", init: num(0) }, + cond: condition("<", ref("j"), ref("line")), + update: set("j", bin("+", ref("j"), num(1))), + body: [store], + }; + const after = ref("o"); + const f = fn( + [ + set("i", ref("unknown")), + set("o", ref("unknown")), + set("line", ref("unknown")), + loop, + observe(after), + ], + ["i", "o", "line", "j", "unknown"], + ); + f.locals.push( + { id: "src", name: "src", type: BYTES_U8, mutable: false }, + { id: "out", name: "out", type: BYTES_U8, mutable: false }, + ); + return { f, loop, store, read, after }; +} + +test("versions a for loop on the positions it advances and the passes it runs", () => { + const { f, loop, store, read, after } = cursors(); + const facts = analyzeIntegerRangeFacts(f); + expect(facts.ranges.get((store as Extract).index)).toBeNull(); + const version = findLoopEntryVersions(f, facts).get(loop)!; + expect([...version.entry.keys()].sort()).toEqual(["i", "line", "o"]); + // Every pass advances each position once, and the passes stay below `line`. + expect(version.loopRanges?.get("o")).toEqual({ + min: LOOP_ENTRY_RANGE.min, + max: 2 * LOOP_ENTRY_RANGE.max, + }); + expect(version.ranges.get((store as Extract).index)).toBeDefined(); + expect( + version.ranges.get((read as Extract).args[0]!), + ).toBeDefined(); + // No fact leaves the loop. + expect(version.ranges.has(after)).toBe(false); +}); + +test("leaves a for loop whose initializer writes an advanced position as it is", () => { + const init: IrStmt = { loc, kind: "assign", localId: "o", value: num(0) }; + const { f, loop } = cursors(init); + expect(findLoopEntryVersions(f, analyzeIntegerRangeFacts(f)).has(loop)).toBe(false); +}); diff --git a/packages/compiler/src/ir/loop-entry-ranges.ts b/packages/compiler/src/ir/loop-entry-ranges.ts index a5b7628be..a9707c670 100644 --- a/packages/compiler/src/ir/loop-entry-ranges.ts +++ b/packages/compiler/src/ir/loop-entry-ranges.ts @@ -1,5 +1,6 @@ import type { IrExpr, IrFunction, IrStmt } from "./ir.js"; import { everyExpr, everyStmt, everyStmtList } from "./traverse.js"; +import { byteNumberAccess } from "./byte-numbers.js"; import { analyzeIntegerRangeFacts, type IntegerRange, @@ -8,14 +9,12 @@ import { type LoopEntryAssumptions, } from "./integer-ranges.js"; -type WhileLoop = Extract; - /** The range a versioned loop requires of each assumed local on entry. A * loop's counters stay below a typed array or string length (at most 2^48), * so sums of a few such values and assumed locals stay exact integers. */ export const LOOP_ENTRY_RANGE: IntegerRange = { min: -(2 ** 48), max: 2 ** 48 }; -/** A copy of a while loop that runs when every local in `entry` holds an +/** A copy of a loop that runs when every local in `entry` holds an * integer in its range on entry (never -0). `ranges` are the facts that * hold for the loop's expressions in that copy and that the function's own * analysis does not prove; `loopRanges` are the loop's bounded bindings @@ -26,17 +25,22 @@ export interface LoopEntryVersion { loopRanges: ReadonlyMap | undefined; } -/** Versions of innermost while loops whose integer work depends on locals - * that enter the loop without range facts: copies of module globals, +/** Versions of innermost while and for loops whose integer work depends on + * locals that enter the loop without range facts: copies of module globals, * parameters, fields or call results (`const base = offset;` before a loop - * that computes `(count + base) % 60`). + * that computes `(count + base) % 60`), or positions carried from one loop + * to the next (`out[o++] = f(src[i++])` in a loop of `line` passes). * * The function is analyzed again with each number local the loop reads * without a fact assumed in LOOP_ENTRY_RANGE at the loop's entry. A loop is - * versioned when that analysis proves both operands of a remainder that the - * function's own analysis leaves unproven. Only remainders qualify: the - * operands of a product or a sum of such locals are as often fractional, - * and a loop whose test fails runs the ordinary copy in a larger function. + * versioned when that analysis proves what the function's own analysis + * leaves unproven: both operands of a remainder, or the index of a typed + * array read or write. Only those qualify: the operands of a product or a + * sum of such locals are as often fractional, while an index that is not + * an integer reads undefined, and a loop whose test fails runs the + * ordinary copy in a larger function. A for loop qualifies only when its + * initializer writes none of the assumed locals, so that they hold their + * entry values when the test reads them. * The entry test covers only the assumed locals that such an operation reads, directly, * through the loop's writes of other locals, or through the loop condition; * when that drops any, the function is analyzed once more with the smaller @@ -58,19 +62,21 @@ export function findLoopEntryVersions( .map((l) => l.id), ); const known = (e: IrExpr): boolean => !!facts.ranges.get(e); - let candidates = new Map>(); + // Keyed by the statement (a while or for loop). + let candidates = new Map>(); everyStmtList(fn.body, { expr: () => true, stmt: (s) => { - if (s.kind !== "while" || containsLoop(s.body)) return true; - const assumed = entryCandidates(s, eligible, known); - if (assumed.size > 0) candidates.set(s, assumed); + const loop: IrStmt = s; + if ((s.kind !== "while" && s.kind !== "for") || containsLoop(s.body)) return true; + const assumed = entryCandidates(loop, eligible, known); + if (assumed.size > 0) candidates.set(loop, assumed); return true; }, }); if (candidates.size === 0) return result; let versioned = analyzeIntegerRangeFacts(fn, slots, assumptionsOf(candidates)); - const pruned = new Map>(); + const pruned = new Map>(); let dropped = false; for (const [loop, entry] of candidates) { const roots = gainedOperands(loop, known, versioned); @@ -104,7 +110,7 @@ export function findLoopEntryVersions( } function assumptionsOf( - candidates: ReadonlyMap>, + candidates: ReadonlyMap>, ): LoopEntryAssumptions { const result = new Map>(); for (const [loop, entry] of candidates) result.set(loop, entry); @@ -112,9 +118,10 @@ function assumptionsOf( } /** Number locals the loop reads without a range fact and does not declare, - * when the loop has a remainder without one. */ + * when the loop has a remainder or a typed array index without one. A for + * loop whose initializer writes such a local has none. */ function entryCandidates( - loop: WhileLoop, + loop: IrStmt, eligible: ReadonlySet, known: (e: IrExpr) => boolean, ): Map { @@ -123,29 +130,64 @@ function entryCandidates( let integerWork = false; everyStmt(loop, { expr: (e) => { - if (e.kind === "varRef" && eligible.has(e.localId) && !known(e)) read.add(e.localId); + if ((e.kind === "varRef" || e.kind === "incDec") && eligible.has(e.localId) && !known(e)) + read.add(e.localId); else if (e.kind === "bin" && e.op === "%" && !known(e)) integerWork = true; + const index = bytesIndex(e); + if (index && !known(index)) integerWork = true; return true; }, stmt: (s) => { if (s.kind === "varDecl") declared.add(s.localId); + if (s.kind === "bytesSet" && !known(s.index)) integerWork = true; return true; }, }); const result = new Map(); if (!integerWork) return result; - for (const id of read) if (!declared.has(id)) result.set(id, LOOP_ENTRY_RANGE); + const initialized = loop.kind === "for" && loop.init ? writtenIn(loop.init) : new Set(); + for (const id of read) { + if (initialized.has(id)) return new Map(); + if (!declared.has(id)) result.set(id, LOOP_ENTRY_RANGE); + } return result; } -/** Operands of the loop's remainders that the versioned analysis proves - * where the function's own analysis does not prove both. */ +/** The index of a typed array element read (`a[i]`, `buf.readInt16LE(i)`). */ +function bytesIndex(e: IrExpr): IrExpr | undefined { + if (e.kind !== "bytesIntrinsic") return undefined; + const numeric = byteNumberAccess(e); + if (numeric) return e.args[numeric.offsetArg]; + return e.method === "get" ? e.args[0] : undefined; +} + +/** The locals `node` assigns, declares or increments. */ +function writtenIn(node: IrStmt): Set { + const ids = new Set(); + everyStmt(node, { + expr: (e) => { + if (e.kind === "assignExpr" || e.kind === "incDec") ids.add(e.localId); + return true; + }, + stmt: (s) => { + if (s.kind === "assign" || s.kind === "varDecl") ids.add(s.localId); + return true; + }, + }); + return ids; +} + +/** Operands of the loop's remainders and typed array indexes that the + * versioned analysis proves where the function's own analysis does not + * prove them. */ function gainedOperands( - loop: WhileLoop, + loop: IrStmt, known: (e: IrExpr) => boolean, versioned: IntegerRangeFacts, ): IrExpr[] { const operands: IrExpr[] = []; + const gained = (e: IrExpr | undefined): boolean => + e !== undefined && !known(e) && !!versioned.ranges.get(e); everyStmt(loop, { expr: (e) => { if ( @@ -156,23 +198,32 @@ function gainedOperands( !!versioned.ranges.get(e.right) ) operands.push(e.left, e.right); + const index = bytesIndex(e); + if (gained(index)) operands.push(index!); + return true; + }, + stmt: (s) => { + if (s.kind === "bytesSet" && gained(s.index)) operands.push(s.index); return true; }, - stmt: () => true, }); return operands; } /** The locals `roots` read, closed over the loop's writes (a local written * from others reads them) and over its condition (a local the condition - * reads makes every local of the condition relevant). */ -function relevantLocals(loop: WhileLoop, roots: readonly IrExpr[]): Set { + * reads makes every local of the condition relevant). A for loop's + * condition bounds its passes, and with them the positions its body + * advances (`o++` in a loop of `line` passes), so its locals are always + * relevant. An increment or assignment reads its own local. */ +function relevantLocals(loop: IrStmt, roots: readonly IrExpr[]): Set { const relevant = new Set(); - const readsOf = (e: IrExpr): string[] => { + const readsOf = (e: IrExpr | null | undefined): string[] => { const ids: string[] = []; + if (!e) return ids; everyExpr(e, { expr: (x) => { - if (x.kind === "varRef") ids.push(x.localId); + if (x.kind === "varRef" || x.kind === "incDec") ids.push(x.localId); return true; }, stmt: () => true, @@ -193,7 +244,7 @@ function relevantLocals(loop: WhileLoop, roots: readonly IrExpr[]): Set return true; }, }); - const condition = readsOf(loop.cond); + const condition = readsOf(loop.kind === "while" || loop.kind === "for" ? loop.cond : null); const pending: string[] = []; const add = (id: string): void => { if (relevant.has(id)) return; @@ -201,6 +252,7 @@ function relevantLocals(loop: WhileLoop, roots: readonly IrExpr[]): Set pending.push(id); }; for (const root of roots) for (const id of readsOf(root)) add(id); + if (loop.kind === "for") for (const id of condition) add(id); while (pending.length > 0) { const id = pending.pop()!; for (const value of written.get(id) ?? []) for (const other of readsOf(value)) add(other); diff --git a/packages/compiler/test/loop-entry-versions.test.ts b/packages/compiler/test/loop-entry-versions.test.ts index 22e41e3e7..78d86a3bb 100644 --- a/packages/compiler/test/loop-entry-versions.test.ts +++ b/packages/compiler/test/loop-entry-versions.test.ts @@ -43,3 +43,33 @@ test("while loops test their entry values and run integer remainders in the fast await rm(dir, { recursive: true, force: true }); } }); + +test("for loops test the positions they advance and step them as integers in the fast copy", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-entry-cursors-")); + try { + const entry = join(import.meta.dirname, "../../../tests/corpus/for-entry-range-cursors.ts"); + const outPath = join(dir, "cursors.ir.json"); + const result = await compile(entry, { outPath, outDir: dir, outputKind: "ir" }); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + const mod = deserializeModule(await readFile(outPath, "utf8")); + const ll = emitLlvmModule(mod, { pointerBits: 64 }); + const body = (name: string): string => + ll.match( + new RegExp(`define internal [^\\n]+ @sc_bf_${name}\\([^\\n]*\\{([\\s\\S]*?)\\n\\}`), + )![1]!; + for (const name of ["lines", "reverse", "find"]) { + const fn = body(name); + const fast = fn.slice(fn.indexOf("\nentry.fast"), fn.indexOf("\nentry.slow")); + const slow = fn.slice(fn.indexOf("\nentry.slow")); + // The fast copy steps i64 shadows of the positions; the ordinary copy + // steps their doubles. + expect(fast, name).toMatch(/= (add|sub) nsw i64 %t\d+, 1\n/); + expect(fn, name).toMatch(/alloca i64 ; integer [io]\n/); + expect(slow, name).toMatch(/= f(add|sub) double %t\d+, 0x3FF0000000000000\n/); + } + // The initializer writes a position, so the loop has one copy. + expect(body("restart")).not.toContain("entry.fast"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/for-entry-range-cursors.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/for-entry-range-cursors.ts.json new file mode 100644 index 000000000..3dcf49574 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/for-entry-range-cursors.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/for-entry-range-cursors.ts" + ], + "diags": [] +} diff --git a/tests/corpus/for-entry-range-cursors.ts b/tests/corpus/for-entry-range-cursors.ts new file mode 100644 index 000000000..71599277f --- /dev/null +++ b/tests/corpus/for-entry-range-cursors.ts @@ -0,0 +1,116 @@ +// For loops that advance positions carried in from outside the loop +// (`out[o++] = f(src[i++])` in a loop of `line` passes). Entry positions and +// pass counts cover integers inside and outside [-2^48, 2^48], fractions, +// -0, NaN and infinities. + +function code(rand: number): number { + if (rand < 0.25) return 97; + if (rand < 0.5) return 99; + if (rand < 0.75) return 103; + return 116; +} + +// Lines of `line` codes, each followed by a newline, from `start` on. +function lines(input: Float64Array, output: Uint8Array, count: number, start: number): number { + let i = start; + let o = start; + while (count > 0) { + const line = Math.min(count, 6); + for (let j = 0; j < line; j++) output[o++] = code(input[i++]!); + output[o++] = 10; + count -= line; + } + return o + i / 1000; +} + +// Writes backwards from `o`, reading forwards from `i`, with prefix steps. +function reverse(input: Float64Array, output: Uint8Array, line: number, i: number, o: number): string { + for (let j = 0; j < line; j++) output[--o] = code(input[++i]!); + return `${i} ${o}`; +} + +// Leaves the loop with a return that reads both positions. +function find(input: Float64Array, line: number, i: number, o: number): number { + for (let j = 0; j < line; j++) { + if (input[i++]! > 0.9) return i * 1000 + o; + o++; + } + return -(i * 1000 + o); +} + +// The initializer writes a carried position. +function restart(input: Float64Array, output: Uint8Array, line: number, o: number): number { + let i = 3; + for (let j = (i = 0); j < line; j++) output[o++] = code(input[i++]!); + return i * 100 + o; +} + +// A callee that throws partway through the loop. +function guarded(input: Float64Array, output: Uint8Array, line: number, i: number): number { + let o = 0; + for (let j = 0; j < line; j++) { + const rand = input[i++]!; + if (rand > 0.91) throw new RangeError(`stop at ${i} ${o}`); + output[o++] = code(rand); + } + return o; +} + +const input = new Float64Array(64); +let seed = 42; +for (let k = 0; k < input.length; k++) { + seed = (seed * 3877 + 29573) % 139968; + input[k] = seed / 139968; +} +function show(bytes: Uint8Array): string { + let text = ""; + for (const b of bytes) text += b === 10 ? "|" : b === 0 ? "." : String.fromCharCode(b); + return text; +} + +const starts = [0, 3, -0, -2, 2.5, NaN, Infinity, -Infinity, 2 ** 48, 2 ** 48 + 2, 2 ** 53, 60]; +for (const start of starts) { + const output = new Uint8Array(40); + const end = lines(input, output, 20, start); + console.log("lines", start, end, show(output)); +} +for (const count of [0, 1, 6, 7, 2.5, -3, NaN]) { + const output = new Uint8Array(24); + console.log("count", count, lines(input, output, count, 0), show(output)); +} +for (const [line, i, o] of [ + [5, 0, 10], + [5, -1, 4], + [3.5, 2, 8], + [NaN, 0, 5], + [4, 0.5, 6], + [4, 2 ** 48, 2 ** 48], + [4, 1, -0], +] as const) { + const output = new Uint8Array(12); + console.log("reverse", line, i, o, reverse(input, output, line, i, o), show(output)); +} +for (const [line, i, o] of [ + [10, 0, 0], + [64, 5, -3], + [3, 2 ** 48 - 1, 7], + [8, 1.5, 0], +] as const) { + console.log("find", line, i, o, find(input, line, i, o)); +} +for (const [line, o] of [ + [4, 0], + [4, 2], + [2, NaN], +] as const) { + const output = new Uint8Array(8); + console.log("restart", line, o, restart(input, output, line, o), show(output)); +} +for (const start of [0, 20, 40, 2.5]) { + const output = new Uint8Array(64); + try { + console.log("guarded", start, guarded(input, output, 30, start), show(output)); + } catch (error) { + console.log("guarded", start, error instanceof RangeError, (error as Error).message); + } +} diff --git a/tests/harness/corpus-costs/for-entry-range-cursors.ts.json b/tests/harness/corpus-costs/for-entry-range-cursors.ts.json new file mode 100644 index 000000000..ec5830b99 --- /dev/null +++ b/tests/harness/corpus-costs/for-entry-range-cursors.ts.json @@ -0,0 +1 @@ +{ "plain": 5800, "san": 6664 } From f553e5790d660cd07b25b5d090d5aeaf9fc63f87 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 07:54:56 -0500 Subject: [PATCH 16/54] Check that recorded units leave held functions and IR data unchanged Function tables and IR data maps hold objects a unit could change in place, which a record does not carry. The serial mode now keeps the encoded content of their entries from the start of each wave and throws an InternalCompilerError naming an entry whose content changed by the wave's end. --- .../frontend/lowering/unit-journal.test.ts | 20 +++++ .../src/frontend/lowering/unit-journal.ts | 81 +++++++++++++++++-- 2 files changed, 93 insertions(+), 8 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/unit-journal.test.ts b/packages/compiler/src/frontend/lowering/unit-journal.test.ts index 74dc57b1a..ff47e5c0a 100644 --- a/packages/compiler/src/frontend/lowering/unit-journal.test.ts +++ b/packages/compiler/src/frontend/lowering/unit-journal.test.ts @@ -280,3 +280,23 @@ test("a unit that changes an entry it did not add has no record", () => { expect(unit.record).toBeNull(); expect(unit.reasons).toEqual(["helpers:replaced"]); }); + +test("a recorded wave throws when a unit changes a held function in place", () => { + const functions = [fn("%shared")]; + const subject: WaveHost = { + tables: [functionTable("lifted", functions, (name) => functions.some((f) => f.name === name))], + scalars: [], + lowerRecorded: (name) => { + if (name === "b") + functions[0]!.locals.push({ id: "x.0", name: "x", type: { kind: "f64" } } as never); + return { fn: fn(`%${name}`), poisoned: false }; + }, + store: () => {}, + statementCounts: () => ({ total: 0, failed: 0, other: 0 }), + resolves: () => true, + }; + lowerWaveRecorded(subject, ["a"], waveStats()); + expect(() => lowerWaveRecorded(subject, ["a", "b"], waveStats())).toThrow( + "a unit changed lifted entry %shared in place", + ); +}); diff --git a/packages/compiler/src/frontend/lowering/unit-journal.ts b/packages/compiler/src/frontend/lowering/unit-journal.ts index 8c8de3001..abb9d90b4 100644 --- a/packages/compiler/src/frontend/lowering/unit-journal.ts +++ b/packages/compiler/src/frontend/lowering/unit-journal.ts @@ -22,6 +22,7 @@ * without a record has a line with its name and reasons. */ import { appendFileSync, mkdirSync } from "node:fs"; import { join } from "node:path"; +import { InternalCompilerError } from "../../errors.js"; import type { IrFunction } from "../../ir/ir.js"; import { deserializeFunction, encodeFunction } from "../../ir/serialize.js"; @@ -90,6 +91,17 @@ export interface JournalTable { readonly truncate?: (from: number) => void; /** The entries the table holds now. */ readonly hold?: () => HeldEntries; + /** The content of the entries the table holds now, for tables whose + * values are objects a unit could change in place. */ + readonly contents?: () => HeldContents; +} + +/** The content of a table's entries at one point (JournalTable.contents). */ +export interface HeldContents { + /** Adds the content of the entries past the ones this holds. */ + extend(): void; + /** The key of an entry this holds whose content has changed, or null. */ + changed(): string | null; } /** A table's entries at one point (JournalTable.hold). */ @@ -277,6 +289,9 @@ export function recordUnit(host: WaveHost, name: string): RecordedUnit { * results its record carries: the entries it added are taken out of their * tables and added again from the record, and its function is the one * decoded from the record. A unit without one keeps what lowering made. + * Throws an InternalCompilerError when a unit changed the content of an + * entry a table held when the unit started (JournalTable.contents), which + * no record carries. * `lines`, when given, receives each unit's record as a JSON line. */ export function lowerWaveRecorded( host: WaveHost, @@ -286,8 +301,12 @@ export function lowerWaveRecorded( ): void { stats.waves++; const tables = host.tables; + // Units keep the content of the entries that tables held when they + // started: a record carries a unit's new entries, not changes to others. + const contents = tables.map((table) => (table.contents === undefined ? null : table.contents())); for (const name of wave) { const unit = recordUnit(host, name); + for (const held of contents) held?.extend(); stats.units++; const record = unit.record; if (record === null) { @@ -307,6 +326,13 @@ export function lowerWaveRecorded( }); host.store(name, record.fn === null ? null : decodeUnitFunction(record.fn)); } + tables.forEach((table, index) => { + const key = contents[index]?.changed() ?? null; + if (key !== null) + throw new InternalCompilerError( + `a unit changed ${table.name} entry ${key} in place, which no unit record carries`, + ); + }); } /** An appendable table over a Map from strings to strings, such as helper @@ -334,15 +360,37 @@ export function countTable(name: string, map: Map): JournalTable * JSON. JSON writes a number that is not finite as null, so a value whose * encoding holds a null does not encode. */ export function irDataTable(name: string, map: Map): JournalTable { - return codecTable( - name, - map, - (value) => { - const text = JSON.stringify(value); - return /[:,[]null\b/.test(text) ? null : text; + const encode = (value: V): string | null => { + const text = JSON.stringify(value); + return /[:,[]null\b/.test(text) ? null : text; + }; + return { + ...codecTable(name, map, encode, (text) => JSON.parse(text) as V), + contents: () => { + const keys: string[] = []; + const texts: (string | null)[] = []; + const held: HeldContents = { + extend: () => { + let index = 0; + for (const [key, value] of map) { + if (index++ < keys.length) continue; + keys.push(key); + texts.push(encode(value)); + } + }, + changed: () => { + for (let index = 0; index < keys.length; index++) { + const key = keys[index]!; + const value = map.get(key); + if (value === undefined || encode(value) !== texts[index]) return key; + } + return null; + }, + }; + held.extend(); + return held; }, - (text) => JSON.parse(text) as V, - ); + }; } /** An appendable table over a Map from strings to values that `encode` @@ -465,6 +513,23 @@ export function functionTable( const held = [...functions]; return { same: () => held.every((fn, index) => functions[index] === fn) }; }, + contents: () => { + const texts: (string | null)[] = []; + const held: HeldContents = { + extend: () => { + for (let index = texts.length; index < functions.length; index++) + texts.push(encodeUnitFunction(functions[index]!)); + }, + changed: () => { + for (let index = 0; index < texts.length && index < functions.length; index++) + if (encodeUnitFunction(functions[index]!) !== texts[index]) + return functions[index]!.name; + return null; + }, + }; + held.extend(); + return held; + }, }; } From c54c4e1d2ab6cdc5cb98889c8ed15185939dd8c9 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 07:57:22 -0500 Subject: [PATCH 17/54] Prove module-level number state int32 through every write - Module globals of type number join the whole-program int32 fixpoint as `global:` candidates, starting non-negative. A candidate needs its module initializer to assign it at the top level (its declaration), and no write by `++`/`--`, for-of or catch; every assignment must evaluate to an int32 under the current facts (an integer literal counts in any function), and a write that may be negative keeps only the int32 fact. - Reads take the proven range through IntegerSlotFacts.locals, so a generator state reduced by a remainder (`seed = (seed * a + c) % m`) takes unsigned integer remainders without the -0 select. --- packages/compiler/src/ir/int32-slots.test.ts | 63 +++++++++++ packages/compiler/src/ir/int32-slots.ts | 104 ++++++++++++++++-- .../test/int32-global-emission.test.ts | 33 ++++++ .../tests/corpus/module-state-int32.ts.json | 6 + tests/corpus/module-state-int32.ts | 71 ++++++++++++ .../corpus-costs/module-state-int32.ts.json | 1 + 6 files changed, 269 insertions(+), 9 deletions(-) create mode 100644 packages/compiler/test/int32-global-emission.test.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/module-state-int32.ts.json create mode 100644 tests/corpus/module-state-int32.ts create mode 100644 tests/harness/corpus-costs/module-state-int32.ts.json diff --git a/packages/compiler/src/ir/int32-slots.test.ts b/packages/compiler/src/ir/int32-slots.test.ts index b94dee341..a51c06900 100644 --- a/packages/compiler/src/ir/int32-slots.test.ts +++ b/packages/compiler/src/ir/int32-slots.test.ts @@ -565,3 +565,66 @@ test("narrowed divisor facts prove remainders only while every divisor is positi expect(zero.facts("caller").call?.("rem") ?? null).toBeNull(); expect(zero.reasons.get("ret:rem")).toMatch(/not int32/); }); + +/** A module with one number global `state`: its initializer assigns `init` + * inside an unlabeled block, as a declaration does, and `step` writes + * `write(l)` with `l` the global's value. */ +function globalModule( + init: IrExpr | null, + write: (l: IrExpr) => IrExpr, + extra: IrStmt[] = [], +): IrModule { + const state = "%g.e.state"; + const block: IrStmt = { + loc, + kind: "block", + body: init ? [{ loc, kind: "assign", localId: state, value: init }] : [], + }; + const step = fn( + "step", + [], + [ + { loc, kind: "varDecl", localId: "l", init: ref(state) }, + { loc, kind: "assign", localId: state, value: write(ref("l")) }, + ...extra, + ], + ); + step.locals.push({ id: "l", name: "l", type: F64, mutable: true }); + const m = mod([], [fn("%init.0", [], [block]), step]); + m.globals = [{ id: state, name: "state", type: F64, mutable: true }]; + return m; +} + +test("proves a module global non-negative through every write", () => { + const m = globalModule(num(42), (l) => + bin("%", bin("+", bin("*", l, num(3877)), num(29573)), num(139968)), + ); + const slots = analyzeInt32Slots(m); + expect(slots.hasGlobal("%g.e.state")).toBe(true); + expect(slots.facts("step").locals?.("%g.e.state")).toEqual({ min: 0, max: 2147483647 }); + // A write that may be negative keeps the int32 fact without the sign. + const signed = analyzeInt32Slots( + globalModule(num(42), (l) => bin("-", bin("&", l, num(255)), num(128))), + ); + expect(signed.facts("step").locals?.("%g.e.state")).toEqual({ + min: -2147483648, + max: 2147483647, + }); +}); + +test("drops module globals with a write the fixpoint cannot prove", () => { + const fraction = analyzeInt32Slots(globalModule(num(42), (l) => bin("/", l, num(2)))); + expect(fraction.hasGlobal("%g.e.state")).toBe(false); + expect(fraction.reasons.get("global:%g.e.state")).toBe("write in step is not int32"); + expect(fraction.facts("step").locals?.("%g.e.state") ?? null).toBeNull(); + // ++/-- writes, and a global its initializer does not assign, are no candidates. + const increment = globalModule(num(42), (l) => l, [ + stmt({ loc, kind: "incDec", op: "+", prefix: false, localId: "%g.e.state", type: F64 }), + ]); + expect(analyzeInt32Slots(increment).hasGlobal("%g.e.state")).toBe(false); + expect(analyzeInt32Slots(globalModule(null, (l) => l)).hasGlobal("%g.e.state")).toBe(false); + // An Int32 contract already covers its global. + const contract = globalModule(num(42), (l) => l); + contract.contracts = { int32Locals: ["\0%g.e.state"] } as NonNullable; + expect(analyzeInt32Slots(contract).hasGlobal("%g.e.state")).toBe(false); +}); diff --git a/packages/compiler/src/ir/int32-slots.ts b/packages/compiler/src/ir/int32-slots.ts index be5f29768..465f6c683 100644 --- a/packages/compiler/src/ir/int32-slots.ts +++ b/packages/compiler/src/ir/int32-slots.ts @@ -118,6 +118,9 @@ export class Int32Slots { private readonly pinned = new Set(); /** Int32 contract locals (`function\0localId`) and globals (`\0id`). */ private readonly int32Locals = new Set(); + /** Module globals of type number whose every write the fixpoint proved + * an int32 (`global:` keys). */ + private readonly globals = new Set(); /** Number-array slots whose arrays hold only int32 elements (Int32[] * contracts, IrContracts.int32Elements keys). */ private readonly int32Elements = new Set(); @@ -167,6 +170,7 @@ export class Int32Slots { this.paramFacts.size === 0 && this.boxes.size === 0 && this.int32Locals.size === 0 && + this.globals.size === 0 && this.int32Elements.size === 0 ) return {}; @@ -181,11 +185,13 @@ export class Int32Slots { return group !== null && this.boxes.has(group) ? level(`box:${group}`) : null; }, }; - if (this.int32Locals.size > 0) + if (this.int32Locals.size > 0 || this.globals.size > 0) result.locals = (localId) => this.int32Locals.has(`${fnName}\0${localId}`) || this.int32Locals.has(`\0${localId}`) ? INT32_RANGE - : null; + : this.globals.has(localId) + ? level(`global:${localId}`) + : null; if (this.int32Elements.size > 0) { result.elements = (arr) => this.hasInt32Elements(fnName, arr); if (this.narrowedReads) result.narrowedReads = true; @@ -251,6 +257,16 @@ export class Int32Slots { addInt32Local(key: string): void { this.int32Locals.add(key); } + addGlobal(id: string): void { + this.globals.add(id); + } + hasGlobal(id: string): boolean { + return this.globals.has(id); + } + dropGlobal(id: string): boolean { + this.nonNegative.delete(`global:${id}`); + return this.globals.delete(id); + } addInt32Elements(key: string): void { this.int32Elements.add(key); } @@ -372,6 +388,7 @@ export class Int32Slots { reasons: [...this.reasons], pinned: [...this.pinned], int32Locals: [...this.int32Locals], + globals: [...this.globals], int32Elements: [...this.int32Elements], pinViolations: this.pinViolations, nonNegative: [...this.nonNegative], @@ -398,6 +415,7 @@ export class Int32Slots { for (const [key, why] of data.reasons) slots.reasons.set(key, why); for (const key of data.pinned) slots.pinned.add(key); for (const key of data.int32Locals) slots.int32Locals.add(key); + for (const id of data.globals) slots.globals.add(id); for (const key of data.int32Elements) slots.int32Elements.add(key); slots.pinViolations.push(...data.pinViolations); for (const key of data.nonNegative) slots.nonNegative.add(key); @@ -422,6 +440,7 @@ export class Int32Slots { `called ${sorted(this.called)}`, `pinned ${sorted(this.pinned)}`, `int32Locals ${sorted(this.int32Locals)}`, + `globals ${sorted(this.globals)}`, `int32Elements ${sorted(this.int32Elements)}`, `nonNegative ${sorted(this.nonNegative)}`, `pinViolations ${sorted(this.pinViolations)}`, @@ -463,6 +482,7 @@ interface Int32SlotsData { reasons: [string, string][]; pinned: string[]; int32Locals: string[]; + globals: string[]; int32Elements: string[]; pinViolations: string[]; nonNegative: string[]; @@ -1208,6 +1228,9 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}): }; const boxedReads: [IrFunction, Set][] = []; const incrementedFamilies: string[] = []; + // Bindings written by `++`/`--` or bound by for-of and catch: a module + // global among them is no candidate. + const unstableGlobals = new Set(); for (const fn of mod.functions) { for (const local of fn.locals) if (local.boxed) { @@ -1251,12 +1274,20 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}): }); } else if (e.kind === "classRef") for (const id of e.captures ?? []) unstableBoxes.add(boxKey(fn.name, id)); - else if (e.kind === "incDec") unstableBoxes.add(boxKey(fn.name, e.localId)); + else if (e.kind === "incDec") { + unstableBoxes.add(boxKey(fn.name, e.localId)); + unstableGlobals.add(e.localId); + } }, stmt: (s) => { - if (s.kind === "forOf") unstableBoxes.add(boxKey(fn.name, s.localId)); - if (s.kind === "tryCatch" && s.catchLocalId) + if (s.kind === "forOf") { + unstableBoxes.add(boxKey(fn.name, s.localId)); + unstableGlobals.add(s.localId); + } + if (s.kind === "tryCatch" && s.catchLocalId) { unstableBoxes.add(boxKey(fn.name, s.catchLocalId)); + unstableGlobals.add(s.catchLocalId); + } }, }); if (read.size > 0) boxedReads.push([fn, read]); @@ -1361,6 +1392,41 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}): const group = slots.boxGroup(fn.name, localId); if (group !== null) reads(`box:${group}`, fn.name); } + + /* ── module globals ── */ + // A module global of type number is a candidate when its module's + // initializer assigns it at its top level or in an unlabeled block there + // (the declaration's initializer, which runs before any read that does + // not throw) and no other write is outside the assignments the fixpoint + // checks. Globals with a TDZ sentinel or a cycle's initialization flag, + // and globals an Int32 contract already covers, keep their ordinary facts. + const initialized = new Set(); + const initializers = (body: readonly IrStmt[]): void => { + for (const s of body) { + if (s.kind === "assign") initialized.add(s.localId); + else if (s.kind === "block" && !s.labels?.length) initializers(s.body); + } + }; + for (const fn of mod.functions) if (fn.name.startsWith("%init")) initializers(fn.body); + for (const global of mod.globals ?? []) { + if ( + global.type.kind !== "f64" || + global.tdz || + global.initFlag !== undefined || + !initialized.has(global.id) || + unstableGlobals.has(global.id) || + contracts?.int32Locals?.includes(`\0${global.id}`) + ) + continue; + slots.addGlobal(global.id); + slots.assumeNonNegative(`global:${global.id}`); + } + const dropGlobal = (id: string, why: string): boolean => { + reason(`global:${id}`, why); + return slots.dropGlobal(id); + }; + for (const [fn, read] of boxedReads) + for (const localId of read) if (slots.hasGlobal(localId)) reads(`global:${localId}`, fn.name); for (const family of incrementedFamilies) dropFamily(family, "++/-- on the field"); /* ── the optimistic fixpoint ── */ @@ -1396,7 +1462,7 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}): eachNodeOfKinds(fn, candidateWrites, (node) => { switch (node.kind) { case "assign": - found = box(node.localId); + found = box(node.localId) || slots.hasGlobal(node.localId); break; case "varDecl": found = node.init !== null && box(node.localId); @@ -1420,7 +1486,7 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}): ); break; case "assignExpr": - found = box(node.localId); + found = box(node.localId) || slots.hasGlobal(node.localId); break; } return !found; @@ -1637,9 +1703,26 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}): if (int32(value)) settle(`box:${group}`, value); else if (dropBox(group, `write in ${fn.name} is not int32`)) invalidate(`box:${group}`); }; + // An integer literal is its own range, also where the analysis proves + // nothing (an async module initializer). + const globalWrite = (localId: string, value: IrExpr): void => { + if (!slots.hasGlobal(localId)) return; + const key = `global:${localId}`; + const range = + ranges.get(value) ?? + (value.kind === "numLit" && Number.isSafeInteger(value.value) && !Object.is(value.value, -0) + ? { min: value.value, max: value.value } + : null); + if (!withinInt32(range)) { + if (dropGlobal(localId, `write in ${fn.name} is not int32`)) invalidate(key); + } else if (range!.min < 0 && slots.demote(key)) invalidate(key); + }; walkFunction(fn, nodes?.scan(CANDIDATE_WRITE_KINDS), { stmt: (s) => { - if (s.kind === "assign") boxWrite(s.localId, s.value); + if (s.kind === "assign") { + boxWrite(s.localId, s.value); + globalWrite(s.localId, s.value); + } else if (s.kind === "varDecl" && s.init) boxWrite(s.localId, s.init); if (s.kind === "fieldSet") { const family = slots.family(s.className, s.field); @@ -1662,7 +1745,10 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}): expr: (e) => { if (e.kind === "call") args(functions.get(e.callee), e.args, 0); else if (e.kind === "new") args(functions.get(`%${e.className}.constructor`), e.args, 1); - else if (e.kind === "assignExpr") boxWrite(e.localId, e.value); + else if (e.kind === "assignExpr") { + boxWrite(e.localId, e.value); + globalWrite(e.localId, e.value); + } }, }); provenByFunction.set(fn.name, proven); diff --git a/packages/compiler/test/int32-global-emission.test.ts b/packages/compiler/test/int32-global-emission.test.ts new file mode 100644 index 000000000..5738fc6d3 --- /dev/null +++ b/packages/compiler/test/int32-global-emission.test.ts @@ -0,0 +1,33 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile, deserializeModule } from "../src/index.js"; +import { emitLlvmModule } from "../src/backend/llvm/emitter.js"; +import { analyzeInt32Slots } from "../src/ir/int32-slots.js"; + +test("module state proven non-negative takes unsigned remainders", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-int32-globals-")); + try { + const entry = join(import.meta.dirname, "../../../tests/corpus/module-state-int32.ts"); + const outPath = join(dir, "state.ir.json"); + const result = await compile(entry, { outPath, outDir: dir, outputKind: "ir" }); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + const mod = deserializeModule(await readFile(outPath, "utf8")); + const slots = analyzeInt32Slots(mod); + expect(slots.hasGlobal("%g.e.seed")).toBe(true); + expect(slots.hasGlobal("%g.e.ticks")).toBe(true); + // A negative or fractional write leaves the global a double. + expect(slots.reasons.get("global:%g.e.balance")).toBe("write in spend is not int32"); + expect(slots.reasons.get("global:%g.e.ratio")).toBe("write in halve is not int32"); + // Read before its declaration runs (a TDZ check): no candidate. + expect(slots.hasGlobal("%g.e.marker")).toBe(false); + const ll = emitLlvmModule(mod, { pointerBits: 64 }); + const fill = ll.match(/define internal [^\n]+ @sc_bf_fill\([^\n]*\{([\s\S]*?)\n\}/)![1]!; + expect(fill).toContain("urem i64"); + expect(fill).not.toContain("srem"); + expect(fill).not.toContain("frem"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/module-state-int32.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/module-state-int32.ts.json new file mode 100644 index 000000000..55613a4a1 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/module-state-int32.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/module-state-int32.ts" + ], + "diags": [] +} diff --git a/tests/corpus/module-state-int32.ts b/tests/corpus/module-state-int32.ts new file mode 100644 index 000000000..5e9a46fa0 --- /dev/null +++ b/tests/corpus/module-state-int32.ts @@ -0,0 +1,71 @@ +// Module-level number state written from several functions: a generator +// state reduced by a remainder, a counter that goes negative, a value that +// becomes a fraction, writes from a closure and an async function, and a +// read before the declaration runs. + +function early(): number { + return marker + 1; +} +try { + console.log("early", early()); +} catch (error) { + console.log("early", (error as Error).name); +} + +let marker = 5; +let seed = 42; +let balance = 10; +let ratio = 3; +let ticks = 0; + +function next(): number { + seed = (seed * 3877 + 29573) % 139968; + return seed; +} + +function fill(buffer: Float64Array): void { + let l = seed; + for (let k = 0; k < buffer.length; k++) { + l = (l * 3877 + 29573) % 139968; + buffer[k] = l / 139968; + } + seed = l; +} + +function spend(amount: number): number { + balance = (balance & 1023) - amount; + return balance; +} + +function halve(): number { + ratio = ratio / 2; + return ratio; +} + +const tick = (): number => { + ticks = (ticks + 7) % 10; + return ticks; +}; + +async function later(): Promise { + await Promise.resolve(); + ticks = 9; + return ticks; +} + +const buffer = new Float64Array(5); +console.log("next", next(), next(), next()); +fill(buffer); +let shown = ""; +for (const v of buffer) shown += ` ${v.toFixed(6)}`; +console.log("fill", shown, seed); +seed = 2147483647; +console.log("large", next(), next()); +seed = 0; +console.log("zero", next()); +console.log("spend", spend(4), spend(30), spend(-2000), spend(5)); +console.log("halve", halve(), halve(), ratio * 4); +console.log("tick", tick(), tick(), tick(), ticks); +console.log("later", await later(), ticks, tick()); +marker = marker * 3; +console.log("marker", early(), marker); diff --git a/tests/harness/corpus-costs/module-state-int32.ts.json b/tests/harness/corpus-costs/module-state-int32.ts.json new file mode 100644 index 000000000..5e1bd9bc5 --- /dev/null +++ b/tests/harness/corpus-costs/module-state-int32.ts.json @@ -0,0 +1 @@ +{ "plain": 5618, "san": 5782 } From 3b374f3201c045c5805faaf0b150ded3d09c4068 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 08:21:48 -0500 Subject: [PATCH 18/54] Check with the shared tsc-ts server by default on Linux and macOS Builds on Linux and macOS now connect to a shared `tsc-ts --api --listen` server by default, which keeps the files it parsed and bound, so a later build in the same working directory does not parse the TypeScript libs and scriptc's declaration files again. SCRIPTC_CHECKER_SERVER=0 turns it off. When CI is set to a value other than 0 or false, builds use their own tsc-ts process unless SCRIPTC_CHECKER_SERVER=1. - The server exits after 600 seconds without a build (was 300). - Its name also digests the compiler executable's identity, so each scriptc installation and build connects only to servers it started. - The server process runs in the root directory, so it does not keep the first build's directory in use; its --cwd argument is unchanged. - When the connection to the server fails after it answered, through a failed transfer or an ended stream, the native CLI runs the command again in a new process with SCRIPTC_CHECKER_SERVER=0 and exits with its status. - SCRIPTC_CHECKER_SERVER no longer changes the frontend cache key: the server's answers are the build's own checker's. The CLI docs describe the background process, its socket directory, when it exits and how to stop it. --- docs/content/docs/cli.mdx | 2 +- .../compiler/native/ts7-process-child.test.c | 12 +++--- packages/compiler/native/ts7-process.c | 8 ++-- packages/compiler/native/ts7-process.test.c | 7 +-- .../src/frontend/ts7/checker-server.test.ts | 11 +++-- .../src/frontend/ts7/checker-server.ts | 38 +++++++++++----- .../compiler/src/frontend/ts7/native-api.ts | 43 ++++++++++++++++++- packages/compiler/src/native/cli.ts | 35 +++++++++++++-- packages/compiler/src/native/prepare.ts | 2 + .../self-hosting/ts7-native-process.ts | 6 ++- 10 files changed, 130 insertions(+), 34 deletions(-) diff --git a/docs/content/docs/cli.mdx b/docs/content/docs/cli.mdx index b1583a617..fd15a000d 100644 --- a/docs/content/docs/cli.mdx +++ b/docs/content/docs/cli.mdx @@ -246,7 +246,7 @@ With `--sparse`, the profile leaves out the functions the runs never reached. Th
The TypeScript 7 API server that checks the program: tsc-ts (the default), which scriptc includes, or tsgo, from the typescript package scriptc also includes. Both give the same diagnostics and generated code.
SCRIPTC_CHECKER_SERVER
-
Set to 1 on Linux and macOS to check with a tsc-ts server that later builds share. It takes effect with a tsc-ts release that serves tsc-ts --api --listen. The tsc-ts 0.0.9 that this version of scriptc includes does not, so each build checks with its own tsc-ts process, as without the variable. A build connects to the server for its working directory, starting it when none answers, and the server keeps the files it parsed, so the next build of the same program loads it faster. A build that finds the server in use by another build, or cannot start or reach it, checks with its own tsc-ts process. The server exits after 5 minutes without a build, or after a build when its resident memory is above 2048 MB (TSC_TS_LISTEN_MEMORY sets another limit in megabytes). Diagnostics and generated code are the same with and without it. It applies to the default checker, tsc-ts.
+
On Linux and macOS, builds check with a tsc-ts server that later builds share. The first build in a working directory starts it: a tsc-ts --api --listen process that keeps running in the background with no terminal and keeps the files it parsed, so the next build in that directory loads its program faster. Each user, working directory and scriptc installation gets its own server, and a build connects only to the server of its own scriptc and tsc-ts. The socket is in $XDG_RUNTIME_DIR/scriptc, or /tmp/scriptc-<uid>, a directory only that user can access. A build that finds the server in use by another build, cannot start or reach it, or loses its connection to it, checks with its own tsc-ts process. The server exits after 10 minutes without a build, or after a build when its resident memory is above 2048 MB (TSC_TS_LISTEN_MEMORY sets another limit in megabytes). To stop it sooner, end that process, for example with pkill -f 'tsc-ts --api --listen'. Set to 0 to check with a tsc-ts process of each build's own. When CI is set to a value other than 0 or false, builds use their own process unless this variable is 1. Diagnostics and generated code are the same with and without the server. It applies to the default checker, tsc-ts.
```console diff --git a/packages/compiler/native/ts7-process-child.test.c b/packages/compiler/native/ts7-process-child.test.c index 4d1cbc081..d387f735a 100644 --- a/packages/compiler/native/ts7-process-child.test.c +++ b/packages/compiler/native/ts7-process-child.test.c @@ -20,8 +20,9 @@ /* `--listen PATH --idle-exit SECONDS`: a stand-in for a shared server. In * mode "listen-exit" it exits as an executable without --listen does. In * mode "listen" it answers each client, in order, with its client number, - * its listen arguments and TSC_TS_LISTEN_MEMORY, then echoes until the - * client closes, and exits after SECONDS (at most 5) without a client. */ + * its listen arguments, TSC_TS_LISTEN_MEMORY and its working directory, then + * echoes until the client closes, and exits after SECONDS (at most 5) without + * a client. */ static int listen_main(const char *mode, char **argv) { if (strcmp(mode, "listen-exit") == 0) return 2; if (strcmp(mode, "listen") != 0) return 73; @@ -42,9 +43,10 @@ static int listen_main(const char *mode, char **argv) { if (poll(&entry, 1, idle * 1000) <= 0) break; int fd = accept(listener, NULL, NULL); if (fd < 0) return 79; - char line[512]; - int size = snprintf(line, sizeof(line), "client %d\n%s %s\n%s\n", client, argv[0], argv[2], - memory == NULL ? "unset" : memory); + char line[1024], cwd[256]; + if (getcwd(cwd, sizeof(cwd)) == NULL) return 81; + int size = snprintf(line, sizeof(line), "client %d\n%s %s\n%s\n%s\n", client, argv[0], argv[2], + memory == NULL ? "unset" : memory, cwd); if (size <= 0 || write(fd, line, (size_t)size) != size) return 80; uint8_t bytes[17]; ssize_t count; diff --git a/packages/compiler/native/ts7-process.c b/packages/compiler/native/ts7-process.c index 8261c341b..bf486c87d 100644 --- a/packages/compiler/native/ts7-process.c +++ b/packages/compiler/native/ts7-process.c @@ -414,8 +414,10 @@ static int connect_server(const char *path) { /* Starts the server in its own process group, so that a terminal's signals * to the build do not reach it, with its standard streams on /dev/null and * no other descriptor of this process, so that it holds no pipe a caller of - * the build waits on. `limit` (TSC_TS_LISTEN_MEMORY=...) is added to the - * environment unless it sets TSC_TS_LISTEN_MEMORY. */ + * the build waits on. It runs in the root directory, so that it does not + * keep the build's directory in use; its working directory for the build's + * files is the --cwd argument. `limit` (TSC_TS_LISTEN_MEMORY=...) is added + * to the environment unless it sets TSC_TS_LISTEN_MEMORY. */ static pid_t start_server(char **args, char *limit) { #ifdef __APPLE__ char **inherited = *_NSGetEnviron(); @@ -440,7 +442,7 @@ static pid_t start_server(char **args, char *limit) { // Only async-signal-safe calls until exec. int null = open("/dev/null", O_RDWR); if (setpgid(0, 0) != 0 || null < 0 || dup2(null, STDIN_FILENO) < 0 || dup2(null, STDOUT_FILENO) < 0 || - dup2(null, STDERR_FILENO) < 0) + dup2(null, STDERR_FILENO) < 0 || chdir("/") != 0) _exit(127); #if defined(__linux__) && defined(SYS_close_range) if (syscall(SYS_close_range, 3U, ~0U, 0U) != 0) diff --git a/packages/compiler/native/ts7-process.test.c b/packages/compiler/native/ts7-process.test.c index 85de63db1..e07a7d10c 100644 --- a/packages/compiler/native/ts7-process.test.c +++ b/packages/compiler/native/ts7-process.test.c @@ -208,9 +208,9 @@ static void attach_cases(void) { uint8_t bytes[4]; snprintf(directory, sizeof(directory), "%s/scriptc", runtime); snprintf(socket_path, sizeof(socket_path), "%s/server.sock", directory); - // The first build starts the server with its idle time and memory limit. - // The server holds none of the build's descriptors, such as an output - // pipe its caller reads to the end. + // The first build starts the server with its idle time and memory limit, + // in the root directory. The server holds none of the build's descriptors, + // such as an output pipe its caller reads to the end. int output[2]; assert(pipe(output) == 0); uint32_t first = attach("server", "listen", 123); @@ -224,6 +224,7 @@ static void attach_cases(void) { snprintf(expected, sizeof(expected), "%s 1\n", socket_path); expect_line(first, expected); expect_line(first, "123\n"); + expect_line(first, "/\n"); write_exact(first, (const uint8_t *)"ping", 4, 0, 4); read_exact(first, bytes, 4); assert(memcmp(bytes, "ping", 4) == 0); diff --git a/packages/compiler/src/frontend/ts7/checker-server.test.ts b/packages/compiler/src/frontend/ts7/checker-server.test.ts index 9a0371fe6..e6cc5d29a 100644 --- a/packages/compiler/src/frontend/ts7/checker-server.test.ts +++ b/packages/compiler/src/frontend/ts7/checker-server.test.ts @@ -12,13 +12,18 @@ const posix = process.platform === "linux" || process.platform === "darwin"; afterAll(() => rmSync(dir, { recursive: true, force: true })); -test("builds use the shared checker server only when they opt in, on Linux and macOS, with tsc-ts", () => { - expect(checkerServerName(executable, dir, false, {})).toBeNull(); +test("builds use the shared checker server by default outside CI, on Linux and macOS, with tsc-ts", () => { expect(checkerServerName(executable, dir, false, { SCRIPTC_CHECKER_SERVER: "0" })).toBeNull(); + expect(checkerServerName(executable, dir, false, { SCRIPTC_CHECKER: "tsgo" })).toBeNull(); expect( checkerServerName(executable, dir, false, { ...enabled, SCRIPTC_CHECKER: "tsgo" }), ).toBeNull(); - const name = checkerServerName(executable, dir, false, enabled); + // CI turns the default off; SCRIPTC_CHECKER_SERVER=1 turns it back on. + expect(checkerServerName(executable, dir, false, { CI: "true" })).toBeNull(); + expect(checkerServerName(executable, dir, false, { CI: "1" })).toBeNull(); + const name = checkerServerName(executable, dir, false, {}); + for (const env of [enabled, { CI: "false" }, { CI: "0" }, { CI: "" }, { ...enabled, CI: "true" }]) + expect(checkerServerName(executable, dir, false, env)).toBe(name); if (!posix) expect(name).toBeNull(); else expect(name).toMatch(/^[0-9a-f]{32}$/); }); diff --git a/packages/compiler/src/frontend/ts7/checker-server.ts b/packages/compiler/src/frontend/ts7/checker-server.ts index 94c80f8db..36a951e4c 100644 --- a/packages/compiler/src/frontend/ts7/checker-server.ts +++ b/packages/compiler/src/frontend/ts7/checker-server.ts @@ -5,23 +5,37 @@ import { TS7_FILE_SYSTEM_CALLBACKS } from "./rpc-filesystem.js"; /** How long a shared checker server waits for the next build before it * exits, in seconds. */ -export const CHECKER_SERVER_IDLE_EXIT = 300; +export const CHECKER_SERVER_IDLE_EXIT = 600; /** The resident memory, in megabytes, above which a shared checker server * exits after a build. tsc-ts never frees a program's syntax trees, types and * symbols, so its memory grows with each build it serves. */ export const CHECKER_SERVER_MEMORY = 2048; +/** Whether builds use a shared checker server: by default, except in CI + * (`CI` set to anything but `0` or `false`), where a job usually builds once + * per directory and would only leave the server behind. + * `SCRIPTC_CHECKER_SERVER=0` turns it off and `SCRIPTC_CHECKER_SERVER=1` turns + * it on in CI. */ +function checkerServerEnabled(env: NodeJS.ProcessEnv): boolean { + const setting = env["SCRIPTC_CHECKER_SERVER"]; + if (setting === "1") return true; + if (setting === "0") return false; + const ci = env["CI"]; + return ci === undefined || ci === "" || ci === "0" || ci === "false"; +} + /** - * With `SCRIPTC_CHECKER_SERVER=1` on Linux and macOS, the native compiler - * checks with a tsc-ts server shared by the builds that follow - * (`tsc-ts --api --listen`): a build connects to it, starting it when none - * answers, and the server keeps the files it parsed for the next build. - * Returns the server's name, a digest of everything a server's answers - * depend on besides the files it reads through the build: the tsc-ts - * executable, the working directory, the file system callbacks, timing, - * whether it reads its library files from the disk, and the `TSC_TS_*` - * environment. Returns null when the shared server is not used. + * On Linux and macOS, the native compiler checks with a tsc-ts server shared + * by the builds that follow (`tsc-ts --api --listen`): a build connects to + * it, starting it when none answers, and the server keeps the files it + * parsed for the next build (see checkerServerEnabled for when). Returns the + * server's name, a digest of everything a server's answers depend on + * besides the files it reads through the build: the tsc-ts executable, the + * working directory, the file system callbacks, timing, whether it reads its + * library files from the disk, and the `TSC_TS_*` environment; and of the + * compiler executable, so that each scriptc build has servers of its own. + * Returns null when the shared server is not used. */ export function checkerServerName( executable: string, @@ -29,8 +43,9 @@ export function checkerServerName( timing: boolean, env: NodeJS.ProcessEnv = process.env, librariesFromDisk = false, + compiler: string = process.execPath, ): string | null { - if (env["SCRIPTC_CHECKER_SERVER"] !== "1") return null; + if (!checkerServerEnabled(env)) return null; if (process.platform !== "linux" && process.platform !== "darwin") return null; if (checkerBackend(env) !== "tsc-ts") return null; const tscTsEnv = Object.keys(env) @@ -41,6 +56,7 @@ export function checkerServerName( .update( JSON.stringify([ checkerExecutableIdentity(executable), + checkerExecutableIdentity(compiler), cwd, TS7_FILE_SYSTEM_CALLBACKS, timing, diff --git a/packages/compiler/src/frontend/ts7/native-api.ts b/packages/compiler/src/frontend/ts7/native-api.ts index fa0690e69..03be8974c 100644 --- a/packages/compiler/src/frontend/ts7/native-api.ts +++ b/packages/compiler/src/frontend/ts7/native-api.ts @@ -4,7 +4,41 @@ import { checkerServerName } from "./checker-server.js"; import { attachNativeTs7Process, spawnNativeTs7Wire } from "./native-process.js"; import { Ts7RpcClient } from "./rpc-client.js"; import type { Ts7FileSystem } from "./rpc-filesystem.js"; -import { Ts7Wire } from "./rpc-wire.js"; +import { Ts7Wire, type Ts7WireIo } from "./rpc-wire.js"; + +let sharedServerLost = false; + +/** Whether a connection to the shared checker server failed in this process + * after the server answered: a transfer failed or the server ended the + * connection. The build then runs again with its own checker (cli.ts). */ +export function checkerServerLost(): boolean { + return sharedServerLost; +} + +/** `io`, recording a failed transfer or an ended stream in sharedServerLost. */ +function watched(io: Ts7WireIo): Ts7WireIo { + return { + read: (buffer, offset, length) => { + try { + const count = io.read(buffer, offset, length); + if (count === 0) sharedServerLost = true; + return count; + } catch (error) { + sharedServerLost = true; + throw error; + } + }, + write: (buffer, offset, length) => { + try { + return io.write(buffer, offset, length); + } catch (error) { + sharedServerLost = true; + throw error; + } + }, + close: () => io.close(), + }; +} /** The native driver supplies its installed TypeScript executable path. * Package discovery is separate from starting and owning a server. */ @@ -32,10 +66,15 @@ export function createNativeTs7Api(options: { try { // A server that ended the connection leaves this build to start its own. rpc.requestText("ping", "null"); - return new Ts7Api(rpc, options.fs, timing); } catch { rpc.close(); + return new Ts7Api( + new Ts7RpcClient(spawnNativeTs7Wire(options.executable, options.cwd, timing, libraries)), + options.fs, + timing, + ); } + return new Ts7Api(new Ts7RpcClient(new Ts7Wire(watched(shared))), options.fs, timing); } return new Ts7Api( new Ts7RpcClient(spawnNativeTs7Wire(options.executable, options.cwd, timing, libraries)), diff --git a/packages/compiler/src/native/cli.ts b/packages/compiler/src/native/cli.ts index 4209ecf0e..961fd02a9 100644 --- a/packages/compiler/src/native/cli.ts +++ b/packages/compiler/src/native/cli.ts @@ -13,7 +13,7 @@ import { setDeclarationRoot } from "../frontend/dts-paths.js"; import { setFrontendDisposedHook } from "../frontend/dispose-hook.js"; import { FrontendServices } from "../frontend/services.js"; import { runTypePatchCommand } from "../frontend/type-patches/command.js"; -import { createNativeTs7Api } from "../frontend/ts7/native-api.js"; +import { checkerServerLost, createNativeTs7Api } from "../frontend/ts7/native-api.js"; import { resolveProvenanceSourcesWithParser } from "../frontend/provenance-core.js"; import { setCompilerReleaseVersion } from "../library/sidecar.js"; import { NativeCompiler } from "./compiler.js"; @@ -41,6 +41,30 @@ declare function compilerNativeLock(): void; declare function compilerNativeUnlock(): void; declare function compilerNativeBoundCycleGrowth(): void; +/** Runs a build. When this process's connection to the shared checker server + * failed during it, the command runs again in a new process that checks with + * its own tsc-ts process (SCRIPTC_CHECKER_SERVER=0), and this process exits + * with that one's status: the new process prints the build's output. */ +async function withCheckerFallback(work: () => Promise): Promise { + let result: T; + try { + result = await work(); + } catch (error) { + if (!checkerServerLost()) throw error; + return runWithOwnChecker(); + } + return checkerServerLost() ? runWithOwnChecker() : result; +} + +function runWithOwnChecker(): never { + const run = spawnSync(process.execPath, process.argv.slice(2), { + stdio: "inherit", + env: { ...process.env, SCRIPTC_CHECKER_SERVER: "0" }, + }); + if (run.error) throw new Error(run.error.message); + process.exit(run.status ?? 1); +} + async function main(): Promise { compilerNativeBoundCycleGrowth(); setFrontendDisposedHook(() => compilerNativeCollectCycles()); @@ -100,9 +124,12 @@ async function main(): Promise { return runCli(process.argv.slice(2), { version: () => loadNativeToolchain(manifestPath).compilerVersion, sourceTargetPlatform: () => getCompiler().toolchain.target.platform, - analyze: (entry, options) => runCompilerTask(() => getCompiler().analyzeCached(entry, options)), - compile: (entry, options) => runCompilerTask(() => getCompiler().compile(entry, options)), - compileLibrary: (options) => runCompilerTask(() => getCompiler().compileLibrary(options)), + analyze: (entry, options) => + withCheckerFallback(() => runCompilerTask(() => getCompiler().analyzeCached(entry, options))), + compile: (entry, options) => + withCheckerFallback(() => runCompilerTask(() => getCompiler().compile(entry, options))), + compileLibrary: (options) => + withCheckerFallback(() => runCompilerTask(() => getCompiler().compileLibrary(options))), resolveProvenanceSources: async (entry) => { const toolchain = getCompiler().toolchain; const services = new FrontendServices((options) => diff --git a/packages/compiler/src/native/prepare.ts b/packages/compiler/src/native/prepare.ts index 339a50faa..33c239c64 100644 --- a/packages/compiler/src/native/prepare.ts +++ b/packages/compiler/src/native/prepare.ts @@ -217,6 +217,8 @@ function implementationIdentity(toolchain: NativeToolchain): string { const FRONTEND_INDEPENDENT_VARIABLES = new Set([ "SCRIPTC_CACHE_DIR", "SCRIPTC_CACHE_MAX_MB", + // The shared checker server answers as the build's own checker does. + "SCRIPTC_CHECKER_SERVER", "SCRIPTC_EMIT_WORKERS", "SCRIPTC_EMIT_WORKER_FUNCTIONS", "SCRIPTC_TIMING", diff --git a/tests/fixtures/self-hosting/ts7-native-process.ts b/tests/fixtures/self-hosting/ts7-native-process.ts index 3b04a784c..8ca9ec8e3 100644 --- a/tests/fixtures/self-hosting/ts7-native-process.ts +++ b/tests/fixtures/self-hosting/ts7-native-process.ts @@ -109,13 +109,15 @@ function readLine(connection: NativeTs7Process): string { } // The stand-in server (ts7-process-child.test.c, mode "listen") answers each -// client with its number, its socket and idle time, and its memory limit. +// client with its number, its socket and idle time, its memory limit and its +// working directory. function checkShared(executable: string): void { const first = attachNativeTs7Process("server", executable, "listen"); check(first !== null, "shared server started"); check(readLine(first!) === "client 1", "first client"); - check(readLine(first!).endsWith("/scriptc/server.sock 300"), "listen arguments"); + check(readLine(first!).endsWith("/scriptc/server.sock 600"), "listen arguments"); check(readLine(first!) === "2048", "memory limit"); + check(readLine(first!) === "/", "server in the root directory"); check(attachNativeTs7Process("server", executable, "listen") === null, "shared server in use"); first!.close(); first!.close(); From 1a20467889615bcf1a18d8aa88c02239126a911e Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 08:31:01 -0500 Subject: [PATCH 19/54] Find the counter loop's fast copy inside its entry version The counter loop's remainder by n now versions the loop on n's entry range, so the int32 slots copy ends where the copy for other entry values starts. --- packages/compiler/test/int32-element-array-emission.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/compiler/test/int32-element-array-emission.test.ts b/packages/compiler/test/int32-element-array-emission.test.ts index fccf06973..69b413b55 100644 --- a/packages/compiler/test/int32-element-array-emission.test.ts +++ b/packages/compiler/test/int32-element-array-emission.test.ts @@ -191,8 +191,10 @@ test("a loop over a counter array tests its kind once on entry", async () => { /icmp eq i8 %\w+, 7\n\s+br i1 %\w+, label %kind\.fast\d+, label %kind\.wide/, ); const rest = counter.slice(counter.search(/^kind\.fast\d+:/m)); - // The fast copy is the last one, before the loop's exit. - const fast = rest.slice(0, rest.search(/^loop\.e\d+:/m)); + // The fast copy is the last one, before the loop's exit or, when the + // loop is also versioned on `n`'s entry range, before the copy for other + // entry values. + const fast = rest.slice(0, rest.search(/^(loop\.e|entry\.slow)\d+:/m)); expect(fast).not.toBe(""); expect(fast).not.toMatch(/icmp eq i8 %\w+, 7/); expect(fast).toMatch(/add nsw i64 %\w+, 1/); From f69ff780e5354fff38bacee39ece9c33ee748646 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 08:33:04 -0500 Subject: [PATCH 20/54] Give a stop command that matches the shared checker server The server's command line puts --cwd and --callbacks between --api and --listen, so `pkill -f 'tsc-ts --api --listen'` matched no server that a build started. The docs now give `pkill -f 'tsc-ts --api .*--listen'`, and name the flags instead of writing them as one command. --- docs/content/docs/cli.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/docs/cli.mdx b/docs/content/docs/cli.mdx index fd15a000d..734c07da6 100644 --- a/docs/content/docs/cli.mdx +++ b/docs/content/docs/cli.mdx @@ -246,7 +246,7 @@ With `--sparse`, the profile leaves out the functions the runs never reached. Th
The TypeScript 7 API server that checks the program: tsc-ts (the default), which scriptc includes, or tsgo, from the typescript package scriptc also includes. Both give the same diagnostics and generated code.
SCRIPTC_CHECKER_SERVER
-
On Linux and macOS, builds check with a tsc-ts server that later builds share. The first build in a working directory starts it: a tsc-ts --api --listen process that keeps running in the background with no terminal and keeps the files it parsed, so the next build in that directory loads its program faster. Each user, working directory and scriptc installation gets its own server, and a build connects only to the server of its own scriptc and tsc-ts. The socket is in $XDG_RUNTIME_DIR/scriptc, or /tmp/scriptc-<uid>, a directory only that user can access. A build that finds the server in use by another build, cannot start or reach it, or loses its connection to it, checks with its own tsc-ts process. The server exits after 10 minutes without a build, or after a build when its resident memory is above 2048 MB (TSC_TS_LISTEN_MEMORY sets another limit in megabytes). To stop it sooner, end that process, for example with pkill -f 'tsc-ts --api --listen'. Set to 0 to check with a tsc-ts process of each build's own. When CI is set to a value other than 0 or false, builds use their own process unless this variable is 1. Diagnostics and generated code are the same with and without the server. It applies to the default checker, tsc-ts.
+
On Linux and macOS, builds check with a tsc-ts server that later builds share. The first build in a working directory starts it: a tsc-ts process, started with --api and --listen, that keeps running in the background with no terminal and keeps the files it parsed, so the next build in that directory loads its program faster. Each user, working directory and scriptc installation gets its own server, and a build connects only to the server of its own scriptc and tsc-ts. The socket is in $XDG_RUNTIME_DIR/scriptc, or /tmp/scriptc-<uid>, a directory only that user can access. A build that finds the server in use by another build, cannot start or reach it, or loses its connection to it, checks with its own tsc-ts process. The server exits after 10 minutes without a build, or after a build when its resident memory is above 2048 MB (TSC_TS_LISTEN_MEMORY sets another limit in megabytes). To stop it sooner, end that process, for example with pkill -f 'tsc-ts --api .*--listen'. Set to 0 to check with a tsc-ts process of each build's own. When CI is set to a value other than 0 or false, builds use their own process unless this variable is 1. Diagnostics and generated code are the same with and without the server. It applies to the default checker, tsc-ts.
```console From 5d3a234d075aa364dc5fb773339dafc3726a546f Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 08:34:10 -0500 Subject: [PATCH 21/54] Reuse the supplied files' diagnostics across Node module sets The stored diagnostics of the supplied declaration files are keyed by the whole supplied set, which includes the fallback declarations of each Node module a program imports. A program that imports a different set of modules checked every supplied file again, even when the cache held an answer for the same libs, scriptc declaration files and options without module files. The module fallback files declare only ambient modules, and the TypeScript libs declare only globals and name no module, so adding module files cannot change what checking the libs reports. Nor can it change what checking scriptc's declaration files reports when the set without module files reports nothing in them and no module file declares a module they declare. On a miss, a program of modules now reuses the answer of the set without module files, stored or prebuilt: - for every file of that set when both conditions hold, and checks only its own files and its module files; - for the libs alone otherwise, and checks its own files, scriptc's declaration files and its module files. The answer for the whole set is then stored as before. Programs with scripts, and sets whose module files declare a global, check in full. The content digests behind prebuilt answers are kept per file identity, so a miss hashes each supplied file once. --- packages/compiler/src/frontend/dts-paths.ts | 5 + .../src/frontend/ts7/lib-diagnostics.test.ts | 177 +++++++++++++++- .../src/frontend/ts7/lib-diagnostics.ts | 198 +++++++++++++++++- .../compiler/src/frontend/ts7/program-host.ts | 23 +- 4 files changed, 391 insertions(+), 12 deletions(-) diff --git a/packages/compiler/src/frontend/dts-paths.ts b/packages/compiler/src/frontend/dts-paths.ts index 7e53a2a9b..810d08311 100644 --- a/packages/compiler/src/frontend/dts-paths.ts +++ b/packages/compiler/src/frontend/dts-paths.ts @@ -220,6 +220,11 @@ export function fallbackModulesNamedIn(text: string): string[] { return [...files]; } +/** True for the fallback declarations of a Node module (node-fallback/). */ +export function isFallbackModuleDtsPath(file: string): boolean { + return tsgoPath(file).startsWith(fallbackModuleRoot() + "/"); +} + /** True for the fallback declarations: the core file and the module files. */ export function isFallbackDtsPath(file: string): boolean { const path = tsgoPath(file); diff --git a/packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts b/packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts index 6d0c0651d..0e1d0c9da 100644 --- a/packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts +++ b/packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts @@ -1,6 +1,6 @@ -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { afterAll, afterEach, expect, test } from "vitest"; import type { SourceFile } from "./ast-types.js"; import { @@ -12,7 +12,9 @@ import { setLibDiagnosticsStore, type LibDiagnosticsStore, type PrebuiltLibDiagnostics, + type SuppliedKind, } from "./lib-diagnostics.js"; +import { hostTscTsExecutable } from "./checker-backend.js"; import type { Ts7DiagnosticData } from "./session-schema.generated.js"; import type { Ts7CheckedFileDiagnosticData, Ts7SessionProgram } from "./session.js"; import { closeSourceParser, parseSourceFile } from "./source-parser-node.js"; @@ -382,3 +384,174 @@ test("prebuilt answer files round-trip, and a malformed one is no answer", () => expect(readPrebuiltLibDiagnostics(path)).toBeNull(); expect(readPrebuiltLibDiagnostics(join(dir, "missing.json"))).toBeNull(); }); + +// Node module fallback files: ambient modules only, or one that also declares a global. +const moduleA = join(dir, "node-fallback-a.d.ts"); +const moduleB = join(dir, "node-fallback-b.d.ts"); +const moduleGlobal = join(dir, "node-fallback-global.d.ts"); +const moduleTexts: Record = { + [moduleA]: 'declare module "node:a" {\n export const a: number;\n}\n', + [moduleB]: 'declare module "node:b" {\n export const b: string;\n}\n', + [moduleGlobal]: + 'declare module "node:g" {\n global {\n var g: number;\n }\n export {};\n}\n', +}; +for (const [name, text] of Object.entries(moduleTexts)) writeFileSync(name, text); +const kind = (name: string): SuppliedKind => + name === lib ? "library" : name.includes("node-fallback-") ? "module" : "declarations"; +const withModules = (...modules: string[]): string[] => [lib, fallback, ...modules, main]; +const suppliedWithModules = (name: string): boolean => name !== main; +const moduleSources = files({ ...moduleTexts, [main]: "export const n: number = 'x';\n" }); + +test("a set with module files reuses the libs' answers of the set without them", () => { + const store = memoryStore(); + setLibDiagnosticsStore(store, "checker"); + const semantic = new Map([ + [lib, [diagnostic(lib, 1)]], + [main, [diagnostic(main, 5)]], + ]); + // Without an answer for the set without module files, a set with them checks in full. + const first = sessionProgram({ semantic }); + reusedLibDiagnostics(first, withModules(moduleA), suppliedWithModules, moduleSources, kind); + expect(first.asked[0]).toBe("getGlobalDiagnostics"); + const base = sessionProgram({ semantic }); + reusedLibDiagnostics(base, withModules(), suppliedWithModules, moduleSources, kind); + expect(base.asked[0]).toBe("getGlobalDiagnostics"); + // That answer reports a diagnostic in a lib, so a set with another module file reuses the + // libs' answers and checks the program, scriptc's declarations and that file. + const second = sessionProgram({ semantic }); + const composed = reusedLibDiagnostics( + second, + withModules(moduleB), + suppliedWithModules, + moduleSources, + kind, + ); + expect(second.asked).toEqual([`checked ${main},${fallback},${moduleB}`]); + expect(composed).toEqual({ semantic: [diagnostic(lib, 1), diagnostic(main, 5)], global: [] }); + // Its own answer is stored, so the same set then checks only the program. + const third = sessionProgram({ semantic }); + reusedLibDiagnostics(third, withModules(moduleB), suppliedWithModules, moduleSources, kind); + expect(third.asked).toEqual([`checked ${main}`]); +}); + +test("a clean answer of the set without module files stands in for all of its files", () => { + const semantic = new Map([[main, []]]); + const store = memoryStore(); + setLibDiagnosticsStore(store, "checker"); + reusedLibDiagnostics( + sessionProgram({ semantic }), + withModules(), + suppliedWithModules, + moduleSources, + kind, + ); + const fromBase = sessionProgram({ semantic }); + reusedLibDiagnostics(fromBase, withModules(moduleA), suppliedWithModules, moduleSources, kind); + expect(fromBase.asked).toEqual([`checked ${main},${moduleA}`]); + // So does a prebuilt answer for it. + const keys = new Set(); + recordPrebuiltLibDiagnostics({ checker: "1.0.0", record: (key) => keys.add(key) }); + setLibDiagnosticsStore(memoryStore(), "build-time checker"); + reusedLibDiagnostics( + sessionProgram({ semantic }), + withModules(), + suppliedWithModules, + moduleSources, + kind, + ); + recordPrebuiltLibDiagnostics(null); + expect(keys.size).toBe(1); + setLibDiagnosticsStore(memoryStore(), "installed checker", () => ({ checker: "1.0.0", keys })); + const prebuilt = sessionProgram({ semantic }); + reusedLibDiagnostics( + prebuilt, + withModules(moduleA, moduleB), + suppliedWithModules, + moduleSources, + kind, + ); + expect(prebuilt.asked).toEqual([`checked ${main},${moduleA},${moduleB}`]); +}); + +test("scriptc's declarations are checked again when they report something or declare a module file's module", () => { + const declared = join(dir, "scriptc-declared.d.ts"); + const moduleX = join(dir, "node-fallback-x.d.ts"); + const texts = { + [declared]: 'declare module "node:x" {\n export const x: number;\n}\n', + [moduleX]: 'declare module "node:x" {\n export const y: number;\n}\n', + }; + for (const [name, text] of Object.entries(texts)) writeFileSync(name, text); + const sources = files({ ...moduleTexts, ...texts, [main]: "export const n: number = 'x';\n" }); + const semantic = new Map([[main, []]]); + setLibDiagnosticsStore(memoryStore(), "checker"); + const base = [lib, fallback, declared, main]; + const set = [lib, fallback, declared, moduleX, main]; + reusedLibDiagnostics(sessionProgram({ semantic }), base, suppliedWithModules, sources, kind); + const merged = sessionProgram({ semantic }); + reusedLibDiagnostics(merged, set, suppliedWithModules, sources, kind); + expect(merged.asked).toEqual([`checked ${main},${fallback},${declared},${moduleX}`]); + // With a diagnostic in scriptc's declarations, only the libs' answers are reused. + const reporting = new Map([ + [main, []], + [fallback, [diagnostic(fallback, 3)]], + ]); + setLibDiagnosticsStore(memoryStore(), "checker"); + reusedLibDiagnostics( + sessionProgram({ semantic: reporting }), + withModules(), + suppliedWithModules, + moduleSources, + kind, + ); + const again = sessionProgram({ semantic: reporting }); + const answer = reusedLibDiagnostics( + again, + withModules(moduleA), + suppliedWithModules, + moduleSources, + kind, + ); + expect(again.asked).toEqual([`checked ${main},${fallback},${moduleA}`]); + expect(answer?.semantic).toEqual([diagnostic(fallback, 3)]); +}); + +test("module files that declare something global, and programs with scripts, check in full", () => { + const semantic = new Map([[main, []]]); + const store = memoryStore(); + setLibDiagnosticsStore(store, "checker"); + reusedLibDiagnostics( + sessionProgram({ semantic }), + withModules(), + suppliedWithModules, + moduleSources, + kind, + ); + const global = sessionProgram({ semantic }); + reusedLibDiagnostics(global, withModules(moduleGlobal), suppliedWithModules, moduleSources, kind); + expect(global.asked[0]).toBe("getGlobalDiagnostics"); + const other = sessionProgram({ semantic }); + reusedLibDiagnostics( + other, + withModules(moduleGlobal, moduleA), + suppliedWithModules, + moduleSources, + kind, + ); + expect(other.asked[0]).toBe("getGlobalDiagnostics"); + const script = sessionProgram({ semantic }); + const scriptSources = files({ ...moduleTexts, [main]: "const total = 1;\n" }); + reusedLibDiagnostics(script, withModules(moduleA), suppliedWithModules, scriptSources, kind); + expect(script.asked[0]).toBe("getGlobalDiagnostics"); +}); + +test("the TypeScript libs declare no ambient module and import none, so module files cannot reach them", () => { + const libDir = dirname(hostTscTsExecutable()); + const libFiles = readdirSync(libDir).filter((name) => /^lib\..*\.d\.ts$/.test(name)); + expect(libFiles.length).toBeGreaterThan(50); + for (const name of libFiles) { + const text = readFileSync(join(libDir, name), "utf8"); + expect(text, name).not.toMatch( + /\bmodule\s+["']|\bimport\s*\(\s*["']|\bfrom\s+["']|\brequire\s*\(/, + ); + } +}); diff --git a/packages/compiler/src/frontend/ts7/lib-diagnostics.ts b/packages/compiler/src/frontend/ts7/lib-diagnostics.ts index ca1f71fc2..b87ca883b 100644 --- a/packages/compiler/src/frontend/ts7/lib-diagnostics.ts +++ b/packages/compiler/src/frontend/ts7/lib-diagnostics.ts @@ -24,7 +24,20 @@ * the distribution was built. A build whose store has no answer uses a * prebuilt one when the checker is the distribution's own and the files' * content key is among them, and stores it, so builds with an empty cache - * skip the check of the supplied files too. */ + * skip the check of the supplied files too. + * + * The supplied set depends on the Node modules a program imports: each one + * adds its fallback declarations. Those files declare ambient modules and + * nothing global, and the TypeScript libs declare only globals and name no + * module, so adding module files cannot change what checking the libs + * reports. Nor can it change what checking scriptc's declaration files + * reports when the set without module files reports nothing in them (every + * name they use resolves among them and the libs) and no module file + * declares a module they declare. A program whose set has no stored answer + * therefore reuses the answers of the set without module files, stored or + * prebuilt: for every file of that set when both hold, for the libs alone + * otherwise. The server then checks the program's other files and the + * supplied files not reused, and the answer for the whole set is stored. */ import { createHash } from "node:crypto"; import { readFileSync, statSync } from "node:fs"; import { basename, dirname, join } from "node:path"; @@ -48,6 +61,11 @@ export interface PrebuiltLibDiagnostics { keys: ReadonlySet; } +/** A supplied file's part in reuse across sets: "library" for the + * TypeScript libs, "module" for a Node module's fallback declarations, and + * "declarations" for scriptc's other declaration files. */ +export type SuppliedKind = "library" | "module" | "declarations"; + const SCHEMA = "scriptc.lib-diagnostics.v2"; const PREBUILT_SCHEMA = "scriptc.lib-diagnostics-prebuilt.v1"; @@ -159,14 +177,16 @@ export function checkerExecutableIdentity(path: string): string { * them, using stored diagnostics for the supplied declaration files; null * when they cannot be reused for this program (then the caller asks for * them as before). `supplied` tells whether a program file is one of the - * supplied declaration files. */ + * supplied declaration files, and `kind` which kind of supplied file it is + * (by default scriptc's declarations, which reuse nothing across sets). */ export function reusedLibDiagnostics( program: Ts7SessionProgram, fileNames: readonly string[], supplied: (fileName: string) => boolean, sourceFile: (fileName: string) => SourceFile | undefined, + kind: (fileName: string) => SuppliedKind = () => "declarations", ): { semantic: Ts7DiagnosticData[]; global: Ts7DiagnosticData[] } | null { - return reuse(program, fileNames, supplied, sourceFile, () => true, false); + return reuse(program, fileNames, supplied, sourceFile, () => true, false, kind); } /** reusedLibDiagnostics for a caller that keeps only the diagnostics of the @@ -180,7 +200,15 @@ export function reusedKeptLibDiagnostics( sourceFile: (fileName: string) => SourceFile | undefined, kept: ReadonlySet, ): { semantic: Ts7DiagnosticData[]; global: Ts7DiagnosticData[] } | null { - return reuse(program, fileNames, supplied, sourceFile, (name) => kept.has(name), true); + return reuse( + program, + fileNames, + supplied, + sourceFile, + (name) => kept.has(name), + true, + () => "declarations", + ); } function reuse( @@ -190,9 +218,11 @@ function reuse( sourceFile: (fileName: string) => SourceFile | undefined, keep: (fileName: string) => boolean, storedOnly: boolean, + kind: (fileName: string) => SuppliedKind, ): { semantic: Ts7DiagnosticData[]; global: Ts7DiagnosticData[] } | null { if (configured === null || process.env["SCRIPTC_LIB_DIAGNOSTICS"] === "0") return null; const { store, checker } = configured; + identities.clear(); const libs: string[] = []; const own: string[] = []; for (const name of fileNames) (supplied(name) ? libs : own).push(name); @@ -240,6 +270,31 @@ function reuse( return { semantic: inFileOrder(byFile), global: checked.global }; } if (storedOnly) return null; + // Another set's answers for the libs (see the header), for a program of + // modules: its scripts' globals would need every supplied file's names. + const kinds = new Map(); + if (stored === null && scripts.length === 0) for (const name of libs) kinds.set(name, kind(name)); + const kindOf = (name: string): SuppliedKind => kinds.get(name) ?? "declarations"; + const libraries = libs.filter((name) => kindOf(name) === "library"); + const modules = libs.filter((name) => kindOf(name) === "module"); + const shared = + modules.length > 0 + ? sharedAnswers(store, checker, options, libs, libraries, modules, sourceFile, kindOf) + : null; + if (shared !== null) { + const checkedNames = [...own, ...libs.filter((name) => !shared.files.has(name))]; + const checked = program.checkedFileDiagnosticData(checkedNames); + if (checked !== null) { + const byFile = new Map(shared.files); + checkedNames.forEach((name, index) => byFile.set(name, checked.files[index] ?? [])); + if (checked.global.length === 0) { + const files: Record = {}; + for (const name of libs) files[name] = byFile.get(name) ?? []; + writeEntry(store, key, null, files); + } + return { semantic: inFileOrder(byFile), global: checked.global }; + } + } // The full check, as before. Its answers for the supplied files are stored // when it reports no global diagnostics and the program's scripts declare // none of the supplied files' globals (so nothing of the program reached @@ -424,8 +479,7 @@ function entryKey(checker: string, options: unknown, libs: readonly string[]): s const files: unknown[] = []; for (const name of libs) { try { - const info = statSync(name); - files.push([name, info.size, info.mtimeMs, info.ino]); + files.push(fileIdentity(name)); } catch { return null; } @@ -435,6 +489,121 @@ function entryKey(checker: string, options: unknown, libs: readonly string[]): s .digest("hex"); } +/** The files' identities (path, size, modification time, inode), kept for + * one reuse: the keys of a set and of the set without its module files + * share most of their files. */ +const identities = new Map(); + +function fileIdentity(name: string): [string, number, number, number] { + let identity = identities.get(name); + if (identity === undefined) { + const info = statSync(name); + identity = [name, info.size, info.mtimeMs, info.ino]; + identities.set(name, identity); + } + return identity; +} + +/** Answers of the set without module files (see the header), stored or + * prebuilt, when the program's module files declare nothing global: for + * every file of that set when its answer reports no diagnostics and the + * module files declare no module its declaration files declare, otherwise + * for the libs alone; null when there is none. */ +function sharedAnswers( + store: LibDiagnosticsStore, + checker: string, + options: unknown, + libs: readonly string[], + libraries: readonly string[], + modules: readonly string[], + sourceFile: (fileName: string) => SourceFile | undefined, + kind: (fileName: string) => SuppliedKind, +): { files: Map } | null { + if (libraries.length === 0) return null; + const base = libs.filter((name) => kind(name) !== "module"); + const key = entryKey(checker, options, base); + const entry = key === null ? null : readEntry(store, key, base); + const prebuilt = configured?.prebuilt?.() ?? null; + const prebuiltClean = (): boolean => { + if (prebuilt === null || prebuilt.keys.size === 0) return false; + const content = contentKey(prebuilt.checker, options, base); + return content !== null && prebuilt.keys.has(content); + }; + const clean = + entry !== null ? [...entry.files.values()].every((list) => list.length === 0) : prebuiltClean(); + if (clean) { + const declared = moduleNames(modules, sourceFile); + if (declared === null) return null; + const own = declaredModuleNames(base.filter((name) => kind(name) === "declarations")); + const reused = own !== null && ![...declared].some((name) => own.has(name)) ? base : libraries; + const files = new Map(); + for (const name of reused) files.set(name, []); + return { files }; + } + if (entry === null || moduleNames(modules, sourceFile) === null) return null; + return { + files: new Map( + libraries.map((name): [string, Ts7DiagnosticData[]] => [name, entry.files.get(name)!]), + ), + }; +} + +/** The ambient modules the module files declare, when each one declares + * only ambient modules (`declare module "x"`) with no `global` augmentation, + * or is a module that declares nothing global: then it adds nothing the libs + * can see. Null otherwise. */ +function moduleNames( + modules: readonly string[], + sourceFile: (fileName: string) => SourceFile | undefined, +): Set | null { + const names = new Set(); + for (const name of modules) { + const file = sourceFile(name); + if (file === undefined) return null; + if (file.externalModuleIndicator !== undefined) { + if (!declaresNothingGlobal(file)) return null; + continue; + } + for (const statement of file.statements.nodes) { + if ( + statement.kind !== SyntaxKind.ModuleDeclaration || + statement.name?.kind !== SyntaxKind.StringLiteral + ) + return null; + names.add(statement.name.text ?? ""); + for (const inner of statement.body?.statements?.nodes ?? []) + if ( + inner.kind === SyntaxKind.ModuleDeclaration && + inner.name?.kind === SyntaxKind.Identifier && + inner.name.text === "global" + ) + return null; + } + } + return names; +} + +/** The module names scriptc's declaration files name in a module + * declaration (`module "x"` or `module 'x'`, in their text, so also in + * comments); null when one of them cannot be read or names a pattern + * (`"*"`), which a module file's module could also match. */ +function declaredModuleNames(files: readonly string[]): Set | null { + const names = new Set(); + for (const name of files) { + let text: string; + try { + text = readFileSync(name, "utf8"); + } catch { + return null; + } + for (const match of text.matchAll(/\bmodule\s+(["'])(.*?)\1/g)) { + if (match[2]!.includes("*")) return null; + names.add(match[2]!); + } + } + return names; +} + /** The supplied files' answer from the distribution's prebuilt answers, * stored under `key` for later builds: every file without diagnostics, when * they hold the files' content key; null otherwise. */ @@ -461,7 +630,7 @@ function contentKey(checker: string, options: unknown, libs: readonly string[]): const files: [string, string][] = []; for (const name of libs) { try { - files.push([basename(name), createHash("sha256").update(readFileSync(name)).digest("hex")]); + files.push([basename(name), contentDigest(name)]); } catch { return null; } @@ -471,6 +640,21 @@ function contentKey(checker: string, options: unknown, libs: readonly string[]): .digest("hex"); } +/** The digests of files' contents by path and file identity: a build that + * misses its answer computes the content keys of its set and of the set + * without module files, which share most of their files. */ +const contentDigests = new Map(); + +function contentDigest(name: string): string { + const identity = JSON.stringify(fileIdentity(name)); + let digest = contentDigests.get(identity); + if (digest === undefined) { + digest = createHash("sha256").update(readFileSync(name)).digest("hex"); + contentDigests.set(identity, digest); + } + return digest; +} + function readEntry( store: LibDiagnosticsStore, key: string, diff --git a/packages/compiler/src/frontend/ts7/program-host.ts b/packages/compiler/src/frontend/ts7/program-host.ts index 2c8ff873f..070f178ac 100644 --- a/packages/compiler/src/frontend/ts7/program-host.ts +++ b/packages/compiler/src/frontend/ts7/program-host.ts @@ -47,8 +47,18 @@ import { } from "../input-tracker.js"; import { packageShimDeclarationText } from "../package-shims.js"; import { typePatchedText, typePatchFileExists } from "../type-patches/layer.js"; -import { ambientDtsPath, isFallbackDtsPath, overridesDtsPath } from "../dts-paths.js"; -import { inFileOrder, reusedKeptLibDiagnostics, reusedLibDiagnostics } from "./lib-diagnostics.js"; +import { + ambientDtsPath, + isFallbackDtsPath, + isFallbackModuleDtsPath, + overridesDtsPath, +} from "../dts-paths.js"; +import { + inFileOrder, + reusedKeptLibDiagnostics, + reusedLibDiagnostics, + type SuppliedKind, +} from "./lib-diagnostics.js"; import { ts7Diagnostic } from "./diagnostic.js"; import type { Ts7DiagnosticData } from "./session-schema.generated.js"; @@ -446,8 +456,15 @@ export class Ts7Program { isFallbackDtsPath(name) || program.getSourceFileMetadataByPath(name)?.isDefaultLibrary === true; const sourceFile = (name: string): SourceFile | undefined => this.getSourceFile(name); + // By path: a supplied file that is not one of scriptc's is a default library. + const kind = (name: string): SuppliedKind => + isFallbackModuleDtsPath(name) + ? "module" + : declarations.has(tsgoPath(name)) || isFallbackDtsPath(name) + ? "declarations" + : "library"; return kept === null - ? reusedLibDiagnostics(program, names, supplied, sourceFile) + ? reusedLibDiagnostics(program, names, supplied, sourceFile, kind) : reusedKeptLibDiagnostics(program, names, supplied, sourceFile, kept); } From 6a30f02a69ab3a2421e0355ff8ab35f84f926da8 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 26 Oct 2026 20:43:11 -0700 Subject: [PATCH 22/54] Answer cross-layout record casts with views of the original record A checked conversion of a capsule of record A into record type B, when A has all of B's fields with the same types in any order, now answers a view: a B object whose two trailing words hold A and A's offsets for B's fields. Every field address of a B value tests the first word (recordViewFieldPtr, the record clone and the JSON walker), so reads and writes through either name reach the same storage. The view's record-source entry owns A's capsule: boxing the view answers that capsule, so identity holds, B's reflective surfaces and spreads see the whole object, and the entry dies with the view, which releases A's capsule. The capsule finds its views again through borrowed cast-list entries. Only the target types of such conversions carry the words and the test. They are planned from the IR before emission (record-views.ts): record types a checked conversion can produce for which another record type in the module has all their fields. Programs without checked conversions are unchanged. This replaces the prefix pointer pass, whose narrower release could not free the source's extra fields. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/backend/llvm/dyn.ts | 128 ++++++------- packages/compiler/src/backend/llvm/emitter.ts | 30 ++- .../compiler/src/backend/llvm/record-views.ts | 171 +++++++++++++++++ packages/compiler/src/backend/llvm/shapes.ts | 34 +++- .../src/backend/llvm/stack-objects.ts | 3 + packages/compiler/src/backend/llvm/walkers.ts | 24 ++- .../tests/corpus/record-layout-views.ts.json | 6 + .../corpus/record-view-extras-release.ts.json | 6 + packages/runtime/src/scr_dyn.h | 4 +- packages/runtime/src/scr_json.c | 181 ++++++++++-------- packages/runtime/src/scr_region.c | 2 +- tests/corpus/record-layout-views.ts | 26 +++ tests/corpus/record-view-extras-release.ts | 44 +++++ .../corpus-costs/record-layout-views.ts.json | 1 + .../record-view-extras-release.ts.json | 1 + 15 files changed, 502 insertions(+), 159 deletions(-) create mode 100644 packages/compiler/src/backend/llvm/record-views.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/record-layout-views.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/record-view-extras-release.ts.json create mode 100644 tests/corpus/record-layout-views.ts create mode 100644 tests/corpus/record-view-extras-release.ts create mode 100644 tests/harness/corpus-costs/record-layout-views.ts.json create mode 100644 tests/harness/corpus-costs/record-view-extras-release.ts.json diff --git a/packages/compiler/src/backend/llvm/dyn.ts b/packages/compiler/src/backend/llvm/dyn.ts index 762bc52f4..5bc71a331 100644 --- a/packages/compiler/src/backend/llvm/dyn.ts +++ b/packages/compiler/src/backend/llvm/dyn.ts @@ -1,5 +1,6 @@ import { emitPerKeyOrder } from "./key-orders.js"; -import { rcMembers, typedRefConstructor } from "./shapes.js"; +import { typedRefConstructor } from "./shapes.js"; +import { recordViewWords } from "./record-views.js"; import { bindLiveArrayOps } from "./expr-stream-bridges.js"; import { BYTES_ELEM_NUM, @@ -45,7 +46,6 @@ import { DYN_HANDLE_KINDS, isDirentShape, KEY_ORDER_FIELD, - shapeHasAccessorSlots, shapeKeyOrders, isDynTypedRefType, isRefCounted, @@ -261,7 +261,7 @@ export class LlDyn { /** Emitted function definitions, in interning order. */ readonly defs: string[] = []; - private readonly prefixSources = new Map(); + private readonly recordViewTables = new Map(); constructor(private readonly host: DynHost) {} @@ -1161,51 +1161,51 @@ export class LlDyn { ); } - /** The record shapes whose values a pointer of shape `targetId` may - * reference unchanged: their struct begins with exactly the target's - * fields (same names, types and representations, so the same offsets), - * and everything the target's release, teardown and collector support - * would get wrong on them is absent. The extra fields hold no - * references (the target's release frees the block without visiting - * them; frees are size-class agnostic), neither shape carries a shape - * word, a cycle header, an overflow map, key order variants or accessor - * slots, and both freeze alike. A record that keeps undeclared source - * properties is refused at run time (scr_dyn_typed_ref_prefix_view). */ - private prefixViewSources(targetId: string): string[] { - const memo = this.prefixSources.get(targetId); - if (memo) return memo; + /** `scr_dyn_record_view(d, sources, n, key, key_len, new, words)` for + * view target `targetId`: the view (+1) of the record a capsule `%d` + * holds when its type is one of the target's sources, else NULL. The + * sources table lists each source's type key and the byte offsets of + * the target's fields in it. */ + private recordViewCall(B: BlockBuilder, targetId: string): string { const host = this.host; - const plain = (shape: IrRecordShape): boolean => - !shape.tuple && - !shape.indexValue && - !isDirentShape(shape) && - !shapeHasAccessorSlots(shape) && - shapeKeyOrders(shape).length === 1 && - !shape.fields.some((field) => field.name === KEY_ORDER_FIELD) && - host.nullableUnions.shapeNumber(shape.id) === 0 && - !host.tracedShapes.has(`record:${shape.id}`); - const target = host.recordsById.get(targetId); - const out: string[] = []; - if (target && plain(target) && target.fields.length > 0) { - const base = host.nullableUnions.fieldBase(targetId); - for (const shape of host.recordsById.values()) { - if (shape.id === targetId || shape.fields.length <= target.fields.length) continue; - if (!plain(shape) || shape.freezable !== target.freezable) continue; - if (host.nullableUnions.fieldBase(shape.id) !== base) continue; - const prefix = target.fields.every( - (field, i) => - shape.fields[i]!.name === field.name && - typeKey(shape.fields[i]!.type) === typeKey(field.type), + let table = this.recordViewTables.get(targetId); + const sources = host.recordViews!.sources(targetId); + const target = host.recordsById.get(targetId)!; + if (table === undefined) { + table = `@sc_rv_${this.recordViewTables.size}`; + this.recordViewTables.set(targetId, table); + const S = this.S; + const entries = sources.map((sourceId, k) => { + const source = host.recordsById.get(sourceId)!; + const struct = mangleRecordStruct(sourceId); + const base = host.nullableUnions.fieldBase(sourceId); + const offsets = target.fields.map((field) => { + const index = source.fields.findIndex((f) => f.name === field.name) + base; + return `${S} ptrtoint (ptr getelementptr (%${struct}, ptr null, i32 0, i32 ${index}) to ${S})`; + }); + const offsetTable = `${table}_${k}`; + this.defs.push( + `${offsetTable} = internal constant [${offsets.length} x ${S}] [${offsets.join(", ")}]`, ); - if (!prefix) continue; - const extra = rcMembers(shape, host.inlineUnions, base).some( - (member) => member.index >= target.fields.length + base && isRefCounted(member.type), - ); - if (!extra) out.push(shape.id); - } + const key = typeKey({ kind: "record", shapeId: sourceId }); + return `{ ptr, ${S}, ptr } { ptr ${host.cstr(key)}, ${S} ${Buffer.byteLength(key, "utf8")}, ptr ${offsetTable} }`; + }); + this.defs.push( + `${table} = internal constant [${entries.length} x { ptr, ${S}, ptr }] [${entries.join(", ")}]`, + ``, + ); } - this.prefixSources.set(targetId, out); - return out; + const key = typeKey({ kind: "record", shapeId: targetId }); + const words = recordViewWords(target.fields.length, host.nullableUnions.fieldBase(targetId)); + const struct = mangleRecordStruct(targetId); + host.declare( + `declare ptr @scr_dyn_record_view(ptr, ptr, ${this.S}, ptr, ${this.S}, ptr, ${this.S})`, + ); + const view = B.tmp(); + B.line( + `${view} = call ptr @scr_dyn_record_view(ptr %d, ptr ${table}, ${this.S} ${sources.length}, ptr ${host.cstr(key)}, ${this.S} ${Buffer.byteLength(key, "utf8")}, ptr @${mangleRecordNew(targetId)}, ${this.S} ptrtoint (ptr getelementptr (%${struct}, ptr null, i32 0, i32 ${words.record}) to ${this.S}))`, + ); + return view; } private typedRefMatchesAny(B: BlockBuilder, types: readonly IrType[]): string { @@ -2222,31 +2222,19 @@ export class LlDyn { B.line(`${ref} = call ptr @scr_dyn_typed_ref_unbox(ptr %d)`); B.terminate(`ret ptr ${ref}`); B.startBlock(lNext); - if (t.kind === "record") { - // A record whose storage begins with exactly this type's storage - // is this type's value too: the conversion passes the pointer, so - // both names keep one object (decision D3). - const sources = this.prefixViewSources(t.shapeId); - if (sources.length > 0) { - const viewed = this.typedRefMatchesAny( - B, - sources.map((shapeId) => ({ kind: "record", shapeId })), - ); - const lView = B.newLabel("dc.prefix"); - const lOther = B.newLabel("dc.noprefix"); - B.condBr(viewed, lView, lOther); - B.startBlock(lView); - host.declare(`declare ptr @scr_dyn_typed_ref_prefix_view(ptr)`); - const view = B.tmp(); - B.line(`${view} = call ptr @scr_dyn_typed_ref_prefix_view(ptr %d)`); - const passed = B.tmp(); - B.line(`${passed} = icmp ne ptr ${view}, null`); - const lPass = B.newLabel("dc.prefix.pass"); - B.condBr(passed, lPass, lOther); - B.startBlock(lPass); - B.terminate(`ret ptr ${view}`); - B.startBlock(lOther); - } + if (t.kind === "record" && host.recordViews?.targets.has(t.shapeId) === true) { + // A capsule of a record with all of this type's fields converts + // to a view of that record: both names keep one object + // (record-views.ts, decision D3). + const view = this.recordViewCall(B, t.shapeId); + const viewed = B.tmp(); + B.line(`${viewed} = icmp ne ptr ${view}, null`); + const lView = B.newLabel("dc.view"); + const lOther = B.newLabel("dc.noview"); + B.condBr(viewed, lView, lOther); + B.startBlock(lView); + B.terminate(`ret ptr ${view}`); + B.startBlock(lOther); } // A constructor-function stream (scr_dyn_builtin_stream.c) carries // its runtime stream. diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index e6c1b98c5..f24e3a862 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -357,6 +357,7 @@ import { import { children, type Node } from "./region-ir.js"; import { LlDyn, recordDynKeepsIdentity, type DynHost } from "./dyn.js"; import { VirtualBorrows } from "./virtual-borrows.js"; +import { planRecordViews, recordViewFieldPtr, recordViewWords } from "./record-views.js"; import { LlvmUnsupportedError } from "./unsupported.js"; import { LlWalkers } from "./walkers.js"; import { isObjectArm, NULLABLE_ABSENT, NULLABLE_NULL, NullableUnions } from "./nullable-unions.js"; @@ -2523,6 +2524,13 @@ export class LlEmitter { this.shapeHost.arenasClosed = this.arenas.allClosed; } } + this.shapeHost.recordViews = planRecordViews(mod, { + traced: (id) => this.tracedShapes.has(`record:${id}`), + excluded: (id) => + this.shapeHost.arenaShapes?.has(id) === true || + this.publishedTypes?.recordGuarded(id) === true || + mod.workers === true, + }); for (const [name, rec] of RUNTIME_ERROR_CLASSES) { const meta = this.classMeta.get(name); if (!meta) break; // hand-written IR without the builtin defs: no stamps @@ -4844,10 +4852,24 @@ export class LlEmitter { throw new InternalCompilerError( `llvm emitter bug: unknown field ${field} on shape ${shapeId}`, ); - const p = this.B.tmp(); - this.B.line( - `${p} = getelementptr inbounds %${mangleRecordStruct(shapeId)}, ptr ${objName}, i64 0, i32 ${idx + this.nullableUnions.fieldBase(shapeId)}`, - ); + const base = this.nullableUnions.fieldBase(shapeId); + let p: string; + if (this.shapeHost.recordViews?.targets.has(shapeId) === true) { + p = recordViewFieldPtr( + this.B, + this.sizeType, + mangleRecordStruct(shapeId), + objName, + idx + base, + idx, + recordViewWords(shape.fields.length, base), + ); + } else { + p = this.B.tmp(); + this.B.line( + `${p} = getelementptr inbounds %${mangleRecordStruct(shapeId)}, ptr ${objName}, i64 0, i32 ${idx + base}`, + ); + } const type = shape.fields[idx]!.type; return { ptr: p, type, inline: this.inlineUnions.record(type) }; } diff --git a/packages/compiler/src/backend/llvm/record-views.ts b/packages/compiler/src/backend/llvm/record-views.ts new file mode 100644 index 000000000..41df2cb71 --- /dev/null +++ b/packages/compiler/src/backend/llvm/record-views.ts @@ -0,0 +1,171 @@ +/* Record views: one object across record layouts (decision D3). + * + * A checked conversion of a capsule of a native record A into another + * record type B whose fields A also has (same names and types, any + * offsets) answers a view: a B-shaped object whose fields live in A. The + * view's storage is B's struct plus two trailing words, the A record + * (borrowed: the view's record-source entry owns A's capsule, which owns + * A) and a table of A's byte offsets for B's fields. Every field address + * of a B value tests the first word (recordViewFieldPtr), so reads and + * writes through either name reach the same storage, and boxing a view + * answers A's capsule (scr_dyn_record_view), so identity holds too. + * + * Only the target types of such conversions carry the words and the + * test. They are found from the IR, before any code is emitted: B is a + * record type a checked conversion can produce (the closure of dynCheck + * targets and of the types live capsules expose) and some other record + * type in the module has all of B's fields. A program without checked + * conversions has none, so its code is unchanged. */ +import { + isDirentShape, + KEY_ORDER_FIELD, + shapeHasAccessorSlots, + shapeKeyOrders, + typeKey, + type IrModule, + type IrRecordShape, + type IrType, +} from "../../ir/ir.js"; +import { everyStmtList } from "../../ir/traverse.js"; +import type { BlockBuilder } from "./blocks.js"; + +export interface RecordViews { + /** Shape ids whose values may be views. */ + readonly targets: ReadonlySet; + /** The record shapes whose capsules convert to a view of `target`. */ + sources(target: string): readonly string[]; +} + +export const NO_RECORD_VIEWS: RecordViews = { targets: new Set(), sources: () => [] }; + +interface ViewOptions { + readonly traced: (shapeId: string) => boolean; + readonly excluded: (shapeId: string) => boolean; +} + +function plainShape(shape: IrRecordShape): boolean { + return ( + !shape.tuple && + !shape.indexValue && + shape.fields.length > 0 && + !isDirentShape(shape) && + !shapeHasAccessorSlots(shape) && + shapeKeyOrders(shape).length === 1 && + !shape.fields.some((field) => field.name === KEY_ORDER_FIELD || field.name.startsWith("%")) + ); +} + +/** Whether every field of `target` is a field of `source` with the same + * type (the view reads A's field with B's representation). */ +function covers(source: IrRecordShape, target: IrRecordShape): boolean { + const fields = new Map(source.fields.map((field) => [field.name, typeKey(field.type)])); + return target.fields.every((field) => fields.get(field.name) === typeKey(field.type)); +} + +export function planRecordViews(mod: IrModule, options: ViewOptions): RecordViews { + const shapes = new Map((mod.records ?? []).map((shape) => [shape.id, shape])); + const unions = new Map((mod.unions ?? []).map((union) => [union.id, union])); + const reachable = new Set(); + const seen = new Set(); + const visit = (t: IrType): void => { + const key = typeKey(t); + if (seen.has(key)) return; + seen.add(key); + switch (t.kind) { + case "record": { + reachable.add(t.shapeId); + for (const field of shapes.get(t.shapeId)?.fields ?? []) visit(field.type); + return; + } + case "array": + visit(t.elem); + return; + case "union": + for (const arm of unions.get(t.unionId)?.arms ?? []) visit(arm); + return; + default: + return; + } + }; + for (const fn of mod.functions) + everyStmtList(fn.body, { + stmt: () => true, + expr: (e) => { + if (e.kind === "dynCheck") visit(e.type); + else if (e.kind === "dynFrom" && e.liveRef === true) visit(e.value.type); + return true; + }, + }); + if (reachable.size === 0) return NO_RECORD_VIEWS; + const targets = new Set(); + const sources = new Map(); + for (const id of reachable) { + const target = shapes.get(id); + if (!target || !plainShape(target) || target.freezable === true) continue; + if (options.traced(id) || options.excluded(id)) continue; + const found: string[] = []; + for (const source of shapes.values()) { + if (source.id === id || !reachable.has(source.id) || !plainShape(source)) continue; + if (covers(source, target)) found.push(source.id); + } + if (found.length === 0) continue; + targets.add(id); + sources.set(id, found); + } + if (targets.size === 0) return NO_RECORD_VIEWS; + return { targets, sources: (target) => sources.get(target) ?? [] }; +} + +/** The struct indices of a view target's two trailing words. */ +export function recordViewWords( + fieldCount: number, + base: number, +): { record: number; offsets: number } { + return { record: fieldCount + base, offsets: fieldCount + base + 1 }; +} + +/** The address of field `position` (declaration index) of the record + * `obj` of view-target shape `struct`: the slot itself, or the source + * record's slot when `obj` is a view. Leaves `B` in a fresh block. */ +export function recordViewFieldPtr( + B: BlockBuilder, + sizeType: "i32" | "i64", + struct: string, + obj: string, + slotIndex: number, + position: number, + words: { record: number; offsets: number }, +): string { + const own = B.tmp(); + B.line(`${own} = getelementptr inbounds %${struct}, ptr ${obj}, i64 0, i32 ${slotIndex}`); + const recordPtr = B.tmp(); + const record = B.tmp(); + const isView = B.tmp(); + B.line( + `${recordPtr} = getelementptr inbounds %${struct}, ptr ${obj}, i64 0, i32 ${words.record}`, + ); + B.line(`${record} = load ptr, ptr ${recordPtr}`); + B.line(`${isView} = icmp ne ptr ${record}, null`); + const from = B.currentLabel(); + const view = B.newLabel("rv.view"); + const join = B.newLabel("rv.join"); + B.condBr(isView, view, join); + B.startBlock(view); + const tablePtr = B.tmp(); + const table = B.tmp(); + const offsetPtr = B.tmp(); + const offset = B.tmp(); + const viewed = B.tmp(); + B.line( + `${tablePtr} = getelementptr inbounds %${struct}, ptr ${obj}, i64 0, i32 ${words.offsets}`, + ); + B.line(`${table} = load ptr, ptr ${tablePtr}`); + B.line(`${offsetPtr} = getelementptr inbounds ${sizeType}, ptr ${table}, i64 ${position}`); + B.line(`${offset} = load ${sizeType}, ptr ${offsetPtr}`); + B.line(`${viewed} = getelementptr inbounds i8, ptr ${record}, ${sizeType} ${offset}`); + B.br(join); + B.startBlock(join); + const ptr = B.tmp(); + B.line(`${ptr} = phi ptr [ ${own}, %${from} ], [ ${viewed}, %${view} ]`); + return ptr; +} diff --git a/packages/compiler/src/backend/llvm/shapes.ts b/packages/compiler/src/backend/llvm/shapes.ts index 25afbad69..8661783b8 100644 --- a/packages/compiler/src/backend/llvm/shapes.ts +++ b/packages/compiler/src/backend/llvm/shapes.ts @@ -69,6 +69,7 @@ import { disposeFieldLines, plainRefType, } from "./arenas.js"; +import { recordViewWords, type RecordViews } from "./record-views.js"; import { recordMayKeepSource } from "./record-sources.js"; import { LlvmUnsupportedError } from "./unsupported.js"; @@ -151,6 +152,8 @@ export interface ShapeHost { /** Record shapes and classes with an arena-aware constructor * (arenas.ts). */ arenaShapes?: ReadonlySet; + /** Record view targets and their sources (record-views.ts). */ + recordViews?: RecordViews; arenaClasses?: ReadonlySet; /** Every arena the program opens is closed (ArenaPlan.allClosed): the * arena-aware constructors take no dispose list. */ @@ -2172,6 +2175,8 @@ export function emitRecordShapes( llFieldType(f.type, host.inlineUnions.record(f.type)), ); if (shape.indexValue) fieldTys.push("ptr"); // the overflow ScrMap * + // A record view target: the viewed record and its offset table. + if (host.recordViews?.targets.has(shape.id) === true) fieldTys.push("ptr", "ptr"); // A tagged shape (an arm of a shape union) carries its shape number. if (host.nullableUnions.shapeNumber(shape.id) !== 0) fieldTys.unshift("i32"); typeDefs.push( @@ -2390,10 +2395,31 @@ export function emitRecordShapes( const index = i + base; const inline = host.inlineUnions.record(field.type); const fieldTy = llFieldType(field.type, inline); - clone.push( - ` %sp${i} = getelementptr inbounds %${struct}, ptr %src, i64 0, i32 ${index}`, - ` %sv${i} = load ${fieldTy}, ptr %sp${i}`, - ); + if (host.recordViews?.targets.has(shape.id) === true) { + // A view's fields live in the viewed record (record-views.ts). + const words = recordViewWords(shape.fields.length, base); + clone.push( + ` %so${i} = getelementptr inbounds %${struct}, ptr %src, i64 0, i32 ${index}`, + ` %sr${i}p = getelementptr inbounds %${struct}, ptr %src, i64 0, i32 ${words.record}`, + ` %sr${i}r = load ptr, ptr %sr${i}p`, + ` %sr${i}v = icmp ne ptr %sr${i}r, null`, + ` br i1 %sr${i}v, label %sv${i}.view, label %sv${i}.join`, + `sv${i}.view:`, + ` %st${i}p = getelementptr inbounds %${struct}, ptr %src, i64 0, i32 ${words.offsets}`, + ` %st${i} = load ptr, ptr %st${i}p`, + ` %sx${i}p = getelementptr inbounds ${host.sizeType}, ptr %st${i}, i64 ${i}`, + ` %sx${i} = load ${host.sizeType}, ptr %sx${i}p`, + ` %sw${i} = getelementptr inbounds i8, ptr %sr${i}r, ${host.sizeType} %sx${i}`, + ` br label %sv${i}.join`, + `sv${i}.join:`, + ` %sp${i} = phi ptr [ %so${i}, %${i === 0 ? "entry" : `sv${i - 1}.join`} ], [ %sw${i}, %sv${i}.view ]`, + ` %sv${i} = load ${fieldTy}, ptr %sp${i}`, + ); + } else + clone.push( + ` %sp${i} = getelementptr inbounds %${struct}, ptr %src, i64 0, i32 ${index}`, + ` %sv${i} = load ${fieldTy}, ptr %sp${i}`, + ); const owned = isRefCounted(field.type) && !inline; const stored = owned ? `%sr${i}` : `%sv${i}`; if (owned) { diff --git a/packages/compiler/src/backend/llvm/stack-objects.ts b/packages/compiler/src/backend/llvm/stack-objects.ts index 36793598b..eaba2d437 100644 --- a/packages/compiler/src/backend/llvm/stack-objects.ts +++ b/packages/compiler/src/backend/llvm/stack-objects.ts @@ -54,6 +54,9 @@ export function canStackObject(host: LlvmEmitterContext, e: IrExpr): e is StackO return false; if (e.type.kind === "record" && host.publishedTypes?.recordGuarded(e.type.shapeId) === true) return false; + // A record view target's frame copy would need its view words zeroed. + if (e.type.kind === "record" && host.shapeHost.recordViews?.targets.has(e.type.shapeId) === true) + return false; if (e.type.kind === "record" && host.recordsById.get(e.type.shapeId)?.freezable === true) return false; // A producer with a result-slot entry builds its fresh result in the diff --git a/packages/compiler/src/backend/llvm/walkers.ts b/packages/compiler/src/backend/llvm/walkers.ts index 7c67e98ae..c688422ab 100644 --- a/packages/compiler/src/backend/llvm/walkers.ts +++ b/packages/compiler/src/backend/llvm/walkers.ts @@ -1,4 +1,5 @@ import { emitPerKeyOrder } from "./key-orders.js"; +import { recordViewFieldPtr, recordViewWords } from "./record-views.js"; import { f64Lit } from "./common.js"; import { InternalCompilerError } from "../../errors.js"; /* Structure-walking helper EMITTERS for the LLVM backend — the .ll mirror @@ -280,10 +281,25 @@ export class LlWalkers { index: number, t: IrType, ): string { - const p = B.tmp(); - B.line( - `${p} = getelementptr inbounds %${mangleRecordStruct(shapeId)}, ptr ${recName}, i64 0, i32 ${index}`, - ); + const shape = this.host.recordsById.get(shapeId)!; + const base = this.host.nullableUnions.fieldBase(shapeId); + let p: string; + if (this.host.recordViews?.targets.has(shapeId) === true) { + p = recordViewFieldPtr( + B, + this.S, + mangleRecordStruct(shapeId), + recName, + index, + index - base, + recordViewWords(shape.fields.length, base), + ); + } else { + p = B.tmp(); + B.line( + `${p} = getelementptr inbounds %${mangleRecordStruct(shapeId)}, ptr ${recName}, i64 0, i32 ${index}`, + ); + } // An inline union field lends a private stack box (ABSENT kept). if (this.host.inlineUnions.record(t)) return loadStructField(B, this.S, p, t, true); const fieldTy = llFieldType(t, false); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/record-layout-views.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/record-layout-views.ts.json new file mode 100644 index 000000000..e413e3ee6 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/record-layout-views.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/record-layout-views.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/record-view-extras-release.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/record-view-extras-release.ts.json new file mode 100644 index 000000000..cbdf42556 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/record-view-extras-release.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/record-view-extras-release.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 3a1e83308..9faf7135b 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -301,6 +301,9 @@ typedef struct ScrDynTypedCast { void *(*retain)(void *); void (*release)(void *); bool traced; + /* A record view (scr_dyn_record_view): the view's own record-source + * entry owns the capsule, so the list neither owns nor releases it. */ + bool borrowed; struct ScrDynTypedCast *next; } ScrDynTypedCast; @@ -1315,7 +1318,6 @@ bool scr_dyn_typed_ref_is( const ScrDyn *d, const char *type_key, size_t type_key_len); bool scr_dyn_typed_ref_is_key(const ScrDyn *d, const ScrStr *type_key); void *scr_dyn_typed_ref_unbox(const ScrDyn *d); /* +1 */ -void *scr_dyn_typed_ref_prefix_view(const ScrDyn *d); /* +1 or NULL */ ScrDyn *scr_dyn_class_view_unavailable(void *ptr); ScrDyn *scr_dyn_typed_ref_materialize(const ScrDyn *d); /* +1 */ ScrDyn *scr_dyn_typed_ref_peek(const ScrDyn *d); /* +1 */ diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index eb40100cf..26f794b83 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -1526,7 +1526,7 @@ static void scr_dyn_dispose(ScrDyn *d, bool collected) { while (d->v.typed_ref.casts) { ScrDynTypedCast *cast = d->v.typed_ref.casts; d->v.typed_ref.casts = cast->next; - if (!collected || !cast->traced) cast->release(cast->ptr); + if (!cast->borrowed && (!collected || !cast->traced)) cast->release(cast->ptr); free(cast); } if (!collected || !d->v.typed_ref.traced) d->v.typed_ref.release(d->v.typed_ref.ptr); @@ -3745,24 +3745,13 @@ typedef struct { const char *names; const char *lens; size_t name_stride, len_stride, count; - /* A record that a narrower record type also references (a prefix view, - * scr_dyn_typed_ref_prefix_view) has an entry with no view and this, its - * own capsule's parameters: every capsule of the record then answers - * for its whole storage (scr_dyn_typed_ref_new), and reflective surfaces - * of the narrower type go through that capsule (scr_record_source_has). - * Owned; NULL for an ordinary source. */ - struct ScrRecordPrefix *prefix; + /* A record view (scr_dyn_record_view) has an entry with no view and + * this: the capsule of the record it views (+1). Boxing the view answers + * that capsule (scr_dyn_typed_ref_new), and the view type's reflective + * surfaces go through it (scr_record_source_has). NULL for an ordinary + * source. */ + ScrDyn *capsule; } ScrRecordSource; -typedef struct ScrRecordPrefix { - void *(*retain)(void *); - void (*release)(void *); - const char *type_key; - size_t type_key_len; - ScrDyn *(*materialize)(void *); - void (*commit)(void *, const ScrDyn *); - const ScrDynTypedArrayOps *ops; /* the record's ScrDynTypedRecordOps */ - bool traced, record; -} ScrRecordPrefix; static SCR_TL ScrRecordSource *scr_record_sources; static SCR_TL size_t scr_record_source_capacity, scr_record_source_count; /* Deleted slots keep this marker so probe runs stay intact without moving @@ -3803,17 +3792,20 @@ static ScrRecordSource *scr_record_source_slot(const void *rec) { return NULL; } -/* The record's kept source, or NULL (also for a prefix-view entry). */ +/* The record's kept source, or NULL (also for a record view's entry). */ static ScrRecordSource *scr_record_source_find(const void *rec) { ScrRecordSource *entry = scr_record_source_slot(rec); - return entry && !entry->prefix ? entry : NULL; + return entry && !entry->capsule ? entry : NULL; } -static const ScrRecordPrefix *scr_record_prefix_find(const void *rec) { +/* The capsule a record view views (borrowed), or NULL. */ +static ScrDyn *scr_record_view_capsule(const void *rec) { ScrRecordSource *entry = scr_record_source_slot(rec); - return entry ? entry->prefix : NULL; + return entry ? entry->capsule : NULL; } +static void scr_record_view_forget(ScrDyn *capsule, const void *view); + /* The retained view of a record with undeclared keys (borrowed), or NULL. */ ScrDyn *scr_record_source_view(const void *rec) { ScrRecordSource *entry = scr_record_source_find(rec); @@ -3829,11 +3821,16 @@ bool scr_record_source_has(const void *rec) { * the dispose hook sees a consistent table. */ static void scr_record_source_drop(ScrRecordSource *found) { ScrDyn *view = found->view; - free(found->prefix); + ScrDyn *capsule = found->capsule; + void *rec = found->rec; *found = (ScrRecordSource){ .rec = SCR_RECORD_SOURCE_DELETED }; scr_record_source_count--; scr_record_source_deleted++; scr_dyn_release(view); + if (capsule) { + scr_record_view_forget(capsule, rec); + scr_dyn_release(capsule); + } } static void scr_record_source_disposed(void *rec) { @@ -3913,8 +3910,8 @@ static void scr_record_source_teardown(void) { scr_record_source_filter = NULL; for (size_t i = 0; i < capacity; i++) { if (table[i].rec && table[i].rec != SCR_RECORD_SOURCE_DELETED) { - free(table[i].prefix); scr_dyn_release(table[i].view); + scr_dyn_release(table[i].capsule); } } free(table); @@ -3991,6 +3988,14 @@ void scr_record_source_note(void *rec, const ScrDyn *src, const char *const *nam void scr_record_source_spread(void *dst, const void *src, const char *const *names, const size_t *lens, size_t count) { if (!scr_record_source_count || !dst || !src) return; + ScrDyn *viewed = scr_record_view_capsule(src); + if (viewed) { + /* A record view spreads every property of the record it views. */ + ScrDyn *snapshot = scr_dyn_typed_ref_peek(viewed); + if (!scr_exc_pending()) scr_record_source_note(dst, snapshot, names, lens, count); + scr_dyn_release(snapshot); + return; + } ScrRecordSource *found = scr_record_source_find(src); if (!found) return; ScrRecordSource probe = { dst, NULL, (const char *)names, (const char *)lens, @@ -4033,9 +4038,9 @@ void scr_record_source_move(void *from, void *to) { if (!scr_record_source_count) return; ScrRecordSource *old = scr_record_source_slot(to); ScrRecordSource *found = scr_record_source_find(from); - if (old && old->prefix) { - /* A prefix-viewed record keeps its entry: its capsule's view holds the - * undeclared properties, and narrower names read them through it. */ + if (old && old->capsule) { + /* A record view keeps its entry: the viewed capsule holds every + * property, and the view's name reads them through it. */ if (found) scr_record_source_drop(found); return; } @@ -4099,7 +4104,7 @@ ScrDyn *scr_record_source_apply(ScrDyn *snapshot, const void *rec) { } static ScrDyn *scr_json_member(const ScrDyn *holder, const ScrStr *key); -static void scr_jb_put_record_prefix(ScrJsonBuf *b, void *rec); +static void scr_jb_put_record_view(ScrJsonBuf *b, const ScrDyn *capsule); static bool scr_jb_try_pure_member(ScrJsonBuf *b, const char *key, size_t key_len, const ScrStr *key_str, const ScrDyn *value, bool *first); static bool scr_jb_put_member(ScrJsonBuf *b, const ScrStr *key, ScrDyn *value, bool *first, @@ -4177,8 +4182,9 @@ static void scr_record_source_key_order(const ScrDyn *view, size_t *out) { bool scr_jb_try_record_source(ScrJsonBuf *b, void *rec, bool (*field)(ScrJsonBuf *, void *, size_t, bool)) { - if (scr_record_prefix_find(rec)) { - scr_jb_put_record_prefix(b, rec); + ScrDyn *viewed = scr_record_view_capsule(rec); + if (viewed) { + scr_jb_put_record_view(b, viewed); return true; } ScrRecordSource *found = scr_record_source_find(rec); @@ -4278,20 +4284,12 @@ static ScrDyn *scr_dyn_typed_ref_new( const char *type_key, size_t type_key_len, ScrDyn *(*materialize)(void *), void (*commit)(void *, const ScrDyn *), bool traced, bool observed) { - const ScrRecordPrefix *prefix = - scr_record_source_count && type_key_len >= 7 && memcmp(type_key, "record:", 7) == 0 - ? scr_record_prefix_find(ptr) - : NULL; - if (prefix) { - /* A record seen through a narrower type: its capsule is its own. */ - retain = prefix->retain; - release = prefix->release; - type_key = prefix->type_key; - type_key_len = prefix->type_key_len; - materialize = prefix->materialize; - commit = prefix->commit; - traced = traced || prefix->traced; - } + /* A record view boxes as the record it views. */ + ScrDyn *viewed = scr_record_source_count && type_key_len >= 7 && + memcmp(type_key, "record:", 7) == 0 + ? scr_record_view_capsule(ptr) + : NULL; + if (viewed) return scr_dyn_retain(viewed); scr_typed_ref_reserve(); size_t slot = scr_typed_ref_bucket(ptr); for (ScrDyn *value; (value = scr_typed_ref_slots[slot]); slot = (slot + 1) & (scr_typed_ref_capacity - 1)) { @@ -4327,11 +4325,6 @@ static ScrDyn *scr_dyn_typed_ref_new( d->v.typed_ref.record = false; d->v.typed_ref.klass = false; d->v.typed_ref.bag_class = false; - if (prefix) { - /* Bound here: the narrower name's bind keeps them (bind_record). */ - d->v.typed_ref.array = prefix->ops; - d->v.typed_ref.record = prefix->record; - } scr_typed_ref_slots[slot] = d; /* the empty slot that ended the probe */ scr_typed_ref_count++; return d; @@ -4378,29 +4371,68 @@ void *scr_dyn_typed_ref_unbox(const ScrDyn *d) { return d->v.typed_ref.retain(d->v.typed_ref.ptr); } -/* A record capsule's native record for a conversion into a type whose - * storage is a prefix of the record's (the compiler's prefix-view sources): - * the pointer itself (+1), so both names keep one object. The record - * remembers its own capsule parameters (ScrRecordPrefix), so every later - * capsule of it, whichever name boxes it, answers for its whole storage. - * NULL while the record keeps undeclared source properties, whose table - * is keyed by the record's own declaration; the conversion then builds - * its copy as before. */ -void *scr_dyn_typed_ref_prefix_view(const ScrDyn *d) { +/* Record views (compiler record-views.ts). A checked conversion of a + * capsule of record A into record type B, when A has all of B's fields, + * answers a view (+1): a B object (allocated by `alloc`, zeroed) whose + * trailing words at byte offset `words` hold A (borrowed) and A's offset + * table for B's fields, so every field access of the B value reaches A. + * `sources` lists, for each A the compiler allows, its type key and offset + * table. The view's record-source entry owns A's capsule: boxing the view + * answers that capsule, and the entry dies with the view. One view per + * capsule and B, found again through the capsule's cast list, where it is + * borrowed. NULL for any other capsule and for a frozen A, whose + * conversion keeps its copy. */ +typedef struct { + const char *key; + size_t key_len; + const size_t *offsets; +} ScrRecordViewSource; +static void *scr_record_view_retain(void *view); + +void *scr_dyn_record_view(ScrDyn *d, const ScrRecordViewSource *sources, size_t count, + const char *key, size_t key_len, void *(*alloc)(void), size_t words) { + if (d->kind != SCR_DYN_TYPED_REF) return NULL; + const ScrRecordViewSource *source = NULL; + for (size_t i = 0; i < count && !source; i++) + if (sources[i].key_len == d->v.typed_ref.type_key_len && + memcmp(sources[i].key, d->v.typed_ref.type_key, sources[i].key_len) == 0) + source = &sources[i]; void *record = d->v.typed_ref.ptr; - if (!scr_record_prefix_find(record)) { - if (scr_record_source_find(record) || !d->v.typed_ref.record) return NULL; - ScrRecordPrefix *prefix = malloc(sizeof *prefix); - if (!prefix) scr_json_oom(); - *prefix = (ScrRecordPrefix){ - d->v.typed_ref.retain, d->v.typed_ref.release, d->v.typed_ref.type_key, - d->v.typed_ref.type_key_len, d->v.typed_ref.materialize, d->v.typed_ref.commit, - d->v.typed_ref.array, d->v.typed_ref.traced, d->v.typed_ref.record, - }; - ScrRecordSource entry = { .rec = record, .prefix = prefix }; - scr_record_source_put(&entry); + if (!source || scr_rc_frozen(record)) return NULL; + for (ScrDynTypedCast *cast = d->v.typed_ref.casts; cast; cast = cast->next) + if (cast->borrowed && cast->type_key_len == key_len && memcmp(cast->type_key, key, key_len) == 0) + return cast->retain(cast->ptr); + void *view = alloc(); + void **slots = (void **)((char *)view + words); + slots[0] = record; + slots[1] = (void *)source->offsets; + ScrRecordSource entry = { .rec = view, .capsule = scr_dyn_retain(d) }; + scr_record_source_put(&entry); + ScrDynTypedCast *cast = malloc(sizeof *cast); + if (!cast) scr_trap("scriptc: out of memory\n"); + *cast = (ScrDynTypedCast){ .type_key = key, .type_key_len = key_len, .ptr = view, + .retain = scr_record_view_retain, .release = NULL, + .borrowed = true, .next = d->v.typed_ref.casts }; + d->v.typed_ref.casts = cast; + return view; +} + +/* A view itself counts like any record (its target's retain). */ +static void *scr_record_view_retain(void *view) { + if (!scr_rc_immortal(view)) SCR_RC_COUNT(view)++; + return view; +} + +/* A view left the table: its capsule's cast list stops naming it. */ +static void scr_record_view_forget(ScrDyn *capsule, const void *view) { + for (ScrDynTypedCast **at = &capsule->v.typed_ref.casts; *at; at = &(*at)->next) { + if ((*at)->borrowed && (*at)->ptr == view) { + ScrDynTypedCast *cast = *at; + *at = cast->next; + free(cast); + return; + } } - return d->v.typed_ref.retain(record); } @@ -4988,7 +5020,7 @@ void *scr_dyn_typed_ref_cached_cast( const ScrDyn *d, const char *type_key, size_t type_key_len) { for (ScrDynTypedCast *cast = d->v.typed_ref.casts; cast; cast = cast->next) { - if (cast->type_key_len == type_key_len && + if (!cast->borrowed && cast->type_key_len == type_key_len && memcmp(cast->type_key, type_key, type_key_len) == 0) { return cast->retain(cast->ptr); } @@ -5008,6 +5040,7 @@ void scr_dyn_typed_ref_cache_cast( cast->retain = retain; cast->release = release; cast->traced = traced; + cast->borrowed = false; cast->next = d->v.typed_ref.casts; d->v.typed_ref.casts = cast; } @@ -14845,11 +14878,9 @@ ScrStr *scr_util_format_j_text(ScrStr *text) { /* The gap of a compact walk; the walks read only its length. */ static const ScrStr scr_json_compact_gap = { .rc = SIZE_MAX, .len = 0 }; -/* JSON for a prefix-viewed record: its own capsule's. */ -static void scr_jb_put_record_prefix(ScrJsonBuf *b, void *rec) { - ScrDyn *capsule = scr_dyn_new_typed_ref(rec, NULL, NULL, "record:", 7, NULL, NULL); +/* JSON for a record view: the viewed capsule's. */ +static void scr_jb_put_record_view(ScrJsonBuf *b, const ScrDyn *capsule) { scr_json_replaced_write(b, capsule, scr_dyn_undefined(), NULL, &scr_json_compact_gap, b->seen_len); - scr_dyn_release(capsule); } #define SCR_JSON_PURE_DEPTH 32 diff --git a/packages/runtime/src/scr_region.c b/packages/runtime/src/scr_region.c index dacd9f163..bbd163259 100644 --- a/packages/runtime/src/scr_region.c +++ b/packages/runtime/src/scr_region.c @@ -871,7 +871,7 @@ static void scr_rgn_walk_dyn(ScrRgnWalk *w, ScrDyn *d) { scr_region_slot(w, d->v.typed_ref.ptr, d->v.typed_ref.release); scr_rgn_enter(w, d->v.typed_ref.materialized, SCR_RGN_W_DYN, NULL); for (ScrDynTypedCast *cast = d->v.typed_ref.casts; cast; cast = cast->next) - scr_region_slot(w, cast->ptr, cast->release); + if (!cast->borrowed) scr_region_slot(w, cast->ptr, cast->release); break; case SCR_DYN_PROXY: scr_rgn_enter(w, d->v.proxy.target, SCR_RGN_W_DYN, NULL); diff --git a/tests/corpus/record-layout-views.ts b/tests/corpus/record-layout-views.ts new file mode 100644 index 000000000..4176f4210 --- /dev/null +++ b/tests/corpus/record-layout-views.ts @@ -0,0 +1,26 @@ +// A native record converted from an untyped value into another record type +// that names some of its fields, in any order, is the same object: writes +// through either name show through the other, identity holds, and the +// untyped views, JSON and spreads of the narrower name see every field. +type Wide = { a: number; b: string; c: boolean; d: number[] }; +type AC = { a: number; c: boolean }; +type DB = { d: number[]; b: string }; +const wide: Wide = { a: 1, b: "x", c: true, d: [1] }; +const o: unknown = wide; +const v = o as AC; +v.c = false; +console.log((o as Wide).c, wide.c, o === v, (v as unknown) === o); +wide.a = 5; +console.log(v.a, JSON.stringify(v), Object.keys(v).join(",")); +const w = o as DB; +w.d.push(2); +w.b = "y"; +w.d = [7, 8]; +console.log(wide.b, wide.d.length, JSON.stringify(o), w === (o as DB), v === (o as AC)); +const v2 = { ...v }; +console.log(JSON.stringify(v2)); +function bump(x: AC): void { x.a += 1; } +bump(v); +console.log(wide.a, v.a); +const list: AC[] = [v, { a: 9, c: true }]; +console.log(list.map((x) => x.a).join(","), JSON.stringify(list)); diff --git a/tests/corpus/record-view-extras-release.ts b/tests/corpus/record-view-extras-release.ts new file mode 100644 index 000000000..b1de5ca82 --- /dev/null +++ b/tests/corpus/record-view-extras-release.ts @@ -0,0 +1,44 @@ +// Views of records whose other fields hold strings, arrays and records: every view and every record is released exactly once, whichever +// name drops the last reference (the sanitized lane checks for leaks). +type Node = { id: number; label: string; children: Node[]; meta: { tag: string }; seen: { key: string; count: number }[] }; +type Head = { id: number; meta: { tag: string } }; +type Tail = { children: Node[]; label: string }; + +function make(id: number): Node { + return { id, label: "n" + id, children: [], meta: { tag: "t" + id }, seen: [{ key: "k", count: id }] }; +} +function asHead(value: unknown): Head { + return value as Head; +} +function asTail(value: unknown): Tail { + return value as Tail; +} + +let total = 0; +for (let round = 0; round < 200; round++) { + const node = make(round); + node.children.push(make(round + 1000)); + const head = asHead(node); + const tail = asTail(node); + head.meta = { tag: "h" + round }; + tail.label = tail.label + "!"; + tail.children.push(make(round + 2000)); + total += head.id + tail.children.length + node.seen[0]!.count; + if (round === 199) console.log(JSON.stringify(head), node.label, node.meta.tag, tail.children.length); +} +console.log(total); + +// The last reference is the view's. +let kept: Head | undefined; +for (let round = 0; round < 100; round++) kept = asHead(make(round)); +if (kept) console.log(kept.id, kept.meta.tag, JSON.stringify(kept)); +kept = undefined; + +// The last reference is the record's. +const records: Node[] = []; +for (let round = 0; round < 100; round++) { + const node = make(round); + asTail(node).label = "v" + round; + records.push(node); +} +console.log(records.map((node) => node.label).slice(0, 3).join(","), records.length); diff --git a/tests/harness/corpus-costs/record-layout-views.ts.json b/tests/harness/corpus-costs/record-layout-views.ts.json new file mode 100644 index 000000000..8c0e66c3c --- /dev/null +++ b/tests/harness/corpus-costs/record-layout-views.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 3600 } diff --git a/tests/harness/corpus-costs/record-view-extras-release.ts.json b/tests/harness/corpus-costs/record-view-extras-release.ts.json new file mode 100644 index 000000000..8c0e66c3c --- /dev/null +++ b/tests/harness/corpus-costs/record-view-extras-release.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 3600 } From fe34846a54761f7f06e6672b6141e77aa7dfe8d4 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 26 Oct 2026 22:23:05 -0700 Subject: [PATCH 23/54] Keep tagged record shapes out of record views and declare the view entry point A tagged shape's values share field reads with the other arms of its shape union, which carry no view words, so tagged shapes keep their copies. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/backend/llvm/emitter.ts | 3 +++ packages/runtime/src/scr_dyn.h | 9 +++++++++ packages/runtime/src/scr_json.c | 5 ----- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index f24e3a862..f2998db78 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -2526,7 +2526,10 @@ export class LlEmitter { } this.shapeHost.recordViews = planRecordViews(mod, { traced: (id) => this.tracedShapes.has(`record:${id}`), + // A tagged shape's values share field reads with the other arms of + // its shape union, which do not carry view words. excluded: (id) => + this.nullableUnions.shapeNumber(id) !== 0 || this.shapeHost.arenaShapes?.has(id) === true || this.publishedTypes?.recordGuarded(id) === true || mod.workers === true, diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 9faf7135b..c90b29419 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -1318,6 +1318,15 @@ bool scr_dyn_typed_ref_is( const ScrDyn *d, const char *type_key, size_t type_key_len); bool scr_dyn_typed_ref_is_key(const ScrDyn *d, const ScrStr *type_key); void *scr_dyn_typed_ref_unbox(const ScrDyn *d); /* +1 */ +/* Record views (scr_json.c): a checked conversion's view of the record a + * capsule holds, or NULL. */ +typedef struct { + const char *key; + size_t key_len; + const size_t *offsets; +} ScrRecordViewSource; +void *scr_dyn_record_view(ScrDyn *d, const ScrRecordViewSource *sources, size_t count, + const char *key, size_t key_len, void *(*alloc)(void), size_t words); ScrDyn *scr_dyn_class_view_unavailable(void *ptr); ScrDyn *scr_dyn_typed_ref_materialize(const ScrDyn *d); /* +1 */ ScrDyn *scr_dyn_typed_ref_peek(const ScrDyn *d); /* +1 */ diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 26f794b83..d3e7431e0 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -4382,11 +4382,6 @@ void *scr_dyn_typed_ref_unbox(const ScrDyn *d) { * capsule and B, found again through the capsule's cast list, where it is * borrowed. NULL for any other capsule and for a frozen A, whose * conversion keeps its copy. */ -typedef struct { - const char *key; - size_t key_len; - const size_t *offsets; -} ScrRecordViewSource; static void *scr_record_view_retain(void *view); void *scr_dyn_record_view(ScrDyn *d, const ScrRecordViewSource *sources, size_t count, From 10185c214796c8821d658060058d0e648d5db82d Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 26 Oct 2026 23:09:38 -0700 Subject: [PATCH 24/54] Mark record views with the frozen bit instead of testing a trailing word A value of a view target type can point at another storage (a shared field read of a shape union, for one), whose block ends before the view words. The view mark is now the rc word's frozen bit, which no other value of a view target has (targets are never freezable; an immortal word is no view), and the trailing words are read only after it. View targets release like freezable shapes. Co-Authored-By: Claude Opus 5.5 --- .../compiler/src/backend/llvm/record-views.ts | 36 ++++++++++++++----- packages/compiler/src/backend/llvm/shapes.ts | 18 ++++++---- packages/runtime/src/scr_json.c | 9 +++-- 3 files changed, 45 insertions(+), 18 deletions(-) diff --git a/packages/compiler/src/backend/llvm/record-views.ts b/packages/compiler/src/backend/llvm/record-views.ts index 41df2cb71..ce27c84a8 100644 --- a/packages/compiler/src/backend/llvm/record-views.ts +++ b/packages/compiler/src/backend/llvm/record-views.ts @@ -5,8 +5,9 @@ * offsets) answers a view: a B-shaped object whose fields live in A. The * view's storage is B's struct plus two trailing words, the A record * (borrowed: the view's record-source entry owns A's capsule, which owns - * A) and a table of A's byte offsets for B's fields. Every field address - * of a B value tests the first word (recordViewFieldPtr), so reads and + * A) and a table of A's byte offsets for B's fields, and its rc word + * carries the frozen bit as the view mark. Every field address of a B + * value tests the mark (recordViewFieldPtr), so reads and * writes through either name reach the same storage, and boxing a view * answers A's capsule (scr_dyn_record_view), so identity holds too. * @@ -116,6 +117,23 @@ export function planRecordViews(mod: IrModule, options: ViewOptions): RecordView return { targets, sources: (target) => sources.get(target) ?? [] }; } +/** Lines setting `dest` (i1) to whether the record `obj` of a view target + * type is a view. A view's rc word carries the frozen bit (scr_rc.h + * SCR_RC_FROZEN), which no other value of such a type has: view targets + * are never freezable, and an immortal word (all ones) is not a view. The + * trailing words are read only after this test, so a value of another + * storage seen through the type is never read past its end. */ +export function recordViewTestLines(sizeType: "i32" | "i64", obj: string, dest: string): string[] { + const bit = sizeType === "i64" ? "4611686018427387904" : "1073741824"; + return [ + ` ${dest}.w = load ${sizeType}, ptr ${obj}`, + ` ${dest}.b = and ${sizeType} ${dest}.w, ${bit}`, + ` ${dest}.m = icmp ne ${sizeType} ${dest}.b, 0`, + ` ${dest}.i = icmp ne ${sizeType} ${dest}.w, -1`, + ` ${dest} = and i1 ${dest}.m, ${dest}.i`, + ]; +} + /** The struct indices of a view target's two trailing words. */ export function recordViewWords( fieldCount: number, @@ -138,19 +156,19 @@ export function recordViewFieldPtr( ): string { const own = B.tmp(); B.line(`${own} = getelementptr inbounds %${struct}, ptr ${obj}, i64 0, i32 ${slotIndex}`); - const recordPtr = B.tmp(); - const record = B.tmp(); const isView = B.tmp(); - B.line( - `${recordPtr} = getelementptr inbounds %${struct}, ptr ${obj}, i64 0, i32 ${words.record}`, - ); - B.line(`${record} = load ptr, ptr ${recordPtr}`); - B.line(`${isView} = icmp ne ptr ${record}, null`); + for (const line of recordViewTestLines(sizeType, obj, isView)) B.line(line.trimStart()); const from = B.currentLabel(); const view = B.newLabel("rv.view"); const join = B.newLabel("rv.join"); B.condBr(isView, view, join); B.startBlock(view); + const recordPtr = B.tmp(); + const record = B.tmp(); + B.line( + `${recordPtr} = getelementptr inbounds %${struct}, ptr ${obj}, i64 0, i32 ${words.record}`, + ); + B.line(`${record} = load ptr, ptr ${recordPtr}`); const tablePtr = B.tmp(); const table = B.tmp(); const offsetPtr = B.tmp(); diff --git a/packages/compiler/src/backend/llvm/shapes.ts b/packages/compiler/src/backend/llvm/shapes.ts index 8661783b8..c5d4526bc 100644 --- a/packages/compiler/src/backend/llvm/shapes.ts +++ b/packages/compiler/src/backend/llvm/shapes.ts @@ -69,7 +69,7 @@ import { disposeFieldLines, plainRefType, } from "./arenas.js"; -import { recordViewWords, type RecordViews } from "./record-views.js"; +import { recordViewTestLines, recordViewWords, type RecordViews } from "./record-views.js"; import { recordMayKeepSource } from "./record-sources.js"; import { LlvmUnsupportedError } from "./unsupported.js"; @@ -454,7 +454,11 @@ export function emitShapeUnionHelpers(host: ShapeHost): string[] { const traced = union.arms.map((arm) => host.tracedShapes.has(`record:${arm.shapeId}`)); // A freezable arm's count is the word without its frozen bit, as in // releaseFastPath: a frozen record's last reference takes the slow path. - const frozen = union.arms.some((arm) => host.recordsById.get(arm.shapeId)?.freezable === true); + const frozen = union.arms.some( + (arm) => + host.recordsById.get(arm.shapeId)?.freezable === true || + host.recordViews?.targets.has(arm.shapeId) === true, + ); const notFrozen = ~recordFrozenBit(S); const release = shapeReleaseFn(union.unionId); if (traced.every((t) => t === traced[0])) @@ -2296,7 +2300,9 @@ export function emitRecordShapes( false, chained, segmented, - shape.freezable === true, + // A record view's word carries the frozen bit as its view mark + // (record-views.ts), so view targets release like freezable shapes. + shape.freezable === true || host.recordViews?.targets.has(shape.id) === true, iterative ? iterativeTeardownDefinitions( host, @@ -2400,11 +2406,11 @@ export function emitRecordShapes( const words = recordViewWords(shape.fields.length, base); clone.push( ` %so${i} = getelementptr inbounds %${struct}, ptr %src, i64 0, i32 ${index}`, - ` %sr${i}p = getelementptr inbounds %${struct}, ptr %src, i64 0, i32 ${words.record}`, - ` %sr${i}r = load ptr, ptr %sr${i}p`, - ` %sr${i}v = icmp ne ptr %sr${i}r, null`, + ...recordViewTestLines(host.sizeType, "%src", `%sr${i}v`), ` br i1 %sr${i}v, label %sv${i}.view, label %sv${i}.join`, `sv${i}.view:`, + ` %sr${i}p = getelementptr inbounds %${struct}, ptr %src, i64 0, i32 ${words.record}`, + ` %sr${i}r = load ptr, ptr %sr${i}p`, ` %st${i}p = getelementptr inbounds %${struct}, ptr %src, i64 0, i32 ${words.offsets}`, ` %st${i} = load ptr, ptr %st${i}p`, ` %sx${i}p = getelementptr inbounds ${host.sizeType}, ptr %st${i}, i64 ${i}`, diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index d3e7431e0..d094f15db 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -4373,9 +4373,9 @@ void *scr_dyn_typed_ref_unbox(const ScrDyn *d) { /* Record views (compiler record-views.ts). A checked conversion of a * capsule of record A into record type B, when A has all of B's fields, - * answers a view (+1): a B object (allocated by `alloc`, zeroed) whose - * trailing words at byte offset `words` hold A (borrowed) and A's offset - * table for B's fields, so every field access of the B value reaches A. + * answers a view (+1): a B object (allocated by `alloc`, zeroed, its rc + * word marked with the frozen bit) whose trailing words at byte offset + * `words` hold A (borrowed) and A's offset table for B's fields, so every field access of the B value reaches A. * `sources` lists, for each A the compiler allows, its type key and offset * table. The view's record-source entry owns A's capsule: boxing the view * answers that capsule, and the entry dies with the view. One view per @@ -4398,6 +4398,9 @@ void *scr_dyn_record_view(ScrDyn *d, const ScrRecordViewSource *sources, size_t if (cast->borrowed && cast->type_key_len == key_len && memcmp(cast->type_key, key, key_len) == 0) return cast->retain(cast->ptr); void *view = alloc(); + /* The frozen bit marks a view: view target types are never freezable, + * and their emitted releases ignore the bit (record-views.ts). */ + *(size_t *)view |= SCR_RC_FROZEN; void **slots = (void **)((char *)view + words); slots[0] = record; slots[1] = (void *)source->offsets; From 7b91c5fe5c4ce396c158091327ede08188ec4a4c Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Tue, 27 Oct 2026 00:53:10 -0700 Subject: [PATCH 25/54] Plan record views before any host copies the shape host The walkers take a narrower host type than the emitter's shape host; in the self-hosted compiler that conversion copies the host, so a plan assigned afterwards never reached them and the native emitter left view guards out. The plan is now made before the copies, and arena shapes leave its shared target set once arenas are planned. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/backend/llvm/emitter.ts | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index f2998db78..f2025bdf2 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -1397,6 +1397,8 @@ export class LlEmitter { readonly dyn: LlDyn; /** Explicit callback boundary for standalone layout and walker stages. */ readonly shapeHost: DynHost; + /** The record view plan's target set (record-views.ts), shared by reference. */ + private recordViewTargets: Set | null = null; /** External declarations, in first-use order. */ readonly decls = new Set(); /** Declared functions referenced as values: each needs an env-signature @@ -2402,6 +2404,20 @@ export class LlEmitter { } } } + // Planned before any host copies the shape host (record-views.ts). + { + const views = planRecordViews(mod, { + traced: (id) => this.tracedShapes.has(`record:${id}`), + // A tagged shape's values share field reads with the other arms of + // its shape union. + excluded: (id) => + this.nullableUnions.shapeNumber(id) !== 0 || + this.publishedTypes?.recordGuarded(id) === true || + mod.workers === true, + }); + this.shapeHost.recordViews = views; + this.recordViewTargets = views.targets as Set; + } this.walkers = new LlWalkers(this.shapeHost); this.dyn = new LlDyn(this.shapeHost); for (const g of mod.globals ?? []) { @@ -2524,16 +2540,13 @@ export class LlEmitter { this.shapeHost.arenasClosed = this.arenas.allClosed; } } - this.shapeHost.recordViews = planRecordViews(mod, { - traced: (id) => this.tracedShapes.has(`record:${id}`), - // A tagged shape's values share field reads with the other arms of - // its shape union, which do not carry view words. - excluded: (id) => - this.nullableUnions.shapeNumber(id) !== 0 || - this.shapeHost.arenaShapes?.has(id) === true || - this.publishedTypes?.recordGuarded(id) === true || - mod.workers === true, - }); + // Arena shapes are known only now; the plan's target set is shared by + // every host that copied the shape host, so dropping them here reaches + // all of them. + const arenaShapes = this.shapeHost.arenaShapes; + const viewTargets = this.recordViewTargets; + if (arenaShapes !== undefined && viewTargets !== null) + for (const id of arenaShapes) viewTargets.delete(id); for (const [name, rec] of RUNTIME_ERROR_CLASSES) { const meta = this.classMeta.get(name); if (!meta) break; // hand-written IR without the builtin defs: no stamps From ae74868e7ac521531bcb87dcae226d8941529a3d Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Tue, 27 Oct 2026 01:45:06 -0700 Subject: [PATCH 26/54] Plan record views only where a checked target meets a live capsule source Targets come from the closure of checked conversion types and sources from the closure of types boxed as live capsules; a conversion anywhere else keeps its copy. Fewer types carry view guards, the compiler's own among them. Co-Authored-By: Claude Opus 5.5 --- .../compiler/src/backend/llvm/record-views.ts | 62 +++++++++++-------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/packages/compiler/src/backend/llvm/record-views.ts b/packages/compiler/src/backend/llvm/record-views.ts index ce27c84a8..135a330ed 100644 --- a/packages/compiler/src/backend/llvm/record-views.ts +++ b/packages/compiler/src/backend/llvm/record-views.ts @@ -14,8 +14,8 @@ * Only the target types of such conversions carry the words and the * test. They are found from the IR, before any code is emitted: B is a * record type a checked conversion can produce (the closure of dynCheck - * targets and of the types live capsules expose) and some other record - * type in the module has all of B's fields. A program without checked + * targets), and some other record type whose values reach untyped code as + * live capsules (the closure of live dynFrom types) has all of B's fields. A program without checked * conversions has none, so its code is unchanged. */ import { isDirentShape, @@ -66,37 +66,49 @@ function covers(source: IrRecordShape, target: IrRecordShape): boolean { export function planRecordViews(mod: IrModule, options: ViewOptions): RecordViews { const shapes = new Map((mod.records ?? []).map((shape) => [shape.id, shape])); const unions = new Map((mod.unions ?? []).map((union) => [union.id, union])); - const reachable = new Set(); - const seen = new Set(); - const visit = (t: IrType): void => { - const key = typeKey(t); - if (seen.has(key)) return; - seen.add(key); - switch (t.kind) { - case "record": { - reachable.add(t.shapeId); - for (const field of shapes.get(t.shapeId)?.fields ?? []) visit(field.type); - return; + // Targets: record types a checked conversion produces. Sources: record + // types whose values reach untyped code as live capsules. A view arises + // only where the two meet; elsewhere a conversion keeps its copy. + const closure = (roots: IrType[]): Set => { + const out = new Set(); + const seen = new Set(); + const visit = (t: IrType): void => { + const key = typeKey(t); + if (seen.has(key)) return; + seen.add(key); + switch (t.kind) { + case "record": { + out.add(t.shapeId); + for (const field of shapes.get(t.shapeId)?.fields ?? []) visit(field.type); + return; + } + case "array": + visit(t.elem); + return; + case "union": + for (const arm of unions.get(t.unionId)?.arms ?? []) visit(arm); + return; + default: + return; } - case "array": - visit(t.elem); - return; - case "union": - for (const arm of unions.get(t.unionId)?.arms ?? []) visit(arm); - return; - default: - return; - } + }; + for (const root of roots) visit(root); + return out; }; + const checked: IrType[] = []; + const boxed: IrType[] = []; for (const fn of mod.functions) everyStmtList(fn.body, { stmt: () => true, expr: (e) => { - if (e.kind === "dynCheck") visit(e.type); - else if (e.kind === "dynFrom" && e.liveRef === true) visit(e.value.type); + if (e.kind === "dynCheck") checked.push(e.type); + else if (e.kind === "dynFrom" && e.liveRef === true) boxed.push(e.value.type); return true; }, }); + if (checked.length === 0 || boxed.length === 0) return NO_RECORD_VIEWS; + const reachable = closure(checked); + const live = closure(boxed); if (reachable.size === 0) return NO_RECORD_VIEWS; const targets = new Set(); const sources = new Map(); @@ -106,7 +118,7 @@ export function planRecordViews(mod: IrModule, options: ViewOptions): RecordView if (options.traced(id) || options.excluded(id)) continue; const found: string[] = []; for (const source of shapes.values()) { - if (source.id === id || !reachable.has(source.id) || !plainShape(source)) continue; + if (source.id === id || !live.has(source.id) || !plainShape(source)) continue; if (covers(source, target)) found.push(source.id); } if (found.length === 0) continue; From 3f772c0ee6878212730f121a6cf572ec1b4b33fc Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Tue, 27 Oct 2026 02:27:46 -0700 Subject: [PATCH 27/54] Look up view slots out of line behind a cold branch The view guard's slot lookup moves into scr_record_view_slot, and the branch to it is weighted as unlikely, so a field access of a view target type adds one rc-word test to its hot path. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/backend/llvm/emitter.ts | 1 + .../compiler/src/backend/llvm/record-views.ts | 21 +++++-------------- packages/compiler/src/backend/llvm/walkers.ts | 1 + packages/runtime/src/scr_dyn.h | 1 + packages/runtime/src/scr_json.c | 9 ++++++++ 5 files changed, 17 insertions(+), 16 deletions(-) diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index f2025bdf2..059190eb5 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -4879,6 +4879,7 @@ export class LlEmitter { idx + base, idx, recordViewWords(shape.fields.length, base), + (decl) => this.declare(decl), ); } else { p = this.B.tmp(); diff --git a/packages/compiler/src/backend/llvm/record-views.ts b/packages/compiler/src/backend/llvm/record-views.ts index 135a330ed..92238fd54 100644 --- a/packages/compiler/src/backend/llvm/record-views.ts +++ b/packages/compiler/src/backend/llvm/record-views.ts @@ -165,6 +165,7 @@ export function recordViewFieldPtr( slotIndex: number, position: number, words: { record: number; offsets: number }, + declare: (decl: string) => void, ): string { const own = B.tmp(); B.line(`${own} = getelementptr inbounds %${struct}, ptr ${obj}, i64 0, i32 ${slotIndex}`); @@ -173,26 +174,14 @@ export function recordViewFieldPtr( const from = B.currentLabel(); const view = B.newLabel("rv.view"); const join = B.newLabel("rv.join"); - B.condBr(isView, view, join); + // Views are rare: the slot lookup stays out of line. + B.condBr(isView, view, join, `, !prof !{!"branch_weights", i32 1, i32 1000}`); B.startBlock(view); - const recordPtr = B.tmp(); - const record = B.tmp(); - B.line( - `${recordPtr} = getelementptr inbounds %${struct}, ptr ${obj}, i64 0, i32 ${words.record}`, - ); - B.line(`${record} = load ptr, ptr ${recordPtr}`); - const tablePtr = B.tmp(); - const table = B.tmp(); - const offsetPtr = B.tmp(); - const offset = B.tmp(); + declare(`declare ptr @scr_record_view_slot(ptr, ${sizeType}, ${sizeType})`); const viewed = B.tmp(); B.line( - `${tablePtr} = getelementptr inbounds %${struct}, ptr ${obj}, i64 0, i32 ${words.offsets}`, + `${viewed} = call ptr @scr_record_view_slot(ptr ${obj}, ${sizeType} ptrtoint (ptr getelementptr (%${struct}, ptr null, i32 0, i32 ${words.record}) to ${sizeType}), ${sizeType} ${position})`, ); - B.line(`${table} = load ptr, ptr ${tablePtr}`); - B.line(`${offsetPtr} = getelementptr inbounds ${sizeType}, ptr ${table}, i64 ${position}`); - B.line(`${offset} = load ${sizeType}, ptr ${offsetPtr}`); - B.line(`${viewed} = getelementptr inbounds i8, ptr ${record}, ${sizeType} ${offset}`); B.br(join); B.startBlock(join); const ptr = B.tmp(); diff --git a/packages/compiler/src/backend/llvm/walkers.ts b/packages/compiler/src/backend/llvm/walkers.ts index c688422ab..47a3c2dd4 100644 --- a/packages/compiler/src/backend/llvm/walkers.ts +++ b/packages/compiler/src/backend/llvm/walkers.ts @@ -293,6 +293,7 @@ export class LlWalkers { index, index - base, recordViewWords(shape.fields.length, base), + (decl) => this.host.declare(decl), ); } else { p = B.tmp(); diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index c90b29419..cc55661e7 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -1325,6 +1325,7 @@ typedef struct { size_t key_len; const size_t *offsets; } ScrRecordViewSource; +void *scr_record_view_slot(void *view, size_t words, size_t position); void *scr_dyn_record_view(ScrDyn *d, const ScrRecordViewSource *sources, size_t count, const char *key, size_t key_len, void *(*alloc)(void), size_t words); ScrDyn *scr_dyn_class_view_unavailable(void *ptr); diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index d094f15db..eba125a7c 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -4415,6 +4415,15 @@ void *scr_dyn_record_view(ScrDyn *d, const ScrRecordViewSource *sources, size_t return view; } +/* The slot of field `position` of a view (emitted view guards, cold): the + * viewed record's slot at its offset table's entry. `words` is the byte + * offset of the view's two trailing words. */ +void *scr_record_view_slot(void *view, size_t words, size_t position) { + void **slots = (void **)((char *)view + words); + const size_t *offsets = slots[1]; + return (char *)slots[0] + offsets[position]; +} + /* A view itself counts like any record (its target's retain). */ static void *scr_record_view_retain(void *view) { if (!scr_rc_immortal(view)) SCR_RC_COUNT(view)++; From 47dfa71ab19641a1fe4d16627ed82454adad8cd7 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Tue, 27 Oct 2026 03:30:28 -0700 Subject: [PATCH 28/54] Keep the view guard to one value in unoptimized builds Unoptimized builds give every value a frame slot, so the inline guard grew the frames of large recursive functions enough for the sanitized self-rebuild to run out of stack. Development builds now call scr_record_view_field for the slot address; optimized builds test the view mark as one range (the frozen bit plus a count) before the cold lookup. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/backend/llvm/emitter.ts | 1 + .../compiler/src/backend/llvm/record-views.ts | 28 ++++++++++++++----- packages/compiler/src/backend/llvm/walkers.ts | 1 + packages/runtime/src/scr_dyn.h | 1 + packages/runtime/src/scr_json.c | 13 +++++++++ 5 files changed, 37 insertions(+), 7 deletions(-) diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 059190eb5..928742b2b 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -4880,6 +4880,7 @@ export class LlEmitter { idx, recordViewWords(shape.fields.length, base), (decl) => this.declare(decl), + this.rcMode === undefined, ); } else { p = this.B.tmp(); diff --git a/packages/compiler/src/backend/llvm/record-views.ts b/packages/compiler/src/backend/llvm/record-views.ts index 92238fd54..052b9f5c8 100644 --- a/packages/compiler/src/backend/llvm/record-views.ts +++ b/packages/compiler/src/backend/llvm/record-views.ts @@ -132,17 +132,19 @@ export function planRecordViews(mod: IrModule, options: ViewOptions): RecordView /** Lines setting `dest` (i1) to whether the record `obj` of a view target * type is a view. A view's rc word carries the frozen bit (scr_rc.h * SCR_RC_FROZEN), which no other value of such a type has: view targets - * are never freezable, and an immortal word (all ones) is not a view. The + * are never freezable, and a word with the program-lifetime bit (an + * immortal one included) is no view, so a view's word is the frozen bit + * plus a count. The * trailing words are read only after this test, so a value of another * storage seen through the type is never read past its end. */ export function recordViewTestLines(sizeType: "i32" | "i64", obj: string, dest: string): string[] { - const bit = sizeType === "i64" ? "4611686018427387904" : "1073741824"; + // The frozen bit with a count and no program-lifetime bit: one range. + const [bit, span] = + sizeType === "i64" ? ["4611686018427387904", "4294967296"] : ["1073741824", "1073741824"]; return [ ` ${dest}.w = load ${sizeType}, ptr ${obj}`, - ` ${dest}.b = and ${sizeType} ${dest}.w, ${bit}`, - ` ${dest}.m = icmp ne ${sizeType} ${dest}.b, 0`, - ` ${dest}.i = icmp ne ${sizeType} ${dest}.w, -1`, - ` ${dest} = and i1 ${dest}.m, ${dest}.i`, + ` ${dest}.d = sub ${sizeType} ${dest}.w, ${bit}`, + ` ${dest} = icmp ult ${sizeType} ${dest}.d, ${span}`, ]; } @@ -166,9 +168,21 @@ export function recordViewFieldPtr( position: number, words: { record: number; offsets: number }, declare: (decl: string) => void, + outOfLine: boolean, ): string { const own = B.tmp(); B.line(`${own} = getelementptr inbounds %${struct}, ptr ${obj}, i64 0, i32 ${slotIndex}`); + const wordsOffset = `${sizeType} ptrtoint (ptr getelementptr (%${struct}, ptr null, i32 0, i32 ${words.record}) to ${sizeType})`; + if (outOfLine) { + // Unoptimized builds give every value a frame slot: one call keeps + // the guard to a single value per access. + declare(`declare ptr @scr_record_view_field(ptr, ptr, ${sizeType}, ${sizeType})`); + const ptr = B.tmp(); + B.line( + `${ptr} = call ptr @scr_record_view_field(ptr ${obj}, ptr ${own}, ${wordsOffset}, ${sizeType} ${position})`, + ); + return ptr; + } const isView = B.tmp(); for (const line of recordViewTestLines(sizeType, obj, isView)) B.line(line.trimStart()); const from = B.currentLabel(); @@ -180,7 +194,7 @@ export function recordViewFieldPtr( declare(`declare ptr @scr_record_view_slot(ptr, ${sizeType}, ${sizeType})`); const viewed = B.tmp(); B.line( - `${viewed} = call ptr @scr_record_view_slot(ptr ${obj}, ${sizeType} ptrtoint (ptr getelementptr (%${struct}, ptr null, i32 0, i32 ${words.record}) to ${sizeType}), ${sizeType} ${position})`, + `${viewed} = call ptr @scr_record_view_slot(ptr ${obj}, ${wordsOffset}, ${sizeType} ${position})`, ); B.br(join); B.startBlock(join); diff --git a/packages/compiler/src/backend/llvm/walkers.ts b/packages/compiler/src/backend/llvm/walkers.ts index 47a3c2dd4..8c32558c2 100644 --- a/packages/compiler/src/backend/llvm/walkers.ts +++ b/packages/compiler/src/backend/llvm/walkers.ts @@ -294,6 +294,7 @@ export class LlWalkers { index - base, recordViewWords(shape.fields.length, base), (decl) => this.host.declare(decl), + this.host.rcMode === undefined, ); } else { p = B.tmp(); diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index cc55661e7..c77218de2 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -1326,6 +1326,7 @@ typedef struct { const size_t *offsets; } ScrRecordViewSource; void *scr_record_view_slot(void *view, size_t words, size_t position); +void *scr_record_view_field(void *value, void *own, size_t words, size_t position); void *scr_dyn_record_view(ScrDyn *d, const ScrRecordViewSource *sources, size_t count, const char *key, size_t key_len, void *(*alloc)(void), size_t words); ScrDyn *scr_dyn_class_view_unavailable(void *ptr); diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index eba125a7c..12e35df93 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -4424,6 +4424,19 @@ void *scr_record_view_slot(void *view, size_t words, size_t position) { return (char *)slots[0] + offsets[position]; } +/* The address of field `position` of a value of a view target type, `own` + * being its own slot (the unoptimized builds' guard): the viewed record's + * slot for a view, whose word is the frozen bit plus a count. */ +void *scr_record_view_field(void *value, void *own, size_t words, size_t position) { + size_t word = *(const size_t *)value; +#if SIZE_MAX > UINT32_MAX + bool view = word - SCR_RC_FROZEN < ((size_t)1 << 32); +#else + bool view = word - SCR_RC_FROZEN < SCR_RC_FROZEN; +#endif + return view ? scr_record_view_slot(value, words, position) : own; +} + /* A view itself counts like any record (its target's retain). */ static void *scr_record_view_retain(void *view) { if (!scr_rc_immortal(view)) SCR_RC_COUNT(view)++; From 85d2a63410eb1ef3677559b717583239ff818096 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Tue, 27 Oct 2026 04:39:28 -0700 Subject: [PATCH 29/54] Keep a view's viewed record in its record-source entry instead of trailing words View target structs no longer grow: every stack copy of such a record grew by the two words, which enlarged unoptimized frames enough for the sanitized self-rebuild to run out of stack. The guard's cold path looks the slot up through the view's record-source entry (scr_record_view_slot), which now holds the viewed record and its offsets, and stack objects of target types are allowed again. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/backend/llvm/dyn.ts | 11 +--- packages/compiler/src/backend/llvm/emitter.ts | 3 +- .../compiler/src/backend/llvm/record-views.ts | 62 +++++++------------ packages/compiler/src/backend/llvm/shapes.ts | 14 +---- .../src/backend/llvm/stack-objects.ts | 3 - packages/compiler/src/backend/llvm/walkers.ts | 4 +- packages/runtime/src/scr_dyn.h | 6 +- packages/runtime/src/scr_json.c | 47 +++++++------- 8 files changed, 58 insertions(+), 92 deletions(-) diff --git a/packages/compiler/src/backend/llvm/dyn.ts b/packages/compiler/src/backend/llvm/dyn.ts index 5bc71a331..dfeffdecb 100644 --- a/packages/compiler/src/backend/llvm/dyn.ts +++ b/packages/compiler/src/backend/llvm/dyn.ts @@ -1,6 +1,5 @@ import { emitPerKeyOrder } from "./key-orders.js"; import { typedRefConstructor } from "./shapes.js"; -import { recordViewWords } from "./record-views.js"; import { bindLiveArrayOps } from "./expr-stream-bridges.js"; import { BYTES_ELEM_NUM, @@ -1161,7 +1160,7 @@ export class LlDyn { ); } - /** `scr_dyn_record_view(d, sources, n, key, key_len, new, words)` for + /** `scr_dyn_record_view(d, sources, n, key, key_len, new)` for * view target `targetId`: the view (+1) of the record a capsule `%d` * holds when its type is one of the target's sources, else NULL. The * sources table lists each source's type key and the byte offsets of @@ -1196,14 +1195,10 @@ export class LlDyn { ); } const key = typeKey({ kind: "record", shapeId: targetId }); - const words = recordViewWords(target.fields.length, host.nullableUnions.fieldBase(targetId)); - const struct = mangleRecordStruct(targetId); - host.declare( - `declare ptr @scr_dyn_record_view(ptr, ptr, ${this.S}, ptr, ${this.S}, ptr, ${this.S})`, - ); + host.declare(`declare ptr @scr_dyn_record_view(ptr, ptr, ${this.S}, ptr, ${this.S}, ptr)`); const view = B.tmp(); B.line( - `${view} = call ptr @scr_dyn_record_view(ptr %d, ptr ${table}, ${this.S} ${sources.length}, ptr ${host.cstr(key)}, ${this.S} ${Buffer.byteLength(key, "utf8")}, ptr @${mangleRecordNew(targetId)}, ${this.S} ptrtoint (ptr getelementptr (%${struct}, ptr null, i32 0, i32 ${words.record}) to ${this.S}))`, + `${view} = call ptr @scr_dyn_record_view(ptr %d, ptr ${table}, ${this.S} ${sources.length}, ptr ${host.cstr(key)}, ${this.S} ${Buffer.byteLength(key, "utf8")}, ptr @${mangleRecordNew(targetId)})`, ); return view; } diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 928742b2b..4f8a3ec29 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -357,7 +357,7 @@ import { import { children, type Node } from "./region-ir.js"; import { LlDyn, recordDynKeepsIdentity, type DynHost } from "./dyn.js"; import { VirtualBorrows } from "./virtual-borrows.js"; -import { planRecordViews, recordViewFieldPtr, recordViewWords } from "./record-views.js"; +import { planRecordViews, recordViewFieldPtr } from "./record-views.js"; import { LlvmUnsupportedError } from "./unsupported.js"; import { LlWalkers } from "./walkers.js"; import { isObjectArm, NULLABLE_ABSENT, NULLABLE_NULL, NullableUnions } from "./nullable-unions.js"; @@ -4878,7 +4878,6 @@ export class LlEmitter { objName, idx + base, idx, - recordViewWords(shape.fields.length, base), (decl) => this.declare(decl), this.rcMode === undefined, ); diff --git a/packages/compiler/src/backend/llvm/record-views.ts b/packages/compiler/src/backend/llvm/record-views.ts index 052b9f5c8..d11cb3170 100644 --- a/packages/compiler/src/backend/llvm/record-views.ts +++ b/packages/compiler/src/backend/llvm/record-views.ts @@ -2,21 +2,20 @@ * * A checked conversion of a capsule of a native record A into another * record type B whose fields A also has (same names and types, any - * offsets) answers a view: a B-shaped object whose fields live in A. The - * view's storage is B's struct plus two trailing words, the A record - * (borrowed: the view's record-source entry owns A's capsule, which owns - * A) and a table of A's byte offsets for B's fields, and its rc word - * carries the frozen bit as the view mark. Every field address of a B - * value tests the mark (recordViewFieldPtr), so reads and - * writes through either name reach the same storage, and boxing a view - * answers A's capsule (scr_dyn_record_view), so identity holds too. + * offsets) answers a view: a B object, its rc word marked with the frozen + * bit, whose record-source entry names A and A's byte offsets for B's + * fields (scr_dyn_record_view). Every field address of a B value tests + * the mark (recordViewFieldPtr) and takes A's slot for a view, so reads + * and writes through either name reach the same storage, and boxing a + * view answers A's capsule, so identity holds too. B's storage itself is + * unchanged. * - * Only the target types of such conversions carry the words and the - * test. They are found from the IR, before any code is emitted: B is a - * record type a checked conversion can produce (the closure of dynCheck - * targets), and some other record type whose values reach untyped code as - * live capsules (the closure of live dynFrom types) has all of B's fields. A program without checked - * conversions has none, so its code is unchanged. */ + * Only the target types of such conversions carry the test. They are + * found from the IR, before any code is emitted: B is a record type a + * checked conversion can produce (the closure of dynCheck targets), and + * some other record type whose values reach untyped code as live capsules + * (the closure of live dynFrom types) has all of B's fields. A program + * without checked conversions has none, so its code is unchanged. */ import { isDirentShape, KEY_ORDER_FIELD, @@ -135,8 +134,7 @@ export function planRecordViews(mod: IrModule, options: ViewOptions): RecordView * are never freezable, and a word with the program-lifetime bit (an * immortal one included) is no view, so a view's word is the frozen bit * plus a count. The - * trailing words are read only after this test, so a value of another - * storage seen through the type is never read past its end. */ + */ export function recordViewTestLines(sizeType: "i32" | "i64", obj: string, dest: string): string[] { // The frozen bit with a count and no program-lifetime bit: one range. const [bit, span] = @@ -148,17 +146,11 @@ export function recordViewTestLines(sizeType: "i32" | "i64", obj: string, dest: ]; } -/** The struct indices of a view target's two trailing words. */ -export function recordViewWords( - fieldCount: number, - base: number, -): { record: number; offsets: number } { - return { record: fieldCount + base, offsets: fieldCount + base + 1 }; -} - /** The address of field `position` (declaration index) of the record - * `obj` of view-target shape `struct`: the slot itself, or the source - * record's slot when `obj` is a view. Leaves `B` in a fresh block. */ + * `obj` of view-target shape `struct`: the slot itself, or the viewed + * record's slot when `obj` is a view. Optimized builds test the mark + * inline and look the slot up out of line; unoptimized builds, which give + * every value a frame slot, make the address one call. */ export function recordViewFieldPtr( B: BlockBuilder, sizeType: "i32" | "i64", @@ -166,23 +158,19 @@ export function recordViewFieldPtr( obj: string, slotIndex: number, position: number, - words: { record: number; offsets: number }, declare: (decl: string) => void, outOfLine: boolean, ): string { - const own = B.tmp(); - B.line(`${own} = getelementptr inbounds %${struct}, ptr ${obj}, i64 0, i32 ${slotIndex}`); - const wordsOffset = `${sizeType} ptrtoint (ptr getelementptr (%${struct}, ptr null, i32 0, i32 ${words.record}) to ${sizeType})`; if (outOfLine) { - // Unoptimized builds give every value a frame slot: one call keeps - // the guard to a single value per access. - declare(`declare ptr @scr_record_view_field(ptr, ptr, ${sizeType}, ${sizeType})`); + declare(`declare ptr @scr_record_view_field(ptr, ${sizeType}, ${sizeType})`); const ptr = B.tmp(); B.line( - `${ptr} = call ptr @scr_record_view_field(ptr ${obj}, ptr ${own}, ${wordsOffset}, ${sizeType} ${position})`, + `${ptr} = call ptr @scr_record_view_field(ptr ${obj}, ${sizeType} ptrtoint (ptr getelementptr (%${struct}, ptr null, i32 0, i32 ${slotIndex}) to ${sizeType}), ${sizeType} ${position})`, ); return ptr; } + const own = B.tmp(); + B.line(`${own} = getelementptr inbounds %${struct}, ptr ${obj}, i64 0, i32 ${slotIndex}`); const isView = B.tmp(); for (const line of recordViewTestLines(sizeType, obj, isView)) B.line(line.trimStart()); const from = B.currentLabel(); @@ -191,11 +179,9 @@ export function recordViewFieldPtr( // Views are rare: the slot lookup stays out of line. B.condBr(isView, view, join, `, !prof !{!"branch_weights", i32 1, i32 1000}`); B.startBlock(view); - declare(`declare ptr @scr_record_view_slot(ptr, ${sizeType}, ${sizeType})`); + declare(`declare ptr @scr_record_view_slot(ptr, ${sizeType})`); const viewed = B.tmp(); - B.line( - `${viewed} = call ptr @scr_record_view_slot(ptr ${obj}, ${wordsOffset}, ${sizeType} ${position})`, - ); + B.line(`${viewed} = call ptr @scr_record_view_slot(ptr ${obj}, ${sizeType} ${position})`); B.br(join); B.startBlock(join); const ptr = B.tmp(); diff --git a/packages/compiler/src/backend/llvm/shapes.ts b/packages/compiler/src/backend/llvm/shapes.ts index c5d4526bc..69ca911d3 100644 --- a/packages/compiler/src/backend/llvm/shapes.ts +++ b/packages/compiler/src/backend/llvm/shapes.ts @@ -69,7 +69,7 @@ import { disposeFieldLines, plainRefType, } from "./arenas.js"; -import { recordViewTestLines, recordViewWords, type RecordViews } from "./record-views.js"; +import { recordViewTestLines, type RecordViews } from "./record-views.js"; import { recordMayKeepSource } from "./record-sources.js"; import { LlvmUnsupportedError } from "./unsupported.js"; @@ -2179,8 +2179,6 @@ export function emitRecordShapes( llFieldType(f.type, host.inlineUnions.record(f.type)), ); if (shape.indexValue) fieldTys.push("ptr"); // the overflow ScrMap * - // A record view target: the viewed record and its offset table. - if (host.recordViews?.targets.has(shape.id) === true) fieldTys.push("ptr", "ptr"); // A tagged shape (an arm of a shape union) carries its shape number. if (host.nullableUnions.shapeNumber(shape.id) !== 0) fieldTys.unshift("i32"); typeDefs.push( @@ -2403,19 +2401,13 @@ export function emitRecordShapes( const fieldTy = llFieldType(field.type, inline); if (host.recordViews?.targets.has(shape.id) === true) { // A view's fields live in the viewed record (record-views.ts). - const words = recordViewWords(shape.fields.length, base); + host.declare(`declare ptr @scr_record_view_slot(ptr, ${host.sizeType})`); clone.push( ` %so${i} = getelementptr inbounds %${struct}, ptr %src, i64 0, i32 ${index}`, ...recordViewTestLines(host.sizeType, "%src", `%sr${i}v`), ` br i1 %sr${i}v, label %sv${i}.view, label %sv${i}.join`, `sv${i}.view:`, - ` %sr${i}p = getelementptr inbounds %${struct}, ptr %src, i64 0, i32 ${words.record}`, - ` %sr${i}r = load ptr, ptr %sr${i}p`, - ` %st${i}p = getelementptr inbounds %${struct}, ptr %src, i64 0, i32 ${words.offsets}`, - ` %st${i} = load ptr, ptr %st${i}p`, - ` %sx${i}p = getelementptr inbounds ${host.sizeType}, ptr %st${i}, i64 ${i}`, - ` %sx${i} = load ${host.sizeType}, ptr %sx${i}p`, - ` %sw${i} = getelementptr inbounds i8, ptr %sr${i}r, ${host.sizeType} %sx${i}`, + ` %sw${i} = call ptr @scr_record_view_slot(ptr %src, ${host.sizeType} ${i})`, ` br label %sv${i}.join`, `sv${i}.join:`, ` %sp${i} = phi ptr [ %so${i}, %${i === 0 ? "entry" : `sv${i - 1}.join`} ], [ %sw${i}, %sv${i}.view ]`, diff --git a/packages/compiler/src/backend/llvm/stack-objects.ts b/packages/compiler/src/backend/llvm/stack-objects.ts index eaba2d437..36793598b 100644 --- a/packages/compiler/src/backend/llvm/stack-objects.ts +++ b/packages/compiler/src/backend/llvm/stack-objects.ts @@ -54,9 +54,6 @@ export function canStackObject(host: LlvmEmitterContext, e: IrExpr): e is StackO return false; if (e.type.kind === "record" && host.publishedTypes?.recordGuarded(e.type.shapeId) === true) return false; - // A record view target's frame copy would need its view words zeroed. - if (e.type.kind === "record" && host.shapeHost.recordViews?.targets.has(e.type.shapeId) === true) - return false; if (e.type.kind === "record" && host.recordsById.get(e.type.shapeId)?.freezable === true) return false; // A producer with a result-slot entry builds its fresh result in the diff --git a/packages/compiler/src/backend/llvm/walkers.ts b/packages/compiler/src/backend/llvm/walkers.ts index 8c32558c2..f475f073b 100644 --- a/packages/compiler/src/backend/llvm/walkers.ts +++ b/packages/compiler/src/backend/llvm/walkers.ts @@ -1,5 +1,5 @@ import { emitPerKeyOrder } from "./key-orders.js"; -import { recordViewFieldPtr, recordViewWords } from "./record-views.js"; +import { recordViewFieldPtr } from "./record-views.js"; import { f64Lit } from "./common.js"; import { InternalCompilerError } from "../../errors.js"; /* Structure-walking helper EMITTERS for the LLVM backend — the .ll mirror @@ -281,7 +281,6 @@ export class LlWalkers { index: number, t: IrType, ): string { - const shape = this.host.recordsById.get(shapeId)!; const base = this.host.nullableUnions.fieldBase(shapeId); let p: string; if (this.host.recordViews?.targets.has(shapeId) === true) { @@ -292,7 +291,6 @@ export class LlWalkers { recName, index, index - base, - recordViewWords(shape.fields.length, base), (decl) => this.host.declare(decl), this.host.rcMode === undefined, ); diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index c77218de2..85ecef504 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -1325,10 +1325,10 @@ typedef struct { size_t key_len; const size_t *offsets; } ScrRecordViewSource; -void *scr_record_view_slot(void *view, size_t words, size_t position); -void *scr_record_view_field(void *value, void *own, size_t words, size_t position); +void *scr_record_view_slot(void *view, size_t position); +void *scr_record_view_field(void *value, size_t offset, size_t position); void *scr_dyn_record_view(ScrDyn *d, const ScrRecordViewSource *sources, size_t count, - const char *key, size_t key_len, void *(*alloc)(void), size_t words); + const char *key, size_t key_len, void *(*alloc)(void)); ScrDyn *scr_dyn_class_view_unavailable(void *ptr); ScrDyn *scr_dyn_typed_ref_materialize(const ScrDyn *d); /* +1 */ ScrDyn *scr_dyn_typed_ref_peek(const ScrDyn *d); /* +1 */ diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 12e35df93..018fde0f3 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -3751,6 +3751,10 @@ typedef struct { * surfaces go through it (scr_record_source_has). NULL for an ordinary * source. */ ScrDyn *capsule; + /* A view's record (borrowed: the capsule holds it) and its byte offsets + * for the view type's fields. */ + void *viewed; + const size_t *offsets; } ScrRecordSource; static SCR_TL ScrRecordSource *scr_record_sources; static SCR_TL size_t scr_record_source_capacity, scr_record_source_count; @@ -4374,18 +4378,18 @@ void *scr_dyn_typed_ref_unbox(const ScrDyn *d) { /* Record views (compiler record-views.ts). A checked conversion of a * capsule of record A into record type B, when A has all of B's fields, * answers a view (+1): a B object (allocated by `alloc`, zeroed, its rc - * word marked with the frozen bit) whose trailing words at byte offset - * `words` hold A (borrowed) and A's offset table for B's fields, so every field access of the B value reaches A. - * `sources` lists, for each A the compiler allows, its type key and offset - * table. The view's record-source entry owns A's capsule: boxing the view - * answers that capsule, and the entry dies with the view. One view per - * capsule and B, found again through the capsule's cast list, where it is - * borrowed. NULL for any other capsule and for a frozen A, whose + * word marked with the frozen bit) whose record-source entry names A + * (borrowed) and A's byte offsets for B's fields, so every field access of + * the B value reaches A. `sources` lists, for each A the compiler allows, + * its type key and offset table. The entry owns A's capsule: boxing the + * view answers that capsule, and the entry dies with the view. One view + * per capsule and B, found again through the capsule's cast list, where it + * is borrowed. NULL for any other capsule and for a frozen A, whose * conversion keeps its copy. */ static void *scr_record_view_retain(void *view); void *scr_dyn_record_view(ScrDyn *d, const ScrRecordViewSource *sources, size_t count, - const char *key, size_t key_len, void *(*alloc)(void), size_t words) { + const char *key, size_t key_len, void *(*alloc)(void)) { if (d->kind != SCR_DYN_TYPED_REF) return NULL; const ScrRecordViewSource *source = NULL; for (size_t i = 0; i < count && !source; i++) @@ -4401,10 +4405,8 @@ void *scr_dyn_record_view(ScrDyn *d, const ScrRecordViewSource *sources, size_t /* The frozen bit marks a view: view target types are never freezable, * and their emitted releases ignore the bit (record-views.ts). */ *(size_t *)view |= SCR_RC_FROZEN; - void **slots = (void **)((char *)view + words); - slots[0] = record; - slots[1] = (void *)source->offsets; - ScrRecordSource entry = { .rec = view, .capsule = scr_dyn_retain(d) }; + ScrRecordSource entry = { .rec = view, .capsule = scr_dyn_retain(d), .viewed = record, + .offsets = source->offsets }; scr_record_source_put(&entry); ScrDynTypedCast *cast = malloc(sizeof *cast); if (!cast) scr_trap("scriptc: out of memory\n"); @@ -4415,26 +4417,23 @@ void *scr_dyn_record_view(ScrDyn *d, const ScrRecordViewSource *sources, size_t return view; } -/* The slot of field `position` of a view (emitted view guards, cold): the - * viewed record's slot at its offset table's entry. `words` is the byte - * offset of the view's two trailing words. */ -void *scr_record_view_slot(void *view, size_t words, size_t position) { - void **slots = (void **)((char *)view + words); - const size_t *offsets = slots[1]; - return (char *)slots[0] + offsets[position]; +/* The slot of field `position` of a view (emitted view guards, cold). */ +void *scr_record_view_slot(void *view, size_t position) { + ScrRecordSource *entry = scr_record_source_slot(view); + return (char *)entry->viewed + entry->offsets[position]; } -/* The address of field `position` of a value of a view target type, `own` - * being its own slot (the unoptimized builds' guard): the viewed record's - * slot for a view, whose word is the frozen bit plus a count. */ -void *scr_record_view_field(void *value, void *own, size_t words, size_t position) { +/* The address of field `position` of a value of a view target type, whose + * own slot is at byte `offset` (the unoptimized builds' guard): the viewed + * record's slot for a view, whose word is the frozen bit plus a count. */ +void *scr_record_view_field(void *value, size_t offset, size_t position) { size_t word = *(const size_t *)value; #if SIZE_MAX > UINT32_MAX bool view = word - SCR_RC_FROZEN < ((size_t)1 << 32); #else bool view = word - SCR_RC_FROZEN < SCR_RC_FROZEN; #endif - return view ? scr_record_view_slot(value, words, position) : own; + return view ? scr_record_view_slot(value, position) : (char *)value + offset; } /* A view itself counts like any record (its target's retain). */ From 660ddfd90ed3078aed5d1dd5aa19f90763f6b0ec Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 08:41:05 -0500 Subject: [PATCH 30/54] Keep the view-header test's loops speculative Module number globals are now proven int32 through their writes, so the test's generator state starts from a value no analysis bounds to keep its loops on the speculative path. Format int32-slots.ts. --- packages/compiler/src/ir/int32-slots.ts | 3 +-- packages/compiler/test/speculative-loop-emission.test.ts | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/compiler/src/ir/int32-slots.ts b/packages/compiler/src/ir/int32-slots.ts index 465f6c683..6d234831d 100644 --- a/packages/compiler/src/ir/int32-slots.ts +++ b/packages/compiler/src/ir/int32-slots.ts @@ -1722,8 +1722,7 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}): if (s.kind === "assign") { boxWrite(s.localId, s.value); globalWrite(s.localId, s.value); - } - else if (s.kind === "varDecl" && s.init) boxWrite(s.localId, s.init); + } else if (s.kind === "varDecl" && s.init) boxWrite(s.localId, s.init); if (s.kind === "fieldSet") { const family = slots.family(s.className, s.field); if (!family || !slots.hasFamily(family)) return; diff --git a/packages/compiler/test/speculative-loop-emission.test.ts b/packages/compiler/test/speculative-loop-emission.test.ts index dcec6d8af..52b3c0c07 100644 --- a/packages/compiler/test/speculative-loop-emission.test.ts +++ b/packages/compiler/test/speculative-loop-emission.test.ts @@ -188,9 +188,10 @@ test("a speculative loop reads its views' header fields once, before its copies" // A view held by a const local or by a parameter is the same view in // every iteration. Its data pointer is read before the branch to the fast // copy, and no copy (ordinary, fast, or the one a failed check resumes - // in) reads it again. + // in) reads it again. `last` starts from a value no analysis bounds, + // so the loops speculate on `l`. const llvm = await emitLlvm(` -let last = 42; +let last = Number(process.argv[2] ?? "42"); function fill(array: ArrayBuffer, count: number): void { const buf = new Float64Array(array); let l = last; From aed7a2070958bb165b5db950768eeef657fe4d4f Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 09:03:39 -0500 Subject: [PATCH 31/54] Let units without a record change the helpers they name in place The serial mode's wave-end check threw when a unit changed a held function in place, even when the unit had no record and its change was its own lowering's: two functions that define computed-key properties on one class both rebuild the class's descriptor guard. Each unit's in-place changes are now checked against the names its IR and new entries quote: a unit with a record that changed such an entry has no record, and a unit without one keeps its changes. The wave-end check throws only for a change to an entry no unit names. --- .../frontend/lowering/unit-journal.test.ts | 44 ++++++- .../src/frontend/lowering/unit-journal.ts | 116 ++++++++++++++---- .../compiler/test/recorded-lowering.test.ts | 2 + .../class-computed-key-descriptors.ts.json | 6 + .../corpus/class-computed-key-descriptors.ts | 19 +++ .../class-computed-key-descriptors.ts.json | 1 + 6 files changed, 165 insertions(+), 23 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/class-computed-key-descriptors.ts.json create mode 100644 tests/corpus/class-computed-key-descriptors.ts create mode 100644 tests/harness/corpus-costs/class-computed-key-descriptors.ts.json diff --git a/packages/compiler/src/frontend/lowering/unit-journal.test.ts b/packages/compiler/src/frontend/lowering/unit-journal.test.ts index ff47e5c0a..439e18fd7 100644 --- a/packages/compiler/src/frontend/lowering/unit-journal.test.ts +++ b/packages/compiler/src/frontend/lowering/unit-journal.test.ts @@ -297,6 +297,48 @@ test("a recorded wave throws when a unit changes a held function in place", () = }; lowerWaveRecorded(subject, ["a"], waveStats()); expect(() => lowerWaveRecorded(subject, ["a", "b"], waveStats())).toThrow( - "a unit changed lifted entry %shared in place", + "a unit changed lifted entry %shared in place without naming it", ); }); + +/** A wave host over one function table holding `%shared` and one checked + * list, whose units change `%shared` in place and name it in their IR. */ +function sharedHost(grows: (name: string) => boolean): { host: WaveHost; checked: string[] } { + const functions = [fn("%shared")]; + const checked: string[] = []; + return { + checked, + host: { + tables: [ + functionTable("lifted", functions, (name) => functions.some((f) => f.name === name)), + checkedTable("checked", () => checked.length), + ], + scalars: [], + lowerRecorded: (name) => { + functions[0]!.body = [{ kind: "exprStmt", note: name } as never]; + if (grows(name)) checked.push(name); + return { + fn: { ...fn(`%${name}`), body: [{ kind: "call", callee: "%shared" } as never] }, + poisoned: false, + }; + }, + store: () => {}, + statementCounts: () => ({ total: 0, failed: 0, other: 0 }), + resolves: () => true, + }, + }; +} + +test("units without a record may change a function they name in place", () => { + const { host: subject } = sharedHost(() => true); + const stats = waveStats(); + lowerWaveRecorded(subject, ["a", "b"], stats); + expect(stats).toMatchObject({ recorded: 0, unrecorded: 2, reasons: { checked: 2 } }); +}); + +test("a unit that changes a function it names in place has no record", () => { + const { host: subject } = sharedHost(() => false); + const stats = waveStats(); + lowerWaveRecorded(subject, ["a"], stats); + expect(stats).toMatchObject({ recorded: 0, unrecorded: 1, reasons: { "lifted:changed": 1 } }); +}); diff --git a/packages/compiler/src/frontend/lowering/unit-journal.ts b/packages/compiler/src/frontend/lowering/unit-journal.ts index abb9d90b4..0992e7544 100644 --- a/packages/compiler/src/frontend/lowering/unit-journal.ts +++ b/packages/compiler/src/frontend/lowering/unit-journal.ts @@ -102,6 +102,31 @@ export interface HeldContents { extend(): void; /** The key of an entry this holds whose content has changed, or null. */ changed(): string | null; + /** The key of an entry among `keys` whose content has changed, or null. */ + changedAmong(keys: ReadonlySet): string | null; + /** Takes the current content of the entries among `keys`. */ + refresh(keys: ReadonlySet): void; +} + +/** The strings quoted in JSON `texts`, unescaped only as far as their + * delimiting quotes: the names a unit's encoded IR and entries refer to. */ +export function quotedStrings(texts: readonly string[]): Set { + const out = new Set(); + for (const text of texts) { + let start = -1; + for (let index = 0; index < text.length; index++) { + const char = text.charCodeAt(index); + if (char === 92 /* \\ */) index++; + else if (char === 34 /* " */) { + if (start < 0) start = index + 1; + else { + out.add(text.slice(start, index)); + start = -1; + } + } + } + } + return out; } /** A table's entries at one point (JournalTable.hold). */ @@ -237,7 +262,11 @@ export interface RecordedUnit { * entries it held before, every field is as it was or a flag that was set, * statement totals grew by the recorded statements alone, no diagnostic was * emitted, every event resolves and the function encodes. */ -export function recordUnit(host: WaveHost, name: string): RecordedUnit { +export function recordUnit( + host: WaveHost, + name: string, + contents: readonly (HeldContents | null)[] | null = null, +): RecordedUnit { const tables = host.tables; const before = tables.map((table) => table.size()); const held = tables.map((table) => (table.hold === undefined ? null : table.hold())); @@ -281,6 +310,26 @@ export function recordUnit(host: WaveHost, name: string): RecordedUnit { if (entries === null) reasons.push(`${table.name}:encoding`); else added[table.name] = entries; } + if (contents !== null) { + // A unit changes in place the entries it uses, which its IR and new + // entries name: a unit with a record must leave them as they were, and + // a unit without one keeps its changes. + const texts: string[] = []; + if (text !== null) texts.push(text); + for (const index of grown) { + const table = tables[index]!; + const entries = added[table.name] ?? table.entries!(before[index]!); + if (entries !== null) for (const entry of entries) texts.push(entry); + } + const names = quotedStrings(texts); + if (reasons.length === 0) + tables.forEach((table, index) => { + if ((contents[index] ?? null)?.changedAmong(names) != null) + reasons.push(`${table.name}:changed`); + }); + if (reasons.length > 0) for (const held of contents) held?.refresh(names); + for (const held of contents) held?.extend(); + } if (reasons.length > 0) return { record: null, reasons, fn, before }; return { record: { name, fn: text, poisoned, events, tables: added }, reasons, fn, before }; } @@ -305,8 +354,7 @@ export function lowerWaveRecorded( // started: a record carries a unit's new entries, not changes to others. const contents = tables.map((table) => (table.contents === undefined ? null : table.contents())); for (const name of wave) { - const unit = recordUnit(host, name); - for (const held of contents) held?.extend(); + const unit = recordUnit(host, name, contents); stats.units++; const record = unit.record; if (record === null) { @@ -330,7 +378,7 @@ export function lowerWaveRecorded( const key = contents[index]?.changed() ?? null; if (key !== null) throw new InternalCompilerError( - `a unit changed ${table.name} entry ${key} in place, which no unit record carries`, + `a unit changed ${table.name} entry ${key} in place without naming it, which no unit record carries`, ); }); } @@ -367,25 +415,30 @@ export function irDataTable(name: string, map: Map): JournalTable return { ...codecTable(name, map, encode, (text) => JSON.parse(text) as V), contents: () => { - const keys: string[] = []; - const texts: (string | null)[] = []; + const texts = new Map(); + const differs = (key: string): boolean => { + const value = map.get(key); + return value === undefined || encode(value) !== texts.get(key); + }; const held: HeldContents = { extend: () => { - let index = 0; - for (const [key, value] of map) { - if (index++ < keys.length) continue; - keys.push(key); - texts.push(encode(value)); - } + if (texts.size === map.size) return; + for (const [key, value] of map) if (!texts.has(key)) texts.set(key, encode(value)); }, changed: () => { - for (let index = 0; index < keys.length; index++) { - const key = keys[index]!; - const value = map.get(key); - if (value === undefined || encode(value) !== texts[index]) return key; - } + for (const key of texts.keys()) if (differs(key)) return key; return null; }, + changedAmong: (keys) => { + for (const key of keys) if (texts.has(key) && differs(key)) return key; + return null; + }, + refresh: (keys) => { + for (const key of keys) { + const value = map.get(key); + if (texts.has(key) && value !== undefined) texts.set(key, encode(value)); + } + }, }; held.extend(); return held; @@ -515,17 +568,36 @@ export function functionTable( }, contents: () => { const texts: (string | null)[] = []; + const indexes = new Map(); + const differs = (index: number): boolean => + index < functions.length && encodeUnitFunction(functions[index]!) !== texts[index]; const held: HeldContents = { extend: () => { - for (let index = texts.length; index < functions.length; index++) - texts.push(encodeUnitFunction(functions[index]!)); + for (let index = texts.length; index < functions.length; index++) { + const fn = functions[index]!; + texts.push(encodeUnitFunction(fn)); + indexes.set(fn.name, index); + } }, changed: () => { - for (let index = 0; index < texts.length && index < functions.length; index++) - if (encodeUnitFunction(functions[index]!) !== texts[index]) - return functions[index]!.name; + for (let index = 0; index < texts.length; index++) + if (differs(index)) return functions[index]!.name; return null; }, + changedAmong: (keys) => { + for (const key of keys) { + const index = indexes.get(key); + if (index !== undefined && differs(index)) return key; + } + return null; + }, + refresh: (keys) => { + for (const key of keys) { + const index = indexes.get(key); + if (index !== undefined && index < functions.length) + texts[index] = encodeUnitFunction(functions[index]!); + } + }, }; held.extend(); return held; diff --git a/packages/compiler/test/recorded-lowering.test.ts b/packages/compiler/test/recorded-lowering.test.ts index a40f33964..6c9b176f2 100644 --- a/packages/compiler/test/recorded-lowering.test.ts +++ b/packages/compiler/test/recorded-lowering.test.ts @@ -20,6 +20,8 @@ const PROGRAMS = [ "6496-key-order-built-and-parsed.ts", "6916-index-record-spread-overrides.ts", "6951-unknown-cast-class-methods.ts", + // Units without records rebuild a shared helper in place. + "class-computed-key-descriptors.ts", ]; async function lower(program: string, dir: string, label: string): Promise { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/class-computed-key-descriptors.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/class-computed-key-descriptors.ts.json new file mode 100644 index 000000000..898e483df --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/class-computed-key-descriptors.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/class-computed-key-descriptors.ts" + ], + "diags": [] +} diff --git a/tests/corpus/class-computed-key-descriptors.ts b/tests/corpus/class-computed-key-descriptors.ts new file mode 100644 index 000000000..d86c035c4 --- /dev/null +++ b/tests/corpus/class-computed-key-descriptors.ts @@ -0,0 +1,19 @@ +// Two functions define computed-key properties on instances of one class. +class Point { + x = 1; + y = 2; +} + +function defineValue(p: Point, key: string, value: number): void { + Object.defineProperty(p, key, { value, writable: true, enumerable: true, configurable: true }); +} + +function defineHidden(p: Point, key: string): void { + Object.defineProperty(p, key, { value: "hidden", writable: false, enumerable: false, configurable: true }); +} + +const p = new Point(); +defineValue(p, "z", 3); +defineHidden(p, "label"); +console.log(JSON.stringify(p)); +console.log(Object.keys(p).join(",")); diff --git a/tests/harness/corpus-costs/class-computed-key-descriptors.ts.json b/tests/harness/corpus-costs/class-computed-key-descriptors.ts.json new file mode 100644 index 000000000..1c61ba33a --- /dev/null +++ b/tests/harness/corpus-costs/class-computed-key-descriptors.ts.json @@ -0,0 +1 @@ +{ "plain": 5600 } From 18196d4df077f8755d198818b96dff8487349f1b Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 09:36:18 -0500 Subject: [PATCH 32/54] Lay out code by context-sensitive profiles --- docs/content/docs/cli.mdx | 21 +++++- native/llvm-codegen/src/emit.cpp | 64 ++++++++++++++++--- native/llvm-codegen/src/emit.h | 4 +- native/llvm-codegen/src/target.cpp | 8 +++ native/llvm-codegen/src/target.h | 4 ++ packages/cli/test/output-options.test.ts | 11 +++- packages/cli/test/profile.test.ts | 4 -- .../compiler/src/backend/link-plan-core.ts | 5 +- packages/compiler/src/backend/link-plan.ts | 2 + .../src/backend/native-codegen.test.ts | 12 +++- .../src/backend/native-profile.test.ts | 56 +++++++++++++++- .../compiler/src/backend/native-profile.ts | 58 ++++++++++++++--- packages/compiler/src/cli/output-options.ts | 3 - packages/compiler/src/index.ts | 17 ++++- packages/compiler/src/native/compiler.ts | 15 ++++- packages/compiler/test/native-profile.test.ts | 60 +++++++++++++---- 16 files changed, 289 insertions(+), 55 deletions(-) diff --git a/docs/content/docs/cli.mdx b/docs/content/docs/cli.mdx index 734c07da6..f2d91f1bd 100644 --- a/docs/content/docs/cli.mdx +++ b/docs/content/docs/cli.mdx @@ -173,10 +173,10 @@ With `--sparse`, the profile leaves out the functions the runs never reached. Th
On x86-64 targets, compile the program's code twice, for the baseline and for x86-64-v3, and choose the copy when the program starts. The executable runs on any x86-64 CPU and uses the x86-64-v3 code on CPUs that support it. Not used with a --target-cpu level above baseline. Valid only for executable builds. See Runtime CPU dispatch.
--profile-generate <pattern>
-
On Linux, build an instrumented executable for profile-guided optimization. Each run writes a raw profile to the path pattern, where %p expands to the process ID; the LLVM_PROFILE_FILE environment variable overrides the pattern at run time. A relative pattern resolves from the current working directory. Instrumented programs run slower. Valid only for executable builds with --optimization=release or speed.
+
On Linux, build an instrumented executable for profile-guided optimization. Each run writes a raw profile to the path pattern, where %p expands to the process ID; the LLVM_PROFILE_FILE environment variable overrides the pattern at run time. A relative pattern resolves from the current working directory. Instrumented programs run slower. With --profile-use, the build is optimized by that profile and instrumented after inlining, the second phase of a context-sensitive profile. Valid only for executable builds with --optimization=release or speed.
--profile-use <file.profdata>
-
Optimize the executable with a profile from scriptc merge-profiles: code is laid out, inlined, and split by how the profiled runs used it. Code the profile does not describe, or describes from an older version of the program, is optimized as without a profile, so a stale profile only loses part of its gain. A new or changed profile compiles every partition of the program again. scriptc build prints how the profile matched the program's functions on stderr; see Profile-guided optimization. Valid only for executable builds with --optimization=release or speed.
+
Optimize the executable with a profile from scriptc merge-profiles: code is laid out, inlined, and split by how the profiled runs used it. Code the profile does not describe, or describes from an older version of the program, is optimized as without a profile, so a stale profile only loses part of its gain. A new or changed profile compiles every partition of the program again. scriptc build prints how the profile matched the program's functions on stderr; see Profile-guided optimization. A profile with context-sensitive counts also orders each function's blocks by them on Linux. Valid only for executable builds with --optimization=release or speed.
--npm-static <pkg[,pkg…]|auto>
Name npm packages to compile even if they don't ship their own type declarations (repeatable). A trailing ..., as in fs-extra..., also names every package that package depends on. auto uses the default automatic selection. Packages that can't compile produce diagnostics. See npm Dependencies.
@@ -287,6 +287,23 @@ scriptc: warning: 3739 of the 8947 functions app.profdata describes (41.8%) chan The build still succeeds. Collecting the profile again with the current program and scriptc version restores the counts. +### Context-sensitive profiles + +A profile from the steps above counts each function's branches before the optimizer inlines calls. A second phase counts them again after inlining, separately for every place a function was inlined into: + +```console +$ scriptc build app.ts --optimization=speed --profile-use app.profdata --profile-generate profiles-cs/%p.profraw -o app-instrumented-cs +$ ./app-instrumented-cs typical-input.json +$ scriptc merge-profiles -o app-cs.profdata app.profdata profiles-cs/*.profraw +$ scriptc build app.ts --optimization=speed --profile-use app-cs.profdata +``` + +- **First phase:** `--profile-generate` alone. It instruments the program before inlining and gives the profile of the steps above. +- **Second phase:** `--profile-use` with the first profile and `--profile-generate` together. The build is optimized by the first profile and instrumented after inlining, and `scriptc merge-profiles` adds its counts to the first profile. +- **Optimized build:** `--profile-use` with the merged profile. On Linux, it also orders each function's blocks by the counts and moves blocks the runs never reached into separate sections, so the code that runs sits together. + +A profile that already holds context-sensitive counts cannot start a second phase. Merge the second phase's raw profiles into the first-phase profile instead. Both phases run the same workloads. + ## Target CPU By default, x86-64 executables run on any x86-64 CPU, so the compiler uses only the baseline instruction set (SSE2). `--target-cpu` compiles the program's code for a higher x86-64 microarchitecture level. Each level includes the ones below it: diff --git a/native/llvm-codegen/src/emit.cpp b/native/llvm-codegen/src/emit.cpp index d3d868fae..98e07e9ce 100644 --- a/native/llvm-codegen/src/emit.cpp +++ b/native/llvm-codegen/src/emit.cpp @@ -111,9 +111,6 @@ std::optional parseEmitOptions(int Argc, char **Argv) { } if (Options.Input.empty() || Options.Outputs.empty()) return std::nullopt; - // A build either collects a profile or uses one. - if (!Options.ProfileGenerate.empty() && !Options.ProfileUse.empty()) - return std::nullopt; return Options; } @@ -150,20 +147,48 @@ PipelineTuningOptions tuningFor(OptimizationLevel Level) { return Tuning; } -// The profile-guided optimization a build asks for: IR instrumentation that -// writes raw profiles to the ProfileGenerate pattern, or annotation from the -// indexed ProfileUse profile. Functions the profile does not describe, or -// whose control flow changed since it was collected, are optimized as without -// one, so a stale profile loses only its gain. +// Whether the ProfileUse profile also holds context-sensitive counts, which +// a second instrumentation after inlining recorded (inspectProfile). +bool ContextProfile = false; + +// The profile-guided optimization a build asks for: +// - IR instrumentation that writes raw profiles to the ProfileGenerate +// pattern; +// - annotation from the indexed ProfileUse profile, and from its +// context-sensitive counts after inlining when it has them; +// - with both, annotation from ProfileUse and a second instrumentation after +// inlining that writes context-sensitive counts to the ProfileGenerate +// pattern. `scriptc merge-profiles` adds them to the first profile. +// Functions the profile does not describe, or whose control flow changed +// since it was collected, are optimized as without one, so a stale profile +// loses only its gain. std::optional profileOptions(const EmitOptions &Options) { + if (!Options.ProfileGenerate.empty() && !Options.ProfileUse.empty()) + return PGOOptions(Options.ProfileUse, Options.ProfileGenerate, "", "", + PGOOptions::IRUse, PGOOptions::CSIRInstr); if (!Options.ProfileGenerate.empty()) return PGOOptions(Options.ProfileGenerate, "", "", "", PGOOptions::IRInstr); if (!Options.ProfileUse.empty()) - return PGOOptions(Options.ProfileUse, "", "", "", PGOOptions::IRUse); + return PGOOptions(Options.ProfileUse, "", "", "", PGOOptions::IRUse, + ContextProfile ? PGOOptions::CSIRUse + : PGOOptions::NoCSAction); return std::nullopt; } +// Block placement by the extended TSP model, which needs the exact block +// counts a context-sensitive profile gives. The option is process-wide; the +// helper sets it once. +void enableExtTspPlacement() { + static const bool Enabled = [] { + if (cl::Option *Option = cl::getRegisteredOptions().lookup( + "enable-ext-tsp-block-placement")) + Option->addOccurrence(0, "enable-ext-tsp-block-placement", "true"); + return true; + }(); + (void)Enabled; +} + // With a profile, the thin link imports the callees of hot call edges up to // 10 times, and of the hottest edges up to 100 times, the usual size limit, // into every partition that calls them. On large programs that triples the @@ -318,7 +343,8 @@ void reportProfileMatch(const ProfileMatch &Match) { } // Reads the profile's header once, before any partition needs it: it must -// hold IR-level counts. +// hold IR-level counts. Context-sensitive counts lay out the code by them; +// a profile that has them cannot take a second set. std::optional inspectProfile(const EmitOptions &Options) { if (Options.ProfileUse.empty()) return std::nullopt; @@ -333,6 +359,16 @@ std::optional inspectProfile(const EmitOptions &Options) { return EmitFailure{"profile_read_failed", Options.ProfileUse + ": not an IR-level instrumentation profile"}; + ContextProfile = (*Reader)->hasCSIRLevelProfile(); + if (ContextProfile && !Options.ProfileGenerate.empty()) + return EmitFailure{"profile_read_failed", + Options.ProfileUse + + ": already holds context-sensitive counts; " + "collect them on top of the IR-level profile"}; + if (ContextProfile && Triple(Options.Target).isOSBinFormatELF()) { + enableProfileLayout(); + enableExtTspPlacement(); + } return std::nullopt; } @@ -804,6 +840,14 @@ compileOptimizedPartitions(const EmitOptions &Options, : Level == OptimizationLevel::O3 ? 3 : 2; Config.PTO = tuningFor(Level); + // The thin backend instruments or annotates context-sensitive counts after + // its inlining (see profileOptions). + if (!Options.ProfileGenerate.empty() && !Options.ProfileUse.empty()) { + Config.RunCSIRInstr = true; + Config.CSIRProfile = Options.ProfileGenerate; + } else if (ContextProfile) { + Config.CSIRProfile = Options.ProfileUse; + } Config.CGFileType = CodeGenFileType::ObjectFile; Config.DefaultTriple = Options.Target; Config.DiagHandler = [&](const DiagnosticInfo &Info) { diff --git a/native/llvm-codegen/src/emit.h b/native/llvm-codegen/src/emit.h index b62a9d01c..1bdc578df 100644 --- a/native/llvm-codegen/src/emit.h +++ b/native/llvm-codegen/src/emit.h @@ -22,7 +22,9 @@ struct EmitOptions { std::string CacheDirectory; // Instruments the program for IR profiling. The instrumented executable // writes raw profiles to this path pattern (LLVM_PROFILE_FILE overrides it - // at run time). Empty: no instrumentation. + // at run time). Empty: no instrumentation. With ProfileUse, the program is + // optimized by that profile and instrumented again after inlining, which + // records context-sensitive counts. std::string ProfileGenerate; // Indexed profile (.profdata) that guides optimization. Empty: none. std::string ProfileUse; diff --git a/native/llvm-codegen/src/target.cpp b/native/llvm-codegen/src/target.cpp index fd2866761..2695d478f 100644 --- a/native/llvm-codegen/src/target.cpp +++ b/native/llvm-codegen/src/target.cpp @@ -118,11 +118,19 @@ CodeGenOptLevel codeGenLevel(StringRef Level) { return CodeGenOptLevel::Default; } +static bool ProfileLayout = false; + +void enableProfileLayout() { ProfileLayout = true; } + TargetOptions targetOptions(const Triple &TargetTriple) { TargetOptions Options; // The Android API 26 runtime pack uses the NDK's emulated TLS ABI. // Program globals and runtime thread instances must use the same ABI. Options.EmulatedTLS = TargetTriple.isAndroid(); + if (ProfileLayout && TargetTriple.isOSBinFormatELF()) { + Options.FunctionSections = true; + Options.EnableMachineFunctionSplitter = true; + } return Options; } diff --git a/native/llvm-codegen/src/target.h b/native/llvm-codegen/src/target.h index 5049d34f5..5ad469bcf 100644 --- a/native/llvm-codegen/src/target.h +++ b/native/llvm-codegen/src/target.h @@ -40,6 +40,10 @@ bool supportsTarget(llvm::StringRef Triple); // The target options and code generation level every helper target machine // uses, shared with the partitioned optimizer's configuration. llvm::TargetOptions targetOptions(const llvm::Triple &TargetTriple); +// Lays out code by a profile's context-sensitive counts from now on: one +// section per function, and cold blocks split into .text.split sections. +// ELF targets only; the helper calls it once, before any target machine. +void enableProfileLayout(); llvm::CodeGenOptLevel codeGenLevel(llvm::StringRef OptLevel); std::unique_ptr createTargetMachine(llvm::StringRef Triple, llvm::StringRef OptLevel, diff --git a/packages/cli/test/output-options.test.ts b/packages/cli/test/output-options.test.ts index 45d5c55b8..d1e157ea7 100644 --- a/packages/cli/test/output-options.test.ts +++ b/packages/cli/test/output-options.test.ts @@ -55,7 +55,6 @@ describe("output option compatibility", () => { [{ emit: "asm", cpuDispatch: true }, /cpu-dispatch/], [{ emit: "obj", profileGenerate: "/tmp/%p.profraw" }, /profile-generate/], [{ emit: "llvm", profileUse: "/tmp/a.profdata" }, /profile-use/], - [{ profileGenerate: "/tmp/%p.profraw", profileUse: "/tmp/a.profdata" }, /cannot be combined/], ] as const)("rejects %j", (override, message) => { const result = resolveOutputOptions("build", { ...BASE, ...override }); expect(result).toEqual({ ok: false, message: expect.stringMatching(message) }); @@ -120,4 +119,14 @@ describe("output option compatibility", () => { outputKind: "exe", }); }); + + test("a second profile phase combines --profile-use and --profile-generate", () => { + expect( + resolveOutputOptions("build", { + ...BASE, + profileGenerate: "/tmp/%p.profraw", + profileUse: "/tmp/a.profdata", + }), + ).toMatchObject({ ok: true, outputKind: "exe" }); + }); }); diff --git a/packages/cli/test/profile.test.ts b/packages/cli/test/profile.test.ts index da40a15b6..ab3f198ac 100644 --- a/packages/cli/test/profile.test.ts +++ b/packages/cli/test/profile.test.ts @@ -109,10 +109,6 @@ test("profile options are refused where they cannot apply", async () => { await writeFile(entry, PROGRAM); for (const [args, message] of [ [["build", entry, "--emit=llvm", "--profile-use", "a.profdata"], "--emit=exe"], - [ - ["build", entry, "--profile-generate", "%p.profraw", "--profile-use", "a.profdata"], - "cannot be combined", - ], [["coverage", entry, "--profile-use", "a.profdata"], "build/run option"], [["merge-profiles", "a.profraw"], "merge-profiles takes -o"], [["merge-profiles", "-o", "out.profdata"], "merge-profiles takes -o"], diff --git a/packages/compiler/src/backend/link-plan-core.ts b/packages/compiler/src/backend/link-plan-core.ts index f810a4a17..847dca065 100644 --- a/packages/compiler/src/backend/link-plan-core.ts +++ b/packages/compiler/src/backend/link-plan-core.ts @@ -8,7 +8,7 @@ import { type NativeTargetSpec, type WindowsSubsystem, } from "./targets.js"; -import { profileGenerateLinkerArgs } from "./native-profile.js"; +import { profileGenerateLinkerArgs, profileLayoutLinkerArgs } from "./native-profile.js"; export interface ExecutableLinkInputs { inputs: string[]; @@ -34,6 +34,8 @@ export function executableLinkInputs(options: { windowsSubsystem?: WindowsSubsystem; /** Instrumented for profile-guided optimization. */ profileGenerate?: boolean; + /** Laid out by a profile's context-sensitive counts (nativeProfileLayout). */ + profileLayout?: boolean; }): ExecutableLinkInputs { if (options.ffiFrameworks.length !== 0 && options.target.platform !== "darwin") throw new Error("FFI frameworks require a Darwin target"); @@ -71,6 +73,7 @@ export function executableLinkInputs(options: { ...executableStaticLinkerArgs(options.target, options.staticLink ?? false), ...windowsSubsystemLinkerArgs(options.target.platform, options.windowsSubsystem), ...profileGenerateLinkerArgs(options.profileGenerate ?? false), + ...profileLayoutLinkerArgs(options.target, options.profileLayout ?? false), ], }; } diff --git a/packages/compiler/src/backend/link-plan.ts b/packages/compiler/src/backend/link-plan.ts index 73139d22a..c3c0cb5a7 100644 --- a/packages/compiler/src/backend/link-plan.ts +++ b/packages/compiler/src/backend/link-plan.ts @@ -44,6 +44,7 @@ export async function createNativeLinkPlan(options: { staticLink?: boolean; windowsSubsystem?: WindowsSubsystem; profileGenerate?: boolean; + profileLayout?: boolean; programObjectDependencies?: readonly NativeArtifactDependency[]; env?: NodeJS.ProcessEnv; resolver?: (specifier: string) => string; @@ -78,6 +79,7 @@ export async function createNativeLinkPlan(options: { ? {} : { windowsSubsystem: options.windowsSubsystem }), ...(options.profileGenerate === true ? { profileGenerate: true } : {}), + ...(options.profileLayout === true ? { profileLayout: true } : {}), }), dependencyPaths: [...runtimePack.dependencyPaths, ...(options.ffi?.libraries ?? [])], programObjectDependencies: [...(options.programObjectDependencies ?? [])], diff --git a/packages/compiler/src/backend/native-codegen.test.ts b/packages/compiler/src/backend/native-codegen.test.ts index 8ef940c6a..ff0224746 100644 --- a/packages/compiler/src/backend/native-codegen.test.ts +++ b/packages/compiler/src/backend/native-codegen.test.ts @@ -186,13 +186,17 @@ test("profiles reach the helper and key the native cache by their bytes", async const used = { ...base, outputPath: join(pkg.root, "used.o"), - profile: { use: { path: "/profiles/a.profdata", digest: "a".repeat(64) } }, + profile: { + use: { path: "/profiles/a.profdata", digest: "a".repeat(64), contextSensitive: false }, + }, }; await emitNativeArtifact(used); await emitNativeArtifact({ ...used, outputPath: join(pkg.root, "changed.o"), - profile: { use: { path: "/profiles/a.profdata", digest: "b".repeat(64) } }, + profile: { + use: { path: "/profiles/a.profdata", digest: "b".repeat(64), contextSensitive: false }, + }, }); await emitNativeArtifact({ ...used, outputPath: join(pkg.root, "again.o") }); const calls = (await readFile(pkg.log, "utf8")).trim().split("\n"); @@ -209,7 +213,9 @@ test("a cached artifact keeps the profile match of the build that produced it", }); const used = { ...request(pkg.root, pkg.packageJson, join(pkg.root, "used.o")), - profile: { use: { path: "/profiles/a.profdata", digest: "a".repeat(64) } }, + profile: { + use: { path: "/profiles/a.profdata", digest: "a".repeat(64), contextSensitive: false }, + }, }; const match = { annotated: 120, mismatched: 7, missing: 40, unknownPartitions: 0 }; expect((await emitNativeArtifact(used)).profileMatch).toEqual(match); diff --git a/packages/compiler/src/backend/native-profile.test.ts b/packages/compiler/src/backend/native-profile.test.ts index 18f0c6f73..4fffcf457 100644 --- a/packages/compiler/src/backend/native-profile.test.ts +++ b/packages/compiler/src/backend/native-profile.test.ts @@ -5,8 +5,11 @@ import { nativeProfileHelperArgs, renderNativeProfileMatch, nativeProfileKey, + nativeProfileLayout, nativeProfileRefusal, profileGenerateLinkerArgs, + profileHasContextCounts, + profileLayoutLinkerArgs, } from "./native-profile.js"; import { NATIVE_TARGETS } from "./targets.js"; @@ -27,10 +30,21 @@ test("profiles are accepted for optimized executables", () => { expect( nativeProfileRefusal({ ...BASE, platform: "darwin", profileUse: "/p/a.profdata" }), ).toBeNull(); + // The second phase of a context-sensitive profile. + expect( + nativeProfileRefusal({ + ...BASE, + profileGenerate: "/p/%p.profraw", + profileUse: "/p/a.profdata", + }), + ).toBeNull(); }); test.each([ - [{ profileGenerate: "/p/%p.profraw", profileUse: "/p/a.profdata" }, /cannot be combined/], + [ + { profileGenerate: "/p/%p.profraw", profileUse: "/p/a.profdata", platform: "darwin" }, + /Linux executables/, + ], [{ profileUse: "/p/a.profdata", outputKind: "obj" }, /--profile-use is only supported/], [{ profileGenerate: "/p/%p.profraw", sanitize: true }, /--sanitize/], [{ profileUse: "/p/a.profdata", optimization: "dev" }, /optimized build/], @@ -48,9 +62,47 @@ test("helper arguments and cache keys name the profile", () => { "/p/%p.profraw", ]); expect(nativeProfileKey({ generate: "/p/%p.profraw" })).toBe("profile-generate:/p/%p.profraw"); - const use = { use: { path: "/p/a.profdata", digest: "d".repeat(64) } }; + const use = { use: { path: "/p/a.profdata", digest: "d".repeat(64), contextSensitive: false } }; expect(nativeProfileHelperArgs(use)).toEqual(["--profile-use", "/p/a.profdata"]); expect(nativeProfileKey(use)).toBe(`profile-use:${"d".repeat(64)}`); + const both = { generate: "/p/%p.profraw", ...use }; + expect(nativeProfileHelperArgs(both)).toEqual([ + "--profile-generate", + "/p/%p.profraw", + "--profile-use", + "/p/a.profdata", + ]); + expect(nativeProfileKey(both)).toBe( + `profile-generate:/p/%p.profraw\0profile-use:${"d".repeat(64)}`, + ); +}); + +test("a profile's context-sensitive flag is read from its header", () => { + const header = (last: number) => + Uint8Array.from([0xff, 0x6c, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x81, 12, 0, 0, 0, 0, 0, 0, last]); + // VARIANT_MASK_IR_PROF (bit 56) alone, then with VARIANT_MASK_CSIR_PROF (bit 57). + expect(profileHasContextCounts(header(0x01))).toBe(false); + expect(profileHasContextCounts(header(0x03))).toBe(true); + expect(profileHasContextCounts(header(0x03).slice(0, 15))).toBe(false); + const raw = header(0x03); + raw[0] = 0x00; + expect(profileHasContextCounts(raw)).toBe(false); +}); + +test("only a context-sensitive profile lays out code and keeps its section kinds apart", () => { + const use = (contextSensitive: boolean) => ({ + use: { path: "/p/a.profdata", digest: "d".repeat(64), contextSensitive }, + }); + expect(nativeProfileLayout(undefined)).toBe(false); + expect(nativeProfileLayout(use(false))).toBe(false); + expect(nativeProfileLayout(use(true))).toBe(true); + expect(nativeProfileLayout({ generate: "/p/%p.profraw", ...use(true) })).toBe(false); + const linux = { platform: "linux", defaultLinker: "clang" }; + expect(profileLayoutLinkerArgs(linux, true)).toEqual(["-Wl,-z,keep-text-section-prefix"]); + expect(profileLayoutLinkerArgs(linux, false)).toEqual([]); + expect(profileLayoutLinkerArgs({ platform: "darwin", defaultLinker: "clang" }, true)).toEqual([]); + // Zig links cross-compiled and glibc-versioned Linux targets and rejects the flag. + expect(profileLayoutLinkerArgs({ platform: "linux", defaultLinker: "zig" }, true)).toEqual([]); }); test("only the instrumented link adds the profile runtime", () => { diff --git a/packages/compiler/src/backend/native-profile.ts b/packages/compiler/src/backend/native-profile.ts index 85b1fdca4..3968ac712 100644 --- a/packages/compiler/src/backend/native-profile.ts +++ b/packages/compiler/src/backend/native-profile.ts @@ -5,14 +5,32 @@ * merge-profiles` turns those raw profiles into one indexed profile, and an * optimized build (`--profile-use`) lays out, inlines, and splits code by it. * The helper applies both in the pre-link pipeline of every partition. + * + * A second phase records context-sensitive counts: a build with both flags + * is optimized by the first profile and instrumented again after inlining, + * and merging its raw profiles into the first profile adds the counts of the + * inlined code. A profile with those counts also orders each function's + * blocks by them and moves cold blocks to separate sections. */ import type { NativeOptimization } from "./optimization.js"; export interface NativeProfileOptions { - /** Raw profile path pattern written by the instrumented executable. */ + /** Raw profile path pattern written by the instrumented executable. With + * `use`, the instrumentation runs after inlining and records + * context-sensitive counts. */ generate?: string; - /** Indexed profile and the digest of its bytes, which key every cache. */ - use?: { path: string; digest: string }; + /** Indexed profile, the digest of its bytes (which key every cache), and + * whether it holds context-sensitive counts. */ + use?: { path: string; digest: string; contextSensitive: boolean }; +} + +/** Whether an indexed profile holds context-sensitive counts: the header's + * version word carries LLVM's VARIANT_MASK_CSIR_PROF flag, bit 57, which is + * bit 1 of its last byte. */ +export function profileHasContextCounts(bytes: Uint8Array): boolean { + const magic = [0xff, 0x6c, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x81]; + if (bytes.length < 16 || magic.some((byte, index) => bytes[index] !== byte)) return false; + return (bytes[15]! & 0x02) !== 0; } /** Why a request cannot use profiles, or null when it can. */ @@ -28,7 +46,6 @@ export function nativeProfileRefusal(options: { const use = options.profileUse !== undefined; if (!generate && !use) return null; const flag = generate ? "--profile-generate" : "--profile-use"; - if (generate && use) return "--profile-generate and --profile-use cannot be combined"; if (options.outputKind !== "exe") return `${flag} is only supported for executable output`; if (options.sanitize) return `${flag} cannot be combined with --sanitize`; if (options.optimization === "dev") @@ -39,9 +56,10 @@ export function nativeProfileRefusal(options: { } export function nativeProfileHelperArgs(profile: NativeProfileOptions | undefined): string[] { - if (profile?.generate !== undefined) return ["--profile-generate", profile.generate]; - if (profile?.use !== undefined) return ["--profile-use", profile.use.path]; - return []; + return [ + ...(profile?.generate === undefined ? [] : ["--profile-generate", profile.generate]), + ...(profile?.use === undefined ? [] : ["--profile-use", profile.use.path]), + ]; } /** How an optimized build's profile matched the program's functions, as the @@ -111,12 +129,32 @@ export function renderNativeProfileMatch(profile: string, match: NativeProfileMa /** Cache-key part; empty without a profile so other keys are unchanged. */ export function nativeProfileKey(profile: NativeProfileOptions | undefined): string { - if (profile?.generate !== undefined) return `profile-generate:${profile.generate}`; - if (profile?.use !== undefined) return `profile-use:${profile.use.digest}`; - return ""; + return [ + ...(profile?.generate === undefined ? [] : [`profile-generate:${profile.generate}`]), + ...(profile?.use === undefined ? [] : [`profile-use:${profile.use.digest}`]), + ].join("\0"); +} + +/** Whether a build lays out its code by context-sensitive counts: it uses a + * profile that has them, and records none. */ +export function nativeProfileLayout(profile: NativeProfileOptions | undefined): boolean { + return profile?.use?.contextSensitive === true && profile.generate === undefined; } /** The instrumented link adds the compiler driver's profile runtime. */ export function profileGenerateLinkerArgs(generate: boolean): string[] { return generate ? ["-fprofile-generate"] : []; } + +/** A link that lays out code by a profile keeps the hot, unlikely and split + * sections the helper writes apart, so each kind of code sits together. Zig, + * which links cross-compiled and glibc-versioned Linux targets, rejects the + * flag, so those links leave it out. */ +export function profileLayoutLinkerArgs( + target: { platform: string; defaultLinker: string }, + layout: boolean, +): string[] { + return layout && target.platform === "linux" && target.defaultLinker !== "zig" + ? ["-Wl,-z,keep-text-section-prefix"] + : []; +} diff --git a/packages/compiler/src/cli/output-options.ts b/packages/compiler/src/cli/output-options.ts index 207878692..53361a9a1 100644 --- a/packages/compiler/src/cli/output-options.ts +++ b/packages/compiler/src/cli/output-options.ts @@ -87,9 +87,6 @@ export function resolveOutputOptions( if (values.cpuDispatch && emit !== "exe") { return { ok: false, message: `--cpu-dispatch is only supported with --emit=exe` }; } - if (values.profileGenerate !== undefined && values.profileUse !== undefined) { - return { ok: false, message: `--profile-generate and --profile-use cannot be combined` }; - } if (values.profileGenerate !== undefined && emit !== "exe") { return { ok: false, message: `--profile-generate is only supported with --emit=exe` }; } diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index e81e4353a..f492ca672 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -112,7 +112,9 @@ import { RuntimePackError } from "./backend/runtime-pack.js"; import { createNativeLinkPlan } from "./backend/link-plan.js"; import { nativeProfileKey, + nativeProfileLayout, nativeProfileRefusal, + profileHasContextCounts, type NativeProfileMatch, type NativeProfileOptions, } from "./backend/native-profile.js"; @@ -619,6 +621,7 @@ async function compileExecutableNative( /** The kept LLVM text's shared name, when `llvmPath` is the build's private copy of it * (BuildSources): the spelling clang sees and native cache keys name. */ llvmVisiblePath?: string, + profileLayout = false, ): Promise { const programIsObject = /\.(?:o|obj)$/.test(llvmPath); const runtimePackTarget = programIsObject && !sanitize ? nativeCodegenTarget() : null; @@ -635,6 +638,7 @@ async function compileExecutableNative( ...(staticLink ? { staticLink: true } : {}), ...(windowsSubsystem === undefined ? {} : { windowsSubsystem }), ...(profileGenerate ? { profileGenerate: true } : {}), + ...(profileLayout ? { profileLayout: true } : {}), programObjectDependencies, }); const cacheableLinker = @@ -819,8 +823,8 @@ async function removeNativeProgramObject( async function readNativeProfile( opts: CompileRequestOptions, ): Promise { - if (opts.profileGenerate !== undefined) return { generate: opts.profileGenerate }; - if (opts.profileUse === undefined) return undefined; + const generate = opts.profileGenerate; + if (opts.profileUse === undefined) return generate === undefined ? undefined : { generate }; const path = resolve(opts.profileUse); let bytes: Buffer; try { @@ -830,7 +834,12 @@ async function readNativeProfile( `cannot read --profile-use file ${path}: ${err instanceof Error ? err.message : String(err)}`, ); } - return { use: { path, digest: createHash("sha256").update(bytes).digest("hex") } }; + const use = { + path, + digest: createHash("sha256").update(bytes).digest("hex"), + contextSensitive: profileHasContextCounts(bytes), + }; + return generate === undefined ? { use } : { generate, use }; } /** The digest of the request's function layout file, which keys the @@ -1879,6 +1888,7 @@ async function compileTracked( nativeProgramObject?.partitionPaths, profile?.generate !== undefined, hitSources.llvmPath, + nativeProfileLayout(profile), ); if (nativeProgramObject !== null && opts.nativeProgramObject === true) { await rename(nativeProgramObject.linkPath, nativeProgramObject.artifactPath); @@ -2026,6 +2036,7 @@ async function compileTracked( nativeProgramObject?.partitionPaths, profile?.generate !== undefined, llvmPath, + nativeProfileLayout(profile), ); timing("native-link"); if (nativeProgramObject !== null && opts.nativeProgramObject === true) { diff --git a/packages/compiler/src/native/compiler.ts b/packages/compiler/src/native/compiler.ts index 0dec85cbd..b46778827 100644 --- a/packages/compiler/src/native/compiler.ts +++ b/packages/compiler/src/native/compiler.ts @@ -64,8 +64,10 @@ import { evaluateNativeComptime } from "./comptime.js"; import { contentDigest, fileDigest, partsDigest, type NativeCache } from "./cache.js"; import { nativeProfileKey, + nativeProfileLayout, nativeProfileRefusal, parseNativeProfileMatch, + profileHasContextCounts, type NativeProfileMatch, type NativeProfileOptions, } from "../backend/native-profile.js"; @@ -113,11 +115,17 @@ function failure(error: unknown, entry: string, sources: Map): C /** The request's profile. An indexed profile is read here so every cache keys * its bytes. */ function nativeProfile(options: CompileRequestOptions): NativeProfileOptions | undefined { - if (options.profileGenerate !== undefined) return { generate: options.profileGenerate }; - if (options.profileUse === undefined) return undefined; + const generate = options.profileGenerate; + if (options.profileUse === undefined) return generate === undefined ? undefined : { generate }; const path = resolve(options.profileUse); try { - return { use: { path, digest: contentDigest(readFileSync(path)) } }; + const bytes = readFileSync(path); + const use = { + path, + digest: contentDigest(bytes), + contextSensitive: profileHasContextCounts(bytes), + }; + return generate === undefined ? { use } : { generate, use }; } catch (error) { throw new NativeCodegenError( "SC3002", @@ -625,6 +633,7 @@ export class NativeCompiler { ? {} : { windowsSubsystem: options.windowsSubsystem }), ...(profile?.generate === undefined ? {} : { profileGenerate: true }), + ...(nativeProfileLayout(profile) ? { profileLayout: true } : {}), }); // Linux links with lld when the driver finds it. const lld = options.sanitize ? [] : lldLinkerArgs(target.platform, toolchain.linker); diff --git a/packages/compiler/test/native-profile.test.ts b/packages/compiler/test/native-profile.test.ts index 9d04739d0..8abca4393 100644 --- a/packages/compiler/test/native-profile.test.ts +++ b/packages/compiler/test/native-profile.test.ts @@ -272,18 +272,54 @@ async function cacheEntries(cache: string): Promise> { } describe.runIf(helper !== null)("LLVM profile-guided optimization", () => { - test("a profile and instrumentation together are a usage error", async () => { - const dir = await mkdtemp(join(tmpdir(), "scriptc-profile-usage-")); - dirs.push(dir); - await expect( - emit(dir, program(), { - outputs: 1, - name: "both", - generate: join(dir, "%p.profraw"), - use: join(dir, "missing.profdata"), - }), - ).rejects.toMatchObject({ stderr: expect.stringContaining('"code":"usage"') }); - }); + test.runIf(profileLinker !== null)( + "a second phase records context-sensitive counts that lay out the code", + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-profile-context-")); + dirs.push(dir); + const first = await collect(dir, inlinedProgram(), 2, "first", inlinedExit); + // The second phase: optimized by the first profile, instrumented after inlining. + const raw = join(dir, "context-raw"); + const instrumented = await emit(dir, inlinedProgram(), { + outputs: 2, + name: "context", + generate: join(raw, "%p.profraw"), + use: first.profile, + }); + const executable = await link(dir, instrumented, "context", true); + expect(await run(executable, 1)).toBe(inlinedExit(1)); + expect(await run(executable, 3)).toBe(inlinedExit(3)); + const raws = (await readdir(raw)).map((file) => join(raw, file)); + expect(raws).toHaveLength(2); + const merged = join(dir, "context.profdata"); + await merge(merged, [first.profile, ...raws]); + // VARIANT_MASK_CSIR_PROF, bit 57 of the header's version word. + expect((await readFile(merged))[15]! & 0x02).toBe(0x02); + expect((await readFile(first.profile))[15]! & 0x02).toBe(0); + // A build with the merged profile gives every function its own section. + const objects = await emit(dir, inlinedProgram(), { + outputs: 2, + name: "laidout", + use: merged, + }); + const bytes = Buffer.concat(await Promise.all(objects.map((object) => readFile(object)))); + expect(bytes.includes(".text.hot.main")).toBe(true); + const optimized = await link(dir, objects, "laidout"); + expect(await run(optimized, 1)).toBe(inlinedExit(1)); + expect(await run(optimized, 3)).toBe(inlinedExit(3)); + // A profile that has context-sensitive counts cannot start another second phase. + await expect( + emit(dir, inlinedProgram(), { + outputs: 1, + name: "again", + generate: join(dir, "again-raw", "%p.profraw"), + use: merged, + }), + ).rejects.toMatchObject({ + stderr: expect.stringContaining("already holds context-sensitive counts"), + }); + }, + ); test("merge-profiles reports an input that is not a profile", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-profile-merge-error-")); From 0927bfd295ba542ffaa6413df6e38b5c3567e126 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Tue, 27 Oct 2026 07:42:33 -0700 Subject: [PATCH 33/54] Expect exact canonical mappings in the dynamic-slot coverage tests With main's canonical-mapping gap fixes (#2340), stage N2 maps the Tool[] field and the (Callbacks | number)[] global exactly, so they are no longer dynamic slots. The hot-slot fixture's Hook gets an intersection arm, which has no canonical form, so it stays a dynamic piece of Plugin[]. Co-Authored-By: Claude Opus 5.5 --- tests/coverage-fixtures/dynamic-slot-hot.ts | 5 +++-- tests/harness/coverage.test.ts | 19 ++++++++----------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/tests/coverage-fixtures/dynamic-slot-hot.ts b/tests/coverage-fixtures/dynamic-slot-hot.ts index 65672ffc1..e92c5261f 100644 --- a/tests/coverage-fixtures/dynamic-slot-hot.ts +++ b/tests/coverage-fixtures/dynamic-slot-hot.ts @@ -1,6 +1,7 @@ // A dynamic slot read in a loop: SC6101 names the slot, and coverage -// counts the dynamic operations on it. -type Hook = (() => number) | AbortSignal | Map string>; +// counts the dynamic operations on it. The intersection arm has no +// canonical form, so `Hook` stays a dynamic piece of `Plugin[]`. +type Hook = (() => number) | (AbortSignal & { tag?: string }) | Map string>; interface Plugin { name: string; diff --git a/tests/harness/coverage.test.ts b/tests/harness/coverage.test.ts index 82e541140..ce3f22d7c 100644 --- a/tests/harness/coverage.test.ts +++ b/tests/harness/coverage.test.ts @@ -431,11 +431,10 @@ test("--explain-dynamic attributes each refusal to the unit the dynamic tier wou test("--explain-dynamic lists the type failures localized failure would turn into dynamic slots", async () => { const { coverage } = analyze(fixture("localized-failure.ts"), { explainDynamic: true }); const report = coverage.tiers!.localizedFailure; - // Locals, fields and parameters degrade (stages L1 and L2), and the - // 'any' field compiles; only the refused WeakRef field and its - // construction remain. + // Locals and parameters degrade (stages L1 and L2), the 'any' field + // compiles, and the `Tool[]` field maps exactly from its canonical form + // (stage N2); only the refused WeakRef field and its construction remain. expect(coverage.dynamicSlots?.map((s) => [s.slot, s.name])).toEqual([ - ["field", "tools"], ["param", "progress"], ["local", "callbacks"], ]); @@ -458,28 +457,26 @@ test("a local or module global whose declared type does not compile becomes a dy expect(coverage.diagnostics).toEqual([]); expect(coverage.dynamicSlots?.map((s) => [s.slot, s.name, s.declaredType, s.code])).toEqual([ ["global", "current", "Callbacks | undefined", "SC2009"], - ["global", "all", "(number | Callbacks)[]", "SC2009"], ["local", "local", "Callbacks | undefined", "SC2009"], ]); - // The loop over `all` is top-level code: its dynamic operations count too. - expect(coverage.dynamicSlots?.find((s) => s.name === "all")?.operations).toBeGreaterThan(0); - expect(coverage.performance?.filter((d) => d.code === "SC6104")).toHaveLength(3); + // `all`'s element type maps exactly from its canonical form (stage N2), so + // the array is no dynamic slot. + expect(coverage.dynamicSlots?.find((s) => s.name === "all")).toBeUndefined(); + expect(coverage.performance?.filter((d) => d.code === "SC6104")).toHaveLength(2); expect(coveragePasses(coverage, "blockers")).toBe(true); expect(coveragePasses(coverage, "dynamic")).toBe(true); expect(coveragePasses(coverage, "dynamic-slots")).toBe(false); const envelope = coverageEnvelope(coverage, { compilerVersion: "test", sourceTexts }); expect(envelope.dynamicSlots?.map((s) => [s.kind, s.name, s.line])).toEqual([ ["global", "current", 10], - ["global", "all", 36], ["local", "local", 14], ]); expect(envelope.diagnostics.filter((d) => d.code === "SC6104").map((d) => d.severity)).toEqual([ "warning", "warning", - "warning", ]); const text = renderCoverage(coverage, { sourceTexts, root: repoRoot }); - expect(text).toContain("dynamic slots 3"); + expect(text).toContain("dynamic slots 2"); expect(text).toContain("module global 'current' (declared 'Callbacks | undefined')"); // --tier=static-only keeps every refusal. const strict = analyze(program, { tier: "static-only" }).coverage; From f12a2b2a23e351f86a7da53526a6ed008361614a Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 10:00:00 -0500 Subject: [PATCH 34/54] Take JavaScript ReadableStreams as fetch bodies and in instanceof by value A JavaScript ReadableStream given to Response, Request or fetch as a body was converted to the string "[object ReadableStream]". The fetch unit now reads such a stream through a reader of its own. A locked or disturbed stream gives Node's TypeError, and the chunks must be Uint8Array when the body is consumed ("Received non-Uint8Array chunk"). instanceof against a web stream class held in a variable or passed as an argument walked no prototype for handles and answered false. It now starts at the class prototype of the handle. The fetch unit's own streams, readers and controllers, such as Response.body, are instances of the JavaScript classes, so `response.body instanceof ReadableStream` is true again. These paths, and the fetch reader functions' hand-off to web stream readers, are in SCR_HANDLE_CLASSES variants of scr_json.c and scr_fetch.c, which only programs with JavaScript web streams link; other programs that use fetch build to the same size as on main. --- .../builtin-apis/streams-and-readline.mdx | 1 + .../compiler/src/backend/native/executable.ts | 5 +- .../corpus/web-stream-fetch-interop.mjs.json | 6 + .../runtime-darwin-arm64/scripts/build.mjs | 1 + .../runtime-pack-matrix.mjs | 22 ++- .../runtime-pack-common/scripts/build.mjs | 1 + packages/runtime/src/scr_dyn.h | 8 + packages/runtime/src/scr_dyn_builtin_class.c | 19 ++- packages/runtime/src/scr_dyn_handle_class.c | 18 ++ packages/runtime/src/scr_fetch.c | 157 +++++++++++++++++- packages/runtime/src/scr_generator.h | 11 +- packages/runtime/src/scr_json.c | 12 +- packages/runtime/src/scr_webstreams.c | 17 +- tests/corpus/web-stream-fetch-interop.mjs | 50 ++++++ .../web-stream-fetch-interop.mjs.json | 1 + 15 files changed, 311 insertions(+), 18 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/web-stream-fetch-interop.mjs.json create mode 100644 tests/corpus/web-stream-fetch-interop.mjs create mode 100644 tests/harness/corpus-costs/web-stream-fetch-interop.mjs.json diff --git a/docs/content/limitations/builtin-apis/streams-and-readline.mdx b/docs/content/limitations/builtin-apis/streams-and-readline.mdx index 25e944994..57a43e67a 100644 --- a/docs/content/limitations/builtin-apis/streams-and-readline.mdx +++ b/docs/content/limitations/builtin-apis/streams-and-readline.mdx @@ -63,4 +63,5 @@ `node:stream/web` exports the same `ReadableStream` class as the global; in TypeScript, its other classes are unsupported. In JavaScript, `ReadableStream`, `ReadableStreamDefaultReader`, `ReadableStreamDefaultController`, `CountQueuingStrategy`, and `ByteLengthQueuingStrategy`, as globals and as `node:stream/web` exports, are constructor values with Node's prototypes, brand checks, and errors. Their underlying sources (`start`, `pull`, `cancel`), queuing strategies with a `size` function, readers, `tee`, `ReadableStream.from`, and async iteration settle promises in Node's order. + A JavaScript `ReadableStream` given as a `Response` or `Request` body, or as `fetch`'s body, is read through a stream of its own: `response.body` is that stream rather than the one passed in, and the stream passed in is locked from the start. In JavaScript, a byte stream (`type: 'bytes'`), `pipeTo`, `pipeThrough`, `Object.getPrototypeOf` on these streams, readers, controllers, and strategies, their use as `WeakMap` and `WeakSet` keys, `ReadableStream.from` of a function that is not iterable, and the other `node:stream/web` classes are unsupported. diff --git a/packages/compiler/src/backend/native/executable.ts b/packages/compiler/src/backend/native/executable.ts index 4b3ee66e7..9bd7d50da 100644 --- a/packages/compiler/src/backend/native/executable.ts +++ b/packages/compiler/src/backend/native/executable.ts @@ -396,8 +396,11 @@ const RUNTIME_DEFINES: readonly { { define: "SCR_JS_TIER", when: (opts) => opts.dynTier === true }, { define: "SCR_WARNING_REPLACE", when: (opts) => opts.warningReplace }, { define: "SCR_STDIO_EXTENDED", when: (opts) => opts.stdioExtended }, - // scr_async.c's native promise reactions, which the web streams unit uses. + // scr_async.c's native promise reactions, which the web streams unit uses, + // scr_json.c's and scr_dyn_builtin_class.c's instanceof over its classes, + // and scr_fetch.c's bodies of its streams. { define: "SCR_NATIVE_REACTIONS", when: (opts) => opts.webStreams }, + { define: "SCR_HANDLE_CLASSES", when: (opts) => opts.webStreams }, { define: "SCR_YIN", when: (opts) => opts.youngNursery }, ]; diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/web-stream-fetch-interop.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/web-stream-fetch-interop.mjs.json new file mode 100644 index 000000000..11116bf4d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/web-stream-fetch-interop.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/web-stream-fetch-interop.mjs" + ], + "diags": [] +} diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index 60b1527f3..ecb2a901b 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -377,6 +377,7 @@ async function build() { "SCR_WARNING_REPLACE", "SCR_STDIO_EXTENDED", "SCR_NATIVE_REACTIONS", + "SCR_HANDLE_CLASSES", "SCR_YIN", ], excluded: ["SCR_RC_AUDIT", "SCR_ASAN_FIBERS"], diff --git a/packages/runtime-pack-common/runtime-pack-matrix.mjs b/packages/runtime-pack-common/runtime-pack-matrix.mjs index 8b54541e7..c22730524 100644 --- a/packages/runtime-pack-common/runtime-pack-matrix.mjs +++ b/packages/runtime-pack-common/runtime-pack-matrix.mjs @@ -198,6 +198,17 @@ export function variantsFor(source) { // other program links the ordinary object, byte for byte. if (source === "scr_json.c") { variants.push({ id: "dyn-tier", when: { dynTier: true }, defines: ["SCR_JS_TIER"] }); + // Programs with web streams: instanceof over their classes' handles. + variants.push({ + id: "handle-classes", + when: { webStreams: true }, + defines: ["SCR_HANDLE_CLASSES"], + }); + variants.push({ + id: "dyn-tier-handle-classes", + when: { dynTier: true, webStreams: true }, + defines: ["SCR_JS_TIER", "SCR_HANDLE_CLASSES"], + }); } if (source === "scr_bytes.c") { variants.push({ @@ -216,7 +227,9 @@ export function variantsFor(source) { }); } // Programs with web streams link the promise unit with native promise - // reactions; every other program keeps the default object. + // reactions, the builtin-class unit whose instanceof knows their classes + // (as scr_json.c's does above), and the fetch unit that takes their + // streams as bodies; every other program keeps the default objects. if (source === "scr_async.c") { variants.push({ id: "native-reactions", @@ -224,6 +237,13 @@ export function variantsFor(source) { defines: ["SCR_NATIVE_REACTIONS"], }); } + if (source === "scr_dyn_builtin_class.c" || source === "scr_fetch.c") { + variants.push({ + id: "handle-classes", + when: { webStreams: true }, + defines: ["SCR_HANDLE_CLASSES"], + }); + } // stdin setEncoding, string data listeners and stdio 'resize' events. if (source === "scr_events.c") { variants.push({ diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index 62fdfbbf1..663fca182 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -480,6 +480,7 @@ async function build() { "SCR_WARNING_REPLACE", "SCR_STDIO_EXTENDED", "SCR_NATIVE_REACTIONS", + "SCR_HANDLE_CLASSES", "SCR_YIN", ], excluded: ["SCR_RC_AUDIT", "SCR_ASAN_FIBERS"], diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index c63d608a2..e9624a932 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -2054,6 +2054,14 @@ ScrDyn *scr_dyn_handle_class_value(ScrDynHandleTag tag); ScrDyn *scr_dyn_handle_class_method(ScrDynHandleTag tag, const char *name); /* The tag's prototype (+1), created on first use. */ ScrDyn *scr_dyn_handle_class_prototype(ScrDynHandleTag tag); +/* Handles of `tag` (another unit's native representation of the same + * interface) are instances of the class of `klass_tag`. */ +void scr_dyn_handle_class_alias(ScrDynHandleTag tag, ScrDynHandleTag klass_tag); +/* The class tag a handle is an instance of, or SCR_DYNH_CAPACITY for a + * handle of no class. */ +ScrDynHandleTag scr_dyn_handle_class_tag_of(const ScrDyn *handle); +/* The prototype of the class a handle is an instance of (+1), or NULL. */ +ScrDyn *scr_dyn_handle_class_prototype_of(const ScrDyn *handle); /* For a class's ScrDynHandleOps.get: a member that is not one of the * class's methods, from the prototype chain with a handle over `h` as the * receiver (+1); NULL when the chain has none or with the exception diff --git a/packages/runtime/src/scr_dyn_builtin_class.c b/packages/runtime/src/scr_dyn_builtin_class.c index b176e1a42..f7bec0938 100644 --- a/packages/runtime/src/scr_dyn_builtin_class.c +++ b/packages/runtime/src/scr_dyn_builtin_class.c @@ -1544,10 +1544,21 @@ ScrDyn *scr_dyn_builtin_class(double kind) { /* OrdinaryHasInstance over the value's prototype chain. */ static bool bc_chain_has(ScrDyn *value, ScrDyn *prototype) { - if (value->kind != SCR_DYN_OBJ && value->kind != SCR_DYN_ARR && value->kind != SCR_DYN_FUNC && - value->kind != SCR_DYN_PROXY) - return false; - ScrDyn *current = scr_dyn_get_prototype(value); + ScrDyn *current; +#ifdef SCR_HANDLE_CLASSES + /* A handle of a class (scr_dyn_handle_class.c) has the class's + * prototype; this build is the one programs with such classes link. */ + if (value->kind == SCR_DYN_HANDLE) { + current = scr_dyn_handle_class_prototype_of(value); + if (!current) return false; + } else +#endif + { + if (value->kind != SCR_DYN_OBJ && value->kind != SCR_DYN_ARR && value->kind != SCR_DYN_FUNC && + value->kind != SCR_DYN_PROXY) + return false; + current = scr_dyn_get_prototype(value); + } while (current && current->kind != SCR_DYN_NULL) { if (scr_dyn_strict_eq(current, prototype)) { scr_dyn_release(current); diff --git a/packages/runtime/src/scr_dyn_handle_class.c b/packages/runtime/src/scr_dyn_handle_class.c index 2539a54d4..42e2884c5 100644 --- a/packages/runtime/src/scr_dyn_handle_class.c +++ b/packages/runtime/src/scr_dyn_handle_class.c @@ -17,6 +17,9 @@ #include static SCR_TL const ScrDynHandleClass *hc_classes[SCR_DYNH_CAPACITY]; +/* The class tag of another tag's handles + 1 (scr_dyn_handle_class_alias); + * 0 = none. */ +static SCR_TL uint16_t hc_aliases[SCR_DYNH_CAPACITY]; static SCR_TL ScrDyn *hc_ctors[SCR_DYNH_CAPACITY]; static SCR_TL ScrDyn *hc_prototypes[SCR_DYNH_CAPACITY]; static SCR_TL bool hc_teardown_registered; @@ -230,6 +233,21 @@ static void hc_create(ScrDynHandleTag tag) { } } +void scr_dyn_handle_class_alias(ScrDynHandleTag tag, ScrDynHandleTag klass_tag) { + hc_aliases[tag] = (uint16_t)(klass_tag + 1); +} + +ScrDynHandleTag scr_dyn_handle_class_tag_of(const ScrDyn *handle) { + ScrDynHandleTag tag = handle->v.handle.tag; + if (hc_classes[tag]) return tag; + return hc_aliases[tag] ? (ScrDynHandleTag)(hc_aliases[tag] - 1) : SCR_DYNH_CAPACITY; +} + +ScrDyn *scr_dyn_handle_class_prototype_of(const ScrDyn *handle) { + ScrDynHandleTag tag = scr_dyn_handle_class_tag_of(handle); + return tag == SCR_DYNH_CAPACITY ? NULL : scr_dyn_handle_class_prototype(tag); +} + void scr_dyn_handle_class_install(ScrDynHandleTag tag, const ScrDynHandleOps *ops, const ScrDynHandleClass *klass) { scr_dyn_handle_install(tag, ops); hc_classes[tag] = klass; diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index 5f2122486..9ca6efde3 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -157,6 +157,11 @@ struct SfStream { ScrDyn *cancel_cb; ScrDyn *source_this; ScrDyn *from_dyn; +#ifdef SCR_HANDLE_CLASSES + /* A body made from a checked-dynamic web ReadableStream: that stream's + * reader (+1), read one chunk per pull through scr_web_stream_bridge. */ + ScrDyn *web_reader; +#endif ScrArr *from_array; ScrBytes *from_bytes; ScrStr *from_string; @@ -923,6 +928,9 @@ static void sf_stream_release(SfStream *s) { scr_dyn_release(s->error); sf_stream_drop_source_callbacks(s, true); scr_dyn_release(s->from_dyn); +#ifdef SCR_HANDLE_CLASSES + scr_dyn_release(s->web_reader); +#endif scr_arr_release(s->from_array); scr_bytes_release(s->from_bytes); scr_str_release(s->from_string); @@ -1168,7 +1176,19 @@ static void sf_stream_drain(SfStream *s) { s->head = c->next; if (!s->head) s->tail = NULL; s->queued--; +#ifdef SCR_HANDLE_CLASSES + ScrBytes *bytes; + /* A body read from a JavaScript ReadableStream takes Uint8Array + * chunks only, as Node's body mixin does. */ + if (s->web_reader && !scr_dyn_bytes_is(c->value, SCR_BYTES_U8)) { + sf_type_error("Received non-Uint8Array chunk"); + bytes = NULL; + } else { + bytes = sf_chunk_bytes(c->value); + } +#else ScrBytes *bytes = sf_chunk_bytes(c->value); +#endif if (!bytes) { sf_chunk_release(c); scr_promise_reject_pending(s->collector->promise); @@ -1520,9 +1540,88 @@ static ScrDyn *sf_controller_box(SfStream *s) { } static bool sf_stream_has_from_source(const SfStream *s) { +#ifdef SCR_HANDLE_CLASSES + if (s->web_reader) return true; +#endif return s->from_dyn || s->from_array || s->from_bytes || s->from_string; } +/* A Response, Request or fetch body of a ReadableStream: the fetch unit's + * own streams, and in the build that programs with JavaScript web streams + * link (SCR_HANDLE_CLASSES), theirs. */ +static bool sf_is_stream_body(const ScrDyn *body) { + if (body->kind != SCR_DYN_HANDLE) return false; +#ifdef SCR_HANDLE_CLASSES + if (body->v.handle.tag == SCR_DYNH_WS_READABLE) return true; +#endif + return body->v.handle.tag == SCR_DYNH_WEB_STREAM; +} + +#ifdef SCR_HANDLE_CLASSES +static void sf_stream_schedule_pull_complete(SfStream *s); + +/* The web stream reader's answer to one pull. */ +static void sf_web_chunk(void *ctx, ScrDyn *chunk) { + SfStream *s = ctx; + if (!s->close_requested && !s->closed && !s->error) { + sf_stream_enqueue_value(s, chunk); + sf_stream_drain(s); + } + sf_stream_schedule_pull_complete(s); + sf_stream_release(s); +} + +static void sf_web_close(void *ctx) { + SfStream *s = ctx; + s->pulling = false; + sf_stream_close(s); + sf_stream_release(s); +} + +static void sf_web_error(void *ctx, ScrDyn *reason) { + SfStream *s = ctx; + s->pulling = false; + sf_stream_error(s, reason ? reason : scr_dyn_undefined()); + sf_stream_release(s); +} + +static const ScrFetchReadSink sf_web_sink = {sf_web_chunk, sf_web_close, sf_web_error}; + +static void sf_stream_pull_from_web(SfStream *s) { + s->pulling = true; + sf_stream_retain(s); + if (!scr_web_stream_bridge->reader_read_into(s->web_reader, &sf_web_sink, s)) { + ScrCaught *caught = scr_exc_take(); + ScrDyn *reason = scr_caught_to_dyn(caught); + scr_caught_release(caught); + sf_web_error(s, reason); + scr_dyn_release(reason); + } +} + +/* A body from a checked-dynamic web ReadableStream (borrowed): a native + * stream that reads it (+1 handle), or NULL with the TypeError pending + * for one that is locked or disturbed. */ +static ScrDyn *sf_web_body(const ScrDyn *body) { + if (!scr_web_stream_bridge || scr_web_stream_bridge->unusable(body)) { + sf_type_error("Response body object should not be disturbed or locked"); + return NULL; + } + ScrDyn *reader = scr_dyn_invoke((ScrDyn *)body, "getReader", NULL, 0, "ReadableStream.getReader"); + if (!reader) return NULL; + SfStream *s = sf_stream_new_native(); + s->web_reader = reader; + sf_stream_schedule_initial_pull(s); + ScrDyn *out = scr_dyn_new_handle(s, SCR_DYNH_WEB_STREAM); + sf_stream_release(s); + return out; +} + +static bool sf_is_web_body(const ScrDyn *body) { + return body && body->kind == SCR_DYN_HANDLE && body->v.handle.tag == SCR_DYNH_WS_READABLE; +} +#endif + static void sf_stream_pull_from_source(SfStream *s) { ScrDyn *value = NULL; if (s->from_array) { @@ -1657,6 +1756,12 @@ static void sf_stream_pull(SfStream *s) { s->pull_again = true; return; } +#ifdef SCR_HANDLE_CLASSES + if (s->web_reader) { + sf_stream_pull_from_web(s); + return; + } +#endif if (sf_stream_has_from_source(s)) { s->pulling = true; sf_stream_pull_from_source(s); @@ -1823,6 +1928,9 @@ static ScrPromise *sf_stream_cancel(SfStream *s, ScrDyn *reason, s->cancel_cb ? scr_dyn_retain(s->cancel_cb) : NULL; ScrDyn *cancel_this = cancel_cb ? scr_dyn_retain(s->source_this) : NULL; +#ifdef SCR_HANDLE_CLASSES + ScrDyn *web_reader = s->web_reader ? scr_dyn_retain(s->web_reader) : NULL; +#endif sf_stream_drop_chunks(s); SfTransfer *response_owner = s->response_owner; if (response_owner && response_owner->client) { @@ -1832,6 +1940,22 @@ static ScrPromise *sf_stream_cancel(SfStream *s, ScrDyn *reason, else sf_stream_close(s); if (response_owner && !response_owner->done) sf_settle(response_owner); +#ifdef SCR_HANDLE_CLASSES + if (cancel_cb || web_reader) { + ScrDyn *args[1] = {reason ? reason : scr_dyn_undefined()}; + ScrDyn *r; + if (web_reader) { + /* The web stream's cancel through its reader. */ + r = scr_dyn_invoke(web_reader, "cancel", args, 1, "ReadableStreamDefaultReader.cancel"); + scr_dyn_release(web_reader); + } else { + scr_dyn_this_push_dyn(cancel_this); + r = scr_dyn_call(cancel_cb, args, 1, "underlyingSource.cancel"); + scr_dyn_this_pop(); + scr_dyn_release(cancel_cb); + scr_dyn_release(cancel_this); + } +#else if (cancel_cb) { ScrDyn *args[1] = {reason ? reason : scr_dyn_undefined()}; scr_dyn_this_push_dyn(cancel_this); @@ -1840,6 +1964,7 @@ static ScrPromise *sf_stream_cancel(SfStream *s, ScrDyn *reason, scr_dyn_this_pop(); scr_dyn_release(cancel_cb); scr_dyn_release(cancel_this); +#endif if (!r) { scr_promise_reject_pending(p); return p; @@ -2266,9 +2391,11 @@ ScrDyn *scr_fetch_stream_from_string(ScrStr *iterable) { } bool scr_fetch_reader_read_into(ScrDyn *reader, const ScrFetchReadSink *sink, void *ctx) { - if (scr_web_reader_read_into_fn && reader && reader->kind == SCR_DYN_HANDLE && +#ifdef SCR_HANDLE_CLASSES + if (scr_web_stream_bridge && reader && reader->kind == SCR_DYN_HANDLE && reader->v.handle.tag == SCR_DYNH_WS_READER) - return scr_web_reader_read_into_fn(reader, sink, ctx); + return scr_web_stream_bridge->reader_read_into(reader, sink, ctx); +#endif if (!reader || reader->kind != SCR_DYN_HANDLE || reader->v.handle.tag != SCR_DYNH_WEB_READER) { sf_type_error("Expected a ReadableStreamDefaultReader"); @@ -2278,6 +2405,7 @@ bool scr_fetch_reader_read_into(ScrDyn *reader, const ScrFetchReadSink *sink, vo } ScrPromise *scr_fetch_reader_read(ScrDyn *reader) { +#ifdef SCR_HANDLE_CLASSES /* A checked-dynamic web stream's reader answers its own read(). */ if (reader && reader->kind == SCR_DYN_HANDLE && reader->v.handle.tag == SCR_DYNH_WS_READER) { ScrDyn *result = scr_dyn_invoke(reader, "read", NULL, 0, "reader.read"); @@ -2290,6 +2418,7 @@ ScrPromise *scr_fetch_reader_read(ScrDyn *reader) { scr_dyn_release(result); return p; } +#endif if (!reader || reader->kind != SCR_DYN_HANDLE || reader->v.handle.tag != SCR_DYNH_WEB_READER) { sf_type_error("Expected a ReadableStreamDefaultReader"); @@ -6169,9 +6298,7 @@ static ScrDyn *sf_response_construct(ScrDyn *body, ScrDyn *init, * string arm therefore runs before ResponseInit's dictionary conversion, * while a ReadableStream is only brand-converted here — its locked/ * disturbed state belongs to the later body-extraction phase. */ - if (!null_body && !json_body && - !(body->kind == SCR_DYN_HANDLE && - body->v.handle.tag == SCR_DYNH_WEB_STREAM)) { + if (!null_body && !json_body && !sf_is_stream_body(body)) { if (body->kind == SCR_DYN_BYTES) { ScrBytes *raw = scr_bytes_raw_view(body->v.bytes); body_bytes = scr_bytes_copy(raw); @@ -6275,6 +6402,13 @@ static ScrDyn *sf_response_construct(ScrDyn *body, ScrDyn *init, goto fail; } sf_stream_retain(stream); +#ifdef SCR_HANDLE_CLASSES + } else if (sf_is_web_body(body)) { + ScrDyn *web = sf_web_body(body); + if (!web) goto fail; + stream = sf_stream_retain((SfStream *)web->v.handle.ptr); + scr_dyn_release(web); +#endif } else { stream = sf_stream_new_native(); light_length_bytes = body_bytes->len; @@ -6519,7 +6653,7 @@ ScrDyn *scr_fetch_request_new(ScrDyn *input, ScrDyn *init) { if (has_body && (sf_name(method_text->data, method_text->len, "GET") || sf_name(method_text->data, method_text->len, "HEAD"))) { scr_dyn_release(body); scr_str_release(method_text); scr_url_release(url); sf_type_error("Request with GET/HEAD method cannot have body."); return NULL; } - if (supplied && body->kind == SCR_DYN_HANDLE && body->v.handle.tag == SCR_DYNH_WEB_STREAM) { + if (supplied && sf_is_stream_body(body)) { ScrDyn *duplex = sf_response_init_get(init, "duplex", 6); ScrStr *text = duplex && duplex->kind != SCR_DYN_UNDEF ? scr_dyn_string_coerce_js(duplex) : NULL; scr_dyn_release(duplex); @@ -6725,6 +6859,17 @@ ScrPromise *scr_fetch_static(ScrStr *url, ScrDyn *init) { * arrays, and plain objects (with the ordinary JS object coercion * protocol), while null and undefined continue to mean no body. */ ScrDyn *coerced_body = NULL; +#ifdef SCR_HANDLE_CLASSES + if (body_present && sf_is_web_body(body)) { + coerced_body = sf_web_body(body); + if (!coerced_body) { + scr_promise_reject_pending(promise); + return promise; + } + body = coerced_body; + body_stream = true; + } +#endif /* A URLSearchParams body is its serialization, sent as * application/x-www-form-urlencoded;charset=UTF-8 (extract a body). */ bool form_body = false; diff --git a/packages/runtime/src/scr_generator.h b/packages/runtime/src/scr_generator.h index 722ee1f88..6b66e8f9d 100644 --- a/packages/runtime/src/scr_generator.h +++ b/packages/runtime/src/scr_generator.h @@ -223,9 +223,16 @@ typedef struct ScrFetchReadSink { /* Queue a sink read request on a reader handle (borrowed). false with a * TypeError pending for a value that is not a reader or a released one. */ bool scr_fetch_reader_read_into(ScrDyn *reader, const ScrFetchReadSink *sink, void *ctx); -/* The same for the readers of checked-dynamic web streams +/* What the fetch unit reaches of checked-dynamic web streams * (scr_webstreams.c installs it); NULL in programs without them. */ -extern SCR_TL bool (*scr_web_reader_read_into_fn)(ScrDyn *reader, const ScrFetchReadSink *sink, void *ctx); +typedef struct ScrWebStreamBridge { + /* scr_fetch_reader_read_into for one of their readers. */ + bool (*reader_read_into)(ScrDyn *reader, const ScrFetchReadSink *sink, void *ctx); + /* Whether a ReadableStream is locked or disturbed, which makes it + * unusable as a body. */ + bool (*unusable)(const ScrDyn *stream); +} ScrWebStreamBridge; +extern SCR_TL const ScrWebStreamBridge *scr_web_stream_bridge; /* The web stream class constructors for checked-dynamic code, by kind * (scr_webstreams.c: ReadableStream, ReadableStreamDefaultReader, * ReadableStreamDefaultController, CountQueuingStrategy, diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index b88271d3a..f06c869c8 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -5621,8 +5621,16 @@ bool scr_bytes_instanceof_at(const ScrDyn *value, const ScrDyn *callee, const Sc if (!prototype) return false; /* the read threw: pending */ if (prototype->kind == SCR_DYN_OBJ) { bool found = false; +#ifdef SCR_HANDLE_CLASSES + /* A handle of a class (scr_dyn_handle_class.c) has the class's + * prototype; this build is the one programs with such classes link. */ + ScrDyn *handle_prototype = value->kind == SCR_DYN_HANDLE ? scr_dyn_handle_class_prototype_of(value) : NULL; + if (handle_prototype || value->kind == SCR_DYN_OBJ || value->kind == SCR_DYN_ARR || value->kind == SCR_DYN_FUNC) { + ScrDyn *current = handle_prototype ? handle_prototype : scr_dyn_get_prototype((ScrDyn *)value); +#else if (value->kind == SCR_DYN_OBJ || value->kind == SCR_DYN_ARR || value->kind == SCR_DYN_FUNC) { ScrDyn *current = scr_dyn_get_prototype((ScrDyn *)value); +#endif while (current && current->kind != SCR_DYN_NULL) { if (current == prototype) { found = true; @@ -6981,8 +6989,8 @@ static ScrDyn *scr_dynh_method_call(ScrClosure *closure, ScrDyn *const *args, si return result; } -/* scr_generator.h: the web stream readers' sink reads (scr_webstreams.c). */ -SCR_TL bool (*scr_web_reader_read_into_fn)(ScrDyn *reader, const ScrFetchReadSink *sink, void *ctx) = NULL; +/* scr_generator.h: what fetch reaches of the web streams (scr_webstreams.c). */ +SCR_TL const ScrWebStreamBridge *scr_web_stream_bridge = NULL; /* The method `key` of the handle's class as a function value (+1), or NULL * when the class has no such method. */ diff --git a/packages/runtime/src/scr_webstreams.c b/packages/runtime/src/scr_webstreams.c index a9b6a0392..e4b192b87 100644 --- a/packages/runtime/src/scr_webstreams.c +++ b/packages/runtime/src/scr_webstreams.c @@ -2011,6 +2011,14 @@ static bool ws_reader_read_into(ScrDyn *reader, const ScrFetchReadSink *sink, vo return true; } +/* A stream that cannot be a Response or Request body. */ +static bool ws_readable_unusable(const ScrDyn *stream) { + WsReadable *s = ws_stream_of(stream); + return !s || ws_readable_locked(s) || s->disturbed; +} + +static const ScrWebStreamBridge ws_fetch_bridge = {ws_reader_read_into, ws_readable_unusable}; + /* ── the handle surfaces ───────────────────────────────────────────── */ static bool ws_name(const char *key, size_t len, const char *want) { @@ -2456,12 +2464,17 @@ static void ws_install(void) { ScrStr *name = scr_str_new("asyncIterator", 13); scr_sym_release(scr_sym_well_known(name)); scr_str_release(name); - scr_web_reader_read_into_fn = ws_reader_read_into; + scr_web_stream_bridge = &ws_fetch_bridge; scr_dyn_handle_class_install(SCR_DYNH_WS_READABLE, &ws_readable_ops, &ws_readable_class); scr_dyn_handle_class_install(SCR_DYNH_WS_READER, &ws_reader_ops, &ws_reader_class); scr_dyn_handle_class_install(SCR_DYNH_WS_RCONTROLLER, &ws_rcontroller_ops, &ws_rcontroller_class); scr_dyn_handle_class_install(SCR_DYNH_WS_COUNT_STRATEGY, &ws_count_strategy_ops, &ws_count_strategy_class); scr_dyn_handle_class_install(SCR_DYNH_WS_BYTE_STRATEGY, &ws_byte_strategy_ops, &ws_byte_strategy_class); + /* The fetch unit's streams, readers and controllers (a Response body) + * are instances of the same classes. */ + scr_dyn_handle_class_alias(SCR_DYNH_WEB_STREAM, SCR_DYNH_WS_READABLE); + scr_dyn_handle_class_alias(SCR_DYNH_WEB_READER, SCR_DYNH_WS_READER); + scr_dyn_handle_class_alias(SCR_DYNH_WEB_CONTROLLER, SCR_DYNH_WS_RCONTROLLER); } /* The classes in the frontend's WEB_STREAM_CLASSES order. */ @@ -2487,7 +2500,7 @@ ScrDyn *scr_webstream_class(double kind) { bool scr_webstream_instanceof(ScrDyn *value, double kind) { ws_install(); ScrDynHandleTag tag = ws_class_tag(kind); - if (value->kind == SCR_DYN_HANDLE) return value->v.handle.tag == tag; + if (value->kind == SCR_DYN_HANDLE) return scr_dyn_handle_class_tag_of(value) == tag; if (value->kind != SCR_DYN_OBJ && value->kind != SCR_DYN_ARR && value->kind != SCR_DYN_FUNC) return false; ScrDyn *prototype = scr_dyn_handle_class_prototype(tag); bool found = false; diff --git a/tests/corpus/web-stream-fetch-interop.mjs b/tests/corpus/web-stream-fetch-interop.mjs new file mode 100644 index 000000000..f6a3877a1 --- /dev/null +++ b/tests/corpus/web-stream-fetch-interop.mjs @@ -0,0 +1,50 @@ +// Web streams with fetch: a JavaScript ReadableStream as a Response or +// Request body, fetch's bodies and readers as instances of the classes, +// and instanceof through constructor values held in variables. +const encoder = new TextEncoder(); +const RS = ReadableStream; +const Reader = ReadableStreamDefaultReader; +const settled = (p) => p.then((v) => `ok:${v}`, (e) => `${e.name}:${e.message}`); + +const body = new ReadableStream({ + start(c) { + c.enqueue(encoder.encode("he")); + c.enqueue(encoder.encode("llo")); + c.close(); + }, +}); +const response = new Response(body); +console.log("response", await response.text(), response.bodyUsed); + +const fetched = new Response("abc"); +console.log("fetch-body", fetched.body instanceof ReadableStream, fetched.body instanceof RS, fetched.body.getReader() instanceof Reader); + +const own = new ReadableStream(); +const ownReader = own.getReader(); +console.log("variables", own instanceof RS, ownReader instanceof Reader, ownReader instanceof ReadableStreamDefaultReader, {} instanceof RS); +const isInstance = (value, C) => value instanceof C; +console.log("parameters", isInstance(own, ReadableStream), isInstance(fetched.body, RS), isInstance(ownReader, RS), isInstance(new Uint8Array(1), Uint8Array)); + +const locked = new ReadableStream(); +locked.getReader(); +try { + new Response(locked); + console.log("locked", "no error"); +} catch (e) { + console.log("locked", e.name, e.message); +} + +const request = new Request("http://example.com/", { + method: "POST", + body: new ReadableStream({ + start(c) { + c.enqueue(new Uint8Array([104, 105])); + c.close(); + }, + }), + duplex: "half", +}); +console.log("request", await request.text()); + +const strings = new Response(new ReadableStream({ start(c) { c.enqueue("str"); c.close(); } })); +console.log("non-bytes", await settled(strings.text())); diff --git a/tests/harness/corpus-costs/web-stream-fetch-interop.mjs.json b/tests/harness/corpus-costs/web-stream-fetch-interop.mjs.json new file mode 100644 index 000000000..dc6f10404 --- /dev/null +++ b/tests/harness/corpus-costs/web-stream-fetch-interop.mjs.json @@ -0,0 +1 @@ +{ "plain": 3100, "san": 4900 } From aed3e84ff5d4d4381910dd6085544ee8d2c85a2a Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 10:04:40 -0500 Subject: [PATCH 35/54] Borrow typed-array pages and literal-ending getter returns --- .../src/backend/llvm/nullable-unions.test.ts | 16 ++++ .../src/backend/llvm/nullable-unions.ts | 20 +++-- .../src/backend/llvm/walk-borrows.test.ts | 21 +++++ .../compiler/src/backend/llvm/walk-borrows.ts | 9 +- .../corpus/typed-array-or-undefined.ts.json | 6 ++ tests/corpus/typed-array-or-undefined.ts | 82 +++++++++++++++++++ .../typed-array-or-undefined.ts.json | 1 + 7 files changed, 144 insertions(+), 11 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/typed-array-or-undefined.ts.json create mode 100644 tests/corpus/typed-array-or-undefined.ts create mode 100644 tests/harness/corpus-costs/typed-array-or-undefined.ts.json diff --git a/packages/compiler/src/backend/llvm/nullable-unions.test.ts b/packages/compiler/src/backend/llvm/nullable-unions.test.ts index f4e6a6866..ea0517b56 100644 --- a/packages/compiler/src/backend/llvm/nullable-unions.test.ts +++ b/packages/compiler/src/backend/llvm/nullable-unions.test.ts @@ -130,3 +130,19 @@ test("function arms are nullable pointers compared by identity root", () => { expect(isPointerIdentityArm(single)).toBe(false); expect(unions.armTypes().get("fu")).toEqual(fn); }); + +test("typed array arms are nullable pointers compared by identity", () => { + const i32: IrType = { kind: "bytes", elem: "i32" }; + const unions = new NullableUnions( + module([ + { id: "iu", arms: [i32, UNDEFINED_T] }, + { id: "nin", arms: [NULL_T, i32, UNDEFINED_T] }, + { id: "if", arms: [i32, F64] }, + ]), + ); + expect(unions.single("iu")).toMatchObject({ refTag: 0, unitTag: 1, nullTag: -1, arm: i32 }); + expect(unions.single("nin")).toMatchObject({ refTag: 1, unitTag: 2, nullTag: 0, arm: i32 }); + expect(unions.get("if")).toBeNull(); + expect(isObjectArm(unions.single("iu")!)).toBe(true); + expect(isPointerIdentityArm(unions.single("iu")!)).toBe(true); +}); diff --git a/packages/compiler/src/backend/llvm/nullable-unions.ts b/packages/compiler/src/backend/llvm/nullable-unions.ts index eedb922b3..587f3c18c 100644 --- a/packages/compiler/src/backend/llvm/nullable-unions.ts +++ b/packages/compiler/src/backend/llvm/nullable-unions.ts @@ -88,17 +88,17 @@ export const NULLABLE_NULL = "@sc_nullable_null"; export const NULLABLE_ABSENT = "@sc_nullable_absent"; /** The reference arms whose instances are never NULL. Class and record - * entry points all tolerate NULL; strings, Stats snapshots, arrays, maps and - * sets retain through a NULL-skipping wrapper (shapes.ts), as do closures - * (function arms). Object arms are always truthy and compare by identity - * (function arms by identity root); string arms are truthy when non-empty - * and compare bytes. */ + * entry points all tolerate NULL; strings, Stats snapshots, arrays, typed + * arrays, maps and sets retain through a NULL-skipping wrapper (shapes.ts), + * as do closures (function arms). Object arms are always truthy and compare + * by identity (function arms by identity root); string arms are truthy when + * non-empty and compare bytes. */ export type NullableRefArm = | { kind: "object"; className: string } | { kind: "record"; shapeId: string } | { kind: "string" } | { kind: "stats" } - | (IrType & { kind: "array" | "map" | "set" | "func" }); + | (IrType & { kind: "array" | "bytes" | "map" | "set" | "func" }); /** The arm is a JS object: always truthy. */ export function isObjectArm(nullable: PointerUnion): boolean { @@ -273,7 +273,13 @@ function nullableShape( return { ...base, arm: { kind: "record", shapeId: arm.shapeId } }; if (arm.kind === "string") return { ...base, arm: { kind: "string" } }; if (arm.kind === "stats") return { ...base, arm: { kind: "stats" } }; - if (arm.kind === "array" || arm.kind === "map" || arm.kind === "set" || arm.kind === "func") + if ( + arm.kind === "array" || + arm.kind === "bytes" || + arm.kind === "map" || + arm.kind === "set" || + arm.kind === "func" + ) return { ...base, arm }; return null; } diff --git a/packages/compiler/src/backend/llvm/walk-borrows.test.ts b/packages/compiler/src/backend/llvm/walk-borrows.test.ts index caa6be94f..4df5f88fe 100644 --- a/packages/compiler/src/backend/llvm/walk-borrows.test.ts +++ b/packages/compiler/src/backend/llvm/walk-borrows.test.ts @@ -139,3 +139,24 @@ test("returns of walks, including calls to borrowed-return callees, return borro ); expect(analyzeWalks(owned, borrowedHost).returnsWalk).toBe(true); }); + +test("a string literal return keeps the body's other walks borrowed", () => { + const ret = (value: IrExpr | null): IrStmt => ({ kind: "return", value, loc }); + const STRING: IrType = { kind: "string" }; + const text: IrExpr = { + kind: "fieldGet", + obj: ref("node"), + className: "Node", + field: "text", + type: STRING, + loc, + }; + const empty: IrExpr = { kind: "strLit", value: "", type: STRING, loc }; + const getter = fn([ret(text), ret(empty)], []); + expect(analyzeWalks(getter, host).returnsWalk).toBe(true); + // The literal needs no unwritten location; the field read still does. + expect(analyzeWalks(getter, { ...host, unwritten: () => false }).returnsWalk).toBe(false); + expect(analyzeWalks(fn([ret(empty)], []), { ...host, unwritten: () => false }).returnsWalk).toBe( + true, + ); +}); diff --git a/packages/compiler/src/backend/llvm/walk-borrows.ts b/packages/compiler/src/backend/llvm/walk-borrows.ts index 86b8f3d21..573eb775f 100644 --- a/packages/compiler/src/backend/llvm/walk-borrows.ts +++ b/packages/compiler/src/backend/llvm/walk-borrows.ts @@ -42,8 +42,9 @@ const WALK_KINDS: readonly IrNodeKind[] = [ export interface WalkFacts { locals: ReadonlySet; - /** Every return yields a walk (or a throw), so the body can return its - * pointer without a reference: no try statement, at least one return. */ + /** Every return yields a walk, a throw or a string literal (an interned + * constant that nothing frees), so the body can return its pointer + * without a reference: no try statement, at least one return. */ returnsWalk: boolean; } @@ -176,6 +177,7 @@ export function analyzeWalks(fn: IrFunction, host: WalkBorrowHost): WalkFacts { }; const isThrow = (e: IrExpr): boolean => e.kind === "libCall" && e.fn === "error.nodeThrow"; const walkOrThrow = (e: IrExpr): boolean => isThrow(e) || walk(e); + const constant = (e: IrExpr): boolean => isThrow(e) || e.kind === "strLit"; for (let changed = true; changed;) { changed = false; for (const id of candidates) { @@ -199,8 +201,7 @@ export function analyzeWalks(fn: IrFunction, host: WalkBorrowHost): WalkFacts { returns.every( (value) => value !== null && - walkOrThrow(value) && - (isThrow(value) || host.unwritten?.(value, candidates) !== false), + (constant(value) || (walk(value) && host.unwritten?.(value, candidates) !== false)), ); return { locals: result, returnsWalk }; } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/typed-array-or-undefined.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/typed-array-or-undefined.ts.json new file mode 100644 index 000000000..48f4fcde4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/typed-array-or-undefined.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/typed-array-or-undefined.ts" + ], + "diags": [] +} diff --git a/tests/corpus/typed-array-or-undefined.ts b/tests/corpus/typed-array-or-undefined.ts new file mode 100644 index 000000000..11bee503f --- /dev/null +++ b/tests/corpus/typed-array-or-undefined.ts @@ -0,0 +1,82 @@ +// Typed arrays in `T | undefined` slots: paged bit sets, optional fields, +// map values, comparisons, printing and JSON. +class PagedBits { + readonly pages: (Int32Array | undefined)[] = []; + has(id: number): boolean { + const page = this.pages[id >> 8]; + return page !== undefined && (page[(id >> 5) & 7]! & (1 << (id & 31))) !== 0; + } + add(id: number): void { + const index = id >> 8; + let page = this.pages[index]; + if (page === undefined) { + page = new Int32Array(8); + while (this.pages.length <= index) this.pages.push(undefined); + this.pages[index] = page; + } + page[(id >> 5) & 7]! |= 1 << (id & 31); + } + clearPage(index: number): void { + this.pages[index] = undefined; + } +} + +const bits = new PagedBits(); +for (let i = 0; i < 4000; i += 13) bits.add(i); +for (let i = 2000; i < 2600; i += 7) bits.add(i); +let count = 0; +for (let i = 0; i < 5000; i++) if (bits.has(i)) count++; +console.log("count", count, "pages", bits.pages.length); +console.log( + "present", + bits.pages.map((p) => (p === undefined ? "-" : p.length)).join(","), +); +bits.clearPage(3); +let after = 0; +for (let i = 0; i < 5000; i++) if (bits.has(i)) after++; +console.log("after clear", after, bits.pages[3] === undefined, bits.pages[99]); + +// A page read past the end and from a hole. +const sparse: (Uint8Array | undefined)[] = []; +sparse[4] = Uint8Array.of(1, 2, 3); +console.log(sparse.length, sparse[0], sparse[4], sparse[9]); +console.log(sparse.findIndex((p) => p !== undefined), sparse.includes(undefined)); + +// Optional fields and parameters. +class Buffered { + data: Float64Array | undefined; + constructor(n: number) { + this.data = n > 0 ? new Float64Array(n).fill(0.5) : undefined; + } + total(): number { + const d = this.data; + if (!d) return -1; + let s = 0; + for (const x of d) s += x; + return s; + } +} +const buffers = [new Buffered(4), new Buffered(0), new Buffered(2)]; +console.log(buffers.map((b) => b.total())); +console.log(buffers.map((b) => b.data?.length ?? "none")); +buffers[1]!.data = buffers[0]!.data; +console.log(buffers[1]!.data === buffers[0]!.data, buffers[2]!.data === buffers[0]!.data); +buffers[0]!.data![0] = 9; +console.log(buffers[1]!.total(), typeof buffers[1]!.data, typeof buffers[2]!.data?.[5]); + +function pick(a: Int16Array | undefined, b: Int16Array | undefined): Int16Array | undefined { + return a ?? b; +} +const one = Int16Array.of(-1, 2); +console.log(pick(undefined, one), pick(one, undefined) === one, pick(undefined, undefined)); + +// Map values and printing. +const byName = new Map(); +byName.set("a", Uint32Array.of(7, 8)); +byName.set("b", undefined); +for (const [k, v] of byName) console.log(k, v, v ? v[1] : "absent"); +console.log(byName.get("c"), byName.has("b")); +const row: (Int32Array | undefined)[] = [Int32Array.of(5), undefined]; +console.log(row, JSON.stringify(row)); +const copy = [...row, Int32Array.of(6)]; +console.log(copy.filter((p) => p !== undefined).length, copy.indexOf(row[0])); diff --git a/tests/harness/corpus-costs/typed-array-or-undefined.ts.json b/tests/harness/corpus-costs/typed-array-or-undefined.ts.json new file mode 100644 index 000000000..ec575788c --- /dev/null +++ b/tests/harness/corpus-costs/typed-array-or-undefined.ts.json @@ -0,0 +1 @@ +{ "plain": 2800 } From 06e9672d9088f3ec59cfb5637cacd96cb148edb1 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 10:32:32 -0500 Subject: [PATCH 36/54] Degrade a canonical form whose union mapped to the checked dynamic Localized failure's canonical fallback classified a failing type as an exact mapping whenever its canonical form mapped to something other than the checked dynamic. A record or array holding a union that mapCanonical collapses to the checked dynamic (an arm such as {} or unknown) then counted as exact, so its slot got no SC6104 warning and its dynamic operations no attribution. mapCanonical now notes such a collapse (CanonMapNotes), and canonicalType refuses the form when it happened, so the type degrades through the pushed-down form like any other. --- packages/compiler/src/frontend/canon-map.ts | 29 +++++++++++++++---- .../compiler/src/frontend/lowering/lowerer.ts | 13 +++++---- .../compiler/test/canon-normalize.test.ts | 21 +++++++++++++- 3 files changed, 51 insertions(+), 12 deletions(-) diff --git a/packages/compiler/src/frontend/canon-map.ts b/packages/compiler/src/frontend/canon-map.ts index 806c06113..25680d1e0 100644 --- a/packages/compiler/src/frontend/canon-map.ts +++ b/packages/compiler/src/frontend/canon-map.ts @@ -25,8 +25,21 @@ import { const MAX_DEPTH = 24; +/** What a mapping did besides its result. */ +export interface CanonMapNotes { + /** A union in the form mapped to the checked dynamic because an arm + * did (`string | {}`): the result holds a dynamic part the form did not + * spell as `unknown`. */ + collapsedToDynamic: boolean; +} + /** The IR type of a canonical node, or null when no rule applies. */ -export function mapCanonical(c: Canon, ctx: TypeMapperCtx, depth = 0): IrType | null { +export function mapCanonical( + c: Canon, + ctx: TypeMapperCtx, + depth = 0, + notes?: CanonMapNotes, +): IrType | null { if (depth > MAX_DEPTH) return null; const viaChecker = (): IrType | null => (c.src === undefined ? null : mapType(c.src, ctx)); switch (c.k) { @@ -55,13 +68,13 @@ export function mapCanonical(c: Canon, ctx: TypeMapperCtx, depth = 0): IrType | case "tuple": return viaChecker(); case "array": { - const elem = mapCanonical(c.elem, ctx, depth + 1); + const elem = mapCanonical(c.elem, ctx, depth + 1, notes); return elem === null ? null : { kind: "array", elem }; } case "union": { const byKey = new Map(); for (const arm of c.arms) { - const mapped = mapCanonical(arm, ctx, depth + 1); + const mapped = mapCanonical(arm, ctx, depth + 1, notes); if (mapped === null || mapped.kind === "void") return null; if (mapped.kind === "union") { const def = ctx.unions.get(mapped.unionId); @@ -71,7 +84,10 @@ export function mapCanonical(c: Canon, ctx: TypeMapperCtx, depth = 0): IrType | } const arms = [...byKey.values()]; if (arms.length === 1) return arms[0]!; - if (arms.some((a) => a.kind === "dyn")) return DYN; + if (arms.some((a) => a.kind === "dyn")) { + if (notes !== undefined) notes.collapsedToDynamic = true; + return DYN; + } arms.sort((a, b) => (typeKey(a) < typeKey(b) ? -1 : 1)); return { kind: "union", unionId: ctx.unions.intern(arms) }; } @@ -86,7 +102,7 @@ export function mapCanonical(c: Canon, ctx: TypeMapperCtx, depth = 0): IrType | if (props.some((p) => p.accessor)) return viaChecker(); // `{}` holds any non-nullish value: the checked dynamic, as mapType. if (props.length === 0) return DYN; - const structural = mapCanonicalRecord(props, ctx, depth); + const structural = mapCanonicalRecord(props, ctx, depth, notes); // An object whose members do not map structurally (an interface over // Map or Set, methods) keeps its per-type recognition. return structural ?? viaChecker(); @@ -100,10 +116,11 @@ function mapCanonicalRecord( props: readonly CanonProp[], ctx: TypeMapperCtx, depth: number, + notes: CanonMapNotes | undefined, ): IrType | null { const fields: { name: string; type: IrType }[] = []; for (const p of props) { - let t = mapCanonical(p.type(), ctx, depth + 1); + let t = mapCanonical(p.type(), ctx, depth + 1, notes); if (t === null) return null; if (p.optional && t.kind !== "dyn") { const armed = withUndefinedArm(t, ctx.unions); diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 020861bdb..e72434c33 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -10,7 +10,7 @@ import { arrayRootOf, lowerWithArraySpecies, } from "./builtins/array-classes.js"; -import { canonShadowHook, mapCanonical } from "../canon-map.js"; +import { canonShadowHook, mapCanonical, type CanonMapNotes } from "../canon-map.js"; import { Normalizer } from "../canon.js"; import { compactDeclOrigins, @@ -10077,16 +10077,19 @@ export class Lowerer { } /** The IR type of a type's canonical form (canon-map.ts), or null when - * the form is irreducible, unbound, or only the checked dynamic. Runs - * only at localized failure's commit points, so a program that compiles - * today never consults it. */ + * the form is irreducible, unbound, only the checked dynamic, or holds a + * union that mapped to the checked dynamic (such a slot is dynamic in + * part, so it degrades like any other). Runs only at localized failure's + * commit points, so a program that compiles today never consults it. */ canonicalType(type: ts.Type): IrType | null { try { const resolve = this.typeCtx.resolveTypeParamTs; const c = new Normalizer(this.checker).canon(type, resolve ? (t) => resolve(t) : new Map()); if (c.k === "opaque" || c.k === "param") return null; const { typeMemo: _strictMemo, ...strict } = this.typeCtx; - const mapped = mapCanonical(c, { ...strict, canMemoizeType: () => false }); + const notes: CanonMapNotes = { collapsedToDynamic: false }; + const mapped = mapCanonical(c, { ...strict, canMemoizeType: () => false }, 0, notes); + if (notes.collapsedToDynamic) return null; return mapped === null || mapped.kind === "void" || mapped.kind === "dyn" ? null : mapped; } catch { return null; diff --git a/packages/compiler/test/canon-normalize.test.ts b/packages/compiler/test/canon-normalize.test.ts index c510b2790..96406513e 100644 --- a/packages/compiler/test/canon-normalize.test.ts +++ b/packages/compiler/test/canon-normalize.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { canonEqual, canonKey, Normalizer, type Canon } from "../src/frontend/canon.js"; -import { mapCanonical } from "../src/frontend/canon-map.js"; +import { mapCanonical, type CanonMapNotes } from "../src/frontend/canon-map.js"; import { mapType, ShapeRegistry, @@ -76,6 +76,9 @@ beforeAll(() => { // Accessor members: one type, and a getter and setter of different types. export interface Gauge { get level(): number; set level(next: number) } export interface Cell { get value(): string; set value(next: string | number) } + // A union with an arm that maps to the checked dynamic, inside a record. + export type Collapsing = { tag: string | {}; name: string }; + export type Tight = { tag: string; name: string }; // The same instantiations, each refined by an object type. export type RefinedMaps = | (Map & { tag: number })[] @@ -319,6 +322,22 @@ describe("mapCanonical", () => { expect(mapCanonical(new Normalizer(checker).canon(cell), ctx())).toBeNull(); expect(key(new Normalizer(checker).canon(cell))).toContain("get value"); }); + test("a union that maps to the checked dynamic inside a record is noted", () => { + const loose: CanonMapNotes = { collapsedToDynamic: false }; + const mapped = mapCanonical( + new Normalizer(checker).canon(aliases.get("Collapsing")!), + ctx(), + 0, + loose, + ); + expect(mapped?.kind).toBe("record"); + expect(loose.collapsedToDynamic).toBe(true); + const tight: CanonMapNotes = { collapsedToDynamic: false }; + expect( + mapCanonical(new Normalizer(checker).canon(aliases.get("Tight")!), ctx(), 0, tight)?.kind, + ).toBe("record"); + expect(tight.collapsedToDynamic).toBe(false); + }); test("shadow mode observes every successful top-level mapping", () => { const seen: string[] = []; const point = { kind: "object", className: "Point" } as const; From 0c954ebfd6d3a035278ecb8476950e31aee891ae Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 10:43:05 -0500 Subject: [PATCH 37/54] Reserve fewer worker arenas before smaller class slices on hosts that limit address space Worker runtimes reserve one region for 32 arenas of 40 class slices. When the host refused the full region (32 arenas of 1 GiB slices, 1.28 TiB of address space), the allocator kept 32 arenas and shrank every slice. Under valgrind, which maps at most 56 GiB, slices dropped to 16 MiB, and every class whose live blocks outgrew 16 MiB took the system allocator. Callgrind profiles then counted system allocations that a native run never makes: tsc-ts checking VS Code made 10.3 M small calloc calls under valgrind and 0.06 M natively. The reservation now tries arena counts from 32 down to 4 at each slice size, from 1 GiB down, before smaller slices, and 2 and 1 arena only when no slice size fits 4. Arenas that are not reserved stay claimed, so threads beyond them use the system allocator, and the region bounds of foreign frees use the arenas reserved. Hosts that map the full region keep 32 arenas of 1 GiB slices. A runtime test limits RLIMIT_AS to 60 GiB and checks that the first thread gets 256 MiB slices for 400,000 blocks of 56 bytes, that three more threads get arenas and a fourth uses the system allocator, and that blocks freed across threads keep their contents. --- packages/runtime/src/scr_alloc.c | 45 ++++++++---- packages/runtime/test/sa-reserve.test.ts | 38 ++++++++++ packages/runtime/test/test_sa_reserve.c | 91 ++++++++++++++++++++++++ 3 files changed, 161 insertions(+), 13 deletions(-) create mode 100644 packages/runtime/test/sa-reserve.test.ts create mode 100644 packages/runtime/test/test_sa_reserve.c diff --git a/packages/runtime/src/scr_alloc.c b/packages/runtime/src/scr_alloc.c index ac80df6e2..13b83a396 100644 --- a/packages/runtime/src/scr_alloc.c +++ b/packages/runtime/src/scr_alloc.c @@ -797,10 +797,12 @@ static void scr_sa_large_note(void *p, size_t n) { * once; arena i spans [region + i * arena_span, + arena_span) and is split * into class slices exactly like the single-threaded reservation. */ #define SCR_SA_ARENAS 32u +#define SCR_SA_MIN_ARENAS 4u /* the fewest arenas a limited host gets before smaller slices */ static pthread_mutex_t scr_sa_region_lock = PTHREAD_MUTEX_INITIALIZER; static _Atomic(uintptr_t) scr_sa_region; /* 0 until reserved */ static uintptr_t scr_sa_arena_span; static unsigned scr_sa_region_shift; +static unsigned scr_sa_region_arenas; /* arenas the region holds, a power of two */ static bool scr_sa_region_refused; static _Atomic(uint32_t) scr_sa_claimed; /* one bit per arena */ static ScrSaState scr_sa_parked[SCR_SA_ARENAS]; /* unowned arenas */ @@ -813,17 +815,34 @@ static bool scr_sa_reserve_region(void) { pthread_mutex_lock(&scr_sa_region_lock); if (!atomic_load_explicit(&scr_sa_region, memory_order_acquire) && !scr_sa_region_refused) { /* Same per-class slice ladder as the single-threaded reservation, for - * every arena at once; hosts that limit address space get smaller - * slices (each slice that fills falls back to the system). */ - for (unsigned shift = 30; shift >= 22; shift -= 2) { - size_t arena = (size_t)SCR_SA_NCLASS << shift; - size_t span = arena * SCR_SA_ARENAS; - void *p = scr_sa_map(span); - if (!p) continue; - scr_sa_arena_span = arena; - scr_sa_region_shift = shift; - atomic_store_explicit(&scr_sa_region, (uintptr_t)p, memory_order_release); - break; + * every arena at once. A host that limits address space (valgrind maps + * at most 56 GiB) gets fewer arenas before smaller slices, down to + * SCR_SA_MIN_ARENAS, and fewer still only when no slice size fits that + * many. A slice that fills falls back to the system, so with smaller + * slices a program that keeps many objects of one class makes system + * allocations that a run with full slices does not, and a profile taken + * under the limit counts them. At least SCR_SA_MIN_ARENAS arenas are + * kept because threads claim them in the order they first allocate: + * the main thread keeps its arena, and a thread that starts later reuses + * one that another thread parked. Threads beyond the arenas reserved use + * the system allocator. */ + for (unsigned least = SCR_SA_MIN_ARENAS; least >= 1 && !scr_sa_region_arenas; least /= 2) { + for (unsigned shift = 30; shift >= 22 && !scr_sa_region_arenas; shift -= 2) { + for (unsigned arenas = least == SCR_SA_MIN_ARENAS ? SCR_SA_ARENAS : least; arenas >= least; + arenas /= 2) { + size_t arena = (size_t)SCR_SA_NCLASS << shift; + void *p = scr_sa_map(arena * arenas); + if (!p) continue; + scr_sa_arena_span = arena; + scr_sa_region_shift = shift; + scr_sa_region_arenas = arenas; + /* The arenas that were not reserved stay claimed. */ + if (arenas < SCR_SA_ARENAS) + atomic_fetch_or_explicit(&scr_sa_claimed, ~((1u << arenas) - 1u), memory_order_relaxed); + atomic_store_explicit(&scr_sa_region, (uintptr_t)p, memory_order_release); + break; + } + } } if (!atomic_load_explicit(&scr_sa_region, memory_order_relaxed)) scr_sa_region_refused = true; } @@ -1743,7 +1762,7 @@ static void scr_sa_park_arena(void) { static size_t scr_sa_foreign_size(const void *p) { uintptr_t region = atomic_load_explicit(&scr_sa_region, memory_order_acquire); uintptr_t off = (uintptr_t)p - region; - if (!region || off >= scr_sa_arena_span * SCR_SA_ARENAS) return 0; + if (!region || off >= scr_sa_arena_span * scr_sa_region_arenas) return 0; return scr_sa_class_size((unsigned)((off % scr_sa_arena_span) >> scr_sa_region_shift)); } @@ -1753,7 +1772,7 @@ void scr_sa_free_foreign(void *p) { #endif uintptr_t region = atomic_load_explicit(&scr_sa_region, memory_order_acquire); uintptr_t off = (uintptr_t)p - region; - if (region && off < scr_sa_arena_span * SCR_SA_ARENAS) { + if (region && off < scr_sa_arena_span * scr_sa_region_arenas) { unsigned i = (unsigned)(off / scr_sa_arena_span); ScrSaBlock *b = p; ScrSaBlock *head = atomic_load_explicit(&scr_sa_remote[i], memory_order_relaxed); diff --git a/packages/runtime/test/sa-reserve.test.ts b/packages/runtime/test/sa-reserve.test.ts new file mode 100644 index 000000000..b6258bb76 --- /dev/null +++ b/packages/runtime/test/sa-reserve.test.ts @@ -0,0 +1,38 @@ +import { execFile } from "node:child_process"; +import { mkdir } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; + +const execFileAsync = promisify(execFile); +const testDir = import.meta.dirname; + +// Under a limited address space, as under valgrind, worker runtimes reserve +// fewer arenas, down to four, before smaller class slices. +test.runIf(process.platform === "linux" && process.arch === "x64")( + "a limited address space reserves fewer arenas before smaller class slices", + async () => { + const buildDir = join(testDir, "build"); + await mkdir(buildDir, { recursive: true }); + const bin = join(buildDir, "test_sa_reserve"); + await execFileAsync("clang", [ + "-std=c11", + "-D_GNU_SOURCE", + "-O1", + "-Wall", + "-Wextra", + "-fsanitize=undefined", + "-DSCR_WORKERS", + "-pthread", + "-o", + bin, + join(testDir, "test_sa_reserve.c"), + join(testDir, "../src/scr_alloc.c"), + ]); + const run = await execFileAsync(bin, [], { + env: { ...process.env, UBSAN_OPTIONS: "halt_on_error=1" }, + }); + expect(run.stdout).toBe("reservation checks passed\n"); + }, + 120_000, +); diff --git a/packages/runtime/test/test_sa_reserve.c b/packages/runtime/test/test_sa_reserve.c new file mode 100644 index 000000000..dbb983b9f --- /dev/null +++ b/packages/runtime/test/test_sa_reserve.c @@ -0,0 +1,91 @@ +/* The worker allocator's reservation under a limited address space. With + * RLIMIT_AS at 60 GiB, too little for 32 arenas of 1 GiB class slices, the + * region holds 4 arenas of 256 MiB slices rather than 32 arenas of 16 MiB + * slices: 400,000 blocks of 56 bytes (more than a 16 MiB slice holds) all + * come from the first thread's arena. Of four more threads alive at once, + * three get the other arenas and one uses the system allocator; blocks + * freed across threads go back to their owners. */ +#include "../src/scr_runtime.h" + +#include +#include +#include +#include +#include +#include +#include + +#define COUNT 400000 +#define SIZE 56 +#define THREADS 4 +#define PER_THREAD 1000 + +_Noreturn void scr_trap(const char *msg) { + fputs(msg, stderr); + exit(EXIT_FAILURE); +} + +static void check(bool condition, const char *message) { + if (condition) return; + fprintf(stderr, "reservation test failed: %s\n", message); + exit(EXIT_FAILURE); +} + +/* Whether p lies in the calling thread's arena. */ +static bool in_arena(const void *p) { return (uintptr_t)p - scr_sa.base < scr_sa.span; } + +static unsigned char *thread_blocks[THREADS][PER_THREAD]; +static atomic_int with_arena, started; +static atomic_bool done; + +/* Allocates PER_THREAD blocks, then stays alive until every thread has. */ +static void *allocate(void *arg) { + unsigned char **blocks = arg; + for (size_t i = 0; i < PER_THREAD; i++) { + blocks[i] = scr_mem_alloc(SIZE); + check(blocks[i] != NULL, "allocate on another thread"); + memset(blocks[i], 0x44, SIZE); + } + bool owned = scr_sa.span != 0; + for (size_t i = 0; i < PER_THREAD; i++) check(in_arena(blocks[i]) == owned, "blocks come from the thread's arena, if any"); + if (owned) atomic_fetch_add(&with_arena, 1); + atomic_fetch_add(&started, 1); + while (!atomic_load(&done)) { + } + return NULL; +} + +int main(void) { + struct rlimit limit = {(rlim_t)60 << 30, (rlim_t)60 << 30}; + check(setrlimit(RLIMIT_AS, &limit) == 0, "limit the address space"); + unsigned char **blocks = malloc(COUNT * sizeof *blocks); + check(blocks != NULL, "block table"); + for (size_t i = 0; i < COUNT; i++) { + blocks[i] = scr_mem_alloc(SIZE); + check(blocks[i] != NULL, "allocate"); + memset(blocks[i], (int)(i & 0xff), SIZE); + } + check(scr_sa.shift == 28, "the class slices are 256 MiB"); + for (size_t i = 0; i < COUNT; i++) check(in_arena(blocks[i]), "every block comes from the arena"); + pthread_t threads[THREADS]; + for (int t = 0; t < THREADS; t++) + check(pthread_create(&threads[t], NULL, allocate, thread_blocks[t]) == 0, "start a thread"); + while (atomic_load(&started) < THREADS) { + } + check(atomic_load(&with_arena) == THREADS - 1, "three more threads get arenas"); + atomic_store(&done, true); + for (int t = 0; t < THREADS; t++) check(pthread_join(threads[t], NULL) == 0, "join a thread"); + for (int t = 0; t < THREADS; t++) + for (size_t i = 0; i < PER_THREAD; i++) { + check(!in_arena(thread_blocks[t][i]), "other threads' blocks are not in this arena"); + for (size_t j = 0; j < SIZE; j++) check(thread_blocks[t][i][j] == 0x44, "other threads' blocks keep their contents"); + scr_mem_free(thread_blocks[t][i]); + } + for (size_t i = 0; i < COUNT; i++) { + for (size_t j = 0; j < SIZE; j++) check(blocks[i][j] == (unsigned char)(i & 0xff), "blocks keep their contents"); + scr_mem_free(blocks[i]); + } + free(blocks); + printf("reservation checks passed\n"); + return 0; +} From b11bc54a2ba3bce41b34e5ff9d5b38c4f603142c Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 10:46:54 -0500 Subject: [PATCH 38/54] Expect the collapsed canonical forms as dynamic slots in the coverage tests The tools field (Tool[]) and the all global ((number | Callbacks)[]) hold unions that canonical mapping maps to the checked dynamic, so the canonical fallback now degrades them: they are dynamic slots again, with SC6104 warnings and attributed operations, as these tests expected before the fallback classified them as exact. --- tests/harness/coverage.test.ts | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/harness/coverage.test.ts b/tests/harness/coverage.test.ts index ce3f22d7c..82e541140 100644 --- a/tests/harness/coverage.test.ts +++ b/tests/harness/coverage.test.ts @@ -431,10 +431,11 @@ test("--explain-dynamic attributes each refusal to the unit the dynamic tier wou test("--explain-dynamic lists the type failures localized failure would turn into dynamic slots", async () => { const { coverage } = analyze(fixture("localized-failure.ts"), { explainDynamic: true }); const report = coverage.tiers!.localizedFailure; - // Locals and parameters degrade (stages L1 and L2), the 'any' field - // compiles, and the `Tool[]` field maps exactly from its canonical form - // (stage N2); only the refused WeakRef field and its construction remain. + // Locals, fields and parameters degrade (stages L1 and L2), and the + // 'any' field compiles; only the refused WeakRef field and its + // construction remain. expect(coverage.dynamicSlots?.map((s) => [s.slot, s.name])).toEqual([ + ["field", "tools"], ["param", "progress"], ["local", "callbacks"], ]); @@ -457,26 +458,28 @@ test("a local or module global whose declared type does not compile becomes a dy expect(coverage.diagnostics).toEqual([]); expect(coverage.dynamicSlots?.map((s) => [s.slot, s.name, s.declaredType, s.code])).toEqual([ ["global", "current", "Callbacks | undefined", "SC2009"], + ["global", "all", "(number | Callbacks)[]", "SC2009"], ["local", "local", "Callbacks | undefined", "SC2009"], ]); - // `all`'s element type maps exactly from its canonical form (stage N2), so - // the array is no dynamic slot. - expect(coverage.dynamicSlots?.find((s) => s.name === "all")).toBeUndefined(); - expect(coverage.performance?.filter((d) => d.code === "SC6104")).toHaveLength(2); + // The loop over `all` is top-level code: its dynamic operations count too. + expect(coverage.dynamicSlots?.find((s) => s.name === "all")?.operations).toBeGreaterThan(0); + expect(coverage.performance?.filter((d) => d.code === "SC6104")).toHaveLength(3); expect(coveragePasses(coverage, "blockers")).toBe(true); expect(coveragePasses(coverage, "dynamic")).toBe(true); expect(coveragePasses(coverage, "dynamic-slots")).toBe(false); const envelope = coverageEnvelope(coverage, { compilerVersion: "test", sourceTexts }); expect(envelope.dynamicSlots?.map((s) => [s.kind, s.name, s.line])).toEqual([ ["global", "current", 10], + ["global", "all", 36], ["local", "local", 14], ]); expect(envelope.diagnostics.filter((d) => d.code === "SC6104").map((d) => d.severity)).toEqual([ "warning", "warning", + "warning", ]); const text = renderCoverage(coverage, { sourceTexts, root: repoRoot }); - expect(text).toContain("dynamic slots 2"); + expect(text).toContain("dynamic slots 3"); expect(text).toContain("module global 'current' (declared 'Callbacks | undefined')"); // --tier=static-only keeps every refusal. const strict = analyze(program, { tier: "static-only" }).coverage; From fa65a3b38509d4a7caac25c8c5ab9c61e68c3e20 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 10:49:22 -0500 Subject: [PATCH 39/54] Specialize calls that omit a constant default and drop unreached originals --- .../src/backend/llvm/repr-report.test.ts | 10 +- .../src/ir/numeric-call-specialization.ts | 210 ++++++++++++++---- .../test/numeric-call-specialization.test.ts | 72 ++++++ .../optional-parameter-defaults.ts.json | 6 + tests/corpus/optional-parameter-defaults.ts | 56 +++++ .../optional-parameter-defaults.ts.json | 1 + 6 files changed, 311 insertions(+), 44 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/optional-parameter-defaults.ts.json create mode 100644 tests/corpus/optional-parameter-defaults.ts create mode 100644 tests/harness/corpus-costs/optional-parameter-defaults.ts.json diff --git a/packages/compiler/src/backend/llvm/repr-report.test.ts b/packages/compiler/src/backend/llvm/repr-report.test.ts index ec7b9b5ed..14bae41f7 100644 --- a/packages/compiler/src/backend/llvm/repr-report.test.ts +++ b/packages/compiler/src/backend/llvm/repr-report.test.ts @@ -50,11 +50,11 @@ console.log(scale(boxes[1]!.id), scale(3, 4), half(boxes.length), boxes[2]!.item expect(row("field", "Box", "hits")?.[4]).toBe("double"); expect(row("field", "Box", "hits")?.[5]).not.toBe(""); expect(row("field", "Box", "items")?.slice(3, 5)).toEqual(["array", "ptr"]); - expect(row("param", "scale", "value")?.[4]).toBe("i32"); - // A defaulted parameter is a `number | undefined` box; the variant for - // calls that pass a number takes an i32. - expect(row("param", "scale", "by")?.slice(3, 5)).toEqual(["f64 | undefined", "box"]); - expect(row("param", "scale.nativeNumber", "by")?.[4]).toBe("i32"); + // Both calls of `scale` take its variant, whose defaulted parameter is + // a number: the one that omits it passes the default. + expect(row("param", "scale", "by")).toBeUndefined(); + expect(row("param", "scale.nativeNumber", "value")?.[4]).toBe("i32"); + expect(row("param", "scale.nativeNumber", "by")?.slice(3, 5)).toEqual(["f64", "i32"]); // An array length may exceed the int32 range. expect(row("param", "half", "value")?.[4]).toBe("double"); expect(row("param", "half", "value")?.[5]).toMatch(/^argument in .* is not int32$/); diff --git a/packages/compiler/src/ir/numeric-call-specialization.ts b/packages/compiler/src/ir/numeric-call-specialization.ts index ecb7e9447..d90c35e2f 100644 --- a/packages/compiler/src/ir/numeric-call-specialization.ts +++ b/packages/compiler/src/ir/numeric-call-specialization.ts @@ -5,17 +5,29 @@ import { type IrFunction, type IrModule, type IrStmt, + type IrType, type IrUnionDef, } from "./ir.js"; +import { exprFunctionReference, moduleFunctionReferences } from "./function-references.js"; import { everyStmtList, mapExprChildren, mapStmtChildren, transformStmtList } from "./traverse.js"; const MAX_FUNCTION_NODES = 1024; -const MAX_SPECIALIZATIONS = 32; +/** The variants' bodies together: at least this many nodes, or a tenth of + * the module's. */ +const MIN_SPECIALIZATION_BUDGET = 32 * MAX_FUNCTION_NODES; + interface NumericParameter { index: number; localId: string; unionId: string; + /** The tag of the number or boolean arm, and that arm's type. */ tag: number; + arm: IrType; + undefinedTag: number; + /** The constant of `param = `, when the parameter's only reads + * are that default's: a call that passes undefined (or omits the + * argument) then calls the variant with the constant instead. */ + fallback: IrExpr | null; } function parameters(fn: IrFunction, unions: ReadonlyMap): NumericParameter[] { @@ -24,41 +36,85 @@ function parameters(fn: IrFunction, unions: ReadonlyMap): Nu fn.params.forEach((param, index) => { if (param.type.kind !== "union") return; const arms = unions.get(param.type.unionId)?.arms; - if (arms?.length !== 2 || !arms.some((arm) => arm.kind === "undefinedT")) return; - const tag = arms.findIndex((arm) => arm.kind === "f64"); - if (tag >= 0) result.push({ index, localId: param.localId, unionId: param.type.unionId, tag }); + if (arms?.length !== 2) return; + const undefinedTag = arms.findIndex((arm) => arm.kind === "undefinedT"); + const tag = arms.findIndex((arm) => arm.kind === "f64" || arm.kind === "bool"); + if (undefinedTag < 0 || tag < 0) return; + result.push({ + index, + localId: param.localId, + unionId: param.type.unionId, + tag, + arm: arms[tag]!.kind === "f64" ? F64 : BOOL, + undefinedTag, + fallback: null, + }); }); if (!result.length) return result; const selected = new Map(result.map((param) => [param.localId, param])); if (fn.locals.some((local) => selected.has(local.id) && (local.boxed || local.tdz))) return []; let size = 0; - function numericWrite(id: string, value: IrExpr | undefined): boolean { + function armWrite(id: string, value: IrExpr | undefined): boolean { const param = selected.get(id); return ( !param || (value?.kind === "unionWrap" && value.unionId === param.unionId && value.tag === param.tag && - value.value.type.kind === "f64") + value.value.type.kind === param.arm.kind) ); } + // Reads of each parameter, and the reads inside a default's declaration. + const reads = new Map(); const eligible = everyStmtList(fn.body, { stmt: (stmt) => { if (++size > MAX_FUNCTION_NODES) return false; - if (stmt.kind === "assign") return numericWrite(stmt.localId, stmt.value); + if (stmt.kind === "assign") return armWrite(stmt.localId, stmt.value); if (stmt.kind === "varDecl" || stmt.kind === "forOf") return !selected.has(stmt.localId); return true; }, expr: (expr) => { if (++size > MAX_FUNCTION_NODES) return false; - if (expr.kind === "assignExpr") return numericWrite(expr.localId, expr.value); + if (expr.kind === "varRef" && selected.has(expr.localId)) + reads.set(expr.localId, (reads.get(expr.localId) ?? 0) + 1); + if (expr.kind === "assignExpr") return armWrite(expr.localId, expr.value); if (expr.kind === "incDec") return !selected.has(expr.localId); if (expr.kind === "closure" || expr.kind === "classRef") return !expr.captures?.some((id) => selected.has(id)); return true; }, }); - return eligible ? result : []; + if (!eligible) return []; + // A default the lowering writes as the body's leading declarations: + // `x = isUndefined(param) ? : narrow(param)`. When those are the + // parameter's only two reads, passing the constant is the same as passing + // undefined. + for (const stmt of fn.body) { + if (stmt.kind !== "varDecl") break; + const init = stmt.init; + if (init?.kind !== "ternary") break; + const { cond, then, else_ } = init; + if (cond.kind !== "unionIsTag" || cond.negated || cond.value.kind !== "varRef") break; + const param = selected.get(cond.value.localId); + if ( + !param || + cond.unionId !== param.unionId || + cond.tag !== param.undefinedTag || + else_.kind !== "unionNarrow" || + else_.tag !== param.tag || + else_.value.kind !== "varRef" || + else_.value.localId !== param.localId || + reads.get(param.localId) !== 2 || + !constantOf(then, param.arm) + ) + break; + param.fallback = then; + } + return result; +} + +function constantOf(expr: IrExpr, arm: IrType): boolean { + return arm.kind === "f64" ? expr.kind === "numLit" : expr.kind === "boolLit"; } function specializeBody(fn: IrFunction, slots: NumericParameter[]): IrStmt[] { @@ -83,7 +139,7 @@ function specializeBody(fn: IrFunction, slots: NumericParameter[]): IrStmt[] { type: BOOL, loc: expr.loc, }; - if (expr.tag === param.tag) return { ...expr.value, type: F64 }; + if (expr.tag === param.tag) return { ...expr.value, type: param.arm }; } } if (expr.kind === "varRef") { @@ -93,7 +149,7 @@ function specializeBody(fn: IrFunction, slots: NumericParameter[]): IrStmt[] { kind: "unionWrap", unionId: param.unionId, tag: param.tag, - value: { ...expr, type: F64 }, + value: { ...expr, type: param.arm }, type: expr.type, loc: expr.loc, }; @@ -105,7 +161,7 @@ function specializeBody(fn: IrFunction, slots: NumericParameter[]): IrStmt[] { kind: "unionWrap", unionId: param.unionId, tag: param.tag, - value: { ...expr, value: expr.value.value, type: F64 }, + value: { ...expr, value: expr.value.value, type: param.arm }, type: expr.type, loc: expr.loc, }; @@ -115,12 +171,47 @@ function specializeBody(fn: IrFunction, slots: NumericParameter[]): IrStmt[] { }); } -/** Direct calls that already carry known numbers need no optional-parameter - * boxes. Keep the ordinary function and its ABI for holes, undefined, - * closures, exports and virtual dispatch. One bounded native variant per - * function preserves the body, argument order and return representation; - * mixed or unknown incoming values retain the original call. The backend - * supplies its symbol spelling to avoid collisions after name mangling. */ +/** An argument the variant takes in place of a parameter's box: the + * wrapped number or boolean, the default's constant for an undefined, or a + * conditional choosing between two such arguments. */ +function nativeArgument(arg: IrExpr | undefined, param: NumericParameter): IrExpr | null { + if (arg?.kind === "ternary") { + const then = nativeArgument(arg.then, param); + const else_ = then && nativeArgument(arg.else_, param); + return else_ ? { ...arg, then: then!, else_, type: param.arm } : null; + } + if (arg?.kind !== "unionWrap" || arg.unionId !== param.unionId) return null; + if (arg.tag === param.tag && arg.value.type.kind === param.arm.kind) return arg.value; + const fallback = param.fallback; + if (arg.tag === param.undefinedTag && arg.value.kind === "unitLit" && fallback !== null) + return { ...fallback, loc: arg.loc }; + return null; +} + +function countNodes(body: IrStmt[]): number { + let size = 0; + everyStmtList(body, { + stmt: () => { + size++; + return true; + }, + expr: () => { + size++; + return true; + }, + }); + return size; +} + +/** Direct calls that already carry known numbers or booleans, or that pass + * undefined to a parameter with a constant default, need no + * optional-parameter boxes. Keep the ordinary function and its ABI for + * other values, closures, exports and virtual dispatch. One native variant + * per function preserves the body, argument order and return + * representation; mixed or unknown incoming values retain the original + * call. An ordinary function no reference reaches after the rewrite is + * dropped. The variants share a node budget. The backend supplies its + * symbol spelling to avoid collisions after name mangling. */ export function specializeNumericCalls( mod: IrModule, symbolName: (name: string) => string = (name) => name, @@ -132,6 +223,9 @@ export function specializeNumericCalls( if (slots.length) candidates.set(fn.name, { fn, slots }); } if (!candidates.size) return mod; + let moduleNodes = 0; + for (const fn of mod.functions) moduleNodes += countNodes(fn.body); + let budget = Math.max(MIN_SPECIALIZATION_BUDGET, moduleNodes / 10); const names = new Set(mod.functions.map((fn) => symbolName(fn.name))); const variants = new Map(); const pending: IrFunction[] = []; @@ -139,15 +233,7 @@ export function specializeNumericCalls( if (expr.kind !== "call") return undefined; const candidate = candidates.get(expr.callee); if (!candidate || candidate.fn.name === mod.entry) return undefined; - return candidate.slots.every((param) => { - const arg = expr.args[param.index]; - return ( - arg?.kind === "unionWrap" && - arg.unionId === param.unionId && - arg.tag === param.tag && - arg.value.type.kind === "f64" - ); - }) + return candidate.slots.every((param) => nativeArgument(expr.args[param.index], param) !== null) ? candidate : undefined; } @@ -159,31 +245,39 @@ export function specializeNumericCalls( const { fn, slots } = candidate; let variant = variants.get(fn.name); if (!variant) { - if (variants.size >= MAX_SPECIALIZATIONS) return node; + const body = specializeBody(fn, slots); + const size = countNodes(body); + if (size > budget) return node; + budget -= size; let name = `${fn.name}.nativeNumber`; while (names.has(symbolName(name))) name += "_"; names.add(symbolName(name)); - const ids = new Set(slots.map((param) => param.localId)); + const types = new Map(slots.map((param) => [param.localId, param.arm])); const { ownsPrototype: _ownsPrototype, ...implementation } = fn; variant = { ...implementation, name, - params: fn.params.map((param) => - ids.has(param.localId) ? { ...param!, type: F64 } : param, - ), - locals: fn.locals.map((local) => (ids.has(local.id) ? { ...local!, type: F64 } : local)), - body: specializeBody(fn, slots), + params: fn.params.map((param) => { + const type = types.get(param.localId); + return type ? { ...param!, type } : param; + }), + locals: fn.locals.map((local) => { + const type = types.get(local.id); + return type ? { ...local!, type } : local; + }), + body, }; variants.set(fn.name, variant); pending.push(variant); } - const indices = new Set(slots.map((param) => param.index)); + const byIndex = new Map(slots.map((param) => [param.index, param])); return { ...node, callee: variant.name, - args: node.args.map((arg, index) => - indices.has(index) && arg.kind === "unionWrap" ? arg.value : arg, - ), + args: node.args.map((arg, index) => { + const param = byIndex.get(index); + return param ? nativeArgument(arg, param)! : arg; + }), }; } function rewriteStmt(stmt: IrStmt): IrStmt { @@ -203,5 +297,43 @@ export function specializeNumericCalls( const fn = pending[i]!; functions.push(rewriteFunction(fn)); } - return variants.size ? { ...mod, functions } : mod; + if (!variants.size) return mod; + const unreferenced = unreferencedOriginals(mod, functions, variants); + return { + ...mod, + functions: unreferenced.size ? functions.filter((fn) => !unreferenced.has(fn.name)) : functions, + }; +} + +/** The ordinary functions of `variants` that nothing reaches any more: no + * call or function reference from another function, no module metadata + * names them, and they are not class members (which dispatch reaches by + * their `%.` names). A function's prototype is observable + * only through the function as a value, which is a reference. */ +function unreferencedOriginals( + mod: IrModule, + functions: readonly IrFunction[], + variants: ReadonlyMap, +): Set { + const result = new Set(); + const members = (mod.classes ?? []).map((cls) => `%${cls.name}.`); + for (const name of variants.keys()) + if (!members.some((prefix) => name.startsWith(prefix))) result.add(name); + for (const name of moduleFunctionReferences(mod)) result.delete(name); + if (!result.size) return result; + for (const fn of functions) + everyStmtList(fn.body, { + stmt: () => true, + expr: (expr) => { + const name = + expr.kind === "call" + ? expr.callee + : expr.kind === "new" + ? `%${expr.className}.constructor` + : exprFunctionReference(expr); + if (name !== undefined && name !== fn.name) result.delete(name); + return true; + }, + }); + return result; } diff --git a/packages/compiler/test/numeric-call-specialization.test.ts b/packages/compiler/test/numeric-call-specialization.test.ts index 751764a15..0311a4fda 100644 --- a/packages/compiler/test/numeric-call-specialization.test.ts +++ b/packages/compiler/test/numeric-call-specialization.test.ts @@ -61,3 +61,75 @@ console.log(scale(5), forward(6), rewrite(-1), scale_nativeNumber(1)); await rm(dir, { recursive: true, force: true }); } }); + +test("calls that omit a constant default or pass booleans take the variant, and unreached originals go", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-presence-calls-")); + try { + const entry = join(dir, "main.ts"), + outPath = join(dir, "main.ir.json"); + await writeFile( + entry, + ` +let ticks = 0; +function tick(): number { ticks++; return ticks; } +function scale(value: number, by: number = 2): number { return value * by; } +function flag(on: boolean = true): string { return on ? "on" : "off"; } +function stepped(value: number, step: number = tick()): number { return value + step; } +function shown(value: number, extra?: number): string { return extra === undefined ? "-" : String(value + extra); } +function held(value: number, by: number = 3): number { return value - by; } +const keep = held; +const values: number[] = [4]; +const on = values.length > 0; +console.log(scale(values.length), scale(1, undefined), scale(2, on ? 5 : undefined), flag(), flag(on)); +console.log(stepped(1), stepped(2, 3), shown(1), shown(1, 2), held(5), keep(6)); +`, + ); + const result = await compile(entry, { outDir: dir, outPath, outputKind: "ir" }); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + const original = deserializeModule(await readFile(outPath, "utf8")); + const specialized = specializeNumericCalls(original); + expect(validateModule(specialized)).toEqual([]); + const names = new Set(specialized.functions.map((fn) => fn.name)); + // Every call of scale and flag reaches the variant, so the originals go; + // held is also a value, and stepped's and shown's omitted arguments are + // not constants, so those keep theirs. + expect( + [...names].filter((name) => /^(scale|flag|stepped|shown|held)/.test(name)).sort(), + ).toEqual([ + "flag.nativeNumber", + "held", + "held.nativeNumber", + "scale.nativeNumber", + "shown", + "shown.nativeNumber", + "stepped", + "stepped.nativeNumber", + ]); + expect( + specialized.functions.find((fn) => fn.name === "flag.nativeNumber")?.params[0]?.type.kind, + ).toBe("bool"); + const calls: [string, string][] = []; + for (const fn of specialized.functions) + everyStmtList(fn.body, { + stmt: () => true, + expr: (expr) => { + if (expr.kind === "call" && /^(scale|flag|stepped|shown)/.test(expr.callee)) + calls.push([expr.callee, expr.args.map((arg) => arg.kind).join(",")]); + return true; + }, + }); + expect(calls).toEqual([ + ["scale.nativeNumber", "arrIntrinsic,numLit"], + ["scale.nativeNumber", "numLit,numLit"], + ["scale.nativeNumber", "numLit,ternary"], + ["flag.nativeNumber", "boolLit"], + ["flag.nativeNumber", "varRef"], + ["stepped", "numLit,unionWrap"], + ["stepped.nativeNumber", "numLit,numLit"], + ["shown", "numLit,unionWrap"], + ["shown.nativeNumber", "numLit,numLit"], + ]); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/optional-parameter-defaults.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/optional-parameter-defaults.ts.json new file mode 100644 index 000000000..c02772f40 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/optional-parameter-defaults.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/optional-parameter-defaults.ts" + ], + "diags": [] +} diff --git a/tests/corpus/optional-parameter-defaults.ts b/tests/corpus/optional-parameter-defaults.ts new file mode 100644 index 000000000..ac5852b7b --- /dev/null +++ b/tests/corpus/optional-parameter-defaults.ts @@ -0,0 +1,56 @@ +// Defaulted and optional number and boolean parameters, called with the +// argument omitted, with an explicit undefined, with a value, and through +// the function as a value: every form must see JavaScript's defaults. + +function scale(value: number, by: number = 2): number { + return value * by; +} + +function flag(on: boolean = true): string { + return on ? "on" : "off"; +} + +function depth(node: number, level: number = 0): number { + // Recursion that omits the argument and passes it. + if (node <= 1) return level; + return node % 2 === 0 ? depth(node / 2, level + 1) : depth(node - 1); +} + +let counter = 0; +function next(): number { + counter++; + return counter * 10; +} + +function sideEffect(base: number, step: number = next()): number { + // A default that is not a constant runs at each call that omits it. + return base + step; +} + +function optional(value: number, extra?: number): string { + return extra === undefined ? `${value}` : `${value}+${extra}`; +} + +function signed(value: number = 5): string { + return Object.is(value, -0) ? "-0" : String(value); +} + +const values = [1, 2.5, -3, 0]; +for (const v of values) { + console.log("scale", scale(v), scale(v, undefined), scale(v, 3), scale(v, NaN)); + // A conditional argument, including one that passes undefined. + console.log("choose", scale(v, v > 1 ? 3 : undefined), flag(v > 0 ? false : undefined), signed(v < 0 ? -0 : v)); +} +console.log("flag", flag(), flag(undefined), flag(false), flag(true)); +console.log("depth", depth(1), depth(12), depth(12, 5), depth(7)); +console.log("side", sideEffect(1), sideEffect(1, 4), sideEffect(2), counter); +console.log("optional", optional(1), optional(2, 3), optional(4, undefined)); +console.log("signed", signed(), signed(-0), signed(0), signed(undefined), signed(-1.5)); + +// The function as a value keeps its defaults. +const viaValue: (value: number, by?: number) => number = scale; +console.log("value", viaValue(4), viaValue(4, 5), viaValue(4, undefined)); +const table = [scale, scale]; +let sum = 0; +for (const f of table) sum += f(3); +console.log("table", sum); diff --git a/tests/harness/corpus-costs/optional-parameter-defaults.ts.json b/tests/harness/corpus-costs/optional-parameter-defaults.ts.json new file mode 100644 index 000000000..88bafb998 --- /dev/null +++ b/tests/harness/corpus-costs/optional-parameter-defaults.ts.json @@ -0,0 +1 @@ +{ "plain": 2700, "san": 3600 } From cdd121405ba5bd60d7f8865a2edebd4d2a70d71e Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Tue, 27 Oct 2026 09:11:51 -0700 Subject: [PATCH 40/54] Convert number | undefined timer delays with ToNumber setTimeout and setInterval passed their delay to the F64 timer slot unconverted, so a delay typed number | undefined failed with SC2003. Such a delay is an optional `ms?: number` passed on, or a number[] element read, whose storage keeps the out-of-range undefined. Node converts the delay with ToNumber, so undefined is NaN, the omitted delay. A union delay now goes through lowerOptionalNumber, as Math arguments already do. Other unions still reach the lib boundary's refusal, and number delays are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../src/frontend/lowering/lower-timers.ts | 8 ++ .../test/timer-delay-lowering.test.ts | 79 ++++++++++++ .../corpus/timer-optional-delays.ts.json | 6 + tests/corpus/timer-optional-delays.ts | 120 ++++++++++++++++++ .../timer-optional-delays.ts.json | 1 + 5 files changed, 214 insertions(+) create mode 100644 packages/compiler/test/timer-delay-lowering.test.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/timer-optional-delays.ts.json create mode 100644 tests/corpus/timer-optional-delays.ts create mode 100644 tests/harness/corpus-costs/timer-optional-delays.ts.json diff --git a/packages/compiler/src/frontend/lowering/lower-timers.ts b/packages/compiler/src/frontend/lowering/lower-timers.ts index 03c9c5c1f..c76b2e667 100644 --- a/packages/compiler/src/frontend/lowering/lower-timers.ts +++ b/packages/compiler/src/frontend/lowering/lower-timers.ts @@ -15,6 +15,7 @@ import { typeKey, } from "../../ir/ir.js"; import { buildFunctionAdapter } from "./coercions/builders.js"; +import { lowerOptionalNumber } from "./lower-exprs.js"; import type { Lowerer } from "./lowerer.js"; /** setTimeout invokes its callback with NO arguments, but @types/node's @@ -409,6 +410,13 @@ function lowerTimerCall( type: F64, loc, }; + } else if (ms.type.kind === "union") { + // A delay that may be undefined: an optional `ms?: number` passed on, + // or an element read (`delays[0]`) whose storage keeps the + // out-of-range undefined. Node converts the delay with ToNumber + // (`after *= 1`), so undefined is NaN, the omitted delay. Other + // unions stay as they are, and the lib boundary refuses them. + ms = lowerOptionalNumber(lowerer, ms, loc, expr.arguments.nodes[1]); } if (member === "setInterval") { return { kind: "libCall", fn: "timers.setInterval", args: [cb, ms], type: F64, loc }; diff --git a/packages/compiler/test/timer-delay-lowering.test.ts b/packages/compiler/test/timer-delay-lowering.test.ts new file mode 100644 index 000000000..e4219c3c2 --- /dev/null +++ b/packages/compiler/test/timer-delay-lowering.test.ts @@ -0,0 +1,79 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile, deserializeModule } from "../src/index.js"; +import type { IrExpr } from "../src/ir/ir.js"; + +/** The delay argument of every setTimeout/setInterval libCall in the module. */ +async function timerDelays(source: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "scriptc-timer-delays-")); + try { + const entry = join(dir, "main.ts"); + await writeFile(entry, source); + const outPath = join(dir, "main.ir.json"); + const result = await compile(entry, { outDir: dir, outPath, outputKind: "ir" }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const mod = deserializeModule(await readFile(outPath, "utf8")); + const delays: IrExpr[] = []; + const visit = (value: unknown): void => { + if (Array.isArray(value)) { + for (const item of value) visit(item); + return; + } + if (value === null || typeof value !== "object") return; + const node = value as { kind?: unknown; fn?: unknown; args?: IrExpr[] }; + if ( + node.kind === "libCall" && + typeof node.fn === "string" && + /^timers\.set(Timeout|TimeoutHandle|Interval)$/.test(node.fn) + ) + delays.push(node.args![1]!); + for (const child of Object.values(value)) visit(child); + }; + visit(mod.functions); + return delays; + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +test("number delays pass to the timer unchanged", async () => { + const delays = await timerDelays( + `function arm(ms: number): void { + setTimeout(() => console.log("a"), ms); + setTimeout(() => console.log("b"), 5); + const id = setInterval(() => clearInterval(id), ms); + } + arm(1); + export {}; + `, + ); + expect(delays.map((delay) => delay.kind)).toEqual(["varRef", "numLit", "varRef"]); + for (const delay of delays) expect(delay.type).toEqual({ kind: "f64" }); +}); + +test("a number | undefined delay converts with ToNumber", async () => { + const delays = await timerDelays( + `interface Options { ms?: number } + function arm(options: Options): void { + setTimeout(() => console.log("a"), options.ms); + const id = setInterval(() => clearInterval(id), options.ms); + } + arm({}); + export {}; + `, + ); + expect(delays).toHaveLength(2); + for (const delay of delays) { + // undefined is NaN, the omitted delay; a number passes through. + expect(delay).toMatchObject({ + kind: "ternary", + type: { kind: "f64" }, + cond: { kind: "unionIsTag" }, + then: { kind: "bin", op: "/" }, + else_: { kind: "unionNarrow", type: { kind: "f64" } }, + }); + } +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/timer-optional-delays.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/timer-optional-delays.ts.json new file mode 100644 index 000000000..f95051960 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/timer-optional-delays.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/timer-optional-delays.ts" + ], + "diags": [] +} diff --git a/tests/corpus/timer-optional-delays.ts b/tests/corpus/timer-optional-delays.ts new file mode 100644 index 000000000..788d6a193 --- /dev/null +++ b/tests/corpus/timer-optional-delays.ts @@ -0,0 +1,120 @@ +// setTimeout and setInterval delays typed `number | undefined`: an optional +// `ms?: number` passed on, a number[] element read (its storage keeps the +// out-of-range undefined), and a read past the end. Node converts the delay +// with ToNumber, so undefined is NaN and the timer waits the default 1ms, +// like an omitted delay. Timers sharing the 1ms bucket fire in registration +// order; the other timers report through a counter so wall-clock timing +// never decides the output. + +interface RefreshOptions { + label: string; + timeoutMs?: number; +} + +const DELAYS = [1, 5]; +const firstBucket: string[] = []; +const events: string[] = []; +let pending = 0; + +function track(): () => void { + pending++; + return () => { + pending--; + if (pending === 0) { + console.log("1ms bucket:", firstBucket.join(", ")); + console.log("events:", events.sort().join(", ")); + } + }; +} + +// An optional field passed on, absent and present; the handle form clears. +function schedule(options: RefreshOptions): ReturnType { + const done = track(); + return setTimeout(() => { + firstBucket.push(options.label); + done(); + }, options.timeoutMs); +} +schedule({ label: "absent" }); +schedule({ label: "one", timeoutMs: 1 }); + +// Element reads in statement position: in range, and past the end. +const doneFirst = track(); +setTimeout(() => { + firstBucket.push("DELAYS[0]"); + doneFirst(); +}, DELAYS[0]); +const doneMissing = track(); +setTimeout(() => { + firstBucket.push("DELAYS[9]"); + doneMissing(); +}, DELAYS[9]); + +const cancelled = schedule({ label: "cancelled", timeoutMs: DELAYS[1] }); +clearTimeout(cancelled); +pending--; + +// A binding initialized from an element read. +const retry = DELAYS[1]; +const doneRetry = track(); +setTimeout(() => { + events.push(`retry after ${retry}`); + doneRetry(); +}, retry); + +// A guard that narrows the controller, not the delay. +function withTimeout(ms?: number): void { + const controller = ms !== undefined ? new AbortController() : undefined; + const done = controller ? track() : undefined; + const timer = controller + ? setTimeout(() => { + controller.abort(); + events.push(`aborted ${controller.signal.aborted}`); + done?.(); + }, ms) + : undefined; + if (timer === undefined) events.push("no timer"); +} +withTimeout(); +withTimeout(3); + +// An interval with an optional delay, cleared through its handle. +function tickTwice(label: string, ms?: number): void { + const done = track(); + let ticks = 0; + const id = setInterval(() => { + ticks++; + if (ticks === 2) { + clearInterval(id); + events.push(`${label} ticked ${ticks}`); + done(); + } + }, ms); +} +tickTwice("default interval"); +tickTwice("element interval", DELAYS[0]); + +// The sleep idiom with an element delay. +async function sleepThenLog(): Promise { + const done = track(); + await new Promise((resolve) => setTimeout(resolve, DELAYS[0])); + events.push("slept"); + done(); +} +void sleepThenLog(); + +// A narrowed delay keeps the plain number path. +function narrowed(ms: number | undefined): void { + if (ms === undefined) return; + const done = track(); + setTimeout(() => { + events.push(`narrowed ${ms + 1}`); + done(); + }, ms); +} +narrowed(2); +narrowed(undefined); + +console.log("scheduled", pending); + +export {}; diff --git a/tests/harness/corpus-costs/timer-optional-delays.ts.json b/tests/harness/corpus-costs/timer-optional-delays.ts.json new file mode 100644 index 000000000..fa2c86926 --- /dev/null +++ b/tests/harness/corpus-costs/timer-optional-delays.ts.json @@ -0,0 +1 @@ +{ "plain": 4000, "san": 73500 } From bb0513c03cea6630b828b999120d7b3040a076dd Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 11:39:28 -0500 Subject: [PATCH 41/54] Collect the native compiler's profile in two phases `scripts/refresh-cli-profile.mjs --context-sensitive` runs the training twice. The second run uses a compiler that the first profile optimizes and that is instrumented again after inlining, and it merges those counts into the first profile. Builds with the result also order each function's blocks by the counts and split cold blocks off (context-sensitive profiles). `--base-profile` runs only the second phase. `scripts/build-native-cli.mts` builds such a compiler when `SCRIPTC_NATIVE_PROFILE_GENERATE` and `SCRIPTC_NATIVE_PROFILE_USE` are both set. The release workflow's cli-profile job collects the profile this way, with a 120-minute timeout. --- .github/workflows/release.yml | 5 +-- scripts/build-native-cli.mts | 12 ++++--- scripts/refresh-cli-profile.mjs | 63 ++++++++++++++++++++++++++++++--- 3 files changed, 68 insertions(+), 12 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a402f912f..1b0efe2da 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -111,7 +111,8 @@ jobs: needs: check-release if: needs.check-release.outputs.should_release == 'true' && needs.check-release.outputs.reuse_artifacts != 'true' runs-on: ubuntu-24.04 - timeout-minutes: 75 + # The layout and the profile's two phases each build an instrumented compiler. + timeout-minutes: 120 continue-on-error: true permissions: contents: read @@ -148,7 +149,7 @@ jobs: env: SCRIPTC_NATIVE_LAYOUT: ${{ runner.temp }}/cli-profile/cli.layout.gz run: >- - node scripts/refresh-cli-profile.mjs --tsc-ts "$RUNNER_TEMP/tsc-ts" --jobs 4 + node scripts/refresh-cli-profile.mjs --tsc-ts "$RUNNER_TEMP/tsc-ts" --jobs 4 --context-sensitive --output "$RUNNER_TEMP/cli-profile/cli.profdata.gz" --work "$RUNNER_TEMP/cli-profile-work" - uses: actions/upload-artifact@v4 with: diff --git a/scripts/build-native-cli.mts b/scripts/build-native-cli.mts index 22b9967ba..de0ff2e2a 100644 --- a/scripts/build-native-cli.mts +++ b/scripts/build-native-cli.mts @@ -24,10 +24,12 @@ const platformPackages = process.argv.includes("--platform-packages"); const optimization = process.env["SCRIPTC_NATIVE_OPTIMIZATION"] === "dev" ? "dev" : "release"; const sanitize = process.env["SCRIPTC_SAN"] === "1"; // SCRIPTC_NATIVE_PROFILE_GENERATE= builds a compiler that writes a -// raw profile per run (scripts/refresh-cli-profile.mjs). Optimized builds -// otherwise use the committed profile, or the indexed profile that -// SCRIPTC_NATIVE_PROFILE_USE names (gzip-compressed when the name ends in -// .gz, as refresh-cli-profile.mjs writes it); an empty +// raw profile per run (scripts/refresh-cli-profile.mjs). With it, a +// SCRIPTC_NATIVE_PROFILE_USE profile optimizes the build too and the +// instrumentation runs after inlining: the second, context-sensitive phase +// of the profile. Optimized builds otherwise use the committed profile, or +// the indexed profile that SCRIPTC_NATIVE_PROFILE_USE names (gzip-compressed +// when the name ends in .gz, as refresh-cli-profile.mjs writes it); an empty // SCRIPTC_NATIVE_PROFILE_USE builds without one. const profileGenerate = process.env["SCRIPTC_NATIVE_PROFILE_GENERATE"] || undefined; // Below this share of matching functions, the build warns that the profile @@ -39,7 +41,7 @@ const PROFILE_MATCH_WARNING = 0.9; const committedProfile = join(root, "packages/compiler/native/cli.profdata.gz"); const requestedProfile = process.env["SCRIPTC_NATIVE_PROFILE_USE"]; const useProfile = - profileGenerate === undefined && + (profileGenerate === undefined || (requestedProfile !== undefined && requestedProfile !== "")) && optimization !== "dev" && !sanitize && requestedProfile !== "" && diff --git a/scripts/refresh-cli-profile.mjs b/scripts/refresh-cli-profile.mjs index a9436b505..e8338b902 100644 --- a/scripts/refresh-cli-profile.mjs +++ b/scripts/refresh-cli-profile.mjs @@ -19,7 +19,14 @@ * use the same layout. When refreshing both, refresh the layout first * (scripts/refresh-cli-layout.mjs). * - * node scripts/refresh-cli-profile.mjs --tsc-ts [--jobs 8] + * With --context-sensitive, a second phase follows: a compiler optimized by + * the first profile and instrumented after inlining runs the same training, + * and its raw profiles are merged into the first profile. The result adds + * the counts of inlined code, by which builds also order each function's + * blocks and move cold blocks apart. --base-profile runs only that second + * phase on a given first profile. + * + * node scripts/refresh-cli-profile.mjs --tsc-ts [--jobs 8] [--context-sensitive] */ import { spawn, spawnSync } from "node:child_process"; import { @@ -37,7 +44,7 @@ import { cpus, tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; -import { constants, gzipSync } from "node:zlib"; +import { constants, gunzipSync, gzipSync } from "node:zlib"; const root = fileURLToPath(new URL("..", import.meta.url)); const { values } = parseArgs({ @@ -47,12 +54,14 @@ const { values } = parseArgs({ output: { type: "string", default: join(root, "packages/compiler/native/cli.profdata.gz") }, work: { type: "string" }, instrumented: { type: "string" }, + "context-sensitive": { type: "boolean", default: false }, + "base-profile": { type: "string" }, help: { type: "boolean", default: false }, }, }); if (values.help || values["tsc-ts"] === undefined) { console.log( - "Usage: node scripts/refresh-cli-profile.mjs --tsc-ts [--jobs N] [--output ] [--work ] [--instrumented ]", + "Usage: node scripts/refresh-cli-profile.mjs --tsc-ts [--jobs N] [--output ] [--work ] [--instrumented ] [--context-sensitive | --base-profile ]", ); process.exit(values.help ? 0 : 2); } @@ -63,6 +72,38 @@ const tscTsPackage = join(tscTs, "packages/tsc-ts"); if (!existsSync(join(tscTsPackage, "src/main.ts"))) throw new Error(`${tscTs} is not a tsc-ts checkout with installed dependencies`); const jobs = Number(values.jobs); +if (values["context-sensitive"]) { + if (values["base-profile"] !== undefined || values.instrumented !== undefined) + throw new Error( + "--context-sensitive runs both phases: it takes no --base-profile or --instrumented", + ); + // The first phase, then the second on its profile, each by a run of this script. + const phases = resolve(values.work ?? mkdtempSync(join(tmpdir(), "scriptc-cli-profile-phases-"))); + const first = join(phases, "first.profdata.gz"); + const common = ["--tsc-ts", tscTs, "--jobs", String(jobs)]; + for (const args of [ + [...common, "--output", first, "--work", join(phases, "first")], + [ + ...common, + "--output", + resolve(values.output), + "--work", + join(phases, "second"), + "--base-profile", + first, + ], + ]) { + const phase = spawnSync(process.execPath, [fileURLToPath(import.meta.url), ...args], { + stdio: "inherit", + }); + if (phase.status !== 0) throw new Error("a profile phase failed"); + } + if (values.work === undefined) rmSync(phases, { recursive: true, force: true }); + process.exit(0); +} +// The first profile, for a run of the second phase. +const baseProfile = + values["base-profile"] === undefined ? undefined : resolve(values["base-profile"]); // The output's directory is created up front, before the instrumented build. const profileOutput = resolve(values.output); mkdirSync(dirname(profileOutput), { recursive: true }); @@ -91,7 +132,11 @@ if (values.instrumented === undefined) { { cwd: root, stdio: "inherit", - env: { ...process.env, SCRIPTC_NATIVE_PROFILE_GENERATE: profileFile }, + env: { + ...process.env, + SCRIPTC_NATIVE_PROFILE_GENERATE: profileFile, + ...(baseProfile === undefined ? {} : { SCRIPTC_NATIVE_PROFILE_USE: baseProfile }), + }, }, ); if (built.status !== 0) throw new Error("the instrumented compiler did not build"); @@ -218,7 +263,15 @@ const indexed = join(work, "cli.profdata"); // The profile keeps the functions that never ran in training, with zero // counts. Builds optimize those as cold and place them after the trained code, // which keeps the compiler's .text smaller than a sparse profile does. -const merged = spawnSync(scriptc, ["merge-profiles", "-o", indexed, ...profiles], { +// A second phase merges its counts into the first profile, uncompressed. +const inputs = [...profiles]; +if (baseProfile !== undefined) { + const base = join(work, "base.profdata"); + const bytes = readFileSync(baseProfile); + writeFileSync(base, baseProfile.endsWith(".gz") ? gunzipSync(bytes) : bytes); + inputs.unshift(base); +} +const merged = spawnSync(scriptc, ["merge-profiles", "-o", indexed, ...inputs], { stdio: "inherit", env: { ...trainingEnv, LLVM_PROFILE_FILE: join(work, "merge.profraw") }, }); From 6d0cb98d7b39f610466e76e607e2b895c3d34184 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 11:39:51 -0500 Subject: [PATCH 42/54] Carry interned unions in unit records A unit's record carries the unions it interned, as their definitions. Applying one interns the union under the key its arms give and checks that its name matches. Recursive unions still leave the unit without a record, and the arm-tag index is a memo. --- .../compiler/src/frontend/lowering/lowerer.ts | 3 +- .../frontend/lowering/unit-journal.test.ts | 4 +- packages/compiler/src/frontend/type-mapper.ts | 118 +++++++++++++++--- 3 files changed, 107 insertions(+), 18 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index d6e72582d..2327446f2 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -4197,7 +4197,8 @@ export class Lowerer { checkedTable("declOrigins", () => this.declOrigins.length), checkedTable("declOriginIds", () => this.declOriginIds.size), sizes("shapes", () => this.shapes.journalSizes()), - sizes("unions", () => this.unions.journalSizes()), + this.unions.journalTable(), + sizes("unions.recIds", () => this.unions.journalSizes()), sizes("keyOrder", () => this.keyOrder.journalSizes()), sizes("deferredModuleInitializers", () => this.deferredModuleInitializers === null diff --git a/packages/compiler/src/frontend/lowering/unit-journal.test.ts b/packages/compiler/src/frontend/lowering/unit-journal.test.ts index 439e18fd7..414184ae1 100644 --- a/packages/compiler/src/frontend/lowering/unit-journal.test.ts +++ b/packages/compiler/src/frontend/lowering/unit-journal.test.ts @@ -24,8 +24,8 @@ const lowerer = (): Lowerer => const isCollection = (value: unknown): boolean => value instanceof Map || value instanceof Set || Array.isArray(value); -/** The registries whose collections journalSizes covers, with the - * collections each one's journalSizes reads. */ +/** The registries whose collections their journalSizes and journalTable + * cover, or that are memos of them, with those collections. */ const sized: Record = { shapes: [ "keyedNameKeys", diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index 1d505f08f..2dee0d62a 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -1,5 +1,6 @@ import { InternalCompilerError } from "../errors.js"; import { keyedHelperName, type KeyedNameOwner } from "./lowering/stable-names.js"; +import type { HeldContents, JournalTable } from "./lowering/unit-journal.js"; import { mockCallRecordType, mockContextType } from "./mock-shapes.js"; import * as ts from "./ts7/adapter.js"; import { bodyReadsArguments } from "./arguments-usage.js"; @@ -700,21 +701,103 @@ export class UnionRegistry { /** All interned unions in first-seen order. */ readonly unions: IrUnionDef[] = []; - /** The sizes of the registry's tables (ShapeRegistry.journalSizes). */ + /** The sizes of the tables recursive unions change, which no unit + * record carries (unit-journal.ts). */ journalSizes(): number[] { - let tags = 0; - for (const byArm of this.armTags.values()) tags += byArm.size; - return [ - this.revision, - this.unions.length, - this.byKey.size, - this.byId.size, - this.keyedNameKeys.size, - this.armTags.size, - tags, - this.recIds.size, - this.pendingRec.size, - ]; + return [this.revision, this.recIds.size, this.pendingRec.size]; + } + + /** The interned unions as a table unit records carry (unit-journal.ts). + * An entry is a union's definition; applying it interns the union under + * the key its arms give. A unit that adds a recursive union changes + * journalSizes, so the unions past a unit's start are interned ones. */ + journalTable(): JournalTable { + return { + name: "unions", + size: () => this.unions.length, + entries: (from) => { + const out: string[] = []; + for (const def of this.unions.slice(from)) { + // JSON writes a number that is not finite as null. + const text = JSON.stringify(def); + if (/[:,[]null\b/.test(text)) return null; + out.push(text); + } + return out; + }, + apply: (entries) => { + for (const entry of entries) { + const def = JSON.parse(entry) as IrUnionDef; + const key = unionKey(def.arms, def.discriminant); + if (this.byKey.has(key)) continue; + if (keyedHelperName(this, "u", key, "") !== def.id) + throw new InternalCompilerError(`union record ${def.id} does not match its arms`); + this.byKey.set(key, def.id); + this.byId.set(def.id, def); + this.unions.push(def); + } + }, + truncate: (from) => { + while (this.unions.length > from) { + const def = this.unions.pop()!; + this.byKey.delete(unionKey(def.arms, def.discriminant)); + this.byId.delete(def.id); + this.keyedNameKeys.delete(def.id); + } + }, + hold: () => { + const held = [...this.unions]; + return { same: () => held.every((def, index) => this.unions[index] === def) }; + }, + contents: () => { + // A recursive placeholder still pending fills in place when its + // outer frame completes, which changes journalSizes. + const texts: (string | null)[] = []; + const indexes = new Map(); + const differs = (index: number): boolean => { + const text = texts[index] ?? null; + return ( + text !== null && + index < this.unions.length && + JSON.stringify(this.unions[index]!) !== text + ); + }; + const held: HeldContents = { + extend: () => { + for (let index = texts.length; index < this.unions.length; index++) { + const def = this.unions[index]!; + texts.push(this.pendingRec.has(def.id) ? null : JSON.stringify(def)); + indexes.set(def.id, index); + } + }, + changed: () => { + for (let index = 0; index < texts.length; index++) + if (differs(index)) return this.unions[index]!.id; + return null; + }, + changedAmong: (keys) => { + for (const key of keys) { + const index = indexes.get(key); + if (index !== undefined && differs(index)) return key; + } + return null; + }, + refresh: (keys) => { + for (const key of keys) { + const index = indexes.get(key); + if ( + index !== undefined && + index < this.unions.length && + (texts[index] ?? null) !== null + ) + texts[index] = JSON.stringify(this.unions[index]!); + } + }, + }; + held.extend(); + return held; + }, + }; } /** ts.Types currently being mapped — a back-reference to one is the * recursive knot passing through a union (`type Tree = Leaf | Branch` @@ -788,7 +871,7 @@ export class UnionRegistry { /** Interns canonical arms and discriminator cases, returning their unionId. */ intern(arms: IrType[], discriminant?: IrUnionDiscriminant): string { - const key = JSON.stringify([arms.map(typeKey), discriminant]); + const key = unionKey(arms, discriminant); let id = this.byKey.get(key); if (id === undefined) { id = keyedHelperName(this, "u", key, ""); @@ -830,6 +913,11 @@ export class UnionRegistry { } } +/** The key UnionRegistry interns a union's arms and discriminant under. */ +function unionKey(arms: readonly IrType[], discriminant: IrUnionDiscriminant | undefined): string { + return JSON.stringify([arms.map(typeKey), discriminant]); +} + /** Human-readable rendering of an IrType for diagnostics (records expand to * their canonical field list, unions to their arms; `checker.typeToString` * can't — it never sees IR types). Bound the traversal as well as its output: From 5431a8c142b9c558b60a0b7f7b012b1ebd75b598 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 11:56:10 -0500 Subject: [PATCH 43/54] Keep class instances that stay in their frame in frame storage A class instance could live in its function's frame only when its class had scalar fields only and CallLifetimes proved the binding was only read. Per-call state objects whose fields own arrays, maps and other objects, and whose methods and helpers store into them, stayed on the heap. frame-escapes.ts proves which locals never leave their frame: every use keeps the value in the frame (region-ir.ts keepsInFrame), a const binding of it in the same function shares its fate, and a closure that captures it stays in the frame and keeps the captured binding there. Parameter and capture facts are solved optimistically over the call graph with a worklist; async functions, generators and module code get none. A frame class is a standalone class whose constructor does not let this escape. Its instance now lives in frame storage when it is bound to such a local, when it is passed in place to a parameter that stays in the callee's frame, or when closures that stay in the frame capture its binding. The frame owns the reference fields: they start empty, union and dyn fields start at undefined, and the binding's scope (or the call's statement frame) releases them on every exit. A cycle-capable class keeps its collector header in front of the object, never a candidate. --- packages/compiler/src/backend/llvm/classes.ts | 28 +- .../src/backend/llvm/emit-statements.ts | 16 +- packages/compiler/src/backend/llvm/emitter.ts | 5 + .../compiler/src/backend/llvm/expr-calls.ts | 20 +- .../src/backend/llvm/frame-escapes.ts | 220 +++++++++++++ .../src/backend/llvm/stack-objects.test.ts | 123 +++++++ .../src/backend/llvm/stack-objects.ts | 127 ++++++-- .../tests/corpus/stack-class-fields.ts.json | 6 + tests/corpus/stack-class-fields.ts | 302 ++++++++++++++++++ 9 files changed, 815 insertions(+), 32 deletions(-) create mode 100644 packages/compiler/src/backend/llvm/frame-escapes.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stack-class-fields.ts.json create mode 100644 tests/corpus/stack-class-fields.ts diff --git a/packages/compiler/src/backend/llvm/classes.ts b/packages/compiler/src/backend/llvm/classes.ts index dd639393f..e4e8bd459 100644 --- a/packages/compiler/src/backend/llvm/classes.ts +++ b/packages/compiler/src/backend/llvm/classes.ts @@ -863,6 +863,20 @@ function undefFieldInits( host: ClassHost, meta: LlClassMeta, nullable: NullableRefFields | undefined, +): string[] { + return classUndefinedFieldInits(host, meta, nullable, "%o", "%uf"); +} + +/** The stores that start an instance's union and dyn fields at undefined, + * for the instance at `object`, with temporaries named `` + * and `v`. A nullable-pointer field's zeroed slot already is + * its unit arm. */ +export function classUndefinedFieldInits( + host: ClassHost, + meta: LlClassMeta, + nullable: NullableRefFields | undefined, + object: string, + prefix: string, ): string[] { const out: string[] = []; meta.def.fields.forEach((f, i) => { @@ -880,17 +894,17 @@ function undefFieldInits( const tag = undefinedArmTag(f.type, host.unionsById); if (tag < 0) return; out.push( - ` %uf${i} = getelementptr inbounds %${mangleClassStruct(meta.def.name)}, ptr %o, i64 0, i32 ${index}, i32 0`, - ` store i32 ${tag}, ptr %uf${i} ; ${llvmCommentText(f.name)} starts undefined`, + ` ${prefix}${i} = getelementptr inbounds %${mangleClassStruct(meta.def.name)}, ptr ${object}, i64 0, i32 ${index}, i32 0`, + ` store i32 ${tag}, ptr ${prefix}${i} ; ${llvmCommentText(f.name)} starts undefined`, ); return; } if (f.type.kind === "dyn") { host.declare("declare ptr @scr_dyn_undefined()"); out.push( - ` %ufv${i} = call ptr @scr_dyn_undefined()`, - ` %uf${i} = getelementptr inbounds %${mangleClassStruct(meta.def.name)}, ptr %o, i64 0, i32 ${index}`, - ` store ptr %ufv${i}, ptr %uf${i} ; ${llvmCommentText(f.name)} starts undefined`, + ` ${prefix}v${i} = call ptr @scr_dyn_undefined()`, + ` ${prefix}${i} = getelementptr inbounds %${mangleClassStruct(meta.def.name)}, ptr ${object}, i64 0, i32 ${index}`, + ` store ptr ${prefix}v${i}, ptr ${prefix}${i} ; ${llvmCommentText(f.name)} starts undefined`, ); return; } @@ -898,8 +912,8 @@ function undefFieldInits( const tag = undefinedArmTag(f.type, host.unionsById); if (tag < 0) return; out.push( - ` %uf${i} = getelementptr inbounds %${mangleClassStruct(meta.def.name)}, ptr %o, i64 0, i32 ${index}`, - ` store ptr ${host.unitInstanceRef(f.type.unionId, tag)}, ptr %uf${i} ; ${llvmCommentText(f.name)} starts undefined`, + ` ${prefix}${i} = getelementptr inbounds %${mangleClassStruct(meta.def.name)}, ptr ${object}, i64 0, i32 ${index}`, + ` store ptr ${host.unitInstanceRef(f.type.unionId, tag)}, ptr ${prefix}${i} ; ${llvmCommentText(f.name)} starts undefined`, ); }); return out; diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index 08f2267bd..de43d3c67 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -153,7 +153,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { break; } const stackObject = emitter.localStackObjects.get(s.localId); - if (stackObject) { + if (stackObject && b.kind !== "boxed") { const value = emitStackObject(emitter, stackObject, (fields) => emitter.ownScopeFields(fields), ); @@ -222,7 +222,11 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { B.line(`store ptr ${stack.box}, ptr ${b.slot}`); if (stack.owner) emitter.scopes[emitter.scopes.length - 1]!.push(stack.owner); if (s.init) { - const value = emitter.emitExpr(s.init); + const value = stackObject + ? emitter.markImmortal( + emitStackObject(emitter, stackObject, (fields) => emitter.ownScopeFields(fields)), + ) + : emitter.emitExpr(s.init); if (isRefCounted(value.type)) emitter.toFrame(value); emitter.boxSet(stack.box, b.type, value.name); } @@ -253,7 +257,13 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { if (sentinel !== null) B.line(`store ptr ${sentinel}, ptr ${emitter.boxSlot(box)}`); break; } - const v = emitter.emitExpr(s.init); + // A frame object's binding box holds its frame storage, which the + // closures that capture the box never keep (frame-escapes.ts). + const v = stackObject + ? emitter.markImmortal( + emitStackObject(emitter, stackObject, (fields) => emitter.ownScopeFields(fields)), + ) + : emitter.emitExpr(s.init); if (isRefCounted(v.type)) emitter.toHeap(v); // the box takes ownership if (b.local!.tdz === true && boxAccess(b.type) !== "ref") { emitter.tdzScalarInit(box, b.type, v.name); diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 2b9b45ed0..f3bff1ac6 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -2,6 +2,7 @@ import type { TargetCpu } from "../target-cpu.js"; import { loopEntryInlines } from "./loop-entry-versions.js"; import type { FunctionLayout } from "./function-layout.js"; import { relieveHeapPressure } from "../../heap-pressure.js"; +import { analyzeFrameEscapes, type FrameEscapes } from "./frame-escapes.js"; import { wasmBoundsChecks } from "./wasm-modules.js"; import { jsTruthy } from "./jsval.js"; import { JsIcSites } from "../../ir/js-ic.js"; @@ -1440,6 +1441,9 @@ export class LlEmitter { /** The current function's fused string-key Map get-then-set pairs (map-entry-fusion.ts). */ mapEntryFusion: MapEntryFusion | null = null; readonly callLifetimes: CallLifetimes; + /** The locals whose value never leaves its frame (frame-escapes.ts), for + * class instances with reference fields in frame storage. */ + frameEscapes: FrameEscapes | null = null; readonly constantCallbacks: Map>; currentConstantCallbacks: ReadonlyMap = new Map(); /** The current function's running maximum and minimum updates, whose @@ -2234,6 +2238,7 @@ export class LlEmitter { }, (wrap) => this.borrowableNullableWrap(wrap) === "ref", ); + if (this.stackObjects) this.frameEscapes = analyzeFrameEscapes(this.mod); if (this.debug === null) { const strings = findLentStringParameters(this.fnByName, this.callLifetimes); this.lendingFacts = { diff --git a/packages/compiler/src/backend/llvm/expr-calls.ts b/packages/compiler/src/backend/llvm/expr-calls.ts index 9b2f2e4d8..2d29022ed 100644 --- a/packages/compiler/src/backend/llvm/expr-calls.ts +++ b/packages/compiler/src/backend/llvm/expr-calls.ts @@ -18,7 +18,12 @@ import { liveRefView } from "./expr-stream-bridges.js"; import { typedRefConstructor, vAdapters } from "./shapes.js"; import { INT32_ABI_SUFFIX, SCALAR_UNION_RESULT } from "./int32-abi.js"; import { dispatchesCall, INTEGER_DISPATCH_SUFFIX } from "./integer-clones.js"; -import { canStackObject, emitStackObject } from "./stack-objects.js"; +import { + canStackObject, + emitStackObject, + frameClass, + type StackObjectSource, +} from "./stack-objects.js"; import { collectFfiRetainedOps, parseFfiCallbackKey } from "../ffi-callbacks.js"; import { mangleAsyncAdoptCall, @@ -129,6 +134,19 @@ export function emitCallExpr( if (borrowed) host.frames.push([]); let suffixes: Writes[] | undefined; const args = e.args.map((a, index) => { + // A fresh instance of a frame class that the parameter keeps in the + // callee's frame (frame-escapes.ts) is built in the caller's frame; + // the call's statement frame releases its reference fields. + if ( + a.kind === "new" && + host.frameEscapes?.parameters.get(e.callee)?.has(index) === true && + frameClass(host, a.className) + ) { + const value = emitStackObject(host, a as StackObjectSource, (fields) => { + for (const field of fields) host.ownSlot(field.ptr, field.type); + }); + return host.markImmortal(value); + } if (borrowed?.has(index)) { // A heap-borrowing helper may retain or return its argument. // Only projection-only parameters may see an unowned stack box, diff --git a/packages/compiler/src/backend/llvm/frame-escapes.ts b/packages/compiler/src/backend/llvm/frame-escapes.ts new file mode 100644 index 000000000..8197e94f5 --- /dev/null +++ b/packages/compiler/src/backend/llvm/frame-escapes.ts @@ -0,0 +1,220 @@ +/* Frame escapes (stack-objects.ts): the locals whose value provably dies + * before their function returns, so that an object bound to one can live in + * the frame. + * + * A local stays in the frame when every use of its value does: field and + * element reads and writes through it (storing into it is fine), length and + * search operations, identity comparisons, a statement that discards it, and + * passing it to a parameter that itself stays in its callee's frame + * (region-ir.ts keepsInFrame). A value that a conditional, a union wrap or + * narrow, a cast or a sequence passes on takes that expression's fate. A + * `const` binding of the value in the same function shares the local's fate: + * it is declared inside the local's scope, so it cannot outlive it. + * + * A closure that captures the local keeps it in the frame when the closure + * itself stays in the frame (it is only invoked, or passed to a parameter + * that stays in its callee's frame) and its body keeps the captured binding + * in the frame. The binding's box then never outlives the frame either. + * + * Everything else escapes: returning or throwing it, storing the value + * itself anywhere (a field, an element, a collection, a module global or + * another binding by assignment), capturing it in a closure that escapes or + * in a local class, converting it, and every call whose target is not a + * known function with a proven parameter. + * + * Parameter and capture facts are solved optimistically over the call + * graph, like CallLifetimes: every reference parameter stays in its + * callee's frame, and every captured binding in its closure's, until a use + * proves otherwise. Async functions, generators and module code have none, + * since their frames can outlive a call or are the program's own. */ +import type { IrFunction, IrModule } from "../../ir/ir.js"; +import { + children, + forEachNode, + keepsInFrame, + same, + refType, + transparent, + type Node, +} from "./region-ir.js"; + +export interface FrameEscapes { + /** The reference locals of `fn` whose value never leaves its frame: + * never assigned after its declaration, captured only by closures that + * keep it in the frame, and every use keeping it in the frame. */ + locals(fn: IrFunction): ReadonlySet; + /** Per function, the reference parameters whose argument stays in the + * callee's frame. */ + readonly parameters: ReadonlyMap>; +} + +const NO_FUNCTIONS: ReadonlySet = new Set(); + +function suspends(fn: IrFunction): boolean { + return fn.async === true || fn.generator !== undefined || /^%(init\.\d+|main)$/.test(fn.name); +} + +/** Parameter and capture facts, solved together (analyzeFrameEscapes). */ +interface Facts { + parameters: Map>; + /** Per lifted function, the captured locals whose value stays in the + * frame of each invocation of the closure. */ + captures: Map>; +} + +/** The locals of `fn` that a use lets leave the frame, or that are assigned + * after their declaration. A local bound by a `const` declaration to + * another local's value shares its fate. A closure shares the fate of the + * locals it captures: they stay in the frame only while the closure does + * and its body keeps them there (Facts.captures). Every other captured + * binding escapes. */ +function escapingLocals(fn: IrFunction, facts: Facts): Set { + const escaping = new Set(); + const own = new Set(fn.locals.map((l) => l.id)); + const group = new Map(); + const find = (id: string): string => { + let root = id; + while (group.has(root)) root = group.get(root)!; + return root; + }; + const unite = (a: string, b: string): void => { + const ra = find(a), + rb = find(b); + if (ra !== rb) group.set(ra, rb); + }; + const declared = new Map(); + // Captured bindings that a closure node of this function captures. + const closureCaptured = new Set(); + let closures = 0; + const parents: Node[] = []; + /** The fate of the value `node` produces: shared with the binding a + * `const` declaration gives it, kept by its consumer, or escaping. */ + const fate = (node: Node, id: string): void => { + let value: Node = node; + let at = parents.length - 1; + while (at >= 0 && transparent(value, parents[at])) value = parents[at--]!; + const parent = at >= 0 ? parents[at] : undefined; + if ( + parent?.kind === "varDecl" && + parent.init !== null && + same(parent.init, value) && + own.has(parent.localId) && + parent.localId !== id + ) + unite(id, parent.localId); + else if (!keepsInFrame(value, parent, facts.parameters)) escaping.add(id); + }; + const walk = (node: Node): void => { + if (node.kind === "varDecl") declared.set(node.localId, (declared.get(node.localId) ?? 0) + 1); + if (node.kind === "assign" || node.kind === "assignExpr" || node.kind === "incDec") + escaping.add(node.localId); + if (node.kind === "classRef") for (const id of node.captures ?? []) escaping.add(id); + if (node.kind === "closure") { + const closure = `%closure.${closures++}`; + fate(node, closure); + const kept = facts.captures.get(node.fnName); + for (const id of node.captures ?? []) { + closureCaptured.add(id); + unite(id, closure); + if (kept?.has(id) !== true) escaping.add(id); + } + } + if (node.kind === "varRef") { + fate(node, node.localId); + return; + } + const kids: Node[] = []; + children(node, kids); + parents.push(node); + for (const kid of kids) walk(kid); + parents.pop(); + }; + for (const stmt of fn.body) walk(stmt); + const lifted = new Set((fn.captures ?? []).map((c) => c.localId)); + for (const local of fn.locals) { + if (local.boxed === true && !lifted.has(local.id) && !closureCaptured.has(local.id)) + escaping.add(local.id); + if ((declared.get(local.id) ?? 0) > 1) escaping.add(local.id); + } + const roots = new Set(); + for (const id of escaping) roots.add(find(id)); + for (const local of fn.locals) if (roots.has(find(local.id))) escaping.add(local.id); + return escaping; +} + +export function analyzeFrameEscapes(mod: IrModule): FrameEscapes { + const facts: Facts = { parameters: new Map(), captures: new Map() }; + for (const fn of mod.functions) { + const safe = new Set(); + const kept = new Set(); + if (!suspends(fn)) { + fn.params.forEach((param, i) => { + if (refType(param.type)) safe.add(i); + }); + for (const capture of fn.captures ?? []) kept.add(capture.localId); + } + facts.parameters.set(fn.name, safe); + facts.captures.set(fn.name, kept); + } + // A fact that turns false can make locals escape in the functions that + // call the function or create its closures, so only those are visited + // again. + const dependents = new Map>(); + const depend = (on: string, fn: IrFunction): void => { + let set = dependents.get(on); + if (!set) dependents.set(on, (set = new Set())); + set.add(fn); + }; + for (const fn of mod.functions) + forEachNode(fn.body, (node) => { + if (node.kind === "call") depend(node.callee, fn); + else if (node.kind === "closure") depend(node.fnName, fn); + }); + const has = (fn: IrFunction): boolean => + facts.parameters.get(fn.name)!.size > 0 || facts.captures.get(fn.name)!.size > 0; + const queued = new Set(mod.functions.filter(has)); + const queue = [...queued]; + while (queue.length > 0) { + const fn = queue.pop()!; + queued.delete(fn); + if (!has(fn)) continue; + const escaping = escapingLocals(fn, facts); + let changed = false; + const safe = facts.parameters.get(fn.name)!; + fn.params.forEach((param, i) => { + if (safe.has(i) && escaping.has(param.localId)) { + safe.delete(i); + changed = true; + } + }); + const kept = facts.captures.get(fn.name)!; + for (const id of [...kept]) + if (escaping.has(id)) { + kept.delete(id); + changed = true; + } + if (changed) + for (const dependent of dependents.get(fn.name) ?? NO_FUNCTIONS) + if (!queued.has(dependent) && has(dependent)) { + queued.add(dependent); + queue.push(dependent); + } + } + const locals = new Map>(); + return { + parameters: facts.parameters, + locals(fn: IrFunction): ReadonlySet { + let kept = locals.get(fn.name); + if (kept === undefined) { + const set = new Set(); + if (!suspends(fn)) { + const escaping = escapingLocals(fn, facts); + for (const local of fn.locals) + if (refType(local.type) && !escaping.has(local.id)) set.add(local.id); + } + locals.set(fn.name, (kept = set)); + } + return kept; + }, + }; +} diff --git a/packages/compiler/src/backend/llvm/stack-objects.test.ts b/packages/compiler/src/backend/llvm/stack-objects.test.ts index c4f2bedc4..43c01f211 100644 --- a/packages/compiler/src/backend/llvm/stack-objects.test.ts +++ b/packages/compiler/src/backend/llvm/stack-objects.test.ts @@ -337,3 +337,126 @@ test("spread updates copy into the binding's slot and release the replaced field /(%t\d+) = load ptr, ptr %s\d+\n\s+(%t\d+) = load %sc_rs_\w+, ptr %s\d+\n\s+store %sc_rs_\w+ \2, ptr %s\d+\n(?:.*\n)*?\s+call void @\w+\(ptr \1\)/, ); }); + +const COLLECTORS = ` +class Tag { readonly name: string; constructor(name: string) { this.name = name; } } +class Collector { + readonly items: string[] = []; + last: string | undefined = undefined; + tag: Tag | null = null; + count = 0; + add(item: string): void { this.items.push(item); this.last = item; this.count++; } + label(name: string): void { this.tag = new Tag(name); } +} +function fill(c: Collector, n: number): void { for (let i = 0; i < n; i++) c.add("w" + i); } +const kept: Collector[] = []; +let outer: Collector | null = null; +export function local(n: number): number { + const c = new Collector(); + fill(c, n); + c.label("x"); + return c.count + c.items.length + (c.tag?.name.length ?? 0); +} +export function stored(n: number): number { + const c = new Collector(); + fill(c, n); + kept.push(c); + return c.count; +} +export function assigned(n: number): number { + const c = new Collector(); + outer = c; + fill(c, n); + return c.count; +} +export function returned(n: number): Collector { + const c = new Collector(); + fill(c, n); + return c; +} +export function inCase(n: number): number { + switch (n) { + case 1: + const c = new Collector(); + fill(c, n); + return c.count; + default: + return 0; + } +} +class Link { next: Link | null = null; readonly id: number; constructor(id: number) { this.id = id; } } +export function chain(n: number): number { + const head = new Link(0); + let tail = new Link(1); + head.next = tail; + for (let i = 2; i < n; i++) { const l = new Link(i); tail.next = l; tail = l; } + tail.next = head.next; + return head.next.id + (head.next.next?.id ?? 0); +} +console.log(local(2), stored(1), assigned(1), returned(1).count, inCase(1), chain(4), kept.length, outer !== null); +`; + +test("class instances that own references live in frame storage while they stay in the frame", async () => { + const ir = emitLlvmModule(await moduleFor(COLLECTORS)); + const local = body(ir, "local"); + expect(local).toMatch(/alloca %sc_o_Collector, align 8/); + expect(local).not.toContain("@sc_new_Collector()"); + expect(local).toContain("store i64 -1, ptr"); + // The frame releases the reference fields when the binding's scope exits. + expect(local).toMatch(/call void @scr_arr_release\(/); + // A cycle-capable class keeps its collector header in front of the object. + const chain = body(ir, "chain"); + expect(chain).toMatch(/alloca \{ \[8 x i8\], %sc_o_Link \}, align 8/); + expect(chain).toContain("store i32 -1, ptr"); + // Stored, assigned to another binding, returned, or declared in a switch + // case: the instance stays on the heap. + for (const name of ["stored", "assigned", "returned", "inCase"]) { + const fn = body(ir, name); + expect(fn, name).not.toMatch(/alloca %sc_o_Collector/); + expect(fn, name).toContain("@sc_new_Collector()"); + } +}); + +test("fresh instances passed in place and instances captured by invoked closures live in frame storage", async () => { + const ir = emitLlvmModule( + await moduleFor(` +class Checker { + readonly names: string[]; + readonly seen = new Set(); + constructor(names: string[]) { this.names = names; } + check(): number { let d = 0; for (const n of this.names) { if (this.seen.has(n)) d++; this.seen.add(n); } return d; } +} +function each(xs: number[], f: (x: number) => void): void { for (const x of xs) f(x); } +class Totals { + readonly seen: number[] = []; + total = 0; + add(x: number): void { this.total += x; this.seen.push(x); } + addAll(xs: number[]): void { each(xs, (x) => this.add(x)); } +} +const handlers: ((x: number) => void)[] = []; +export function inPlace(names: string[]): number { return new Checker(names).check(); } +export function captured(xs: number[]): number { + const t = new Totals(); + t.addAll(xs); + each(xs, (x) => t.add(x * 2)); + return t.total; +} +export function kept(xs: number[]): number { + const t = new Totals(); + handlers.push((x) => t.add(x)); + return t.total; +} +console.log(inPlace(["a", "a"]), captured([1, 2]), kept([3]), handlers.length); +`), + ); + const inPlace = body(ir, "inPlace"); + expect(inPlace).toMatch(/alloca %sc_o_Checker, align 8/); + expect(inPlace).not.toContain("@sc_new_Checker()"); + const captured = body(ir, "captured"); + expect(captured).toMatch(/alloca %sc_o_Totals, align 8/); + expect(captured).not.toContain("@sc_new_Totals()"); + // A closure that outlives the call keeps its captured instance on the heap. + const kept = body(ir, "kept"); + expect(kept).not.toMatch(/alloca %sc_o_Totals/); + expect(kept).toContain("@sc_new_Totals()"); +}); diff --git a/packages/compiler/src/backend/llvm/stack-objects.ts b/packages/compiler/src/backend/llvm/stack-objects.ts index 36793598b..a5fcc9ce2 100644 --- a/packages/compiler/src/backend/llvm/stack-objects.ts +++ b/packages/compiler/src/backend/llvm/stack-objects.ts @@ -1,3 +1,4 @@ +import { constructorEscapePoint } from "../cycle-mutability.js"; import { VOID, isRefCounted, @@ -15,8 +16,8 @@ import { } from "./record-results.js"; import { everyStmtList } from "../../ir/traverse.js"; import { mangleClassStruct, mangleRecordStruct } from "../mangle.js"; -import { undefinedArmTag } from "../../ir/analysis.js"; -import { classFieldIndex } from "./classes.js"; +import { classFieldIndex, classUndefinedFieldInits } from "./classes.js"; +import { CYC_BUF, CYC_HDR } from "./alloc.js"; import type { CallLifetimes } from "./call-lifetimes.js"; import { emitCallExpr, emitConstruction } from "./expr-calls.js"; import type { LlValue, LlvmEmitterContext } from "./expr-context.js"; @@ -83,8 +84,39 @@ export function canStackObject(host: LlvmEmitterContext, e: IrExpr): e is StackO ); } +/** A standalone class whose instance can live in the frame of a binding + * that frame-escapes.ts proves stays in the frame. Unlike + * scalarStandaloneClass, its fields may own references: the frame owns + * them and releases them when the binding's scope exits. Classes in a + * hierarchy keep heap storage, as do the kinds canStackObject refuses. */ +export function frameClass(host: LlvmEmitterContext, className: string): boolean { + const meta = host.classMeta.get(className); + const ctor = host.fnByName.get(`%${className}.constructor`); + if (meta === undefined || ctor === undefined) return false; + const cls = meta.def; + if ( + meta.hierarchy || + meta.compact || + meta.threadSlot === true || + host.programLifetime.has(className) || + host.publishedTypes?.classGuarded(className) === true || + cls.runtime || + cls.base !== undefined || + cls.arrayElem !== undefined || + cls.localCaptures !== undefined || + cls.tracksOwnFields || + (cls.symbolFields?.length ?? 0) > 0 || + ctor.async || + ctor.generator + ) + return false; + const self = ctor.params[0]?.localId; + return self !== undefined && constructorEscapePoint(ctor, self, null) === ctor.body.length; +} + /** Whether a frame object of this type has fields that own references. */ function ownsReferences(host: LlvmEmitterContext, type: IrType): boolean { + if (type.kind === "object") return classReferenceFields(host, type.className).length > 0; return ( type.kind === "record" && host @@ -93,6 +125,24 @@ function ownsReferences(host: LlvmEmitterContext, type: IrType): boolean { ); } +/** The fields of a frame class that own references: refcounted, not inline + * unions, and not compressed references to program-lifetime instances. */ +function classReferenceFields( + host: LlvmEmitterContext, + className: string, +): { name: string; index: number; type: IrType }[] { + const meta = host.classMeta.get(className); + if (meta === undefined) return []; + const out: { name: string; index: number; type: IrType }[] = []; + for (const field of meta.def.fields) { + if (!isRefCounted(field.type) || host.inlineUnions.classField(className, field.name)) continue; + const { index, compressed } = classFieldIndex(meta, field.name, "compressed"); + if (compressed) continue; + out.push({ name: field.name, index, type: field.type }); + } + return out; +} + /** Locals declared directly in a switch case. The cases share one scope * whose exit runs whichever case was entered, so a scope-owned field would * be released without its declaration having run. */ @@ -112,7 +162,10 @@ function caseScopedLocals(fn: IrFunction): Set { /** Locals initialized once by a stack-eligible object and only projected * afterwards (CallLifetimes.locals): field reads, and arguments to - * parameters that are themselves only projected. */ + * parameters that are themselves only projected. Also locals initialized + * once by a `new` of a frame class (frameClass) whose value stays in the + * frame (frame-escapes.ts): their methods and callees may store into the + * instance. */ export function findLocalStackObjects( host: LlvmEmitterContext, fn: IrFunction, @@ -120,19 +173,28 @@ export function findLocalStackObjects( ): Map { const result = new Map(); const safe = lifetimes.locals.get(fn.name); - if (!safe?.size || fn.async || fn.generator) return result; + if (fn.async || fn.generator) return result; + // Frame facts are computed only for functions that construct a frame class. + let frame: ReadonlySet | undefined; + const frameLocal = (localId: string): boolean => { + if (host.frameEscapes === null) return false; + frame ??= host.frameEscapes.locals(fn); + return frame.has(localId); + }; const caseScoped = caseScopedLocals(fn); everyStmtList(fn.body, { expr: () => true, stmt: (stmt) => { if ( stmt.kind === "varDecl" && - safe.has(stmt.localId) && stmt.init && - canStackObject(host, stmt.init) && + ((safe?.has(stmt.localId) === true && canStackObject(host, stmt.init)) || + (stmt.init.kind === "new" && + frameClass(host, stmt.init.className) && + frameLocal(stmt.localId))) && !(caseScoped.has(stmt.localId) && ownsReferences(host, stmt.init.type)) ) - result.set(stmt.localId, stmt.init); + result.set(stmt.localId, stmt.init as StackObjectSource); return true; }, }); @@ -194,17 +256,29 @@ export function stackObjectStruct(type: IrType): string { /** The reference fields of the frame object in `slot`, addressed from the * entry block so that every cleanup path can reach them. Inline union - * fields copy bits and own nothing; class objects in frame storage are - * scalar. */ + * fields copy bits and own nothing, and so do a class's compressed + * references to program-lifetime instances. */ export function stackObjectFields( host: LlvmEmitterContext, type: IrType, slot: string, ): readonly StackObjectField[] { - if (type.kind !== "record") return []; + if (type.kind !== "record" && type.kind !== "object") return []; const cached = host.stackObjectFieldPtrs.get(slot); if (cached) return cached; const struct = stackObjectStruct(type); + if (type.kind === "object") { + const fields: StackObjectField[] = []; + for (const field of classReferenceFields(host, type.className)) { + const ptr = host.B.slot(); + host.B.entryAllocas.push( + `${ptr} = getelementptr inbounds ${struct}, ptr ${slot}, i64 0, i32 ${field.index}`, + ); + fields.push({ name: field.name, ptr, type: field.type }); + } + host.stackObjectFieldPtrs.set(slot, fields); + return fields; + } const base = host.nullableUnions.fieldBase(type.shapeId); const fields: StackObjectField[] = []; host.recordShape(type.shapeId).fields.forEach((field, index) => { @@ -272,7 +346,19 @@ export function emitStackObject( let slot = into; if (slot === undefined) { slot = B.slot(); - B.entryAllocas.push(`${slot} = alloca ${struct}, align 8`); + if (type.kind === "object" && host.shapeHost.tracedShapes.has(`object:${type.className}`)) { + // A cycle-capable class keeps its collector header in front of the + // object, never a candidate (SCR_CYC_ACYCLIC), as arena objects do. + const frame = B.slot(); + B.entryAllocas.push( + `${frame} = alloca { [${CYC_HDR} x i8], ${struct} }, align 8`, + `${slot} = getelementptr inbounds { [${CYC_HDR} x i8], ${struct} }, ptr ${frame}, i64 0, i32 1`, + ); + const buf = B.tmp(); + B.line(`store i16 0, ptr ${frame} ; ScrCycHdr.id`); + B.line(`${buf} = getelementptr inbounds i8, ptr ${frame}, i64 ${CYC_BUF}`); + B.line(`store i32 -1, ptr ${buf} ; ScrCycHdr.buf = SCR_CYC_ACYCLIC`); + } else B.entryAllocas.push(`${slot} = alloca ${struct}, align 8`); } // A literal's interned string fields are immortal and need no owner. A // binding's own slot (`into`) keeps an owner for every field: later @@ -293,16 +379,15 @@ export function emitStackObject( const ctor = host.fnByName.get(`%${type.className}.constructor`)!; if (!constructorInitializesFields(meta.def, ctor)) { B.line(`store ${struct} zeroinitializer, ptr ${slot}`); - // Inline union fields start undefined, as the class's `new` sets them. - for (const field of meta.def.fields) { - const tag = undefinedArmTag(field.type, host.unionsById); - if (tag < 0 || !host.inlineUnions.classField(type.className, field.name)) continue; - const p = B.tmp(); - B.line( - `${p} = getelementptr inbounds ${struct}, ptr ${slot}, i64 0, i32 ${classFieldIndex(meta, field.name).index}, i32 0`, - ); - B.line(`store i32 ${tag}, ptr ${p}`); - } + // Union and dyn fields start undefined, as the class's `new` sets them. + for (const line of classUndefinedFieldInits( + host.shapeHost, + meta, + host.nullableFields, + slot, + `${B.tmp()}.uf`, + )) + B.line(line.trim()); } } B.line(`store ${host.sizeType} -1, ptr ${slot}`); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stack-class-fields.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stack-class-fields.ts.json new file mode 100644 index 000000000..eac351760 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stack-class-fields.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/stack-class-fields.ts" + ], + "diags": [] +} diff --git a/tests/corpus/stack-class-fields.ts b/tests/corpus/stack-class-fields.ts new file mode 100644 index 000000000..a7f0389a4 --- /dev/null +++ b/tests/corpus/stack-class-fields.ts @@ -0,0 +1,302 @@ +// Class instances with reference fields that never leave their function: +// builders and per-call state objects whose methods read and write their +// fields, overwrite them, throw while they live, and are created per loop +// iteration. +class Tag { + readonly name: string; + constructor(name: string) { + this.name = name; + } +} + +class Collector { + readonly items: string[] = []; + readonly seen = new Map(); + last: string | undefined = undefined; + tag: Tag | null = null; + count = 0; + + add(item: string): void { + this.items.push(item); + this.seen.set(item, (this.seen.get(item) ?? 0) + 1); + this.last = item; + this.count++; + } + + label(name: string): void { + this.tag = new Tag(name); + } + + summary(): string { + return `${this.count}:${this.items.join(",")}:${this.last ?? "-"}:${this.tag?.name ?? "-"}`; + } +} + +function collect(words: string[]): string { + const c = new Collector(); + for (const w of words) c.add(w); + c.label("first"); + c.label("second"); + return c.summary() + ` dup=${c.seen.get("a") ?? 0}`; +} +console.log(collect(["a", "b", "a", "c"])); + +function fill(c: Collector, n: number): void { + for (let i = 0; i < n; i++) c.add(`w${i % 3}`); +} + +function readCount(c: Collector): number { + return c.count + c.items.length; +} + +function viaHelpers(n: number): string { + const c = new Collector(); + fill(c, n); + return `${readCount(c)} ${c.summary()}`; +} +console.log(viaHelpers(5)); + +// A field value can leave; the object stays. +function firstItems(n: number): string[] { + const c = new Collector(); + fill(c, n); + return c.items.slice(0, 2); +} +console.log(firstItems(4).join("|")); + +// One instance per iteration. +let total = 0; +for (let round = 0; round < 1000; round++) { + const c = new Collector(); + fill(c, round % 7); + total += c.count + (c.last?.length ?? 0); +} +console.log("total", total); + +// Throwing while the instance lives releases its fields once. +class Checked { + readonly values: string[] = []; + owner: Tag; + constructor(owner: string) { + this.owner = new Tag(owner); + } + push(value: string): void { + if (value === "bad") throw new Error(`bad value after ${this.values.join("+")}`); + this.values.push(value); + } +} + +function checkAll(values: string[]): string { + const c = new Checked("checker"); + for (const v of values) c.push(v); + return `${c.owner.name} ${c.values.length}`; +} +for (const input of [["x", "y"], ["x", "bad", "z"], []]) { + try { + console.log(checkAll(input)); + } catch (e) { + console.log("caught", (e as Error).message); + } +} + +// A constructor that throws after storing some fields. +class Partial { + readonly first: string[]; + readonly second: string[]; + constructor(n: number) { + this.first = [`n${n}`]; + if (n < 0) throw new RangeError(`negative ${n}`); + this.second = [`m${n}`]; + } + both(): string { + return this.first.concat(this.second).join("/"); + } +} + +function build(n: number): string { + const p = new Partial(n); + return p.both(); +} +for (const n of [1, -1, 2]) { + try { + console.log(build(n)); + } catch (e) { + console.log("caught", (e as Error).name, (e as Error).message); + } +} + +// Early exits from the binding's scope. +function search(words: string[], target: string): number { + for (let i = 0; i < 3; i++) { + const c = new Collector(); + for (const w of words) { + c.add(w); + if (w === target) return c.count * 10 + i; + } + if (c.count > 2) break; + } + return -1; +} +console.log(search(["p", "q", "r"], "q"), search(["p", "q", "r"], "z")); + +// A cycle-capable class: its nodes can point back to each other. +class Node { + next: Node | null = null; + readonly id: number; + constructor(id: number) { + this.id = id; + } +} + +class Ring { + readonly nodes: Node[] = []; + head: Node | null = null; + add(id: number): void { + const n = new Node(id); + if (this.head !== null) n.next = this.head; + this.head = n; + this.nodes.push(n); + } + close(): void { + if (this.nodes.length > 0) this.nodes[0]!.next = this.head; + } + walk(steps: number): number { + let sum = 0; + let at = this.head; + for (let i = 0; i < steps && at !== null; i++) { + sum += at.id; + at = at.next; + } + return sum; + } +} + +function ring(n: number): number { + const r = new Ring(); + for (let i = 1; i <= n; i++) r.add(i); + r.close(); + return r.walk(n * 2); +} +console.log(ring(4), ring(0)); + +// A cycle-capable class in frame storage: its own field holds heap +// instances of the same class, and those form a cycle among themselves. +class Link { + next: Link | null = null; + label: string; + constructor(label: string) { + this.label = label; + } +} + +function chain(n: number): string { + const head = new Link("head"); + let tail = new Link("t0"); + head.next = tail; + for (let i = 1; i < n; i++) { + const link = new Link(`t${i}`); + tail.next = link; + tail = link; + } + tail.next = head.next; + const labels: string[] = [head.label]; + let at: Link | null = head.next; + for (let i = 0; i < n + 2 && at !== null; i++) { + labels.push(at.label); + at = at.next; + } + return labels.join(">"); +} +console.log(chain(3)); +for (let i = 0; i < 200; i++) chain(i % 5 + 1); +console.log(chain(1)); + +// A fresh instance used in place as a receiver or argument, never kept. +class Overloads { + readonly names: string[]; + readonly seen = new Set(); + duplicates = 0; + constructor(names: string[]) { + this.names = names; + } + check(): number { + for (const n of this.names) { + if (this.seen.has(n)) this.duplicates++; + this.seen.add(n); + } + return this.duplicates; + } +} + +function countDuplicates(o: Overloads): number { + return o.check() + o.names.length; +} + +let duplicates = 0; +for (let i = 0; i < 100; i++) { + duplicates += new Overloads(["f", "g", "f", `h${i % 3}`, "g"]).check(); + duplicates += countDuplicates(new Overloads(["x", "x"])); +} +console.log("duplicates", duplicates); + +// Closures that capture the instance (or `this`) and are only invoked by +// the callee they are passed to. +function each(xs: number[], f: (x: number) => void): void { + for (const x of xs) f(x); +} + +class Totals { + readonly seen: number[] = []; + total = 0; + add(x: number): void { + this.total += x; + this.seen.push(x); + } + addAll(xs: number[]): void { + each(xs, (x) => this.add(x)); + } +} + +const handlers: ((x: number) => void)[] = []; +function totals(xs: number[], keep: boolean): number { + const t = new Totals(); + t.addAll(xs); + each(xs, (x) => t.add(x * 2)); + if (keep) handlers.push((x) => t.add(x)); + return t.total + t.seen.length; +} +console.log(totals([1, 2, 3], false), totals([4], true)); +for (const h of handlers) h(5); +console.log("handlers", handlers.length); + +function doubled(xs: number[]): number { + const t = new Totals(); + t.addAll(xs); + each(xs, (x) => t.add(x * 2)); + return t.total + t.seen.length; +} +let doubledSum = 0; +for (let i = 0; i < 50; i++) doubledSum += doubled([i, i + 1]); +console.log("doubled", doubledSum); + +// try/finally around the instance's use: the finally block runs and the +// fields are released on every path. +function guarded(words: string[], failAt: number): string { + const log: string[] = []; + try { + const c = new Collector(); + try { + for (let i = 0; i < words.length; i++) { + if (i === failAt) throw new Error(`stop at ${words[i]}`); + c.add(words[i]!); + } + return c.summary(); + } finally { + log.push(`finally ${c.count}`); + } + } catch (e) { + return `${(e as Error).message} ${log.join(",")}`; + } +} +console.log(guarded(["a", "b", "c"], -1)); +console.log(guarded(["a", "b", "c"], 1)); From 551e9adcef9bac18158e89800d2f374cb791a6ac Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Tue, 27 Oct 2026 07:21:29 -0700 Subject: [PATCH 44/54] Keep personality routines off split LLVM declarations A shard declares every function another shard defines. The declaration copied the definition's personality routine, which LLVM rejects on a declaration, so on Linux (where functions with landing pads carry one) every shard compile failed and large dev builds silently fell back to one single-threaded translation unit. The sanitized bootstrap seed spent about 5.5 minutes in that one clang process. Co-Authored-By: Claude Opus 5.5 --- .../compiler/src/backend/llvm/split.test.ts | 27 +++++++++++++++++++ packages/compiler/src/backend/llvm/split.ts | 7 ++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/packages/compiler/src/backend/llvm/split.test.ts b/packages/compiler/src/backend/llvm/split.test.ts index 2e0cfa5a5..0b28a78da 100644 --- a/packages/compiler/src/backend/llvm/split.test.ts +++ b/packages/compiler/src/backend/llvm/split.test.ts @@ -324,6 +324,33 @@ test("debug metadata stays on definitions when splitting LLVM modules", async () } }); +test("personality routines stay on definitions when splitting LLVM modules", async () => { + // Functions with landing pads (Linux unwinding) carry a personality; every + // other shard declares them, and a declaration must not repeat it. + const source = + SAMPLE.replaceAll(") #0 {", ") #0 personality ptr @__gcc_personality_v0 {") + + "declare i32 @__gcc_personality_v0(...)\n"; + const split = splitLlvmProgram(source, { minimumBytes: 0 }); + expect(split).not.toBeNull(); + const dir = await mkdtemp(join(tmpdir(), "scriptc-personality-split-")); + scratch.push(dir); + for (const shard of split!.shards) { + expect(shard.source()).not.toMatch(/^declare.* personality ptr/m); + const path = join(dir, shard.name); + await writeFile(path, shard.source()); + const result = spawnSync("clang", ["-Wno-override-module", "-c", path, "-o", `${path}.o`], { + encoding: "utf8", + }); + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).toBe(""); + } + expect( + split!.shards.flatMap((shard) => [ + ...shard.source().matchAll(/^define .* personality ptr @__gcc_personality_v0 \{/gm), + ]).length, + ).toBe(3); +}); + test("dev libraries split at the measured 2MB crossover while executables retain 4MB", () => { const body = Array.from( { length: 700 }, diff --git a/packages/compiler/src/backend/llvm/split.ts b/packages/compiler/src/backend/llvm/split.ts index aecef8df8..619eadfa7 100644 --- a/packages/compiler/src/backend/llvm/split.ts +++ b/packages/compiler/src/backend/llvm/split.ts @@ -104,7 +104,12 @@ function functionHeader( // attributes, zeroext, and varargs stay byte-exact. // A definition's DISubprogram cannot be attached to a declaration in a // different shard. Its owning definition retains the original metadata. - const attrs = match[5]!.replace(/\s+!dbg !\d+/g, ""); + // A personality routine belongs to a definition too: LLVM rejects it on a + // declaration, which would fail every shard that calls a function with + // landing pads (Linux unwinding) and fall back to the single-TU compile. + const attrs = match[5]! + .replace(/\s+!dbg !\d+/g, "") + .replace(/\s+personality\s+ptr\s+@[-$._A-Za-z][-$.\w]*/g, ""); const declaration = `declare ${linkage}${match[2]} ${match[3]}(${match[4]})${attrs}`.trimEnd(); return { line, declaration, symbol, promoted }; } From 3b330b850a352b99e954c25ed49e52676cfa2d4b Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 12:21:27 -0500 Subject: [PATCH 45/54] Release a frame object's nullable class field as the class pointer it holds A nullable class field (Tag | null) stores the class pointer itself, whatever form the union takes elsewhere. When the same union was also a Set element or a Map key, it was a boxed union outside class fields, and the frame's cleanup of a frame class released the field's raw pointer with scr_union_release. classReferenceFields now gives each field its storage type (NullableRefFields.storageType), as the heap teardown does, so the scope exit calls the class's release. --- .../src/backend/llvm/stack-objects.test.ts | 33 +++++++++++++++++++ .../src/backend/llvm/stack-objects.ts | 10 ++++-- tests/corpus/stack-class-fields.ts | 29 ++++++++++++++++ 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/packages/compiler/src/backend/llvm/stack-objects.test.ts b/packages/compiler/src/backend/llvm/stack-objects.test.ts index 43c01f211..b4cd1025e 100644 --- a/packages/compiler/src/backend/llvm/stack-objects.test.ts +++ b/packages/compiler/src/backend/llvm/stack-objects.test.ts @@ -460,3 +460,36 @@ console.log(inPlace(["a", "a"]), captured([1, 2]), kept([3]), handlers.length); expect(kept).not.toMatch(/alloca %sc_o_Totals/); expect(kept).toContain("@sc_new_Totals()"); }); + +test("a nullable class field in frame storage is released as the class pointer it holds", async () => { + const ir = emitLlvmModule( + await moduleFor(` +class Tag { readonly name: string; constructor(name: string) { this.name = name; } } +const seen = new Set(); +const counts = new Map(); +class Holder { + tag: Tag | null = null; + set(name: string): void { this.tag = new Tag(name); } +} +export function label(names: string[]): string { + const h = new Holder(); + for (const n of names) h.set(n); + seen.add(h.tag); + counts.set(h.tag, 1); + return h.tag?.name ?? "-"; +} +console.log(label(["a"]), seen.size, counts.size); +`), + ); + const label = body(ir, "label"); + expect(label).toMatch(/alloca %sc_o_Holder, align 8/); + // The union is boxed where it is a Set element or Map key, but the field + // keeps the raw pointer: its release at scope exit is the class release. + const field = /(%s\d+) = getelementptr inbounds %sc_o_Holder, ptr %s\d+, i64 0, i32 1/.exec( + label, + )![1]!; + const release = new RegExp( + `(%t\\d+) = load ptr, ptr ${field}\\n\\s+call void (@\\w+)\\(ptr \\1\\)`, + ).exec(label); + expect(release?.[2]).toBe("@sc_release_Tag"); +}); diff --git a/packages/compiler/src/backend/llvm/stack-objects.ts b/packages/compiler/src/backend/llvm/stack-objects.ts index a5fcc9ce2..d4c46ac11 100644 --- a/packages/compiler/src/backend/llvm/stack-objects.ts +++ b/packages/compiler/src/backend/llvm/stack-objects.ts @@ -126,7 +126,9 @@ function ownsReferences(host: LlvmEmitterContext, type: IrType): boolean { } /** The fields of a frame class that own references: refcounted, not inline - * unions, and not compressed references to program-lifetime instances. */ + * unions, and not compressed references to program-lifetime instances. + * Each has its storage type: a nullable class field holds the class pointer + * itself, whatever form its union takes elsewhere (nullable-fields.ts). */ function classReferenceFields( host: LlvmEmitterContext, className: string, @@ -138,7 +140,11 @@ function classReferenceFields( if (!isRefCounted(field.type) || host.inlineUnions.classField(className, field.name)) continue; const { index, compressed } = classFieldIndex(meta, field.name, "compressed"); if (compressed) continue; - out.push({ name: field.name, index, type: field.type }); + out.push({ + name: field.name, + index, + type: host.nullableFields.storageType(className, field.name, field.type), + }); } return out; } diff --git a/tests/corpus/stack-class-fields.ts b/tests/corpus/stack-class-fields.ts index a7f0389a4..37dbfb597 100644 --- a/tests/corpus/stack-class-fields.ts +++ b/tests/corpus/stack-class-fields.ts @@ -300,3 +300,32 @@ function guarded(words: string[], failAt: number): string { } console.log(guarded(["a", "b", "c"], -1)); console.log(guarded(["a", "b", "c"], 1)); + +// A nullable class field whose union is a Set element and a Map key +// elsewhere: the field holds the class pointer itself, and the frame +// releases it as one. +class Badge { + readonly name: string; + constructor(name: string) { + this.name = name; + } +} +const seenBadges = new Set(); +const badgeCounts = new Map(); +class BadgeHolder { + badge: Badge | null = null; + count = 0; + set(name: string): void { + this.badge = new Badge(name); + this.count++; + } +} +function badgeLabel(names: string[]): string { + const h = new BadgeHolder(); + for (const n of names) h.set(n); + seenBadges.add(h.badge); + badgeCounts.set(h.badge, (badgeCounts.get(h.badge) ?? 0) + h.count); + return `${h.badge?.name ?? "-"}:${h.count}`; +} +for (let i = 0; i < 100; i++) badgeLabel(["a", `b${i % 3}`]); +console.log(badgeLabel(["x", "y"]), badgeLabel([]), seenBadges.size, badgeCounts.size); From ad3b4f9e12e6eb43b7c8245a0309f9e999e130ad Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 12:22:03 -0500 Subject: [PATCH 46/54] Record a linker job again when a path it names changes A Linux build that links with lld records the driver's ld.lld job with the identities of the files and directories it names, and later links run that job directly while those identities match. When one of them changed, for example when a system update replaced libc or the linker, the link went back to the driver, but the entry stayed: record() kept the existing entry, so every later link of that shape went through the driver until the entry was pruned. An entry whose job names a changed path now counts as missing. A listed link records the job again with the driver's -v -### and runs it; a traced link through the driver records it from its -v output. Entries without a job, for links that cannot run as one ld.lld job, are kept as before. --- .../compiler/src/native/linker-job.test.ts | 38 +++++++++++++++++-- packages/compiler/src/native/linker-job.ts | 21 +++++++--- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/packages/compiler/src/native/linker-job.test.ts b/packages/compiler/src/native/linker-job.test.ts index b9d718c03..96d186baf 100644 --- a/packages/compiler/src/native/linker-job.test.ts +++ b/packages/compiler/src/native/linker-job.test.ts @@ -212,7 +212,7 @@ test.skipIf(process.platform === "win32")( ); test.skipIf(process.platform === "win32")( - "a changed startup object, library directory or linker sends the link to the driver", + "a changed startup object, library directory or linker records the job again", () => { for (const change of ["crt", "lib", "ld"] as const) { const f = fixture(); @@ -230,12 +230,44 @@ test.skipIf(process.platform === "win32")( } const second = f.stage(); const count = f.calls().length; - expect(open().run(second.args, second.paths, second.directory, true)).toBeNull(); - expect(f.calls().length).toBe(count); + expect(open().run(second.args, second.paths, second.directory, true)).not.toBeNull(); + expect(f.calls().slice(count)).toEqual([ + `driver ${second.args.join(" ")} -v -###`, + `ld.lld -pie ${f.crt} -L${f.lib} -pthread ${second.paths[0]} -lm -o ${second.output}`, + ]); } }, ); +test.skipIf(process.platform === "win32")( + "a traced link whose recorded job names a changed path records it again", + () => { + const f = fixture(); + const open = () => LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!; + const traced = (s: ReturnType) => [...s.args, "-v", "-Wl,-t"]; + const linkThroughDriver = (s: ReturnType) => { + const link = open().link(s.paths, s.directory); + expect(link.run(traced(s), false)).toBeNull(); + const linked = spawnSync(f.driver, traced(s), { encoding: "utf8" }); + expect(linked.status).toBe(0); + link.record(traced(s), linked.stdout + "\n" + linked.stderr); + }; + linkThroughDriver(f.stage()); + const before = statSync(f.crt); + writeFileSync(f.crt, "new startup object"); + utimesSync(f.crt, before.atime, new Date(before.mtimeMs + 1000)); + // The recorded job names the old startup object: the link runs through the driver once, + // and its job is recorded again. + linkThroughDriver(f.stage()); + const third = f.stage(); + const count = f.calls().length; + expect(open().link(third.paths, third.directory).run(traced(third), false)).not.toBeNull(); + expect(f.calls().slice(count)).toEqual([ + `ld.lld -pie ${f.crt} -L${f.lib} -pthread ${third.paths[0]} -lm -o ${third.output} -t`, + ]); + }, +); + test.skipIf(process.platform === "win32")("a changed driver records the job again", () => { const f = fixture(); const first = f.stage(); diff --git a/packages/compiler/src/native/linker-job.ts b/packages/compiler/src/native/linker-job.ts index e1195e4c4..f006cb233 100644 --- a/packages/compiler/src/native/linker-job.ts +++ b/packages/compiler/src/native/linker-job.ts @@ -164,10 +164,11 @@ export class LinkerJobs { * Runs the link that the driver arguments `args` describe as its recorded * job. `paths` are the link's own inputs and output among `args`, which may * differ from build to build; `stage` is the directory they are in. Without - * an entry, `list` records the job first with the driver's `-v -###`; + * an entry, or with one whose job names a path that changed since it was + * recorded, `list` records the job first with the driver's `-v -###`; * otherwise the caller links through the driver and passes its `-v` output - * to record(). Null when the link has to run through the driver: no job is - * recorded, a recorded path changed, or the job failed. + * to record(), which records it again. Null when the link has to run + * through the driver: no job is recorded, or the job failed. */ run( args: readonly string[], @@ -180,7 +181,7 @@ export class LinkerJobs { const roots = [resolve(stage), realpathSync(stage)]; const key = this.key(args, paths, roots); if (key === null) return null; - let entry = this.read(key, paths.length); + let entry = this.fresh(this.read(key, paths.length)); if (entry === null && list) { const result = spawnSync(this.driver, [...args, "-v", "-###"], { encoding: "utf8", @@ -190,7 +191,7 @@ export class LinkerJobs { return null; entry = this.store(key, result.stdout + "\n" + result.stderr, paths, roots); } - if (entry === null || entry.job === null || !stillMatches(entry.inputs)) return null; + if (entry === null || entry.job === null) return null; const job = entry.job.map((token) => (typeof token === "number" ? paths[token]! : token)); const result = spawnSync(job[0]!, job.slice(1), { encoding: "utf8", @@ -206,6 +207,14 @@ export class LinkerJobs { } } + /** `entry`, or null when a path its job names changed since it was + * recorded: the next link through the driver records the job again. An + * entry without a job, a link the driver cannot hand to one ld.lld job, + * stays. */ + private fresh(entry: LinkerJobEntry | null): LinkerJobEntry | null { + return entry !== null && entry.job !== null && !stillMatches(entry.inputs) ? null : entry; + } + private read(key: string, paths: number): LinkerJobEntry | null { const bytes = this.cache.read("linker-job", key, MAX_ENTRY_BYTES); if (bytes === null) return null; @@ -231,7 +240,7 @@ export class LinkerJobs { try { const roots = [resolve(stage), realpathSync(stage)]; const key = this.key(args, paths, roots); - if (key !== null && this.read(key, paths.length) === null) + if (key !== null && this.fresh(this.read(key, paths.length)) === null) this.store(key, output, paths, roots, driverInputs); } catch { /* The next link records it. */ From 74b1d43e523e92f14f395f332cc1dd64bd6e4edd Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 12:53:04 -0500 Subject: [PATCH 47/54] Keep record views off sources that may be frozen or hold the view A view writes its source's slots through the target type's stores, which have no frozen-write guard, so a write through a view made before Object.freeze(source) changed the frozen record. A view's record-source entry also owns the source's capsule, an edge the cycle analysis does not see: a source that stored its own view, or a value reaching it, kept both records until process exit. A source now takes part in a view only when it is not freezable and its fields, elements, map entries, union arms and overflow properties cannot hold a value of the target type. A class instance, function, untyped value or other reference that can reach arbitrary objects counts as holding every type, and a value of a view target type counts as holding the records it may view, so two views cannot close a cycle either. Other conversions keep their copy. --- .../src/backend/llvm/record-views.test.ts | 130 ++++++++++++++++++ .../compiler/src/backend/llvm/record-views.ts | 90 +++++++++++- 2 files changed, 214 insertions(+), 6 deletions(-) create mode 100644 packages/compiler/src/backend/llvm/record-views.test.ts diff --git a/packages/compiler/src/backend/llvm/record-views.test.ts b/packages/compiler/src/backend/llvm/record-views.test.ts new file mode 100644 index 000000000..02a82d873 --- /dev/null +++ b/packages/compiler/src/backend/llvm/record-views.test.ts @@ -0,0 +1,130 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile, deserializeModule } from "../../index.js"; +import type { IrModule } from "../../ir/ir.js"; +import { computeTraced } from "../cycle-analysis.js"; +import { planRecordViews } from "./record-views.js"; + +/** The views planned for `source`'s module, as target field names → source + * field names (each list sorted). */ +async function plannedViews(source: string): Promise> { + const dir = await mkdtemp(join(tmpdir(), "scriptc-record-views-")); + try { + const entry = join(dir, "main.ts"), + outPath = join(dir, "main.ir.json"); + await writeFile(entry, source); + const result = await compile(entry, { outDir: dir, outPath, outputKind: "ir" }); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + const mod: IrModule = deserializeModule(await readFile(outPath, "utf8")); + const traced = computeTraced(mod).shapes; + const views = planRecordViews(mod, { + traced: (id) => traced.has(`record:${id}`), + excluded: () => false, + }); + const names = new Map( + (mod.records ?? []).map((shape) => [shape.id, shape.fields.map((f) => f.name).join(",")]), + ); + return new Map( + [...views.targets].map((target) => [ + names.get(target)!, + views.sources(target).map((id) => names.get(id)!), + ]), + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +const conversions = ` +const o: unknown = wide; +const v = o as AC; +v.a = 2; +console.log(wide.a, v.a, o === v); +`; + +test("a checked conversion of a live record capsule into a record type it covers plans a view", async () => { + const views = await plannedViews(` +type Wide = { a: number; b: string; c: boolean }; +type AC = { a: number; c: boolean }; +const wide: Wide = { a: 1, b: "x", c: true }; +${conversions}`); + expect(views).toEqual(new Map([["a,c", ["a,b,c"]]])); +}); + +test("a source whose fields can hold the target type keeps the copy", async () => { + // A view stored in its own source would be a cycle through the view's + // capsule reference, which the cycle analysis does not see. + const views = await plannedViews(` +type AC = { a: number; c: boolean }; +type Holder = { items: AC[] }; +type Wide = { a: number; b: string; c: boolean; self: AC | undefined; holder: Holder }; +const wide: Wide = { a: 1, b: "x", c: true, self: undefined, holder: { items: [] } }; +${conversions} +wide.self = v; +wide.holder.items.push(v);`); + expect(views).toEqual(new Map()); +}); + +test("a source that holds the target type through a class instance keeps the copy", async () => { + const views = await plannedViews(` +type AC = { a: number; c: boolean }; +class Base { note = 0; } +class Keeper extends Base { kept: AC | undefined = undefined; } +type Wide = { a: number; b: string; c: boolean; keeper: Base }; +const wide: Wide = { a: 1, b: "x", c: true, keeper: new Keeper() }; +${conversions}`); + expect(views).toEqual(new Map()); +}); + +test("a source that holds itself still views a type it cannot hold", async () => { + const views = await plannedViews(` +type Tree = { a: number; b: string; c: boolean; children: Tree[] }; +type AC = { a: number; c: boolean }; +const wide: Tree = { a: 1, b: "x", c: true, children: [] }; +wide.children.push({ a: 2, b: "y", c: false, children: [] }); +${conversions}`); + expect(views).toEqual(new Map([["a,c", ["a,b,c,children"]]])); +}); + +test("of two sources that can hold each other's views, one keeps the copy", async () => { + // Each source only holds the other's target type, but a view of that type + // names the other source, so the two views could form a cycle. Without + // Parent's view, Kid holds only a copy and its own view is safe. + const views = await plannedViews(` +type PView = { id: number; name: string }; +type KView = { id: number; tag: string }; +type Parent = { id: number; name: string; kids: KView[] }; +type Kid = { id: number; tag: string; parent: PView | undefined }; +const parent: Parent = { id: 1, name: "p", kids: [] }; +const kid: Kid = { id: 2, tag: "k", parent: undefined }; +const p: unknown = parent; +const k: unknown = kid; +kid.parent = p as PView; +parent.kids.push(k as KView); +console.log(parent.kids.length, kid.parent === p);`); + expect(views).toEqual(new Map([["id,tag", ["id,parent,tag"]]])); +}); + +test("a source with a function field keeps the copy", async () => { + // A function can capture anything, the view included. + const views = await plannedViews(` +type AC = { a: number; c: boolean }; +type Wide = { a: number; b: string; c: boolean; f: () => number }; +const wide: Wide = { a: 1, b: "x", c: true, f: () => 1 }; +${conversions}`); + expect(views).toEqual(new Map()); +}); + +test("a source that may be frozen keeps the copy", async () => { + // Writes through a view take the target type's stores, which have no + // frozen-write guard. + const views = await plannedViews(` +type Wide = { a: number; b: string; c: boolean }; +type AC = { a: number; c: boolean }; +const wide: Wide = { a: 1, b: "x", c: true }; +${conversions} +Object.freeze(wide);`); + expect(views).toEqual(new Map()); +}); diff --git a/packages/compiler/src/backend/llvm/record-views.ts b/packages/compiler/src/backend/llvm/record-views.ts index d11cb3170..5574d1dde 100644 --- a/packages/compiler/src/backend/llvm/record-views.ts +++ b/packages/compiler/src/backend/llvm/record-views.ts @@ -43,6 +43,24 @@ interface ViewOptions { readonly excluded: (shapeId: string) => boolean; } +/** Value kinds that hold no reference to a record. */ +const LEAF_KINDS: ReadonlySet = new Set([ + "f64", + "bool", + "string", + "bigint", + "date", + "regex", + "bytes", + "symbol", + "url", + "searchParams", + "stats", + "undefinedT", + "nullT", + "void", +]); + function plainShape(shape: IrRecordShape): boolean { return ( !shape.tuple && @@ -94,6 +112,47 @@ export function planRecordViews(mod: IrModule, options: ViewOptions): RecordView for (const root of roots) visit(root); return out; }; + // The views planned so far: target shape → the sources it may view. + const sources = new Map(); + // Whether a value of a record can hold a value of another record type: + // through its fields, elements, map entries, union arms and overflow + // properties, and through the record a view of it names. Class instances, + // functions, untyped values, promises and the other kinds that can + // reference arbitrary objects count as holding every type. + const held = new Map | "any">(); + const holds = (shapeId: string): Set | "any" => { + const known = held.get(shapeId); + if (known) return known; + const out = new Set(); + const seen = new Set(); + const visit = (t: IrType): boolean => { + const key = typeKey(t); + if (seen.has(key)) return true; + seen.add(key); + switch (t.kind) { + case "record": { + out.add(t.shapeId); + const shape = shapes.get(t.shapeId); + if (shape?.indexValue !== undefined && !visit(shape.indexValue)) return false; + if (!(sources.get(t.shapeId) ?? []).every((id) => visit({ kind: "record", shapeId: id }))) + return false; + return (shape?.fields ?? []).every((field) => visit(field.type)); + } + case "array": + case "set": + return visit(t.elem); + case "map": + return visit(t.key) && visit(t.value); + case "union": + return (unions.get(t.unionId)?.arms ?? []).every(visit); + default: + return LEAF_KINDS.has(t.kind); + } + }; + const result = visit({ kind: "record", shapeId }) ? out : "any"; + held.set(shapeId, result); + return result; + }; const checked: IrType[] = []; const boxed: IrType[] = []; for (const fn of mod.functions) @@ -109,8 +168,6 @@ export function planRecordViews(mod: IrModule, options: ViewOptions): RecordView const reachable = closure(checked); const live = closure(boxed); if (reachable.size === 0) return NO_RECORD_VIEWS; - const targets = new Set(); - const sources = new Map(); for (const id of reachable) { const target = shapes.get(id); if (!target || !plainShape(target) || target.freezable === true) continue; @@ -118,13 +175,34 @@ export function planRecordViews(mod: IrModule, options: ViewOptions): RecordView const found: string[] = []; for (const source of shapes.values()) { if (source.id === id || !live.has(source.id) || !plainShape(source)) continue; + // A view writes the source's slots through the target type, which has + // no frozen-write guard, so a source that may be frozen keeps the copy. + if (source.freezable === true) continue; if (covers(source, target)) found.push(source.id); } - if (found.length === 0) continue; - targets.add(id); - sources.set(id, found); + if (found.length > 0) sources.set(id, found); } - if (targets.size === 0) return NO_RECORD_VIEWS; + // A view's record-source entry owns the source's capsule, an edge the + // cycle analysis does not see. A view stays off every cycle only when its + // source cannot hold a value of the target type, such as the view itself, + // including through other views. Dropping a pair removes edges, so the + // check repeats until no pair is dropped. + for (let changed = true; changed;) { + changed = false; + held.clear(); + for (const [id, list] of sources) { + const kept = list.filter((source) => { + const sourceHolds = holds(source); + return sourceHolds !== "any" && !sourceHolds.has(id); + }); + if (kept.length === list.length) continue; + changed = true; + if (kept.length > 0) sources.set(id, kept); + else sources.delete(id); + } + } + if (sources.size === 0) return NO_RECORD_VIEWS; + const targets = new Set(sources.keys()); return { targets, sources: (target) => sources.get(target) ?? [] }; } From 9773dccee5cacb56b90efd84895067e692a70f5e Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Tue, 27 Oct 2026 09:35:44 -0700 Subject: [PATCH 48/54] Compile executable program shards as a pool Dev executables compiled their program shards in fixed batches of up to eight, each batch waiting for its slowest shard before the next one started. Keep up to eight clangs busy instead, each taking the next shard when it finishes. Co-Authored-By: Claude Opus 5.5 --- .../compiler/src/backend/native/executable.ts | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/packages/compiler/src/backend/native/executable.ts b/packages/compiler/src/backend/native/executable.ts index 60e61ff0b..18c577bb5 100644 --- a/packages/compiler/src/backend/native/executable.ts +++ b/packages/compiler/src/backend/native/executable.ts @@ -1485,12 +1485,18 @@ export async function compileCInternal( missed: false, }; }); - const shardWidth = Math.min(8, availableParallelism()); - for (let i = 0; i < entries.length; i += shardWidth) { - await Promise.all( - entries.slice(i, i + shardWidth).map(async (entry) => { + // Up to eight clangs (one per available core), each taking the next + // shard as it finishes: shards differ in size, and a fixed batch + // would wait for its slowest member before starting the next. A + // failed shard stops the others from taking more work. + let nextShard = 0; + let failed = false; + await Promise.all( + Array.from({ length: Math.min(entries.length, 8, availableParallelism()) }, async () => { + while (!failed && nextShard < entries.length) { + const entry = entries[nextShard++]!; await writeFile(entry.sourcePath, entry.source()); - if (await copyValidCachedFile(entry.cachePath, entry.staged)) return; + if (await copyValidCachedFile(entry.cachePath, entry.staged)) continue; entry.missed = true; await runClang([ ...programCompilerArgs, @@ -1499,10 +1505,13 @@ export async function compileCInternal( entry.sourcePath, "-o", entry.staged, - ]); - }), - ); - } + ]).catch((error: unknown) => { + failed = true; + throw error; + }); + } + }), + ); const arArgv = isZigDriver(driver) ? [driver.argv[0]!, "ar"] : ["ar"]; const merged = await localizeLibraryObjects( driver, From 9bdb36ed46e20e2bf68bae9d2e6cc1a4293637a1 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 12:57:41 -0500 Subject: [PATCH 49/54] Report published values as frozen, sealed and not extensible Under Node.js, publish() deep-freezes everything it reaches, so Object.isFrozen and Object.isSealed return true and Object.isExtensible returns false for a published object, array, Map or Set. Native builds already refused writes into published values, but answered these queries as for an unpublished value: publication marks an object immortal and sets no frozen bit. - record.isFrozen, array.isFrozen and the extensibility test of Maps and Sets also read an immortal rc word as published, the test of the published write guards, for the types the program may publish (PublishedTypes). Other types emit the code they did. - scr_dyn_is_frozen, scr_dyn_is_sealed and scr_dyn_is_extensible answer for a published object, array, Map or Set, or a checked view of one, before materializing the view. --- .../compiler/src/backend/llvm/lib-dispatch.ts | 49 +++++++- packages/compiler/src/backend/publication.ts | 5 + .../published-integrity-queries.ts.json | 6 + packages/runtime/src/scr_json.c | 15 +++ tests/corpus/published-integrity-queries.ts | 107 ++++++++++++++++++ 5 files changed, 178 insertions(+), 4 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/published-integrity-queries.ts.json create mode 100644 tests/corpus/published-integrity-queries.ts diff --git a/packages/compiler/src/backend/llvm/lib-dispatch.ts b/packages/compiler/src/backend/llvm/lib-dispatch.ts index c3acf766f..80ec73e5a 100644 --- a/packages/compiler/src/backend/llvm/lib-dispatch.ts +++ b/packages/compiler/src/backend/llvm/lib-dispatch.ts @@ -24,7 +24,7 @@ import { } from "./lib-shared.js"; import { isDynUndefined } from "../../ir/runtime-features.js"; import { DYN_KIND } from "./dyn.js"; -import { caBytes } from "./lib-abi.js"; +import { caBytes, callRuntime, ptr } from "./lib-abi.js"; import { emitInlineAtomicsOp } from "./expr-bytes.js"; import { regexLiteralCaptures } from "./regex-matchers.js"; import { registerReplay } from "./emit-journal.js"; @@ -1110,14 +1110,44 @@ export function emitLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue { if (e.fn === "record.isFrozen") { // The frozen bit of the record's rc word (scr_rc.h SCR_RC_FROZEN) is // its second-highest bit; an immortal word sets the highest one too. - const value = host.emitReadReceiver(e.args[0]!); + // Where the record's shape may be published, an immortal word reads as + // frozen: publish() freezes what it reaches, as Node's deep freeze does. + const receiver = e.args[0]!; + const value = host.emitReadReceiver(receiver); const S = host.sizeType; const rc = host.B.tmp(), - top = host.B.tmp(), - t = host.B.tmp(); + top = host.B.tmp(); + let t = host.B.tmp(); host.B.line(`${rc} = load ${S}, ptr ${value.name}`); host.B.line(`${top} = lshr ${S} ${rc}, ${S === "i64" ? 62 : 30}`); host.B.line(`${t} = icmp eq ${S} ${top}, 1`); + if ( + receiver.type.kind === "record" && + host.publishedTypes?.recordGuarded(receiver.type.shapeId) === true + ) + t = orPublished(host, t, rc); + return { name: t, type: e.type }; + } + if ( + (e.fn === "array.isFrozen" || e.fn === "collection.isExtensible") && + host.publishedTypes?.containerPublished(e.args[0]!.type) === true + ) { + // publish() freezes what it reaches, as Node's deep freeze does: a + // published array is frozen, and a published Map or Set is not extensible. + const value = host.emitReadReceiver(e.args[0]!); + const answer = callRuntime( + host, + e.fn === "array.isFrozen" ? "scr_arr_is_frozen" : "scr_map_is_extensible", + "i1", + [ptr(value.name)], + ); + const rc = host.B.tmp(); + host.B.line(`${rc} = load ${host.sizeType}, ptr ${value.name}`); + if (e.fn === "array.isFrozen") return { name: orPublished(host, answer, rc), type: e.type }; + const mortal = host.B.tmp(), + t = host.B.tmp(); + host.B.line(`${mortal} = icmp ne ${host.sizeType} ${rc}, -1`); + host.B.line(`${t} = and i1 ${answer}, ${mortal}`); return { name: t, type: e.type }; } const prefix = e.fn.slice(0, e.fn.indexOf(".")) as LibCallPrefix; @@ -1346,3 +1376,14 @@ function emitPromiseAllTuple(host: LlvmEmitterContext, e: LibCallExpr): LlValue B.line(`call void @scr_promise_release(ptr ${all})`); return host.own({ name: result, type: e.type }); } + +/** `frozen` or'ed with "the object whose rc word is `rc` is published": an + * immortal word (all ones), the test of the published write guards + * (emitter.ts emitPublishedTest). */ +function orPublished(host: LlvmEmitterContext, frozen: string, rc: string): string { + const immortal = host.B.tmp(), + t = host.B.tmp(); + host.B.line(`${immortal} = icmp eq ${host.sizeType} ${rc}, -1`); + host.B.line(`${t} = or i1 ${frozen}, ${immortal}`); + return t; +} diff --git a/packages/compiler/src/backend/publication.ts b/packages/compiler/src/backend/publication.ts index 38e429a47..607ee3139 100644 --- a/packages/compiler/src/backend/publication.ts +++ b/packages/compiler/src/backend/publication.ts @@ -73,6 +73,11 @@ export class PublishedTypes { return this.containers.has(key) || this.frozenArrays.has(key); } + /** Whether an array, Map or Set of type `t` may be published. */ + containerPublished(t: IrType): boolean { + return this.containers.has(typeKey(t)); + } + /** Whether an array of type `t` may be published, and whether it may be * frozen: the two states its write guard tests. */ arrayGuards(t: IrType): { published: boolean; frozen: boolean } { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/published-integrity-queries.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/published-integrity-queries.ts.json new file mode 100644 index 000000000..2e5570a24 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/published-integrity-queries.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/published-integrity-queries.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index eb40100cf..93a0bc1ae 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -2967,6 +2967,18 @@ static bool scr_dyn_typed_ref_view_answer(ScrDyn *object, bool (*query)(const Sc return true; } +/* Whether `object` is a published object, array, Map or Set, or the checked + * view of one (its referent's rc word is immortal, the test of the published + * write guards). publish() freezes what it reaches, as Node's deep freeze + * does: such a value is frozen, sealed and not extensible. */ +static bool scr_dyn_published_object(const ScrDyn *object) { + if (object->kind == SCR_DYN_TYPED_REF) + return object->v.typed_ref.ptr != NULL && SCR_RC_PUBLISHED(object->v.typed_ref.ptr); + if (object->kind == SCR_DYN_OBJ || object->kind == SCR_DYN_ARR) return object->rc == SIZE_MAX; + const ScrMap *collection = scr_dyn_native_collection(object); + return collection != NULL && SCR_RC_PUBLISHED(collection); +} + static ScrDyn *scr_bytes_properties(const ScrDyn *v); static bool scr_dyn_is_extensible_view(const ScrDyn *view) { if (view->kind == SCR_DYN_BYTES) return !view->v.bytes->properties || !view->v.bytes->properties->non_extensible; @@ -3043,6 +3055,7 @@ ScrDyn *scr_dyn_prevent_extensions(ScrDyn *object) { bool scr_dyn_is_extensible(ScrDyn *object) { if (scr_dyn_class_reflection_fence(object)) return false; if (object->kind == SCR_DYN_PROXY) { scr_dyn_proxy_unsupported("Object.isExtensible"); return false; } + if (scr_dyn_published_object(object)) return false; bool answer; if (scr_dyn_typed_ref_view_answer(object, scr_dyn_is_extensible_view, &answer)) return answer; ScrMap *collection = scr_dyn_native_collection(object); @@ -3094,6 +3107,7 @@ ScrDyn *scr_dyn_seal(ScrDyn *object) { bool scr_dyn_is_sealed(const ScrDyn *object) { if (scr_dyn_class_reflection_fence(object)) return false; if (object->kind == SCR_DYN_PROXY) { scr_dyn_proxy_unsupported("Object.isSealed"); return false; } + if (scr_dyn_published_object(object)) return true; bool answer; if (scr_dyn_typed_ref_view_answer((ScrDyn *)object, scr_dyn_is_sealed, &answer)) return answer; ScrMap *collection = scr_dyn_native_collection(object); @@ -19103,6 +19117,7 @@ ScrDyn *scr_dyn_freeze_data(ScrDyn *value) { bool scr_dyn_is_frozen(const ScrDyn *value) { if (scr_dyn_class_reflection_fence(value)) return false; + if (scr_dyn_published_object(value)) return true; bool answer; if (scr_dyn_typed_ref_view_answer((ScrDyn *)value, scr_dyn_is_frozen, &answer)) return answer; ScrMap *collection = scr_dyn_native_collection(value); diff --git a/tests/corpus/published-integrity-queries.ts b/tests/corpus/published-integrity-queries.ts new file mode 100644 index 000000000..9ec0c7cf3 --- /dev/null +++ b/tests/corpus/published-integrity-queries.ts @@ -0,0 +1,107 @@ +import { publish } from "@scriptc/threads"; + +// Object.isFrozen, Object.isSealed and Object.isExtensible on published +// values. Under Node.js, publish() deep-freezes the graph, so every object it +// reaches is frozen, sealed and not extensible. Unpublished values of the same +// types keep their own state. + +class Box { + items: number[]; + constructor(items: number[]) { + this.items = items; + } +} + +class Link { + next: Link | undefined = undefined; + label = "link"; +} + +interface Point { + x: number; + inner: { y: number }; +} + +function report(name: string, value: object): void { + console.log(name, Object.isFrozen(value), Object.isSealed(value), Object.isExtensible(value)); +} + +const box = publish(new Box([])); +report("class", box); +report("class field array", box.items); +report("numbers", publish([1, 2])); +report("strings", publish(["a", "b"])); +const point: Point = publish({ x: 1, inner: { y: 2 } }); +report("record", point); +report("nested record", point.inner); +report("map", publish(new Map([["a", 1]]))); +report("set", publish(new Set([1]))); +const chain = new Link(); +chain.next = new Link(); +publish(chain); +report("chain", chain); +report("chain next", chain.next!); +const loose: unknown = publish({ z: [3] }); +report("unknown", loose as object); +report("unknown field", (loose as { z: number[] }).z); + +// The same queries on statically typed values. +const numbers = publish([4, 5]); +const pair = publish({ x: 3, inner: { y: 4 } }); +const table = publish(new Map([["b", 2]])); +const members = publish(new Set(["s"])); +console.log( + "typed", + Object.isFrozen(numbers), + Object.isFrozen(pair), + Object.isFrozen(pair.inner), + Object.isFrozen(table), + Object.isFrozen(members), + Object.isFrozen(box), + Object.isFrozen(box.items), +); +console.log("typed sealed", Object.isSealed(table), Object.isSealed(members), Object.isSealed(box)); +console.log( + "typed extensible", + Object.isExtensible(table), + Object.isExtensible(members), + Object.isExtensible(box), +); +const otherNumbers = [4, 5]; +const otherPair = { x: 3, inner: { y: 4 } }; +const otherTable = new Map([["b", 2]]); +console.log( + "typed unpublished", + Object.isFrozen(otherNumbers), + Object.isFrozen(otherPair), + Object.isFrozen(otherTable), + Object.isExtensible(otherTable), + Object.isFrozen(new Box([])), +); + +// Unpublished values of the same types. +report("unpublished class", new Box([1])); +report("unpublished numbers", [1, 2]); +report("unpublished record", { x: 1, inner: { y: 2 } }); +report("unpublished map", new Map()); +report("unpublished set", new Set()); +const fresh = new Link(); +report("unpublished chain", fresh); +fresh.next = new Link(); +report("unpublished chain next", fresh.next); + +// Frozen without publishing. +report("frozen record", Object.freeze({ x: 1, inner: { y: 2 } })); +report("frozen numbers", Object.freeze([1, 2])); + +// Writes into published values still throw. +try { + box.items.push(1); +} catch (e) { + console.log((e as Error).name, (e as Error).message); +} +try { + (chain as { label: string }).label = "changed"; +} catch (e) { + console.log((e as Error).name, (e as Error).message); +} From a897079be1b4282dd897cc4bb2fc817c6e010da7 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 13:00:48 -0500 Subject: [PATCH 50/54] Capture onto the target evaluated before a constructorOpt that reassigns it When Error.captureStackTrace evaluates a JavaScript constructorOpt (a getter, a call, or this.constructor with a constructor getter), V8 has already evaluated the target. The lowering kept the target in a local only when its expression was not a plain read, so a constructorOpt that assigned the target variable made the capture install the stack on the new value. The target is now always kept in a local before the constructorOpt runs. --- .../src/frontend/lowering/builtins/errors.ts | 13 ++++++----- ...6874-error-capture-stack-trace-objects.cjs | 22 +++++++++++++++++++ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/builtins/errors.ts b/packages/compiler/src/frontend/lowering/builtins/errors.ts index 7e2bb6d37..4cd72b2b2 100644 --- a/packages/compiler/src/frontend/lowering/builtins/errors.ts +++ b/packages/compiler/src/frontend/lowering/builtins/errors.ts @@ -2,7 +2,7 @@ import * as ts from "../../ts7/adapter.js"; import { type Lowerer } from "../lowerer.js"; import { isJsSourceFile, locOf } from "../../program.js"; import { dynUndefinedExpr } from "../../../ir/build.js"; -import { isSafeToDiscard, isSafeToRepeat } from "../expressions/evaluation-safety.js"; +import { isSafeToDiscard } from "../expressions/evaluation-safety.js"; import { BOOL, CAUGHT, @@ -541,11 +541,12 @@ export function lowerErrorCaptureStackTraceCall( const stmts: IrStmt[] = []; let receiver = lowerer.coerceToExpected(target, DYN); if (evaluated !== null) { - if (!isSafeToRepeat(receiver)) { - const local = lowerer.declareHiddenLocal("%captureTarget", DYN); - stmts.push({ kind: "varDecl", localId: local.id, init: receiver, loc }); - receiver = { kind: "varRef", localId: local.id, type: DYN, loc }; - } + // V8 captures onto the target value it evaluated first. The constructor + // argument may assign the variable or field the target was read from, so + // even a plain read is kept in a local before the argument runs. + const local = lowerer.declareHiddenLocal("%captureTarget", DYN); + stmts.push({ kind: "varDecl", localId: local.id, init: receiver, loc }); + receiver = { kind: "varRef", localId: local.id, type: DYN, loc }; stmts.push({ kind: "exprStmt", expr: evaluated, loc }); } const capture: IrExpr = { diff --git a/tests/corpus/6874-error-capture-stack-trace-objects.cjs b/tests/corpus/6874-error-capture-stack-trace-objects.cjs index 456874b4e..c575760c3 100644 --- a/tests/corpus/6874-error-capture-stack-trace-objects.cjs +++ b/tests/corpus/6874-error-capture-stack-trace-objects.cjs @@ -150,3 +150,25 @@ try { } catch (e) { console.log('getter threw', e instanceof RangeError, e.message); } + +// The stack goes on the target as evaluated before constructorOpt, even when +// constructorOpt assigns the variable or field the target was read from. +let target = { message: 'original' }; +const replacement = { message: 'replacement' }; +const swapping = { + get ctor() { + target = replacement; + return inner; + }, +}; +const original = target; +Error.captureStackTrace(target, swapping.ctor); +console.log('reassigned variable', 'stack' in original, 'stack' in replacement, target === replacement); +const box = { error: { message: 'boxed' } }; +const boxed = box.error; +function swapField() { + box.error = { message: 'swapped in' }; + return inner; +} +Error.captureStackTrace(box.error, swapField()); +console.log('reassigned field', 'stack' in boxed, 'stack' in box.error); From f01a913ed4836debf335288fedf9750948c33026 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 13:04:18 -0500 Subject: [PATCH 51/54] Solve frame-escape facts on demand from one walk per function analyzeFrameEscapes walked every function's body on each visit of its parameter and capture fixpoint, and once more for each function's locals: 1.8 s on tsc-ts's IR, plus 0.5 s for the locals. Each function's body is now walked once into a summary: its value groups, the groups that escape whatever the facts say, and per group the calls and closures whose facts decide it. A query explores only the facts its answer depends on (a parameter's fact depends on the uses of its own group) and solves that part of the graph to its greatest fixpoint; solved facts are kept for later queries. The answers are unchanged: on tsc-ts every parameter fact and every function's frame locals are identical, and the emitter's queries take about 0.35 s. --- .../compiler/src/backend/llvm/expr-calls.ts | 2 +- .../src/backend/llvm/frame-escapes.ts | 251 +++++++++++------- 2 files changed, 152 insertions(+), 101 deletions(-) diff --git a/packages/compiler/src/backend/llvm/expr-calls.ts b/packages/compiler/src/backend/llvm/expr-calls.ts index 2d29022ed..a088cca01 100644 --- a/packages/compiler/src/backend/llvm/expr-calls.ts +++ b/packages/compiler/src/backend/llvm/expr-calls.ts @@ -139,7 +139,7 @@ export function emitCallExpr( // the call's statement frame releases its reference fields. if ( a.kind === "new" && - host.frameEscapes?.parameters.get(e.callee)?.has(index) === true && + host.frameEscapes?.parameterStays(e.callee, index) === true && frameClass(host, a.className) ) { const value = emitStackObject(host, a as StackObjectSource, (fields) => { diff --git a/packages/compiler/src/backend/llvm/frame-escapes.ts b/packages/compiler/src/backend/llvm/frame-escapes.ts index 8197e94f5..6156f6fc3 100644 --- a/packages/compiler/src/backend/llvm/frame-escapes.ts +++ b/packages/compiler/src/backend/llvm/frame-escapes.ts @@ -22,15 +22,19 @@ * in a local class, converting it, and every call whose target is not a * known function with a proven parameter. * - * Parameter and capture facts are solved optimistically over the call - * graph, like CallLifetimes: every reference parameter stays in its - * callee's frame, and every captured binding in its closure's, until a use - * proves otherwise. Async functions, generators and module code have none, - * since their frames can outlive a call or are the program's own. */ -import type { IrFunction, IrModule } from "../../ir/ir.js"; + * Parameter and capture facts are solved optimistically, like + * CallLifetimes: every reference parameter stays in its callee's frame, and + * every captured binding in its closure's, until a use proves otherwise. + * They are solved on demand: a query explores only the facts its answer + * depends on (a parameter's fact depends on the calls its own value reaches), + * and each function's body is walked once into a summary of those uses. + * Async functions, generators and module code have no facts, since their + * frames can outlive a call or are the program's own. */ +import type { IrExpr, IrFunction, IrModule, IrStmt } from "../../ir/ir.js"; +import { everyExprChild, everyStmtChild } from "../../ir/traverse.js"; import { - children, - forEachNode, + isExpr, + indexOfNode, keepsInFrame, same, refType, @@ -43,33 +47,43 @@ export interface FrameEscapes { * never assigned after its declaration, captured only by closures that * keep it in the frame, and every use keeping it in the frame. */ locals(fn: IrFunction): ReadonlySet; - /** Per function, the reference parameters whose argument stays in the - * callee's frame. */ - readonly parameters: ReadonlyMap>; + /** Whether argument `index` of a call of `callee` stays in the callee's + * frame. */ + parameterStays(callee: string, index: number): boolean; } -const NO_FUNCTIONS: ReadonlySet = new Set(); - function suspends(fn: IrFunction): boolean { return fn.async === true || fn.generator !== undefined || /^%(init\.\d+|main)$/.test(fn.name); } -/** Parameter and capture facts, solved together (analyzeFrameEscapes). */ -interface Facts { - parameters: Map>; - /** Per lifted function, the captured locals whose value stays in the - * frame of each invocation of the closure. */ - captures: Map>; +/** A use whose fate a fact decides: argument `index` of a call of `fn` + * (`index` >= 0), or a binding `id` that a closure of lifted function `fn` + * captures. */ +interface FactUse { + fn: string; + index: number; + id: string; } -/** The locals of `fn` that a use lets leave the frame, or that are assigned - * after their declaration. A local bound by a `const` declaration to - * another local's value shares its fate. A closure shares the fate of the - * locals it captures: they stay in the frame only while the closure does - * and its body keeps them there (Facts.captures). Every other captured - * binding escapes. */ -function escapingLocals(fn: IrFunction, facts: Facts): Set { - const escaping = new Set(); +/** What one walk of a function's body finds, by value group: a `const` + * binding of a value shares its fate, and a closure that of the locals it + * captures. */ +interface Summary { + /** The representative of each grouped value. */ + root: Map; + /** Groups that escape whatever the facts say. */ + escaping: Set; + /** Per group, the uses whose facts decide it. */ + uses: Map; +} + +const NO_USES: readonly FactUse[] = []; +const NO_PARAMETERS: ReadonlyMap> = new Map(); + +/** Walks `fn` once. */ +function summarize(fn: IrFunction): Summary { + const escapes = new Set(); + const facts: { id: string; use: FactUse }[] = []; const own = new Set(fn.locals.map((l) => l.id)); const group = new Map(); const find = (id: string): string => { @@ -88,7 +102,8 @@ function escapingLocals(fn: IrFunction, facts: Facts): Set { let closures = 0; const parents: Node[] = []; /** The fate of the value `node` produces: shared with the binding a - * `const` declaration gives it, kept by its consumer, or escaping. */ + * `const` declaration gives it, kept by its consumer, decided by a + * parameter fact, or escaping. */ const fate = (node: Node, id: string): void => { let value: Node = node; let at = parents.length - 1; @@ -102,115 +117,151 @@ function escapingLocals(fn: IrFunction, facts: Facts): Set { parent.localId !== id ) unite(id, parent.localId); - else if (!keepsInFrame(value, parent, facts.parameters)) escaping.add(id); + else if (parent?.kind === "call") { + const index = indexOfNode(parent.args, value); + if (index >= 0) facts.push({ id, use: { fn: parent.callee, index, id: "" } }); + else escapes.add(id); + } else if (!keepsInFrame(value, parent, NO_PARAMETERS)) escapes.add(id); }; const walk = (node: Node): void => { if (node.kind === "varDecl") declared.set(node.localId, (declared.get(node.localId) ?? 0) + 1); if (node.kind === "assign" || node.kind === "assignExpr" || node.kind === "incDec") - escaping.add(node.localId); - if (node.kind === "classRef") for (const id of node.captures ?? []) escaping.add(id); + escapes.add(node.localId); + if (node.kind === "classRef") for (const id of node.captures ?? []) escapes.add(id); if (node.kind === "closure") { const closure = `%closure.${closures++}`; fate(node, closure); - const kept = facts.captures.get(node.fnName); for (const id of node.captures ?? []) { closureCaptured.add(id); unite(id, closure); - if (kept?.has(id) !== true) escaping.add(id); + facts.push({ id, use: { fn: node.fnName, index: -1, id } }); } } if (node.kind === "varRef") { fate(node, node.localId); return; } - const kids: Node[] = []; - children(node, kids); parents.push(node); - for (const kid of kids) walk(kid); + if (isExpr(node)) everyExprChild(node, visitExpr, visitStmt); + else everyStmtChild(node, visitExpr, visitStmt); parents.pop(); }; + const visitExpr = (child: IrExpr): boolean => { + walk(child); + return true; + }; + const visitStmt = (child: IrStmt): boolean => { + walk(child); + return true; + }; for (const stmt of fn.body) walk(stmt); const lifted = new Set((fn.captures ?? []).map((c) => c.localId)); for (const local of fn.locals) { if (local.boxed === true && !lifted.has(local.id) && !closureCaptured.has(local.id)) - escaping.add(local.id); - if ((declared.get(local.id) ?? 0) > 1) escaping.add(local.id); + escapes.add(local.id); + if ((declared.get(local.id) ?? 0) > 1) escapes.add(local.id); } - const roots = new Set(); - for (const id of escaping) roots.add(find(id)); - for (const local of fn.locals) if (roots.has(find(local.id))) escaping.add(local.id); - return escaping; + const root = new Map(); + for (const id of group.keys()) root.set(id, find(id)); + const rootOf = (id: string): string => root.get(id) ?? id; + const escaping = new Set(); + for (const id of escapes) escaping.add(rootOf(id)); + const uses = new Map(); + for (const { id, use } of facts) { + const r = rootOf(id); + if (escaping.has(r)) continue; + let list = uses.get(r); + if (!list) uses.set(r, (list = [])); + list.push(use); + } + return { root, escaping, uses }; } export function analyzeFrameEscapes(mod: IrModule): FrameEscapes { - const facts: Facts = { parameters: new Map(), captures: new Map() }; - for (const fn of mod.functions) { - const safe = new Set(); - const kept = new Set(); - if (!suspends(fn)) { - fn.params.forEach((param, i) => { - if (refType(param.type)) safe.add(i); - }); - for (const capture of fn.captures ?? []) kept.add(capture.localId); + const fns = new Map(mod.functions.map((fn) => [fn.name, fn])); + const summaries = new Map(); + const summaryOf = (fn: IrFunction): Summary => { + let summary = summaries.get(fn.name); + if (summary === undefined) { + summary = summarize(fn); + summaries.set(fn.name, summary); } - facts.parameters.set(fn.name, safe); - facts.captures.set(fn.name, kept); - } - // A fact that turns false can make locals escape in the functions that - // call the function or create its closures, so only those are visited - // again. - const dependents = new Map>(); - const depend = (on: string, fn: IrFunction): void => { - let set = dependents.get(on); - if (!set) dependents.set(on, (set = new Set())); - set.add(fn); + return summary; }; - for (const fn of mod.functions) - forEachNode(fn.body, (node) => { - if (node.kind === "call") depend(node.callee, fn); - else if (node.kind === "closure") depend(node.fnName, fn); - }); - const has = (fn: IrFunction): boolean => - facts.parameters.get(fn.name)!.size > 0 || facts.captures.get(fn.name)!.size > 0; - const queued = new Set(mod.functions.filter(has)); - const queue = [...queued]; - while (queue.length > 0) { - const fn = queue.pop()!; - queued.delete(fn); - if (!has(fn)) continue; - const escaping = escapingLocals(fn, facts); - let changed = false; - const safe = facts.parameters.get(fn.name)!; - fn.params.forEach((param, i) => { - if (safe.has(i) && escaping.has(param.localId)) { - safe.delete(i); - changed = true; - } - }); - const kept = facts.captures.get(fn.name)!; - for (const id of [...kept]) - if (escaping.has(id)) { - kept.delete(id); - changed = true; - } - if (changed) - for (const dependent of dependents.get(fn.name) ?? NO_FUNCTIONS) - if (!queued.has(dependent) && has(dependent)) { - queued.add(dependent); - queue.push(dependent); + /** A fact's key, the binding in its function that it is about, or null + * when the fact is false from the start. */ + const subject = (use: FactUse): { key: string; fn: IrFunction; id: string } | null => { + const fn = fns.get(use.fn); + if (fn === undefined || suspends(fn)) return null; + if (use.index >= 0) { + const param = fn.params[use.index]; + if (param === undefined || !refType(param.type)) return null; + return { key: `${use.fn}#${use.index}`, fn, id: param.localId }; + } + if (!(fn.captures ?? []).some((c) => c.localId === use.id)) return null; + return { key: `${use.fn}@${use.id}`, fn, id: use.id }; + }; + const solved = new Map(); + /** The facts `uses` decide, solved over the facts they depend on that are + * not solved yet: a greatest fixpoint on that part of the graph. */ + const solve = (uses: readonly FactUse[]): void => { + const pending = new Map(); + const stack: FactUse[] = [...uses]; + while (stack.length > 0) { + const use = stack.pop()!; + const s = subject(use); + if (s === null || solved.has(s.key) || pending.has(s.key)) continue; + const summary = summaryOf(s.fn); + const r = summary.root.get(s.id) ?? s.id; + const node = { ok: !summary.escaping.has(r), deps: [] as string[] }; + pending.set(s.key, node); + if (!node.ok) continue; + for (const dep of summary.uses.get(r) ?? NO_USES) { + const d = subject(dep); + if (d === null) { + node.ok = false; + break; } - } + node.deps.push(d.key); + stack.push(dep); + } + } + for (let changed = true; changed;) { + changed = false; + for (const node of pending.values()) + if (node.ok && node.deps.some((d) => !(solved.get(d) ?? pending.get(d)!.ok))) { + node.ok = false; + changed = true; + } + } + for (const [key, node] of pending) solved.set(key, node.ok); + }; + /** Whether every use in `uses` keeps its value in the frame. */ + const allStay = (uses: readonly FactUse[]): boolean => { + solve(uses); + for (const use of uses) { + const s = subject(use); + if (s === null || solved.get(s.key) !== true) return false; + } + return true; + }; const locals = new Map>(); return { - parameters: facts.parameters, + parameterStays(callee: string, index: number): boolean { + return allStay([{ fn: callee, index, id: "" }]); + }, locals(fn: IrFunction): ReadonlySet { let kept = locals.get(fn.name); if (kept === undefined) { const set = new Set(); if (!suspends(fn)) { - const escaping = escapingLocals(fn, facts); - for (const local of fn.locals) - if (refType(local.type) && !escaping.has(local.id)) set.add(local.id); + const summary = summaryOf(fn); + for (const local of fn.locals) { + if (!refType(local.type)) continue; + const r = summary.root.get(local.id) ?? local.id; + if (!summary.escaping.has(r) && allStay(summary.uses.get(r) ?? NO_USES)) + set.add(local.id); + } } locals.set(fn.name, (kept = set)); } From 4bd4c43efdaae81f727b0921d662538e7c382f99 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 13:28:33 -0500 Subject: [PATCH 52/54] Check Uint8Array chunks in clones of JavaScript ReadableStream bodies Response.clone and Request.clone tee the body. The tee branches of a body read from a JavaScript ReadableStream did not keep that body's chunk check, so consuming the original or the clone took string chunks as text. The check is now a flag on the body's stream that its tee branches inherit, and consuming either branch rejects with Node's TypeError "Received non-Uint8Array chunk". --- packages/runtime/src/scr_fetch.c | 14 +++++++++++--- tests/corpus/web-stream-fetch-interop.mjs | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index 9ca6efde3..4416d99a9 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -161,6 +161,9 @@ struct SfStream { /* A body made from a checked-dynamic web ReadableStream: that stream's * reader (+1), read one chunk per pull through scr_web_stream_bridge. */ ScrDyn *web_reader; + /* Consuming the body takes Uint8Array chunks only: set on a body from a + * web ReadableStream and on the tee branches of such a body. */ + bool u8_chunks; #endif ScrArr *from_array; ScrBytes *from_bytes; @@ -1178,9 +1181,10 @@ static void sf_stream_drain(SfStream *s) { s->queued--; #ifdef SCR_HANDLE_CLASSES ScrBytes *bytes; - /* A body read from a JavaScript ReadableStream takes Uint8Array - * chunks only, as Node's body mixin does. */ - if (s->web_reader && !scr_dyn_bytes_is(c->value, SCR_BYTES_U8)) { + /* A body read from a JavaScript ReadableStream, or from a clone or + * tee branch of one, takes Uint8Array chunks only, as Node's body + * mixin does. */ + if (s->u8_chunks && !scr_dyn_bytes_is(c->value, SCR_BYTES_U8)) { sf_type_error("Received non-Uint8Array chunk"); bytes = NULL; } else { @@ -1611,6 +1615,7 @@ static ScrDyn *sf_web_body(const ScrDyn *body) { if (!reader) return NULL; SfStream *s = sf_stream_new_native(); s->web_reader = reader; + s->u8_chunks = true; sf_stream_schedule_initial_pull(s); ScrDyn *out = scr_dyn_new_handle(s, SCR_DYNH_WEB_STREAM); sf_stream_release(s); @@ -2228,6 +2233,9 @@ static void sf_stream_tee(SfStream *s, SfStream *out[2]) { b->cancel_cb = sf_tee_func( t, i == 0 ? &sf_tee_cancel_thunk_0 : &sf_tee_cancel_thunk_1, "cancel"); b->source_this = scr_dyn_retain(scr_dyn_undefined()); +#ifdef SCR_HANDLE_CLASSES + b->u8_chunks = s->u8_chunks; +#endif sf_stream_track_callbacks(b); t->branch[i] = sf_stream_retain(b); out[i] = b; diff --git a/tests/corpus/web-stream-fetch-interop.mjs b/tests/corpus/web-stream-fetch-interop.mjs index f6a3877a1..16ee91c23 100644 --- a/tests/corpus/web-stream-fetch-interop.mjs +++ b/tests/corpus/web-stream-fetch-interop.mjs @@ -48,3 +48,17 @@ console.log("request", await request.text()); const strings = new Response(new ReadableStream({ start(c) { c.enqueue("str"); c.close(); } })); console.log("non-bytes", await settled(strings.text())); + +// Clones go through an untyped parameter: Response.clone and Request.clone +// run on checked-dynamic handles. +const cloneOf = (value) => value.clone(); +const stringSource = () => new ReadableStream({ start(c) { c.enqueue("str"); c.close(); } }); +const original = new Response(stringSource()); +const clone = cloneOf(original); +console.log("clone", await settled(original.text()), await settled(clone.text())); +const cloneOfClone = cloneOf(cloneOf(new Response(stringSource()))); +console.log("clone-of-clone", await settled(cloneOfClone.text())); +const requestClone = cloneOf(new Request("http://example.com/", { method: "POST", body: stringSource(), duplex: "half" })); +console.log("request-clone", await settled(requestClone.text())); +const bytesClone = cloneOf(new Response(new ReadableStream({ start(c) { c.enqueue(encoder.encode("ok")); c.close(); } }))); +console.log("bytes-clone", await settled(bytesClone.text())); From 4faabded7f889058067d5e44c864c6dc69778e99 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 13:31:11 -0500 Subject: [PATCH 53/54] List the held entries a unit record reads, and leave units that register units or add class fields without one A record now has a reads section: the entries of its function tables that the unit's IR and new entries name and that the tables held before the unit, each with a digest of its content (contentDigest). An applier checks that it holds each one with the same content. This catches helpers named in creation order, such as %dyn.class.protocol., which another process can number differently. Two lowering effects had no table a record could check. A unit that collects a function-local class registers the class's members as reachable units, and Object.freeze of a class instance adds hidden fields to the class layout. The lowerer now counts both (unitRegistrations, classLayoutChanges), and a unit that changes either count has no record. --- .../src/frontend/lowering/class-attributes.ts | 11 +-- .../compiler/src/frontend/lowering/lowerer.ts | 10 +++ .../frontend/lowering/unit-journal.test.ts | 32 +++++++ .../src/frontend/lowering/unit-journal.ts | 88 ++++++++++++++++--- .../compiler/test/recorded-lowering.test.ts | 33 +++++++ 5 files changed, 157 insertions(+), 17 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/class-attributes.ts b/packages/compiler/src/frontend/lowering/class-attributes.ts index cdd53efa6..e9f730cc9 100644 --- a/packages/compiler/src/frontend/lowering/class-attributes.ts +++ b/packages/compiler/src/frontend/lowering/class-attributes.ts @@ -53,13 +53,14 @@ function attributeClassNamed(lowerer: Lowerer, className: string): ClassInfo | n /** Adds a zero-initialized f64 to every class of the hierarchy, at the same * offset, like the property bag. */ -function addHiddenField(info: ClassInfo, name: string): void { +function addHiddenField(lowerer: Lowerer, info: ClassInfo, name: string): void { let root = info; while (root.base && !root.base.def.runtime) root = root.base; if (root.fields.has(name)) return; const index = root.def.fields.length; const add = (current: ClassInfo): void => { if (!current.fields.has(name)) { + lowerer.classLayoutChanges++; current.fields.set(name, F64); current.def.fields.splice(index, 0, { name, type: F64 }); } @@ -78,7 +79,7 @@ export function addUntypedIntegrityWords(lowerer: Lowerer): void { if (!lowerer.untypedIntegrity) return; for (const [className, info] of lowerer.classes) { if (isDynTypedRefType({ kind: "object", className }) && attributeClassNamed(lowerer, className)) - addHiddenField(info, CLASS_ATTRIBUTES); + addHiddenField(lowerer, info, CLASS_ATTRIBUTES); } } @@ -97,7 +98,7 @@ export function lowerClassIntegrity( ): IrExpr | null { const info = attributeClass(lowerer, target); if (!info) return null; - addHiddenField(info, CLASS_ATTRIBUTES); + addHiddenField(lowerer, info, CLASS_ATTRIBUTES); const loc = locOf(call); const type = target.type as ClassObjectType; const bits = @@ -204,8 +205,8 @@ export function lowerClassFieldDefine( ) return null; const attributesField = classFieldAttributesName(field); - addHiddenField(info, CLASS_ATTRIBUTES); - addHiddenField(info, attributesField); + addHiddenField(lowerer, info, CLASS_ATTRIBUTES); + addHiddenField(lowerer, info, attributesField); const loc = locOf(call); const type = target.type as ClassObjectType; const receiverSlot = lowerer.declareHiddenLocal("%defineReceiver", type); diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 2327446f2..07f5dc700 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -3798,6 +3798,11 @@ export class Lowerer { /** The events of the unit being recorded (unit-journal.ts); null * otherwise. */ unitEvents: UnitEvents | null = null; + /** Fields added to collected classes (class-attributes.ts) and reachable + * units registered while units lower: a unit that changes either has no + * record (journalScalars). */ + classLayoutChanges = 0; + unitRegistrations = 0; classCanBeConstructed(info: ClassInfo): boolean { return ( @@ -4440,6 +4445,8 @@ export class Lowerer { fixed("onExprClassCollected", () => this.onExprClassCollected), fixed("onLateClassCollected", () => this.onLateClassCollected), fixed("onDeclarationRegistered", () => this.onDeclarationRegistered), + fixed("classLayoutChanges", () => this.classLayoutChanges), + fixed("unitRegistrations", () => this.unitRegistrations), fixed("liftDynFieldPolicy", () => this.liftDynFieldPolicy), fixed("coercionOriginNode", () => this.coercionOriginNode), fixed("coercionOriginUnion", () => this.coercionOriginUnion), @@ -9855,6 +9862,7 @@ export class Lowerer { lower: () => IrFunction | null, ): void => { units.set(name, { order: unitOrder++, roots, lower }); + this.unitRegistrations++; if (reachable.has(name)) queue.push(name); metadataPriority.set(name, [3, expressionMetadataOrder++]); }; @@ -9895,6 +9903,7 @@ export class Lowerer { roots: functionRoots(decl), lower: () => this.lowerFunction(decl), }); + this.unitRegistrations++; if (reachable.has(sig.name)) queue.push(sig.name); metadataPriority.set(sig.name, [3, expressionMetadataOrder++]); } else if (ts.isClassDeclaration(decl)) { @@ -9915,6 +9924,7 @@ export class Lowerer { roots: functionRoots(entry.member), lower: () => lowerStaticMethod(this, info, name), }); + this.unitRegistrations++; if (reachable.has(unit)) queue.push(unit); metadataPriority.set(unit, [3, expressionMetadataOrder++]); } diff --git a/packages/compiler/src/frontend/lowering/unit-journal.test.ts b/packages/compiler/src/frontend/lowering/unit-journal.test.ts index 414184ae1..f92f11dc7 100644 --- a/packages/compiler/src/frontend/lowering/unit-journal.test.ts +++ b/packages/compiler/src/frontend/lowering/unit-journal.test.ts @@ -3,6 +3,8 @@ import type { Program, SourceFile } from "../ts7/adapter.js"; import { Lowerer } from "./lowerer.js"; import { checkedTable, + contentDigest, + encodeUnitFunction, flagScalar, functionTable, lowerWaveRecorded, @@ -342,3 +344,33 @@ test("a unit that changes a function it names in place has no record", () => { lowerWaveRecorded(subject, ["a"], stats); expect(stats).toMatchObject({ recorded: 0, unrecorded: 1, reasons: { "lifted:changed": 1 } }); }); + +test("a record lists the held functions it names, with their digests", () => { + const functions = [fn("%shared"), fn("%unused")]; + const subject: WaveHost = { + tables: [functionTable("lifted", functions, (name) => functions.some((f) => f.name === name))], + scalars: [], + lowerRecorded: () => { + functions.push(fn("%own")); + return { + fn: { + ...fn("%a"), + body: [ + { kind: "call", callee: "%shared" }, + { kind: "call", callee: "%own" }, + ] as never, + }, + poisoned: false, + }; + }, + store: () => {}, + statementCounts: () => ({ total: 0, failed: 0, other: 0 }), + resolves: () => true, + }; + const { record } = recordUnit(subject, "a"); + // The function the unit added is in its entries, not its reads. + expect(record!.reads).toEqual({ + lifted: [["%shared", contentDigest(encodeUnitFunction(functions[0]!)!)]], + }); + expect(Object.keys(record!.tables)).toEqual(["lifted"]); +}); diff --git a/packages/compiler/src/frontend/lowering/unit-journal.ts b/packages/compiler/src/frontend/lowering/unit-journal.ts index 0992e7544..991b1368c 100644 --- a/packages/compiler/src/frontend/lowering/unit-journal.ts +++ b/packages/compiler/src/frontend/lowering/unit-journal.ts @@ -72,6 +72,10 @@ export interface UnitRecord { /** The encoded entries the unit added, by table name, for the tables it * grew. */ readonly tables: Readonly>; + /** The entries the unit's IR and new entries name that its tables held + * before it, by table name, each with a digest of its content. Applying + * the record needs each to be held with the same content. */ + readonly reads: Readonly>; } /** A table a unit's lowering can grow. An appendable table only adds @@ -87,6 +91,9 @@ export interface JournalTable { readonly entries?: (from: number) => string[] | null; /** Adds the encoded entries whose keys this process's table lacks. */ readonly apply?: (entries: readonly string[]) => void; + /** A digest of the content of the entry under `key`, when the table + * holds one before position `before`; null otherwise. */ + readonly digest?: (key: string, before: number) => string | null; /** Removes the entries from position `from` on. */ readonly truncate?: (from: number) => void; /** The entries the table holds now. */ @@ -108,6 +115,18 @@ export interface HeldContents { refresh(keys: ReadonlySet): void; } +/** A digest of `text`: its length and two 32-bit FNV-1a hashes. */ +export function contentDigest(text: string): string { + let a = 0x811c9dc5; + let b = 0x01000193 ^ 0x5bd1e995; + for (let i = 0; i < text.length; i++) { + const c = text.charCodeAt(i); + a = Math.imul(a ^ c, 0x01000193); + b = Math.imul(b ^ c, 0x01000193) ^ (b >>> 13); + } + return `${text.length}:${(a >>> 0).toString(16)}${(b >>> 0).toString(16)}`; +} + /** The strings quoted in JSON `texts`, unescaped only as far as their * delimiting quotes: the names a unit's encoded IR and entries refer to. */ export function quotedStrings(texts: readonly string[]): Set { @@ -310,18 +329,19 @@ export function recordUnit( if (entries === null) reasons.push(`${table.name}:encoding`); else added[table.name] = entries; } + // A unit uses the entries its IR and new entries name: it reads them, and + // a unit that changes one in place changes one it names. + const texts: string[] = []; + if (text !== null) texts.push(text); + for (const index of grown) { + const table = tables[index]!; + const entries = added[table.name] ?? table.entries!(before[index]!); + if (entries !== null) for (const entry of entries) texts.push(entry); + } + const names = quotedStrings(texts); if (contents !== null) { - // A unit changes in place the entries it uses, which its IR and new - // entries name: a unit with a record must leave them as they were, and - // a unit without one keeps its changes. - const texts: string[] = []; - if (text !== null) texts.push(text); - for (const index of grown) { - const table = tables[index]!; - const entries = added[table.name] ?? table.entries!(before[index]!); - if (entries !== null) for (const entry of entries) texts.push(entry); - } - const names = quotedStrings(texts); + // A unit with a record must leave the entries it names as they were; a + // unit without one keeps its changes. if (reasons.length === 0) tables.forEach((table, index) => { if ((contents[index] ?? null)?.changedAmong(names) != null) @@ -330,8 +350,24 @@ export function recordUnit( if (reasons.length > 0) for (const held of contents) held?.refresh(names); for (const held of contents) held?.extend(); } + const reads: Record = {}; + if (reasons.length === 0) + tables.forEach((table, index) => { + if (table.digest === undefined) return; + const read: (readonly [string, string])[] = []; + for (const key of names) { + const digest = table.digest(key, before[index]!); + if (digest !== null) read.push([key, digest]); + } + if (read.length > 0) reads[table.name] = read; + }); if (reasons.length > 0) return { record: null, reasons, fn, before }; - return { record: { name, fn: text, poisoned, events, tables: added }, reasons, fn, before }; + return { + record: { name, fn: text, poisoned, events, tables: added, reads }, + reasons, + fn, + before, + }; } /** Lowers `wave` with every unit recorded. A unit with a record keeps the @@ -539,9 +575,37 @@ export function functionTable( has: (name: string) => boolean, changed: () => void = () => {}, ): JournalTable { + // The position of each function by name, for the functions indexed so far. + let indexes = new Map(); + let indexed = 0; + const lookup = (key: string): number | undefined => { + for (; indexed < functions.length; indexed++) indexes.set(functions[indexed]!.name, indexed); + const index = indexes.get(key); + return index !== undefined && functions[index]?.name === key ? index : undefined; + }; + const indexOf = (key: string): number | undefined => { + if (indexed > functions.length) { + indexes = new Map(); + indexed = 0; + } + const index = lookup(key); + if (index !== undefined || !has(key)) return index; + // A list that was cut and grown again holds other functions at the + // positions indexed before. + indexes = new Map(); + indexed = 0; + return lookup(key); + }; return { name, size: () => functions.length, + digest: (key, before) => { + if (!has(key)) return null; + const index = indexOf(key); + if (index === undefined || index >= before) return null; + const text = encodeUnitFunction(functions[index]!); + return text === null ? null : contentDigest(text); + }, entries: (from) => { const out: string[] = []; for (const fn of functions.slice(from)) { diff --git a/packages/compiler/test/recorded-lowering.test.ts b/packages/compiler/test/recorded-lowering.test.ts index 6c9b176f2..b3a172dc3 100644 --- a/packages/compiler/test/recorded-lowering.test.ts +++ b/packages/compiler/test/recorded-lowering.test.ts @@ -70,3 +70,36 @@ test("two builds of a program write the same records", async () => { await rm(dir, { recursive: true, force: true }); } }, 300_000); + +/** The reasons in a program's records that leave units without one. */ +async function recordReasons(program: string, dir: string): Promise> { + process.env["SCRIPTC_LOWER_JOURNAL_DUMP"] = join(dir, program); + try { + await lower(program, dir, "dump"); + } finally { + delete process.env["SCRIPTC_LOWER_JOURNAL_DUMP"]; + } + const reasons = new Set(); + for (const line of (await readFile(join(dir, program, "records.jsonl"), "utf8")) + .trimEnd() + .split("\n")) + for (const reason of (JSON.parse(line) as { reasons?: string[] }).reasons ?? []) + reasons.add(reason); + return reasons; +} + +test("units that register reachable units or add class fields have no record", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-recorded-lowering-reasons-")); + try { + // A function-local class registers its members as reachable units. + expect(await recordReasons("5945-local-class-observed-methods.ts", dir)).toContain( + "unitRegistrations", + ); + // Object.freeze of a class instance adds the class's integrity field. + expect(await recordReasons("6004-class-instance-integrity.ts", dir)).toContain( + "classLayoutChanges", + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}, 300_000); From 76d08ef73e542583ab8d2c9eac7b8992d0034c1f Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Tue, 27 Oct 2026 11:36:29 -0700 Subject: [PATCH 54/54] Lower function values with extra optional parameters into narrower slots A function value with more parameters than the slot it flows into is assignable in TypeScript exactly when the extra trailing parameters are optional, and a call through the slot passes no value for them, so the callee sees undefined as it does in JavaScript. funcCoerceAdapter refused every such pair, so a callback parameter forwarded into a narrower slot fell through to requireExactShape and failed with SC2003 ("union conversion has no unambiguous layout mapping") even though the checker accepted the program. The adapter now supplies each omitted extra parameter with omittedArgFor (the undefined arm of the parameter's ABI union, or the checked-dynamic undefined). A pair whose extra parameter cannot hold undefined stays refused, and an extra required parameter remains a checker error. Only pairs that previously returned null reach the new code, so programs that compiled before are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../lowering/coercions/function-adapters.ts | 22 ++- .../function-extra-optional-params.test.ts | 162 ++++++++++++++++++ .../function-extra-optional-params.ts.json | 6 + .../corpus/function-extra-optional-params.ts | 145 ++++++++++++++++ .../function-extra-optional-params.ts.json | 1 + 5 files changed, 334 insertions(+), 2 deletions(-) create mode 100644 packages/compiler/test/function-extra-optional-params.test.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/function-extra-optional-params.ts.json create mode 100644 tests/corpus/function-extra-optional-params.ts create mode 100644 tests/harness/corpus-costs/function-extra-optional-params.ts.json diff --git a/packages/compiler/src/frontend/lowering/coercions/function-adapters.ts b/packages/compiler/src/frontend/lowering/coercions/function-adapters.ts index fe3c14605..78c273b66 100644 --- a/packages/compiler/src/frontend/lowering/coercions/function-adapters.ts +++ b/packages/compiler/src/frontend/lowering/coercions/function-adapters.ts @@ -16,6 +16,7 @@ import { import { typeKey } from "../../type-mapper.js"; import { newFnCtx } from "../function-context.js"; import type { Lowerer } from "../lowerer.js"; +import { omittedArgFor } from "../call-arguments.js"; import { maybePromiseUnionOf } from "../maybe-promise.js"; /** Wrap a zero-argument function whose array result needs an element @@ -273,7 +274,22 @@ export function funcCoerceAdapter( if (toT.rest === true && toT.restAbi === undefined && toT.params.length === 0) return funcIntoDynRestAdapter(lowerer, fromT, toT, loc); if (fromT.rest === true || toT.rest === true) return null; - if (fromT.params.length > toT.params.length) return null; + // A source with MORE parameters than the slot: the checker admits it only + // when every extra trailing parameter is optional, so a call through the + // slot hands them undefined (JavaScript ignores surplus arguments). Pass + // that absent value explicitly. A pair whose extra parameter cannot hold + // undefined stays refused. + const omittedParams = new Map(); + for (let i = toT.params.length; i < fromT.params.length; i++) { + const param = fromT.params[i]!; + const absent = + omittedArgFor(lowerer, param, loc) ?? + (typeEquals(param, UNDEFINED_T) + ? ({ kind: "unitLit", unit: "undefined", type: UNDEFINED_T, loc } as IrExpr) + : null); + if (!absent) return null; + omittedParams.set(i, absent); + } // Piece dispositions beyond coercibleValue, all CHECKER-APPROVED // function compatibilities (bivariant method params under the suite's // non-strict settings, `() => never` throwers displayed as void by @@ -290,7 +306,7 @@ export function funcCoerceAdapter( let strandParams = false; const narrowedParams = new Map>(); const partialDynParams = new Map(); - for (let i = 0; i < fromT.params.length; i++) { + for (let i = 0; i < toT.params.length && i < fromT.params.length; i++) { const actual = toT.params[i]!; const expected = fromT.params[i]!; if (lowerer.coercibleValue(actual, expected)) continue; @@ -412,6 +428,8 @@ export function funcCoerceAdapter( ]; } else { const args = fromT.params.map((pt, i) => { + const omitted = omittedParams.get(i); + if (omitted) return omitted; const aRef: IrExpr = { kind: "varRef", localId: `a.${i}`, type: toT.params[i]!, loc }; const partial = partialDynParams.get(i); if (partial && aRef.type.kind === "union") { diff --git a/packages/compiler/test/function-extra-optional-params.test.ts b/packages/compiler/test/function-extra-optional-params.test.ts new file mode 100644 index 000000000..dc8263410 --- /dev/null +++ b/packages/compiler/test/function-extra-optional-params.test.ts @@ -0,0 +1,162 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile, deserializeModule, type ScrDiagnostic } from "../src/index.js"; +import type { IrExpr, IrModule } from "../src/ir/ir.js"; +import { everyStmtList } from "../src/ir/traverse.js"; + +interface Lowered { + ok: boolean; + diagnostics: ScrDiagnostic[]; + module: IrModule | null; +} + +async function lowerSource(source: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "scriptc-fn-param-")); + try { + const entry = join(dir, "main.ts"); + await writeFile(entry, source); + const outPath = join(dir, "main.ir.json"); + const result = await compile(entry, { outPath, outDir: dir, outputKind: "ir" }); + if (!result.ok) return { ok: false, diagnostics: result.diagnostics, module: null }; + return { + ok: true, + diagnostics: [], + module: deserializeModule(await readFile(outPath, "utf8")), + }; + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +/** Every indirect `callValue` in the module, flattened. */ +function callValues(mod: IrModule): IrExpr[] { + const out: IrExpr[] = []; + for (const fn of mod.functions) { + everyStmtList(fn.body, { + expr: (e) => { + if (e.kind === "callValue") out.push(e); + return true; + }, + stmt: () => true, + }); + } + return out; +} + +/** The adapter call that invokes a source function with MORE parameters than + * the destination slot it was converted into. */ +function extraParamAdapterCalls(mod: IrModule) { + return callValues(mod).filter( + (e): e is IrExpr & { kind: "callValue" } => + e.kind === "callValue" && + e.callee.type.kind === "func" && + !e.callee.type.rest && + e.callee.type.params.length === 2 && + e.args.length === 2, + ); +} + +test("an extra optional parameter receives undefined through the adapter", async () => { + const { ok, diagnostics, module } = await lowerSource(` +type Slot = (value: string) => void; +type Source = (value: string | undefined, extra?: number) => void; + +function take(callback: Slot): void { + callback("x"); +} + +function forward(callback: Source): void { + take(callback); +} + +forward((value, extra) => { + console.log(value, extra); +}); +export {}; +`); + expect(diagnostics).toEqual([]); + expect(ok).toBe(true); + if (!module) return; + + const adapted = extraParamAdapterCalls(module); + expect(adapted).toHaveLength(1); + const call = adapted[0]!; + if (call.kind !== "callValue") return; + + // The slot's `string` parameter still widens into the source's + // `string | undefined`. + expect(call.args[0]!.kind).toBe("unionWrap"); + expect(call.args[0]!.type.kind).toBe("union"); + // The absent extra parameter is the undefined arm of its `number | + // undefined` ABI union, wrapping a unit `undefined`. + expect(call.args[1]).toMatchObject({ + kind: "unionWrap", + value: { kind: "unitLit", unit: "undefined" }, + }); + expect(call.args[1]!.type.kind).toBe("union"); +}); + +test("an extra checked-dynamic optional parameter receives undefined", async () => { + const { ok, diagnostics, module } = await lowerSource(` +function take(callback: (value: string) => void): void { + callback("x"); +} + +function forward(callback: (value: string, meta?: any) => void): void { + take(callback); +} + +forward((value, meta) => { + console.log(value, meta === undefined); +}); +export {}; +`); + expect(diagnostics).toEqual([]); + expect(ok).toBe(true); + if (!module) return; + + const adapted = extraParamAdapterCalls(module); + expect(adapted).toHaveLength(1); + const call = adapted[0]!; + if (call.kind !== "callValue") return; + expect(call.args[1]!.type.kind).toBe("dyn"); +}); + +test("an extra REQUIRED parameter stays a checker refusal", async () => { + const { ok, diagnostics } = await lowerSource(` +function take(callback: (value: string) => void): void { + callback("x"); +} + +function two(value: string, count: number): void { + console.log(value, count); +} + +take(two); +export {}; +`); + expect(ok).toBe(false); + expect(diagnostics.map((d) => d.code)).toContain("SC0001"); +}); + +test("a typed-rest source stays served by the rest-packing adapter", async () => { + // The typed-rest source is admitted by TypeScript (the rest may be empty) + // and already lowering through the rest-packing adapter, not the + // extra-parameter path this fix touches. + const { ok, diagnostics } = await lowerSource(` +function take(callback: (value: string) => void): void { + callback("x"); +} + +function rest(value: string, ...others: number[]): void { + console.log(value, others.length); +} + +take(rest); +export {}; +`); + expect(diagnostics).toEqual([]); + expect(ok).toBe(true); +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/function-extra-optional-params.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/function-extra-optional-params.ts.json new file mode 100644 index 000000000..807bcce66 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/function-extra-optional-params.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/function-extra-optional-params.ts" + ], + "diags": [] +} diff --git a/tests/corpus/function-extra-optional-params.ts b/tests/corpus/function-extra-optional-params.ts new file mode 100644 index 000000000..016efc82c --- /dev/null +++ b/tests/corpus/function-extra-optional-params.ts @@ -0,0 +1,145 @@ +// A function value with MORE parameters than the slot it flows into: +// TypeScript admits the assignment when the extra trailing parameters are +// optional, and a JavaScript caller simply leaves them undefined. The +// lowering forwards the slot's arguments and supplies undefined for the +// extras. The program also covers both legal parameter-variance directions +// for `string | undefined` and `string` (the widening function-type +// direction and the narrowing bivariant method-parameter direction; the +// narrowing function-type direction is a TypeScript error, not a scriptc +// one). +// +// The converted values are callback PARAMETERS (the shape a library hands +// to another slot), because a fresh function literal with optional +// parameters carries its own completed signature at birth. + +type Done = (selectedValue?: string) => void; +type DoneWithOptions = ( + selectedValue?: string, + options?: { navigateTo?: string }, +) => void; + +function doneString(label: string, done: Done): void { + done(label); +} + +function doneNothing(run: Done): void { + run(); +} + +function donePlain(callback: (value: string) => void): void { + callback("plain"); +} + +// 1. A callback parameter with an extra optional record parameter forwarded +// into a one-parameter slot, invoked with and without an argument. +function forwardOptions(done: DoneWithOptions): void { + doneString("one", done); + doneNothing(done); +} +forwardOptions((selectedValue, options) => { + console.log( + "forwarded", + selectedValue ?? "none", + options === undefined ? "absent" : (options.navigateTo ?? "no-target"), + ); +}); + +// 2. The same value created under its own two-parameter type, then named and +// forwarded later. +const named: DoneWithOptions = function (selectedValue, options) { + console.log( + "named", + selectedValue ?? "none", + options === undefined ? "absent" : (options.navigateTo ?? "no-target"), + ); +}; +forwardOptions(named); + +// 3. Several extra optional parameters of different representations. +type Wide = (value: string, count?: number, name?: string, options?: { flag?: boolean }) => void; + +function forwardWide(callback: Wide): void { + donePlain(callback); +} +forwardWide((value, count, name, options) => { + console.log("many", value, count === undefined, name === undefined, options === undefined); +}); + +// 4. The slot's parameter also needs a widening conversion into the callee's +// `string | undefined`. +type WidenExtra = (value: string | undefined, extra?: number) => void; + +function forwardWidenExtra(callback: WidenExtra): void { + donePlain(callback); +} +forwardWidenExtra((value, extra) => { + console.log("widen-extra", value ?? "none", extra === undefined ? "no-extra" : extra); +}); + +// 5. An object field holding the converted callback. +interface Settings { + onApply: (value: string) => void; +} +function makeSettings(callback: (value: string, count?: number) => void): Settings { + return { onApply: callback }; +} +const settings = makeSettings((value, count) => { + console.log("field", value, count === undefined ? "no-count" : count); +}); +settings.onApply("five"); + +// 6. An optional callback field. +interface OptionalSlot { + onDone?: (value?: string) => void; +} +function makeOptional(callback: (value: string | undefined, target?: string) => void): OptionalSlot { + return { onDone: callback }; +} +const optional = makeOptional((value, target) => { + console.log("optional", value ?? "none", target === undefined ? "no-target" : target); +}); +optional.onDone?.("six"); + +// 7. Class methods passed as callbacks: one widens its parameter to the +// slot's `string`, the other has an extra optional parameter. +class Handler { + widen(value: string | undefined): void { + console.log("method-widen", value ?? "none"); + } + extra(value: string, detail?: { level: number }): void { + console.log("method-extra", value, detail === undefined ? "no-detail" : detail.level); + } +} +const handler = new Handler(); +donePlain(handler.widen); +donePlain(handler.extra); + +// 8. An extra optional checked-dynamic parameter. +function withAny(value: string, meta?: any): void { + console.log("any-extra", value, meta === undefined ? "absent" : "present"); +} +donePlain(withAny); + +// 9. The narrowing parameter direction in a bivariant method-parameter +// position: a `(value: string) => void` implementation in a slot declared +// `(value: string | undefined) => void`. +interface Options { + onChange(value: string | undefined): void; +} +const narrowing: Options = { + onChange: (value: string) => { + console.log("narrow", value); + }, +}; +narrowing.onChange("nine"); + +// 10. The control that already worked: fewer source parameters than the +// slot, whose surplus arguments JavaScript ignores. +function takesThree(callback: (a: string, b: number, c: boolean) => void): void { + callback("fewer", 1, true); +} +takesThree((a: string) => { + console.log("fewer", a); +}); + +export {}; diff --git a/tests/harness/corpus-costs/function-extra-optional-params.ts.json b/tests/harness/corpus-costs/function-extra-optional-params.ts.json new file mode 100644 index 000000000..0301bf275 --- /dev/null +++ b/tests/harness/corpus-costs/function-extra-optional-params.ts.json @@ -0,0 +1 @@ +{ "plain": 6048, "san": 122291 }