From 1e2f9b2dfcbe82867161d745b8a2d47c053599c8 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 23:52:50 -0500 Subject: [PATCH 01/20] Resolve computed createRequire specifiers at run time In JavaScript dependencies, a createRequire require call with a computed specifier, and require.resolve called with a computed request or read as a value, now compile. The new runtime unit scr_require.c resolves the specifier with Node 24's CommonJS algorithm (builtins, relative and absolute paths with Node's extension search, package "main", "exports" and self-reference, node_modules and NODE_PATH lookup) when the call runs: - require.resolve returns the id or real path Node returns, and throws Node's MODULE_NOT_FOUND error when nothing resolves. - require throws the same MODULE_NOT_FOUND error when nothing resolves, so packages that probe for optional addons, such as pi-tui, take their absent branch as in Node. When the specifier resolves to a native addon, a builtin or another file, Node would load that module: the program prints an SC2020 error naming it and exits with status 1, without running the catch clause around the call. Coverage lists these calls as deferred to runtime. TypeScript sources keep the compile-time refusal. Refs #1777. --- docs/content/limitations/native-addons.mdx | 10 + .../src/backend/executable-features.ts | 1 + .../compiler/src/backend/llvm/lib-shared.ts | 2 + .../src/backend/native-link-info-core.ts | 3 + .../compiler/src/backend/native/contracts.ts | 3 + .../compiler/src/backend/native/executable.ts | 2 + .../compiler/src/backend/runtime-pack-core.ts | 2 + .../compiler/src/diagnostics/diagnostic.ts | 15 + .../src/executable/executable-cache.ts | 1 + .../lowering/builtins/module-resolution.ts | 158 +++- .../lowering/expressions/identifiers.ts | 8 +- .../lowering/expressions/property-access.ts | 14 + .../src/frontend/lowering/import-meta.ts | 13 +- packages/compiler/src/frontend/program.ts | 63 +- packages/compiler/src/index.ts | 1 + packages/compiler/src/ir/builtin-effects.ts | 2 + .../compiler/src/ir/builtin-signatures.ts | 12 + packages/compiler/src/ir/runtime-features.ts | 4 + .../compiler/src/native/sanitizer.test.ts | 1 + .../main.ts.json | 6 + .../main.ts.json | 6 + .../main.ts.json | 6 + .../runtime-pack-matrix.mjs | 2 + packages/runtime/src/scr_path.h | 19 +- packages/runtime/src/scr_require.c | 895 ++++++++++++++++++ .../8210-require-computed-missing/main.ts | 7 + .../node_modules/optional-probe/index.d.ts | 2 + .../node_modules/optional-probe/index.js | 46 + .../node_modules/optional-probe/package.json | 1 + .../8211-require-resolve-computed/main.ts | 7 + .../node_modules/@scope/scoped/lib/node.js | 1 + .../node_modules/@scope/scoped/lib/other.js | 1 + .../node_modules/@scope/scoped/package.json | 1 + .../node_modules/plain-index/index.js | 1 + .../node_modules/resolve-probe/data.json | 1 + .../node_modules/resolve-probe/index.d.ts | 1 + .../node_modules/resolve-probe/index.js | 50 + .../node_modules/resolve-probe/package.json | 1 + .../node_modules/with-exports/cjs.cjs | 1 + .../node_modules/with-exports/esm.mjs | 1 + .../node_modules/with-exports/lib/feature.js | 1 + .../node_modules/with-exports/lib/helper.js | 1 + .../node_modules/with-exports/main.js | 1 + .../node_modules/with-exports/package.json | 1 + .../node_modules/with-main/lib/entry.js | 1 + .../node_modules/with-main/package.json | 1 + .../main.ts | 18 + .../native-helper/dist/index.d.ts | 11 + .../node_modules/native-helper/dist/index.js | 2 + .../native-helper/dist/native-module-path.js | 20 + .../native-helper/dist/native-platform.js | 48 + .../linux-arm64/linux-helper-x11.node | 1 + .../prebuilds/linux-x64/linux-helper-x11.node | 1 + .../node_modules/native-helper/package.json | 1 + tests/fixtures/npm-static/addon-probe-cli.ts | 5 + .../npm-static/addon-probe-installed-cli.ts | 4 + .../node_modules/addon-probe/index.d.ts | 3 + .../node_modules/addon-probe/index.js | 38 + .../node_modules/addon-probe/installed.js | 1 + .../addon-probe/native/installed.node | 1 + .../node_modules/addon-probe/package.json | 7 + .../main.ts.json | 1 + .../main.ts.json | 1 + .../main.ts.json | 1 + tests/harness/npm-static.test.ts | 92 +- 65 files changed, 1597 insertions(+), 36 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8210-require-computed-missing/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8211-require-resolve-computed/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8212-native-helper-probe-no-display/main.ts.json create mode 100644 packages/runtime/src/scr_require.c create mode 100644 tests/corpus/8210-require-computed-missing/main.ts create mode 100644 tests/corpus/8210-require-computed-missing/node_modules/optional-probe/index.d.ts create mode 100644 tests/corpus/8210-require-computed-missing/node_modules/optional-probe/index.js create mode 100644 tests/corpus/8210-require-computed-missing/node_modules/optional-probe/package.json create mode 100644 tests/corpus/8211-require-resolve-computed/main.ts create mode 100644 tests/corpus/8211-require-resolve-computed/node_modules/@scope/scoped/lib/node.js create mode 100644 tests/corpus/8211-require-resolve-computed/node_modules/@scope/scoped/lib/other.js create mode 100644 tests/corpus/8211-require-resolve-computed/node_modules/@scope/scoped/package.json create mode 100644 tests/corpus/8211-require-resolve-computed/node_modules/plain-index/index.js create mode 100644 tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/data.json create mode 100644 tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/index.d.ts create mode 100644 tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/index.js create mode 100644 tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/package.json create mode 100644 tests/corpus/8211-require-resolve-computed/node_modules/with-exports/cjs.cjs create mode 100644 tests/corpus/8211-require-resolve-computed/node_modules/with-exports/esm.mjs create mode 100644 tests/corpus/8211-require-resolve-computed/node_modules/with-exports/lib/feature.js create mode 100644 tests/corpus/8211-require-resolve-computed/node_modules/with-exports/lib/helper.js create mode 100644 tests/corpus/8211-require-resolve-computed/node_modules/with-exports/main.js create mode 100644 tests/corpus/8211-require-resolve-computed/node_modules/with-exports/package.json create mode 100644 tests/corpus/8211-require-resolve-computed/node_modules/with-main/lib/entry.js create mode 100644 tests/corpus/8211-require-resolve-computed/node_modules/with-main/package.json create mode 100644 tests/corpus/8212-native-helper-probe-no-display/main.ts create mode 100644 tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/index.d.ts create mode 100644 tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/index.js create mode 100644 tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/native-module-path.js create mode 100644 tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/native-platform.js create mode 100644 tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/native/linux/prebuilds/linux-arm64/linux-helper-x11.node create mode 100644 tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/native/linux/prebuilds/linux-x64/linux-helper-x11.node create mode 100644 tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/package.json create mode 100644 tests/fixtures/npm-static/addon-probe-cli.ts create mode 100644 tests/fixtures/npm-static/addon-probe-installed-cli.ts create mode 100644 tests/fixtures/npm-static/node_modules/addon-probe/index.d.ts create mode 100644 tests/fixtures/npm-static/node_modules/addon-probe/index.js create mode 100644 tests/fixtures/npm-static/node_modules/addon-probe/installed.js create mode 100644 tests/fixtures/npm-static/node_modules/addon-probe/native/installed.node create mode 100644 tests/fixtures/npm-static/node_modules/addon-probe/package.json create mode 100644 tests/harness/corpus-costs/8210-require-computed-missing/main.ts.json create mode 100644 tests/harness/corpus-costs/8211-require-resolve-computed/main.ts.json create mode 100644 tests/harness/corpus-costs/8212-native-helper-probe-no-display/main.ts.json diff --git a/docs/content/limitations/native-addons.mdx b/docs/content/limitations/native-addons.mdx index 469d1cdd2..1df9470cd 100644 --- a/docs/content/limitations/native-addons.mdx +++ b/docs/content/limitations/native-addons.mdx @@ -2,5 +2,15 @@ Node-API (N-API) and V8 `.node` addons depend on Node.js itself, so compiled pro Direct `createRequire` calls to local addons are rejected at compile time. A package that requires an addon needs a JavaScript implementation that compiles, or a replacement through native FFI. +A package that looks for an optional addon at run time compiles with `--npm-static`. +In JavaScript dependencies, a `createRequire` require call with a computed specifier, and `require.resolve` called with a computed request or read as a value, resolve the specifier with the Node.js algorithm when the program runs. +`require.resolve` returns the path Node.js returns. +When nothing resolves, both throw the `MODULE_NOT_FOUND` error that Node.js throws, so a package that probes for a missing addon takes the same branch as in Node.js. +When the specifier of a require call resolves to a native addon, a builtin, or a file that is not part of the compiled program, the program prints an SC2020 error that names it and exits with status 1, because Node.js would load that module. +A `catch` clause around the call does not run in that case. +Resolution steps that the runtime does not reproduce, such as package `imports` specifiers, `require.resolve` options, and subpaths that a package does not export, end the program the same way. +Coverage lists these calls as deferred to runtime. +This applies until compiled programs can load addons through Node.js. + Use [Native FFI](/docs/ffi#replacing-a-node-api-addon) to call a plain C ABI function linked from an object or static archive. The guide includes a replacement for a small Node-API helper. diff --git a/packages/compiler/src/backend/executable-features.ts b/packages/compiler/src/backend/executable-features.ts index a9076af22..0eae21565 100644 --- a/packages/compiler/src/backend/executable-features.ts +++ b/packages/compiler/src/backend/executable-features.ts @@ -33,6 +33,7 @@ export function executableLinkFeatures(mod: IrModule): NativeLinkFeatures { ...(features.eventsOnceSignal ? { eventsOnceSignal: true } : {}), ...(features.dynBuffer ? { dynBuffer: true } : {}), ...(features.builtinValues ? { builtinValues: true } : {}), + ...(features.computedRequire ? { computedRequire: true } : {}), events: features.processEvents, emitter: features.emitter, symbol: features.symbol, diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 014191314..49d0bed68 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -270,6 +270,8 @@ export const LIB_FN_SYMS: Record = { "module.id": "scr_module_id", "module.path": "scr_module_path", "module.paths": "scr_module_paths", + "module.requireComputed": "scr_require_computed", + "module.resolveComputed": "scr_require_resolve", "module.children": "scr_module_children", "module.parent": "scr_module_parent", "module.loaded": "scr_module_loaded", diff --git a/packages/compiler/src/backend/native-link-info-core.ts b/packages/compiler/src/backend/native-link-info-core.ts index c073bb9a6..3ff6b0423 100644 --- a/packages/compiler/src/backend/native-link-info-core.ts +++ b/packages/compiler/src/backend/native-link-info-core.ts @@ -62,6 +62,9 @@ export interface NativeLinkFeatures { /** Builtin functions read as values (Buffer.byteLength, Buffer.alloc, * fs.realpathSync) link scr_builtin_values.c; absent (false) otherwise. */ builtinValues?: boolean; + /** createRequire's computed require and require.resolve link + * scr_require.c; absent (false) otherwise. */ + computedRequire?: boolean; events: boolean; emitter: boolean; symbol: boolean; diff --git a/packages/compiler/src/backend/native/contracts.ts b/packages/compiler/src/backend/native/contracts.ts index 53e160984..6606745a3 100644 --- a/packages/compiler/src/backend/native/contracts.ts +++ b/packages/compiler/src/backend/native/contracts.ts @@ -131,6 +131,9 @@ export interface CcOptions { /** Buffer.prototype methods called on checked-dynamic values: * compiles scr_bytes_dyn.c. */ dynBuffer?: boolean; + /** createRequire's computed require and require.resolve: compiles + * scr_require.c. */ + computedRequire?: boolean; /** Builtin functions read as values: compiles scr_builtin_values.c. */ builtinValues?: boolean; /** The program constructs EventTarget/Event/CustomEvent values or diff --git a/packages/compiler/src/backend/native/executable.ts b/packages/compiler/src/backend/native/executable.ts index 1578a0c80..8fde247a7 100644 --- a/packages/compiler/src/backend/native/executable.ts +++ b/packages/compiler/src/backend/native/executable.ts @@ -186,6 +186,8 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [ unit((u) => u.opts.dynBuffer === true && u.opts.dynInvoke, "scr_bytes_dyn.c"), // Builtin functions read as values in JavaScript dependencies. unit((u) => u.opts.builtinValues === true, "scr_builtin_values.c"), + // createRequire's computed require and require.resolve. + unit((u) => u.opts.computedRequire === true, "scr_require.c"), unit((u) => u.opts.dc, "scr_dc.c"), unit((u) => u.opts.dynTier, "scr_js_value.c", "scr_js_object.c"), unit((u) => u.opts.eventTarget || u.nativeFetch, "scr_event_target.c"), diff --git a/packages/compiler/src/backend/runtime-pack-core.ts b/packages/compiler/src/backend/runtime-pack-core.ts index 75953be59..4b8d20bac 100644 --- a/packages/compiler/src/backend/runtime-pack-core.ts +++ b/packages/compiler/src/backend/runtime-pack-core.ts @@ -91,6 +91,7 @@ export interface RuntimeFeatureSet extends NativeLinkFeatures { eventsOnceSignal: boolean; dynBuffer: boolean; builtinValues: boolean; + computedRequire: boolean; warningReplace: boolean; stdioExtended: boolean; nativeFetch: boolean; @@ -289,6 +290,7 @@ export function effectiveRuntimeFeatures( eventsOnceSignal: features.eventsOnceSignal === true, dynBuffer: features.dynBuffer === true, builtinValues: features.builtinValues === true, + computedRequire: features.computedRequire === true, warningReplace: features.warningReplace === true, stdioExtended: features.stdioExtended === true, nativeFetch, diff --git a/packages/compiler/src/diagnostics/diagnostic.ts b/packages/compiler/src/diagnostics/diagnostic.ts index 2fd8b6fb0..4579f637f 100644 --- a/packages/compiler/src/diagnostics/diagnostic.ts +++ b/packages/compiler/src/diagnostics/diagnostic.ts @@ -776,6 +776,21 @@ export function nativeAddonDiag(specifier: string, loc: SrcLoc): ScrDiagnostic { }; } +/** A createRequire require or require.resolve call whose specifier is + * computed at run time, where the runtime resolves it like Node + * (scr_require.c). Coverage lists the call as deferred: a require whose + * specifier resolves to a module ends the program with this code, because + * the module graph is fixed at build time. */ +export function computedRequireDiag(surface: string, loc: SrcLoc): ScrDiagnostic { + return { + code: "SC2020", + message: `${surface} with a computed specifier resolves at run time; a require of a specifier that resolves to a module, such as a native addon, ends the program`, + loc, + milestone: "later", + hint: 'require("") compiles the module into the program; native addons need a replacement through native FFI (https://scriptc.dev/ffi#replacing-a-node-api-addon)', + }; +} + /** Records are monomorphic structs, so a value's shape must match the * expected shape exactly OR narrow through the width-copy family * (recordWidthPlan / the overflow capture: target fields copied off the diff --git a/packages/compiler/src/executable/executable-cache.ts b/packages/compiler/src/executable/executable-cache.ts index 50f783761..a82fe967f 100644 --- a/packages/compiler/src/executable/executable-cache.ts +++ b/packages/compiler/src/executable/executable-cache.ts @@ -76,6 +76,7 @@ export interface EarlyExecutableNativeFeatures { eventsOnceSignal?: boolean; dynBuffer?: boolean; builtinValues?: boolean; + computedRequire?: boolean; events: boolean; emitter: boolean; symbol: boolean; diff --git a/packages/compiler/src/frontend/lowering/builtins/module-resolution.ts b/packages/compiler/src/frontend/lowering/builtins/module-resolution.ts index 6b2e9e589..f00e78b0a 100644 --- a/packages/compiler/src/frontend/lowering/builtins/module-resolution.ts +++ b/packages/compiler/src/frontend/lowering/builtins/module-resolution.ts @@ -1,8 +1,8 @@ -import { nodeThrowExpr, boolLit, strLit, varRef } from "../../../ir/build.js"; +import { dynUndefinedExpr, nodeThrowExpr, boolLit, strLit, varRef } from "../../../ir/build.js"; import { dirname, isAbsolute, resolve } from "node:path"; import * as ts from "../../ts7/adapter.js"; import { type Lowerer, PoisonError } from "../lowerer.js"; -import { canonicalBuiltinModule, isNodeEsmFile, locOf } from "../../program.js"; +import { canonicalBuiltinModule, isJsSourceFile, isNodeEsmFile, locOf } from "../../program.js"; import { isRelativeSpecifier } from "../../workspace-registry.js"; import { probeNodeRequireRefusal } from "../../npm.js"; import { isNpmStaticPackage } from "../../npm-static.js"; @@ -15,21 +15,25 @@ import { type RuntimeResolveResult, } from "../../runtime-resolve.js"; import { + computedRequireDiag, invalidJsonModuleDiag, nativeAddonDiag, unsupportedPackageImportDiag, } from "../../../diagnostics/diagnostic.js"; -import { NODE_BUILTIN_MODULES_V24 } from "../surfaces.js"; +import { NODE_BUILTIN_MODULES_V24, builtinModulesArrayLit } from "../surfaces.js"; +import { moduleFileName } from "../import-meta.js"; import { BOOL, DYN, type IrExpr, + type IrStmt, type IrType, NULL_T, STRING, type SrcLoc, UNDEFINED_T, arrayOf, + funcOf, } from "../../../ir/ir.js"; import { lowerFfiMemoryModule } from "../native-ffi.js"; import { @@ -38,6 +42,7 @@ import { createRequireCalleeFileOf, } from "./module-bindings.js"; import { stripTypeCasts } from "./arguments.js"; +import { keyedHelperName } from "../stable-names.js"; /** node:module's two compiler-only calls. isBuiltin compares one evaluated * string against the pinned Node 24 list (bare builtins also accept their @@ -134,6 +139,102 @@ export function lowerNodeModuleCall( }; } +/** The run-time arguments of scr_require.c's computed require and + * require.resolve after the specifier: the requiring file (Node's require + * stack names it), the call's "file:line" for the SC2020 refusal, and the + * builtin ids. The call joins the runtime-fence ledger, so coverage lists + * it as deferred to runtime. */ +function computedRequireArgs( + lowerer: Lowerer, + surface: string, + node: ts.Node, + baseFile: ts.SourceFile, +): IrExpr[] { + const loc = locOf(node); + lowerer.runtimeFences.push(computedRequireDiag(surface, loc)); + const line = ts.getLineAndCharacterOfPosition(ts.getSourceFileOfNode(node), loc.start).line + 1; + return [ + strLit(moduleFileName(lowerer, baseFile), loc), + strLit(`${loc.file}:${line}`, loc), + builtinModulesArrayLit(loc), + ]; +} + +/** A createRequire loader's require.resolve with a computed request, in + * JavaScript source: Node's resolution at run time (scr_require.c), with + * the options argument checked there. */ +function computedResolveCall( + lowerer: Lowerer, + node: ts.Node, + baseFile: ts.SourceFile, + request: IrExpr, + options: IrExpr, + loc: SrcLoc, +): IrExpr { + return { + kind: "libCall", + fn: "module.resolveComputed", + args: [ + request, + options, + ...computedRequireArgs(lowerer, "createRequire's require.resolve", node, baseFile), + ], + type: STRING, + loc, + }; +} + +/** A createRequire loader's `resolve` read as a value in JavaScript source + * (passed on, or picked by `??`): a function that resolves its request at + * run time, so `(options.resolve ?? require.resolve)(id)` behaves as in + * Node. */ +export function createRequireResolveValue( + lowerer: Lowerer, + access: ts.PropertyAccessExpression, + loc: SrcLoc, +): IrExpr { + const baseFile = createRequireCalleeFileOf(lowerer, access.expression); + if (baseFile === null || !isJsSourceFile(ts.getSourceFileOfNode(access))) { + lowerer.noLowering( + "createRequire's require.resolve as a value", + access, + 'call it with a string literal (require.resolve("")), which resolves at build time', + ); + } + const name = keyedHelperName(lowerer, "%createRequire.resolve", `${loc.file}:${loc.start}`); + if (!lowerer.liftedFns.some((fn) => fn.name === name)) { + const call = computedResolveCall( + lowerer, + access, + baseFile, + varRef("request", DYN, loc), + varRef("options", DYN, loc), + loc, + ); + lowerer.liftedFns.push({ + name, + params: [ + { localId: "request", name: "request", type: DYN }, + { localId: "options", name: "options", type: DYN }, + ], + locals: [ + { id: "request", name: "request", type: DYN, mutable: false }, + { id: "options", name: "options", type: DYN, mutable: false }, + ], + returnType: STRING, + body: [{ kind: "return", value: call, loc }], + loc, + }); + } + return { + kind: "dynFrom", + value: { kind: "closure", fnName: name, captures: [], type: funcOf([DYN, DYN], STRING), loc }, + fnName: access.name.text, + type: DYN, + loc, + }; +} + export function lowerCreateRequireCall( lowerer: Lowerer, call: ts.CallExpression, @@ -142,10 +243,35 @@ export function lowerCreateRequireCall( const cr = createRequireSpecOf(lowerer, call); if (cr === null) return null; if (cr.spec === null) { - lowerer.noLowering( - "createRequire's require with this argument shape", + const args = call.arguments.nodes; + if (!isJsSourceFile(ts.getSourceFileOfNode(call)) || args.some((a) => ts.isSpreadElement(a))) { + lowerer.noLowering( + "createRequire's require with this argument shape", + call, + 'the compiled module graph is fixed at build time — the one lowered form is require("")', + ); + } + // Node evaluates every argument, then requires the first. + const stmts: IrStmt[] = []; + let id = args.length > 0 ? lowerer.lowerExprExpecting(args[0]!, DYN) : dynUndefinedExpr(loc); + if (args.length > 1) { + const slot = lowerer.declareHiddenLocal("%require.id", DYN); + stmts.push({ kind: "varDecl", localId: slot.id, init: id, loc }); + for (const arg of args.slice(1)) + stmts.push({ kind: "exprStmt", expr: lowerer.lowerExpr(arg), loc: locOf(arg) }); + id = varRef(slot.id, DYN, loc); + } + const require: IrExpr = { + kind: "libCall", + fn: "module.requireComputed", + args: [id, ...computedRequireArgs(lowerer, "createRequire's require", call, cr.baseFile)], + type: DYN, + loc, + }; + return lowerer.coerceInto( call, - 'the compiled module graph is fixed at build time — the one lowered form is require("")', + stmts.length === 0 ? require : { kind: "seqExpr", stmts, result: require, type: DYN, loc }, + lowerer.irTypeOf(call), ); } const spec = cr.spec; @@ -380,6 +506,26 @@ export function lowerRequireResolveCall(lowerer: Lowerer, call: ts.CallExpressio ); } const specifier = staticString(call.arguments.nodes[0]); + // A computed request (or options) through a createRequire loader in + // JavaScript source resolves at run time. + if ( + (specifier === null || staticResolveOptionPaths(call.arguments.nodes[1]) === null) && + !pathsCall && + createRequireCalleeFileOf(lowerer, receiver) !== null && + isJsSourceFile(ts.getSourceFileOfNode(call)) + ) { + const [request, options] = call.arguments.nodes; + return computedResolveCall( + lowerer, + call, + baseFile, + lowerer.lowerExprExpecting(request!, DYN), + options === undefined + ? dynUndefinedExpr(locOf(call)) + : lowerer.lowerExprExpecting(options, DYN), + locOf(call), + ); + } if (specifier === null) { lowerer.noLowering( `${pathsCall ? "require.resolve.paths" : "require.resolve"} with a runtime-computed request`, diff --git a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts index f7ba7a115..159873630 100644 --- a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts +++ b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts @@ -88,7 +88,7 @@ import { lowerGlobalValue } from "../lower-global-value.js"; import { fenceEnumObjectValue } from "../lower-enums.js"; import { mixinFnOfCallee } from "../lower-mixins.js"; import { isNavigatorGlobal, refuseNavigator } from "../builtins/navigator.js"; -import { wasiGuestPath } from "../../../wasi-paths.js"; +import { moduleFileName } from "../import-meta.js"; export function lowerIdentifierExpr(lowerer: Lowerer, expr: ts.Identifier, loc: SrcLoc): IrExpr { if (lowerer.isStdlibGlobal(expr, "console")) { @@ -1014,12 +1014,6 @@ export function lowerIdentifierExpr(lowerer: Lowerer, expr: ts.Identifier, loc: ); } -function moduleFileName(lowerer: Lowerer, sf: ts.SourceFile): string { - return lowerer.targetPlatform === "wasi" - ? (wasiGuestPath(sf.fileName) ?? sf.fileName.replace(/\\/g, "/")) - : sf.fileName; -} - /** A computed-key access (`value[key]` after `typeof value === "object"`) * of a runtime-optional binding the checker narrowed to one record arm of a * wider union. The binding re-tags to that record or undefined, so the keyed diff --git a/packages/compiler/src/frontend/lowering/expressions/property-access.ts b/packages/compiler/src/frontend/lowering/expressions/property-access.ts index 1a4883e5a..a26caf4d9 100644 --- a/packages/compiler/src/frontend/lowering/expressions/property-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/property-access.ts @@ -146,6 +146,8 @@ import { nullishGenericBindingUnitOf } from "../binding-analysis.js"; import { blockedBindingUseDiag } from "../../../diagnostics/diagnostic.js"; import { npmStaticPackageOfPath } from "../../npm-static.js"; import { importMetaField, isImportMeta } from "../import-meta.js"; +import { createRequireCalleeFileOf } from "../builtins/module-bindings.js"; +import { createRequireResolveValue } from "../builtins/module-resolution.js"; import { builtinPrototypeMethod, lowerArrayIsArrayValue, @@ -286,6 +288,14 @@ const DGRAM_METHOD_VALUES = new Set([ "unref", ]); +function isCreateRequireResolveValue(lowerer: Lowerer, expr: ts.PropertyAccessExpression): boolean { + return ( + expr.name.text === "resolve" && + !expr.questionDotToken && + createRequireCalleeFileOf(lowerer, expr.expression) !== null + ); +} + export function lowerPropertyAccessExpr( lowerer: Lowerer, expr: ts.PropertyAccessExpression, @@ -1275,6 +1285,10 @@ export function lowerPropertyAccessExpr( return declared && declared.kind !== "void" ? lowerer.coerceToExpected(read, declared) : read; } } + // A createRequire loader's resolve read as a value (passed on, or + // picked by `??`): a function that resolves its request at run time. + if (isCreateRequireResolveValue(lowerer, expr)) + return createRequireResolveValue(lowerer, expr, loc); // The lib fence's PROPERTY chokepoint: a stdlib-declared member that // no lowering above claimed ([1,2].entries, Math.SQRT2, Promise.all, // re.exec as a value, ...) reports SC2020 here. diff --git a/packages/compiler/src/frontend/lowering/import-meta.ts b/packages/compiler/src/frontend/lowering/import-meta.ts index 796e64b69..ccc86bfba 100644 --- a/packages/compiler/src/frontend/lowering/import-meta.ts +++ b/packages/compiler/src/frontend/lowering/import-meta.ts @@ -17,14 +17,19 @@ export function isImportMeta(expr: ts.Expression): boolean { ); } +/** The module's file name as the compiled program reports it + * (import.meta.filename): the WASI guest path on wasi targets. */ +export function moduleFileName(lowerer: Lowerer, sf: ts.SourceFile): string { + return lowerer.targetPlatform === "wasi" + ? (wasiGuestPath(sf.fileName) ?? sf.fileName.replace(/\\/g, "/")) + : sf.fileName; +} + /** File-backed metadata is fixed by the compiled module graph. Reads share * one implementation across dot access, folded keys and selected bindings. */ export function importMetaField(lowerer: Lowerer, node: ts.Node, name: string): IrExpr { const sf = ts.getSourceFileOfNode(node); - const fileName = - lowerer.targetPlatform === "wasi" - ? (wasiGuestPath(sf.fileName) ?? sf.fileName.replace(/\\/g, "/")) - : sf.fileName; + const fileName = moduleFileName(lowerer, sf); const loc = locOf(node); switch (name) { case "env": diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index 8075379b8..517b9accf 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -2519,12 +2519,29 @@ export function isCreateRequireBinding7(program: ts.Program, callee: ts.Identifi ); } -/** Only literal calls and literal resolution queries can erase the loader - * value. In particular, assignments, exported aliases, shorthand properties, - * optional calls, and passing the loader to user code cannot pass this test. */ -function staticCreateRequireUse7(expr: ts.Expression): boolean { +/** How a use of a createRequire loader lowers: + * - "static": a literal call or a literal resolution query, which erases + * the loader value. + * - "runtime": in JavaScript source, a call with a computed specifier, a + * require.resolve call with a computed request, or `resolve` read as a + * value. The runtime resolves the specifier like Node + * (module-resolution.ts, scr_require.c). + * - "computed": the same uses in TypeScript source. + * - "escaping": the loader itself as a value, writes, optional calls, + * spread arguments, `resolve.paths` with a computed request or as a + * value, and other members. */ +type CreateRequireUse7 = "static" | "runtime" | "computed" | "escaping"; + +function createRequireUse7(expr: ts.Expression): CreateRequireUse7 { + const use = createRequireUseShape7(expr); + return use === "runtime" && !isJsSourceFile(ts.getSourceFileOfNode(expr)) ? "computed" : use; +} + +function createRequireUseShape7(expr: ts.Expression): CreateRequireUse7 { let use: ts.Node = expr; while (ts.isParenthesizedExpression(use.parent)) use = use.parent; + let member = false; + let paths = false; if ( ts.isPropertyAccessExpression(use.parent) && use.parent.expression === use && @@ -2532,21 +2549,35 @@ function staticCreateRequireUse7(expr: ts.Expression): boolean { !use.parent.questionDotToken ) { use = use.parent; + member = true; if ( ts.isPropertyAccessExpression(use.parent) && use.parent.expression === use && use.parent.name.text === "paths" && !use.parent.questionDotToken - ) + ) { use = use.parent; + paths = true; + } } const call = use.parent; + if (ts.isCallExpression(call) && call.expression === use) { + if (call.questionDotToken) return "escaping"; + const args = call.arguments.nodes; + if (args.length === 1 && ts.isStringLiteralLike(args[0]!)) return "static"; + if (paths || args.some((arg) => ts.isSpreadElement(arg))) return "escaping"; + return member && (args.length === 0 || args.length > 2) ? "escaping" : "runtime"; + } + return member && !paths && !isAssignmentTarget7(use) ? "runtime" : "escaping"; +} + +function isAssignmentTarget7(node: ts.Node): boolean { + const parent = node.parent; return ( - ts.isCallExpression(call) && - call.expression === use && - !call.questionDotToken && - call.arguments.nodes.length === 1 && - ts.isStringLiteralLike(call.arguments.nodes[0]!) + ts.isBinaryExpression(parent) && + parent.left === node && + parent.operatorToken.kind >= ts.SyntaxKind.FirstAssignment && + parent.operatorToken.kind <= ts.SyntaxKind.LastAssignment ); } @@ -2582,9 +2613,11 @@ function stableCreateRequireBindingReason7( : checker.getSymbolAtLocation(ref); if (target && (target.flags & ts.SymbolFlags.Alias) !== 0) target = checker.getAliasedSymbol(target); - if (target === symbol && !staticCreateRequireUse7(ref)) { + if (target !== symbol) continue; + const use = createRequireUse7(ref); + if (use === "computed") return "its createRequire binding is reassigned, escapes, or uses a computed specifier"; - } + if (use === "escaping") return "its createRequire binding is reassigned or escapes"; } const preceding = stmt.declarationList.declarations.nodes.slice( 0, @@ -2647,10 +2680,12 @@ function npmStaticModuleImportReason7( checker.prefetchSymbolNodesExact(identifierOccurrences7(sf, specifier.name.text)); let reason: string | null = null; const checkUse = (expr: ts.Expression): void => { - if (!staticCreateRequireUse7(expr)) { + const use = createRequireUse7(expr); + if (use === "computed") reason = "its createRequire result escapes or is used with a computed specifier (static requires need literal specifiers)"; - } + else if (use === "escaping") + reason = "its createRequire result escapes (static requires need literal specifiers)"; }; ts.walkPreorder(sf, (node) => { if ( diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index c0b02e8db..63f6d2945 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -609,6 +609,7 @@ async function compileExecutableNative( ...(features.eventsOnceSignal ? { eventsOnceSignal: true } : {}), ...(features.dynBuffer ? { dynBuffer: true } : {}), ...(features.builtinValues ? { builtinValues: true } : {}), + ...(features.computedRequire ? { computedRequire: true } : {}), events: features.events, emitter: features.emitter, symbol: features.symbol, diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index 390c7fbc3..fbeb05dd3 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -679,6 +679,8 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "process.builtinId", "process.builtinModule", "process.builtinUnsupported", + "module.requireComputed", + "module.resolveComputed", "fs.callbackValue", "fs.callbackCall", "dyn.keyDelete", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index e6ad0614a..a3efc12c8 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -378,6 +378,18 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "module.parent": { argTypes: [F64], result: F64 }, "module.loaded": { argTypes: [F64], result: BOOL }, "module.cacheGet": { argTypes: [STRING], result: F64 }, + /** createRequire's require(id) and require.resolve(request, options) + * with a specifier computed at run time (scr_require.c): the arguments, + * then the requiring file, the call's "file:line" and the builtin ids. + * Both resolve like Node at run time. requireComputed always throws + * (Node's argument errors or MODULE_NOT_FOUND), or ends the program with + * an SC2020 refusal when the specifier resolves; resolveComputed answers + * the resolved id or real path. */ + "module.requireComputed": { argTypes: [DYN, STRING, STRING, arrayOf(STRING)], result: DYN }, + "module.resolveComputed": { + argTypes: [DYN, DYN, STRING, STRING, arrayOf(STRING)], + result: STRING, + }, "module.cacheHas": { argTypes: [STRING], result: BOOL }, "module.cacheKeys": { argTypes: [], result: arrayOf(STRING) }, "timers.setTimeout": { argTypes: [{ kind: "func", params: [], ret: VOID }, F64], result: VOID }, diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index 2ae19375f..54c5ee5be 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -946,6 +946,9 @@ const RUNTIME_FEATURE_RULES = { fns: ["buffer.prototypeMember"], expr: (node, _tables) => node.kind === "dynInvoke" && DYN_BUFFER_METHODS.has(node.method), }), + /** createRequire's require and require.resolve with specifiers computed + * at run time (scr_require.c). */ + computedRequire: rule({ fns: ["module.requireComputed", "module.resolveComputed"] }), /** Builtin functions read as values (scr_builtin_values.c). */ builtinValues: rule({ fns: [ @@ -1182,6 +1185,7 @@ function noRuntimeFeatures(): RuntimeFeatures { dynForAwait: false, dynBuffer: false, builtinValues: false, + computedRequire: false, symbol: false, tls: false, tlsCa: false, diff --git a/packages/compiler/src/native/sanitizer.test.ts b/packages/compiler/src/native/sanitizer.test.ts index 33a4c19b9..c6063b2db 100644 --- a/packages/compiler/src/native/sanitizer.test.ts +++ b/packages/compiler/src/native/sanitizer.test.ts @@ -84,6 +84,7 @@ test.skipIf(process.platform !== "linux" && process.platform !== "darwin")( eventsOnceSignal: false, dynBuffer: false, builtinValues: false, + computedRequire: false, warningReplace: false, stdioExtended: false, regex: true, diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8210-require-computed-missing/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8210-require-computed-missing/main.ts.json new file mode 100644 index 000000000..659ee27aa --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8210-require-computed-missing/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8210-require-computed-missing/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8211-require-resolve-computed/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8211-require-resolve-computed/main.ts.json new file mode 100644 index 000000000..44f6cb75f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8211-require-resolve-computed/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8211-require-resolve-computed/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8212-native-helper-probe-no-display/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8212-native-helper-probe-no-display/main.ts.json new file mode 100644 index 000000000..2a83db362 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8212-native-helper-probe-no-display/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8212-native-helper-probe-no-display/main.ts" + ], + "diags": [] +} diff --git a/packages/runtime-pack-common/runtime-pack-matrix.mjs b/packages/runtime-pack-common/runtime-pack-matrix.mjs index 0ab467236..0fc9fdf79 100644 --- a/packages/runtime-pack-common/runtime-pack-matrix.mjs +++ b/packages/runtime-pack-common/runtime-pack-matrix.mjs @@ -67,6 +67,8 @@ export const OPTIONAL_RUNTIME_SOURCES = [ ["scr_bytes_dyn.c", all("dynBuffer", "dynInvoke")], // Builtin functions read as values in JavaScript dependencies. ["scr_builtin_values.c", "builtinValues"], + // createRequire's computed require and require.resolve. + ["scr_require.c", "computedRequire"], ["scr_dc.c", "dc"], // The dynamic tier's jsval values (two-tier design I3): linked only into // programs whose IR has dynamic-tier operations. diff --git a/packages/runtime/src/scr_path.h b/packages/runtime/src/scr_path.h index 9600b5e95..de5e85491 100644 --- a/packages/runtime/src/scr_path.h +++ b/packages/runtime/src/scr_path.h @@ -1,5 +1,6 @@ -/* scriptc runtime — node:path (scr_path.c) and path globs (scr_glob.c). - * Included by scr_runtime.h, which says where new declarations go. */ +/* scriptc runtime — node:path (scr_path.c), path globs (scr_glob.c), and + * module resolution at run time (scr_require.c). Included by scr_runtime.h, + * which says where new declarations go. */ #ifndef SCR_PATH_H #define SCR_PATH_H @@ -89,4 +90,18 @@ void scr_fs_glob_callback(ScrDyn *args); ScrArr *scr_fsp_glob_strings(const ScrDyn *pattern, const ScrDyn *options); ScrDyn *scr_fsp_glob_error(void); +/* createRequire's require(id) and require.resolve(request, options) with a + * specifier computed at run time (scr_require.c). Both resolve with Node + * 24's CommonJS algorithm from `importer`, the requiring file; `builtins` + * lists the builtin ids (process.getBuiltinModule's table), and `site` is + * the call's "file:line" for the SC2020 refusal. All arguments are + * borrowed. scr_require_computed always leaves an exception pending and + * returns NULL: Node's argument errors, or MODULE_NOT_FOUND when nothing + * resolves; a specifier that resolves ends the program with the refusal. + * scr_require_resolve answers the resolved id or real path (+1), or NULL + * with Node's error pending. */ +ScrDyn *scr_require_computed(ScrDyn *id, ScrStr *importer, ScrStr *site, ScrArr *builtins); +ScrStr *scr_require_resolve(ScrDyn *request, ScrDyn *options, ScrStr *importer, ScrStr *site, + ScrArr *builtins); + #endif /* SCR_PATH_H */ diff --git a/packages/runtime/src/scr_require.c b/packages/runtime/src/scr_require.c new file mode 100644 index 000000000..645273bce --- /dev/null +++ b/packages/runtime/src/scr_require.c @@ -0,0 +1,895 @@ +/* createRequire's require(id) and require.resolve(request) with a specifier + * computed at run time (module-resolution.ts lowers both calls here). + * + * The compiled module graph is fixed at build time, so neither call loads a + * module. Both resolve the specifier with Node 24's CommonJS algorithm + * (lib/internal/modules/cjs/loader.js: Module._resolveFilename and + * Module._findPath, with package.json "main", "exports" and self-reference) + * against the file system at run time: + * + * - require.resolve returns what Node returns: the builtin's id, or the real + * path of the resolved file. It throws Node's MODULE_NOT_FOUND error when + * nothing resolves. + * - require throws the same MODULE_NOT_FOUND error when nothing resolves, so + * packages that probe for optional files take their absent branch as in + * Node. When the specifier resolves (a builtin, or a file such as a native + * addon), Node would load the module, which a compiled program cannot do: + * the program ends with an SC2020 refusal naming the module. The refusal is + * not an exception, so a `catch` clause cannot take a fallback that Node + * would not take. + * + * Resolution steps whose Node result this file does not reproduce (package + * "imports", invalid package configs or export targets, unexported subpaths, + * percent-encoded export targets, an export target that names no file, a + * "main" that names no file) end the program with the same SC2020 refusal, + * naming the step. */ +#include "scr_runtime.h" + +#include +#include +#include + +typedef struct { + ScrStr *request; /* borrowed */ + ScrStr *importer; /* borrowed: the requiring file */ + ScrArr *builtins; /* borrowed: Node 24's require-able builtin ids */ + /* Set when resolution reaches a step it does not reproduce; the caller + * refuses with this reason. Static text. */ + const char *refusal; +} ScrRequire; + +static ScrStr *req_cstr(const char *s) { return scr_str_new(s, strlen(s)); } + +static bool req_has_prefix(const ScrStr *s, const char *prefix) { + size_t n = strlen(prefix); + return s->len >= n && memcmp(s->data, prefix, n) == 0; +} + +static bool req_has_suffix(const ScrStr *s, const char *suffix) { + size_t n = strlen(suffix); + return s->len >= n && memcmp(s->data + s->len - n, suffix, n) == 0; +} + +static bool req_eq(const ScrStr *s, const char *text) { + size_t n = strlen(text); + return s->len == n && memcmp(s->data, text, n) == 0; +} + +static ScrStr *req_concat(const ScrStr *a, const char *b, size_t b_len) { + ScrJsonBuf out; + scr_jb_init(&out); + scr_jb_write(&out, a->data, a->len); + scr_jb_write(&out, b, b_len); + return scr_jb_finish(&out); +} + +static ScrStr *req_resolve2(ScrStr *a, ScrStr *b) { + ScrArr *parts = scr_arr_new(SCR_ELEM_STR, 2); + scr_arr_push_ref(parts, scr_str_retain(a)); + scr_arr_push_ref(parts, scr_str_retain(b)); +#ifdef _WIN32 + ScrStr *out = scr_path_win32_resolve(parts); +#else + ScrStr *out = scr_path_resolve(parts); +#endif + scr_arr_release(parts); + return out; +} + +static ScrStr *req_dirname(ScrStr *path) { +#ifdef _WIN32 + return scr_path_win32_dirname(path); +#else + return scr_path_dirname(path); +#endif +} + +static bool req_is_absolute(ScrStr *path) { +#ifdef _WIN32 + return scr_path_win32_is_absolute(path); +#else + return scr_path_is_absolute(path); +#endif +} + +/* Node's internalModuleStat: 0 for a file, 1 for a directory, -1 when the + * path does not exist or cannot be read. Never throws. */ +static int req_stat(ScrStr *path) { + ScrStats *stats = scr_fs_stat_probe(path, false, false); + if (stats == NULL) { + if (scr_exc_pending()) scr_exc_clear(); + return -1; + } + int kind = scr_stats_is_dir(stats) ? 1 : 0; + scr_stats_release(stats); + return kind; +} + +/* tryFile: the real path of `path` when it is a file (+1), else NULL. */ +static ScrStr *req_try_file(ScrStr *path) { + if (req_stat(path) != 0) return NULL; + ScrStr *real = scr_fs_realpath(path); + if (real != NULL) return real; + if (scr_exc_pending()) scr_exc_clear(); + return scr_str_retain(path); +} + +/* tryExtensions with Module._extensions' keys, in registration order. */ +static ScrStr *req_try_extensions(ScrStr *base) { + static const char *const extensions[] = { ".js", ".json", ".node" }; + for (size_t i = 0; i < sizeof extensions / sizeof extensions[0]; i++) { + ScrStr *candidate = req_concat(base, extensions[i], strlen(extensions[i])); + ScrStr *found = req_try_file(candidate); + scr_str_release(candidate); + if (found != NULL) return found; + } + return NULL; +} + +/* ── package.json ────────────────────────────────────────────────────── */ + +/* The package.json at `pjson` parsed (+1), or NULL when there is none. + * A file Node rejects as an invalid package config (unparsable JSON, a + * non-object document, a non-string "name" or "type") sets the refusal. */ +static ScrDyn *req_read_package(ScrRequire *r, ScrStr *pjson) { + if (req_stat(pjson) != 0) return NULL; + ScrStr *text = scr_fs_read_file(pjson); + if (text == NULL) { + if (scr_exc_pending()) scr_exc_clear(); + return NULL; + } + ScrDyn *pkg = scr_json_parse(text); + scr_str_release(text); + if (pkg == NULL) { + if (scr_exc_pending()) scr_exc_clear(); + r->refusal = "a package.json that is not valid JSON"; + return NULL; + } + const ScrDyn *name = pkg->kind == SCR_DYN_OBJ ? scr_dyn_obj_get(pkg, "name", 4) : NULL; + const ScrDyn *type = pkg->kind == SCR_DYN_OBJ ? scr_dyn_obj_get(pkg, "type", 4) : NULL; + if (pkg->kind != SCR_DYN_OBJ || (name != NULL && name->kind != SCR_DYN_STR) || + (type != NULL && type->kind != SCR_DYN_STR)) { + scr_dyn_release(pkg); + r->refusal = "an invalid package.json"; + return NULL; + } + return pkg; +} + +/* A string field of a parsed package.json (borrowed), or NULL. */ +static ScrStr *req_package_string(const ScrDyn *pkg, const char *key) { + const ScrDyn *value = scr_dyn_obj_get(pkg, key, strlen(key)); + return value != NULL && value->kind == SCR_DYN_STR ? value->v.str : NULL; +} + +/* "exports" or "imports" as Node keeps them: a string, an object or an + * array (borrowed); NULL for every other value. */ +static const ScrDyn *req_package_map(const ScrDyn *pkg, const char *key) { + const ScrDyn *value = scr_dyn_obj_get(pkg, key, strlen(key)); + if (value == NULL) return NULL; + return value->kind == SCR_DYN_STR || value->kind == SCR_DYN_OBJ || value->kind == SCR_DYN_ARR ? value : NULL; +} + +/* tryPackage: a directory's "main" entry, then its index files. */ +static ScrStr *req_try_package(ScrRequire *r, ScrStr *dir) { + ScrStr *pjson_name = req_cstr("package.json"); + ScrStr *pjson = req_resolve2(dir, pjson_name); + scr_str_release(pjson_name); + ScrDyn *pkg = req_read_package(r, pjson); + scr_str_release(pjson); + if (r->refusal != NULL) return NULL; + ScrStr *main = pkg != NULL ? req_package_string(pkg, "main") : NULL; + ScrStr *index_name = req_cstr("index"); + ScrStr *index = req_resolve2(dir, index_name); + ScrStr *found = NULL; + if (main == NULL) { + found = req_try_extensions(index); + } else { + ScrStr *filename = req_resolve2(dir, main); + found = req_try_file(filename); + if (found == NULL) found = req_try_extensions(filename); + if (found == NULL) { + ScrStr *main_index = req_resolve2(filename, index_name); + found = req_try_extensions(main_index); + scr_str_release(main_index); + } + scr_str_release(filename); + if (found == NULL) { + /* Node falls back to the index with a DEP0128 warning, or throws an + * error that names the "main" entry. */ + r->refusal = "a package.json \"main\" entry that names no file"; + } + } + scr_str_release(index); + scr_str_release(index_name); + if (pkg != NULL) scr_dyn_release(pkg); + return found; +} + +/* ── package exports (lib/internal/modules/esm/resolve.js) ───────────── */ + +/* A target or subpath segment Node rejects or warns about: "", ".", "..", + * "node_modules" in any case, and their percent-encoded spellings. The + * caller refuses '%' separately, so the plain spellings suffice. */ +static bool req_invalid_segment(const char *s, size_t len) { + size_t start = 0; + for (size_t i = 0; i <= len; i++) { + if (i < len && s[i] != '/' && s[i] != '\\') continue; + size_t n = i - start; + const char *seg = s + start; + if (n == 0 && i < len) return true; + if ((n == 1 && seg[0] == '.') || (n == 2 && seg[0] == '.' && seg[1] == '.')) return true; + if (n == 12) { + static const char nm[] = "node_modules"; + bool same = true; + for (size_t k = 0; k < 12 && same; k++) { + char c = seg[k]; + if (c >= 'A' && c <= 'Z') c = (char)(c - 'A' + 'a'); + same = c == nm[k]; + } + if (same) return true; + } + start = i + 1; + } + return false; +} + +/* Characters whose URL parsing changes the path Node resolves: percent + * escapes, backslashes, query and fragment delimiters, and the control + * characters the URL parser strips. */ +static bool req_url_sensitive(const char *s, size_t len) { + for (size_t i = 0; i < len; i++) { + unsigned char c = (unsigned char)s[i]; + if (c == '%' || c == '\\' || c == '?' || c == '#' || c < 0x20 || c == 0x7f) return true; + } + return false; +} + +typedef enum { REQ_TARGET_PATH, REQ_TARGET_NULL, REQ_TARGET_UNDEFINED, REQ_TARGET_INVALID } ScrRequireTarget; + +static bool req_condition(const ScrStr *key) { + return req_eq(key, "default") || req_eq(key, "require") || req_eq(key, "node") || + req_eq(key, "node-addons") || req_eq(key, "module-sync"); +} + +static bool req_array_index(const ScrStr *key) { + if (key->len == 0 || key->len > 10) return false; + if (key->len > 1 && key->data[0] == '0') return false; + double n = 0; + for (size_t i = 0; i < key->len; i++) { + if (key->data[i] < '0' || key->data[i] > '9') return false; + n = n * 10 + (key->data[i] - '0'); + } + return n < 4294967295.0; +} + +/* resolvePackageTarget for an "exports" target. `pkg_dir` is the package + * directory, `subpath` the part the target receives. A path result is in + * `*out` (+1, not yet checked against the file system). */ +static ScrRequireTarget req_resolve_target(ScrRequire *r, ScrStr *pkg_dir, const ScrDyn *target, + const char *subpath, size_t subpath_len, bool pattern, + ScrStr **out) { + if (target->kind == SCR_DYN_STR) { + ScrStr *t = target->v.str; + if (subpath_len != 0 && !pattern && !req_has_suffix(t, "/")) return REQ_TARGET_INVALID; + if (!req_has_prefix(t, "./")) return REQ_TARGET_INVALID; + if (req_url_sensitive(t->data, t->len) || req_url_sensitive(subpath, subpath_len) || + req_invalid_segment(t->data + 2, t->len - 2) || + (subpath_len != 0 && req_invalid_segment(subpath, subpath_len))) { + r->refusal = "a package \"exports\" target that Node rejects, warns about or decodes"; + return REQ_TARGET_INVALID; + } + if (pattern && memchr(pkg_dir->data, '*', pkg_dir->len) != NULL) { + /* Node substitutes the pattern in the whole URL, directory included. */ + r->refusal = "a package \"exports\" pattern in a directory whose path contains '*'"; + return REQ_TARGET_INVALID; + } + ScrJsonBuf path; + scr_jb_init(&path); + scr_jb_write(&path, pkg_dir->data, pkg_dir->len); + const char *rest = t->data + 1; /* keeps the '/' after "." */ + size_t rest_len = t->len - 1; + if (pattern) { + for (size_t i = 0; i < rest_len; i++) { + if (rest[i] == '*') + scr_jb_write(&path, subpath, subpath_len); + else + scr_jb_putc(&path, rest[i]); + } + } else { + scr_jb_write(&path, rest, rest_len); + scr_jb_write(&path, subpath, subpath_len); + } + ScrStr *joined = scr_jb_finish(&path); +#ifdef _WIN32 + *out = scr_path_win32_normalize(joined); +#else + *out = scr_path_normalize(joined); +#endif + scr_str_release(joined); + return REQ_TARGET_PATH; + } + if (target->kind == SCR_DYN_ARR) { + double n = scr_dyn_arr_len(target); + if (n == 0) return REQ_TARGET_NULL; + ScrRequireTarget last = REQ_TARGET_UNDEFINED; + for (double i = 0; i < n; i++) { + ScrDyn *item = scr_dyn_arr_at(target, i); + ScrRequireTarget result = req_resolve_target(r, pkg_dir, item, subpath, subpath_len, pattern, out); + scr_dyn_release(item); + if (r->refusal != NULL) return REQ_TARGET_INVALID; + if (result == REQ_TARGET_PATH) return result; + if (result == REQ_TARGET_INVALID) last = REQ_TARGET_INVALID; + else if (result == REQ_TARGET_NULL) last = REQ_TARGET_NULL; + } + return last == REQ_TARGET_UNDEFINED ? REQ_TARGET_UNDEFINED : last; + } + if (target->kind == SCR_DYN_OBJ) { + ScrDyn *keys = scr_dyn_obj_own_keys(target); + double n = scr_dyn_arr_len(keys); + ScrRequireTarget result = REQ_TARGET_UNDEFINED; + for (double i = 0; i < n; i++) { + ScrDyn *key = scr_dyn_arr_at(keys, i); + bool index = req_array_index(key->v.str); + scr_dyn_release(key); + if (index) { + r->refusal = "numeric keys in a package's \"exports\""; + scr_dyn_release(keys); + return REQ_TARGET_INVALID; + } + } + for (double i = 0; i < n && result == REQ_TARGET_UNDEFINED; i++) { + ScrDyn *key = scr_dyn_arr_at(keys, i); + if (req_condition(key->v.str)) { + const ScrDyn *value = scr_dyn_obj_get(target, key->v.str->data, key->v.str->len); + result = req_resolve_target(r, pkg_dir, value, subpath, subpath_len, pattern, out); + if (r->refusal != NULL) result = REQ_TARGET_INVALID; + } + scr_dyn_release(key); + } + scr_dyn_release(keys); + return result; + } + if (target->kind == SCR_DYN_NULL) return REQ_TARGET_NULL; + return REQ_TARGET_INVALID; +} + +/* patternKeyCompare(a, b) === 1: `b` is the better pattern match. */ +static bool req_pattern_better(const ScrStr *a, const ScrStr *b) { + const char *as = a ? memchr(a->data, '*', a->len) : NULL; + const char *bs = memchr(b->data, '*', b->len); + size_t a_len = a ? a->len : 0; + size_t base_a = as ? (size_t)(as - a->data) + 1 : a_len; + size_t base_b = bs ? (size_t)(bs - b->data) + 1 : b->len; + if (base_a > base_b) return false; + if (base_b > base_a) return true; + if (as == NULL) return true; + if (bs == NULL) return false; + return b->len > a_len; +} + +/* packageExportsResolve for `subpath` ("." or "./..."), then + * finalizeEsmResolution's file check. +1 real path, or NULL with the + * refusal set. */ +static ScrStr *req_exports_resolve(ScrRequire *r, ScrStr *pkg_dir, const ScrDyn *exports, ScrStr *subpath) { + bool sugar; + if (exports->kind != SCR_DYN_OBJ) { + sugar = true; + } else { + sugar = false; + ScrDyn *keys = scr_dyn_obj_own_keys(exports); + double n = scr_dyn_arr_len(keys); + for (double i = 0; i < n; i++) { + ScrDyn *key = scr_dyn_arr_at(keys, i); + bool conditional = key->v.str->len == 0 || key->v.str->data[0] != '.'; + scr_dyn_release(key); + if (i == 0) { + sugar = conditional; + } else if (sugar != conditional) { + scr_dyn_release(keys); + r->refusal = "a package \"exports\" object that mixes subpaths and conditions"; + return NULL; + } + } + scr_dyn_release(keys); + } + if (exports->kind == SCR_DYN_STR && exports->v.str->len > 0 && + (exports->v.str->data[0] == '[' || exports->v.str->data[0] == '{')) { + r->refusal = "a package \"exports\" string that Node parses as JSON"; + return NULL; + } + ScrStr *path = NULL; + ScrRequireTarget result = REQ_TARGET_UNDEFINED; + bool has_star = memchr(subpath->data, '*', subpath->len) != NULL; + if (sugar) { + if (req_eq(subpath, ".")) + result = req_resolve_target(r, pkg_dir, exports, "", 0, false, &path); + } else { + const ScrDyn *exact = has_star || req_has_suffix(subpath, "/") + ? NULL + : scr_dyn_obj_get(exports, subpath->data, subpath->len); + if (exact != NULL) { + result = req_resolve_target(r, pkg_dir, exact, "", 0, false, &path); + } else { + ScrStr *best = NULL; + size_t best_start = 0, best_end = 0; + ScrDyn *keys = scr_dyn_obj_own_keys(exports); + double n = scr_dyn_arr_len(keys); + for (double i = 0; i < n; i++) { + ScrDyn *key_value = scr_dyn_arr_at(keys, i); + ScrStr *key = key_value->v.str; + const char *star = memchr(key->data, '*', key->len); + size_t at = star ? (size_t)(star - key->data) : 0; + if (star != NULL && subpath->len >= at && memcmp(subpath->data, key->data, at) == 0) { + size_t trailer = key->len - at - 1; + bool single = memchr(star + 1, '*', trailer) == NULL; + if (subpath->len >= key->len && + memcmp(subpath->data + subpath->len - trailer, star + 1, trailer) == 0 && + req_pattern_better(best, key) && single) { + if (best) scr_str_release(best); + best = scr_str_retain(key); + best_start = at; + best_end = subpath->len - trailer; + } + } + scr_dyn_release(key_value); + } + scr_dyn_release(keys); + if (best != NULL) { + if (req_has_suffix(subpath, "/")) { + r->refusal = "a package \"exports\" pattern matched by a subpath ending in '/'"; + } else { + const ScrDyn *target = scr_dyn_obj_get(exports, best->data, best->len); + result = req_resolve_target(r, pkg_dir, target, subpath->data + best_start, + best_end - best_start, true, &path); + } + scr_str_release(best); + } + } + } + if (r->refusal != NULL) { + if (path) scr_str_release(path); + return NULL; + } + if (result != REQ_TARGET_PATH) { + r->refusal = result == REQ_TARGET_INVALID ? "an invalid package \"exports\" target" + : "a subpath that the package's \"exports\" do not export"; + return NULL; + } + ScrStr *found = req_try_file(path); + scr_str_release(path); + if (found == NULL) r->refusal = "a package \"exports\" target that names no file"; + return found; +} + +/* ── Module._findPath ────────────────────────────────────────────────── */ + +/* EXPORTS_PATTERN: /^((?:@[^/\\%]+\/)?[^./\\%][^/\\%]*)(\/.*)?$/ — the + * package name's length in `request`, or 0 when it does not match. */ +static size_t req_package_name_len(const ScrStr *request) { + const char *s = request->data; + size_t len = request->len, i = 0; + if (len > 0 && s[0] == '@') { + i = 1; + size_t start = i; + while (i < len && s[i] != '/' && s[i] != '\\' && s[i] != '%') i++; + if (i == start || i >= len || s[i] != '/') i = 0; + else i++; + } + if (i >= len || s[i] == '.' || s[i] == '/' || s[i] == '\\' || s[i] == '%') return 0; + while (i < len && s[i] != '/' && s[i] != '\\' && s[i] != '%') i++; + if (i < len && s[i] != '/') return 0; + return i; +} + +/* resolveExports: a package under `nm_dir` whose package.json has + * "exports". +1 real path; NULL when the package has no "exports" (or the + * refusal is set). */ +static ScrStr *req_resolve_exports(ScrRequire *r, ScrStr *nm_dir) { + size_t name_len = req_package_name_len(r->request); + if (name_len == 0) return NULL; + ScrStr *name = scr_str_new(r->request->data, name_len); + ScrStr *pkg_dir = req_resolve2(nm_dir, name); + scr_str_release(name); + ScrStr *pjson = req_concat(pkg_dir, "/package.json", 13); + ScrDyn *pkg = req_read_package(r, pjson); + scr_str_release(pjson); + ScrStr *found = NULL; + const ScrDyn *exports = pkg != NULL ? req_package_map(pkg, "exports") : NULL; + if (exports != NULL) { + ScrJsonBuf sub; + scr_jb_init(&sub); + scr_jb_putc(&sub, '.'); + scr_jb_write(&sub, r->request->data + name_len, r->request->len - name_len); + ScrStr *subpath = scr_jb_finish(&sub); + found = req_exports_resolve(r, pkg_dir, exports, subpath); + scr_str_release(subpath); + } + if (pkg != NULL) scr_dyn_release(pkg); + scr_str_release(pkg_dir); + return found; +} + +/* isRelative in Module._findPath: ".", "..", and "./" or "../" prefixes. */ +static bool req_is_relative(const ScrStr *request) { + if (req_eq(request, ".") || req_eq(request, "..")) return true; + if (req_has_prefix(request, "./") || req_has_prefix(request, "../")) return true; +#ifdef _WIN32 + if (req_has_prefix(request, ".\\") || req_has_prefix(request, "..\\")) return true; +#endif + return false; +} + +static bool req_trailing_slash(const ScrStr *request) { + const char *s = request->data; + size_t n = request->len; + if (n == 0) return false; + if (s[n - 1] == '/') return true; + if (s[n - 1] != '.') return false; + if (n == 1 || s[n - 2] == '/') return true; + return s[n - 2] == '.' && (n == 2 || s[n - 3] == '/'); +} + +/* Module._findPath over `paths` (borrowed strings). +1 real path or NULL. */ +static ScrStr *req_find_path(ScrRequire *r, ScrStr *const *paths, size_t count) { + ScrStr *request = r->request; + bool absolute = req_is_absolute(request); + ScrStr *empty = NULL; + if (absolute) { + empty = scr_str_new("", 0); + paths = ∅ + count = 1; + } + bool trailing = req_trailing_slash(request); + bool inside = true; + if (req_is_relative(request)) { +#ifdef _WIN32 + ScrStr *normalized = scr_path_win32_normalize(request); +#else + ScrStr *normalized = scr_path_normalize(request); +#endif + inside = !req_has_prefix(normalized, ".."); + scr_str_release(normalized); + } + ScrStr *found = NULL; + for (size_t i = 0; i < count && found == NULL && r->refusal == NULL; i++) { + ScrStr *dir = paths[i]; + if (inside && dir->len > 0 && req_stat(dir) < 1) continue; + if (!absolute) { + found = req_resolve_exports(r, dir); + if (found != NULL || r->refusal != NULL) break; + } + ScrStr *base = req_resolve2(dir, request); + int kind = req_stat(base); + if (!trailing) { + if (kind == 0) found = req_try_file(base); + if (found == NULL) found = req_try_extensions(base); + } + if (found == NULL && kind == 1) found = req_try_package(r, base); + scr_str_release(base); + } + if (empty) scr_str_release(empty); + return found; +} + +/* Module._nodeModulePaths(from) followed by Module.globalPaths' entries + * that come from the environment (NODE_PATH, then the home directory's + * .node_modules and .node_libraries). Node also searches the lib/node + * directory of its own installation prefix, which a compiled program does + * not have. Fills `out` (+1 each) and answers the count. */ +static size_t req_lookup_paths(ScrStr *from, ScrStr ***out) { + size_t cap = 16, n = 0; + ScrStr **paths = malloc(cap * sizeof *paths); + if (!paths) scr_trap("scriptc: out of memory\n"); +#define REQ_PUSH(value) \ + do { \ + if (n == cap) { \ + cap *= 2; \ + ScrStr **grown = realloc(paths, cap * sizeof *paths); \ + if (!grown) scr_trap("scriptc: out of memory\n"); \ + paths = grown; \ + } \ + paths[n++] = (value); \ + } while (0) + ScrStr *dot = scr_str_new(".", 1); + ScrStr *abs = req_resolve2(from, dot); + scr_str_release(dot); + static const char nm[] = "node_modules"; + const char *s = abs->data; +#ifdef _WIN32 + if (abs->len >= 2 && s[abs->len - 1] == '\\' && s[abs->len - 2] == ':') { + REQ_PUSH(req_concat(abs, "node_modules", 12)); + } else { + for (long i = (long)abs->len - 1, p = 0, last = (long)abs->len; i >= 0; --i) { + char c = s[i]; + if (c == '\\' || c == '/' || c == ':') { + if (p != 12) { + ScrStr *prefix = scr_str_new(s, (size_t)last); + REQ_PUSH(req_concat(prefix, "\\node_modules", 13)); + scr_str_release(prefix); + } + last = i; + p = 0; + } else if (p != -1) { + p = (p < 12 && nm[11 - p] == c) ? p + 1 : -1; + } + } + } +#else + if (abs->len == 1 && s[0] == '/') { + REQ_PUSH(scr_str_new("/node_modules", 13)); + } else { + for (long i = (long)abs->len - 1, p = 0, last = (long)abs->len; i >= 0; --i) { + char c = s[i]; + if (c == '/') { + if (p != 12) { + ScrStr *prefix = scr_str_new(s, (size_t)last); + REQ_PUSH(req_concat(prefix, "/node_modules", 13)); + scr_str_release(prefix); + } + last = i; + p = 0; + } else if (p != -1) { + p = (p < 12 && nm[11 - p] == c) ? p + 1 : -1; + } + } + REQ_PUSH(scr_str_new("/node_modules", 13)); + } +#endif + scr_str_release(abs); +#ifdef _WIN32 + const char *node_path = scr_getenv("NODE_PATH"); + const char *home = scr_getenv("USERPROFILE"); + const char delimiter = ';'; +#else + const char *node_path = scr_getenv("NODE_PATH"); + const char *home = scr_getenv("HOME"); + const char delimiter = ':'; +#endif + if (node_path != NULL) { + for (const char *p = node_path; *p;) { + const char *end = strchr(p, delimiter); + size_t len = end ? (size_t)(end - p) : strlen(p); + if (len > 0) REQ_PUSH(scr_str_new(p, len)); + p += len; + if (*p) p++; + } + } + if (home != NULL && *home) { + ScrStr *home_dir = req_cstr(home); + static const char *const globals[] = { ".node_modules", ".node_libraries" }; + for (size_t i = 0; i < 2; i++) { + ScrStr *name = req_cstr(globals[i]); + REQ_PUSH(req_resolve2(home_dir, name)); + scr_str_release(name); + } + scr_str_release(home_dir); + } +#undef REQ_PUSH + *out = paths; + return n; +} + +/* trySelf: the nearest package.json above the importer (Node stops below + * the root and at a node_modules directory) resolves the request through + * its own "exports" when its "name" prefixes it. */ +static ScrStr *req_try_self(ScrRequire *r) { + ScrStr *dir = req_dirname(r->importer); + ScrStr *found = NULL; + for (;;) { + ScrStr *parent = req_dirname(dir); + bool root = scr_str_eq(parent, dir); + scr_str_release(parent); + if (root) break; + ScrStr *dot = scr_str_new(".", 1); + ScrStr *abs = req_resolve2(dir, dot); + scr_str_release(dot); + bool boundary = req_has_suffix(abs, "/node_modules") +#ifdef _WIN32 + || req_has_suffix(abs, "\\node_modules") +#endif + ; + scr_str_release(abs); + if (boundary) break; + ScrStr *pjson_name = req_cstr("package.json"); + ScrStr *pjson = req_resolve2(dir, pjson_name); + scr_str_release(pjson_name); + ScrDyn *pkg = req_read_package(r, pjson); + scr_str_release(pjson); + if (r->refusal != NULL) break; + if (pkg != NULL) { + ScrStr *name = req_package_string(pkg, "name"); + const ScrDyn *exports = req_package_map(pkg, "exports"); + if (req_has_prefix(r->request, "#") && req_package_map(pkg, "imports") != NULL) { + r->refusal = "a package \"imports\" specifier"; + } else if (name != NULL && exports != NULL) { + bool whole = scr_str_eq(name, r->request); + bool prefixed = r->request->len > name->len && + memcmp(r->request->data, name->data, name->len) == 0 && + r->request->data[name->len] == '/'; + if (whole || prefixed) { + ScrJsonBuf sub; + scr_jb_init(&sub); + scr_jb_putc(&sub, '.'); + scr_jb_write(&sub, r->request->data + name->len, r->request->len - name->len); + ScrStr *subpath = scr_jb_finish(&sub); + found = req_exports_resolve(r, dir, exports, subpath); + scr_str_release(subpath); + } + } + scr_dyn_release(pkg); + break; + } + ScrStr *next = req_dirname(dir); + scr_str_release(dir); + dir = next; + } + scr_str_release(dir); + return found; +} + +/* The builtin id `request` names (+1), or NULL. */ +static ScrStr *req_builtin(ScrRequire *r) { + ScrDyn *id = scr_dyn_new_str(r->request); + ScrStr *bare = scr_process_builtin_id(id, r->builtins); + scr_dyn_release(id); + if (bare == NULL || bare->len == 0) { + if (bare) scr_str_release(bare); + return NULL; + } + scr_str_release(bare); + return scr_str_retain(r->request); +} + +/* Module._resolveFilename: +1 resolved id or path; NULL when nothing + * resolves (or the refusal is set). */ +static ScrStr *req_resolve_filename(ScrRequire *r) { + ScrStr *builtin = req_builtin(r); + if (builtin != NULL) return builtin; + ScrStr *found = req_try_self(r); + if (found != NULL || r->refusal != NULL) return found; + bool lookup = r->request->len == 0 || r->request->data[0] != '.' || + (r->request->len > 1 && r->request->data[1] != '.' && r->request->data[1] != '/' +#ifdef _WIN32 + && r->request->data[1] != '\\' +#endif + ); + if (lookup) { + ScrStr *dir = req_dirname(r->importer); + ScrStr **paths = NULL; + size_t count = req_lookup_paths(dir, &paths); + scr_str_release(dir); + found = req_find_path(r, paths, count); + for (size_t i = 0; i < count; i++) scr_str_release(paths[i]); + free(paths); + } else { + ScrStr *dir = req_dirname(r->importer); + found = req_find_path(r, &dir, 1); + scr_str_release(dir); + } + return found; +} + +/* ── entry points ────────────────────────────────────────────────────── */ + +/* Ends the program with the SC2020 refusal: it names the native addon, the + * builtin or file the specifier resolves to, or the resolution step, then + * the program exits with status 1 as after an uncaught error, running the + * 'exit' listeners. */ +static void req_refuse(ScrRequire *r, const char *call, const ScrStr *found, bool builtin, ScrStr *site) { + ScrJsonBuf text; + scr_jb_init(&text); + if (r->refusal != NULL) { + scr_jb_puts(&text, call); + scr_jb_puts(&text, " of '"); + scr_jb_write(&text, r->request->data, r->request->len); + scr_jb_puts(&text, "' reaches "); + scr_jb_puts(&text, r->refusal); + scr_jb_puts(&text, ", which scriptc does not resolve at run time"); + } else if (builtin) { + scr_jb_puts(&text, call); + scr_jb_puts(&text, " of the builtin '"); + scr_jb_write(&text, found->data, found->len); + scr_jb_puts(&text, "' with a computed specifier is not supported; require it with a string literal"); + } else if (req_has_suffix(found, ".node")) { + scr_jb_puts(&text, "native addon '"); + scr_jb_write(&text, found->data, found->len); + scr_jb_puts(&text, "' requires Node's addon runtime, which is unavailable in scriptc"); + } else { + scr_jb_puts(&text, call); + scr_jb_puts(&text, " of '"); + scr_jb_write(&text, r->request->data, r->request->len); + scr_jb_puts(&text, "' resolves to '"); + scr_jb_write(&text, found->data, found->len); + scr_jb_puts(&text, "', which is not part of the compiled program: the module graph is fixed at build time"); + } + scr_jb_puts(&text, " [SC2020 at "); + scr_jb_write(&text, site->data, site->len); + scr_jb_puts(&text, "]\n"); + ScrStr *message = scr_jb_finish(&text); + fflush(stdout); + fputs("scriptc: ", stderr); + fwrite(message->data, 1, message->len, stderr); + fflush(stderr); + scr_str_release(message); + scr_process_exit(1); +} + +static void req_throw(const char *code, const char *prefix, ScrRequire *r, bool stack) { + ScrJsonBuf text; + scr_jb_init(&text); + scr_jb_puts(&text, prefix); + scr_jb_write(&text, r->request->data, r->request->len); + if (stack) { + scr_jb_puts(&text, "'\nRequire stack:\n- "); + scr_jb_write(&text, r->importer->data, r->importer->len); + } + ScrStr *message = scr_jb_finish(&text); + scr_throw_error_msg_code(SCR_ERR_ERROR, message->data, message->len, code); + scr_str_release(message); +} + +ScrDyn *scr_require_computed(ScrDyn *id, ScrStr *importer, ScrStr *site, ScrArr *builtins) { + if (id->kind != SCR_DYN_STR) { + scr_dyn_arg_type_fail("id", "of type string", id); + return NULL; + } + if (id->v.str->len == 0) { + static const char message[] = "The argument 'id' must be a non-empty string. Received ''"; + scr_throw_error_msg_code(SCR_ERR_TYPE, message, sizeof message - 1, "ERR_INVALID_ARG_VALUE"); + return NULL; + } + ScrRequire r = { id->v.str, importer, builtins, NULL }; + ScrStr *builtin = req_builtin(&r); + if (builtin == NULL && req_has_prefix(r.request, "node:")) { + req_throw("ERR_UNKNOWN_BUILTIN_MODULE", "No such built-in module: ", &r, false); + return NULL; + } + ScrStr *found = builtin != NULL ? builtin : req_resolve_filename(&r); + if (found == NULL && r.refusal == NULL) { + req_throw("MODULE_NOT_FOUND", "Cannot find module '", &r, true); + return NULL; + } + req_refuse(&r, "createRequire's require", found, builtin != NULL, site); + if (found) scr_str_release(found); + return NULL; +} + +/* require.resolve's options: Node reads `paths` and `conditions` from an + * object; other values select the default lookup. */ +static const char *req_options_refusal(const ScrDyn *options) { + switch (options->kind) { + case SCR_DYN_UNDEF: + case SCR_DYN_NULL: + case SCR_DYN_NUM: + case SCR_DYN_BOOL: + case SCR_DYN_STR: + return NULL; + case SCR_DYN_OBJ: { + const ScrDyn *paths = scr_dyn_obj_get(options, "paths", 5); + const ScrDyn *conditions = scr_dyn_obj_get(options, "conditions", 10); + if (paths != NULL && paths->kind != SCR_DYN_UNDEF) return "require.resolve's paths option"; + if (conditions != NULL && conditions->kind != SCR_DYN_UNDEF) return "require.resolve's conditions option"; + return NULL; + } + default: + return "require.resolve options that are not a plain object"; + } +} + +ScrStr *scr_require_resolve(ScrDyn *request, ScrDyn *options, ScrStr *importer, ScrStr *site, + ScrArr *builtins) { + if (request->kind != SCR_DYN_STR) { + scr_dyn_arg_type_fail("request", "of type string", request); + return NULL; + } + ScrRequire r = { request->v.str, importer, builtins, req_options_refusal(options) }; + ScrStr *found = r.refusal == NULL ? req_resolve_filename(&r) : NULL; + if (r.refusal != NULL) { + if (found) scr_str_release(found); + req_refuse(&r, "createRequire's require.resolve", NULL, false, site); + return NULL; + } + if (found == NULL) req_throw("MODULE_NOT_FOUND", "Cannot find module '", &r, true); + return found; +} diff --git a/tests/corpus/8210-require-computed-missing/main.ts b/tests/corpus/8210-require-computed-missing/main.ts new file mode 100644 index 000000000..685222157 --- /dev/null +++ b/tests/corpus/8210-require-computed-missing/main.ts @@ -0,0 +1,7 @@ +// A package that probes for optional files through createRequire with +// computed specifiers. A specifier that names no file throws Node's error, +// so the package takes its absent branch as in Node. +import { probeAddon, requireErrors } from "optional-probe"; + +console.log(probeAddon("clipboard")); +for (const line of requireErrors()) console.log(line); diff --git a/tests/corpus/8210-require-computed-missing/node_modules/optional-probe/index.d.ts b/tests/corpus/8210-require-computed-missing/node_modules/optional-probe/index.d.ts new file mode 100644 index 000000000..4eb024b0c --- /dev/null +++ b/tests/corpus/8210-require-computed-missing/node_modules/optional-probe/index.d.ts @@ -0,0 +1,2 @@ +export declare function probeAddon(name: string): string; +export declare function requireErrors(): string[]; diff --git a/tests/corpus/8210-require-computed-missing/node_modules/optional-probe/index.js b/tests/corpus/8210-require-computed-missing/node_modules/optional-probe/index.js new file mode 100644 index 000000000..c824302ba --- /dev/null +++ b/tests/corpus/8210-require-computed-missing/node_modules/optional-probe/index.js @@ -0,0 +1,46 @@ +import { createRequire } from "node:module"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const importer = fileURLToPath(import.meta.url); +const load = createRequire(import.meta.url); +const here = dirname(importer); + +// Looks for a prebuilt addon in each packaging location. +export function probeAddon(name) { + const codes = []; + const dirs = [join(here, "prebuilds", `${process.platform}-${process.arch}`), join(here, "native"), here]; + for (const dir of dirs) { + try { + const addon = load(join(dir, `${name}.node`)); + return `loaded ${typeof addon}`; + } catch (error) { + codes.push(error.code); + } + } + return `absent: ${codes.join(", ")}`; +} + +function describe(spec) { + try { + load(spec); + return "loaded"; + } catch (error) { + return `${error.name} ${error.code}: ${error.message.replaceAll(importer, "")}`; + } +} + +export function requireErrors() { + const specs = [ + "./missing.js", + "./missing", + "../missing-directory/", + "scriptc-corpus-missing-package", + "@scriptc-corpus/missing/sub", + "node:not-a-builtin", + "", + 42, + undefined, + ]; + return specs.map((spec) => `${typeof spec === "string" ? JSON.stringify(spec) : String(spec)} -> ${describe(spec)}`); +} diff --git a/tests/corpus/8210-require-computed-missing/node_modules/optional-probe/package.json b/tests/corpus/8210-require-computed-missing/node_modules/optional-probe/package.json new file mode 100644 index 000000000..50ad53129 --- /dev/null +++ b/tests/corpus/8210-require-computed-missing/node_modules/optional-probe/package.json @@ -0,0 +1 @@ +{ "name": "optional-probe", "version": "1.0.0", "type": "module", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/corpus/8211-require-resolve-computed/main.ts b/tests/corpus/8211-require-resolve-computed/main.ts new file mode 100644 index 000000000..d8e441d2f --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/main.ts @@ -0,0 +1,7 @@ +// require.resolve with requests computed at run time, called directly and +// read as a value, resolves like Node: builtins, relative files with +// Node's extension search, package "main" and "exports" entries, a +// package's own name, and MODULE_NOT_FOUND for requests that name nothing. +import { resolveAll } from "resolve-probe"; + +for (const line of resolveAll()) console.log(line); diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/@scope/scoped/lib/node.js b/tests/corpus/8211-require-resolve-computed/node_modules/@scope/scoped/lib/node.js new file mode 100644 index 000000000..c414b6b26 --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/@scope/scoped/lib/node.js @@ -0,0 +1 @@ +module.exports = "node"; diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/@scope/scoped/lib/other.js b/tests/corpus/8211-require-resolve-computed/node_modules/@scope/scoped/lib/other.js new file mode 100644 index 000000000..7f4090ad9 --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/@scope/scoped/lib/other.js @@ -0,0 +1 @@ +module.exports = "other"; diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/@scope/scoped/package.json b/tests/corpus/8211-require-resolve-computed/node_modules/@scope/scoped/package.json new file mode 100644 index 000000000..25326e6e5 --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/@scope/scoped/package.json @@ -0,0 +1 @@ +{ "name": "@scope/scoped", "version": "1.0.0", "exports": { "node": "./lib/node.js", "default": "./lib/other.js" } } diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/plain-index/index.js b/tests/corpus/8211-require-resolve-computed/node_modules/plain-index/index.js new file mode 100644 index 000000000..0d24e266f --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/plain-index/index.js @@ -0,0 +1 @@ +module.exports = "index"; diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/data.json b/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/data.json new file mode 100644 index 000000000..6d94e13bd --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/data.json @@ -0,0 +1 @@ +{ "kind": "data" } diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/index.d.ts b/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/index.d.ts new file mode 100644 index 000000000..1b8cc8a0e --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/index.d.ts @@ -0,0 +1 @@ +export declare function resolveAll(): string[]; diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/index.js b/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/index.js new file mode 100644 index 000000000..870a14401 --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/index.js @@ -0,0 +1,50 @@ +import { createRequire } from "node:module"; +import { dirname, relative } from "node:path"; +import { fileURLToPath } from "node:url"; + +const importer = fileURLToPath(import.meta.url); +const root = dirname(dirname(dirname(importer))); +const load = createRequire(import.meta.url); + +function show(resolved) { + return resolved.startsWith("/") || /^[A-Za-z]:\\/.test(resolved) ? relative(root, resolved).replaceAll("\\", "/") : resolved; +} + +function describe(resolve, request) { + try { + return show(resolve(request)); + } catch (error) { + return `${error.name} ${error.code}: ${error.message.replaceAll(importer, "")}`; + } +} + +const requests = [ + "fs", + "node:fs", + "node:test", + "./data.json", + "./data", + "./index", + "resolve-probe", + "resolve-probe/data", + "with-exports", + "with-exports/feature", + "with-exports/lib/helper", + "with-main", + "plain-index", + "@scope/scoped", + "./missing", + "scriptc-corpus-missing-package", + "", + 42, +]; + +export function resolveAll(options = {}) { + const resolve = options.resolve ?? load.resolve; + const label = (request) => (typeof request === "string" ? JSON.stringify(request) : String(request)); + return [ + ...requests.map((request) => `call ${label(request)} -> ${describe((r) => load.resolve(r), request)}`), + ...requests.slice(0, 4).map((request) => `value ${label(request)} -> ${describe(resolve, request)}`), + `options ${describe((r) => load.resolve(r, {}), "with-main")}`, + ]; +} diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/package.json b/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/package.json new file mode 100644 index 000000000..d0935f646 --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/package.json @@ -0,0 +1 @@ +{ "name": "resolve-probe", "version": "1.0.0", "type": "module", "exports": { ".": { "types": "./index.d.ts", "default": "./index.js" }, "./data": "./data.json" } } diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/cjs.cjs b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/cjs.cjs new file mode 100644 index 000000000..2cd122bdc --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/cjs.cjs @@ -0,0 +1 @@ +module.exports = "cjs"; diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/esm.mjs b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/esm.mjs new file mode 100644 index 000000000..06c676e54 --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/esm.mjs @@ -0,0 +1 @@ +export default "esm"; diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/lib/feature.js b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/lib/feature.js new file mode 100644 index 000000000..33535fc6f --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/lib/feature.js @@ -0,0 +1 @@ +module.exports = "feature"; diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/lib/helper.js b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/lib/helper.js new file mode 100644 index 000000000..4f51f7bfa --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/lib/helper.js @@ -0,0 +1 @@ +module.exports = "helper"; diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/main.js b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/main.js new file mode 100644 index 000000000..0470f8201 --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/main.js @@ -0,0 +1 @@ +module.exports = "main"; diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/package.json b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/package.json new file mode 100644 index 000000000..09d74787c --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/with-exports/package.json @@ -0,0 +1 @@ +{ "name": "with-exports", "version": "1.0.0", "main": "./main.js", "exports": { ".": { "import": "./esm.mjs", "require": "./cjs.cjs" }, "./feature": "./lib/feature.js", "./lib/*": "./lib/*.js", "./private/*": null } } diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/with-main/lib/entry.js b/tests/corpus/8211-require-resolve-computed/node_modules/with-main/lib/entry.js new file mode 100644 index 000000000..6555fae7c --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/with-main/lib/entry.js @@ -0,0 +1 @@ +module.exports = "entry"; diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/with-main/package.json b/tests/corpus/8211-require-resolve-computed/node_modules/with-main/package.json new file mode 100644 index 000000000..8c473085d --- /dev/null +++ b/tests/corpus/8211-require-resolve-computed/node_modules/with-main/package.json @@ -0,0 +1 @@ +{ "name": "with-main", "version": "1.0.0", "main": "lib/entry" } diff --git a/tests/corpus/8212-native-helper-probe-no-display/main.ts b/tests/corpus/8212-native-helper-probe-no-display/main.ts new file mode 100644 index 000000000..a9e021017 --- /dev/null +++ b/tests/corpus/8212-native-helper-probe-no-display/main.ts @@ -0,0 +1,18 @@ +// A terminal package that loads prebuilt native helpers the way pi-tui does: +// it builds candidate paths (through require.resolve read as a value) and +// requires each one inside a try block. On Linux without DISPLAY the +// clipboard helper is never loaded, and on other platforms no helper for +// the platform is installed, so the package uses its JavaScript fallback as +// in Node. +import { getNativeClipboard, getNativeModuleCandidates, isNativeModifierPressed } from "native-helper"; +import { relative } from "node:path"; + +delete process.env["DISPLAY"]; +console.log("clipboard", typeof getNativeClipboard()); +console.log("modifier", isNativeModifierPressed("shift")); +const root = import.meta.dirname; +for (const candidate of getNativeModuleCandidates("native/helper.node")) { + console.log("candidate", relative(root, candidate).replaceAll("\\", "/")); +} +const custom = getNativeModuleCandidates("native/helper.node", { resolvePackage: () => "/opt/helper/dist/index.js" }); +console.log("custom", custom[0]!.replaceAll("\\", "/")); diff --git a/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/index.d.ts b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/index.d.ts new file mode 100644 index 000000000..8803ba552 --- /dev/null +++ b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/index.d.ts @@ -0,0 +1,11 @@ +export interface NativeHelper { + getText(): string; + getImage(): Uint8Array | undefined; + isModifierPressed?(key: string): boolean; +} +export declare function getNativeClipboard(): NativeHelper | undefined; +export declare function isNativeModifierPressed(key: string): boolean; +export declare function getNativeModuleCandidates( + nativePath: string, + options?: { moduleUrl?: string; resolvePackage?: (id: string) => string; execPath?: string }, +): string[]; diff --git a/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/index.js b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/index.js new file mode 100644 index 000000000..874130379 --- /dev/null +++ b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/index.js @@ -0,0 +1,2 @@ +export { getNativeClipboard, isNativeModifierPressed } from "./native-platform.js"; +export { getNativeModuleCandidates } from "./native-module-path.js"; diff --git a/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/native-module-path.js b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/native-module-path.js new file mode 100644 index 000000000..2a82b41f7 --- /dev/null +++ b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/native-module-path.js @@ -0,0 +1,20 @@ +import { createRequire } from "node:module"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const moduleRequire = createRequire(import.meta.url); + +// Where a prebuilt helper can live: next to the installed package, next to +// this module, and next to the executable. +export function getNativeModuleCandidates(nativePath, options = {}) { + const moduleDir = dirname(fileURLToPath(options.moduleUrl ?? import.meta.url)); + const candidates = []; + try { + const entry = (options.resolvePackage ?? moduleRequire.resolve)("native-helper"); + candidates.push(join(dirname(entry), "..", nativePath)); + } catch { + // A standalone executable has no installed package. + } + candidates.push(join(moduleDir, "..", nativePath), join(moduleDir, nativePath)); + return Array.from(new Set(candidates)); +} diff --git a/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/native-platform.js b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/native-platform.js new file mode 100644 index 000000000..9aaed1dc1 --- /dev/null +++ b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/dist/native-platform.js @@ -0,0 +1,48 @@ +import { createRequire } from "node:module"; +import * as path from "node:path"; +import { getNativeModuleCandidates } from "./native-module-path.js"; + +const load = createRequire(import.meta.url); +const helpers = new Map(); + +function loadHelper(platform, suffix = "") { + const arch = process.arch; + if (arch !== "x64" && arch !== "arm64") return undefined; + const nativePath = path.join("native", platform, "prebuilds", `${platform}-${arch}`, `${platform}-helper${suffix}.node`); + if (helpers.has(nativePath)) return helpers.get(nativePath); + for (const modulePath of getNativeModuleCandidates(nativePath)) { + try { + const helper = load(modulePath); + if (typeof helper?.getText === "function") { + helpers.set(nativePath, helper); + return helper; + } + } catch { + // Try the next location. + } + } + helpers.set(nativePath, undefined); + return undefined; +} + +function getPlatformHelper() { + if (process.platform !== "darwin" && process.platform !== "win32") return undefined; + return loadHelper(process.platform); +} + +// The clipboard helper opens the X display, so it loads only when one is set. +export function getNativeClipboard() { + if (process.platform !== "linux") return getPlatformHelper(); + if (!process.env.DISPLAY) return undefined; + return loadHelper("linux", "-x11"); +} + +export function isNativeModifierPressed(key) { + const helper = getPlatformHelper(); + if (!helper?.isModifierPressed) return false; + try { + return helper.isModifierPressed(key) === true; + } catch { + return false; + } +} diff --git a/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/native/linux/prebuilds/linux-arm64/linux-helper-x11.node b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/native/linux/prebuilds/linux-arm64/linux-helper-x11.node new file mode 100644 index 000000000..72d2c39bd --- /dev/null +++ b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/native/linux/prebuilds/linux-arm64/linux-helper-x11.node @@ -0,0 +1 @@ +placeholder for a prebuilt X11 clipboard helper diff --git a/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/native/linux/prebuilds/linux-x64/linux-helper-x11.node b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/native/linux/prebuilds/linux-x64/linux-helper-x11.node new file mode 100644 index 000000000..72d2c39bd --- /dev/null +++ b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/native/linux/prebuilds/linux-x64/linux-helper-x11.node @@ -0,0 +1 @@ +placeholder for a prebuilt X11 clipboard helper diff --git a/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/package.json b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/package.json new file mode 100644 index 000000000..475c1d5af --- /dev/null +++ b/tests/corpus/8212-native-helper-probe-no-display/node_modules/native-helper/package.json @@ -0,0 +1 @@ +{ "name": "native-helper", "version": "1.0.0", "type": "module", "main": "dist/index.js", "types": "dist/index.d.ts" } diff --git a/tests/fixtures/npm-static/addon-probe-cli.ts b/tests/fixtures/npm-static/addon-probe-cli.ts new file mode 100644 index 000000000..4515fb5bf --- /dev/null +++ b/tests/fixtures/npm-static/addon-probe-cli.ts @@ -0,0 +1,5 @@ +import { probe, resolveHelper } from "addon-probe"; +import { join } from "node:path"; + +console.log(join("a", "b"), [probe("clipboard"), probe("keys"), resolveHelper("addon-probe-helper")].join(",")); +console.log(resolveHelper("other", (id) => `custom:${id}`)); diff --git a/tests/fixtures/npm-static/addon-probe-installed-cli.ts b/tests/fixtures/npm-static/addon-probe-installed-cli.ts new file mode 100644 index 000000000..5568ce086 --- /dev/null +++ b/tests/fixtures/npm-static/addon-probe-installed-cli.ts @@ -0,0 +1,4 @@ +import { loadInstalled } from "addon-probe"; + +console.log("before"); +console.log(loadInstalled(process.argv[2] === "module" ? "module" : "addon")); diff --git a/tests/fixtures/npm-static/node_modules/addon-probe/index.d.ts b/tests/fixtures/npm-static/node_modules/addon-probe/index.d.ts new file mode 100644 index 000000000..7de7bc1fd --- /dev/null +++ b/tests/fixtures/npm-static/node_modules/addon-probe/index.d.ts @@ -0,0 +1,3 @@ +export declare function probe(name: string): string; +export declare function resolveHelper(name: string, resolvePackage?: (id: string) => string): string; +export declare function loadInstalled(kind: "addon" | "module"): string; diff --git a/tests/fixtures/npm-static/node_modules/addon-probe/index.js b/tests/fixtures/npm-static/node_modules/addon-probe/index.js new file mode 100644 index 000000000..b718d7e98 --- /dev/null +++ b/tests/fixtures/npm-static/node_modules/addon-probe/index.js @@ -0,0 +1,38 @@ +import { createRequire } from "node:module"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const load = createRequire(import.meta.url); +const here = dirname(fileURLToPath(import.meta.url)); + +// Looks for a prebuilt addon in each packaging location, the way +// packages with optional native helpers do. +export function probe(name) { + for (const dir of ["prebuilds", "native"]) { + try { + const addon = load(join(here, dir, `${name}.node`)); + if (typeof addon?.run === "function") return "loaded"; + } catch { + // Try the next location. + } + } + return "absent"; +} + +export function resolveHelper(name, resolvePackage) { + try { + return (resolvePackage ?? load.resolve)(name); + } catch { + return "unresolved"; + } +} + +// Requires a file that is installed: an addon, or a module that is not part +// of the compiled program. +export function loadInstalled(kind) { + try { + return typeof load(join(here, kind === "addon" ? "native/installed.node" : "installed.js")); + } catch { + return "fallback"; + } +} diff --git a/tests/fixtures/npm-static/node_modules/addon-probe/installed.js b/tests/fixtures/npm-static/node_modules/addon-probe/installed.js new file mode 100644 index 000000000..8476dc24e --- /dev/null +++ b/tests/fixtures/npm-static/node_modules/addon-probe/installed.js @@ -0,0 +1 @@ +export const installed = true; diff --git a/tests/fixtures/npm-static/node_modules/addon-probe/native/installed.node b/tests/fixtures/npm-static/node_modules/addon-probe/native/installed.node new file mode 100644 index 000000000..3d1ca6ec7 --- /dev/null +++ b/tests/fixtures/npm-static/node_modules/addon-probe/native/installed.node @@ -0,0 +1 @@ +placeholder for a prebuilt addon diff --git a/tests/fixtures/npm-static/node_modules/addon-probe/package.json b/tests/fixtures/npm-static/node_modules/addon-probe/package.json new file mode 100644 index 000000000..a2f7406f5 --- /dev/null +++ b/tests/fixtures/npm-static/node_modules/addon-probe/package.json @@ -0,0 +1,7 @@ +{ + "name": "addon-probe", + "version": "1.0.0", + "type": "module", + "main": "index.js", + "types": "index.d.ts" +} diff --git a/tests/harness/corpus-costs/8210-require-computed-missing/main.ts.json b/tests/harness/corpus-costs/8210-require-computed-missing/main.ts.json new file mode 100644 index 000000000..ef439d096 --- /dev/null +++ b/tests/harness/corpus-costs/8210-require-computed-missing/main.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 4500 } diff --git a/tests/harness/corpus-costs/8211-require-resolve-computed/main.ts.json b/tests/harness/corpus-costs/8211-require-resolve-computed/main.ts.json new file mode 100644 index 000000000..ef439d096 --- /dev/null +++ b/tests/harness/corpus-costs/8211-require-resolve-computed/main.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 4500 } diff --git a/tests/harness/corpus-costs/8212-native-helper-probe-no-display/main.ts.json b/tests/harness/corpus-costs/8212-native-helper-probe-no-display/main.ts.json new file mode 100644 index 000000000..ef439d096 --- /dev/null +++ b/tests/harness/corpus-costs/8212-native-helper-probe-no-display/main.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 4500 } diff --git a/tests/harness/npm-static.test.ts b/tests/harness/npm-static.test.ts index 388da2138..962dc75b8 100644 --- a/tests/harness/npm-static.test.ts +++ b/tests/harness/npm-static.test.ts @@ -423,13 +423,99 @@ describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { }, ); + test.each(["llvm"] as const)( + "a package probing for optional addons through createRequire resolves at run time (%s)", + async (backend) => { + const entry = join(pilotRoot, "addon-probe-cli.ts"); + const { coverage } = analyze(entry, { npmStatic: "auto" }); + expect(coverage.npmStatic).toEqual([{ package: "addon-probe", status: "static" }]); + expect(coverage.diagnostics).toHaveLength(0); + expect( + (coverage.runtimeFences ?? []).filter((f) => f.code === "SC2020").map((f) => f.message), + ).toEqual([ + expect.stringContaining( + "createRequire's require with a computed specifier resolves at run time", + ), + expect.stringContaining( + "createRequire's require.resolve with a computed specifier resolves at run time", + ), + ]); + const outDir = join(cacheDir, "addon-probe", sanitize ? "san" : "plain", backend); + const result = await compile(entry, { + backend, + npmStatic: "auto", + sanitize, + outDir, + outPath: join(outDir, "program"), + }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const [nodeRes, nativeRes] = await Promise.all([ + runBinary(process.execPath, [entry]), + runBinary(result.binaryPath, []), + ]); + expect(nativeRes.stdout.toString()).toBe("a/b absent,absent,unresolved\ncustom:other\n"); + expect(nativeRes.stdout).toEqual(nodeRes.stdout); + expect(comparableStderr(nativeRes.stderr)).toEqual(nodeRes.stderr); + expect(nativeRes.exitCode).toBe(nodeRes.exitCode); + }, + ); + + // Node would load the installed file, which the compiled program cannot: + // the require ends the program with the refusal instead of throwing, so + // the package's catch clause does not take its fallback. test.each([ [ - "computed specifier", - 'const spec = "node:path"; const path = load(spec);', - "computed specifier", + "addon", + /^scriptc: native addon '.*[\\/]native[\\/]installed\.node' requires Node's addon runtime, which is unavailable in scriptc \[SC2020 at .*index\.js:\d+\]\n$/, ], + [ + "module", + /^scriptc: createRequire's require of '.*[\\/]installed\.js' resolves to '.*[\\/]installed\.js', which is not part of the compiled program: the module graph is fixed at build time \[SC2020 at .*index\.js:\d+\]\n$/, + ], + ] as const)( + "a computed require of an installed %s ends the program with a refusal", + async (kind, stderr) => { + const entry = join(pilotRoot, "addon-probe-installed-cli.ts"); + const outDir = join(cacheDir, "addon-probe-installed", sanitize ? "san" : "plain"); + const result = await compile(entry, { + backend: "llvm", + npmStatic: "auto", + sanitize, + outDir, + outPath: join(outDir, "program"), + }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const nativeRes = await runBinary(result.binaryPath, [kind]); + expect(nativeRes.stdout.toString()).toBe("before\n"); + expect(comparableStderr(nativeRes.stderr).toString()).toMatch(stderr); + expect(nativeRes.exitCode).toBe(1); + }, + ); + + test.each([ ["escaped require", 'const alias = load; const path = alias("node:path");', "escapes"], + [ + "escaped require in a try block", + 'try { const alias = load; alias("node:path"); } catch {} const path = load("node:path");', + "escapes", + ], + [ + "computed resolve.paths request", + 'const spec = "node:path"; load.resolve.paths(spec); const path = load("node:path");', + "escapes", + ], + [ + "resolve.paths value", + 'const paths = load.resolve.paths; const path = load("node:path");', + "escapes", + ], + [ + "spread require arguments", + 'const specs = ["node:path"]; load(...specs); const path = load("node:path");', + "escapes", + ], [ "reassigned require", 'let load = createLoader(import.meta.url); load = () => ({}); const path = load("node:path");', From 9e4f1253c620270543a190b4f2f24e29c9744c28 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 10:11:29 -0500 Subject: [PATCH 02/20] Keep unchanged partitions cached across edits --- native/llvm-codegen/README.md | 2 +- native/llvm-codegen/src/emit.cpp | 39 ++++++++++++++++++- packages/compiler/src/backend/llvm/emitter.ts | 28 ++++++++----- .../src/backend/llvm/expr-callbacks.ts | 30 +++++++------- .../src/backend/llvm/expr-containers.ts | 3 +- .../src/backend/llvm/expr-primitives.ts | 2 +- .../src/backend/llvm/expr-stream-callbacks.ts | 12 +++--- .../compiler/src/backend/llvm/lib-connect.ts | 2 +- .../compiler/src/backend/llvm/lib-datagram.ts | 4 +- .../compiler/src/backend/llvm/lib-dispatch.ts | 2 +- .../compiler/src/backend/llvm/lib-http2.ts | 2 +- .../compiler/src/backend/llvm/lib-lookup.ts | 2 +- .../compiler/src/backend/llvm/lib-process.ts | 2 +- packages/compiler/src/backend/mangle.ts | 9 ----- .../lowering/class-dynamic-dispatch.ts | 3 +- 15 files changed, 89 insertions(+), 53 deletions(-) diff --git a/native/llvm-codegen/README.md b/native/llvm-codegen/README.md index 0e8abbc47..d72aebdfd 100644 --- a/native/llvm-codegen/README.md +++ b/native/llvm-codegen/README.md @@ -37,7 +37,7 @@ Repeating `--output` divides the program into that many partitions, which are co WebAssembly code generation differs from LLVM's default pipeline in two ways that do not change its output. The pass that gives multiple-entry loops a single entry runs only on functions whose control flow is irreducible; its reachability analysis grows with the square of a function's block count even when it has nothing to change. Object emission also leaves out the module pass that collects external symbols for assembly declarations, so each function's machine code is freed once it is printed instead of when the whole module is compiled. -`--cache-dir ` names a private directory the caller owns for one helper build. Simplified partitions are stored under the hash of their exact source and objects under the hash of the exact module after importing, each with a trailing digest and published by rename, so a rebuild recompiles only the partitions an edit can affect. +`--cache-dir ` names a private directory the caller owns for one helper build. Simplified partitions are stored under the hash of their exact source and objects under the hash of the exact module after importing, each with a trailing digest and published by rename, so a rebuild recompiles only the partitions an edit can affect. The locals ThinLTO promotes for importing are named after their partition rather than a hash of its bitcode, so an edit to one partition leaves the partitions that import from it unchanged. `--import-bitcode `, given once per runtime-pack unit that ships bitcode, imports runtime functions for inlining; scriptc's program builds do not pass it. Small runtime functions reachable from the program's runtime calls are imported as `available_externally` so LLVM can inline them; the runtime objects remain the only definitions. The helper never runs a sanitizer pass, so with import it also drops the emitter's inert `sanitize_address` attributes, which would otherwise block that inlining. Without `--import-bitcode` the module is optimized exactly as emitted. diff --git a/native/llvm-codegen/src/emit.cpp b/native/llvm-codegen/src/emit.cpp index 9cae164d3..4e8c7a30c 100644 --- a/native/llvm-codegen/src/emit.cpp +++ b/native/llvm-codegen/src/emit.cpp @@ -170,6 +170,38 @@ void limitProfileImports() { (void)Limited; } +// ThinLTO renames the locals it promotes so that the code other partitions +// import can refer to them: switch tables, constants and the other private +// symbols optimization creates. By default the new name carries a hash of +// the defining partition's whole bitcode, so an edit anywhere in a partition +// renames them in every partition that imports from it, and those partitions +// miss the object cache although their own code is unchanged. Named after +// the partition instead (PartitionSourceName), they keep their names while +// the partition changes. The option is process-wide; the helper sets it once. +void stablePromotedNames() { + static const bool Set = [] { + StringRef Name = "use-source-filename-for-promoted-locals"; + if (cl::Option *Option = cl::getRegisteredOptions().lookup(Name)) + Option->addOccurrence(0, Name, "true"); + return true; + }(); + (void)Set; +} + +// Gives a partition's module its own source file name, which the names of +// the locals ThinLTO promotes derive from (stablePromotedNames). It runs +// after the pre-link optimization, so profile annotation still matches the +// program's local functions under their source path. +struct PartitionSourceName : PassInfoMixin { + explicit PartitionSourceName(std::string Name) : Name(std::move(Name)) {} + std::string Name; + PreservedAnalyses run(Module &M, ModuleAnalysisManager &) { + M.setSourceFileName(Name); + return PreservedAnalyses::all(); + } + static bool isRequired() { return true; } +}; + // Reads the profile's header once, before any partition needs it: it must // hold IR-level counts. std::optional inspectProfile(const EmitOptions &Options) { @@ -559,8 +591,12 @@ compileOptimizedPartitions(const EmitOptions &Options, size_t Count = Plan.Modules.size(); std::vector> Summaries(Count); SharedFailure Failures; + stablePromotedNames(); forEachPartition(Count, [&](size_t I) { - std::string Key = hashParts({Salt, "summary", Plan.Modules[I]}); + // The partition's index names its promoted locals, so it keys the + // summary with the partition's source. + std::string Key = + hashParts({Salt, "summary", std::to_string(I), Plan.Modules[I]}); if ((Summaries[I] = Cache.lookup(Key, "bc"))) return; std::string LookupError; @@ -593,6 +629,7 @@ compileOptimizedPartitions(const EmitOptions &Options, // link; the post-link optimization reads the annotated weights. Pipeline P(*Machine, Level, profileOptions(Options)); ModulePassManager MPM = P.PB.buildThinLTOPreLinkDefaultPipeline(Level); + MPM.addPass(PartitionSourceName("scriptc-partition-" + std::to_string(I))); MPM.addPass(BitcodeWriterPass(Stream, /*ShouldPreserveUseListOrder=*/false, /*EmitSummaryIndex=*/true, /*EmitModuleHash=*/true)); diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 59b4b64a4..69ca09e71 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -2746,12 +2746,13 @@ export class LlEmitter { // ── plumbing (frame and scope ownership) ────── - /** A symbol derived from the constant's text rather than its first-use - * index, so an edit that interns one more string leaves every other - * constant's symbol, and the code referencing it, unchanged. This lets - * native builds reuse the compiled partitions an edit does not touch. A - * collision retries with a counter, still in first-use order. */ - private contentSymbol(prefix: string, text: string): string { + /** A symbol derived from a constant's text, or a helper's key, rather + * than its first-use index, so an edit that interns one more string or + * helper leaves every other symbol, and the code referencing it, + * unchanged. This lets native builds reuse the compiled partitions an + * edit does not touch. A collision retries with a counter, still in + * first-use order. */ + contentSymbol(prefix: string, text: string): string { for (let attempt = 0; ; attempt++) { const digest = createHash("sha256") .update(attempt === 0 ? text : `${attempt}\0${text}`) @@ -2877,7 +2878,7 @@ export class LlEmitter { const key = `${unionId}:${tag}`; let sym = this.unitInstances.get(key); if (!sym) { - sym = `sc_unit_${this.unitInstances.size}`; + sym = this.contentSymbol("sc_unit_", key); this.unitInstances.set(key, sym); } this.journal?.entry("unit", key, sym); @@ -4997,8 +4998,15 @@ export class LlEmitter { case "retainWrapper": addEntry(this.nullableUnions.retainWrappers, key, value as string); return; - case "unit": - return addNumbered(this.unitInstances, key, value as string, "sc_unit_", names); + case "unit": { + let own = this.unitInstances.get(key); + if (own === undefined) { + own = this.contentSymbol("sc_unit_", key); + this.unitInstances.set(key, own); + } + if (own !== value) names.set(value as string, own); + return; + } case "absent": return addNumbered(this.absentInstances, key, value as string, "sc_absent_", names); case "retype": @@ -5006,7 +5014,7 @@ export class LlEmitter { case "slots": return addNumbered(this.threadSlotsBlocks, key, value as string, "@sc_slots_", names); case "regex": - return addNumberedSymbol(this.regexInstances, key, value as never, "sc_re_", names); + return this.addContentEntry(this.regexInstances, key, value as never, "sc_re_", names); case "templateStrings": return addNumberedSymbol( this.templateStringsInstances, diff --git a/packages/compiler/src/backend/llvm/expr-callbacks.ts b/packages/compiler/src/backend/llvm/expr-callbacks.ts index 398aebb21..3d5a53402 100644 --- a/packages/compiler/src/backend/llvm/expr-callbacks.ts +++ b/packages/compiler/src/backend/llvm/expr-callbacks.ts @@ -8,7 +8,7 @@ import { typeEquals, typeKey, } from "../../ir/ir.js"; -import { mangleGenResThunk, mangleRecordNew, mangleRecordStruct } from "../mangle.js"; +import { mangleRecordNew, mangleRecordStruct } from "../mangle.js"; import { FN_ATTRS, llFieldType, releaseSym, retainSym, traceArg, vAdapters } from "./shapes.js"; import { LlvmUnsupportedError } from "./unsupported.js"; import type { LlvmEmitterContext } from "./expr-context.js"; @@ -54,7 +54,7 @@ export function raceAdapterFor(host: LlvmEmitterContext, from: IrType, to: IrTyp const key = `race:${typeKey(from)}=>${typeKey(to)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_race_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_race_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "raceAdapterFor", [from, to]); const toArms = to.kind === "union" ? (host.unionsById.get(to.unionId)?.arms ?? []) : []; @@ -217,7 +217,7 @@ export function genResultThunkFor( const key = `gr:${typeKey(genT)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = mangleGenResThunk(host.resolveThunks.size); + sym = host.contentSymbol("sc_gres_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "genResultThunkFor", [genT, recT]); const shape = host.recordsById.get(recT.shapeId); @@ -474,7 +474,7 @@ export function childExitThunkFor(host: LlvmEmitterContext, param: IrType): stri const key = `cx:${param.unionId}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_cx_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_cx_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "childExitThunkFor", [param]); const def = host.unionsById.get(param.unionId); @@ -519,7 +519,7 @@ export function childExitSignalThunkFor( const key = `cx2:${codeParam.unionId}+${sigParam.unionId}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_cx_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_cx_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "childExitSignalThunkFor", [codeParam, sigParam]); const codeDef = host.unionsById.get(codeParam.unionId); @@ -580,7 +580,7 @@ export function execFileThunkFor(host: LlvmEmitterContext, cbT: IrType & { kind: const key = `exec:${typeKey(cbT)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_exec_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_exec_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "execFileThunkFor", [cbT]); const errorParam = cbT.params[0]; @@ -652,7 +652,7 @@ export function ipcMessageThunkFor( const key = `ipcmsg:${typeKey(cbT)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_ipc_msg_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_ipc_msg_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "ipcMessageThunkFor", [cbT]); const param = cbT.params[0]; @@ -703,7 +703,7 @@ export function ipcSendThunkFor(host: LlvmEmitterContext, cbT: IrType & { kind: const key = `ipcsend:${typeKey(cbT)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_ipc_send_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_ipc_send_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "ipcSendThunkFor", [cbT]); const param = cbT.params[0]; @@ -764,7 +764,7 @@ export function childDataThunkFor(host: LlvmEmitterContext, param: IrType): stri const key = `cd:${param.unionId}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_cd_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_cd_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "childDataThunkFor", [param]); const def = host.unionsById.get(param.unionId); @@ -799,7 +799,7 @@ export function childDataDynThunkFor(host: LlvmEmitterContext, chunk: "bytes" | const key = `cdd:${chunk}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_cdd_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_cdd_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "childDataDynThunkFor", [chunk]); const box = chunk === "bytes" ? "scr_dyn_new_buffer" : "scr_dyn_new_str"; @@ -834,7 +834,7 @@ export function emitterFixedAdapter( const key = `ee:${typeKey(cbT)}`; let sym = host.resolveThunks.get(key); if (sym) return { fn: sym, shim }; - sym = `sc_ee_ad_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_ee_ad_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "emitterFixedAdapter", [cbT]); host.declare(`declare ptr @scr_box_get_ref(ptr)`); @@ -905,7 +905,7 @@ export function emitterOnceAdapter( : `eeOnce:${fire}:${typeKey(boxerT)}`; let sym = host.resolveThunks.get(key); if (sym) return { fn: sym, shim }; - sym = `sc_ee_once_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_ee_once_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "emitterOnceAdapter", [boxerT, fire]); host.declare(`declare ptr @scr_box_get_ref(ptr)`); @@ -1015,7 +1015,7 @@ export function closeBindThunkFor( const key = `ncb:${cbUnion.unionId}:${retServer ? "srv" : "void"}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_ncb_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_ncb_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "closeBindThunkFor", [cbUnion, retServer]); const def = host.unionsById.get(cbUnion.unionId); @@ -1121,7 +1121,7 @@ export function closeOverrideWrapFor( const key = `ncw:${cbUnion.unionId}:${retServer ? "srv" : "void"}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_ncw_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_ncw_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "closeOverrideWrapFor", [cbUnion, retServer]); const def = host.unionsById.get(cbUnion.unionId); @@ -1169,7 +1169,7 @@ export function settledStoreFor( const key = `settled:${typeKey(from)}=>${typeKey(to)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_settled_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_settled_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "settledStoreFor", [from, to]); const def = host.unionsById.get(to.unionId); diff --git a/packages/compiler/src/backend/llvm/expr-containers.ts b/packages/compiler/src/backend/llvm/expr-containers.ts index 279b8c044..8ffc8727e 100644 --- a/packages/compiler/src/backend/llvm/expr-containers.ts +++ b/packages/compiler/src/backend/llvm/expr-containers.ts @@ -3,7 +3,6 @@ import { arraySearchHelperName, arraySearchKind } from "./array-search.js"; import { InternalCompilerError } from "../../errors.js"; import { isStableReceiverOperand, undefinedArmTag } from "../../ir/analysis.js"; import { F64, type IrExpr, type IrType, isRefCounted, typeEquals, typeKey } from "../../ir/ir.js"; -import { mangleResolveThunk } from "../mangle.js"; import { arrOps, elemAccess, @@ -71,7 +70,7 @@ export function resolveThunkFor(host: LlvmEmitterContext, inner: IrType): string const key = typeKey(inner); let sym = host.resolveThunks.get(key); if (!sym) { - sym = mangleResolveThunk(host.resolveThunks.size); + sym = host.contentSymbol("sc_rslv_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "resolveThunkFor", [inner]); const v = vAdapters(host.shapeHost, inner); diff --git a/packages/compiler/src/backend/llvm/expr-primitives.ts b/packages/compiler/src/backend/llvm/expr-primitives.ts index ccbe3edcc..8ecf147f6 100644 --- a/packages/compiler/src/backend/llvm/expr-primitives.ts +++ b/packages/compiler/src/backend/llvm/expr-primitives.ts @@ -589,7 +589,7 @@ export function regexLiteralTemplate( let re = host.regexInstances.get(key); if (!re) { re = { - sym: `sc_re_${host.regexInstances.size}`, + sym: host.contentSymbol("sc_re_", key), src: host.internLiteral(e.pattern), fl: host.internLiteral(e.flags), pattern: e.pattern, diff --git a/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts b/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts index b368c91f4..285f9ca9a 100644 --- a/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts +++ b/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts @@ -13,7 +13,7 @@ export function streamDataAdapter( const key = `eed:${typeKey(cbT)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_ee_dad_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_ee_dad_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "streamDataAdapter", [cbT]); const p = cbT.params[0]; @@ -221,7 +221,7 @@ export function streamDoneFnFor( const key = `sd:${kind}:${typeKey(doneT)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_sd_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_sd_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "streamDoneFnFor", [kind, doneT]); const errT = doneT.params[0]; @@ -365,7 +365,7 @@ export function fsRenameThunkFor(host: LlvmEmitterContext, cbT: IrType & { kind: const key = `fsren:${typeKey(cbT)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_fsren_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_fsren_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "fsRenameThunkFor", [cbT]); const param = cbT.params[0]!; @@ -454,7 +454,7 @@ export function cryptoIntThunkFor( const key = `cryptoint:${typeKey(cbT)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_cryptoint_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_cryptoint_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "cryptoIntThunkFor", [cbT]); const d: string[] = [ @@ -516,7 +516,7 @@ export function zlibBytesThunkFor( const key = `${family}bytes:${arrayBuffer ? "arraybuffer:" : ""}${mode}${typeKey(cbT)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_${family}bytes_${host.resolveThunks.size}`; + sym = host.contentSymbol(`sc_${family}bytes_`, key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "zlibBytesThunkFor", [cbT, arrayBuffer, family, valueMode]); const d: string[] = [ @@ -644,7 +644,7 @@ export function streamCbThunkFor( const key = `scb:${kind}:${typeKey(cbT)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_scb_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_scb_", key); host.resolveThunks.set(key, sym); host.journal?.allocated(sym, "streamCbThunkFor", [kind, cbT]); const runtimeParams = diff --git a/packages/compiler/src/backend/llvm/lib-connect.ts b/packages/compiler/src/backend/llvm/lib-connect.ts index 4d151fc7e..2ab72c188 100644 --- a/packages/compiler/src/backend/llvm/lib-connect.ts +++ b/packages/compiler/src/backend/llvm/lib-connect.ts @@ -17,7 +17,7 @@ function connectAdapter(host: LlvmEmitterContext, cb: LlValue, h2: boolean): str const key = `connect:${h2}:${typeKey(type)}`; const previous = host.resolveThunks.get(key); if (previous !== undefined) return previous; - const symbol = `sc_connect_${host.resolveThunks.size}`; + const symbol = host.contentSymbol("sc_connect_", key); host.resolveThunks.set(key, symbol); host.journal?.allocated(symbol, "connectAdapter", [cb, h2]); const retain = h2 ? "scr_http_res_retain_v" : "scr_net_sock_retain_v"; diff --git a/packages/compiler/src/backend/llvm/lib-datagram.ts b/packages/compiler/src/backend/llvm/lib-datagram.ts index 07de4e54e..7a83f1d30 100644 --- a/packages/compiler/src/backend/llvm/lib-datagram.ts +++ b/packages/compiler/src/backend/llvm/lib-datagram.ts @@ -19,7 +19,7 @@ function messageAdapter(host: LlvmEmitterContext, type: IrType): string { const key = "dgram:" + type.shapeId; const previous = host.resolveThunks.get(key); if (previous !== undefined) return previous; - const symbol = `sc_dgram_message_${host.resolveThunks.size}`; + const symbol = host.contentSymbol("sc_dgram_message_", key); host.resolveThunks.set(key, symbol); host.journal?.allocated(symbol, "messageAdapter", [type]); const shape = host.recordsById.get(type.shapeId); @@ -68,7 +68,7 @@ function lookupAdapter(host: LlvmEmitterContext, cb: LlValue): string { const key = `dns:${error.unionId}:${arity}`; const previous = host.resolveThunks.get(key); if (previous !== undefined) return previous; - const symbol = `sc_dns_lookup_${host.resolveThunks.size}`; + const symbol = host.contentSymbol("sc_dns_lookup_", key); host.resolveThunks.set(key, symbol); host.journal?.allocated(symbol, "lookupAdapter", [cb]); host.declare("declare ptr @scr_error_new(i32, ptr)"); diff --git a/packages/compiler/src/backend/llvm/lib-dispatch.ts b/packages/compiler/src/backend/llvm/lib-dispatch.ts index 9cd11feb1..6ab83016d 100644 --- a/packages/compiler/src/backend/llvm/lib-dispatch.ts +++ b/packages/compiler/src/backend/llvm/lib-dispatch.ts @@ -1085,7 +1085,7 @@ function emitPromiseAllTuple(host: LlvmEmitterContext, e: LibCallExpr): LlValue const key = `allTuple:${build.fnName}`; let adapter = host.resolveThunks.get(key); if (adapter === undefined) { - adapter = `sc_all_tuple_${host.resolveThunks.size}`; + adapter = host.contentSymbol("sc_all_tuple_", key); host.resolveThunks.set(key, adapter); const rc = vAdapters(host.shapeHost, tuple); host.declare(`declare ptr @scr_promise_payload_ref(ptr)`); diff --git a/packages/compiler/src/backend/llvm/lib-http2.ts b/packages/compiler/src/backend/llvm/lib-http2.ts index 321f1d005..1808200a0 100644 --- a/packages/compiler/src/backend/llvm/lib-http2.ts +++ b/packages/compiler/src/backend/llvm/lib-http2.ts @@ -21,7 +21,7 @@ function sniAnswer(host: LlvmEmitterContext, type: IrType): string { const key = "sni:" + typeKey(type); const previous = host.resolveThunks.get(key); if (previous !== undefined) return previous; - const symbol = `sc_sni_answer_${host.resolveThunks.size}`; + const symbol = host.contentSymbol("sc_sni_answer_", key); host.resolveThunks.set(key, symbol); host.journal?.allocated(symbol, "sniAnswer", [type]); const params = ["ptr %self"], diff --git a/packages/compiler/src/backend/llvm/lib-lookup.ts b/packages/compiler/src/backend/llvm/lib-lookup.ts index 56e94461f..620833819 100644 --- a/packages/compiler/src/backend/llvm/lib-lookup.ts +++ b/packages/compiler/src/backend/llvm/lib-lookup.ts @@ -37,7 +37,7 @@ function lookupAnswer(host: LlvmEmitterContext, answer: IrType & { kind: "func" const key = "lookup:" + typeKey(answer); let symbol = host.resolveThunks.get(key); if (symbol === undefined) { - symbol = `sc_lookup_answer_${host.resolveThunks.size}`; + symbol = host.contentSymbol("sc_lookup_answer_", key); host.resolveThunks.set(key, symbol); host.journal?.allocated(symbol, "lookupAnswer", [answer]); for (const declaration of [ diff --git a/packages/compiler/src/backend/llvm/lib-process.ts b/packages/compiler/src/backend/llvm/lib-process.ts index 1a3973b07..21205c9f6 100644 --- a/packages/compiler/src/backend/llvm/lib-process.ts +++ b/packages/compiler/src/backend/llvm/lib-process.ts @@ -489,7 +489,7 @@ function boundCallbackThunk(host: LlvmEmitterContext, cbT: IrType & { kind: "fun const key = `alsbound:${typeKey(cbT)}`; let sym = host.resolveThunks.get(key); if (sym) return sym; - sym = `sc_alsbound_${host.resolveThunks.size}`; + sym = host.contentSymbol("sc_alsbound_", key); host.resolveThunks.set(key, sym); host.declare("declare ptr @scr_als_bound_target(ptr)"); host.declare("declare ptr @scr_als_bound_enter(ptr)"); diff --git a/packages/compiler/src/backend/mangle.ts b/packages/compiler/src/backend/mangle.ts index 14736ead7..8542a1c64 100644 --- a/packages/compiler/src/backend/mangle.ts +++ b/packages/compiler/src/backend/mangle.ts @@ -221,15 +221,6 @@ export function mangleGenDrop(fnName: string): string { export function mangleGenSourceClose(fnName: string): string { return `sc_gc_${sanitize(fnName)}`; } -/** Emitted ref-kind resolve thunk, interned per inner-type key. */ -export function mangleResolveThunk(n: number): string { - return `sc_rslv_${n}`; -} -/** Emitted generator-resume result builder (the IteratorResult record), - * interned per generator-type key. */ -export function mangleGenResThunk(n: number): string { - return `sc_gres_${n}`; -} /** Emitted Promise.race fulfillment adapter (entry inner type → result * inner type), interned per type pair. */ export function mangleRaceThunk(n: number): string { diff --git a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts index fc6d0a922..0789b6d4f 100644 --- a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts +++ b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts @@ -80,6 +80,7 @@ import { isClassCallback } from "./class-callbacks.js"; import { CaughtValueDispatch } from "./caught-value-dispatch.js"; import { refreshDescriptorGuards } from "./class-descriptors.js"; import { ClassConstructionDispatch } from "./class-construction.js"; +import { keyedHelperName } from "./stable-names.js"; import { DispatchSites } from "./dispatch-sites.js"; import { emitterRooted, @@ -3909,7 +3910,7 @@ export class ClassDynamicDispatch { if (!dispatch.fn.params.some((param) => param.localId === dispatch.keyLocal)) params.push({ localId: dispatch.keyLocal, name: "key", type: STRING }); const helper: IrFunction = { - name: `%dyn.class.keys.${lowerer.liftedFns.length}`, + name: keyedHelperName(lowerer, "%dyn.class.keys", helperKey), params, returnType: dispatch.fn.returnType, locals: params.map((p) => ({ From fe6dd9a5c5fa057f799fc7072e92bc2772c96785 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 17:38:25 -0500 Subject: [PATCH 03/20] Skip a byte order mark at the start of package.json The run-time resolver parsed package.json as it read it, so a file that starts with a UTF-8 byte order mark failed to parse and computed require and require.resolve ended with the invalid-package-config refusal. Node's reader skips the mark; the resolver now does too. The corpus program writes the package with the marked package.json to a fresh directory at run time and resolves it by its absolute path, so no committed file contains the mark. --- packages/runtime/src/scr_require.c | 7 +++++++ .../8211-require-resolve-computed/main.ts | 6 ++++-- .../node_modules/resolve-probe/index.js | 21 ++++++++++++++++++- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/packages/runtime/src/scr_require.c b/packages/runtime/src/scr_require.c index 645273bce..d1ede2186 100644 --- a/packages/runtime/src/scr_require.c +++ b/packages/runtime/src/scr_require.c @@ -138,6 +138,13 @@ static ScrDyn *req_read_package(ScrRequire *r, ScrStr *pjson) { if (scr_exc_pending()) scr_exc_clear(); return NULL; } + /* Node's reader skips a leading UTF-8 byte order mark. */ + if (text->len >= 3 && (unsigned char)text->data[0] == 0xEF && + (unsigned char)text->data[1] == 0xBB && (unsigned char)text->data[2] == 0xBF) { + ScrStr *body = scr_str_new(text->data + 3, text->len - 3); + scr_str_release(text); + text = body; + } ScrDyn *pkg = scr_json_parse(text); scr_str_release(text); if (pkg == NULL) { diff --git a/tests/corpus/8211-require-resolve-computed/main.ts b/tests/corpus/8211-require-resolve-computed/main.ts index d8e441d2f..ab94d0625 100644 --- a/tests/corpus/8211-require-resolve-computed/main.ts +++ b/tests/corpus/8211-require-resolve-computed/main.ts @@ -1,7 +1,9 @@ // require.resolve with requests computed at run time, called directly and // read as a value, resolves like Node: builtins, relative files with -// Node's extension search, package "main" and "exports" entries, a -// package's own name, and MODULE_NOT_FOUND for requests that name nothing. +// Node's extension search, package "main" and "exports" entries (a +// package.json that starts with a byte order mark included, written at run +// time), a package's own name, and MODULE_NOT_FOUND for requests that name +// nothing. import { resolveAll } from "resolve-probe"; for (const line of resolveAll()) console.log(line); diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/index.js b/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/index.js index 870a14401..4be08c224 100644 --- a/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/index.js +++ b/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/index.js @@ -1,5 +1,7 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { createRequire } from "node:module"; -import { dirname, relative } from "node:path"; +import { tmpdir } from "node:os"; +import { dirname, join, relative } from "node:path"; import { fileURLToPath } from "node:url"; const importer = fileURLToPath(import.meta.url); @@ -39,6 +41,22 @@ const requests = [ 42, ]; +// A package whose package.json starts with a byte order mark, written to a +// fresh directory and resolved by its absolute path: "main" is still read. +function resolveWithByteOrderMark() { + const dir = mkdtempSync(join(tmpdir(), "scriptc-bom-")); + try { + const pkg = join(dir, "with-bom"); + mkdirSync(join(pkg, "lib"), { recursive: true }); + const manifest = JSON.stringify({ name: "with-bom", main: "lib/entry.js" }); + writeFileSync(join(pkg, "package.json"), "\uFEFF" + manifest); + writeFileSync(join(pkg, "lib", "entry.js"), 'module.exports = "with-bom";\n'); + return relative(pkg, load.resolve(pkg)); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + export function resolveAll(options = {}) { const resolve = options.resolve ?? load.resolve; const label = (request) => (typeof request === "string" ? JSON.stringify(request) : String(request)); @@ -46,5 +64,6 @@ export function resolveAll(options = {}) { ...requests.map((request) => `call ${label(request)} -> ${describe((r) => load.resolve(r), request)}`), ...requests.slice(0, 4).map((request) => `value ${label(request)} -> ${describe(resolve, request)}`), `options ${describe((r) => load.resolve(r, {}), "with-main")}`, + `byte order mark -> ${resolveWithByteOrderMark()}`, ]; } From 65555dace3072de3546caeb9f525cea72dbfb221 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 17:47:58 -0500 Subject: [PATCH 04/20] Drop struct type names before the thin link instead of renaming promoted locals The partitions of a program build have no locals for ThinLTO to promote: symbols that other partitions refer to are already hidden. What changed the modules after importing, and with them the object cache keys, of partitions an edit did not touch was the numbering of struct types. Every partition declares the program's types under the same names, and the thin link loads the importing partition and the partitions it imports from into one context, which tells same-named types apart by appending a counter that runs across the whole link. An edit that adds a type anywhere shifted the counter. Partitions now drop the names of their struct types before their summary is written; importing matches unnamed types by their bodies. The promoted-locals option, the per-partition source file name and the partition index in the summary key are removed. --- native/llvm-codegen/README.md | 2 +- native/llvm-codegen/src/emit.cpp | 45 +++++++++----------------------- 2 files changed, 14 insertions(+), 33 deletions(-) diff --git a/native/llvm-codegen/README.md b/native/llvm-codegen/README.md index d72aebdfd..70c118202 100644 --- a/native/llvm-codegen/README.md +++ b/native/llvm-codegen/README.md @@ -37,7 +37,7 @@ Repeating `--output` divides the program into that many partitions, which are co WebAssembly code generation differs from LLVM's default pipeline in two ways that do not change its output. The pass that gives multiple-entry loops a single entry runs only on functions whose control flow is irreducible; its reachability analysis grows with the square of a function's block count even when it has nothing to change. Object emission also leaves out the module pass that collects external symbols for assembly declarations, so each function's machine code is freed once it is printed instead of when the whole module is compiled. -`--cache-dir ` names a private directory the caller owns for one helper build. Simplified partitions are stored under the hash of their exact source and objects under the hash of the exact module after importing, each with a trailing digest and published by rename, so a rebuild recompiles only the partitions an edit can affect. The locals ThinLTO promotes for importing are named after their partition rather than a hash of its bitcode, so an edit to one partition leaves the partitions that import from it unchanged. +`--cache-dir ` names a private directory the caller owns for one helper build. Simplified partitions are stored under the hash of their exact source and objects under the hash of the exact module after importing, each with a trailing digest and published by rename, so a rebuild recompiles only the partitions an edit can affect. Partitions drop the names of their struct types before the thin link, which would otherwise number same-named types from different partitions in an order that any edit can shift, changing the module after importing in partitions the edit did not touch. `--import-bitcode `, given once per runtime-pack unit that ships bitcode, imports runtime functions for inlining; scriptc's program builds do not pass it. Small runtime functions reachable from the program's runtime calls are imported as `available_externally` so LLVM can inline them; the runtime objects remain the only definitions. The helper never runs a sanitizer pass, so with import it also drops the emitter's inert `sanitize_address` attributes, which would otherwise block that inlining. Without `--import-bitcode` the module is optimized exactly as emitted. diff --git a/native/llvm-codegen/src/emit.cpp b/native/llvm-codegen/src/emit.cpp index 4e8c7a30c..c087dead8 100644 --- a/native/llvm-codegen/src/emit.cpp +++ b/native/llvm-codegen/src/emit.cpp @@ -170,33 +170,18 @@ void limitProfileImports() { (void)Limited; } -// ThinLTO renames the locals it promotes so that the code other partitions -// import can refer to them: switch tables, constants and the other private -// symbols optimization creates. By default the new name carries a hash of -// the defining partition's whole bitcode, so an edit anywhere in a partition -// renames them in every partition that imports from it, and those partitions -// miss the object cache although their own code is unchanged. Named after -// the partition instead (PartitionSourceName), they keep their names while -// the partition changes. The option is process-wide; the helper sets it once. -void stablePromotedNames() { - static const bool Set = [] { - StringRef Name = "use-source-filename-for-promoted-locals"; - if (cl::Option *Option = cl::getRegisteredOptions().lookup(Name)) - Option->addOccurrence(0, Name, "true"); - return true; - }(); - (void)Set; -} - -// Gives a partition's module its own source file name, which the names of -// the locals ThinLTO promotes derive from (stablePromotedNames). It runs -// after the pre-link optimization, so profile annotation still matches the -// program's local functions under their source path. -struct PartitionSourceName : PassInfoMixin { - explicit PartitionSourceName(std::string Name) : Name(std::move(Name)) {} - std::string Name; +// Drops the names of a partition's struct types before its summary is +// written. Every partition declares the program's types under the same +// names, and a thin link loads the importing partition and the partitions +// it imports from into one context, which tells same-named types apart by +// appending a counter that runs across the whole link. An edit that adds a +// type anywhere shifts the counter, so the module after importing, the +// object cache key, changed in partitions whose code had not. Unnamed types +// carry no such suffix; importing still matches them by their bodies. +struct UnnameStructTypes : PassInfoMixin { PreservedAnalyses run(Module &M, ModuleAnalysisManager &) { - M.setSourceFileName(Name); + for (StructType *Type : M.getIdentifiedStructTypes()) + Type->setName(""); return PreservedAnalyses::all(); } static bool isRequired() { return true; } @@ -591,12 +576,8 @@ compileOptimizedPartitions(const EmitOptions &Options, size_t Count = Plan.Modules.size(); std::vector> Summaries(Count); SharedFailure Failures; - stablePromotedNames(); forEachPartition(Count, [&](size_t I) { - // The partition's index names its promoted locals, so it keys the - // summary with the partition's source. - std::string Key = - hashParts({Salt, "summary", std::to_string(I), Plan.Modules[I]}); + std::string Key = hashParts({Salt, "summary", Plan.Modules[I]}); if ((Summaries[I] = Cache.lookup(Key, "bc"))) return; std::string LookupError; @@ -629,7 +610,7 @@ compileOptimizedPartitions(const EmitOptions &Options, // link; the post-link optimization reads the annotated weights. Pipeline P(*Machine, Level, profileOptions(Options)); ModulePassManager MPM = P.PB.buildThinLTOPreLinkDefaultPipeline(Level); - MPM.addPass(PartitionSourceName("scriptc-partition-" + std::to_string(I))); + MPM.addPass(UnnameStructTypes()); MPM.addPass(BitcodeWriterPass(Stream, /*ShouldPreserveUseListOrder=*/false, /*EmitSummaryIndex=*/true, /*EmitModuleHash=*/true)); From 4eb3a16a0b0de657f7b4de5253e2545f1a0a2682 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 05:52:34 -0500 Subject: [PATCH 05/20] Match the Promise.all tuple adapter by its key-derived name in the chunked-emission test The adapter is named from its key since the helpers moved to contentSymbol, so the test no longer finds the counter name sc_all_tuple_0. It now matches any sc_all_tuple_ name; the serial and chunked emissions are still compared in full. --- packages/compiler/test/chunked-emission.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/compiler/test/chunked-emission.test.ts b/packages/compiler/test/chunked-emission.test.ts index 6d2b2e06a..703669648 100644 --- a/packages/compiler/test/chunked-emission.test.ts +++ b/packages/compiler/test/chunked-emission.test.ts @@ -95,7 +95,8 @@ test("a Promise.all tuple adapter requested in a range is defined", async () => throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); const ir = await readFile(outPath, "utf8"); const serial = text(emitLlvmModuleSource(deserializeModule(ir), {})); - expect(serial).toContain("define internal void @sc_all_tuple_0("); + // The adapter is named from its key (contentSymbol). + expect(serial).toMatch(/define internal void @sc_all_tuple_\d+\(/); const chunked = text(emitLlvmModuleSource(deserializeModule(ir), { functionChunks: 2 })); expect(chunked === serial).toBe(true); } finally { From c5131cb958e4331770db8e3b23a6aec1ba733be1 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 13:39:37 -0500 Subject: [PATCH 06/20] Name the byte order mark fixture's directory by its real path The 8211 program resolved a package written to a fresh directory under os.tmpdir() and printed the result relative to that directory. On macOS the temporary directory is a symbolic link (/var to /private/var), so the resolved real path printed relative to the linked name included the random directory suffix, which differs between the Node run and the compiled run. The fixture now names the directory by its real path, and the result is lib/entry.js on every host. --- .../node_modules/resolve-probe/index.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/index.js b/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/index.js index 4be08c224..8d581c90e 100644 --- a/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/index.js +++ b/tests/corpus/8211-require-resolve-computed/node_modules/resolve-probe/index.js @@ -1,4 +1,4 @@ -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs"; import { createRequire } from "node:module"; import { tmpdir } from "node:os"; import { dirname, join, relative } from "node:path"; @@ -43,8 +43,10 @@ const requests = [ // A package whose package.json starts with a byte order mark, written to a // fresh directory and resolved by its absolute path: "main" is still read. +// The directory is named by its real path (the temporary directory is a +// symbolic link on macOS), so the resolved path is relative to it. function resolveWithByteOrderMark() { - const dir = mkdtempSync(join(tmpdir(), "scriptc-bom-")); + const dir = realpathSync(mkdtempSync(join(tmpdir(), "scriptc-bom-"))); try { const pkg = join(dir, "with-bom"); mkdirSync(join(pkg, "lib"), { recursive: true }); From 5a3d6d79037bf839e50f70adc828e7c6625b658f Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sat, 24 Oct 2026 14:53:54 -0700 Subject: [PATCH 07/20] Call function tables without boxing or reference counting Values typed `F | undefined` for a function type F are now plain closure pointers (nullable-pointer unions), so a keyed read of a `Record` or a Map.get no longer allocates a union box. Keyed record reads, string-keyed Map.get and checked extractions lend the stored closure as a receiver or callee when ReferenceWrites proves the call cannot write the locations it was read from, and optional-chain receivers lend under the same proof. Owned record overflow reads use the inline entry lookup. Indirect closure calls skip the ambient receiver bracket for closures whose bodies never observe it (new ScrClosure.function_kind bit SCR_FN_RECEIVER_FREE; signature adapters that only forward the receiver copy it from their original), and a callValue that forwards `dyn.this` needs no bracket at all. Co-Authored-By: Claude Opus 5.5 --- benchmarks/runtime/README.md | 1 + .../runtime/workloads/handler-tables.json | 6 + .../runtime/workloads/handler-tables.ts | 108 ++++++++++++++++ .../src/backend/llvm/checked-narrows.ts | 9 +- packages/compiler/src/backend/llvm/common.ts | 4 + .../backend/llvm/constant-callbacks.test.ts | 41 +++++++ packages/compiler/src/backend/llvm/emitter.ts | 61 ++++++++-- .../src/backend/llvm/expr-callbacks.ts | 20 ++- .../compiler/src/backend/llvm/expr-calls.ts | 115 ++++++++++++++++-- .../compiler/src/backend/llvm/expr-control.ts | 33 +++-- .../compiler/src/backend/llvm/expr-dynamic.ts | 6 +- .../compiler/src/backend/llvm/expr-records.ts | 107 ++++++++++++---- .../compiler/src/backend/llvm/lib-http2.ts | 5 +- .../compiler/src/backend/llvm/lib-network.ts | 2 +- .../src/backend/llvm/map-read-lifetimes.ts | 64 ++++++++++ .../src/backend/llvm/nullable-unions.test.ts | 21 +++- .../src/backend/llvm/nullable-unions.ts | 22 +++- .../compiler/src/backend/llvm/receivers.ts | 40 +++++- .../src/backend/llvm/reference-writes.test.ts | 43 +++++++ .../src/backend/llvm/reference-writes.ts | 13 ++ .../src/backend/llvm/stack-callbacks.ts | 6 +- packages/compiler/src/ir/receiver-readers.ts | 92 ++++++++++---- .../8230-function-record-dispatch.ts.json | 6 + .../corpus/8231-function-table-shapes.ts.json | 6 + .../8232-optional-function-values.ts.json | 6 + packages/runtime/src/scr_closure.h | 9 +- packages/runtime/src/scr_json.c | 9 +- tests/corpus/8230-function-record-dispatch.ts | 107 ++++++++++++++++ tests/corpus/8231-function-table-shapes.ts | 108 ++++++++++++++++ tests/corpus/8232-optional-function-values.ts | 75 ++++++++++++ .../8230-function-record-dispatch.ts.json | 1 + .../8231-function-table-shapes.ts.json | 1 + .../8232-optional-function-values.ts.json | 1 + 33 files changed, 1042 insertions(+), 106 deletions(-) create mode 100644 benchmarks/runtime/workloads/handler-tables.json create mode 100644 benchmarks/runtime/workloads/handler-tables.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8230-function-record-dispatch.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8231-function-table-shapes.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8232-optional-function-values.ts.json create mode 100644 tests/corpus/8230-function-record-dispatch.ts create mode 100644 tests/corpus/8231-function-table-shapes.ts create mode 100644 tests/corpus/8232-optional-function-values.ts create mode 100644 tests/harness/corpus-costs/8230-function-record-dispatch.ts.json create mode 100644 tests/harness/corpus-costs/8231-function-table-shapes.ts.json create mode 100644 tests/harness/corpus-costs/8232-optional-function-values.ts.json diff --git a/benchmarks/runtime/README.md b/benchmarks/runtime/README.md index af3d3d239..4c6cf45ed 100644 --- a/benchmarks/runtime/README.md +++ b/benchmarks/runtime/README.md @@ -16,6 +16,7 @@ Application-shaped workloads for measuring the speed of compiled executables. Ea | `dynamic-methods` | cpu | untyped JavaScript calling builtin methods on values it does not type: string `slice`/`indexOf`/`toUpperCase`/`padStart`, array `push`/`pop`/`includes`, `Map` and `Set`, and an object's own methods | | `dynamic-properties` | cpu | untyped JavaScript reading and writing properties of values it does not type: records built by object literals in two key orders, nested records, and instances of a class and its subclass | | `functional-pipeline` | cpu | `filter`/`map`/`reduce`/`flatMap` chains, closures, small records | +| `handler-tables` | cpu | dispatch tables of functions: `Record` read as `t[k]!(…)`, `t[k]?.(…)` and through a checked local, a `Map`, an array of handlers, and an object literal of methods; some handlers declare fewer parameters than the table type | | `intern-table`, `intern-table-map`, `intern-table-1`, `intern-table-map-1` | cpu | worker threads interning the same type-like objects under string keys and records under number keys, then hits and misses: one shared `InternTable` per kind of key, or a module-level `Map` per thread (the `-map` entries), with 8 threads or 1 (the `-1` entries); under Node.js every thread has its own tables | | `json-records` | cpu | `JSON.stringify`, typed `JSON.parse`, Map aggregation | | `json-unions` | cpu | `JSON.parse` into discriminated unions: an event log with eight kinds and nested expression trees with nine, evaluated after the cast | diff --git a/benchmarks/runtime/workloads/handler-tables.json b/benchmarks/runtime/workloads/handler-tables.json new file mode 100644 index 000000000..1408ae91e --- /dev/null +++ b/benchmarks/runtime/workloads/handler-tables.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/handler-tables.ts", + "args": [ + "4" + ] +} diff --git a/benchmarks/runtime/workloads/handler-tables.ts b/benchmarks/runtime/workloads/handler-tables.ts new file mode 100644 index 000000000..b0a61fcfe --- /dev/null +++ b/benchmarks/runtime/workloads/handler-tables.ts @@ -0,0 +1,108 @@ +// Dispatch tables of functions: a `Record` of opcode handlers +// called as `ops[name]!(…)`, `ops[name]?.(…)` and through a checked local, +// a `Map` of the same handlers, an array of handlers indexed by +// opcode number, and an object literal of methods. Some handlers declare +// fewer parameters than the table's function type. +type Op = (acc: number, arg: number) => number; + +const ops: Record = { + add: (acc, arg) => acc + arg, + sub: (acc, arg) => acc - arg, + mul: (acc, arg) => (acc * arg) % 1000003, + neg: (acc) => -acc, + half: (acc) => Math.floor(acc / 2), + mix: (acc, arg) => (acc ^ arg) & 0xffff, +}; + +const byName = new Map(Object.entries(ops)); +const byCode: Op[] = Object.values(ops); +const names = Object.keys(ops); + +const methods = { + add(acc: number, arg: number): number { + return acc + arg; + }, + sub(acc: number, arg: number): number { + return acc - arg; + }, + mul(acc: number, arg: number): number { + return (acc * arg) % 1000003; + }, +}; + +interface Insn { + name: string; + code: number; + arg: number; +} + +function program(length: number): Insn[] { + const out: Insn[] = []; + let seed = 12345; + for (let i = 0; i < length; i++) { + seed = (seed * 1103515245 + 12345) % 2147483648; + const code = seed % (names.length + 1); + // One in seven instructions names an opcode no table holds. + out.push({ name: code < names.length ? names[code]! : "nop", code, arg: (seed >> 8) % 97 }); + } + return out; +} + +function runAsserted(insns: Insn[]): number { + let acc = 1; + for (const insn of insns) if (insn.name !== "nop") acc = ops[insn.name]!(acc, insn.arg); + return acc; +} + +function runOptional(insns: Insn[]): number { + let acc = 1; + for (const insn of insns) acc = ops[insn.name]?.(acc, insn.arg) ?? acc + 1; + return acc; +} + +function runChecked(insns: Insn[]): number { + let acc = 1; + for (const insn of insns) { + const handler = ops[insn.name]; + if (handler) acc = handler(acc, insn.arg); + } + return acc; +} + +function runMap(insns: Insn[]): number { + let acc = 1; + for (const insn of insns) { + const handler = byName.get(insn.name); + if (handler !== undefined) acc = handler(acc, insn.arg); + } + return acc; +} + +function runArray(insns: Insn[]): number { + let acc = 1; + for (const insn of insns) if (insn.code < byCode.length) acc = byCode[insn.code]!(acc, insn.arg); + return acc; +} + +function runMethods(insns: Insn[]): number { + let acc = 1; + for (const insn of insns) { + if (insn.code === 0) acc = methods.add(acc, insn.arg); + else if (insn.code === 1) acc = methods.sub(acc, insn.arg); + else if (insn.code === 2) acc = methods["mul"](acc, insn.arg); + } + return acc; +} + +const rounds = Number(process.argv[2] ?? "4"); +const insns = program(200_000); +const totals = [0, 0, 0, 0, 0, 0]; +for (let round = 0; round < rounds * 10; round++) { + totals[0] = (totals[0]! + runAsserted(insns)) % 1e9; + totals[1] = (totals[1]! + runOptional(insns)) % 1e9; + totals[2] = (totals[2]! + runChecked(insns)) % 1e9; + totals[3] = (totals[3]! + runMap(insns)) % 1e9; + totals[4] = (totals[4]! + runArray(insns)) % 1e9; + totals[5] = (totals[5]! + runMethods(insns)) % 1e9; +} +console.log(totals.join(" ")); diff --git a/packages/compiler/src/backend/llvm/checked-narrows.ts b/packages/compiler/src/backend/llvm/checked-narrows.ts index c1514df1e..5fbaf6d08 100644 --- a/packages/compiler/src/backend/llvm/checked-narrows.ts +++ b/packages/compiler/src/backend/llvm/checked-narrows.ts @@ -101,6 +101,13 @@ export class CheckedNarrows { } } +/** A call to a checked extraction whose helper borrows its argument, so + * emitCheckedNarrowReceiver can lend the payload instead of calling it. */ +export function borrowsCheckedNarrow(host: LlvmEmitterContext, e: IrExpr): boolean { + if (e.kind !== "call" || host.checkedNarrows.get(e) === null) return false; + return host.callLifetimes.borrowed.get(e.callee)?.has(0) === true && host.mayThrow.has(e.callee); +} + /** A checked extraction consumed at once by a field or tag load. The * successful arm borrows the payload from the union, which itself borrows * from its source: a dense array read keeps the element in the array slot, @@ -113,7 +120,7 @@ export function emitCheckedNarrowReceiver( narrow: CheckedNarrowCall, ): LlValue | null { const callee = narrow.call.callee; - if (!host.callLifetimes.borrowed.get(callee)?.has(0) || !host.mayThrow.has(callee)) return null; + if (!borrowsCheckedNarrow(host, narrow.call)) return null; const B = host.B; const read = host.optionalArrayReads.get(narrow.value); const union = read diff --git a/packages/compiler/src/backend/llvm/common.ts b/packages/compiler/src/backend/llvm/common.ts index cca1b603b..f1bdeb557 100644 --- a/packages/compiler/src/backend/llvm/common.ts +++ b/packages/compiler/src/backend/llvm/common.ts @@ -277,6 +277,10 @@ export function emitFieldAbsentTest(B: BlockBuilder, value: string, undefinedTag return absent; } +/** ScrClosure.function_kind's SCR_FN_RECEIVER_FREE bit (scr_closure.h): + * the body never observes the ambient receiver. */ +export const CLOSURE_RECEIVER_FREE = 16; + /** The right-hand side of a JS function-identity comparison: two closures * are one function when their identity roots match (a signature adapter * stands for the function it adapts). Both operands must be closures. */ diff --git a/packages/compiler/src/backend/llvm/constant-callbacks.test.ts b/packages/compiler/src/backend/llvm/constant-callbacks.test.ts index e8ca5571f..8bdab0408 100644 --- a/packages/compiler/src/backend/llvm/constant-callbacks.test.ts +++ b/packages/compiler/src/backend/llvm/constant-callbacks.test.ts @@ -299,6 +299,47 @@ test("receiver readers propagate backwards through direct calls and recursion", ).toBe(false); }); +test("signature adapters that only forward the receiver ignore it with their original", () => { + const forward = (receiver: IrExpr): IrStmt => ({ + kind: "return", + value: { + kind: "callValue", + callee: ref("f.0"), + receiver, + args: [{ kind: "varRef", localId: "a.0", type: F64, loc }], + type: BOOL, + loc, + }, + loc, + }); + const ambient: IrExpr = { kind: "libCall", fn: "dyn.this", args: [], type: { kind: "dyn" }, loc }; + const adapter: IrFunction = { + name: "%fn.adapt.x.impl", + params: [{ localId: "a.0", name: "a", type: F64 }], + captures: [{ localId: "f.0", name: "f", type: callback }], + locals: [ + { id: "f.0", name: "f", type: callback, mutable: false, boxed: true }, + { id: "a.0", name: "a", type: F64, mutable: false }, + ], + returnType: BOOL, + body: [forward(ambient)], + loc, + }; + const readers = (): AmbientReceiverReaders => + new AmbientReceiverReaders(new Map([[adapter.name, adapter]])); + // The adapter itself reads the receiver, only to forward it. + expect(readers().ignores(adapter)).toBe(false); + expect(readers().forwardsOnly(adapter)).toBe(true); + // Any other read of the receiver, or forwarding something else, does not. + adapter.body = [thisRead, forward(ambient)]; + expect(readers().forwardsOnly(adapter)).toBe(false); + adapter.body = [forward(ref("a.0", F64))]; + expect(readers().forwardsOnly(adapter)).toBe(false); + adapter.body = [forward(ambient)]; + adapter.captures = []; + expect(readers().forwardsOnly(adapter)).toBe(false); +}); + test("virtual calls admit every same-named method", () => { const call: IrStmt = { kind: "exprStmt", diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 1a2c8bef8..4c15ae61d 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -183,14 +183,14 @@ import { unionBoxBits, } from "./inline-unions.js"; import { OptionalArrayReads, type LocalArrayRead } from "./local-array-reads.js"; -import { CheckedNarrows } from "./checked-narrows.js"; +import { borrowsCheckedNarrow, CheckedNarrows } from "./checked-narrows.js"; import { emitDenseArraySet, emitDenseSlotUnlink, SCR_ELEM_I32, type ArrayAccess, } from "./dense-array-access.js"; -import { matchMapRead, type MapReadLifetimes } from "./map-read-lifetimes.js"; +import { matchMapRead, pointerMapGet, type MapReadLifetimes } from "./map-read-lifetimes.js"; import { type MapEntryFusion } from "./map-entry-fusion.js"; import { emitBorrowedFieldSequence, guardedValue } from "./borrowed-receivers.js"; import { emitBorrowedInputs } from "./borrowed-inputs.js"; @@ -213,7 +213,13 @@ import { } from "./string-slices.js"; import { LazyCaptures } from "./lazy-captures.js"; import { StackCaptures } from "./stack-captures.js"; -import { emitFieldAbsentTest, f64Lit, SCR_ARR_FIELD, SCR_ARR_FROZEN } from "./common.js"; +import { + CLOSURE_RECEIVER_FREE, + emitFieldAbsentTest, + f64Lit, + SCR_ARR_FIELD, + SCR_ARR_FROZEN, +} from "./common.js"; import { ffiExtendsNarrowIntegers } from "../targets.js"; import { emitLiteralExpr, @@ -285,7 +291,12 @@ import { emitBytesSet, emitBytesIntrinsic, } from "./expr-bytes.js"; -import { emitRegexIntrinsic, emitRecordKeyGet, keyedRecordReadInto } from "./expr-records.js"; +import { + emitRecordKeyGet, + emitRegexIntrinsic, + keyedRecordReadInto, + pointerRecordKeyGet, +} from "./expr-records.js"; import { dynPromiseAdapter, streamTypedRefCommitAdapter, @@ -2540,7 +2551,7 @@ export class LlEmitter { ret === "void" ? ` ${call}` : ` %r = ${call}`, ret === "void" ? ` ret void` : ` ret ${ret} %r`, `}`, - `@${mangleFnClosure(name)} = internal ${this.mod.workers === true || this.mod.lib?.threadInstances === true ? "thread_local " : ""}global %ScrClosure { ${this.sizeType} -1, ptr @${mangleWrapper(name)}, ${this.sizeType} 0, ptr null, i32 ${(fn.generator ? 1 : 0) + (fn.async ? 2 : 0) + (fn.ownsPrototype ? 4 : 0)}, i32 0, ptr null, ptr ${lending ? `@${mangleBorrowingWrapper(name)}` : "null"} }`, + `@${mangleFnClosure(name)} = internal ${this.mod.workers === true || this.mod.lib?.threadInstances === true ? "thread_local " : ""}global %ScrClosure { ${this.sizeType} -1, ptr @${mangleWrapper(name)}, ${this.sizeType} 0, ptr null, i32 ${this.closureFunctionKind(fn)}, i32 0, ptr null, ptr ${lending ? `@${mangleBorrowingWrapper(name)}` : "null"} }`, ``, ); if (lending) @@ -5715,7 +5726,10 @@ export class LlEmitter { unwrittenRead(e: IrExpr, writes: Writes): boolean { const rw = this.referenceWrites; if (rw === null) return false; - const keys = rw.readKeys(e, (call) => this.borrowedReturnKeys(call.callee)); + // A checked extraction reads nothing besides the union it narrows. + const keys = rw.readKeys(e, (call) => + borrowsCheckedNarrow(this, call) ? [] : this.borrowedReturnKeys(call.callee), + ); return keys !== null && !rw.conflicts(keys, writes); } @@ -5992,7 +6006,21 @@ export class LlEmitter { } case "fieldGet": case "recordGet": + case "recordKeyGet": return this.borrowsWithoutOwning(e) && this.unownedWalk(e.obj); + case "mapIntrinsic": { + // A Map binding's own pointer (maps are not plainReference types). + if (!this.borrowsWithoutOwning(e) || e.receiver.kind !== "varRef") return false; + const binding = this.binding(e.receiver.localId); + return ( + binding.kind === "global" || + (binding.kind === "local" && + binding.local !== undefined && + !binding.local.boxed && + !binding.local.tdz && + !this.localUnionStorage.has(e.receiver.localId)) + ); + } case "unionNarrow": case "downcast": case "upcast": @@ -7244,6 +7272,11 @@ export class LlEmitter { return this.nullableFieldGet(e) === null; case "recordGet": return true; + // A keyed read whose hits are all the result's own pointer. + case "recordKeyGet": + return pointerRecordKeyGet(this, e); + case "mapIntrinsic": + return pointerMapGet(this, e); // emitReadReceiver lends a required element of a borrowed array when // computing the index removes no edge. case "arrayGet": @@ -7389,8 +7422,20 @@ export class LlEmitter { return wrapEmitterListener(this, target, adapterFn); } - unwrapNullableClosure(u: string, funcTag: number): string { - return unwrapNullableClosure(this, u, funcTag); + /** ScrClosure.function_kind for a closure over `fn`: generator 1, async + * 2, own prototype 4, and SCR_FN_RECEIVER_FREE (16) when its body never + * observes the ambient receiver, so indirect callers skip the bracket. */ + closureFunctionKind(fn: IrFunction): number { + return ( + (fn.generator ? 1 : 0) + + (fn.async ? 2 : 0) + + (fn.ownsPrototype ? 4 : 0) + + (this.receiverReaders.ignores(fn) ? CLOSURE_RECEIVER_FREE : 0) + ); + } + + unwrapNullableClosure(u: string, unionId: string, funcTag: number): string { + return unwrapNullableClosure(this, u, unionId, funcTag); } closeBindThunkFor(cbUnion: IrType, retServer: boolean): string { diff --git a/packages/compiler/src/backend/llvm/expr-callbacks.ts b/packages/compiler/src/backend/llvm/expr-callbacks.ts index 398aebb21..ad827c4bf 100644 --- a/packages/compiler/src/backend/llvm/expr-callbacks.ts +++ b/packages/compiler/src/backend/llvm/expr-callbacks.ts @@ -974,10 +974,20 @@ export function wrapEmitterListener( export function unwrapNullableClosure( host: LlvmEmitterContext, u: string, + unionId: string, funcTag: number, ): string { const B = host.B; host.declare(`declare ptr @scr_closure_retain_v(ptr)`); + const nullable = host.nullableUnions.get(unionId); + if (nullable !== null) { + // The union is the closure pointer itself (NULL for the unit arm). + if (nullable.repr !== "single" || nullable.nullTag >= 0 || nullable.refTag !== funcTag) + throw new InternalCompilerError("llvm emitter bug: nullable callback union shape"); + const r = B.tmp(); + B.line(`${r} = call ptr ${retainSym(host.shapeHost, { kind: "union", unionId })}(ptr ${u})`); + return r; + } const slot = B.slot(); B.entryAllocas.push(`${slot} = alloca ptr`); B.line(`store ptr null, ptr ${slot}`); @@ -1026,9 +1036,9 @@ export function closeBindThunkFor( "llvm emitter bug: bound-close callback union lacks its func arm", ); const oneParam = funcArm.params.length === 1; + const cbNullable = host.nullableUnions.get(cbUnion.unionId); host.declare(`declare ptr @scr_box_get_ref(ptr)`); host.declare(`declare void @scr_closure_release(ptr)`); - host.declare(`declare void @scr_union_release(ptr)`); host.declare(`declare ptr @scr_closure_retain_v(ptr)`); host.declare(`declare void @scr_net_server_close_direct(ptr, ptr)`); let trampoline: string | null = null; @@ -1072,13 +1082,11 @@ export function closeBindThunkFor( ` %capsp = getelementptr inbounds %ScrClosure, ptr %self, i64 1`, ` %bx = load ptr, ptr %capsp`, ` %srv = call ptr @scr_box_get_ref(ptr %bx) ; +1`, - ` %tagp = getelementptr inbounds %ScrUnion, ptr %cbu, i64 0, i32 1`, - ` %tag = load i32, ptr %tagp`, + ...unionTagLines(cbNullable, "%cbu", "%tag"), ` %isfn = icmp eq i32 %tag, ${funcTag}`, ` br i1 %isfn, label %fn, label %go`, `fn:`, - ` %pp = getelementptr inbounds %ScrUnion, ptr %cbu, i64 0, i32 5`, - ` %pv = load ptr, ptr %pp`, + ...unionPeekLines(cbNullable, "%cbu", "%pv"), ]; if (oneParam) { d.push( @@ -1097,7 +1105,7 @@ export function closeBindThunkFor( ` br label %go`, `go:`, ` %regv = load ptr, ptr %regslot`, - ` call void @scr_union_release(ptr %cbu) ; the callee owns its +1 param`, + ` call void ${releaseSym(host.shapeHost, cbUnion)}(ptr %cbu) ; the callee owns its +1 param`, ` call void @scr_net_server_close_direct(ptr %srv, ptr %regv) ; reg moves`, ); if (retServer) { diff --git a/packages/compiler/src/backend/llvm/expr-calls.ts b/packages/compiler/src/backend/llvm/expr-calls.ts index d411d9172..3ecf33a79 100644 --- a/packages/compiler/src/backend/llvm/expr-calls.ts +++ b/packages/compiler/src/backend/llvm/expr-calls.ts @@ -42,7 +42,13 @@ import { } from "./classes.js"; import { LlvmUnsupportedError } from "./unsupported.js"; import type { LlvmEmitterContext, ExprOf, LlValue } from "./expr-context.js"; -import { f64Lit, ffiNativeTypeLl, ffiNativeParamLl, ffiNativeReturnLl } from "./common.js"; +import { + CLOSURE_RECEIVER_FREE, + f64Lit, + ffiNativeTypeLl, + ffiNativeParamLl, + ffiNativeReturnLl, +} from "./common.js"; import { canStackUnion, emitStackUnion } from "./stack-unions.js"; import { emitStackRecordKeyGet, scalarRecordKeyGet } from "./expr-records.js"; import { emitCallArrayRead } from "./local-array-reads.js"; @@ -761,8 +767,7 @@ export function emitCallExpr( `${c} = call ptr @scr_closure_new(ptr @${host.callTarget(e.fnName)}, ${host.sizeType} ${e.captures.length})`, ); const out = host.own({ name: c, type: e.type }); - const functionKind = - (target.generator ? 1 : 0) + (target.async ? 2 : 0) + (target.ownsPrototype ? 4 : 0); + const functionKind = host.closureFunctionKind(target); if (functionKind) { const kindPtr = B.tmp(); B.line(`${kindPtr} = getelementptr inbounds %ScrClosure, ptr ${c}, i64 0, i32 4`); @@ -800,6 +805,25 @@ export function emitCallExpr( : 0; host.declare(`declare void @scr_closure_adopt_identity(ptr, ptr, i32)`); B.line(`call void @scr_closure_adopt_identity(ptr ${c}, ptr ${original}, i32 ${length})`); + if (host.receiverReaders.forwardsOnly(target)) { + // The adapter forwards the ambient receiver to its original, + // so it ignores the receiver exactly when the original does. + const fromPtr = B.tmp(), + from = B.tmp(), + bit = B.tmp(), + toPtr = B.tmp(), + to = B.tmp(), + kind = B.tmp(); + B.line( + `${fromPtr} = getelementptr inbounds %ScrClosure, ptr ${original}, i64 0, i32 4`, + ); + B.line(`${from} = load i32, ptr ${fromPtr}`); + B.line(`${bit} = and i32 ${from}, ${CLOSURE_RECEIVER_FREE}`); + B.line(`${toPtr} = getelementptr inbounds %ScrClosure, ptr ${c}, i64 0, i32 4`); + B.line(`${to} = load i32, ptr ${toPtr}`); + B.line(`${kind} = or i32 ${to}, ${bit}`); + B.line(`store i32 ${kind}, ptr ${toPtr}`); + } } }); return out; @@ -813,7 +837,18 @@ export function emitCallExpr( const ft = e.callee.type; if (e.args.length !== ft.params.length) throw new LlvmUnsupportedError("callValue:arity", e.loc); - const callee = host.stackCallbacks.emit(host, e.callee) ?? emitBorrowedInput(host, e.callee); + // A receiver that reads the ambient `this` binds what the callee + // would see without a binding: the call forwards it unchanged and + // needs no receiver bracket (signature adapters forward this way). + const forwardsReceiver = + e.receiver?.kind === "libCall" && + e.receiver.fn === "dyn.this" && + e.receiver.args.length === 0; + const callee = + host.stackCallbacks.emit(host, e.callee) ?? + (lendsCallee(host, e, forwardsReceiver) + ? host.emitReadReceiver(e.callee) + : emitBorrowedInput(host, e.callee)); // A class instance bound as the callee's `this` (`h.fn(x)`) is boxed // only if the callee reads it: the receiver window records the // instance (owned by this frame for the whole call) with its typed-ref @@ -825,7 +860,10 @@ export function emitCallExpr( !host.shapeHost.isErrorClass(e.receiver.value.type.className) ? host.emitExpr(e.receiver.value) : null; - const receiver = e.receiver === undefined || lazyReceiver ? null : host.emitExpr(e.receiver); + const receiver = + e.receiver === undefined || lazyReceiver || forwardsReceiver + ? null + : host.emitExpr(e.receiver); const pushReceiver = (): void => { if (lazyReceiver) { const t = lazyReceiver.type; @@ -865,7 +903,7 @@ export function emitCallExpr( ? borrowableInputs(host, e.args, false, host.referenceWrites?.summary(known!.name)) : []; let dynamic = false; - if (!direct && e.receiver === undefined) { + if (!direct && (e.receiver === undefined || forwardsReceiver)) { const references = new Set(); ft.params.forEach((param, index) => { if (isRefCounted(param)) references.add(index); @@ -898,16 +936,51 @@ export function emitCallExpr( lent.has(i) ? { name: host.retainFrame(a.name, a.type), type: a.type } : a, ) : args; - const invoke = (target: string, needsReceiver: boolean, values = args): string => { + // An entry through the closure's own pointer brackets the call only + // when the closure may observe the receiver (SCR_FN_RECEIVER_FREE). + const invoke = ( + target: string, + bracket: boolean, + values = args, + indirect = false, + ): string => { + const needsReceiver = bracket && !forwardsReceiver; + let observes: string | null = null; if (needsReceiver) { host.declare("declare void @scr_dyn_this_pop()"); - pushReceiver(); + if (indirect) { + const kindPtr = B.tmp(), + kind = B.tmp(), + free = B.tmp(); + observes = B.tmp(); + B.line( + `${kindPtr} = getelementptr inbounds %ScrClosure, ptr ${callee.name}, i64 0, i32 4`, + ); + B.line(`${kind} = load i32, ptr ${kindPtr}`); + B.line(`${free} = and i32 ${kind}, ${CLOSURE_RECEIVER_FREE}`); + B.line(`${observes} = icmp eq i32 ${free}, 0`); + const push = B.newLabel("receiver.push"), + pushed = B.newLabel("receiver.call"); + B.condBr(observes, push, pushed); + B.startBlock(push); + pushReceiver(); + B.br(pushed); + B.startBlock(pushed); + } else pushReceiver(); } const result = e.type.kind === "void" ? "" : B.tmp(); B.line( `${result ? `${result} = ` : ""}call ${host.llType(e.type)} ${target}(${argListOf(values)})`, ); - if (needsReceiver) B.line("call void @scr_dyn_this_pop()"); + if (observes !== null) { + const pop = B.newLabel("receiver.pop"), + popped = B.newLabel("receiver.done"); + B.condBr(observes, pop, popped); + B.startBlock(pop); + B.line("call void @scr_dyn_this_pop()"); + B.br(popped); + B.startBlock(popped); + } else if (needsReceiver) B.line("call void @scr_dyn_this_pop()"); return result; }; const lending = @@ -953,11 +1026,11 @@ export function emitCallExpr( B.line(`${has} = icmp ne ptr ${entry}, null`); B.condBr(has, borrowing, owning); B.startBlock(borrowing); - const lentResult = invoke(entry, true); + const lentResult = invoke(entry, true, args, true); const borrowingEnd = B.currentLabel(); B.br(done); B.startBlock(owning); - const ownedResult = invoke(fn, true, ownedArgs()); + const ownedResult = invoke(fn, true, ownedArgs(), true); const owningEnd = B.currentLabel(); B.br(done); B.startBlock(done); @@ -966,7 +1039,7 @@ export function emitCallExpr( B.line( `${t} = phi ${host.llType(e.type)} [ ${lentResult}, %${borrowingEnd} ], [ ${ownedResult}, %${owningEnd} ]`, ); - } else if (!direct) t = invoke(fn, true, ownedArgs()); + } else if (!direct) t = invoke(fn, true, ownedArgs(), true); else { // A global initializer may not have run, or a callback may enter // through a boundary the census cannot prove. Validate the actual @@ -984,7 +1057,7 @@ export function emitCallExpr( const fastEnd = B.currentLabel(); B.br(done); B.startBlock(fallback); - const indirect = invoke(fn, true, ownedArgs()); + const indirect = invoke(fn, true, ownedArgs(), true); const fallbackEnd = B.currentLabel(); B.br(done); B.startBlock(done); @@ -1358,6 +1431,22 @@ function ctorArguments( * walk consumes without owning it (LlEmitter.canBorrowReceiver proved every * reference argument borrowable and every argument edge-preserving). The * result stays reachable from the borrowed arguments until the consumer. */ +/** A closure value read from a location the call cannot write (a keyed + * table entry, a field, a checked extraction of either) is entered without + * a reference of its own: the location keeps the closure alive until the + * call returns. Plain stable bindings already lend (emitBorrowedInput). */ +function lendsCallee( + host: LlvmEmitterContext, + e: ExprOf<"callValue">, + forwardsReceiver: boolean, +): boolean { + const writes = host.referenceWrites; + if (writes === null || host.canBorrowCallArgument(e.callee) || !host.canBorrowReceiver(e.callee)) + return false; + const later = e.receiver === undefined || forwardsReceiver ? e.args : [e.receiver, ...e.args]; + return borrowableInputs(host, [e.callee, ...later], false, writes.ownWrites(e))[0] === true; +} + export function emitBorrowedResultCall(host: LlvmEmitterContext, e: ExprOf<"call">): LlValue { const callee = host.fnByName.get(e.callee); if (!callee || !host.borrowedReturns.has(e.callee)) diff --git a/packages/compiler/src/backend/llvm/expr-control.ts b/packages/compiler/src/backend/llvm/expr-control.ts index 2f7c31813..9ce5540a6 100644 --- a/packages/compiler/src/backend/llvm/expr-control.ts +++ b/packages/compiler/src/backend/llvm/expr-control.ts @@ -491,10 +491,7 @@ export function emitOptionalChainReceiver( // needs no reference of its own while the body cannot remove an edge // (so nothing can overwrite the projection before the body finishes). const borrowReceiver = - !multiple && - isRefCounted(narrowed) && - host.canBorrowReceiver(e.receiver) && - host.referenceEffects.preserves(e.body); + !multiple && isRefCounted(narrowed) && borrowsChainReceiver(host, e.receiver, e.body); // An optional array read of a nullable-pointer union is the element // pointer itself, read inline and owned by the statement frame until // the body finishes; the bind slot borrows it like an emitExpr value. @@ -554,6 +551,16 @@ export function emitOptionalChainReceiver( }; } +/** An optional chain's receiver can lend its payload to the bind slot when + * the body leaves the receiver's locations alone: it removes no edge at + * all, or it writes none of the locations the receiver reads. */ +function borrowsChainReceiver(host: LlvmEmitterContext, receiver: IrExpr, body: IrExpr): boolean { + if (!host.canBorrowReceiver(receiver)) return false; + if (host.referenceEffects.preserves(body)) return true; + const writes = host.referenceWrites; + return writes !== null && host.unwrittenRead(receiver, writes.expr(body)); +} + /** `a?.b ?? d` narrowed to the chain body's present arm. The body wraps a * value of the result type in a non-unit arm, so the default runs exactly * when the receiver is nullish and the intermediate result box is never @@ -580,16 +587,21 @@ function emitNullishOptionalChain(host: LlvmEmitterContext, e: ExprOf<"nullish"> if (unitTags.length === 0 || present.length !== 1) return null; const B = host.B; const narrowed = receiverDef.arms[present[0]!]!; - const r = host.canStackReceiver(chain.receiver) - ? host.emitReadReceiver(chain.receiver) - : host.emitExpr(chain.receiver); + // A receiver read from a location the body cannot write lends its + // payload to the bind slot for the whole body (`table[k]?.(…) ?? d`). + const borrowed = + isRefCounted(narrowed) && borrowsChainReceiver(host, chain.receiver, chain.body.value); + const r = + borrowed || host.canStackReceiver(chain.receiver) + ? host.emitReadReceiver(chain.receiver) + : host.emitExpr(chain.receiver); const bindTy = host.llType(narrowed); const bind = B.slot(); B.entryAllocas.push(`${bind} = alloca ${bindTy}`); B.line( `store ${bindTy} ${bindTy === "ptr" ? "null" : bindTy === "double" ? f64Lit(0) : "false"}, ptr ${bind}`, ); - host.ownSlot(bind, narrowed); + if (!borrowed) host.ownSlot(bind, narrowed); const isUnit = host.tagInSet(r.name, receiverDef.id, unitTags); const ty = host.llType(e.type); const slot = B.slot(); @@ -602,7 +614,10 @@ function emitNullishOptionalChain(host: LlvmEmitterContext, e: ExprOf<"nullish"> host.emitBranchInto(slot, e.right); B.br(lj); B.startBlock(lb); - B.line(`store ${bindTy} ${host.unionExtract(r.name, receiverDef.id, narrowed)}, ptr ${bind}`); + const payload = borrowed + ? host.unionPeek(r.name, receiverDef.id) + : host.unionExtract(r.name, receiverDef.id, narrowed); + B.line(`store ${bindTy} ${payload}, ptr ${bind}`); host.chainSlots.set(chain.id, { name: bind, type: narrowed, slot: true }); host.emitBranchInto(slot, chain.body.value); host.chainSlots.delete(chain.id); diff --git a/packages/compiler/src/backend/llvm/expr-dynamic.ts b/packages/compiler/src/backend/llvm/expr-dynamic.ts index 55bbb828b..7f4b7e48a 100644 --- a/packages/compiler/src/backend/llvm/expr-dynamic.ts +++ b/packages/compiler/src/backend/llvm/expr-dynamic.ts @@ -11,7 +11,7 @@ import { bindLiveArrayOps } from "./expr-stream-bridges.js"; /* Focused LLVM expression emission extracted from emitter.ts. */ import { InternalCompilerError } from "../../errors.js"; import { emitNullableIsTag, emitNullablePresent } from "./union-repr.js"; -import { isObjectArm } from "./nullable-unions.js"; +import { isPointerIdentityArm } from "./nullable-unions.js"; import { streamTypedRefEligible } from "../../ir/analysis.js"; import { DYN, @@ -908,13 +908,13 @@ export function emitDynamicExpr( if (!def) throw new InternalCompilerError(`llvm emitter bug: equality of unknown union ${e.unionId}`); const nullable = host.nullableUnions.get(def.id); - if (nullable && isObjectArm(nullable)) { + if (nullable && isPointerIdentityArm(nullable)) { // Object identity, and NULL === NULL for the unit arm: one compare. const eq = B.tmp(); B.line(`${eq} = icmp ${e.negated ? "ne" : "eq"} ptr ${l.name}, ${r.name}`); return { name: eq, type: e.type }; } - if (nullable) { + if (nullable && nullable.repr === "single" && nullable.arm.kind === "string") { // String arm: identical pointers are equal (the same string or the // same unit encoding); otherwise equal only when both hold strings // with the same bytes. diff --git a/packages/compiler/src/backend/llvm/expr-records.ts b/packages/compiler/src/backend/llvm/expr-records.ts index 59ab625e5..597cda630 100644 --- a/packages/compiler/src/backend/llvm/expr-records.ts +++ b/packages/compiler/src/backend/llvm/expr-records.ts @@ -9,6 +9,7 @@ import { borrowableInputs, emitBorrowedInputs, emitBorrowedInput } from "./borro import { regexLiteralTemplate } from "./expr-primitives.js"; import { isScalarUnionDef, loadInlineUnion, presentBits } from "./inline-unions.js"; import { preservesRegexInputs } from "./reference-effects.js"; +import { emitNumberMapSlot, emitStringMapEntry } from "./map-number-lookup.js"; /** Widen one tagged read. The source is borrowed unless the overflow map * supplied an owned value. Each selected reference payload is retained @@ -343,8 +344,8 @@ export function emitRecordKeyGet( e: IrExpr & { kind: "recordKeyGet" }, ): LlValue { const B = host.B; - const obj = host.emitExpr(e.obj); - const k = host.emitExpr(e.key); + // The read runs no user code: both inputs lend for its duration. + const [obj, k] = emitBorrowedInputs(host, [e.obj, e.key]) as [LlValue, LlValue]; const ty = host.llType(e.type); const slot = B.slot(); B.entryAllocas.push(`${slot} = alloca ${ty}`); @@ -366,6 +367,80 @@ export function emitRecordKeyGet( return host.own({ name: t, type: e.type }); } +/** A keyed read whose every hit is the result's own pointer: the result + * is a nullable-pointer union (nullable-unions.ts) with an undefined arm, + * and each declared field and the overflow value hold exactly the result + * type or its reference arm in a plain (non-inline) slot. A miss is the + * immortal undefined encoding. Such a read can lend the stored pointer: + * borrowedRecordKeyGet reads it without a reference. */ +export function pointerRecordKeyGet(host: LlvmEmitterContext, e: IrExpr): boolean { + if (e.kind !== "recordKeyGet" || e.missNaN === true || e.type.kind !== "union") return false; + const resultType = e.type; + const nullable = host.nullableUnions.single(resultType.unionId); + if (nullable === null || undefinedArmTag(resultType, host.unionsById) < 0) return false; + const exact = (t: IrType): boolean => typeEquals(t, resultType) || typeEquals(t, nullable.arm); + const shape = host.recordShape(e.shapeId); + if (e.overflowOnly !== true) { + for (const f of shape.fields) + if (host.recordFieldInline(e.shapeId, f.name) || !exact(f.type)) return false; + } + return shape.indexValue === undefined || typeEquals(shape.indexValue, nullable.arm); +} + +/** A pointerRecordKeyGet read without a reference: the declared field or + * overflow entry the key names, or the undefined encoding for a miss. The + * record (and through it the stored value) must stay unchanged and alive + * until the consumer is done with the pointer (canBorrowReceiver). */ +export function emitBorrowedRecordKeyGet( + host: LlvmEmitterContext, + e: IrExpr & { kind: "recordKeyGet" }, +): LlValue { + const B = host.B; + if (e.type.kind !== "union") + throw new InternalCompilerError("llvm emitter bug: borrowed keyed read of a non-union"); + const resultType = e.type; + const obj = host.emitReadReceiver(e.obj); + const key = host.canBorrowReceiver(e.key) ? host.emitReadReceiver(e.key) : host.emitExpr(e.key); + const slot = B.slot(); + B.entryAllocas.push(`${slot} = alloca ptr`); + const join = B.newLabel("rkb.j"); + const shape = host.recordShape(e.shapeId); + for (const f of e.overflowOnly === true ? [] : shape.fields) { + const hit = B.tmp(); + B.line( + `${hit} = call zeroext i1 ${host.literalEquals(f.name)}(ptr ${key.name}) ; ${llvmCommentText(f.name)}`, + ); + const lh = B.newLabel("rkb.h"); + const ln = B.newLabel("rkb.n"); + B.condBr(hit, lh, ln); + B.startBlock(lh); + const { ptr, type } = host.recordFieldPtr(obj.name, e.shapeId, f.name); + B.line(`store ptr ${host.loadRecordField(ptr, type, false)}, ptr ${slot}`); + B.br(join); + B.startBlock(ln); + } + if (shape.indexValue) { + const ovf = host.recordOvfPtr(obj.name, e.shapeId); + const entry = emitStringMapEntry(host, ovf, key.name); + const found = B.tmp(); + B.line(`${found} = icmp ne ptr ${entry}, null`); + const lh = B.newLabel("rkb.h"); + const ln = B.newLabel("rkb.n"); + B.condBr(found, lh, ln); + B.startBlock(lh); + B.line(`store ptr ${emitNumberMapSlot(host, entry, "ptr")}, ptr ${slot}`); + B.br(join); + B.startBlock(ln); + } + const undefTag = undefinedArmTag(resultType, host.unionsById); + B.line(`store ptr ${host.unitInstanceRef(resultType.unionId, undefTag)}, ptr ${slot}`); + B.br(join); + B.startBlock(join); + const t = B.tmp(); + B.line(`${t} = load ptr, ptr ${slot}`); + return { name: t, type: e.type }; +} + /** A keyed read whose result is a scalar union (`r[k]` on a * `Record`, at `number | undefined`) and whose every hit * is a number or boolean arm of it, or an inline field of the same union. @@ -660,27 +735,17 @@ export function keyedRecordReadInto( B.br(join); B.startBlock(ln); } else { - host.declare(`declare ptr @scr_map_get_str_ref(ptr, ptr)`); + // The entry lookup filters absent keys inline; a hit retains the + // stored value with its own type's retain (map-number-lookup.ts). A + // nullable value's NULL is a stored unit arm: presence decides. const lh = B.newLabel("rkg.h"); const ln = B.newLabel("rkg.n"); - let raw: string; - if (iv.kind === "union" && host.nullableUnions.has(iv.unionId)) { - // NULL is a stored unit arm of a nullable value: presence decides. - host.declare(`declare zeroext i1 @scr_map_has_str(ptr, ptr)`); - const has = B.tmp(); - B.line(`${has} = call zeroext i1 @scr_map_has_str(ptr ${ovf}, ptr ${keyName})`); - B.condBr(has, lh, ln); - B.startBlock(lh); - raw = B.tmp(); - B.line(`${raw} = call ptr @scr_map_get_str_ref(ptr ${ovf}, ptr ${keyName})`); - } else { - raw = B.tmp(); - const isnull = B.tmp(); - B.line(`${raw} = call ptr @scr_map_get_str_ref(ptr ${ovf}, ptr ${keyName})`); - B.line(`${isnull} = icmp eq ptr ${raw}, null`); - B.condBr(isnull, ln, lh); - B.startBlock(lh); - } + const entry = emitStringMapEntry(host, ovf, keyName); + const found = B.tmp(); + B.line(`${found} = icmp ne ptr ${entry}, null`); + B.condBr(found, lh, ln); + B.startBlock(lh); + const raw = host.retainHeap(emitNumberMapSlot(host, entry, "ptr"), iv); B.line(`store ${ty} ${surface(iv, raw, true)}, ptr ${slot}`); B.br(join); B.startBlock(ln); diff --git a/packages/compiler/src/backend/llvm/lib-http2.ts b/packages/compiler/src/backend/llvm/lib-http2.ts index d750fb7c5..e9f60cdfa 100644 --- a/packages/compiler/src/backend/llvm/lib-http2.ts +++ b/packages/compiler/src/backend/llvm/lib-http2.ts @@ -95,11 +95,12 @@ export function sniCallbackArgs(host: LlvmEmitterContext, cb: LlValue): AbiArgum let type = cb.type, value = cb.name; if (type.kind === "union") { - const arms = host.unionsById.get(type.unionId)?.arms ?? []; + const unionId = type.unionId; + const arms = host.unionsById.get(unionId)?.arms ?? []; const tag = arms.findIndex((arm) => arm.kind === "func"); if (tag < 0) throw new InternalCompilerError("SNI callback union lacks its function arm"); type = arms[tag]!; - value = host.unwrapNullableClosure(value, tag); + value = host.unwrapNullableClosure(value, unionId, tag); } else host.toHeap(cb); if (type.kind !== "func" || type.params[1] === undefined) throw new InternalCompilerError("SNI callback lacks its answer function"); diff --git a/packages/compiler/src/backend/llvm/lib-network.ts b/packages/compiler/src/backend/llvm/lib-network.ts index 35eb885cd..63661ab4c 100644 --- a/packages/compiler/src/backend/llvm/lib-network.ts +++ b/packages/compiler/src/backend/llvm/lib-network.ts @@ -165,7 +165,7 @@ export function emitNetworkHttpLibCall(host: LlvmEmitterContext, e: LibCallExpr) const funcTag = def ? def.arms.findIndex((a) => a.kind === "func") : -1; if (funcTag < 0) throw new InternalCompilerError(`llvm emitter bug: ${e.fn} union lacks its func arm`); - cb = host.unwrapNullableClosure(args[cbIndex]!.name, funcTag); + cb = host.unwrapNullableClosure(args[cbIndex]!.name, cbT.unionId, funcTag); } } const valueCount = reusePort ? 5 : 4; diff --git a/packages/compiler/src/backend/llvm/map-read-lifetimes.ts b/packages/compiler/src/backend/llvm/map-read-lifetimes.ts index ae741463d..fbb21dcd2 100644 --- a/packages/compiler/src/backend/llvm/map-read-lifetimes.ts +++ b/packages/compiler/src/backend/llvm/map-read-lifetimes.ts @@ -12,13 +12,18 @@ import type { CallLifetimes } from "./call-lifetimes.js"; import type { ReferenceEffects } from "./reference-effects.js"; import type { LlValue, LlvmEmitterContext } from "./expr-context.js"; import { + emitCompactMapSlot, emitNumberMapGetRef, emitNumberMapGetScalar, + emitNumberMapSlot, + emitStringMapEntry, emitStringMapGetScalar, emitStringMapGetRef, compactMapValues, noteNumberMapKey, } from "./map-number-lookup.js"; +import { InternalCompilerError } from "../../errors.js"; +import { undefinedArmTag } from "../../ir/analysis.js"; import { mapKeyAccess, mapKeyParamType } from "./shapes.js"; import { rememberMapEntry } from "./map-entry-fusion.js"; @@ -490,3 +495,62 @@ export function emitStackMapRead(host: LlvmEmitterContext, read: LocalMapRead): B.line(`store i32 ${selected}, ptr ${tag}`); return { value: { name: box, type: read.type }, owner }; } + +/** A string-keyed Map.get whose result is a nullable-pointer union over the + * map's own value type (`Map` read as `F | undefined`): a hit is + * the stored pointer itself and a miss the immortal undefined encoding, so + * the read can lend the stored value (emitBorrowedMapGet). */ +export function pointerMapGet(host: LlvmEmitterContext, e: IrExpr): boolean { + if ( + e.kind !== "mapIntrinsic" || + e.method !== "get" || + e.args.length !== 1 || + e.type.kind !== "union" || + e.receiver.type.kind !== "map" || + e.receiver.type.key.kind !== "string" || + !borrowsMapReadInputs(e) + ) + return false; + const nullable = host.nullableUnions.single(e.type.unionId); + return ( + nullable !== null && + typeEquals(nullable.arm, e.receiver.type.value) && + undefinedArmTag(e.type, host.unionsById) >= 0 + ); +} + +/** A pointerMapGet read without a reference: the map (and through it the + * stored value) must stay unchanged and alive until the consumer is done + * with the pointer (canBorrowReceiver). */ +export function emitBorrowedMapGet( + host: LlvmEmitterContext, + e: IrExpr & { kind: "mapIntrinsic" }, +): LlValue { + const B = host.B; + const mapType = e.receiver.type; + if (e.type.kind !== "union" || mapType.kind !== "map") + throw new InternalCompilerError("llvm emitter bug: borrowed Map.get shape"); + const receiver = host.emitReadReceiver(e.receiver); + const keyExpr = e.args[0]!; + const key = host.canBorrowReceiver(keyExpr) + ? host.emitReadReceiver(keyExpr) + : host.emitExpr(keyExpr); + const compact = compactMapValues(host, mapType.key, mapType.value); + const entry = emitStringMapEntry(host, receiver.name, key.name, compact); + const found = B.tmp(); + B.line(`${found} = icmp ne ptr ${entry}, null`); + const hit = B.newLabel("mapb.hit"), + miss = B.newLabel("mapb.miss"), + join = B.newLabel("mapb.join"); + B.condBr(found, hit, miss); + B.startBlock(hit); + const value = compact ? emitCompactMapSlot(host, entry) : emitNumberMapSlot(host, entry, "ptr"); + B.br(join); + B.startBlock(miss); + B.br(join); + B.startBlock(join); + const absent = host.unitInstanceRef(e.type.unionId, undefinedArmTag(e.type, host.unionsById)); + const result = B.tmp(); + B.line(`${result} = phi ptr [ ${value}, %${hit} ], [ ${absent}, %${miss} ]`); + return { name: result, type: e.type }; +} diff --git a/packages/compiler/src/backend/llvm/nullable-unions.test.ts b/packages/compiler/src/backend/llvm/nullable-unions.test.ts index bf0941652..f4e6a6866 100644 --- a/packages/compiler/src/backend/llvm/nullable-unions.test.ts +++ b/packages/compiler/src/backend/llvm/nullable-unions.test.ts @@ -7,7 +7,7 @@ import { type IrType, type IrUnionDef, } from "../../ir/ir.js"; -import { NullableUnions } from "./nullable-unions.js"; +import { isObjectArm, isPointerIdentityArm, NullableUnions } from "./nullable-unions.js"; const loc = { file: "nullable-unions.ts", start: 0, end: 0 }; const a: IrType = { kind: "record", shapeId: "a" }; @@ -111,3 +111,22 @@ test("library, lifetime-region and debug builds keep record unions boxed", () => expect(new NullableUnions(module(unions, { lifetimeRegions: true })).get("ab")).toBeNull(); expect(new NullableUnions(module(unions)).shapes("ab")).not.toBeNull(); }); + +test("function arms are nullable pointers compared by identity root", () => { + const fn: IrType = { kind: "func", params: [F64], ret: F64 }; + const unions = new NullableUnions( + module([ + { id: "fu", arms: [fn, UNDEFINED_T] }, + { id: "fnu", arms: [NULL_T, fn, UNDEFINED_T] }, + { id: "ff", arms: [fn, F64] }, + ]), + ); + expect(unions.single("fu")).toMatchObject({ refTag: 0, unitTag: 1, nullTag: -1, arm: fn }); + expect(unions.single("fnu")).toMatchObject({ refTag: 1, unitTag: 2, nullTag: 0 }); + expect(unions.get("ff")).toBeNull(); + const single = unions.single("fu")!; + // Always truthy, but `===` compares adapters by the function they adapt. + expect(isObjectArm(single)).toBe(true); + expect(isPointerIdentityArm(single)).toBe(false); + expect(unions.armTypes().get("fu")).toEqual(fn); +}); diff --git a/packages/compiler/src/backend/llvm/nullable-unions.ts b/packages/compiler/src/backend/llvm/nullable-unions.ts index c51f3b5f6..eedb922b3 100644 --- a/packages/compiler/src/backend/llvm/nullable-unions.ts +++ b/packages/compiler/src/backend/llvm/nullable-unions.ts @@ -89,21 +89,30 @@ export const NULLABLE_ABSENT = "@sc_nullable_absent"; /** The reference arms whose instances are never NULL. Class and record * entry points all tolerate NULL; strings, Stats snapshots, arrays, maps and - * sets retain through a NULL-skipping wrapper (shapes.ts). Object arms are - * always truthy and compare by identity; string arms are truthy when - * non-empty and compare bytes. */ + * sets retain through a NULL-skipping wrapper (shapes.ts), as do closures + * (function arms). Object arms are always truthy and compare by identity + * (function arms by identity root); string arms are truthy when non-empty + * and compare bytes. */ export type NullableRefArm = | { kind: "object"; className: string } | { kind: "record"; shapeId: string } | { kind: "string" } | { kind: "stats" } - | (IrType & { kind: "array" | "map" | "set" }); + | (IrType & { kind: "array" | "map" | "set" | "func" }); -/** The arm is a JS object: always truthy, compared by identity. */ +/** The arm is a JS object: always truthy. */ export function isObjectArm(nullable: PointerUnion): boolean { return nullable.repr === "shapes" || nullable.arm.kind !== "string"; } +/** The arm compares by pointer identity. Function arms compare identity + * roots instead: a signature adapter is the function it adapts. */ +export function isPointerIdentityArm(nullable: PointerUnion): boolean { + return ( + nullable.repr === "shapes" || (nullable.arm.kind !== "string" && nullable.arm.kind !== "func") + ); +} + /** The representation decision for every union of one module. It is a * pure function of the module, so serialized-IR and in-memory emission * agree. */ @@ -264,7 +273,8 @@ function nullableShape( return { ...base, arm: { kind: "record", shapeId: arm.shapeId } }; if (arm.kind === "string") return { ...base, arm: { kind: "string" } }; if (arm.kind === "stats") return { ...base, arm: { kind: "stats" } }; - if (arm.kind === "array" || arm.kind === "map" || arm.kind === "set") return { ...base, arm }; + if (arm.kind === "array" || arm.kind === "map" || arm.kind === "set" || arm.kind === "func") + return { ...base, arm }; return null; } diff --git a/packages/compiler/src/backend/llvm/receivers.ts b/packages/compiler/src/backend/llvm/receivers.ts index 2fe9dd66b..8524a51fb 100644 --- a/packages/compiler/src/backend/llvm/receivers.ts +++ b/packages/compiler/src/backend/llvm/receivers.ts @@ -11,14 +11,24 @@ import type { LlValue } from "./expr-context.js"; import { guardedValue } from "./borrowed-receivers.js"; import { InternalCompilerError } from "../../errors.js"; import { emitBorrowedArrayRead, emitProjectedArrayRead } from "./local-array-reads.js"; -import { emitStackMapRead, matchMapRead } from "./map-read-lifetimes.js"; +import { + emitBorrowedMapGet, + emitStackMapRead, + matchMapRead, + pointerMapGet, +} from "./map-read-lifetimes.js"; import { emitBorrowedResultCall, emitCallExpr } from "./expr-calls.js"; -import { emitCheckedNarrowReceiver } from "./checked-narrows.js"; +import { borrowsCheckedNarrow, emitCheckedNarrowReceiver } from "./checked-narrows.js"; import { canStackUnion, emitStackUnion } from "./stack-unions.js"; import { emitDowncastGuard, emitUnionNarrowGuard } from "./narrow-guards.js"; import { emitScalarOptional, isScalarOptionalSource } from "./local-union-storage.js"; import { borrowableInputs } from "./borrowed-inputs.js"; -import { emitStackRecordKeyGet, scalarRecordKeyGet } from "./expr-records.js"; +import { + emitBorrowedRecordKeyGet, + emitStackRecordKeyGet, + pointerRecordKeyGet, + scalarRecordKeyGet, +} from "./expr-records.js"; import { type WalkBorrowHost, analyzeWalks } from "./walk-borrows.js"; /** Read a receiver whose value is consumed immediately by a field or tag @@ -92,6 +102,10 @@ export function emitReadReceiver(emitter: LlEmitter, e: IrExpr): LlValue { return emitBorrowedResultCall(emitter, e); const arrayRead = emitter.optionalArrayReads.get(e); if (arrayRead) return emitProjectedArrayRead(emitter, arrayRead); + if (e.kind === "recordKeyGet" && emitter.canBorrowReceiver(e)) + return emitBorrowedRecordKeyGet(emitter, e); + if (e.kind === "mapIntrinsic" && emitter.canBorrowReceiver(e)) + return emitBorrowedMapGet(emitter, e); const narrow = emitter.checkedNarrows.get(e); const narrowed = narrow ? emitCheckedNarrowReceiver(emitter, narrow) : null; if (narrowed) return narrowed; @@ -294,7 +308,27 @@ export function canBorrowReceiver(emitter: LlEmitter, e: IrExpr): boolean { // snapshot, so neither the callee nor a later argument may replace // what it was read from: a callee that preserves every edge cannot, // and any other one must leave the argument's locations unwritten. + // A keyed read lends the stored pointer while the record keeps it. + case "recordKeyGet": + return ( + pointerRecordKeyGet(emitter, e) && + emitter.canBorrowReceiver(e.obj) && + emitter.referenceEffects.preserves(e.key) + ); + // So does a Map.get while the map keeps it. + case "mapIntrinsic": + return ( + pointerMapGet(emitter, e) && + emitter.canBorrowReceiver(e.receiver) && + emitter.referenceEffects.preserves(e.args[0]!) + ); case "call": { + // A checked extraction lends the payload of the union it narrows + // (emitCheckedNarrowReceiver). + if (borrowsCheckedNarrow(emitter, e)) { + const narrow = emitter.checkedNarrows.get(e)!; + return emitter.canBorrowReceiver(narrow.value); + } if (!emitter.borrowedReturns.has(e.callee)) return false; if (!e.args.every((arg) => emitter.referenceEffects.preserves(arg))) return false; const preserving = emitter.referenceEffects.functions.has(e.callee); diff --git a/packages/compiler/src/backend/llvm/reference-writes.test.ts b/packages/compiler/src/backend/llvm/reference-writes.test.ts index cbbe49d3b..db575babe 100644 --- a/packages/compiler/src/backend/llvm/reference-writes.test.ts +++ b/packages/compiler/src/backend/llvm/reference-writes.test.ts @@ -97,3 +97,46 @@ test("an array that is read back or escapes keeps its element writes", () => { const input = fill("input", [store("input", element("input"))]); expect(writes(input).conflicts(["a"], writes(input).summary("input"))).toBe(true); }); + +test("a keyed record read depends on every field and keyed entry of records", () => { + const handler: IrType = { kind: "func", params: [F64], ret: F64 }; + const table: IrType = { kind: "record", shapeId: "t" }; + const key: IrExpr = { kind: "strLit", value: "k", type: { kind: "string" }, loc }; + const read: IrExpr = { + kind: "recordKeyGet", + obj: ref("table", table), + shapeId: "t", + key, + type: { kind: "union", unionId: "hu" }, + loc, + }; + const keyedWrite: IrStmt = { + kind: "recordKeySet", + obj: ref("table", table), + shapeId: "t", + key, + value: ref("h", handler), + loc, + }; + const fn = (name: string, body: IrStmt[]): IrFunction => ({ + name, + loc, + returnType: F64, + params: [{ localId: "table", name: "table", type: table }], + locals: [ + { id: "table", name: "table", type: table, mutable: false }, + { id: "h", name: "h", type: handler, mutable: false }, + ], + body, + }); + const records = [{ id: "t", fields: [{ name: "add", type: handler }], indexValue: handler }]; + const quiet = fn("quiet", []); + const writer = fn("writer", [keyedWrite]); + const functions = new Map([quiet, writer].map((f) => [f.name, f])); + const rw = new ReferenceWrites(functions, [], () => false, new Set(), { records }); + rw.enter(quiet); + const keys = rw.readKeys(read); + expect(keys).toEqual(expect.arrayContaining(["r:*", "r:add"])); + expect(rw.conflicts(keys!, rw.summary("quiet"))).toBe(false); + expect(rw.conflicts(keys!, rw.summary("writer"))).toBe(true); +}); diff --git a/packages/compiler/src/backend/llvm/reference-writes.ts b/packages/compiler/src/backend/llvm/reference-writes.ts index 89132a715..d64beafc5 100644 --- a/packages/compiler/src/backend/llvm/reference-writes.ts +++ b/packages/compiler/src/backend/llvm/reference-writes.ts @@ -92,6 +92,8 @@ const INERT_LIB_PREFIXES = [ ]; const INERT_LIB_CALLS = new Set([ "dyn.compare", + // Reads the ambient receiver (a fresh reference to the bound value). + "dyn.this", "error.new", "error.nodeThrow", "dyn.typedRefIs", @@ -922,6 +924,17 @@ export class ReferenceWrites { case "arrayGet": out.push(ARRAY_KEY); return this.expr(node.index) === null && walk(node.arr); + case "recordKeyGet": + // Any declared field or overflow entry the key may name. + out.push(ANY_RECORD_KEY); + for (const f of this.records.get(node.shapeId)?.fields ?? []) out.push(`r:${f.name}`); + return this.expr(node.key) === null && walk(node.obj); + case "mapIntrinsic": + // A typed Map.get: any entry of any map. + if (node.method !== "get" || node.args.length !== 1 || !borrowsMapReadInputs(node)) + return false; + out.push(MAP_KEY); + return this.expr(node.args[0]!) === null && walk(node.receiver); case "unionNarrow": case "upcast": case "downcast": diff --git a/packages/compiler/src/backend/llvm/stack-callbacks.ts b/packages/compiler/src/backend/llvm/stack-callbacks.ts index 5c404896a..20de50372 100644 --- a/packages/compiler/src/backend/llvm/stack-callbacks.ts +++ b/packages/compiler/src/backend/llvm/stack-callbacks.ts @@ -32,10 +32,12 @@ export class StackCallbacks { const count = B.tmp(); B.line(`${count} = getelementptr inbounds %ScrClosure, ptr ${closure}, i32 0, i32 2`); B.line(`store ${host.sizeType} ${value.captures.length}, ptr ${count}`); - if (this.functions.get(value.fnName)?.ownsPrototype) { + const target = this.functions.get(value.fnName); + const functionKind = target ? host.closureFunctionKind(target) : 0; + if (functionKind !== 0) { const kind = B.tmp(); B.line(`${kind} = getelementptr inbounds %ScrClosure, ptr ${closure}, i32 0, i32 4`); - B.line(`store i32 4, ptr ${kind}`); + B.line(`store i32 ${functionKind}, ptr ${kind}`); } value.captures.forEach((localId, index) => { const box = host.retainBox(host.captureBox(localId)); diff --git a/packages/compiler/src/ir/receiver-readers.ts b/packages/compiler/src/ir/receiver-readers.ts index f5270aa7d..067c474ca 100644 --- a/packages/compiler/src/ir/receiver-readers.ts +++ b/packages/compiler/src/ir/receiver-readers.ts @@ -1,6 +1,7 @@ import { isBorrowSafeMath } from "./analysis.js"; import type { IrExpr, IrFunction, IrLibFn, IrStmt } from "./ir.js"; import { eachNodeOfKinds, type IrNodeIndex } from "./node-index.js"; +import { everyExprChild, everyStmtChild } from "./traverse.js"; /** Expression kinds that run no user code of their own and never consult * the ambient receiver stack. Calls are admitted here and resolved through @@ -151,6 +152,7 @@ function opaqueOperand(e: IrExpr): boolean { * this set needs no receiver bracket. */ export class AmbientReceiverReaders { private readers: Set | null = null; + private byMethod: Map | null = null; constructor( private readonly functions: ReadonlyMap, @@ -163,8 +165,63 @@ export class AmbientReceiverReaders { return !this.compute().has(fn.name); } - private compute(): Set { - if (this.readers) return this.readers; + /** A signature adapter body that observes the ambient receiver only by + * forwarding it, unchanged, to the original it captures (a callValue on + * that capture whose receiver is the plain `this` read). Such an adapter + * ignores the receiver exactly when its original does. */ + forwardsOnly(fn: IrFunction): boolean { + if (fn.async || fn.generator) return false; + const original = fn.captures?.[0]?.localId; + if (original === undefined) return false; + const readers = this.compute(); + const edge = (name: string): boolean => this.functions.has(name) && !readers.has(name); + const stmt = (node: IrStmt): boolean => + RECEIVER_FREE_STMTS.has(node.kind) && everyStmtChild(node, expr, stmt); + const expr = (node: IrExpr): boolean => { + if (node.kind === "callValue" && node.receiver !== undefined) { + const receiver = node.receiver; + return ( + receiver.kind === "libCall" && + receiver.fn === "dyn.this" && + receiver.args.length === 0 && + node.callee.kind === "varRef" && + node.callee.localId === original && + node.args.every(expr) + ); + } + return this.admits(node, edge) && everyExprChild(node, expr, stmt); + }; + return fn.body.every(stmt); + } + + /** Whether one expression node, apart from its children, leaves the + * ambient receiver alone; `edge` admits a direct callee. */ + private admits(expr: IrExpr, edge: (name: string) => boolean): boolean { + if (!RECEIVER_FREE_EXPRS.has(expr.kind)) return false; + switch (expr.kind) { + case "call": + return edge(expr.callee); + case "new": + return edge(`%${expr.className}.constructor`); + case "virtualCall": { + // Any class's override may run: admit only when every + // same-named method is itself receiver-free. + const targets = this.methods().get(expr.method) ?? []; + return targets.length > 0 && targets.every(edge); + } + case "libCall": + return isBorrowSafeMath(expr.fn) || RECEIVER_FREE_LIB_FNS.has(expr.fn); + case "toString": + return !opaqueOperand(expr.operand); + default: { + const operands = dataIntrinsicOperands(expr); + return operands === null || !operands.some(opaqueOperand); + } + } + } + + private methods(): Map { + if (this.byMethod) return this.byMethod; const byMethod = new Map(); for (const name of this.functions.keys()) { const dot = name.lastIndexOf("."); @@ -174,6 +231,12 @@ export class AmbientReceiverReaders { list.push(name); byMethod.set(method, list); } + this.byMethod = byMethod; + return byMethod; + } + + private compute(): Set { + if (this.readers) return this.readers; const readers = new Set(); const callers = new Map(); const scan = this.nodes?.scanExcept(TRIVIALLY_RECEIVER_FREE); @@ -185,34 +248,11 @@ export class AmbientReceiverReaders { callees.add(name); return true; }; - const expr = (expr: IrExpr): boolean => { - if (!RECEIVER_FREE_EXPRS.has(expr.kind)) return false; - switch (expr.kind) { - case "call": - return edge(expr.callee); - case "new": - return edge(`%${expr.className}.constructor`); - case "virtualCall": { - // Any class's override may run: admit only when every - // same-named method is itself receiver-free. - const targets = byMethod.get(expr.method) ?? []; - return targets.length > 0 && targets.every(edge); - } - case "libCall": - return isBorrowSafeMath(expr.fn) || RECEIVER_FREE_LIB_FNS.has(expr.fn); - case "toString": - return !opaqueOperand(expr.operand); - default: { - const operands = dataIntrinsicOperands(expr); - return operands === null || !operands.some(opaqueOperand); - } - } - }; if (!reads) eachNodeOfKinds(fn, scan, (node, statement) => { const free = statement ? RECEIVER_FREE_STMTS.has((node as IrStmt).kind) - : expr(node as IrExpr); + : this.admits(node as IrExpr, edge); if (!free) reads = true; return free; }); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8230-function-record-dispatch.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8230-function-record-dispatch.ts.json new file mode 100644 index 000000000..1670b97b4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8230-function-record-dispatch.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8230-function-record-dispatch.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8231-function-table-shapes.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8231-function-table-shapes.ts.json new file mode 100644 index 000000000..8f1963ef5 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8231-function-table-shapes.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8231-function-table-shapes.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8232-optional-function-values.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8232-optional-function-values.ts.json new file mode 100644 index 000000000..671364bc6 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8232-optional-function-values.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8232-optional-function-values.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_closure.h b/packages/runtime/src/scr_closure.h index 9644722ce..cfca90313 100644 --- a/packages/runtime/src/scr_closure.h +++ b/packages/runtime/src/scr_closure.h @@ -98,7 +98,7 @@ typedef struct ScrClosure { * (like the dyn→closure edge): a cycle through it is merely never * collected. */ ScrBox *props; - uint32_t function_kind; /* low bits: 0 ordinary, 1 generator, 2 async, 3 async generator; bit 2: ordinary own prototype; bit 3: checked callable adapter in caps[0] */ + uint32_t function_kind; /* low bits: 0 ordinary, 1 generator, 2 async, 3 async generator; bit 2: ordinary own prototype; bit 3: checked callable adapter in caps[0]; bit 4: SCR_FN_RECEIVER_FREE */ /* The root's `length` as boxed values report it; meaningful only when * identity is set (the root's own boxes carry their arity). */ uint32_t identity_length; @@ -117,6 +117,13 @@ typedef struct ScrClosure { ScrBox *caps[]; } ScrClosure; +/* function_kind bit 4: the body never observes the ambient receiver (the + * `this` an unbound call binds), so compiled callers skip the receiver + * bracket around an indirect call. Set only by the compiler; the kind + * proper is function_kind & SCR_FN_KIND_MASK. */ +#define SCR_FN_RECEIVER_FREE 16u +#define SCR_FN_KIND_MASK 15u + ScrClosure *scr_closure_new(void *fn, size_t ncaps); /* +1; caller fills caps with +1 box refs */ static inline ScrClosure *scr_closure_retain(ScrClosure *c) { diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index cd9eeecde..cef3be673 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -6880,7 +6880,7 @@ ScrDyn *scr_dyn_class_base_prototype(ScrDyn *constructor) { default: break; } if (constructor->kind != SCR_DYN_FUNC || constructor->v.fn.class_obj || - scr_closure_identity(constructor->v.fn.clo)->function_kind != 4) { + (scr_closure_identity(constructor->v.fn.clo)->function_kind & SCR_FN_KIND_MASK) != 4) { static const char message[] = "Class extends value is not a native ordinary constructor"; scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); return NULL; @@ -7028,7 +7028,8 @@ void scr_dyn_name_class_prototype(ScrDyn *prototype, const ScrStr *name) { /* Ordinary native function constructors retain their live prototype table. * A constructor may replace the fresh receiver with another object. */ ScrDyn *scr_dyn_construct(const ScrDyn *callee, const ScrDyn *args, const ScrStr *what) { - if (callee->kind != SCR_DYN_FUNC || callee->v.fn.class_obj || scr_closure_identity(callee->v.fn.clo)->function_kind != 4) + if (callee->kind != SCR_DYN_FUNC || callee->v.fn.class_obj || + (scr_closure_identity(callee->v.fn.clo)->function_kind & SCR_FN_KIND_MASK) != 4) return scr_bytes_construct(callee, args, what); ScrDyn *prototype = scr_dyn_fn_get(callee, "prototype", 9); if (!prototype) return NULL; @@ -9144,7 +9145,9 @@ bool scr_dyn_is_callable(const ScrDyn *value) { int scr_dyn_function_kind(const ScrDyn *value) { while (value->kind == SCR_DYN_PROXY) value = value->v.proxy.target; - return value->kind == SCR_DYN_FUNC ? scr_closure_identity(value->v.fn.clo)->function_kind : 0; + return value->kind == SCR_DYN_FUNC + ? (int)(scr_closure_identity(value->v.fn.clo)->function_kind & SCR_FN_KIND_MASK) + : 0; } void scr_dyn_adopt_identity(ScrClosure *adapter, const ScrDyn *value) { diff --git a/tests/corpus/8230-function-record-dispatch.ts b/tests/corpus/8230-function-record-dispatch.ts new file mode 100644 index 000000000..3acde5bd7 --- /dev/null +++ b/tests/corpus/8230-function-record-dispatch.ts @@ -0,0 +1,107 @@ +// Records of functions used as dispatch tables: keyed reads consumed at once +// (`t[k]!(…)`, `t[k]?.(…)`, a checked local), missing keys, handlers that +// declare fewer parameters than the table's type, handlers that replace, +// delete or re-add table entries (their own included) while they run, and +// key order after such mutation. +type Handler = (x: number, y: number) => number; + +let log: string[] = []; + +function make(tag: string): Handler { + let calls = 0; + return (x, y) => { + calls++; + log.push(tag + ":" + calls); + return x * 10 + y; + }; +} + +const table: Record = { + add: (x, y) => x + y, + neg: (x) => -x, + pow: (x, y) => x ** y, + made: make("made"), +}; + +function keys(): string[] { + return process.argv.length > 99 ? [] : ["add", "neg", "pow", "made", "missing", ""]; +} + +for (const k of keys()) { + const h = table[k] as Handler | undefined; + console.log(JSON.stringify(k), typeof h, h === undefined, h !== undefined && h === table[k]); + console.log(" optional", table[k]?.(3, 4) ?? "none"); + if (h) console.log(" checked", h(2, 5)); + try { + console.log(" asserted", table[k]!(5, 2)); + } catch (e) { + console.log(" asserted threw", (e as Error).name); + } +} + +// A handler that replaces its own entry while it runs: the replaced closure +// keeps running with its captured state. +function selfReplacing(): Handler { + const state = { hits: 0, label: "first" }; + return (x, y) => { + table["swap"] = make("second"); + state.hits++; + log.push(state.label + " still running after replace, hits " + state.hits); + return x + y + state.hits; + }; +} +table["swap"] = selfReplacing(); +console.log("swap 1", table["swap"]!(1, 2)); +console.log("swap 2", table["swap"]?.(1, 2)); +console.log("swap 3", table["swap"]!(1, 2)); + +// A handler that deletes its own entry. +function selfDeleting(name: string): Handler { + const captured = [name, "captured"]; + return (x) => { + delete table[name]; + log.push(captured.join(" ") + " after delete"); + return x; + }; +} +table["gone"] = selfDeleting("gone"); +const goneKey = keys()[0] === "add" ? "gone" : "add"; +console.log("gone", table[goneKey]?.(7, 0) ?? "none", table[goneKey]?.(7, 0) ?? "none"); + +// A checked local keeps its closure after the entry changes. +table["kept"] = make("kept-old"); +const kept = table["kept"] as Handler | undefined; +table["kept"] = make("kept-new"); +delete table["kept"]; +if (kept) console.log("kept", kept(4, 4)); +console.log("kept entry", table["kept"] === undefined); + +// Iteration while handlers add and delete keys: key order follows inserts. +const order: Record = { + b: (x) => x + 1, + a: (x) => { + delete order["c"]; + order["z"] = (v) => v * 2; + return x + 2; + }, + c: (x) => x + 3, + "1": (x) => x + 4, +}; +for (const k in order) console.log("in", k, order[k]?.(1, 0)); +console.log("keys", Object.keys(order).join(",")); +for (const k of Object.keys(order)) { + order[k + k] = (x) => x; + console.log("of", k, order[k]!(10, 0)); +} +console.log("keys", Object.keys(order).join(",")); + +// Reassigning the table binding itself from inside a handler. +let current: Record = { + go: (x) => { + current = { go: (y) => y * 100 }; + const local = [x, x]; + return local[0]! + local[1]!; + }, +}; +console.log("rebind", current["go"]!(3, 0), current["go"]!(3, 0)); +console.log(log.join("\n")); diff --git a/tests/corpus/8231-function-table-shapes.ts b/tests/corpus/8231-function-table-shapes.ts new file mode 100644 index 000000000..108c4cee0 --- /dev/null +++ b/tests/corpus/8231-function-table-shapes.ts @@ -0,0 +1,108 @@ +// Function dispatch through other table shapes: Map, arrays of +// handlers, object literals of methods (with and without `this`), handler +// mutation of the Map during calls, and adapters for handlers that declare +// fewer parameters than the table type. +type Op = (acc: number, arg: number) => number; + +const trace: string[] = []; +const byName = new Map(); +byName.set("add", (acc, arg) => acc + arg); +byName.set("half", (acc) => acc / 2); +byName.set("reset", () => { + byName.delete("reset"); + byName.set("reset", (acc) => acc - 1); + trace.push("reset replaced itself"); + return 0; +}); +byName.set("clear", (acc) => { + const keep = [acc, "before clear"]; + byName.clear(); + trace.push(keep.join(" ") + ", size " + byName.size); + return acc; +}); + +function program(): string[] { + return process.argv.length > 99 ? [] : ["add", "half", "nop", "reset", "reset", "add", "clear", "add"]; +} + +let acc = 8; +for (const name of program()) { + const handler = byName.get(name); + if (handler !== undefined) acc = handler(acc, 3); + else trace.push("no " + name); + console.log(name, acc, byName.get(name)?.(1, 1) ?? "-"); +} + +const byCode: Op[] = [(a, b) => a - b, (a) => a * 3, (a, b) => Math.max(a, b)]; +let total = 1; +for (let i = 0; i < 9; i++) { + total = byCode[i % 4]?.(total, i) ?? total + 100; + if (i % 4 < byCode.length) total = byCode[i % 4]!(total, 1); +} +console.log("array", total); + +const counter = { + base: 5, + bump(by: number): number { + this.base += by; + return this.base; + }, + twice(by: number): number { + return this.bump(by) + this.bump(by); + }, + plain(by: number): number { + return by * 2; + }, +}; +console.log("methods", counter.bump(1), counter.twice(2), counter.plain(3), counter["plain"](4)); + +// Methods that read `this`, called through computed keys and through a +// signature adapter (a method declaring fewer parameters than its type). +interface Account { + balance: number; + deposit: (amount: number, note: string) => number; + withdraw: (amount: number, note: string) => number; + peek: (amount: number, note: string) => number; +} +const account: Account = { + balance: 10, + deposit(amount) { + this.balance += amount; + return this.balance; + }, + withdraw(amount, note) { + this.balance -= amount; + trace.push("withdraw " + note); + return this.balance; + }, + peek: (amount) => amount, +}; +for (const op of ["deposit", "withdraw", "peek", "deposit"] as const) { + console.log(op, account[op](5, "n-" + op), account.balance); +} +const detached = account.peek; +console.log("detached", detached(3, "x")); + +interface Machine { + state: string; + on: Record string>; +} +const machine: Machine = { + state: "idle", + on: { + idle: (m, input) => (input > 0 ? "running" : m.state), + running: (m, input) => { + if (input < 0) { + m.on["stopped"] = () => "idle"; + return "stopped"; + } + return m.state; + }, + }, +}; +for (const input of [0, 1, 2, -1, 5, 3]) { + const next = machine.on[machine.state]?.(machine, input) ?? machine.state; + console.log("machine", machine.state, input, "->", next); + machine.state = next; +} +console.log(trace.join("\n")); diff --git a/tests/corpus/8232-optional-function-values.ts b/tests/corpus/8232-optional-function-values.ts new file mode 100644 index 000000000..5035c5f51 --- /dev/null +++ b/tests/corpus/8232-optional-function-values.ts @@ -0,0 +1,75 @@ +// Values typed `F | undefined` for a function type F: identity and +// equality (signature adapters compare as the function they adapt), typeof, +// truthiness, storage in arrays, records, class fields and Maps, optional +// parameters and returns. +type Unary = (x: number) => number; +type Binary = (x: number, y: number) => number; + +const double: Unary = (x) => x * 2; +const asBinary: Binary = double; +const maybe = (flag: boolean): Unary | undefined => (flag ? double : undefined); + +const a = maybe(true); +const b = maybe(true); +const none = maybe(false); +console.log(a === b, a === none, none === undefined, typeof a, typeof none, !!a, !!none); +console.log(a === double, (asBinary as unknown) === (double as unknown)); + +const slots: (Unary | undefined)[] = [double, undefined, (x) => x + 1]; +console.log(slots.map((f) => (f ? f(5) : "empty")).join(" "), slots.length); +console.log(slots.indexOf(double), slots.includes(undefined), slots.filter((f) => f !== undefined).length); + +interface Hooks { + before?: Unary; + after?: Unary; +} +const hooks: Hooks = { after: (x) => x - 1 }; +console.log(hooks.before?.(1) ?? "no before", hooks.after?.(1) ?? "no after", "before" in hooks); + +class Button { + onClick: Unary | undefined; + clicks = 0; + click(): number { + this.clicks++; + return this.onClick?.(this.clicks) ?? -1; + } +} +const button = new Button(); +console.log(button.click()); +button.onClick = (n) => n * 100; +console.log(button.click(), button.click()); +button.onClick = undefined; +console.log(button.click(), button.onClick === undefined); + +const registry = new Map([ + ["set", double], + ["unset", undefined], +]); +console.log(registry.get("set")?.(4), registry.get("unset")?.(4), registry.has("unset"), registry.get("other") === undefined); + +function run(x: number, transform?: Unary, combine?: Binary): number { + const t = transform ? transform(x) : x; + return combine === undefined ? t : combine(t, x); +} +console.log(run(3), run(3, double), run(3, undefined, (p, q) => p * q), run(3, (x) => -x, asBinary)); + +function pick(name: string): Binary | undefined { + switch (name) { + case "mul": + return (x, y) => x * y; + case "first": + return (x) => x; + default: + return undefined; + } +} +for (const name of ["mul", "first", "none"]) { + const f = pick(name); + console.log(name, f?.(6, 7), f === undefined ? "undefined" : typeof f); +} + +let late: Unary | undefined; +const callLate = (): number => late?.(9) ?? 0; +console.log(callLate()); +late = (x) => x + 0.5; +console.log(callLate()); diff --git a/tests/harness/corpus-costs/8230-function-record-dispatch.ts.json b/tests/harness/corpus-costs/8230-function-record-dispatch.ts.json new file mode 100644 index 000000000..1dd4b287a --- /dev/null +++ b/tests/harness/corpus-costs/8230-function-record-dispatch.ts.json @@ -0,0 +1 @@ +{ "plain": 4700, "san": 5800 } diff --git a/tests/harness/corpus-costs/8231-function-table-shapes.ts.json b/tests/harness/corpus-costs/8231-function-table-shapes.ts.json new file mode 100644 index 000000000..1dd4b287a --- /dev/null +++ b/tests/harness/corpus-costs/8231-function-table-shapes.ts.json @@ -0,0 +1 @@ +{ "plain": 4700, "san": 5800 } diff --git a/tests/harness/corpus-costs/8232-optional-function-values.ts.json b/tests/harness/corpus-costs/8232-optional-function-values.ts.json new file mode 100644 index 000000000..1dd4b287a --- /dev/null +++ b/tests/harness/corpus-costs/8232-optional-function-values.ts.json @@ -0,0 +1 @@ +{ "plain": 4700, "san": 5800 } From 52abff7c147f5429310136513d17fb72880dd3f8 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 13:33:51 -0700 Subject: [PATCH 08/20] Clone union records whose override is an empty array literal Spreading a union of records and replacing a shared field keeps each value's variant (lowerUnionRecordClone), but only when every override's type lifts into the field. An empty array literal is typed `never[]`, which maps to nothing, so `{ ...entry, content: [] }` over a union fell back to the ordinary literal path. That path cannot build a union of spread records, and the literal was refused (SC2001). An empty array literal now builds an empty array of each arm's own element type. This covers a field that is an array and a field that is a union with exactly one array arm (`string | Part[]`). pi's session-manager repairs messages with missing content this way (`return [{ ...message, content: [] }]`). The refusal rejected the whole SessionManager class. Co-Authored-By: Claude Opus 5.5 --- .../lowering/expressions/object-literals.ts | 19 +++- .../union-clone-empty-array-overrides.ts.json | 6 ++ .../union-clone-empty-array-overrides.ts | 92 +++++++++++++++++++ .../union-clone-empty-array-overrides.ts.json | 1 + 4 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-clone-empty-array-overrides.ts.json create mode 100644 tests/corpus/union-clone-empty-array-overrides.ts create mode 100644 tests/harness/corpus-costs/union-clone-empty-array-overrides.ts.json diff --git a/packages/compiler/src/frontend/lowering/expressions/object-literals.ts b/packages/compiler/src/frontend/lowering/expressions/object-literals.ts index 70766949a..ae82133bf 100644 --- a/packages/compiler/src/frontend/lowering/expressions/object-literals.ts +++ b/packages/compiler/src/frontend/lowering/expressions/object-literals.ts @@ -2513,6 +2513,20 @@ export function lowerObjectLiteral( : { kind: "seqExpr", stmts: spreadStmts, result, type, loc }; } +/** The field type is an array, or a union with exactly one array arm. */ +function singleArrayArm(lowerer: Lowerer, type: IrType): boolean { + if (type.kind === "array") return true; + if (type.kind !== "union") return false; + const arms = lowerer.unions.get(type.unionId)?.arms ?? []; + return arms.filter((arm) => arm.kind === "array").length === 1; +} + +function isEmptyArrayLiteral(node: ts.Node): boolean { + let inner = node; + while (ts.isParenthesizedExpression(inner)) inner = inner.expression; + return ts.isArrayLiteralExpression(inner) && inner.elements.nodes.length === 0; +} + /** A union of plain records can preserve its existing variant while * replacing shared fields. Dispatch before constructing the copy: only * the selected arm evaluates overrides, and recordClone snapshots fields @@ -2561,8 +2575,11 @@ function lowerUnionRecordClone(lowerer: Lowerer, expr: ts.ObjectLiteralExpressio for (let index = 0; index < props.length; index++) { const prop = props[index]!; const node = ts.isPropertyAssignment(prop) ? prop.initializer : prop.name; - const valueType = lowerer.mapTypeOf(lowerer.typeOf(node)); const fieldType = shape.fields.find((field) => field.name === names[index])!.type; + // An empty array literal (`content: []`, typed `never[]`) builds a + // fresh array of each arm's own element type. + if (isEmptyArrayLiteral(node) && singleArrayArm(lowerer, fieldType)) continue; + const valueType = lowerer.mapTypeOf(lowerer.typeOf(node)); const lift = valueType ? lowerer.widthLiftPlan(valueType, fieldType) : null; // The inferred result must retain an override's own fields and // aliases. A structural copy into the source field's narrower shape diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-clone-empty-array-overrides.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-clone-empty-array-overrides.ts.json new file mode 100644 index 000000000..cb11c5098 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-clone-empty-array-overrides.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/union-clone-empty-array-overrides.ts" + ], + "diags": [] +} diff --git a/tests/corpus/union-clone-empty-array-overrides.ts b/tests/corpus/union-clone-empty-array-overrides.ts new file mode 100644 index 000000000..98ddac59c --- /dev/null +++ b/tests/corpus/union-clone-empty-array-overrides.ts @@ -0,0 +1,92 @@ +// Spreading a union of records and overriding a shared field with an empty +// array literal keeps each value's own variant: the `[]` (typed `never[]`) +// becomes an empty array of the selected arm's element type, also when that +// arm's field is a union with one array arm (`string | Part[]`). + +interface TextPart { + type: "text"; + text: string; +} +interface ImagePart { + type: "image"; + data: string; +} +interface UserEntry { + role: "user"; + content: string | (TextPart | ImagePart)[]; + timestamp: number; +} +interface AssistantEntry { + role: "assistant"; + content: TextPart[]; + model: string; + timestamp: number; +} +interface ToolEntry { + role: "tool"; + toolCallId: string; + content: (TextPart | ImagePart)[]; + isError: boolean; + timestamp: number; +} +interface SystemEntry { + role: "system"; + content: string; + timestamp: number; +} +interface NoteEntry { + role: "note"; + text: string; + timestamp: number; +} +type Entry = UserEntry | AssistantEntry | ToolEntry | SystemEntry | NoteEntry; + +function repair(entry: Entry, clear: boolean): Entry[] { + if (entry.role === "system" && clear) return [{ ...entry, content: "" }]; + if ((entry.role === "user" || entry.role === "assistant" || entry.role === "tool") && clear) { + return [{ ...entry, content: [] }]; + } + return [entry]; +} + +interface Circle { + kind: "circle"; + tags: string[]; + radius: number; +} +interface Square { + kind: "square"; + tags: string[]; + side: number; +} +type Shape = Circle | Square; + +function cleared(shape: Shape): Shape { + return { ...shape, tags: [] }; +} + +const entries: Entry[] = [ + { role: "user", content: [{ type: "text", text: "hello" }], timestamp: 1 }, + { role: "assistant", content: [{ type: "text", text: "hi" }], model: "m", timestamp: 2 }, + { role: "tool", toolCallId: "t1", content: [{ type: "image", data: "AA==" }], isError: false, timestamp: 3 }, + { role: "note", text: "hi", timestamp: 4 }, + { role: "system", content: "be brief", timestamp: 5 }, + { role: "user", content: "kept", timestamp: 6 }, +]; +entries.forEach((entry, index) => { + for (const fixed of repair(entry, index % 2 === 0 || entry.role === "system")) { + console.log(fixed.role, JSON.stringify(fixed), fixed === entry); + } +}); +console.log(JSON.stringify(entries[0])); + +const shapes: Shape[] = [ + { kind: "circle", tags: ["round", "red"], radius: 2 }, + { kind: "square", tags: ["blue"], side: 3 }, +]; +for (const shape of shapes) { + const copy = cleared(shape); + copy.tags.push("fresh"); + const area = copy.kind === "circle" ? copy.radius * copy.radius * 3 : copy.side * copy.side; + console.log(copy.kind, area, copy.tags.join("+"), shape.tags.join("+"), copy !== shape); +} diff --git a/tests/harness/corpus-costs/union-clone-empty-array-overrides.ts.json b/tests/harness/corpus-costs/union-clone-empty-array-overrides.ts.json new file mode 100644 index 000000000..1254aa6e4 --- /dev/null +++ b/tests/harness/corpus-costs/union-clone-empty-array-overrides.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } From be0e825ac1e2a1f8e2c88ce9bfafbb8d2788635a Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 14:35:56 -0700 Subject: [PATCH 09/20] Map `any` parameters of interface method and call signatures as checked values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A parameter a TypeScript program types `any` maps to the checked-dynamic slot for function declarations and spelled function types, but not for method or call signatures of interfaces and type literals. So `interface Tool { execute(id: string, params: any): Promise }` was refused (SC2009), along with every value of the interface, whenever no class implementing it gave it a class view. Method and call signatures in program source now take the same rule. pi's ToolDefinition declares `execute(toolCallId: string, params: Static, …)`. Under --node-islands, typebox's types resolve to `any`, so the parameter is `any`. The refusal rejected the AgentSession class through its `_customTools: ToolDefinition[]` field, and CreateAgentSessionRuntimeResult with it. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/frontend/type-mapper.ts | 12 ++++-- .../method-signature-any-params.ts.json | 6 +++ tests/corpus/method-signature-any-params.ts | 39 +++++++++++++++++++ .../method-signature-any-params.ts.json | 1 + 4 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/method-signature-any-params.ts.json create mode 100644 tests/corpus/method-signature-any-params.ts create mode 100644 tests/harness/corpus-costs/method-signature-any-params.ts.json diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index 17f9d0d11..aa3af2b14 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -5218,10 +5218,12 @@ function mapCallSignature(sig: ts.Signature, ctx: TypeMapperCtx): IrType | null // A parameter a TypeScript program types `any` takes whatever the // caller passes: the checked-dynamic slot, as an `any`-typed binding // already is (dynFallbackType). Declaration files keep their paths. - // Only function declarations, spelled function types and lambda - // parameters whose default is the only source of their type (`(n = - // lookup() ?? 5) => ...` with an `any` lookup): other lambdas and - // methods in object literals keep their existing checked-dynamic story. + // Only function declarations, spelled function types, method and call + // signatures of interfaces and type literals (`execute(params: any): + // Promise`), and lambda parameters whose default is the only source + // of their type (`(n = lookup() ?? 5) => ...` with an `any` lookup): + // other lambdas and methods in object literals keep their existing + // checked-dynamic story. if ( !pt && !rest && @@ -5230,6 +5232,8 @@ function mapCallSignature(sig: ts.Signature, ctx: TypeMapperCtx): IrType | null decl.parent !== undefined && (ts.isFunctionDeclaration(decl.parent) || ts.isFunctionTypeNode(decl.parent) || + ts.isMethodSignatureDeclaration(decl.parent) || + ts.isCallSignatureDeclaration(decl.parent) || ((ts.isArrowFunction(decl.parent) || ts.isFunctionExpression(decl.parent)) && decl.initializer !== undefined && decl.type === undefined && diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/method-signature-any-params.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/method-signature-any-params.ts.json new file mode 100644 index 000000000..b3f7ed165 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/method-signature-any-params.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/method-signature-any-params.ts" + ], + "diags": [] +} diff --git a/tests/corpus/method-signature-any-params.ts b/tests/corpus/method-signature-any-params.ts new file mode 100644 index 000000000..e505ae78c --- /dev/null +++ b/tests/corpus/method-signature-any-params.ts @@ -0,0 +1,39 @@ +// A method or call signature of an interface whose parameter is typed `any` +// (directly, or through an alias that resolves to `any`) takes whatever the +// caller passes, as a function type's `any` parameter already does: the +// interface compiles, and implementations receive the checked-dynamic value. + +type Loose = any; + +interface Tool { + name: string; + execute(id: string, params: any, extra?: Loose): Promise; + describe(value: Loose): string; +} + +interface Formatter { + (value: any): string; +} + +const tools: Tool[] = [ + { + name: "count", + execute: async (id, params) => `${id} has ${Object.keys(params ?? {}).length} keys`, + describe: (value) => `count(${Array.isArray(value) ? "array" : typeof value})`, + }, +]; + +const format: Formatter = (value) => (typeof value === "number" ? value.toFixed(1) : String(value)); + +async function main(): Promise { + const inputs: unknown[] = [{ a: 1, b: [2, 3] }, "text", 42, null, [1, 2]]; + for (const tool of tools) { + for (const input of inputs) { + console.log(tool.name, await tool.execute("call", input), tool.describe(input)); + } + console.log(tool.name, await tool.execute("extra", { k: true }, { more: "yes" })); + } + console.log(format(3), format("x"), format(undefined)); +} + +main().catch((e: unknown) => console.log("failed", e instanceof Error ? e.message : String(e))); diff --git a/tests/harness/corpus-costs/method-signature-any-params.ts.json b/tests/harness/corpus-costs/method-signature-any-params.ts.json new file mode 100644 index 000000000..1254aa6e4 --- /dev/null +++ b/tests/harness/corpus-costs/method-signature-any-params.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } From 3375304945e1b6e548d501b3c9d403f1c5d155fa Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 14:40:32 -0700 Subject: [PATCH 10/20] Erase lambdas that implement a generic method through their context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A lambda with no type parameters of its own, contextually typed by a generic signature (`send: (m) => …` for an interface's `send(m: Message): string`), takes that signature's type parameters: its parameter is `Message`. A generic lambda written with its own type parameters already lowers once, erased, with them bound to `unknown`, but this one tried to map `Message` with `T` unbound and was refused (SC2009). It now lowers the same way, erased, binding the contextual signature's type parameters to `unknown`. Parameters bound in the enclosing scope keep their bindings. Co-Authored-By: Claude Opus 5.5 --- .../src/frontend/lowering/lower-functions.ts | 42 +++++++++++++++++++ .../generic-member-lambda-impls.ts.json | 6 +++ tests/corpus/generic-member-lambda-impls.ts | 40 ++++++++++++++++++ .../generic-member-lambda-impls.ts.json | 1 + 4 files changed, 89 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-member-lambda-impls.ts.json create mode 100644 tests/corpus/generic-member-lambda-impls.ts create mode 100644 tests/harness/corpus-costs/generic-member-lambda-impls.ts.json diff --git a/packages/compiler/src/frontend/lowering/lower-functions.ts b/packages/compiler/src/frontend/lowering/lower-functions.ts index bd9b5f55d..7f4c8dfb7 100644 --- a/packages/compiler/src/frontend/lowering/lower-functions.ts +++ b/packages/compiler/src/frontend/lowering/lower-functions.ts @@ -167,6 +167,23 @@ function declareFunctionArguments( } } +/** The type parameters a lambda without its own takes from a generic + * contextual signature, unbound in the current scope. */ +function contextualTypeParameterSymbols( + lowerer: Lowerer, + node: ts.ArrowFunction | ts.FunctionExpression, +): ts.Symbol[] { + const signatures = lowerer.checker.getCallSignatures(lowerer.typeOf(node)); + if (signatures.length !== 1) return []; + const symbols: ts.Symbol[] = []; + for (const parameter of signatures[0]!.getTypeParameters()) { + const symbol = parameter.getSymbol(); + if (!symbol || lowerer.typeParamBindings?.has(symbol)) return []; + symbols.push(symbol); + } + return symbols; +} + /** Lifts an arrow function / function expression / nested declaration / * object-literal shorthand method to a module-level function and yields * the `closure` expression creating it. */ @@ -202,6 +219,31 @@ export function lowerLambda( lowerer.typeParamBindings = previous; } } + // A lambda contextually typed by a GENERIC signature (`send: (m) => …` + // for `send(m: Msg): string`) takes that signature's type + // parameters as its own; it lowers once, erased, like a generic lambda. + if ( + (ts.isArrowFunction(node) || ts.isFunctionExpression(node)) && + !node.typeParameters?.nodes.length && + !lowerer.erasingLambdas.has(node) + ) { + const contextual = contextualTypeParameterSymbols(lowerer, node); + if (contextual.length > 0) { + const previous = lowerer.typeParamBindings; + const bindings = new Map(previous ?? []); + for (const symbol of contextual) bindings.set(symbol, DYN); + lowerer.typeParamBindings = bindings; + lowerer.erasingLambdas.add(node); + lowerer.erasingDepth++; + try { + return lowerLambda(lowerer, node, checkedReceiver); + } finally { + lowerer.erasingDepth--; + lowerer.erasingLambdas.delete(node); + lowerer.typeParamBindings = previous; + } + } + } const prepared = ts.isArrowFunction(node) || ts.isFunctionExpression(node) ? lowerer.preparedClassFactories.get(node) diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-member-lambda-impls.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-member-lambda-impls.ts.json new file mode 100644 index 000000000..bc87c6ea0 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-member-lambda-impls.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/generic-member-lambda-impls.ts" + ], + "diags": [] +} diff --git a/tests/corpus/generic-member-lambda-impls.ts b/tests/corpus/generic-member-lambda-impls.ts new file mode 100644 index 000000000..fd25bae96 --- /dev/null +++ b/tests/corpus/generic-member-lambda-impls.ts @@ -0,0 +1,40 @@ +// An object literal implements an interface's generic method with a plain +// lambda (`send: (m) => …` for `send(m: Message): string`). The +// lambda takes the contextual signature's type parameters as its own and +// lowers once, erased, like a generic lambda: its parameters hold whatever +// each call passes. + +interface Message { + content: string; + details?: T; +} + +interface Sender { + name: string; + send(message: Message): string; + notify(message: Message): void; + pick(keys: K[], fallback: K): K; +} + +const log: string[] = []; + +function makeSender(name: string): Sender { + return { + name, + send: (message) => `${name} sent ${message.content} (${message.details === undefined ? "no details" : JSON.stringify(message.details)})`, + notify: (message) => { + log.push(`${name} notified ${message.content}`); + }, + pick: (keys, fallback) => keys[0] ?? fallback, + }; +} + +const senders: Sender[] = [makeSender("first"), makeSender("second")]; +for (const sender of senders) { + console.log(sender.send({ content: "hi", details: { n: 3 } })); + console.log(sender.send({ content: "bare" })); + console.log(sender.send({ content: "list", details: [1, 2] })); + sender.notify({ content: "ping", details: true }); + console.log(sender.pick(["a", "b"], "z"), sender.pick([], "z")); +} +console.log(log.join("; ")); diff --git a/tests/harness/corpus-costs/generic-member-lambda-impls.ts.json b/tests/harness/corpus-costs/generic-member-lambda-impls.ts.json new file mode 100644 index 000000000..1254aa6e4 --- /dev/null +++ b/tests/harness/corpus-costs/generic-member-lambda-impls.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } From 171214b2c214e71c6dda67b9401859dbf8b0e967 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 14:40:32 -0700 Subject: [PATCH 11/20] Discard the result of void generic methods called through class-backed interfaces A generic method called through an interface value that classes implement dispatches by name on the checked-dynamic receiver, and the result converts to the call's instantiated return type. For a void method, that conversion (`unknown` to `void`) was refused (SC1100). The result is now discarded, as JavaScript does. The dispatch helper selected overriding subclasses' erased instances with a ternary, which has no void form (an internal compiler error once the call got that far). A void method now selects them with if statements. Co-Authored-By: Claude Opus 5.5 --- .../lowering/class-dynamic-dispatch.ts | 50 ++++++++++++------- .../src/frontend/lowering/lower-calls.ts | 13 ++++- .../generic-interface-void-calls.ts.json | 6 +++ tests/corpus/generic-interface-void-calls.ts | 45 +++++++++++++++++ .../generic-interface-void-calls.ts.json | 1 + 5 files changed, 96 insertions(+), 19 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-interface-void-calls.ts.json create mode 100644 tests/corpus/generic-interface-void-calls.ts create mode 100644 tests/harness/corpus-costs/generic-interface-void-calls.ts.json diff --git a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts index 2af84f8fb..7992f7fec 100644 --- a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts +++ b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts @@ -6793,24 +6793,41 @@ export class ClassDynamicDispatch { type: result, loc, }; + // A void method dispatches its overrides with statements: a ternary + // has no void form. + let voidDispatch: IrStmt[] = [{ kind: "exprStmt", expr: call, loc }]; for (const override of overrides.toReversed()) { const subType: IrType = { kind: "object", className: override.cls.def.name }; + const isOverride: IrExpr = { + kind: "instanceOf", + value: receiver, + className: override.cls.def.name, + type: BOOL, + loc, + }; + const overrideCall: IrExpr = { + kind: "call", + callee: override.callee, + args: [{ kind: "downcast", value: receiver, type: subType, loc }, ...args], + type: result, + loc, + }; + if (result.kind === "void") { + voidDispatch = [ + { + kind: "if", + cond: isOverride, + then: [{ kind: "exprStmt", expr: overrideCall, loc }], + else_: voidDispatch, + loc, + }, + ]; + continue; + } call = { kind: "ternary", - cond: { - kind: "instanceOf", - value: receiver, - className: override.cls.def.name, - type: BOOL, - loc, - }, - then: { - kind: "call", - callee: override.callee, - args: [{ kind: "downcast", value: receiver, type: subType, loc }, ...args], - type: result, - loc, - }, + cond: isOverride, + then: overrideCall, else_: call, type: result, loc, @@ -6818,10 +6835,7 @@ export class ClassDynamicDispatch { } let body: IrStmt[]; if (result.kind === "void") - body = [ - { kind: "exprStmt", expr: call, loc }, - { kind: "return", value: dynUndefinedExpr(loc), loc }, - ]; + body = [...voidDispatch, { kind: "return", value: dynUndefinedExpr(loc), loc }]; else { const boxed = lowerer.coerceToExpected(call, DYN); if (boxed.type.kind !== "dyn") return fence(); diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index c8f02562e..e2f7ade90 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -10073,7 +10073,18 @@ function lowerObjLitGenericMethodCall( type: DYN, loc, }; - return lowerer.coerceInto(call, invoked, lowerer.irTypeOf(call)); + const resultType = lowerer.irTypeOf(call); + // A void method's runtime result is discarded, as JavaScript does. + if (resultType.kind === "void") { + return { + kind: "seqExpr", + stmts: [{ kind: "exprStmt", expr: invoked, loc }], + result: { kind: "libCall", fn: "console.noop", args: [], type: VOID, loc }, + type: VOID, + loc, + }; + } + return lowerer.coerceInto(call, invoked, resultType); } } if (onInterface) { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-interface-void-calls.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-interface-void-calls.ts.json new file mode 100644 index 000000000..a5c69b714 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-interface-void-calls.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/generic-interface-void-calls.ts" + ], + "diags": [] +} diff --git a/tests/corpus/generic-interface-void-calls.ts b/tests/corpus/generic-interface-void-calls.ts new file mode 100644 index 000000000..d810383f7 --- /dev/null +++ b/tests/corpus/generic-interface-void-calls.ts @@ -0,0 +1,45 @@ +// Calling a void generic method through an interface value that classes +// implement (the receiver is a checked-dynamic object, and the call +// dispatches by name to the class's erased method) discards the result, as +// JavaScript does, instead of converting it to `void`. + +interface Message { + content: string; + details?: T; +} + +interface Channel { + name: string; + publish(message: Message): void; + count(items: T[]): number; +} + +const delivered: string[] = []; + +class LogChannel implements Channel { + name: string; + constructor(name: string) { + this.name = name; + } + publish(message: Message): void { + delivered.push(`${this.name}:${message.content}:${message.details === undefined ? "-" : JSON.stringify(message.details)}`); + } + count(items: T[]): number { + return items.length; + } +} + +class LoudChannel extends LogChannel { + publish(message: Message): void { + super.publish({ content: message.content.toUpperCase(), details: message.details }); + } +} + +const channels: Channel[] = [new LogChannel("quiet"), new LoudChannel("loud")]; +for (const channel of channels) { + channel.publish({ content: "hello", details: [1, 2] }); + channel.publish({ content: "bare" }); + const result = channel.publish({ content: "typed", details: "x" }); + console.log(channel.name, result === undefined, channel.count(["a", "b", "c"])); +} +console.log(delivered.join(" | ")); diff --git a/tests/harness/corpus-costs/generic-interface-void-calls.ts.json b/tests/harness/corpus-costs/generic-interface-void-calls.ts.json new file mode 100644 index 000000000..1254aa6e4 --- /dev/null +++ b/tests/harness/corpus-costs/generic-interface-void-calls.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } From 062700bdde6125d6f38192d6d636dc9484c17b61 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 10:08:00 -0700 Subject: [PATCH 12/20] Convert asserted values that no ordinary conversion reaches `x as T` erases (or, record to record, converts), and the value then meets its slot with x's representation. Where no conversion reaches the slot the program was refused: - a union whose arms only partly convert into T (`api as Partial>` on `S | Partial> | undefined`) failed at the next typed slot with SC2003; - a record or array whose member types the assertion narrows (`models as Model<"one">[]` on a union-element array, `item as Narrow` with a narrower field) failed with SC2002 or SC2003. The lowerer now remembers the asserted type of an erased value, and a slot that would refuse it converts it as asserted: the arms T excludes throw the catchable TypeError of a lying assertion and the others convert, and records and arrays take the checked deep conversion, which validates the value member by member and names the deepest failing path, as a cast of an unknown value does. The conversions run only where the slot would otherwise refuse the value, so everything that already lowered keeps its code. Co-Authored-By: Claude Opus 5.5 --- .../frontend/lowering/coercions/asserted.ts | 87 +++++++++++++++++++ .../lowering/expressions/narrowing.ts | 10 +++ .../compiler/src/frontend/lowering/lowerer.ts | 9 +- .../checked-union-record-assertions.ts.json | 6 ++ .../corpus/checked-union-record-assertions.ts | 69 +++++++++++++++ .../checked-union-record-assertions.ts.json | 1 + tests/harness/dyncheck.test.ts | 26 ++++++ 7 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 packages/compiler/src/frontend/lowering/coercions/asserted.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/checked-union-record-assertions.ts.json create mode 100644 tests/corpus/checked-union-record-assertions.ts create mode 100644 tests/harness/corpus-costs/checked-union-record-assertions.ts.json diff --git a/packages/compiler/src/frontend/lowering/coercions/asserted.ts b/packages/compiler/src/frontend/lowering/coercions/asserted.ts new file mode 100644 index 000000000..43c447cb0 --- /dev/null +++ b/packages/compiler/src/frontend/lowering/coercions/asserted.ts @@ -0,0 +1,87 @@ +/* Slots reached only through what a type assertion claims. */ +import type { IrExpr, IrType, SrcLoc } from "../../../ir/ir.js"; +import { canDynCheckTo, DYN, isUnitType, typeEquals } from "../../../ir/ir.js"; +import { lowerRecordOvfCaptureHelper } from "../containers/indexed-objects.js"; +import type { Lowerer } from "../lowerer.js"; +import { unionRetagHelper } from "./unions.js"; + +/** A value an erased assertion produced (`x as T`, lowerer.assertedValues) + * flowing into a slot that no ordinary conversion reaches. Runs only where + * the slot would otherwise refuse the value (SC2002/SC2003), so every + * conversion that already lowered keeps its code. + * + * - A union whose arms only partly convert (`api as Partial>` on `S | Partial> | undefined`): the arms that convert + * into T convert into the slot, and an arm the assertion excludes (no + * conversion into T) throws the catchable TypeError of a lying assertion. + * - A record or array whose member types the assertion narrows (`models as + * Model<"one">[]` on a union-element array): the checked deep conversion + * a cast of an unknown value takes, which validates the value member by + * member and names the deepest failing path. */ +export function assertedConversion( + lowerer: Lowerer, + expr: IrExpr, + expected: IrType, +): IrExpr | null { + const asserted = lowerer.assertedValues.get(expr); + if (!asserted) return null; + const loc = expr.loc; + if (expr.type.kind === "union") { + const from = lowerer.unions.get(expr.type.unionId); + if (!from) return null; + const values = from.arms.filter((arm) => !isUnitType(arm)); + const kept = values.filter((arm) => convertsInto(lowerer, arm, asserted, loc)); + if (kept.length === 0 || kept.length === values.length) return null; + if (asserted.kind === "union" && expected.kind === "union") { + // The arms T excludes trap; the others (units included) re-tag. + const trappable = new Set(); + from.arms.forEach((arm, tag) => { + if (!isUnitType(arm) && !kept.includes(arm)) trappable.add(tag); + }); + const helper = unionRetagHelper(lowerer, from.id, expected.unionId, loc, trappable); + return helper ? { kind: "call", callee: helper, args: [expr], type: expected, loc } : null; + } + // A record or array T holds one arm's values: extract that arm (every + // other arm, a unit too, contradicts the assertion) and convert it. + if (asserted.kind === "union" || kept.length !== 1) return null; + const arm = kept[0]!; + const helper = lowerer.narrowedArmHelper(from.id, arm, loc); + if (!helper) return null; + const narrowed = lowerer.checkedNarrowCall(helper, expr, arm, loc); + if (typeEquals(arm, expected)) return narrowed; + const converted = lowerer.coerceToExpected(narrowed, expected); + return typeEquals(converted.type, expected) ? converted : null; + } + if (expected.kind !== "record" && expected.kind !== "array") return null; + if (expected.kind === "record" && lowerer.shapes.get(expected.shapeId)?.indexValue !== undefined) + return null; + if ( + !lowerer.dynConvertible(expr.type) || + !canDynCheckTo( + expected, + (id) => lowerer.shapes.get(id), + (id) => lowerer.unions.get(id), + ) + ) + return null; + return { kind: "dynCheck", value: lowerer.coerceToExpected(expr, DYN), type: expected, loc }; +} + +/** True when a value of `arm` has a structural conversion into `target`: + * the same type, one of its arms, a width lift, or (into an + * index-signature record) the overflow capture. */ +function convertsInto(lowerer: Lowerer, arm: IrType, target: IrType, loc: SrcLoc): boolean { + if (typeEquals(arm, target)) return true; + if ( + target.kind === "union" && + lowerer.unions.get(target.unionId)?.arms.some((candidate) => typeEquals(candidate, arm)) + ) + return true; + if (lowerer.widthLiftPlan(arm, target) !== null) return true; + return ( + arm.kind === "record" && + target.kind === "record" && + lowerer.shapes.get(target.shapeId)?.indexValue !== undefined && + lowerRecordOvfCaptureHelper(lowerer, arm.shapeId, target.shapeId, loc) !== null + ); +} diff --git a/packages/compiler/src/frontend/lowering/expressions/narrowing.ts b/packages/compiler/src/frontend/lowering/expressions/narrowing.ts index 9f0559672..19f879bcf 100644 --- a/packages/compiler/src/frontend/lowering/expressions/narrowing.ts +++ b/packages/compiler/src/frontend/lowering/expressions/narrowing.ts @@ -1481,6 +1481,7 @@ export function lowerAsExpression( target?.kind === "record" && inner.type.shapeId !== target.shapeId ) { + lowerer.assertedValues.set(inner, target); return lowerer.coerceInto(expr, inner, target); } // `u as Arm` on a UNION value is `u!`'s spelling with a named arm @@ -1515,6 +1516,15 @@ export function lowerAsExpression( const asserted = checkedClassUnionAssertion(lowerer, inner, target, locOf(expr)); if (asserted) return asserted; } + // The erased value still answers to the assertion: a slot it cannot + // reach by an ordinary conversion converts it as asserted + // (assertedConversion). + if ( + target && + (target.kind === "union" || target.kind === "record" || target.kind === "array") && + (inner.type.kind === "union" || inner.type.kind === "array") + ) + lowerer.assertedValues.set(inner, target); return inner; // erasure, unchanged } diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index feca34427..491ca7c11 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -118,6 +118,7 @@ import { deferredReadHelper, widthNarrowHelper, } from "./coercions/unions.js"; +import { assertedConversion } from "./coercions/asserted.js"; import { lowerUtilTypeValue, pathModuleValue, @@ -2247,6 +2248,10 @@ export class Lowerer { ts.Type, { tsBindings: Map; instance: ts.Type }[] >(); + /** Values an erased type assertion produced, with the asserted type + * (`x as T` keeps x's representation): a slot that no ordinary + * conversion reaches converts them as asserted (assertedConversion). */ + readonly assertedValues = new WeakMap(); /** Non-null while an IMPLICIT-ANY instance body lowers (npm-static JS; * see generic-functions.ts): bound param symbol → * the call site's checker type, consulted by typeOf for identifier @@ -10599,6 +10604,8 @@ export class Lowerer { if (!typeEquals(expr.type, expected)) { const w = this.widthCoerce(expr, expected); if (w) return w; + const asserted = assertedConversion(this, expr, expected); + if (asserted) return asserted; // A UNIT (null/undefined) flowing into a plain non-nullable slot // the checker approved (`null!` casts, non-strict assignments): // the stranded-source stance (divergence 38) without a union in @@ -10617,7 +10624,7 @@ export class Lowerer { if (helper) { return { kind: "call", callee: helper, args: [expr], type: expected, loc: expr.loc }; } - return expr; + return assertedConversion(this, expr, expected) ?? expr; } if (expr.type.kind === "void") { // A void CALL RESULT flowing into a union with an undefined arm diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/checked-union-record-assertions.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/checked-union-record-assertions.ts.json new file mode 100644 index 000000000..ff77d1a95 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/checked-union-record-assertions.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/checked-union-record-assertions.ts" + ], + "diags": [] +} diff --git a/tests/corpus/checked-union-record-assertions.ts b/tests/corpus/checked-union-record-assertions.ts new file mode 100644 index 000000000..f5d43cee1 --- /dev/null +++ b/tests/corpus/checked-union-record-assertions.ts @@ -0,0 +1,69 @@ +// Assertions whose target no static conversion reaches: a union asserted to +// the one layout some of its arms convert into, and records or arrays whose +// member types the assertion narrows. The value is checked against the +// asserted type; these programs assert truthfully, so they run as in Node. + +type Api = "alpha" | "beta" | (string & {}); + +interface Streams { + stream: (x: number) => string; + fetch?: (x: number) => string; +} + +interface Options { + name: string; + api?: Streams | Partial>; +} + +function isStreams(value: unknown): value is Streams { + return typeof value === "object" && value !== null && "stream" in value && typeof value.stream === "function"; +} + +function run(input: Options): string[] { + const single = input.api && isStreams(input.api) ? input.api : undefined; + const byApi = single || !input.api ? undefined : (input.api as Partial>); + const all = single ? [single] : Object.values(byApi ?? {}).filter((entry) => entry !== undefined); + return all.map((entry) => `${input.name}:${entry.stream(all.length)}`); +} + +const shared: Streams = { stream: (x) => `shared${x}` }; +console.log(run({ name: "single", api: shared })); +console.log(run<"alpha" | "beta">({ name: "map", api: { alpha: { stream: (x) => `alpha${x}` }, beta: shared } })); +console.log(run<"alpha">({ name: "none" })); + +type KnownKind = "alpha" | "beta" | "gamma"; + +interface AlphaOpts { + fast?: boolean; +} +interface BetaOpts { + depth?: number; +} + +interface Item { + id: string; + kind: TKind; + opts?: TKind extends "alpha" ? AlphaOpts : TKind extends "beta" ? BetaOpts : never; +} + +interface Other { + id: string; + kind: string; + size: number; +} + +const alpha: Item<"alpha"> = { id: "a1", kind: "alpha", opts: { fast: true } }; +const gamma: Item<"gamma"> = { id: "g1", kind: "gamma" }; +const other: Other = { id: "o1", kind: "other", size: 2 }; + +const everything: (Item | Other)[] = [alpha, gamma, other, { id: "g2", kind: "gamma" }]; +const gammas = everything.filter((entry) => entry.kind === "gamma") as Item<"gamma">[]; +console.log(gammas.length, gammas.map((entry) => `${entry.id}:${entry.opts === undefined}`).join(",")); + +const items: Item[] = [gamma, alpha]; +const first = items[0]! as Item<"gamma">; +console.log(first.id, first.kind, first.opts === undefined); + +const stored: { list: (Item | Other)[] } = { list: [gamma] }; +const restored = stored.list as Item<"gamma">[]; +console.log(JSON.stringify(restored)); diff --git a/tests/harness/corpus-costs/checked-union-record-assertions.ts.json b/tests/harness/corpus-costs/checked-union-record-assertions.ts.json new file mode 100644 index 000000000..fc3725628 --- /dev/null +++ b/tests/harness/corpus-costs/checked-union-record-assertions.ts.json @@ -0,0 +1 @@ +{ "plain": 14493, "san": 16666 } diff --git a/tests/harness/dyncheck.test.ts b/tests/harness/dyncheck.test.ts index e1a869048..dfd037570 100644 --- a/tests/harness/dyncheck.test.ts +++ b/tests/harness/dyncheck.test.ts @@ -114,6 +114,32 @@ describe(`dynamic-boundary checks (scriptc-only${sanitize ? ", sanitized" : ""}) expect(r.stderr).toBe(""); }); + test("assertions narrowing record members or union arms check the value", async () => { + const r = await compileAndRun( + "narrowing-record-assertions", + ` + interface AlphaOpts { fast?: boolean } + interface Item { id: string; kind: K; opts?: K extends "alpha" ? AlphaOpts : never } + const items: Item<"alpha" | "gamma">[] = [{ id: "g1", kind: "gamma" }, { id: "a1", kind: "alpha", opts: { fast: true } }]; + try { const all = items as Item<"gamma">[]; console.log(all.length); } + catch (error) { console.log(error instanceof TypeError, (error as Error).message); } + try { const one = items[1]! as Item<"gamma">; console.log(one.id); } + catch (error) { console.log(error instanceof TypeError, (error as Error).message); } + interface S { stream: () => string } + function keys(u: S | { a?: S }): string[] { const m = u as Partial>; return Object.keys(m); } + try { console.log(keys({ a: { stream: () => "a" } }).join()); console.log(keys({ stream: () => "s" })); } + catch (error) { console.log(error instanceof TypeError); } + `, + ); + expect(r.exitCode).toBe(0); + expect(r.stdout).toBe( + "true expected null | undefined at $[1].opts, got object\n" + + "true expected null | undefined at $.opts, got object\n" + + "a\ntrue\n", + ); + expect(r.stderr).toBe(""); + }); + test("typed Promise exits validate fulfillment and preserve rejection", async () => { const result = await compileAndRun( "promise-exit-payload", From d33b083a3b4f49c258e4ff2a911235bcd3abda8a Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 17:30:44 -0500 Subject: [PATCH 13/20] Cache the zone offset at UTC midnights for local-time conversions Local setters and the field constructor convert a local time value to UTC by bracketing it with the zone offset a day before and after, then checking the candidate instants: three to five localtime(3) calls per conversion. In glibc each call takes the zone lock and, for dates past the zone file's last transition, parses the zone's rule string again. The offsets now come from a per-thread cache of offsets at UTC midnights: a day whose two midnights agree has that offset throughout, given at most one transition per day, which the conversion already assumes. A day with a transition still asks localtime for the instant. A change of process.env.TZ clears the cache. --- packages/runtime/src/scr_lib.c | 58 ++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/packages/runtime/src/scr_lib.c b/packages/runtime/src/scr_lib.c index a9fd25915..1a625cdf1 100644 --- a/packages/runtime/src/scr_lib.c +++ b/packages/runtime/src/scr_lib.c @@ -11588,8 +11588,8 @@ double scr_date_get_timezone_offset(double ms) { * from local fields reads those fields back. */ /* The host zone's offset (local - UTC) in ms at UTC instant `ms` (an - * integral time value within about ±8.65e15). */ -static bool scr_date_offset_at(double ms, double *out) { + * integral time value within about ±8.65e15), from the host's localtime. */ +static bool scr_date_offset_at_uncached(double ms, double *out) { long long cms = (long long)ms; double secd = (double)(cms / 1000 - (cms % 1000 < 0)); struct tm tmv; @@ -11611,6 +11611,60 @@ static bool scr_date_offset_at(double ms, double *out) { return true; } +/* Offsets at UTC midnights, for scr_date_offset_at. A day whose two + * midnights have the same offset has that offset throughout, given at most + * one transition per day (scr_date_local_to_utc already takes transitions + * to be at least a day apart). Each local setter asks for three to five + * offsets around one time value, and consecutive ones land on the same or + * neighbouring days, so most answers come from here instead of a + * localtime(3) call: glibc takes its zone lock in each and, for dates past + * the zone file's last transition, parses the zone's rule string again. A + * change of process.env.TZ starts a new generation. */ +#define SCR_DATE_MIDNIGHTS 64 +typedef struct { + long long day; + double offset; + bool valid; +} ScrDateMidnight; +typedef struct { + bool valid; + unsigned generation; + ScrDateMidnight entries[SCR_DATE_MIDNIGHTS]; +} ScrDateMidnightCache; +static SCR_TL ScrDateMidnightCache scr_date_midnight_cache; + +static bool scr_date_midnight_offset(ScrDateMidnightCache *c, long long day, double *out) { + ScrDateMidnight *e = &c->entries[(unsigned long long)day % SCR_DATE_MIDNIGHTS]; + if (e->valid && e->day == day) { + *out = e->offset; + return true; + } + if (!scr_date_offset_at_uncached((double)day * 86400000.0, out)) return false; + e->valid = true; + e->day = day; + e->offset = *out; + return true; +} + +static bool scr_date_offset_at(double ms, double *out) { + ScrDateMidnightCache *c = &scr_date_midnight_cache; + unsigned generation = scr_date_tz_generation; + if (!c->valid || c->generation != generation) { + memset(c, 0, sizeof *c); + c->valid = true; + c->generation = generation; + } + long long cms = (long long)ms; + long long day = cms / 86400000 - (cms % 86400000 < 0); + double start, end; + if (scr_date_midnight_offset(c, day, &start) && scr_date_midnight_offset(c, day + 1, &end) && + start == end) { + *out = start; + return true; + } + return scr_date_offset_at_uncached(ms, out); +} + static volatile const double scr_date_day_ms = 86400000.0; static double scr_date_local_to_utc(double t) { From cc08abcbe72c4082a81a195d2627eb258c49119d Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 17:42:53 -0500 Subject: [PATCH 14/20] Match the pending-flag fallback edge after the callback's receiver block An indirect callback call now gives the closure an undefined receiver when it reads one, so the fallback's edge into the pending-flag phi comes from the receiver block instead of callback.indirect. The flag is still false on the direct call and true on the fallback. --- packages/compiler/test/callback-pending-emission.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/compiler/test/callback-pending-emission.test.ts b/packages/compiler/test/callback-pending-emission.test.ts index 3b2ed3823..7e7996c83 100644 --- a/packages/compiler/test/callback-pending-emission.test.ts +++ b/packages/compiler/test/callback-pending-emission.test.ts @@ -40,9 +40,11 @@ try { const module = deserializeModule(readFileSync(output, "utf8")); expect(validateModule(module)).toEqual([]); const llvm = emitLlvmModule(module, {}); + // The fallback calls the closure's entry, after giving it an undefined + // receiver when it reads one, so its edge comes from receiver.done then. const guards = [ ...llvm.matchAll( - /(%t\d+) = phi i1 \[ false, %callback\.direct\d+ \], \[ true, %callback\.indirect\d+ \]/g, + /(%t\d+) = phi i1 \[ false, %callback\.direct\d+ \], \[ true, %(?:callback\.indirect|receiver\.done)\d+ \]/g, ), ]; // The quiet every and map callbacks; not the throwing one. From 5b40498c22c8ac7f4096e05e724a0b0bff177568 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 17:52:20 -0500 Subject: [PATCH 15/20] Keep asserted values in a map the lowerer owns compiler-weak-collections.test.ts refuses weak collections in compiler sources: the compiled compiler must not depend on which of its values can serve as weak keys. The lowerer owns the assertedValues cache for its pass, so an ordinary Map holds it. --- packages/compiler/src/frontend/lowering/lowerer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 491ca7c11..d665cc3ff 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -2251,7 +2251,7 @@ export class Lowerer { /** Values an erased type assertion produced, with the asserted type * (`x as T` keeps x's representation): a slot that no ordinary * conversion reaches converts them as asserted (assertedConversion). */ - readonly assertedValues = new WeakMap(); + readonly assertedValues = new Map(); /** Non-null while an IMPLICIT-ANY instance body lowers (npm-static JS; * see generic-functions.ts): bound param symbol → * the call site's checker type, consulted by typeOf for identifier From 30cf205e90108ea02cd109b71ad0cc6c439655ef Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 18:10:32 -0500 Subject: [PATCH 16/20] Run untyped pipeline function stages and later iterable stages A pipeline stage that arrives as an untyped value now follows Node's stage ladder for more shapes. As the first stage, a function is a source function: the pipeline calls it with { signal } while it is set up and pumps its result, which must be iterable (ERR_INVALID_RETURN_VALUE otherwise), and a { readable, writable } pair pumps its readable side. After the first stage, the pipeline call is lowered once per combination of its first three untyped stages being functions or not, and a typeof test picks the branch at run time. A function stage is called with the previous stage and { signal }; before the last stage its result must be async iterable, and as the last stage a promise resolves the pipeline's value and an async iterable feeds the PassThrough pipeline() returns (SCR_PIPE_DEST_DYN). Any other value that is not a stream becomes Duplex.from of it, which the next stage reads from without the previous stage being piped into it, and Duplex.from's ERR_INVALID_ARG_TYPE for the values it rejects. An untyped stage followed by an untyped function stage hands its value to that stage through a cell created per pipeline call, so the function receives the previous stage's own iterator, as Node's makeAsyncIterable answers it, instead of an async generator over it. Untyped web streams, pairs after the first stage, and function stages that return a stream throw the not-supported error. RUNTIME_ABI_VERSION goes from 45 to 46 for the new stream exports. --- .../builtin-apis/streams-and-readline.mdx | 8 +- packages/cli/test/native-link-info.test.ts | 4 +- .../compiler/src/backend/llvm/lib-dispatch.ts | 12 +- .../compiler/src/backend/llvm/lib-shared.ts | 3 + .../compiler/src/backend/llvm/lib-stream.ts | 2 +- packages/compiler/src/backend/runtime-abi.ts | 4 +- .../src/backend/runtime-pack-targets.test.ts | 2 +- .../compiler/src/backend/runtime-pack.test.ts | 2 +- .../src/frontend/lowering/lower-stream.ts | 667 +++++++++++++++++- packages/compiler/src/ir/builtin-effects.ts | 3 + .../compiler/src/ir/builtin-signatures.ts | 30 +- packages/compiler/src/ir/runtime-features.ts | 4 + packages/compiler/src/ir/validate.ts | 6 +- .../test/native-codegen-integration.test.ts | 4 +- ...pipeline-untyped-source-functions.mjs.json | 6 + ...-pipeline-untyped-function-stages.mjs.json | 6 + ...-pipeline-untyped-later-iterables.mjs.json | 6 + .../7511-pipeline-untyped-stages.cjs.json | 6 + .../corpus/7512-pipeline-any-stages.ts.json | 6 + ...513-pipeline-untyped-stage-errors.mjs.json | 6 + .../runtime-darwin-arm64/scripts/build.mjs | 2 +- .../runtime-pack-common/scripts/build.mjs | 2 +- packages/runtime/src/scr_console.c | 2 +- packages/runtime/src/scr_core.h | 2 +- packages/runtime/src/scr_dyn.h | 3 + packages/runtime/src/scr_dyn_builtin_stream.c | 8 + packages/runtime/src/scr_stream.c | 272 ++++++- packages/runtime/src/scr_stream.h | 12 + ...7508-pipeline-untyped-source-functions.mjs | 85 +++ .../7509-pipeline-untyped-function-stages.mjs | 112 +++ .../7510-pipeline-untyped-later-iterables.mjs | 64 ++ tests/corpus/7511-pipeline-untyped-stages.cjs | 50 ++ tests/corpus/7512-pipeline-any-stages.ts | 45 ++ .../7513-pipeline-untyped-stage-errors.mjs | 86 +++ ...pipeline-untyped-source-functions.mjs.json | 1 + ...-pipeline-untyped-function-stages.mjs.json | 1 + ...-pipeline-untyped-later-iterables.mjs.json | 1 + .../7511-pipeline-untyped-stages.cjs.json | 1 + .../7512-pipeline-any-stages.ts.json | 1 + ...513-pipeline-untyped-stage-errors.mjs.json | 1 + 40 files changed, 1466 insertions(+), 72 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7508-pipeline-untyped-source-functions.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7509-pipeline-untyped-function-stages.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7510-pipeline-untyped-later-iterables.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7511-pipeline-untyped-stages.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7512-pipeline-any-stages.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7513-pipeline-untyped-stage-errors.mjs.json create mode 100644 tests/corpus/7508-pipeline-untyped-source-functions.mjs create mode 100644 tests/corpus/7509-pipeline-untyped-function-stages.mjs create mode 100644 tests/corpus/7510-pipeline-untyped-later-iterables.mjs create mode 100644 tests/corpus/7511-pipeline-untyped-stages.cjs create mode 100644 tests/corpus/7512-pipeline-any-stages.ts create mode 100644 tests/corpus/7513-pipeline-untyped-stage-errors.mjs create mode 100644 tests/harness/corpus-costs/7508-pipeline-untyped-source-functions.mjs.json create mode 100644 tests/harness/corpus-costs/7509-pipeline-untyped-function-stages.mjs.json create mode 100644 tests/harness/corpus-costs/7510-pipeline-untyped-later-iterables.mjs.json create mode 100644 tests/harness/corpus-costs/7511-pipeline-untyped-stages.cjs.json create mode 100644 tests/harness/corpus-costs/7512-pipeline-any-stages.ts.json create mode 100644 tests/harness/corpus-costs/7513-pipeline-untyped-stage-errors.mjs.json diff --git a/docs/content/limitations/builtin-apis/streams-and-readline.mdx b/docs/content/limitations/builtin-apis/streams-and-readline.mdx index 561840268..25427a307 100644 --- a/docs/content/limitations/builtin-apis/streams-and-readline.mdx +++ b/docs/content/limitations/builtin-apis/streams-and-readline.mdx @@ -4,7 +4,13 @@ When an untyped value reaches `NodeJS.ReadableStream` or `NodeJS.WritableStream`, those two interfaces hold any value, so JavaScript stream objects (constructor-function streams, plain objects with `on` and `pipe`) pass through TypeScript functions typed with them and come back to JavaScript as the same objects. Member calls on such a value use the native stream when it is one and call the object's own members otherwise; `readable.pipe()` into a JavaScript destination follows Node's pipe protocol. The first `pipeline` stage may arrive as an untyped value that is a native stream, a `Cipheriv` or `Decipheriv` value, or an iterable: a string, an array, a generator object, or any value with a `Symbol.asyncIterator` or `Symbol.iterator` method, which the pipeline pumps into the next stage as Node does. - Later `pipeline` stages and `for await` sources that arrive as untyped values must be native streams, or `Cipheriv` and `Decipheriv` values, at run time; other values throw a not-supported error. + It may also be a source function, which the pipeline calls with `{ signal }` and whose iterable result it pumps, or a `{ readable, writable }` pair, whose readable side it pumps. + Later `pipeline` stages that arrive as untyped values may be native streams, `Cipheriv` and `Decipheriv` values, functions, and iterables. + A function is called with the previous stage and `{ signal }` as a function stage: before the last stage it must return an async iterable, and as the last stage it may return a promise, whose value resolves the pipeline, or an async iterable, which feeds the PassThrough that `pipeline()` returns. + An untyped source function, function stage, or destination that returns a stream is unsupported. + An iterable becomes `Duplex.from` of it, which the next stage reads from while the stages before it are not read, as in Node. + A pipeline branches at run time on whether each of its first three untyped stages after the first is a function; a function in a later untyped stage, an untyped web stream, and a `{ readable, writable }` pair after the first stage throw a not-supported error. + `for await` sources that arrive as untyped values must be native streams, or `Cipheriv` and `Decipheriv` values, at run time; other values throw a not-supported error. Native streams expose no `emit` to JavaScript, so a JavaScript stream whose own `pipe()` emits on its destination (the legacy `Stream` pipe) cannot pipe into one. `Readable.from` accepts strings, Buffers, arrays, Sets, Maps, Map and Set iterators, generators, async generators, and class iterables and async iterables, and in JavaScript any iterable or async iterable. In JavaScript, a sync iterator value with a `then` method that is not a promise, and an object whose `[Symbol.asyncIterator]` or `[Symbol.iterator]` method is a generator method, are unsupported. diff --git a/packages/cli/test/native-link-info.test.ts b/packages/cli/test/native-link-info.test.ts index f63cc790c..24db2c4c2 100644 --- a/packages/cli/test/native-link-info.test.ts +++ b/packages/cli/test/native-link-info.test.ts @@ -72,7 +72,7 @@ describe.runIf(supported)("macOS arm64 native link info", () => { minimum_os: "14.0", }, program: { object, entry_symbol: "main" }, - runtime_abi: { version: 45, marker: "scr_runtime_abi_v45" }, + runtime_abi: { version: 46, marker: "scr_runtime_abi_v46" }, runtime_pack: { kind: "precompiled", package: "@scriptc/runtime-darwin-arm64", @@ -165,6 +165,6 @@ describe.runIf(supported)("macOS arm64 native link info", () => { () => null, (failure: { stderr?: string }) => failure, ); - expect(error?.stderr).toContain("scr_runtime_abi_v45"); + expect(error?.stderr).toContain("scr_runtime_abi_v46"); }); }); diff --git a/packages/compiler/src/backend/llvm/lib-dispatch.ts b/packages/compiler/src/backend/llvm/lib-dispatch.ts index 8cefda8f0..30eef4b99 100644 --- a/packages/compiler/src/backend/llvm/lib-dispatch.ts +++ b/packages/compiler/src/backend/llvm/lib-dispatch.ts @@ -671,11 +671,13 @@ export function emitGenericLibCall(host: LlvmEmitterContext, e: LibCallExpr): Ll // which also takes those streams. e.fn === "stream.unboxDyn" && host.usesDynBuiltinStream ? "scr_dyn_builtin_stream_unbox" - : // for await over a checked-dynamic value iterates a native - // stream behind it (scr_dyn_for_await_stream.c) - e.fn === "dyn.asyncIteratorOpen" && host.dynForAwaitStreams - ? "scr_stream_dyn_async_iterator_open" - : LIB_FN_SYMS[e.fn]); + : e.fn === "stream.unboxDynStage" && host.usesDynBuiltinStream + ? "scr_dyn_builtin_stream_unbox_stage" + : // for await over a checked-dynamic value iterates a native + // stream behind it (scr_dyn_for_await_stream.c) + e.fn === "dyn.asyncIteratorOpen" && host.dynForAwaitStreams + ? "scr_stream_dyn_async_iterator_open" + : LIB_FN_SYMS[e.fn]); if (sym === undefined) throw new LlvmUnsupportedError(`libCall:${e.fn}`, e.loc); // The runtime's Date functions take and answer time values: a Date // operand passes its time value, read at once from the borrowed object, diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 4b3f34b80..016815819 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -818,6 +818,9 @@ export const LIB_FN_SYMS: Record = { "stream.stdioWritable": "scr_stream_stdio_writable", "stream.unboxDyn": "scr_stream_unbox_dyn", "stream.dynIterableSource": "scr_stream_dyn_iterable_source", + "stream.dynSourceValue": "scr_stream_dyn_source_value", + "stream.unboxDynStage": "scr_stream_unbox_dyn_stage", + "stream.dynTransformResult": "scr_stream_dyn_transform_result", "stream.handleStream": "scr_stream_handle", "stream.pipeDyn": "scr_stream_pipe_dyn", "stream.unpipeDyn": "scr_stream_unpipe_dyn", diff --git a/packages/compiler/src/backend/llvm/lib-stream.ts b/packages/compiler/src/backend/llvm/lib-stream.ts index fdbf13d4f..53f455964 100644 --- a/packages/compiler/src/backend/llvm/lib-stream.ts +++ b/packages/compiler/src/backend/llvm/lib-stream.ts @@ -743,7 +743,7 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV const helpers = packed(5 + n + k, helperCount); let at = 5 + n + k + helperCount; const take = (present: boolean): string => (present ? args[at++]!.name : "null"); - const box = take(((flags >> 1) & 3) === 2); + const box = take(((flags >> 1) & 3) >= 2); const signal = take((flags & 32) !== 0); const abort = take((flags & 32) !== 0); const head = diff --git a/packages/compiler/src/backend/runtime-abi.ts b/packages/compiler/src/backend/runtime-abi.ts index fe13879a2..892e9281e 100644 --- a/packages/compiler/src/backend/runtime-abi.ts +++ b/packages/compiler/src/backend/runtime-abi.ts @@ -2,8 +2,8 @@ * of the link contract: a program object keeps this symbol undefined and the * matching runtime defines it, so incompatible manual links fail before the * program can start. */ -export const RUNTIME_ABI_VERSION = 45 as const; -export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v45" as const; +export const RUNTIME_ABI_VERSION = 46 as const; +export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v46" as const; /** Object consumption is useful today, but the complete scr_* surface may * still change before 1.0. The versioned marker prevents accidental mixing; diff --git a/packages/compiler/src/backend/runtime-pack-targets.test.ts b/packages/compiler/src/backend/runtime-pack-targets.test.ts index 492b71217..9c5b61f58 100644 --- a/packages/compiler/src/backend/runtime-pack-targets.test.ts +++ b/packages/compiler/src/backend/runtime-pack-targets.test.ts @@ -52,7 +52,7 @@ async function fixture(target: NativeTargetSpec, workers = false) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 45, marker: "scr_runtime_abi_v45" }, + runtime_abi: { version: 46, marker: "scr_runtime_abi_v46" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: ["SCR_LIB"], sanitizer: "external-toolchain-required" }, flavors: { diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index fdb0ba29e..812fd9531 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -129,7 +129,7 @@ async function fixture(target: NativeTargetSpec = MACOS_ARM64_TARGET) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 45, marker: "scr_runtime_abi_v45" }, + runtime_abi: { version: 46, marker: "scr_runtime_abi_v46" }, compiler: { command: "clang", identity: "fixture clang", diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index c8699d856..2ae952aaa 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -2837,20 +2837,24 @@ export function streamClassAliasDecl( const PIPELINE_STAGE_EXPECTED = "of type function or an instance of Blob, ReadableStream, WritableStream, Stream, Iterable, AsyncIterable, or Promise or { readable, writable } pair"; -/** A checked-dynamic pipeline stage (an untyped source or destination): - * a boxed native stream unwraps at run time; other values (functions, - * pairs) throw the not-supported Error there. */ +/** A checked-dynamic pipeline stage taken as a stream: a boxed native + * stream unwraps at run time, and a value that is not stream-like or a + * function (an iterable after the first stage) becomes Duplex.from of it, + * as pipelineImpl's last branch does. Stream-like objects, web streams and + * { readable, writable } pairs throw the not-supported Error there, and so + * do functions, which reach this lowering only past + * MAX_DYN_FUNCTION_STAGES. */ function unboxDynStage(lowerer: Lowerer, v: IrExpr, node: ts.Expression): IrExpr { noteUntypedStreamFlow(lowerer, null); return { kind: "libCall", - fn: "stream.unboxDyn", + fn: "stream.unboxDynStage", args: [ v, strLit( lowerer.dynNodeStreams - ? "pipeline stages other than native streams (JavaScript stream objects, iterables, generators, plain functions, and { readable, writable } pairs) are not supported yet" - : "pipeline stages other than native streams (iterables, generators, plain functions, and { readable, writable } pairs) are not supported yet", + ? "pipeline stages other than native streams, iterables and functions (JavaScript stream objects, web streams, { readable, writable } pairs after the first stage, and functions after the first three untyped stages) are not supported yet" + : "pipeline stages other than native streams, iterables and functions (web streams, { readable, writable } pairs after the first stage, and functions after the first three untyped stages) are not supported yet", locOf(node), ), ], @@ -2930,6 +2934,102 @@ const BOXED_SOURCE_KINDS: ReadonlySet = new Set([ "string", ]); +/** The most checked-dynamic stages after the first that a pipeline + * branches on as functions; each one doubles the lowered pipeline calls. */ +const MAX_DYN_FUNCTION_STAGES = 3; + +/** A pipeline's arguments for one of the run-time branches + * lowerDynStagePipeline lowers. */ +interface DynStagePlan { + /** the first stage when it is checked-dynamic */ + source: DynPipelineSource | null; + /** argument values evaluated before the branches, in argument order */ + bound: ReadonlyMap; + /** a typed iterable first stage's source, lowered before the branches */ + preSource?: { closure: IrExpr; out: SyncGenType } | undefined; + /** the checked-dynamic stages this branch takes as functions */ + fns: ReadonlySet; +} + +/** pipeline() whose stages after the first include untyped values: + * pipelineImpl calls a stage that is a function with the previous stage + * and { signal }, and pipes into or takes any other value, so the call is + * lowered once per combination of those stages being functions or not, + * up to MAX_DYN_FUNCTION_STAGES of them, and a typeof test on each + * picks the branch at run time. The arguments other than function stages + * are evaluated first, in argument order. */ +function lowerDynStagePipeline( + lowerer: Lowerer, + stageArgs: readonly ts.Expression[], + src: DynPipelineSource | null, + loc: SrcLoc, + build: (plan: DynStagePlan) => IrExpr, +): IrExpr { + const candidates: number[] = []; + for (let j = 1; j < stageArgs.length; j++) { + const arg = stageArgs[j]!; + if (pipelineStageFunction(lowerer, arg) !== null || isValueStream(lowerer, arg)) continue; + if ((lowerer.typeOf(arg).flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0) { + candidates.push(j); + } + } + if (candidates.length === 0) return build({ source: src, bound: new Map(), fns: new Set() }); + const stmts: IrStmt[] = []; + const bind = (init: IrExpr): IrExpr => { + const local = lowerer.declareHiddenLocal("%pipelineArg", init.type); + stmts.push({ kind: "varDecl", localId: local.id, init, loc: init.loc }); + return varRef(local.id, init.type, init.loc); + }; + const source = src !== null && !src.iterable ? { expr: bind(src.expr), iterable: false } : src; + const bound = new Map(); + let preSource: DynStagePlan["preSource"]; + for (let j = src !== null ? 1 : 0; j < stageArgs.length; j++) { + const arg = stageArgs[j]!; + if (pipelineStageFunction(lowerer, arg) !== null) continue; + if (j === 0 && iterableSourceType(lowerer, arg) !== null) { + const lowered = lowerIterableSource(lowerer, arg, null, loc); + if (lowered === null) lowerer.noLowering("pipeline over this source", arg); + preSource = { closure: bind(lowered!.closure), out: lowered!.out }; + continue; + } + let value = lowerer.lowerExpr(arg); + // an untyped array, string, Set, Map or generator object that keeps + // its static type is taken through its checked-dynamic value + if ( + candidates.includes(j) && + BOXED_SOURCE_KINDS.has(value.type.kind) && + lowerer.dynConvertible(value.type) + ) { + value = { kind: "dynFrom", value, type: DYN, loc: value.loc }; + } + bound.set(j, bind(value)); + } + const dynStages = candidates + .filter((j) => bound.get(j)!.type.kind === "dyn") + .slice(0, MAX_DYN_FUNCTION_STAGES); + const branch = (i: number, fns: ReadonlySet): IrExpr => { + if (i === dynStages.length) return build({ source, bound, preSource, fns }); + const j = dynStages[i]!; + const asStream = branch(i + 1, fns); + const asFunction = branch(i + 1, new Set([...fns, j])); + // a function destination's promise resolves to its value + const type = + asFunction.type.kind === "promise" && asFunction.type.inner.kind === "dyn" + ? asFunction.type + : asStream.type; + return { + kind: "ternary", + cond: { kind: "dynTest", test: "function", value: bound.get(j)!, type: BOOL, loc }, + then: lowerer.coerceToExpected(asFunction, type), + else_: lowerer.coerceToExpected(asStream, type), + type, + loc, + }; + }; + const result = branch(0, new Set()); + return { kind: "seqExpr", stmts, result, type: result.type, loc }; +} + /** lowerPipelineStages' source arguments for a checked-dynamic first stage. */ function stagesSource( src: DynPipelineSource | null, @@ -2938,6 +3038,22 @@ function stagesSource( return src.iterable ? [undefined, src.expr] : [new Map([[0, src.expr]]), undefined]; } +/** A pipeline stage taken as a stream under a DynStagePlan: the + * checked-dynamic first stage's stream, a value bound before the + * branches, or the argument lowered here. */ +function planStreamArg( + lowerer: Lowerer, + plan: DynStagePlan, + node: ts.Expression, + i: number, +): IrExpr { + if (i === 0 && plan.source !== null) return plan.source.expr; + const bound = plan.bound.get(i); + return bound !== undefined + ? streamArgOf(lowerer, node, bound, "pipeline") + : lowerStreamArg(lowerer, node, "pipeline"); +} + /** A stream-rooted argument, lowered — or the pointed fence (pipeline's * iterable/generator/function stages have no lowering yet). `what` picks * the validation ladder: finished() rejects EVERY non-stream through @@ -3304,7 +3420,9 @@ export function lowerStreamModuleCall( : null; const stageArgs = opts ? args.slice(0, -1) : args; const build = (src: DynPipelineSource | null): IrExpr => - lowerPromisePipeline(lowerer, stageArgs, opts, src, loc); + lowerDynStagePipeline(lowerer, stageArgs, src, loc, (plan) => + lowerPromisePipeline(lowerer, stageArgs, opts, plan, loc), + ); return lowerDynSourcePipeline(lowerer, stageArgs[0]!, loc, build) ?? build(null); } return null; @@ -3312,17 +3430,23 @@ export function lowerStreamModuleCall( return lowerStreamModuleCallbackCall(lowerer, call, bi, loc); } -/** stream/promises pipeline(...stages[, options]), with the first stage - * pre-lowered when it is checked-dynamic (lowerDynSourcePipeline). */ +/** stream/promises pipeline(...stages[, options]), for one branch of + * lowerDynStagePipeline's plan. */ function lowerPromisePipeline( lowerer: Lowerer, stageArgs: readonly ts.Expression[], opts: ReturnType | null, - src: DynPipelineSource | null, + plan: DynStagePlan, loc: SrcLoc, ): IrExpr { - if (src?.iterable === true || hasPipelineFunctionStage(lowerer, stageArgs)) { - const staged = lowerPipelineStages(lowerer, stageArgs, loc, ...stagesSource(src)); + const src = plan.source; + if ( + src?.iterable === true || + plan.fns.size > 0 || + plan.preSource !== undefined || + hasPipelineFunctionStage(lowerer, stageArgs) + ) { + const staged = lowerPipelineStages(lowerer, stageArgs, loc, ...stagesSource(src), plan); const type = staged.destPromise ?? { kind: "promise", inner: VOID }; // A promise destination's value settles the result (same type). if (!opts) { @@ -3346,9 +3470,7 @@ function lowerPromisePipeline( } let streams: IrExpr[]; try { - streams = stageArgs.map((a, i) => - i === 0 && src !== null ? src.expr : lowerStreamArg(lowerer, a, "pipeline"), - ); + streams = stageArgs.map((a, i) => planStreamArg(lowerer, plan, a, i)); } catch (e) { if (e instanceof StreamArgTypeThrow) return e.expr; throw e; @@ -3530,7 +3652,9 @@ function lowerStreamModuleCallbackCall( ); } const build = (src: DynPipelineSource | null): IrExpr => - lowerCallbackPipeline(lowerer, call, args, src, loc); + lowerDynStagePipeline(lowerer, args.slice(0, -1), src, loc, (plan) => + lowerCallbackPipeline(lowerer, call, args, plan, loc), + ); return lowerDynSourcePipeline(lowerer, args[0]!, loc, build) ?? build(null); } @@ -3544,23 +3668,32 @@ function lowerStreamModuleCallbackCall( return null; } -/** pipeline(...stages, callback), with the first stage pre-lowered when it - * is checked-dynamic (lowerDynSourcePipeline). */ +/** pipeline(...stages, callback), for one branch of lowerDynStagePipeline's + * plan. */ function lowerCallbackPipeline( lowerer: Lowerer, call: ts.CallExpression, args: readonly ts.Expression[], - src: DynPipelineSource | null, + plan: DynStagePlan, loc: SrcLoc, ): IrExpr { - if (src?.iterable === true || hasPipelineFunctionStage(lowerer, args.slice(0, -1))) { - const staged = lowerPipelineStages(lowerer, args.slice(0, -1), loc, ...stagesSource(src)); - const cbNode = args[args.length - 1]!; - const cbParams = - ts.isArrowFunction(cbNode) || ts.isFunctionExpression(cbNode) - ? cbNode.parameters.nodes.length - : 0; - if (staged.dest === 2 || (staged.dest === 1 && cbParams > 1)) { + const src = plan.source; + const cbNode = args[args.length - 1]!; + const cbParams = + ts.isArrowFunction(cbNode) || ts.isFunctionExpression(cbNode) + ? cbNode.parameters.nodes.length + : 0; + // A branch over untyped stages whose destination is a stream: the + // callback still receives (err, value), value undefined, as in Node. + const dynCallback = plan.bound.size > 0 && cbParams > 1; + if ( + src?.iterable === true || + plan.fns.size > 0 || + plan.preSource !== undefined || + hasPipelineFunctionStage(lowerer, args.slice(0, -1)) + ) { + const staged = lowerPipelineStages(lowerer, args.slice(0, -1), loc, ...stagesSource(src), plan); + if (staged.dest >= 2 || (staged.dest === 1 && cbParams > 1) || dynCallback) { // callback(err, value): the destination's value (undefined for // an async generator) reaches the callback through the // checked-dynamic path. @@ -3595,15 +3728,15 @@ function lowerCallbackPipeline( } let streams: IrExpr[]; try { - streams = args - .slice(0, -1) - .map((a, i) => (i === 0 && src !== null ? src.expr : lowerStreamArg(lowerer, a, "pipeline"))); + streams = args.slice(0, -1).map((a, i) => planStreamArg(lowerer, plan, a, i)); } catch (e) { if (e instanceof StreamArgTypeThrow) return e.expr; throw e; } const last = streams[streams.length - 1]!; - const { cb, dyn } = lowerEosCallback(lowerer, "pipeline", args[args.length - 1]!, last.type); + const { cb, dyn } = dynCallback + ? { cb: lowerer.lowerExprExpecting(cbNode, DYN), dyn: true } + : lowerEosCallback(lowerer, "pipeline", cbNode, last.type); return { kind: "libCall", fn: dyn ? "stream.pipelineDyn" : "stream.pipeline", @@ -4213,6 +4346,397 @@ function lowerDynIterableSource( }; } +/** The async generator a checked-dynamic stage's iterable is pumped + * through. */ +function dynStageGenType(): SyncGenType { + return { kind: "generator", async: true, yieldT: DYN, retT: VOID, nextT: UNDEFINED_T }; +} + +/** A checked-dynamic pipeline stage after the first that is a function at + * run time, as the stage closure the runtime calls with the previous + * stage and { signal } (pipelineImpl's `typeof stream === 'function'` + * branch): it calls the function with the previous stream or generator + * as an untyped value. Before the last stage the result must be async + * iterable (ERR_INVALID_RETURN_VALUE otherwise) and is pumped on through + * lowerDynIterableSource's generator; the last stage answers the result, + * which the runtime classifies (SCR_PIPE_DEST_DYN). A previous stage + * whose type has no untyped form throws the not-supported Error. */ +function lowerDynFunctionStage( + lowerer: Lowerer, + fnValue: IrExpr, + srcT: IrType, + j: number, + isLast: boolean, + loc: SrcLoc, + handoff?: DynStageHandoff, +): { closure: IrExpr; out: IrType } { + const out: IrType = isLast ? DYN : dynStageGenType(); + const stageT = funcOf([srcT, DYN], out); + const take = handoff?.take === true; + const give = handoff?.give === true && !isLast; + const key = `dynFunctionStage:${isLast ? "dest" : "transform"}:${typeKey(srcT)}:${take ? 1 : 0}${give ? 1 : 0}`; + let maker = lowerer.valueHelpers.get(key); + if (!maker) { + maker = `%stream.dynFunctionStage.${lowerer.valueHelpers.size}`; + lowerer.valueHelpers.set(key, maker); + const stageName = `${maker}.stage`; + const frame = new LiftedFrame(loc); + const fnId = frame.local("fn", DYN, false); + const labelId = frame.local("label", STRING, false); + const cellId = frame.local("cell", DYN, false); + const srcId = frame.param("src", srcT); + const optsId = frame.param("opts", DYN); + const inputId = frame.local("input", DYN, false); + const resultId = frame.local("result", DYN, false); + const nextId = frame.local("next", DYN, false); + const cell = (): IrExpr => varRef(cellId, DYN, loc); + const body: IrStmt[] = []; + if (take) { + // the previous untyped stage's value, handed over as it is + // (makeAsyncIterable answers an iterable itself) + body.push( + { kind: "varDecl", localId: inputId, init: handoffTake(cell(), loc), loc }, + handoffPut(cell(), dynUndefinedExpr(loc), loc), + ); + } + if (!take && !lowerer.dynConvertible(srcT)) { + body.push({ + kind: "return", + value: nodeThrowExpr( + 0, + "SC2020", + `pipeline function stages after a stage of type '${lowerer.fmt(srcT)}' are not supported yet`, + out, + loc, + ), + loc, + }); + } else { + body.push({ + kind: "varDecl", + localId: resultId, + init: { + kind: "dynCall", + callee: varRef(fnId, DYN, loc), + receiver: dynUndefinedExpr(loc), + calleeName: "transform", + args: [ + take + ? varRef(inputId, DYN, loc) + : { kind: "dynFrom", value: varRef(srcId, srcT, loc), type: DYN, loc }, + varRef(optsId, DYN, loc), + ], + type: DYN, + loc, + }, + loc, + }); + if (isLast) { + body.push({ kind: "return", value: varRef(resultId, DYN, loc), loc }); + } else { + body.push({ + kind: "varDecl", + localId: nextId, + init: { + kind: "libCall", + fn: "stream.dynTransformResult", + args: [varRef(resultId, DYN, loc), varRef(labelId, STRING, loc)], + type: DYN, + loc, + }, + loc, + }); + // the next untyped function stage takes the result itself; the + // generator returned for the runtime is then never read + if (give) body.push(handoffPut(cell(), varRef(nextId, DYN, loc), loc)); + const iterable = lowerDynIterableSource(lowerer, varRef(nextId, DYN, loc), loc); + if (!iterable) + throw new InternalCompilerError("dyn iterable generator without a result record"); + body.push({ + kind: "return", + value: { kind: "callValue", callee: iterable.closure, args: [], type: out, loc }, + loc, + }); + } + } + const captured = take || give ? [fnId, labelId, cellId] : [fnId, labelId]; + for (const id of captured) frame.locals.find((l) => l.id === id)!.boxed = true; + lowerer.liftedFns.push({ + name: stageName, + params: frame.params, + returnType: out, + captures: [ + { localId: fnId, name: "fn", type: DYN }, + { localId: labelId, name: "label", type: STRING }, + ...(take || give ? [{ localId: cellId, name: "cell", type: DYN }] : []), + ], + locals: frame.locals, + body, + loc, + }); + const makerFn: IrLocal = { id: "fn.0", name: "fn", type: DYN, mutable: false, boxed: true }; + const makerLabel: IrLocal = { + id: "label.1", + name: "label", + type: STRING, + mutable: false, + boxed: true, + }; + const makerCell: IrLocal = { + id: "cell.2", + name: "cell", + type: DYN, + mutable: false, + boxed: true, + }; + const makerLocals = take || give ? [makerFn, makerLabel, makerCell] : [makerFn, makerLabel]; + lowerer.liftedFns.push({ + name: maker, + params: makerLocals.map((l) => ({ localId: l.id, name: l.name, type: l.type })), + returnType: stageT, + locals: makerLocals, + body: [ + { + kind: "return", + value: { + kind: "closure", + fnName: stageName, + captures: makerLocals.map((l) => l.id), + type: stageT, + loc, + }, + loc, + }, + ], + loc, + }); + } + return { + closure: { + kind: "call", + callee: maker, + args: [fnValue, strLit(`transform[${j - 1}]`, loc), ...(take || give ? [handoff!.cell] : [])], + type: stageT, + loc, + }, + out, + }; +} + +/** The cell that carries an untyped stage's value to the untyped function + * stage after it, created per pipeline call. A stage that gives stores its + * value; the next stage takes it instead of the generator the runtime + * passes, as Node's makeAsyncIterable answers an iterable itself. */ +interface DynStageHandoff { + cell: IrExpr; + take: boolean; + give: boolean; +} + +function handoffPut(cell: IrExpr, value: IrExpr, loc: SrcLoc): IrStmt { + return { + kind: "exprStmt", + expr: { + kind: "libCall", + fn: "dyn.keySet", + args: [cell, strLit("v", loc), value], + type: VOID, + loc, + }, + loc, + }; +} + +function handoffTake(cell: IrExpr, loc: SrcLoc): IrExpr { + return { kind: "dynKeyGet", value: cell, key: strLit("v", loc), type: DYN, loc }; +} + +/** `(value) => generator over value`: the async generator an untyped + * destination function's async iterable result is pumped through + * (SCR_PIPE_DEST_DYN's box), interned. */ +function dynStageIterHelper(lowerer: Lowerer, loc: SrcLoc): IrExpr { + const genT = dynStageGenType(); + const type = funcOf([DYN], genT); + const key = "dynStageIter"; + let name = lowerer.valueHelpers.get(key); + if (!name) { + name = `%stream.dynStageIter.${lowerer.valueHelpers.size}`; + lowerer.valueHelpers.set(key, name); + const v: IrLocal = { id: "value.0", name: "value", type: DYN, mutable: false }; + const iterable = lowerDynIterableSource(lowerer, varRef(v.id, DYN, loc), loc); + if (!iterable) + throw new InternalCompilerError("dyn iterable generator without a result record"); + lowerer.liftedFns.push({ + name, + params: [{ localId: v.id, name: v.name, type: DYN }], + returnType: genT, + locals: [v], + body: [ + { + kind: "return", + value: { kind: "callValue", callee: iterable.closure, args: [], type: genT, loc }, + loc, + }, + ], + loc, + }); + } + return { kind: "closure", fnName: name, captures: [], type, loc }; +} + +/** A checked-dynamic first pipeline stage pumped as an iterable + * (pipelineImpl's i === 0 branch), as the stage closure the runtime calls + * with { signal } while it sets the pipeline up: a function is called + * with { signal } and its result must be iterable, a { readable, writable } + * pair pumps its readable side, and any other value is pumped itself, + * through lowerDynIterableSource's generator. */ +function lowerDynSourceStage( + lowerer: Lowerer, + value: IrExpr, + loc: SrcLoc, + cell?: IrExpr, +): { closure: IrExpr; out: SyncGenType } | null { + const out: SyncGenType = { + kind: "generator", + async: true, + yieldT: DYN, + retT: VOID, + nextT: UNDEFINED_T, + }; + const stageT = funcOf([DYN], out); + const give = cell !== undefined; + const key = give ? "dynSourceStage:give" : "dynSourceStage"; + let maker = lowerer.valueHelpers.get(key); + if (!maker) { + maker = `%stream.dynSourceStage.${lowerer.valueHelpers.size}`; + const stageName = `${maker}.stage`; + const frame = new LiftedFrame(loc); + const srcId = frame.local("src", DYN, false); + const cellId = frame.local("cell", DYN, false); + const optsId = frame.param("opts", DYN); + const valueId = frame.local("value", DYN, false); + const genId = frame.local("gen", out, false); + const src = (): IrExpr => varRef(srcId, DYN, loc); + const sourceValue = (v: IrExpr, returned: boolean): IrExpr => ({ + kind: "libCall", + fn: "stream.dynSourceValue", + args: [v, boolLit(returned, loc)], + type: DYN, + loc, + }); + const iterable = lowerDynIterableSource(lowerer, varRef(valueId, DYN, loc), loc); + if (!iterable) return null; + lowerer.valueHelpers.set(key, maker); + const body: IrStmt[] = [ + { + kind: "varDecl", + localId: valueId, + init: { + kind: "ternary", + cond: { kind: "dynTest", test: "function", value: src(), type: BOOL, loc }, + then: sourceValue( + { + kind: "dynCall", + callee: src(), + receiver: dynUndefinedExpr(loc), + calleeName: "source", + args: [varRef(optsId, DYN, loc)], + type: DYN, + loc, + }, + true, + ), + else_: sourceValue(src(), false), + type: DYN, + loc, + }, + loc, + }, + { + kind: "varDecl", + localId: genId, + init: { kind: "callValue", callee: iterable.closure, args: [], type: out, loc }, + loc, + }, + ]; + if (give) { + // the untyped function stage after the source takes the iterable + // itself, as Node's makeAsyncIterable answers it; a stream (a pair's + // readable side) goes as the generator over it, as Node iterates a + // stream there + const v = varRef(valueId, DYN, loc); + body.push( + handoffPut( + varRef(cellId, DYN, loc), + { + kind: "ternary", + cond: { kind: "libCall", fn: "stream.dynIterableSource", args: [v], type: BOOL, loc }, + then: v, + else_: { kind: "dynFrom", value: varRef(genId, out, loc), type: DYN, loc }, + type: DYN, + loc, + }, + loc, + ), + ); + } + body.push({ kind: "return", value: varRef(genId, out, loc), loc }); + const captured = give ? [srcId, cellId] : [srcId]; + for (const id of captured) frame.locals.find((l) => l.id === id)!.boxed = true; + lowerer.liftedFns.push({ + name: stageName, + params: frame.params, + returnType: out, + captures: [ + { localId: srcId, name: "src", type: DYN }, + ...(give ? [{ localId: cellId, name: "cell", type: DYN }] : []), + ], + locals: frame.locals, + body, + loc, + }); + const makerSrc: IrLocal = { id: "src.0", name: "src", type: DYN, mutable: false, boxed: true }; + const makerCell: IrLocal = { + id: "cell.1", + name: "cell", + type: DYN, + mutable: false, + boxed: true, + }; + const makerLocals = give ? [makerSrc, makerCell] : [makerSrc]; + lowerer.liftedFns.push({ + name: maker, + params: makerLocals.map((l) => ({ localId: l.id, name: l.name, type: l.type })), + returnType: stageT, + locals: makerLocals, + body: [ + { + kind: "return", + value: { + kind: "closure", + fnName: stageName, + captures: makerLocals.map((l) => l.id), + type: stageT, + loc, + }, + loc, + }, + ], + loc, + }); + } + return { + closure: { + kind: "call", + callee: maker, + args: give ? [value, cell!] : [value], + type: stageT, + loc, + }, + out, + }; +} + function lowerIterableSource( lowerer: Lowerer, node: ts.Expression, @@ -4356,7 +4880,7 @@ function lowerIterableSource( interface PipelineStagesLowering { args: IrExpr[]; last: IrType; - dest: 0 | 1 | 2; + dest: 0 | 1 | 2 | 3; destPromise: (IrType & { kind: "promise" }) | null; wrap: (call: IrExpr) => IrExpr; } @@ -4377,6 +4901,7 @@ function lowerPipelineStages( loc: SrcLoc, preLowered?: ReadonlyMap, dynSource?: IrExpr, + plan?: DynStagePlan, ): PipelineStagesLowering { if (stageArgs.length > 52) { lowerer.noLowering( @@ -4390,7 +4915,7 @@ function lowerPipelineStages( const helperArgs: IrExpr[] = []; let positions = 0; let mask = 0; - let dest: 0 | 1 | 2 = 0; + let dest: 0 | 1 | 2 | 3 = 0; let destPromise: (IrType & { kind: "promise" }) | null = null; // What the next function stage receives: null before the first stage, // else the previous stream's type or the previous stage's generator. @@ -4399,17 +4924,69 @@ function lowerPipelineStages( // before all stages, so when a stage precedes a stream the arguments // are bound to locals first, keeping JavaScript's evaluation order. const argOrder: { list: IrExpr[]; index: number }[] = []; + // An untyped stage followed by an untyped function stage hands its value + // over through one cell per call (DynStageHandoff). + let handoffCell: IrLocal | null = null; + const cellRef = (): IrExpr => { + handoffCell ??= lowerer.declareHiddenLocal("%pipelineHandoff", DYN); + return varRef(handoffCell.id, DYN, loc); + }; + let takes = false; for (let j = 0; j < stageArgs.length; j++) { const arg = stageArgs[j]!; + const givesToNext = plan?.fns.has(j + 1) === true; if (j === 0 && dynSource !== undefined) { - const source = lowerDynIterableSource(lowerer, dynSource, loc); + const source = lowerDynSourceStage( + lowerer, + dynSource, + loc, + givesToNext ? cellRef() : undefined, + ); + takes = givesToNext; if (source === null) lowerer.noLowering("pipeline over this source", arg); + // a source function takes { signal } + mask |= 1 << stages.length; positions += 1; argOrder.push({ list: stages, index: stages.length }); stages.push(source!.closure); srcT = source!.out; continue; } + if (plan?.fns.has(j) === true) { + // a checked-dynamic stage that is a function at run time + const isLast = j === stageArgs.length - 1; + const take = takes; + const give = givesToNext && !isLast; + takes = give; + const lowered = lowerDynFunctionStage( + lowerer, + plan.bound.get(j)!, + srcT!, + j, + isLast, + loc, + take || give ? { cell: cellRef(), take, give } : undefined, + ); + mask |= 1 << stages.length; + positions += 2 ** j; + argOrder.push({ list: stages, index: stages.length }); + stages.push(lowered.closure); + if (isLast) { + // its result is classified at run time: a promise settles the + // pipeline's value, and an async iterable is pumped into the + // PassThrough + const genT = dynStageGenType(); + const helpers = fromGenHelpers(lowerer, genT, loc); + if (!helpers) lowerer.noLowering("a pipeline destination with this generator type", arg); + helperArgs.push(helpers![0]!, helpers![1]!, helpers![2]!); + dest = 3; + destPromise = { kind: "promise", inner: DYN }; + srcT = null; + } else { + srcT = lowered.out; + } + continue; + } // compose's head and tail functions arrive as Duplex.from streams const fn = preLowered?.has(j) ? null : pipelineStageFunction(lowerer, arg); const syncSourceFn = @@ -4423,7 +5000,9 @@ function lowerPipelineStages( !preLowered?.has(j) && (syncSourceFn || (fn === null && iterableSourceType(lowerer, arg) !== null)) ) { - const source = lowerIterableSource(lowerer, arg, syncSourceFn ? fn : null, loc); + const source = + (syncSourceFn ? undefined : plan?.preSource) ?? + lowerIterableSource(lowerer, arg, syncSourceFn ? fn : null, loc); if (source === null) lowerer.noLowering("pipeline over this source", arg); positions += 1; argOrder.push({ list: stages, index: stages.length }); @@ -4448,7 +5027,12 @@ function lowerPipelineStages( } let stream: IrExpr; try { - stream = preLowered?.get(j) ?? lowerStreamArg(lowerer, arg, "pipeline"); + const bound = plan?.bound.get(j); + stream = + preLowered?.get(j) ?? + (bound !== undefined + ? streamArgOf(lowerer, arg, bound, "pipeline") + : lowerStreamArg(lowerer, arg, "pipeline")); } catch (e) { if (e instanceof StreamArgTypeThrow) lowerer.noLowering("pipeline over this value", arg); throw e; @@ -4527,6 +5111,8 @@ function lowerPipelineStages( ); } helperArgs.push(pipelineBoxHelper(lowerer, destPromise!, loc)); + } else if (dest === 3) { + helperArgs.push(dynStageIterHelper(lowerer, loc)); } const argDecls: IrStmt[] = []; const firstStage = argOrder.findIndex((a) => a.list === stages); @@ -4538,6 +5124,15 @@ function lowerPipelineStages( a.list[a.index] = varRef(local.id, init.type, init.loc); } } + if (handoffCell !== null) { + const cell: IrLocal = handoffCell; + argDecls.unshift({ + kind: "varDecl", + localId: cell.id, + init: { kind: "dynObjLit", fields: [], type: DYN, loc }, + loc, + }); + } const bindArgs = (call: IrExpr): IrExpr => argDecls.length === 0 ? call diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index b0cc96687..86f9ad233 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -92,6 +92,9 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "child.stdioStream", "stream.unboxDyn", "stream.dynIterableSource", + "stream.dynSourceValue", + "stream.unboxDynStage", + "stream.dynTransformResult", "stream.handleStream", "stream.pipeDyn", "stream.unpipeDyn", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index e230932ba..b7ef7d660 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -2870,9 +2870,27 @@ function buildLibFnSignatures() { result: { kind: "object", className: "%Duplex" }, }, /** Whether pipeline() pumps a checked-dynamic first stage as an - * iterable (a string, or a value with an iterator method that is not a - * stream with pipe() and on()) instead of piping it as a stream. */ + * iterable (a string, a value with an iterator method that is not a + * stream with pipe() and on(), a { readable, writable } pair, or a + * source function's result) instead of piping it as a stream. */ "stream.dynIterableSource": { argTypes: [DYN], result: BOOL }, + /** The value pipeline() pumps for a checked-dynamic first stage: a + * source function's result (the flag set), which must be iterable, else + * a { readable, writable } pair's readable side or the value itself. */ + "stream.dynSourceValue": { argTypes: [DYN, BOOL], result: DYN }, + /** An untyped pipeline function stage's result before the last stage, + * which must be async iterable: ERR_INVALID_RETURN_VALUE naming the + * stage ("transform[i]") otherwise. Answers the value (+1). */ + "stream.dynTransformResult": { argTypes: [DYN, STRING], result: DYN }, + /** A checked-dynamic pipeline() stage taken as a stream: a native stream + * (+1), or Duplex.from of a value that is not stream-like or a function + * (an iterable), which the pipeline uses without piping into it. Other + * values throw: Duplex.from's ERR_INVALID_ARG_TYPE, or the given + * not-supported message. */ + "stream.unboxDynStage": { + argTypes: [DYN, STRING], + result: { kind: "object", className: "%Duplex" }, + }, /** The Transform a Cipheriv or Decipheriv reads and writes as a stream * (+1, created on first use and owned by the cipher). */ "stream.handleStream": { @@ -5237,10 +5255,12 @@ export type IrLibFn = keyof ReturnType; /** The function-stage runs of a pipeline of `total` stages that a stream * follows (positions bit j: stage j is a function), and an async * generator destination's run (flags bits 1-2 = 1, pumped into the - * PassThrough pipeline() answers): each pumps its last generator through - * one (next, read, return) triple. */ + * PassThrough pipeline() answers, or 3, an untyped destination whose + * result may be pumped there): each pumps its last generator through one + * (next, read, return) triple. */ export function pipelinePumpedRuns(total: number, positions: number, flags: number): number { - let runs = ((flags >> 1) & 3) === 1 ? 1 : 0; + const dest = (flags >> 1) & 3; + let runs = dest === 1 || dest === 3 ? 1 : 0; for (let j = 0; j + 1 < total; j++) { if (Math.floor(positions / 2 ** j) % 2 === 1 && Math.floor(positions / 2 ** (j + 1)) % 2 === 0) runs++; diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index 494c4e3e8..630ebdc67 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -278,6 +278,9 @@ const STREAM_LIB_FNS: ReadonlySet = new Set([ "stream.stdioWritable", "stream.unboxDyn", "stream.dynIterableSource", + "stream.dynSourceValue", + "stream.dynTransformResult", + "stream.unboxDynStage", "stream.handleStream", "stream.unpipeDyn", "stream.workerStdio", @@ -314,6 +317,7 @@ const READABLE_DYN_ASYNC_LIB_FNS: ReadonlySet = new Set([ "stream.duplexFromAsyncFn", "stream.duplexFromGenSignal", "stream.duplexFromAsyncFnSignal", + "stream.unboxDynStage", ]); /** libCall name prefixes of the Readable entry points that await through the dyn-async unit. */ diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index 45ebf5c79..84751cc06 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -5761,7 +5761,7 @@ function validateFunction( } const lastIsFunction = Math.floor(positions! / 2 ** (total - 1)) % 2 === 1; const helperCount = 3 * pipelinePumpedRuns(total, positions!, flags!); - const groups = helperCount + (destKind === 2 ? 1 : 0) + (flags! & 32 ? 2 : 0); + const groups = helperCount + (destKind >= 2 ? 1 : 0) + (flags! & 32 ? 2 : 0); const cbArgs = e.fn === "sp.pipelineStagesOpts" ? 1 + (flags! & 64 ? 1 : 0) @@ -5776,7 +5776,7 @@ function validateFunction( total > 52 || positions! >= 2 ** total || functions !== k || - destKind === 3 || + (destKind === 3 && e.fn === "stream.composeStages") || (destKind !== 0) !== lastIsFunction || total < 2 || e.args.length !== 5 + n! + k! + groups + cbArgs @@ -5795,7 +5795,7 @@ function validateFunction( err(`libCall ${e.fn} stage function ${i} must be a func`, e.loc); } at2 += k!; - const helpers = helperCount + (destKind === 2 ? 1 : 0); + const helpers = helperCount + (destKind >= 2 ? 1 : 0); for (let i = at2; i < at2 + helpers; i++) { if (e.args[i]?.type.kind !== "func") err(`libCall ${e.fn} helper ${i} must be a func`, e.loc); diff --git a/packages/compiler/test/native-codegen-integration.test.ts b/packages/compiler/test/native-codegen-integration.test.ts index a57d96f01..bdad62203 100644 --- a/packages/compiler/test/native-codegen-integration.test.ts +++ b/packages/compiler/test/native-codegen-integration.test.ts @@ -378,7 +378,7 @@ done: expect(await symbols(helperObject, ["-u"])).toEqual(await symbols(clangObject, ["-u"])); expect(await symbols(helperObject, ["-gU"])).toEqual(await symbols(clangObject, ["-gU"])); expect(await symbols(helperObject, ["-gU"])).toEqual(["0000000000000000 T _main"]); - expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v45"); + expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v46"); const clangExe = join(dir, "clang-program"); const linkDriver = join(dir, "clang-driver.c"); @@ -712,7 +712,7 @@ done: .split("\n"); expect(undefinedSymbols).toContain("_sf_scale"); expect(undefinedSymbols).toContain("_sf_callback_mix"); - expect(undefinedSymbols).toContain("_scr_runtime_abi_v45"); + expect(undefinedSymbols).toContain("_scr_runtime_abi_v46"); }); test("helper and clang object paths retain the same outbound FFI C ABI", async () => { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7508-pipeline-untyped-source-functions.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7508-pipeline-untyped-source-functions.mjs.json new file mode 100644 index 000000000..4027063d4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7508-pipeline-untyped-source-functions.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7508-pipeline-untyped-source-functions.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7509-pipeline-untyped-function-stages.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7509-pipeline-untyped-function-stages.mjs.json new file mode 100644 index 000000000..028a5c1c6 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7509-pipeline-untyped-function-stages.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7509-pipeline-untyped-function-stages.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7510-pipeline-untyped-later-iterables.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7510-pipeline-untyped-later-iterables.mjs.json new file mode 100644 index 000000000..be18c801a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7510-pipeline-untyped-later-iterables.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7510-pipeline-untyped-later-iterables.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7511-pipeline-untyped-stages.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7511-pipeline-untyped-stages.cjs.json new file mode 100644 index 000000000..2353cc44c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7511-pipeline-untyped-stages.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7511-pipeline-untyped-stages.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7512-pipeline-any-stages.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7512-pipeline-any-stages.ts.json new file mode 100644 index 000000000..acea0e98d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7512-pipeline-any-stages.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7512-pipeline-any-stages.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7513-pipeline-untyped-stage-errors.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7513-pipeline-untyped-stage-errors.mjs.json new file mode 100644 index 000000000..39a5dba57 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7513-pipeline-untyped-stage-errors.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7513-pipeline-untyped-stage-errors.mjs" + ], + "diags": [] +} diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index 3305d69de..f8f5d0bf1 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -366,7 +366,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: RUNTIME_PACK_MATRIX.target, - runtime_abi: { version: 45, marker: "scr_runtime_abi_v45" }, + runtime_abi: { version: 46, marker: "scr_runtime_abi_v46" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index 8c0863b04..ed32f43fb 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -469,7 +469,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: matrix.target, - runtime_abi: { version: 45, marker: "scr_runtime_abi_v45" }, + runtime_abi: { version: 46, marker: "scr_runtime_abi_v46" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime/src/scr_console.c b/packages/runtime/src/scr_console.c index 67d4aca15..18c715cb2 100644 --- a/packages/runtime/src/scr_console.c +++ b/packages/runtime/src/scr_console.c @@ -148,7 +148,7 @@ static void scr_enable_utf8_console_output(void) { } #endif -void scr_runtime_abi_v45(void) {} +void scr_runtime_abi_v46(void) {} void scr_init(void) { #ifdef SCR_WORKERS diff --git a/packages/runtime/src/scr_core.h b/packages/runtime/src/scr_core.h index 206463e18..92392897a 100644 --- a/packages/runtime/src/scr_core.h +++ b/packages/runtime/src/scr_core.h @@ -81,7 +81,7 @@ void scr_init(void); /* Program objects emitted by the bundled LLVM helper reference this symbol. * Its versioned spelling makes a mismatched manual runtime link fail before * the program can start. */ -void scr_runtime_abi_v45(void); +void scr_runtime_abi_v46(void); void scr_runtime_workers_v8(void); /* ── the trap funnel (scr_console.c; scr_library.c under -DSCR_LIB) ────── diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 2d916a825..63dc0d209 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -1548,6 +1548,9 @@ ScrDyn *scr_dyn_builtin_stream_object(struct ScrStream *s); ScrDyn *scr_dyn_builtin_stream_view(struct ScrStream *s); struct ScrStream *scr_dyn_builtin_stream_check(const ScrDyn *value, double kind); struct ScrStream *scr_dyn_builtin_stream_unbox(const ScrDyn *value, const ScrStr *message); +/* stream.unboxDynStage in those programs (scr_stream_unbox_dyn_stage's + * contract, also taking constructor-function streams). */ +struct ScrStream *scr_dyn_builtin_stream_unbox_stage(const ScrDyn *value, const ScrStr *message); /* stream.dynNative in those programs: scr_stream_dyn_native's contract, * also taking constructor-function streams that keep the runtime's write * and end (+1 or NULL). */ diff --git a/packages/runtime/src/scr_dyn_builtin_stream.c b/packages/runtime/src/scr_dyn_builtin_stream.c index 2875cfa06..52c1e7269 100644 --- a/packages/runtime/src/scr_dyn_builtin_stream.c +++ b/packages/runtime/src/scr_dyn_builtin_stream.c @@ -1752,3 +1752,11 @@ ScrStream *scr_dyn_builtin_stream_unbox(const ScrDyn *value, const ScrStr *messa } return scr_stream_unbox_dyn(value, message); } + +/* stream.unboxDynStage in programs that link this unit: a + * constructor-function stream as scr_dyn_builtin_stream_unbox takes it, + * any other value as scr_stream_unbox_dyn_stage does. */ +ScrStream *scr_dyn_builtin_stream_unbox_stage(const ScrDyn *value, const ScrStr *message) { + if (bcs_of(value) && value->kind == SCR_DYN_OBJ) return scr_dyn_builtin_stream_unbox(value, message); + return scr_stream_unbox_dyn_stage(value, message); +} diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index 201d6e34b..a7593b551 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -93,6 +93,10 @@ struct ScrStreamState { bool is_transform; /* the writable side routes through transform/flush */ bool allow_half_open; bool auto_destroy, emit_close; + /* Duplex.from of a pipeline() argument that is not a stream: the + * pipeline takes it as that stage's stream without wiring the stage + * (scr_stream_unbox_dyn_stage) */ + bool pipeline_body; /* shared lifecycle */ bool destroyed, destroy_calling, error_emitted, close_emitted, error_scheduled, close_scheduled; @@ -3850,7 +3854,9 @@ typedef ScrPromise *(*ScrPipeNextFn)(ScrClosure *, ScrGen *); typedef ScrDyn *(*ScrPipeReadFn)(ScrClosure *, void *); typedef ScrDyn *(*ScrPipeBoxFn)(ScrClosure *, ScrPromise *); -enum { SCR_PIPE_DEST_STREAM, SCR_PIPE_DEST_GEN, SCR_PIPE_DEST_PROMISE }; +/* SCR_PIPE_DEST_DYN: an untyped destination function's result, a + * checked-dynamic value the pipeline classifies (scr_pipe_dest_dyn_kind) */ +enum { SCR_PIPE_DEST_STREAM, SCR_PIPE_DEST_GEN, SCR_PIPE_DEST_PROMISE, SCR_PIPE_DEST_DYN }; /* flags bit 7: the callback is typed code's (stream.pipelineStagesDyn) */ enum { SCR_PIPE_TYPED_CB = 128 }; @@ -3868,7 +3874,9 @@ typedef struct { * async generator destination's PassThrough included), in pipeline * order, or NULL */ ScrClosure **helpers; - ScrClosure *box; /* SCR_PIPE_DEST_PROMISE: boxes its promise as a dyn promise */ + /* SCR_PIPE_DEST_PROMISE: boxes its promise as a dyn promise; + * SCR_PIPE_DEST_DYN: answers the async generator over a dyn iterable */ + ScrClosure *box; } ScrPipeStages; static bool scr_pipe_is_stage(const ScrPipeStages *fs, size_t j) { @@ -4234,6 +4242,71 @@ static void scr_pipe_dest_promise(ScrStreamPipeCtx *ctx, ScrPromise *p /*moves*/ scr_promise_release(scr_async_spawn_after(d->boxed->v.promise, &scr_pipe_dest_entry, d)); } +/* An untyped destination's promise: scr_pipe_dest_promise over a promise + * that already holds dyn values. */ +static void scr_pipe_dest_dyn_promise(ScrStreamPipeCtx *ctx, ScrDyn *boxed /*moves*/, size_t pi, + bool end) { + ScrPipeDest *d = malloc(sizeof *d); + if (!d) scr_stream_oom(); + d->ctx = scr_stream_pipectx_retain_v(ctx); + d->typed = scr_promise_retain(boxed->v.promise); + d->boxed = boxed; + d->pi = pi; + d->end = end; + scr_promise_release(scr_async_spawn_after(d->boxed->v.promise, &scr_pipe_dest_entry, d)); +} + +typedef ScrGen *(*ScrPipeIterFn)(ScrClosure *, ScrDyn *); +static bool scr_stream_dyn_has_method(const ScrDyn *value, const char *key); +static bool scr_stream_dyn_transform_like(const ScrDyn *value); +static void scr_stream_throw_web_stage(void); +static bool scr_stream_dyn_stage_returned_stream(const ScrDyn *value); +static void scr_stream_throw_invalid_return(const char *expected, const char *name, + const ScrDyn *got); + +/* What pipelineImpl does with an untyped destination function's result: + * a promise settles the pipeline's value (the `then` branch), and an + * async iterable, or a { readable, writable } pair's readable side, is + * pumped into the PassThrough. *out answers the promise or that + * iterable's async generator (+1). Other values throw + * ERR_INVALID_RETURN_VALUE, and thenables that are not promises and web + * streams throw the not-supported Error; -1 then. */ +enum { SCR_PIPE_DYN_PROMISE, SCR_PIPE_DYN_ITER }; +static int scr_pipe_dest_dyn_kind(const ScrPipeStages *fs, ScrDyn *r, void **out) { + if (r->kind == SCR_DYN_PROMISE) { + *out = scr_dyn_retain(r); + return SCR_PIPE_DYN_PROMISE; + } + if (r->kind == SCR_DYN_OBJ && scr_stream_dyn_has_method(r, "then")) { + static const char msg[] = + "pipeline destinations that return a thenable other than a promise are not supported yet"; + scr_throw_error_msg_code(SCR_ERR_ERROR, msg, sizeof msg - 1, "SC2020"); + return -1; + } + if (scr_exc_pending() || scr_stream_dyn_stage_returned_stream(r)) return -1; + ScrDyn *read = NULL; + int kind = scr_dyn_iterable_kind(r); + if (kind < 0) return -1; + if (r->kind == SCR_DYN_HANDLE && + (r->v.handle.tag == SCR_DYNH_WEB_STREAM || r->v.handle.tag == SCR_DYNH_WEB_TRANSFORM)) { + scr_stream_throw_web_stage(); + return -1; + } + if (kind == 2) { + read = scr_dyn_retain(r); + } else if (scr_stream_dyn_transform_like(r)) { + read = scr_dyn_obj_read(r, "readable", 8); + } + if (!read) { + if (!scr_exc_pending()) scr_stream_throw_invalid_return("AsyncIterable or Promise", "destination", r); + return -1; + } + ScrGen *g = ((ScrPipeIterFn)fs->box->fn)(fs->box, read); + if (!g) return -1; + *out = g; + return SCR_PIPE_DYN_ITER; +} + static ScrStream *scr_stream_pipeline_ex(size_t n, ScrStream **streams, ScrClosure *cb, ScrStreamErrInv inv, bool end_opt, ScrDyn *signal, ScrAbortAddFn add, const ScrPipeStages *fs, @@ -4294,8 +4367,18 @@ static ScrStream *scr_stream_pipeline_ex(size_t n, ScrStream **streams, ScrClosu : (void *)scr_stream_retain(ret); ret_gen = NULL; void *out = scr_pipe_call_stage(fs, k++, src); + int dyn_kind = -1; + if (!scr_exc_pending() && dest && fs->dest == SCR_PIPE_DEST_DYN) { + ScrDyn *r = out ? out : scr_dyn_retain(scr_dyn_undefined()); + out = NULL; + dyn_kind = scr_pipe_dest_dyn_kind(fs, r, &out); + scr_dyn_release(r); + } if (scr_exc_pending()) { - if (out && dest) scr_promise_release(out); + if (out && dest && fs->dest == SCR_PIPE_DEST_DYN) { + if (dyn_kind == SCR_PIPE_DYN_ITER) scr_gen_release(out); + else scr_dyn_release(out); + } else if (out && dest) scr_promise_release(out); else if (out) scr_gen_release(out); scr_stream_pipectx_drop(ctx); scr_stream_pipectx_release_v(ctx); @@ -4315,9 +4398,12 @@ static ScrStream *scr_stream_pipeline_ex(size_t n, ScrStream **streams, ScrClosu size_t pi = ctx->n++; ctx->streams[pi] = pt; ctx->finish_count++; - if (fs->dest == SCR_PIPE_DEST_GEN) { + if (fs->dest == SCR_PIPE_DEST_GEN || + (fs->dest == SCR_PIPE_DEST_DYN && dyn_kind == SCR_PIPE_DYN_ITER)) { /* pumpToNode(ret, pt, finish, { end }) */ scr_pump_start(ctx, out, fs->helpers + 3 * h++, pt, end); + } else if (fs->dest == SCR_PIPE_DEST_DYN) { + scr_pipe_dest_dyn_promise(ctx, out, pi, end); } else { scr_pipe_dest_promise(ctx, out, fs->box, pi, end); } @@ -4332,6 +4418,20 @@ static ScrStream *scr_stream_pipeline_ex(size_t n, ScrStream **streams, ScrClosu continue; } ScrStream *s = streams[i]; + if (s->st->pipeline_body) { + /* ret = Duplex.from(stream) for an argument that is not a stream: + * no destroyer or 'error' listener, and the previous stage is + * neither piped nor pumped into it */ + s->st->pipeline_body = false; + if (ret_gen) { + scr_gen_release(ret_gen); + ret_gen = NULL; + h++; + } + ret = s; + i++; + continue; + } bool writing = j > 0; if (reading && !scr_pipe_is_stage(fs, j + 1) && (streams[i + 1]->st->close_emitted || streams[i + 1]->st->destroyed)) { @@ -8593,10 +8693,90 @@ ScrStream *scr_stream_unbox_dyn(const ScrDyn *value, const ScrStr *message) { return NULL; } -/* pipeline.js's first stage: a value with pipe() and on() methods is a - * readable node stream, which the stage ladder tests before isIterable; - * a string, or a value with a Symbol.asyncIterator or Symbol.iterator - * method, is an iterable the pipeline pumps into the next stage. */ +/* Whether a checked-dynamic object's member is of type "object". */ +static bool scr_stream_dyn_member_object(const ScrDyn *value, const char *key) { + ScrDyn *m = scr_dyn_obj_read(value, key, strlen(key)); + if (!m) return false; + ScrStr *t = scr_dyn_typeof(m); + bool object = t->len == 6 && memcmp(t->data, "object", 6) == 0; + scr_str_release(t); + scr_dyn_release(m); + return object; +} + +/* pipeline.js's isTransformStream for a value that is not a stream: an + * object whose readable and writable members are both of type "object". */ +static bool scr_stream_dyn_transform_like(const ScrDyn *value) { + if (value->kind == SCR_DYN_HANDLE) return value->v.handle.tag == SCR_DYNH_WEB_TRANSFORM; + return value->kind == SCR_DYN_OBJ && scr_stream_dyn_member_object(value, "readable") && + !scr_exc_pending() && scr_stream_dyn_member_object(value, "writable"); +} + +/* Whether a checked-dynamic object has a function member `key`. */ +static bool scr_stream_dyn_has_method(const ScrDyn *value, const char *key) { + ScrDyn *m = scr_dyn_obj_read(value, key, strlen(key)); + if (!m) return false; + bool fn = m->kind == SCR_DYN_FUNC; + scr_dyn_release(m); + return fn; +} + +/* pipeline.js's isNodeStream and isWebStream for a checked-dynamic value + * that is not a native stream: a stream-like object or a web stream, which + * the pipeline would pipe or pump into and has no lowering for. */ +static bool scr_stream_dyn_stream_like(const ScrDyn *value) { + if (value->kind == SCR_DYN_HANDLE) { + int tag = value->v.handle.tag; + return tag == SCR_DYNH_WEB_STREAM || tag == SCR_DYNH_WEB_TRANSFORM || tag == SCR_DYNH_WEB_WRITABLE; + } + if (value->kind != SCR_DYN_OBJ) return false; + static const char *const state_keys[] = {"_readableState", "_writableState"}; + for (size_t i = 0; i < 2; i++) { + ScrDyn *v = scr_dyn_obj_read(value, state_keys[i], strlen(state_keys[i])); + bool set = v && scr_dyn_truthy(v); + if (v) scr_dyn_release(v); + if (set) return true; + } + bool on = scr_stream_dyn_has_method(value, "on"); + if (on && (scr_stream_dyn_has_method(value, "write") || scr_stream_dyn_has_method(value, "pipe"))) + return true; + if (scr_stream_dyn_has_method(value, "pipeThrough") && + scr_stream_dyn_has_method(value, "getReader") && scr_stream_dyn_has_method(value, "cancel")) + return true; + if (scr_stream_dyn_has_method(value, "getWriter") && scr_stream_dyn_has_method(value, "abort")) + return true; + return scr_stream_dyn_transform_like(value); +} + +/* A checked-dynamic pipeline() stage that the pipeline takes as a stream + * (the stream.unboxDynStage libCall): a boxed native stream or a stream + * handle answers its stream (+1). Any other value that is not stream-like + * or a function takes pipelineImpl's last branch, Duplex.from(stream): + * an iterable answers a readable Duplex the pipeline uses without piping + * the previous stage into it, and values Duplex.from rejects throw its + * ERR_INVALID_ARG_TYPE. Stream-like objects, web streams and functions + * throw `message` as the not-supported Error. */ +ScrStream *scr_stream_unbox_dyn_stage(const ScrDyn *value, const ScrStr *message) { + ScrStream *stream = scr_stream_of_handle(value); + if (!stream) stream = scr_stream_boxed(value); + if (stream) return stream; + if (value->kind == SCR_DYN_FUNC || scr_stream_dyn_stream_like(value)) { + if (!scr_exc_pending()) + scr_throw_error_msg_code(SCR_ERR_ERROR, message->data, message->len, "SC2020"); + return NULL; + } + if (scr_exc_pending()) return NULL; + ScrStream *d = scr_stream_duplex_from_dyn((ScrDyn *)value); + if (d) d->st->pipeline_body = true; + return d; +} + +/* pipeline.js's first stage: a function is a source function, which the + * pipeline calls; a value with pipe() and on() methods is a readable node + * stream, which the stage ladder tests before isIterable; a string, a + * value with a Symbol.asyncIterator or Symbol.iterator method, or a + * { readable, writable } pair is an iterable the pipeline pumps into the + * next stage. */ bool scr_stream_dyn_iterable_source(const ScrDyn *value) { ScrStream *stream = scr_stream_of_handle(value); if (!stream) stream = scr_stream_boxed(value); @@ -8604,7 +8784,8 @@ bool scr_stream_dyn_iterable_source(const ScrDyn *value) { scr_stream_release(stream); return false; } - if (value->kind == SCR_DYN_STR) return true; + if (value->kind == SCR_DYN_STR || value->kind == SCR_DYN_FUNC) return true; + if (value->kind == SCR_DYN_HANDLE && value->v.handle.tag == SCR_DYNH_WEB_TRANSFORM) return true; if (value->kind == SCR_DYN_OBJ) { ScrDyn *pipe = scr_dyn_obj_read(value, "pipe", 4); if (!pipe) return false; @@ -8617,6 +8798,79 @@ bool scr_stream_dyn_iterable_source(const ScrDyn *value) { scr_dyn_release(on); if (streamish) return false; } + if (scr_stream_dyn_transform_like(value)) return true; + if (scr_exc_pending()) return false; } return scr_dyn_iterable_kind(value) > 0; } + +/* Untyped web streams as pipeline stages have no lowering. */ +static void scr_stream_throw_web_stage(void) { + static const char msg[] = "pipeline stages that are untyped web streams are not supported yet"; + scr_throw_error_msg_code(SCR_ERR_ERROR, msg, sizeof msg - 1, "SC2020"); +} + +/* A native stream returned by an untyped pipeline function, which Node + * pipes or passes on as the stream: throws the not-supported Error. */ +static bool scr_stream_dyn_stage_returned_stream(const ScrDyn *value) { + ScrStream *stream = scr_stream_of_handle(value); + if (!stream) stream = scr_stream_boxed(value); + if (!stream) return false; + scr_stream_release(stream); + static const char msg[] = "pipeline function stages that return a stream are not supported yet"; + scr_throw_error_msg_code(SCR_ERR_ERROR, msg, sizeof msg - 1, "SC2020"); + return true; +} + +/* Node's ERR_INVALID_RETURN_VALUE TypeError: "Expected to be + * returned from the "" function but got ." */ +static void scr_stream_throw_invalid_return(const char *expected, const char *name, + const ScrDyn *got) { + char detail[160]; + const char *type = scr_dyn_specific_type(got, detail, sizeof detail); + char msg[384]; + int n = snprintf(msg, sizeof msg, "Expected %s to be returned from the \"%s\" function but got %s.", + expected, name, type); + if (n < 0) n = 0; + if ((size_t)n >= sizeof msg) n = (int)sizeof msg - 1; + scr_throw_error_msg_code(SCR_ERR_TYPE, msg, (size_t)n, "ERR_INVALID_RETURN_VALUE"); +} + +/* An untyped pipeline function stage's result before the last stage + * (pipelineImpl's `isIterable(ret, true)` check): an async iterable + * answers itself (+1); other values throw ERR_INVALID_RETURN_VALUE naming + * the stage, as `label` ("transform[i]") does. */ +ScrDyn *scr_stream_dyn_transform_result(const ScrDyn *value, const ScrStr *label) { + if (scr_stream_dyn_stage_returned_stream(value)) return NULL; + int kind = scr_dyn_iterable_kind(value); + if (kind < 0) return NULL; + if (kind != 2) { + scr_stream_throw_invalid_return("AsyncIterable", label->data, value); + return NULL; + } + return scr_dyn_retain((ScrDyn *)value); +} + +/* The value a checked-dynamic first stage pumps (pipelineImpl's i === 0 + * branch): for `returned`, a source function's result, which must be + * iterable; otherwise a { readable, writable } pair's readable side, or + * the value itself. +1, or NULL with an exception pending. */ +ScrDyn *scr_stream_dyn_source_value(const ScrDyn *value, bool returned) { + if (!returned) { + if (value->kind == SCR_DYN_HANDLE && value->v.handle.tag == SCR_DYNH_WEB_TRANSFORM) { + scr_stream_throw_web_stage(); + return NULL; + } + if (scr_stream_dyn_transform_like(value)) return scr_dyn_obj_read(value, "readable", 8); + if (scr_exc_pending()) return NULL; + return scr_dyn_retain((ScrDyn *)value); + } + if (scr_stream_dyn_stage_returned_stream(value)) return NULL; + int kind = scr_dyn_iterable_kind(value); + if (kind < 0) return NULL; + if (kind == 0) { + scr_stream_throw_invalid_return("Iterable, AsyncIterable or Stream", "source", value); + return NULL; + } + return scr_dyn_retain((ScrDyn *)value); +} diff --git a/packages/runtime/src/scr_stream.h b/packages/runtime/src/scr_stream.h index 7acad0eeb..c10978298 100644 --- a/packages/runtime/src/scr_stream.h +++ b/packages/runtime/src/scr_stream.h @@ -149,6 +149,18 @@ ScrStream *scr_stream_unbox_dyn(const struct ScrDyn *value, const ScrStr *messag * to pump rather than as a stream (false with an exception pending when * reading the value's members throws). */ bool scr_stream_dyn_iterable_source(const struct ScrDyn *value); +/* A checked-dynamic pipeline() stage the pipeline takes as a stream: a + * native stream (+1), or Duplex.from of a value that is not stream-like + * or a function, which the pipeline uses without piping into it; NULL + * with an exception otherwise (`message` for the unsupported shapes). */ +ScrStream *scr_stream_unbox_dyn_stage(const struct ScrDyn *value, const ScrStr *message); +/* The iterable pipeline() pumps for a checked-dynamic first stage: a + * source function's result when `returned`, else a { readable, writable } + * pair's readable side or the value (+1, or NULL with an exception). */ +struct ScrDyn *scr_stream_dyn_source_value(const struct ScrDyn *value, bool returned); +/* An untyped function stage's result before the last pipeline stage: the + * async iterable itself (+1), or NULL with ERR_INVALID_RETURN_VALUE. */ +struct ScrDyn *scr_stream_dyn_transform_result(const struct ScrDyn *value, const ScrStr *label); /* Handles that are Transforms (ScrDynHandleOps.transform: Cipheriv and * Decipheriv). scr_stream_of_handle: the handle's Transform (+1), built on * first use, or NULL when the value is not such a handle. diff --git a/tests/corpus/7508-pipeline-untyped-source-functions.mjs b/tests/corpus/7508-pipeline-untyped-source-functions.mjs new file mode 100644 index 000000000..987221ffd --- /dev/null +++ b/tests/corpus/7508-pipeline-untyped-source-functions.mjs @@ -0,0 +1,85 @@ +// pipeline() over untyped first stages that are functions or +// { readable, writable } pairs: a source function is called with +// { signal } while the pipeline is set up and its iterable result is +// pumped; a result that is not iterable throws ERR_INVALID_RETURN_VALUE +// from the callback form and rejects the promise form; a pair pumps its +// readable side. +import { pipeline, Writable, Readable, PassThrough } from "node:stream"; +import { pipeline as pipelineP } from "node:stream/promises"; + +function pick(v) { + return v; +} +function sink(label) { + return new Writable({ + objectMode: true, + write(chunk, _enc, cb) { + console.log(label, "write", JSON.stringify(String(chunk))); + cb(); + }, + final(cb) { + console.log(label, "final"); + cb(); + }, + }); +} + +const arraySource = pick(function (opts) { + console.log("array source called", typeof opts, typeof opts.signal, opts.signal.aborted); + return ["a1", "a2"]; +}); +const before = pipelineP(arraySource, sink("array")); +console.log("after pipeline() returned"); +await before; +console.log("array resolved"); + +await pipelineP(pick(() => "xy"), sink("string")); + +await pipelineP( + pick(async function* ({ signal }) { + console.log("generator source", typeof signal.aborted); + yield "g1"; + await new Promise((r) => setTimeout(r, 1)); + yield "g2"; + }), + pick(async function* (source) { + for await (const chunk of source) yield `[${chunk}]`; + }), + sink("generator"), +); + +await new Promise((resolve) => { + pipeline(pick(() => new Set(["s1", "s2"])), sink("set"), (err) => { + console.log("set done", err); + resolve(undefined); + }); +}); + +// a result that is not iterable +try { + pipeline(pick(() => 5), sink("five"), () => console.log("five callback")); +} catch (err) { + console.log("five threw", err.name, err.code, err.message); +} +try { + await pipelineP(pick(() => ({ value: 1 })), sink("object")); +} catch (err) { + console.log("object rejected", err.name, err.code, err.message); +} + +// a source function that throws +try { + await pipelineP( + pick(() => { + throw new Error("source failed"); + }), + sink("throws"), + ); +} catch (err) { + console.log("throws rejected", err.message); +} + +// a { readable, writable } pair pumps its readable side +const side = new PassThrough(); +await pipelineP(pick({ readable: Readable.from(["p1", "p2"]), writable: side }), sink("pair")); +console.log("done"); diff --git a/tests/corpus/7509-pipeline-untyped-function-stages.mjs b/tests/corpus/7509-pipeline-untyped-function-stages.mjs new file mode 100644 index 000000000..18749e55a --- /dev/null +++ b/tests/corpus/7509-pipeline-untyped-function-stages.mjs @@ -0,0 +1,112 @@ +// pipeline() with untyped stages after the first that are functions: a +// transform is called with the previous stream or generator and +// { signal } while the pipeline is set up, and must return an async +// iterable; an async function destination's value resolves the promise +// form and reaches the callback; an async generator destination feeds +// the PassThrough that pipeline() returns. Results of the wrong kind throw +// ERR_INVALID_RETURN_VALUE naming the stage. A stage after an untyped +// source function or transform receives that stage's own iterator, as +// Node's makeAsyncIterable answers an iterable itself. +import { pipeline, Writable, Readable, PassThrough } from "node:stream"; +import { pipeline as pipelineP } from "node:stream/promises"; + +function pick(v) { + return v; +} +function sink(label) { + return new Writable({ + objectMode: true, + write(chunk, _enc, cb) { + console.log(label, "write", JSON.stringify(String(chunk))); + cb(); + }, + }); +} +const upper = pick(async function* (source, { signal }) { + console.log("upper called", source instanceof Readable, typeof signal.aborted); + for await (const chunk of source) yield String(chunk).toUpperCase(); +}); +const join = pick(async (source) => { + let text = ""; + for await (const chunk of source) text += chunk; + return text; +}); + +await pipelineP(Readable.from(["a", "b"]), upper, sink("transform")); +console.log("joined", await pipelineP(Readable.from(["x", "y"]), upper, join)); +console.log("two transforms", await pipelineP(pick(["m", "n"]), upper, upper, join)); + +await new Promise((resolve) => { + pipeline(Readable.from(["c1", "c2"]), upper, join, (err, value) => { + console.log("callback", err, value); + resolve(undefined); + }); +}); + +// an async generator destination +const wrap = pick(async function* (source) { + for await (const chunk of source) yield `<${chunk}>`; +}); +const out = pipeline(Readable.from(["p1", "p2"]), wrap, (err, value) => { + console.log("generator destination done", err, value); +}); +console.log("returns a PassThrough", out instanceof PassThrough); +for await (const chunk of out) console.log("passthrough", chunk); + +// a destination whose promise rejects +try { + await pipelineP( + Readable.from(["r"]), + pick(async (source) => { + for await (const chunk of source) console.log("rejecting after", chunk); + throw new Error("destination failed"); + }), + ); +} catch (err) { + console.log("rejected", err.message); +} + +// results of the wrong kind +try { + await pipelineP(Readable.from(["i"]), pick(() => 5), sink("number")); +} catch (err) { + console.log("transform result", err.name, err.code, err.message); +} +try { + await pipelineP(Readable.from(["i"]), upper, pick(() => ["sync"]), sink("array")); +} catch (err) { + console.log("second transform result", err.code, err.message); +} +try { + pipeline(Readable.from(["i"]), pick(() => 7), () => console.log("never")); +} catch (err) { + console.log("destination result", err.name, err.code, err.message); +} +// the iterators handed from stage to stage +let transformIter = null; +const handed = await pipelineP( + pick(function* () { + yield "s1"; + yield "s2"; + }), + pick((source) => { + // a sync generator, not an async iterator over it + console.log( + "transform gets the source's iterator", + typeof source[Symbol.asyncIterator], + typeof source[Symbol.iterator], + ); + transformIter = (async function* () { + for await (const chunk of source) yield `${chunk}!`; + })(); + return transformIter; + }), + pick(async (source) => { + console.log("destination gets the transform's iterator", source === transformIter); + let text = ""; + for await (const chunk of source) text += chunk; + return text; + }), +); +console.log("handed", handed); +console.log("done"); diff --git a/tests/corpus/7510-pipeline-untyped-later-iterables.mjs b/tests/corpus/7510-pipeline-untyped-later-iterables.mjs new file mode 100644 index 000000000..7fe6e27b3 --- /dev/null +++ b/tests/corpus/7510-pipeline-untyped-later-iterables.mjs @@ -0,0 +1,64 @@ +// pipeline() with an untyped stage after the first that is not a stream or +// a function: Node takes Duplex.from of it, so an iterable becomes the +// readable stage the next stream is piped from, the stages before it are +// not read and are destroyed when the pipeline finishes, and a value +// Duplex.from rejects throws ERR_INVALID_ARG_TYPE. +import { pipeline, Writable, Readable, Duplex } from "node:stream"; +import { pipeline as pipelineP } from "node:stream/promises"; + +function pick(v) { + return v; +} +function sink(label) { + return new Writable({ + objectMode: true, + write(chunk, _enc, cb) { + console.log(label, "write", JSON.stringify(String(chunk))); + cb(); + }, + }); +} +function watched(label, values) { + const source = Readable.from(values); + source.on("error", (err) => console.log(label, "error", err.code)); + source.on("close", () => console.log(label, "close", source.destroyed)); + return source; +} + +await pipelineP(watched("first", ["unread"]), pick(["z1", "z2"]), sink("array")); +console.log("array resolved"); +await new Promise((r) => setTimeout(r, 5)); + +await new Promise((resolve) => { + pipeline(watched("second", ["unread"]), pick("text"), sink("string"), (err) => { + console.log("string done", err); + resolve(undefined); + }); +}); +await new Promise((r) => setTimeout(r, 5)); + +// a generator stage before the iterable is never read +await pipelineP( + Readable.from(["s"]), + pick(async function* (source) { + console.log("generator stage called"); + for await (const chunk of source) yield chunk; + }), + pick(new Set(["set1", "set2"])), + sink("set"), +); +console.log("set resolved"); + +// the iterable as the last stage is what pipeline() returns +const last = pipeline(Readable.from(["u"]), pick(["tail"]), () => console.log("never")); +console.log(last instanceof Duplex, last.readable, last.writable); +for await (const chunk of last) console.log("last", chunk); + +for (const value of [42, { plain: true }, null]) { + try { + await pipelineP(Readable.from(["i"]), pick(value), sink("rejected")); + } catch (err) { + console.log(err.name, err.code, err.message); + } +} +console.log("done"); diff --git a/tests/corpus/7511-pipeline-untyped-stages.cjs b/tests/corpus/7511-pipeline-untyped-stages.cjs new file mode 100644 index 000000000..124372e12 --- /dev/null +++ b/tests/corpus/7511-pipeline-untyped-stages.cjs @@ -0,0 +1,50 @@ +// A CommonJS helper that passes its untyped parameters to pipeline(): the +// same call runs over streams, function stages and iterables, whichever +// each argument is at run time. +const { pipeline, Transform, Writable, Readable } = require("node:stream"); + +function run(label, source, middle, destination) { + return new Promise((resolve) => { + pipeline(source, middle, destination, (err, value) => { + console.log(label, "done", err ? err.code || err.message : err, value); + resolve(undefined); + }); + }); +} +function sink(label) { + return new Writable({ + objectMode: true, + write(chunk, _enc, cb) { + console.log(label, "write", String(chunk)); + cb(); + }, + }); +} +const double = new Transform({ + objectMode: true, + transform(chunk, _enc, cb) { + cb(null, String(chunk) + String(chunk)); + }, +}); +async function* exclaim(source) { + for await (const chunk of source) yield `${chunk}!`; +} +async function count(source) { + let n = 0; + for await (const _ of source) n++; + return n; +} + +(async () => { + await run("streams", Readable.from(["a", "b"]), double, sink("streams")); + await run("function middle", Readable.from(["c"]), exclaim, sink("function middle")); + await run("function ends", () => ["d", "e", "f"], exclaim, count); + await run("iterable middle", Readable.from(["skipped"]), ["g"], sink("iterable middle")); + await run("generator into stream", ["h"], exclaim, new Transform({ + objectMode: true, + transform(chunk, _enc, cb) { + console.log("transform saw", chunk); + cb(null, chunk); + }, + }).on("data", (d) => console.log("data", d))); +})(); diff --git a/tests/corpus/7512-pipeline-any-stages.ts b/tests/corpus/7512-pipeline-any-stages.ts new file mode 100644 index 000000000..a2039a533 --- /dev/null +++ b/tests/corpus/7512-pipeline-any-stages.ts @@ -0,0 +1,45 @@ +// pipeline() in TypeScript with stages after the first typed any or +// unknown: functions run as function stages, streams are piped, and an +// iterable is taken as Duplex.from of it. +import { pipeline, Readable, Writable, PassThrough } from "node:stream"; +import { pipeline as pipelineP } from "node:stream/promises"; + +function sink(label: string): Writable { + return new Writable({ + objectMode: true, + write(chunk: unknown, _enc, cb) { + console.log(label, JSON.stringify(chunk)); + cb(); + }, + }); +} + +async function main(): Promise { + const shout: any = async function* (source: any) { + for await (const chunk of source) yield String(chunk).toUpperCase(); + }; + await pipelineP(Readable.from(["a", "b"]), shout, sink("function")); + + const total: unknown = async (source: any) => { + let sum = 0; + for await (const chunk of source) sum += Number(chunk); + return sum; + }; + const sum = await pipelineP(Readable.from(["1", "2", "3"]), total as any); + console.log("sum", sum); + + const relay: any = new PassThrough({ objectMode: true }); + await pipelineP(Readable.from([{ k: 1 }]), relay, sink("stream")); + + const values: any = ["v1", "v2"]; + await pipelineP(Readable.from(["unread"]), values, sink("iterable")); + + await new Promise((resolve) => { + pipeline(Readable.from(["x"]), shout, sink("callback"), (err: NodeJS.ErrnoException | null) => { + console.log("callback done", err == null); + resolve(); + }); + }); +} + +main().then(() => console.log("done")); diff --git a/tests/corpus/7513-pipeline-untyped-stage-errors.mjs b/tests/corpus/7513-pipeline-untyped-stage-errors.mjs new file mode 100644 index 000000000..ade4370b4 --- /dev/null +++ b/tests/corpus/7513-pipeline-untyped-stage-errors.mjs @@ -0,0 +1,86 @@ +// Untyped pipeline function stages next to typed stages: an untyped stage +// after a typed generator stage or a Transform, the { signal } an untyped +// stage receives aborting when the pipeline fails, and an untyped stage +// that throws while the pipeline is set up. +import { pipeline, Writable, Readable, Transform } from "node:stream"; +import { pipeline as pipelineP } from "node:stream/promises"; + +function pick(v) { + return v; +} +function sink(label) { + return new Writable({ + objectMode: true, + write(chunk, _enc, cb) { + console.log(label, "write", JSON.stringify(String(chunk))); + cb(); + }, + }); +} + +await pipelineP( + Readable.from(["a", "b"]), + async function* (source) { + for await (const chunk of source) yield `${chunk}1`; + }, + pick(async function* (source) { + for await (const chunk of source) yield `${chunk}2`; + }), + sink("after generator"), +); + +await pipelineP( + Readable.from(["q"]), + new Transform({ + transform(chunk, _enc, cb) { + cb(null, chunk); + }, + }), + pick(async (source) => { + for await (const chunk of source) console.log("destination saw", String(chunk)); + }), +); + +let seen; +try { + await pipelineP( + Readable.from(["x", "y"]), + pick(async function* (source, { signal }) { + seen = signal; + signal.addEventListener("abort", () => console.log("signal abort event")); + for await (const chunk of source) yield chunk; + }), + new Writable({ + write(_chunk, _enc, cb) { + cb(new Error("write failed")); + }, + }), + ); +} catch (err) { + console.log("rejected", err.message, seen.aborted); +} + +try { + pipeline( + Readable.from(["t"]), + pick(() => { + throw new Error("stage threw"); + }), + sink("never"), + () => console.log("never called"), + ); +} catch (err) { + console.log("callback form threw", err.message); +} +try { + await pipelineP( + Readable.from(["t"]), + pick(() => { + throw new Error("stage threw again"); + }), + sink("never"), + ); +} catch (err) { + console.log("promise form rejected", err.message); +} +console.log("done"); diff --git a/tests/harness/corpus-costs/7508-pipeline-untyped-source-functions.mjs.json b/tests/harness/corpus-costs/7508-pipeline-untyped-source-functions.mjs.json new file mode 100644 index 000000000..805df1394 --- /dev/null +++ b/tests/harness/corpus-costs/7508-pipeline-untyped-source-functions.mjs.json @@ -0,0 +1 @@ +{ "plain": 4900, "san": 6400 } diff --git a/tests/harness/corpus-costs/7509-pipeline-untyped-function-stages.mjs.json b/tests/harness/corpus-costs/7509-pipeline-untyped-function-stages.mjs.json new file mode 100644 index 000000000..93a0d0f3b --- /dev/null +++ b/tests/harness/corpus-costs/7509-pipeline-untyped-function-stages.mjs.json @@ -0,0 +1 @@ +{ "plain": 6100, "san": 8500 } diff --git a/tests/harness/corpus-costs/7510-pipeline-untyped-later-iterables.mjs.json b/tests/harness/corpus-costs/7510-pipeline-untyped-later-iterables.mjs.json new file mode 100644 index 000000000..e1920ef92 --- /dev/null +++ b/tests/harness/corpus-costs/7510-pipeline-untyped-later-iterables.mjs.json @@ -0,0 +1 @@ +{ "plain": 5100, "san": 8600 } diff --git a/tests/harness/corpus-costs/7511-pipeline-untyped-stages.cjs.json b/tests/harness/corpus-costs/7511-pipeline-untyped-stages.cjs.json new file mode 100644 index 000000000..eafd03265 --- /dev/null +++ b/tests/harness/corpus-costs/7511-pipeline-untyped-stages.cjs.json @@ -0,0 +1 @@ +{ "plain": 5300, "san": 6900 } diff --git a/tests/harness/corpus-costs/7512-pipeline-any-stages.ts.json b/tests/harness/corpus-costs/7512-pipeline-any-stages.ts.json new file mode 100644 index 000000000..ac0e8497d --- /dev/null +++ b/tests/harness/corpus-costs/7512-pipeline-any-stages.ts.json @@ -0,0 +1 @@ +{ "plain": 4500, "san": 6100 } diff --git a/tests/harness/corpus-costs/7513-pipeline-untyped-stage-errors.mjs.json b/tests/harness/corpus-costs/7513-pipeline-untyped-stage-errors.mjs.json new file mode 100644 index 000000000..f363efcf0 --- /dev/null +++ b/tests/harness/corpus-costs/7513-pipeline-untyped-stage-errors.mjs.json @@ -0,0 +1 @@ +{ "plain": 4700, "san": 6300 } From 693b038b15432b25d446b1352391858e7c545e9f Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 18:17:32 -0500 Subject: [PATCH 17/20] Keep the result of void generic methods called through class-backed interfaces A class may implement a void interface method with a method that returns a value, and JavaScript passes that value on (`console.log(c.publish(m))`). The call now keeps the dispatch's untyped result, as a call through a non-generic void member already does, instead of replacing it with undefined: printing or interpolating it shows the value, and binding it to a void-typed name checks it. --- .../src/frontend/lowering/lower-calls.ts | 15 +++++--------- tests/corpus/generic-interface-void-calls.ts | 20 +++++++++++++++++++ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index e2f7ade90..265b7553b 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -10074,16 +10074,11 @@ function lowerObjLitGenericMethodCall( loc, }; const resultType = lowerer.irTypeOf(call); - // A void method's runtime result is discarded, as JavaScript does. - if (resultType.kind === "void") { - return { - kind: "seqExpr", - stmts: [{ kind: "exprStmt", expr: invoked, loc }], - result: { kind: "libCall", fn: "console.noop", args: [], type: VOID, loc }, - type: VOID, - loc, - }; - } + // A void method's result stays the runtime's untyped value, as a + // call through a non-generic void member's dispatch keeps it: a class + // may implement the member with a method that returns a value, which + // JavaScript passes on (`console.log(c.publish(m))`). + if (resultType.kind === "void") return invoked; return lowerer.coerceInto(call, invoked, resultType); } } diff --git a/tests/corpus/generic-interface-void-calls.ts b/tests/corpus/generic-interface-void-calls.ts index d810383f7..63c137c10 100644 --- a/tests/corpus/generic-interface-void-calls.ts +++ b/tests/corpus/generic-interface-void-calls.ts @@ -43,3 +43,23 @@ for (const channel of channels) { console.log(channel.name, result === undefined, channel.count(["a", "b", "c"])); } console.log(delivered.join(" | ")); + +// A class may implement the void member with a method that returns a value; +// JavaScript passes that value on. +class CountingChannel implements Channel { + name = "counting"; + sent = 0; + publish(message: Message): number { + this.sent++; + return message.content.length; + } + count(items: T[]): number { + return items.length * 2; + } +} + +const counted: Channel[] = [new CountingChannel(), new LogChannel("plain")]; +for (const channel of counted) { + console.log(channel.name, channel.publish({ content: "four" })); + console.log(`sent ${channel.publish({ content: "again", details: 2 })}`); +} From a848a7b349338d91ec3000ccebd3d5bd5d1242b0 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 18:44:18 -0500 Subject: [PATCH 18/20] Resolve the pipeline promise to undefined for an untyped iterable destination When an untyped destination function returned an async iterable, the promise form settled its result without a payload, and code that read the awaited value as an untyped value dereferenced NULL. The promise form for an untyped destination now resolves to undefined when the destination returned no promise, or its promise carries no value, as Node does. --- packages/runtime/src/scr_stream.c | 24 +++++++++++++++++-- .../7509-pipeline-untyped-function-stages.mjs | 5 ++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index a7593b551..fb416e5c3 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -5312,13 +5312,31 @@ static void scr_sp_value_inv(ScrClosure *cb, ScrError *err, ScrDyn *value, ScrPr scr_promise_release(p); } +/* scr_sp_value_inv for an untyped destination (SCR_PIPE_DEST_DYN), whose + * result promise carries a dyn value: when the destination returned an + * async iterable (no promise), or its promise carries no payload, the + * result is undefined, as in Node. */ +static void scr_sp_dyn_value_inv(ScrClosure *cb, ScrError *err, ScrDyn *value, + ScrPromise *value_p) { + if (err != NULL || (value_p != NULL && scr_promise_payload_kind(value_p) != SCR_EXC_NONE)) { + scr_sp_value_inv(cb, err, value, value_p); + return; + } + ScrPromise *p = scr_box_get_ref(cb->caps[0]); /* +1 */ + scr_promise_fulfill_ref(p, scr_dyn_retain(scr_dyn_undefined()), scr_dyn_retain_v, + scr_dyn_release_v, NULL); + scr_promise_release(p); +} + ScrPromise *scr_sp_pipeline_stages(double n, ScrStream **streams, double count, ScrClosure **stages, double positions, double flags, double mask, ScrClosure **helpers, ScrClosure *box, ScrDyn *signal, ScrClosure *abort) { ScrPromise *p = scr_promise_new(); ScrPipeStages fs = scr_pipe_stages_of(count, stages, positions, flags, mask, helpers, box, signal); - ScrPipeValInv vi = fs.dest != SCR_PIPE_DEST_STREAM ? &scr_sp_value_inv : NULL; + ScrPipeValInv vi = fs.dest == SCR_PIPE_DEST_STREAM ? NULL + : fs.dest == SCR_PIPE_DEST_DYN ? &scr_sp_dyn_value_inv + : &scr_sp_value_inv; ScrStream *dst = scr_stream_pipeline_ex((size_t)n, streams, scr_sp_watcher(p), &scr_sp_settle_inv, true, NULL, NULL, &fs, abort, vi); if (dst) scr_stream_release(dst); @@ -5370,7 +5388,9 @@ ScrPromise *scr_sp_pipeline_stages_opts(double n, ScrStream **streams, double co ScrPromise *p = scr_promise_new(); if (!scr_sp_opts_signal(p, &outer)) return p; ScrPipeStages fs = scr_pipe_stages_of(count, stages, positions, flags, mask, helpers, box, signal); - ScrPipeValInv vi = fs.dest != SCR_PIPE_DEST_STREAM ? &scr_sp_value_inv : NULL; + ScrPipeValInv vi = fs.dest == SCR_PIPE_DEST_STREAM ? NULL + : fs.dest == SCR_PIPE_DEST_DYN ? &scr_sp_dyn_value_inv + : &scr_sp_value_inv; ScrStream *dst = scr_stream_pipeline_ex((size_t)n, streams, scr_sp_watcher(p), &scr_sp_settle_inv, end, outer, add, &fs, abort, vi); if (dst) scr_stream_release(dst); diff --git a/tests/corpus/7509-pipeline-untyped-function-stages.mjs b/tests/corpus/7509-pipeline-untyped-function-stages.mjs index 18749e55a..590334f90 100644 --- a/tests/corpus/7509-pipeline-untyped-function-stages.mjs +++ b/tests/corpus/7509-pipeline-untyped-function-stages.mjs @@ -53,6 +53,11 @@ const out = pipeline(Readable.from(["p1", "p2"]), wrap, (err, value) => { console.log("returns a PassThrough", out instanceof PassThrough); for await (const chunk of out) console.log("passthrough", chunk); +// an async generator destination in the promise form: the PassThrough +// takes its values and the promise resolves to undefined +const promised = await pipelineP(Readable.from(["q1", "q2"]), wrap); +console.log("generator destination promise", promised, typeof promised); + // a destination whose promise rejects try { await pipelineP( From ebf36f94a179db97fe183f48d59d206541999859 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 19:12:43 -0500 Subject: [PATCH 19/20] Type the typed-arguments parameter list as IrParam[] declareTypedArguments appends the hidden positional slots of a typed-arguments function expression to the parameter list that declareParams returns. Its parameter was typed `{ localId: string; name: string; type: IrType }[]`, while the list is an `IrParam[]` (which also declares `owned?: true`). The native compiler passes an array whose element type has another layout as a converted copy, so the slots were appended to the copy and the lifted function kept no parameters. Building tests/corpus/6380 to 6384 (variadic wrappers that forward `arguments`) with the native compiler then stopped with SC9001 "closure ...: func type does not match target signature" in 6380, 6381, 6382 and 6384. The parameter is now typed `IrParam[]`, the type of the list the caller passes, and the native compiler builds all five programs with output that matches Node. --- .../src/frontend/lowering/typed-arguments.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/typed-arguments.ts b/packages/compiler/src/frontend/lowering/typed-arguments.ts index 2fbd83482..e415add7c 100644 --- a/packages/compiler/src/frontend/lowering/typed-arguments.ts +++ b/packages/compiler/src/frontend/lowering/typed-arguments.ts @@ -2,7 +2,15 @@ import * as ts from "../ts7/adapter.js"; import { bodyReadsArguments } from "../arguments-usage.js"; import { isJsSourceFile, locOf } from "../program.js"; import { restTupleElementTypes, unitOnlyUnion } from "../type-mapper.js"; -import { DYN, type IrExpr, type IrLocal, type IrStmt, type IrType, arrayOf } from "../../ir/ir.js"; +import { + DYN, + type IrExpr, + type IrLocal, + type IrParam, + type IrStmt, + type IrType, + arrayOf, +} from "../../ir/ir.js"; import type { Lowerer } from "./lowerer.js"; import { captureContextArguments } from "./function-context.js"; import { isThisParameter, type ParamShape } from "./call-signatures.js"; @@ -156,7 +164,7 @@ export function declareTypedArguments( lowerer: Lowerer, node: ts.FunctionExpression, shapes: readonly ParamShape[], - params: { localId: string; name: string; type: IrType }[], + params: IrParam[], prologue: IrStmt[], ): void { const loc = locOf(node); From d9bf6c59d2f0129708805adcf49c24414ae7aa20 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 19:24:27 -0500 Subject: [PATCH 20/20] Ship the lib diagnostics answers with the native distribution A build with an empty cache type-checked scriptc's own declaration files, the TypeScript libraries and scriptc's declarations, before reusing the answer in later builds. That check was 18 ms of a 105 ms first build of hello world. - build-native-cli.mts has the built compiler build a module without imports and a module with a type error, each in an empty cache, and records the content keys of the declaration file sets the checker reported no diagnostics in: the checker release, the compiler options, and each file's name and sha256. They ship in lib/declarations/lib-diagnostics.json, named by the toolchain manifest's lib_diagnostics. - A build whose cache has no answer computes the files' content key and, when the shipped answers hold it, uses no diagnostics for those files and stores that answer in the cache. The shipped answers apply only while the checker is the distribution's own tsc-ts and its package version is the release they were recorded with; a changed file has another key. - A compiler the build host cannot run ships without them. --- .../src/frontend/ts7/lib-diagnostics.test.ts | 75 ++++++++- .../src/frontend/ts7/lib-diagnostics.ts | 157 +++++++++++++++++- packages/compiler/src/native/cli.ts | 21 ++- .../compiler/src/native/toolchain.test.ts | 34 ++++ packages/compiler/src/native/toolchain.ts | 12 ++ scripts/build-native-cli.mts | 62 ++++++- scripts/verify-native-cli.mjs | 7 + 7 files changed, 356 insertions(+), 12 deletions(-) diff --git a/packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts b/packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts index 83abaeddc..6d0c0651d 100644 --- a/packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts +++ b/packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts @@ -4,10 +4,14 @@ import { join } from "node:path"; import { afterAll, afterEach, expect, test } from "vitest"; import type { SourceFile } from "./ast-types.js"; import { + prebuiltLibDiagnosticsText, + readPrebuiltLibDiagnostics, + recordPrebuiltLibDiagnostics, reusedKeptLibDiagnostics, reusedLibDiagnostics, setLibDiagnosticsStore, type LibDiagnosticsStore, + type PrebuiltLibDiagnostics, } from "./lib-diagnostics.js"; import type { Ts7DiagnosticData } from "./session-schema.generated.js"; import type { Ts7CheckedFileDiagnosticData, Ts7SessionProgram } from "./session.js"; @@ -24,7 +28,10 @@ const main = join(dir, "main.ts"); const util = join(dir, "\u{1F600}.ts"); const other = join(dir, "�.ts"); -afterEach(() => setLibDiagnosticsStore(null)); +afterEach(() => { + setLibDiagnosticsStore(null); + recordPrebuiltLibDiagnostics(null); +}); afterAll(() => { closeSourceParser(); rmSync(dir, { recursive: true, force: true }); @@ -309,3 +316,69 @@ test("a script that uses an answer stored without global names collects and stor expect(store.entries.size).toBe(1); expect(JSON.parse([...store.entries.values()][0]!).globals).not.toBeNull(); }); + +/** The content keys a full check of the program records, with checker release `checker`. */ +function recorded( + checker: string, + semantic: Map, +): PrebuiltLibDiagnostics { + const keys = new Set(); + recordPrebuiltLibDiagnostics({ checker, record: (key) => keys.add(key) }); + setLibDiagnosticsStore(memoryStore(), "build-time checker"); + reusedLibDiagnostics(sessionProgram({ semantic }), names, supplied, moduleFile); + recordPrebuiltLibDiagnostics(null); + return { checker, keys }; +} + +test("a prebuilt answer stands in for checking the supplied files in an empty store", () => { + const semantic = new Map([[main, [diagnostic(main, 5)]]]); + const prebuilt = recorded("1.0.0", semantic); + expect(prebuilt.keys.size).toBe(1); + // Another installation: its own store, its own checker identity. + const store = memoryStore(); + setLibDiagnosticsStore(store, "installed checker", () => prebuilt); + const program = sessionProgram({ semantic }); + expect(reusedLibDiagnostics(program, names, supplied, moduleFile)).toEqual({ + semantic: [diagnostic(main, 5)], + global: [], + }); + expect(program.asked).toEqual([`checked ${main}`]); + // The answer is stored, so the next build reads it from the store. + expect(store.entries.size).toBe(1); + const next = sessionProgram({ semantic }); + setLibDiagnosticsStore(store, "installed checker", () => { + throw new Error("the store answers first"); + }); + reusedLibDiagnostics(next, names, supplied, moduleFile); + expect(next.asked).toEqual([`checked ${main}`]); +}); + +test("prebuilt answers apply only to the same checker release and file contents", () => { + const semantic = new Map([[main, []]]); + const prebuilt = recorded("1.0.0", semantic); + const other: PrebuiltLibDiagnostics = { checker: "1.0.1", keys: prebuilt.keys }; + setLibDiagnosticsStore(memoryStore(), "installed checker", () => other); + const release = sessionProgram({ semantic }); + reusedLibDiagnostics(release, names, supplied, moduleFile); + expect(release.asked[0]).toBe("getGlobalDiagnostics"); + writeFileSync(fallback, fallbackText + "// edited\n"); + setLibDiagnosticsStore(memoryStore(), "installed checker", () => prebuilt); + const edited = sessionProgram({ semantic }); + reusedLibDiagnostics(edited, names, supplied, moduleFile); + expect(edited.asked[0]).toBe("getGlobalDiagnostics"); + writeFileSync(fallback, fallbackText); +}); + +test("a full check that reports diagnostics in a supplied file records no prebuilt answer", () => { + const prebuilt = recorded("1.0.0", new Map([[lib, [diagnostic(lib, 1)]]])); + expect(prebuilt.keys.size).toBe(0); +}); + +test("prebuilt answer files round-trip, and a malformed one is no answer", () => { + const path = join(dir, "lib-diagnostics.json"); + writeFileSync(path, prebuiltLibDiagnosticsText("1.0.0", ["b", "a", "b"])); + expect(readPrebuiltLibDiagnostics(path)).toEqual({ checker: "1.0.0", keys: new Set(["a", "b"]) }); + writeFileSync(path, '{ "schema": "other", "checker": "1.0.0", "keys": [] }'); + expect(readPrebuiltLibDiagnostics(path)).toBeNull(); + expect(readPrebuiltLibDiagnostics(join(dir, "missing.json"))).toBeNull(); +}); diff --git a/packages/compiler/src/frontend/ts7/lib-diagnostics.ts b/packages/compiler/src/frontend/ts7/lib-diagnostics.ts index c3e563fef..ca1f71fc2 100644 --- a/packages/compiler/src/frontend/ts7/lib-diagnostics.ts +++ b/packages/compiler/src/frontend/ts7/lib-diagnostics.ts @@ -17,9 +17,17 @@ * which a program of modules does not otherwise read (about 4 MB at the * peak of a small build). An answer stored by such a program leaves them * out; the first program with scripts that uses the answer collects them - * and stores them with it. */ + * and stores them with it. + * + * A native distribution also ships prebuilt answers: the content keys of + * the supplied file sets the checker reports no diagnostics in, recorded when + * the distribution was built. A build whose store has no answer uses a + * prebuilt one when the checker is the distribution's own and the files' + * content key is among them, and stores it, so builds with an empty cache + * skip the check of the supplied files too. */ import { createHash } from "node:crypto"; -import { statSync } from "node:fs"; +import { readFileSync, statSync } from "node:fs"; +import { basename, dirname, join } from "node:path"; import type { Node, SourceFile } from "./ast-types.js"; import type { Ts7DiagnosticData } from "./session-schema.generated.js"; import type { Ts7SessionProgram } from "./session.js"; @@ -32,15 +40,108 @@ export interface LibDiagnosticsStore { write(key: string, text: string): void; } -const SCHEMA = "scriptc.lib-diagnostics.v2"; +/** The answers a distribution ships: the content keys (contentKey) of the + * supplied file sets whose full check reported no diagnostics with the + * checker release `checker`. */ +export interface PrebuiltLibDiagnostics { + checker: string; + keys: ReadonlySet; +} -let configured: { store: LibDiagnosticsStore; checker: string } | null = null; +const SCHEMA = "scriptc.lib-diagnostics.v2"; +const PREBUILT_SCHEMA = "scriptc.lib-diagnostics-prebuilt.v1"; + +let configured: { + store: LibDiagnosticsStore; + checker: string; + /** Read on the first store miss; null when there are none. */ + prebuilt: (() => PrebuiltLibDiagnostics | null) | null; +} | null = null; /** Has later builds of this process reuse the diagnostics of the supplied * declaration files through `store`, for the checker identified by - * `checker` (its executable's identity), or stop reusing them (null). */ -export function setLibDiagnosticsStore(store: LibDiagnosticsStore | null, checker = ""): void { - configured = store === null ? null : { store, checker }; + * `checker` (its executable's identity), or stop reusing them (null). + * `prebuilt` reads the answers the distribution ships for that checker. */ +export function setLibDiagnosticsStore( + store: LibDiagnosticsStore | null, + checker = "", + prebuilt: (() => PrebuiltLibDiagnostics | null) | null = null, +): void { + if (store === null) { + configured = null; + return; + } + let loaded: PrebuiltLibDiagnostics | null | undefined; + configured = { + store, + checker, + prebuilt: + prebuilt === null ? null : () => (loaded === undefined ? (loaded = prebuilt()) : loaded), + }; +} + +let prebuiltRecorder: { checker: string; record: (key: string) => void } | null = null; + +/** Has later full checks that report no diagnostics in the supplied files + * pass their content key (contentKey, for the checker release `checker`) to + * `record`, or stop (null). A distribution build collects its prebuilt + * answers this way. */ +export function recordPrebuiltLibDiagnostics( + recorder: { checker: string; record: (key: string) => void } | null, +): void { + prebuiltRecorder = recorder; +} + +/** Reads a prebuilt answers file (`{ schema, checker, keys }`); null when + * it is missing or malformed. */ +export function readPrebuiltLibDiagnostics(path: string): PrebuiltLibDiagnostics | null { + try { + const value = JSON.parse(readFileSync(path, "utf8")) as { + schema?: unknown; + checker?: unknown; + keys?: unknown; + }; + if ( + value.schema !== PREBUILT_SCHEMA || + typeof value.checker !== "string" || + !Array.isArray(value.keys) || + !value.keys.every((key) => typeof key === "string") + ) + return null; + return { checker: value.checker, keys: new Set(value.keys as string[]) }; + } catch { + return null; + } +} + +/** A distribution's prebuilt answers at `path`, for its tsc-ts at `checker` + * (`/lib/tsc-ts`): null unless their checker release is the + * version of the package the executable belongs to. */ +export function distributionPrebuiltLibDiagnostics( + path: string, + checker: string, +): PrebuiltLibDiagnostics | null { + const prebuilt = readPrebuiltLibDiagnostics(path); + return prebuilt !== null && checkerPackageVersion(checker) === prebuilt.checker ? prebuilt : null; +} + +/** The version of the tsc-ts package whose executable is `checker` + * (`/lib/tsc-ts`), or null when it cannot be read. */ +export function checkerPackageVersion(checker: string): string | null { + try { + const pkg = JSON.parse( + readFileSync(join(dirname(dirname(checker)), "package.json"), "utf8"), + ) as { version?: unknown }; + return typeof pkg.version === "string" ? pkg.version : null; + } catch { + return null; + } +} + +/** The text of a prebuilt answers file holding `keys` for the checker + * release `checker`. */ +export function prebuiltLibDiagnosticsText(checker: string, keys: Iterable): string { + return `${JSON.stringify({ schema: PREBUILT_SCHEMA, checker, keys: [...new Set(keys)].sort() }, null, 1)}\n`; } /** A checker executable's identity for the stored diagnostics' key: its @@ -120,7 +221,7 @@ function reuse( const options = { ...program.getCompilerOptions(), configFilePath: undefined }; const key = entryKey(checker, options, libs); if (key === null) return null; - const stored = readEntry(store, key, libs); + const stored = readEntry(store, key, libs) ?? prebuiltEntry(store, key, options, libs); // An answer stored without the supplied files' global names: a program // with scripts collects them and stores them with the answer. let globals = stored?.globals ?? null; @@ -152,6 +253,10 @@ function reuse( const files: Record = {}; for (const name of libs) files[name] = program.diagnosticData("getSemanticDiagnostics", name); writeEntry(store, key, names, files); + if (prebuiltRecorder !== null && Object.values(files).every((list) => list.length === 0)) { + const content = contentKey(prebuiltRecorder.checker, options, libs); + if (content !== null) prebuiltRecorder.record(content); + } } } return { semantic, global }; @@ -330,6 +435,42 @@ function entryKey(checker: string, options: unknown, libs: readonly string[]): s .digest("hex"); } +/** The supplied files' answer from the distribution's prebuilt answers, + * stored under `key` for later builds: every file without diagnostics, when + * they hold the files' content key; null otherwise. */ +function prebuiltEntry( + store: LibDiagnosticsStore, + key: string, + options: unknown, + libs: readonly string[], +): { files: Map; globals: Set | null } | null { + const prebuilt = configured?.prebuilt?.() ?? null; + if (prebuilt === null || prebuilt.keys.size === 0) return null; + const content = contentKey(prebuilt.checker, options, libs); + if (content === null || !prebuilt.keys.has(content)) return null; + const files: Record = {}; + for (const name of libs) files[name] = []; + writeEntry(store, key, null, files); + return { files: new Map(Object.entries(files)), globals: null }; +} + +/** A supplied file set's key independent of where the files are installed: + * the checker release, the compiler options and each file's name and + * content digest, in program order; null when a file cannot be read. */ +function contentKey(checker: string, options: unknown, libs: readonly string[]): string | null { + const files: [string, string][] = []; + for (const name of libs) { + try { + files.push([basename(name), createHash("sha256").update(readFileSync(name)).digest("hex")]); + } catch { + return null; + } + } + return createHash("sha256") + .update(JSON.stringify({ schema: PREBUILT_SCHEMA, checker, options, files })) + .digest("hex"); +} + function readEntry( store: LibDiagnosticsStore, key: string, diff --git a/packages/compiler/src/native/cli.ts b/packages/compiler/src/native/cli.ts index b3de05846..b87e8b735 100644 --- a/packages/compiler/src/native/cli.ts +++ b/packages/compiler/src/native/cli.ts @@ -1,5 +1,5 @@ import { spawnSync } from "node:child_process"; -import { mkdtempSync, rmSync } from "node:fs"; +import { appendFileSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { runCli } from "../cli/command.js"; @@ -19,6 +19,9 @@ import { NativeCompiler } from "./compiler.js"; import { NATIVE_STRING_BYTES, openNativeCache } from "./cache.js"; import { checkerExecutableIdentity, + checkerPackageVersion, + distributionPrebuiltLibDiagnostics, + recordPrebuiltLibDiagnostics, setLibDiagnosticsStore, } from "../frontend/ts7/lib-diagnostics.js"; import { setCheckerFeatureStore } from "../frontend/ts7/checker-features.js"; @@ -67,7 +70,23 @@ async function main(): Promise { write: (key, text) => store.write("lib-diagnostics", key, text), }, checkerExecutableIdentity(toolchain.ts7Executable), + toolchain.prebuiltLibDiagnostics === undefined + ? null + : () => + distributionPrebuiltLibDiagnostics( + toolchain.prebuiltLibDiagnostics!, + toolchain.ts7Executable, + ), ); + // A distribution build records its prebuilt lib diagnostics (build-native-cli.mts). + const recordPath = process.env["SCRIPTC_LIB_DIAGNOSTICS_RECORD"]; + const checkerVersion = + recordPath === undefined ? null : checkerPackageVersion(toolchain.ts7Executable); + if (recordPath !== undefined && checkerVersion !== null) + recordPrebuiltLibDiagnostics({ + checker: checkerVersion, + record: (key) => appendFileSync(recordPath, `${key}\n`), + }); setCheckerFeatureStore({ read: (key) => store.read("checker-features", key, NATIVE_STRING_BYTES)?.toString("utf8") ?? null, diff --git a/packages/compiler/src/native/toolchain.test.ts b/packages/compiler/src/native/toolchain.test.ts index ad4570c0c..c64bce76b 100644 --- a/packages/compiler/src/native/toolchain.test.ts +++ b/packages/compiler/src/native/toolchain.test.ts @@ -54,6 +54,40 @@ test("native toolchain installation paths relocate with the manifest", () => { } }); +test("prebuilt lib diagnostics apply while the checker is the distribution's tsc-ts", () => { + const root = realpathSync( + mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-native-tools-")), + ); + const path = join(root, "toolchain.json"); + const manifest: NativeToolchainManifest = { + schema: "scriptc.native-toolchain.v1", + compiler_version: "1.2.3", + target: "macos-arm64", + ts7: "typescript/tsc", + tsc_ts: "tsc-ts/lib/tsc-ts", + llvm_package: "llvm", + runtime_pack: "runtime", + linker: "tools/linker", + linker_args: [], + dsymutil: "dsymutil", + lib_diagnostics: "declarations/lib-diagnostics.json", + }; + try { + writeFileSync(path, JSON.stringify(manifest)); + expect(loadNativeToolchain(path).prebuiltLibDiagnostics).toBe( + join(root, "declarations/lib-diagnostics.json"), + ); + // tsgo checks without them. + expect( + loadNativeToolchain(path, { SCRIPTC_CHECKER: "tsgo" }).prebuiltLibDiagnostics, + ).toBeUndefined(); + writeFileSync(path, JSON.stringify({ ...manifest, lib_diagnostics: "" })); + expect(() => loadNativeToolchain(path)).toThrow(); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + test("native toolchain assets resolve from the physical manifest through package links", () => { const root = realpathSync( mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-linked-tools-")), diff --git a/packages/compiler/src/native/toolchain.ts b/packages/compiler/src/native/toolchain.ts index 5abbb329e..c9167ba14 100644 --- a/packages/compiler/src/native/toolchain.ts +++ b/packages/compiler/src/native/toolchain.ts @@ -30,6 +30,9 @@ export interface NativeToolchain { declarationsRoot?: string; comptimeExecutable?: string; wasiNodeRunner?: string; + /** The prebuilt lib diagnostics the distribution ships, present while the + * checker is the distribution's own tsc-ts (lib-diagnostics.ts). */ + prebuiltLibDiagnostics?: string; } /** Distribution paths are relative to this manifest, so moving an installed @@ -54,6 +57,8 @@ export interface NativeToolchainManifest { relocatable_linker?: string; comptime?: string; wasi_node_runner?: string; + /** Answers for the supplied declaration files, computed with `tsc_ts`. */ + lib_diagnostics?: string; } function pathFrom(root: string, value: string): string { @@ -120,6 +125,7 @@ export function loadNativeToolchain( "relocatable_linker", "comptime", "wasi_node_runner", + "lib_diagnostics", ]) { if (value[name] !== undefined && (typeof value[name] !== "string" || value[name] === "")) throw new Error(`native toolchain requires a nonempty string for ${name}`); @@ -221,5 +227,11 @@ export function loadNativeToolchain( ...(manifest.wasi_node_runner === undefined ? {} : { wasiNodeRunner: pathFrom(root, manifest.wasi_node_runner) }), + // The prebuilt answers are the distribution's tsc-ts's. + ...(manifest.lib_diagnostics === undefined || + manifest.tsc_ts === undefined || + checkerBackend(env) === "tsgo" + ? {} + : { prebuiltLibDiagnostics: pathFrom(root, manifest.lib_diagnostics) }), }; } diff --git a/scripts/build-native-cli.mts b/scripts/build-native-cli.mts index 4d3f00ff5..05f36cf68 100644 --- a/scripts/build-native-cli.mts +++ b/scripts/build-native-cli.mts @@ -1,12 +1,14 @@ /** Build the production CLI seed and its relocatable compiler assets. */ -import { execFileSync } from "node:child_process"; -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { execFileSync, spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; import { basename, dirname, join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { gunzipSync } from "node:zlib"; import { compile } from "../packages/compiler/src/index.js"; import { NATIVE_TARGETS, nativeCodegenTarget } from "../packages/compiler/src/backend/targets.js"; import { tsgoExecutable } from "../packages/compiler/src/frontend/ts7/rpc-api.js"; +import { prebuiltLibDiagnosticsText } from "../packages/compiler/src/frontend/ts7/lib-diagnostics.js"; import { stageNativeCliAssets } from "./native-cli-assets.mjs"; import type { NativeToolchainManifest } from "../packages/compiler/src/native/toolchain.js"; @@ -181,4 +183,60 @@ const manifest: NativeToolchainManifest = { comptime: relative(bin, comptime), }; writeFileSync(executable + ".json", JSON.stringify(manifest, null, 2) + "\n"); +// The prebuilt lib diagnostics (frontend/ts7/lib-diagnostics.ts): the +// declaration file sets the distribution's tsc-ts reports no diagnostics in +// for programs without a tsconfig, recorded by the built compiler itself. A +// compiler this host cannot run ships without them. +const prebuiltKeys = recordPrebuiltLibDiagnostics(executable); +if (prebuiltKeys === null) { + console.warn( + "warning: the built compiler did not run here; it ships without prebuilt lib diagnostics", + ); +} else { + const prebuilt = join(lib, "declarations", "lib-diagnostics.json"); + const checker = ( + JSON.parse(readFileSync(join(lib, "tsc-ts", "package.json"), "utf8")) as { version: string } + ).version; + writeFileSync(prebuilt, prebuiltLibDiagnosticsText(checker, prebuiltKeys)); + manifest.lib_diagnostics = relative(bin, prebuilt); + writeFileSync(executable + ".json", JSON.stringify(manifest, null, 2) + "\n"); +} console.log(executable); + +/** The content keys the built compiler records building, each in an empty + * cache, a module without imports (the program it lowers) and a module with + * a type error (which also checks the program without scriptc's overrides); + * null when it cannot run. */ +function recordPrebuiltLibDiagnostics(compiler: string): string[] | null { + const work = mkdtempSync(join(tmpdir(), "scriptc-lib-diagnostics-")); + try { + const record = join(work, "keys.txt"); + const probes = [ + { name: "clean", source: "export {};\n", status: 0 }, + { name: "error", source: 'export const n: number = "x";\n', status: 1 }, + ]; + for (const probe of probes) { + const dir = join(work, probe.name); + mkdirSync(dir); + writeFileSync(join(dir, "main.ts"), probe.source); + const env: NodeJS.ProcessEnv = { + ...process.env, + SCRIPTC_CACHE_DIR: join(dir, "cache"), + SCRIPTC_LIB_DIAGNOSTICS_RECORD: record, + }; + delete env["SCRIPTC_TIMING"]; + const run = spawnSync( + compiler, + ["build", join(dir, "main.ts"), "--optimization", "dev", "-o", join(dir, "main")], + { cwd: dir, env, encoding: "utf8" }, + ); + if (run.status !== probe.status) return null; + } + if (!existsSync(record)) return null; + return readFileSync(record, "utf8") + .split("\n") + .filter((key) => key !== ""); + } finally { + rmSync(work, { recursive: true, force: true }); + } +} diff --git a/scripts/verify-native-cli.mjs b/scripts/verify-native-cli.mjs index 92de57938..310cc04a9 100644 --- a/scripts/verify-native-cli.mjs +++ b/scripts/verify-native-cli.mjs @@ -41,6 +41,13 @@ for (const name of [ throw new Error(`native toolchain needs a relative ${name}`); within(resolve(dirname(binary), hostPath(value))); } +// Optional: a compiler this host could not run while building ships without it. +if (toolchain.lib_diagnostics !== undefined) { + const value = toolchain.lib_diagnostics; + if (typeof value !== "string" || absolutePath(value)) + throw new Error("native toolchain needs a relative lib_diagnostics"); + within(resolve(dirname(binary), hostPath(value))); +} const pack = JSON.parse( readFileSync( resolve(dirname(binary), hostPath(toolchain.runtime_pack), "runtime-pack.json"),