From aaca1f1e2b838e0ef15d91743b06bfd5803d6dc0 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 18:17:48 -0500 Subject: [PATCH 01/17] Count the arguments a function receives when it is called through a value `arguments.length` in a typed function folds to a constant. In a function declaration it was the declared parameter count even when the function is called through a value with more arguments: Node prints 4 for `[1].map(f)` with `function f(v) { return v + arguments.length; }`, the compiled program printed 2. A top-level function declaration that reads `arguments` and that the program uses as a value now gets a count slot. Its value is an adapter that stores the parameter count of the function type the value is passed as (3 for map, 4 for reduce, 2 for a sort comparator) and forwards; the function reads the slot on entry and clears it, so a direct call sees the declared count. Function expressions and declarations passed where the expected function type has optional or rest parameters (a replace callback) are called with a varying count and are refused with SC1090 instead of folding to a wrong count, as are methods, nested functions, and arrows inside a function used as a value. --- .../src/frontend/lowering/argument-counts.ts | 230 ++++++++++++++++++ .../lowering/expressions/identifiers.ts | 13 +- .../lowering/expressions/property-access.ts | 50 ++-- .../src/frontend/lowering/function-context.ts | 3 + .../src/frontend/lowering/lower-functions.ts | 2 + .../compiler/src/frontend/lowering/lowerer.ts | 3 + ...81-function-value-arguments-length.ts.json | 6 + .../function-value-arguments-length.ts.json | 6 + .../7381-function-value-arguments-length.ts | 27 ++ .../function-value-arguments-length.ts | 36 +++ .../function-value-arguments-length.ts.txt | 43 ++++ ...81-function-value-arguments-length.ts.json | 1 + 12 files changed, 398 insertions(+), 22 deletions(-) create mode 100644 packages/compiler/src/frontend/lowering/argument-counts.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7381-function-value-arguments-length.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/function-value-arguments-length.ts.json create mode 100644 tests/corpus/7381-function-value-arguments-length.ts create mode 100644 tests/diagnostics/function-value-arguments-length.ts create mode 100644 tests/harness/__snapshots__/function-value-arguments-length.ts.txt create mode 100644 tests/harness/corpus-costs/7381-function-value-arguments-length.ts.json diff --git a/packages/compiler/src/frontend/lowering/argument-counts.ts b/packages/compiler/src/frontend/lowering/argument-counts.ts new file mode 100644 index 000000000..ec1753160 --- /dev/null +++ b/packages/compiler/src/frontend/lowering/argument-counts.ts @@ -0,0 +1,230 @@ +import * as ts from "../ts7/adapter.js"; +import { bodyReadsArguments } from "../arguments-usage.js"; +import { isJsSourceFile, locOf } from "../program.js"; +import { BOOL, F64, type IrExpr, type IrStmt, type IrType } from "../../ir/ir.js"; +import { varRef } from "../../ir/build.js"; +import { typeKey } from "../type-mapper.js"; +import type { Lowerer } from "./lowerer.js"; +import { isThisParameter } from "./call-signatures.js"; +import { buildFunctionAdapter } from "./coercions/builders.js"; +import { keyedHelperName } from "./stable-names.js"; + +/** `arguments.length` in a top-level function declaration that the program + * also uses as a value. A direct call passes the declared parameters (tsc + * checks the arity), while a call through the value passes the parameter + * count of the function type the value was passed as (`[1].map(f)` calls f + * with 3). The value is an adapter that stores that count in a global slot + * before forwarding; the function reads the slot on entry and clears it, + * so a direct call (the cleared slot) sees the declared count. */ + +/** The count slot of `decl`, or null when its `arguments.length` folds to + * the declared count (or the declaration does not qualify). */ +export function argumentCountSlot(lowerer: Lowerer, decl: ts.FunctionDeclaration): string | null { + const cached = lowerer.argumentCountSlots.get(decl); + if (cached !== undefined) return cached; + let slot: string | null = null; + if ( + decl.body !== undefined && + decl.name !== undefined && + decl.asteriskToken === undefined && + !decl.typeParameters?.nodes.length && + lowerer.isTopLevelFnSymbol(decl.name) && + decl.modifiers?.nodes.some((m) => m.kind === ts.SyntaxKind.AsyncKeyword) !== true && + !isJsSourceFile(ts.getSourceFileOfNode(decl)) && + fixedArity(decl) && + bodyReadsArguments(decl) && + usedAsValue(lowerer, decl) + ) { + const sig = lowerer.fnSigOf(decl.name); + if (sig) { + slot = `%g.argc.${sig.name}`; + lowerer.globalsList.push({ id: slot, name: "%argc", type: F64, mutable: true }); + } + } + lowerer.argumentCountSlots.set(decl, slot); + return slot; +} + +/** Reads `decl`'s count slot into a hidden local on entry and clears it; the + * local holds the declared count when the slot was clear. */ +export function declareArgumentCount( + lowerer: Lowerer, + decl: ts.FunctionDeclaration, + prologue: IrStmt[], +): void { + const slot = argumentCountSlot(lowerer, decl); + if (slot === null) return; + const loc = locOf(decl); + const local = lowerer.declareHiddenLocal("%argc", F64); + const stored = varRef(slot, F64, loc); + prologue.push( + { + kind: "varDecl", + localId: local.id, + init: { + kind: "ternary", + cond: { + kind: "bin", + op: "===", + left: stored, + right: { kind: "numLit", value: 0, type: F64, loc }, + type: BOOL, + loc, + }, + then: { kind: "numLit", value: declaredCount(decl), type: F64, loc }, + else_: stored, + type: F64, + loc, + }, + loc, + }, + { kind: "assign", localId: slot, value: { kind: "numLit", value: 0, type: F64, loc }, loc }, + ); + lowerer.ctx.argumentCount = local; +} + +/** `decl` as a value at `reference`: the plain closure when every call + * through the expected function type passes the declared count, otherwise + * an adapter that stores the expected type's count in the slot first. */ +export function argumentCountValue( + lowerer: Lowerer, + decl: ts.FunctionDeclaration, + reference: ts.Expression, + closure: IrExpr & { type: IrType & { kind: "func" } }, +): IrExpr { + const slot = argumentCountSlot(lowerer, decl); + if (slot === null) return closure; + const count = contextualCallCount(lowerer, reference); + if (count === null || count === "varies") { + lowerer.unsupported( + "SC1090", + reference, + count === null + ? "functions that read 'arguments.length' used as values where no function type is expected (the count each call passes is not known)" + : "functions that read 'arguments.length' used as values where the expected function type has optional or rest parameters (the count varies per call)", + ); + } + const declared = declaredCount(decl); + if (count <= declared) return closure; + const loc = locOf(reference); + const type = closure.type; + const key = `argc:${slot}:${count}:${typeKey(type)}`; + let name = lowerer.arrHofHelpers.get(key); + if (name === undefined) { + name = keyedHelperName(lowerer, "%argc.adapter", key); + lowerer.arrHofHelpers.set(key, name); + const params = type.params.map((t, i) => ({ localId: `p${i}.0`, name: `p${i}`, type: t })); + lowerer.liftedFns.push( + ...buildFunctionAdapter( + name, + type, + type, + params, + [], + [ + { + kind: "assign", + localId: slot, + value: { kind: "numLit", value: count, type: F64, loc }, + loc, + }, + { + kind: "return", + value: { + kind: "callValue", + callee: varRef("f.0", type, loc), + args: params.map((p) => varRef(p.localId, p.type, loc)), + type: type.ret, + loc, + }, + loc, + }, + ], + loc, + ), + ); + } + return { kind: "call", callee: name, args: [closure], type, loc }; +} + +/** The argument count every call through the function type `node` is + * expected as passes: that type's parameter count when it has one call + * signature with a fixed parameter list (a `this` parameter is type-only), + * "varies" when it has optional or rest parameters, null when there is no + * such type. */ +export function contextualCallCount( + lowerer: Lowerer, + node: ts.Expression, +): number | "varies" | null { + const contextual = lowerer.checker.getContextualType(node); + if (contextual === undefined) return null; + const signatures = lowerer.checker.getCallSignatures( + lowerer.checker.getNonNullableType(contextual), + ); + if (signatures.length !== 1) return null; + const declaration = signatures[0]!.declaration; + if (declaration === undefined) return null; + let count = 0; + for (const parameter of (declaration.parameters?.nodes ?? []) as ts.ParameterDeclaration[]) { + if (isThisParameter(parameter)) continue; + if (parameter.questionToken || parameter.initializer || parameter.dotDotDotToken) + return "varies"; + count++; + } + return count; +} + +/** Whether a program reference to the function declaration or method `fn` + * reads it as a value (`xs.map(f)`, `const g = f`, `xs.map(C.parse)`) rather + * than calling it. Export specifiers and default exports only name a + * function; the importing files' references resolve to it and are checked + * the same way. */ +export function usedAsValue( + lowerer: Lowerer, + fn: ts.FunctionDeclaration | ts.MethodDeclaration, +): boolean { + const name = fn.name; + if (!name || !(ts.isIdentifier(name) || ts.isPrivateIdentifier(name))) return false; + const method = ts.isMethodDeclaration(fn); + const symbolAt = (n: ts.Node): ts.Symbol | null | undefined => + method + ? lowerer.checker.getSymbolAtLocation(n) + : lowerer.resolveValueSymbol(n as ts.Identifier); + const symbol = symbolAt(name); + if (!symbol) return false; + let found = false; + for (const sf of lowerer.program.getSourceFiles()) { + if (sf.isDeclarationFile) continue; + ts.walkPreorder(sf, (n) => { + if (found) return "stop"; + if (ts.isTypeNode(n)) return "skip"; + if (n === name || !(ts.isIdentifier(n) || ts.isPrivateIdentifier(n))) return undefined; + if (n.text !== name.text || !n.parent) return undefined; + const parent = n.parent; + // A method is read through its member name (`C.parse`, `this.#m`). + const read = ts.isPropertyAccessExpression(parent) && parent.name === n ? parent : n; + if (method && read === n && !ts.isBindingElement(parent)) return undefined; + const use = read.parent; + if (use && ts.isCallExpression(use) && use.expression === read) return undefined; + if (ts.isExportSpecifier(parent) || ts.isExportAssignment(parent)) return undefined; + if (ts.isImportSpecifier(parent) || ts.isImportClause(parent)) return undefined; + if (symbolAt(n) === symbol) found = true; + return undefined; + }); + if (found) break; + } + return found; +} + +function fixedArity(fn: ts.FunctionLikeDeclaration): boolean { + return fn.parameters.nodes.every( + (p) => + p.questionToken === undefined && + p.initializer === undefined && + p.dotDotDotToken === undefined, + ); +} + +function declaredCount(fn: ts.FunctionLikeDeclaration): number { + return fn.parameters.nodes.filter((p) => !isThisParameter(p)).length; +} diff --git a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts index 63175d50a..ff6f65f26 100644 --- a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts +++ b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts @@ -25,6 +25,7 @@ import { lowerNodeModuleIdentifier } from "../lower-node-module.js"; import { lowerCjsExportsObjectValue } from "../cjs-exports-object.js"; import { captureContextArguments } from "../function-context.js"; import { argumentsOwner, typedArgumentsCandidate } from "../typed-arguments.js"; +import { argumentCountValue } from "../argument-counts.js"; import { isCjsExportTableLiteral, isCjsJsFile, @@ -424,7 +425,17 @@ export function lowerIdentifierExpr(lowerer: Lowerer, expr: ts.Identifier, loc: // semantics); the rest marker carries the trailing dyn-array ABI. const funcType: IrType = funcTypeFromParamShapes(sig.params, sig.returnType); lowerer.requireExactArityValue(expr, expr, sig.params, funcType); - return { kind: "closure", fnName: sig.name, captures: [], type: funcType, loc }; + const closure = { + kind: "closure" as const, + fnName: sig.name, + captures: [], + type: funcType, + loc, + }; + const decl = lowerer.resolveValueSymbol(expr)?.valueDeclaration; + return decl && ts.isFunctionDeclaration(decl) && funcType.kind === "func" + ? argumentCountValue(lowerer, decl, expr, { ...closure, type: funcType }) + : closure; } { // A generic function as a VALUE: monomorphized by flow — the diff --git a/packages/compiler/src/frontend/lowering/expressions/property-access.ts b/packages/compiler/src/frontend/lowering/expressions/property-access.ts index 5df3c7fe0..5f1683ff7 100644 --- a/packages/compiler/src/frontend/lowering/expressions/property-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/property-access.ts @@ -6,6 +6,7 @@ import { lowerHistogramProperty } from "../builtins/performance.js"; import { staticFunctionNameRead } from "../static-function-names.js"; import { typedArgumentsArity } from "../typed-arguments.js"; import { isThisParameter } from "../call-signatures.js"; +import { argumentCountSlot, contextualCallCount, usedAsValue } from "../argument-counts.js"; import { checkedPrivateValue, localPrivateBrandClass, @@ -669,7 +670,31 @@ export function lowerPropertyAccessExpr( const declared = (fn as ts.FunctionLikeDeclaration).parameters.nodes.filter( (p) => !isThisParameter(p), ).length; - const count = Math.max(declared, contextualArgumentCount(lowerer, fn) ?? 0); + const contextual = contextualArgumentCount(lowerer, fn); + if (contextual === "varies") { + lowerer.unsupported( + "SC1090", + expr, + "'arguments.length' in function expressions passed where the expected function type has optional or rest parameters (the count varies per call)", + ); + } + // A top-level declaration also used as a value reads the count its + // caller stored (argument-counts.ts). + if (ts.isFunctionDeclaration(fn) && lowerer.ctx.argumentCount !== undefined) + return varRef(lowerer.ctx.argumentCount.id, F64, loc); + if ( + (ts.isFunctionDeclaration(fn) || ts.isMethodDeclaration(fn)) && + usedAsValue(lowerer, fn) + ) { + lowerer.unsupported( + "SC1090", + expr, + ts.isFunctionDeclaration(fn) && argumentCountSlot(lowerer, fn) !== null + ? "'arguments.length' in an arrow function inside a function that is also used as a value" + : "'arguments.length' in methods, nested, async and generator functions used as values (a caller of the value can pass more arguments than the function declares)", + ); + } + const count = Math.max(declared, contextual ?? 0); // Calls through the expression's own name pass the declared count, // calls through the wider slot pass more: no constant answers both. if (count > declared && referencesOwnName(lowerer, fn)) { @@ -4469,24 +4494,7 @@ function referencesOwnName(lowerer: Lowerer, fn: ts.Node): boolean { } /** The argument count every caller passes a function expression whose - * value flows into one fixed-arity function type, an array method's - * callback or an annotated slot: that type's parameter count. Null when - * there is no such type or its arity varies. */ -function contextualArgumentCount(lowerer: Lowerer, fn: ts.Node): number | null { - if (!ts.isFunctionExpression(fn)) return null; - const contextual = lowerer.checker.getContextualType(fn); - if (contextual === undefined) return null; - const signatures = lowerer.checker.getCallSignatures( - lowerer.checker.getNonNullableType(contextual), - ); - if (signatures.length !== 1) return null; - const declaration = signatures[0]!.declaration; - if (declaration === undefined) return null; - let count = 0; - for (const parameter of (declaration.parameters?.nodes ?? []) as ts.ParameterDeclaration[]) { - if (isThisParameter(parameter)) continue; - if (parameter.questionToken || parameter.initializer || parameter.dotDotDotToken) return null; - count++; - } - return count; + * value flows into a function type (argument-counts.ts contextualCallCount). */ +function contextualArgumentCount(lowerer: Lowerer, fn: ts.Node): number | "varies" | null { + return ts.isFunctionExpression(fn) ? contextualCallCount(lowerer, fn) : null; } diff --git a/packages/compiler/src/frontend/lowering/function-context.ts b/packages/compiler/src/frontend/lowering/function-context.ts index 92042b7d7..702574b63 100644 --- a/packages/compiler/src/frontend/lowering/function-context.ts +++ b/packages/compiler/src/frontend/lowering/function-context.ts @@ -46,6 +46,9 @@ export interface FnCtx { * rest param): the synthetic trailing dyn-array param `arguments` * reads resolve to. */ argumentsLocal?: IrLocal | null; + /** The count `arguments.length` reads in a top-level function declaration + * that is also used as a value (argument-counts.ts). */ + argumentCount?: IrLocal; /** Arrows inherit the nearest ordinary function's arguments binding. */ inheritsArguments?: boolean; /** A function lowered on demand on top of an unrelated caller's frames diff --git a/packages/compiler/src/frontend/lowering/lower-functions.ts b/packages/compiler/src/frontend/lowering/lower-functions.ts index c1cca9c64..c1682f7c8 100644 --- a/packages/compiler/src/frontend/lowering/lower-functions.ts +++ b/packages/compiler/src/frontend/lowering/lower-functions.ts @@ -1,5 +1,6 @@ import type { ScrDiagnostic } from "../../diagnostics/diagnostic.js"; import { declareTypedArguments } from "./typed-arguments.js"; +import { declareArgumentCount } from "./argument-counts.js"; import { dynUndefinedExpr, varRef } from "../../ir/build.js"; import { everyStmtList, transformStmtList } from "../../ir/traverse.js"; import { bindingNeverReassigned } from "./binding-analysis.js"; @@ -551,6 +552,7 @@ export function lowerFunction(lowerer: Lowerer, decl: ts.FunctionDeclaration): I (sig.params[sig.params.length - 1]!.mode === "dynRest" || sig.params[sig.params.length - 1]!.mode === "arguments"), ); + declareArgumentCount(lowerer, decl, prologue); const bodyBlock = blockBodyOf(decl); if (!bodyBlock) { lowerer.unsupported( diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 4add3193c..4b60d9e1b 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -2335,6 +2335,9 @@ export class Lowerer { * every name destructured from a present source optional. */ readonly runtimeOptionalPatternTypes = new Map(); readonly runtimeOptionalReduceTypes = new Map(); + /** Each function declaration's `arguments.length` count slot, or null + * (argument-counts.ts argumentCountSlot). */ + readonly argumentCountSlots = new Map(); /** Non-null assertions whose value is forwarded into a slot promoted to * hold undefined. `x!` has no runtime effect, so these keep the stored * undefined state instead of trapping before any dereference. */ diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7381-function-value-arguments-length.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7381-function-value-arguments-length.ts.json new file mode 100644 index 000000000..b9e5eacd1 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7381-function-value-arguments-length.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7381-function-value-arguments-length.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/function-value-arguments-length.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/function-value-arguments-length.ts.json new file mode 100644 index 000000000..6aa894918 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/function-value-arguments-length.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/diagnostics/function-value-arguments-length.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7381-function-value-arguments-length.ts b/tests/corpus/7381-function-value-arguments-length.ts new file mode 100644 index 000000000..059ff1487 --- /dev/null +++ b/tests/corpus/7381-function-value-arguments-length.ts @@ -0,0 +1,27 @@ +// `arguments.length` in a function declaration counts the arguments its +// caller passes: a direct call passes the declared parameters, and a call +// through the function as a value passes the parameters of the function +// type it was passed as (3 for map and filter, 4 for reduce, 2 for sort). +function scaled(value: number): number { + return value * 10 + arguments.length; +} +function total(acc: number, value: number): number { + return acc + value + arguments.length; +} +function compare(a: number, b: number): number { + return arguments.length === 2 ? a - b : 0; +} +console.log([1, 2].map(scaled).join(), scaled(3)); +console.log([5, 6].filter(scaled).length, [1, 2].reduce(total, 0), total(1, 2)); +console.log([3, 1, 2].sort(compare).join()); + +function apply(f: (a: number, b: number, c: number) => number): number { + return f(1, 2, 3); +} +console.log(apply(scaled)); + +// A direct call inside a call through the value sees its own count. +function depth(n: number): number { + return n <= 0 ? arguments.length : depth(n - 1) + arguments.length; +} +console.log([2].map(depth).join(), depth(2)); diff --git a/tests/diagnostics/function-value-arguments-length.ts b/tests/diagnostics/function-value-arguments-length.ts new file mode 100644 index 000000000..ab0b53f82 --- /dev/null +++ b/tests/diagnostics/function-value-arguments-length.ts @@ -0,0 +1,36 @@ +// `arguments.length` has no single value when the calls pass varying +// counts: a function expression or declaration passed where the expected +// function type has optional or rest parameters. Methods and nested +// functions used as values, and arrows inside a function used as a value, +// keep the fence. +const replaced = "a-b".replace(/-/, function (match) { + return String(arguments.length) + match; +}); +function call(f: (a: number, b?: number) => number): number { + return f(1, 2); +} +const counted = call(function (a) { + return arguments.length + a; +}); +function offset(value: number): number { + return value + arguments.length; +} +const offsets = call(offset); +class Scale { + static by(value: number): number { + return value * arguments.length; + } +} +const scaled = [2].map(Scale.by); +function outer(): number[] { + function inner(value: number): number { + return value + arguments.length; + } + return [1].map(inner); +} +function deferred(value: number): number { + const read = (): number => arguments.length; + return value + read(); +} +const late = [1].map(deferred); +export { replaced, counted, offsets, scaled, outer, late }; diff --git a/tests/harness/__snapshots__/function-value-arguments-length.ts.txt b/tests/harness/__snapshots__/function-value-arguments-length.ts.txt new file mode 100644 index 000000000..adcb48d2a --- /dev/null +++ b/tests/harness/__snapshots__/function-value-arguments-length.ts.txt @@ -0,0 +1,43 @@ +function-value-arguments-length.ts:7:17 - error SC1090: 'arguments.length' in function expressions passed where the expected function type has optional or rest parameters (the count varies per call) are not supported yet + + 6 | const replaced = "a-b".replace(/-/, function (match) { + 7 | return String(arguments.length) + match; + | ^~~~~~~~~~~~~~~~ + 8 | }); + +function-value-arguments-length.ts:13:10 - error SC1090: 'arguments.length' in function expressions passed where the expected function type has optional or rest parameters (the count varies per call) are not supported yet + + 12 | const counted = call(function (a) { + 13 | return arguments.length + a; + | ^~~~~~~~~~~~~~~~ + 14 | }); + +function-value-arguments-length.ts:18:22 - error SC1090: functions that read 'arguments.length' used as values where the expected function type has optional or rest parameters (the count varies per call) are not supported yet + + 17 | } + 18 | const offsets = call(offset); + | ^~~~~~ + 19 | class Scale { + +function-value-arguments-length.ts:21:20 - error SC1090: 'arguments.length' in methods, nested, async and generator functions used as values (a caller of the value can pass more arguments than the function declares) are not supported yet + + 20 | static by(value: number): number { + 21 | return value * arguments.length; + | ^~~~~~~~~~~~~~~~ + 22 | } + +function-value-arguments-length.ts:32:30 - error SC1090: 'arguments.length' in an arrow function inside a function that is also used as a value is not supported yet + + 31 | function deferred(value: number): number { + 32 | const read = (): number => arguments.length; + | ^~~~~~~~~~~~~~~~ + 33 | return value + read(); + +function-value-arguments-length.ts:33:18 - error SC2004: uses of 'read' inherit the blocker on its declaration + + 32 | const read = (): number => arguments.length; + 33 | return value + read(); + | ^~~~ + 34 | } + + hint: the declaration of 'read' did not compile — fix the diagnostic reported there and these sites clear with it \ No newline at end of file diff --git a/tests/harness/corpus-costs/7381-function-value-arguments-length.ts.json b/tests/harness/corpus-costs/7381-function-value-arguments-length.ts.json new file mode 100644 index 000000000..b4659b9a0 --- /dev/null +++ b/tests/harness/corpus-costs/7381-function-value-arguments-length.ts.json @@ -0,0 +1 @@ +{ "plain": 3400, "san": 3300 } From d99fc851e66ffc3eadb95b30b226dbdcce5745c0 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 01:22:42 -0500 Subject: [PATCH 02/17] Type untyped JavaScript Maps, Sets and arrays from their uses In a JavaScript file, `const m = new Map()`, `new Set()` and `[]` type as Map, Set and any[], so every use lowered to the checked-dynamic collection operations: boxed keys and values, generic method dispatch, and dynamic arithmetic on what they returned. js-collection-types.ts proves types for such bindings from their uses. A `const` qualifies when every reference is a direct use of the collection: - for a Map: get/has/delete/clear, size, and set as a statement; - for a Set: has/delete/clear, size, and add as a statement; - for an array: push/pop/shift/at/includes/indexOf/join and length. The value then never leaves the binding. Its key, value or element type is the one primitive (string, number or boolean) of every expression the program writes. That is found by a fixed point that types `m.get(k) ?? d`, arithmetic, comparisons and constants defined from them, and it requires lookups of the same type. Any other reference (passing, returning, storing, exporting, iterating, inspecting, an optional call), a direct `eval` in the file, or any doubt keeps the binding untyped. A proven binding answers the lowerer's typeOf with a carrier type declared in scriptc-overrides.d.ts, which only the lowering program includes. `new Map()` and `new Set()` adopt it, so the binding lowers as the typed Map, Set or T[], as a JSDoc @type would make it. 7985 covers the typed cases and every refusal. --- .../compiler/ambient/scriptc-overrides.d.ts | 21 + .../frontend/lowering/js-collection-types.ts | 519 ++++++++++++++++++ .../src/frontend/lowering/lower-classes.ts | 10 +- .../compiler/src/frontend/lowering/lowerer.ts | 8 + .../compiler/test/js-collection-types.test.ts | 62 +++ .../7985-untyped-collection-types.mjs.json | 6 + .../corpus/7985-untyped-collection-types.mjs | 116 ++++ .../7985-untyped-collection-types.mjs.json | 1 + 8 files changed, 741 insertions(+), 2 deletions(-) create mode 100644 packages/compiler/src/frontend/lowering/js-collection-types.ts create mode 100644 packages/compiler/test/js-collection-types.test.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7985-untyped-collection-types.mjs.json create mode 100644 tests/corpus/7985-untyped-collection-types.mjs create mode 100644 tests/harness/corpus-costs/7985-untyped-collection-types.mjs.json diff --git a/packages/compiler/ambient/scriptc-overrides.d.ts b/packages/compiler/ambient/scriptc-overrides.d.ts index 2dafbcdf0..376eba9d5 100644 --- a/packages/compiler/ambient/scriptc-overrides.d.ts +++ b/packages/compiler/ambient/scriptc-overrides.d.ts @@ -70,3 +70,24 @@ interface PromiseConstructor { reject: (reason: Error) => void; }; } + +/* Carriers of the types the lowerer gives untyped JavaScript collections + * whose uses prove them (js-collection-types.ts). Nothing reads them at run + * time. */ +declare namespace __scriptcInferredCollections { + const map_string_string: Map; + const map_string_number: Map; + const map_string_boolean: Map; + const map_number_string: Map; + const map_number_number: Map; + const map_number_boolean: Map; + const map_boolean_string: Map; + const map_boolean_number: Map; + const map_boolean_boolean: Map; + const set_string: Set; + const set_number: Set; + const set_boolean: Set; + const array_string: string[]; + const array_number: number[]; + const array_boolean: boolean[]; +} diff --git a/packages/compiler/src/frontend/lowering/js-collection-types.ts b/packages/compiler/src/frontend/lowering/js-collection-types.ts new file mode 100644 index 000000000..2be3c067f --- /dev/null +++ b/packages/compiler/src/frontend/lowering/js-collection-types.ts @@ -0,0 +1,519 @@ +/* Types of untyped JavaScript collections that their uses prove. + * + * In a JavaScript file, `const m = new Map()`, `new Set()` and `[]` type as + * Map, Set and any[] to the checker, so every use lowers to + * the checked-dynamic collection operations. When every reference of such a + * `const` is a direct use of the collection, the value never leaves the + * binding. If its keys, values or elements are then each one primitive type + * wherever the program writes and looks them up, the binding lowers as the + * typed Map, Set or T[], as a JSDoc @type would make it: + * + * - Map: `m.get(k)`, `m.has(k)`, `m.delete(k)`, `m.clear()`, `m.size`, and + * `m.set(k, v)` as a statement (its result is the map). + * - Set: `s.add(k)` as a statement, `s.has(k)`, `s.delete(k)`, `s.clear()`, + * `s.size`. + * - Array: `a.push(...)`, `a.pop()`, `a.shift()`, `a.at(i)`, `a.includes(v)`, + * `a.indexOf(v)`, `a.join(separator)`, `a.length`. + * + * Any other reference (passing, returning, storing, exporting, iterating, + * spreading, inspecting the collection, an optional call), a direct `eval` + * in the file, or any doubt about a type keeps the binding untyped. + * + * The types come from carrier declarations in scriptc-overrides.d.ts, which + * only the lowering program includes. */ + +import * as ts from "../ts7/adapter.js"; +import { isJsSourceFile } from "../program.js"; +import { overridesDtsPath } from "../dts-paths.js"; +import type { Lowerer } from "./lowerer.js"; + +type Primitive = "string" | "number" | "boolean"; +const PRIMITIVES: readonly Primitive[] = ["string", "number", "boolean"]; + +type Collection = "map" | "set" | "array"; + +interface Candidate { + readonly kind: Collection; + readonly declaration: ts.VariableDeclaration; + /** Every identifier that names the binding, its declaration's included. */ + readonly names: ts.Identifier[]; + /** Map keys or Set members written by set/add. */ + readonly keyWrites: ts.Expression[]; + /** Map values written by set. Array elements written by push. */ + readonly valueWrites: ts.Expression[]; + /** Arguments compared with keys or elements: get/has/delete, includes/indexOf. */ + readonly lookups: ts.Expression[]; + /** Array indexes passed to at(). */ + readonly indexes: ts.Expression[]; + /** Separators passed to join(). */ + readonly separators: ts.Expression[]; + key: Primitive | undefined; + value: Primitive | undefined; +} + +const MAP_LOOKUPS = new Set(["get", "has", "delete"]); +const SET_LOOKUPS = new Set(["has", "delete"]); +const ARRAY_READS = new Set(["pop", "shift"]); +const ARRAY_SEARCHES = new Set(["includes", "indexOf"]); + +/** The inferred type of the binding a `new Map()` or `new Set()` initializes, + * which the construction adopts. */ +export function initializedCollectionType( + lowerer: Lowerer, + expr: ts.NewExpression, +): ts.Type | null { + const declaration = expr.parent; + return declaration !== undefined && + ts.isVariableDeclaration(declaration) && + declaration.initializer === expr && + ts.isIdentifier(declaration.name) + ? inferredCollectionType(lowerer, declaration.name) + : null; +} + +/** The inferred collection type of an identifier that names a proven + * binding, or null. Computed for every JavaScript file of the program on + * first use. */ +export function inferredCollectionType(lowerer: Lowerer, node: ts.Identifier): ts.Type | null { + const types = (lowerer.inferredCollectionTypes ??= inferProgram(lowerer)); + return types.size === 0 ? null : (types.get(node) ?? null); +} + +function inferProgram(lowerer: Lowerer): Map { + const out = new Map(); + const carriers = carrierTypes(lowerer); + if (carriers === null) return out; + for (const file of lowerer.program.getSourceFiles()) { + if (file.isDeclarationFile || !isJsSourceFile(file)) continue; + for (const candidate of inferFile(lowerer, file)) { + const name = + candidate.kind === "map" + ? `map_${candidate.key}_${candidate.value}` + : candidate.kind === "set" + ? `set_${candidate.key}` + : `array_${candidate.value}`; + const type = carriers.get(name); + if (type === undefined) continue; + for (const id of candidate.names) out.set(id, type); + } + } + return out; +} + +/** The carrier declarations' types, by name (`map_string_number`, + * `set_number`, `array_string`, ...), or null when the lowering program does + * not include them. */ +function carrierTypes(lowerer: Lowerer): Map | null { + const file = lowerer.program.getSourceFile(overridesDtsPath()); + if (file === undefined) return null; + const out = new Map(); + for (const statement of file.statements.nodes) { + if ( + !ts.isModuleDeclaration(statement) || + !ts.isIdentifier(statement.name) || + statement.name.text !== "__scriptcInferredCollections" || + statement.body === undefined || + !ts.isModuleBlock(statement.body) + ) + continue; + for (const member of statement.body.statements.nodes) { + if (!ts.isVariableStatement(member)) continue; + for (const declaration of member.declarationList.declarations.nodes) + if (ts.isIdentifier(declaration.name)) + out.set(declaration.name.text, lowerer.checker.getTypeAtLocation(declaration.name)); + } + } + return out.size > 0 ? out : null; +} + +function inferFile(lowerer: Lowerer, file: ts.SourceFile): Candidate[] { + const candidates = new Map(); + let directEval = false; + const declarations = (node: ts.Node): void => { + if ( + ts.isCallExpression(node) && + ts.isIdentifier(node.expression) && + node.expression.text === "eval" + ) + directEval = true; + if (ts.isVariableDeclaration(node)) { + const kind = candidateKind(lowerer, node); + const symbol = kind !== null ? lowerer.checker.getSymbolAtLocation(node.name) : undefined; + if (kind !== null && symbol !== undefined) + candidates.set(symbol, { + kind, + declaration: node, + names: [node.name as ts.Identifier], + keyWrites: [], + valueWrites: [], + lookups: [], + indexes: [], + separators: [], + key: undefined, + value: undefined, + }); + } + ts.forEachChild(node, declarations); + }; + declarations(file); + if (directEval || candidates.size === 0) return []; + + const names = new Set( + [...candidates.values()].map((c) => (c.declaration.name as ts.Identifier).text), + ); + const refused = new Set(); + const references = (node: ts.Node): void => { + if (ts.isIdentifier(node) && names.has(node.text)) { + const symbol = referencedSymbol(lowerer, node); + const candidate = symbol !== undefined ? candidates.get(symbol) : undefined; + if (symbol !== undefined && candidate !== undefined && node !== candidate.declaration.name) { + if (recordUse(candidate, node)) candidate.names.push(node); + else refused.add(symbol); + } + } + ts.forEachChild(node, references); + }; + references(file); + for (const symbol of refused) candidates.delete(symbol); + const proven = [...candidates.values()].filter( + (c) => c.valueWrites.length > 0 || c.keyWrites.length > 0, + ); + solve(lowerer, proven); + return proven.filter((c) => + c.kind === "map" ? c.key && c.value : c.kind === "set" ? c.key : c.value, + ); +} + +/** The collection a `const` declaration starts empty, when the checker + * types it as an untyped Map, Set or array. */ +function candidateKind(lowerer: Lowerer, declaration: ts.VariableDeclaration): Collection | null { + if (!ts.isIdentifier(declaration.name) || declaration.initializer === undefined) return null; + const list = declaration.parent; + if (!ts.isVariableDeclarationList(list) || (list.flags & ts.NodeFlags.Const) === 0) return null; + const statement = list.parent; + if ( + ts.isVariableStatement(statement) && + ts.getModifiers(statement)?.some((m) => m.kind === ts.SyntaxKind.ExportKeyword) + ) + return null; + const init = declaration.initializer; + let kind: Collection | null = null; + if (ts.isArrayLiteralExpression(init) && init.elements.nodes.length === 0) kind = "array"; + else if ( + ts.isNewExpression(init) && + init.typeArguments === undefined && + (init.arguments?.nodes.length ?? 0) === 0 + ) { + if (lowerer.isStdlibGlobal(init.expression, "Map")) kind = "map"; + else if (lowerer.isStdlibGlobal(init.expression, "Set")) kind = "set"; + } + if (kind === null) return null; + // A JSDoc type, or anything else the checker already knows, stands. + const type = lowerer.checker.getTypeAtLocation(declaration.name); + const args = + kind === "array" + ? lowerer.checker.isArrayType(type) + ? lowerer.checker.getTypeArguments(type as ts.TypeReference) + : [] + : lowerer.checker.getTypeArguments(type as ts.TypeReference); + const arity = kind === "map" ? 2 : 1; + if (args.length !== arity || !args.every((a) => (a.flags & ts.TypeFlags.Any) !== 0)) return null; + return kind; +} + +function referencedSymbol(lowerer: Lowerer, id: ts.Identifier): ts.Symbol | undefined { + const parent = id.parent; + if (parent !== undefined && ts.isShorthandPropertyAssignment(parent) && parent.name === id) + return lowerer.checker.getShorthandAssignmentValueSymbol(parent); + return lowerer.checker.getSymbolAtLocation(id); +} + +/** Records a reference's use, or answers false when the reference is not + * one of the collection uses this inference proves. */ +function recordUse(candidate: Candidate, id: ts.Identifier): boolean { + const access = id.parent; + if ( + access === undefined || + !ts.isPropertyAccessExpression(access) || + access.expression !== id || + access.questionDotToken !== undefined + ) + return false; + const name = access.name.text; + const call = access.parent; + if (call !== undefined && ts.isCallExpression(call) && call.expression === access) { + if (call.questionDotToken !== undefined) return false; + const args = call.arguments.nodes; + if (args.some(ts.isSpreadElement)) return false; + switch (candidate.kind) { + case "map": + if (MAP_LOOKUPS.has(name) && args.length === 1) + return (candidate.lookups.push(args[0]!), true); + if (name === "clear") return args.length === 0; + if (name === "set" && args.length === 2 && isStatement(call)) { + candidate.keyWrites.push(args[0]!); + candidate.valueWrites.push(args[1]!); + return true; + } + return false; + case "set": + if (SET_LOOKUPS.has(name) && args.length === 1) + return (candidate.lookups.push(args[0]!), true); + if (name === "clear") return args.length === 0; + if (name === "add" && args.length === 1 && isStatement(call)) + return (candidate.keyWrites.push(args[0]!), true); + return false; + case "array": + if (name === "push") return (candidate.valueWrites.push(...args), true); + if (ARRAY_READS.has(name)) return args.length === 0; + if (name === "at" && args.length === 1) return (candidate.indexes.push(args[0]!), true); + if (ARRAY_SEARCHES.has(name) && args.length === 1) + return (candidate.lookups.push(args[0]!), true); + if (name === "join" && args.length <= 1) return (candidate.separators.push(...args), true); + return false; + } + } + return (candidate.kind === "array" ? name === "length" : name === "size") && isPlainRead(access); +} + +function isStatement(call: ts.CallExpression): boolean { + let node: ts.Node = call; + while (node.parent !== undefined && ts.isParenthesizedExpression(node.parent)) node = node.parent; + return node.parent !== undefined && ts.isExpressionStatement(node.parent); +} + +/** A read that neither writes nor deletes the property. */ +function isPlainRead(access: ts.PropertyAccessExpression): boolean { + let node: ts.Node = access; + while (node.parent !== undefined && ts.isParenthesizedExpression(node.parent)) node = node.parent; + const parent = node.parent; + if (parent === undefined) return false; + if ( + ts.isBinaryExpression(parent) && + parent.left === node && + isAssignment(parent.operatorToken.kind) + ) + return false; + if ( + (ts.isPrefixUnaryExpression(parent) || ts.isPostfixUnaryExpression(parent)) && + (parent.operator === ts.SyntaxKind.PlusPlusToken || + parent.operator === ts.SyntaxKind.MinusMinusToken) + ) + return false; + return !ts.isDeleteExpression(parent); +} + +function isAssignment(kind: ts.SyntaxKind): boolean { + return kind >= ts.SyntaxKind.FirstAssignment && kind <= ts.SyntaxKind.LastAssignment; +} + +/** Decides each candidate's key and value types: a fixed point in which a + * decided candidate's reads type the expressions built from them. */ +function solve(lowerer: Lowerer, candidates: Candidate[]): void { + const bySymbol = new Map(); + for (const c of candidates) { + const symbol = lowerer.checker.getSymbolAtLocation(c.declaration.name); + if (symbol) bySymbol.set(symbol, c); + } + for (let changed = true; changed;) { + changed = false; + for (const c of candidates) { + if (c.kind !== "array" && c.key === undefined) { + const key = uniform(lowerer, bySymbol, c.keyWrites); + if (key !== null) { + c.key = key; + changed = true; + } + } + if (c.kind !== "set" && c.value === undefined) { + // Each candidate type is assumed for this collection's own reads. + const accepted = PRIMITIVES.filter((assumed) => { + c.value = assumed; + const found = uniform(lowerer, bySymbol, c.valueWrites); + c.value = undefined; + return found === assumed; + }); + if (accepted.length === 1) { + c.value = accepted[0]; + changed = true; + } + } + } + } + for (const c of candidates) { + const element = c.kind === "array" ? c.value : c.key; + // Lookups compare with the stored type; an index is a number. + const lookupsAgree = + element !== undefined && + c.lookups.every((e) => primitiveOf(lowerer, bySymbol, e, new Set()) === element) && + c.indexes.every((e) => primitiveOf(lowerer, bySymbol, e, new Set()) === "number") && + c.separators.every((e) => primitiveOf(lowerer, bySymbol, e, new Set()) === "string"); + if (!lookupsAgree) { + c.key = undefined; + c.value = undefined; + } + } +} + +/** The one primitive type of every expression, or null. */ +function uniform( + lowerer: Lowerer, + bySymbol: Map, + exprs: readonly ts.Expression[], +): Primitive | null { + let found: Primitive | null = null; + for (const e of exprs) { + const p = primitiveOf(lowerer, bySymbol, e, new Set()); + if (p === null || (found !== null && p !== found)) return null; + found = p; + } + return found; +} + +function checkerPrimitive(type: ts.Type): Primitive | null { + const flags = type.flags; + if (flags & (ts.TypeFlags.String | ts.TypeFlags.StringLiteral)) return "string"; + if (flags & (ts.TypeFlags.Number | ts.TypeFlags.NumberLiteral)) return "number"; + if (flags & (ts.TypeFlags.Boolean | ts.TypeFlags.BooleanLiteral)) return "boolean"; + return null; +} + +/** The primitive type `e` always evaluates to, or null. The checker answers + * where it knows; where it types `e` as `any`, the value's structure does: + * a read of a decided candidate, arithmetic and comparisons over such + * values, and constants defined from them. */ +function primitiveOf( + lowerer: Lowerer, + bySymbol: Map, + e: ts.Expression, + visiting: Set, +): Primitive | null { + const type = lowerer.checker.getTypeAtLocation(e); + if ((type.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown)) === 0) return checkerPrimitive(type); + const of = (x: ts.Expression): Primitive | null => primitiveOf(lowerer, bySymbol, x, visiting); + if (ts.isParenthesizedExpression(e)) return of(e.expression); + if (ts.isPrefixUnaryExpression(e)) { + if (e.operator === ts.SyntaxKind.ExclamationToken) return "boolean"; + return e.operator !== ts.SyntaxKind.PlusPlusToken && + e.operator !== ts.SyntaxKind.MinusMinusToken && + of(e.operand) === "number" + ? "number" + : null; + } + if (ts.isConditionalExpression(e)) { + const yes = of(e.whenTrue); + return yes !== null && yes === of(e.whenFalse) ? yes : null; + } + if (ts.isBinaryExpression(e)) { + const op = e.operatorToken.kind; + switch (op) { + case ts.SyntaxKind.LessThanToken: + case ts.SyntaxKind.GreaterThanToken: + case ts.SyntaxKind.LessThanEqualsToken: + case ts.SyntaxKind.GreaterThanEqualsToken: + case ts.SyntaxKind.EqualsEqualsEqualsToken: + case ts.SyntaxKind.ExclamationEqualsEqualsToken: + case ts.SyntaxKind.EqualsEqualsToken: + case ts.SyntaxKind.ExclamationEqualsToken: + return "boolean"; + case ts.SyntaxKind.QuestionQuestionToken: + case ts.SyntaxKind.BarBarToken: { + // `m.get(k) ?? d` and `a.pop() || d`: the stored type or a default of it. + const stored = storedRead(lowerer, bySymbol, e.left); + const fallback = of(e.right); + return stored !== null && stored === fallback ? stored : null; + } + case ts.SyntaxKind.PlusToken: { + const left = of(e.left); + const right = of(e.right); + if (left === "number" && right === "number") return "number"; + return left !== null && right !== null && (left === "string" || right === "string") + ? "string" + : null; + } + case ts.SyntaxKind.MinusToken: + case ts.SyntaxKind.AsteriskToken: + case ts.SyntaxKind.SlashToken: + case ts.SyntaxKind.PercentToken: + case ts.SyntaxKind.AsteriskAsteriskToken: + case ts.SyntaxKind.BarToken: + case ts.SyntaxKind.AmpersandToken: + case ts.SyntaxKind.CaretToken: + case ts.SyntaxKind.LessThanLessThanToken: + case ts.SyntaxKind.GreaterThanGreaterThanToken: + case ts.SyntaxKind.GreaterThanGreaterThanGreaterThanToken: + return of(e.left) === "number" && of(e.right) === "number" ? "number" : null; + default: + return null; + } + } + if (ts.isCallExpression(e) || ts.isPropertyAccessExpression(e)) + return candidateRead(lowerer, bySymbol, e); + // A constant defined from such an expression. + if (ts.isIdentifier(e) && !visiting.has(e)) { + const symbol = lowerer.checker.getSymbolAtLocation(e); + const declaration = symbol?.valueDeclaration; + if ( + declaration !== undefined && + ts.isVariableDeclaration(declaration) && + declaration.initializer !== undefined && + ts.isVariableDeclarationList(declaration.parent) && + (declaration.parent.flags & ts.NodeFlags.Const) !== 0 + ) { + visiting.add(e); + const p = primitiveOf(lowerer, bySymbol, declaration.initializer, visiting); + visiting.delete(e); + return p; + } + } + return null; +} + +/** The candidate a use reads, when `e` is `c.member` or `c.member(...)` of a + * candidate `c`. */ +function candidateOf( + lowerer: Lowerer, + bySymbol: Map, + e: ts.Expression, +): { candidate: Candidate; member: string } | null { + const access = ts.isCallExpression(e) ? e.expression : e; + if (!ts.isPropertyAccessExpression(access) || !ts.isIdentifier(access.expression)) return null; + const symbol = lowerer.checker.getSymbolAtLocation(access.expression); + const candidate = symbol !== undefined ? bySymbol.get(symbol) : undefined; + return candidate !== undefined ? { candidate, member: access.name.text } : null; +} + +/** The type of a candidate's read that may answer undefined: `m.get(k)`, + * `a.pop()`, `a.shift()`, `a.at(i)`. */ +function storedRead( + lowerer: Lowerer, + bySymbol: Map, + e: ts.Expression, +): Primitive | null { + while (ts.isParenthesizedExpression(e)) e = e.expression; + if (!ts.isCallExpression(e)) return null; + const read = candidateOf(lowerer, bySymbol, e); + if (read === null) return null; + const { candidate, member } = read; + if (candidate.kind === "map" && member === "get") return candidate.value ?? null; + if (candidate.kind === "array" && (ARRAY_READS.has(member) || member === "at")) + return candidate.value ?? null; + return null; +} + +/** The type of a candidate's read that always answers a value: size, + * length, has, includes, indexOf, push. */ +function candidateRead( + lowerer: Lowerer, + bySymbol: Map, + e: ts.Expression, +): Primitive | null { + const read = candidateOf(lowerer, bySymbol, e); + if (read === null) return null; + const { candidate, member } = read; + if (!ts.isCallExpression(e)) return member === "size" || member === "length" ? "number" : null; + if (member === "has" || member === "includes" || member === "delete") return "boolean"; + if (member === "indexOf" || (candidate.kind === "array" && member === "push")) return "number"; + if (candidate.kind === "array" && member === "join") return "string"; + return null; +} diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index b829b6ddc..c4dd6bce1 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -113,6 +113,7 @@ import { PoisonError, dynFallbackType, jsFuncNameOf, newFnCtx, own } from "./low import { isAssertClassNew } from "./assert-instances.js"; import { isCjsRequireGlobal } from "./lower-node-module.js"; import { lowerArrayConstructor } from "./containers/array-construction.js"; +import { initializedCollectionType } from "./js-collection-types.js"; import { lowerMapSeedNew, lowerSetSeedNew } from "./containers/collection-methods.js"; import { lowerObjectConstructor } from "./containers/indexed-objects.js"; import { knownBufEncoding, lowerBufferConstruct } from "./containers/bytes.js"; @@ -10568,7 +10569,11 @@ export function lowerNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr { // expression). The CONTEXTUAL type carries the annotation — adopt // it when it is a supported map. TS type arguments keep winning: // their expression type already maps. - if (mapped?.kind !== "map" || contextualSeed) { + const inferred = initializedCollectionType(lowerer, expr); + if (inferred) { + tsType = inferred; + mapped = lowerer.mapTypeOf(inferred); + } else if (mapped?.kind !== "map" || contextualSeed) { const ctx = lowerer.checker.getContextualType(expr); const ctxMapped = ctx ? lowerer.mapTypeOf(ctx) : null; if (ctx && ctxMapped?.kind === "map") { @@ -10708,7 +10713,8 @@ export function lowerNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr { // `new Set()`: construct + bulk add from supported values, // including live collection cursors. Duplicates collapse in insertion order. if (ts.isIdentifier(identifier) && lowerer.isStdlibGlobal(identifier, "Set")) { - const tsType = lowerer.typeOf(expr); + const inferred = initializedCollectionType(lowerer, expr); + const tsType = inferred ?? lowerer.typeOf(expr); let mapped = lowerer.mapTypeOf(tsType); const fieldType = assignedThisFieldType(lowerer, expr); if (mapped?.kind !== "set" && fieldType?.kind === "set") mapped = fieldType; diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index af89c61a4..78d1fafad 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -156,6 +156,7 @@ import { trackClassFieldCreation, } from "./class-dynamic-dispatch.js"; import { guardClassAttributeWrites } from "./class-attributes.js"; +import { inferredCollectionType } from "./js-collection-types.js"; import { guardFrozenWrites, markSloppyRecordWrites } from "./frozen-records.js"; import { applyHandleProperties } from "./handle-properties.js"; import { installBufferValue } from "./containers/bytes.js"; @@ -2846,6 +2847,9 @@ export class Lowerer { /** Parameters read as `any` in the error-path copy of an error-first * callback. */ readonly anySymbols = new Set(); + /** The collection types that uses prove for untyped JavaScript bindings, + * by every identifier naming one (js-collection-types.ts). */ + inferredCollectionTypes: Map | undefined = undefined; /** overload-choice's answers per call, binding and function. */ readonly dynamicOverloadAnswers = new Map(); /** Error parameters of an error-first callback with a known value in @@ -9055,6 +9059,10 @@ export class Lowerer { } typeOf(node: ts.Node): ts.Type { + if (ts.isIdentifier(node)) { + const inferred = inferredCollectionType(this, node); + if (inferred) return inferred; + } if (this.anySymbols.size > 0 && this.readsAnySymbol(node)) return this.checker.getAnyType(); // A checked-dynamic argument that another overload could take at run // time leaves the resolved overload's return a guess, and JS bindings diff --git a/packages/compiler/test/js-collection-types.test.ts b/packages/compiler/test/js-collection-types.test.ts new file mode 100644 index 000000000..88e2006b7 --- /dev/null +++ b/packages/compiler/test/js-collection-types.test.ts @@ -0,0 +1,62 @@ +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile, deserializeModule, validateModule } from "../src/index.js"; + +// Corpus 7985 exercises every refusal: only the collections whose uses +// prove their types lower as typed collections; the rest stay untyped. +test("untyped JavaScript collections take the types their uses prove, and only those", async () => { + const dir = mkdtempSync(join(tmpdir(), "scriptc-js-collection-types-")); + try { + const entry = join( + import.meta.dirname, + "../../../tests/corpus/7985-untyped-collection-types.mjs", + ); + const output = join(dir, "main.ir.json"); + const lowered = await compile(entry, { outputKind: "ir", outDir: dir, outPath: output }); + if (!lowered.ok) throw new Error(JSON.stringify(lowered.diagnostics)); + const module = deserializeModule(readFileSync(output, "utf8")); + expect(validateModule(module)).toEqual([]); + const types = new Map(module.globals.map((g) => [g.name, JSON.stringify(g.type)])); + const map = (key: string, value: string) => + JSON.stringify({ kind: "map", key: { kind: key }, value: { kind: value } }); + const set = (elem: string) => JSON.stringify({ kind: "set", elem: { kind: elem } }); + const array = (elem: string) => JSON.stringify({ kind: "array", elem: { kind: elem } }); + // Untyped collections hold checked-dynamic keys, values and elements. + const typed = (t: string) => !t.includes('"dyn"'); + expect(Object.fromEntries([...types].filter(([, t]) => typed(t)))).toEqual({ + counts: map("string", "f64"), + seen: set("f64"), + stack: array("f64"), + words: array("string"), + byNumber: map("f64", "string"), + nums: array("f64"), + flags: set("bool"), + cache: map("string", "f64"), + "%loaded": '{"kind":"bool"}', + }); + for (const untyped of [ + "strings", + "numbers", + "mixedKeys", + "mixedValues", + "mixedElements", + "passed", + "stored", + "iterated", + "eachSet", + "shown", + "chained", + "optional", + "indexed", + "truncated", + "spread", + "unread", + "exported", + ]) + expect(typed(types.get(untyped)!), untyped).toBe(false); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7985-untyped-collection-types.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7985-untyped-collection-types.mjs.json new file mode 100644 index 000000000..a807063cc --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7985-untyped-collection-types.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7985-untyped-collection-types.mjs" + ], + "diags": [] +} diff --git a/tests/corpus/7985-untyped-collection-types.mjs b/tests/corpus/7985-untyped-collection-types.mjs new file mode 100644 index 000000000..a60a2ea04 --- /dev/null +++ b/tests/corpus/7985-untyped-collection-types.mjs @@ -0,0 +1,116 @@ +// Untyped Maps, Sets and arrays whose uses prove their key, value and +// element types compile as typed collections; any other use keeps them +// untyped. Both must answer as Node does, including SameValueZero keys, +// lookups of another type, and every use that refuses the typing. + +// Typed: counters, membership sets and stacks used only through their +// methods. +const counts = new Map(); +const seen = new Set(); +const stack = []; +const words = []; +for (let i = 0; i < 40; i++) { + const key = "k" + (i % 7); + counts.set(key, (counts.get(key) ?? 0) + 1); + seen.add(i % 5); + stack.push(i * 2); + if (stack.length > 6) stack.shift(); + words.push(key); +} +console.log(counts.size, counts.get("k3"), counts.get("missing"), counts.has("k6"), counts.delete("k0"), counts.size); +console.log(seen.size, seen.has(4), seen.has(5), seen.delete(4), seen.has(4)); +console.log(stack.length, stack.at(-1), stack.at(0), stack.pop(), stack.includes(70), stack.indexOf(72), words.join("|").length); + +// SameValueZero keys and NaN elements on typed collections. +const byNumber = new Map(); +byNumber.set(NaN, "nan"); +byNumber.set(-0, "zero"); +byNumber.set(1.5, "x"); +console.log(byNumber.get(NaN), byNumber.get(0), byNumber.get(-0), byNumber.size, byNumber.has(1.5)); +const nums = []; +nums.push(NaN); +nums.push(-0); +console.log(nums.includes(NaN), nums.indexOf(NaN), nums.includes(0), nums.indexOf(0)); +const flags = new Set(); +flags.add(true); +console.log(flags.has(true), flags.has(false), flags.size); + +// Captured by a closure that only uses it. +const cache = new Map(); +const latest = () => cache.get("c") ?? -1; +console.log(latest()); +for (let i = 0; i < 3; i++) cache.set("c", i * 10); +console.log(latest(), cache.size); + +// Lookups of another type: the typing refuses, and the answers stay Node's. +const strings = new Map(); +strings.set("1", "one"); +console.log(strings.get(1), strings.get("1"), strings.has(1)); +const numbers = []; +numbers.push(1); +console.log(numbers.includes("1"), numbers.indexOf("1"), numbers.at("0")); + +// Mixed key, value and element types. +const mixedKeys = new Map(); +mixedKeys.set(1, "number"); +mixedKeys.set("1", "string"); +console.log(mixedKeys.size, mixedKeys.get(1), mixedKeys.get("1")); +const mixedValues = new Map(); +mixedValues.set("a", 1); +mixedValues.set("b", "two"); +console.log(mixedValues.get("a"), mixedValues.get("b")); +const mixedElements = []; +mixedElements.push(1); +mixedElements.push("two"); +console.log(mixedElements.join(","), mixedElements.length); + +// Escapes and other uses that keep the collection untyped. +const passed = new Map(); +passed.set("x", 1); +function sizeOf(m) { + return m.size; +} +console.log(sizeOf(passed)); +function make() { + const made = new Set(); + made.add("y"); + return made; +} +console.log(make().has("y")); +const stored = []; +stored.push(3); +const holder = { list: stored }; +holder.list.push(4); +console.log(stored.length, holder.list.join("+")); +const iterated = new Map(); +iterated.set("a", 1); +iterated.set("b", 2); +for (const [k, v] of iterated) console.log("entry", k, v); +const eachSet = new Set(); +eachSet.add(3); +eachSet.forEach((v) => console.log("member", v)); +const shown = new Map(); +shown.set("s", 1); +console.log(shown); +const chained = new Map(); +chained.set("a", 1).set("b", 2); +console.log(chained.size); +const optional = new Map(); +optional.set("o", 1); +console.log(optional?.get("o"), optional.get?.("o")); +const indexed = []; +indexed.push(1); +indexed[3] = 2; +console.log(indexed.length, indexed.join(",")); +const truncated = []; +truncated.push(1, 2, 3); +truncated.length = 1; +console.log(truncated.length, truncated.at(0)); +const spread = []; +spread.push(5); +console.log(Math.max(...spread)); +const unread = new Map(); +console.log(unread.size, unread.get("none")); +export const exported = new Map(); +exported.set("e", 1); +console.log(exported.get("e")); diff --git a/tests/harness/corpus-costs/7985-untyped-collection-types.mjs.json b/tests/harness/corpus-costs/7985-untyped-collection-types.mjs.json new file mode 100644 index 000000000..6915a85fe --- /dev/null +++ b/tests/harness/corpus-costs/7985-untyped-collection-types.mjs.json @@ -0,0 +1 @@ +{ "plain": 2400, "san": 2600 } From 854f3ae8103d5573f9c39e829f850b8a7f76661f Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 06:41:00 -0500 Subject: [PATCH 03/17] Construct readline.Interface and builtin classes bound through require() members in JavaScript --- .../lowering/builtins/module-bindings.ts | 26 ++++++++++ .../frontend/lowering/builtins/readline.ts | 36 ++++++++----- .../src/frontend/lowering/lower-classes.ts | 16 ++++-- packages/compiler/src/frontend/tsc-codes.ts | 6 +++ ...30-readline-interface-constructor.cjs.json | 6 +++ .../8231-builtin-member-binding-new.cjs.json | 6 +++ .../8230-readline-interface-constructor.cjs | 51 +++++++++++++++++++ .../8231-builtin-member-binding-new.cjs | 35 +++++++++++++ ...30-readline-interface-constructor.cjs.json | 1 + .../8231-builtin-member-binding-new.cjs.json | 1 + 10 files changed, 167 insertions(+), 17 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8230-readline-interface-constructor.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8231-builtin-member-binding-new.cjs.json create mode 100644 tests/corpus/8230-readline-interface-constructor.cjs create mode 100644 tests/corpus/8231-builtin-member-binding-new.cjs create mode 100644 tests/harness/corpus-costs/8230-readline-interface-constructor.cjs.json create mode 100644 tests/harness/corpus-costs/8231-builtin-member-binding-new.cjs.json diff --git a/packages/compiler/src/frontend/lowering/builtins/module-bindings.ts b/packages/compiler/src/frontend/lowering/builtins/module-bindings.ts index 5465aeafa..4ded700fe 100644 --- a/packages/compiler/src/frontend/lowering/builtins/module-bindings.ts +++ b/packages/compiler/src/frontend/lowering/builtins/module-bindings.ts @@ -185,6 +185,32 @@ export function builtinMemberRequireDecl( return spec !== null && canonicalBuiltinModule(spec) !== null; } +/** The member's own symbol for a variable bound to a builtin module's + * member (`const SD = require("string_decoder").StringDecoder`, + * builtinMemberRequireDecl) when the member is a class, so the variable + * resolves as a named import of the class does. The symbol itself + * otherwise. */ +export function builtinMemberBindingTarget( + lowerer: Lowerer, + symbol: ts.Symbol | null, +): ts.Symbol | null { + const decl = symbol?.valueDeclaration; + if ( + decl === undefined || + !ts.isVariableDeclaration(decl) || + !builtinMemberRequireDecl(decl.name, decl.initializer) + ) + return symbol; + const init = decl.initializer as ts.PropertyAccessExpression; + let member = lowerer.checker.getSymbolAtLocation(init.name); + if (member && member.flags & ts.SymbolFlags.Alias) + member = lowerer.checker.getAliasedSymbol(member); + // Only a class member: `require("events").EventEmitter` is a static + // property of the exported class, which builtinEmitterInfoOf follows + // from the variable itself. + return member?.declarations.some(ts.isClassDeclaration) ? member : symbol; +} + /** The supported builtin module when `decl` destructures a builtin * NAMESPACE binding (`const crypto = require('crypto'); const { * createSign, sign: mySign } = crypto;` — test/common/crypto.js's diff --git a/packages/compiler/src/frontend/lowering/builtins/readline.ts b/packages/compiler/src/frontend/lowering/builtins/readline.ts index aca209017..c2b6984ce 100644 --- a/packages/compiler/src/frontend/lowering/builtins/readline.ts +++ b/packages/compiler/src/frontend/lowering/builtins/readline.ts @@ -302,6 +302,15 @@ function interfaceClassModule(lowerer: Lowerer, node: ts.Expression): string | n : null; } +/** `new readline.Interface(...)` and `new readlinePromises.Interface(...)`: + * the constructor createInterface calls, with the same arguments. Null for + * other constructors. */ +export function lowerReadlineInterfaceNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr | null { + const module = interfaceClassModule(lowerer, expr.expression); + if (module === null) return null; + return lowerReadlineCreateCall(lowerer, expr, locOf(expr), module === "readline/promises"); +} + /** `rl instanceof readline.Interface` and `rl instanceof * readlinePromises.Interface` over a readline Interface: an interface is * an instance of the class of the module that created it (Node's @@ -1059,10 +1068,12 @@ function lowerOutput(lowerer: Lowerer, node: ts.Expression): IrExpr | "stdout" | * goes through rl.terminal with the completer. */ export function lowerReadlineCreateCall( lowerer: Lowerer, - expr: ts.CallExpression, + expr: ts.CallExpression | ts.NewExpression, loc: SrcLoc, promises: boolean, ): IrExpr { + const argNodes = expr.arguments?.nodes ?? []; + const what = ts.isNewExpression(expr) ? "new Interface" : "createInterface"; let input: IrExpr | null = null; let output: IrExpr | "stdout" | "none" = "none"; let crlf: IrExpr = numLit(0, loc); @@ -1112,39 +1123,38 @@ export function lowerReadlineCreateCall( }); terminal = { kind: "varRef", localId: local.id, type: BOOL, loc }; }; - const optsNode = expr.arguments.nodes[0]; + const optsNode = argNodes[0]; if (optsNode === undefined) { lowerer.noLowering( - "createInterface without arguments", + `${what} without arguments`, expr, "pass { input: process.stdin, output: process.stdout } or createInterface(input, output)", ); } if (!ts.isObjectLiteralExpression(optsNode)) { // createInterface(input, output?, completer?, terminal?) - if (expr.arguments.nodes.length > 4) { + if (argNodes.length > 4) { lowerer.noLowering( - `createInterface with ${expr.arguments.nodes.length} arguments`, - expr.arguments.nodes[4]!, + `${what} with ${argNodes.length} arguments`, + argNodes[4]!, "pass createInterface(input, output, completer, terminal) or one options object", ); } input = lowerInput(lowerer, optsNode); - if (expr.arguments.nodes[1] !== undefined) - output = lowerOutput(lowerer, expr.arguments.nodes[1]); - const completerNode = expr.arguments.nodes[2]; + if (argNodes[1] !== undefined) output = lowerOutput(lowerer, argNodes[1]); + const completerNode = argNodes[2]; if ( completerNode !== undefined && !(ts.isIdentifier(completerNode) && completerNode.text === "undefined") ) { setCompleter(completerNode); } - if (expr.arguments.nodes[3] !== undefined) setTerminal(expr.arguments.nodes[3]); + if (argNodes[3] !== undefined) setTerminal(argNodes[3]); } else { - if (expr.arguments.nodes.length > 1) { + if (argNodes.length > 1) { lowerer.noLowering( - "createInterface with more than one argument", - expr.arguments.nodes[1]!, + `${what} with more than one argument`, + argNodes[1]!, "pass every option in the one options object", ); } diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index b829b6ddc..8bd5e57d7 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -18,8 +18,11 @@ import { superIntoResultSlot, } from "./constructor-returns.js"; import { checkedIterableSpread } from "./checked-iterable-spread.js"; -import { lowerReadlineUiNew } from "./builtins/readline.js"; -import { builtinMemberRequireDecl } from "./builtins/module-bindings.js"; +import { lowerReadlineInterfaceNew, lowerReadlineUiNew } from "./builtins/readline.js"; +import { + builtinMemberBindingTarget, + builtinMemberRequireDecl, +} from "./builtins/module-bindings.js"; import { checkedClassUnionAssertion } from "./class-assertions.js"; import { collectClassMethodMutations } from "./class-method-mutations.js"; import { InternalCompilerError } from "../../errors.js"; @@ -9819,7 +9822,7 @@ export function lowerNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr { } const worker = lowerWorkerNew(lowerer, expr); if (worker) return worker; - const readlineUi = lowerReadlineUiNew(lowerer, expr); + const readlineUi = lowerReadlineUiNew(lowerer, expr) ?? lowerReadlineInterfaceNew(lowerer, expr); if (readlineUi) return readlineUi; const channel = lowerMessageChannelNew(lowerer, expr) ?? lowerBroadcastChannelNew(lowerer, expr); if (channel) return channel; @@ -10132,7 +10135,12 @@ export function lowerNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr { // (a no-op for every non-import= binding). if (ts.isIdentifier(expr.expression)) fenceEarlyAliasUse(lowerer, expr.expression, expr); const identifier = ts.isIdentifier(expr.expression) ? expr.expression : expr.expression.name; - const symbol = ts.isIdentifier(identifier) ? lowerer.resolveValueSymbol(identifier) : null; + // `const SD = require("string_decoder").StringDecoder; new SD()` + // constructs the member, as a named import of it does. + const symbol = builtinMemberBindingTarget( + lowerer, + ts.isIdentifier(identifier) ? lowerer.resolveValueSymbol(identifier) : null, + ); if (ts.isIdentifier(identifier) && lowerer.isStdlibGlobal(identifier, "Array")) { return lowerArrayConstructor(lowerer, expr, expr.arguments?.nodes ?? []); } diff --git a/packages/compiler/src/frontend/tsc-codes.ts b/packages/compiler/src/frontend/tsc-codes.ts index e6c0be7cd..8816615cf 100644 --- a/packages/compiler/src/frontend/tsc-codes.ts +++ b/packages/compiler/src/frontend/tsc-codes.ts @@ -128,6 +128,12 @@ export const JS_RELAXED_TSC_CODES: ReadonlySet = new Set([ // JavaScript subclasses may widen fields or add method parameters. // Class lowering checks native storage and override ABIs at each use. 2415, + // a private or protected constructor (`new readline.Interface(...)`, + // whose declaration is protected): JavaScript has no constructor + // accessibility, and Node constructs the class. Each `new` meets the + // lowerer's per-site constructor fences. + 2673, + 2674, // unresolvable modules: Node throws WHEN the require executes — the // binding types as any and reached uses fence instead of passing unchecked 2307, diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8230-readline-interface-constructor.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8230-readline-interface-constructor.cjs.json new file mode 100644 index 000000000..05573974c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8230-readline-interface-constructor.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8230-readline-interface-constructor.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8231-builtin-member-binding-new.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8231-builtin-member-binding-new.cjs.json new file mode 100644 index 000000000..1f54f4886 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8231-builtin-member-binding-new.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8231-builtin-member-binding-new.cjs" + ], + "diags": [] +} diff --git a/tests/corpus/8230-readline-interface-constructor.cjs b/tests/corpus/8230-readline-interface-constructor.cjs new file mode 100644 index 000000000..6d7eea0e4 --- /dev/null +++ b/tests/corpus/8230-readline-interface-constructor.cjs @@ -0,0 +1,51 @@ +// `new readline.Interface(...)`: Node's createInterface calls this +// constructor with the same arguments, and a JavaScript file may call it +// directly. Its declaration is protected, which JavaScript does not +// enforce. +"use strict"; +const readline = require("readline"); +const readlinePromises = require("readline/promises"); +const { Interface } = require("readline"); +const { PassThrough } = require("stream"); + +// The options form. +const a = new PassThrough(); +const rlA = new readline.Interface({ input: a, terminal: false }); +console.log("instanceof", rlA instanceof readline.Interface); +const linesA = []; +rlA.on("line", (line) => { + linesA.push(line); +}); +rlA.on("close", () => { + console.log("options", JSON.stringify(linesA)); +}); +a.write("one\ntwo\n"); +a.end("three"); + +// The positional form, through a destructured binding. +const b = new PassThrough(); +const outB = new PassThrough(); +const rlB = new Interface(b, outB); +const linesB = []; +rlB.on("line", (line) => { + linesB.push(line); +}); +rlB.on("close", () => { + console.log("positional", JSON.stringify(linesB)); +}); +b.write("four\r\nfive\n"); +b.end(); + +// readline/promises' Interface. +const c = new PassThrough(); +const rlC = new readlinePromises.Interface({ input: c, crlfDelay: Infinity }); +console.log("promises instanceof", rlC instanceof readlinePromises.Interface); +const linesC = []; +rlC.on("line", (line) => { + linesC.push(line); +}); +rlC.on("close", () => { + console.log("promises", JSON.stringify(linesC)); +}); +c.end("six\nseven"); + diff --git a/tests/corpus/8231-builtin-member-binding-new.cjs b/tests/corpus/8231-builtin-member-binding-new.cjs new file mode 100644 index 000000000..1549425f7 --- /dev/null +++ b/tests/corpus/8231-builtin-member-binding-new.cjs @@ -0,0 +1,35 @@ +// `new` of a builtin class bound through a member of a require() result +// (`const SD = require("string_decoder").StringDecoder`, the spelling of +// Node's own tests): the binding resolves to the class, as the +// destructured `const { StringDecoder } = require(...)` does. +"use strict"; +const SD = require("string_decoder").StringDecoder; +const Readable = require("stream").Readable; +const PassThrough = require("stream").PassThrough; +const Transform = require("stream").Transform; + +const decoder = new SD("utf8"); +console.log(decoder.encoding); +console.log(JSON.stringify(decoder.write(Buffer.from([0xe2, 0x82])))); +console.log(decoder.end(Buffer.from([0xac]))); +const hex = new SD("hex"); +console.log(hex.encoding, hex.write(Buffer.from("hi"))); +console.log(new SD().encoding); + +const r = new Readable({ read() {} }); +r.on("data", (chunk) => console.log("readable", String(chunk))); +r.on("end", () => console.log("readable end")); +r.push("abc"); +r.push(null); + +const p = new PassThrough(); +p.on("data", (chunk) => console.log("passthrough", String(chunk))); +p.end("xyz"); + +const upper = new Transform({ + transform(chunk, encoding, callback) { + callback(null, String(chunk).toUpperCase()); + }, +}); +upper.on("data", (chunk) => console.log("transform", String(chunk))); +upper.end("shout"); diff --git a/tests/harness/corpus-costs/8230-readline-interface-constructor.cjs.json b/tests/harness/corpus-costs/8230-readline-interface-constructor.cjs.json new file mode 100644 index 000000000..9b459042f --- /dev/null +++ b/tests/harness/corpus-costs/8230-readline-interface-constructor.cjs.json @@ -0,0 +1 @@ +{ "plain": 3500, "san": 3000 } diff --git a/tests/harness/corpus-costs/8231-builtin-member-binding-new.cjs.json b/tests/harness/corpus-costs/8231-builtin-member-binding-new.cjs.json new file mode 100644 index 000000000..54e11997a --- /dev/null +++ b/tests/harness/corpus-costs/8231-builtin-member-binding-new.cjs.json @@ -0,0 +1 @@ +{ "plain": 2500, "san": 2500 } From 53baad4ebee91f746f0e4c310c889cfa1cff4dde Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 10:12:06 -0500 Subject: [PATCH 04/17] Pump untyped iterable pipeline sources A pipeline whose first stage is a checked-dynamic value now checks it at run time, as Node's stage ladder does: a stream is piped as before, and a string or a value with a Symbol.asyncIterator or Symbol.iterator method is pumped into the next stage through an async generator over its iterator. A typed Set, Map, generator or async generator object as the first stage takes the same path. This covers the callback and stream/promises forms, with or without function stages and pipeline options. A write whose callback fails synchronously no longer sets needDrain, so no 'drain' follows the error and a pumped source stops at the failed write, as in Node. RUNTIME_ABI_VERSION goes from 37 to 42 for scr_stream_dyn_iterable_source. --- .../builtin-apis/streams-and-readline.mdx | 5 +- packages/cli/test/native-link-info.test.ts | 4 +- .../compiler/src/backend/llvm/lib-shared.ts | 1 + packages/compiler/src/backend/runtime-abi.ts | 4 +- .../src/backend/runtime-pack-targets.test.ts | 2 +- .../compiler/src/backend/runtime-pack.test.ts | 2 +- .../src/frontend/lowering/lower-stream.ts | 626 ++++++++++++++---- packages/compiler/src/ir/builtin-effects.ts | 1 + .../compiler/src/ir/builtin-signatures.ts | 4 + packages/compiler/src/ir/runtime-features.ts | 1 + .../test/native-codegen-integration.test.ts | 4 +- .../7503-pipeline-untyped-sources.mjs.json | 6 + ...04-pipeline-untyped-source-errors.mjs.json | 6 + .../7505-pipeline-untyped-sources.cjs.json | 6 + .../corpus/7506-pipeline-any-source.ts.json | 6 + .../7507-stream-write-error-drain.mjs.json | 6 + .../runtime-darwin-arm64/scripts/build.mjs | 2 +- .../runtime-pack-common/scripts/build.mjs | 2 +- packages/runtime/src/scr_console.c | 2 +- packages/runtime/src/scr_core.h | 2 +- packages/runtime/src/scr_stream.c | 38 +- packages/runtime/src/scr_stream.h | 4 + .../corpus/7503-pipeline-untyped-sources.mjs | 102 +++ .../7504-pipeline-untyped-source-errors.mjs | 90 +++ .../corpus/7505-pipeline-untyped-sources.cjs | 40 ++ tests/corpus/7506-pipeline-any-source.ts | 48 ++ .../corpus/7507-stream-write-error-drain.mjs | 37 ++ .../7503-pipeline-untyped-sources.mjs.json | 1 + ...04-pipeline-untyped-source-errors.mjs.json | 1 + .../7505-pipeline-untyped-sources.cjs.json | 1 + .../7506-pipeline-any-source.ts.json | 1 + .../7507-stream-write-error-drain.mjs.json | 1 + .../self-hosting-native-toolchain.test.ts | 2 +- 33 files changed, 911 insertions(+), 147 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7503-pipeline-untyped-sources.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7504-pipeline-untyped-source-errors.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7505-pipeline-untyped-sources.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7506-pipeline-any-source.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7507-stream-write-error-drain.mjs.json create mode 100644 tests/corpus/7503-pipeline-untyped-sources.mjs create mode 100644 tests/corpus/7504-pipeline-untyped-source-errors.mjs create mode 100644 tests/corpus/7505-pipeline-untyped-sources.cjs create mode 100644 tests/corpus/7506-pipeline-any-source.ts create mode 100644 tests/corpus/7507-stream-write-error-drain.mjs create mode 100644 tests/harness/corpus-costs/7503-pipeline-untyped-sources.mjs.json create mode 100644 tests/harness/corpus-costs/7504-pipeline-untyped-source-errors.mjs.json create mode 100644 tests/harness/corpus-costs/7505-pipeline-untyped-sources.cjs.json create mode 100644 tests/harness/corpus-costs/7506-pipeline-any-source.ts.json create mode 100644 tests/harness/corpus-costs/7507-stream-write-error-drain.mjs.json diff --git a/docs/content/limitations/builtin-apis/streams-and-readline.mdx b/docs/content/limitations/builtin-apis/streams-and-readline.mdx index f9d69e3db..561840268 100644 --- a/docs/content/limitations/builtin-apis/streams-and-readline.mdx +++ b/docs/content/limitations/builtin-apis/streams-and-readline.mdx @@ -3,7 +3,8 @@ `readable.pipe(process.stdout)` and `readable.pipe(process.stderr)` compile and, as in Node, do not end the process stream. When an untyped value reaches `NodeJS.ReadableStream` or `NodeJS.WritableStream`, those two interfaces hold any value, so JavaScript stream objects (constructor-function streams, plain objects with `on` and `pipe`) pass through TypeScript functions typed with them and come back to JavaScript as the same objects. Member calls on such a value use the native stream when it is one and call the object's own members otherwise; `readable.pipe()` into a JavaScript destination follows Node's pipe protocol. - A `pipeline` stage and a `for await` source that arrive as untyped values must be native streams, or `Cipheriv` and `Decipheriv` values, at run time; other values throw a not-supported error. + The first `pipeline` stage may arrive as an untyped value that is a native stream, a `Cipheriv` or `Decipheriv` value, or an iterable: a string, an array, a generator object, or any value with a `Symbol.asyncIterator` or `Symbol.iterator` method, which the pipeline pumps into the next stage as Node does. + Later `pipeline` stages and `for await` sources that arrive as untyped values must be native streams, or `Cipheriv` and `Decipheriv` values, at run time; other values throw a not-supported error. Native streams expose no `emit` to JavaScript, so a JavaScript stream whose own `pipe()` emits on its destination (the legacy `Stream` pipe) cannot pipe into one. `Readable.from` accepts strings, Buffers, arrays, Sets, Maps, Map and Set iterators, generators, async generators, and class iterables and async iterables, and in JavaScript any iterable or async iterable. In JavaScript, a sync iterator value with a `then` method that is not a promise, and an object whose `[Symbol.asyncIterator]` or `[Symbol.iterator]` method is a generator method, are unsupported. @@ -14,7 +15,7 @@ `node:stream/consumers` and readline interfaces read object-mode streams of strings and Buffers; other values throw a not-supported error. `pipeline` accepts function stages anywhere among its streams: async generator functions (`async function* (source)`, iterating `source` with `for await`), an async generator source function (`async function* ({ signal })`), an async generator function as the destination, and an async function destination whose value resolves the promise form and reaches the callback's second argument. A stage is written inline, or named by a function declaration or a `const` bound to a function expression that nothing reassigns. - The source may also be an array, a string, a synchronous generator object, or a synchronous generator function without parameters; their elements reach the next stage on the same ticks as in Node. + The source may also be an array, a string, a `Set`, a `Map`, a generator or async generator object, or a synchronous generator function without parameters; their elements reach the next stage on the same ticks as in Node. Stages may take the `{ signal }` argument. With a function destination, `pipeline()` returns an object-mode PassThrough that receives an async generator destination's values, or an async function destination's value unless it is `null` or `undefined`. `@types/node` types that result as `NodeJS.WritableStream`, so `for await` over it needs the `PassThrough` type (`pipeline(...) as PassThrough`, or a JSDoc `@type`). diff --git a/packages/cli/test/native-link-info.test.ts b/packages/cli/test/native-link-info.test.ts index 8ac579449..5e862e126 100644 --- a/packages/cli/test/native-link-info.test.ts +++ b/packages/cli/test/native-link-info.test.ts @@ -72,7 +72,7 @@ describe.runIf(supported)("macOS arm64 native link info", () => { minimum_os: "14.0", }, program: { object, entry_symbol: "main" }, - runtime_abi: { version: 37, marker: "scr_runtime_abi_v37" }, + runtime_abi: { version: 42, marker: "scr_runtime_abi_v42" }, runtime_pack: { kind: "precompiled", package: "@scriptc/runtime-darwin-arm64", @@ -165,6 +165,6 @@ describe.runIf(supported)("macOS arm64 native link info", () => { () => null, (failure: { stderr?: string }) => failure, ); - expect(error?.stderr).toContain("scr_runtime_abi_v37"); + expect(error?.stderr).toContain("scr_runtime_abi_v42"); }); }); diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 7a16c37e8..9c38c0983 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -795,6 +795,7 @@ export const LIB_FN_SYMS: Record = { "procStream.write": "scr_proc_stream_write", "stream.stdioWritable": "scr_stream_stdio_writable", "stream.unboxDyn": "scr_stream_unbox_dyn", + "stream.dynIterableSource": "scr_stream_dyn_iterable_source", "stream.handleStream": "scr_stream_handle", "stream.pipeDyn": "scr_stream_pipe_dyn", "stream.unpipeDyn": "scr_stream_unpipe_dyn", diff --git a/packages/compiler/src/backend/runtime-abi.ts b/packages/compiler/src/backend/runtime-abi.ts index 11bde6c2b..0eda18a5d 100644 --- a/packages/compiler/src/backend/runtime-abi.ts +++ b/packages/compiler/src/backend/runtime-abi.ts @@ -2,8 +2,8 @@ * of the link contract: a program object keeps this symbol undefined and the * matching runtime defines it, so incompatible manual links fail before the * program can start. */ -export const RUNTIME_ABI_VERSION = 37 as const; -export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v37" as const; +export const RUNTIME_ABI_VERSION = 42 as const; +export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v42" as const; /** Object consumption is useful today, but the complete scr_* surface may * still change before 1.0. The versioned marker prevents accidental mixing; diff --git a/packages/compiler/src/backend/runtime-pack-targets.test.ts b/packages/compiler/src/backend/runtime-pack-targets.test.ts index 9846e8d89..6f39c6d08 100644 --- a/packages/compiler/src/backend/runtime-pack-targets.test.ts +++ b/packages/compiler/src/backend/runtime-pack-targets.test.ts @@ -52,7 +52,7 @@ async function fixture(target: NativeTargetSpec, workers = false) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 37, marker: "scr_runtime_abi_v37" }, + runtime_abi: { version: 42, marker: "scr_runtime_abi_v42" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: ["SCR_LIB"], sanitizer: "external-toolchain-required" }, flavors: { diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index e2b24f8ca..4561fad54 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -128,7 +128,7 @@ async function fixture(target: NativeTargetSpec = MACOS_ARM64_TARGET) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 37, marker: "scr_runtime_abi_v37" }, + runtime_abi: { version: 42, marker: "scr_runtime_abi_v42" }, compiler: { command: "clang", identity: "fixture clang", diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index 04c2fa043..72bea5dd7 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -2837,6 +2837,107 @@ export function streamClassAliasDecl( const PIPELINE_STAGE_EXPECTED = "of type function or an instance of Blob, ReadableStream, WritableStream, Stream, Iterable, AsyncIterable, or Promise or { readable, writable } pair"; +/** A checked-dynamic pipeline stage (an untyped source or destination): + * a boxed native stream unwraps at run time; other values (functions, + * pairs) throw the not-supported Error there. */ +function unboxDynStage(lowerer: Lowerer, v: IrExpr, node: ts.Expression): IrExpr { + noteUntypedStreamFlow(lowerer, null); + return { + kind: "libCall", + fn: "stream.unboxDyn", + args: [ + v, + strLit( + lowerer.dynNodeStreams + ? "pipeline stages other than native streams (JavaScript stream objects, iterables, generators, plain functions, and { readable, writable } pairs) are not supported yet" + : "pipeline stages other than native streams (iterables, generators, plain functions, and { readable, writable } pairs) are not supported yet", + locOf(node), + ), + ], + type: { kind: "object", className: "%Duplex" }, + loc: locOf(node), + }; +} + +/** A pipeline whose first stage is checked-dynamic, lowered with that + * stage already evaluated: `expr` is the stream it unboxes to, or, when + * `iterable` is set, the value to pump as an iterable source. */ +interface DynPipelineSource { + expr: IrExpr; + iterable: boolean; +} + +/** pipeline() whose first stage is a stream argument (not a function + * stage, an iterable of a static type, or a cipher value), lowered here. + * When the value is checked-dynamic, Node keeps a stream there and pipes + * it, and keeps an iterable (a string, an array, a generator, any value + * with an iterator method) and pumps it into the next stage. The value is + * evaluated once, and stream.dynIterableSource picks the lowering at run + * time. Null when the first stage is not such an argument. */ +function lowerDynSourcePipeline( + lowerer: Lowerer, + node: ts.Expression, + loc: SrcLoc, + build: (source: DynPipelineSource) => IrExpr, +): IrExpr | null { + if (pipelineStageFunction(lowerer, node) !== null) return null; + if (iterableSourceType(lowerer, node) !== null || isValueStream(lowerer, node)) return null; + let value = lowerer.lowerExpr(node); + // A Set, a Map or a generator object is iterated through its + // checked-dynamic value. + if (BOXED_SOURCE_KINDS.has(value.type.kind) && lowerer.dynConvertible(value.type)) { + value = { kind: "dynFrom", value, type: DYN, loc: value.loc }; + } + if (value.type.kind !== "dyn") { + let stream: IrExpr; + try { + stream = streamArgOf(lowerer, node, value, "pipeline"); + } catch (e) { + if (e instanceof StreamArgTypeThrow) return e.expr; + throw e; + } + return build({ expr: stream, iterable: false }); + } + const local = lowerer.declareHiddenLocal("%pipelineSource", DYN); + const ref = varRef(local.id, DYN, loc); + const stream = build({ expr: unboxDynStage(lowerer, ref, node), iterable: false }); + const iterable = build({ expr: ref, iterable: true }); + const type = stream.type; + return { + kind: "seqExpr", + stmts: [{ kind: "varDecl", localId: local.id, init: value, loc }], + result: { + kind: "ternary", + cond: { kind: "libCall", fn: "stream.dynIterableSource", args: [ref], type: BOOL, loc }, + then: lowerer.coerceToExpected(iterable, type), + else_: stream, + type, + loc, + }, + type, + loc, + }; +} + +/** The static types of a pipeline source that lowerDynSourcePipeline boxes + * as checked-dynamic values. An array or a string reaches it when the + * checker types the argument as any while its value keeps a static type. */ +const BOXED_SOURCE_KINDS: ReadonlySet = new Set([ + "set", + "map", + "generator", + "array", + "string", +]); + +/** lowerPipelineStages' source arguments for a checked-dynamic first stage. */ +function stagesSource( + src: DynPipelineSource | null, +): [ReadonlyMap | undefined, IrExpr | undefined] { + if (src === null) return [undefined, undefined]; + return src.iterable ? [undefined, src.expr] : [new Map([[0, src.expr]]), undefined]; +} + /** A stream-rooted argument, lowered — or the pointed fence (pipeline's * iterable/generator/function stages have no lowering yet). `what` picks * the validation ladder: finished() rejects EVERY non-stream through @@ -2848,7 +2949,16 @@ function lowerStreamArg( node: ts.Expression, what: "finished" | "pipeline" | "addAbortSignal" | "compose", ): IrExpr { - const lowered = lowerer.lowerExpr(node); + return streamArgOf(lowerer, node, lowerer.lowerExpr(node), what); +} + +/** lowerStreamArg over the argument's lowered value. */ +function streamArgOf( + lowerer: Lowerer, + node: ts.Expression, + lowered: IrExpr, + what: "finished" | "pipeline" | "addAbortSignal" | "compose", +): IrExpr { // A Hash, Hmac, Cipheriv or Decipheriv stage is its Transform; pipeline // and compose stages keep the value alive until the stream settles. const v = isValueStream(lowerer, node) @@ -2857,24 +2967,7 @@ function lowerStreamArg( // A checked-dynamic pipeline stage (an untyped source or destination): // a boxed native stream unwraps at run time; other values (iterables, // functions) throw the not-supported Error there. - if (v.type.kind === "dyn" && what === "pipeline") { - noteUntypedStreamFlow(lowerer, null); - return { - kind: "libCall", - fn: "stream.unboxDyn", - args: [ - v, - strLit( - lowerer.dynNodeStreams - ? "pipeline stages other than native streams (JavaScript stream objects, iterables, generators, plain functions, and { readable, writable } pairs) are not supported yet" - : "pipeline stages other than native streams (iterables, generators, plain functions, and { readable, writable } pairs) are not supported yet", - locOf(node), - ), - ], - type: { kind: "object", className: "%Duplex" }, - loc: locOf(node), - }; - } + if (v.type.kind === "dyn" && what === "pipeline") return unboxDynStage(lowerer, v, node); const info = v.type.kind === "object" ? lowerer.classes.get(v.type.className) : undefined; if (!streamSidesOf(lowerer, info)) { // A provably-non-stream value in a JS source (the invalid-input @@ -3210,57 +3303,82 @@ export function lowerStreamModuleCall( ? pipelineOptions(lowerer, lastArg) : null; const stageArgs = opts ? args.slice(0, -1) : args; - if (hasPipelineFunctionStage(lowerer, stageArgs)) { - const staged = lowerPipelineStages(lowerer, stageArgs, loc); - const type = staged.destPromise ?? { kind: "promise", inner: VOID }; - // A promise destination's value settles the result (same type). - if (!opts) { - return staged.wrap({ - kind: "libCall", - fn: "sp.pipelineStages", - args: staged.args, - type, - loc, - }); - } - // the options: end, and the signal (flags bit 6) - const flagsArg = staged.args[3]!; - if (flagsArg.kind !== "numLit") - throw new InternalCompilerError("pipeline flags not a literal"); - const args = [...staged.args, boolLit(opts.end, loc)]; - if (opts.signal) { - args[3] = numLit(flagsArg.value | 64, loc); - args.push(opts.signal); - } - return staged.wrap({ kind: "libCall", fn: "sp.pipelineStagesOpts", args, type, loc }); - } - let streams: IrExpr[]; - try { - streams = stageArgs.map((a) => lowerStreamArg(lowerer, a, "pipeline")); - } catch (e) { - if (e instanceof StreamArgTypeThrow) return e.expr; - throw e; - } - if (opts) { - const head = [numLit(streams.length, loc), ...streams, boolLit(opts.end, loc)]; - return { - kind: "libCall", - fn: opts.signal ? "sp.pipelineOptsSignal" : "sp.pipelineOpts", - args: opts.signal ? [...head, opts.signal] : head, - type: { kind: "promise", inner: VOID }, - loc, - }; - } - return { - kind: "libCall", - fn: "sp.pipeline", - args: [numLit(streams.length, loc), ...streams], - type: { kind: "promise", inner: VOID }, - loc, - }; + const build = (src: DynPipelineSource | null): IrExpr => + lowerPromisePipeline(lowerer, stageArgs, opts, src, loc); + return lowerDynSourcePipeline(lowerer, stageArgs[0]!, loc, build) ?? build(null); } return null; } + return lowerStreamModuleCallbackCall(lowerer, call, bi, loc); +} + +/** stream/promises pipeline(...stages[, options]), with the first stage + * pre-lowered when it is checked-dynamic (lowerDynSourcePipeline). */ +function lowerPromisePipeline( + lowerer: Lowerer, + stageArgs: readonly ts.Expression[], + opts: ReturnType | null, + src: DynPipelineSource | null, + loc: SrcLoc, +): IrExpr { + if (src?.iterable === true || hasPipelineFunctionStage(lowerer, stageArgs)) { + const staged = lowerPipelineStages(lowerer, stageArgs, loc, ...stagesSource(src)); + const type = staged.destPromise ?? { kind: "promise", inner: VOID }; + // A promise destination's value settles the result (same type). + if (!opts) { + return staged.wrap({ + kind: "libCall", + fn: "sp.pipelineStages", + args: staged.args, + type, + loc, + }); + } + // the options: end, and the signal (flags bit 6) + const flagsArg = staged.args[3]!; + if (flagsArg.kind !== "numLit") throw new InternalCompilerError("pipeline flags not a literal"); + const args = [...staged.args, boolLit(opts.end, loc)]; + if (opts.signal) { + args[3] = numLit(flagsArg.value | 64, loc); + args.push(opts.signal); + } + return staged.wrap({ kind: "libCall", fn: "sp.pipelineStagesOpts", args, type, loc }); + } + let streams: IrExpr[]; + try { + streams = stageArgs.map((a, i) => + i === 0 && src !== null ? src.expr : lowerStreamArg(lowerer, a, "pipeline"), + ); + } catch (e) { + if (e instanceof StreamArgTypeThrow) return e.expr; + throw e; + } + if (opts) { + const head = [numLit(streams.length, loc), ...streams, boolLit(opts.end, loc)]; + return { + kind: "libCall", + fn: opts.signal ? "sp.pipelineOptsSignal" : "sp.pipelineOpts", + args: opts.signal ? [...head, opts.signal] : head, + type: { kind: "promise", inner: VOID }, + loc, + }; + } + return { + kind: "libCall", + fn: "sp.pipeline", + args: [numLit(streams.length, loc), ...streams], + type: { kind: "promise", inner: VOID }, + loc, + }; +} + +/** The node:stream and stream/consumers members of lowerStreamModuleCall. */ +function lowerStreamModuleCallbackCall( + lowerer: Lowerer, + call: ts.CallExpression, + bi: { module: string; member: string }, + loc: SrcLoc, +): IrExpr | null { if (bi.module === "stream/consumers") { const args = call.arguments.nodes; // text/json/buffer/arrayBuffer(stream) → a pending promise the @@ -3411,67 +3529,9 @@ export function lowerStreamModuleCall( "the supported form is pipeline(source, ...streams, callback)", ); } - if (hasPipelineFunctionStage(lowerer, args.slice(0, -1))) { - const staged = lowerPipelineStages(lowerer, args.slice(0, -1), loc); - const cbNode = args[args.length - 1]!; - const cbParams = - ts.isArrowFunction(cbNode) || ts.isFunctionExpression(cbNode) - ? cbNode.parameters.nodes.length - : 0; - if (staged.dest === 2 || (staged.dest === 1 && cbParams > 1)) { - // callback(err, value): the destination's value (undefined for - // an async generator) reaches the callback through the - // checked-dynamic path. - const cb = lowerer.lowerExprExpecting(args[args.length - 1]!, DYN); - // SCR_PIPE_TYPED_CB: a typed callback whose error parameter cannot - // hold undefined takes null for no error; others take undefined. - const stagedArgs = [...staged.args]; - const flagsArg = stagedArgs[3]!; - if (flagsArg.kind !== "numLit") - throw new InternalCompilerError("pipeline flags not a literal"); - if ( - !isJsSourceFile(ts.getSourceFileOfNode(call)) && - !callbackErrorAdmitsUndefined(lowerer, cb) - ) - stagedArgs[3] = numLit(flagsArg.value | 128, loc); - return staged.wrap({ - kind: "libCall", - fn: "stream.pipelineStagesDyn", - args: [...stagedArgs, cb], - type: staged.last, - loc, - }); - } - const { cb, dyn } = lowerEosCallback( - lowerer, - "pipeline", - args[args.length - 1]!, - staged.last, - ); - return staged.wrap({ - kind: "libCall", - fn: dyn ? "stream.pipelineStagesDyn" : "stream.pipelineStages", - args: [...staged.args, cb], - type: staged.last, - loc, - }); - } - let streams: IrExpr[]; - try { - streams = args.slice(0, -1).map((a) => lowerStreamArg(lowerer, a, "pipeline")); - } catch (e) { - if (e instanceof StreamArgTypeThrow) return e.expr; - throw e; - } - const last = streams[streams.length - 1]!; - const { cb, dyn } = lowerEosCallback(lowerer, "pipeline", args[args.length - 1]!, last.type); - return { - kind: "libCall", - fn: dyn ? "stream.pipelineDyn" : "stream.pipeline", - args: [numLit(streams.length, loc), ...streams, cb], - type: last.type, - loc, - }; + const build = (src: DynPipelineSource | null): IrExpr => + lowerCallbackPipeline(lowerer, call, args, src, loc); + return lowerDynSourcePipeline(lowerer, args[0]!, loc, build) ?? build(null); } if (bi.member === "compose") { @@ -3484,6 +3544,75 @@ export function lowerStreamModuleCall( return null; } +/** pipeline(...stages, callback), with the first stage pre-lowered when it + * is checked-dynamic (lowerDynSourcePipeline). */ +function lowerCallbackPipeline( + lowerer: Lowerer, + call: ts.CallExpression, + args: readonly ts.Expression[], + src: DynPipelineSource | null, + loc: SrcLoc, +): IrExpr { + if (src?.iterable === true || hasPipelineFunctionStage(lowerer, args.slice(0, -1))) { + const staged = lowerPipelineStages(lowerer, args.slice(0, -1), loc, ...stagesSource(src)); + const cbNode = args[args.length - 1]!; + const cbParams = + ts.isArrowFunction(cbNode) || ts.isFunctionExpression(cbNode) + ? cbNode.parameters.nodes.length + : 0; + if (staged.dest === 2 || (staged.dest === 1 && cbParams > 1)) { + // callback(err, value): the destination's value (undefined for + // an async generator) reaches the callback through the + // checked-dynamic path. + const cb = lowerer.lowerExprExpecting(args[args.length - 1]!, DYN); + // SCR_PIPE_TYPED_CB: a typed callback whose error parameter cannot + // hold undefined takes null for no error; others take undefined. + const stagedArgs = [...staged.args]; + const flagsArg = stagedArgs[3]!; + if (flagsArg.kind !== "numLit") + throw new InternalCompilerError("pipeline flags not a literal"); + if ( + !isJsSourceFile(ts.getSourceFileOfNode(call)) && + !callbackErrorAdmitsUndefined(lowerer, cb) + ) + stagedArgs[3] = numLit(flagsArg.value | 128, loc); + return staged.wrap({ + kind: "libCall", + fn: "stream.pipelineStagesDyn", + args: [...stagedArgs, cb], + type: staged.last, + loc, + }); + } + const { cb, dyn } = lowerEosCallback(lowerer, "pipeline", args[args.length - 1]!, staged.last); + return staged.wrap({ + kind: "libCall", + fn: dyn ? "stream.pipelineStagesDyn" : "stream.pipelineStages", + args: [...staged.args, cb], + type: staged.last, + loc, + }); + } + let streams: IrExpr[]; + try { + streams = args + .slice(0, -1) + .map((a, i) => (i === 0 && src !== null ? src.expr : lowerStreamArg(lowerer, a, "pipeline"))); + } catch (e) { + if (e instanceof StreamArgTypeThrow) return e.expr; + throw e; + } + const last = streams[streams.length - 1]!; + const { cb, dyn } = lowerEosCallback(lowerer, "pipeline", args[args.length - 1]!, last.type); + return { + kind: "libCall", + fn: dyn ? "stream.pipelineDyn" : "stream.pipeline", + args: [numLit(streams.length, loc), ...streams, cb], + type: last.type, + loc, + }; +} + /** A pipeline() function stage: an inline function or arrow function, or * an identifier bound to a function declaration or to a `const` * initialized with one, which nothing reassigns. */ @@ -3788,6 +3917,237 @@ function lowerWebStreamSource( }; } +/** A checked-dynamic pipeline source that is an iterable at run time + * (stream.dynIterableSource), as an async generator closure that iterates + * it as Node's pipeline does with `for await`: the value's async iterator, + * or its sync iterator stepped like the typed iterable sources. Each value + * is yielded; leaving early closes the iterator through its return(). */ +function lowerDynIterableSource( + lowerer: Lowerer, + value: IrExpr, + loc: SrcLoc, +): { closure: IrExpr; out: SyncGenType } | null { + const out: SyncGenType = { + kind: "generator", + async: true, + yieldT: DYN, + retT: VOID, + nextT: UNDEFINED_T, + }; + const genResultT = genResultRecord(DYN, VOID, lowerer.shapes, lowerer.unions); + if (!genResultT) return null; + const key = "dynIterableSource"; + let maker = lowerer.valueHelpers.get(key); + if (!maker) { + maker = `%stream.dynIterableSource.${lowerer.valueHelpers.size}`; + const genName = `${maker}.gen`; + const frame = new LiftedFrame(loc); + const srcId = frame.local("src", DYN, false); + const itId = frame.local("it", DYN, false); + const syncId = frame.local("sync", BOOL, false); + const nextId = frame.local("next", DYN, false); + const stepId = frame.local("step", DYN, true); + const activeId = frame.local("active", BOOL, true); + const closeId = frame.local("close", DYN, false); + const it = (): IrExpr => varRef(itId, DYN, loc); + const sync = (): IrExpr => varRef(syncId, BOOL, loc); + const step = (): IrExpr => varRef(stepId, DYN, loc); + const get = (obj: IrExpr, name: string): IrExpr => ({ + kind: "dynKeyGet", + value: obj, + key: strLit(name, loc), + type: DYN, + loc, + }); + const awaitDyn = (v: IrExpr): IrExpr => ({ + kind: "libCall", + fn: "async.awaitDyn", + args: [v], + type: DYN, + loc, + }); + const call = (callee: IrExpr, calleeName: string): IrExpr => ({ + kind: "dynCall", + callee, + receiver: it(), + calleeName, + args: [], + type: DYN, + loc, + }); + const setActive = (on: boolean): IrStmt => ({ + kind: "assign", + localId: activeId, + value: boolLit(on, loc), + loc, + }); + const nextCall = (): IrExpr => call(varRef(nextId, DYN, loc), "iterator method"); + const body: IrStmt[] = [ + { + kind: "varDecl", + localId: itId, + init: { + kind: "libCall", + fn: "dyn.iteratorResult", + args: [ + { + kind: "libCall", + fn: "dyn.asyncIteratorOpen", + args: [varRef(srcId, DYN, loc), strLit("", loc)], + type: DYN, + loc, + }, + ], + type: DYN, + loc, + }, + loc, + }, + { + kind: "varDecl", + localId: syncId, + init: { kind: "libCall", fn: "dyn.asyncIteratorWasSync", args: [], type: BOOL, loc }, + loc, + }, + { kind: "varDecl", localId: nextId, init: get(it(), "next"), loc }, + { kind: "varDecl", localId: stepId, init: dynUndefinedExpr(loc), loc }, + { kind: "varDecl", localId: activeId, init: boolLit(false, loc), loc }, + { + kind: "tryCatch", + tryBody: [ + { + kind: "while", + cond: boolLit(true, loc), + body: [ + setActive(false), + { + kind: "assign", + localId: stepId, + value: { + kind: "libCall", + fn: "dyn.iteratorResult", + args: [ + { + kind: "ternary", + cond: sync(), + then: nextCall(), + else_: awaitDyn(nextCall()), + type: DYN, + loc, + }, + ], + type: DYN, + loc, + }, + loc, + }, + { + kind: "if", + cond: { + kind: "dynTest", + test: "truthy", + value: get(step(), "done"), + type: BOOL, + loc, + }, + then: [{ kind: "break", loc }], + else_: null, + loc, + }, + setActive(true), + { + kind: "exprStmt", + expr: { kind: "yieldExpr", value: get(step(), "value"), type: VOID, loc }, + loc, + }, + ], + loc, + }, + ], + catchBody: null, + catchLocalId: null, + finallyBody: [ + { + kind: "if", + cond: varRef(activeId, BOOL, loc), + then: [ + { kind: "varDecl", localId: closeId, init: get(it(), "return"), loc }, + { + kind: "if", + cond: { + kind: "unary", + op: "!", + operand: { + kind: "dynTest", + test: "nullish", + value: varRef(closeId, DYN, loc), + type: BOOL, + loc, + }, + type: BOOL, + loc, + }, + then: [ + { + kind: "exprStmt", + expr: awaitDyn(call(varRef(closeId, DYN, loc), "iterator return")), + loc, + }, + ], + else_: null, + loc, + }, + ], + else_: null, + loc, + }, + ], + loc, + }, + { kind: "return", value: null, loc }, + ]; + const srcLocal = frame.locals.find((l) => l.id === srcId)!; + srcLocal.boxed = true; + lowerer.valueHelpers.set(key, maker); + lowerer.liftedFns.push({ + name: genName, + params: [], + returnType: VOID, + async: true, + captures: [{ localId: srcId, name: "src", type: DYN }], + locals: frame.locals, + body, + generator: { yieldT: DYN, nextT: UNDEFINED_T, resultType: genResultT }, + loc, + }); + const makerSrc: IrLocal = { id: "src.0", name: "src", type: DYN, mutable: false, boxed: true }; + lowerer.liftedFns.push({ + name: maker, + params: [{ localId: makerSrc.id, name: makerSrc.name, type: DYN }], + returnType: funcOf([], out), + locals: [makerSrc], + body: [ + { + kind: "return", + value: { + kind: "closure", + fnName: genName, + captures: [makerSrc.id], + type: funcOf([], out), + loc, + }, + loc, + }, + ], + loc, + }); + } + return { + closure: { kind: "call", callee: maker, args: [value], type: funcOf([], out), loc }, + out, + }; +} + function lowerIterableSource( lowerer: Lowerer, node: ts.Expression, @@ -3951,6 +4311,7 @@ function lowerPipelineStages( stageArgs: readonly ts.Expression[], loc: SrcLoc, preLowered?: ReadonlyMap, + dynSource?: IrExpr, ): PipelineStagesLowering { if (stageArgs.length > 52) { lowerer.noLowering( @@ -3975,6 +4336,15 @@ function lowerPipelineStages( const argOrder: { list: IrExpr[]; index: number }[] = []; for (let j = 0; j < stageArgs.length; j++) { const arg = stageArgs[j]!; + if (j === 0 && dynSource !== undefined) { + const source = lowerDynIterableSource(lowerer, dynSource, loc); + if (source === null) lowerer.noLowering("pipeline over this source", arg); + positions += 1; + argOrder.push({ list: stages, index: stages.length }); + stages.push(source!.closure); + srcT = source!.out; + continue; + } // compose's head and tail functions arrive as Duplex.from streams const fn = preLowered?.has(j) ? null : pipelineStageFunction(lowerer, arg); const syncSourceFn = diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index 390c7fbc3..1821b6a6b 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -91,6 +91,7 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "procStream.write", "child.stdioStream", "stream.unboxDyn", + "stream.dynIterableSource", "stream.handleStream", "stream.pipeDyn", "stream.unpipeDyn", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 216e46a4d..2cc5c708e 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -2786,6 +2786,10 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ argTypes: [DYN, STRING], result: { kind: "object", className: "%Duplex" }, }, + /** Whether pipeline() pumps a checked-dynamic first stage as an + * iterable (a string, or a value with an iterator method that is not a + * stream with pipe() and on()) instead of piping it as a stream. */ + "stream.dynIterableSource": { argTypes: [DYN], result: BOOL }, /** The Transform a Cipheriv or Decipheriv reads and writes as a stream * (+1, created on first use and owned by the cipher). */ "stream.handleStream": { diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index 85b21af7b..f66697567 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -263,6 +263,7 @@ const STREAM_LIB_FNS: ReadonlySet = new Set([ "stream.prop", "stream.stdioWritable", "stream.unboxDyn", + "stream.dynIterableSource", "stream.handleStream", "stream.unpipeDyn", "stream.workerStdio", diff --git a/packages/compiler/test/native-codegen-integration.test.ts b/packages/compiler/test/native-codegen-integration.test.ts index cecebb49c..1be1b7a3f 100644 --- a/packages/compiler/test/native-codegen-integration.test.ts +++ b/packages/compiler/test/native-codegen-integration.test.ts @@ -378,7 +378,7 @@ done: expect(await symbols(helperObject, ["-u"])).toEqual(await symbols(clangObject, ["-u"])); expect(await symbols(helperObject, ["-gU"])).toEqual(await symbols(clangObject, ["-gU"])); expect(await symbols(helperObject, ["-gU"])).toEqual(["0000000000000000 T _main"]); - expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v37"); + expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v42"); const clangExe = join(dir, "clang-program"); const linkDriver = join(dir, "clang-driver.c"); @@ -712,7 +712,7 @@ done: .split("\n"); expect(undefinedSymbols).toContain("_sf_scale"); expect(undefinedSymbols).toContain("_sf_callback_mix"); - expect(undefinedSymbols).toContain("_scr_runtime_abi_v37"); + expect(undefinedSymbols).toContain("_scr_runtime_abi_v42"); }); test("helper and clang object paths retain the same outbound FFI C ABI", async () => { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7503-pipeline-untyped-sources.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7503-pipeline-untyped-sources.mjs.json new file mode 100644 index 000000000..4d93af132 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7503-pipeline-untyped-sources.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7503-pipeline-untyped-sources.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7504-pipeline-untyped-source-errors.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7504-pipeline-untyped-source-errors.mjs.json new file mode 100644 index 000000000..19402586b --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7504-pipeline-untyped-source-errors.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7504-pipeline-untyped-source-errors.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7505-pipeline-untyped-sources.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7505-pipeline-untyped-sources.cjs.json new file mode 100644 index 000000000..29d85351d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7505-pipeline-untyped-sources.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7505-pipeline-untyped-sources.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7506-pipeline-any-source.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7506-pipeline-any-source.ts.json new file mode 100644 index 000000000..942a2b3e8 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7506-pipeline-any-source.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7506-pipeline-any-source.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7507-stream-write-error-drain.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7507-stream-write-error-drain.mjs.json new file mode 100644 index 000000000..6fa55d09d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7507-stream-write-error-drain.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7507-stream-write-error-drain.mjs" + ], + "diags": [] +} diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index b6b855c05..56f5ca559 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -366,7 +366,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: RUNTIME_PACK_MATRIX.target, - runtime_abi: { version: 37, marker: "scr_runtime_abi_v37" }, + runtime_abi: { version: 42, marker: "scr_runtime_abi_v42" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index 479d8aad4..c571a25c5 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -469,7 +469,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: matrix.target, - runtime_abi: { version: 37, marker: "scr_runtime_abi_v37" }, + runtime_abi: { version: 42, marker: "scr_runtime_abi_v42" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime/src/scr_console.c b/packages/runtime/src/scr_console.c index 086340014..6c15f932c 100644 --- a/packages/runtime/src/scr_console.c +++ b/packages/runtime/src/scr_console.c @@ -148,7 +148,7 @@ static void scr_enable_utf8_console_output(void) { } #endif -void scr_runtime_abi_v37(void) {} +void scr_runtime_abi_v42(void) {} void scr_init(void) { #ifdef SCR_WORKERS diff --git a/packages/runtime/src/scr_core.h b/packages/runtime/src/scr_core.h index 3707b8d50..5415ccdc6 100644 --- a/packages/runtime/src/scr_core.h +++ b/packages/runtime/src/scr_core.h @@ -81,7 +81,7 @@ void scr_init(void); /* Program objects emitted by the bundled LLVM helper reference this symbol. * Its versioned spelling makes a mismatched manual runtime link fail before * the program can start. */ -void scr_runtime_abi_v37(void); +void scr_runtime_abi_v42(void); void scr_runtime_workers_v8(void); /* ── the trap funnel (scr_console.c; scr_library.c under -DSCR_LIB) ────── diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index ec061e10c..be6d3635f 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -6866,10 +6866,12 @@ static bool scr_stream_write_chunk_enc(ScrStream *s, ScrBytes *chunk /*borrowed* scr_stream_do_write(s, scr_stream_wchunk_retain(st, chunk), cb, enc); } /* Node computes the answer AFTER the (possibly synchronous) write: - * a sync completion has already drained its bytes from length. */ - bool ret = st->w.length < st->w.hwm && !st->destroyed && !st->errored; - if (!ret) st->w.need_drain = true; - return ret; + * a sync completion has already drained its bytes from length. Only + * the length sets needDrain; an errored or destroyed stream answers + * false without waiting for 'drain'. */ + bool below = st->w.length < st->w.hwm; + if (!below) st->w.need_drain = true; + return below && !st->destroyed && !st->errored; } static bool scr_stream_write_chunk(ScrStream *s, ScrBytes *chunk /*borrowed*/, ScrClosure *cb /*moves*/) { @@ -8407,3 +8409,31 @@ ScrStream *scr_stream_unbox_dyn(const ScrDyn *value, const ScrStr *message) { scr_throw_error_msg_code(SCR_ERR_ERROR, message->data, message->len, "SC2020"); return NULL; } + +/* pipeline.js's first stage: a value with pipe() and on() methods is a + * readable node stream, which the stage ladder tests before isIterable; + * a string, or a value with a Symbol.asyncIterator or Symbol.iterator + * method, is an iterable the pipeline pumps into the next stage. */ +bool scr_stream_dyn_iterable_source(const ScrDyn *value) { + ScrStream *stream = scr_stream_of_handle(value); + if (!stream) stream = scr_stream_boxed(value); + if (stream) { + scr_stream_release(stream); + return false; + } + if (value->kind == SCR_DYN_STR) return true; + if (value->kind == SCR_DYN_OBJ) { + ScrDyn *pipe = scr_dyn_obj_read(value, "pipe", 4); + if (!pipe) return false; + bool streamish = pipe->kind == SCR_DYN_FUNC; + scr_dyn_release(pipe); + if (streamish) { + ScrDyn *on = scr_dyn_obj_read(value, "on", 2); + if (!on) return false; + streamish = on->kind == SCR_DYN_FUNC; + scr_dyn_release(on); + if (streamish) return false; + } + } + return scr_dyn_iterable_kind(value) > 0; +} diff --git a/packages/runtime/src/scr_stream.h b/packages/runtime/src/scr_stream.h index a4798c629..7acad0eeb 100644 --- a/packages/runtime/src/scr_stream.h +++ b/packages/runtime/src/scr_stream.h @@ -145,6 +145,10 @@ ScrStream *scr_stream_pipe_general(ScrStream *src, ScrStream *dst, bool end); /* A boxed native stream out of a checked-dynamic pipeline stage (+1), or * NULL with `message` thrown as the not-supported Error. */ ScrStream *scr_stream_unbox_dyn(const struct ScrDyn *value, const ScrStr *message); +/* Whether pipeline() keeps a checked-dynamic first stage as an iterable + * to pump rather than as a stream (false with an exception pending when + * reading the value's members throws). */ +bool scr_stream_dyn_iterable_source(const struct ScrDyn *value); /* Handles that are Transforms (ScrDynHandleOps.transform: Cipheriv and * Decipheriv). scr_stream_of_handle: the handle's Transform (+1), built on * first use, or NULL when the value is not such a handle. diff --git a/tests/corpus/7503-pipeline-untyped-sources.mjs b/tests/corpus/7503-pipeline-untyped-sources.mjs new file mode 100644 index 000000000..3dbc90eee --- /dev/null +++ b/tests/corpus/7503-pipeline-untyped-sources.mjs @@ -0,0 +1,102 @@ +// pipeline() over untyped iterable sources: arrays, strings, Sets, generator +// objects, async generator objects, class iterables and object values, +// into Writables, a Transform chain, a function stage and the promise form. +// Each source is pumped as Node pumps an iterable, on the same ticks. +import { pipeline, Writable, Transform } from "node:stream"; +import { pipeline as pipelineP } from "node:stream/promises"; + +const log = []; +function show(chunk) { + return typeof chunk === "string" ? chunk : Buffer.isBuffer(chunk) ? chunk.toString() : JSON.stringify(chunk); +} +function objectSink(label) { + return new Writable({ + objectMode: true, + write(chunk, _enc, cb) { + log.push(`${label} write ${show(chunk)}`); + cb(); + }, + final(cb) { log.push(`${label} final`); cb(); }, + }); +} +function sink(label, objectMode = false) { + if (objectMode) return objectSink(label); + return new Writable({ + write(chunk, _enc, cb) { + log.push(`${label} write ${typeof chunk === "string" ? chunk : Buffer.isBuffer(chunk) ? chunk.toString() : JSON.stringify(chunk)}`); + cb(); + }, + final(cb) { log.push(`${label} final`); cb(); }, + }); +} +function* gen() { + log.push("gen start"); + yield "g1"; + log.push("gen mid"); + yield "g2"; + log.push("gen end"); +} +async function* agen() { + yield "a1"; + await null; + yield "a2"; +} +const sources = { + array: () => ["x", "y"], + string: () => "hey", + set: () => new Set(["s1", "s2"]), + gen: () => gen(), + agen: () => agen(), + iterable: () => ({ *[Symbol.iterator]() { yield "i1"; yield "i2"; } }), + objects: () => [{ n: 1 }, { n: 2 }], +}; +const names = Object.keys(sources); +async function run(name) { + const src = sources[name](); + const objectMode = name === "objects"; + await new Promise((resolve) => { + queueMicrotask(() => log.push(`${name} microtask`)); + pipeline(src, sink(name, objectMode), (err) => { + log.push(`${name} done ${err ? err.code ?? err.message : err}`); + resolve(); + }); + log.push(`${name} after call`); + }); +} +for (const n of names) await run(n); + +// a transform chain and a function stage after an untyped source +const src2 = ["p", "q"]; +await new Promise((resolve) => + pipeline( + src2, + new Transform({ transform(c, _e, cb) { cb(null, c.toString().toUpperCase()); } }), + sink("transform"), + (err) => { log.push(`transform done ${err}`); resolve(); }, + ), +); +const src3 = new Set(["f1", "f2"]); +await new Promise((resolve) => + pipeline( + src3, + async function* (source) { for await (const c of source) yield c + "!"; }, + sink("fnstage"), + (err) => { log.push(`fnstage done ${err}`); resolve(); }, + ), +); +// promises form +const src4 = ["m", "n"]; +await pipelineP(src4, sink("promises")); +log.push("promises resolved"); +// an error thrown by the source +function* bad() { yield "ok"; throw new Error("source failed"); } +const src5 = bad(); +await new Promise((resolve) => + pipeline(src5, sink("bad"), (err) => { log.push(`bad done ${err && err.message}`); resolve(); }), +); +// a source of non-string values into a byte-mode writable +const src6 = [1, 2]; +await new Promise((resolve) => + pipeline(src6, sink("numbers"), (err) => { log.push(`numbers done ${err && err.code}`); resolve(); }), +); +console.log(log.join("\n")); diff --git a/tests/corpus/7504-pipeline-untyped-source-errors.mjs b/tests/corpus/7504-pipeline-untyped-source-errors.mjs new file mode 100644 index 000000000..fcbfa9d60 --- /dev/null +++ b/tests/corpus/7504-pipeline-untyped-source-errors.mjs @@ -0,0 +1,90 @@ +// pipeline() over untyped iterable sources: a destination that fails closes +// the source, backpressure waits for drain, an abort signal stops the pump, +// and async generator and Map sources reach object-mode destinations. +import { pipeline, Writable, PassThrough } from "node:stream"; +import { pipeline as pipelineP } from "node:stream/promises"; + +function* counted(label, n) { + try { + for (let i = 0; i < n; i++) { + console.log(label, "yield", i); + yield `${label}${i}`; + } + } finally { + console.log(label, "finally"); + } +} +function pick(v) { + return v; +} + +// the destination fails on its second write: the source's finally runs +await new Promise((resolve) => { + let n = 0; + const dst = new Writable({ + write(chunk, _e, cb) { + n++; + console.log("fail write", String(chunk)); + cb(n === 2 ? new Error("write failed") : null); + }, + }); + pipeline(pick(counted("f", 5)), dst, (err) => { + console.log("fail done", err && err.message); + resolve(); + }); +}); + +// backpressure: a one-byte highWaterMark and asynchronous writes +await new Promise((resolve) => { + const dst = new Writable({ + highWaterMark: 1, + write(chunk, _e, cb) { + console.log("slow write", String(chunk)); + setTimeout(cb, 1); + }, + }); + pipeline(pick(counted("b", 3)), dst, (err) => { + console.log("slow done", err); + resolve(); + }); +}); + +// an abort signal +const ac = new AbortController(); +try { + await pipelineP( + pick(counted("s", 10)), + new Writable({ + write(chunk, _e, cb) { + console.log("signal write", String(chunk)); + if (String(chunk) === "s1") ac.abort(); + cb(); + }, + }), + { signal: ac.signal }, + ); +} catch (err) { + console.log("signal rejected", err.name); +} + +// an untyped async generator into a PassThrough read with for await +const out = new PassThrough({ objectMode: true }); +const both = pipelineP( + pick((async function* () { + yield "u1"; + yield "u2"; + })()), + out, +); +for await (const c of out) console.log("passthrough", c); +await both; +console.log("passthrough resolved"); + +// a Map source of entries into an object-mode sink +await new Promise((resolve) => { + pipeline( + pick(new Map([["k", 1]])), + new Writable({ objectMode: true, write(c, _e, cb) { console.log("map write", JSON.stringify(c)); cb(); } }), + (err) => { console.log("map done", err); resolve(); }, + ); +}); diff --git a/tests/corpus/7505-pipeline-untyped-sources.cjs b/tests/corpus/7505-pipeline-untyped-sources.cjs new file mode 100644 index 000000000..6dd67b4ff --- /dev/null +++ b/tests/corpus/7505-pipeline-untyped-sources.cjs @@ -0,0 +1,40 @@ +// pipeline() over untyped sources in CommonJS: a source that a function +// answers, which may be an array, a generator, a stream, a string, or a +// Buffer, whose bytes are numbers that a byte-mode Writable refuses. +const { pipeline, Readable, Writable } = require("node:stream"); + +function sink(label) { + const parts = []; + return new Writable({ + write(chunk, _enc, cb) { + parts.push(String(chunk)); + cb(); + }, + final(cb) { + console.log(label, parts.join("|")); + cb(); + }, + }); +} + +function* letters() { + yield "a"; + yield "b"; +} + +const sources = [ + () => ["one", "two"], + () => letters(), + () => Readable.from(["r1", "r2"]), + () => "xyz", + () => Buffer.from("buf"), +]; + +function run(i) { + if (i === sources.length) return; + pipeline(sources[i](), sink(`source ${i}`), (err) => { + console.log(`source ${i} done`, err === undefined ? "ok" : err.code); + run(i + 1); + }); +} +run(0); diff --git a/tests/corpus/7506-pipeline-any-source.ts b/tests/corpus/7506-pipeline-any-source.ts new file mode 100644 index 000000000..84eec0cf4 --- /dev/null +++ b/tests/corpus/7506-pipeline-any-source.ts @@ -0,0 +1,48 @@ +// pipeline() in TypeScript over sources typed any or unknown, and over a +// Set, a Map and an async generator object, which are pumped as iterables. +import { pipeline, Writable } from "node:stream"; +import { pipeline as pipelineP } from "node:stream/promises"; + +function sink(label: string): Writable { + const parts: string[] = []; + return new Writable({ + objectMode: true, + write(chunk: unknown, _enc, cb) { + parts.push(JSON.stringify(chunk)); + cb(); + }, + final(cb) { + console.log(label, parts.join(" ")); + cb(); + }, + }); +} + +async function* ticks(): AsyncGenerator { + yield "t1"; + await new Promise((r) => setTimeout(r, 1)); + yield "t2"; +} + +async function main(): Promise { + const anySource: any = ["any1", "any2"]; + await pipelineP(anySource, sink("any")); + const unknownSource: unknown = new Set([1, 2, 3]); + await pipelineP(unknownSource as any, sink("unknown")); + await pipelineP(new Set(["s"]), sink("set")); + await pipelineP( + new Map([ + ["a", 1], + ["b", 2], + ]), + sink("map"), + ); + await pipelineP(ticks(), sink("async generator")); + await new Promise((resolve) => { + pipeline(anySource, sink("callback"), (err?: Error | null) => { + console.log("callback done", err); + resolve(); + }); + }); +} +void main(); diff --git a/tests/corpus/7507-stream-write-error-drain.mjs b/tests/corpus/7507-stream-write-error-drain.mjs new file mode 100644 index 000000000..8298bcb42 --- /dev/null +++ b/tests/corpus/7507-stream-write-error-drain.mjs @@ -0,0 +1,37 @@ +// A write whose callback fails synchronously: write() answers false and +// writableNeedDrain stays false, so no 'drain' follows the error, and a +// pipeline pumping a generator stops at the failed write. +import { pipeline, Writable } from "node:stream"; + +const w = new Writable({ + write(_chunk, _enc, cb) { + cb(new Error("bad write")); + }, +}); +w.on("error", (e) => console.log("error event", e.message)); +w.on("drain", () => console.log("drain")); +const r = w.write("a"); +console.log("returned", r, "needDrain", w.writableNeedDrain, "errored", w.errored.message); + +function* source() { + try { + for (let i = 0; i < 4; i++) { + console.log("yield", i); + yield `c${i}`; + } + } finally { + console.log("source finally"); + } +} +let writes = 0; +pipeline( + source(), + new Writable({ + write(chunk, _enc, cb) { + writes++; + console.log("write", String(chunk)); + cb(writes === 2 ? new Error("second write failed") : null); + }, + }), + (err) => console.log("pipeline done", err.message), +); diff --git a/tests/harness/corpus-costs/7503-pipeline-untyped-sources.mjs.json b/tests/harness/corpus-costs/7503-pipeline-untyped-sources.mjs.json new file mode 100644 index 000000000..d2d431b0f --- /dev/null +++ b/tests/harness/corpus-costs/7503-pipeline-untyped-sources.mjs.json @@ -0,0 +1 @@ +{ "plain": 5499, "san": 7259 } diff --git a/tests/harness/corpus-costs/7504-pipeline-untyped-source-errors.mjs.json b/tests/harness/corpus-costs/7504-pipeline-untyped-source-errors.mjs.json new file mode 100644 index 000000000..4fd20681f --- /dev/null +++ b/tests/harness/corpus-costs/7504-pipeline-untyped-source-errors.mjs.json @@ -0,0 +1 @@ +{ "plain": 3431, "san": 4529 } diff --git a/tests/harness/corpus-costs/7505-pipeline-untyped-sources.cjs.json b/tests/harness/corpus-costs/7505-pipeline-untyped-sources.cjs.json new file mode 100644 index 000000000..3bb5df076 --- /dev/null +++ b/tests/harness/corpus-costs/7505-pipeline-untyped-sources.cjs.json @@ -0,0 +1 @@ +{ "plain": 3197, "san": 4220 } diff --git a/tests/harness/corpus-costs/7506-pipeline-any-source.ts.json b/tests/harness/corpus-costs/7506-pipeline-any-source.ts.json new file mode 100644 index 000000000..454dcb2c0 --- /dev/null +++ b/tests/harness/corpus-costs/7506-pipeline-any-source.ts.json @@ -0,0 +1 @@ +{ "plain": 3250, "san": 4741 } diff --git a/tests/harness/corpus-costs/7507-stream-write-error-drain.mjs.json b/tests/harness/corpus-costs/7507-stream-write-error-drain.mjs.json new file mode 100644 index 000000000..35f298bef --- /dev/null +++ b/tests/harness/corpus-costs/7507-stream-write-error-drain.mjs.json @@ -0,0 +1 @@ +{ "plain": 2735, "san": 4076 } diff --git a/tests/harness/self-hosting-native-toolchain.test.ts b/tests/harness/self-hosting-native-toolchain.test.ts index c5476328e..f4121bf88 100644 --- a/tests/harness/self-hosting-native-toolchain.test.ts +++ b/tests/harness/self-hosting-native-toolchain.test.ts @@ -109,7 +109,7 @@ for (const backend of ["llvm"] as const) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 37, marker: "scr_runtime_abi_v37" }, + runtime_abi: { version: 42, marker: "scr_runtime_abi_v42" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: [], sanitizer: "external-toolchain-required" }, flavors: { From 396c6944ecad5bda1d8ab9dd0179181aa0cf97a2 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 11:20:47 -0500 Subject: [PATCH 05/17] Report how well the native compiler's profile matches its functions The native compiler is built with a committed optimization profile. A function whose control flow changed since the profile was collected is optimized as without one, so the profile loses its gain as compiler code changes, with nothing to show it. With --profile-use, the helper now counts what the profile annotation reports while it applies the profile: functions that took the profile's counts, functions whose control flow no longer matches (hash mismatch), and functions the profile has no record of. It sums the counts over partitions, keeps each partition's counts beside its cached summary, and prints them as one line of JSON after a successful emit. The compiler reads that line (an older helper prints none) and returns it as the executable artifact's profileMatch. build-native-cli prints the share of profiled functions that match, and warns below 90% that the profile should be refreshed. On main aa1c26845, a profile refreshed on that commit matches 92.7% of the compiler's profiled functions, and the one committed about 200 commits earlier 87.2%; that one kept about half of the fresh profile's gain on a cold tsc-ts build. --- native/llvm-codegen/src/emit.cpp | 157 +++++++++++++++++- .../compiler/src/backend/native-codegen.ts | 10 +- .../src/backend/native-profile.test.ts | 13 ++ .../compiler/src/backend/native-profile.ts | 39 +++++ packages/compiler/src/compile-types.ts | 3 + packages/compiler/src/index.ts | 5 + scripts/build-native-cli.mts | 25 +++ 7 files changed, 242 insertions(+), 10 deletions(-) diff --git a/native/llvm-codegen/src/emit.cpp b/native/llvm-codegen/src/emit.cpp index 9cae164d3..35d7ce4f0 100644 --- a/native/llvm-codegen/src/emit.cpp +++ b/native/llvm-codegen/src/emit.cpp @@ -19,6 +19,7 @@ #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/IR/DiagnosticHandler.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/DiagnosticPrinter.h" #include "llvm/IR/LegacyPassManager.h" @@ -41,11 +42,13 @@ #include "llvm/Support/SourceMgr.h" #include "llvm/Support/ThreadPool.h" #include "llvm/Support/Threading.h" +#include "llvm/Support/JSON.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" #include "llvm/TargetParser/Triple.h" #include "llvm/Transforms/Utils/SplitModule.h" +#include #include #include #include @@ -170,6 +173,108 @@ void limitProfileImports() { (void)Limited; } +// How the --profile-use profile matched the functions it annotated, summed +// over partitions: functions that received its counts, functions whose +// control flow changed since it was collected (hash mismatch), and functions +// it has no record of (new, or never run in training: the merged profile is +// sparse). A partition reused from the cache without its counts is counted +// in Unknown. +struct ProfileMatch { + std::atomic Annotated{0}; + std::atomic Mismatched{0}; + std::atomic Missing{0}; + std::atomic Unknown{0}; + + void add(const ProfileMatch &Other) { + Annotated += Other.Annotated; + Mismatched += Other.Mismatched; + Missing += Other.Missing; + } + + std::string serialize() const { + return formatv("{0} {1} {2}", Annotated.load(), Mismatched.load(), + Missing.load()) + .str(); + } + + bool deserialize(StringRef Text) { + SmallVector Parts; + Text.split(Parts, ' '); + uint64_t Values[3]; + if (Parts.size() != 3) + return false; + for (size_t I = 0; I < 3; ++I) + if (Parts[I].getAsInteger(10, Values[I])) + return false; + Annotated += Values[0]; + Mismatched += Values[1]; + Missing += Values[2]; + return true; + } +}; + +// Counts the profile annotation's hash-mismatch and missing-function +// warnings instead of printing them. Every other diagnostic takes the +// context's default handling. +class ProfileMatchHandler : public DiagnosticHandler { +public: + explicit ProfileMatchHandler(ProfileMatch &Match) : Match(Match) {} + + bool handleDiagnostics(const DiagnosticInfo &Info) override { + if (Info.getKind() != DK_PGOProfile) + return false; + std::string Message; + raw_string_ostream Stream(Message); + DiagnosticPrinterRawOStream Printer(Stream); + Info.print(Printer); + if (StringRef(Message).contains("hash mismatch")) + ++Match.Mismatched; + else if (StringRef(Message).contains_insensitive("no profile data")) + ++Match.Missing; + else + return false; + return true; + } + +private: + ProfileMatch &Match; +}; + +// The annotation warns about functions the profile does not describe only +// when asked; the match counts need those warnings. Process-wide, set once. +void reportMissingProfiles() { + static const bool Reported = [] { + if (cl::Option *Option = + cl::getRegisteredOptions().lookup("pgo-warn-missing-function")) + Option->addOccurrence(0, "pgo-warn-missing-function", "true"); + return true; + }(); + (void)Reported; +} + +// Functions of `M` that the profile annotated with counts. +uint64_t annotatedFunctions(const Module &M) { + uint64_t Count = 0; + for (const Function &F : M) { + if (F.isDeclaration()) + continue; + std::optional Entry = F.getEntryCount(); + if (Entry && !Entry->isSynthetic()) + ++Count; + } + return Count; +} + +// Prints the profile match as the successful emit's one line of JSON. +void reportProfileMatch(const ProfileMatch &Match) { + json::Object Profile{{"annotated", Match.Annotated.load()}, + {"mismatched", Match.Mismatched.load()}, + {"missing", Match.Missing.load()}, + {"unknown_partitions", Match.Unknown.load()}}; + json::Object Response{{"ok", true}, {"profile", std::move(Profile)}}; + outs() << formatv("{0}\n", json::Value(std::move(Response))); +} + // Reads the profile's header once, before any partition needs it: it must // hold IR-level counts. std::optional inspectProfile(const EmitOptions &Options) { @@ -387,9 +492,16 @@ std::optional publish(StringRef TemporaryPath, std::optional emitModule(Module &M, TargetMachine &Machine, OptimizationLevel Level, std::optional Profile, CodeGenFileType Type, - StringRef OutputPath, std::vector> &Temporaries) { + StringRef OutputPath, std::vector> &Temporaries, + ProfileMatch &Match) { + bool Annotating = Profile && Profile->Action == PGOOptions::IRUse; + if (Annotating) + M.getContext().setDiagnosticHandler( + std::make_unique(Match)); Pipeline P(Machine, Level, std::move(Profile)); P.PB.buildPerModuleDefaultPipeline(Level).run(M, P.MAM); + if (Annotating) + Match.Annotated += annotatedFunctions(M); if (std::optional Failure = verify(M, "post_optimization_verification_failed")) return Failure; @@ -555,20 +667,32 @@ std::optional compileOptimizedPartitions(const EmitOptions &Options, const PartitionPlan &Plan, OptimizationLevel Level, StringRef Salt, const ArtifactCache &Cache, - std::vector> &Objects) { + std::vector> &Objects, + ProfileMatch &Match) { size_t Count = Plan.Modules.size(); + bool Annotating = !Options.ProfileUse.empty(); std::vector> Summaries(Count); SharedFailure Failures; forEachPartition(Count, [&](size_t I) { std::string Key = hashParts({Salt, "summary", Plan.Modules[I]}); - if ((Summaries[I] = Cache.lookup(Key, "bc"))) + if ((Summaries[I] = Cache.lookup(Key, "bc"))) { + if (Annotating) { + std::unique_ptr Counts = Cache.lookup(Key, "match"); + if (!Counts || !Match.deserialize(Counts->getBuffer())) + ++Match.Unknown; + } return; + } std::string LookupError; std::unique_ptr Machine = createTargetMachine(Options.Target, Options.OptLevel, LookupError); if (!Machine) return Failures.set({"target_machine_failed", LookupError}); LLVMContext Context; + ProfileMatch PartitionMatch; + if (Annotating) + Context.setDiagnosticHandler( + std::make_unique(PartitionMatch)); Expected> Loaded = loadPartition(Options, Plan.Modules[I], I, Context, *Machine); if (!Loaded) @@ -598,6 +722,12 @@ compileOptimizedPartitions(const EmitOptions &Options, /*EmitModuleHash=*/true)); MPM.run(Mod, P.MAM); } + if (Annotating) { + PartitionMatch.Annotated += annotatedFunctions(Mod); + Match.add(PartitionMatch); + // Stored before the summary, so a reused summary finds its counts. + Cache.store(Key, "match", PartitionMatch.serialize()); + } StringRef Contents(Bitcode.data(), Bitcode.size()); Cache.store(Key, "bc", Contents); Summaries[I] = MemoryBuffer::getMemBufferCopy(Contents); @@ -744,7 +874,8 @@ compileOptimizedPartitions(const EmitOptions &Options, std::optional compilePartitions(const EmitOptions &Options, const PartitionPlan &Plan, - std::vector> &Objects) { + std::vector> &Objects, + ProfileMatch &Match) { // Everything besides a partition's own bytes that determines its // artifacts. The caller's cache directory already identifies the exact // helper build; source content and partition count arrive per module. @@ -778,7 +909,7 @@ compilePartitions(const EmitOptions &Options, const PartitionPlan &Plan, return compileUnoptimizedPartitions(Options, Plan, Salt, Cache, Objects); return compileOptimizedPartitions(Options, Plan, optimizationLevel(Options.OptLevel), Salt, - Cache, Objects); + Cache, Objects, Match); } // Publishes in-memory partition objects through private sibling files. @@ -819,6 +950,8 @@ int emitPartitions(const EmitOptions &Options) { std::vector> Objects; std::optional Failure; bool Compiled = false; + // A failed attempt's counts are discarded with its objects. + auto Match = std::make_unique(); { ErrorOr> Input = MemoryBuffer::getFile(Options.Input, /*IsText=*/false, @@ -830,15 +963,16 @@ int emitPartitions(const EmitOptions &Options) { Input->reset(); PartitionPlan Plan{std::move(Text->Sources), std::move(Text->PublicSymbols)}; - Compiled = !compilePartitions(Options, Plan, Objects); + Compiled = !compilePartitions(Options, Plan, Objects, *Match); } } } if (!Compiled) { + Match = std::make_unique(); PartitionPlan Plan; Failure = splitParsedModule(Options, Plan); if (!Failure) - Failure = compilePartitions(Options, Plan, Objects); + Failure = compilePartitions(Options, Plan, Objects, *Match); } std::vector> Temporaries; if (!Failure) @@ -851,6 +985,8 @@ int emitPartitions(const EmitOptions &Options) { return reportError(Failure->Code, Failure->Message, Options.DiagnosticFormat); } + if (!Options.ProfileUse.empty()) + reportProfileMatch(*Match); return 0; } @@ -895,6 +1031,8 @@ int emit(const EmitOptions &Options) { return reportError("invalid_import", "runtime bitcode import requires an optimized build", Options.DiagnosticFormat); + if (!Options.ProfileUse.empty()) + reportMissingProfiles(); if (std::optional Failure = inspectProfile(Options)) return reportError(Failure->Code, Failure->Message, Options.DiagnosticFormat); @@ -957,9 +1095,10 @@ int emit(const EmitOptions &Options) { ? CodeGenFileType::ObjectFile : CodeGenFileType::AssemblyFile; std::vector> Temporaries; + ProfileMatch Match; std::optional Failure = emitModule(*Mod, *Machine, Level, profileOptions(Options), Type, - Options.Outputs[0], Temporaries); + Options.Outputs[0], Temporaries, Match); if (!Failure) Failure = publish(Temporaries[0], Options.Outputs[0]); if (Failure) { @@ -968,6 +1107,8 @@ int emit(const EmitOptions &Options) { return reportError(Failure->Code, Failure->Message, Options.DiagnosticFormat); } + if (!Options.ProfileUse.empty()) + reportProfileMatch(Match); return 0; } diff --git a/packages/compiler/src/backend/native-codegen.ts b/packages/compiler/src/backend/native-codegen.ts index 5b91c60e7..413fb7e02 100644 --- a/packages/compiler/src/backend/native-codegen.ts +++ b/packages/compiler/src/backend/native-codegen.ts @@ -30,6 +30,8 @@ import { compilerReleaseVersion } from "../library/sidecar.js"; import { nativeProfileHelperArgs, nativeProfileKey, + parseNativeProfileMatch, + type NativeProfileMatch, type NativeProfileOptions, } from "./native-profile.js"; import { writeTextParts } from "../text-parts.js"; @@ -69,6 +71,8 @@ export interface NativeCodegenArtifact { dependencies: NativeArtifactDependency[]; /** Every emitted file, starting with the requested output path. */ outputPaths: string[]; + /** How the build's profile matched, when it used one and the helper ran. */ + profileMatch?: NativeProfileMatch; } const resolvedHelperCache = new Map>(); @@ -458,7 +462,7 @@ export async function emitNativeArtifact( try { if (isLlvmFile(options.llvm)) await copyFile(options.llvm.file, input); else await writeLlvmSource(input, options.llvm, 0o600); - await invoke(helper.binaryPath, [ + const { stdout } = await invoke(helper.binaryPath, [ "emit", "--input", input, @@ -502,7 +506,9 @@ export async function emitNativeArtifact( for (let index = 0; index < stages.length; index++) await rename(stages[index]!, outputs[index]!); await pruneBuildCache(root); - return artifact; + const profileMatch = + options.profile?.use === undefined ? null : parseNativeProfileMatch(stdout); + return profileMatch === null ? artifact : { ...artifact, profileMatch }; } finally { await Promise.all([ ...stages.map((stage) => rm(stage, { force: true }).catch(() => undefined)), diff --git a/packages/compiler/src/backend/native-profile.test.ts b/packages/compiler/src/backend/native-profile.test.ts index 590c39eef..b2cb0c75d 100644 --- a/packages/compiler/src/backend/native-profile.test.ts +++ b/packages/compiler/src/backend/native-profile.test.ts @@ -1,6 +1,7 @@ import { expect, test } from "vitest"; import { executableLinkInputs } from "./link-plan-core.js"; import { + parseNativeProfileMatch, nativeProfileHelperArgs, nativeProfileKey, nativeProfileRefusal, @@ -71,3 +72,15 @@ test("only the instrumented link adds the profile runtime", () => { "-fprofile-generate", ); }); + +test("the helper's profile match is read from its output", () => { + expect( + parseNativeProfileMatch( + '{"ok":true,"profile":{"annotated":120,"mismatched":7,"missing":40,"unknown_partitions":1}}\n', + ), + ).toEqual({ annotated: 120, mismatched: 7, missing: 40, unknownPartitions: 1 }); + // A helper that predates the report prints nothing. + expect(parseNativeProfileMatch("")).toBeNull(); + expect(parseNativeProfileMatch('{"ok":true}\n')).toBeNull(); + expect(parseNativeProfileMatch('{"ok":true,"profile":{"annotated":-1}}\n')).toBeNull(); +}); diff --git a/packages/compiler/src/backend/native-profile.ts b/packages/compiler/src/backend/native-profile.ts index 4cd766319..904994a7d 100644 --- a/packages/compiler/src/backend/native-profile.ts +++ b/packages/compiler/src/backend/native-profile.ts @@ -44,6 +44,45 @@ export function nativeProfileHelperArgs(profile: NativeProfileOptions | undefine return []; } +/** How an optimized build's profile matched the program's functions, as the + * helper reports it: functions that took the profile's counts, functions + * whose control flow changed since the profile was collected, functions the + * profile has no record of (new, or never run while it was collected), and + * partitions reused from the cache without their counts. */ +export interface NativeProfileMatch { + annotated: number; + mismatched: number; + missing: number; + unknownPartitions: number; +} + +/** The profile match in a successful helper emit's output, or null when the + * helper reports none (a helper that predates the report). */ +export function parseNativeProfileMatch(stdout: string): NativeProfileMatch | null { + for (const line of stdout.split("\n")) { + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch { + continue; + } + const profile = (parsed as { profile?: Record } | null)?.profile; + if (typeof profile !== "object" || profile === null) continue; + const count = (key: string): number | null => { + const value = profile[key]; + return typeof value === "number" && Number.isInteger(value) && value >= 0 ? value : null; + }; + const annotated = count("annotated"); + const mismatched = count("mismatched"); + const missing = count("missing"); + const unknownPartitions = count("unknown_partitions"); + if (annotated === null || mismatched === null || missing === null || unknownPartitions === null) + continue; + return { annotated, mismatched, missing, unknownPartitions }; + } + return null; +} + /** Cache-key part; empty without a profile so other keys are unchanged. */ export function nativeProfileKey(profile: NativeProfileOptions | undefined): string { if (profile?.generate !== undefined) return `profile-generate:${profile.generate}`; diff --git a/packages/compiler/src/compile-types.ts b/packages/compiler/src/compile-types.ts index c50931c6a..2a5cea5ea 100644 --- a/packages/compiler/src/compile-types.ts +++ b/packages/compiler/src/compile-types.ts @@ -2,6 +2,7 @@ import type { TargetCpu } from "./backend/target-cpu.js"; import type { WindowsSubsystem } from "./backend/targets.js"; import type { NativeLinkInfo } from "./backend/native-link-info.js"; import type { NativeOptimization } from "./backend/optimization.js"; +import type { NativeProfileMatch } from "./backend/native-profile.js"; import type { ScrDiagnostic } from "./diagnostics/diagnostic.js"; import type { CoverageInput } from "./coverage/report.js"; import type { TierMode } from "./frontend/tier-analysis.js"; @@ -100,6 +101,8 @@ export type CompileArtifact = path: string; translationUnitPath: string; backend: "llvm"; + /** How a `profileUse` build's profile matched the program's functions. */ + profileMatch?: NativeProfileMatch; }; export type CompileFailure = { diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index e2d1c97cc..607298427 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -94,6 +94,7 @@ import { createNativeLinkPlan } from "./backend/link-plan.js"; import { nativeProfileKey, nativeProfileRefusal, + type NativeProfileMatch, type NativeProfileOptions, } from "./backend/native-profile.js"; import { @@ -659,6 +660,7 @@ interface NativeProgramObject { partitionPaths: string[]; artifactPath: string; dependencies: NativeArtifactDependency[]; + profileMatch?: NativeProfileMatch; } async function emitNativeProgramObject( @@ -694,6 +696,7 @@ async function emitNativeProgramObject( partitionPaths: artifact.outputPaths.slice(1), artifactPath, dependencies: artifact.dependencies, + ...(artifact.profileMatch === undefined ? {} : { profileMatch: artifact.profileMatch }), }; } catch (error) { await removeNativeProgramObject(linkPath, nativePartitionPaths(linkPath, partitions)); @@ -1851,6 +1854,7 @@ async function compileTracked( } await pruneBuildCache(cacheRoot); timing("complete"); + const profileMatch = nativeProgramObject?.profileMatch; return { ok: true, artifact: { @@ -1858,6 +1862,7 @@ async function compileTracked( path: opts.outPath, translationUnitPath: llvmPath, backend, + ...(profileMatch === undefined ? {} : { profileMatch }), }, binaryPath: opts.outPath, llvmPath, diff --git a/scripts/build-native-cli.mts b/scripts/build-native-cli.mts index e539cea1d..30e0ae39b 100644 --- a/scripts/build-native-cli.mts +++ b/scripts/build-native-cli.mts @@ -20,6 +20,12 @@ const sanitize = process.env["SCRIPTC_SAN"] === "1"; // SCRIPTC_NATIVE_PROFILE_USE names; an empty SCRIPTC_NATIVE_PROFILE_USE // builds without one. const profileGenerate = process.env["SCRIPTC_NATIVE_PROFILE_GENERATE"] || undefined; +// Below this share of matching functions, the build warns that the profile +// has lost most of its gain. A profile refreshed on the same commit matched +// 92.7% of the native compiler's profiled functions, one refreshed 1.5 days +// and about 200 commits earlier 87.2%, and that one kept about half of the +// fresh profile's gain on a cold tsc-ts build. +const PROFILE_MATCH_WARNING = 0.9; const committedProfile = join(root, "packages/compiler/native/cli.profdata.gz"); const requestedProfile = process.env["SCRIPTC_NATIVE_PROFILE_USE"]; const useProfile = @@ -127,6 +133,25 @@ if (!result.ok) ) .join("\n"), ); +// How well the profile still describes the compiler: a function whose +// control flow changed since the profile was collected is optimized as +// without one. The share falls as compiler code changes, and with it the +// profile's gain (refresh it with scripts/refresh-cli-profile.mjs). +const match = useProfile && "artifact" in result ? result.artifact.profileMatch : undefined; +if (match !== undefined) { + const profiled = match.annotated + match.mismatched; + const share = profiled === 0 ? 0 : match.annotated / profiled; + console.log( + `native compiler profile: ${match.annotated} of ${profiled} profiled functions match (${(100 * share).toFixed(1)}%); ${match.missing} functions have no profile` + + (match.unknownPartitions > 0 + ? `; ${match.unknownPartitions} partitions reused without counts` + : ""), + ); + if (match.unknownPartitions === 0 && share < PROFILE_MATCH_WARNING) + console.warn( + `warning: the native compiler profile matches ${(100 * share).toFixed(1)}% of the functions it describes; run scripts/refresh-cli-profile.mjs`, + ); +} const compilerVersion = ( JSON.parse(readFileSync(join(root, "packages/compiler/package.json"), "utf8")) as { version: string; From 41664c903b18e943bb26e94c0cd743692b364036 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 12:03:38 -0500 Subject: [PATCH 06/17] Walk files iteratively in the untyped collection inference The declaration and reference sweeps recursed through ts.forEachChild, two JS frames per AST level, so a 7000-level expression overflowed the stack before the expression lowering's 200-level SC1090 fence. Both sweeps use ts.walkPreorder now, and primitiveOf answers null past the same 200 levels. --- .../frontend/lowering/js-collection-types.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/js-collection-types.ts b/packages/compiler/src/frontend/lowering/js-collection-types.ts index 2be3c067f..8820ade4a 100644 --- a/packages/compiler/src/frontend/lowering/js-collection-types.ts +++ b/packages/compiler/src/frontend/lowering/js-collection-types.ts @@ -55,6 +55,8 @@ const MAP_LOOKUPS = new Set(["get", "has", "delete"]); const SET_LOOKUPS = new Set(["has", "delete"]); const ARRAY_READS = new Set(["pop", "shift"]); const ARRAY_SEARCHES = new Set(["includes", "indexOf"]); +/** The expression lowering refuses deeper nesting (SC1090). */ +const MAX_DEPTH = 200; /** The inferred type of the binding a `new Map()` or `new Set()` initializes, * which the construction adopts. */ @@ -153,9 +155,8 @@ function inferFile(lowerer: Lowerer, file: ts.SourceFile): Candidate[] { value: undefined, }); } - ts.forEachChild(node, declarations); }; - declarations(file); + ts.walkPreorder(file, declarations); if (directEval || candidates.size === 0) return []; const names = new Set( @@ -171,9 +172,8 @@ function inferFile(lowerer: Lowerer, file: ts.SourceFile): Candidate[] { else refused.add(symbol); } } - ts.forEachChild(node, references); }; - references(file); + ts.walkPreorder(file, references); for (const symbol of refused) candidates.delete(symbol); const proven = [...candidates.values()].filter( (c) => c.valueWrites.length > 0 || c.keyWrites.length > 0, @@ -381,16 +381,20 @@ function checkerPrimitive(type: ts.Type): Primitive | null { /** The primitive type `e` always evaluates to, or null. The checker answers * where it knows; where it types `e` as `any`, the value's structure does: * a read of a decided candidate, arithmetic and comparisons over such - * values, and constants defined from them. */ + * values, and constants defined from them. An expression nested deeper than + * the lowering's nesting limit answers null. */ function primitiveOf( lowerer: Lowerer, bySymbol: Map, e: ts.Expression, visiting: Set, + depth = 0, ): Primitive | null { + if (depth > MAX_DEPTH) return null; const type = lowerer.checker.getTypeAtLocation(e); if ((type.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown)) === 0) return checkerPrimitive(type); - const of = (x: ts.Expression): Primitive | null => primitiveOf(lowerer, bySymbol, x, visiting); + const of = (x: ts.Expression): Primitive | null => + primitiveOf(lowerer, bySymbol, x, visiting, depth + 1); if (ts.isParenthesizedExpression(e)) return of(e.expression); if (ts.isPrefixUnaryExpression(e)) { if (e.operator === ts.SyntaxKind.ExclamationToken) return "boolean"; @@ -461,7 +465,7 @@ function primitiveOf( (declaration.parent.flags & ts.NodeFlags.Const) !== 0 ) { visiting.add(e); - const p = primitiveOf(lowerer, bySymbol, declaration.initializer, visiting); + const p = primitiveOf(lowerer, bySymbol, declaration.initializer, visiting, depth + 1); visiting.delete(e); return p; } From 55740d094a6b461abb886be66f395a577fe21205 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 14:33:02 -0500 Subject: [PATCH 07/17] Resolve corpus promises 7503 and 7504 with an explicit undefined tsgo checks JavaScript calls to an untyped Promise's resolve against its one parameter, so resolve() with no argument is an error there while tsc-ts accepts it. The two programs pass undefined, which both checkers accept and which settles the promise the same way. --- tests/corpus/7503-pipeline-untyped-sources.mjs | 10 +++++----- tests/corpus/7504-pipeline-untyped-source-errors.mjs | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/corpus/7503-pipeline-untyped-sources.mjs b/tests/corpus/7503-pipeline-untyped-sources.mjs index 3dbc90eee..e676fd607 100644 --- a/tests/corpus/7503-pipeline-untyped-sources.mjs +++ b/tests/corpus/7503-pipeline-untyped-sources.mjs @@ -58,7 +58,7 @@ async function run(name) { queueMicrotask(() => log.push(`${name} microtask`)); pipeline(src, sink(name, objectMode), (err) => { log.push(`${name} done ${err ? err.code ?? err.message : err}`); - resolve(); + resolve(undefined); }); log.push(`${name} after call`); }); @@ -72,7 +72,7 @@ await new Promise((resolve) => src2, new Transform({ transform(c, _e, cb) { cb(null, c.toString().toUpperCase()); } }), sink("transform"), - (err) => { log.push(`transform done ${err}`); resolve(); }, + (err) => { log.push(`transform done ${err}`); resolve(undefined); }, ), ); const src3 = new Set(["f1", "f2"]); @@ -81,7 +81,7 @@ await new Promise((resolve) => src3, async function* (source) { for await (const c of source) yield c + "!"; }, sink("fnstage"), - (err) => { log.push(`fnstage done ${err}`); resolve(); }, + (err) => { log.push(`fnstage done ${err}`); resolve(undefined); }, ), ); // promises form @@ -92,11 +92,11 @@ log.push("promises resolved"); function* bad() { yield "ok"; throw new Error("source failed"); } const src5 = bad(); await new Promise((resolve) => - pipeline(src5, sink("bad"), (err) => { log.push(`bad done ${err && err.message}`); resolve(); }), + pipeline(src5, sink("bad"), (err) => { log.push(`bad done ${err && err.message}`); resolve(undefined); }), ); // a source of non-string values into a byte-mode writable const src6 = [1, 2]; await new Promise((resolve) => - pipeline(src6, sink("numbers"), (err) => { log.push(`numbers done ${err && err.code}`); resolve(); }), + pipeline(src6, sink("numbers"), (err) => { log.push(`numbers done ${err && err.code}`); resolve(undefined); }), ); console.log(log.join("\n")); diff --git a/tests/corpus/7504-pipeline-untyped-source-errors.mjs b/tests/corpus/7504-pipeline-untyped-source-errors.mjs index fcbfa9d60..9c677c011 100644 --- a/tests/corpus/7504-pipeline-untyped-source-errors.mjs +++ b/tests/corpus/7504-pipeline-untyped-source-errors.mjs @@ -30,7 +30,7 @@ await new Promise((resolve) => { }); pipeline(pick(counted("f", 5)), dst, (err) => { console.log("fail done", err && err.message); - resolve(); + resolve(undefined); }); }); @@ -45,7 +45,7 @@ await new Promise((resolve) => { }); pipeline(pick(counted("b", 3)), dst, (err) => { console.log("slow done", err); - resolve(); + resolve(undefined); }); }); @@ -85,6 +85,6 @@ await new Promise((resolve) => { pipeline( pick(new Map([["k", 1]])), new Writable({ objectMode: true, write(c, _e, cb) { console.log("map write", JSON.stringify(c)); cb(); } }), - (err) => { console.log("map done", err); resolve(); }, + (err) => { console.log("map done", err); resolve(undefined); }, ); }); From 70f565177e1823c1a67853b9fe46da50e8572fae Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 16:05:36 -0500 Subject: [PATCH 08/17] Count profile-annotated functions when the annotation runs The match report counted the functions with profile counts after the whole pipeline, so an annotated function that inlining or dead code elimination later removed was not counted, while the annotation's mismatch and missing-function warnings were. A pass-instrumentation callback now counts the annotated functions right after PGOInstrumentationUse runs, in the one-module and the partitioned pipelines. A new test builds helpers that reach the annotation and are then inlined into main and deleted, and checks that the report counts them. --- native/llvm-codegen/src/emit.cpp | 23 +++- packages/compiler/test/native-profile.test.ts | 100 +++++++++++++++++- 2 files changed, 114 insertions(+), 9 deletions(-) diff --git a/native/llvm-codegen/src/emit.cpp b/native/llvm-codegen/src/emit.cpp index 35d7ce4f0..14bcd1522 100644 --- a/native/llvm-codegen/src/emit.cpp +++ b/native/llvm-codegen/src/emit.cpp @@ -296,6 +296,7 @@ std::optional inspectProfile(const EmitOptions &Options) { // Owns the analysis managers a pipeline needs for one module. struct Pipeline { + PassInstrumentationCallbacks Callbacks; LoopAnalysisManager LAM; FunctionAnalysisManager FAM; CGSCCAnalysisManager CGAM; @@ -304,13 +305,26 @@ struct Pipeline { Pipeline(TargetMachine &Machine, OptimizationLevel Level, std::optional Profile = std::nullopt) - : PB(&Machine, tuningFor(Level), std::move(Profile)) { + : PB(&Machine, tuningFor(Level), std::move(Profile), &Callbacks) { PB.registerModuleAnalyses(MAM); PB.registerCGSCCAnalyses(CGAM); PB.registerFunctionAnalyses(FAM); PB.registerLoopAnalyses(LAM); PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); } + + // Adds to Count the functions the profile annotation gave counts to, as + // soon as that pass has run: inlining and dead code elimination later in + // the pipeline delete annotated functions. + void countAnnotated(std::atomic &Count) { + Callbacks.registerAfterPassCallback( + [&Count](StringRef PassID, Any IR, const PreservedAnalyses &) { + if (PassID != "PGOInstrumentationUse") + return; + if (const auto *M = llvm::any_cast(&IR)) + Count += annotatedFunctions(**M); + }); + } }; std::optional verify(Module &M, StringRef Code) { @@ -499,9 +513,9 @@ emitModule(Module &M, TargetMachine &Machine, OptimizationLevel Level, M.getContext().setDiagnosticHandler( std::make_unique(Match)); Pipeline P(Machine, Level, std::move(Profile)); - P.PB.buildPerModuleDefaultPipeline(Level).run(M, P.MAM); if (Annotating) - Match.Annotated += annotatedFunctions(M); + P.countAnnotated(Match.Annotated); + P.PB.buildPerModuleDefaultPipeline(Level).run(M, P.MAM); if (std::optional Failure = verify(M, "post_optimization_verification_failed")) return Failure; @@ -716,6 +730,8 @@ compileOptimizedPartitions(const EmitOptions &Options, // Profile instrumentation or annotation happens here, before the thin // link; the post-link optimization reads the annotated weights. Pipeline P(*Machine, Level, profileOptions(Options)); + if (Annotating) + P.countAnnotated(PartitionMatch.Annotated); ModulePassManager MPM = P.PB.buildThinLTOPreLinkDefaultPipeline(Level); MPM.addPass(BitcodeWriterPass(Stream, /*ShouldPreserveUseListOrder=*/false, /*EmitSummaryIndex=*/true, @@ -723,7 +739,6 @@ compileOptimizedPartitions(const EmitOptions &Options, MPM.run(Mod, P.MAM); } if (Annotating) { - PartitionMatch.Annotated += annotatedFunctions(Mod); Match.add(PartitionMatch); // Stored before the summary, so a reused summary finds its counts. Cache.store(Key, "match", PartitionMatch.serialize()); diff --git a/packages/compiler/test/native-profile.test.ts b/packages/compiler/test/native-profile.test.ts index 53d077017..9d04739d0 100644 --- a/packages/compiler/test/native-profile.test.ts +++ b/packages/compiler/test/native-profile.test.ts @@ -110,6 +110,65 @@ function expectedExit(argc: number, edit = 0): number { return Number(total % 251n); } +const INLINED = 6; + +/** A program whose helpers are too large for the inliner that runs before the + * profile annotation and each called twice, so they reach the annotation, + * and which the profile-guided inliner then inlines into their hot call + * sites in `main` and deletes. */ +function inlinedProgram(): string { + const lines = ["; profiled program, helpers inlined after annotation"]; + for (let index = 0; index < INLINED; index++) { + lines.push( + `define internal i64 @g${index}(i64 %n) {`, + "entry:", + " %x0 = add i64 %n, " + index, + ); + for (let step = 1; step <= 40; step++) + lines.push(` %x${step} = ${step % 2 === 0 ? "xor" : "mul"} i64 %x${step - 1}, ${step + 3}`); + lines.push(" ret i64 %x40", "}"); + } + lines.push( + "define i32 @main(i32 %argc, ptr %argv) {", + "entry:", + " %wide = sext i32 %argc to i64", + " %limit = mul i64 %wide, 1000", + " br label %loop", + "loop:", + " %i = phi i64 [ 0, %entry ], [ %next, %loop ]", + " %total = phi i64 [ 0, %entry ], [ %grown, %loop ]", + ...Array.from({ length: INLINED }, (_, index) => [ + ` %a${index} = call i64 @g${index}(i64 %i)`, + ` %b${index} = call i64 @g${index}(i64 %a${index})`, + ` %t${index} = add i64 ${index === 0 ? "%total" : `%t${index - 1}`}, %b${index}`, + ]).flat(), + ` %grown = add i64 %t${INLINED - 1}, 0`, + " %next = add i64 %i, 1", + " %again = icmp ult i64 %next, %limit", + " br i1 %again, label %loop, label %exit", + "exit:", + " %low = urem i64 %grown, 251", + " %code = trunc i64 %low to i32", + " ret i32 %code", + "}", + ); + return lines.join("\n") + "\n"; +} + +function inlinedExit(argc: number): number { + const mask = (1n << 64n) - 1n; + const g = (index: number, n: bigint) => { + let x = (n + BigInt(index)) & mask; + for (let step = 1; step <= 40; step++) + x = step % 2 === 0 ? x ^ BigInt(step + 3) : (x * BigInt(step + 3)) & mask; + return x; + }; + let total = 0n; + for (let i = 0n; i < BigInt(argc * 1000); i++) + for (let index = 0; index < INLINED; index++) total = (total + g(index, g(index, i))) & mask; + return Number(total % 251n); +} + type EmitOptions = { outputs: number; name: string; @@ -119,12 +178,17 @@ type EmitOptions = { }; async function emit(dir: string, source: string, options: EmitOptions): Promise { + return (await emitReporting(dir, source, options)).outputs; +} + +/** `emit`, also returning the helper's stdout (its profile match report). */ +async function emitReporting(dir: string, source: string, options: EmitOptions) { const input = join(dir, `${options.name}.ll`); await writeFile(input, source); const outputs = Array.from({ length: options.outputs }, (_, index) => join(dir, `${options.name}.${index}.o`), ); - await execFileAsync(helper!, [ + const { stdout } = await execFileAsync(helper!, [ "emit", "--input", input, @@ -145,7 +209,7 @@ async function emit(dir: string, source: string, options: EmitOptions): Promise< ...(options.use === undefined ? [] : ["--profile-use", options.use]), ...(options.cache === undefined ? [] : ["--cache-dir", options.cache]), ]); - return outputs; + return { outputs, stdout }; } async function link(dir: string, objects: string[], name: string, instrumented = false) { @@ -181,12 +245,18 @@ async function merge(output: string, inputs: string[]) { /** An instrumented build of `source` run with 1 and 3 arguments, its raw * profiles merged into one indexed profile. */ -async function collect(dir: string, source: string, outputs: number, name: string) { +async function collect( + dir: string, + source: string, + outputs: number, + name: string, + expected: (argc: number) => number = expectedExit, +) { const raw = join(dir, `${name}-raw`); const objects = await emit(dir, source, { outputs, name, generate: join(raw, "%p.profraw") }); const executable = await link(dir, objects, name, true); - expect(await run(executable, 1)).toBe(expectedExit(1)); - expect(await run(executable, 3)).toBe(expectedExit(3)); + expect(await run(executable, 1)).toBe(expected(1)); + expect(await run(executable, 3)).toBe(expected(3)); const profiles = (await readdir(raw)).map((file) => join(raw, file)); expect(profiles).toHaveLength(2); const profile = join(dir, `${name}.profdata`); @@ -286,6 +356,26 @@ describe.runIf(helper !== null)("LLVM profile-guided optimization", () => { expect(await run(await link(dir, objects, "edited"), 2)).toBe(expectedExit(2, 7)); }); + test("the match report counts annotated functions that inlining removes", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-profile-match-")); + dirs.push(dir); + const { profile } = await collect(dir, inlinedProgram(), 1, "train", inlinedExit); + const { outputs, stdout } = await emitReporting(dir, inlinedProgram(), { + outputs: 1, + name: "guided", + use: profile, + }); + expect(await run(await link(dir, outputs, "guided"), 2)).toBe(inlinedExit(2)); + // Every g and main received counts, though optimization then + // inlined the helpers into main and deleted them. + expect(JSON.parse(stdout.trim().split("\n").at(-1)!)).toEqual({ + ok: true, + profile: { annotated: INLINED + 1, mismatched: 0, missing: 0, unknown_partitions: 0 }, + }); + const symbols = await execFileAsync("nm", [outputs[0]!]); + expect(symbols.stdout).not.toMatch(/\bg\d\b/); + }); + test("the profile keys the partition cache", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-profile-cache-")); dirs.push(dir); From 530bd9534ffc45ca8a29db5c971b533c407d55d6 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 16:10:00 -0500 Subject: [PATCH 09/17] Await the values of untyped sync pipeline sources An untyped pipeline source that is a sync iterable passed each value on as it came, so a promise reached the next stage as the promise: a byte-mode destination refused it and an object-mode one received it. for await over a sync iterator, which Node's pipeline pumps it with, awaits each value. The source's generator now awaits a value that is an object (a promise included) before yielding it, and a rejection closes the source, as Node's async-from-sync iterator does. --- .../src/frontend/lowering/lower-stream.ts | 44 ++++++++++++++++++- .../7504-pipeline-untyped-source-errors.mjs | 27 +++++++++++- 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index 72bea5dd7..b519ce378 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -3947,11 +3947,13 @@ function lowerDynIterableSource( const syncId = frame.local("sync", BOOL, false); const nextId = frame.local("next", DYN, false); const stepId = frame.local("step", DYN, true); + const valueId = frame.local("value", DYN, true); const activeId = frame.local("active", BOOL, true); const closeId = frame.local("close", DYN, false); const it = (): IrExpr => varRef(itId, DYN, loc); const sync = (): IrExpr => varRef(syncId, BOOL, loc); const step = (): IrExpr => varRef(stepId, DYN, loc); + const value = (): IrExpr => varRef(valueId, DYN, loc); const get = (obj: IrExpr, name: string): IrExpr => ({ kind: "dynKeyGet", value: obj, @@ -4011,6 +4013,7 @@ function lowerDynIterableSource( }, { kind: "varDecl", localId: nextId, init: get(it(), "next"), loc }, { kind: "varDecl", localId: stepId, init: dynUndefinedExpr(loc), loc }, + { kind: "varDecl", localId: valueId, init: dynUndefinedExpr(loc), loc }, { kind: "varDecl", localId: activeId, init: boolLit(false, loc), loc }, { kind: "tryCatch", @@ -4055,9 +4058,48 @@ function lowerDynIterableSource( loc, }, setActive(true), + { kind: "assign", localId: valueId, value: get(step(), "value"), loc }, + // for await over a sync iterator awaits each value + // (CreateAsyncFromSyncIterator): a promise or a thenable + // passes on its result, and a rejection closes the iterator + { + kind: "if", + cond: { + kind: "logical", + op: "&&", + left: sync(), + right: { + kind: "logical", + op: "||", + left: { kind: "dynTest", test: "promise", value: value(), type: BOOL, loc }, + right: { + kind: "logical", + op: "&&", + left: { + kind: "dynTest", + test: "nullish", + negated: true, + value: value(), + type: BOOL, + loc, + }, + right: { kind: "dynTest", test: "object", value: value(), type: BOOL, loc }, + type: BOOL, + loc, + }, + type: BOOL, + loc, + }, + type: BOOL, + loc, + }, + then: [{ kind: "assign", localId: valueId, value: awaitDyn(value()), loc }], + else_: null, + loc, + }, { kind: "exprStmt", - expr: { kind: "yieldExpr", value: get(step(), "value"), type: VOID, loc }, + expr: { kind: "yieldExpr", value: value(), type: VOID, loc }, loc, }, ], diff --git a/tests/corpus/7504-pipeline-untyped-source-errors.mjs b/tests/corpus/7504-pipeline-untyped-source-errors.mjs index 9c677c011..3ee984547 100644 --- a/tests/corpus/7504-pipeline-untyped-source-errors.mjs +++ b/tests/corpus/7504-pipeline-untyped-source-errors.mjs @@ -1,6 +1,7 @@ // pipeline() over untyped iterable sources: a destination that fails closes // the source, backpressure waits for drain, an abort signal stops the pump, -// and async generator and Map sources reach object-mode destinations. +// async generator and Map sources reach object-mode destinations, and a +// sync source's promise values are awaited. import { pipeline, Writable, PassThrough } from "node:stream"; import { pipeline as pipelineP } from "node:stream/promises"; @@ -88,3 +89,27 @@ await new Promise((resolve) => { (err) => { console.log("map done", err); resolve(undefined); }, ); }); + +// sync sources whose values are promises: each value is awaited, as +// for await over a sync iterator does, and a rejection closes the source +await pipelineP( + pick([Promise.resolve("p1"), "p2", new Promise((r) => setTimeout(() => r("p3"), 1))]), + new Writable({ write(c, _e, cb) { console.log("byte write", String(c)); cb(); } }), +); +function* promised() { + try { + yield Promise.resolve({ n: 1 }); + yield Promise.reject(new Error("value rejected")); + yield "never"; + } finally { + console.log("promised finally"); + } +} +try { + await pipelineP( + pick(promised()), + new Writable({ objectMode: true, write(c, _e, cb) { console.log("object write", JSON.stringify(c)); cb(); } }), + ); +} catch (err) { + console.log("promised rejected", err.message); +} From af33be8046e3a3833d44dcf63880082cccea4257 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 16:16:49 -0500 Subject: [PATCH 10/17] Treat a collection exported through an export list as escaping A reference in `export { m }` resolved to the export's alias symbol, not to the local binding, so the inference did not see the export and typed the collection from its local uses. An importing module then used it as an untyped Map and failed to compile (SC9001). An unrenamed export specifier now resolves to the binding it exports, which refuses the inference as `export const` does. Corpus 7985 adds `export { listed, renamed as alias }`. --- .../src/frontend/lowering/js-collection-types.ts | 12 +++++++++++- packages/compiler/test/js-collection-types.test.ts | 2 ++ tests/corpus/7985-untyped-collection-types.mjs | 9 +++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/compiler/src/frontend/lowering/js-collection-types.ts b/packages/compiler/src/frontend/lowering/js-collection-types.ts index 8820ade4a..a4c182523 100644 --- a/packages/compiler/src/frontend/lowering/js-collection-types.ts +++ b/packages/compiler/src/frontend/lowering/js-collection-types.ts @@ -225,7 +225,17 @@ function referencedSymbol(lowerer: Lowerer, id: ts.Identifier): ts.Symbol | unde const parent = id.parent; if (parent !== undefined && ts.isShorthandPropertyAssignment(parent) && parent.name === id) return lowerer.checker.getShorthandAssignmentValueSymbol(parent); - return lowerer.checker.getSymbolAtLocation(id); + const symbol = lowerer.checker.getSymbolAtLocation(id); + // `export { m }` names the export, an alias of the local binding. + if ( + symbol !== undefined && + parent !== undefined && + ts.isExportSpecifier(parent) && + parent.propertyName === undefined && + (symbol.flags & ts.SymbolFlags.Alias) !== 0 + ) + return lowerer.checker.getAliasedSymbol(symbol); + return symbol; } /** Records a reference's use, or answers false when the reference is not diff --git a/packages/compiler/test/js-collection-types.test.ts b/packages/compiler/test/js-collection-types.test.ts index 88e2006b7..6a6f65798 100644 --- a/packages/compiler/test/js-collection-types.test.ts +++ b/packages/compiler/test/js-collection-types.test.ts @@ -54,6 +54,8 @@ test("untyped JavaScript collections take the types their uses prove, and only t "spread", "unread", "exported", + "listed", + "renamed", ]) expect(typed(types.get(untyped)!), untyped).toBe(false); } finally { diff --git a/tests/corpus/7985-untyped-collection-types.mjs b/tests/corpus/7985-untyped-collection-types.mjs index a60a2ea04..de4e581c2 100644 --- a/tests/corpus/7985-untyped-collection-types.mjs +++ b/tests/corpus/7985-untyped-collection-types.mjs @@ -114,3 +114,12 @@ console.log(unread.size, unread.get("none")); export const exported = new Map(); exported.set("e", 1); console.log(exported.get("e")); +// Exported through an export list, under its own name or another: an +// importer sees an untyped Map. +const listed = new Map(); +listed.set("l", 1); +console.log(listed.get("l")); +const renamed = new Map(); +renamed.set("r", 1); +console.log(renamed.get("r")); +export { listed, renamed as alias }; From d9fdd81ee92ed8b60dbceb6a311843e119207059 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 16:32:40 -0500 Subject: [PATCH 11/17] Construct X509Certificate bound through a require() member `const X = require("crypto").X509Certificate; new X(data)` now resolves to the builtin class, so the standard-library constructor fence reported SC2020 for it before the X509Certificate lowering ran. That lowering now runs ahead of the fence. Corpus 8231 constructs a certificate through the binding. --- .../src/frontend/lowering/lower-classes.ts | 189 +++++++++--------- .../8231-builtin-member-binding-new.cjs | 27 ++- 2 files changed, 122 insertions(+), 94 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index 8bd5e57d7..fc89dc6b1 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -10122,6 +10122,102 @@ export function lowerNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr { const agent = lowerHttpAgentNew(lowerer, expr); if (agent) return agent; } + // `new crypto.X509Certificate(data)` — the Dirent-style data record: + // the certificate's lowered members (fingerprint — the SHA-1 of the + // DER, uppercase colon-separated — plus the validFrom/validTo + // validity window in Node's ASN1_TIME_print shape) compute AT + // CONSTRUCTION, when Node parses too, so unparseable input throws + // Node's exact PEM error here (ERR_OSSL_PEM_NO_START_LINE) and the + // handle never exists. Both import forms (`crypto.X509Certificate` + // through the namespace, named `X509Certificate`); Buffer input only + // — the readFileSync idiom. mapType interns the matching record, so + // locals and the composed member reads all flow. The data argument + // feeds THREE field computations, so construction goes through an + // interned helper whose parameter evaluates it exactly once. + // It runs before the standard-library constructor fence below, which + // also sees `const X = require("crypto").X509Certificate` as the + // builtin class. + { + const callee = expr.expression; + const isX509 = + (ts.isPropertyAccessExpression(callee) && + callee.name.text === "X509Certificate" && + lowerer.builtinNamespaceModuleOf(callee.expression) === "crypto") || + (ts.isIdentifier(callee) && + (() => { + const bi = lowerer.builtinImportOf(callee); + return bi?.module === "crypto" && bi.member === "X509Certificate"; + })()); + if (isX509) { + const args: readonly ts.Expression[] = expr.arguments?.nodes ?? []; + if (args.length !== 1) { + lowerer.noLowering( + "X509Certificate with this argument shape", + expr, + "the supported form is new X509Certificate(readFileSync(path))", + ); + } + const data = lowerer.lowerExpr(args[0]!); + const isBytes = data.type.kind === "bytes" && data.type.elem === "u8"; + if (!isBytes && data.type.kind !== "string") { + lowerer.noLowering( + `X509Certificate over '${lowerer.fmt(data.type)}' data`, + args[0]!, + "pass the certificate Buffer or PEM string (an fs.readFileSync result)", + ); + } + const t = lowerer.mapTypeOf(lowerer.typeOf(expr)); + if (t?.kind !== "record") lowerer.badType(expr, lowerer.typeOf(expr)); + const key = `x509.record:${isBytes ? "bytes" : "str"}`; + let helper = lowerer.valueHelpers.get(key); + if (!helper) { + helper = keyedHelperName(lowerer, "%x509.record", key); + lowerer.valueHelpers.set(key, helper); + const dataT = data.type; + const dRef: IrExpr = { kind: "varRef", localId: "d.0", type: dataT, loc }; + const field = ( + name: string, + fn: + | "crypto.x509Fingerprint" + | "crypto.x509FingerprintStr" + | "crypto.x509ValidFrom" + | "crypto.x509ValidFromStr" + | "crypto.x509ValidTo" + | "crypto.x509ValidToStr", + ): { name: string; value: IrExpr } => ({ + name, + value: { kind: "libCall", fn, args: [dRef], type: STRING, loc }, + }); + lowerer.liftedFns.push({ + name: helper, + params: [{ localId: "d.0", name: "d", type: dataT }], + returnType: t, + locals: [{ id: "d.0", name: "d", type: dataT, mutable: false }], + body: [ + { + kind: "return", + value: { + kind: "recordLit", + fields: [ + field( + "fingerprint", + isBytes ? "crypto.x509Fingerprint" : "crypto.x509FingerprintStr", + ), + field("validFrom", isBytes ? "crypto.x509ValidFrom" : "crypto.x509ValidFromStr"), + field("validTo", isBytes ? "crypto.x509ValidTo" : "crypto.x509ValidToStr"), + ], + type: t, + loc, + }, + loc, + }, + ], + loc, + }); + } + return { kind: "call", callee: helper, args: [data], type: t, loc }; + } + } if ( ts.isIdentifier(expr.expression) || (ts.isPropertyAccessExpression(expr.expression) && @@ -10907,99 +11003,6 @@ export function lowerNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr { ); } } - // `new crypto.X509Certificate(data)` — the Dirent-style data record: - // the certificate's lowered members (fingerprint — the SHA-1 of the - // DER, uppercase colon-separated — plus the validFrom/validTo - // validity window in Node's ASN1_TIME_print shape) compute AT - // CONSTRUCTION, when Node parses too, so unparseable input throws - // Node's exact PEM error here (ERR_OSSL_PEM_NO_START_LINE) and the - // handle never exists. Both import forms (`crypto.X509Certificate` - // through the namespace, named `X509Certificate`); Buffer input only - // — the readFileSync idiom. mapType interns the matching record, so - // locals and the composed member reads all flow. The data argument - // feeds THREE field computations, so construction goes through an - // interned helper whose parameter evaluates it exactly once. - { - const callee = expr.expression; - const isX509 = - (ts.isPropertyAccessExpression(callee) && - callee.name.text === "X509Certificate" && - lowerer.builtinNamespaceModuleOf(callee.expression) === "crypto") || - (ts.isIdentifier(callee) && - (() => { - const bi = lowerer.builtinImportOf(callee); - return bi?.module === "crypto" && bi.member === "X509Certificate"; - })()); - if (isX509) { - const args: readonly ts.Expression[] = expr.arguments?.nodes ?? []; - if (args.length !== 1) { - lowerer.noLowering( - "X509Certificate with this argument shape", - expr, - "the supported form is new X509Certificate(readFileSync(path))", - ); - } - const data = lowerer.lowerExpr(args[0]!); - const isBytes = data.type.kind === "bytes" && data.type.elem === "u8"; - if (!isBytes && data.type.kind !== "string") { - lowerer.noLowering( - `X509Certificate over '${lowerer.fmt(data.type)}' data`, - args[0]!, - "pass the certificate Buffer or PEM string (an fs.readFileSync result)", - ); - } - const t = lowerer.mapTypeOf(lowerer.typeOf(expr)); - if (t?.kind !== "record") lowerer.badType(expr, lowerer.typeOf(expr)); - const key = `x509.record:${isBytes ? "bytes" : "str"}`; - let helper = lowerer.valueHelpers.get(key); - if (!helper) { - helper = keyedHelperName(lowerer, "%x509.record", key); - lowerer.valueHelpers.set(key, helper); - const dataT = data.type; - const dRef: IrExpr = { kind: "varRef", localId: "d.0", type: dataT, loc }; - const field = ( - name: string, - fn: - | "crypto.x509Fingerprint" - | "crypto.x509FingerprintStr" - | "crypto.x509ValidFrom" - | "crypto.x509ValidFromStr" - | "crypto.x509ValidTo" - | "crypto.x509ValidToStr", - ): { name: string; value: IrExpr } => ({ - name, - value: { kind: "libCall", fn, args: [dRef], type: STRING, loc }, - }); - lowerer.liftedFns.push({ - name: helper, - params: [{ localId: "d.0", name: "d", type: dataT }], - returnType: t, - locals: [{ id: "d.0", name: "d", type: dataT, mutable: false }], - body: [ - { - kind: "return", - value: { - kind: "recordLit", - fields: [ - field( - "fingerprint", - isBytes ? "crypto.x509Fingerprint" : "crypto.x509FingerprintStr", - ), - field("validFrom", isBytes ? "crypto.x509ValidFrom" : "crypto.x509ValidFromStr"), - field("validTo", isBytes ? "crypto.x509ValidTo" : "crypto.x509ValidToStr"), - ], - type: t, - loc, - }, - loc, - }, - ], - loc, - }); - } - return { kind: "call", callee: helper, args: [data], type: t, loc }; - } - } // `new X(...)` through a class VALUE (a classval-typed binding, array // element, map read, param): the newValue dispatch through the class // object's construct thunk. Arguments complete against the STATIC diff --git a/tests/corpus/8231-builtin-member-binding-new.cjs b/tests/corpus/8231-builtin-member-binding-new.cjs index 1549425f7..df1b2ebb3 100644 --- a/tests/corpus/8231-builtin-member-binding-new.cjs +++ b/tests/corpus/8231-builtin-member-binding-new.cjs @@ -1,12 +1,14 @@ // `new` of a builtin class bound through a member of a require() result // (`const SD = require("string_decoder").StringDecoder`, the spelling of // Node's own tests): the binding resolves to the class, as the -// destructured `const { StringDecoder } = require(...)` does. +// destructured `const { StringDecoder } = require(...)` does, also for the +// X509Certificate record. "use strict"; const SD = require("string_decoder").StringDecoder; const Readable = require("stream").Readable; const PassThrough = require("stream").PassThrough; const Transform = require("stream").Transform; +const X509 = require("crypto").X509Certificate; const decoder = new SD("utf8"); console.log(decoder.encoding); @@ -33,3 +35,26 @@ const upper = new Transform({ }); upper.on("data", (chunk) => console.log("transform", String(chunk))); upper.end("shout"); + +// The X509Certificate record constructs through the same binding. +const PEM = `-----BEGIN CERTIFICATE----- +MIIDEzCCAfugAwIBAgIUMsBCFtPck8TATMJzfikgW963q3AwDQYJKoZIhvcNAQEL +BQAwGDEWMBQGA1UEAwwNdHNuYXRpdmUtdGVzdDAgFw0yNjA3MTcxNzUyMTFaGA8y +MTI2MDYyMzE3NTIxMVowGDEWMBQGA1UEAwwNdHNuYXRpdmUtdGVzdDCCASIwDQYJ +KoZIhvcNAQEBBQADggEPADCCAQoCggEBALRspe2XjtKvNU7Ws5JfqzmTCPmWtfw0 +1Uha271hNp/30VstOxWLbpqPrXDvIwJMN4o8CRp92Y1hcmS36m9QS5qfXJFw+n+S +SWGpFdMNvp6hro0OoNBA+anuKWoTsMyHSwe8ND0fyobQbgAXJ0+snmuK0E2ii+CM +IoUqYbK57nHeyHfISLupgrRoNGXjDHv4v+RSr3MJDag6BLA+/yVjkBOBOT6kaE4K +oyNXYYsfFxGnB/4pDSanV/UsfJ4wHXkVn18ndZHIzTsZMUkV4BecF8UaMrP+u8NI +c5wce0uWLFFEneK4q15iVvk9TaY0g1Uj69ryRT3L58W7RYYuHu7cVB0CAwEAAaNT +MFEwHQYDVR0OBBYEFLT2GqaTmxDhntPxMWaUZvA3s9V5MB8GA1UdIwQYMBaAFLT2 +GqaTmxDhntPxMWaUZvA3s9V5MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL +BQADggEBAFcTLVJIyhbIKHfOnGM5T+Kzu5IoDlqzr+Rdx/0IKFLVDqEA9ihEme07 +K/d246pOW5LBceVDY2eMikDpIOjYPEoe9C+Ftv5zrCaDw1wtXrq8a7/eS5B+3TzL +k8c1SVVDPIbmViAZNRmnkwCPBsIF9xIfMFhxz4BTJrm/Lt2Hv1GcRBQx/yRK89sj +JiGnsTna//4mPGRGYw+dRCq4tbyr8HbBKdJmbzOCb4CPKfwAMKUpd4e0Ao7jh/BU +EML7B3BEzv5Jw/BT7B39wAfDyhq32PnRnZefF+7pAmpkklHbckBtZtWn02zwQazv +5z62tszZee2i4BouJiyFnb30F6Pebs4= +-----END CERTIFICATE-----`; +const cert = new X509(PEM); +console.log(cert.fingerprint, cert.validFrom, cert.validTo); From 9406ba5b892b16b1675ba55fbfbcd5c1a762e514 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 17:04:14 -0500 Subject: [PATCH 12/17] Run row sums two rows per pass A loop nest that sums one term per row, for (let i = start; i < end; i++) { let t = 0; for (let j = from; j < to; j++) t += term; out[i] = t; } where the term only reads numbers, the counters and typed-array elements, now runs two rows per pass when both row bounds are int32, the start is at most the end, and the stored array shares no byte with any array the term reads. Each row keeps its order of additions, so each sum has the bits of the loop as written; an odd row count runs its last row as written, and any other case runs the nest as written. Element reads that do not depend on the row are read once per column for both rows. With two independent sums in the inner loop, neither waits on the other's addition, and LLVM computes both rows' terms in one vector. bytes.disjoint compares two typed arrays' element ranges in memory. --- packages/compiler/src/backend/llvm/emitter.ts | 19 +- .../compiler/src/backend/llvm/lib-dispatch.ts | 34 +- .../compiler/src/ir/builtin-signatures.ts | 3 + packages/compiler/src/ir/row-pairs.test.ts | 147 ++++++ packages/compiler/src/ir/row-pairs.ts | 438 ++++++++++++++++++ .../tests/corpus/8194-row-pair-sums.ts.json | 6 + tests/corpus/8194-row-pair-sums.ts | 72 +++ 7 files changed, 710 insertions(+), 9 deletions(-) create mode 100644 packages/compiler/src/ir/row-pairs.test.ts create mode 100644 packages/compiler/src/ir/row-pairs.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8194-row-pair-sums.ts.json create mode 100644 tests/corpus/8194-row-pair-sums.ts diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index f6f9ea313..0009d5229 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -81,6 +81,7 @@ import { import { specializeNumericCalls } from "../../ir/numeric-call-specialization.js"; import { versionLoopBoundParameters } from "../../ir/parameter-versions.js"; import { inlineExpressionFunctions } from "../../ir/expression-inlining.js"; +import { pairRowSums } from "../../ir/row-pairs.js"; import { allocatedLiteral, fillAllocationLength, @@ -565,14 +566,16 @@ function moduleEmitter(input: IrModule, options: LlvmTargetOptions): LlEmitter { scalarizeNumericRecords( specializeCallbackCalls( specializeNumericCalls( - inlineExpressionFunctions( - foldConstantGlobals( - copyStableCaptures( - resolveSelfCaptures( - hoistFinalFieldReads( - freezeInitOnlyGlobals( - holdCharLocalsAsUnits( - dropUnwrittenPropertyBags(runPairedWhileLoopsAsFor(mod)), + pairRowSums( + inlineExpressionFunctions( + foldConstantGlobals( + copyStableCaptures( + resolveSelfCaptures( + hoistFinalFieldReads( + freezeInitOnlyGlobals( + holdCharLocalsAsUnits( + dropUnwrittenPropertyBags(runPairedWhileLoopsAsFor(mod)), + ), ), ), ), diff --git a/packages/compiler/src/backend/llvm/lib-dispatch.ts b/packages/compiler/src/backend/llvm/lib-dispatch.ts index d8ca5add8..f4ce9e405 100644 --- a/packages/compiler/src/backend/llvm/lib-dispatch.ts +++ b/packages/compiler/src/backend/llvm/lib-dispatch.ts @@ -10,7 +10,7 @@ import { FN_ATTRS, arrNewCall, traceArg, vAdapters } from "./shapes.js"; import { InternalCompilerError } from "../../errors.js"; import { MAY_THROW_LIB_FNS } from "../../ir/builtin-effects.js"; -import { F64, type IrExpr, type IrType } from "../../ir/ir.js"; +import { BYTES_ELEMENT_SIZE, F64, type IrExpr, type IrType } from "../../ir/ir.js"; import { LlvmUnsupportedError } from "./unsupported.js"; import { emitJsLibCall } from "./jsval.js"; import type { LlvmEmitterContext, LibCallExpr, LibCallPrefix, LlValue } from "./expr-context.js"; @@ -936,6 +936,38 @@ export function emitLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue { return { name: t, type: e.type }; } if (e.fn === "bytes.absent") return host.own({ name: "null", type: e.type }); + if (e.fn === "bytes.disjoint") { + // [data, data + len * size) of each view, compared as addresses. + const B = host.B; + const range = (arg: IrExpr): { start: string; end: string } => { + const value = host.emitReadReceiver(arg); + const size = arg.type.kind === "bytes" ? BYTES_ELEMENT_SIZE[arg.type.elem] : 1; + const dataSlot = B.tmp(), + data = B.tmp(), + lenSlot = B.tmp(), + len = B.tmp(), + bytes = B.tmp(), + start = B.tmp(), + end = B.tmp(); + B.line(`${dataSlot} = getelementptr inbounds %ScrBytes, ptr ${value.name}, i64 0, i32 3`); + B.line(`${data} = load ptr, ptr ${dataSlot}`); + B.line(`${lenSlot} = getelementptr inbounds %ScrBytes, ptr ${value.name}, i64 0, i32 1`); + B.line(`${len} = load ${host.sizeType}, ptr ${lenSlot}`); + B.line(`${bytes} = mul ${host.sizeType} ${len}, ${size}`); + B.line(`${start} = ptrtoint ptr ${data} to ${host.sizeType}`); + B.line(`${end} = add ${host.sizeType} ${start}, ${bytes}`); + return { start, end }; + }; + const left = range(e.args[0]!), + right = range(e.args[1]!); + const before = B.tmp(), + after = B.tmp(), + t = B.tmp(); + B.line(`${before} = icmp ule ${host.sizeType} ${left.end}, ${right.start}`); + B.line(`${after} = icmp ule ${host.sizeType} ${right.end}, ${left.start}`); + B.line(`${t} = or i1 ${before}, ${after}`); + return { name: t, type: e.type }; + } if (e.fn === "record.isFrozen") { // The frozen bit of the record's rc word (scr_rc.h SCR_RC_FROZEN) is // its second-highest bit; an immortal word sets the highest one too. diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 1109f7f95..ef75b1b6d 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -4930,6 +4930,9 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ * and crypto callback adapters pass NULL after an error. Borrowed arg; * never throws. */ "bytes.isAbsent": { argTypes: [BYTES_U8], result: BOOL }, + /** Whether two typed arrays share no byte: their element ranges in + * memory do not overlap. Borrowed args of any element kind; never throws. */ + "bytes.disjoint": { argTypes: [null, null], result: BOOL }, /** The absent Buffer result itself (NULL), where an error-path callback * body passes its undefined result on into a Buffer slot. Owned (+1); * never throws. */ diff --git a/packages/compiler/src/ir/row-pairs.test.ts b/packages/compiler/src/ir/row-pairs.test.ts new file mode 100644 index 000000000..f9db5b89d --- /dev/null +++ b/packages/compiler/src/ir/row-pairs.test.ts @@ -0,0 +1,147 @@ +import { expect, test } from "vitest"; +import { BOOL, F64, VOID, type IrExpr, type IrFunction, type IrModule, type IrStmt } from "./ir.js"; +import { pairRowSums } from "./row-pairs.js"; + +const loc = { file: "row-pairs.ts", start: 0, end: 0 }; +const BYTES = { kind: "bytes", elem: "f64" } as const; +const num = (value: number): IrExpr => ({ loc, kind: "numLit", value, type: F64 }); +const ref = (localId: string): IrExpr => ({ loc, kind: "varRef", localId, type: F64 }); +const bytes = (localId: string): IrExpr & { kind: "varRef" } => ({ + loc, + kind: "varRef", + localId, + type: BYTES, +}); +const bin = (op: "+" | "/" | "<", left: IrExpr, right: IrExpr): IrExpr => ({ + loc, + kind: "bin", + op, + left, + right, + type: op === "<" ? BOOL : F64, +}); +const step = (id: string): IrStmt => ({ + loc, + kind: "assign", + localId: id, + value: bin("+", ref(id), num(1)), +}); +const counted = ( + id: string, + start: IrExpr, + end: IrExpr, + body: IrStmt[], +): IrStmt & { kind: "for" } => ({ + loc, + kind: "for", + init: { loc, kind: "varDecl", localId: id, init: start }, + cond: bin("<", ref(id), end), + update: step(id), + body, +}); +const get = (array: string, index: IrExpr): IrExpr => ({ + loc, + kind: "bytesIntrinsic", + method: "get", + receiver: bytes(array), + args: [index], + type: F64, + invalidNaN: true, +}); + +/** for (i = start; i < end; i++) { t = 0; for (j = 0; j < n; j++) t += u[j] / (i + j + 1); v[i] = t } */ +function rowSum( + term: IrExpr = bin("/", get("u", ref("j")), bin("+", bin("+", ref("i"), ref("j")), num(1))), +) { + return counted("i", ref("start"), ref("end"), [ + { loc, kind: "varDecl", localId: "t", init: num(0) }, + counted("j", num(0), ref("n"), [ + { loc, kind: "assign", localId: "t", value: bin("+", ref("t"), term) }, + ]), + { loc, kind: "bytesSet", arr: bytes("v"), index: ref("i"), value: ref("t") }, + ]); +} + +const local = (id: string, type: IrExpr["type"] = F64) => ({ id, name: id, type, mutable: true }); +function mod(body: IrStmt[]): IrModule { + const fn: IrFunction = { + loc, + name: "rows", + params: [], + locals: [ + ...["i", "j", "t", "start", "end", "n"].map((id) => local(id)), + local("u", BYTES), + local("v", BYTES), + ], + returnType: VOID, + body, + }; + return { irVersion: 17, sourceFile: "row-pairs.ts", entry: "rows", functions: [fn] }; +} + +test("a row-sum nest runs two rows per pass behind its bounds and overlap test", () => { + const result = pairRowSums(mod([rowSum()])); + const top = result.functions[0]!.body[0] as IrStmt & { kind: "block" }; + expect(top.kind).toBe("block"); + const [go, from, branch, rest] = top.body as [ + IrStmt & { kind: "varDecl" }, + IrStmt & { kind: "varDecl" }, + IrStmt & { kind: "if" }, + IrStmt & { kind: "for" }, + ]; + // The test: int32 bounds, start <= end, and the stored array disjoint + // from the array the term reads. + expect(JSON.stringify(go.init)).toContain('"fn":"bytes.disjoint"'); + expect(branch.else_).toBeNull(); + const pairs = branch.then[0] as IrStmt & { kind: "for" }; + expect(pairs.body.filter((s) => s.kind === "bytesSet")).toHaveLength(2); + const inner = pairs.body[2] as IrStmt & { kind: "for" }; + // One read of u[j] for both rows, then the two sums. + expect(inner.body).toHaveLength(3); + expect(inner.body[0]).toMatchObject({ kind: "varDecl", init: { kind: "bytesIntrinsic" } }); + // The second row's term reads `row + 1`. + const row = pairs.init?.kind === "varDecl" ? pairs.init.localId : ""; + expect((inner.body[2] as IrStmt & { kind: "assign" }).value).toMatchObject({ + op: "+", + right: { + op: "/", + left: { kind: "varRef" }, + right: { + op: "+", + left: { + op: "+", + left: { op: "+", left: { kind: "varRef", localId: row }, right: { value: 1 } }, + }, + }, + }, + }); + // The loop as written runs from `end - ((end - start) & 1)` after the + // pairs, or from the start when the test failed: the nest's one other copy. + expect(from.init).toMatchObject({ + kind: "ternary", + then: { op: "-" }, + else_: { localId: "start" }, + }); + expect(rest.init).toMatchObject({ + kind: "varDecl", + localId: "i", + init: { localId: from.localId }, + }); + expect(top.body).toHaveLength(4); + expect( + result.functions[0]!.locals.map((l) => l.id).filter((id) => id.startsWith("%pair.")), + ).toHaveLength(7); +}); + +test("nests whose term calls, writes or reads the sum keep their loops", () => { + const calls: IrExpr = { loc, kind: "call", callee: "f", args: [ref("j")], type: F64 }; + const readsSum = bin("+", ref("t"), get("u", ref("j"))); + for (const term of [calls, readsSum]) { + const m = mod([rowSum(term)]); + expect(pairRowSums(m)).toBe(m); + } + const stored = rowSum(); + stored.body[2] = { loc, kind: "bytesSet", arr: bytes("v"), index: ref("j"), value: ref("t") }; + const m = mod([stored]); + expect(pairRowSums(m)).toBe(m); +}); diff --git a/packages/compiler/src/ir/row-pairs.ts b/packages/compiler/src/ir/row-pairs.ts new file mode 100644 index 000000000..ed71166a8 --- /dev/null +++ b/packages/compiler/src/ir/row-pairs.ts @@ -0,0 +1,438 @@ +import { + BOOL, + F64, + type IrExpr, + type IrFunction, + type IrLocal, + type IrModule, + type IrStmt, +} from "./ir.js"; +import { everyExpr, everyExprChild, mapExprChildren, transformStmtList } from "./traverse.js"; + +/** Inner-loop expressions above this many nodes keep the loop as written. */ +const MAX_TERM_NODES = 64; + +/** The number operators a term may use: they run no other code and cannot + * throw. */ +const PURE_BINARY = new Set([ + "+", + "-", + "*", + "/", + "%", + "**", + "&", + "|", + "^", + "<<", + ">>", + ">>>", +]); + +/** A loop nest that sums one term per row: + * + * for (let i = start; i < end; i++) { + * let t = ; + * for (let j = from; j < to; j++) t = t + ; + * out[i] = t; + * } + */ +interface RowSum { + row: string; + start: IrExpr; + end: IrExpr; + sum: string; + init: IrExpr & { kind: "numLit" }; + inner: IrStmt & { kind: "for" }; + column: string; + term: IrExpr; + out: IrExpr & { kind: "varRef" }; + /** The typed arrays the term reads. */ + reads: (IrExpr & { kind: "varRef" })[]; +} + +function isRef(e: IrExpr | null | undefined, id: string): boolean { + return e?.kind === "varRef" && e.localId === id; +} + +/** `id = id + 1`, or `id++` as a statement. */ +function unitStep(update: IrStmt | null, id: string): boolean { + if (update?.kind === "exprStmt") + return update.expr.kind === "incDec" && update.expr.localId === id && update.expr.op === "+"; + return ( + update?.kind === "assign" && + update.localId === id && + update.value.kind === "bin" && + update.value.op === "+" && + isRef(update.value.left, id) && + update.value.right.kind === "numLit" && + update.value.right.value === 1 + ); +} + +/** A plain number local: unboxed and outside the temporal dead zone. */ +function plainNumber(locals: ReadonlyMap, id: string): boolean { + const local = locals.get(id); + return local !== undefined && local.type.kind === "f64" && !local.boxed && !local.tdz; +} + +function readsLocal(e: IrExpr, id: string): boolean { + return !everyExpr(e, { stmt: () => true, expr: (x) => !isRef(x, id) }); +} + +/** The row-sum shape of a `for` statement, or null. Bounds and receivers + * must be literals or locals the nest never writes. */ +function matchRowSum( + s: IrStmt & { kind: "for" }, + locals: ReadonlyMap, +): RowSum | null { + if (s.labels?.length || s.init?.kind !== "varDecl" || !s.init.init) return null; + const row = s.init.localId; + if (!plainNumber(locals, row) || !unitStep(s.update, row)) return null; + const cond = s.cond; + if (cond?.kind !== "bin" || cond.op !== "<" || !isRef(cond.left, row)) return null; + const start = s.init.init, + end = cond.right; + if (s.body.length !== 3) return null; + const [decl, inner, store] = s.body as [IrStmt, IrStmt, IrStmt]; + if (decl.kind !== "varDecl" || decl.init?.kind !== "numLit") return null; + const sum = decl.localId; + if (!plainNumber(locals, sum)) return null; + if (inner.kind !== "for" || inner.labels?.length) return null; + if (inner.init?.kind !== "varDecl" || !inner.init.init) return null; + const column = inner.init.localId; + if (!plainNumber(locals, column) || !unitStep(inner.update, column)) return null; + const innerCond = inner.cond; + if (innerCond?.kind !== "bin" || innerCond.op !== "<" || !isRef(innerCond.left, column)) + return null; + if (inner.body.length !== 1) return null; + const add = inner.body[0]!; + if ( + add.kind !== "assign" || + add.localId !== sum || + add.value.kind !== "bin" || + add.value.op !== "+" || + !isRef(add.value.left, sum) + ) + return null; + const term = add.value.right; + if ( + store.kind !== "bytesSet" || + store.arr.kind !== "varRef" || + store.arr.type.kind !== "bytes" || + !isRef(store.index, row) || + !isRef(store.value, sum) + ) + return null; + const out = store.arr; + // Values the nest reads but never writes: the row counter, the column + // counter and the sum are its only writes. + const written = new Set([row, sum, column]); + // A typed array's length is fixed, so a stable receiver's length is + // stable too. + const stable = (e: IrExpr): boolean => + e.kind === "numLit" || + (e.kind === "varRef" && + !written.has(e.localId) && + locals.get(e.localId) !== undefined && + !locals.get(e.localId)!.boxed && + !locals.get(e.localId)!.tdz) || + (e.kind === "bytesIntrinsic" && + e.method === "length" && + e.args.length === 0 && + e.receiver.kind === "varRef" && + e.receiver.type.kind === "bytes" && + stable(e.receiver)); + if (!stable(start) || !stable(end) || !stable(out)) return null; + if (!stable(inner.init.init) || !stable(innerCond.right)) return null; + const reads: (IrExpr & { kind: "varRef" })[] = []; + let nodes = 0; + const pure = (e: IrExpr): boolean => { + if (++nodes > MAX_TERM_NODES) return false; + switch (e.kind) { + case "numLit": + return e.type.kind === "f64"; + case "varRef": + return e.type.kind === "f64" && (e.localId === row || e.localId === column || stable(e)); + case "bin": + return e.type.kind === "f64" && PURE_BINARY.has(e.op) && pure(e.left) && pure(e.right); + case "unary": + return e.type.kind === "f64" && (e.op === "-" || e.op === "~") && pure(e.operand); + case "bytesIntrinsic": + if (stable(e)) return true; + if ( + e.method !== "get" || + e.args.length !== 1 || + e.receiver.kind !== "varRef" || + e.receiver.type.kind !== "bytes" || + !stable(e.receiver) + ) + return false; + reads.push(e.receiver); + return pure(e.args[0]!); + default: + return false; + } + }; + if (!pure(term) || readsLocal(term, sum)) return null; + return { row, start, end, sum, init: decl.init, inner, column, term, out, reads }; +} + +/** A fresh copy of `e` with `substitute` applied to its local reads and + * `replace` to the nodes it answers for. */ +function copy( + e: IrExpr, + substitute: (ref: IrExpr & { kind: "varRef" }) => IrExpr, + replace: (node: IrExpr) => IrExpr | null = () => null, +): IrExpr { + const replaced = replace(e); + if (replaced !== null) return replaced; + if (e.kind === "varRef") return substitute(e); + return mapExprChildren( + { ...e }, + (child) => copy(child, substitute, replace), + (stmt) => stmt, + ); +} + +const bin = ( + op: "+" | "-" | "&" | "|", + left: IrExpr, + right: IrExpr, + loc: IrExpr["loc"], +): IrExpr => ({ + kind: "bin", + op, + left, + right, + type: F64, + loc, +}); + +/** Runs the row sums of `match` two rows per pass when the test holds and + * as written otherwise. */ +function pairRows( + s: IrStmt & { kind: "for" }, + match: RowSum, + fresh: (name: string) => string, + done: Set, +): IrStmt { + const loc = s.loc; + const ref = (id: string): IrExpr => ({ kind: "varRef", localId: id, type: F64, loc }); + const num = (value: number): IrExpr => ({ kind: "numLit", value, type: F64, loc }); + const again = (e: IrExpr): IrExpr => copy(e, (r) => ({ ...r })); + const row = fresh("row"), + sum = fresh("sum"), + next = fresh("next"), + column = fresh("column"); + const atColumn = (r: IrExpr & { kind: "varRef" }): IrExpr => + r.localId === match.column ? { ...r, localId: column } : { ...r }; + // Element reads that do not depend on the row are read once per column + // and shared by both rows. + const shared = new Map(); + const elements: IrStmt[] = []; + const collect = (e: IrExpr): boolean => { + if (e.kind === "bytesIntrinsic" && e.method === "get" && !readsLocal(e, match.row)) { + const id = fresh("element"); + shared.set(e, id); + elements.push({ kind: "varDecl", localId: id, init: copy(e, atColumn), loc }); + return true; + } + return everyExprChild(e, collect, () => true); + }; + collect(match.term); + // The second row's term reads `row + 1`, so its counter sums stay visible + // to the counted-loop analysis. + const termAt = (rowExpr: () => IrExpr): IrExpr => + copy( + match.term, + (r) => (r.localId === match.row ? rowExpr() : atColumn(r)), + (node) => { + const id = shared.get(node); + return id === undefined ? null : { kind: "varRef", localId: id, type: node.type, loc }; + }, + ); + const innerCond = match.inner.cond as IrExpr & { kind: "bin" }; + const pairs: IrStmt = { + kind: "for", + init: { kind: "varDecl", localId: row, init: again(match.start), loc }, + cond: { + kind: "bin", + op: "<", + left: ref(row), + right: bin("-", again(match.end), num(1), loc), + type: BOOL, + loc, + }, + update: { kind: "assign", localId: row, value: bin("+", ref(row), num(2), loc), loc }, + body: [ + { kind: "varDecl", localId: sum, init: { ...match.init }, loc }, + { kind: "varDecl", localId: next, init: { ...match.init }, loc }, + { + kind: "for", + init: { + kind: "varDecl", + localId: column, + init: again((match.inner.init as IrStmt & { kind: "varDecl" }).init!), + loc, + }, + cond: { ...innerCond, left: ref(column), right: again(innerCond.right) }, + update: { kind: "assign", localId: column, value: bin("+", ref(column), num(1), loc), loc }, + body: [ + ...elements, + { + kind: "assign", + localId: sum, + value: bin( + "+", + ref(sum), + termAt(() => ref(row)), + loc, + ), + loc, + }, + { + kind: "assign", + localId: next, + value: bin( + "+", + ref(next), + termAt(() => bin("+", ref(row), num(1), loc)), + loc, + ), + loc, + }, + ], + loc, + }, + { kind: "bytesSet", arr: { ...match.out }, index: ref(row), value: ref(sum), loc }, + { + kind: "bytesSet", + arr: { ...match.out }, + index: bin("+", ref(row), num(1), loc), + value: ref(next), + loc, + }, + ], + loc, + }; + // Both bounds are int32, so `row < end - 1` is `row + 1 < end`, and the + // start is at most the end. Rows are independent when the written array + // shares no byte with an array the term reads. + const int32 = (e: IrExpr): IrExpr => ({ + kind: "bin", + op: "===", + left: bin("|", again(e), num(0), loc), + right: again(e), + type: BOOL, + loc, + }); + const tests: IrExpr[] = [ + int32(match.start), + int32(match.end), + { kind: "bin", op: "<=", left: again(match.start), right: again(match.end), type: BOOL, loc }, + ...match.reads.map((read): IrExpr => ({ + kind: "libCall", + fn: "bytes.disjoint", + args: [{ ...match.out }, { ...read }], + type: BOOL, + loc, + })), + ]; + const test = tests.reduce((left, right) => ({ + kind: "logical", + op: "&&", + left, + right, + type: BOOL, + loc, + })); + // The loop as written then runs the rows the pairs left: the last one when + // the count is odd (from `end - ((end - start) & 1)`), or all of them when + // the test failed. It is the nest's only other copy. + const go = fresh("go"), + from = fresh("from"); + const paired: IrExpr = { kind: "bin", op: "===", left: ref(go), right: num(1), type: BOOL, loc }; + const rest: IrStmt = { + ...s, + init: { kind: "varDecl", localId: match.row, init: ref(from), loc }, + }; + done.add(rest); + return { + kind: "block", + body: [ + { + kind: "varDecl", + localId: go, + init: { kind: "ternary", cond: test, then: num(1), else_: num(0), type: F64, loc }, + loc, + }, + { + kind: "varDecl", + localId: from, + init: { + kind: "ternary", + cond: paired, + then: bin( + "-", + again(match.end), + bin("&", bin("-", again(match.end), again(match.start), loc), num(1), loc), + loc, + ), + else_: again(match.start), + type: F64, + loc, + }, + loc, + }, + { kind: "if", cond: { ...paired }, then: [pairs], else_: null, loc }, + rest, + ], + loc, + }; +} + +/** Runs loop nests that sum one term per row (spectral-norm's `au`) two rows + * per pass. + * + * Each row's sum keeps its order of additions, so every sum has the bits + * the nest computes as written: the second row's term is the same + * expression of `row + 1`. The term only reads numbers, the counters and + * typed-array elements, so it runs no other code and cannot throw. The + * nest runs paired when both row bounds are int32 (so `row < end - 1` is + * `row + 1 < end`), the start is at most the end, and the array the rows + * store into shares no byte with any array the term reads, so no row reads + * another row's result. An odd row count runs its last row as written. + * Otherwise the nest runs as written. Two independent sums take turns in + * the inner loop, so neither waits on the other's addition. */ +export function pairRowSums(mod: IrModule): IrModule { + let changed = false; + const functions = mod.functions.map((fn): IrFunction => { + if (fn.async || fn.generator) return fn; + const locals = new Map(fn.locals.map((local) => [local.id, local])); + const added: IrLocal[] = []; + const ids = new Set(fn.locals.map((local) => local.id)); + const fresh = (name: string): string => { + let id = `%pair.${name}.${ids.size}`; + while (ids.has(id)) id += "_"; + ids.add(id); + added.push({ id, name: `%pair.${name}`, type: F64, mutable: true }); + return id; + }; + const paired = new Set(); + const body = transformStmtList(fn.body, { + expr: (e) => e, + stmt: (s) => { + if (s.kind !== "for" || paired.has(s)) return s; + const match = matchRowSum(s, locals); + if (match === null) return s; + paired.add(s); + return pairRows(s, match, fresh, paired); + }, + }); + if (added.length === 0) return fn; + changed = true; + return { ...fn, locals: [...fn.locals, ...added], body }; + }); + return changed ? { ...mod, functions } : mod; +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8194-row-pair-sums.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8194-row-pair-sums.ts.json new file mode 100644 index 000000000..afb53a915 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8194-row-pair-sums.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8194-row-pair-sums.ts" + ], + "diags": [] +} diff --git a/tests/corpus/8194-row-pair-sums.ts b/tests/corpus/8194-row-pair-sums.ts new file mode 100644 index 000000000..9aa22b9bd --- /dev/null +++ b/tests/corpus/8194-row-pair-sums.ts @@ -0,0 +1,72 @@ +// Row sums (`for i { t = 0; for j t += term; out[i] = t }`) over typed +// arrays, run with even and odd row counts, an empty range, fractional, +// negative-zero and out-of-int32 bounds, and output arrays that are the +// input array or overlap it in one buffer, so a row may read an earlier +// row's result. +function rows(u: Float64Array, v: Float64Array, start: number, end: number, n: number): void { + for (let i = start; i < end; i++) { + let t = 0; + for (let j = 0; j < n; j++) t += u[j]! / ((((i + j) * (i + j + 1)) >>> 1) + i + 1); + v[i] = t; + } +} + +function weighted(u: Float32Array, w: Int32Array, v: Float64Array, start: number, end: number): void { + for (let i = start; i < end; i++) { + let t = 0.5; + for (let j = 0; j < u.length; j++) t += u[j]! * w[(i + j) % w.length]! - i * 0.25; + v[i] = t; + } +} + +const show = (label: string, v: Float64Array): void => { + const parts: string[] = []; + for (const x of v) parts.push(Object.is(x, -0) ? "-0" : x.toPrecision(17)); + console.log(label, parts.join(" ")); +}; + +const n = Number(process.argv[2] ?? 7); +const u = new Float64Array(n).map((_, k) => k + 1); +for (const [start, end] of [ + [0, n], + [0, n - 1], + [1, n], + [2, 2], + [3, 1], + [-0, 3], + [0.5, 4], + [0, 4.5], + [2 ** 31 - 3, 2 ** 31 + 1], + [NaN, 3], +]) { + const v = new Float64Array(n + 2); + rows(u, v, start!, end!, n); + show(`rows ${start} ${end}`, v); +} + +// The output is the input: each row reads the rows stored before it. +const same = new Float64Array(n).map((_, k) => k + 1); +rows(same, same, 0, n, n); +show("same", same); + +// Overlapping views of one buffer, offset by one element. +const buffer = new ArrayBuffer(8 * (n + 1)); +const lo = new Float64Array(buffer, 0, n).map((_, k) => k + 1); +const hi = new Float64Array(buffer, 8, n); +rows(lo, hi, 0, n, n); +show("overlap", new Float64Array(buffer)); + +// Disjoint views of one buffer. +const shared = new ArrayBuffer(16 * n); +const a = new Float64Array(shared, 0, n).map((_, k) => n - k); +const b = new Float64Array(shared, 8 * n, n); +rows(a, b, 0, n, n); +show("disjoint", b); + +const f = new Float32Array(n).map((_, k) => k / 3); +const w = new Int32Array([3, -1, 4, 1, -5]); +const out = new Float64Array(n); +weighted(f, w, out, 0, n); +show("weighted", out); +weighted(f, w, out, 1, n - 1); +show("weighted inner", out); From adbd9cd1e9b85fbe6b997e6ba2170b2f1d5a1858 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 17:18:30 -0500 Subject: [PATCH 13/17] Lower each closure argument of an implicit-any call once To key an implicit-any function's instance, storedImplicitArgumentType lowers a JavaScript arrow or function-expression argument to read its function type, then discards the value; the call's argument list then lowers the closure again. Every such closure was lowered twice, and a closure nested in another one four times, and each lowering lifted its own copy of the body. The first lowering now keeps its value (Lowerer.speculatedClosures), and the argument's own lowering in the same function and with the same type bindings takes it instead of lowering the closure again. A lowering that reported diagnostics is not kept. --- .../frontend/lowering/generic-functions.ts | 17 ++++++++++-- .../compiler/src/frontend/lowering/lowerer.ts | 26 +++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/generic-functions.ts b/packages/compiler/src/frontend/lowering/generic-functions.ts index 7896da578..0f9e41723 100644 --- a/packages/compiler/src/frontend/lowering/generic-functions.ts +++ b/packages/compiler/src/frontend/lowering/generic-functions.ts @@ -2082,6 +2082,10 @@ function storedImplicitArgumentType(lowerer: Lowerer, arg: ts.Expression): IrTyp isJsSourceFile(ts.getSourceFileOfNode(arg)) && (ts.isArrowFunction(arg) || ts.isFunctionExpression(arg)) ) { + // A closure lowered here before (another call site's instance key, or + // a repeated demand) keeps its value for the argument's own lowering. + const known = lowerer.speculatedClosure(arg); + if (known !== null) return known.type; const saved = lowerer.diagSink; const diagnostics: typeof lowerer.diags = []; lowerer.diagSink = diagnostics; @@ -2092,8 +2096,17 @@ function storedImplicitArgumentType(lowerer: Lowerer, arg: ts.Expression): IrTyp lowerer.diagSink = saved; } // Direct inferred helpers can receive native callbacks even when their - // result is an opaque handle that cannot cross checked storage. - if (diagnostics.length === 0 && value?.type.kind === "func") return value.type; + // result is an opaque handle that cannot cross checked storage. The + // argument's own lowering then takes this value instead of lowering + // the closure a second time. + if (diagnostics.length === 0 && value?.type.kind === "func") { + lowerer.speculatedClosures.set(arg, { + ctx: lowerer.ctx, + bindings: lowerer.typeParamBindings, + value, + }); + return value.type; + } } if (arg.kind === ts.SyntaxKind.ThisKeyword) return lowerer.resolveThis()?.type ?? null; if (ts.isAwaitExpression(arg)) { diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 6fd6d863a..dd07d0075 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -3174,6 +3174,14 @@ export class Lowerer { * (builtins/fs-paths.ts): each lowers to the hidden local that holds its * value, which is null until the path's lowering has saved it. */ readonly stagedArguments = new Map(); + /** Closure arguments an implicit-any call already lowered to learn their + * type (storedImplicitArgumentType), by node: the argument's own lowering + * in the same function and with the same type bindings takes that value + * once instead of lowering the closure again. */ + readonly speculatedClosures = new Map< + ts.Expression, + { ctx: FnCtx; bindings: Map | null; value: IrExpr } + >(); /** What each fs path argument of a URL-path call lowers to (the plans * of builtins/fs-paths.ts). */ readonly fsPathPlans = new Map(); @@ -12938,12 +12946,30 @@ export class Lowerer { /* ── expressions ──────────────────────────────────────────────────── */ lowerExpr(expr: ts.Expression): IrExpr { + if (this.speculatedClosures.size > 0) { + const speculated = this.speculatedClosure(expr); + if (speculated !== null) { + this.speculatedClosures.delete(expr); + return speculated; + } + } const streamValue = this.valueStreamReceivers.get(expr); if (streamValue) return valueStreamOf(varRef(streamValue.localId, streamValue.type, locOf(expr))); return this.stagedArgument(expr) ?? lowerExpr(this, expr); } + /** The value storedImplicitArgumentType lowered for `expr` in the active + * function and type bindings (speculatedClosures), or null. */ + speculatedClosure(expr: ts.Expression): IrExpr | null { + const entry = this.speculatedClosures.get(expr); + return entry !== undefined && + entry.ctx === this.ctx && + entry.bindings === this.typeParamBindings + ? entry.value + : null; + } + /** The saved value of a staged fs argument (stagedArguments, keyed * without type-only wrappers), or null for every other expression. */ private stagedArgument(node: ts.Expression): IrExpr | null { From cc6b1144681da6c0d29649d44ee203703e9db25d Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 17:23:08 -0500 Subject: [PATCH 14/17] Take the argument count before parameters bind A function that reads arguments.length and is called through a value reads the count its adapter stored, then clears the slot. That read ran after the parameter prologue, so a parameter binding that threw would leave the slot set for the next direct call. The count is now taken, and the slot cleared, before any parameter binds. --- .../src/frontend/lowering/lower-functions.ts | 7 ++++++- ...3-argument-count-destructured-parameter.ts.json | 6 ++++++ .../8223-argument-count-destructured-parameter.ts | 14 ++++++++++++++ ...3-argument-count-destructured-parameter.ts.json | 1 + 4 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8223-argument-count-destructured-parameter.ts.json create mode 100644 tests/corpus/8223-argument-count-destructured-parameter.ts create mode 100644 tests/harness/corpus-costs/8223-argument-count-destructured-parameter.ts.json diff --git a/packages/compiler/src/frontend/lowering/lower-functions.ts b/packages/compiler/src/frontend/lowering/lower-functions.ts index c1682f7c8..bd9b5f55d 100644 --- a/packages/compiler/src/frontend/lowering/lower-functions.ts +++ b/packages/compiler/src/frontend/lowering/lower-functions.ts @@ -535,12 +535,18 @@ export function lowerFunction(lowerer: Lowerer, decl: ts.FunctionDeclaration): I const diagsBefore = lowerer.diags.length; lowerer.fnStack.push(ctx); try { + // The count a call through a value stored is taken (and its slot + // cleared) before the parameters bind: a destructuring default that + // throws must not leave the slot set for the next direct call. + const argumentCount: IrStmt[] = []; + declareArgumentCount(lowerer, decl, argumentCount); const receiver = decl.parameters.nodes.some(isThisParameter) || isJsSourceFile(ts.getSourceFileOfNode(decl)) ? declareCallReceiver(lowerer, decl, sig.generator !== undefined) : null; const { params, prologue } = lowerer.declareParams(decl.parameters.nodes, sig.params); if (receiver) prologue.unshift(receiver); + prologue.unshift(...argumentCount); declareFunctionArguments( lowerer, decl, @@ -552,7 +558,6 @@ export function lowerFunction(lowerer: Lowerer, decl: ts.FunctionDeclaration): I (sig.params[sig.params.length - 1]!.mode === "dynRest" || sig.params[sig.params.length - 1]!.mode === "arguments"), ); - declareArgumentCount(lowerer, decl, prologue); const bodyBlock = blockBodyOf(decl); if (!bodyBlock) { lowerer.unsupported( diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8223-argument-count-destructured-parameter.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8223-argument-count-destructured-parameter.ts.json new file mode 100644 index 000000000..6cae202d3 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8223-argument-count-destructured-parameter.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8223-argument-count-destructured-parameter.ts" + ], + "diags": [] +} diff --git a/tests/corpus/8223-argument-count-destructured-parameter.ts b/tests/corpus/8223-argument-count-destructured-parameter.ts new file mode 100644 index 000000000..def8c5ba8 --- /dev/null +++ b/tests/corpus/8223-argument-count-destructured-parameter.ts @@ -0,0 +1,14 @@ +// A function with a destructured parameter that reads arguments.length, +// called through a value (map passes three arguments) and directly. The +// count is taken before the parameters bind, so each call sees its own. + +function probe({ v, w }: { v: number; w: string }): string { + return `${arguments.length}:${v}${w}`; +} + +console.log([{ v: 1, w: "a" }, { v: 2, w: "b" }].map(probe)); +console.log(probe({ v: 3, w: "c" })); +const call = (f: (item: { v: number; w: string }, index: number) => string): string => + f({ v: 4, w: "d" }, 0); +console.log(call(probe)); +console.log(probe({ v: 5, w: "e" })); diff --git a/tests/harness/corpus-costs/8223-argument-count-destructured-parameter.ts.json b/tests/harness/corpus-costs/8223-argument-count-destructured-parameter.ts.json new file mode 100644 index 000000000..664b49416 --- /dev/null +++ b/tests/harness/corpus-costs/8223-argument-count-destructured-parameter.ts.json @@ -0,0 +1 @@ +{ "plain": 4700, "san": 4200 } From 6e4bfb3c913bf82bd9119243495f9074bced4ea0 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 17:34:40 -0500 Subject: [PATCH 15/17] Await the final value of an untyped sync pipeline source for await over a sync iterator awaits the value of the result that ends it too, so a sync iterator that returns a rejected promise rejects the pump. The source's generator skipped that value and reported success. It now awaits a final value that is an object (a promise included) before it stops, without closing the finished iterator. --- .../src/frontend/lowering/lower-stream.ts | 80 +++++++++++-------- .../7504-pipeline-untyped-source-errors.mjs | 24 +++++- 2 files changed, 70 insertions(+), 34 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index b519ce378..d3883427a 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -3954,6 +3954,38 @@ function lowerDynIterableSource( const sync = (): IrExpr => varRef(syncId, BOOL, loc); const step = (): IrExpr => varRef(stepId, DYN, loc); const value = (): IrExpr => varRef(valueId, DYN, loc); + // for await over a sync iterator awaits each value, the final one + // included (CreateAsyncFromSyncIterator): an object (a promise or a + // thenable) passes on its result + const awaitsValue = (): IrExpr => ({ + kind: "logical", + op: "&&", + left: sync(), + right: { + kind: "logical", + op: "||", + left: { kind: "dynTest", test: "promise", value: value(), type: BOOL, loc }, + right: { + kind: "logical", + op: "&&", + left: { + kind: "dynTest", + test: "nullish", + negated: true, + value: value(), + type: BOOL, + loc, + }, + right: { kind: "dynTest", test: "object", value: value(), type: BOOL, loc }, + type: BOOL, + loc, + }, + type: BOOL, + loc, + }, + type: BOOL, + loc, + }); const get = (obj: IrExpr, name: string): IrExpr => ({ kind: "dynKeyGet", value: obj, @@ -4053,46 +4085,28 @@ function lowerDynIterableSource( type: BOOL, loc, }, - then: [{ kind: "break", loc }], + // the final value is awaited too: a rejection rejects the + // pump, and the finished iterator is not closed + then: [ + { kind: "assign", localId: valueId, value: get(step(), "value"), loc }, + { + kind: "if", + cond: awaitsValue(), + then: [{ kind: "exprStmt", expr: awaitDyn(value()), loc }], + else_: null, + loc, + }, + { kind: "break", loc }, + ], else_: null, loc, }, setActive(true), { kind: "assign", localId: valueId, value: get(step(), "value"), loc }, - // for await over a sync iterator awaits each value - // (CreateAsyncFromSyncIterator): a promise or a thenable - // passes on its result, and a rejection closes the iterator + // a rejection here closes the iterator { kind: "if", - cond: { - kind: "logical", - op: "&&", - left: sync(), - right: { - kind: "logical", - op: "||", - left: { kind: "dynTest", test: "promise", value: value(), type: BOOL, loc }, - right: { - kind: "logical", - op: "&&", - left: { - kind: "dynTest", - test: "nullish", - negated: true, - value: value(), - type: BOOL, - loc, - }, - right: { kind: "dynTest", test: "object", value: value(), type: BOOL, loc }, - type: BOOL, - loc, - }, - type: BOOL, - loc, - }, - type: BOOL, - loc, - }, + cond: awaitsValue(), then: [{ kind: "assign", localId: valueId, value: awaitDyn(value()), loc }], else_: null, loc, diff --git a/tests/corpus/7504-pipeline-untyped-source-errors.mjs b/tests/corpus/7504-pipeline-untyped-source-errors.mjs index 3ee984547..5d6faded1 100644 --- a/tests/corpus/7504-pipeline-untyped-source-errors.mjs +++ b/tests/corpus/7504-pipeline-untyped-source-errors.mjs @@ -1,7 +1,7 @@ // pipeline() over untyped iterable sources: a destination that fails closes // the source, backpressure waits for drain, an abort signal stops the pump, // async generator and Map sources reach object-mode destinations, and a -// sync source's promise values are awaited. +// sync source's promise values, its final value included, are awaited. import { pipeline, Writable, PassThrough } from "node:stream"; import { pipeline as pipelineP } from "node:stream/promises"; @@ -113,3 +113,25 @@ try { } catch (err) { console.log("promised rejected", err.message); } + +// a sync source's final value is awaited too: a rejected one rejects the +// pipeline, and a resolved one is not written +function* finalRejects() { + yield "f1"; + return Promise.reject(new Error("final value rejected")); +} +function* finalResolves() { + yield "r1"; + return Promise.resolve("not written"); +} +for (const [label, source] of [["finalRejects", finalRejects], ["finalResolves", finalResolves]]) { + try { + await pipelineP( + pick(source()), + new Writable({ objectMode: true, write(c, _e, cb) { console.log(label, "write", c); cb(); } }), + ); + console.log(label, "resolved"); + } catch (err) { + console.log(label, "rejected", err.message); + } +} From 01f5a3455a215634771f94ab4e813803013253f9 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 17:38:44 -0500 Subject: [PATCH 16/17] Test row-pair overlaps only for nests of two or more rows The paired copy's test read the stored and read arrays' headers whatever the row range, so a nest whose arrays the program assigns only for a nonempty range dereferenced their NULL headers on an empty range, where the loop as written never reads them. The overlap tests now run only when the nest has at least two rows, and an array that is not there answers that it is not disjoint, so the loop runs as written. --- .../compiler/src/backend/llvm/lib-dispatch.ts | 35 ++++++++++++++----- packages/compiler/src/ir/row-pairs.ts | 17 ++++++--- tests/corpus/8194-row-pair-sums.ts | 18 ++++++++++ 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/packages/compiler/src/backend/llvm/lib-dispatch.ts b/packages/compiler/src/backend/llvm/lib-dispatch.ts index f4ce9e405..95cee3860 100644 --- a/packages/compiler/src/backend/llvm/lib-dispatch.ts +++ b/packages/compiler/src/backend/llvm/lib-dispatch.ts @@ -937,10 +937,24 @@ export function emitLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue { } if (e.fn === "bytes.absent") return host.own({ name: "null", type: e.type }); if (e.fn === "bytes.disjoint") { - // [data, data + len * size) of each view, compared as addresses. + // [data, data + len * size) of each view, compared as addresses. A view + // that is not there (a binding the nest never assigned, NULL) answers + // false without reading its header, so the caller keeps its loop. const B = host.B; - const range = (arg: IrExpr): { start: string; end: string } => { - const value = host.emitReadReceiver(arg); + const values = e.args.map((arg) => ({ arg, value: host.emitReadReceiver(arg) })); + const present = values.map(({ value }) => { + const t = B.tmp(); + B.line(`${t} = icmp ne ptr ${value.name}, null`); + return t; + }); + const both = B.tmp(); + B.line(`${both} = and i1 ${present[0]}, ${present[1]}`); + const compare = B.newLabel("disjoint.compare"), + done = B.newLabel("disjoint.done"); + const from = B.currentLabel(); + B.condBr(both, compare, done); + B.startBlock(compare); + const range = (arg: IrExpr, pointer: string): { start: string; end: string } => { const size = arg.type.kind === "bytes" ? BYTES_ELEMENT_SIZE[arg.type.elem] : 1; const dataSlot = B.tmp(), data = B.tmp(), @@ -949,23 +963,28 @@ export function emitLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue { bytes = B.tmp(), start = B.tmp(), end = B.tmp(); - B.line(`${dataSlot} = getelementptr inbounds %ScrBytes, ptr ${value.name}, i64 0, i32 3`); + B.line(`${dataSlot} = getelementptr inbounds %ScrBytes, ptr ${pointer}, i64 0, i32 3`); B.line(`${data} = load ptr, ptr ${dataSlot}`); - B.line(`${lenSlot} = getelementptr inbounds %ScrBytes, ptr ${value.name}, i64 0, i32 1`); + B.line(`${lenSlot} = getelementptr inbounds %ScrBytes, ptr ${pointer}, i64 0, i32 1`); B.line(`${len} = load ${host.sizeType}, ptr ${lenSlot}`); B.line(`${bytes} = mul ${host.sizeType} ${len}, ${size}`); B.line(`${start} = ptrtoint ptr ${data} to ${host.sizeType}`); B.line(`${end} = add ${host.sizeType} ${start}, ${bytes}`); return { start, end }; }; - const left = range(e.args[0]!), - right = range(e.args[1]!); + const left = range(values[0]!.arg, values[0]!.value.name), + right = range(values[1]!.arg, values[1]!.value.name); const before = B.tmp(), after = B.tmp(), + apart = B.tmp(), t = B.tmp(); B.line(`${before} = icmp ule ${host.sizeType} ${left.end}, ${right.start}`); B.line(`${after} = icmp ule ${host.sizeType} ${right.end}, ${left.start}`); - B.line(`${t} = or i1 ${before}, ${after}`); + B.line(`${apart} = or i1 ${before}, ${after}`); + const compared = B.currentLabel(); + B.br(done); + B.startBlock(done); + B.line(`${t} = phi i1 [ false, %${from} ], [ ${apart}, %${compared} ]`); return { name: t, type: e.type }; } if (e.fn === "record.isFrozen") { diff --git a/packages/compiler/src/ir/row-pairs.ts b/packages/compiler/src/ir/row-pairs.ts index ed71166a8..2e189f42e 100644 --- a/packages/compiler/src/ir/row-pairs.ts +++ b/packages/compiler/src/ir/row-pairs.ts @@ -317,8 +317,10 @@ function pairRows( loc, }; // Both bounds are int32, so `row < end - 1` is `row + 1 < end`, and the - // start is at most the end. Rows are independent when the written array - // shares no byte with an array the term reads. + // nest has at least two rows. The overlap tests run only then, so they never + // read an array the program assigns only when there are rows. Rows are + // independent when the written array shares no byte with an array the + // term reads; an array that is not there answers false. const int32 = (e: IrExpr): IrExpr => ({ kind: "bin", op: "===", @@ -330,7 +332,14 @@ function pairRows( const tests: IrExpr[] = [ int32(match.start), int32(match.end), - { kind: "bin", op: "<=", left: again(match.start), right: again(match.end), type: BOOL, loc }, + { + kind: "bin", + op: "<", + left: bin("+", again(match.start), num(1), loc), + right: again(match.end), + type: BOOL, + loc, + }, ...match.reads.map((read): IrExpr => ({ kind: "libCall", fn: "bytes.disjoint", @@ -400,7 +409,7 @@ function pairRows( * expression of `row + 1`. The term only reads numbers, the counters and * typed-array elements, so it runs no other code and cannot throw. The * nest runs paired when both row bounds are int32 (so `row < end - 1` is - * `row + 1 < end`), the start is at most the end, and the array the rows + * `row + 1 < end`), there are at least two rows, and the array the rows * store into shares no byte with any array the term reads, so no row reads * another row's result. An odd row count runs its last row as written. * Otherwise the nest runs as written. Two independent sums take turns in diff --git a/tests/corpus/8194-row-pair-sums.ts b/tests/corpus/8194-row-pair-sums.ts index 9aa22b9bd..f28242ff5 100644 --- a/tests/corpus/8194-row-pair-sums.ts +++ b/tests/corpus/8194-row-pair-sums.ts @@ -70,3 +70,21 @@ weighted(f, w, out, 0, n); show("weighted", out); weighted(f, w, out, 1, n - 1); show("weighted inner", out); + +// Arrays a program assigns only when the row range is nonempty: an empty or +// one-row range never reads them, and the paired copy does not either. +function lazyRows(n: number, start: number, end: number): number { + let u!: Float64Array; + let v!: Float64Array; + if (end > start) { + u = new Float64Array(n).fill(1); + v = new Float64Array(end); + } + for (let i = start; i < end; i++) { + let t = 0; + for (let j = 0; j < n; j++) t += u[j]! / (i + j + 1); + v[i] = t; + } + return end > start ? v[end - 1]! : -1; +} +console.log("lazy", lazyRows(4, 3, 3), lazyRows(4, 2, 3), lazyRows(4, 0, 5), lazyRows(0, 5, 5)); From fcc81c034cbef0e3fbcea1835a131d43359e8692 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 17:59:56 -0500 Subject: [PATCH 17/17] Read the final result's value only for sync pipeline sources for await reads the value of a done result only through the async-from-sync wrapper; an async iterator's done result is not read. The source's generator read it for both, so an async iterator whose done result has a throwing value getter rejected the pipeline. It now reads and awaits the final value only when the iterator is sync. --- .../src/frontend/lowering/lower-stream.ts | 19 ++++++++--- .../7504-pipeline-untyped-source-errors.mjs | 34 +++++++++++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index d3883427a..d67cf257e 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -4085,14 +4085,23 @@ function lowerDynIterableSource( type: BOOL, loc, }, - // the final value is awaited too: a rejection rejects the - // pump, and the finished iterator is not closed + // a sync iterator's final value is awaited too: a rejection + // rejects the pump, and the finished iterator is not closed. + // An async iterator's final value is not read. then: [ - { kind: "assign", localId: valueId, value: get(step(), "value"), loc }, { kind: "if", - cond: awaitsValue(), - then: [{ kind: "exprStmt", expr: awaitDyn(value()), loc }], + cond: sync(), + then: [ + { kind: "assign", localId: valueId, value: get(step(), "value"), loc }, + { + kind: "if", + cond: awaitsValue(), + then: [{ kind: "exprStmt", expr: awaitDyn(value()), loc }], + else_: null, + loc, + }, + ], else_: null, loc, }, diff --git a/tests/corpus/7504-pipeline-untyped-source-errors.mjs b/tests/corpus/7504-pipeline-untyped-source-errors.mjs index 5d6faded1..9dd671606 100644 --- a/tests/corpus/7504-pipeline-untyped-source-errors.mjs +++ b/tests/corpus/7504-pipeline-untyped-source-errors.mjs @@ -135,3 +135,37 @@ for (const [label, source] of [["finalRejects", finalRejects], ["finalResolves", console.log(label, "rejected", err.message); } } + +// the final result's value is read for a sync iterator only: an async +// iterator's done result is not read, so a throwing value getter there is +// never called +function finalResult(label, i) { + if (i === 1) return { value: `${label}1`, done: false }; + return { + done: true, + get value() { + throw new Error(`${label} final value read`); + }, + }; +} +function asyncFinal() { + let i = 0; + return { [Symbol.asyncIterator]() { return { async next() { i++; return finalResult("async", i); } }; } }; +} +function syncFinal() { + let i = 0; + return { [Symbol.iterator]() { return { next() { i++; return finalResult("sync", i); } }; } }; +} +async function runFinal(label, source) { + try { + await pipelineP( + pick(source), + new Writable({ objectMode: true, write(c, _e, cb) { console.log(label, "write", c); cb(); } }), + ); + console.log(label, "resolved"); + } catch (err) { + console.log(label, "rejected", err.message); + } +} +await runFinal("async final", asyncFinal()); +await runFinal("sync final", syncFinal());